View Javadoc
1   package org.apache.maven.index.cli;
2   
3   /*
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *   http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing,
15   * software distributed under the License is distributed on an
16   * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
17   * KIND, either express or implied.  See the License for the
18   * specific language governing permissions and limitations
19   * under the License.
20   */
21  
22  import java.io.BufferedInputStream;
23  import java.io.File;
24  import java.io.FileInputStream;
25  import java.io.IOException;
26  import java.io.InputStream;
27  import java.lang.reflect.Proxy;
28  import java.util.ArrayList;
29  import java.util.List;
30  import java.util.Properties;
31  import java.util.concurrent.TimeUnit;
32  
33  import com.google.inject.Guice;
34  import com.google.inject.Module;
35  import org.apache.commons.cli.CommandLine;
36  import org.apache.commons.cli.DefaultParser;
37  import org.apache.commons.cli.HelpFormatter;
38  import org.apache.commons.cli.Option;
39  import org.apache.commons.cli.Options;
40  import org.apache.commons.cli.ParseException;
41  import org.apache.lucene.search.IndexSearcher;
42  import org.apache.lucene.store.FSDirectory;
43  import org.apache.maven.index.ArtifactContext;
44  import org.apache.maven.index.ArtifactInfo;
45  import org.apache.maven.index.ArtifactScanningListener;
46  import org.apache.maven.index.ScanningResult;
47  import org.apache.maven.index.context.IndexCreator;
48  import org.apache.maven.index.context.IndexingContext;
49  import org.apache.maven.index.context.UnsupportedExistingLuceneIndexException;
50  import org.apache.maven.index.packer.IndexPacker;
51  import org.apache.maven.index.packer.IndexPackingRequest;
52  import org.apache.maven.index.packer.IndexPackingRequest.IndexFormat;
53  import org.apache.maven.index.updater.DefaultIndexUpdater;
54  import org.eclipse.sisu.launch.Main;
55  import org.eclipse.sisu.space.BeanScanning;
56  
57  import static java.util.Objects.requireNonNull;
58  
59  /**
60   * A command line tool that can be used to index local Maven repository.
61   * <p/>
62   * The following command line options are supported:
63   * <ul>
64   * <li>-repository <path> : required path to repository to be indexed</li>
65   * <li>-index <path> : required index folder used to store created index or where previously created index is
66   * stored</li>
67   * <li>-name <path> : required repository name/id</li>
68   * <li>-target <path> : optional folder name where to save produced index files</li>
69   * <li>-type <path> : optional indexer types</li>
70   * <li>-format <path> : optional indexer formats</li>
71   * </ul>
72   * When index folder contains previously created index, the tool will use it as a base line and will generate chunks for
73   * the incremental updates.
74   * <p/>
75   * The indexer types could be one of default, min or full. You can also specify list of comma-separated custom index
76   * creators. An index creator should be a regular Plexus component, see
77   * {@link org.apache.maven.index.creator.MinimalArtifactInfoIndexCreator} and
78   * {@link org.apache.maven.index.creator.JarFileContentsIndexCreator}.
79   */
80  public class NexusIndexerCli
81  {
82  
83      // Generic command line options
84  
85      public static final String QUIET = "q";
86  
87      public static final String DEBUG = "X";
88  
89      public static final String HELP = "h";
90  
91      public static final String VERSION = "v";
92  
93      // Command line options
94  
95      public static final String REPO = "r";
96  
97      public static final String INDEX = "i";
98  
99      public static final String NAME = "n";
100 
101     public static final String TYPE = "t";
102 
103     public static final String TARGET_DIR = "d";
104 
105     public static final String CREATE_INCREMENTAL_CHUNKS = "c";
106 
107     public static final String CREATE_FILE_CHECKSUMS = "s";
108 
109     public static final String INCREMENTAL_CHUNK_KEEP_COUNT = "k";
110 
111     public static final String UNPACK = "u";
112 
113     private static final long MB = 1024 * 1024;
114 
115     private Options options;
116 
117     public static void main( String[] args )
118     {
119         System.exit( new NexusIndexerCli().execute( args ) );
120     }
121 
122     /**
123      * Visible for testing.
124      */
125     int execute( String[] args )
126     {
127         CommandLine cli;
128 
129         try
130         {
131             cli =  new DefaultParser().parse( buildCliOptions(), cleanArgs( args ) );
132         }
133         catch ( ParseException e )
134         {
135             System.err.println( "Unable to parse command line options: " + e.getMessage() );
136 
137             displayHelp();
138 
139             return 1;
140         }
141 
142         boolean debug = cli.hasOption( DEBUG );
143 
144         if ( cli.hasOption( HELP ) )
145         {
146             displayHelp();
147 
148             return 0;
149         }
150 
151         if ( cli.hasOption( VERSION ) )
152         {
153             showVersion();
154 
155             return 0;
156         }
157         else if ( debug )
158         {
159             showVersion();
160         }
161 
162         final Module app = Main.wire(
163                 BeanScanning.INDEX
164         );
165 
166         Components components =
167                 Guice.createInjector( app ).getInstance( Components.class );
168 
169         if ( cli.hasOption( UNPACK ) )
170         {
171             try
172             {
173                 return unpack( cli, components );
174             }
175             catch ( Exception e )
176             {
177                 e.printStackTrace( System.err );
178                 return 1;
179             }
180         }
181         else if ( cli.hasOption( INDEX ) && cli.hasOption( REPO ) )
182         {
183             try
184             {
185                 return index( cli, components );
186             }
187             catch ( Exception e )
188             {
189                 e.printStackTrace( System.err );
190                 return 1;
191             }
192         }
193         else
194         {
195             System.out.println();
196             System.out.println( "Use either unpack (\"" + UNPACK + "\") or index (\"" + INDEX + "\" and \"" + REPO
197                     + "\") options, but none has been found!" );
198             System.out.println();
199             displayHelp();
200             return 1;
201         }
202     }
203 
204     /**
205      * Visible for testing.
206      */
207     Options buildCliOptions()
208     {
209         this.options = new Options();
210 
211         options.addOption( Option.builder( QUIET ).longOpt( "quiet" )
212                 .desc( "Quiet output - only show errors" ).build() );
213 
214         options.addOption( Option.builder( DEBUG ).longOpt( "debug" )
215                 .desc( "Produce execution debug output" ).build() );
216 
217         options.addOption( Option.builder( VERSION ).longOpt( "version" )
218                 .desc( "Display version information" ).build() );
219 
220         options.addOption( Option.builder( HELP ).longOpt( "help" )
221                 .desc( "Display help information" ).build() );
222 
223 
224         options.addOption( Option.builder( INDEX ).longOpt( "index" ).argName( "path" ).hasArg()
225                 .desc( "Path to the index folder" ).build() );
226 
227         options.addOption( Option.builder( TARGET_DIR ).longOpt( "destination" ).argName( "path" ).hasArg()
228                 .desc( "Target folder" ).build() );
229 
230         options.addOption( Option.builder( REPO ).longOpt( "repository" ).argName( "path" ).hasArg()
231                 .desc( "Path to the Maven repository" ).build() );
232 
233         options.addOption( Option.builder( NAME ).longOpt( "name" ).argName( "string" ).hasArg()
234                 .desc( "Repository name" ).build() );
235 
236         options.addOption( Option.builder( CREATE_INCREMENTAL_CHUNKS ).longOpt( "chunks" )
237                 .desc( "Create incremental chunks" ).build() );
238 
239         options.addOption( Option.builder( INCREMENTAL_CHUNK_KEEP_COUNT ).longOpt( "keep" ).argName( "num" ).hasArg()
240                 .desc( "Number of incremental chunks to keep" ).build() );
241 
242         options.addOption( Option.builder( CREATE_FILE_CHECKSUMS ).longOpt( "checksums" )
243                 .desc( "Create checksums for all files (sha1, md5)" ).build() );
244 
245         options.addOption( Option.builder( TYPE ).longOpt( "type" ).argName( "type" ).hasArg()
246                 .desc( "Indexer type (default, min, full or comma separated list of custom types)" ).build() );
247 
248         options.addOption( Option.builder( UNPACK ).longOpt( "unpack" )
249                 .desc( "Unpack an index file" ).build() );
250 
251         return options;
252     }
253 
254     private String[] cleanArgs( String[] args )
255     {
256         List<String> cleaned = new ArrayList<>();
257 
258         StringBuilder currentArg = null;
259 
260         for ( String arg : args )
261         {
262             boolean addedToBuffer = false;
263 
264             if ( arg.startsWith( "\"" ) )
265             {
266                 // if we're in the process of building up another arg, push it and start over.
267                 // this is for the case: "-Dfoo=bar "-Dfoo2=bar two" (note the first unterminated quote)
268                 if ( currentArg != null )
269                 {
270                     cleaned.add( currentArg.toString() );
271                 }
272 
273                 // start building an argument here.
274                 currentArg = new StringBuilder( arg.substring( 1 ) );
275 
276                 addedToBuffer = true;
277             }
278 
279             // this has to be a separate "if" statement, to capture the case of: "-Dfoo=bar"
280             if ( arg.endsWith( "\"" ) )
281             {
282                 String cleanArgPart = arg.substring( 0, arg.length() - 1 );
283 
284                 // if we're building an argument, keep doing so.
285                 if ( currentArg != null )
286                 {
287                     // if this is the case of "-Dfoo=bar", then we need to adjust the buffer.
288                     if ( addedToBuffer )
289                     {
290                         currentArg.setLength( currentArg.length() - 1 );
291                     }
292                     // otherwise, we trim the trailing " and append to the buffer.
293                     else
294                     {
295                         // TODO: introducing a space here...not sure what else to do but collapse whitespace
296                         currentArg.append( ' ' ).append( cleanArgPart );
297                     }
298 
299                     // we're done with this argument, so add it.
300                     cleaned.add( currentArg.toString() );
301                 }
302                 else
303                 {
304                     // this is a simple argument...just add it.
305                     cleaned.add( cleanArgPart );
306                 }
307 
308                 // the currentArg MUST be finished when this completes.
309                 currentArg = null;
310 
311                 continue;
312             }
313 
314             // if we haven't added this arg to the buffer, and we ARE building an argument
315             // buffer, then append it with a preceding space...again, not sure what else to
316             // do other than collapse whitespace.
317             // NOTE: The case of a trailing quote is handled by nullifying the arg buffer.
318             if ( !addedToBuffer )
319             {
320                 // append to the argument we're building, collapsing whitespace to a single space.
321                 if ( currentArg != null )
322                 {
323                     currentArg.append( ' ' ).append( arg );
324                 }
325                 // this is a loner, just add it directly.
326                 else
327                 {
328                     cleaned.add( arg );
329                 }
330             }
331         }
332 
333         // clean up.
334         if ( currentArg != null )
335         {
336             cleaned.add( currentArg.toString() );
337         }
338 
339         int cleanedSz = cleaned.size();
340         String[] cleanArgs;
341 
342         if ( cleanedSz == 0 )
343         {
344             // if we didn't have any arguments to clean, simply pass the original array through
345             cleanArgs = args;
346         }
347         else
348         {
349             cleanArgs = cleaned.toArray( new String[cleanedSz] );
350         }
351 
352         return cleanArgs;
353     }
354 
355     private void displayHelp()
356     {
357         System.out.println();
358 
359         HelpFormatter formatter = new HelpFormatter();
360 
361         formatter.printHelp( "nexus-indexer [options]", "\nOptions:", options, "\n" );
362     }
363 
364     private void showVersion()
365     {
366         InputStream is;
367 
368         try
369         {
370             Properties properties = new Properties();
371 
372             is = getClass().getClassLoader().getResourceAsStream(
373                     "META-INF/maven/org.apache.maven.indexer/indexer-core/pom.properties" );
374 
375             if ( is == null )
376             {
377                 System.err.println( "Unable determine version from JAR file." );
378 
379                 return;
380             }
381 
382             properties.load( is );
383 
384             if ( properties.getProperty( "builtOn" ) != null )
385             {
386                 System.out.println( "Version: " + properties.getProperty( "version", "unknown" )
387                         + " built on " + properties.getProperty( "builtOn" ) );
388             }
389             else
390             {
391                 System.out.println( "Version: " + properties.getProperty( "version", "unknown" ) );
392             }
393         }
394         catch ( IOException e )
395         {
396             System.err.println( "Unable determine version from JAR file: " + e.getMessage() );
397         }
398     }
399 
400     private int index( final CommandLine cli, Components components )
401             throws IOException, UnsupportedExistingLuceneIndexException
402     {
403         String indexDirectoryName = cli.getOptionValue( INDEX );
404 
405         File indexFolder = new File( indexDirectoryName );
406 
407         String outputDirectoryName = cli.getOptionValue( TARGET_DIR, "." );
408 
409         File outputFolder = new File( outputDirectoryName );
410 
411         File repositoryFolder = new File( cli.getOptionValue( REPO ) );
412 
413         String repositoryName = cli.getOptionValue( NAME, indexFolder.getName() );
414 
415         List<IndexCreator> indexers = getIndexers( cli, components );
416 
417         boolean createChecksums = cli.hasOption( CREATE_FILE_CHECKSUMS );
418 
419         boolean createIncrementalChunks = cli.hasOption( CREATE_INCREMENTAL_CHUNKS );
420 
421         boolean debug = cli.hasOption( DEBUG );
422 
423         boolean quiet = cli.hasOption( QUIET );
424 
425         Integer chunkCount = cli.hasOption( INCREMENTAL_CHUNK_KEEP_COUNT )
426                 ? Integer.parseInt( cli.getOptionValue( INCREMENTAL_CHUNK_KEEP_COUNT ) )
427                 : null;
428 
429         if ( !quiet )
430         {
431             System.err.printf( "Repository Folder: %s\n", repositoryFolder.getAbsolutePath() );
432             System.err.printf( "Index Folder:      %s\n", indexFolder.getAbsolutePath() );
433             System.err.printf( "Output Folder:     %s\n", outputFolder.getAbsolutePath() );
434             System.err.printf( "Repository name:   %s\n", repositoryName );
435             System.err.printf( "Indexers: %s\n", indexers );
436 
437             if ( createChecksums )
438             {
439                 System.err.print( "Will create checksum files for all published files (sha1, md5).\n" );
440             }
441             else
442             {
443                 System.err.print( "Will not create checksum files.\n" );
444             }
445 
446             if ( createIncrementalChunks )
447             {
448                 System.err.print( "Will create incremental chunks for changes, along with baseline file.\n" );
449             }
450             else
451             {
452                 System.err.print( "Will create baseline file.\n" );
453             }
454         }
455 
456         long tstart = System.currentTimeMillis();
457 
458         IndexingContext context = components.indexer.addIndexingContext( //
459                 repositoryName, // context id
460                 repositoryName, // repository id
461                 repositoryFolder, // repository folder
462                 indexFolder, // index folder
463                 null, // repositoryUrl
464                 null, // index update url
465                 indexers );
466 
467         try
468         {
469             ArtifactScanningListener listener = new IndexerListener( context, debug, quiet );
470 
471             components.indexer.scan( context, listener, true );
472 
473             IndexSearcher indexSearcher = context.acquireIndexSearcher();
474 
475             try
476             {
477                 IndexPackingRequest request =
478                         new IndexPackingRequest( context, indexSearcher.getIndexReader(), outputFolder );
479 
480                 request.setCreateChecksumFiles( createChecksums );
481 
482                 request.setCreateIncrementalChunks( createIncrementalChunks );
483 
484                 request.setFormats( List.of( IndexFormat.FORMAT_V1 ) );
485 
486                 if ( chunkCount != null )
487                 {
488                     request.setMaxIndexChunks( chunkCount );
489                 }
490 
491                 packIndex( components.packer, request, debug, quiet );
492             }
493             finally
494             {
495                 context.releaseIndexSearcher( indexSearcher );
496             }
497 
498             if ( !quiet )
499             {
500                 printStats( tstart );
501             }
502         }
503         finally
504         {
505             components.indexer.removeIndexingContext( context, false );
506         }
507         return 0;
508     }
509 
510     private int unpack( CommandLine cli, Components components )
511             throws IOException
512     {
513         final String indexDirectoryName = cli.getOptionValue( INDEX, "." );
514         final File indexFolder = new File( indexDirectoryName ).getCanonicalFile();
515         final File indexArchive = new File( indexFolder, IndexingContext.INDEX_FILE_PREFIX + ".gz" );
516 
517         final String outputDirectoryName = cli.getOptionValue( TARGET_DIR, "." );
518         final File outputFolder = new File( outputDirectoryName ).getCanonicalFile();
519 
520         final boolean quiet = cli.hasOption( QUIET );
521         if ( !quiet )
522         {
523             System.err.printf( "Index Folder:      %s\n", indexFolder.getAbsolutePath() );
524             System.err.printf( "Output Folder:     %s\n", outputFolder.getAbsolutePath() );
525         }
526 
527         long tstart = System.currentTimeMillis();
528 
529         final List<IndexCreator> indexers = getIndexers( cli, components );
530 
531 
532         try ( BufferedInputStream is = new BufferedInputStream( new FileInputStream( indexArchive ) ); //
533               FSDirectory directory = FSDirectory.open( outputFolder.toPath() ) )
534         {
535             DefaultIndexUpdater.unpackIndexData( is, 4, directory, (IndexingContext) Proxy.newProxyInstance(
536                     getClass().getClassLoader(), new Class[] {IndexingContext.class}, new PartialImplementation()
537                     {
538                         public List<IndexCreator> getIndexCreators()
539                         {
540                             return indexers;
541                         }
542                     } )
543 
544             );
545         }
546 
547         if ( !quiet )
548         {
549             printStats( tstart );
550         }
551         return 0;
552     }
553 
554     private List<IndexCreator> getIndexers( final CommandLine cli, Components components )
555     {
556         String type = "default";
557 
558         if ( cli.hasOption( TYPE ) )
559         {
560             type = cli.getOptionValue( TYPE );
561         }
562 
563         List<IndexCreator> indexers = new ArrayList<>(); // NexusIndexer.DEFAULT_INDEX;
564 
565         if ( "default".equals( type ) )
566         {
567             indexers.add( requireNonNull( components.allIndexCreators.get( "min" ) ) );
568             indexers.add( requireNonNull( components.allIndexCreators.get( "jarContent" ) ) );
569         }
570         else if ( "full".equals( type ) )
571         {
572             indexers.addAll( components.allIndexCreators.values() );
573         }
574         else
575         {
576             for ( String name : type.split( "," ) )
577             {
578                 indexers.add( requireNonNull( components.allIndexCreators.get( name ) ) );
579             }
580         }
581         return indexers;
582     }
583 
584     private void packIndex( IndexPacker packer, IndexPackingRequest request, boolean debug, boolean quiet )
585     {
586         try
587         {
588             packer.packIndex( request );
589         }
590         catch ( IOException e )
591         {
592             if ( !quiet )
593             {
594                 System.err.printf( "Cannot zip index: %s\n", e.getMessage() );
595 
596                 if ( debug )
597                 {
598                     e.printStackTrace();
599                 }
600             }
601         }
602     }
603 
604     private void printStats( final long startTimeInMillis )
605     {
606         long t = System.currentTimeMillis() - startTimeInMillis;
607 
608         long s = TimeUnit.MILLISECONDS.toSeconds( t );
609         if ( t > TimeUnit.MINUTES.toMillis( 1 ) )
610         {
611             long m = TimeUnit.MILLISECONDS.toMinutes( t );
612 
613             System.err.printf( "Total time:   %d min %d sec\n", m, s - ( m * 60 ) );
614         }
615         else
616         {
617             System.err.printf( "Total time:   %d sec\n", s );
618         }
619 
620         Runtime r = Runtime.getRuntime();
621 
622         System.err.printf( "Final memory: %dM/%dM\n", //
623                 ( r.totalMemory() - r.freeMemory() ) / MB, r.totalMemory() / MB );
624     }
625 
626     /**
627      * Scanner listener
628      */
629     private static final class IndexerListener
630             implements ArtifactScanningListener
631     {
632         private final IndexingContext context;
633 
634         private final boolean debug;
635 
636         private final boolean quiet;
637 
638         private long ts = System.currentTimeMillis();
639 
640         private int count;
641 
642         IndexerListener( IndexingContext context, boolean debug, boolean quiet )
643         {
644             this.context = context;
645             this.debug = debug;
646             this.quiet = quiet;
647         }
648 
649         @Override
650         public void scanningStarted( IndexingContext context )
651         {
652             if ( !quiet )
653             {
654                 System.err.println( "Scanning started" );
655             }
656         }
657 
658         @Override
659         public void artifactDiscovered( ArtifactContext ac )
660         {
661             count++;
662 
663             long t = System.currentTimeMillis();
664 
665             ArtifactInfo ai = ac.getArtifactInfo();
666 
667             if ( !quiet && debug && "maven-plugin".equals( ai.getPackaging() ) )
668             {
669                 System.err.printf( "Plugin: %s:%s:%s - %s %s\n", //
670                         ai.getGroupId(), ai.getArtifactId(), ai.getVersion(), ai.getPrefix(), "" + ai.getGoals() );
671             }
672 
673             if ( !quiet && ( debug || ( t - ts ) > 2000L ) )
674             {
675                 System.err.printf( "  %6d %s\n", count, formatFile( ac.getPom() ) );
676                 ts = t;
677             }
678         }
679 
680         @Override
681         public void artifactError( ArtifactContext ac, Exception e )
682         {
683             if ( !quiet )
684             {
685                 System.err.printf( "! %6d %s - %s\n", count, formatFile( ac.getPom() ), e.getMessage() );
686 
687                 System.err.printf( "         %s\n", formatFile( ac.getArtifact() ) );
688 
689                 if ( debug )
690                 {
691                     e.printStackTrace();
692                 }
693             }
694 
695             ts = System.currentTimeMillis();
696         }
697 
698         private String formatFile( File file )
699         {
700             return file.getAbsolutePath().substring( context.getRepository().getAbsolutePath().length() + 1 );
701         }
702 
703         @Override
704         public void scanningFinished( IndexingContext context, ScanningResult result )
705         {
706             if ( !quiet )
707             {
708                 if ( result.hasExceptions() )
709                 {
710                     System.err.printf( "Scanning errors:   %s\n", result.getExceptions().size() );
711                 }
712 
713                 System.err.printf( "Artifacts added:   %s\n", result.getTotalFiles() );
714                 System.err.printf( "Artifacts deleted: %s\n", result.getDeletedFiles() );
715             }
716         }
717     }
718 
719 }