1 package org.apache.maven.index.cli;
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22 import java.io.BufferedInputStream;
23 import java.io.File;
24 import java.io.FileInputStream;
25 import java.io.IOException;
26 import java.io.InputStream;
27 import java.lang.reflect.Proxy;
28 import java.util.ArrayList;
29 import java.util.List;
30 import java.util.Properties;
31 import java.util.concurrent.TimeUnit;
32
33 import com.google.inject.Guice;
34 import com.google.inject.Module;
35 import org.apache.commons.cli.CommandLine;
36 import org.apache.commons.cli.DefaultParser;
37 import org.apache.commons.cli.HelpFormatter;
38 import org.apache.commons.cli.Option;
39 import org.apache.commons.cli.Options;
40 import org.apache.commons.cli.ParseException;
41 import org.apache.lucene.search.IndexSearcher;
42 import org.apache.lucene.store.FSDirectory;
43 import org.apache.maven.index.ArtifactContext;
44 import org.apache.maven.index.ArtifactInfo;
45 import org.apache.maven.index.ArtifactScanningListener;
46 import org.apache.maven.index.ScanningResult;
47 import org.apache.maven.index.context.IndexCreator;
48 import org.apache.maven.index.context.IndexingContext;
49 import org.apache.maven.index.context.UnsupportedExistingLuceneIndexException;
50 import org.apache.maven.index.packer.IndexPacker;
51 import org.apache.maven.index.packer.IndexPackingRequest;
52 import org.apache.maven.index.packer.IndexPackingRequest.IndexFormat;
53 import org.apache.maven.index.updater.DefaultIndexUpdater;
54 import org.eclipse.sisu.launch.Main;
55 import org.eclipse.sisu.space.BeanScanning;
56
57 import static java.util.Objects.requireNonNull;
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80 public class NexusIndexerCli
81 {
82
83
84
85 public static final String QUIET = "q";
86
87 public static final String DEBUG = "X";
88
89 public static final String HELP = "h";
90
91 public static final String VERSION = "v";
92
93
94
95 public static final String REPO = "r";
96
97 public static final String INDEX = "i";
98
99 public static final String NAME = "n";
100
101 public static final String TYPE = "t";
102
103 public static final String TARGET_DIR = "d";
104
105 public static final String CREATE_INCREMENTAL_CHUNKS = "c";
106
107 public static final String CREATE_FILE_CHECKSUMS = "s";
108
109 public static final String INCREMENTAL_CHUNK_KEEP_COUNT = "k";
110
111 public static final String UNPACK = "u";
112
113 private static final long MB = 1024 * 1024;
114
115 private Options options;
116
117 public static void main( String[] args )
118 {
119 System.exit( new NexusIndexerCli().execute( args ) );
120 }
121
122
123
124
125 int execute( String[] args )
126 {
127 CommandLine cli;
128
129 try
130 {
131 cli = new DefaultParser().parse( buildCliOptions(), cleanArgs( args ) );
132 }
133 catch ( ParseException e )
134 {
135 System.err.println( "Unable to parse command line options: " + e.getMessage() );
136
137 displayHelp();
138
139 return 1;
140 }
141
142 boolean debug = cli.hasOption( DEBUG );
143
144 if ( cli.hasOption( HELP ) )
145 {
146 displayHelp();
147
148 return 0;
149 }
150
151 if ( cli.hasOption( VERSION ) )
152 {
153 showVersion();
154
155 return 0;
156 }
157 else if ( debug )
158 {
159 showVersion();
160 }
161
162 final Module app = Main.wire(
163 BeanScanning.INDEX
164 );
165
166 Components components =
167 Guice.createInjector( app ).getInstance( Components.class );
168
169 if ( cli.hasOption( UNPACK ) )
170 {
171 try
172 {
173 return unpack( cli, components );
174 }
175 catch ( Exception e )
176 {
177 e.printStackTrace( System.err );
178 return 1;
179 }
180 }
181 else if ( cli.hasOption( INDEX ) && cli.hasOption( REPO ) )
182 {
183 try
184 {
185 return index( cli, components );
186 }
187 catch ( Exception e )
188 {
189 e.printStackTrace( System.err );
190 return 1;
191 }
192 }
193 else
194 {
195 System.out.println();
196 System.out.println( "Use either unpack (\"" + UNPACK + "\") or index (\"" + INDEX + "\" and \"" + REPO
197 + "\") options, but none has been found!" );
198 System.out.println();
199 displayHelp();
200 return 1;
201 }
202 }
203
204
205
206
207 Options buildCliOptions()
208 {
209 this.options = new Options();
210
211 options.addOption( Option.builder( QUIET ).longOpt( "quiet" )
212 .desc( "Quiet output - only show errors" ).build() );
213
214 options.addOption( Option.builder( DEBUG ).longOpt( "debug" )
215 .desc( "Produce execution debug output" ).build() );
216
217 options.addOption( Option.builder( VERSION ).longOpt( "version" )
218 .desc( "Display version information" ).build() );
219
220 options.addOption( Option.builder( HELP ).longOpt( "help" )
221 .desc( "Display help information" ).build() );
222
223
224 options.addOption( Option.builder( INDEX ).longOpt( "index" ).argName( "path" ).hasArg()
225 .desc( "Path to the index folder" ).build() );
226
227 options.addOption( Option.builder( TARGET_DIR ).longOpt( "destination" ).argName( "path" ).hasArg()
228 .desc( "Target folder" ).build() );
229
230 options.addOption( Option.builder( REPO ).longOpt( "repository" ).argName( "path" ).hasArg()
231 .desc( "Path to the Maven repository" ).build() );
232
233 options.addOption( Option.builder( NAME ).longOpt( "name" ).argName( "string" ).hasArg()
234 .desc( "Repository name" ).build() );
235
236 options.addOption( Option.builder( CREATE_INCREMENTAL_CHUNKS ).longOpt( "chunks" )
237 .desc( "Create incremental chunks" ).build() );
238
239 options.addOption( Option.builder( INCREMENTAL_CHUNK_KEEP_COUNT ).longOpt( "keep" ).argName( "num" ).hasArg()
240 .desc( "Number of incremental chunks to keep" ).build() );
241
242 options.addOption( Option.builder( CREATE_FILE_CHECKSUMS ).longOpt( "checksums" )
243 .desc( "Create checksums for all files (sha1, md5)" ).build() );
244
245 options.addOption( Option.builder( TYPE ).longOpt( "type" ).argName( "type" ).hasArg()
246 .desc( "Indexer type (default, min, full or comma separated list of custom types)" ).build() );
247
248 options.addOption( Option.builder( UNPACK ).longOpt( "unpack" )
249 .desc( "Unpack an index file" ).build() );
250
251 return options;
252 }
253
254 private String[] cleanArgs( String[] args )
255 {
256 List<String> cleaned = new ArrayList<>();
257
258 StringBuilder currentArg = null;
259
260 for ( String arg : args )
261 {
262 boolean addedToBuffer = false;
263
264 if ( arg.startsWith( "\"" ) )
265 {
266
267
268 if ( currentArg != null )
269 {
270 cleaned.add( currentArg.toString() );
271 }
272
273
274 currentArg = new StringBuilder( arg.substring( 1 ) );
275
276 addedToBuffer = true;
277 }
278
279
280 if ( arg.endsWith( "\"" ) )
281 {
282 String cleanArgPart = arg.substring( 0, arg.length() - 1 );
283
284
285 if ( currentArg != null )
286 {
287
288 if ( addedToBuffer )
289 {
290 currentArg.setLength( currentArg.length() - 1 );
291 }
292
293 else
294 {
295
296 currentArg.append( ' ' ).append( cleanArgPart );
297 }
298
299
300 cleaned.add( currentArg.toString() );
301 }
302 else
303 {
304
305 cleaned.add( cleanArgPart );
306 }
307
308
309 currentArg = null;
310
311 continue;
312 }
313
314
315
316
317
318 if ( !addedToBuffer )
319 {
320
321 if ( currentArg != null )
322 {
323 currentArg.append( ' ' ).append( arg );
324 }
325
326 else
327 {
328 cleaned.add( arg );
329 }
330 }
331 }
332
333
334 if ( currentArg != null )
335 {
336 cleaned.add( currentArg.toString() );
337 }
338
339 int cleanedSz = cleaned.size();
340 String[] cleanArgs;
341
342 if ( cleanedSz == 0 )
343 {
344
345 cleanArgs = args;
346 }
347 else
348 {
349 cleanArgs = cleaned.toArray( new String[cleanedSz] );
350 }
351
352 return cleanArgs;
353 }
354
355 private void displayHelp()
356 {
357 System.out.println();
358
359 HelpFormatter formatter = new HelpFormatter();
360
361 formatter.printHelp( "nexus-indexer [options]", "\nOptions:", options, "\n" );
362 }
363
364 private void showVersion()
365 {
366 InputStream is;
367
368 try
369 {
370 Properties properties = new Properties();
371
372 is = getClass().getClassLoader().getResourceAsStream(
373 "META-INF/maven/org.apache.maven.indexer/indexer-core/pom.properties" );
374
375 if ( is == null )
376 {
377 System.err.println( "Unable determine version from JAR file." );
378
379 return;
380 }
381
382 properties.load( is );
383
384 if ( properties.getProperty( "builtOn" ) != null )
385 {
386 System.out.println( "Version: " + properties.getProperty( "version", "unknown" )
387 + " built on " + properties.getProperty( "builtOn" ) );
388 }
389 else
390 {
391 System.out.println( "Version: " + properties.getProperty( "version", "unknown" ) );
392 }
393 }
394 catch ( IOException e )
395 {
396 System.err.println( "Unable determine version from JAR file: " + e.getMessage() );
397 }
398 }
399
400 private int index( final CommandLine cli, Components components )
401 throws IOException, UnsupportedExistingLuceneIndexException
402 {
403 String indexDirectoryName = cli.getOptionValue( INDEX );
404
405 File indexFolder = new File( indexDirectoryName );
406
407 String outputDirectoryName = cli.getOptionValue( TARGET_DIR, "." );
408
409 File outputFolder = new File( outputDirectoryName );
410
411 File repositoryFolder = new File( cli.getOptionValue( REPO ) );
412
413 String repositoryName = cli.getOptionValue( NAME, indexFolder.getName() );
414
415 List<IndexCreator> indexers = getIndexers( cli, components );
416
417 boolean createChecksums = cli.hasOption( CREATE_FILE_CHECKSUMS );
418
419 boolean createIncrementalChunks = cli.hasOption( CREATE_INCREMENTAL_CHUNKS );
420
421 boolean debug = cli.hasOption( DEBUG );
422
423 boolean quiet = cli.hasOption( QUIET );
424
425 Integer chunkCount = cli.hasOption( INCREMENTAL_CHUNK_KEEP_COUNT )
426 ? Integer.parseInt( cli.getOptionValue( INCREMENTAL_CHUNK_KEEP_COUNT ) )
427 : null;
428
429 if ( !quiet )
430 {
431 System.err.printf( "Repository Folder: %s\n", repositoryFolder.getAbsolutePath() );
432 System.err.printf( "Index Folder: %s\n", indexFolder.getAbsolutePath() );
433 System.err.printf( "Output Folder: %s\n", outputFolder.getAbsolutePath() );
434 System.err.printf( "Repository name: %s\n", repositoryName );
435 System.err.printf( "Indexers: %s\n", indexers );
436
437 if ( createChecksums )
438 {
439 System.err.print( "Will create checksum files for all published files (sha1, md5).\n" );
440 }
441 else
442 {
443 System.err.print( "Will not create checksum files.\n" );
444 }
445
446 if ( createIncrementalChunks )
447 {
448 System.err.print( "Will create incremental chunks for changes, along with baseline file.\n" );
449 }
450 else
451 {
452 System.err.print( "Will create baseline file.\n" );
453 }
454 }
455
456 long tstart = System.currentTimeMillis();
457
458 IndexingContext context = components.indexer.addIndexingContext(
459 repositoryName,
460 repositoryName,
461 repositoryFolder,
462 indexFolder,
463 null,
464 null,
465 indexers );
466
467 try
468 {
469 ArtifactScanningListener listener = new IndexerListener( context, debug, quiet );
470
471 components.indexer.scan( context, listener, true );
472
473 IndexSearcher indexSearcher = context.acquireIndexSearcher();
474
475 try
476 {
477 IndexPackingRequest request =
478 new IndexPackingRequest( context, indexSearcher.getIndexReader(), outputFolder );
479
480 request.setCreateChecksumFiles( createChecksums );
481
482 request.setCreateIncrementalChunks( createIncrementalChunks );
483
484 request.setFormats( List.of( IndexFormat.FORMAT_V1 ) );
485
486 if ( chunkCount != null )
487 {
488 request.setMaxIndexChunks( chunkCount );
489 }
490
491 packIndex( components.packer, request, debug, quiet );
492 }
493 finally
494 {
495 context.releaseIndexSearcher( indexSearcher );
496 }
497
498 if ( !quiet )
499 {
500 printStats( tstart );
501 }
502 }
503 finally
504 {
505 components.indexer.removeIndexingContext( context, false );
506 }
507 return 0;
508 }
509
510 private int unpack( CommandLine cli, Components components )
511 throws IOException
512 {
513 final String indexDirectoryName = cli.getOptionValue( INDEX, "." );
514 final File indexFolder = new File( indexDirectoryName ).getCanonicalFile();
515 final File indexArchive = new File( indexFolder, IndexingContext.INDEX_FILE_PREFIX + ".gz" );
516
517 final String outputDirectoryName = cli.getOptionValue( TARGET_DIR, "." );
518 final File outputFolder = new File( outputDirectoryName ).getCanonicalFile();
519
520 final boolean quiet = cli.hasOption( QUIET );
521 if ( !quiet )
522 {
523 System.err.printf( "Index Folder: %s\n", indexFolder.getAbsolutePath() );
524 System.err.printf( "Output Folder: %s\n", outputFolder.getAbsolutePath() );
525 }
526
527 long tstart = System.currentTimeMillis();
528
529 final List<IndexCreator> indexers = getIndexers( cli, components );
530
531
532 try ( BufferedInputStream is = new BufferedInputStream( new FileInputStream( indexArchive ) );
533 FSDirectory directory = FSDirectory.open( outputFolder.toPath() ) )
534 {
535 DefaultIndexUpdater.unpackIndexData( is, 4, directory, (IndexingContext) Proxy.newProxyInstance(
536 getClass().getClassLoader(), new Class[] {IndexingContext.class}, new PartialImplementation()
537 {
538 public List<IndexCreator> getIndexCreators()
539 {
540 return indexers;
541 }
542 } )
543
544 );
545 }
546
547 if ( !quiet )
548 {
549 printStats( tstart );
550 }
551 return 0;
552 }
553
554 private List<IndexCreator> getIndexers( final CommandLine cli, Components components )
555 {
556 String type = "default";
557
558 if ( cli.hasOption( TYPE ) )
559 {
560 type = cli.getOptionValue( TYPE );
561 }
562
563 List<IndexCreator> indexers = new ArrayList<>();
564
565 if ( "default".equals( type ) )
566 {
567 indexers.add( requireNonNull( components.allIndexCreators.get( "min" ) ) );
568 indexers.add( requireNonNull( components.allIndexCreators.get( "jarContent" ) ) );
569 }
570 else if ( "full".equals( type ) )
571 {
572 indexers.addAll( components.allIndexCreators.values() );
573 }
574 else
575 {
576 for ( String name : type.split( "," ) )
577 {
578 indexers.add( requireNonNull( components.allIndexCreators.get( name ) ) );
579 }
580 }
581 return indexers;
582 }
583
584 private void packIndex( IndexPacker packer, IndexPackingRequest request, boolean debug, boolean quiet )
585 {
586 try
587 {
588 packer.packIndex( request );
589 }
590 catch ( IOException e )
591 {
592 if ( !quiet )
593 {
594 System.err.printf( "Cannot zip index: %s\n", e.getMessage() );
595
596 if ( debug )
597 {
598 e.printStackTrace();
599 }
600 }
601 }
602 }
603
604 private void printStats( final long startTimeInMillis )
605 {
606 long t = System.currentTimeMillis() - startTimeInMillis;
607
608 long s = TimeUnit.MILLISECONDS.toSeconds( t );
609 if ( t > TimeUnit.MINUTES.toMillis( 1 ) )
610 {
611 long m = TimeUnit.MILLISECONDS.toMinutes( t );
612
613 System.err.printf( "Total time: %d min %d sec\n", m, s - ( m * 60 ) );
614 }
615 else
616 {
617 System.err.printf( "Total time: %d sec\n", s );
618 }
619
620 Runtime r = Runtime.getRuntime();
621
622 System.err.printf( "Final memory: %dM/%dM\n",
623 ( r.totalMemory() - r.freeMemory() ) / MB, r.totalMemory() / MB );
624 }
625
626
627
628
629 private static final class IndexerListener
630 implements ArtifactScanningListener
631 {
632 private final IndexingContext context;
633
634 private final boolean debug;
635
636 private final boolean quiet;
637
638 private long ts = System.currentTimeMillis();
639
640 private int count;
641
642 IndexerListener( IndexingContext context, boolean debug, boolean quiet )
643 {
644 this.context = context;
645 this.debug = debug;
646 this.quiet = quiet;
647 }
648
649 @Override
650 public void scanningStarted( IndexingContext context )
651 {
652 if ( !quiet )
653 {
654 System.err.println( "Scanning started" );
655 }
656 }
657
658 @Override
659 public void artifactDiscovered( ArtifactContext ac )
660 {
661 count++;
662
663 long t = System.currentTimeMillis();
664
665 ArtifactInfo ai = ac.getArtifactInfo();
666
667 if ( !quiet && debug && "maven-plugin".equals( ai.getPackaging() ) )
668 {
669 System.err.printf( "Plugin: %s:%s:%s - %s %s\n",
670 ai.getGroupId(), ai.getArtifactId(), ai.getVersion(), ai.getPrefix(), "" + ai.getGoals() );
671 }
672
673 if ( !quiet && ( debug || ( t - ts ) > 2000L ) )
674 {
675 System.err.printf( " %6d %s\n", count, formatFile( ac.getPom() ) );
676 ts = t;
677 }
678 }
679
680 @Override
681 public void artifactError( ArtifactContext ac, Exception e )
682 {
683 if ( !quiet )
684 {
685 System.err.printf( "! %6d %s - %s\n", count, formatFile( ac.getPom() ), e.getMessage() );
686
687 System.err.printf( " %s\n", formatFile( ac.getArtifact() ) );
688
689 if ( debug )
690 {
691 e.printStackTrace();
692 }
693 }
694
695 ts = System.currentTimeMillis();
696 }
697
698 private String formatFile( File file )
699 {
700 return file.getAbsolutePath().substring( context.getRepository().getAbsolutePath().length() + 1 );
701 }
702
703 @Override
704 public void scanningFinished( IndexingContext context, ScanningResult result )
705 {
706 if ( !quiet )
707 {
708 if ( result.hasExceptions() )
709 {
710 System.err.printf( "Scanning errors: %s\n", result.getExceptions().size() );
711 }
712
713 System.err.printf( "Artifacts added: %s\n", result.getTotalFiles() );
714 System.err.printf( "Artifacts deleted: %s\n", result.getDeletedFiles() );
715 }
716 }
717 }
718
719 }