1 package org.apache.maven.index.cli;
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22 import org.apache.commons.cli.CommandLine;
23 import org.apache.commons.cli.HelpFormatter;
24 import org.apache.commons.cli.OptionBuilder;
25 import org.apache.commons.cli.Options;
26 import org.apache.commons.cli.ParseException;
27 import org.apache.lucene.search.IndexSearcher;
28 import org.apache.lucene.store.FSDirectory;
29 import org.apache.maven.index.ArtifactContext;
30 import org.apache.maven.index.ArtifactInfo;
31 import org.apache.maven.index.ArtifactScanningListener;
32 import org.apache.maven.index.NexusIndexer;
33 import org.apache.maven.index.ScanningResult;
34 import org.apache.maven.index.context.IndexCreator;
35 import org.apache.maven.index.context.IndexingContext;
36 import org.apache.maven.index.context.UnsupportedExistingLuceneIndexException;
37 import org.apache.maven.index.packer.IndexPacker;
38 import org.apache.maven.index.packer.IndexPackingRequest;
39 import org.apache.maven.index.packer.IndexPackingRequest.IndexFormat;
40 import org.apache.maven.index.updater.DefaultIndexUpdater;
41 import org.codehaus.plexus.DefaultContainerConfiguration;
42 import org.codehaus.plexus.DefaultPlexusContainer;
43 import org.codehaus.plexus.PlexusConstants;
44 import org.codehaus.plexus.PlexusContainer;
45 import org.codehaus.plexus.classworlds.ClassWorld;
46 import org.codehaus.plexus.component.repository.exception.ComponentLookupException;
47 import org.codehaus.plexus.logging.Logger;
48 import org.codehaus.plexus.logging.LoggerManager;
49 import org.codehaus.plexus.tools.cli.AbstractCli;
50
51 import java.io.BufferedInputStream;
52 import java.io.File;
53 import java.io.FileInputStream;
54 import java.io.IOException;
55 import java.lang.reflect.Proxy;
56 import java.util.ArrayList;
57 import java.util.Arrays;
58 import java.util.List;
59 import java.util.concurrent.TimeUnit;
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82 public class NexusIndexerCli
83 extends AbstractCli
84 {
85
86
87 public static final char REPO = 'r';
88
89 public static final char INDEX = 'i';
90
91 public static final char NAME = 'n';
92
93 public static final char TYPE = 't';
94
95 public static final char TARGET_DIR = 'd';
96
97 public static final char CREATE_INCREMENTAL_CHUNKS = 'c';
98
99 public static final char CREATE_FILE_CHECKSUMS = 's';
100
101 public static final char INCREMENTAL_CHUNK_KEEP_COUNT = 'k';
102
103 public static final char UNPACK = 'u';
104
105 private static final long MB = 1024 * 1024;
106
107 private Options options;
108
109 private int status = 0;
110
111 public static void main( String[] args )
112 throws Exception
113 {
114 NexusIndexerCli cli = new NexusIndexerCli();
115
116 cli.execute( args );
117
118 System.exit( cli.status );
119 }
120
121 @Override
122 public int execute( String[] arg0, ClassWorld arg1 )
123 {
124 int value = super.execute( arg0, arg1 );
125
126 if ( status == 0 )
127 {
128 status = value;
129 }
130
131 return status;
132 }
133
134 @Override
135 public int execute( String[] args )
136 {
137 int value = super.execute( args );
138
139 if ( status == 0 )
140 {
141 status = value;
142 }
143
144 return status;
145 }
146
147 @Override
148 protected void showError( String message, Exception e, boolean show )
149 {
150 status = 1;
151 super.showError( message, e, show );
152 }
153
154 @Override
155 protected int showFatalError( String message, Exception e, boolean show )
156 {
157 status = 1;
158 return super.showFatalError( message, e, show );
159 }
160
161 @Override
162 public CommandLine parse( String[] args )
163 throws ParseException
164 {
165 try
166 {
167 return super.parse( args );
168 }
169 catch ( ParseException e )
170 {
171 status = 1;
172 throw e;
173 }
174 }
175
176 @Override
177 public String getPomPropertiesPath()
178 {
179 return "META-INF/maven/org.sonatype.nexus/nexus-indexer/pom.properties";
180 }
181
182 @Override
183 @SuppressWarnings( "static-access" )
184 public Options buildCliOptions( Options options )
185 {
186 this.options = options;
187
188 options.addOption( OptionBuilder.withLongOpt( "index" ).hasArg()
189 .withDescription( "Path to the index folder." ).create( INDEX ) );
190
191 options.addOption( OptionBuilder.withLongOpt( "destination" ).hasArg()
192 .withDescription( "Target folder." ).create( TARGET_DIR ) );
193
194 options.addOption( OptionBuilder.withLongOpt( "repository" ).hasArg()
195 .withDescription( "Path to the Maven repository." ).create( REPO ) );
196
197 options.addOption( OptionBuilder.withLongOpt( "name" ).hasArg()
198 .withDescription( "Repository name." ).create( NAME ) );
199
200 options.addOption( OptionBuilder.withLongOpt( "chunks" )
201 .withDescription( "Create incremental chunks." ).create( CREATE_INCREMENTAL_CHUNKS ) );
202
203 options.addOption( OptionBuilder.withLongOpt( "keep" ).hasArg().withDescription(
204 "Number of incremental chunks to keep." ).create( INCREMENTAL_CHUNK_KEEP_COUNT ) );
205
206 options.addOption( OptionBuilder.withLongOpt( "checksums" )
207 .withDescription( "Create checksums for all files (sha1, md5)." ).create( CREATE_FILE_CHECKSUMS ) );
208
209 options.addOption( OptionBuilder.withLongOpt( "type" ).hasArg()
210 .withDescription( "Indexer type (default, min, full or comma separated list of custom types)." )
211 .create( TYPE ) );
212
213 options.addOption( OptionBuilder.withLongOpt( "unpack" )
214 .withDescription( "Unpack an index file" ).create( UNPACK ) );
215
216 return options;
217 }
218
219 @Override
220 public void displayHelp()
221 {
222 System.out.println();
223
224 HelpFormatter formatter = new HelpFormatter();
225
226 formatter.printHelp( "nexus-indexer [options]", "\nOptions:", options, "\n" );
227 }
228
229 public void displayHelp( String message )
230 {
231 System.out.println();
232
233 System.out.println( message );
234
235 System.out.println();
236
237 displayHelp();
238 }
239
240 @Override
241 public void invokePlexusComponent( final CommandLine cli, PlexusContainer plexus )
242 throws Exception
243 {
244 final DefaultContainerConfiguration configuration = new DefaultContainerConfiguration();
245 configuration.setClassWorld( ( (DefaultPlexusContainer) plexus ).getClassWorld() );
246 configuration.setClassPathScanning( PlexusConstants.SCANNING_INDEX );
247
248
249
250 plexus = new DefaultPlexusContainer( configuration );
251
252 if ( cli.hasOption( QUIET ) )
253 {
254 setLogLevel( plexus, Logger.LEVEL_DISABLED );
255 }
256 else if ( cli.hasOption( DEBUG ) )
257 {
258 setLogLevel( plexus, Logger.LEVEL_DEBUG );
259 }
260 else if ( cli.hasOption( ERRORS ) )
261 {
262 setLogLevel( plexus, Logger.LEVEL_ERROR );
263 }
264
265 if ( cli.hasOption( UNPACK ) )
266 {
267 unpack( cli, plexus );
268 }
269 else if ( cli.hasOption( INDEX ) && cli.hasOption( REPO ) )
270 {
271 index( cli, plexus );
272 }
273 else
274 {
275 status = 1;
276
277 displayHelp( "Use either unpack (\"" + UNPACK + "\") or index (\"" + INDEX + "\" and \"" + REPO
278 + "\") options, but none has been found!" );
279 }
280 }
281
282 private void setLogLevel( PlexusContainer plexus, int logLevel )
283 throws ComponentLookupException
284 {
285 plexus.lookup( LoggerManager.class ).setThresholds( logLevel );
286 }
287
288 private void index( final CommandLine cli, PlexusContainer plexus )
289 throws ComponentLookupException, IOException, UnsupportedExistingLuceneIndexException
290 {
291 String indexDirectoryName = cli.getOptionValue( INDEX );
292
293 File indexFolder = new File( indexDirectoryName );
294
295 String outputDirectoryName = cli.getOptionValue( TARGET_DIR, "." );
296
297 File outputFolder = new File( outputDirectoryName );
298
299 File repositoryFolder = new File( cli.getOptionValue( REPO ) );
300
301 String repositoryName = cli.getOptionValue( NAME, indexFolder.getName() );
302
303 List<IndexCreator> indexers = getIndexers( cli, plexus );
304
305 boolean createChecksums = cli.hasOption( CREATE_FILE_CHECKSUMS );
306
307 boolean createIncrementalChunks = cli.hasOption( CREATE_INCREMENTAL_CHUNKS );
308
309 boolean debug = cli.hasOption( DEBUG );
310
311 boolean quiet = cli.hasOption( QUIET );
312
313 Integer chunkCount = cli.hasOption( INCREMENTAL_CHUNK_KEEP_COUNT )
314 ? Integer.parseInt( cli.getOptionValue( INCREMENTAL_CHUNK_KEEP_COUNT ) )
315 : null;
316
317 if ( !quiet )
318 {
319 System.err.printf( "Repository Folder: %s\n", repositoryFolder.getAbsolutePath() );
320 System.err.printf( "Index Folder: %s\n", indexFolder.getAbsolutePath() );
321 System.err.printf( "Output Folder: %s\n", outputFolder.getAbsolutePath() );
322 System.err.printf( "Repository name: %s\n", repositoryName );
323 System.err.printf( "Indexers: %s\n", indexers.toString() );
324
325 if ( createChecksums )
326 {
327 System.err.printf( "Will create checksum files for all published files (sha1, md5).\n" );
328 }
329 else
330 {
331 System.err.printf( "Will not create checksum files.\n" );
332 }
333
334 if ( createIncrementalChunks )
335 {
336 System.err.printf( "Will create incremental chunks for changes, along with baseline file.\n" );
337 }
338 else
339 {
340 System.err.printf( "Will create baseline file.\n" );
341 }
342 }
343
344 NexusIndexer indexer = plexus.lookup( NexusIndexer.class );
345
346 long tstart = System.currentTimeMillis();
347
348 IndexingContext context = indexer.addIndexingContext(
349 repositoryName,
350 repositoryName,
351 repositoryFolder,
352 indexFolder,
353 null,
354 null,
355 indexers );
356
357 try
358 {
359 IndexPacker packer = plexus.lookup( IndexPacker.class );
360
361 ArtifactScanningListener listener = new IndexerListener( context, debug, quiet );
362
363 indexer.scan( context, listener, true );
364
365 IndexSearcher indexSearcher = context.acquireIndexSearcher();
366
367 try
368 {
369 IndexPackingRequest request =
370 new IndexPackingRequest( context, indexSearcher.getIndexReader(), outputFolder );
371
372 request.setCreateChecksumFiles( createChecksums );
373
374 request.setCreateIncrementalChunks( createIncrementalChunks );
375
376 request.setFormats( Arrays.asList( IndexFormat.FORMAT_V1 ) );
377
378 if ( chunkCount != null )
379 {
380 request.setMaxIndexChunks( chunkCount.intValue() );
381 }
382
383 packIndex( packer, request, debug, quiet );
384 }
385 finally
386 {
387 context.releaseIndexSearcher( indexSearcher );
388 }
389
390 if ( !quiet )
391 {
392 printStats( tstart );
393 }
394 }
395 finally
396 {
397 indexer.removeIndexingContext( context, false );
398 }
399 }
400
401 private void unpack( CommandLine cli, PlexusContainer plexus )
402 throws ComponentLookupException, IOException
403 {
404 final String indexDirectoryName = cli.getOptionValue( INDEX, "." );
405 final File indexFolder = new File( indexDirectoryName ).getCanonicalFile();
406 final File indexArchive = new File( indexFolder, IndexingContext.INDEX_FILE_PREFIX + ".gz" );
407
408 final String outputDirectoryName = cli.getOptionValue( TARGET_DIR, "." );
409 final File outputFolder = new File( outputDirectoryName ).getCanonicalFile();
410
411 final boolean quiet = cli.hasOption( QUIET );
412 if ( !quiet )
413 {
414 System.err.printf( "Index Folder: %s\n", indexFolder.getAbsolutePath() );
415 System.err.printf( "Output Folder: %s\n", outputFolder.getAbsolutePath() );
416 }
417
418 long tstart = System.currentTimeMillis();
419
420 final List<IndexCreator> indexers = getIndexers( cli, plexus );
421
422 try ( BufferedInputStream is = new BufferedInputStream( new FileInputStream( indexArchive ) );
423 FSDirectory directory = FSDirectory.open( outputFolder.toPath() ) )
424 {
425 DefaultIndexUpdater.unpackIndexData( is, directory, (IndexingContext) Proxy.newProxyInstance(
426 getClass().getClassLoader(), new Class[] { IndexingContext.class }, new PartialImplementation()
427 {
428 public List<IndexCreator> getIndexCreators()
429 {
430 return indexers;
431 }
432 } )
433
434 );
435 }
436
437 if ( !quiet )
438 {
439 printStats( tstart );
440 }
441 }
442
443 private List<IndexCreator> getIndexers( final CommandLine cli, PlexusContainer plexus )
444 throws ComponentLookupException
445 {
446 String type = "default";
447
448 if ( cli.hasOption( TYPE ) )
449 {
450 type = cli.getOptionValue( TYPE );
451 }
452
453 List<IndexCreator> indexers = new ArrayList<IndexCreator>();
454
455 if ( "default".equals( type ) )
456 {
457 indexers.add( plexus.lookup( IndexCreator.class, "min" ) );
458 indexers.add( plexus.lookup( IndexCreator.class, "jarContent" ) );
459 }
460 else if ( "full".equals( type ) )
461 {
462 for ( Object component : plexus.lookupList( IndexCreator.class ) )
463 {
464 indexers.add( (IndexCreator) component );
465 }
466 }
467 else
468 {
469 for ( String hint : type.split( "," ) )
470 {
471 indexers.add( plexus.lookup( IndexCreator.class, hint ) );
472 }
473 }
474 return indexers;
475 }
476
477 private void packIndex( IndexPacker packer, IndexPackingRequest request, boolean debug, boolean quiet )
478 {
479 try
480 {
481 packer.packIndex( request );
482 }
483 catch ( IOException e )
484 {
485 if ( !quiet )
486 {
487 System.err.printf( "Cannot zip index; \n", e.getMessage() );
488
489 if ( debug )
490 {
491 e.printStackTrace();
492 }
493 }
494 }
495 }
496
497 private void printStats( final long startTimeInMillis )
498 {
499 long t = System.currentTimeMillis() - startTimeInMillis;
500
501 long s = TimeUnit.MILLISECONDS.toSeconds( t );
502 if ( t > TimeUnit.MINUTES.toMillis( 1 ) )
503 {
504 long m = TimeUnit.MILLISECONDS.toMinutes( t );
505
506 System.err.printf( "Total time: %d min %d sec\n", m, s - ( m * 60 ) );
507 }
508 else
509 {
510 System.err.printf( "Total time: %d sec\n", s );
511 }
512
513 Runtime r = Runtime.getRuntime();
514
515 System.err.printf( "Final memory: %dM/%dM\n",
516 ( r.totalMemory() - r.freeMemory() ) / MB, r.totalMemory() / MB );
517 }
518
519
520
521
522 private static final class IndexerListener
523 implements ArtifactScanningListener
524 {
525 private final IndexingContext context;
526
527 private final boolean debug;
528
529 private boolean quiet;
530
531 private long ts = System.currentTimeMillis();
532
533 private int count;
534
535 IndexerListener( IndexingContext context, boolean debug, boolean quiet )
536 {
537 this.context = context;
538 this.debug = debug;
539 this.quiet = quiet;
540 }
541
542 public void scanningStarted( IndexingContext context )
543 {
544 if ( !quiet )
545 {
546 System.err.println( "Scanning started" );
547 }
548 }
549
550 public void artifactDiscovered( ArtifactContext ac )
551 {
552 count++;
553
554 long t = System.currentTimeMillis();
555
556 ArtifactInfo ai = ac.getArtifactInfo();
557
558 if ( !quiet && debug && "maven-plugin".equals( ai.getPackaging() ) )
559 {
560 System.err.printf( "Plugin: %s:%s:%s - %s %s\n",
561 ai.getGroupId(), ai.getArtifactId(), ai.getVersion(), ai.getPrefix(), "" + ai.getGoals() );
562 }
563
564 if ( !quiet && ( debug || ( t - ts ) > 2000L ) )
565 {
566 System.err.printf( " %6d %s\n", count, formatFile( ac.getPom() ) );
567 ts = t;
568 }
569 }
570
571 public void artifactError( ArtifactContext ac, Exception e )
572 {
573 if ( !quiet )
574 {
575 System.err.printf( "! %6d %s - %s\n", count, formatFile( ac.getPom() ), e.getMessage() );
576
577 System.err.printf( " %s\n", formatFile( ac.getArtifact() ) );
578
579 if ( debug )
580 {
581 e.printStackTrace();
582 }
583 }
584
585 ts = System.currentTimeMillis();
586 }
587
588 private String formatFile( File file )
589 {
590 return file.getAbsolutePath().substring( context.getRepository().getAbsolutePath().length() + 1 );
591 }
592
593 public void scanningFinished( IndexingContext context, ScanningResult result )
594 {
595 if ( !quiet )
596 {
597 if ( result.hasExceptions() )
598 {
599 System.err.printf( "Scanning errors: %s\n", result.getExceptions().size() );
600 }
601
602 System.err.printf( "Artifacts added: %s\n", result.getTotalFiles() );
603 System.err.printf( "Artifacts deleted: %s\n", result.getDeletedFiles() );
604 }
605 }
606 }
607
608 }