1 package org.apache.maven.index.cli;
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22 import org.apache.commons.cli.CommandLine;
23 import org.apache.commons.cli.HelpFormatter;
24 import org.apache.commons.cli.OptionBuilder;
25 import org.apache.commons.cli.Options;
26 import org.apache.commons.cli.ParseException;
27 import org.apache.lucene.store.FSDirectory;
28 import org.apache.maven.index.ArtifactContext;
29 import org.apache.maven.index.ArtifactInfo;
30 import org.apache.maven.index.ArtifactScanningListener;
31 import org.apache.maven.index.NexusIndexer;
32 import org.apache.maven.index.ScanningResult;
33 import org.apache.maven.index.context.DefaultIndexingContext;
34 import org.apache.maven.index.context.IndexCreator;
35 import org.apache.maven.index.context.IndexingContext;
36 import org.apache.maven.index.context.UnsupportedExistingLuceneIndexException;
37 import org.apache.maven.index.packer.IndexPacker;
38 import org.apache.maven.index.packer.IndexPackingRequest;
39 import org.apache.maven.index.packer.IndexPackingRequest.IndexFormat;
40 import org.apache.maven.index.updater.DefaultIndexUpdater;
41 import org.codehaus.plexus.PlexusContainer;
42 import org.codehaus.plexus.classworlds.ClassWorld;
43 import org.codehaus.plexus.component.repository.exception.ComponentLookupException;
44 import org.codehaus.plexus.logging.Logger;
45 import org.codehaus.plexus.logging.LoggerManager;
46 import org.codehaus.plexus.tools.cli.AbstractCli;
47 import org.codehaus.plexus.util.IOUtil;
48
49 import java.io.BufferedInputStream;
50 import java.io.File;
51 import java.io.FileInputStream;
52 import java.io.IOException;
53 import java.lang.reflect.Proxy;
54 import java.util.ArrayList;
55 import java.util.Arrays;
56 import java.util.List;
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77 public class NexusIndexerCli
78 extends AbstractCli
79 {
80
81
82 public static final char REPO = 'r';
83
84 public static final char INDEX = 'i';
85
86 public static final char NAME = 'n';
87
88 public static final char TYPE = 't';
89
90 public static final char TARGET_DIR = 'd';
91
92 public static final char CREATE_INCREMENTAL_CHUNKS = 'c';
93
94 public static final char CREATE_FILE_CHECKSUMS = 's';
95
96 public static final char INCREMENTAL_CHUNK_KEEP_COUNT = 'k';
97
98 public static final char LEGACY = 'l';
99
100 public static final char UNPACK = 'u';
101
102 private static final long MB = 1024 * 1024;
103
104 private Options options;
105
106 private int status = 0;
107
108 public static void main( String[] args )
109 throws Exception
110 {
111 NexusIndexerCli cli = new NexusIndexerCli();
112
113 cli.execute( args );
114
115 System.exit( cli.status );
116 }
117
118 @Override
119 public int execute( String[] arg0, ClassWorld arg1 )
120 {
121 int value = super.execute( arg0, arg1 );
122
123 if ( status == 0 )
124 {
125 status = value;
126 }
127
128 return status;
129 }
130
131 @Override
132 public int execute( String[] args )
133 {
134 int value = super.execute( args );
135
136 if ( status == 0 )
137 {
138 status = value;
139 }
140
141 return status;
142 }
143
144 @Override
145 protected void showError( String message, Exception e, boolean show )
146 {
147 status = 1;
148 super.showError( message, e, show );
149 }
150
151 @Override
152 protected int showFatalError( String message, Exception e, boolean show )
153 {
154 status = 1;
155 return super.showFatalError( message, e, show );
156 }
157
158 @Override
159 public CommandLine parse( String[] args )
160 throws ParseException
161 {
162 try
163 {
164 return super.parse( args );
165 }
166 catch ( ParseException e )
167 {
168 status = 1;
169 throw e;
170 }
171 }
172
173 @Override
174 public String getPomPropertiesPath()
175 {
176 return "META-INF/maven/org.sonatype.nexus/nexus-indexer/pom.properties";
177 }
178
179 @Override
180 @SuppressWarnings( "static-access" )
181 public Options buildCliOptions( Options options )
182 {
183 this.options = options;
184
185 options.addOption( OptionBuilder.withLongOpt( "index" ).hasArg()
186 .withDescription( "Path to the index folder." ).create( INDEX ) );
187
188 options.addOption( OptionBuilder.withLongOpt( "destination" ).hasArg()
189 .withDescription( "Target folder." ).create( TARGET_DIR ) );
190
191 options.addOption( OptionBuilder.withLongOpt( "repository" ).hasArg()
192 .withDescription( "Path to the Maven repository." ).create( REPO ) );
193
194 options.addOption( OptionBuilder.withLongOpt( "name" ).hasArg()
195 .withDescription( "Repository name." ).create( NAME ) );
196
197 options.addOption( OptionBuilder.withLongOpt( "chunks" )
198 .withDescription( "Create incremental chunks." ).create( CREATE_INCREMENTAL_CHUNKS ) );
199
200 options.addOption( OptionBuilder.withLongOpt( "keep" ).hasArg().withDescription(
201 "Number of incremental chunks to keep." ).create( INCREMENTAL_CHUNK_KEEP_COUNT ) );
202
203 options.addOption( OptionBuilder.withLongOpt( "checksums" )
204 .withDescription( "Create checksums for all files (sha1, md5)." ).create(
205 CREATE_FILE_CHECKSUMS ) );
206
207 options.addOption( OptionBuilder.withLongOpt( "type" ).hasArg()
208 .withDescription(
209 "Indexer type (default, min, full or coma separated list of custom types)." ).create(
210 TYPE ) );
211
212 options.addOption( OptionBuilder.withLongOpt( "legacy" )
213 .withDescription( "Build legacy .zip index file" ).create( LEGACY ) );
214
215 options.addOption( OptionBuilder.withLongOpt( "unpack" )
216 .withDescription( "Unpack an index file" ).create( UNPACK ) );
217
218 return options;
219 }
220
221 @Override
222 public void displayHelp()
223 {
224 System.out.println();
225
226 HelpFormatter formatter = new HelpFormatter();
227
228 formatter.printHelp( "nexus-indexer [options]", "\nOptions:", options, "\n" );
229 }
230
231 public void displayHelp( String message )
232 {
233 System.out.println();
234
235 System.out.println( message );
236
237 System.out.println();
238
239 displayHelp();
240 }
241
242 @Override
243 public void invokePlexusComponent( final CommandLine cli, PlexusContainer plexus )
244 throws Exception
245 {
246 if ( cli.hasOption( QUIET ) )
247 {
248 setLogLevel( plexus, Logger.LEVEL_DISABLED );
249 }
250 else if ( cli.hasOption( DEBUG ) )
251 {
252 setLogLevel( plexus, Logger.LEVEL_DEBUG );
253 }
254 else if ( cli.hasOption( ERRORS ) )
255 {
256 setLogLevel( plexus, Logger.LEVEL_ERROR );
257 }
258
259 if ( cli.hasOption( UNPACK ) )
260 {
261 unpack( cli, plexus );
262 }
263 else if ( cli.hasOption( INDEX ) && cli.hasOption( REPO ) )
264 {
265 index( cli, plexus );
266 }
267 else
268 {
269 status = 1;
270
271 displayHelp( "Use either unpack (\"" + UNPACK + "\") or index (\"" + INDEX + "\" and \"" + REPO
272 + "\") options, but none has been found!" );
273 }
274 }
275
276 private void setLogLevel( PlexusContainer plexus, int logLevel )
277 throws ComponentLookupException
278 {
279 plexus.lookup( LoggerManager.class ).setThresholds( logLevel );
280 }
281
282 private void index( final CommandLine cli, PlexusContainer plexus )
283 throws ComponentLookupException, IOException, UnsupportedExistingLuceneIndexException
284 {
285 String indexDirectoryName = cli.getOptionValue( INDEX );
286
287 File indexFolder = new File( indexDirectoryName );
288
289 String outputDirectoryName = cli.getOptionValue( TARGET_DIR, "." );
290
291 File outputFolder = new File( outputDirectoryName );
292
293 File repositoryFolder = new File( cli.getOptionValue( REPO ) );
294
295 String repositoryName = cli.getOptionValue( NAME, indexFolder.getName() );
296
297 List<IndexCreator> indexers = getIndexers( cli, plexus );
298
299 boolean createChecksums = cli.hasOption( CREATE_FILE_CHECKSUMS );
300
301 boolean createIncrementalChunks = cli.hasOption( CREATE_INCREMENTAL_CHUNKS );
302
303 boolean createLegacyIndex = cli.hasOption( LEGACY );
304
305 boolean debug = cli.hasOption( DEBUG );
306
307 boolean quiet = cli.hasOption( QUIET );
308
309 Integer chunkCount = cli.hasOption( INCREMENTAL_CHUNK_KEEP_COUNT ) ? Integer.parseInt(
310 cli.getOptionValue( INCREMENTAL_CHUNK_KEEP_COUNT ) ) : null;
311
312 if ( !quiet )
313 {
314 System.err.printf( "Repository Folder: %s\n", repositoryFolder.getAbsolutePath() );
315 System.err.printf( "Index Folder: %s\n", indexFolder.getAbsolutePath() );
316 System.err.printf( "Output Folder: %s\n", outputFolder.getAbsolutePath() );
317 System.err.printf( "Repository name: %s\n", repositoryName );
318 System.err.printf( "Indexers: %s\n", indexers.toString() );
319
320 if ( createChecksums )
321 {
322 System.err.printf( "Will create checksum files for all published files (sha1, md5).\n" );
323 }
324 else
325 {
326 System.err.printf( "Will not create checksum files.\n" );
327 }
328
329 if ( createIncrementalChunks )
330 {
331 System.err.printf( "Will create incremental chunks for changes, along with baseline file.\n" );
332 }
333 else
334 {
335 System.err.printf( "Will create baseline file.\n" );
336 }
337
338 if ( createLegacyIndex )
339 {
340 System.err.printf( "Will also create legacy .zip index file.\n" );
341 }
342 }
343
344 NexusIndexer indexer = plexus.lookup( NexusIndexer.class );
345
346 DefaultIndexingContext.BLOCKING_COMMIT = true;
347
348 long tstart = System.currentTimeMillis();
349
350 IndexingContext context = indexer.addIndexingContext(
351 repositoryName,
352 repositoryName,
353 repositoryFolder,
354 indexFolder,
355 null,
356 null,
357 indexers );
358
359 try
360 {
361 IndexPacker packer = plexus.lookup( IndexPacker.class );
362
363 ArtifactScanningListener listener = new IndexerListener( context, debug, quiet );
364
365 indexer.scan( context, listener, true );
366
367 IndexPackingRequest request = new IndexPackingRequest( context, outputFolder );
368
369 request.setCreateChecksumFiles( createChecksums );
370
371 request.setCreateIncrementalChunks( createIncrementalChunks );
372
373 if ( createLegacyIndex )
374 {
375 request.setFormats( Arrays.asList( IndexFormat.FORMAT_LEGACY, IndexFormat.FORMAT_V1 ) );
376 }
377 else
378 {
379 request.setFormats( Arrays.asList( IndexFormat.FORMAT_V1 ) );
380 }
381
382 if ( chunkCount != null )
383 {
384 request.setMaxIndexChunks( chunkCount.intValue() );
385 }
386
387 packIndex( packer, request, debug, quiet );
388
389 if ( !quiet )
390 {
391 printStats( tstart );
392 }
393 }
394 finally
395 {
396 indexer.removeIndexingContext( context, false );
397 }
398 }
399
400 private void unpack( CommandLine cli, PlexusContainer plexus )
401 throws ComponentLookupException, IOException
402 {
403 final String indexDirectoryName = cli.getOptionValue( INDEX, "." );
404 final File indexFolder = new File( indexDirectoryName ).getCanonicalFile();
405 final File indexArchive = new File( indexFolder, IndexingContext.INDEX_FILE_PREFIX + ".gz" );
406
407 final String outputDirectoryName = cli.getOptionValue( TARGET_DIR, "." );
408 final File outputFolder = new File( outputDirectoryName ).getCanonicalFile();
409
410 final boolean quiet = cli.hasOption( QUIET );
411 if ( !quiet )
412 {
413 System.err.printf( "Index Folder: %s\n", indexFolder.getAbsolutePath() );
414 System.err.printf( "Output Folder: %s\n", outputFolder.getAbsolutePath() );
415 }
416
417 long tstart = System.currentTimeMillis();
418
419 final FSDirectory directory = FSDirectory.open( outputFolder );
420
421 final List<IndexCreator> indexers = getIndexers( cli, plexus );
422
423 BufferedInputStream is = null;
424 try
425 {
426 is = new BufferedInputStream( new FileInputStream( indexArchive ) );
427 DefaultIndexUpdater.unpackIndexData( is, directory,
428 (IndexingContext) Proxy.newProxyInstance( getClass().getClassLoader(),
429 new Class[]{
430 IndexingContext.class },
431 new PartialImplementation()
432 {
433 public List<IndexCreator> getIndexCreators()
434 {
435 return indexers;
436 }
437 } )
438
439 );
440 }
441 finally
442 {
443 IOUtil.close( is );
444 if ( directory != null )
445 {
446 directory.close();
447 }
448 }
449
450 if ( !quiet )
451 {
452 printStats( tstart );
453 }
454 }
455
456 private List<IndexCreator> getIndexers( final CommandLine cli, PlexusContainer plexus )
457 throws ComponentLookupException
458 {
459 String type = "default";
460
461 if ( cli.hasOption( TYPE ) )
462 {
463 type = cli.getOptionValue( TYPE );
464 }
465
466 List<IndexCreator> indexers = new ArrayList<IndexCreator>();
467
468 if ( "default".equals( type ) )
469 {
470 indexers.add( plexus.lookup( IndexCreator.class, "min" ) );
471 indexers.add( plexus.lookup( IndexCreator.class, "jarContent" ) );
472 }
473 else if ( "full".equals( type ) )
474 {
475 for ( Object component : plexus.lookupList( IndexCreator.class ) )
476 {
477 indexers.add( (IndexCreator) component );
478 }
479 }
480 else
481 {
482 for ( String hint : type.split( "," ) )
483 {
484 indexers.add( plexus.lookup( IndexCreator.class, hint ) );
485 }
486 }
487 return indexers;
488 }
489
490 private void packIndex( IndexPacker packer, IndexPackingRequest request, boolean debug, boolean quiet )
491 {
492 try
493 {
494 packer.packIndex( request );
495 }
496 catch ( IOException e )
497 {
498 if ( !quiet )
499 {
500 System.err.printf( "Cannot zip index; \n", e.getMessage() );
501
502 if ( debug )
503 {
504 e.printStackTrace();
505 }
506 }
507 }
508 }
509
510 private void printStats( final long startTimeInMillis )
511 {
512 long t = System.currentTimeMillis() - startTimeInMillis;
513
514 long s = t / 1000L;
515 if ( t > 60 * 1000 )
516 {
517 long m = t / 1000L / 60L;
518
519 System.err.printf( "Total time: %d min %d sec\n", m, s - ( m * 60 ) );
520 }
521 else
522 {
523 System.err.printf( "Total time: %d sec\n", s );
524 }
525
526 Runtime r = Runtime.getRuntime();
527
528 System.err.printf( "Final memory: %dM/%dM\n",
529 ( r.totalMemory() - r.freeMemory() ) / MB, r.totalMemory() / MB );
530 }
531
532
533
534
535 private static final class IndexerListener
536 implements ArtifactScanningListener
537 {
538 private final IndexingContext context;
539
540 private final boolean debug;
541
542 private boolean quiet;
543
544 private long ts = System.currentTimeMillis();
545
546 private int count;
547
548 IndexerListener( IndexingContext context, boolean debug, boolean quiet )
549 {
550 this.context = context;
551 this.debug = debug;
552 this.quiet = quiet;
553 }
554
555 public void scanningStarted( IndexingContext context )
556 {
557 if ( !quiet )
558 {
559 System.err.println( "Scanning started" );
560 }
561 }
562
563 public void artifactDiscovered( ArtifactContext ac )
564 {
565 count++;
566
567 long t = System.currentTimeMillis();
568
569 ArtifactInfo ai = ac.getArtifactInfo();
570
571 if ( !quiet && debug && "maven-plugin".equals( ai.packaging ) )
572 {
573 System.err.printf( "Plugin: %s:%s:%s - %s %s\n",
574 ai.groupId, ai.artifactId, ai.version, ai.prefix, "" + ai.goals );
575 }
576
577 if ( !quiet && ( debug || ( t - ts ) > 2000L ) )
578 {
579 System.err.printf( " %6d %s\n", count, formatFile( ac.getPom() ) );
580 ts = t;
581 }
582 }
583
584 public void artifactError( ArtifactContext ac, Exception e )
585 {
586 if ( !quiet )
587 {
588 System.err.printf( "! %6d %s - %s\n", count, formatFile( ac.getPom() ), e.getMessage() );
589
590 System.err.printf( " %s\n", formatFile( ac.getArtifact() ) );
591
592 if ( debug )
593 {
594 e.printStackTrace();
595 }
596 }
597
598 ts = System.currentTimeMillis();
599 }
600
601 private String formatFile( File file )
602 {
603 return file.getAbsolutePath().substring( context.getRepository().getAbsolutePath().length() + 1 );
604 }
605
606 public void scanningFinished( IndexingContext context, ScanningResult result )
607 {
608 if ( !quiet )
609 {
610 if ( result.hasExceptions() )
611 {
612 System.err.printf( "Scanning errors: %s\n", result.getExceptions().size() );
613 }
614
615 System.err.printf( "Artifacts added: %s\n", result.getTotalFiles() );
616 System.err.printf( "Artifacts deleted: %s\n", result.getDeletedFiles() );
617 }
618 }
619 }
620
621 }