1 package org.apache.maven.index.updater;
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22 import javax.inject.Inject;
23 import javax.inject.Named;
24 import javax.inject.Singleton;
25 import java.io.BufferedInputStream;
26 import java.io.BufferedOutputStream;
27 import java.io.BufferedReader;
28 import java.io.File;
29 import java.io.FileInputStream;
30 import java.io.FileNotFoundException;
31 import java.io.FileOutputStream;
32 import java.io.IOException;
33 import java.io.InputStream;
34 import java.io.InputStreamReader;
35 import java.io.OutputStream;
36 import java.io.OutputStreamWriter;
37 import java.io.Writer;
38 import java.text.ParseException;
39 import java.text.SimpleDateFormat;
40 import java.util.ArrayList;
41 import java.util.Date;
42 import java.util.List;
43 import java.util.Properties;
44 import java.util.Set;
45 import java.util.TimeZone;
46
47 import org.apache.lucene.document.Document;
48 import org.apache.lucene.index.DirectoryReader;
49 import org.apache.lucene.index.IndexReader;
50 import org.apache.lucene.index.IndexWriter;
51 import org.apache.lucene.index.MultiFields;
52 import org.apache.lucene.store.Directory;
53 import org.apache.lucene.util.Bits;
54 import org.apache.maven.index.context.DocumentFilter;
55 import org.apache.maven.index.context.IndexUtils;
56 import org.apache.maven.index.context.IndexingContext;
57 import org.apache.maven.index.context.NexusAnalyzer;
58 import org.apache.maven.index.context.NexusIndexWriter;
59 import org.apache.maven.index.fs.Lock;
60 import org.apache.maven.index.fs.Locker;
61 import org.apache.maven.index.incremental.IncrementalHandler;
62 import org.apache.maven.index.updater.IndexDataReader.IndexDataReadResult;
63 import org.codehaus.plexus.util.FileUtils;
64 import org.codehaus.plexus.util.io.RawInputStreamFacade;
65 import org.slf4j.Logger;
66 import org.slf4j.LoggerFactory;
67
68
69
70
71
72
73
74 @Singleton
75 @Named
76 public class DefaultIndexUpdater
77 implements IndexUpdater
78 {
79
80 private final Logger logger = LoggerFactory.getLogger( getClass() );
81
82 protected Logger getLogger()
83 {
84 return logger;
85 }
86
87 private final IncrementalHandler incrementalHandler;
88
89 private final List<IndexUpdateSideEffect> sideEffects;
90
91
92 @Inject
93 public DefaultIndexUpdater( final IncrementalHandler incrementalHandler,
94 final List<IndexUpdateSideEffect> sideEffects )
95 {
96 this.incrementalHandler = incrementalHandler;
97 this.sideEffects = sideEffects;
98 }
99
100 public IndexUpdateResult fetchAndUpdateIndex( final IndexUpdateRequest updateRequest )
101 throws IOException
102 {
103 IndexUpdateResult result = new IndexUpdateResult();
104
105 IndexingContext context = updateRequest.getIndexingContext();
106
107 ResourceFetcher fetcher = null;
108
109 if ( !updateRequest.isOffline() )
110 {
111 fetcher = updateRequest.getResourceFetcher();
112
113
114
115 if ( fetcher == null )
116 {
117 throw new IOException( "Update of the index without provided ResourceFetcher is impossible." );
118 }
119
120 fetcher.connect( context.getId(), context.getIndexUpdateUrl() );
121 }
122
123 File cacheDir = updateRequest.getLocalIndexCacheDir();
124 Locker locker = updateRequest.getLocker();
125 Lock lock = locker != null && cacheDir != null ? locker.lock( cacheDir ) : null;
126 try
127 {
128 if ( cacheDir != null )
129 {
130 LocalCacheIndexAdaptor cache = new LocalCacheIndexAdaptor( cacheDir, result );
131
132 if ( !updateRequest.isOffline() )
133 {
134 cacheDir.mkdirs();
135
136 try
137 {
138 if ( fetchAndUpdateIndex( updateRequest, fetcher, cache ).isSuccessful() )
139 {
140 cache.commit();
141 }
142 }
143 finally
144 {
145 fetcher.disconnect();
146 }
147 }
148
149 fetcher = cache.getFetcher();
150 }
151 else if ( updateRequest.isOffline() )
152 {
153 throw new IllegalArgumentException( "LocalIndexCacheDir can not be null in offline mode" );
154 }
155
156 try
157 {
158 if ( !updateRequest.isCacheOnly() )
159 {
160 LuceneIndexAdaptor target = new LuceneIndexAdaptor( updateRequest );
161 result = fetchAndUpdateIndex( updateRequest, fetcher, target );
162
163 if ( result.isSuccessful() )
164 {
165 target.commit();
166 }
167 }
168 }
169 finally
170 {
171 fetcher.disconnect();
172 }
173 }
174 finally
175 {
176 if ( lock != null )
177 {
178 lock.release();
179 }
180 }
181
182 return result;
183 }
184
185 private Date loadIndexDirectory( final IndexUpdateRequest updateRequest, final ResourceFetcher fetcher,
186 final boolean merge, final String remoteIndexFile )
187 throws IOException
188 {
189 File indexDir = File.createTempFile( remoteIndexFile, ".dir" );
190 indexDir.delete();
191 indexDir.mkdirs();
192
193 try ( BufferedInputStream is = new BufferedInputStream( fetcher.retrieve( remoteIndexFile ) );
194 Directory directory = updateRequest.getFSDirectoryFactory().open( indexDir ) )
195 {
196 Date timestamp = null;
197
198 Set<String> rootGroups = null;
199 Set<String> allGroups = null;
200 if ( remoteIndexFile.endsWith( ".gz" ) )
201 {
202 IndexDataReadResult result = unpackIndexData( is, directory, updateRequest.getIndexingContext() );
203 timestamp = result.getTimestamp();
204 rootGroups = result.getRootGroups();
205 allGroups = result.getAllGroups();
206 }
207 else
208 {
209
210 throw new IllegalArgumentException( "The legacy format is no longer supported "
211 + "by this version of maven-indexer." );
212 }
213
214 if ( updateRequest.getDocumentFilter() != null )
215 {
216 filterDirectory( directory, updateRequest.getDocumentFilter() );
217 }
218
219 if ( merge )
220 {
221 updateRequest.getIndexingContext().merge( directory );
222 }
223 else
224 {
225 updateRequest.getIndexingContext().replace( directory, rootGroups, allGroups );
226 }
227 if ( sideEffects != null && sideEffects.size() > 0 )
228 {
229 getLogger().info( IndexUpdateSideEffect.class.getName() + " extensions found: " + sideEffects.size() );
230 for ( IndexUpdateSideEffect sideeffect : sideEffects )
231 {
232 sideeffect.updateIndex( directory, updateRequest.getIndexingContext(), merge );
233 }
234 }
235
236 return timestamp;
237 }
238 finally
239 {
240 try
241 {
242 FileUtils.deleteDirectory( indexDir );
243 }
244 catch ( IOException ex )
245 {
246
247 }
248 }
249 }
250
251 private static void filterDirectory( final Directory directory, final DocumentFilter filter )
252 throws IOException
253 {
254 IndexReader r = null;
255 IndexWriter w = null;
256 try
257 {
258 r = DirectoryReader.open( directory );
259 w = new NexusIndexWriter( directory, new NexusAnalyzer(), false );
260
261 Bits liveDocs = MultiFields.getLiveDocs( r );
262
263 int numDocs = r.maxDoc();
264
265 for ( int i = 0; i < numDocs; i++ )
266 {
267 if ( liveDocs != null && !liveDocs.get( i ) )
268 {
269 continue;
270 }
271
272 Document d = r.document( i );
273
274 if ( !filter.accept( d ) )
275 {
276 boolean success = w.tryDeleteDocument( r, i );
277
278 }
279 }
280 w.commit();
281 }
282 finally
283 {
284 IndexUtils.close( r );
285 IndexUtils.close( w );
286 }
287
288 w = null;
289 try
290 {
291
292 w = new NexusIndexWriter( directory, new NexusAnalyzer(), false );
293
294 w.commit();
295 }
296 finally
297 {
298 IndexUtils.close( w );
299 }
300 }
301
302 private Properties loadIndexProperties( final File indexDirectoryFile, final String remoteIndexPropertiesName )
303 {
304 File indexProperties = new File( indexDirectoryFile, remoteIndexPropertiesName );
305
306 try ( FileInputStream fis = new FileInputStream( indexProperties ) )
307 {
308 Properties properties = new Properties();
309
310 properties.load( fis );
311
312 return properties;
313 }
314 catch ( IOException e )
315 {
316 getLogger().debug( "Unable to read remote properties stored locally", e );
317 }
318 return null;
319 }
320
321 private void storeIndexProperties( final File dir, final String indexPropertiesName, final Properties properties )
322 throws IOException
323 {
324 File file = new File( dir, indexPropertiesName );
325
326 if ( properties != null )
327 {
328 try ( OutputStream os = new BufferedOutputStream( new FileOutputStream( file ) ) )
329 {
330 properties.store( os, null );
331 }
332 }
333 else
334 {
335 file.delete();
336 }
337 }
338
339 private Properties downloadIndexProperties( final ResourceFetcher fetcher )
340 throws IOException
341 {
342 try ( InputStream fis = fetcher.retrieve( IndexingContext.INDEX_REMOTE_PROPERTIES_FILE ) )
343 {
344 Properties properties = new Properties();
345
346 properties.load( fis );
347
348 return properties;
349 }
350 }
351
352 public Date getTimestamp( final Properties properties, final String key )
353 {
354 String indexTimestamp = properties.getProperty( key );
355
356 if ( indexTimestamp != null )
357 {
358 try
359 {
360 SimpleDateFormat df = new SimpleDateFormat( IndexingContext.INDEX_TIME_FORMAT );
361 df.setTimeZone( TimeZone.getTimeZone( "GMT" ) );
362 return df.parse( indexTimestamp );
363 }
364 catch ( ParseException ex )
365 {
366 }
367 }
368 return null;
369 }
370
371
372
373
374
375
376
377
378 public static IndexDataReadResult unpackIndexData( final InputStream is, final Directory d,
379 final IndexingContext context )
380 throws IOException
381 {
382 NexusIndexWriter w = new NexusIndexWriter( d, new NexusAnalyzer(), true );
383 try
384 {
385 IndexDataReader dr = new IndexDataReader( is );
386
387 return dr.readIndex( w, context );
388 }
389 finally
390 {
391 IndexUtils.close( w );
392 }
393 }
394
395
396
397
398 public static class FileFetcher
399 implements ResourceFetcher
400 {
401 private final File basedir;
402
403 public FileFetcher( File basedir )
404 {
405 this.basedir = basedir;
406 }
407
408 public void connect( String id, String url )
409 throws IOException
410 {
411
412 }
413
414 public void disconnect()
415 throws IOException
416 {
417
418 }
419
420 public void retrieve( String name, File targetFile )
421 throws IOException, FileNotFoundException
422 {
423 FileUtils.copyFile( getFile( name ), targetFile );
424
425 }
426
427 public InputStream retrieve( String name )
428 throws IOException, FileNotFoundException
429 {
430 return new FileInputStream( getFile( name ) );
431 }
432
433 private File getFile( String name )
434 {
435 return new File( basedir, name );
436 }
437
438 }
439
440 private abstract class IndexAdaptor
441 {
442 protected final File dir;
443
444 protected Properties properties;
445
446 protected IndexAdaptor( File dir )
447 {
448 this.dir = dir;
449 }
450
451 public abstract Properties getProperties();
452
453 public abstract void storeProperties()
454 throws IOException;
455
456 public abstract void addIndexChunk( ResourceFetcher source, String filename )
457 throws IOException;
458
459 public abstract Date setIndexFile( ResourceFetcher source, String string )
460 throws IOException;
461
462 public Properties setProperties( ResourceFetcher source )
463 throws IOException
464 {
465 this.properties = downloadIndexProperties( source );
466 return properties;
467 }
468
469 public abstract Date getTimestamp();
470
471 public void commit()
472 throws IOException
473 {
474 storeProperties();
475 }
476 }
477
478 private class LuceneIndexAdaptor
479 extends IndexAdaptor
480 {
481 private final IndexUpdateRequest updateRequest;
482
483 LuceneIndexAdaptor( IndexUpdateRequest updateRequest )
484 {
485 super( updateRequest.getIndexingContext().getIndexDirectoryFile() );
486 this.updateRequest = updateRequest;
487 }
488
489 public Properties getProperties()
490 {
491 if ( properties == null )
492 {
493 properties = loadIndexProperties( dir, IndexingContext.INDEX_UPDATER_PROPERTIES_FILE );
494 }
495 return properties;
496 }
497
498 public void storeProperties()
499 throws IOException
500 {
501 storeIndexProperties( dir, IndexingContext.INDEX_UPDATER_PROPERTIES_FILE, properties );
502 }
503
504 public Date getTimestamp()
505 {
506 return updateRequest.getIndexingContext().getTimestamp();
507 }
508
509 public void addIndexChunk( ResourceFetcher source, String filename )
510 throws IOException
511 {
512 loadIndexDirectory( updateRequest, source, true, filename );
513 }
514
515 public Date setIndexFile( ResourceFetcher source, String filename )
516 throws IOException
517 {
518 return loadIndexDirectory( updateRequest, source, false, filename );
519 }
520
521 public void commit()
522 throws IOException
523 {
524 super.commit();
525
526 updateRequest.getIndexingContext().commit();
527 }
528
529 }
530
531 private class LocalCacheIndexAdaptor
532 extends IndexAdaptor
533 {
534 private static final String CHUNKS_FILENAME = "chunks.lst";
535
536 private static final String CHUNKS_FILE_ENCODING = "UTF-8";
537
538 private final IndexUpdateResult result;
539
540 private final ArrayList<String> newChunks = new ArrayList<String>();
541
542 LocalCacheIndexAdaptor( File dir, IndexUpdateResult result )
543 {
544 super( dir );
545 this.result = result;
546 }
547
548 public Properties getProperties()
549 {
550 if ( properties == null )
551 {
552 properties = loadIndexProperties( dir, IndexingContext.INDEX_REMOTE_PROPERTIES_FILE );
553 }
554 return properties;
555 }
556
557 public void storeProperties()
558 throws IOException
559 {
560 storeIndexProperties( dir, IndexingContext.INDEX_REMOTE_PROPERTIES_FILE, properties );
561 }
562
563 public Date getTimestamp()
564 {
565 Properties properties = getProperties();
566 if ( properties == null )
567 {
568 return null;
569 }
570
571 Date timestamp = DefaultIndexUpdater.this.getTimestamp( properties, IndexingContext.INDEX_TIMESTAMP );
572
573 if ( timestamp == null )
574 {
575 timestamp = DefaultIndexUpdater.this.getTimestamp( properties, IndexingContext.INDEX_LEGACY_TIMESTAMP );
576 }
577
578 return timestamp;
579 }
580
581 public void addIndexChunk( ResourceFetcher source, String filename )
582 throws IOException
583 {
584 File chunk = new File( dir, filename );
585 FileUtils.copyStreamToFile( new RawInputStreamFacade( source.retrieve( filename ) ), chunk );
586 newChunks.add( filename );
587 }
588
589 public Date setIndexFile( ResourceFetcher source, String filename )
590 throws IOException
591 {
592 cleanCacheDirectory( dir );
593
594 result.setFullUpdate( true );
595
596 File target = new File( dir, filename );
597 FileUtils.copyStreamToFile( new RawInputStreamFacade( source.retrieve( filename ) ), target );
598
599 return null;
600 }
601
602 @Override
603 public void commit()
604 throws IOException
605 {
606 File chunksFile = new File( dir, CHUNKS_FILENAME );
607 try ( BufferedOutputStream os = new BufferedOutputStream( new FileOutputStream( chunksFile, true ) );
608 Writer w = new OutputStreamWriter( os, CHUNKS_FILE_ENCODING ) )
609 {
610 for ( String filename : newChunks )
611 {
612 w.write( filename + "\n" );
613 }
614 w.flush();
615 }
616 super.commit();
617 }
618
619 public List<String> getChunks()
620 throws IOException
621 {
622 ArrayList<String> chunks = new ArrayList<String>();
623
624 File chunksFile = new File( dir, CHUNKS_FILENAME );
625 try ( BufferedReader r =
626 new BufferedReader( new InputStreamReader( new FileInputStream( chunksFile ), CHUNKS_FILE_ENCODING ) ) )
627 {
628 String str;
629 while ( ( str = r.readLine() ) != null )
630 {
631 chunks.add( str );
632 }
633 }
634 return chunks;
635 }
636
637 public ResourceFetcher getFetcher()
638 {
639 return new LocalIndexCacheFetcher( dir )
640 {
641 @Override
642 public List<String> getChunks()
643 throws IOException
644 {
645 return LocalCacheIndexAdaptor.this.getChunks();
646 }
647 };
648 }
649 }
650
651 abstract static class LocalIndexCacheFetcher
652 extends FileFetcher
653 {
654 LocalIndexCacheFetcher( File basedir )
655 {
656 super( basedir );
657 }
658
659 public abstract List<String> getChunks()
660 throws IOException;
661 }
662
663 private IndexUpdateResult fetchAndUpdateIndex( final IndexUpdateRequest updateRequest, ResourceFetcher source,
664 IndexAdaptor target )
665 throws IOException
666 {
667 IndexUpdateResult result = new IndexUpdateResult();
668
669 if ( !updateRequest.isForceFullUpdate() )
670 {
671 Properties localProperties = target.getProperties();
672 Date localTimestamp = null;
673
674 if ( localProperties != null )
675 {
676 localTimestamp = getTimestamp( localProperties, IndexingContext.INDEX_TIMESTAMP );
677 }
678
679
680
681 Properties remoteProperties = target.setProperties( source );
682
683 Date updateTimestamp = getTimestamp( remoteProperties, IndexingContext.INDEX_TIMESTAMP );
684
685
686 if ( updateTimestamp != null )
687 {
688 List<String> filenames =
689 incrementalHandler.loadRemoteIncrementalUpdates( updateRequest, localProperties, remoteProperties );
690
691
692 if ( filenames != null )
693 {
694 for ( String filename : filenames )
695 {
696 target.addIndexChunk( source, filename );
697 }
698
699 result.setTimestamp( updateTimestamp );
700 result.setSuccessful( true );
701 return result;
702 }
703 }
704 else
705 {
706 updateTimestamp = getTimestamp( remoteProperties, IndexingContext.INDEX_LEGACY_TIMESTAMP );
707 }
708
709
710
711
712 if ( localTimestamp != null )
713 {
714
715
716
717 if ( updateTimestamp != null && localTimestamp != null && !updateTimestamp.after( localTimestamp ) )
718 {
719
720 result.setSuccessful( true );
721 return result;
722 }
723 }
724 }
725 else
726 {
727
728 target.setProperties( source );
729 }
730
731 if ( !updateRequest.isIncrementalOnly() )
732 {
733 Date timestamp = null;
734 try
735 {
736 timestamp = target.setIndexFile( source, IndexingContext.INDEX_FILE_PREFIX + ".gz" );
737 if ( source instanceof LocalIndexCacheFetcher )
738 {
739
740
741 for ( String filename : ( (LocalIndexCacheFetcher) source ).getChunks() )
742 {
743 target.addIndexChunk( source, filename );
744 }
745 }
746 }
747 catch ( IOException ex )
748 {
749
750 try
751 {
752 timestamp = target.setIndexFile( source, IndexingContext.INDEX_FILE_PREFIX + ".zip" );
753 }
754 catch ( IOException ex2 )
755 {
756 getLogger().error( "Fallback to *.zip also failed: " + ex2 );
757
758 throw ex;
759 }
760 }
761
762 result.setTimestamp( timestamp );
763 result.setSuccessful( true );
764 result.setFullUpdate( true );
765 }
766
767 return result;
768 }
769
770
771
772
773 protected void cleanCacheDirectory( File dir )
774 throws IOException
775 {
776 File[] members = dir.listFiles();
777 if ( members == null )
778 {
779 return;
780 }
781
782 for ( File member : members )
783 {
784 if ( !Locker.LOCK_FILE.equals( member.getName() ) )
785 {
786 FileUtils.forceDelete( member );
787 }
788 }
789 }
790
791 }