View Javadoc

1   package org.apache.maven.index.context;
2   
3   /*
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *   http://www.apache.org/licenses/LICENSE-2.0    
13   *
14   * Unless required by applicable law or agreed to in writing,
15   * software distributed under the License is distributed on an
16   * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
17   * KIND, either express or implied.  See the License for the
18   * specific language governing permissions and limitations
19   * under the License.
20   */
21  
22  import java.io.File;
23  import java.io.IOException;
24  import java.util.Arrays;
25  import java.util.Collection;
26  import java.util.Collections;
27  import java.util.Date;
28  import java.util.LinkedHashSet;
29  import java.util.List;
30  import java.util.Set;
31  import java.util.concurrent.locks.ReadWriteLock;
32  import java.util.concurrent.locks.ReentrantReadWriteLock;
33  
34  import org.apache.lucene.analysis.Analyzer;
35  import org.apache.lucene.document.Document;
36  import org.apache.lucene.document.Field;
37  import org.apache.lucene.index.CorruptIndexException;
38  import org.apache.lucene.index.IndexFileNameFilter;
39  import org.apache.lucene.index.IndexReader;
40  import org.apache.lucene.index.IndexWriter;
41  import org.apache.lucene.index.SerialMergeScheduler;
42  import org.apache.lucene.index.Term;
43  import org.apache.lucene.search.IndexSearcher;
44  import org.apache.lucene.search.TermQuery;
45  import org.apache.lucene.search.TopDocs;
46  import org.apache.lucene.search.TopScoreDocCollector;
47  import org.apache.lucene.store.Directory;
48  import org.apache.lucene.store.FSDirectory;
49  import org.apache.maven.index.ArtifactInfo;
50  import org.apache.maven.index.artifact.GavCalculator;
51  import org.apache.maven.index.artifact.M2GavCalculator;
52  import org.codehaus.plexus.util.StringUtils;
53  
54  /**
55   * The default {@link IndexingContext} implementation.
56   * 
57   * @author Jason van Zyl
58   * @author Tamas Cservenak
59   */
60  public class DefaultIndexingContext
61      extends AbstractIndexingContext
62  {
63      /**
64       * A standard location for indices served up by a webserver.
65       */
66      private static final String INDEX_DIRECTORY = ".index";
67  
68      public static final String FLD_DESCRIPTOR = "DESCRIPTOR";
69  
70      private static final String FLD_DESCRIPTOR_CONTENTS = "NexusIndex";
71  
72      private static final String FLD_IDXINFO = "IDXINFO";
73  
74      private static final String VERSION = "1.0";
75  
76      private static final Term DESCRIPTOR_TERM = new Term( FLD_DESCRIPTOR, FLD_DESCRIPTOR_CONTENTS );
77  
78      private Directory indexDirectory;
79  
80      private File indexDirectoryFile;
81  
82      private String id;
83  
84      private boolean searchable;
85  
86      private String repositoryId;
87  
88      private File repository;
89  
90      private String repositoryUrl;
91  
92      private String indexUpdateUrl;
93  
94      private IndexReader indexReader;
95  
96      private NexusIndexSearcher indexSearcher;
97  
98      // disabled for now, see getReadOnlyIndexSearcher() method for explanation
99      // private NexusIndexSearcher readOnlyIndexSearcher;
100 
101     private NexusIndexWriter indexWriter;
102 
103     private Date timestamp;
104 
105     private List<? extends IndexCreator> indexCreators;
106 
107     /**
108      * Currently nexus-indexer knows only M2 reposes
109      * <p>
110      * XXX move this into a concrete Scanner implementation
111      */
112     private GavCalculator gavCalculator;
113 
114     private ReadWriteLock indexMaintenanceLock = new ReentrantReadWriteLock();
115 
116     private Thread bottleWarmerThread;
117 
118     private DefaultIndexingContext( String id,
119                                     String repositoryId,
120                                     File repository, //
121                                     String repositoryUrl, String indexUpdateUrl,
122                                     List<? extends IndexCreator> indexCreators, Directory indexDirectory,
123                                     boolean reclaimIndex )
124         throws UnsupportedExistingLuceneIndexException, IOException
125     {
126         this.id = id;
127 
128         this.searchable = true;
129 
130         this.repositoryId = repositoryId;
131 
132         this.repository = repository;
133 
134         this.repositoryUrl = repositoryUrl;
135 
136         this.indexUpdateUrl = indexUpdateUrl;
137 
138         this.indexReader = null;
139 
140         this.indexWriter = null;
141 
142         this.indexCreators = indexCreators;
143 
144         this.indexDirectory = indexDirectory;
145 
146         // eh?
147         // Guice does NOT initialize these, and we have to do manually?
148         // While in Plexus, all is well, but when in guice-shim,
149         // these objects are still LazyHintedBeans or what not and IndexerFields are NOT registered!
150         for ( IndexCreator indexCreator : indexCreators )
151         {
152             indexCreator.getIndexerFields();
153         }
154 
155         this.gavCalculator = new M2GavCalculator();
156 
157         prepareIndex( reclaimIndex );
158 
159         installBottleWarmer();
160     }
161 
162     public DefaultIndexingContext( String id, String repositoryId, File repository, File indexDirectoryFile,
163                                    String repositoryUrl, String indexUpdateUrl,
164                                    List<? extends IndexCreator> indexCreators, boolean reclaimIndex )
165         throws IOException, UnsupportedExistingLuceneIndexException
166     {
167         this( id, repositoryId, repository, repositoryUrl, indexUpdateUrl, indexCreators,
168             FSDirectory.open( indexDirectoryFile ), reclaimIndex );
169 
170         this.indexDirectoryFile = indexDirectoryFile;
171     }
172 
173     public DefaultIndexingContext( String id, String repositoryId, File repository, Directory indexDirectory,
174                                    String repositoryUrl, String indexUpdateUrl,
175                                    List<? extends IndexCreator> indexCreators, boolean reclaimIndex )
176         throws IOException, UnsupportedExistingLuceneIndexException
177     {
178         this( id, repositoryId, repository, repositoryUrl, indexUpdateUrl, indexCreators, indexDirectory, reclaimIndex );
179 
180         if ( indexDirectory instanceof FSDirectory )
181         {
182             this.indexDirectoryFile = ( (FSDirectory) indexDirectory ).getFile();
183         }
184     }
185 
186     public void lock()
187     {
188         indexMaintenanceLock.readLock().lock();
189     }
190 
191     public void unlock()
192     {
193         indexMaintenanceLock.readLock().unlock();
194     }
195 
196     public void lockExclusively()
197     {
198         indexMaintenanceLock.writeLock().lock();
199     }
200 
201     public void unlockExclusively()
202     {
203         indexMaintenanceLock.writeLock().unlock();
204     }
205 
206     public Directory getIndexDirectory()
207     {
208         return indexDirectory;
209     }
210 
211     public File getIndexDirectoryFile()
212     {
213         return indexDirectoryFile;
214     }
215 
216     private void prepareIndex( boolean reclaimIndex )
217         throws IOException, UnsupportedExistingLuceneIndexException
218     {
219         if ( IndexReader.indexExists( indexDirectory ) )
220         {
221             try
222             {
223                 // unlock the dir forcibly
224                 if ( IndexWriter.isLocked( indexDirectory ) )
225                 {
226                     IndexWriter.unlock( indexDirectory );
227                 }
228 
229                 openAndWarmup();
230 
231                 checkAndUpdateIndexDescriptor( reclaimIndex );
232             }
233             catch ( IOException e )
234             {
235                 if ( reclaimIndex )
236                 {
237                     prepareCleanIndex( true );
238                 }
239                 else
240                 {
241                     throw e;
242                 }
243             }
244         }
245         else
246         {
247             prepareCleanIndex( false );
248         }
249 
250         timestamp = IndexUtils.getTimestamp( indexDirectory );
251     }
252 
253     private void prepareCleanIndex( boolean deleteExisting )
254         throws IOException
255     {
256         if ( deleteExisting )
257         {
258             closeReaders();
259 
260             // unlock the dir forcibly
261             if ( IndexWriter.isLocked( indexDirectory ) )
262             {
263                 IndexWriter.unlock( indexDirectory );
264             }
265 
266             deleteIndexFiles( true );
267         }
268 
269         openAndWarmup();
270 
271         if ( StringUtils.isEmpty( getRepositoryId() ) )
272         {
273             throw new IllegalArgumentException( "The repositoryId cannot be null when creating new repository!" );
274         }
275 
276         storeDescriptor();
277     }
278 
279     private void checkAndUpdateIndexDescriptor( boolean reclaimIndex )
280         throws IOException, UnsupportedExistingLuceneIndexException
281     {
282         if ( reclaimIndex )
283         {
284             // forcefully "reclaiming" the ownership of the index as ours
285             storeDescriptor();
286             return;
287         }
288 
289         // check for descriptor if this is not a "virgin" index
290         if ( getIndexReader().numDocs() > 0 )
291         {
292             TopScoreDocCollector collector = TopScoreDocCollector.create( 1, false );
293 
294             getIndexSearcher().search( new TermQuery( DESCRIPTOR_TERM ), collector );
295 
296             if ( collector.getTotalHits() == 0 )
297             {
298                 throw new UnsupportedExistingLuceneIndexException( "The existing index has no NexusIndexer descriptor" );
299             }
300 
301             if ( collector.getTotalHits() > 1 )
302             {
303                 // eh? this is buggy index it seems, just iron it out then
304                 storeDescriptor();
305                 return;
306             }
307             else
308             {
309                 // good, we have one descriptor as should
310                 Document descriptor = getIndexSearcher().doc( collector.topDocs().scoreDocs[0].doc );
311                 String[] h = StringUtils.split( descriptor.get( FLD_IDXINFO ), ArtifactInfo.FS );
312                 // String version = h[0];
313                 String repoId = h[1];
314 
315                 // // compare version
316                 // if ( !VERSION.equals( version ) )
317                 // {
318                 // throw new UnsupportedExistingLuceneIndexException(
319                 // "The existing index has version [" + version + "] and not [" + VERSION + "] version!" );
320                 // }
321 
322                 if ( getRepositoryId() == null )
323                 {
324                     repositoryId = repoId;
325                 }
326                 else if ( !getRepositoryId().equals( repoId ) )
327                 {
328                     throw new UnsupportedExistingLuceneIndexException( "The existing index is for repository " //
329                         + "[" + repoId + "] and not for repository [" + getRepositoryId() + "]" );
330                 }
331             }
332         }
333     }
334 
335     private void storeDescriptor()
336         throws IOException
337     {
338         Document hdr = new Document();
339 
340         hdr.add( new Field( FLD_DESCRIPTOR, FLD_DESCRIPTOR_CONTENTS, Field.Store.YES, Field.Index.NOT_ANALYZED ) );
341 
342         hdr.add( new Field( FLD_IDXINFO, VERSION + ArtifactInfo.FS + getRepositoryId(), Field.Store.YES, Field.Index.NO ) );
343 
344         IndexWriter w = getIndexWriter();
345 
346         w.updateDocument( DESCRIPTOR_TERM, hdr );
347 
348         w.commit();
349     }
350 
351     private void deleteIndexFiles( boolean full )
352         throws IOException
353     {
354         if ( indexDirectory != null )
355         {
356             String[] names = indexDirectory.listAll();
357 
358             if ( names != null )
359             {
360                 IndexFileNameFilter filter = IndexFileNameFilter.getFilter();
361 
362                 for ( int i = 0; i < names.length; i++ )
363                 {
364                     if ( filter.accept( null, names[i] ) )
365                     {
366                         indexDirectory.deleteFile( names[i] );
367                     }
368                 }
369             }
370 
371             if ( full )
372             {
373                 if ( indexDirectory.fileExists( INDEX_PACKER_PROPERTIES_FILE ) )
374                 {
375                     indexDirectory.deleteFile( INDEX_PACKER_PROPERTIES_FILE );
376                 }
377 
378                 if ( indexDirectory.fileExists( INDEX_UPDATER_PROPERTIES_FILE ) )
379                 {
380                     indexDirectory.deleteFile( INDEX_UPDATER_PROPERTIES_FILE );
381                 }
382             }
383 
384             IndexUtils.deleteTimestamp( indexDirectory );
385         }
386     }
387 
388     public boolean isSearchable()
389     {
390         return searchable;
391     }
392 
393     public void setSearchable( boolean searchable )
394     {
395         this.searchable = searchable;
396     }
397 
398     public String getId()
399     {
400         return id;
401     }
402 
403     public void updateTimestamp()
404         throws IOException
405     {
406         updateTimestamp( false );
407     }
408 
409     public void updateTimestamp( boolean save )
410         throws IOException
411     {
412         updateTimestamp( save, new Date() );
413     }
414 
415     public void updateTimestamp( boolean save, Date timestamp )
416         throws IOException
417     {
418         this.timestamp = timestamp;
419 
420         if ( save )
421         {
422             IndexUtils.updateTimestamp( indexDirectory, getTimestamp() );
423         }
424     }
425 
426     public Date getTimestamp()
427     {
428         return timestamp;
429     }
430 
431     public int getSize()
432         throws IOException
433     {
434         return getIndexReader().numDocs();
435     }
436 
437     public String getRepositoryId()
438     {
439         return repositoryId;
440     }
441 
442     public File getRepository()
443     {
444         return repository;
445     }
446 
447     public String getRepositoryUrl()
448     {
449         return repositoryUrl;
450     }
451 
452     public String getIndexUpdateUrl()
453     {
454         if ( repositoryUrl != null )
455         {
456             if ( indexUpdateUrl == null || indexUpdateUrl.trim().length() == 0 )
457             {
458                 return repositoryUrl + ( repositoryUrl.endsWith( "/" ) ? "" : "/" ) + INDEX_DIRECTORY;
459             }
460         }
461         return indexUpdateUrl;
462     }
463 
464     public Analyzer getAnalyzer()
465     {
466         return new NexusAnalyzer();
467     }
468 
469     protected void openAndWarmup()
470         throws IOException
471     {
472         // IndexWriter (close)
473         if ( indexWriter != null )
474         {
475             indexWriter.close();
476 
477             indexWriter = null;
478         }
479         // IndexSearcher (close only, since we did supply this.indexReader explicitly)
480         if ( indexSearcher != null )
481         {
482             indexSearcher.close();
483 
484             indexSearcher = null;
485         }
486         // IndexReader
487         if ( indexReader != null )
488         {
489             indexReader.close();
490 
491             indexReader = null;
492         }
493 
494         // IndexWriter open
495         final boolean create = !IndexReader.indexExists( indexDirectory );
496 
497         indexWriter = new NexusIndexWriter( getIndexDirectory(), new NexusAnalyzer(), create );
498 
499         indexWriter.setRAMBufferSizeMB( 2 );
500 
501         indexWriter.setMergeScheduler( new SerialMergeScheduler() );
502 
503         indexWriter.commit(); // LUCENE-2386
504 
505         openAndWarmupReaders();
506     }
507 
508     protected void openAndWarmupReaders()
509         throws IOException
510     {
511         if ( indexReader != null && indexReader.isCurrent() )
512         {
513             return;
514         }
515 
516         // IndexReader open
517         IndexReader newIndexReader = IndexReader.open( indexDirectory, true );
518 
519         // IndexSearcher open, but with new reader
520         NexusIndexSearcher newIndexSearcher = new NexusIndexSearcher( this, newIndexReader );
521 
522         // warm up
523         warmUp( newIndexSearcher );
524 
525         lockExclusively();
526 
527         try
528         {
529             // IndexSearcher (close only, since we did supply this.indexReader explicitly)
530             if ( indexSearcher != null )
531             {
532                 indexSearcher.close();
533             }
534             // IndexReader
535             if ( indexReader != null )
536             {
537                 indexReader.close();
538             }
539 
540             indexReader = newIndexReader;
541 
542             indexSearcher = newIndexSearcher;
543         }
544         finally
545         {
546             unlockExclusively();
547         }
548     }
549 
550     protected void warmUp( NexusIndexSearcher searcher )
551         throws IOException
552     {
553         try
554         {
555             // TODO: figure this out better and non blocking
556             searcher.search( new TermQuery( new Term( "g", "org.apache" ) ), 1000 );
557         }
558         catch ( IOException e )
559         {
560             close( false );
561 
562             throw e;
563         }
564     }
565 
566     public IndexWriter getIndexWriter()
567         throws IOException
568     {
569         lock();
570 
571         try
572         {
573             return indexWriter;
574         }
575         finally
576         {
577             unlock();
578         }
579     }
580 
581     public IndexReader getIndexReader()
582         throws IOException
583     {
584         lock();
585 
586         try
587         {
588             return indexReader;
589         }
590         finally
591         {
592             unlock();
593         }
594     }
595 
596     public IndexSearcher getIndexSearcher()
597         throws IOException
598     {
599         lock();
600 
601         try
602         {
603             return indexSearcher;
604         }
605         finally
606         {
607             unlock();
608         }
609     }
610 
611     public void commit()
612         throws IOException
613     {
614         // TODO: detect is writer "dirty"?
615         if ( true )
616         {
617             if ( BLOCKING_COMMIT )
618             {
619                 lockExclusively();
620             }
621             else
622             {
623                 lock();
624             }
625 
626             try
627             {
628                 doCommit( BLOCKING_COMMIT );
629             }
630             finally
631             {
632                 if ( BLOCKING_COMMIT )
633                 {
634                     unlockExclusively();
635                 }
636                 else
637                 {
638                     unlock();
639                 }
640             }
641         }
642     }
643 
644     protected void doCommit( boolean blocking )
645         throws IOException
646     {
647         try
648         {
649             // TODO: is this needed? Why not put the commit() call into synchronized
650             // since all callers of doCommit() aside of commit() already possess exclusive lock
651             synchronized ( this )
652             {
653                 getIndexWriter().commit();
654             }
655 
656             // TODO: define some treshold or requirement
657             // for reopening readers (is expensive)
658             // For example: by inserting 1 record among 1M, do we really want to reopen?
659             if ( true )
660             {
661                 if ( blocking )
662                 {
663                     openAndWarmupReaders();
664                 }
665                 else
666                 {
667                     flagNeedsReopen();
668                 }
669             }
670         }
671         catch ( CorruptIndexException e )
672         {
673             close( false );
674 
675             throw e;
676         }
677         catch ( IOException e )
678         {
679             close( false );
680 
681             throw e;
682         }
683     }
684 
685     public void rollback()
686         throws IOException
687     {
688         // detect is writer "dirty"?
689         if ( true )
690         {
691             lock();
692 
693             try
694             {
695                 IndexWriter w = getIndexWriter();
696 
697                 try
698                 {
699                     synchronized ( this )
700                     {
701                         w.rollback();
702                     }
703                 }
704                 catch ( CorruptIndexException e )
705                 {
706                     close( false );
707 
708                     throw e;
709                 }
710                 catch ( IOException e )
711                 {
712                     close( false );
713 
714                     throw e;
715                 }
716             }
717             finally
718             {
719                 unlock();
720             }
721         }
722     }
723 
724     public void optimize()
725         throws CorruptIndexException, IOException
726     {
727         lockExclusively();
728 
729         try
730         {
731             IndexWriter w = getIndexWriter();
732 
733             try
734             {
735                 w.optimize();
736 
737                 doCommit( true );
738             }
739             catch ( CorruptIndexException e )
740             {
741                 close( false );
742 
743                 throw e;
744             }
745             catch ( IOException e )
746             {
747                 close( false );
748 
749                 throw e;
750             }
751         }
752         finally
753         {
754             unlockExclusively();
755         }
756     }
757 
758     public void close( boolean deleteFiles )
759         throws IOException
760     {
761         lockExclusively();
762 
763         try
764         {
765             if ( indexDirectory != null )
766             {
767                 IndexUtils.updateTimestamp( indexDirectory, getTimestamp() );
768 
769                 closeReaders();
770 
771                 if ( deleteFiles )
772                 {
773                     deleteIndexFiles( true );
774                 }
775 
776                 indexDirectory.close();
777             }
778 
779             // TODO: this will prevent from reopening them, but needs better solution
780             // Needed to make bottleWarmerThread die off
781             indexDirectory = null;
782         }
783         finally
784         {
785             unlockExclusively();
786         }
787     }
788 
789     public void purge()
790         throws IOException
791     {
792         lockExclusively();
793 
794         try
795         {
796             closeReaders();
797 
798             deleteIndexFiles( true );
799 
800             openAndWarmup();
801 
802             try
803             {
804                 prepareIndex( true );
805             }
806             catch ( UnsupportedExistingLuceneIndexException e )
807             {
808                 // just deleted it
809             }
810 
811             rebuildGroups();
812 
813             updateTimestamp( true, null );
814         }
815         finally
816         {
817             unlockExclusively();
818         }
819     }
820 
821     public void replace( Directory directory )
822         throws IOException
823     {
824         lockExclusively();
825 
826         try
827         {
828             Date ts = IndexUtils.getTimestamp( directory );
829 
830             closeReaders();
831 
832             deleteIndexFiles( false );
833 
834             IndexUtils.copyDirectory( directory, indexDirectory );
835 
836             openAndWarmup();
837 
838             // reclaim the index as mine
839             storeDescriptor();
840 
841             updateTimestamp( true, ts );
842 
843             optimize();
844         }
845         finally
846         {
847             unlockExclusively();
848         }
849     }
850 
851     public void merge( Directory directory )
852         throws IOException
853     {
854         merge( directory, null );
855     }
856 
857     public void merge( Directory directory, DocumentFilter filter )
858         throws IOException
859     {
860         lockExclusively();
861 
862         try
863         {
864             IndexWriter w = getIndexWriter();
865 
866             IndexSearcher s = getIndexSearcher();
867 
868             IndexReader directoryReader = IndexReader.open( directory, true );
869 
870             TopScoreDocCollector collector = null;
871 
872             try
873             {
874                 int numDocs = directoryReader.maxDoc();
875 
876                 for ( int i = 0; i < numDocs; i++ )
877                 {
878                     if ( directoryReader.isDeleted( i ) )
879                     {
880                         continue;
881                     }
882 
883                     Document d = directoryReader.document( i );
884 
885                     if ( filter != null && !filter.accept( d ) )
886                     {
887                         continue;
888                     }
889 
890                     String uinfo = d.get( ArtifactInfo.UINFO );
891 
892                     if ( uinfo != null )
893                     {
894                         collector = TopScoreDocCollector.create( 1, false );
895 
896                         s.search( new TermQuery( new Term( ArtifactInfo.UINFO, uinfo ) ), collector );
897 
898                         if ( collector.getTotalHits() == 0 )
899                         {
900                             w.addDocument( IndexUtils.updateDocument( d, this, false ) );
901                         }
902                     }
903                     else
904                     {
905                         String deleted = d.get( ArtifactInfo.DELETED );
906 
907                         if ( deleted != null )
908                         {
909                             // Deleting the document loses history that it was delete,
910                             // so incrementals wont work. Therefore, put the delete
911                             // document in as well
912                             w.deleteDocuments( new Term( ArtifactInfo.UINFO, deleted ) );
913                             w.addDocument( d );
914                         }
915                     }
916                 }
917 
918             }
919             finally
920             {
921                 directoryReader.close();
922 
923                 doCommit( true );
924             }
925 
926             rebuildGroups();
927 
928             Date mergedTimestamp = IndexUtils.getTimestamp( directory );
929 
930             if ( getTimestamp() != null && mergedTimestamp != null && mergedTimestamp.after( getTimestamp() ) )
931             {
932                 // we have both, keep the newest
933                 updateTimestamp( true, mergedTimestamp );
934             }
935             else
936             {
937                 updateTimestamp( true );
938             }
939 
940             optimize();
941         }
942         finally
943         {
944             unlockExclusively();
945         }
946     }
947 
948     private void closeReaders()
949         throws CorruptIndexException, IOException
950     {
951         if ( indexWriter != null )
952         {
953             indexWriter.close();
954 
955             indexWriter = null;
956         }
957         if ( indexSearcher != null )
958         {
959             indexSearcher.close();
960 
961             indexSearcher = null;
962         }
963         if ( indexReader != null )
964         {
965             indexReader.close();
966 
967             indexReader = null;
968         }
969     }
970 
971     public GavCalculator getGavCalculator()
972     {
973         return gavCalculator;
974     }
975 
976     public List<IndexCreator> getIndexCreators()
977     {
978         return Collections.unmodifiableList( indexCreators );
979     }
980 
981     // groups
982 
983     public void rebuildGroups()
984         throws IOException
985     {
986         lockExclusively();
987 
988         try
989         {
990             IndexReader r = getIndexReader();
991 
992             Set<String> rootGroups = new LinkedHashSet<String>();
993             Set<String> allGroups = new LinkedHashSet<String>();
994 
995             int numDocs = r.maxDoc();
996 
997             for ( int i = 0; i < numDocs; i++ )
998             {
999                 if ( r.isDeleted( i ) )
1000                 {
1001                     continue;
1002                 }
1003 
1004                 Document d = r.document( i );
1005 
1006                 String uinfo = d.get( ArtifactInfo.UINFO );
1007 
1008                 if ( uinfo != null )
1009                 {
1010                     ArtifactInfo info = IndexUtils.constructArtifactInfo( d, this );
1011                     rootGroups.add( info.getRootGroup() );
1012                     allGroups.add( info.groupId );
1013                 }
1014             }
1015 
1016             setRootGroups( rootGroups );
1017             setAllGroups( allGroups );
1018 
1019             optimize();
1020         }
1021         finally
1022         {
1023             unlockExclusively();
1024         }
1025     }
1026 
1027     public Set<String> getAllGroups()
1028         throws IOException
1029     {
1030         lock();
1031 
1032         try
1033         {
1034             return getGroups( ArtifactInfo.ALL_GROUPS, ArtifactInfo.ALL_GROUPS_VALUE, ArtifactInfo.ALL_GROUPS_LIST );
1035         }
1036         finally
1037         {
1038             unlock();
1039         }
1040     }
1041 
1042     public void setAllGroups( Collection<String> groups )
1043         throws IOException
1044     {
1045         lockExclusively();
1046 
1047         try
1048         {
1049             setGroups( groups, ArtifactInfo.ALL_GROUPS, ArtifactInfo.ALL_GROUPS_VALUE, ArtifactInfo.ALL_GROUPS_LIST );
1050 
1051             doCommit( true );
1052         }
1053         finally
1054         {
1055             unlockExclusively();
1056         }
1057     }
1058 
1059     public Set<String> getRootGroups()
1060         throws IOException
1061     {
1062         lock();
1063 
1064         try
1065         {
1066             return getGroups( ArtifactInfo.ROOT_GROUPS, ArtifactInfo.ROOT_GROUPS_VALUE, ArtifactInfo.ROOT_GROUPS_LIST );
1067         }
1068         finally
1069         {
1070             unlock();
1071         }
1072     }
1073 
1074     public void setRootGroups( Collection<String> groups )
1075         throws IOException
1076     {
1077         lockExclusively();
1078 
1079         try
1080         {
1081             setGroups( groups, ArtifactInfo.ROOT_GROUPS, ArtifactInfo.ROOT_GROUPS_VALUE, ArtifactInfo.ROOT_GROUPS_LIST );
1082 
1083             doCommit( true );
1084         }
1085         finally
1086         {
1087             unlockExclusively();
1088         }
1089     }
1090 
1091     protected Set<String> getGroups( String field, String filedValue, String listField )
1092         throws IOException, CorruptIndexException
1093     {
1094         TopScoreDocCollector collector = TopScoreDocCollector.create( 1, false );
1095 
1096         getIndexSearcher().search( new TermQuery( new Term( field, filedValue ) ), collector );
1097 
1098         TopDocs topDocs = collector.topDocs();
1099 
1100         Set<String> groups = new LinkedHashSet<String>( Math.max( 10, topDocs.totalHits ) );
1101 
1102         if ( topDocs.totalHits > 0 )
1103         {
1104             Document doc = getIndexSearcher().doc( topDocs.scoreDocs[0].doc );
1105 
1106             String groupList = doc.get( listField );
1107 
1108             if ( groupList != null )
1109             {
1110                 groups.addAll( Arrays.asList( groupList.split( "\\|" ) ) );
1111             }
1112         }
1113 
1114         return groups;
1115     }
1116 
1117     protected void setGroups( Collection<String> groups, String groupField, String groupFieldValue,
1118                               String groupListField )
1119         throws IOException, CorruptIndexException
1120     {
1121         IndexWriter w = getIndexWriter();
1122 
1123         w.updateDocument( new Term( groupField, groupFieldValue ),
1124             createGroupsDocument( groups, groupField, groupFieldValue, groupListField ) );
1125     }
1126 
1127     protected Document createGroupsDocument( Collection<String> groups, String field, String fieldValue,
1128                                              String listField )
1129     {
1130         Document groupDoc = new Document();
1131 
1132         groupDoc.add( new Field( field, //
1133             fieldValue, Field.Store.YES, Field.Index.NOT_ANALYZED ) );
1134 
1135         groupDoc.add( new Field( listField, //
1136             ArtifactInfo.lst2str( groups ), Field.Store.YES, Field.Index.NO ) );
1137 
1138         return groupDoc;
1139     }
1140 
1141     @Override
1142     public String toString()
1143     {
1144         return id + " : " + timestamp;
1145     }
1146 
1147     // ==
1148 
1149     private volatile boolean needsReaderReopen = false;
1150 
1151     protected void flagNeedsReopen()
1152     {
1153         needsReaderReopen = true;
1154     }
1155 
1156     protected void unflagNeedsReopen()
1157     {
1158         needsReaderReopen = false;
1159     }
1160 
1161     protected boolean isReopenNeeded()
1162     {
1163         return needsReaderReopen;
1164     }
1165 
1166     protected void installBottleWarmer()
1167     {
1168         if ( BLOCKING_COMMIT )
1169         {
1170             return;
1171         }
1172 
1173         Runnable bottleWarmer = new Runnable()
1174         {
1175             public void run()
1176             {
1177                 // die off when context is closed
1178                 while ( indexDirectory != null )
1179                 {
1180                     try
1181                     {
1182                         if ( isReopenNeeded() )
1183                         {
1184                             openAndWarmupReaders();
1185 
1186                             unflagNeedsReopen();
1187                         }
1188 
1189                         Thread.sleep( 1000 );
1190                     }
1191                     catch ( Exception e )
1192                     {
1193                         e.printStackTrace();
1194                     }
1195                 }
1196             }
1197         };
1198 
1199         bottleWarmerThread = new Thread( bottleWarmer, "Index-BottleWarmer-" + id );
1200         bottleWarmerThread.setDaemon( true );
1201         bottleWarmerThread.start();
1202     }
1203 
1204     /**
1205      * A flag useful for tests, to make this IndexingContext implementation blocking. If this flag is true, context will
1206      * block the commit() calls and will return from it when Lucene commit done AND all the readers are reopened and are
1207      * current. TODO: this is currently inherently unsafe (is not final), and is meant to be used in Unit tests only!
1208      * Think something and tie this knot properly.
1209      */
1210     public static boolean BLOCKING_COMMIT = false;
1211 }