1 package org.apache.maven.index.packer;
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22 import java.io.BufferedOutputStream;
23 import java.io.File;
24 import java.io.FileInputStream;
25 import java.io.FileOutputStream;
26 import java.io.IOException;
27 import java.io.OutputStream;
28 import java.text.SimpleDateFormat;
29 import java.util.Date;
30 import java.util.List;
31 import java.util.Properties;
32 import java.util.TimeZone;
33 import java.util.zip.ZipEntry;
34 import java.util.zip.ZipOutputStream;
35
36 import org.apache.lucene.document.Document;
37 import org.apache.lucene.document.Field;
38 import org.apache.lucene.index.CorruptIndexException;
39 import org.apache.lucene.index.IndexReader;
40 import org.apache.lucene.index.IndexWriter;
41 import org.apache.lucene.store.Directory;
42 import org.apache.lucene.store.FSDirectory;
43 import org.apache.lucene.store.IndexInput;
44 import org.apache.lucene.store.LockObtainFailedException;
45 import org.apache.maven.index.ArtifactInfo;
46 import org.apache.maven.index.context.IndexCreator;
47 import org.apache.maven.index.context.IndexUtils;
48 import org.apache.maven.index.context.IndexingContext;
49 import org.apache.maven.index.context.NexusIndexWriter;
50 import org.apache.maven.index.context.NexusLegacyAnalyzer;
51 import org.apache.maven.index.creator.LegacyDocumentUpdater;
52 import org.apache.maven.index.incremental.IncrementalHandler;
53 import org.apache.maven.index.updater.IndexDataWriter;
54 import org.codehaus.plexus.component.annotations.Component;
55 import org.codehaus.plexus.component.annotations.Requirement;
56 import org.codehaus.plexus.logging.AbstractLogEnabled;
57 import org.codehaus.plexus.util.FileUtils;
58 import org.codehaus.plexus.util.IOUtil;
59
60
61
62
63
64
65
66 @Component( role = IndexPacker.class )
67 public class DefaultIndexPacker
68 extends AbstractLogEnabled
69 implements IndexPacker
70 {
71 @Requirement( role = IncrementalHandler.class )
72 IncrementalHandler incrementalHandler;
73
74 public void packIndex( IndexPackingRequest request )
75 throws IOException, IllegalArgumentException
76 {
77 if ( request.getTargetDir() == null )
78 {
79 throw new IllegalArgumentException( "The target dir is null" );
80 }
81
82 if ( request.getTargetDir().exists() )
83 {
84 if ( !request.getTargetDir().isDirectory() )
85 {
86 throw new IllegalArgumentException(
87 String.format( "Specified target path %s is not a directory",
88 request.getTargetDir().getAbsolutePath() ) );
89 }
90 if ( !request.getTargetDir().canWrite() )
91 {
92 throw new IllegalArgumentException( String.format( "Specified target path %s is not writtable",
93 request.getTargetDir().getAbsolutePath() ) );
94 }
95 }
96 else
97 {
98 if ( !request.getTargetDir().mkdirs() )
99 {
100 throw new IllegalArgumentException( "Can't create " + request.getTargetDir().getAbsolutePath() );
101 }
102 }
103
104
105 File legacyFile = new File( request.getTargetDir(), IndexingContext.INDEX_FILE_PREFIX + ".zip" );
106 File v1File = new File( request.getTargetDir(), IndexingContext.INDEX_FILE_PREFIX + ".gz" );
107
108 Properties info = null;
109
110 final IndexingContext context = request.getContext();
111
112 context.lock();
113
114 try
115 {
116 try
117 {
118
119
120 info = readIndexProperties( request );
121
122 if ( request.isCreateIncrementalChunks() )
123 {
124 List<Integer> chunk = incrementalHandler.getIncrementalUpdates( request, info );
125
126 if ( chunk == null )
127 {
128 getLogger().debug( "Problem with Chunks, forcing regeneration of whole index" );
129 incrementalHandler.initializeProperties( info );
130 }
131 else if ( chunk.isEmpty() )
132 {
133 getLogger().debug( "No incremental changes, not writing new incremental chunk" );
134 }
135 else
136 {
137 File file =
138 new File( request.getTargetDir(),
139 IndexingContext.INDEX_FILE_PREFIX + "."
140 + info.getProperty( IndexingContext.INDEX_CHUNK_COUNTER ) + ".gz" );
141
142 writeIndexData( request.getContext(),
143 chunk, file );
144
145 if ( request.isCreateChecksumFiles() )
146 {
147 FileUtils.fileWrite(
148 new File( file.getParentFile(), file.getName() + ".sha1" ).getAbsolutePath(),
149 DigesterUtils.getSha1Digest( file ) );
150
151 FileUtils.fileWrite(
152 new File( file.getParentFile(), file.getName() + ".md5" ).getAbsolutePath(),
153 DigesterUtils.getMd5Digest( file ) );
154 }
155 }
156 }
157 }
158 catch ( IOException e )
159 {
160 getLogger().info( "Unable to read properties file, will force index regeneration" );
161 info = new Properties();
162 incrementalHandler.initializeProperties( info );
163 }
164
165 Date timestamp = request.getContext().getTimestamp();
166
167 if ( timestamp == null )
168 {
169 timestamp = new Date( 0 );
170 }
171
172 if ( request.getFormats().contains( IndexPackingRequest.IndexFormat.FORMAT_LEGACY ) )
173 {
174 info.setProperty( IndexingContext.INDEX_LEGACY_TIMESTAMP, format( timestamp ) );
175
176 writeIndexArchive( request.getContext(), legacyFile );
177
178 if ( request.isCreateChecksumFiles() )
179 {
180 FileUtils.fileWrite(
181 new File( legacyFile.getParentFile(), legacyFile.getName() + ".sha1" ).getAbsolutePath(),
182 DigesterUtils.getSha1Digest( legacyFile ) );
183
184 FileUtils.fileWrite(
185 new File( legacyFile.getParentFile(), legacyFile.getName() + ".md5" ).getAbsolutePath(),
186 DigesterUtils.getMd5Digest( legacyFile ) );
187 }
188 }
189
190 if ( request.getFormats().contains( IndexPackingRequest.IndexFormat.FORMAT_V1 ) )
191 {
192 info.setProperty( IndexingContext.INDEX_TIMESTAMP, format( timestamp ) );
193
194 writeIndexData( request.getContext(), null, v1File );
195
196 if ( request.isCreateChecksumFiles() )
197 {
198 FileUtils.fileWrite(
199 new File( v1File.getParentFile(), v1File.getName() + ".sha1" ).getAbsolutePath(),
200 DigesterUtils.getSha1Digest( v1File ) );
201
202 FileUtils.fileWrite(
203 new File( v1File.getParentFile(), v1File.getName() + ".md5" ).getAbsolutePath(),
204 DigesterUtils.getMd5Digest( v1File ) );
205 }
206 }
207
208 writeIndexProperties( request, info );
209 }
210 finally
211 {
212 context.unlock();
213 }
214 }
215
216 private Properties readIndexProperties( IndexPackingRequest request )
217 throws IOException
218 {
219 File file = null;
220
221 if ( request.isUseTargetProperties() )
222 {
223 file = new File( request.getTargetDir(), IndexingContext.INDEX_REMOTE_PROPERTIES_FILE );
224 }
225 else
226 {
227 file =
228 new File( request.getContext().getIndexDirectoryFile(), IndexingContext.INDEX_PACKER_PROPERTIES_FILE );
229 }
230
231 Properties properties = new Properties();
232
233 FileInputStream fos = null;
234
235 try
236 {
237 fos = new FileInputStream( file );
238 properties.load( fos );
239 }
240 finally
241 {
242 if ( fos != null )
243 {
244 fos.close();
245 }
246 }
247
248 return properties;
249 }
250
251 void writeIndexArchive( IndexingContext context, File targetArchive )
252 throws IOException
253 {
254 if ( targetArchive.exists() )
255 {
256 targetArchive.delete();
257 }
258
259 OutputStream os = null;
260
261 try
262 {
263 os = new BufferedOutputStream( new FileOutputStream( targetArchive ), 4096 );
264
265 packIndexArchive( context, os );
266 }
267 finally
268 {
269 IOUtil.close( os );
270 }
271 }
272
273
274
275
276 public static void packIndexArchive( IndexingContext context, OutputStream os )
277 throws IOException
278 {
279 File indexArchive = File.createTempFile( "nexus-index", "" );
280
281 File indexDir = new File( indexArchive.getAbsoluteFile().getParentFile(), indexArchive.getName() + ".dir" );
282
283 indexDir.mkdirs();
284
285 FSDirectory fdir = FSDirectory.open( indexDir );
286
287 try
288 {
289
290 IndexUtils.updateTimestamp( context.getIndexDirectory(), context.getTimestamp() );
291 IndexUtils.updateTimestamp( fdir, context.getTimestamp() );
292
293 copyLegacyDocuments( context.getIndexReader(), fdir, context );
294 packDirectory( fdir, os );
295 }
296 finally
297 {
298 IndexUtils.close( fdir );
299 indexArchive.delete();
300 IndexUtils.delete( indexDir );
301 }
302 }
303
304 static void copyLegacyDocuments( IndexReader r, Directory targetdir, IndexingContext context )
305 throws CorruptIndexException, LockObtainFailedException, IOException
306 {
307 IndexWriter w = null;
308 try
309 {
310 w = new NexusIndexWriter( targetdir, new NexusLegacyAnalyzer(), true );
311
312 for ( int i = 0; i < r.maxDoc(); i++ )
313 {
314 if ( !r.isDeleted( i ) )
315 {
316 w.addDocument( updateLegacyDocument( r.document( i ), context ) );
317 }
318 }
319
320 w.optimize();
321 w.commit();
322 }
323 finally
324 {
325 IndexUtils.close( w );
326 }
327 }
328
329 static Document updateLegacyDocument( Document doc, IndexingContext context )
330 {
331 ArtifactInfo ai = IndexUtils.constructArtifactInfo( doc, context );
332 if ( ai == null )
333 {
334 return doc;
335 }
336
337 Document document = new Document();
338 document.add( new Field( ArtifactInfo.UINFO, ai.getUinfo(), Field.Store.YES, Field.Index.NOT_ANALYZED ) );
339
340 for ( IndexCreator ic : context.getIndexCreators() )
341 {
342 if ( ic instanceof LegacyDocumentUpdater )
343 {
344 ( (LegacyDocumentUpdater) ic ).updateLegacyDocument( ai, document );
345 }
346 }
347
348 return document;
349 }
350
351 static void packDirectory( Directory directory, OutputStream os )
352 throws IOException
353 {
354 ZipOutputStream zos = null;
355 try
356 {
357 zos = new ZipOutputStream( os );
358 zos.setLevel( 9 );
359
360 String[] names = directory.listAll();
361
362 boolean savedTimestamp = false;
363
364 byte[] buf = new byte[8192];
365
366 for ( int i = 0; i < names.length; i++ )
367 {
368 String name = names[i];
369
370 writeFile( name, zos, directory, buf );
371
372 if ( name.equals( IndexUtils.TIMESTAMP_FILE ) )
373 {
374 savedTimestamp = true;
375 }
376 }
377
378
379 if ( !savedTimestamp && directory.fileExists( IndexUtils.TIMESTAMP_FILE ) )
380 {
381 writeFile( IndexUtils.TIMESTAMP_FILE, zos, directory, buf );
382 }
383 }
384 finally
385 {
386 IndexUtils.close( zos );
387 }
388 }
389
390 static void writeFile( String name, ZipOutputStream zos, Directory directory, byte[] buf )
391 throws IOException
392 {
393 ZipEntry e = new ZipEntry( name );
394
395 zos.putNextEntry( e );
396
397 IndexInput in = directory.openInput( name );
398
399 try
400 {
401 int toRead = 0;
402
403 int bytesLeft = (int) in.length();
404
405 while ( bytesLeft > 0 )
406 {
407 toRead = ( bytesLeft >= buf.length ) ? buf.length : bytesLeft;
408 bytesLeft -= toRead;
409
410 in.readBytes( buf, 0, toRead, false );
411
412 zos.write( buf, 0, toRead );
413 }
414 }
415 finally
416 {
417 IndexUtils.close( in );
418 }
419
420 zos.flush();
421
422 zos.closeEntry();
423 }
424
425 void writeIndexData( IndexingContext context, List<Integer> docIndexes, File targetArchive )
426 throws IOException
427 {
428 if ( targetArchive.exists() )
429 {
430 targetArchive.delete();
431 }
432
433 OutputStream os = null;
434
435 try
436 {
437 os = new FileOutputStream( targetArchive );
438
439 IndexDataWriter dw = new IndexDataWriter( os );
440 dw.write( context, docIndexes );
441
442 os.flush();
443 }
444 finally
445 {
446 IOUtil.close( os );
447 }
448 }
449
450 void writeIndexProperties( IndexPackingRequest request, Properties info )
451 throws IOException
452 {
453 File propertyFile =
454 new File( request.getContext().getIndexDirectoryFile(), IndexingContext.INDEX_PACKER_PROPERTIES_FILE );
455 File targetPropertyFile = new File( request.getTargetDir(), IndexingContext.INDEX_REMOTE_PROPERTIES_FILE );
456
457 info.setProperty( IndexingContext.INDEX_ID, request.getContext().getId() );
458
459 OutputStream os = null;
460
461 try
462 {
463 os = new FileOutputStream( propertyFile );
464
465 info.store( os, null );
466 }
467 finally
468 {
469 IOUtil.close( os );
470 }
471
472 try
473 {
474 os = new FileOutputStream( targetPropertyFile );
475
476 info.store( os, null );
477 }
478 finally
479 {
480 IOUtil.close( os );
481 }
482
483 if ( request.isCreateChecksumFiles() )
484 {
485 FileUtils.fileWrite(
486 new File( targetPropertyFile.getParentFile(), targetPropertyFile.getName() + ".sha1" ).getAbsolutePath(),
487 DigesterUtils.getSha1Digest( targetPropertyFile ) );
488
489 FileUtils.fileWrite(
490 new File( targetPropertyFile.getParentFile(), targetPropertyFile.getName() + ".md5" ).getAbsolutePath(),
491 DigesterUtils.getMd5Digest( targetPropertyFile ) );
492 }
493 }
494
495 private String format( Date d )
496 {
497 SimpleDateFormat df = new SimpleDateFormat( IndexingContext.INDEX_TIME_FORMAT );
498 df.setTimeZone( TimeZone.getTimeZone( "GMT" ) );
499 return df.format( d );
500 }
501 }