1 | |
package org.apache.maven.index.packer; |
2 | |
|
3 | |
|
4 | |
|
5 | |
|
6 | |
|
7 | |
|
8 | |
|
9 | |
|
10 | |
|
11 | |
|
12 | |
|
13 | |
|
14 | |
|
15 | |
|
16 | |
|
17 | |
|
18 | |
|
19 | |
|
20 | |
|
21 | |
|
22 | |
import java.io.BufferedOutputStream; |
23 | |
import java.io.File; |
24 | |
import java.io.FileInputStream; |
25 | |
import java.io.FileOutputStream; |
26 | |
import java.io.IOException; |
27 | |
import java.io.OutputStream; |
28 | |
import java.text.SimpleDateFormat; |
29 | |
import java.util.Date; |
30 | |
import java.util.List; |
31 | |
import java.util.Properties; |
32 | |
import java.util.TimeZone; |
33 | |
import java.util.zip.ZipEntry; |
34 | |
import java.util.zip.ZipOutputStream; |
35 | |
|
36 | |
import org.apache.lucene.document.Document; |
37 | |
import org.apache.lucene.document.Field; |
38 | |
import org.apache.lucene.index.CorruptIndexException; |
39 | |
import org.apache.lucene.index.IndexReader; |
40 | |
import org.apache.lucene.index.IndexWriter; |
41 | |
import org.apache.lucene.store.Directory; |
42 | |
import org.apache.lucene.store.FSDirectory; |
43 | |
import org.apache.lucene.store.IndexInput; |
44 | |
import org.apache.lucene.store.LockObtainFailedException; |
45 | |
import org.apache.maven.index.ArtifactInfo; |
46 | |
import org.apache.maven.index.context.IndexCreator; |
47 | |
import org.apache.maven.index.context.IndexUtils; |
48 | |
import org.apache.maven.index.context.IndexingContext; |
49 | |
import org.apache.maven.index.context.NexusIndexWriter; |
50 | |
import org.apache.maven.index.context.NexusLegacyAnalyzer; |
51 | |
import org.apache.maven.index.creator.LegacyDocumentUpdater; |
52 | |
import org.apache.maven.index.incremental.IncrementalHandler; |
53 | |
import org.apache.maven.index.updater.IndexDataWriter; |
54 | |
import org.codehaus.plexus.component.annotations.Component; |
55 | |
import org.codehaus.plexus.component.annotations.Requirement; |
56 | |
import org.codehaus.plexus.logging.AbstractLogEnabled; |
57 | |
import org.codehaus.plexus.util.FileUtils; |
58 | |
import org.codehaus.plexus.util.IOUtil; |
59 | |
|
60 | |
|
61 | |
|
62 | |
|
63 | |
|
64 | |
|
65 | |
|
66 | |
@Component( role = IndexPacker.class ) |
67 | 47 | public class DefaultIndexPacker |
68 | |
extends AbstractLogEnabled |
69 | |
implements IndexPacker |
70 | |
{ |
71 | |
@Requirement( role = IncrementalHandler.class ) |
72 | |
IncrementalHandler incrementalHandler; |
73 | |
|
74 | |
public void packIndex( IndexPackingRequest request ) |
75 | |
throws IOException, IllegalArgumentException |
76 | |
{ |
77 | 196 | if ( request.getTargetDir() == null ) |
78 | |
{ |
79 | 0 | throw new IllegalArgumentException( "The target dir is null" ); |
80 | |
} |
81 | |
|
82 | 196 | if ( request.getTargetDir().exists() ) |
83 | |
{ |
84 | 189 | if ( !request.getTargetDir().isDirectory() ) |
85 | |
{ |
86 | 0 | throw new IllegalArgumentException( |
87 | |
String.format( "Specified target path %s is not a directory", |
88 | |
request.getTargetDir().getAbsolutePath() ) ); |
89 | |
} |
90 | 189 | if ( !request.getTargetDir().canWrite() ) |
91 | |
{ |
92 | 0 | throw new IllegalArgumentException( String.format( "Specified target path %s is not writtable", |
93 | |
request.getTargetDir().getAbsolutePath() ) ); |
94 | |
} |
95 | |
} |
96 | |
else |
97 | |
{ |
98 | 7 | if ( !request.getTargetDir().mkdirs() ) |
99 | |
{ |
100 | 0 | throw new IllegalArgumentException( "Can't create " + request.getTargetDir().getAbsolutePath() ); |
101 | |
} |
102 | |
} |
103 | |
|
104 | |
|
105 | 196 | File legacyFile = new File( request.getTargetDir(), IndexingContext.INDEX_FILE_PREFIX + ".zip" ); |
106 | 196 | File v1File = new File( request.getTargetDir(), IndexingContext.INDEX_FILE_PREFIX + ".gz" ); |
107 | |
|
108 | 196 | Properties info = null; |
109 | |
|
110 | 196 | final IndexingContext context = request.getContext(); |
111 | |
|
112 | 196 | context.lock(); |
113 | |
|
114 | |
try |
115 | |
{ |
116 | |
try |
117 | |
{ |
118 | |
|
119 | |
|
120 | 196 | info = readIndexProperties( request ); |
121 | |
|
122 | 159 | if ( request.isCreateIncrementalChunks() ) |
123 | |
{ |
124 | 22 | List<Integer> chunk = incrementalHandler.getIncrementalUpdates( request, info ); |
125 | |
|
126 | 22 | if ( chunk == null ) |
127 | |
{ |
128 | 0 | getLogger().debug( "Problem with Chunks, forcing regeneration of whole index" ); |
129 | 0 | incrementalHandler.initializeProperties( info ); |
130 | |
} |
131 | 22 | else if ( chunk.isEmpty() ) |
132 | |
{ |
133 | 0 | getLogger().debug( "No incremental changes, not writing new incremental chunk" ); |
134 | |
} |
135 | |
else |
136 | |
{ |
137 | 22 | File file = |
138 | |
new File( request.getTargetDir(), |
139 | |
IndexingContext.INDEX_FILE_PREFIX + "." |
140 | |
+ info.getProperty( IndexingContext.INDEX_CHUNK_COUNTER ) + ".gz" ); |
141 | |
|
142 | 22 | writeIndexData( request.getContext(), |
143 | |
chunk, file ); |
144 | |
|
145 | 22 | if ( request.isCreateChecksumFiles() ) |
146 | |
{ |
147 | 0 | FileUtils.fileWrite( |
148 | |
new File( file.getParentFile(), file.getName() + ".sha1" ).getAbsolutePath(), |
149 | |
DigesterUtils.getSha1Digest( file ) ); |
150 | |
|
151 | 0 | FileUtils.fileWrite( |
152 | |
new File( file.getParentFile(), file.getName() + ".md5" ).getAbsolutePath(), |
153 | |
DigesterUtils.getMd5Digest( file ) ); |
154 | |
} |
155 | |
} |
156 | |
} |
157 | |
} |
158 | 37 | catch ( IOException e ) |
159 | |
{ |
160 | 37 | getLogger().info( "Unable to read properties file, will force index regeneration" ); |
161 | 37 | info = new Properties(); |
162 | 37 | incrementalHandler.initializeProperties( info ); |
163 | 159 | } |
164 | |
|
165 | 196 | Date timestamp = request.getContext().getTimestamp(); |
166 | |
|
167 | 196 | if ( timestamp == null ) |
168 | |
{ |
169 | 0 | timestamp = new Date( 0 ); |
170 | |
} |
171 | |
|
172 | 196 | if ( request.getFormats().contains( IndexPackingRequest.IndexFormat.FORMAT_LEGACY ) ) |
173 | |
{ |
174 | 188 | info.setProperty( IndexingContext.INDEX_LEGACY_TIMESTAMP, format( timestamp ) ); |
175 | |
|
176 | 188 | writeIndexArchive( request.getContext(), legacyFile ); |
177 | |
|
178 | 188 | if ( request.isCreateChecksumFiles() ) |
179 | |
{ |
180 | 0 | FileUtils.fileWrite( |
181 | |
new File( legacyFile.getParentFile(), legacyFile.getName() + ".sha1" ).getAbsolutePath(), |
182 | |
DigesterUtils.getSha1Digest( legacyFile ) ); |
183 | |
|
184 | 0 | FileUtils.fileWrite( |
185 | |
new File( legacyFile.getParentFile(), legacyFile.getName() + ".md5" ).getAbsolutePath(), |
186 | |
DigesterUtils.getMd5Digest( legacyFile ) ); |
187 | |
} |
188 | |
} |
189 | |
|
190 | 196 | if ( request.getFormats().contains( IndexPackingRequest.IndexFormat.FORMAT_V1 ) ) |
191 | |
{ |
192 | 196 | info.setProperty( IndexingContext.INDEX_TIMESTAMP, format( timestamp ) ); |
193 | |
|
194 | 196 | writeIndexData( request.getContext(), null, v1File ); |
195 | |
|
196 | 196 | if ( request.isCreateChecksumFiles() ) |
197 | |
{ |
198 | 0 | FileUtils.fileWrite( |
199 | |
new File( v1File.getParentFile(), v1File.getName() + ".sha1" ).getAbsolutePath(), |
200 | |
DigesterUtils.getSha1Digest( v1File ) ); |
201 | |
|
202 | 0 | FileUtils.fileWrite( |
203 | |
new File( v1File.getParentFile(), v1File.getName() + ".md5" ).getAbsolutePath(), |
204 | |
DigesterUtils.getMd5Digest( v1File ) ); |
205 | |
} |
206 | |
} |
207 | |
|
208 | 196 | writeIndexProperties( request, info ); |
209 | |
} |
210 | |
finally |
211 | |
{ |
212 | 196 | context.unlock(); |
213 | 196 | } |
214 | 196 | } |
215 | |
|
216 | |
private Properties readIndexProperties( IndexPackingRequest request ) |
217 | |
throws IOException |
218 | |
{ |
219 | 196 | File file = null; |
220 | |
|
221 | 196 | if ( request.isUseTargetProperties() ) |
222 | |
{ |
223 | 8 | file = new File( request.getTargetDir(), IndexingContext.INDEX_REMOTE_PROPERTIES_FILE ); |
224 | |
} |
225 | |
else |
226 | |
{ |
227 | 188 | file = |
228 | |
new File( request.getContext().getIndexDirectoryFile(), IndexingContext.INDEX_PACKER_PROPERTIES_FILE ); |
229 | |
} |
230 | |
|
231 | 196 | Properties properties = new Properties(); |
232 | |
|
233 | 196 | FileInputStream fos = null; |
234 | |
|
235 | |
try |
236 | |
{ |
237 | 196 | fos = new FileInputStream( file ); |
238 | 159 | properties.load( fos ); |
239 | |
} |
240 | |
finally |
241 | |
{ |
242 | 196 | if ( fos != null ) |
243 | |
{ |
244 | 159 | fos.close(); |
245 | |
} |
246 | |
} |
247 | |
|
248 | 159 | return properties; |
249 | |
} |
250 | |
|
251 | |
void writeIndexArchive( IndexingContext context, File targetArchive ) |
252 | |
throws IOException |
253 | |
{ |
254 | 188 | if ( targetArchive.exists() ) |
255 | |
{ |
256 | 168 | targetArchive.delete(); |
257 | |
} |
258 | |
|
259 | 188 | OutputStream os = null; |
260 | |
|
261 | |
try |
262 | |
{ |
263 | 188 | os = new BufferedOutputStream( new FileOutputStream( targetArchive ), 4096 ); |
264 | |
|
265 | 188 | packIndexArchive( context, os ); |
266 | |
} |
267 | |
finally |
268 | |
{ |
269 | 188 | IOUtil.close( os ); |
270 | 188 | } |
271 | 188 | } |
272 | |
|
273 | |
|
274 | |
|
275 | |
|
276 | |
public static void packIndexArchive( IndexingContext context, OutputStream os ) |
277 | |
throws IOException |
278 | |
{ |
279 | 192 | File indexArchive = File.createTempFile( "nexus-index", "" ); |
280 | |
|
281 | 192 | File indexDir = new File( indexArchive.getAbsoluteFile().getParentFile(), indexArchive.getName() + ".dir" ); |
282 | |
|
283 | 192 | indexDir.mkdirs(); |
284 | |
|
285 | 192 | FSDirectory fdir = FSDirectory.open( indexDir ); |
286 | |
|
287 | |
try |
288 | |
{ |
289 | |
|
290 | 192 | IndexUtils.updateTimestamp( context.getIndexDirectory(), context.getTimestamp() ); |
291 | 192 | IndexUtils.updateTimestamp( fdir, context.getTimestamp() ); |
292 | |
|
293 | 192 | copyLegacyDocuments( context.getIndexReader(), fdir, context ); |
294 | 192 | packDirectory( fdir, os ); |
295 | |
} |
296 | |
finally |
297 | |
{ |
298 | 192 | IndexUtils.close( fdir ); |
299 | 192 | indexArchive.delete(); |
300 | 192 | IndexUtils.delete( indexDir ); |
301 | 192 | } |
302 | 192 | } |
303 | |
|
304 | |
static void copyLegacyDocuments( IndexReader r, Directory targetdir, IndexingContext context ) |
305 | |
throws CorruptIndexException, LockObtainFailedException, IOException |
306 | |
{ |
307 | 192 | IndexWriter w = null; |
308 | |
try |
309 | |
{ |
310 | 192 | w = new NexusIndexWriter( targetdir, new NexusLegacyAnalyzer(), true ); |
311 | |
|
312 | 17640 | for ( int i = 0; i < r.maxDoc(); i++ ) |
313 | |
{ |
314 | 17448 | if ( !r.isDeleted( i ) ) |
315 | |
{ |
316 | 17398 | w.addDocument( updateLegacyDocument( r.document( i ), context ) ); |
317 | |
} |
318 | |
} |
319 | |
|
320 | 192 | w.optimize(); |
321 | 192 | w.commit(); |
322 | |
} |
323 | |
finally |
324 | |
{ |
325 | 192 | IndexUtils.close( w ); |
326 | 192 | } |
327 | 192 | } |
328 | |
|
329 | |
static Document updateLegacyDocument( Document doc, IndexingContext context ) |
330 | |
{ |
331 | 17398 | ArtifactInfo ai = IndexUtils.constructArtifactInfo( doc, context ); |
332 | 17398 | if ( ai == null ) |
333 | |
{ |
334 | 1014 | return doc; |
335 | |
} |
336 | |
|
337 | 16384 | Document document = new Document(); |
338 | 16384 | document.add( new Field( ArtifactInfo.UINFO, ai.getUinfo(), Field.Store.YES, Field.Index.NOT_ANALYZED ) ); |
339 | |
|
340 | 16384 | for ( IndexCreator ic : context.getIndexCreators() ) |
341 | |
{ |
342 | 49365 | if ( ic instanceof LegacyDocumentUpdater ) |
343 | |
{ |
344 | 16530 | ( (LegacyDocumentUpdater) ic ).updateLegacyDocument( ai, document ); |
345 | |
} |
346 | |
} |
347 | |
|
348 | 16384 | return document; |
349 | |
} |
350 | |
|
351 | |
static void packDirectory( Directory directory, OutputStream os ) |
352 | |
throws IOException |
353 | |
{ |
354 | 192 | ZipOutputStream zos = null; |
355 | |
try |
356 | |
{ |
357 | 192 | zos = new ZipOutputStream( os ); |
358 | 192 | zos.setLevel( 9 ); |
359 | |
|
360 | 192 | String[] names = directory.listAll(); |
361 | |
|
362 | 192 | boolean savedTimestamp = false; |
363 | |
|
364 | 192 | byte[] buf = new byte[8192]; |
365 | |
|
366 | 2304 | for ( int i = 0; i < names.length; i++ ) |
367 | |
{ |
368 | 2112 | String name = names[i]; |
369 | |
|
370 | 2112 | writeFile( name, zos, directory, buf ); |
371 | |
|
372 | 2112 | if ( name.equals( IndexUtils.TIMESTAMP_FILE ) ) |
373 | |
{ |
374 | 192 | savedTimestamp = true; |
375 | |
} |
376 | |
} |
377 | |
|
378 | |
|
379 | 192 | if ( !savedTimestamp && directory.fileExists( IndexUtils.TIMESTAMP_FILE ) ) |
380 | |
{ |
381 | 0 | writeFile( IndexUtils.TIMESTAMP_FILE, zos, directory, buf ); |
382 | |
} |
383 | |
} |
384 | |
finally |
385 | |
{ |
386 | 192 | IndexUtils.close( zos ); |
387 | 192 | } |
388 | 192 | } |
389 | |
|
390 | |
static void writeFile( String name, ZipOutputStream zos, Directory directory, byte[] buf ) |
391 | |
throws IOException |
392 | |
{ |
393 | 2112 | ZipEntry e = new ZipEntry( name ); |
394 | |
|
395 | 2112 | zos.putNextEntry( e ); |
396 | |
|
397 | 2112 | IndexInput in = directory.openInput( name ); |
398 | |
|
399 | |
try |
400 | |
{ |
401 | 2112 | int toRead = 0; |
402 | |
|
403 | 2112 | int bytesLeft = (int) in.length(); |
404 | |
|
405 | 4538 | while ( bytesLeft > 0 ) |
406 | |
{ |
407 | 2426 | toRead = ( bytesLeft >= buf.length ) ? buf.length : bytesLeft; |
408 | 2426 | bytesLeft -= toRead; |
409 | |
|
410 | 2426 | in.readBytes( buf, 0, toRead, false ); |
411 | |
|
412 | 2426 | zos.write( buf, 0, toRead ); |
413 | |
} |
414 | |
} |
415 | |
finally |
416 | |
{ |
417 | 2112 | IndexUtils.close( in ); |
418 | 2112 | } |
419 | |
|
420 | 2112 | zos.flush(); |
421 | |
|
422 | 2112 | zos.closeEntry(); |
423 | 2112 | } |
424 | |
|
425 | |
void writeIndexData( IndexingContext context, List<Integer> docIndexes, File targetArchive ) |
426 | |
throws IOException |
427 | |
{ |
428 | 218 | if ( targetArchive.exists() ) |
429 | |
{ |
430 | 169 | targetArchive.delete(); |
431 | |
} |
432 | |
|
433 | 218 | OutputStream os = null; |
434 | |
|
435 | |
try |
436 | |
{ |
437 | 218 | os = new FileOutputStream( targetArchive ); |
438 | |
|
439 | 218 | IndexDataWriter dw = new IndexDataWriter( os ); |
440 | 218 | dw.write( context, docIndexes ); |
441 | |
|
442 | 218 | os.flush(); |
443 | |
} |
444 | |
finally |
445 | |
{ |
446 | 218 | IOUtil.close( os ); |
447 | 218 | } |
448 | 218 | } |
449 | |
|
450 | |
void writeIndexProperties( IndexPackingRequest request, Properties info ) |
451 | |
throws IOException |
452 | |
{ |
453 | 196 | File propertyFile = |
454 | |
new File( request.getContext().getIndexDirectoryFile(), IndexingContext.INDEX_PACKER_PROPERTIES_FILE ); |
455 | 196 | File targetPropertyFile = new File( request.getTargetDir(), IndexingContext.INDEX_REMOTE_PROPERTIES_FILE ); |
456 | |
|
457 | 196 | info.setProperty( IndexingContext.INDEX_ID, request.getContext().getId() ); |
458 | |
|
459 | 196 | OutputStream os = null; |
460 | |
|
461 | |
try |
462 | |
{ |
463 | 196 | os = new FileOutputStream( propertyFile ); |
464 | |
|
465 | 196 | info.store( os, null ); |
466 | |
} |
467 | |
finally |
468 | |
{ |
469 | 196 | IOUtil.close( os ); |
470 | 196 | } |
471 | |
|
472 | |
try |
473 | |
{ |
474 | 196 | os = new FileOutputStream( targetPropertyFile ); |
475 | |
|
476 | 196 | info.store( os, null ); |
477 | |
} |
478 | |
finally |
479 | |
{ |
480 | 196 | IOUtil.close( os ); |
481 | 196 | } |
482 | |
|
483 | 196 | if ( request.isCreateChecksumFiles() ) |
484 | |
{ |
485 | 0 | FileUtils.fileWrite( |
486 | |
new File( targetPropertyFile.getParentFile(), targetPropertyFile.getName() + ".sha1" ).getAbsolutePath(), |
487 | |
DigesterUtils.getSha1Digest( targetPropertyFile ) ); |
488 | |
|
489 | 0 | FileUtils.fileWrite( |
490 | |
new File( targetPropertyFile.getParentFile(), targetPropertyFile.getName() + ".md5" ).getAbsolutePath(), |
491 | |
DigesterUtils.getMd5Digest( targetPropertyFile ) ); |
492 | |
} |
493 | 196 | } |
494 | |
|
495 | |
private String format( Date d ) |
496 | |
{ |
497 | 384 | SimpleDateFormat df = new SimpleDateFormat( IndexingContext.INDEX_TIME_FORMAT ); |
498 | 384 | df.setTimeZone( TimeZone.getTimeZone( "GMT" ) ); |
499 | 384 | return df.format( d ); |
500 | |
} |
501 | |
} |