View Javadoc
1   package org.apache.archiva.repository.scanner;
2   
3   /*
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *  http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing,
15   * software distributed under the License is distributed on an
16   * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
17   * KIND, either express or implied.  See the License for the
18   * specific language governing permissions and limitations
19   * under the License.
20   */
21  
22  import junit.framework.TestCase;
23  import org.apache.archiva.admin.model.beans.ManagedRepository;
24  import org.apache.archiva.admin.model.beans.RemoteRepository;
25  import org.apache.archiva.consumers.InvalidRepositoryContentConsumer;
26  import org.apache.archiva.consumers.KnownRepositoryContentConsumer;
27  import org.apache.archiva.test.utils.ArchivaSpringJUnit4ClassRunner;
28  import org.apache.commons.io.FileUtils;
29  import org.junit.Test;
30  import org.junit.runner.RunWith;
31  import org.springframework.context.ApplicationContext;
32  import org.springframework.test.context.ContextConfiguration;
33  
34  import javax.inject.Inject;
35  import java.io.File;
36  import java.io.IOException;
37  import java.nio.file.Paths;
38  import java.text.ParseException;
39  import java.text.SimpleDateFormat;
40  import java.util.ArrayList;
41  import java.util.Arrays;
42  import java.util.List;
43  import java.util.Locale;
44  import java.util.TimeZone;
45  
46  /**
47   * RepositoryScannerTest
48   */
49  @RunWith(ArchivaSpringJUnit4ClassRunner.class)
50  @ContextConfiguration(locations = { "classpath*:/META-INF/spring-context.xml" })
51  public class RepositoryScannerTest
52      extends TestCase
53  {
54  
55      @Inject
56      ApplicationContext applicationContext;
57  
58      protected ManagedRepository createRepository( String id, String name, File location )
59      {
60          ManagedRepository repo = new ManagedRepository();
61          repo.setId( id );
62          repo.setName( name );
63          repo.setLocation( location.getAbsolutePath() );
64          return repo;
65      }
66  
67      protected RemoteRepository createRemoteRepository( String id, String name, String url )
68      {
69          RemoteRepository repo = new RemoteRepository();
70          repo.setId( id );
71          repo.setName( name );
72          repo.setUrl( url );
73          return repo;
74      }
75  
76      private static final String[] ARTIFACT_PATTERNS =
77          new String[]{ "**/*.jar", "**/*.pom", "**/*.rar", "**/*.zip", "**/*.war", "**/*.tar.gz" };
78  
79      private ManagedRepository createDefaultRepository()
80      {
81          File repoDir =
82              Paths.get( System.getProperty( "basedir" ), "src/test/repositories/default-repository" ).toFile();
83  
84          assertTrue( "Default Test Repository should exist.", repoDir.exists() && repoDir.isDirectory() );
85  
86          return createRepository( "testDefaultRepo", "Test Default Repository", repoDir );
87      }
88  
89      private ManagedRepository createSimpleRepository()
90          throws IOException, ParseException
91      {
92          File srcDir = Paths.get( System.getProperty( "basedir" ),  "src/test/repositories/simple-repository" ).toFile();
93  
94          File repoDir = Paths.get( System.getProperty( "basedir" ),  "target/test-repos/simple-repository" ).toFile();
95  
96          FileUtils.deleteDirectory( repoDir );
97  
98          FileUtils.copyDirectory( srcDir, repoDir );
99  
100         File repoFile = new File( repoDir,
101                                   "groupId/snapshot-artifact/1.0-alpha-1-SNAPSHOT/snapshot-artifact-1.0-alpha-1-20050611.202024-1.pom" );
102         repoFile.setLastModified( getTimestampAsMillis( "20050611.202024" ) );
103 
104         assertTrue( "Simple Test Repository should exist.", repoDir.exists() && repoDir.isDirectory() );
105 
106         return createRepository( "testSimpleRepo", "Test Simple Repository", repoDir );
107     }
108 
109     private static long getTimestampAsMillis( String timestamp )
110         throws ParseException
111     {
112         SimpleDateFormat fmt = new SimpleDateFormat( "yyyyMMdd.HHmmss", Locale.US );
113         fmt.setTimeZone( TimeZone.getTimeZone( "UTC" ) );
114         return fmt.parse( timestamp ).getTime();
115     }
116 
117     private ManagedRepository createLegacyRepository()
118     {
119         File repoDir = Paths.get( System.getProperty( "basedir" ),  "src/test/repositories/legacy-repository" ).toFile();
120 
121         assertTrue( "Legacy Test Repository should exist.", repoDir.exists() && repoDir.isDirectory() );
122 
123         ManagedRepository repo = createRepository( "testLegacyRepo", "Test Legacy Repository", repoDir );
124         repo.setLayout( "legacy" );
125 
126         return repo;
127     }
128 
129     private void assertMinimumHits( String msg, int minimumHitCount, long actualCount )
130     {
131         if ( actualCount < minimumHitCount )
132         {
133             fail( "Minimum hit count on " + msg + " not satisfied.  Expected more than <" + minimumHitCount
134                       + ">, but actually got <" + actualCount + ">." );
135         }
136     }
137 
138     private RepositoryScanner lookupRepositoryScanner()
139         throws Exception
140     {
141         return applicationContext.getBean( RepositoryScanner.class );
142     }
143 
144     private List<String> getIgnoreList()
145     {
146         List<String> ignores = new ArrayList<>();
147         ignores.addAll( Arrays.asList( RepositoryScanner.IGNORABLE_CONTENT ) );
148         return ignores;
149     }
150 
151     @Test
152     public void testTimestampRepositoryScanner()
153         throws Exception
154     {
155         ManagedRepository repository = createSimpleRepository();
156 
157         List<KnownRepositoryContentConsumer> knownConsumers = new ArrayList<>();
158         KnownScanConsumer consumer = new KnownScanConsumer();
159         consumer.setIncludes( ARTIFACT_PATTERNS );
160         knownConsumers.add( consumer );
161 
162         List<InvalidRepositoryContentConsumer> invalidConsumers = new ArrayList<>();
163         InvalidScanConsumer badconsumer = new InvalidScanConsumer();
164         invalidConsumers.add( badconsumer );
165 
166         RepositoryScanner scanner = lookupRepositoryScanner();
167 
168         RepositoryScanStatistics stats = scanner.scan( repository, knownConsumers, invalidConsumers, getIgnoreList(),
169                                                        getTimestampAsMillis( "20061101.000000" ) );
170 
171         assertNotNull( "Stats should not be null.", stats );
172         assertEquals( "Stats.totalFileCount", 4, stats.getTotalFileCount() );
173         assertEquals( "Stats.newFileCount", 3, stats.getNewFileCount() );
174         assertEquals( "Processed Count", 2, consumer.getProcessCount() );
175         assertEquals( "Processed Count (of invalid items)", 1, badconsumer.getProcessCount() );
176     }
177 
178     @Test
179     public void testTimestampRepositoryScannerFreshScan()
180         throws Exception
181     {
182         ManagedRepository repository = createSimpleRepository();
183 
184         List<KnownRepositoryContentConsumer> knownConsumers = new ArrayList<>();
185         KnownScanConsumer consumer = new KnownScanConsumer();
186         consumer.setIncludes( ARTIFACT_PATTERNS );
187         knownConsumers.add( consumer );
188 
189         List<InvalidRepositoryContentConsumer> invalidConsumers = new ArrayList<>();
190         InvalidScanConsumer badconsumer = new InvalidScanConsumer();
191         invalidConsumers.add( badconsumer );
192 
193         RepositoryScanner scanner = lookupRepositoryScanner();
194         RepositoryScanStatistics stats =
195             scanner.scan( repository, knownConsumers, invalidConsumers, getIgnoreList(), RepositoryScanner.FRESH_SCAN );
196 
197         assertNotNull( "Stats should not be null.", stats );
198         assertEquals( "Stats.totalFileCount", 4, stats.getTotalFileCount() );
199         assertEquals( "Stats.newFileCount", 4, stats.getNewFileCount() );
200         assertEquals( "Processed Count", 3, consumer.getProcessCount() );
201         assertEquals( "Processed Count (of invalid items)", 1, badconsumer.getProcessCount() );
202     }
203 
204     @Test
205     public void testTimestampRepositoryScannerProcessUnmodified()
206         throws Exception
207     {
208         ManagedRepository repository = createSimpleRepository();
209 
210         List<KnownRepositoryContentConsumer> knownConsumers = new ArrayList<>();
211         KnownScanConsumer consumer = new KnownScanConsumer();
212         consumer.setProcessUnmodified( true );
213         consumer.setIncludes( ARTIFACT_PATTERNS );
214         knownConsumers.add( consumer );
215 
216         List<InvalidRepositoryContentConsumer> invalidConsumers = new ArrayList<>();
217         InvalidScanConsumer badconsumer = new InvalidScanConsumer();
218         invalidConsumers.add( badconsumer );
219 
220         RepositoryScanner scanner = lookupRepositoryScanner();
221         RepositoryScanStatistics stats = scanner.scan( repository, knownConsumers, invalidConsumers, getIgnoreList(),
222                                                        getTimestampAsMillis( "20061101.000000" ) );
223 
224         assertNotNull( "Stats should not be null.", stats );
225         assertEquals( "Stats.totalFileCount", 4, stats.getTotalFileCount() );
226         assertEquals( "Stats.newFileCount", 3, stats.getNewFileCount() );
227         assertEquals( "Processed Count", 3, consumer.getProcessCount() );
228         assertEquals( "Processed Count (of invalid items)", 1, badconsumer.getProcessCount() );
229     }
230 
231     @Test
232     public void testDefaultRepositoryScanner()
233         throws Exception
234     {
235         ManagedRepository repository = createDefaultRepository();
236 
237         List<KnownRepositoryContentConsumer> knownConsumers = new ArrayList<>();
238         KnownScanConsumer consumer = new KnownScanConsumer();
239         consumer.setIncludes(
240             new String[]{ "**/*.jar", "**/*.war", "**/*.pom", "**/maven-metadata.xml", "**/*-site.xml", "**/*.zip",
241                 "**/*.tar.gz", "**/*.sha1", "**/*.md5" }
242         );
243         knownConsumers.add( consumer );
244 
245         List<InvalidRepositoryContentConsumer> invalidConsumers = new ArrayList<>();
246         InvalidScanConsumer badconsumer = new InvalidScanConsumer();
247         invalidConsumers.add( badconsumer );
248 
249         RepositoryScanner scanner = lookupRepositoryScanner();
250         RepositoryScanStatistics stats =
251             scanner.scan( repository, knownConsumers, invalidConsumers, getIgnoreList(), RepositoryScanner.FRESH_SCAN );
252 
253         assertNotNull( "Stats should not be null.", stats );
254         assertMinimumHits( "Stats.totalFileCount", 17, stats.getTotalFileCount() );
255         assertMinimumHits( "Processed Count", 17, consumer.getProcessCount() );
256         assertEquals( "Processed Count (of invalid items):" + badconsumer.getPaths(), 6, badconsumer.getProcessCount() );
257     }
258 
259     @Test
260     public void testDefaultRepositoryArtifactScanner()
261         throws Exception
262     {
263         List<String> actualArtifactPaths = new ArrayList<>();
264 
265         actualArtifactPaths.add( "invalid/invalid/1.0-20050611.123456-1/invalid-1.0-20050611.123456-1.jar" );
266         actualArtifactPaths.add( "invalid/invalid/1.0-SNAPSHOT/invalid-1.0.jar" );
267         actualArtifactPaths.add( "invalid/invalid/1.0/invalid-1.0b.jar" );
268         actualArtifactPaths.add( "invalid/invalid/1.0/invalid-2.0.jar" );
269         actualArtifactPaths.add( "invalid/invalid-1.0.jar" );
270         actualArtifactPaths.add( "org/apache/maven/test/1.0-SNAPSHOT/wrong-artifactId-1.0-20050611.112233-1.jar" );
271         actualArtifactPaths.add( "org/apache/maven/test/1.0-SNAPSHOT/test-1.0-20050611.112233-1-javadoc.jar" );
272         actualArtifactPaths.add( "org/apache/maven/test/1.0-SNAPSHOT/test-1.0-20050611.112233-1.jar" );
273         actualArtifactPaths.add( "org/apache/maven/A/1.0/A-1.0.war" );
274         actualArtifactPaths.add( "org/apache/maven/A/1.0/A-1.0.pom" );
275         actualArtifactPaths.add( "org/apache/maven/B/2.0/B-2.0.pom" );
276         actualArtifactPaths.add( "org/apache/maven/B/1.0/B-1.0.pom" );
277         actualArtifactPaths.add( "org/apache/maven/some-ejb/1.0/some-ejb-1.0-client.jar" );
278         actualArtifactPaths.add( "org/apache/maven/C/1.0/C-1.0.war" );
279         actualArtifactPaths.add( "org/apache/maven/C/1.0/C-1.0.pom" );
280         actualArtifactPaths.add( "org/apache/maven/update/test-not-updated/1.0/test-not-updated-1.0.pom" );
281         actualArtifactPaths.add( "org/apache/maven/update/test-not-updated/1.0/test-not-updated-1.0.jar" );
282         actualArtifactPaths.add( "org/apache/maven/update/test-updated/1.0/test-updated-1.0.pom" );
283         actualArtifactPaths.add( "org/apache/maven/update/test-updated/1.0/test-updated-1.0.jar" );
284         actualArtifactPaths.add( "org/apache/maven/discovery/1.0/discovery-1.0.pom" );
285         actualArtifactPaths.add( "org/apache/maven/testing/1.0/testing-1.0-test-sources.jar" );
286         actualArtifactPaths.add( "org/apache/maven/testing/1.0/testing-1.0.jar" );
287         actualArtifactPaths.add( "org/apache/maven/testing/1.0/testing-1.0-sources.jar" );
288         actualArtifactPaths.add( "org/apache/maven/testing/1.0/testing-1.0.zip" );
289         actualArtifactPaths.add( "org/apache/maven/testing/1.0/testing-1.0.tar.gz" );
290         actualArtifactPaths.add( "org/apache/maven/samplejar/2.0/samplejar-2.0.pom" );
291         actualArtifactPaths.add( "org/apache/maven/samplejar/2.0/samplejar-2.0.jar" );
292         actualArtifactPaths.add( "org/apache/maven/samplejar/1.0/samplejar-1.0.pom" );
293         actualArtifactPaths.add( "org/apache/maven/samplejar/1.0/samplejar-1.0.jar" );
294         actualArtifactPaths.add( "org/apache/testgroup/discovery/1.0/discovery-1.0.pom" );
295         actualArtifactPaths.add( "javax/sql/jdbc/2.0/jdbc-2.0.jar" );
296 
297         ManagedRepository repository = createDefaultRepository();
298 
299         List<KnownRepositoryContentConsumer> knownConsumers = new ArrayList<>();
300         KnownScanConsumer consumer = new KnownScanConsumer();
301         consumer.setIncludes( ARTIFACT_PATTERNS );
302         knownConsumers.add( consumer );
303 
304         List<InvalidRepositoryContentConsumer> invalidConsumers = new ArrayList<>();
305         InvalidScanConsumer badconsumer = new InvalidScanConsumer();
306         invalidConsumers.add( badconsumer );
307 
308         RepositoryScanner scanner = lookupRepositoryScanner();
309         RepositoryScanStatistics stats =
310             scanner.scan( repository, knownConsumers, invalidConsumers, getIgnoreList(), RepositoryScanner.FRESH_SCAN );
311 
312         assertNotNull( "Stats should not be null.", stats );
313         assertMinimumHits( "Stats.totalFileCount", actualArtifactPaths.size(), stats.getTotalFileCount() );
314         assertMinimumHits( "Processed Count", actualArtifactPaths.size(), consumer.getProcessCount() );
315     }
316 
317     @Test
318     public void testDefaultRepositoryMetadataScanner()
319         throws Exception
320     {
321         List<String> actualMetadataPaths = new ArrayList<>();
322 
323         actualMetadataPaths.add( "org/apache/maven/some-ejb/1.0/maven-metadata.xml" );
324         actualMetadataPaths.add( "org/apache/maven/update/test-not-updated/maven-metadata.xml" );
325         actualMetadataPaths.add( "org/apache/maven/update/test-updated/maven-metadata.xml" );
326         actualMetadataPaths.add( "org/apache/maven/maven-metadata.xml" );
327         actualMetadataPaths.add( "org/apache/testgroup/discovery/1.0/maven-metadata.xml" );
328         actualMetadataPaths.add( "org/apache/testgroup/discovery/maven-metadata.xml" );
329         actualMetadataPaths.add( "javax/sql/jdbc/2.0/maven-metadata-repository.xml" );
330         actualMetadataPaths.add( "javax/sql/jdbc/maven-metadata-repository.xml" );
331         actualMetadataPaths.add( "javax/sql/maven-metadata-repository.xml" );
332         actualMetadataPaths.add( "javax/maven-metadata.xml" );
333 
334         ManagedRepository repository = createDefaultRepository();
335 
336         List<KnownRepositoryContentConsumer> knownConsumers = new ArrayList<>();
337         KnownScanConsumer knownConsumer = new KnownScanConsumer();
338         knownConsumer.setIncludes( new String[]{ "**/maven-metadata*.xml" } );
339         knownConsumers.add( knownConsumer );
340 
341         List<InvalidRepositoryContentConsumer> invalidConsumers = new ArrayList<>();
342         InvalidScanConsumer badconsumer = new InvalidScanConsumer();
343         invalidConsumers.add( badconsumer );
344 
345         RepositoryScanner scanner = lookupRepositoryScanner();
346         RepositoryScanStatistics stats =
347             scanner.scan( repository, knownConsumers, invalidConsumers, getIgnoreList(), RepositoryScanner.FRESH_SCAN );
348 
349         assertNotNull( "Stats should not be null.", stats );
350         assertMinimumHits( "Stats.totalFileCount", actualMetadataPaths.size(), stats.getTotalFileCount() );
351         assertMinimumHits( "Processed Count", actualMetadataPaths.size(), knownConsumer.getProcessCount() );
352     }
353 
354     @Test
355     public void testDefaultRepositoryProjectScanner()
356         throws Exception
357     {
358         List<String> actualProjectPaths = new ArrayList<>();
359 
360         actualProjectPaths.add( "org/apache/maven/A/1.0/A-1.0.pom" );
361         actualProjectPaths.add( "org/apache/maven/B/2.0/B-2.0.pom" );
362         actualProjectPaths.add( "org/apache/maven/B/1.0/B-1.0.pom" );
363         actualProjectPaths.add( "org/apache/maven/C/1.0/C-1.0.pom" );
364         actualProjectPaths.add( "org/apache/maven/update/test-not-updated/1.0/test-not-updated-1.0.pom" );
365         actualProjectPaths.add( "org/apache/maven/update/test-updated/1.0/test-updated-1.0.pom" );
366         actualProjectPaths.add( "org/apache/maven/discovery/1.0/discovery-1.0.pom" );
367         actualProjectPaths.add( "org/apache/maven/samplejar/2.0/samplejar-2.0.pom" );
368         actualProjectPaths.add( "org/apache/maven/samplejar/1.0/samplejar-1.0.pom" );
369         actualProjectPaths.add( "org/apache/testgroup/discovery/1.0/discovery-1.0.pom" );
370 
371         ManagedRepository repository = createDefaultRepository();
372 
373         List<KnownRepositoryContentConsumer> knownConsumers = new ArrayList<>();
374         KnownScanConsumer consumer = new KnownScanConsumer();
375         consumer.setIncludes( new String[]{ "**/*.pom" } );
376         knownConsumers.add( consumer );
377 
378         List<InvalidRepositoryContentConsumer> invalidConsumers = new ArrayList<>();
379         InvalidScanConsumer badconsumer = new InvalidScanConsumer();
380         invalidConsumers.add( badconsumer );
381 
382         RepositoryScanner scanner = lookupRepositoryScanner();
383         RepositoryScanStatistics stats =
384             scanner.scan( repository, knownConsumers, invalidConsumers, getIgnoreList(), RepositoryScanner.FRESH_SCAN );
385 
386         assertNotNull( "Stats should not be null.", stats );
387         assertMinimumHits( "Stats.totalFileCount", actualProjectPaths.size(), stats.getTotalFileCount() );
388         assertMinimumHits( "Processed Count", actualProjectPaths.size(), consumer.getProcessCount() );
389     }
390 
391     @Test
392     public void testLegacyRepositoryArtifactScanner()
393         throws Exception
394     {
395         List<String> actualArtifactPaths = new ArrayList<>();
396 
397         actualArtifactPaths.add( "invalid/jars/1.0/invalid-1.0.jar" );
398         actualArtifactPaths.add( "invalid/jars/invalid-1.0.rar" );
399         actualArtifactPaths.add( "invalid/jars/invalid.jar" );
400         actualArtifactPaths.add( "invalid/invalid-1.0.jar" );
401         actualArtifactPaths.add( "javax.sql/jars/jdbc-2.0.jar" );
402         actualArtifactPaths.add( "org.apache.maven/jars/some-ejb-1.0-client.jar" );
403         actualArtifactPaths.add( "org.apache.maven/jars/testing-1.0.jar" );
404         actualArtifactPaths.add( "org.apache.maven/jars/testing-1.0-sources.jar" );
405         actualArtifactPaths.add( "org.apache.maven/jars/testing-UNKNOWN.jar" );
406         actualArtifactPaths.add( "org.apache.maven/jars/testing-1.0.zip" );
407         actualArtifactPaths.add( "org.apache.maven/jars/testing-1.0-20050611.112233-1.jar" );
408         actualArtifactPaths.add( "org.apache.maven/jars/testing-1.0.tar.gz" );
409         actualArtifactPaths.add( "org.apache.maven.update/jars/test-not-updated-1.0.jar" );
410         actualArtifactPaths.add( "org.apache.maven.update/jars/test-updated-1.0.jar" );
411 
412         ManagedRepository repository = createLegacyRepository();
413 
414         List<KnownRepositoryContentConsumer> knownConsumers = new ArrayList<>();
415         KnownScanConsumer consumer = new KnownScanConsumer();
416         consumer.setIncludes( ARTIFACT_PATTERNS );
417         knownConsumers.add( consumer );
418 
419         List<InvalidRepositoryContentConsumer> invalidConsumers = new ArrayList<>();
420         InvalidScanConsumer badconsumer = new InvalidScanConsumer();
421         invalidConsumers.add( badconsumer );
422 
423         RepositoryScanner scanner = lookupRepositoryScanner();
424         RepositoryScanStatistics stats =
425             scanner.scan( repository, knownConsumers, invalidConsumers, getIgnoreList(), RepositoryScanner.FRESH_SCAN );
426 
427         assertNotNull( "Stats should not be null.", stats );
428         assertMinimumHits( "Stats.totalFileCount", actualArtifactPaths.size(), stats.getTotalFileCount() );
429         assertMinimumHits( "Processed Count", actualArtifactPaths.size(), consumer.getProcessCount() );
430     }
431 }