View Javadoc
1   package org.eclipse.aether.internal.impl.collect.bf;
2   
3   /*
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *  http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing,
15   * software distributed under the License is distributed on an
16   * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
17   * KIND, either express or implied.  See the License for the
18   * specific language governing permissions and limitations
19   * under the License.
20   */
21  
22  import javax.inject.Inject;
23  import javax.inject.Named;
24  import javax.inject.Singleton;
25  
26  import java.util.ArrayDeque;
27  import java.util.ArrayList;
28  import java.util.Collections;
29  import java.util.LinkedHashMap;
30  import java.util.List;
31  import java.util.Map;
32  import java.util.Optional;
33  import java.util.Queue;
34  import java.util.Set;
35  import java.util.concurrent.Callable;
36  import java.util.concurrent.ConcurrentHashMap;
37  import java.util.concurrent.ExecutorService;
38  import java.util.concurrent.Future;
39  import java.util.concurrent.LinkedBlockingQueue;
40  import java.util.concurrent.ThreadPoolExecutor;
41  import java.util.concurrent.TimeUnit;
42  import java.util.stream.Collectors;
43  import java.util.stream.Stream;
44  
45  import org.apache.commons.lang3.concurrent.ConcurrentUtils;
46  import org.eclipse.aether.RepositorySystemSession;
47  import org.eclipse.aether.RequestTrace;
48  import org.eclipse.aether.artifact.Artifact;
49  import org.eclipse.aether.artifact.ArtifactType;
50  import org.eclipse.aether.artifact.DefaultArtifact;
51  import org.eclipse.aether.collection.CollectRequest;
52  import org.eclipse.aether.collection.DependencyManager;
53  import org.eclipse.aether.collection.DependencySelector;
54  import org.eclipse.aether.collection.DependencyTraverser;
55  import org.eclipse.aether.collection.VersionFilter;
56  import org.eclipse.aether.graph.DefaultDependencyNode;
57  import org.eclipse.aether.graph.Dependency;
58  import org.eclipse.aether.graph.DependencyNode;
59  import org.eclipse.aether.impl.ArtifactDescriptorReader;
60  import org.eclipse.aether.impl.RemoteRepositoryManager;
61  import org.eclipse.aether.impl.VersionRangeResolver;
62  import org.eclipse.aether.internal.impl.collect.DataPool;
63  import org.eclipse.aether.internal.impl.collect.DefaultDependencyCollectionContext;
64  import org.eclipse.aether.internal.impl.collect.DefaultVersionFilterContext;
65  import org.eclipse.aether.internal.impl.collect.DependencyCollectorDelegate;
66  import org.eclipse.aether.internal.impl.collect.PremanagedDependency;
67  import org.eclipse.aether.repository.RemoteRepository;
68  import org.eclipse.aether.resolution.ArtifactDescriptorException;
69  import org.eclipse.aether.resolution.ArtifactDescriptorRequest;
70  import org.eclipse.aether.resolution.ArtifactDescriptorResult;
71  import org.eclipse.aether.resolution.VersionRangeRequest;
72  import org.eclipse.aether.resolution.VersionRangeResult;
73  import org.eclipse.aether.spi.locator.Service;
74  import org.eclipse.aether.util.ConfigUtils;
75  import org.eclipse.aether.util.artifact.ArtifactIdUtils;
76  import org.eclipse.aether.util.concurrency.WorkerThreadFactory;
77  import org.eclipse.aether.util.graph.manager.DependencyManagerUtils;
78  import org.eclipse.aether.version.Version;
79  import org.slf4j.Logger;
80  import org.slf4j.LoggerFactory;
81  
82  import static org.eclipse.aether.internal.impl.collect.DefaultDependencyCycle.find;
83  
84  /**
85   * Breadth-first {@link org.eclipse.aether.impl.DependencyCollector}
86   *
87   * @since 1.8.0
88   */
89  @Singleton
90  @Named( BfDependencyCollector.NAME )
91  public class BfDependencyCollector
92      extends DependencyCollectorDelegate implements Service
93  {
94      public static final String NAME = "bf";
95  
96      /**
97       * The key in the repository session's {@link RepositorySystemSession#getConfigProperties()
98       * configuration properties} used to store a {@link Boolean} flag controlling the resolver's skip mode.
99       *
100      * @since 1.8.0
101      */
102     static final String CONFIG_PROP_SKIPPER = "aether.dependencyCollector.bf.skipper";
103 
104     /**
105      * The default value for {@link #CONFIG_PROP_SKIPPER}, {@code true}.
106      *
107      * @since 1.8.0
108      */
109     static final boolean CONFIG_PROP_SKIPPER_DEFAULT = true;
110 
111     /**
112      * The count of threads to be used when collecting POMs in parallel, default value 5.
113      *
114      * @since 1.9.0
115      */
116     static final String CONFIG_PROP_THREADS = "aether.dependencyCollector.bf.threads";
117 
118     /**
119      * Default ctor for SL.
120      *
121      * @deprecated Will be dropped once SL gone.
122      */
123     @Deprecated
124     public BfDependencyCollector()
125     {
126         // enables default constructor
127     }
128 
129     @Inject
130     BfDependencyCollector( RemoteRepositoryManager remoteRepositoryManager,
131                            ArtifactDescriptorReader artifactDescriptorReader,
132                            VersionRangeResolver versionRangeResolver )
133     {
134         super( remoteRepositoryManager, artifactDescriptorReader, versionRangeResolver );
135     }
136 
137     @SuppressWarnings( "checkstyle:parameternumber" )
138     @Override
139     protected void doCollectDependencies( RepositorySystemSession session, RequestTrace trace, DataPool pool,
140                                           DefaultDependencyCollectionContext context,
141                                           DefaultVersionFilterContext versionContext,
142                                           CollectRequest request, DependencyNode node,
143                                           List<RemoteRepository> repositories, List<Dependency> dependencies,
144                                           List<Dependency> managedDependencies, Results results )
145     {
146         boolean useSkip = ConfigUtils.getBoolean(
147                 session, CONFIG_PROP_SKIPPER_DEFAULT, CONFIG_PROP_SKIPPER
148         );
149         if ( useSkip )
150         {
151             logger.debug( "Collector skip mode enabled" );
152         }
153 
154         Args args =
155                 new Args( session, pool, context, versionContext, request,
156                         useSkip ? DependencyResolutionSkipper.defaultSkipper()
157                                 : DependencyResolutionSkipper.neverSkipper(),
158                         new ParallelDescriptorResolver( session ) );
159 
160         DependencySelector rootDepSelector = session.getDependencySelector() != null
161                 ? session.getDependencySelector().deriveChildSelector( context ) : null;
162         DependencyManager rootDepManager = session.getDependencyManager() != null
163                 ? session.getDependencyManager().deriveChildManager( context ) : null;
164         DependencyTraverser rootDepTraverser = session.getDependencyTraverser() != null
165                 ? session.getDependencyTraverser().deriveChildTraverser( context ) : null;
166         VersionFilter rootVerFilter = session.getVersionFilter() != null
167                 ? session.getVersionFilter().deriveChildFilter( context ) : null;
168 
169         List<DependencyNode> parents = Collections.singletonList( node );
170         for ( Dependency dependency : dependencies )
171         {
172             RequestTrace childTrace = collectStepTrace( trace, args.request.getRequestContext(), parents,
173                     dependency );
174             DependencyProcessingContext processingContext =
175                     new DependencyProcessingContext( rootDepSelector, rootDepManager, rootDepTraverser,
176                             rootVerFilter, childTrace, repositories, managedDependencies, parents, dependency,
177                             PremanagedDependency.create( rootDepManager, dependency,
178                                     false, args.premanagedState ) );
179             if ( !filter( processingContext ) )
180             {
181                 processingContext.withDependency( processingContext.premanagedDependency.getManagedDependency() );
182                 resolveArtifactDescriptorAsync( args, processingContext, results );
183                 args.dependencyProcessingQueue.add( processingContext );
184             }
185         }
186 
187         while ( !args.dependencyProcessingQueue.isEmpty() )
188         {
189             processDependency( args, results, args.dependencyProcessingQueue.remove(), Collections.emptyList(),
190                     false );
191         }
192 
193         args.resolver.shutdown();
194         args.skipper.report();
195     }
196 
197     @SuppressWarnings( "checkstyle:parameternumber" )
198     private void processDependency( Args args, Results results,
199                                     DependencyProcessingContext context, List<Artifact> relocations,
200                                     boolean disableVersionManagement )
201     {
202         Dependency dependency = context.dependency;
203         PremanagedDependency preManaged = context.premanagedDependency;
204 
205         boolean noDescriptor = isLackingDescriptor( dependency.getArtifact() );
206         boolean traverse =
207                 !noDescriptor && ( context.depTraverser == null || context.depTraverser.traverseDependency(
208                         dependency ) );
209 
210         Future<DescriptorResolutionResult> resolutionResultFuture = args.resolver.find( dependency.getArtifact() );
211         DescriptorResolutionResult resolutionResult;
212         VersionRangeResult rangeResult;
213         try
214         {
215             resolutionResult = resolutionResultFuture.get();
216             rangeResult = resolutionResult.rangeResult;
217         }
218         catch ( Exception e )
219         {
220             results.addException( dependency, e, context.parents );
221             return;
222         }
223 
224         Set<Version> versions = resolutionResult.descriptors.keySet();
225         for ( Version version : versions )
226         {
227             Artifact originalArtifact = dependency.getArtifact().setVersion( version.toString() );
228             Dependency d = dependency.setArtifact( originalArtifact );
229 
230             final ArtifactDescriptorResult descriptorResult = resolutionResult.descriptors.get( version );
231             if ( descriptorResult != null )
232             {
233                 d = d.setArtifact( descriptorResult.getArtifact() );
234 
235                 int cycleEntry = find( context.parents, d.getArtifact() );
236                 if ( cycleEntry >= 0 )
237                 {
238                     results.addCycle( context.parents, cycleEntry, d );
239                     DependencyNode cycleNode = context.parents.get( cycleEntry );
240                     if ( cycleNode.getDependency() != null )
241                     {
242                         DefaultDependencyNode child =
243                                 createDependencyNode( relocations, preManaged, rangeResult, version, d,
244                                         descriptorResult, cycleNode );
245                         context.getParent().getChildren().add( child );
246                         continue;
247                     }
248                 }
249 
250                 if ( !descriptorResult.getRelocations().isEmpty() )
251                 {
252                     boolean disableVersionManagementSubsequently =
253                         originalArtifact.getGroupId().equals( d.getArtifact().getGroupId() )
254                             && originalArtifact.getArtifactId().equals( d.getArtifact().getArtifactId() );
255 
256                     PremanagedDependency premanagedDependency =
257                             PremanagedDependency.create( context.depManager, d, disableVersionManagementSubsequently,
258                                     args.premanagedState );
259                     DependencyProcessingContext relocatedContext =
260                             new DependencyProcessingContext( context.depSelector, context.depManager,
261                                     context.depTraverser, context.verFilter,
262                                     context.trace, context.repositories, descriptorResult.getManagedDependencies(),
263                                     context.parents,
264                                     d, premanagedDependency );
265 
266                     if ( !filter( relocatedContext ) )
267                     {
268                         relocatedContext.withDependency( premanagedDependency.getManagedDependency() );
269                         resolveArtifactDescriptorAsync( args, relocatedContext, results );
270                         processDependency( args, results, relocatedContext, descriptorResult.getRelocations(),
271                                 disableVersionManagementSubsequently );
272                     }
273 
274                     return;
275                 }
276                 else
277                 {
278                     d = args.pool.intern( d.setArtifact( args.pool.intern( d.getArtifact() ) ) );
279 
280                     List<RemoteRepository> repos =
281                         getRemoteRepositories( rangeResult.getRepository( version ), context.repositories );
282 
283                     DefaultDependencyNode child =
284                         createDependencyNode( relocations, preManaged, rangeResult, version, d,
285                                               descriptorResult.getAliases(), repos, args.request.getRequestContext() );
286 
287                     context.getParent().getChildren().add( child );
288 
289                     boolean recurse = traverse && !descriptorResult.getDependencies().isEmpty();
290                     DependencyProcessingContext parentContext = context.withDependency( d );
291                     if ( recurse )
292                     {
293                         doRecurse( args, parentContext, descriptorResult, child, results,
294                                 disableVersionManagement );
295                     }
296                     else if ( !args.skipper.skipResolution( child, parentContext.parents ) )
297                     {
298                         List<DependencyNode> parents = new ArrayList<>( parentContext.parents.size() + 1 );
299                         parents.addAll( parentContext.parents );
300                         parents.add( child );
301                         args.skipper.cache( child, parents );
302                     }
303                 }
304             }
305             else
306             {
307                 List<RemoteRepository> repos =
308                     getRemoteRepositories( rangeResult.getRepository( version ), context.repositories );
309                 DefaultDependencyNode child =
310                     createDependencyNode( relocations, preManaged, rangeResult, version, d, null, repos,
311                                           args.request.getRequestContext() );
312                 context.getParent().getChildren().add( child );
313             }
314         }
315     }
316 
317     @SuppressWarnings( "checkstyle:parameternumber" )
318     private void doRecurse( Args args, DependencyProcessingContext parentContext,
319                             ArtifactDescriptorResult descriptorResult, DefaultDependencyNode child, Results results,
320                             boolean disableVersionManagement )
321     {
322         DefaultDependencyCollectionContext context = args.collectionContext;
323         context.set( parentContext.dependency, descriptorResult.getManagedDependencies() );
324 
325         DependencySelector childSelector =
326                 parentContext.depSelector != null ? parentContext.depSelector.deriveChildSelector( context ) : null;
327         DependencyManager childManager =
328                 parentContext.depManager != null ? parentContext.depManager.deriveChildManager( context ) : null;
329         DependencyTraverser childTraverser =
330                 parentContext.depTraverser != null ? parentContext.depTraverser.deriveChildTraverser( context ) : null;
331         VersionFilter childFilter =
332                 parentContext.verFilter != null ? parentContext.verFilter.deriveChildFilter( context ) : null;
333 
334         final List<RemoteRepository> childRepos =
335                 args.ignoreRepos
336                         ? parentContext.repositories
337                         : remoteRepositoryManager.aggregateRepositories( args.session, parentContext.repositories,
338                         descriptorResult.getRepositories(), true );
339 
340         Object key =
341                 args.pool.toKey( parentContext.dependency.getArtifact(), childRepos, childSelector, childManager,
342                         childTraverser, childFilter );
343 
344         List<DependencyNode> children = args.pool.getChildren( key );
345         if ( children == null )
346         {
347             boolean skipResolution = args.skipper.skipResolution( child, parentContext.parents );
348             if ( !skipResolution )
349             {
350                 List<DependencyNode> parents = new ArrayList<>( parentContext.parents.size() + 1 );
351                 parents.addAll( parentContext.parents );
352                 parents.add( child );
353                 for ( Dependency dependency : descriptorResult.getDependencies() )
354                 {
355                     RequestTrace childTrace =
356                             collectStepTrace( parentContext.trace, args.request.getRequestContext(), parents,
357                                     dependency );
358                     PremanagedDependency premanagedDependency =
359                             PremanagedDependency.create( childManager, dependency, disableVersionManagement,
360                                     args.premanagedState );
361                     DependencyProcessingContext processingContext =
362                             new DependencyProcessingContext( childSelector, childManager, childTraverser, childFilter,
363                                     childTrace, childRepos, descriptorResult.getManagedDependencies(), parents,
364                                     dependency, premanagedDependency );
365                     if ( !filter( processingContext ) )
366                     {
367                         //resolve descriptors ahead for managed dependency
368                         processingContext.withDependency(
369                                 processingContext.premanagedDependency.getManagedDependency() );
370                         resolveArtifactDescriptorAsync( args, processingContext, results );
371                         args.dependencyProcessingQueue.add( processingContext );
372                     }
373                 }
374                 args.pool.putChildren( key, child.getChildren() );
375                 args.skipper.cache( child, parents );
376             }
377         }
378         else
379         {
380             child.setChildren( children );
381         }
382     }
383 
384     private boolean filter( DependencyProcessingContext context )
385     {
386         return context.depSelector != null && !context.depSelector.selectDependency( context.dependency );
387     }
388 
389 
390     private void resolveArtifactDescriptorAsync( Args args, DependencyProcessingContext context,
391                                                  Results results )
392     {
393         Dependency dependency = context.dependency;
394         args.resolver.resolveDescriptors( dependency.getArtifact(), () ->
395         {
396             VersionRangeRequest rangeRequest =
397                     createVersionRangeRequest( args.request.getRequestContext(), context.trace, context.repositories,
398                             dependency );
399             VersionRangeResult rangeResult = cachedResolveRangeResult( rangeRequest, args.pool, args.session );
400             List<? extends Version> versions = filterVersions( dependency, rangeResult, context.verFilter,
401                     args.versionContext );
402 
403             //resolve newer version first to maximize benefits of skipper
404             Collections.reverse( versions );
405 
406             Map<Version, ArtifactDescriptorResult> descriptors = new ConcurrentHashMap<>( versions.size() );
407             Stream<? extends Version> stream = versions.size() > 1 ? versions.parallelStream() : versions.stream();
408             stream.forEach( version ->
409                     Optional.ofNullable( resolveDescriptorForVersion( args, context, results, dependency, version ) )
410                             .ifPresent( r -> descriptors.put( version, r ) )
411             );
412 
413             DescriptorResolutionResult resolutionResult =
414                     new DescriptorResolutionResult( dependency.getArtifact(), rangeResult );
415             //keep original sequence
416             versions.forEach( version -> resolutionResult.descriptors.put( version, descriptors.get( version ) ) );
417             //populate for versions in version range
418             resolutionResult.flatten().forEach( dr -> args.resolver.cacheVersionRangeDescriptor( dr.artifact, dr ) );
419 
420             return resolutionResult;
421         } );
422     }
423 
424     private ArtifactDescriptorResult resolveDescriptorForVersion( Args args, DependencyProcessingContext context,
425                                                                   Results results, Dependency dependency,
426                                                                   Version version )
427     {
428         Artifact original = dependency.getArtifact();
429         Artifact newArtifact = new DefaultArtifact( original.getGroupId(),
430                 original.getArtifactId(), original.getClassifier(), original.getExtension(),
431                 version.toString(), original.getProperties(), (ArtifactType) null );
432         Dependency newDependency = new Dependency( newArtifact, dependency.getScope(), dependency.isOptional(),
433                 dependency.getExclusions() );
434         DependencyProcessingContext newContext = context.copy();
435 
436         ArtifactDescriptorRequest descriptorRequest =
437                 createArtifactDescriptorRequest( args.request.getRequestContext(), context.trace,
438                         newContext.repositories, newDependency );
439         return isLackingDescriptor( newArtifact )
440                 ? new ArtifactDescriptorResult( descriptorRequest )
441                 : resolveCachedArtifactDescriptor( args.pool, descriptorRequest, args.session,
442                 newContext.withDependency( newDependency ), results );
443     }
444 
445     private ArtifactDescriptorResult resolveCachedArtifactDescriptor( DataPool pool,
446                                                                       ArtifactDescriptorRequest descriptorRequest,
447                                                                       RepositorySystemSession session,
448                                                                       DependencyProcessingContext context,
449                                                                       Results results )
450     {
451         Object key = pool.toKey( descriptorRequest );
452         ArtifactDescriptorResult descriptorResult = pool.getDescriptor( key, descriptorRequest );
453         if ( descriptorResult == null )
454         {
455             try
456             {
457                 descriptorResult = descriptorReader.readArtifactDescriptor( session, descriptorRequest );
458                 pool.putDescriptor( key, descriptorResult );
459             }
460             catch ( ArtifactDescriptorException e )
461             {
462                 results.addException( context.dependency, e, context.parents );
463                 pool.putDescriptor( key, e );
464                 return null;
465             }
466 
467         }
468         else if ( descriptorResult == DataPool.NO_DESCRIPTOR )
469         {
470             return null;
471         }
472 
473         return descriptorResult;
474     }
475 
476     static class ParallelDescriptorResolver
477     {
478         final ExecutorService executorService;
479 
480         /**
481          * Artifact ID -> Future of DescriptorResolutionResult
482          */
483         final Map<String, Future<DescriptorResolutionResult>> results = new ConcurrentHashMap<>( 256 );
484         final Logger logger = LoggerFactory.getLogger( getClass() );
485 
486         ParallelDescriptorResolver( RepositorySystemSession session )
487         {
488             this.executorService = getExecutorService( session );
489         }
490 
491         void resolveDescriptors( Artifact artifact, Callable<DescriptorResolutionResult> callable )
492         {
493             results.computeIfAbsent( ArtifactIdUtils.toId( artifact ),
494                     key -> this.executorService.submit( callable ) );
495         }
496 
497         void cacheVersionRangeDescriptor( Artifact artifact, DescriptorResolutionResult resolutionResult )
498         {
499             results.computeIfAbsent( ArtifactIdUtils.toId( artifact ),
500                     key -> ConcurrentUtils.constantFuture( resolutionResult ) );
501         }
502 
503         Future<DescriptorResolutionResult> find( Artifact artifact )
504         {
505             return results.get( ArtifactIdUtils.toId( artifact ) );
506         }
507 
508         void shutdown()
509         {
510             executorService.shutdown();
511         }
512 
513         private ExecutorService getExecutorService( RepositorySystemSession session )
514         {
515             int nThreads = ConfigUtils.getInteger( session, 5, CONFIG_PROP_THREADS, "maven.artifact.threads" );
516             logger.debug( "Created thread pool with {} threads to resolve descriptors.", nThreads );
517             return new ThreadPoolExecutor( nThreads, nThreads, 3L, TimeUnit.SECONDS, new LinkedBlockingQueue<>(),
518                     new WorkerThreadFactory( getClass().getSimpleName() ) );
519         }
520     }
521 
522     static class DescriptorResolutionResult
523     {
524         Artifact artifact;
525 
526         VersionRangeResult rangeResult;
527 
528         Map<Version, ArtifactDescriptorResult> descriptors;
529 
530         DescriptorResolutionResult( Artifact artifact, VersionRangeResult rangeResult )
531         {
532             this.artifact = artifact;
533             this.rangeResult = rangeResult;
534             this.descriptors = new LinkedHashMap<>( rangeResult.getVersions().size() );
535         }
536 
537         DescriptorResolutionResult( VersionRangeResult rangeResult,
538                                     Version version, ArtifactDescriptorResult descriptor )
539         {
540             this( descriptor.getArtifact(), rangeResult );
541             this.descriptors.put( version, descriptor );
542         }
543 
544         List<DescriptorResolutionResult> flatten()
545         {
546             if ( descriptors.size() > 1 )
547             {
548                 return descriptors.entrySet().stream()
549                         .map( e -> new DescriptorResolutionResult( rangeResult, e.getKey(), e.getValue() ) )
550                         .collect( Collectors.toList() );
551             }
552             else
553             {
554                 return Collections.emptyList();
555             }
556         }
557     }
558 
559     static class Args
560     {
561 
562         final RepositorySystemSession session;
563 
564         final boolean ignoreRepos;
565 
566         final boolean premanagedState;
567 
568         final DataPool pool;
569 
570         final Queue<DependencyProcessingContext> dependencyProcessingQueue = new ArrayDeque<>( 128 );
571 
572         final DefaultDependencyCollectionContext collectionContext;
573 
574         final DefaultVersionFilterContext versionContext;
575 
576         final CollectRequest request;
577 
578         final DependencyResolutionSkipper skipper;
579 
580         final ParallelDescriptorResolver resolver;
581 
582         Args( RepositorySystemSession session, DataPool pool,
583               DefaultDependencyCollectionContext collectionContext, DefaultVersionFilterContext versionContext,
584               CollectRequest request, DependencyResolutionSkipper skipper,
585               ParallelDescriptorResolver resolver )
586         {
587             this.session = session;
588             this.request = request;
589             this.ignoreRepos = session.isIgnoreArtifactDescriptorRepositories();
590             this.premanagedState = ConfigUtils.getBoolean( session, false, DependencyManagerUtils.CONFIG_PROP_VERBOSE );
591             this.pool = pool;
592             this.collectionContext = collectionContext;
593             this.versionContext = versionContext;
594             this.skipper = skipper;
595             this.resolver = resolver;
596         }
597 
598     }
599 
600 }