Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *   http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing,
013 * software distributed under the License is distributed on an
014 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
015 * KIND, either express or implied.  See the License for the
016 * specific language governing permissions and limitations
017 * under the License.
018 */
019package org.eclipse.aether.internal.impl.collect.bf;
020
021import javax.inject.Inject;
022import javax.inject.Named;
023import javax.inject.Singleton;
024
025import java.io.Closeable;
026import java.util.ArrayDeque;
027import java.util.ArrayList;
028import java.util.Collections;
029import java.util.LinkedHashMap;
030import java.util.List;
031import java.util.Map;
032import java.util.Optional;
033import java.util.Queue;
034import java.util.Set;
035import java.util.concurrent.Callable;
036import java.util.concurrent.ConcurrentHashMap;
037import java.util.concurrent.ExecutionException;
038import java.util.concurrent.ExecutorService;
039import java.util.concurrent.Future;
040import java.util.concurrent.TimeUnit;
041import java.util.concurrent.TimeoutException;
042import java.util.stream.Collectors;
043import java.util.stream.Stream;
044
045import org.eclipse.aether.RepositorySystemSession;
046import org.eclipse.aether.RequestTrace;
047import org.eclipse.aether.artifact.Artifact;
048import org.eclipse.aether.collection.CollectRequest;
049import org.eclipse.aether.collection.DependencyManager;
050import org.eclipse.aether.collection.DependencySelector;
051import org.eclipse.aether.collection.DependencyTraverser;
052import org.eclipse.aether.collection.VersionFilter;
053import org.eclipse.aether.graph.DefaultDependencyNode;
054import org.eclipse.aether.graph.Dependency;
055import org.eclipse.aether.graph.DependencyNode;
056import org.eclipse.aether.impl.ArtifactDescriptorReader;
057import org.eclipse.aether.impl.RemoteRepositoryManager;
058import org.eclipse.aether.impl.VersionRangeResolver;
059import org.eclipse.aether.internal.impl.collect.DataPool;
060import org.eclipse.aether.internal.impl.collect.DefaultDependencyCollectionContext;
061import org.eclipse.aether.internal.impl.collect.DefaultVersionFilterContext;
062import org.eclipse.aether.internal.impl.collect.DependencyCollectorDelegate;
063import org.eclipse.aether.internal.impl.collect.PremanagedDependency;
064import org.eclipse.aether.repository.RemoteRepository;
065import org.eclipse.aether.resolution.ArtifactDescriptorException;
066import org.eclipse.aether.resolution.ArtifactDescriptorRequest;
067import org.eclipse.aether.resolution.ArtifactDescriptorResult;
068import org.eclipse.aether.resolution.VersionRangeRequest;
069import org.eclipse.aether.resolution.VersionRangeResult;
070import org.eclipse.aether.spi.locator.Service;
071import org.eclipse.aether.util.ConfigUtils;
072import org.eclipse.aether.util.artifact.ArtifactIdUtils;
073import org.eclipse.aether.util.concurrency.ExecutorUtils;
074import org.eclipse.aether.util.graph.manager.DependencyManagerUtils;
075import org.eclipse.aether.version.Version;
076
077import static org.eclipse.aether.internal.impl.collect.DefaultDependencyCycle.find;
078
079/**
080 * Breadth-first {@link org.eclipse.aether.impl.DependencyCollector}
081 *
082 * @since 1.8.0
083 */
084@Singleton
085@Named(BfDependencyCollector.NAME)
086public class BfDependencyCollector extends DependencyCollectorDelegate implements Service {
087    public static final String NAME = "bf";
088
089    /**
090     * The key in the repository session's {@link RepositorySystemSession#getConfigProperties()
091     * configuration properties} used to store a {@link Boolean} flag controlling the resolver's skip mode.
092     *
093     * @since 1.8.0
094     */
095    static final String CONFIG_PROP_SKIPPER = "aether.dependencyCollector.bf.skipper";
096
097    /**
098     * The default value for {@link #CONFIG_PROP_SKIPPER}, {@code true}.
099     *
100     * @since 1.8.0
101     */
102    static final boolean CONFIG_PROP_SKIPPER_DEFAULT = true;
103
104    /**
105     * The count of threads to be used when collecting POMs in parallel, default value 5.
106     *
107     * @since 1.9.0
108     */
109    static final String CONFIG_PROP_THREADS = "aether.dependencyCollector.bf.threads";
110
111    /**
112     * Default ctor for SL.
113     *
114     * @deprecated Will be dropped once SL gone.
115     */
116    @Deprecated
117    public BfDependencyCollector() {
118        // enables default constructor
119    }
120
121    @Inject
122    public BfDependencyCollector(
123            RemoteRepositoryManager remoteRepositoryManager,
124            ArtifactDescriptorReader artifactDescriptorReader,
125            VersionRangeResolver versionRangeResolver) {
126        super(remoteRepositoryManager, artifactDescriptorReader, versionRangeResolver);
127    }
128
129    @SuppressWarnings("checkstyle:parameternumber")
130    @Override
131    protected void doCollectDependencies(
132            RepositorySystemSession session,
133            RequestTrace trace,
134            DataPool pool,
135            DefaultDependencyCollectionContext context,
136            DefaultVersionFilterContext versionContext,
137            CollectRequest request,
138            DependencyNode node,
139            List<RemoteRepository> repositories,
140            List<Dependency> dependencies,
141            List<Dependency> managedDependencies,
142            Results results) {
143        boolean useSkip = ConfigUtils.getBoolean(session, CONFIG_PROP_SKIPPER_DEFAULT, CONFIG_PROP_SKIPPER);
144        int nThreads = ExecutorUtils.threadCount(session, 5, CONFIG_PROP_THREADS, "maven.artifact.threads");
145        logger.debug("Using thread pool with {} threads to resolve descriptors.", nThreads);
146
147        if (useSkip) {
148            logger.debug("Collector skip mode enabled");
149        }
150
151        try (DependencyResolutionSkipper skipper = useSkip
152                        ? DependencyResolutionSkipper.defaultSkipper()
153                        : DependencyResolutionSkipper.neverSkipper();
154                ParallelDescriptorResolver parallelDescriptorResolver = new ParallelDescriptorResolver(nThreads)) {
155            Args args = new Args(session, pool, context, versionContext, request, skipper, parallelDescriptorResolver);
156
157            DependencySelector rootDepSelector = session.getDependencySelector() != null
158                    ? session.getDependencySelector().deriveChildSelector(context)
159                    : null;
160            DependencyManager rootDepManager = session.getDependencyManager() != null
161                    ? session.getDependencyManager().deriveChildManager(context)
162                    : null;
163            DependencyTraverser rootDepTraverser = session.getDependencyTraverser() != null
164                    ? session.getDependencyTraverser().deriveChildTraverser(context)
165                    : null;
166            VersionFilter rootVerFilter = session.getVersionFilter() != null
167                    ? session.getVersionFilter().deriveChildFilter(context)
168                    : null;
169
170            List<DependencyNode> parents = Collections.singletonList(node);
171            for (Dependency dependency : dependencies) {
172                RequestTrace childTrace =
173                        collectStepTrace(trace, args.request.getRequestContext(), parents, dependency);
174                DependencyProcessingContext processingContext = new DependencyProcessingContext(
175                        rootDepSelector,
176                        rootDepManager,
177                        rootDepTraverser,
178                        rootVerFilter,
179                        childTrace,
180                        repositories,
181                        managedDependencies,
182                        parents,
183                        dependency,
184                        PremanagedDependency.create(rootDepManager, dependency, false, args.premanagedState));
185                if (!filter(processingContext)) {
186                    processingContext.withDependency(processingContext.premanagedDependency.getManagedDependency());
187                    resolveArtifactDescriptorAsync(args, processingContext, results);
188                    args.dependencyProcessingQueue.add(processingContext);
189                }
190            }
191
192            while (!args.dependencyProcessingQueue.isEmpty()) {
193                processDependency(
194                        args, results, args.dependencyProcessingQueue.remove(), Collections.emptyList(), false);
195            }
196        }
197    }
198
199    @SuppressWarnings("checkstyle:parameternumber")
200    private void processDependency(
201            Args args,
202            Results results,
203            DependencyProcessingContext context,
204            List<Artifact> relocations,
205            boolean disableVersionManagement) {
206        Dependency dependency = context.dependency;
207        PremanagedDependency preManaged = context.premanagedDependency;
208
209        boolean noDescriptor = isLackingDescriptor(dependency.getArtifact());
210        boolean traverse =
211                !noDescriptor && (context.depTraverser == null || context.depTraverser.traverseDependency(dependency));
212
213        Future<DescriptorResolutionResult> resolutionResultFuture = args.resolver.find(dependency.getArtifact());
214        DescriptorResolutionResult resolutionResult;
215        VersionRangeResult rangeResult;
216        try {
217            resolutionResult = resolutionResultFuture.get();
218            rangeResult = resolutionResult.rangeResult;
219        } catch (Exception e) {
220            results.addException(dependency, e, context.parents);
221            return;
222        }
223
224        Set<Version> versions = resolutionResult.descriptors.keySet();
225        for (Version version : versions) {
226            Artifact originalArtifact = dependency.getArtifact().setVersion(version.toString());
227            Dependency d = dependency.setArtifact(originalArtifact);
228
229            final ArtifactDescriptorResult descriptorResult = resolutionResult.descriptors.get(version);
230            if (descriptorResult != null) {
231                d = d.setArtifact(descriptorResult.getArtifact());
232
233                int cycleEntry = find(context.parents, d.getArtifact());
234                if (cycleEntry >= 0) {
235                    results.addCycle(context.parents, cycleEntry, d);
236                    DependencyNode cycleNode = context.parents.get(cycleEntry);
237                    if (cycleNode.getDependency() != null) {
238                        DefaultDependencyNode child = createDependencyNode(
239                                relocations, preManaged, rangeResult, version, d, descriptorResult, cycleNode);
240                        context.getParent().getChildren().add(child);
241                        continue;
242                    }
243                }
244
245                if (!descriptorResult.getRelocations().isEmpty()) {
246                    boolean disableVersionManagementSubsequently =
247                            originalArtifact.getGroupId().equals(d.getArtifact().getGroupId())
248                                    && originalArtifact
249                                            .getArtifactId()
250                                            .equals(d.getArtifact().getArtifactId());
251
252                    PremanagedDependency premanagedDependency = PremanagedDependency.create(
253                            context.depManager, d, disableVersionManagementSubsequently, args.premanagedState);
254                    DependencyProcessingContext relocatedContext = new DependencyProcessingContext(
255                            context.depSelector,
256                            context.depManager,
257                            context.depTraverser,
258                            context.verFilter,
259                            context.trace,
260                            context.repositories,
261                            descriptorResult.getManagedDependencies(),
262                            context.parents,
263                            d,
264                            premanagedDependency);
265
266                    if (!filter(relocatedContext)) {
267                        relocatedContext.withDependency(premanagedDependency.getManagedDependency());
268                        resolveArtifactDescriptorAsync(args, relocatedContext, results);
269                        processDependency(
270                                args,
271                                results,
272                                relocatedContext,
273                                descriptorResult.getRelocations(),
274                                disableVersionManagementSubsequently);
275                    }
276
277                    return;
278                } else {
279                    d = args.pool.intern(d.setArtifact(args.pool.intern(d.getArtifact())));
280
281                    List<RemoteRepository> repos =
282                            getRemoteRepositories(rangeResult.getRepository(version), context.repositories);
283
284                    DefaultDependencyNode child = createDependencyNode(
285                            relocations,
286                            preManaged,
287                            rangeResult,
288                            version,
289                            d,
290                            descriptorResult.getAliases(),
291                            repos,
292                            args.request.getRequestContext());
293
294                    context.getParent().getChildren().add(child);
295
296                    boolean recurse =
297                            traverse && !descriptorResult.getDependencies().isEmpty();
298                    DependencyProcessingContext parentContext = context.withDependency(d);
299                    if (recurse) {
300                        doRecurse(args, parentContext, descriptorResult, child, results, disableVersionManagement);
301                    } else if (!args.skipper.skipResolution(child, parentContext.parents)) {
302                        List<DependencyNode> parents = new ArrayList<>(parentContext.parents.size() + 1);
303                        parents.addAll(parentContext.parents);
304                        parents.add(child);
305                        args.skipper.cache(child, parents);
306                    }
307                }
308            } else {
309                List<RemoteRepository> repos =
310                        getRemoteRepositories(rangeResult.getRepository(version), context.repositories);
311                DefaultDependencyNode child = createDependencyNode(
312                        relocations,
313                        preManaged,
314                        rangeResult,
315                        version,
316                        d,
317                        null,
318                        repos,
319                        args.request.getRequestContext());
320                context.getParent().getChildren().add(child);
321            }
322        }
323    }
324
325    @SuppressWarnings("checkstyle:parameternumber")
326    private void doRecurse(
327            Args args,
328            DependencyProcessingContext parentContext,
329            ArtifactDescriptorResult descriptorResult,
330            DefaultDependencyNode child,
331            Results results,
332            boolean disableVersionManagement) {
333        DefaultDependencyCollectionContext context = args.collectionContext;
334        context.set(parentContext.dependency, descriptorResult.getManagedDependencies());
335
336        DependencySelector childSelector =
337                parentContext.depSelector != null ? parentContext.depSelector.deriveChildSelector(context) : null;
338        DependencyManager childManager =
339                parentContext.depManager != null ? parentContext.depManager.deriveChildManager(context) : null;
340        DependencyTraverser childTraverser =
341                parentContext.depTraverser != null ? parentContext.depTraverser.deriveChildTraverser(context) : null;
342        VersionFilter childFilter =
343                parentContext.verFilter != null ? parentContext.verFilter.deriveChildFilter(context) : null;
344
345        final List<RemoteRepository> childRepos = args.ignoreRepos
346                ? parentContext.repositories
347                : remoteRepositoryManager.aggregateRepositories(
348                        args.session, parentContext.repositories, descriptorResult.getRepositories(), true);
349
350        Object key = args.pool.toKey(
351                parentContext.dependency.getArtifact(),
352                childRepos,
353                childSelector,
354                childManager,
355                childTraverser,
356                childFilter);
357
358        List<DependencyNode> children = args.pool.getChildren(key);
359        if (children == null) {
360            boolean skipResolution = args.skipper.skipResolution(child, parentContext.parents);
361            if (!skipResolution) {
362                List<DependencyNode> parents = new ArrayList<>(parentContext.parents.size() + 1);
363                parents.addAll(parentContext.parents);
364                parents.add(child);
365                for (Dependency dependency : descriptorResult.getDependencies()) {
366                    RequestTrace childTrace = collectStepTrace(
367                            parentContext.trace, args.request.getRequestContext(), parents, dependency);
368                    PremanagedDependency premanagedDependency = PremanagedDependency.create(
369                            childManager, dependency, disableVersionManagement, args.premanagedState);
370                    DependencyProcessingContext processingContext = new DependencyProcessingContext(
371                            childSelector,
372                            childManager,
373                            childTraverser,
374                            childFilter,
375                            childTrace,
376                            childRepos,
377                            descriptorResult.getManagedDependencies(),
378                            parents,
379                            dependency,
380                            premanagedDependency);
381                    if (!filter(processingContext)) {
382                        // resolve descriptors ahead for managed dependency
383                        processingContext.withDependency(processingContext.premanagedDependency.getManagedDependency());
384                        resolveArtifactDescriptorAsync(args, processingContext, results);
385                        args.dependencyProcessingQueue.add(processingContext);
386                    }
387                }
388                args.pool.putChildren(key, child.getChildren());
389                args.skipper.cache(child, parents);
390            }
391        } else {
392            child.setChildren(children);
393        }
394    }
395
396    private boolean filter(DependencyProcessingContext context) {
397        return context.depSelector != null && !context.depSelector.selectDependency(context.dependency);
398    }
399
400    private void resolveArtifactDescriptorAsync(Args args, DependencyProcessingContext context, Results results) {
401        Dependency dependency = context.dependency;
402        args.resolver.resolveDescriptors(dependency.getArtifact(), () -> {
403            VersionRangeRequest rangeRequest = createVersionRangeRequest(
404                    args.request.getRequestContext(), context.trace, context.repositories, dependency);
405            VersionRangeResult rangeResult = cachedResolveRangeResult(rangeRequest, args.pool, args.session);
406            List<? extends Version> versions =
407                    filterVersions(dependency, rangeResult, context.verFilter, args.versionContext);
408
409            // resolve newer version first to maximize benefits of skipper
410            Collections.reverse(versions);
411
412            Map<Version, ArtifactDescriptorResult> descriptors = new ConcurrentHashMap<>(versions.size());
413            Stream<? extends Version> stream = versions.size() > 1 ? versions.parallelStream() : versions.stream();
414            stream.forEach(version -> Optional.ofNullable(
415                            resolveDescriptorForVersion(args, context, results, dependency, version))
416                    .ifPresent(r -> descriptors.put(version, r)));
417
418            DescriptorResolutionResult resolutionResult =
419                    new DescriptorResolutionResult(dependency.getArtifact(), rangeResult);
420            // keep original sequence
421            versions.forEach(version -> resolutionResult.descriptors.put(version, descriptors.get(version)));
422            // populate for versions in version range
423            resolutionResult.flatten().forEach(dr -> args.resolver.cacheVersionRangeDescriptor(dr.artifact, dr));
424
425            return resolutionResult;
426        });
427    }
428
429    private ArtifactDescriptorResult resolveDescriptorForVersion(
430            Args args, DependencyProcessingContext context, Results results, Dependency dependency, Version version) {
431        Artifact original = dependency.getArtifact();
432        Artifact newArtifact = original.setVersion(version.toString());
433        Dependency newDependency =
434                new Dependency(newArtifact, dependency.getScope(), dependency.isOptional(), dependency.getExclusions());
435        DependencyProcessingContext newContext = context.copy();
436
437        ArtifactDescriptorRequest descriptorRequest = createArtifactDescriptorRequest(
438                args.request.getRequestContext(), context.trace, newContext.repositories, newDependency);
439        return isLackingDescriptor(newArtifact)
440                ? new ArtifactDescriptorResult(descriptorRequest)
441                : resolveCachedArtifactDescriptor(
442                        args.pool, descriptorRequest, args.session, newContext.withDependency(newDependency), results);
443    }
444
445    private ArtifactDescriptorResult resolveCachedArtifactDescriptor(
446            DataPool pool,
447            ArtifactDescriptorRequest descriptorRequest,
448            RepositorySystemSession session,
449            DependencyProcessingContext context,
450            Results results) {
451        Object key = pool.toKey(descriptorRequest);
452        ArtifactDescriptorResult descriptorResult = pool.getDescriptor(key, descriptorRequest);
453        if (descriptorResult == null) {
454            try {
455                descriptorResult = descriptorReader.readArtifactDescriptor(session, descriptorRequest);
456                pool.putDescriptor(key, descriptorResult);
457            } catch (ArtifactDescriptorException e) {
458                results.addException(context.dependency, e, context.parents);
459                pool.putDescriptor(key, e);
460                return null;
461            }
462
463        } else if (descriptorResult == DataPool.NO_DESCRIPTOR) {
464            return null;
465        }
466
467        return descriptorResult;
468    }
469
470    static class ParallelDescriptorResolver implements Closeable {
471        private final ExecutorService executorService;
472
473        /**
474         * Artifact ID -> Future of DescriptorResolutionResult
475         */
476        private final Map<String, Future<DescriptorResolutionResult>> results = new ConcurrentHashMap<>(256);
477
478        ParallelDescriptorResolver(int threads) {
479            this.executorService = ExecutorUtils.threadPool(threads, getClass().getSimpleName() + "-");
480        }
481
482        void resolveDescriptors(Artifact artifact, Callable<DescriptorResolutionResult> callable) {
483            results.computeIfAbsent(ArtifactIdUtils.toId(artifact), key -> this.executorService.submit(callable));
484        }
485
486        void cacheVersionRangeDescriptor(Artifact artifact, DescriptorResolutionResult resolutionResult) {
487            results.computeIfAbsent(ArtifactIdUtils.toId(artifact), key -> new DoneFuture<>(resolutionResult));
488        }
489
490        Future<DescriptorResolutionResult> find(Artifact artifact) {
491            return results.get(ArtifactIdUtils.toId(artifact));
492        }
493
494        @Override
495        public void close() {
496            executorService.shutdown();
497        }
498    }
499
500    static class DoneFuture<V> implements Future<V> {
501        private final V v;
502
503        DoneFuture(V v) {
504            this.v = v;
505        }
506
507        @Override
508        public boolean cancel(boolean mayInterruptIfRunning) {
509            return false;
510        }
511
512        @Override
513        public boolean isCancelled() {
514            return false;
515        }
516
517        @Override
518        public boolean isDone() {
519            return true;
520        }
521
522        @Override
523        public V get() throws InterruptedException, ExecutionException {
524            return v;
525        }
526
527        @Override
528        public V get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException {
529            return v;
530        }
531    }
532
533    static class DescriptorResolutionResult {
534        Artifact artifact;
535
536        VersionRangeResult rangeResult;
537
538        Map<Version, ArtifactDescriptorResult> descriptors;
539
540        DescriptorResolutionResult(Artifact artifact, VersionRangeResult rangeResult) {
541            this.artifact = artifact;
542            this.rangeResult = rangeResult;
543            this.descriptors = new LinkedHashMap<>(rangeResult.getVersions().size());
544        }
545
546        DescriptorResolutionResult(
547                VersionRangeResult rangeResult, Version version, ArtifactDescriptorResult descriptor) {
548            // NOTE: In case of A1 -> A2 relocation this happens:
549            // ArtifactDescriptorResult read by ArtifactDescriptorResultReader for A1
550            // will return instance that will have artifact = A2 (as RelocatedArtifact).
551            // So to properly "key" this instance, we need to use "originally requested" A1 instead!
552            // In short:
553            // ArtifactDescriptorRequest.artifact != ArtifactDescriptorResult.artifact WHEN relocation in play
554            // otherwise (no relocation), they are EQUAL.
555            this(descriptor.getRequest().getArtifact(), rangeResult);
556            this.descriptors.put(version, descriptor);
557        }
558
559        List<DescriptorResolutionResult> flatten() {
560            if (descriptors.size() > 1) {
561                return descriptors.entrySet().stream()
562                        .map(e -> new DescriptorResolutionResult(rangeResult, e.getKey(), e.getValue()))
563                        .collect(Collectors.toList());
564            } else {
565                return Collections.emptyList();
566            }
567        }
568    }
569
570    static class Args {
571
572        final RepositorySystemSession session;
573
574        final boolean ignoreRepos;
575
576        final boolean premanagedState;
577
578        final DataPool pool;
579
580        final Queue<DependencyProcessingContext> dependencyProcessingQueue = new ArrayDeque<>(128);
581
582        final DefaultDependencyCollectionContext collectionContext;
583
584        final DefaultVersionFilterContext versionContext;
585
586        final CollectRequest request;
587
588        final DependencyResolutionSkipper skipper;
589
590        final ParallelDescriptorResolver resolver;
591
592        Args(
593                RepositorySystemSession session,
594                DataPool pool,
595                DefaultDependencyCollectionContext collectionContext,
596                DefaultVersionFilterContext versionContext,
597                CollectRequest request,
598                DependencyResolutionSkipper skipper,
599                ParallelDescriptorResolver resolver) {
600            this.session = session;
601            this.request = request;
602            this.ignoreRepos = session.isIgnoreArtifactDescriptorRepositories();
603            this.premanagedState = ConfigUtils.getBoolean(session, false, DependencyManagerUtils.CONFIG_PROP_VERBOSE);
604            this.pool = pool;
605            this.collectionContext = collectionContext;
606            this.versionContext = versionContext;
607            this.skipper = skipper;
608            this.resolver = resolver;
609        }
610    }
611}