Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *   http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing,
013 * software distributed under the License is distributed on an
014 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
015 * KIND, either express or implied.  See the License for the
016 * specific language governing permissions and limitations
017 * under the License.
018 */
019package org.eclipse.aether.internal.impl.collect.bf;
020
021import javax.inject.Inject;
022import javax.inject.Named;
023import javax.inject.Singleton;
024
025import java.io.Closeable;
026import java.util.ArrayDeque;
027import java.util.ArrayList;
028import java.util.Collections;
029import java.util.LinkedHashMap;
030import java.util.List;
031import java.util.Map;
032import java.util.Optional;
033import java.util.Queue;
034import java.util.Set;
035import java.util.concurrent.Callable;
036import java.util.concurrent.ConcurrentHashMap;
037import java.util.concurrent.ExecutionException;
038import java.util.concurrent.ExecutorService;
039import java.util.concurrent.Future;
040import java.util.concurrent.TimeUnit;
041import java.util.concurrent.TimeoutException;
042import java.util.stream.Collectors;
043import java.util.stream.Stream;
044
045import org.eclipse.aether.RepositorySystemSession;
046import org.eclipse.aether.RequestTrace;
047import org.eclipse.aether.artifact.Artifact;
048import org.eclipse.aether.artifact.ArtifactType;
049import org.eclipse.aether.artifact.DefaultArtifact;
050import org.eclipse.aether.collection.CollectRequest;
051import org.eclipse.aether.collection.DependencyManager;
052import org.eclipse.aether.collection.DependencySelector;
053import org.eclipse.aether.collection.DependencyTraverser;
054import org.eclipse.aether.collection.VersionFilter;
055import org.eclipse.aether.graph.DefaultDependencyNode;
056import org.eclipse.aether.graph.Dependency;
057import org.eclipse.aether.graph.DependencyNode;
058import org.eclipse.aether.impl.ArtifactDescriptorReader;
059import org.eclipse.aether.impl.RemoteRepositoryManager;
060import org.eclipse.aether.impl.VersionRangeResolver;
061import org.eclipse.aether.internal.impl.collect.DataPool;
062import org.eclipse.aether.internal.impl.collect.DefaultDependencyCollectionContext;
063import org.eclipse.aether.internal.impl.collect.DefaultVersionFilterContext;
064import org.eclipse.aether.internal.impl.collect.DependencyCollectorDelegate;
065import org.eclipse.aether.internal.impl.collect.PremanagedDependency;
066import org.eclipse.aether.repository.RemoteRepository;
067import org.eclipse.aether.resolution.ArtifactDescriptorException;
068import org.eclipse.aether.resolution.ArtifactDescriptorRequest;
069import org.eclipse.aether.resolution.ArtifactDescriptorResult;
070import org.eclipse.aether.resolution.VersionRangeRequest;
071import org.eclipse.aether.resolution.VersionRangeResult;
072import org.eclipse.aether.util.ConfigUtils;
073import org.eclipse.aether.util.artifact.ArtifactIdUtils;
074import org.eclipse.aether.util.concurrency.ExecutorUtils;
075import org.eclipse.aether.util.graph.manager.DependencyManagerUtils;
076import org.eclipse.aether.version.Version;
077
078import static org.eclipse.aether.internal.impl.collect.DefaultDependencyCycle.find;
079
080/**
081 * Breadth-first {@link org.eclipse.aether.impl.DependencyCollector}
082 *
083 * @since 1.8.0
084 */
085@Singleton
086@Named(BfDependencyCollector.NAME)
087public class BfDependencyCollector extends DependencyCollectorDelegate {
088    public static final String NAME = "bf";
089
090    /**
091     * The key in the repository session's {@link RepositorySystemSession#getConfigProperties()
092     * configuration properties} used to store a {@link Boolean} flag controlling the resolver's skip mode.
093     *
094     * @since 1.8.0
095     */
096    static final String CONFIG_PROP_SKIPPER = "aether.dependencyCollector.bf.skipper";
097
098    /**
099     * The default value for {@link #CONFIG_PROP_SKIPPER}, {@code true}.
100     *
101     * @since 1.8.0
102     */
103    static final boolean CONFIG_PROP_SKIPPER_DEFAULT = true;
104
105    /**
106     * The count of threads to be used when collecting POMs in parallel, default value 5.
107     *
108     * @since 1.9.0
109     */
110    static final String CONFIG_PROP_THREADS = "aether.dependencyCollector.bf.threads";
111
112    @Inject
113    public BfDependencyCollector(
114            RemoteRepositoryManager remoteRepositoryManager,
115            ArtifactDescriptorReader artifactDescriptorReader,
116            VersionRangeResolver versionRangeResolver) {
117        super(remoteRepositoryManager, artifactDescriptorReader, versionRangeResolver);
118    }
119
120    @SuppressWarnings("checkstyle:parameternumber")
121    @Override
122    protected void doCollectDependencies(
123            RepositorySystemSession session,
124            RequestTrace trace,
125            DataPool pool,
126            DefaultDependencyCollectionContext context,
127            DefaultVersionFilterContext versionContext,
128            CollectRequest request,
129            DependencyNode node,
130            List<RemoteRepository> repositories,
131            List<Dependency> dependencies,
132            List<Dependency> managedDependencies,
133            Results results) {
134        boolean useSkip = ConfigUtils.getBoolean(session, CONFIG_PROP_SKIPPER_DEFAULT, CONFIG_PROP_SKIPPER);
135        int nThreads = ExecutorUtils.threadCount(session, 5, CONFIG_PROP_THREADS);
136        logger.debug("Using thread pool with {} threads to resolve descriptors.", nThreads);
137
138        if (useSkip) {
139            logger.debug("Collector skip mode enabled");
140        }
141
142        try (DependencyResolutionSkipper skipper = useSkip
143                        ? DependencyResolutionSkipper.defaultSkipper()
144                        : DependencyResolutionSkipper.neverSkipper();
145                ParallelDescriptorResolver parallelDescriptorResolver = new ParallelDescriptorResolver(nThreads)) {
146            Args args = new Args(session, pool, context, versionContext, request, skipper, parallelDescriptorResolver);
147
148            DependencySelector rootDepSelector = session.getDependencySelector() != null
149                    ? session.getDependencySelector().deriveChildSelector(context)
150                    : null;
151            DependencyManager rootDepManager = session.getDependencyManager() != null
152                    ? session.getDependencyManager().deriveChildManager(context)
153                    : null;
154            DependencyTraverser rootDepTraverser = session.getDependencyTraverser() != null
155                    ? session.getDependencyTraverser().deriveChildTraverser(context)
156                    : null;
157            VersionFilter rootVerFilter = session.getVersionFilter() != null
158                    ? session.getVersionFilter().deriveChildFilter(context)
159                    : null;
160
161            List<DependencyNode> parents = Collections.singletonList(node);
162            for (Dependency dependency : dependencies) {
163                RequestTrace childTrace =
164                        collectStepTrace(trace, args.request.getRequestContext(), parents, dependency);
165                DependencyProcessingContext processingContext = new DependencyProcessingContext(
166                        rootDepSelector,
167                        rootDepManager,
168                        rootDepTraverser,
169                        rootVerFilter,
170                        childTrace,
171                        repositories,
172                        managedDependencies,
173                        parents,
174                        dependency,
175                        PremanagedDependency.create(rootDepManager, dependency, false, args.premanagedState));
176                if (!filter(processingContext)) {
177                    processingContext.withDependency(processingContext.premanagedDependency.getManagedDependency());
178                    resolveArtifactDescriptorAsync(args, processingContext, results);
179                    args.dependencyProcessingQueue.add(processingContext);
180                }
181            }
182
183            while (!args.dependencyProcessingQueue.isEmpty()) {
184                processDependency(
185                        args, results, args.dependencyProcessingQueue.remove(), Collections.emptyList(), false);
186            }
187        }
188    }
189
190    @SuppressWarnings("checkstyle:parameternumber")
191    private void processDependency(
192            Args args,
193            Results results,
194            DependencyProcessingContext context,
195            List<Artifact> relocations,
196            boolean disableVersionManagement) {
197        Dependency dependency = context.dependency;
198        PremanagedDependency preManaged = context.premanagedDependency;
199
200        boolean noDescriptor = isLackingDescriptor(dependency.getArtifact());
201        boolean traverse =
202                !noDescriptor && (context.depTraverser == null || context.depTraverser.traverseDependency(dependency));
203
204        Future<DescriptorResolutionResult> resolutionResultFuture = args.resolver.find(dependency.getArtifact());
205        DescriptorResolutionResult resolutionResult;
206        VersionRangeResult rangeResult;
207        try {
208            resolutionResult = resolutionResultFuture.get();
209            rangeResult = resolutionResult.rangeResult;
210        } catch (Exception e) {
211            results.addException(dependency, e, context.parents);
212            return;
213        }
214
215        Set<Version> versions = resolutionResult.descriptors.keySet();
216        for (Version version : versions) {
217            Artifact originalArtifact = dependency.getArtifact().setVersion(version.toString());
218            Dependency d = dependency.setArtifact(originalArtifact);
219
220            final ArtifactDescriptorResult descriptorResult = resolutionResult.descriptors.get(version);
221            if (descriptorResult != null) {
222                d = d.setArtifact(descriptorResult.getArtifact());
223
224                int cycleEntry = find(context.parents, d.getArtifact());
225                if (cycleEntry >= 0) {
226                    results.addCycle(context.parents, cycleEntry, d);
227                    DependencyNode cycleNode = context.parents.get(cycleEntry);
228                    if (cycleNode.getDependency() != null) {
229                        DefaultDependencyNode child = createDependencyNode(
230                                relocations, preManaged, rangeResult, version, d, descriptorResult, cycleNode);
231                        context.getParent().getChildren().add(child);
232                        continue;
233                    }
234                }
235
236                if (!descriptorResult.getRelocations().isEmpty()) {
237                    boolean disableVersionManagementSubsequently =
238                            originalArtifact.getGroupId().equals(d.getArtifact().getGroupId())
239                                    && originalArtifact
240                                            .getArtifactId()
241                                            .equals(d.getArtifact().getArtifactId());
242
243                    PremanagedDependency premanagedDependency = PremanagedDependency.create(
244                            context.depManager, d, disableVersionManagementSubsequently, args.premanagedState);
245                    DependencyProcessingContext relocatedContext = new DependencyProcessingContext(
246                            context.depSelector,
247                            context.depManager,
248                            context.depTraverser,
249                            context.verFilter,
250                            context.trace,
251                            context.repositories,
252                            descriptorResult.getManagedDependencies(),
253                            context.parents,
254                            d,
255                            premanagedDependency);
256
257                    if (!filter(relocatedContext)) {
258                        relocatedContext.withDependency(premanagedDependency.getManagedDependency());
259                        resolveArtifactDescriptorAsync(args, relocatedContext, results);
260                        processDependency(
261                                args,
262                                results,
263                                relocatedContext,
264                                descriptorResult.getRelocations(),
265                                disableVersionManagementSubsequently);
266                    }
267
268                    return;
269                } else {
270                    d = args.pool.intern(d.setArtifact(args.pool.intern(d.getArtifact())));
271
272                    List<RemoteRepository> repos =
273                            getRemoteRepositories(rangeResult.getRepository(version), context.repositories);
274
275                    DefaultDependencyNode child = createDependencyNode(
276                            relocations,
277                            preManaged,
278                            rangeResult,
279                            version,
280                            d,
281                            descriptorResult.getAliases(),
282                            repos,
283                            args.request.getRequestContext());
284
285                    context.getParent().getChildren().add(child);
286
287                    boolean recurse =
288                            traverse && !descriptorResult.getDependencies().isEmpty();
289                    DependencyProcessingContext parentContext = context.withDependency(d);
290                    if (recurse) {
291                        doRecurse(args, parentContext, descriptorResult, child, results, disableVersionManagement);
292                    } else if (!args.skipper.skipResolution(child, parentContext.parents)) {
293                        List<DependencyNode> parents = new ArrayList<>(parentContext.parents.size() + 1);
294                        parents.addAll(parentContext.parents);
295                        parents.add(child);
296                        args.skipper.cache(child, parents);
297                    }
298                }
299            } else {
300                List<RemoteRepository> repos =
301                        getRemoteRepositories(rangeResult.getRepository(version), context.repositories);
302                DefaultDependencyNode child = createDependencyNode(
303                        relocations,
304                        preManaged,
305                        rangeResult,
306                        version,
307                        d,
308                        null,
309                        repos,
310                        args.request.getRequestContext());
311                context.getParent().getChildren().add(child);
312            }
313        }
314    }
315
316    @SuppressWarnings("checkstyle:parameternumber")
317    private void doRecurse(
318            Args args,
319            DependencyProcessingContext parentContext,
320            ArtifactDescriptorResult descriptorResult,
321            DefaultDependencyNode child,
322            Results results,
323            boolean disableVersionManagement) {
324        DefaultDependencyCollectionContext context = args.collectionContext;
325        context.set(parentContext.dependency, descriptorResult.getManagedDependencies());
326
327        DependencySelector childSelector =
328                parentContext.depSelector != null ? parentContext.depSelector.deriveChildSelector(context) : null;
329        DependencyManager childManager =
330                parentContext.depManager != null ? parentContext.depManager.deriveChildManager(context) : null;
331        DependencyTraverser childTraverser =
332                parentContext.depTraverser != null ? parentContext.depTraverser.deriveChildTraverser(context) : null;
333        VersionFilter childFilter =
334                parentContext.verFilter != null ? parentContext.verFilter.deriveChildFilter(context) : null;
335
336        final List<RemoteRepository> childRepos = args.ignoreRepos
337                ? parentContext.repositories
338                : remoteRepositoryManager.aggregateRepositories(
339                        args.session, parentContext.repositories, descriptorResult.getRepositories(), true);
340
341        Object key = args.pool.toKey(
342                parentContext.dependency.getArtifact(),
343                childRepos,
344                childSelector,
345                childManager,
346                childTraverser,
347                childFilter);
348
349        List<DependencyNode> children = args.pool.getChildren(key);
350        if (children == null) {
351            boolean skipResolution = args.skipper.skipResolution(child, parentContext.parents);
352            if (!skipResolution) {
353                List<DependencyNode> parents = new ArrayList<>(parentContext.parents.size() + 1);
354                parents.addAll(parentContext.parents);
355                parents.add(child);
356                for (Dependency dependency : descriptorResult.getDependencies()) {
357                    RequestTrace childTrace = collectStepTrace(
358                            parentContext.trace, args.request.getRequestContext(), parents, dependency);
359                    PremanagedDependency premanagedDependency = PremanagedDependency.create(
360                            childManager, dependency, disableVersionManagement, args.premanagedState);
361                    DependencyProcessingContext processingContext = new DependencyProcessingContext(
362                            childSelector,
363                            childManager,
364                            childTraverser,
365                            childFilter,
366                            childTrace,
367                            childRepos,
368                            descriptorResult.getManagedDependencies(),
369                            parents,
370                            dependency,
371                            premanagedDependency);
372                    if (!filter(processingContext)) {
373                        // resolve descriptors ahead for managed dependency
374                        processingContext.withDependency(processingContext.premanagedDependency.getManagedDependency());
375                        resolveArtifactDescriptorAsync(args, processingContext, results);
376                        args.dependencyProcessingQueue.add(processingContext);
377                    }
378                }
379                args.pool.putChildren(key, child.getChildren());
380                args.skipper.cache(child, parents);
381            }
382        } else {
383            child.setChildren(children);
384        }
385    }
386
387    private boolean filter(DependencyProcessingContext context) {
388        return context.depSelector != null && !context.depSelector.selectDependency(context.dependency);
389    }
390
391    private void resolveArtifactDescriptorAsync(Args args, DependencyProcessingContext context, Results results) {
392        Dependency dependency = context.dependency;
393        args.resolver.resolveDescriptors(dependency.getArtifact(), () -> {
394            VersionRangeRequest rangeRequest = createVersionRangeRequest(
395                    args.request.getRequestContext(), context.trace, context.repositories, dependency);
396            VersionRangeResult rangeResult = cachedResolveRangeResult(rangeRequest, args.pool, args.session);
397            List<? extends Version> versions =
398                    filterVersions(dependency, rangeResult, context.verFilter, args.versionContext);
399
400            // resolve newer version first to maximize benefits of skipper
401            Collections.reverse(versions);
402
403            Map<Version, ArtifactDescriptorResult> descriptors = new ConcurrentHashMap<>(versions.size());
404            Stream<? extends Version> stream = versions.size() > 1 ? versions.parallelStream() : versions.stream();
405            stream.forEach(version -> Optional.ofNullable(
406                            resolveDescriptorForVersion(args, context, results, dependency, version))
407                    .ifPresent(r -> descriptors.put(version, r)));
408
409            DescriptorResolutionResult resolutionResult =
410                    new DescriptorResolutionResult(dependency.getArtifact(), rangeResult);
411            // keep original sequence
412            versions.forEach(version -> resolutionResult.descriptors.put(version, descriptors.get(version)));
413            // populate for versions in version range
414            resolutionResult.flatten().forEach(dr -> args.resolver.cacheVersionRangeDescriptor(dr.artifact, dr));
415
416            return resolutionResult;
417        });
418    }
419
420    private ArtifactDescriptorResult resolveDescriptorForVersion(
421            Args args, DependencyProcessingContext context, Results results, Dependency dependency, Version version) {
422        Artifact original = dependency.getArtifact();
423        Artifact newArtifact = new DefaultArtifact(
424                original.getGroupId(),
425                original.getArtifactId(),
426                original.getClassifier(),
427                original.getExtension(),
428                version.toString(),
429                original.getProperties(),
430                (ArtifactType) null);
431        Dependency newDependency =
432                new Dependency(newArtifact, dependency.getScope(), dependency.isOptional(), dependency.getExclusions());
433        DependencyProcessingContext newContext = context.copy();
434
435        ArtifactDescriptorRequest descriptorRequest = createArtifactDescriptorRequest(
436                args.request.getRequestContext(), context.trace, newContext.repositories, newDependency);
437        return isLackingDescriptor(newArtifact)
438                ? new ArtifactDescriptorResult(descriptorRequest)
439                : resolveCachedArtifactDescriptor(
440                        args.pool, descriptorRequest, args.session, newContext.withDependency(newDependency), results);
441    }
442
443    private ArtifactDescriptorResult resolveCachedArtifactDescriptor(
444            DataPool pool,
445            ArtifactDescriptorRequest descriptorRequest,
446            RepositorySystemSession session,
447            DependencyProcessingContext context,
448            Results results) {
449        Object key = pool.toKey(descriptorRequest);
450        ArtifactDescriptorResult descriptorResult = pool.getDescriptor(key, descriptorRequest);
451        if (descriptorResult == null) {
452            try {
453                descriptorResult = descriptorReader.readArtifactDescriptor(session, descriptorRequest);
454                pool.putDescriptor(key, descriptorResult);
455            } catch (ArtifactDescriptorException e) {
456                results.addException(context.dependency, e, context.parents);
457                pool.putDescriptor(key, e);
458                return null;
459            }
460
461        } else if (descriptorResult == DataPool.NO_DESCRIPTOR) {
462            return null;
463        }
464
465        return descriptorResult;
466    }
467
468    static class ParallelDescriptorResolver implements Closeable {
469        private final ExecutorService executorService;
470
471        /**
472         * Artifact ID -> Future of DescriptorResolutionResult
473         */
474        private final Map<String, Future<DescriptorResolutionResult>> results = new ConcurrentHashMap<>(256);
475
476        ParallelDescriptorResolver(int threads) {
477            this.executorService = ExecutorUtils.threadPool(threads, getClass().getSimpleName() + "-");
478        }
479
480        void resolveDescriptors(Artifact artifact, Callable<DescriptorResolutionResult> callable) {
481            results.computeIfAbsent(ArtifactIdUtils.toId(artifact), key -> this.executorService.submit(callable));
482        }
483
484        void cacheVersionRangeDescriptor(Artifact artifact, DescriptorResolutionResult resolutionResult) {
485            results.computeIfAbsent(ArtifactIdUtils.toId(artifact), key -> new DoneFuture<>(resolutionResult));
486        }
487
488        Future<DescriptorResolutionResult> find(Artifact artifact) {
489            return results.get(ArtifactIdUtils.toId(artifact));
490        }
491
492        @Override
493        public void close() {
494            executorService.shutdown();
495        }
496    }
497
498    static class DoneFuture<V> implements Future<V> {
499        private final V v;
500
501        DoneFuture(V v) {
502            this.v = v;
503        }
504
505        @Override
506        public boolean cancel(boolean mayInterruptIfRunning) {
507            return false;
508        }
509
510        @Override
511        public boolean isCancelled() {
512            return false;
513        }
514
515        @Override
516        public boolean isDone() {
517            return true;
518        }
519
520        @Override
521        public V get() throws InterruptedException, ExecutionException {
522            return v;
523        }
524
525        @Override
526        public V get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException {
527            return v;
528        }
529    }
530
531    static class DescriptorResolutionResult {
532        Artifact artifact;
533
534        VersionRangeResult rangeResult;
535
536        Map<Version, ArtifactDescriptorResult> descriptors;
537
538        DescriptorResolutionResult(Artifact artifact, VersionRangeResult rangeResult) {
539            this.artifact = artifact;
540            this.rangeResult = rangeResult;
541            this.descriptors = new LinkedHashMap<>(rangeResult.getVersions().size());
542        }
543
544        DescriptorResolutionResult(
545                VersionRangeResult rangeResult, Version version, ArtifactDescriptorResult descriptor) {
546            // NOTE: In case of A1 -> A2 relocation this happens:
547            // ArtifactDescriptorResult read by ArtifactDescriptorResultReader for A1
548            // will return instance that will have artifact = A2 (as RelocatedArtifact).
549            // So to properly "key" this instance, we need to use "originally requested" A1 instead!
550            // In short:
551            // ArtifactDescriptorRequest.artifact != ArtifactDescriptorResult.artifact WHEN relocation in play
552            // otherwise (no relocation), they are EQUAL.
553            this(descriptor.getRequest().getArtifact(), rangeResult);
554            this.descriptors.put(version, descriptor);
555        }
556
557        List<DescriptorResolutionResult> flatten() {
558            if (descriptors.size() > 1) {
559                return descriptors.entrySet().stream()
560                        .map(e -> new DescriptorResolutionResult(rangeResult, e.getKey(), e.getValue()))
561                        .collect(Collectors.toList());
562            } else {
563                return Collections.emptyList();
564            }
565        }
566    }
567
568    static class Args {
569
570        final RepositorySystemSession session;
571
572        final boolean ignoreRepos;
573
574        final boolean premanagedState;
575
576        final DataPool pool;
577
578        final Queue<DependencyProcessingContext> dependencyProcessingQueue = new ArrayDeque<>(128);
579
580        final DefaultDependencyCollectionContext collectionContext;
581
582        final DefaultVersionFilterContext versionContext;
583
584        final CollectRequest request;
585
586        final DependencyResolutionSkipper skipper;
587
588        final ParallelDescriptorResolver resolver;
589
590        Args(
591                RepositorySystemSession session,
592                DataPool pool,
593                DefaultDependencyCollectionContext collectionContext,
594                DefaultVersionFilterContext versionContext,
595                CollectRequest request,
596                DependencyResolutionSkipper skipper,
597                ParallelDescriptorResolver resolver) {
598            this.session = session;
599            this.request = request;
600            this.ignoreRepos = session.isIgnoreArtifactDescriptorRepositories();
601            this.premanagedState = ConfigUtils.getBoolean(session, false, DependencyManagerUtils.CONFIG_PROP_VERBOSE);
602            this.pool = pool;
603            this.collectionContext = collectionContext;
604            this.versionContext = versionContext;
605            this.skipper = skipper;
606            this.resolver = resolver;
607        }
608    }
609}