Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *   http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing,
013 * software distributed under the License is distributed on an
014 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
015 * KIND, either express or implied.  See the License for the
016 * specific language governing permissions and limitations
017 * under the License.
018 */
019package org.eclipse.aether.internal.impl.collect.bf;
020
021import javax.inject.Inject;
022import javax.inject.Named;
023import javax.inject.Singleton;
024
025import java.io.Closeable;
026import java.util.ArrayDeque;
027import java.util.ArrayList;
028import java.util.Collections;
029import java.util.LinkedHashMap;
030import java.util.List;
031import java.util.Map;
032import java.util.Optional;
033import java.util.Queue;
034import java.util.Set;
035import java.util.concurrent.Callable;
036import java.util.concurrent.ConcurrentHashMap;
037import java.util.concurrent.ExecutionException;
038import java.util.concurrent.ExecutorService;
039import java.util.concurrent.Future;
040import java.util.concurrent.TimeUnit;
041import java.util.concurrent.TimeoutException;
042import java.util.concurrent.atomic.AtomicReference;
043import java.util.stream.Collectors;
044import java.util.stream.Stream;
045
046import org.eclipse.aether.RepositorySystemSession;
047import org.eclipse.aether.RequestTrace;
048import org.eclipse.aether.artifact.Artifact;
049import org.eclipse.aether.collection.CollectRequest;
050import org.eclipse.aether.collection.DependencyCollectionException;
051import org.eclipse.aether.collection.DependencyManager;
052import org.eclipse.aether.collection.DependencySelector;
053import org.eclipse.aether.collection.DependencyTraverser;
054import org.eclipse.aether.collection.VersionFilter;
055import org.eclipse.aether.graph.DefaultDependencyNode;
056import org.eclipse.aether.graph.Dependency;
057import org.eclipse.aether.graph.DependencyNode;
058import org.eclipse.aether.impl.ArtifactDescriptorReader;
059import org.eclipse.aether.impl.RemoteRepositoryManager;
060import org.eclipse.aether.impl.VersionRangeResolver;
061import org.eclipse.aether.internal.impl.collect.DataPool;
062import org.eclipse.aether.internal.impl.collect.DefaultDependencyCollectionContext;
063import org.eclipse.aether.internal.impl.collect.DefaultDependencyCollector;
064import org.eclipse.aether.internal.impl.collect.DefaultVersionFilterContext;
065import org.eclipse.aether.internal.impl.collect.DependencyCollectorDelegate;
066import org.eclipse.aether.internal.impl.collect.PremanagedDependency;
067import org.eclipse.aether.repository.RemoteRepository;
068import org.eclipse.aether.resolution.ArtifactDescriptorRequest;
069import org.eclipse.aether.resolution.ArtifactDescriptorResult;
070import org.eclipse.aether.resolution.VersionRangeRequest;
071import org.eclipse.aether.resolution.VersionRangeResult;
072import org.eclipse.aether.spi.artifact.decorator.ArtifactDecoratorFactory;
073import org.eclipse.aether.util.ConfigUtils;
074import org.eclipse.aether.util.artifact.ArtifactIdUtils;
075import org.eclipse.aether.util.concurrency.ExecutorUtils;
076import org.eclipse.aether.util.graph.manager.DependencyManagerUtils;
077import org.eclipse.aether.version.Version;
078
079import static org.eclipse.aether.internal.impl.collect.DefaultDependencyCycle.find;
080
081/**
082 * Breadth-first {@link org.eclipse.aether.impl.DependencyCollector}
083 *
084 * @since 1.8.0
085 */
086@Singleton
087@Named(BfDependencyCollector.NAME)
088public class BfDependencyCollector extends DependencyCollectorDelegate {
089    public static final String NAME = "bf";
090
091    private static final String CONFIG_PROPS_PREFIX = DefaultDependencyCollector.CONFIG_PROPS_PREFIX + NAME + ".";
092
093    /**
094     * The key in the repository session's {@link RepositorySystemSession#getConfigProperties()
095     * configuration properties} used to store a {@link Boolean} flag controlling the resolver's skip mode.
096     *
097     * @since 1.8.0
098     * @configurationSource {@link RepositorySystemSession#getConfigProperties()}
099     * @configurationType {@link java.lang.Boolean}
100     * @configurationDefaultValue {@link #DEFAULT_SKIPPER}
101     */
102    public static final String CONFIG_PROP_SKIPPER = CONFIG_PROPS_PREFIX + "skipper";
103
104    /**
105     * The default value for {@link #CONFIG_PROP_SKIPPER}, {@code true}.
106     *
107     * @since 1.8.0
108     */
109    public static final boolean DEFAULT_SKIPPER = true;
110
111    /**
112     * The count of threads to be used when collecting POMs in parallel.
113     *
114     * @since 1.9.0
115     * @configurationSource {@link RepositorySystemSession#getConfigProperties()}
116     * @configurationType {@link java.lang.Integer}
117     * @configurationDefaultValue {@link #DEFAULT_THREADS}
118     */
119    public static final String CONFIG_PROP_THREADS = CONFIG_PROPS_PREFIX + "threads";
120
121    /**
122     * The default value for {@link #CONFIG_PROP_THREADS}, default value 5.
123     *
124     * @since 1.9.0
125     */
126    public static final int DEFAULT_THREADS = 5;
127
128    @Inject
129    public BfDependencyCollector(
130            RemoteRepositoryManager remoteRepositoryManager,
131            ArtifactDescriptorReader artifactDescriptorReader,
132            VersionRangeResolver versionRangeResolver,
133            Map<String, ArtifactDecoratorFactory> artifactDecoratorFactories) {
134        super(remoteRepositoryManager, artifactDescriptorReader, versionRangeResolver, artifactDecoratorFactories);
135    }
136
137    @SuppressWarnings("checkstyle:parameternumber")
138    @Override
139    protected void doCollectDependencies(
140            RepositorySystemSession session,
141            RequestTrace trace,
142            DataPool pool,
143            DefaultDependencyCollectionContext context,
144            DefaultVersionFilterContext versionContext,
145            CollectRequest request,
146            DependencyNode node,
147            List<RemoteRepository> repositories,
148            List<Dependency> dependencies,
149            List<Dependency> managedDependencies,
150            Results results)
151            throws DependencyCollectionException {
152        boolean useSkip = ConfigUtils.getBoolean(session, DEFAULT_SKIPPER, CONFIG_PROP_SKIPPER);
153        int nThreads = ExecutorUtils.threadCount(session, DEFAULT_THREADS, CONFIG_PROP_THREADS);
154        logger.debug("Using thread pool with {} threads to resolve descriptors.", nThreads);
155
156        if (useSkip) {
157            logger.debug("Collector skip mode enabled");
158        }
159
160        try (DependencyResolutionSkipper skipper = useSkip
161                        ? DependencyResolutionSkipper.defaultSkipper()
162                        : DependencyResolutionSkipper.neverSkipper();
163                ParallelDescriptorResolver parallelDescriptorResolver = new ParallelDescriptorResolver(nThreads)) {
164            Args args = new Args(session, pool, context, versionContext, request, skipper, parallelDescriptorResolver);
165
166            DependencySelector rootDepSelector = session.getDependencySelector() != null
167                    ? session.getDependencySelector().deriveChildSelector(context)
168                    : null;
169            DependencyManager rootDepManager = session.getDependencyManager() != null
170                    ? session.getDependencyManager().deriveChildManager(context)
171                    : null;
172            DependencyTraverser rootDepTraverser = session.getDependencyTraverser() != null
173                    ? session.getDependencyTraverser().deriveChildTraverser(context)
174                    : null;
175            VersionFilter rootVerFilter = session.getVersionFilter() != null
176                    ? session.getVersionFilter().deriveChildFilter(context)
177                    : null;
178
179            List<DependencyNode> parents = Collections.singletonList(node);
180            for (Dependency dependency : dependencies) {
181                RequestTrace childTrace =
182                        collectStepTrace(trace, args.request.getRequestContext(), parents, dependency);
183                DependencyProcessingContext processingContext = new DependencyProcessingContext(
184                        rootDepSelector,
185                        rootDepManager,
186                        rootDepTraverser,
187                        rootVerFilter,
188                        childTrace,
189                        repositories,
190                        managedDependencies,
191                        parents,
192                        dependency,
193                        PremanagedDependency.create(rootDepManager, dependency, false, args.premanagedState));
194                if (!filter(processingContext)) {
195                    processingContext.withDependency(processingContext.premanagedDependency.getManagedDependency());
196                    resolveArtifactDescriptorAsync(args, processingContext, results);
197                    args.dependencyProcessingQueue.add(processingContext);
198                }
199            }
200
201            while (!args.dependencyProcessingQueue.isEmpty()) {
202                processDependency(
203                        args, results, args.dependencyProcessingQueue.remove(), Collections.emptyList(), false);
204            }
205
206            if (args.interruptedException.get() != null) {
207                throw new DependencyCollectionException(
208                        results.getResult(), "Collection interrupted", args.interruptedException.get());
209            }
210        }
211    }
212
213    @SuppressWarnings("checkstyle:parameternumber")
214    private void processDependency(
215            Args args,
216            Results results,
217            DependencyProcessingContext context,
218            List<Artifact> relocations,
219            boolean disableVersionManagement) {
220        if (Thread.interrupted()) {
221            args.interruptedException.set(new InterruptedException());
222        }
223        if (args.interruptedException.get() != null) {
224            return;
225        }
226        Dependency dependency = context.dependency;
227        PremanagedDependency preManaged = context.premanagedDependency;
228
229        boolean noDescriptor = isLackingDescriptor(args.session, dependency.getArtifact());
230        boolean traverse =
231                !noDescriptor && (context.depTraverser == null || context.depTraverser.traverseDependency(dependency));
232
233        Future<DescriptorResolutionResult> resolutionResultFuture = args.resolver.find(dependency.getArtifact());
234        DescriptorResolutionResult resolutionResult;
235        VersionRangeResult rangeResult;
236        try {
237            resolutionResult = resolutionResultFuture.get();
238            rangeResult = resolutionResult.rangeResult;
239        } catch (Exception e) {
240            results.addException(dependency, e, context.parents);
241            return;
242        }
243
244        Set<Version> versions = resolutionResult.descriptors.keySet();
245        for (Version version : versions) {
246            Artifact originalArtifact = dependency.getArtifact().setVersion(version.toString());
247            Dependency d = dependency.setArtifact(originalArtifact);
248
249            final ArtifactDescriptorResult descriptorResult = resolutionResult.descriptors.get(version);
250            if (descriptorResult != null) {
251                d = d.setArtifact(descriptorResult.getArtifact());
252
253                int cycleEntry = find(context.parents, d.getArtifact());
254                if (cycleEntry >= 0) {
255                    results.addCycle(context.parents, cycleEntry, d);
256                    DependencyNode cycleNode = context.parents.get(cycleEntry);
257                    if (cycleNode.getDependency() != null) {
258                        DefaultDependencyNode child = createDependencyNode(
259                                relocations, preManaged, rangeResult, version, d, descriptorResult, cycleNode);
260                        context.getParent().getChildren().add(child);
261                        continue;
262                    }
263                }
264
265                if (!descriptorResult.getRelocations().isEmpty()) {
266                    boolean disableVersionManagementSubsequently =
267                            originalArtifact.getGroupId().equals(d.getArtifact().getGroupId())
268                                    && originalArtifact
269                                            .getArtifactId()
270                                            .equals(d.getArtifact().getArtifactId());
271
272                    PremanagedDependency premanagedDependency = PremanagedDependency.create(
273                            context.depManager, d, disableVersionManagementSubsequently, args.premanagedState);
274                    DependencyProcessingContext relocatedContext = new DependencyProcessingContext(
275                            context.depSelector,
276                            context.depManager,
277                            context.depTraverser,
278                            context.verFilter,
279                            context.trace,
280                            context.repositories,
281                            descriptorResult.getManagedDependencies(),
282                            context.parents,
283                            d,
284                            premanagedDependency);
285
286                    if (!filter(relocatedContext)) {
287                        relocatedContext.withDependency(premanagedDependency.getManagedDependency());
288                        resolveArtifactDescriptorAsync(args, relocatedContext, results);
289                        processDependency(
290                                args,
291                                results,
292                                relocatedContext,
293                                descriptorResult.getRelocations(),
294                                disableVersionManagementSubsequently);
295                    }
296
297                    return;
298                } else {
299                    d = args.pool.intern(d.setArtifact(args.pool.intern(d.getArtifact())));
300
301                    List<RemoteRepository> repos =
302                            getRemoteRepositories(rangeResult.getRepository(version), context.repositories);
303
304                    DefaultDependencyNode child = createDependencyNode(
305                            relocations,
306                            preManaged,
307                            rangeResult,
308                            version,
309                            d,
310                            descriptorResult.getAliases(),
311                            repos,
312                            args.request.getRequestContext());
313
314                    context.getParent().getChildren().add(child);
315
316                    boolean recurse =
317                            traverse && !descriptorResult.getDependencies().isEmpty();
318                    DependencyProcessingContext parentContext = context.withDependency(d);
319                    if (recurse) {
320                        doRecurse(args, parentContext, descriptorResult, child, results, disableVersionManagement);
321                    } else if (!args.skipper.skipResolution(child, parentContext.parents)) {
322                        List<DependencyNode> parents = new ArrayList<>(parentContext.parents.size() + 1);
323                        parents.addAll(parentContext.parents);
324                        parents.add(child);
325                        args.skipper.cache(child, parents);
326                    }
327                }
328            } else {
329                List<RemoteRepository> repos =
330                        getRemoteRepositories(rangeResult.getRepository(version), context.repositories);
331                DefaultDependencyNode child = createDependencyNode(
332                        relocations,
333                        preManaged,
334                        rangeResult,
335                        version,
336                        d,
337                        null,
338                        repos,
339                        args.request.getRequestContext());
340                context.getParent().getChildren().add(child);
341            }
342        }
343    }
344
345    @SuppressWarnings("checkstyle:parameternumber")
346    private void doRecurse(
347            Args args,
348            DependencyProcessingContext parentContext,
349            ArtifactDescriptorResult descriptorResult,
350            DefaultDependencyNode child,
351            Results results,
352            boolean disableVersionManagement) {
353        DefaultDependencyCollectionContext context = args.collectionContext;
354        context.set(parentContext.dependency, descriptorResult.getManagedDependencies());
355
356        DependencySelector childSelector =
357                parentContext.depSelector != null ? parentContext.depSelector.deriveChildSelector(context) : null;
358        DependencyManager childManager =
359                parentContext.depManager != null ? parentContext.depManager.deriveChildManager(context) : null;
360        DependencyTraverser childTraverser =
361                parentContext.depTraverser != null ? parentContext.depTraverser.deriveChildTraverser(context) : null;
362        VersionFilter childFilter =
363                parentContext.verFilter != null ? parentContext.verFilter.deriveChildFilter(context) : null;
364
365        final List<RemoteRepository> childRepos = args.ignoreRepos
366                ? parentContext.repositories
367                : remoteRepositoryManager.aggregateRepositories(
368                        args.session, parentContext.repositories, descriptorResult.getRepositories(), true);
369
370        Object key = args.pool.toKey(
371                parentContext.dependency.getArtifact(),
372                childRepos,
373                childSelector,
374                childManager,
375                childTraverser,
376                childFilter);
377
378        List<DependencyNode> children = args.pool.getChildren(key);
379        if (children == null) {
380            boolean skipResolution = args.skipper.skipResolution(child, parentContext.parents);
381            if (!skipResolution) {
382                List<DependencyNode> parents = new ArrayList<>(parentContext.parents.size() + 1);
383                parents.addAll(parentContext.parents);
384                parents.add(child);
385                for (Dependency dependency : descriptorResult.getDependencies()) {
386                    RequestTrace childTrace = collectStepTrace(
387                            parentContext.trace, args.request.getRequestContext(), parents, dependency);
388                    PremanagedDependency premanagedDependency = PremanagedDependency.create(
389                            childManager, dependency, disableVersionManagement, args.premanagedState);
390                    DependencyProcessingContext processingContext = new DependencyProcessingContext(
391                            childSelector,
392                            childManager,
393                            childTraverser,
394                            childFilter,
395                            childTrace,
396                            childRepos,
397                            descriptorResult.getManagedDependencies(),
398                            parents,
399                            dependency,
400                            premanagedDependency);
401                    if (!filter(processingContext)) {
402                        // resolve descriptors ahead for managed dependency
403                        processingContext.withDependency(processingContext.premanagedDependency.getManagedDependency());
404                        resolveArtifactDescriptorAsync(args, processingContext, results);
405                        args.dependencyProcessingQueue.add(processingContext);
406                    }
407                }
408                args.pool.putChildren(key, child.getChildren());
409                args.skipper.cache(child, parents);
410            }
411        } else {
412            child.setChildren(children);
413        }
414    }
415
416    private boolean filter(DependencyProcessingContext context) {
417        return context.depSelector != null && !context.depSelector.selectDependency(context.dependency);
418    }
419
420    private void resolveArtifactDescriptorAsync(Args args, DependencyProcessingContext context, Results results) {
421        Dependency dependency = context.dependency;
422        args.resolver.resolveDescriptors(dependency.getArtifact(), () -> {
423            VersionRangeRequest rangeRequest = createVersionRangeRequest(
424                    args.request.getRequestContext(), context.trace, context.repositories, dependency);
425            VersionRangeResult rangeResult = cachedResolveRangeResult(rangeRequest, args.pool, args.session);
426            List<? extends Version> versions =
427                    filterVersions(dependency, rangeResult, context.verFilter, args.versionContext);
428
429            // resolve newer version first to maximize benefits of skipper
430            Collections.reverse(versions);
431
432            Map<Version, ArtifactDescriptorResult> descriptors = new ConcurrentHashMap<>(versions.size());
433            Stream<? extends Version> stream = versions.size() > 1 ? versions.parallelStream() : versions.stream();
434            stream.forEach(version -> Optional.ofNullable(
435                            resolveDescriptorForVersion(args, context, results, dependency, version))
436                    .ifPresent(r -> descriptors.put(version, r)));
437
438            DescriptorResolutionResult resolutionResult =
439                    new DescriptorResolutionResult(dependency.getArtifact(), rangeResult);
440            // keep original sequence
441            versions.forEach(version -> resolutionResult.descriptors.put(version, descriptors.get(version)));
442            // populate for versions in version range
443            resolutionResult.flatten().forEach(dr -> args.resolver.cacheVersionRangeDescriptor(dr.artifact, dr));
444
445            return resolutionResult;
446        });
447    }
448
449    private ArtifactDescriptorResult resolveDescriptorForVersion(
450            Args args, DependencyProcessingContext context, Results results, Dependency dependency, Version version) {
451        Artifact original = dependency.getArtifact();
452        Artifact newArtifact = original.setVersion(version.toString());
453        Dependency newDependency =
454                new Dependency(newArtifact, dependency.getScope(), dependency.isOptional(), dependency.getExclusions());
455        DependencyProcessingContext newContext = context.copy();
456
457        ArtifactDescriptorRequest descriptorRequest = createArtifactDescriptorRequest(
458                args.request.getRequestContext(), context.trace, newContext.repositories, newDependency);
459        return isLackingDescriptor(args.session, newArtifact)
460                ? new ArtifactDescriptorResult(descriptorRequest)
461                : resolveCachedArtifactDescriptor(
462                        args.pool,
463                        descriptorRequest,
464                        args.session,
465                        newContext.withDependency(newDependency).dependency,
466                        results,
467                        context.parents);
468    }
469
470    static class ParallelDescriptorResolver implements Closeable {
471        private final ExecutorService executorService;
472
473        /**
474         * Artifact ID -> Future of DescriptorResolutionResult
475         */
476        private final Map<String, Future<DescriptorResolutionResult>> results = new ConcurrentHashMap<>(256);
477
478        ParallelDescriptorResolver(int threads) {
479            this.executorService = ExecutorUtils.threadPool(threads, getClass().getSimpleName() + "-");
480        }
481
482        void resolveDescriptors(Artifact artifact, Callable<DescriptorResolutionResult> callable) {
483            results.computeIfAbsent(ArtifactIdUtils.toId(artifact), key -> this.executorService.submit(callable));
484        }
485
486        void cacheVersionRangeDescriptor(Artifact artifact, DescriptorResolutionResult resolutionResult) {
487            results.computeIfAbsent(ArtifactIdUtils.toId(artifact), key -> new DoneFuture<>(resolutionResult));
488        }
489
490        Future<DescriptorResolutionResult> find(Artifact artifact) {
491            return results.get(ArtifactIdUtils.toId(artifact));
492        }
493
494        @Override
495        public void close() {
496            executorService.shutdown();
497        }
498    }
499
500    static class DoneFuture<V> implements Future<V> {
501        private final V v;
502
503        DoneFuture(V v) {
504            this.v = v;
505        }
506
507        @Override
508        public boolean cancel(boolean mayInterruptIfRunning) {
509            return false;
510        }
511
512        @Override
513        public boolean isCancelled() {
514            return false;
515        }
516
517        @Override
518        public boolean isDone() {
519            return true;
520        }
521
522        @Override
523        public V get() throws InterruptedException, ExecutionException {
524            return v;
525        }
526
527        @Override
528        public V get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException {
529            return v;
530        }
531    }
532
533    static class DescriptorResolutionResult {
534        Artifact artifact;
535
536        VersionRangeResult rangeResult;
537
538        Map<Version, ArtifactDescriptorResult> descriptors;
539
540        DescriptorResolutionResult(Artifact artifact, VersionRangeResult rangeResult) {
541            this.artifact = artifact;
542            this.rangeResult = rangeResult;
543            this.descriptors = new LinkedHashMap<>(rangeResult.getVersions().size());
544        }
545
546        DescriptorResolutionResult(
547                VersionRangeResult rangeResult, Version version, ArtifactDescriptorResult descriptor) {
548            // NOTE: In case of A1 -> A2 relocation this happens:
549            // ArtifactDescriptorResult read by ArtifactDescriptorResultReader for A1
550            // will return instance that will have artifact = A2 (as RelocatedArtifact).
551            // So to properly "key" this instance, we need to use "originally requested" A1 instead!
552            // In short:
553            // ArtifactDescriptorRequest.artifact != ArtifactDescriptorResult.artifact WHEN relocation in play
554            // otherwise (no relocation), they are EQUAL.
555            this(descriptor.getRequest().getArtifact(), rangeResult);
556            this.descriptors.put(version, descriptor);
557        }
558
559        List<DescriptorResolutionResult> flatten() {
560            if (descriptors.size() > 1) {
561                return descriptors.entrySet().stream()
562                        .map(e -> new DescriptorResolutionResult(rangeResult, e.getKey(), e.getValue()))
563                        .collect(Collectors.toList());
564            } else {
565                return Collections.emptyList();
566            }
567        }
568    }
569
570    static class Args {
571
572        final RepositorySystemSession session;
573
574        final boolean ignoreRepos;
575
576        final boolean premanagedState;
577
578        final DataPool pool;
579
580        final Queue<DependencyProcessingContext> dependencyProcessingQueue = new ArrayDeque<>(128);
581
582        final DefaultDependencyCollectionContext collectionContext;
583
584        final DefaultVersionFilterContext versionContext;
585
586        final CollectRequest request;
587
588        final DependencyResolutionSkipper skipper;
589
590        final ParallelDescriptorResolver resolver;
591
592        final AtomicReference<InterruptedException> interruptedException;
593
594        Args(
595                RepositorySystemSession session,
596                DataPool pool,
597                DefaultDependencyCollectionContext collectionContext,
598                DefaultVersionFilterContext versionContext,
599                CollectRequest request,
600                DependencyResolutionSkipper skipper,
601                ParallelDescriptorResolver resolver) {
602            this.session = session;
603            this.request = request;
604            this.ignoreRepos = session.isIgnoreArtifactDescriptorRepositories();
605            this.premanagedState = ConfigUtils.getBoolean(session, false, DependencyManagerUtils.CONFIG_PROP_VERBOSE);
606            this.pool = pool;
607            this.collectionContext = collectionContext;
608            this.versionContext = versionContext;
609            this.skipper = skipper;
610            this.resolver = resolver;
611            this.interruptedException = new AtomicReference<>(null);
612        }
613    }
614}