View Javadoc
1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *   http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing,
13   * software distributed under the License is distributed on an
14   * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15   * KIND, either express or implied.  See the License for the
16   * specific language governing permissions and limitations
17   * under the License.
18   */
19  package org.eclipse.aether.internal.impl;
20  
21  import javax.inject.Inject;
22  import javax.inject.Named;
23  import javax.inject.Singleton;
24  
25  import java.io.IOException;
26  import java.io.UncheckedIOException;
27  import java.nio.file.Files;
28  import java.nio.file.Path;
29  import java.util.ArrayList;
30  import java.util.Collection;
31  import java.util.Collections;
32  import java.util.HashMap;
33  import java.util.List;
34  import java.util.Map;
35  import java.util.concurrent.Executor;
36  
37  import org.eclipse.aether.ConfigurationProperties;
38  import org.eclipse.aether.RepositoryEvent;
39  import org.eclipse.aether.RepositoryEvent.EventType;
40  import org.eclipse.aether.RepositorySystemSession;
41  import org.eclipse.aether.RequestTrace;
42  import org.eclipse.aether.SyncContext;
43  import org.eclipse.aether.impl.MetadataResolver;
44  import org.eclipse.aether.impl.OfflineController;
45  import org.eclipse.aether.impl.RemoteRepositoryFilterManager;
46  import org.eclipse.aether.impl.RemoteRepositoryManager;
47  import org.eclipse.aether.impl.RepositoryConnectorProvider;
48  import org.eclipse.aether.impl.RepositoryEventDispatcher;
49  import org.eclipse.aether.impl.UpdateCheck;
50  import org.eclipse.aether.impl.UpdateCheckManager;
51  import org.eclipse.aether.metadata.Metadata;
52  import org.eclipse.aether.repository.ArtifactRepository;
53  import org.eclipse.aether.repository.LocalMetadataRegistration;
54  import org.eclipse.aether.repository.LocalMetadataRequest;
55  import org.eclipse.aether.repository.LocalMetadataResult;
56  import org.eclipse.aether.repository.LocalRepository;
57  import org.eclipse.aether.repository.LocalRepositoryManager;
58  import org.eclipse.aether.repository.RemoteRepository;
59  import org.eclipse.aether.repository.RepositoryPolicy;
60  import org.eclipse.aether.resolution.MetadataRequest;
61  import org.eclipse.aether.resolution.MetadataResult;
62  import org.eclipse.aether.spi.connector.MetadataDownload;
63  import org.eclipse.aether.spi.connector.RepositoryConnector;
64  import org.eclipse.aether.spi.connector.filter.RemoteRepositoryFilter;
65  import org.eclipse.aether.spi.io.PathProcessor;
66  import org.eclipse.aether.spi.synccontext.SyncContextFactory;
67  import org.eclipse.aether.transfer.MetadataNotFoundException;
68  import org.eclipse.aether.transfer.MetadataTransferException;
69  import org.eclipse.aether.transfer.NoRepositoryConnectorException;
70  import org.eclipse.aether.transfer.RepositoryOfflineException;
71  import org.eclipse.aether.util.concurrency.ExecutorUtils;
72  import org.eclipse.aether.util.concurrency.RunnableErrorForwarder;
73  
74  import static java.util.Objects.requireNonNull;
75  
76  /**
77   */
78  @Singleton
79  @Named
80  public class DefaultMetadataResolver implements MetadataResolver {
81      private static final String CONFIG_PROPS_PREFIX = ConfigurationProperties.PREFIX_AETHER + "metadataResolver.";
82  
83      /**
84       * Number of threads to use in parallel for resolving metadata.
85       *
86       * @since 0.9.0.M4
87       * @configurationSource {@link RepositorySystemSession#getConfigProperties()}
88       * @configurationType {@link java.lang.Integer}
89       * @configurationDefaultValue {@link #DEFAULT_THREADS}
90       */
91      public static final String CONFIG_PROP_THREADS = CONFIG_PROPS_PREFIX + "threads";
92  
93      public static final int DEFAULT_THREADS = 4;
94  
95      private final RepositoryEventDispatcher repositoryEventDispatcher;
96  
97      private final UpdateCheckManager updateCheckManager;
98  
99      private final RepositoryConnectorProvider repositoryConnectorProvider;
100 
101     private final RemoteRepositoryManager remoteRepositoryManager;
102 
103     private final SyncContextFactory syncContextFactory;
104 
105     private final OfflineController offlineController;
106 
107     private final RemoteRepositoryFilterManager remoteRepositoryFilterManager;
108 
109     private final PathProcessor pathProcessor;
110 
111     @SuppressWarnings("checkstyle:parameternumber")
112     @Inject
113     public DefaultMetadataResolver(
114             RepositoryEventDispatcher repositoryEventDispatcher,
115             UpdateCheckManager updateCheckManager,
116             RepositoryConnectorProvider repositoryConnectorProvider,
117             RemoteRepositoryManager remoteRepositoryManager,
118             SyncContextFactory syncContextFactory,
119             OfflineController offlineController,
120             RemoteRepositoryFilterManager remoteRepositoryFilterManager,
121             PathProcessor pathProcessor) {
122         this.repositoryEventDispatcher =
123                 requireNonNull(repositoryEventDispatcher, "repository event dispatcher cannot be null");
124         this.updateCheckManager = requireNonNull(updateCheckManager, "update check manager cannot be null");
125         this.repositoryConnectorProvider =
126                 requireNonNull(repositoryConnectorProvider, "repository connector provider cannot be null");
127         this.remoteRepositoryManager =
128                 requireNonNull(remoteRepositoryManager, "remote repository provider cannot be null");
129         this.syncContextFactory = requireNonNull(syncContextFactory, "sync context factory cannot be null");
130         this.offlineController = requireNonNull(offlineController, "offline controller cannot be null");
131         this.remoteRepositoryFilterManager =
132                 requireNonNull(remoteRepositoryFilterManager, "remote repository filter manager cannot be null");
133         this.pathProcessor = requireNonNull(pathProcessor, "path processor cannot be null");
134     }
135 
136     @Override
137     public List<MetadataResult> resolveMetadata(
138             RepositorySystemSession session, Collection<? extends MetadataRequest> requests) {
139         requireNonNull(session, "session cannot be null");
140         requireNonNull(requests, "requests cannot be null");
141         try (SyncContext shared = syncContextFactory.newInstance(session, true);
142                 SyncContext exclusive = syncContextFactory.newInstance(session, false)) {
143             Collection<Metadata> metadata = new ArrayList<>(requests.size());
144             for (MetadataRequest request : requests) {
145                 metadata.add(request.getMetadata());
146             }
147 
148             return resolve(shared, exclusive, metadata, session, requests);
149         }
150     }
151 
152     @SuppressWarnings("checkstyle:methodlength")
153     private List<MetadataResult> resolve(
154             SyncContext shared,
155             SyncContext exclusive,
156             Collection<Metadata> subjects,
157             RepositorySystemSession session,
158             Collection<? extends MetadataRequest> requests) {
159         SyncContext current = shared;
160         try {
161             while (true) {
162                 current.acquire(null, subjects);
163 
164                 final List<MetadataResult> results = new ArrayList<>(requests.size());
165                 final List<ResolveTask> tasks = new ArrayList<>(requests.size());
166                 final Map<Path, Long> localLastUpdates = new HashMap<>();
167                 final RemoteRepositoryFilter remoteRepositoryFilter =
168                         remoteRepositoryFilterManager.getRemoteRepositoryFilter(session);
169 
170                 for (MetadataRequest request : requests) {
171                     RequestTrace trace = RequestTrace.newChild(request.getTrace(), request);
172 
173                     MetadataResult result = new MetadataResult(request);
174                     results.add(result);
175 
176                     Metadata metadata = request.getMetadata();
177                     RemoteRepository repository = request.getRepository();
178 
179                     if (repository == null) {
180                         LocalRepository localRepo =
181                                 session.getLocalRepositoryManager().getRepository();
182 
183                         metadataResolving(session, trace, metadata, localRepo);
184 
185                         Path localFile = getLocalFile(session, metadata);
186 
187                         if (localFile != null) {
188                             metadata = metadata.setPath(localFile);
189                             result.setMetadata(metadata);
190                         } else {
191                             result.setException(new MetadataNotFoundException(metadata, localRepo));
192                         }
193 
194                         metadataResolved(session, trace, metadata, localRepo, result.getException());
195                         continue;
196                     }
197 
198                     if (remoteRepositoryFilter != null) {
199                         RemoteRepositoryFilter.Result filterResult =
200                                 remoteRepositoryFilter.acceptMetadata(repository, metadata);
201                         if (!filterResult.isAccepted()) {
202                             result.setException(
203                                     new MetadataNotFoundException(metadata, repository, filterResult.reasoning()));
204                             continue;
205                         }
206                     }
207 
208                     List<RemoteRepository> repositories =
209                             getEnabledSourceRepositories(repository, metadata.getNature());
210 
211                     if (repositories.isEmpty()) {
212                         continue;
213                     }
214 
215                     metadataResolving(session, trace, metadata, repository);
216                     LocalRepositoryManager lrm = session.getLocalRepositoryManager();
217                     LocalMetadataRequest localRequest =
218                             new LocalMetadataRequest(metadata, repository, request.getRequestContext());
219                     LocalMetadataResult lrmResult = lrm.find(session, localRequest);
220 
221                     Path metadataPath = lrmResult.getPath();
222 
223                     try {
224                         Utils.checkOffline(session, offlineController, repository);
225                     } catch (RepositoryOfflineException e) {
226                         if (metadataPath != null) {
227                             metadata = metadata.setPath(metadataPath);
228                             result.setMetadata(metadata);
229                         } else {
230                             String msg = "Cannot access " + repository.getId() + " (" + repository.getUrl()
231                                     + ") in offline mode and the metadata " + metadata
232                                     + " has not been downloaded from it before";
233                             result.setException(new MetadataNotFoundException(metadata, repository, msg, e));
234                         }
235 
236                         metadataResolved(session, trace, metadata, repository, result.getException());
237                         continue;
238                     }
239 
240                     Long localLastUpdate = null;
241                     if (request.isFavorLocalRepository()) {
242                         Path localPath = getLocalFile(session, metadata);
243                         localLastUpdate = localLastUpdates.get(localPath);
244                         if (localLastUpdate == null) {
245                             localLastUpdate = localPath != null ? pathProcessor.lastModified(localPath, 0L) : 0L;
246                             localLastUpdates.put(localPath, localLastUpdate);
247                         }
248                     }
249 
250                     List<UpdateCheck<Metadata, MetadataTransferException>> checks = new ArrayList<>();
251                     Exception exception = null;
252                     for (RemoteRepository repo : repositories) {
253                         RepositoryPolicy policy = getPolicy(session, repo, metadata.getNature());
254 
255                         UpdateCheck<Metadata, MetadataTransferException> check = new UpdateCheck<>();
256                         check.setLocalLastUpdated((localLastUpdate != null) ? localLastUpdate : 0);
257                         check.setItem(metadata);
258 
259                         // use 'main' installation file for the check (-> use requested repository)
260                         Path checkPath = session.getLocalRepository()
261                                 .getBasePath()
262                                 .resolve(session.getLocalRepositoryManager()
263                                         .getPathForRemoteMetadata(metadata, repository, request.getRequestContext()));
264                         check.setPath(checkPath);
265                         check.setRepository(repository);
266                         check.setAuthoritativeRepository(repo);
267                         check.setArtifactPolicy(policy.getArtifactUpdatePolicy());
268                         check.setMetadataPolicy(policy.getMetadataUpdatePolicy());
269 
270                         if (lrmResult.isStale()) {
271                             checks.add(check);
272                         } else {
273                             updateCheckManager.checkMetadata(session, check);
274                             if (check.isRequired()) {
275                                 checks.add(check);
276                             } else if (exception == null) {
277                                 exception = check.getException();
278                             }
279                         }
280                     }
281 
282                     if (!checks.isEmpty()) {
283                         RepositoryPolicy policy = getPolicy(session, repository, metadata.getNature());
284 
285                         // install path may be different from lookup path
286                         Path installPath = session.getLocalRepository()
287                                 .getBasePath()
288                                 .resolve(session.getLocalRepositoryManager()
289                                         .getPathForRemoteMetadata(
290                                                 metadata, request.getRepository(), request.getRequestContext()));
291 
292                         ResolveTask task = new ResolveTask(
293                                 session, trace, result, installPath, checks, policy.getChecksumPolicy());
294                         tasks.add(task);
295                     } else {
296                         result.setException(exception);
297                         if (metadataPath != null) {
298                             metadata = metadata.setPath(metadataPath);
299                             result.setMetadata(metadata);
300                         }
301                         metadataResolved(session, trace, metadata, repository, result.getException());
302                     }
303                 }
304 
305                 if (!tasks.isEmpty() && current == shared) {
306                     current.close();
307                     current = exclusive;
308                     continue;
309                 }
310 
311                 if (!tasks.isEmpty()) {
312                     int threads = ExecutorUtils.threadCount(session, DEFAULT_THREADS, CONFIG_PROP_THREADS);
313                     Executor executor = ExecutorUtils.executor(
314                             Math.min(tasks.size(), threads), getClass().getSimpleName() + '-');
315                     try {
316                         RunnableErrorForwarder errorForwarder = new RunnableErrorForwarder();
317 
318                         for (ResolveTask task : tasks) {
319                             metadataDownloading(
320                                     task.session, task.trace, task.request.getMetadata(), task.request.getRepository());
321 
322                             executor.execute(errorForwarder.wrap(task));
323                         }
324 
325                         errorForwarder.await();
326 
327                         for (ResolveTask task : tasks) {
328                             /*
329                              * NOTE: Touch after registration with local repo to ensure concurrent resolution is not
330                              * rejected with "already updated" via session data when actual update to local repo is
331                              * still pending.
332                              */
333                             for (UpdateCheck<Metadata, MetadataTransferException> check : task.checks) {
334                                 updateCheckManager.touchMetadata(task.session, check.setException(task.exception));
335                             }
336 
337                             metadataDownloaded(
338                                     session,
339                                     task.trace,
340                                     task.request.getMetadata(),
341                                     task.request.getRepository(),
342                                     task.metadataPath,
343                                     task.exception);
344 
345                             task.result.setException(task.exception);
346                         }
347                     } finally {
348                         ExecutorUtils.shutdown(executor);
349                     }
350                     for (ResolveTask task : tasks) {
351                         Metadata metadata = task.request.getMetadata();
352                         // re-lookup metadata for resolve
353                         LocalMetadataRequest localRequest = new LocalMetadataRequest(
354                                 metadata, task.request.getRepository(), task.request.getRequestContext());
355                         Path metadataPath = session.getLocalRepositoryManager()
356                                 .find(session, localRequest)
357                                 .getPath();
358                         if (metadataPath != null) {
359                             metadata = metadata.setPath(metadataPath);
360                             task.result.setMetadata(metadata);
361                         }
362                         if (task.result.getException() == null) {
363                             task.result.setUpdated(true);
364                         }
365                         metadataResolved(
366                                 session,
367                                 task.trace,
368                                 metadata,
369                                 task.request.getRepository(),
370                                 task.result.getException());
371                     }
372                 }
373 
374                 return results;
375             }
376         } finally {
377             current.close();
378         }
379     }
380 
381     private Path getLocalFile(RepositorySystemSession session, Metadata metadata) {
382         LocalRepositoryManager lrm = session.getLocalRepositoryManager();
383         LocalMetadataResult localResult = lrm.find(session, new LocalMetadataRequest(metadata, null, null));
384         return localResult.getPath();
385     }
386 
387     private List<RemoteRepository> getEnabledSourceRepositories(RemoteRepository repository, Metadata.Nature nature) {
388         List<RemoteRepository> repositories = new ArrayList<>();
389 
390         if (repository.isRepositoryManager()) {
391             for (RemoteRepository repo : repository.getMirroredRepositories()) {
392                 if (isEnabled(repo, nature)) {
393                     repositories.add(repo);
394                 }
395             }
396         } else if (isEnabled(repository, nature)) {
397             repositories.add(repository);
398         }
399 
400         return repositories;
401     }
402 
403     private boolean isEnabled(RemoteRepository repository, Metadata.Nature nature) {
404         if (!Metadata.Nature.SNAPSHOT.equals(nature)
405                 && repository.getPolicy(false).isEnabled()) {
406             return true;
407         }
408         return !Metadata.Nature.RELEASE.equals(nature)
409                 && repository.getPolicy(true).isEnabled();
410     }
411 
412     private RepositoryPolicy getPolicy(
413             RepositorySystemSession session, RemoteRepository repository, Metadata.Nature nature) {
414         boolean releases = !Metadata.Nature.SNAPSHOT.equals(nature);
415         boolean snapshots = !Metadata.Nature.RELEASE.equals(nature);
416         return remoteRepositoryManager.getPolicy(session, repository, releases, snapshots);
417     }
418 
419     private void metadataResolving(
420             RepositorySystemSession session, RequestTrace trace, Metadata metadata, ArtifactRepository repository) {
421         RepositoryEvent.Builder event = new RepositoryEvent.Builder(session, EventType.METADATA_RESOLVING);
422         event.setTrace(trace);
423         event.setMetadata(metadata);
424         event.setRepository(repository);
425 
426         repositoryEventDispatcher.dispatch(event.build());
427     }
428 
429     private void metadataResolved(
430             RepositorySystemSession session,
431             RequestTrace trace,
432             Metadata metadata,
433             ArtifactRepository repository,
434             Exception exception) {
435         RepositoryEvent.Builder event = new RepositoryEvent.Builder(session, EventType.METADATA_RESOLVED);
436         event.setTrace(trace);
437         event.setMetadata(metadata);
438         event.setRepository(repository);
439         event.setException(exception);
440         event.setPath(metadata.getPath());
441 
442         repositoryEventDispatcher.dispatch(event.build());
443     }
444 
445     private void metadataDownloading(
446             RepositorySystemSession session, RequestTrace trace, Metadata metadata, ArtifactRepository repository) {
447         RepositoryEvent.Builder event = new RepositoryEvent.Builder(session, EventType.METADATA_DOWNLOADING);
448         event.setTrace(trace);
449         event.setMetadata(metadata);
450         event.setRepository(repository);
451 
452         repositoryEventDispatcher.dispatch(event.build());
453     }
454 
455     private void metadataDownloaded(
456             RepositorySystemSession session,
457             RequestTrace trace,
458             Metadata metadata,
459             ArtifactRepository repository,
460             Path path,
461             Exception exception) {
462         RepositoryEvent.Builder event = new RepositoryEvent.Builder(session, EventType.METADATA_DOWNLOADED);
463         event.setTrace(trace);
464         event.setMetadata(metadata);
465         event.setRepository(repository);
466         event.setException(exception);
467         event.setPath(path);
468 
469         repositoryEventDispatcher.dispatch(event.build());
470     }
471 
472     class ResolveTask implements Runnable {
473         final RepositorySystemSession session;
474 
475         final RequestTrace trace;
476 
477         final MetadataResult result;
478 
479         final MetadataRequest request;
480 
481         final Path metadataPath;
482 
483         final String policy;
484 
485         final List<UpdateCheck<Metadata, MetadataTransferException>> checks;
486 
487         volatile MetadataTransferException exception;
488 
489         ResolveTask(
490                 RepositorySystemSession session,
491                 RequestTrace trace,
492                 MetadataResult result,
493                 Path metadataPath,
494                 List<UpdateCheck<Metadata, MetadataTransferException>> checks,
495                 String policy) {
496             this.session = session;
497             this.trace = trace;
498             this.result = result;
499             this.request = result.getRequest();
500             this.metadataPath = metadataPath;
501             this.policy = policy;
502             this.checks = checks;
503         }
504 
505         public void run() {
506             Metadata metadata = request.getMetadata();
507             RemoteRepository requestRepository = request.getRepository();
508 
509             try {
510                 List<RemoteRepository> repositories = new ArrayList<>();
511                 for (UpdateCheck<Metadata, MetadataTransferException> check : checks) {
512                     repositories.add(check.getAuthoritativeRepository());
513                 }
514 
515                 MetadataDownload download = new MetadataDownload();
516                 download.setMetadata(metadata);
517                 download.setRequestContext(request.getRequestContext());
518                 download.setPath(metadataPath);
519                 download.setChecksumPolicy(policy);
520                 download.setRepositories(repositories);
521                 download.setListener(SafeTransferListener.wrap(session));
522                 download.setTrace(trace);
523 
524                 try (RepositoryConnector connector =
525                         repositoryConnectorProvider.newRepositoryConnector(session, requestRepository)) {
526                     connector.get(null, Collections.singletonList(download));
527                 }
528 
529                 exception = download.getException();
530 
531                 if (exception == null) {
532 
533                     List<String> contexts = Collections.singletonList(request.getRequestContext());
534                     LocalMetadataRegistration registration =
535                             new LocalMetadataRegistration(metadata, requestRepository, contexts);
536 
537                     session.getLocalRepositoryManager().add(session, registration);
538                 } else if (request.isDeleteLocalCopyIfMissing() && exception instanceof MetadataNotFoundException) {
539                     try {
540                         Files.deleteIfExists(download.getPath());
541                     } catch (IOException e) {
542                         throw new UncheckedIOException(e);
543                     }
544                 }
545             } catch (NoRepositoryConnectorException e) {
546                 exception = new MetadataTransferException(metadata, requestRepository, e);
547             }
548         }
549     }
550 }