001 package org.apache.archiva.proxy;
002
003 /*
004 * Licensed to the Apache Software Foundation (ASF) under one
005 * or more contributor license agreements. See the NOTICE file
006 * distributed with this work for additional information
007 * regarding copyright ownership. The ASF licenses this file
008 * to you under the Apache License, Version 2.0 (the
009 * "License"); you may not use this file except in compliance
010 * with the License. You may obtain a copy of the License at
011 *
012 * http://www.apache.org/licenses/LICENSE-2.0
013 *
014 * Unless required by applicable law or agreed to in writing,
015 * software distributed under the License is distributed on an
016 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
017 * KIND, either express or implied. See the License for the
018 * specific language governing permissions and limitations
019 * under the License.
020 */
021
022 import com.google.common.io.Files;
023 import org.apache.archiva.admin.model.RepositoryAdminException;
024 import org.apache.archiva.admin.model.beans.NetworkProxy;
025 import org.apache.archiva.admin.model.beans.ProxyConnectorRuleType;
026 import org.apache.archiva.admin.model.beans.RemoteRepository;
027 import org.apache.archiva.admin.model.networkproxy.NetworkProxyAdmin;
028 import org.apache.archiva.configuration.ArchivaConfiguration;
029 import org.apache.archiva.configuration.Configuration;
030 import org.apache.archiva.configuration.ConfigurationNames;
031 import org.apache.archiva.configuration.NetworkProxyConfiguration;
032 import org.apache.archiva.configuration.ProxyConnectorConfiguration;
033 import org.apache.archiva.configuration.ProxyConnectorRuleConfiguration;
034 import org.apache.archiva.model.ArtifactReference;
035 import org.apache.archiva.model.Keys;
036 import org.apache.archiva.model.RepositoryURL;
037 import org.apache.archiva.policies.DownloadErrorPolicy;
038 import org.apache.archiva.policies.DownloadPolicy;
039 import org.apache.archiva.policies.PolicyConfigurationException;
040 import org.apache.archiva.policies.PolicyViolationException;
041 import org.apache.archiva.policies.PostDownloadPolicy;
042 import org.apache.archiva.policies.PreDownloadPolicy;
043 import org.apache.archiva.policies.ProxyDownloadException;
044 import org.apache.archiva.policies.urlcache.UrlFailureCache;
045 import org.apache.archiva.proxy.common.WagonFactory;
046 import org.apache.archiva.proxy.common.WagonFactoryException;
047 import org.apache.archiva.proxy.common.WagonFactoryRequest;
048 import org.apache.archiva.proxy.model.ProxyConnector;
049 import org.apache.archiva.proxy.model.RepositoryProxyConnectors;
050 import org.apache.archiva.redback.components.registry.Registry;
051 import org.apache.archiva.redback.components.registry.RegistryListener;
052 import org.apache.archiva.redback.components.taskqueue.TaskQueueException;
053 import org.apache.archiva.repository.ManagedRepositoryContent;
054 import org.apache.archiva.repository.RemoteRepositoryContent;
055 import org.apache.archiva.repository.RepositoryContentFactory;
056 import org.apache.archiva.repository.RepositoryException;
057 import org.apache.archiva.repository.RepositoryNotFoundException;
058 import org.apache.archiva.repository.metadata.MetadataTools;
059 import org.apache.archiva.repository.metadata.RepositoryMetadataException;
060 import org.apache.archiva.scheduler.ArchivaTaskScheduler;
061 import org.apache.archiva.scheduler.repository.model.RepositoryTask;
062 import org.apache.commons.collections.CollectionUtils;
063 import org.apache.commons.io.FileUtils;
064 import org.apache.commons.io.FilenameUtils;
065 import org.apache.commons.lang.StringUtils;
066 import org.apache.commons.lang.SystemUtils;
067 import org.apache.maven.wagon.ConnectionException;
068 import org.apache.maven.wagon.ResourceDoesNotExistException;
069 import org.apache.maven.wagon.Wagon;
070 import org.apache.maven.wagon.WagonException;
071 import org.apache.maven.wagon.authentication.AuthenticationException;
072 import org.apache.maven.wagon.authentication.AuthenticationInfo;
073 import org.apache.maven.wagon.proxy.ProxyInfo;
074 import org.apache.maven.wagon.repository.Repository;
075 import org.apache.tools.ant.types.selectors.SelectorUtils;
076 import org.slf4j.Logger;
077 import org.slf4j.LoggerFactory;
078 import org.slf4j.MarkerFactory;
079 import org.springframework.stereotype.Service;
080
081 import javax.annotation.PostConstruct;
082 import javax.inject.Inject;
083 import javax.inject.Named;
084 import java.io.File;
085 import java.io.IOException;
086 import java.util.ArrayList;
087 import java.util.Collections;
088 import java.util.HashMap;
089 import java.util.LinkedHashMap;
090 import java.util.List;
091 import java.util.Map;
092 import java.util.Map.Entry;
093 import java.util.Properties;
094 import java.util.concurrent.ConcurrentHashMap;
095
096 /**
097 * DefaultRepositoryProxyConnectors
098 *
099 * @todo exception handling needs work - "not modified" is not really an exceptional case, and it has more layers than
100 * your average brown onion
101 */
102 @Service("repositoryProxyConnectors#default")
103 public class DefaultRepositoryProxyConnectors
104 implements RepositoryProxyConnectors, RegistryListener
105 {
106 private Logger log = LoggerFactory.getLogger( DefaultRepositoryProxyConnectors.class );
107
108 /**
109 *
110 */
111 @Inject
112 @Named(value = "archivaConfiguration#default")
113 private ArchivaConfiguration archivaConfiguration;
114
115 /**
116 *
117 */
118 @Inject
119 @Named(value = "repositoryContentFactory#default")
120 private RepositoryContentFactory repositoryFactory;
121
122 /**
123 *
124 */
125 @Inject
126 @Named(value = "metadataTools#default")
127 private MetadataTools metadataTools;
128
129 /**
130 *
131 */
132 @Inject
133 private Map<String, PreDownloadPolicy> preDownloadPolicies;
134
135 /**
136 *
137 */
138 @Inject
139 private Map<String, PostDownloadPolicy> postDownloadPolicies;
140
141 /**
142 *
143 */
144 @Inject
145 private Map<String, DownloadErrorPolicy> downloadErrorPolicies;
146
147 /**
148 *
149 */
150 @Inject
151 private UrlFailureCache urlFailureCache;
152
153 private Map<String, List<ProxyConnector>> proxyConnectorMap = new HashMap<String, List<ProxyConnector>>();
154
155 private Map<String, ProxyInfo> networkProxyMap = new ConcurrentHashMap<String, ProxyInfo>();
156
157 /**
158 *
159 */
160 @Inject
161 private WagonFactory wagonFactory;
162
163 /**
164 *
165 */
166 @Inject
167 @Named(value = "archivaTaskScheduler#repository")
168 private ArchivaTaskScheduler scheduler;
169
170 @Inject
171 private NetworkProxyAdmin networkProxyAdmin;
172
173 @PostConstruct
174 public void initialize()
175 {
176 initConnectorsAndNetworkProxies();
177 archivaConfiguration.addChangeListener( this );
178
179 }
180
181 @SuppressWarnings("unchecked")
182 private void initConnectorsAndNetworkProxies()
183 {
184
185 ProxyConnectorOrderComparator proxyOrderSorter = new ProxyConnectorOrderComparator();
186 this.proxyConnectorMap.clear();
187
188 Configuration configuration = archivaConfiguration.getConfiguration();
189
190 List<ProxyConnectorRuleConfiguration> allProxyConnectorRuleConfigurations =
191 configuration.getProxyConnectorRuleConfigurations();
192
193 List<ProxyConnectorConfiguration> proxyConfigs = configuration.getProxyConnectors();
194 for ( ProxyConnectorConfiguration proxyConfig : proxyConfigs )
195 {
196 String key = proxyConfig.getSourceRepoId();
197
198 try
199 {
200 // Create connector object.
201 ProxyConnector connector = new ProxyConnector();
202
203 connector.setSourceRepository(
204 repositoryFactory.getManagedRepositoryContent( proxyConfig.getSourceRepoId() ) );
205 connector.setTargetRepository(
206 repositoryFactory.getRemoteRepositoryContent( proxyConfig.getTargetRepoId() ) );
207
208 connector.setProxyId( proxyConfig.getProxyId() );
209 connector.setPolicies( proxyConfig.getPolicies() );
210 connector.setOrder( proxyConfig.getOrder() );
211 connector.setDisabled( proxyConfig.isDisabled() );
212
213 // Copy any blacklist patterns.
214 List<String> blacklist = new ArrayList<String>( 0 );
215 if ( CollectionUtils.isNotEmpty( proxyConfig.getBlackListPatterns() ) )
216 {
217 blacklist.addAll( proxyConfig.getBlackListPatterns() );
218 }
219 connector.setBlacklist( blacklist );
220
221 // Copy any whitelist patterns.
222 List<String> whitelist = new ArrayList<String>( 0 );
223 if ( CollectionUtils.isNotEmpty( proxyConfig.getWhiteListPatterns() ) )
224 {
225 whitelist.addAll( proxyConfig.getWhiteListPatterns() );
226 }
227 connector.setWhitelist( whitelist );
228
229 List<ProxyConnectorRuleConfiguration> proxyConnectorRuleConfigurations =
230 findProxyConnectorRules( connector.getSourceRepository().getId(),
231 connector.getTargetRepository().getId(),
232 allProxyConnectorRuleConfigurations );
233
234 if ( !proxyConnectorRuleConfigurations.isEmpty() )
235 {
236 for ( ProxyConnectorRuleConfiguration proxyConnectorRuleConfiguration : proxyConnectorRuleConfigurations )
237 {
238 if ( StringUtils.equals( proxyConnectorRuleConfiguration.getRuleType(),
239 ProxyConnectorRuleType.BLACK_LIST.getRuleType() ) )
240 {
241 connector.getBlacklist().add( proxyConnectorRuleConfiguration.getPattern() );
242 }
243
244 if ( StringUtils.equals( proxyConnectorRuleConfiguration.getRuleType(),
245 ProxyConnectorRuleType.WHITE_LIST.getRuleType() ) )
246 {
247 connector.getWhitelist().add( proxyConnectorRuleConfiguration.getPattern() );
248 }
249 }
250 }
251
252 // Get other connectors
253 List<ProxyConnector> connectors = this.proxyConnectorMap.get( key );
254 if ( connectors == null )
255 {
256 // Create if we are the first.
257 connectors = new ArrayList<ProxyConnector>( 1 );
258 }
259
260 // Add the connector.
261 connectors.add( connector );
262
263 // Ensure the list is sorted.
264 Collections.sort( connectors, proxyOrderSorter );
265
266 // Set the key to the list of connectors.
267 this.proxyConnectorMap.put( key, connectors );
268 }
269 catch ( RepositoryNotFoundException e )
270 {
271 log.warn( "Unable to use proxy connector: " + e.getMessage(), e );
272 }
273 catch ( RepositoryException e )
274 {
275 log.warn( "Unable to use proxy connector: " + e.getMessage(), e );
276 }
277
278
279 }
280
281 this.networkProxyMap.clear();
282
283 List<NetworkProxyConfiguration> networkProxies = archivaConfiguration.getConfiguration().getNetworkProxies();
284 for ( NetworkProxyConfiguration networkProxyConfig : networkProxies )
285 {
286 String key = networkProxyConfig.getId();
287
288 ProxyInfo proxy = new ProxyInfo();
289
290 proxy.setType( networkProxyConfig.getProtocol() );
291 proxy.setHost( networkProxyConfig.getHost() );
292 proxy.setPort( networkProxyConfig.getPort() );
293 proxy.setUserName( networkProxyConfig.getUsername() );
294 proxy.setPassword( networkProxyConfig.getPassword() );
295
296 this.networkProxyMap.put( key, proxy );
297 }
298
299 }
300
301 private List<ProxyConnectorRuleConfiguration> findProxyConnectorRules( String sourceRepository,
302 String targetRepository,
303 List<ProxyConnectorRuleConfiguration> all )
304 {
305 List<ProxyConnectorRuleConfiguration> proxyConnectorRuleConfigurations =
306 new ArrayList<ProxyConnectorRuleConfiguration>();
307
308 for ( ProxyConnectorRuleConfiguration proxyConnectorRuleConfiguration : all )
309 {
310 for ( ProxyConnectorConfiguration proxyConnector : proxyConnectorRuleConfiguration.getProxyConnectors() )
311 {
312 if ( StringUtils.equals( sourceRepository, proxyConnector.getSourceRepoId() ) && StringUtils.equals(
313 targetRepository, proxyConnector.getTargetRepoId() ) )
314 {
315 proxyConnectorRuleConfigurations.add( proxyConnectorRuleConfiguration );
316 }
317 }
318 }
319
320 return proxyConnectorRuleConfigurations;
321 }
322
323 public File fetchFromProxies( ManagedRepositoryContent repository, ArtifactReference artifact )
324 throws ProxyDownloadException
325 {
326 File localFile = toLocalFile( repository, artifact );
327
328 Properties requestProperties = new Properties();
329 requestProperties.setProperty( "filetype", "artifact" );
330 requestProperties.setProperty( "version", artifact.getVersion() );
331 requestProperties.setProperty( "managedRepositoryId", repository.getId() );
332
333 List<ProxyConnector> connectors = getProxyConnectors( repository );
334 Map<String, Exception> previousExceptions = new LinkedHashMap<String, Exception>();
335 for ( ProxyConnector connector : connectors )
336 {
337 if ( connector.isDisabled() )
338 {
339 continue;
340 }
341
342 RemoteRepositoryContent targetRepository = connector.getTargetRepository();
343 requestProperties.setProperty( "remoteRepositoryId", targetRepository.getId() );
344
345 String targetPath = targetRepository.toPath( artifact );
346
347 if ( SystemUtils.IS_OS_WINDOWS )
348 {
349 // toPath use system PATH_SEPARATOR so on windows url are \ which doesn't work very well :-)
350 targetPath = FilenameUtils.separatorsToUnix( targetPath );
351 }
352
353 try
354 {
355 File downloadedFile =
356 transferFile( connector, targetRepository, targetPath, repository, localFile, requestProperties,
357 true );
358
359 if ( fileExists( downloadedFile ) )
360 {
361 log.debug( "Successfully transferred: {}", downloadedFile.getAbsolutePath() );
362 return downloadedFile;
363 }
364 }
365 catch ( NotFoundException e )
366 {
367 log.debug( "Artifact {} not found on repository \"{}\".", Keys.toKey( artifact ),
368 targetRepository.getRepository().getId() );
369 }
370 catch ( NotModifiedException e )
371 {
372 log.debug( "Artifact {} not updated on repository \"{}\".", Keys.toKey( artifact ),
373 targetRepository.getRepository().getId() );
374 }
375 catch ( ProxyException e )
376 {
377 validatePolicies( this.downloadErrorPolicies, connector.getPolicies(), requestProperties, artifact,
378 targetRepository, localFile, e, previousExceptions );
379 }
380 catch ( RepositoryAdminException e )
381 {
382 validatePolicies( this.downloadErrorPolicies, connector.getPolicies(), requestProperties, artifact,
383 targetRepository, localFile, e, previousExceptions );
384 }
385 }
386
387 if ( !previousExceptions.isEmpty() )
388 {
389 throw new ProxyDownloadException( "Failures occurred downloading from some remote repositories",
390 previousExceptions );
391 }
392
393 log.debug( "Exhausted all target repositories, artifact {} not found.", Keys.toKey( artifact ) );
394
395 return null;
396 }
397
398 public File fetchFromProxies( ManagedRepositoryContent repository, String path )
399 {
400 File localFile = new File( repository.getRepoRoot(), path );
401
402 // no update policies for these paths
403 if ( localFile.exists() )
404 {
405 return null;
406 }
407
408 Properties requestProperties = new Properties();
409 requestProperties.setProperty( "filetype", "resource" );
410 requestProperties.setProperty( "managedRepositoryId", repository.getId() );
411
412 List<ProxyConnector> connectors = getProxyConnectors( repository );
413 for ( ProxyConnector connector : connectors )
414 {
415 if ( connector.isDisabled() )
416 {
417 continue;
418 }
419
420 RemoteRepositoryContent targetRepository = connector.getTargetRepository();
421 requestProperties.setProperty( "remoteRepositoryId", targetRepository.getId() );
422
423 String targetPath = path;
424
425 try
426 {
427 File downloadedFile =
428 transferFile( connector, targetRepository, targetPath, repository, localFile, requestProperties,
429 false );
430
431 if ( fileExists( downloadedFile ) )
432 {
433 log.debug( "Successfully transferred: {}", downloadedFile.getAbsolutePath() );
434 return downloadedFile;
435 }
436 }
437 catch ( NotFoundException e )
438 {
439 log.debug( "Resource {} not found on repository \"{}\".", path,
440 targetRepository.getRepository().getId() );
441 }
442 catch ( NotModifiedException e )
443 {
444 log.debug( "Resource {} not updated on repository \"{}\".", path,
445 targetRepository.getRepository().getId() );
446 }
447 catch ( ProxyException e )
448 {
449 log.warn(
450 "Transfer error from repository \"" + targetRepository.getRepository().getId() + "\" for resource "
451 + path + ", continuing to next repository. Error message: {}", e.getMessage() );
452 log.debug( MarkerFactory.getDetachedMarker( "transfer.error" ),
453 "Transfer error from repository \"" + targetRepository.getRepository().getId()
454 + "\" for resource " + path + ", continuing to next repository. Error message: {}",
455 e.getMessage(), e );
456 }
457 catch ( RepositoryAdminException e )
458 {
459 log.debug( MarkerFactory.getDetachedMarker( "transfer.error" ),
460 "Transfer error from repository \"" + targetRepository.getRepository().getId()
461 + "\" for resource " + path + ", continuing to next repository. Error message: {}",
462 e.getMessage(), e );
463 log.debug( MarkerFactory.getDetachedMarker( "transfer.error" ), "Full stack trace", e );
464 }
465 }
466
467 log.debug( "Exhausted all target repositories, resource {} not found.", path );
468
469 return null;
470 }
471
472 public File fetchMetatadaFromProxies( ManagedRepositoryContent repository, String logicalPath )
473 {
474 File localFile = new File( repository.getRepoRoot(), logicalPath );
475
476 Properties requestProperties = new Properties();
477 requestProperties.setProperty( "filetype", "metadata" );
478 boolean metadataNeedsUpdating = false;
479 long originalTimestamp = getLastModified( localFile );
480
481 List<ProxyConnector> connectors = getProxyConnectors( repository );
482 for ( ProxyConnector connector : connectors )
483 {
484 if ( connector.isDisabled() )
485 {
486 continue;
487 }
488
489 RemoteRepositoryContent targetRepository = connector.getTargetRepository();
490
491 File localRepoFile = toLocalRepoFile( repository, targetRepository, logicalPath );
492 long originalMetadataTimestamp = getLastModified( localRepoFile );
493
494 try
495 {
496 transferFile( connector, targetRepository, logicalPath, repository, localRepoFile, requestProperties,
497 true );
498
499 if ( hasBeenUpdated( localRepoFile, originalMetadataTimestamp ) )
500 {
501 metadataNeedsUpdating = true;
502 }
503 }
504 catch ( NotFoundException e )
505 {
506
507 log.debug( "Metadata {} not found on remote repository '{}'.", logicalPath,
508 targetRepository.getRepository().getId(), e );
509
510 }
511 catch ( NotModifiedException e )
512 {
513
514 log.debug( "Metadata {} not updated on remote repository '{}'.", logicalPath,
515 targetRepository.getRepository().getId(), e );
516
517 }
518 catch ( ProxyException e )
519 {
520 log.warn( "Transfer error from repository \"" + targetRepository.getRepository().getId()
521 + "\" for versioned Metadata " + logicalPath
522 + ", continuing to next repository. Error message: " + e.getMessage() );
523 log.debug( "Full stack trace", e );
524 }
525 catch ( RepositoryAdminException e )
526 {
527 log.warn( "Transfer error from repository \"" + targetRepository.getRepository().getId()
528 + "\" for versioned Metadata " + logicalPath
529 + ", continuing to next repository. Error message: " + e.getMessage() );
530 log.debug( "Full stack trace", e );
531 }
532 }
533
534 if ( hasBeenUpdated( localFile, originalTimestamp ) )
535 {
536 metadataNeedsUpdating = true;
537 }
538
539 if ( metadataNeedsUpdating || !localFile.exists() )
540 {
541 try
542 {
543 metadataTools.updateMetadata( repository, logicalPath );
544 }
545 catch ( RepositoryMetadataException e )
546 {
547 log.warn( "Unable to update metadata " + localFile.getAbsolutePath() + ": " + e.getMessage(), e );
548 }
549 }
550
551 if ( fileExists( localFile ) )
552 {
553 return localFile;
554 }
555
556 return null;
557 }
558
559 /**
560 * @param connector
561 * @param remoteRepository
562 * @param tmpMd5
563 * @param tmpSha1
564 * @param tmpResource
565 * @param url
566 * @param remotePath
567 * @param resource
568 * @param workingDirectory
569 * @param repository
570 * @throws ProxyException
571 * @throws NotModifiedException
572 */
573 protected void transferResources( ProxyConnector connector, RemoteRepositoryContent remoteRepository, File tmpMd5,
574 File tmpSha1, File tmpResource, String url, String remotePath, File resource,
575 File workingDirectory, ManagedRepositoryContent repository )
576 throws ProxyException, NotModifiedException, RepositoryAdminException
577 {
578 Wagon wagon = null;
579 try
580 {
581 RepositoryURL repoUrl = remoteRepository.getURL();
582 String protocol = repoUrl.getProtocol();
583 NetworkProxy networkProxy = null;
584 if ( StringUtils.isNotBlank( connector.getProxyId() ) )
585 {
586 networkProxy = networkProxyAdmin.getNetworkProxy( connector.getProxyId() );
587 }
588 WagonFactoryRequest wagonFactoryRequest = new WagonFactoryRequest( "wagon#" + protocol,
589 remoteRepository.getRepository().getExtraHeaders() ).networkProxy(
590 networkProxy );
591 wagon = wagonFactory.getWagon( wagonFactoryRequest );
592 if ( wagon == null )
593 {
594 throw new ProxyException( "Unsupported target repository protocol: " + protocol );
595 }
596
597 if ( wagon == null )
598 {
599 throw new ProxyException( "Unsupported target repository protocol: " + protocol );
600 }
601
602 boolean connected = connectToRepository( connector, wagon, remoteRepository );
603 if ( connected )
604 {
605 transferArtifact( wagon, remoteRepository, remotePath, repository, resource, workingDirectory,
606 tmpResource );
607
608 // TODO: these should be used to validate the download based on the policies, not always downloaded
609 // to
610 // save on connections since md5 is rarely used
611 transferChecksum( wagon, remoteRepository, remotePath, repository, resource, workingDirectory, ".sha1",
612 tmpSha1 );
613 transferChecksum( wagon, remoteRepository, remotePath, repository, resource, workingDirectory, ".md5",
614 tmpMd5 );
615 }
616 }
617 catch ( NotFoundException e )
618 {
619 urlFailureCache.cacheFailure( url );
620 throw e;
621 }
622 catch ( NotModifiedException e )
623 {
624 // Do not cache url here.
625 throw e;
626 }
627 catch ( ProxyException e )
628 {
629 urlFailureCache.cacheFailure( url );
630 throw e;
631 }
632 catch ( WagonFactoryException e )
633 {
634 throw new ProxyException( e.getMessage(), e );
635 }
636 finally
637 {
638 if ( wagon != null )
639 {
640 try
641 {
642 wagon.disconnect();
643 }
644 catch ( ConnectionException e )
645 {
646 log.warn( "Unable to disconnect wagon.", e );
647 }
648 }
649 }
650 }
651
652 private void transferArtifact( Wagon wagon, RemoteRepositoryContent remoteRepository, String remotePath,
653 ManagedRepositoryContent repository, File resource, File tmpDirectory,
654 File destFile )
655 throws ProxyException
656 {
657 transferSimpleFile( wagon, remoteRepository, remotePath, repository, resource, destFile );
658 }
659
660 private long getLastModified( File file )
661 {
662 if ( !file.exists() || !file.isFile() )
663 {
664 return 0;
665 }
666
667 return file.lastModified();
668 }
669
670 private boolean hasBeenUpdated( File file, long originalLastModified )
671 {
672 if ( !file.exists() || !file.isFile() )
673 {
674 return false;
675 }
676
677 long currentLastModified = getLastModified( file );
678 return ( currentLastModified > originalLastModified );
679 }
680
681 private File toLocalRepoFile( ManagedRepositoryContent repository, RemoteRepositoryContent targetRepository,
682 String targetPath )
683 {
684 String repoPath = metadataTools.getRepositorySpecificName( targetRepository, targetPath );
685 return new File( repository.getRepoRoot(), repoPath );
686 }
687
688 /**
689 * Test if the provided ManagedRepositoryContent has any proxies configured for it.
690 */
691 public boolean hasProxies( ManagedRepositoryContent repository )
692 {
693 synchronized ( this.proxyConnectorMap )
694 {
695 return this.proxyConnectorMap.containsKey( repository.getId() );
696 }
697 }
698
699 private File toLocalFile( ManagedRepositoryContent repository, ArtifactReference artifact )
700 {
701 return repository.toFile( artifact );
702 }
703
704 /**
705 * Simple method to test if the file exists on the local disk.
706 *
707 * @param file the file to test. (may be null)
708 * @return true if file exists. false if the file param is null, doesn't exist, or is not of type File.
709 */
710 private boolean fileExists( File file )
711 {
712 if ( file == null )
713 {
714 return false;
715 }
716
717 if ( !file.exists() )
718 {
719 return false;
720 }
721
722 if ( !file.isFile() )
723 {
724 return false;
725 }
726
727 return true;
728 }
729
730 /**
731 * Perform the transfer of the file.
732 *
733 * @param connector the connector configuration to use.
734 * @param remoteRepository the remote repository get the resource from.
735 * @param remotePath the path in the remote repository to the resource to get.
736 * @param repository the managed repository that will hold the file
737 * @param resource the local file to place the downloaded resource into
738 * @param requestProperties the request properties to utilize for policy handling.
739 * @param executeConsumers whether to execute the consumers after proxying
740 * @return the local file that was downloaded, or null if not downloaded.
741 * @throws NotFoundException if the file was not found on the remote repository.
742 * @throws NotModifiedException if the localFile was present, and the resource was present on remote repository, but
743 * the remote resource is not newer than the local File.
744 * @throws ProxyException if transfer was unsuccessful.
745 */
746 private File transferFile( ProxyConnector connector, RemoteRepositoryContent remoteRepository, String remotePath,
747 ManagedRepositoryContent repository, File resource, Properties requestProperties,
748 boolean executeConsumers )
749 throws ProxyException, NotModifiedException, RepositoryAdminException
750 {
751 String url = remoteRepository.getURL().getUrl();
752 if ( !url.endsWith( "/" ) )
753 {
754 url = url + "/";
755 }
756 url = url + remotePath;
757 requestProperties.setProperty( "url", url );
758
759 // Is a whitelist defined?
760 if ( CollectionUtils.isNotEmpty( connector.getWhitelist() ) )
761 {
762 // Path must belong to whitelist.
763 if ( !matchesPattern( remotePath, connector.getWhitelist() ) )
764 {
765 log.debug( "Path [{}] is not part of defined whitelist (skipping transfer from repository [{}]).",
766 remotePath, remoteRepository.getRepository().getName() );
767 return null;
768 }
769 }
770
771 // Is target path part of blacklist?
772 if ( matchesPattern( remotePath, connector.getBlacklist() ) )
773 {
774 log.debug( "Path [{}] is part of blacklist (skipping transfer from repository [{}]).", remotePath,
775 remoteRepository.getRepository().getName() );
776 return null;
777 }
778
779 // Handle pre-download policy
780 try
781 {
782 validatePolicies( this.preDownloadPolicies, connector.getPolicies(), requestProperties, resource );
783 }
784 catch ( PolicyViolationException e )
785 {
786 String emsg = "Transfer not attempted on " + url + " : " + e.getMessage();
787 if ( fileExists( resource ) )
788 {
789 log.debug( "{} : using already present local file.", emsg );
790 return resource;
791 }
792
793 log.debug( emsg );
794 return null;
795 }
796
797 File workingDirectory = createWorkingDirectory( repository );
798 File tmpResource = new File( workingDirectory, resource.getName() );
799 File tmpMd5 = new File( workingDirectory, resource.getName() + ".md5" );
800 File tmpSha1 = new File( workingDirectory, resource.getName() + ".sha1" );
801
802 try
803 {
804 Wagon wagon = null;
805
806 transferResources( connector, remoteRepository, tmpMd5, tmpSha1, tmpResource, url, remotePath, resource,
807 workingDirectory, repository );
808
809 // Handle post-download policies.
810 try
811 {
812 validatePolicies( this.postDownloadPolicies, connector.getPolicies(), requestProperties, tmpResource );
813 }
814 catch ( PolicyViolationException e )
815 {
816 log.warn( "Transfer invalidated from {} : {}", url, e.getMessage() );
817 executeConsumers = false;
818 if ( !fileExists( tmpResource ) )
819 {
820 resource = null;
821 }
822 }
823
824 if ( resource != null )
825 {
826 synchronized ( resource.getAbsolutePath().intern() )
827 {
828 File directory = resource.getParentFile();
829 moveFileIfExists( tmpMd5, directory );
830 moveFileIfExists( tmpSha1, directory );
831 moveFileIfExists( tmpResource, directory );
832 }
833 }
834 }
835 finally
836 {
837 FileUtils.deleteQuietly( workingDirectory );
838 }
839
840 if ( executeConsumers )
841 {
842 // Just-in-time update of the index and database by executing the consumers for this artifact
843 //consumers.executeConsumers( connector.getSourceRepository().getRepository(), resource );
844 queueRepositoryTask( connector.getSourceRepository().getRepository().getId(), resource );
845 }
846
847 return resource;
848 }
849
850 private void queueRepositoryTask( String repositoryId, File localFile )
851 {
852 RepositoryTask task = new RepositoryTask();
853 task.setRepositoryId( repositoryId );
854 task.setResourceFile( localFile );
855 task.setUpdateRelatedArtifacts( true );
856 task.setScanAll( true );
857
858 try
859 {
860 scheduler.queueTask( task );
861 }
862 catch ( TaskQueueException e )
863 {
864 log.error( "Unable to queue repository task to execute consumers on resource file ['" + localFile.getName()
865 + "']." );
866 }
867 }
868
869 /**
870 * Moves the file into repository location if it exists
871 *
872 * @param fileToMove this could be either the main artifact, sha1 or md5 checksum file.
873 * @param directory directory to write files to
874 */
875 private void moveFileIfExists( File fileToMove, File directory )
876 throws ProxyException
877 {
878 if ( fileToMove != null && fileToMove.exists() )
879 {
880 File newLocation = new File( directory, fileToMove.getName() );
881 moveTempToTarget( fileToMove, newLocation );
882 }
883 }
884
885 /**
886 * <p>
887 * Quietly transfer the checksum file from the remote repository to the local file.
888 * </p>
889 *
890 * @param wagon the wagon instance (should already be connected) to use.
891 * @param remoteRepository the remote repository to transfer from.
892 * @param remotePath the remote path to the resource to get.
893 * @param repository the managed repository that will hold the file
894 * @param resource the local file that should contain the downloaded contents
895 * @param tmpDirectory the temporary directory to download to
896 * @param ext the type of checksum to transfer (example: ".md5" or ".sha1")
897 * @throws ProxyException if copying the downloaded file into place did not succeed.
898 */
899 private void transferChecksum( Wagon wagon, RemoteRepositoryContent remoteRepository, String remotePath,
900 ManagedRepositoryContent repository, File resource, File tmpDirectory, String ext,
901 File destFile )
902 throws ProxyException
903 {
904 String url = remoteRepository.getURL().getUrl() + remotePath + ext;
905
906 // Transfer checksum does not use the policy.
907 if ( urlFailureCache.hasFailedBefore( url ) )
908 {
909 return;
910 }
911
912 try
913 {
914 transferSimpleFile( wagon, remoteRepository, remotePath + ext, repository, resource, destFile );
915 log.debug( "Checksum {} Downloaded: {} to move to {}", url, destFile, resource );
916 }
917 catch ( NotFoundException e )
918 {
919 urlFailureCache.cacheFailure( url );
920 log.debug( "Transfer failed, checksum not found: {}", url );
921 // Consume it, do not pass this on.
922 }
923 catch ( NotModifiedException e )
924 {
925 log.debug( "Transfer skipped, checksum not modified: {}", url );
926 // Consume it, do not pass this on.
927 }
928 catch ( ProxyException e )
929 {
930 urlFailureCache.cacheFailure( url );
931 log.warn( "Transfer failed on checksum: " + url + " : " + e.getMessage(), e );
932 // Critical issue, pass it on.
933 throw e;
934 }
935 }
936
937 /**
938 * Perform the transfer of the remote file to the local file specified.
939 *
940 * @param wagon the wagon instance to use.
941 * @param remoteRepository the remote repository to use
942 * @param remotePath the remote path to attempt to get
943 * @param repository the managed repository that will hold the file
944 * @param origFile the local file to save to
945 * @return The local file that was transfered.
946 * @throws ProxyException if there was a problem moving the downloaded file into place.
947 * @throws WagonException if there was a problem tranfering the file.
948 */
949 private void transferSimpleFile( Wagon wagon, RemoteRepositoryContent remoteRepository, String remotePath,
950 ManagedRepositoryContent repository, File origFile, File destFile )
951 throws ProxyException
952 {
953 assert ( remotePath != null );
954
955 // Transfer the file.
956 try
957 {
958 boolean success = false;
959
960 if ( !origFile.exists() )
961 {
962 log.debug( "Retrieving {} from {}", remotePath, remoteRepository.getRepository().getName() );
963 wagon.get( addParameters( remotePath, remoteRepository.getRepository() ), destFile );
964 success = true;
965
966 // You wouldn't get here on failure, a WagonException would have been thrown.
967 log.debug( "Downloaded successfully." );
968 }
969 else
970 {
971 log.debug( "Retrieving {} from {} if updated", remotePath, remoteRepository.getRepository().getName() );
972 success = wagon.getIfNewer( addParameters( remotePath, remoteRepository.getRepository() ), destFile,
973 origFile.lastModified() );
974 if ( !success )
975 {
976 throw new NotModifiedException(
977 "Not downloaded, as local file is newer than remote side: " + origFile.getAbsolutePath() );
978 }
979
980 if ( destFile.exists() )
981 {
982 log.debug( "Downloaded successfully." );
983 }
984 }
985 }
986 catch ( ResourceDoesNotExistException e )
987 {
988 throw new NotFoundException(
989 "Resource [" + remoteRepository.getURL() + "/" + remotePath + "] does not exist: " + e.getMessage(),
990 e );
991 }
992 catch ( WagonException e )
993 {
994 // TODO: shouldn't have to drill into the cause, but TransferFailedException is often not descriptive enough
995
996 String msg =
997 "Download failure on resource [" + remoteRepository.getURL() + "/" + remotePath + "]:" + e.getMessage();
998 if ( e.getCause() != null )
999 {
1000 msg += " (cause: " + e.getCause() + ")";
1001 }
1002 throw new ProxyException( msg, e );
1003 }
1004 }
1005
1006 /**
1007 * Apply the policies.
1008 *
1009 * @param policies the map of policies to execute. (Map of String policy keys, to {@link DownloadPolicy} objects)
1010 * @param settings the map of settings for the policies to execute. (Map of String policy keys, to String policy
1011 * setting)
1012 * @param request the request properties (utilized by the {@link DownloadPolicy#applyPolicy(String, Properties, File)}
1013 * )
1014 * @param localFile the local file (utilized by the {@link DownloadPolicy#applyPolicy(String, Properties, File)})
1015 */
1016 private void validatePolicies( Map<String, ? extends DownloadPolicy> policies, Map<String, String> settings,
1017 Properties request, File localFile )
1018 throws PolicyViolationException
1019 {
1020 for ( Entry<String, ? extends DownloadPolicy> entry : policies.entrySet() )
1021 {
1022 // olamy with spring rolehint is now downloadPolicy#hint
1023 // so substring after last # to get the hint as with plexus
1024 String key = StringUtils.substringAfterLast( entry.getKey(), "#" );
1025 DownloadPolicy policy = entry.getValue();
1026 String defaultSetting = policy.getDefaultOption();
1027
1028 String setting = StringUtils.defaultString( settings.get( key ), defaultSetting );
1029
1030 log.debug( "Applying [{}] policy with [{}]", key, setting );
1031 try
1032 {
1033 policy.applyPolicy( setting, request, localFile );
1034 }
1035 catch ( PolicyConfigurationException e )
1036 {
1037 log.error( e.getMessage(), e );
1038 }
1039 }
1040 }
1041
1042 private void validatePolicies( Map<String, DownloadErrorPolicy> policies, Map<String, String> settings,
1043 Properties request, ArtifactReference artifact, RemoteRepositoryContent content,
1044 File localFile, Exception exception, Map<String, Exception> previousExceptions )
1045 throws ProxyDownloadException
1046 {
1047 boolean process = true;
1048 for ( Entry<String, ? extends DownloadErrorPolicy> entry : policies.entrySet() )
1049 {
1050
1051 // olamy with spring rolehint is now downloadPolicy#hint
1052 // so substring after last # to get the hint as with plexus
1053 String key = StringUtils.substringAfterLast( entry.getKey(), "#" );
1054 DownloadErrorPolicy policy = entry.getValue();
1055 String defaultSetting = policy.getDefaultOption();
1056 String setting = StringUtils.defaultString( settings.get( key ), defaultSetting );
1057
1058 log.debug( "Applying [{}] policy with [{}]", key, setting );
1059 try
1060 {
1061 // all policies must approve the exception, any can cancel
1062 process = policy.applyPolicy( setting, request, localFile, exception, previousExceptions );
1063 if ( !process )
1064 {
1065 break;
1066 }
1067 }
1068 catch ( PolicyConfigurationException e )
1069 {
1070 log.error( e.getMessage(), e );
1071 }
1072 }
1073
1074 if ( process )
1075 {
1076 // if the exception was queued, don't throw it
1077 if ( !previousExceptions.containsKey( content.getId() ) )
1078 {
1079 throw new ProxyDownloadException(
1080 "An error occurred in downloading from the remote repository, and the policy is to fail immediately",
1081 content.getId(), exception );
1082 }
1083 }
1084 else
1085 {
1086 // if the exception was queued, but cancelled, remove it
1087 previousExceptions.remove( content.getId() );
1088 }
1089
1090 log.warn(
1091 "Transfer error from repository \"" + content.getRepository().getId() + "\" for artifact " + Keys.toKey(
1092 artifact ) + ", continuing to next repository. Error message: " + exception.getMessage() );
1093 log.debug( "Full stack trace", exception );
1094 }
1095
1096 /**
1097 * Creates a working directory
1098 *
1099 * @param repository
1100 * @return file location of working directory
1101 * @throws IOException
1102 */
1103 private File createWorkingDirectory( ManagedRepositoryContent repository )
1104 {
1105 return Files.createTempDir();
1106 }
1107
1108 /**
1109 * Used to move the temporary file to its real destination. This is patterned from the way WagonManager handles its
1110 * downloaded files.
1111 *
1112 * @param temp The completed download file
1113 * @param target The final location of the downloaded file
1114 * @throws ProxyException when the temp file cannot replace the target file
1115 */
1116 private void moveTempToTarget( File temp, File target )
1117 throws ProxyException
1118 {
1119 if ( target.exists() && !target.delete() )
1120 {
1121 throw new ProxyException( "Unable to overwrite existing target file: " + target.getAbsolutePath() );
1122 }
1123
1124 target.getParentFile().mkdirs();
1125 if ( !temp.renameTo( target ) )
1126 {
1127 log.warn( "Unable to rename tmp file to its final name... resorting to copy command." );
1128
1129 try
1130 {
1131 FileUtils.copyFile( temp, target );
1132 }
1133 catch ( IOException e )
1134 {
1135 if ( target.exists() )
1136 {
1137 log.debug( "Tried to copy file {} to {} but file with this name already exists.", temp.getName(),
1138 target.getAbsolutePath() );
1139 }
1140 else
1141 {
1142 throw new ProxyException(
1143 "Cannot copy tmp file " + temp.getAbsolutePath() + " to its final location", e );
1144 }
1145 }
1146 finally
1147 {
1148 FileUtils.deleteQuietly( temp );
1149 }
1150 }
1151 }
1152
1153 /**
1154 * Using wagon, connect to the remote repository.
1155 *
1156 * @param connector the connector configuration to utilize (for obtaining network proxy configuration from)
1157 * @param wagon the wagon instance to establish the connection on.
1158 * @param remoteRepository the remote repository to connect to.
1159 * @return true if the connection was successful. false if not connected.
1160 */
1161 private boolean connectToRepository( ProxyConnector connector, Wagon wagon,
1162 RemoteRepositoryContent remoteRepository )
1163 {
1164 boolean connected = false;
1165
1166 final ProxyInfo networkProxy =
1167 connector.getProxyId() == null ? null : this.networkProxyMap.get( connector.getProxyId() );
1168
1169 if ( log.isDebugEnabled() )
1170 {
1171 if ( networkProxy != null )
1172 {
1173 // TODO: move to proxyInfo.toString()
1174 String msg = "Using network proxy " + networkProxy.getHost() + ":" + networkProxy.getPort()
1175 + " to connect to remote repository " + remoteRepository.getURL();
1176 if ( networkProxy.getNonProxyHosts() != null )
1177 {
1178 msg += "; excluding hosts: " + networkProxy.getNonProxyHosts();
1179 }
1180 if ( StringUtils.isNotBlank( networkProxy.getUserName() ) )
1181 {
1182 msg += "; as user: " + networkProxy.getUserName();
1183 }
1184 log.debug( msg );
1185 }
1186 }
1187
1188 AuthenticationInfo authInfo = null;
1189 String username = remoteRepository.getRepository().getUserName();
1190 String password = remoteRepository.getRepository().getPassword();
1191
1192 if ( StringUtils.isNotBlank( username ) && StringUtils.isNotBlank( password ) )
1193 {
1194 log.debug( "Using username {} to connect to remote repository {}", username, remoteRepository.getURL() );
1195 authInfo = new AuthenticationInfo();
1196 authInfo.setUserName( username );
1197 authInfo.setPassword( password );
1198 }
1199
1200 // Convert seconds to milliseconds
1201 int timeoutInMilliseconds = remoteRepository.getRepository().getTimeout() * 1000;
1202
1203 // Set timeout read and connect
1204 // FIXME olamy having 2 config values
1205 wagon.setReadTimeout( timeoutInMilliseconds );
1206 wagon.setTimeout( timeoutInMilliseconds );
1207
1208 try
1209 {
1210 Repository wagonRepository =
1211 new Repository( remoteRepository.getId(), remoteRepository.getURL().toString() );
1212 wagon.connect( wagonRepository, authInfo, networkProxy );
1213 connected = true;
1214 }
1215 catch ( ConnectionException e )
1216 {
1217 log.warn( "Could not connect to " + remoteRepository.getRepository().getName() + ": " + e.getMessage() );
1218 connected = false;
1219 }
1220 catch ( AuthenticationException e )
1221 {
1222 log.warn( "Could not connect to " + remoteRepository.getRepository().getName() + ": " + e.getMessage() );
1223 connected = false;
1224 }
1225
1226 return connected;
1227 }
1228
1229 /**
1230 * Tests whitelist and blacklist patterns against path.
1231 *
1232 * @param path the path to test.
1233 * @param patterns the list of patterns to check.
1234 * @return true if the path matches at least 1 pattern in the provided patterns list.
1235 */
1236 private boolean matchesPattern( String path, List<String> patterns )
1237 {
1238 if ( CollectionUtils.isEmpty( patterns ) )
1239 {
1240 return false;
1241 }
1242
1243 if ( !path.startsWith( "/" ) )
1244 {
1245 path = "/" + path;
1246 }
1247
1248 for ( String pattern : patterns )
1249 {
1250 if ( !pattern.startsWith( "/" ) )
1251 {
1252 pattern = "/" + pattern;
1253 }
1254
1255 if ( SelectorUtils.matchPath( pattern, path, false ) )
1256 {
1257 return true;
1258 }
1259 }
1260
1261 return false;
1262 }
1263
1264 /**
1265 * TODO: Ensure that list is correctly ordered based on configuration. See MRM-477
1266 */
1267 public List<ProxyConnector> getProxyConnectors( ManagedRepositoryContent repository )
1268 {
1269 synchronized ( this.proxyConnectorMap )
1270 {
1271 List<ProxyConnector> ret = this.proxyConnectorMap.get( repository.getId() );
1272 if ( ret == null )
1273 {
1274 return Collections.emptyList();
1275 }
1276
1277 Collections.sort( ret, ProxyConnectorOrderComparator.getInstance() );
1278 return ret;
1279 }
1280 }
1281
1282 public void afterConfigurationChange( Registry registry, String propertyName, Object propertyValue )
1283 {
1284 if ( ConfigurationNames.isNetworkProxy( propertyName ) || ConfigurationNames.isManagedRepositories(
1285 propertyName ) || ConfigurationNames.isRemoteRepositories( propertyName )
1286 || ConfigurationNames.isProxyConnector( propertyName ) )
1287 {
1288 initConnectorsAndNetworkProxies();
1289 }
1290 }
1291
1292 protected String addParameters( String path, RemoteRepository remoteRepository )
1293 {
1294 if ( remoteRepository.getExtraParameters().isEmpty() )
1295 {
1296 return path;
1297 }
1298
1299 boolean question = false;
1300
1301 StringBuilder res = new StringBuilder( path == null ? "" : path );
1302
1303 for ( Entry<String, String> entry : remoteRepository.getExtraParameters().entrySet() )
1304 {
1305 if ( !question )
1306 {
1307 res.append( '?' ).append( entry.getKey() ).append( '=' ).append( entry.getValue() );
1308 }
1309 }
1310
1311 return res.toString();
1312 }
1313
1314
1315 public void beforeConfigurationChange( Registry registry, String propertyName, Object propertyValue )
1316 {
1317 /* do nothing */
1318 }
1319
1320 public ArchivaConfiguration getArchivaConfiguration()
1321 {
1322 return archivaConfiguration;
1323 }
1324
1325 public void setArchivaConfiguration( ArchivaConfiguration archivaConfiguration )
1326 {
1327 this.archivaConfiguration = archivaConfiguration;
1328 }
1329
1330 public RepositoryContentFactory getRepositoryFactory()
1331 {
1332 return repositoryFactory;
1333 }
1334
1335 public void setRepositoryFactory( RepositoryContentFactory repositoryFactory )
1336 {
1337 this.repositoryFactory = repositoryFactory;
1338 }
1339
1340 public MetadataTools getMetadataTools()
1341 {
1342 return metadataTools;
1343 }
1344
1345 public void setMetadataTools( MetadataTools metadataTools )
1346 {
1347 this.metadataTools = metadataTools;
1348 }
1349
1350 public UrlFailureCache getUrlFailureCache()
1351 {
1352 return urlFailureCache;
1353 }
1354
1355 public void setUrlFailureCache( UrlFailureCache urlFailureCache )
1356 {
1357 this.urlFailureCache = urlFailureCache;
1358 }
1359
1360 public WagonFactory getWagonFactory()
1361 {
1362 return wagonFactory;
1363 }
1364
1365 public void setWagonFactory( WagonFactory wagonFactory )
1366 {
1367 this.wagonFactory = wagonFactory;
1368 }
1369
1370 public Map<String, PreDownloadPolicy> getPreDownloadPolicies()
1371 {
1372 return preDownloadPolicies;
1373 }
1374
1375 public void setPreDownloadPolicies( Map<String, PreDownloadPolicy> preDownloadPolicies )
1376 {
1377 this.preDownloadPolicies = preDownloadPolicies;
1378 }
1379
1380 public Map<String, PostDownloadPolicy> getPostDownloadPolicies()
1381 {
1382 return postDownloadPolicies;
1383 }
1384
1385 public void setPostDownloadPolicies( Map<String, PostDownloadPolicy> postDownloadPolicies )
1386 {
1387 this.postDownloadPolicies = postDownloadPolicies;
1388 }
1389
1390 public Map<String, DownloadErrorPolicy> getDownloadErrorPolicies()
1391 {
1392 return downloadErrorPolicies;
1393 }
1394
1395 public void setDownloadErrorPolicies( Map<String, DownloadErrorPolicy> downloadErrorPolicies )
1396 {
1397 this.downloadErrorPolicies = downloadErrorPolicies;
1398 }
1399 }