001 package org.apache.archiva.scheduler.indexing; 002 /* 003 * Licensed to the Apache Software Foundation (ASF) under one 004 * or more contributor license agreements. See the NOTICE file 005 * distributed with this work for additional information 006 * regarding copyright ownership. The ASF licenses this file 007 * to you under the Apache License, Version 2.0 (the 008 * "License"); you may not use this file except in compliance 009 * with the License. You may obtain a copy of the License at 010 * 011 * http://www.apache.org/licenses/LICENSE-2.0 012 * 013 * Unless required by applicable law or agreed to in writing, 014 * software distributed under the License is distributed on an 015 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 016 * KIND, either express or implied. See the License for the 017 * specific language governing permissions and limitations 018 * under the License. 019 */ 020 021 import org.apache.archiva.admin.model.RepositoryAdminException; 022 import org.apache.archiva.admin.model.beans.NetworkProxy; 023 import org.apache.archiva.admin.model.beans.RemoteRepository; 024 import org.apache.archiva.admin.model.remote.RemoteRepositoryAdmin; 025 import org.apache.archiva.proxy.common.WagonFactory; 026 import org.apache.archiva.proxy.common.WagonFactoryException; 027 import org.apache.archiva.proxy.common.WagonFactoryRequest; 028 import org.apache.commons.io.FileUtils; 029 import org.apache.commons.lang.time.StopWatch; 030 import org.apache.maven.index.context.IndexingContext; 031 import org.apache.maven.index.updater.IndexUpdateRequest; 032 import org.apache.maven.index.updater.IndexUpdater; 033 import org.apache.maven.index.updater.ResourceFetcher; 034 import org.apache.maven.wagon.ConnectionException; 035 import org.apache.maven.wagon.ResourceDoesNotExistException; 036 import org.apache.maven.wagon.StreamWagon; 037 import org.apache.maven.wagon.TransferFailedException; 038 import org.apache.maven.wagon.Wagon; 039 import org.apache.maven.wagon.authentication.AuthenticationException; 040 import org.apache.maven.wagon.authentication.AuthenticationInfo; 041 import org.apache.maven.wagon.authorization.AuthorizationException; 042 import org.apache.maven.wagon.events.TransferEvent; 043 import org.apache.maven.wagon.events.TransferListener; 044 import org.apache.maven.wagon.proxy.ProxyInfo; 045 import org.apache.maven.wagon.repository.Repository; 046 import org.apache.maven.wagon.shared.http4.AbstractHttpClientWagon; 047 import org.apache.maven.wagon.shared.http4.HttpConfiguration; 048 import org.apache.maven.wagon.shared.http4.HttpMethodConfiguration; 049 import org.slf4j.Logger; 050 import org.slf4j.LoggerFactory; 051 052 import java.io.File; 053 import java.io.FileInputStream; 054 import java.io.FileNotFoundException; 055 import java.io.IOException; 056 import java.io.InputStream; 057 import java.net.MalformedURLException; 058 import java.net.URL; 059 import java.util.List; 060 import java.util.Map; 061 062 /** 063 * @author Olivier Lamy 064 * @since 1.4-M1 065 */ 066 public class DownloadRemoteIndexTask 067 implements Runnable 068 { 069 private Logger log = LoggerFactory.getLogger( getClass() ); 070 071 private RemoteRepository remoteRepository; 072 073 private RemoteRepositoryAdmin remoteRepositoryAdmin; 074 075 private WagonFactory wagonFactory; 076 077 private NetworkProxy networkProxy; 078 079 private boolean fullDownload; 080 081 private List<String> runningRemoteDownloadIds; 082 083 private IndexUpdater indexUpdater; 084 085 086 public DownloadRemoteIndexTask( DownloadRemoteIndexTaskRequest downloadRemoteIndexTaskRequest, 087 List<String> runningRemoteDownloadIds ) 088 { 089 this.remoteRepository = downloadRemoteIndexTaskRequest.getRemoteRepository(); 090 this.wagonFactory = downloadRemoteIndexTaskRequest.getWagonFactory(); 091 this.networkProxy = downloadRemoteIndexTaskRequest.getNetworkProxy(); 092 this.fullDownload = downloadRemoteIndexTaskRequest.isFullDownload(); 093 this.runningRemoteDownloadIds = runningRemoteDownloadIds; 094 this.indexUpdater = downloadRemoteIndexTaskRequest.getIndexUpdater(); 095 this.remoteRepositoryAdmin = downloadRemoteIndexTaskRequest.getRemoteRepositoryAdmin(); 096 } 097 098 public void run() 099 { 100 101 // so short lock : not sure we need it 102 synchronized ( this.runningRemoteDownloadIds ) 103 { 104 if ( this.runningRemoteDownloadIds.contains( this.remoteRepository.getId() ) ) 105 { 106 // skip it as it's running 107 log.info( "skip download index remote for repo {} it's already running", 108 this.remoteRepository.getId() ); 109 return; 110 } 111 this.runningRemoteDownloadIds.add( this.remoteRepository.getId() ); 112 } 113 File tempIndexDirectory = null; 114 StopWatch stopWatch = new StopWatch(); 115 stopWatch.start(); 116 try 117 { 118 log.info( "start download remote index for remote repository {}", this.remoteRepository.getId() ); 119 IndexingContext indexingContext = remoteRepositoryAdmin.createIndexContext( this.remoteRepository ); 120 121 // create a temp directory to download files 122 tempIndexDirectory = new File( indexingContext.getIndexDirectoryFile().getParent(), ".tmpIndex" ); 123 File indexCacheDirectory = new File( indexingContext.getIndexDirectoryFile().getParent(), ".indexCache" ); 124 indexCacheDirectory.mkdirs(); 125 if ( tempIndexDirectory.exists() ) 126 { 127 FileUtils.deleteDirectory( tempIndexDirectory ); 128 } 129 tempIndexDirectory.mkdirs(); 130 tempIndexDirectory.deleteOnExit(); 131 String baseIndexUrl = indexingContext.getIndexUpdateUrl(); 132 133 String wagonProtocol = new URL( this.remoteRepository.getUrl() ).getProtocol(); 134 135 final StreamWagon wagon = (StreamWagon) wagonFactory.getWagon( 136 new WagonFactoryRequest( wagonProtocol, this.remoteRepository.getExtraHeaders() ).networkProxy( 137 this.networkProxy ) ); 138 int timeoutInMilliseconds = remoteRepository.getTimeout() * 1000; 139 // FIXME olamy having 2 config values 140 wagon.setReadTimeout( timeoutInMilliseconds ); 141 wagon.setTimeout( timeoutInMilliseconds ); 142 143 if ( wagon instanceof AbstractHttpClientWagon ) 144 { 145 HttpConfiguration httpConfiguration = new HttpConfiguration(); 146 HttpMethodConfiguration httpMethodConfiguration = new HttpMethodConfiguration(); 147 httpMethodConfiguration.setUsePreemptive( true ); 148 httpMethodConfiguration.setReadTimeout( timeoutInMilliseconds ); 149 httpConfiguration.setGet( httpMethodConfiguration ); 150 ( (AbstractHttpClientWagon) wagon ).setHttpConfiguration( httpConfiguration ); 151 } 152 153 wagon.addTransferListener( new DownloadListener() ); 154 ProxyInfo proxyInfo = null; 155 if ( this.networkProxy != null ) 156 { 157 proxyInfo = new ProxyInfo(); 158 proxyInfo.setHost( this.networkProxy.getHost() ); 159 proxyInfo.setPort( this.networkProxy.getPort() ); 160 proxyInfo.setUserName( this.networkProxy.getUsername() ); 161 proxyInfo.setPassword( this.networkProxy.getPassword() ); 162 } 163 AuthenticationInfo authenticationInfo = null; 164 if ( this.remoteRepository.getUserName() != null ) 165 { 166 authenticationInfo = new AuthenticationInfo(); 167 authenticationInfo.setUserName( this.remoteRepository.getUserName() ); 168 authenticationInfo.setPassword( this.remoteRepository.getPassword() ); 169 } 170 wagon.connect( new Repository( this.remoteRepository.getId(), baseIndexUrl ), authenticationInfo, 171 proxyInfo ); 172 173 File indexDirectory = indexingContext.getIndexDirectoryFile(); 174 if ( !indexDirectory.exists() ) 175 { 176 indexDirectory.mkdirs(); 177 } 178 179 ResourceFetcher resourceFetcher = 180 new WagonResourceFetcher( log, tempIndexDirectory, wagon, remoteRepository ); 181 IndexUpdateRequest request = new IndexUpdateRequest( indexingContext, resourceFetcher ); 182 request.setForceFullUpdate( this.fullDownload ); 183 request.setLocalIndexCacheDir( indexCacheDirectory ); 184 185 this.indexUpdater.fetchAndUpdateIndex( request ); 186 stopWatch.stop(); 187 log.info( "time update index from remote for repository {}: {} s", this.remoteRepository.getId(), 188 ( stopWatch.getTime() / 1000 ) ); 189 190 // index packing optionnal ?? 191 //IndexPackingRequest indexPackingRequest = 192 // new IndexPackingRequest( indexingContext, indexingContext.getIndexDirectoryFile() ); 193 //indexPacker.packIndex( indexPackingRequest ); 194 indexingContext.updateTimestamp( true ); 195 196 } 197 catch ( MalformedURLException e ) 198 { 199 log.error( e.getMessage(), e ); 200 throw new RuntimeException( e.getMessage(), e ); 201 } 202 catch ( WagonFactoryException e ) 203 { 204 log.error( e.getMessage(), e ); 205 throw new RuntimeException( e.getMessage(), e ); 206 } 207 catch ( ConnectionException e ) 208 { 209 log.error( e.getMessage(), e ); 210 throw new RuntimeException( e.getMessage(), e ); 211 } 212 catch ( AuthenticationException e ) 213 { 214 log.error( e.getMessage(), e ); 215 throw new RuntimeException( e.getMessage(), e ); 216 } 217 catch ( IOException e ) 218 { 219 log.error( e.getMessage(), e ); 220 throw new RuntimeException( e.getMessage(), e ); 221 } 222 catch ( RepositoryAdminException e ) 223 { 224 log.error( e.getMessage(), e ); 225 throw new RuntimeException( e.getMessage(), e ); 226 } 227 finally 228 { 229 deleteDirectoryQuiet( tempIndexDirectory ); 230 this.runningRemoteDownloadIds.remove( this.remoteRepository.getId() ); 231 } 232 log.info( "end download remote index for remote repository " + this.remoteRepository.getId() ); 233 } 234 235 private void deleteDirectoryQuiet( File f ) 236 { 237 try 238 { 239 FileUtils.deleteDirectory( f ); 240 } 241 catch ( IOException e ) 242 { 243 log.warn( "skip error delete {} : {}", f, e.getMessage() ); 244 } 245 } 246 247 248 private static final class DownloadListener 249 implements TransferListener 250 { 251 private Logger log = LoggerFactory.getLogger( getClass() ); 252 253 private String resourceName; 254 255 private long startTime; 256 257 private int totalLength = 0; 258 259 public void transferInitiated( TransferEvent transferEvent ) 260 { 261 startTime = System.currentTimeMillis(); 262 resourceName = transferEvent.getResource().getName(); 263 log.debug( "initiate transfer of {}", resourceName ); 264 } 265 266 public void transferStarted( TransferEvent transferEvent ) 267 { 268 this.totalLength = 0; 269 resourceName = transferEvent.getResource().getName(); 270 log.info( "start transfer of {}", transferEvent.getResource().getName() ); 271 } 272 273 public void transferProgress( TransferEvent transferEvent, byte[] buffer, int length ) 274 { 275 log.debug( "transfer of {} : {}/{}", transferEvent.getResource().getName(), buffer.length, length ); 276 this.totalLength += length; 277 } 278 279 public void transferCompleted( TransferEvent transferEvent ) 280 { 281 resourceName = transferEvent.getResource().getName(); 282 long endTime = System.currentTimeMillis(); 283 log.info( "end of transfer file {} {} kb: {}s", transferEvent.getResource().getName(), 284 this.totalLength / 1024, ( endTime - startTime ) / 1000 ); 285 } 286 287 public void transferError( TransferEvent transferEvent ) 288 { 289 log.info( "error of transfer file {}: {}", transferEvent.getResource().getName(), 290 transferEvent.getException().getMessage(), transferEvent.getException() ); 291 } 292 293 public void debug( String message ) 294 { 295 log.debug( "transfer debug {}", message ); 296 } 297 } 298 299 private static class WagonResourceFetcher 300 implements ResourceFetcher 301 { 302 303 Logger log; 304 305 File tempIndexDirectory; 306 307 Wagon wagon; 308 309 RemoteRepository remoteRepository; 310 311 private WagonResourceFetcher( Logger log, File tempIndexDirectory, Wagon wagon, 312 RemoteRepository remoteRepository ) 313 { 314 this.log = log; 315 this.tempIndexDirectory = tempIndexDirectory; 316 this.wagon = wagon; 317 this.remoteRepository = remoteRepository; 318 } 319 320 public void connect( String id, String url ) 321 throws IOException 322 { 323 //no op 324 } 325 326 public void disconnect() 327 throws IOException 328 { 329 // no op 330 } 331 332 public InputStream retrieve( String name ) 333 throws IOException, FileNotFoundException 334 { 335 try 336 { 337 log.info( "index update retrieve file, name:{}", name ); 338 File file = new File( tempIndexDirectory, name ); 339 if ( file.exists() ) 340 { 341 file.delete(); 342 } 343 file.deleteOnExit(); 344 wagon.get( addParameters( name, this.remoteRepository ), file ); 345 return new FileInputStream( file ); 346 } 347 catch ( AuthorizationException e ) 348 { 349 throw new IOException( e.getMessage(), e ); 350 } 351 catch ( TransferFailedException e ) 352 { 353 throw new IOException( e.getMessage(), e ); 354 } 355 catch ( ResourceDoesNotExistException e ) 356 { 357 FileNotFoundException fnfe = new FileNotFoundException( e.getMessage() ); 358 fnfe.initCause( e ); 359 throw fnfe; 360 } 361 } 362 363 // FIXME remove crappy copy/paste 364 protected String addParameters( String path, RemoteRepository remoteRepository ) 365 { 366 if ( remoteRepository.getExtraParameters().isEmpty() ) 367 { 368 return path; 369 } 370 371 boolean question = false; 372 373 StringBuilder res = new StringBuilder( path == null ? "" : path ); 374 375 for ( Map.Entry<String, String> entry : remoteRepository.getExtraParameters().entrySet() ) 376 { 377 if ( !question ) 378 { 379 res.append( '?' ).append( entry.getKey() ).append( '=' ).append( entry.getValue() ); 380 } 381 } 382 383 return res.toString(); 384 } 385 386 } 387 388 389 } 390 391