001 package org.apache.archiva.scheduler.indexing;
002 /*
003 * Licensed to the Apache Software Foundation (ASF) under one
004 * or more contributor license agreements. See the NOTICE file
005 * distributed with this work for additional information
006 * regarding copyright ownership. The ASF licenses this file
007 * to you under the Apache License, Version 2.0 (the
008 * "License"); you may not use this file except in compliance
009 * with the License. You may obtain a copy of the License at
010 *
011 * http://www.apache.org/licenses/LICENSE-2.0
012 *
013 * Unless required by applicable law or agreed to in writing,
014 * software distributed under the License is distributed on an
015 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
016 * KIND, either express or implied. See the License for the
017 * specific language governing permissions and limitations
018 * under the License.
019 */
020
021 import org.apache.archiva.admin.model.RepositoryAdminException;
022 import org.apache.archiva.admin.model.beans.NetworkProxy;
023 import org.apache.archiva.admin.model.beans.RemoteRepository;
024 import org.apache.archiva.admin.model.remote.RemoteRepositoryAdmin;
025 import org.apache.archiva.proxy.common.WagonFactory;
026 import org.apache.archiva.proxy.common.WagonFactoryException;
027 import org.apache.archiva.proxy.common.WagonFactoryRequest;
028 import org.apache.commons.io.FileUtils;
029 import org.apache.commons.lang.time.StopWatch;
030 import org.apache.maven.index.context.IndexingContext;
031 import org.apache.maven.index.updater.IndexUpdateRequest;
032 import org.apache.maven.index.updater.IndexUpdater;
033 import org.apache.maven.index.updater.ResourceFetcher;
034 import org.apache.maven.wagon.ConnectionException;
035 import org.apache.maven.wagon.ResourceDoesNotExistException;
036 import org.apache.maven.wagon.StreamWagon;
037 import org.apache.maven.wagon.TransferFailedException;
038 import org.apache.maven.wagon.Wagon;
039 import org.apache.maven.wagon.authentication.AuthenticationException;
040 import org.apache.maven.wagon.authentication.AuthenticationInfo;
041 import org.apache.maven.wagon.authorization.AuthorizationException;
042 import org.apache.maven.wagon.events.TransferEvent;
043 import org.apache.maven.wagon.events.TransferListener;
044 import org.apache.maven.wagon.proxy.ProxyInfo;
045 import org.apache.maven.wagon.repository.Repository;
046 import org.apache.maven.wagon.shared.http4.AbstractHttpClientWagon;
047 import org.apache.maven.wagon.shared.http4.HttpConfiguration;
048 import org.apache.maven.wagon.shared.http4.HttpMethodConfiguration;
049 import org.slf4j.Logger;
050 import org.slf4j.LoggerFactory;
051
052 import java.io.File;
053 import java.io.FileInputStream;
054 import java.io.FileNotFoundException;
055 import java.io.IOException;
056 import java.io.InputStream;
057 import java.net.MalformedURLException;
058 import java.net.URL;
059 import java.util.List;
060 import java.util.Map;
061
062 /**
063 * @author Olivier Lamy
064 * @since 1.4-M1
065 */
066 public class DownloadRemoteIndexTask
067 implements Runnable
068 {
069 private Logger log = LoggerFactory.getLogger( getClass() );
070
071 private RemoteRepository remoteRepository;
072
073 private RemoteRepositoryAdmin remoteRepositoryAdmin;
074
075 private WagonFactory wagonFactory;
076
077 private NetworkProxy networkProxy;
078
079 private boolean fullDownload;
080
081 private List<String> runningRemoteDownloadIds;
082
083 private IndexUpdater indexUpdater;
084
085
086 public DownloadRemoteIndexTask( DownloadRemoteIndexTaskRequest downloadRemoteIndexTaskRequest,
087 List<String> runningRemoteDownloadIds )
088 {
089 this.remoteRepository = downloadRemoteIndexTaskRequest.getRemoteRepository();
090 this.wagonFactory = downloadRemoteIndexTaskRequest.getWagonFactory();
091 this.networkProxy = downloadRemoteIndexTaskRequest.getNetworkProxy();
092 this.fullDownload = downloadRemoteIndexTaskRequest.isFullDownload();
093 this.runningRemoteDownloadIds = runningRemoteDownloadIds;
094 this.indexUpdater = downloadRemoteIndexTaskRequest.getIndexUpdater();
095 this.remoteRepositoryAdmin = downloadRemoteIndexTaskRequest.getRemoteRepositoryAdmin();
096 }
097
098 public void run()
099 {
100
101 // so short lock : not sure we need it
102 synchronized ( this.runningRemoteDownloadIds )
103 {
104 if ( this.runningRemoteDownloadIds.contains( this.remoteRepository.getId() ) )
105 {
106 // skip it as it's running
107 log.info( "skip download index remote for repo {} it's already running",
108 this.remoteRepository.getId() );
109 return;
110 }
111 this.runningRemoteDownloadIds.add( this.remoteRepository.getId() );
112 }
113 File tempIndexDirectory = null;
114 StopWatch stopWatch = new StopWatch();
115 stopWatch.start();
116 try
117 {
118 log.info( "start download remote index for remote repository {}", this.remoteRepository.getId() );
119 IndexingContext indexingContext = remoteRepositoryAdmin.createIndexContext( this.remoteRepository );
120
121 // create a temp directory to download files
122 tempIndexDirectory = new File( indexingContext.getIndexDirectoryFile().getParent(), ".tmpIndex" );
123 File indexCacheDirectory = new File( indexingContext.getIndexDirectoryFile().getParent(), ".indexCache" );
124 indexCacheDirectory.mkdirs();
125 if ( tempIndexDirectory.exists() )
126 {
127 FileUtils.deleteDirectory( tempIndexDirectory );
128 }
129 tempIndexDirectory.mkdirs();
130 tempIndexDirectory.deleteOnExit();
131 String baseIndexUrl = indexingContext.getIndexUpdateUrl();
132
133 String wagonProtocol = new URL( this.remoteRepository.getUrl() ).getProtocol();
134
135 final StreamWagon wagon = (StreamWagon) wagonFactory.getWagon(
136 new WagonFactoryRequest( wagonProtocol, this.remoteRepository.getExtraHeaders() ).networkProxy(
137 this.networkProxy ) );
138 int timeoutInMilliseconds = remoteRepository.getTimeout() * 1000;
139 // FIXME olamy having 2 config values
140 wagon.setReadTimeout( timeoutInMilliseconds );
141 wagon.setTimeout( timeoutInMilliseconds );
142
143 if ( wagon instanceof AbstractHttpClientWagon )
144 {
145 HttpConfiguration httpConfiguration = new HttpConfiguration();
146 HttpMethodConfiguration httpMethodConfiguration = new HttpMethodConfiguration();
147 httpMethodConfiguration.setUsePreemptive( true );
148 httpMethodConfiguration.setReadTimeout( timeoutInMilliseconds );
149 httpConfiguration.setGet( httpMethodConfiguration );
150 ( (AbstractHttpClientWagon) wagon ).setHttpConfiguration( httpConfiguration );
151 }
152
153 wagon.addTransferListener( new DownloadListener() );
154 ProxyInfo proxyInfo = null;
155 if ( this.networkProxy != null )
156 {
157 proxyInfo = new ProxyInfo();
158 proxyInfo.setHost( this.networkProxy.getHost() );
159 proxyInfo.setPort( this.networkProxy.getPort() );
160 proxyInfo.setUserName( this.networkProxy.getUsername() );
161 proxyInfo.setPassword( this.networkProxy.getPassword() );
162 }
163 AuthenticationInfo authenticationInfo = null;
164 if ( this.remoteRepository.getUserName() != null )
165 {
166 authenticationInfo = new AuthenticationInfo();
167 authenticationInfo.setUserName( this.remoteRepository.getUserName() );
168 authenticationInfo.setPassword( this.remoteRepository.getPassword() );
169 }
170 wagon.connect( new Repository( this.remoteRepository.getId(), baseIndexUrl ), authenticationInfo,
171 proxyInfo );
172
173 File indexDirectory = indexingContext.getIndexDirectoryFile();
174 if ( !indexDirectory.exists() )
175 {
176 indexDirectory.mkdirs();
177 }
178
179 ResourceFetcher resourceFetcher =
180 new WagonResourceFetcher( log, tempIndexDirectory, wagon, remoteRepository );
181 IndexUpdateRequest request = new IndexUpdateRequest( indexingContext, resourceFetcher );
182 request.setForceFullUpdate( this.fullDownload );
183 request.setLocalIndexCacheDir( indexCacheDirectory );
184
185 this.indexUpdater.fetchAndUpdateIndex( request );
186 stopWatch.stop();
187 log.info( "time update index from remote for repository {}: {} s", this.remoteRepository.getId(),
188 ( stopWatch.getTime() / 1000 ) );
189
190 // index packing optionnal ??
191 //IndexPackingRequest indexPackingRequest =
192 // new IndexPackingRequest( indexingContext, indexingContext.getIndexDirectoryFile() );
193 //indexPacker.packIndex( indexPackingRequest );
194 indexingContext.updateTimestamp( true );
195
196 }
197 catch ( MalformedURLException e )
198 {
199 log.error( e.getMessage(), e );
200 throw new RuntimeException( e.getMessage(), e );
201 }
202 catch ( WagonFactoryException e )
203 {
204 log.error( e.getMessage(), e );
205 throw new RuntimeException( e.getMessage(), e );
206 }
207 catch ( ConnectionException e )
208 {
209 log.error( e.getMessage(), e );
210 throw new RuntimeException( e.getMessage(), e );
211 }
212 catch ( AuthenticationException e )
213 {
214 log.error( e.getMessage(), e );
215 throw new RuntimeException( e.getMessage(), e );
216 }
217 catch ( IOException e )
218 {
219 log.error( e.getMessage(), e );
220 throw new RuntimeException( e.getMessage(), e );
221 }
222 catch ( RepositoryAdminException e )
223 {
224 log.error( e.getMessage(), e );
225 throw new RuntimeException( e.getMessage(), e );
226 }
227 finally
228 {
229 deleteDirectoryQuiet( tempIndexDirectory );
230 this.runningRemoteDownloadIds.remove( this.remoteRepository.getId() );
231 }
232 log.info( "end download remote index for remote repository " + this.remoteRepository.getId() );
233 }
234
235 private void deleteDirectoryQuiet( File f )
236 {
237 try
238 {
239 FileUtils.deleteDirectory( f );
240 }
241 catch ( IOException e )
242 {
243 log.warn( "skip error delete {} : {}", f, e.getMessage() );
244 }
245 }
246
247
248 private static final class DownloadListener
249 implements TransferListener
250 {
251 private Logger log = LoggerFactory.getLogger( getClass() );
252
253 private String resourceName;
254
255 private long startTime;
256
257 private int totalLength = 0;
258
259 public void transferInitiated( TransferEvent transferEvent )
260 {
261 startTime = System.currentTimeMillis();
262 resourceName = transferEvent.getResource().getName();
263 log.debug( "initiate transfer of {}", resourceName );
264 }
265
266 public void transferStarted( TransferEvent transferEvent )
267 {
268 this.totalLength = 0;
269 resourceName = transferEvent.getResource().getName();
270 log.info( "start transfer of {}", transferEvent.getResource().getName() );
271 }
272
273 public void transferProgress( TransferEvent transferEvent, byte[] buffer, int length )
274 {
275 log.debug( "transfer of {} : {}/{}", transferEvent.getResource().getName(), buffer.length, length );
276 this.totalLength += length;
277 }
278
279 public void transferCompleted( TransferEvent transferEvent )
280 {
281 resourceName = transferEvent.getResource().getName();
282 long endTime = System.currentTimeMillis();
283 log.info( "end of transfer file {} {} kb: {}s", transferEvent.getResource().getName(),
284 this.totalLength / 1024, ( endTime - startTime ) / 1000 );
285 }
286
287 public void transferError( TransferEvent transferEvent )
288 {
289 log.info( "error of transfer file {}: {}", transferEvent.getResource().getName(),
290 transferEvent.getException().getMessage(), transferEvent.getException() );
291 }
292
293 public void debug( String message )
294 {
295 log.debug( "transfer debug {}", message );
296 }
297 }
298
299 private static class WagonResourceFetcher
300 implements ResourceFetcher
301 {
302
303 Logger log;
304
305 File tempIndexDirectory;
306
307 Wagon wagon;
308
309 RemoteRepository remoteRepository;
310
311 private WagonResourceFetcher( Logger log, File tempIndexDirectory, Wagon wagon,
312 RemoteRepository remoteRepository )
313 {
314 this.log = log;
315 this.tempIndexDirectory = tempIndexDirectory;
316 this.wagon = wagon;
317 this.remoteRepository = remoteRepository;
318 }
319
320 public void connect( String id, String url )
321 throws IOException
322 {
323 //no op
324 }
325
326 public void disconnect()
327 throws IOException
328 {
329 // no op
330 }
331
332 public InputStream retrieve( String name )
333 throws IOException, FileNotFoundException
334 {
335 try
336 {
337 log.info( "index update retrieve file, name:{}", name );
338 File file = new File( tempIndexDirectory, name );
339 if ( file.exists() )
340 {
341 file.delete();
342 }
343 file.deleteOnExit();
344 wagon.get( addParameters( name, this.remoteRepository ), file );
345 return new FileInputStream( file );
346 }
347 catch ( AuthorizationException e )
348 {
349 throw new IOException( e.getMessage(), e );
350 }
351 catch ( TransferFailedException e )
352 {
353 throw new IOException( e.getMessage(), e );
354 }
355 catch ( ResourceDoesNotExistException e )
356 {
357 FileNotFoundException fnfe = new FileNotFoundException( e.getMessage() );
358 fnfe.initCause( e );
359 throw fnfe;
360 }
361 }
362
363 // FIXME remove crappy copy/paste
364 protected String addParameters( String path, RemoteRepository remoteRepository )
365 {
366 if ( remoteRepository.getExtraParameters().isEmpty() )
367 {
368 return path;
369 }
370
371 boolean question = false;
372
373 StringBuilder res = new StringBuilder( path == null ? "" : path );
374
375 for ( Map.Entry<String, String> entry : remoteRepository.getExtraParameters().entrySet() )
376 {
377 if ( !question )
378 {
379 res.append( '?' ).append( entry.getKey() ).append( '=' ).append( entry.getValue() );
380 }
381 }
382
383 return res.toString();
384 }
385
386 }
387
388
389 }
390
391