Source code

001package org.apache.archiva.scheduler.indexing;
002/*
003 * Licensed to the Apache Software Foundation (ASF) under one
004 * or more contributor license agreements.  See the NOTICE file
005 * distributed with this work for additional information
006 * regarding copyright ownership.  The ASF licenses this file
007 * to you under the Apache License, Version 2.0 (the
008 * "License"); you may not use this file except in compliance
009 * with the License.  You may obtain a copy of the License at
010 *
011 *   http://www.apache.org/licenses/LICENSE-2.0
012 *
013 * Unless required by applicable law or agreed to in writing,
014 * software distributed under the License is distributed on an
015 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
016 * KIND, either express or implied.  See the License for the
017 * specific language governing permissions and limitations
018 * under the License.
019 */
020
021import org.apache.archiva.admin.model.RepositoryAdminException;
022import org.apache.archiva.admin.model.beans.NetworkProxy;
023import org.apache.archiva.admin.model.beans.RemoteRepository;
024import org.apache.archiva.admin.model.remote.RemoteRepositoryAdmin;
025import org.apache.archiva.proxy.common.WagonFactory;
026import org.apache.archiva.proxy.common.WagonFactoryException;
027import org.apache.archiva.proxy.common.WagonFactoryRequest;
028import org.apache.commons.io.FileUtils;
029import org.apache.commons.lang.time.StopWatch;
030import org.apache.maven.index.context.IndexingContext;
031import org.apache.maven.index.updater.IndexUpdateRequest;
032import org.apache.maven.index.updater.IndexUpdater;
033import org.apache.maven.index.updater.ResourceFetcher;
034import org.apache.maven.wagon.ConnectionException;
035import org.apache.maven.wagon.ResourceDoesNotExistException;
036import org.apache.maven.wagon.StreamWagon;
037import org.apache.maven.wagon.TransferFailedException;
038import org.apache.maven.wagon.Wagon;
039import org.apache.maven.wagon.authentication.AuthenticationException;
040import org.apache.maven.wagon.authentication.AuthenticationInfo;
041import org.apache.maven.wagon.authorization.AuthorizationException;
042import org.apache.maven.wagon.events.TransferEvent;
043import org.apache.maven.wagon.events.TransferListener;
044import org.apache.maven.wagon.providers.http.AbstractHttpClientWagon;
045import org.apache.maven.wagon.providers.http.HttpConfiguration;
046import org.apache.maven.wagon.providers.http.HttpMethodConfiguration;
047import org.apache.maven.wagon.proxy.ProxyInfo;
048import org.apache.maven.wagon.repository.Repository;
049import org.slf4j.Logger;
050import org.slf4j.LoggerFactory;
051
052import java.io.File;
053import java.io.FileNotFoundException;
054import java.io.IOException;
055import java.io.InputStream;
056import java.net.MalformedURLException;
057import java.net.URL;
058import java.nio.file.Files;
059import java.util.List;
060import java.util.Map;
061
062/**
063 * @author Olivier Lamy
064 * @since 1.4-M1
065 */
066public class DownloadRemoteIndexTask
067    implements Runnable
068{
069    private Logger log = LoggerFactory.getLogger( getClass() );
070
071    private RemoteRepository remoteRepository;
072
073    private RemoteRepositoryAdmin remoteRepositoryAdmin;
074
075    private WagonFactory wagonFactory;
076
077    private NetworkProxy networkProxy;
078
079    private boolean fullDownload;
080
081    private List<String> runningRemoteDownloadIds;
082
083    private IndexUpdater indexUpdater;
084
085
086    public DownloadRemoteIndexTask( DownloadRemoteIndexTaskRequest downloadRemoteIndexTaskRequest,
087                                    List<String> runningRemoteDownloadIds )
088    {
089        this.remoteRepository = downloadRemoteIndexTaskRequest.getRemoteRepository();
090        this.wagonFactory = downloadRemoteIndexTaskRequest.getWagonFactory();
091        this.networkProxy = downloadRemoteIndexTaskRequest.getNetworkProxy();
092        this.fullDownload = downloadRemoteIndexTaskRequest.isFullDownload();
093        this.runningRemoteDownloadIds = runningRemoteDownloadIds;
094        this.indexUpdater = downloadRemoteIndexTaskRequest.getIndexUpdater();
095        this.remoteRepositoryAdmin = downloadRemoteIndexTaskRequest.getRemoteRepositoryAdmin();
096    }
097
098    @Override
099    public void run()
100    {
101
102        // so short lock : not sure we need it
103        synchronized ( this.runningRemoteDownloadIds )
104        {
105            if ( this.runningRemoteDownloadIds.contains( this.remoteRepository.getId() ) )
106            {
107                // skip it as it's running
108                log.info( "skip download index remote for repo {} it's already running",
109                          this.remoteRepository.getId() );
110                return;
111            }
112            this.runningRemoteDownloadIds.add( this.remoteRepository.getId() );
113        }
114        File tempIndexDirectory = null;
115        StopWatch stopWatch = new StopWatch();
116        stopWatch.start();
117        try
118        {
119            log.info( "start download remote index for remote repository {}", this.remoteRepository.getId() );
120            IndexingContext indexingContext = remoteRepositoryAdmin.createIndexContext( this.remoteRepository );
121
122            // create a temp directory to download files
123            tempIndexDirectory = new File( indexingContext.getIndexDirectoryFile().getParent(), ".tmpIndex" );
124            File indexCacheDirectory = new File( indexingContext.getIndexDirectoryFile().getParent(), ".indexCache" );
125            indexCacheDirectory.mkdirs();
126            if ( tempIndexDirectory.exists() )
127            {
128                FileUtils.deleteDirectory( tempIndexDirectory );
129            }
130            tempIndexDirectory.mkdirs();
131            tempIndexDirectory.deleteOnExit();
132            String baseIndexUrl = indexingContext.getIndexUpdateUrl();
133
134            String wagonProtocol = new URL( this.remoteRepository.getUrl() ).getProtocol();
135
136            final StreamWagon wagon = (StreamWagon) wagonFactory.getWagon(
137                new WagonFactoryRequest( wagonProtocol, this.remoteRepository.getExtraHeaders() ).networkProxy(
138                    this.networkProxy )
139            );
140            // FIXME olamy having 2 config values
141            wagon.setReadTimeout( remoteRepository.getRemoteDownloadTimeout() * 1000 );
142            wagon.setTimeout( remoteRepository.getTimeout() * 1000 );
143
144            if ( wagon instanceof AbstractHttpClientWagon )
145            {
146                HttpConfiguration httpConfiguration = new HttpConfiguration();
147                HttpMethodConfiguration httpMethodConfiguration = new HttpMethodConfiguration();
148                httpMethodConfiguration.setUsePreemptive( true );
149                httpMethodConfiguration.setReadTimeout( remoteRepository.getRemoteDownloadTimeout() * 1000 );
150                httpConfiguration.setGet( httpMethodConfiguration );
151                AbstractHttpClientWagon.class.cast( wagon ).setHttpConfiguration( httpConfiguration );
152            }
153
154            wagon.addTransferListener( new DownloadListener() );
155            ProxyInfo proxyInfo = null;
156            if ( this.networkProxy != null )
157            {
158                proxyInfo = new ProxyInfo();
159                proxyInfo.setType( this.networkProxy.getProtocol() );
160                proxyInfo.setHost( this.networkProxy.getHost() );
161                proxyInfo.setPort( this.networkProxy.getPort() );
162                proxyInfo.setUserName( this.networkProxy.getUsername() );
163                proxyInfo.setPassword( this.networkProxy.getPassword() );
164            }
165            AuthenticationInfo authenticationInfo = null;
166            if ( this.remoteRepository.getUserName() != null )
167            {
168                authenticationInfo = new AuthenticationInfo();
169                authenticationInfo.setUserName( this.remoteRepository.getUserName() );
170                authenticationInfo.setPassword( this.remoteRepository.getPassword() );
171            }
172            wagon.connect( new Repository( this.remoteRepository.getId(), baseIndexUrl ), authenticationInfo,
173                           proxyInfo );
174
175            File indexDirectory = indexingContext.getIndexDirectoryFile();
176            if ( !indexDirectory.exists() )
177            {
178                indexDirectory.mkdirs();
179            }
180
181            ResourceFetcher resourceFetcher =
182                new WagonResourceFetcher( log, tempIndexDirectory, wagon, remoteRepository );
183            IndexUpdateRequest request = new IndexUpdateRequest( indexingContext, resourceFetcher );
184            request.setForceFullUpdate( this.fullDownload );
185            request.setLocalIndexCacheDir( indexCacheDirectory );
186
187            this.indexUpdater.fetchAndUpdateIndex( request );
188            stopWatch.stop();
189            log.info( "time update index from remote for repository {}: {} s", this.remoteRepository.getId(),
190                      ( stopWatch.getTime() / 1000 ) );
191
192            // index packing optionnal ??
193            //IndexPackingRequest indexPackingRequest =
194            //    new IndexPackingRequest( indexingContext, indexingContext.getIndexDirectoryFile() );
195            //indexPacker.packIndex( indexPackingRequest );
196            indexingContext.updateTimestamp( true );
197
198        }
199        catch ( MalformedURLException e )
200        {
201            log.error( e.getMessage(), e );
202            throw new RuntimeException( e.getMessage(), e );
203        }
204        catch ( WagonFactoryException e )
205        {
206            log.error( e.getMessage(), e );
207            throw new RuntimeException( e.getMessage(), e );
208        }
209        catch ( ConnectionException e )
210        {
211            log.error( e.getMessage(), e );
212            throw new RuntimeException( e.getMessage(), e );
213        }
214        catch ( AuthenticationException e )
215        {
216            log.error( e.getMessage(), e );
217            throw new RuntimeException( e.getMessage(), e );
218        }
219        catch ( IOException e )
220        {
221            log.error( e.getMessage(), e );
222            throw new RuntimeException( e.getMessage(), e );
223        }
224        catch ( RepositoryAdminException e )
225        {
226            log.error( e.getMessage(), e );
227            throw new RuntimeException( e.getMessage(), e );
228        }
229        finally
230        {
231            deleteDirectoryQuiet( tempIndexDirectory );
232            this.runningRemoteDownloadIds.remove( this.remoteRepository.getId() );
233        }
234        log.info( "end download remote index for remote repository {}", this.remoteRepository.getId() );
235    }
236
237    private void deleteDirectoryQuiet( File f )
238    {
239        try
240        {
241            FileUtils.deleteDirectory( f );
242        }
243        catch ( IOException e )
244        {
245            log.warn( "skip error delete {} : {}", f, e.getMessage() );
246        }
247    }
248
249
250    private static final class DownloadListener
251        implements TransferListener
252    {
253        private Logger log = LoggerFactory.getLogger( getClass() );
254
255        private String resourceName;
256
257        private long startTime;
258
259        private int totalLength = 0;
260
261        @Override
262        public void transferInitiated( TransferEvent transferEvent )
263        {
264            startTime = System.currentTimeMillis();
265            resourceName = transferEvent.getResource().getName();
266            log.debug( "initiate transfer of {}", resourceName );
267        }
268
269        @Override
270        public void transferStarted( TransferEvent transferEvent )
271        {
272            this.totalLength = 0;
273            resourceName = transferEvent.getResource().getName();
274            log.info( "start transfer of {}", transferEvent.getResource().getName() );
275        }
276
277        @Override
278        public void transferProgress( TransferEvent transferEvent, byte[] buffer, int length )
279        {
280            log.debug( "transfer of {} : {}/{}", transferEvent.getResource().getName(), buffer.length, length );
281            this.totalLength += length;
282        }
283
284        @Override
285        public void transferCompleted( TransferEvent transferEvent )
286        {
287            resourceName = transferEvent.getResource().getName();
288            long endTime = System.currentTimeMillis();
289            log.info( "end of transfer file {} {} kb: {}s", transferEvent.getResource().getName(),
290                      this.totalLength / 1024, ( endTime - startTime ) / 1000 );
291        }
292
293        @Override
294        public void transferError( TransferEvent transferEvent )
295        {
296            log.info( "error of transfer file {}: {}", transferEvent.getResource().getName(),
297                      transferEvent.getException().getMessage(), transferEvent.getException() );
298        }
299
300        @Override
301        public void debug( String message )
302        {
303            log.debug( "transfer debug {}", message );
304        }
305    }
306
307    private static class WagonResourceFetcher
308        implements ResourceFetcher
309    {
310
311        Logger log;
312
313        File tempIndexDirectory;
314
315        Wagon wagon;
316
317        RemoteRepository remoteRepository;
318
319        private WagonResourceFetcher( Logger log, File tempIndexDirectory, Wagon wagon,
320                                      RemoteRepository remoteRepository )
321        {
322            this.log = log;
323            this.tempIndexDirectory = tempIndexDirectory;
324            this.wagon = wagon;
325            this.remoteRepository = remoteRepository;
326        }
327
328        @Override
329        public void connect( String id, String url )
330            throws IOException
331        {
332            //no op  
333        }
334
335        @Override
336        public void disconnect()
337            throws IOException
338        {
339            // no op
340        }
341
342        @Override
343        public InputStream retrieve( String name )
344            throws IOException, FileNotFoundException
345        {
346            try
347            {
348                log.info( "index update retrieve file, name:{}", name );
349                File file = new File( tempIndexDirectory, name );
350                Files.deleteIfExists( file.toPath() );
351                file.deleteOnExit();
352                wagon.get( addParameters( name, this.remoteRepository ), file );
353                return Files.newInputStream( file.toPath() );
354            }
355            catch ( AuthorizationException | TransferFailedException e )
356            {
357                throw new IOException( e.getMessage(), e );
358            }
359            catch ( ResourceDoesNotExistException e )
360            {
361                FileNotFoundException fnfe = new FileNotFoundException( e.getMessage() );
362                fnfe.initCause( e );
363                throw fnfe;
364            }
365        }
366
367        // FIXME remove crappy copy/paste
368        protected String addParameters( String path, RemoteRepository remoteRepository )
369        {
370            if ( remoteRepository.getExtraParameters().isEmpty() )
371            {
372                return path;
373            }
374
375            boolean question = false;
376
377            StringBuilder res = new StringBuilder( path == null ? "" : path );
378
379            for ( Map.Entry<String, String> entry : remoteRepository.getExtraParameters().entrySet() )
380            {
381                if ( !question )
382                {
383                    res.append( '?' ).append( entry.getKey() ).append( '=' ).append( entry.getValue() );
384                }
385            }
386
387            return res.toString();
388        }
389
390    }
391
392
393}
394