View Javadoc
1   package org.apache.maven.wagon.shared.http;
2   
3   /*
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *   http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing,
15   * software distributed under the License is distributed on an
16   * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
17   * KIND, either express or implied.  See the License for the
18   * specific language governing permissions and limitations
19   * under the License.
20   */
21  
22  import org.apache.http.Header;
23  import org.apache.http.HttpEntity;
24  import org.apache.http.HttpException;
25  import org.apache.http.HttpHost;
26  import org.apache.http.HttpStatus;
27  import org.apache.http.auth.AuthSchemeProvider;
28  import org.apache.http.auth.AuthScope;
29  import org.apache.http.auth.ChallengeState;
30  import org.apache.http.auth.Credentials;
31  import org.apache.http.auth.NTCredentials;
32  import org.apache.http.auth.UsernamePasswordCredentials;
33  import org.apache.http.client.AuthCache;
34  import org.apache.http.client.CredentialsProvider;
35  import org.apache.http.client.HttpRequestRetryHandler;
36  import org.apache.http.client.ServiceUnavailableRetryStrategy;
37  import org.apache.http.client.config.AuthSchemes;
38  import org.apache.http.client.config.CookieSpecs;
39  import org.apache.http.client.config.RequestConfig;
40  import org.apache.http.client.methods.CloseableHttpResponse;
41  import org.apache.http.client.methods.HttpGet;
42  import org.apache.http.client.methods.HttpHead;
43  import org.apache.http.client.methods.HttpPut;
44  import org.apache.http.client.methods.HttpUriRequest;
45  import org.apache.http.client.protocol.HttpClientContext;
46  import org.apache.http.client.utils.DateUtils;
47  import org.apache.http.config.Registry;
48  import org.apache.http.config.RegistryBuilder;
49  import org.apache.http.conn.HttpClientConnectionManager;
50  import org.apache.http.conn.socket.ConnectionSocketFactory;
51  import org.apache.http.conn.socket.PlainConnectionSocketFactory;
52  import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
53  import org.apache.http.conn.ssl.SSLContextBuilder;
54  import org.apache.http.conn.ssl.SSLInitializationException;
55  import org.apache.http.entity.AbstractHttpEntity;
56  import org.apache.http.impl.auth.BasicScheme;
57  import org.apache.http.impl.auth.BasicSchemeFactory;
58  import org.apache.http.impl.auth.DigestSchemeFactory;
59  import org.apache.http.impl.auth.NTLMSchemeFactory;
60  import org.apache.http.impl.client.BasicAuthCache;
61  import org.apache.http.impl.client.BasicCredentialsProvider;
62  import org.apache.http.impl.client.CloseableHttpClient;
63  import org.apache.http.impl.client.DefaultHttpRequestRetryHandler;
64  import org.apache.http.impl.client.DefaultServiceUnavailableRetryStrategy;
65  import org.apache.http.impl.client.HttpClientBuilder;
66  import org.apache.http.impl.client.StandardHttpRequestRetryHandler;
67  import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
68  import org.apache.http.message.BasicHeader;
69  import org.apache.http.protocol.HTTP;
70  import org.apache.http.util.EntityUtils;
71  import org.apache.maven.wagon.InputData;
72  import org.apache.maven.wagon.OutputData;
73  import org.apache.maven.wagon.PathUtils;
74  import org.apache.maven.wagon.ResourceDoesNotExistException;
75  import org.apache.maven.wagon.StreamWagon;
76  import org.apache.maven.wagon.TransferFailedException;
77  import org.apache.maven.wagon.Wagon;
78  import org.apache.maven.wagon.authorization.AuthorizationException;
79  import org.apache.maven.wagon.events.TransferEvent;
80  import org.apache.maven.wagon.proxy.ProxyInfo;
81  import org.apache.maven.wagon.repository.Repository;
82  import org.apache.maven.wagon.resource.Resource;
83  import org.codehaus.plexus.util.StringUtils;
84  
85  import javax.net.ssl.HttpsURLConnection;
86  import javax.net.ssl.SSLContext;
87  import java.io.Closeable;
88  import java.io.File;
89  import java.io.FileInputStream;
90  import java.io.IOException;
91  import java.io.InputStream;
92  import java.io.OutputStream;
93  import java.io.RandomAccessFile;
94  import java.nio.Buffer;
95  import java.nio.ByteBuffer;
96  import java.nio.channels.Channels;
97  import java.nio.channels.ReadableByteChannel;
98  import java.nio.charset.StandardCharsets;
99  import java.text.SimpleDateFormat;
100 import java.util.ArrayList;
101 import java.util.Collection;
102 import java.util.Date;
103 import java.util.List;
104 import java.util.Locale;
105 import java.util.Map;
106 import java.util.Properties;
107 import java.util.TimeZone;
108 import java.util.concurrent.TimeUnit;
109 
110 import static org.apache.maven.wagon.shared.http.HttpMessageUtils.formatAuthorizationMessage;
111 import static org.apache.maven.wagon.shared.http.HttpMessageUtils.formatResourceDoesNotExistMessage;
112 import static org.apache.maven.wagon.shared.http.HttpMessageUtils.formatTransferDebugMessage;
113 import static org.apache.maven.wagon.shared.http.HttpMessageUtils.formatTransferFailedMessage;
114 
115 /**
116  * @author <a href="michal.maczka@dimatics.com">Michal Maczka</a>
117  * @author <a href="mailto:james@atlassian.com">James William Dumay</a>
118  */
119 public abstract class AbstractHttpClientWagon
120     extends StreamWagon
121 {
122     final class WagonHttpEntity
123         extends AbstractHttpEntity
124     {
125         private final Resource resource;
126 
127         private final Wagon wagon;
128 
129         private InputStream stream;
130 
131         private File source;
132 
133         private long length = -1;
134 
135         private boolean repeatable;
136 
137         private WagonHttpEntity( final InputStream stream, final Resource resource, final Wagon wagon,
138                                              final File source )
139             throws TransferFailedException
140         {
141             if ( source != null )
142             {
143                 this.source = source;
144                 this.repeatable = true;
145             }
146             else
147             {
148                 this.stream = stream;
149                 this.repeatable = false;
150             }
151             this.resource = resource;
152             this.length = resource == null ? -1 : resource.getContentLength();
153 
154             this.wagon = wagon;
155         }
156 
157         public Resource getResource()
158         {
159             return resource;
160         }
161 
162         public Wagon getWagon()
163         {
164             return wagon;
165         }
166 
167         public InputStream getContent()
168             throws IOException, IllegalStateException
169         {
170             if ( this.source != null )
171             {
172                 return new FileInputStream( this.source );
173             }
174             return stream;
175         }
176 
177         public File getSource()
178         {
179             return source;
180         }
181 
182         public long getContentLength()
183         {
184             return length;
185         }
186 
187         public boolean isRepeatable()
188         {
189             return repeatable;
190         }
191 
192         public void writeTo( final OutputStream output )
193             throws IOException
194         {
195             if ( output == null )
196             {
197                 throw new NullPointerException( "output cannot be null" );
198             }
199             TransferEvent transferEvent =
200                 new TransferEvent( wagon, resource, TransferEvent.TRANSFER_PROGRESS, TransferEvent.REQUEST_PUT );
201             transferEvent.setTimestamp( System.currentTimeMillis() );
202 
203             try ( ReadableByteChannel input = ( this.source != null )
204                     ? new RandomAccessFile( this.source, "r" ).getChannel()
205                     : Channels.newChannel( stream ) )
206             {
207                 ByteBuffer buffer = ByteBuffer.allocate( getBufferCapacityForTransfer( this.length ) );
208                 int halfBufferCapacity = buffer.capacity() / 2;
209 
210                 long remaining = this.length < 0L ? Long.MAX_VALUE : this.length;
211                 while ( remaining > 0L )
212                 {
213                     int read = input.read( buffer );
214                     if ( read == -1 )
215                     {
216                         // EOF, but some data has not been written yet.
217                         if ( ( (Buffer) buffer ).position() != 0 )
218                         {
219                             ( (Buffer) buffer ).flip();
220                             fireTransferProgress( transferEvent, buffer.array(), ( (Buffer) buffer ).limit() );
221                             output.write( buffer.array(), 0, ( (Buffer) buffer ).limit() );
222                             ( (Buffer) buffer ).clear();
223                         }
224 
225                         break;
226                     }
227 
228                     // Prevent minichunking/fragmentation: when less than half the buffer is utilized,
229                     // read some more bytes before writing and firing progress.
230                     if ( ( (Buffer) buffer ).position() < halfBufferCapacity )
231                     {
232                         continue;
233                     }
234 
235                     ( (Buffer) buffer ).flip();
236                     fireTransferProgress( transferEvent, buffer.array(), ( (Buffer) buffer ).limit() );
237                     output.write( buffer.array(), 0, ( (Buffer) buffer ).limit() );
238                     remaining -= ( (Buffer) buffer ).limit();
239                     ( (Buffer) buffer ).clear();
240 
241                 }
242                 output.flush();
243             }
244         }
245 
246         public boolean isStreaming()
247         {
248             return true;
249         }
250     }
251 
252     private static final TimeZone GMT_TIME_ZONE = TimeZone.getTimeZone( "GMT" );
253 
254     /**
255      * use http(s) connection pool mechanism.
256      * <b>enabled by default</b>
257      */
258     private static boolean persistentPool =
259         Boolean.valueOf( System.getProperty( "maven.wagon.http.pool", "true" ) );
260 
261     /**
262      * skip failure on certificate validity checks.
263      * <b>disabled by default</b>
264      */
265     private static final boolean SSL_INSECURE =
266         Boolean.valueOf( System.getProperty( "maven.wagon.http.ssl.insecure", "false" ) );
267 
268     /**
269      * if using sslInsecure, certificate date issues will be ignored
270      * <b>disabled by default</b>
271      */
272     private static final boolean IGNORE_SSL_VALIDITY_DATES =
273         Boolean.valueOf( System.getProperty( "maven.wagon.http.ssl.ignore.validity.dates", "false" ) );
274 
275     /**
276      * If enabled, ssl hostname verifier does not check hostname. Disable this will use a browser compat hostname
277      * verifier <b>disabled by default</b>
278      */
279     private static final boolean SSL_ALLOW_ALL =
280         Boolean.valueOf( System.getProperty( "maven.wagon.http.ssl.allowall", "false" ) );
281 
282 
283     /**
284      * Maximum concurrent connections per distinct route.
285      * <b>20 by default</b>
286      */
287     private static final int MAX_CONN_PER_ROUTE =
288         Integer.parseInt( System.getProperty( "maven.wagon.httpconnectionManager.maxPerRoute", "20" ) );
289 
290     /**
291      * Maximum concurrent connections in total.
292      * <b>40 by default</b>
293      */
294     private static final int MAX_CONN_TOTAL =
295         Integer.parseInt( System.getProperty( "maven.wagon.httpconnectionManager.maxTotal", "40" ) );
296 
297     /**
298      * Time to live in seconds for an HTTP connection. After that time, the connection will be dropped.
299      * Intermediates tend to drop connections after some idle period. Set to -1 to maintain connections
300      * indefinitely. This value defaults to 300 seconds.
301      *
302      * @since 3.2
303      */
304     private static final long CONN_TTL =
305         Long.getLong( "maven.wagon.httpconnectionManager.ttlSeconds", 300L );
306 
307     /**
308      * Internal connection manager
309      */
310     private static HttpClientConnectionManager httpClientConnectionManager = createConnManager();
311 
312 
313     /**
314      * See RFC6585
315      */
316     protected static final int SC_TOO_MANY_REQUESTS = 429;
317 
318     /**
319      * For exponential backoff.
320      */
321 
322     /**
323      * Initial seconds to back off when a HTTP 429 received.
324      * Subsequent 429 responses result in exponental backoff.
325      * <b>5 by default</b>
326      *
327      * @since 2.7
328      */
329     private int initialBackoffSeconds =
330         Integer.parseInt( System.getProperty( "maven.wagon.httpconnectionManager.backoffSeconds", "5" ) );
331 
332     /**
333      * The maximum amount of time we want to back off in the case of
334      * repeated HTTP 429 response codes.
335      *
336      * @since 2.7
337      */
338     private static final int MAX_BACKOFF_WAIT_SECONDS =
339         Integer.parseInt( System.getProperty( "maven.wagon.httpconnectionManager.maxBackoffSeconds", "180" ) );
340 
341     protected int backoff( int wait, String url )
342         throws InterruptedException, TransferFailedException
343     {
344         TimeUnit.SECONDS.sleep( wait );
345         int nextWait = wait * 2;
346         if ( nextWait >= getMaxBackoffWaitSeconds() )
347         {
348             throw new TransferFailedException( formatTransferFailedMessage( url, SC_TOO_MANY_REQUESTS,
349                     null, getProxyInfo() ) );
350         }
351         return nextWait;
352     }
353 
354     @SuppressWarnings( "checkstyle:linelength" )
355     private static PoolingHttpClientConnectionManager createConnManager()
356     {
357 
358         String sslProtocolsStr = System.getProperty( "https.protocols" );
359         String cipherSuitesStr = System.getProperty( "https.cipherSuites" );
360         String[] sslProtocols = sslProtocolsStr != null ? sslProtocolsStr.split( " *, *" ) : null;
361         String[] cipherSuites = cipherSuitesStr != null ? cipherSuitesStr.split( " *, *" ) : null;
362 
363         SSLConnectionSocketFactory sslConnectionSocketFactory;
364         if ( SSL_INSECURE )
365         {
366             try
367             {
368                 SSLContext sslContext = new SSLContextBuilder().useSSL().loadTrustMaterial( null,
369                                                                                             new RelaxedTrustStrategy(
370                                                                                                 IGNORE_SSL_VALIDITY_DATES ) ).build();
371                 sslConnectionSocketFactory = new SSLConnectionSocketFactory( sslContext, sslProtocols, cipherSuites,
372                                                                              SSL_ALLOW_ALL
373                                                                                  ? SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER
374                                                                                  : SSLConnectionSocketFactory.BROWSER_COMPATIBLE_HOSTNAME_VERIFIER );
375             }
376             catch ( Exception ex )
377             {
378                 throw new SSLInitializationException( ex.getMessage(), ex );
379             }
380         }
381         else
382         {
383             sslConnectionSocketFactory =
384                 new SSLConnectionSocketFactory( HttpsURLConnection.getDefaultSSLSocketFactory(), sslProtocols,
385                                                 cipherSuites,
386                                                 SSLConnectionSocketFactory.BROWSER_COMPATIBLE_HOSTNAME_VERIFIER );
387         }
388 
389         Registry<ConnectionSocketFactory> registry = RegistryBuilder.<ConnectionSocketFactory>create().register( "http",
390                                                                                                                  PlainConnectionSocketFactory.INSTANCE ).register(
391             "https", sslConnectionSocketFactory ).build();
392 
393         PoolingHttpClientConnectionManager connManager =
394             new PoolingHttpClientConnectionManager( registry, null, null, null, CONN_TTL, TimeUnit.SECONDS );
395         if ( persistentPool )
396         {
397             connManager.setDefaultMaxPerRoute( MAX_CONN_PER_ROUTE );
398             connManager.setMaxTotal( MAX_CONN_TOTAL );
399         }
400         else
401         {
402             connManager.setMaxTotal( 1 );
403         }
404         return connManager;
405     }
406 
407     /**
408      * The type of the retry handler, defaults to {@code standard}.
409      * Values can be {@link default DefaultHttpRequestRetryHandler},
410      * or {@link standard StandardHttpRequestRetryHandler},
411      * or a fully qualified name class with a no-arg.
412      *
413      * @since 3.2
414      */
415     private static final String RETRY_HANDLER_CLASS =
416             System.getProperty( "maven.wagon.http.retryHandler.class", "standard" );
417 
418     /**
419      * Whether or not methods that have successfully sent their request will be retried,
420      * defaults to {@code false}.
421      * Note: only used for default and standard retry handlers.
422      *
423      * @since 3.2
424      */
425     private static final boolean RETRY_HANDLER_REQUEST_SENT_ENABLED =
426             Boolean.getBoolean( "maven.wagon.http.retryHandler.requestSentEnabled" );
427 
428     /**
429      * Number of retries for the retry handler, defaults to 3.
430      * Note: only used for default and standard retry handlers.
431      *
432      * @since 3.2
433      */
434     private static final int RETRY_HANDLER_COUNT =
435             Integer.getInteger( "maven.wagon.http.retryHandler.count", 3 );
436 
437     /**
438      * Comma-separated list of non-retryable exception classes.
439      * Note: only used for default retry handler.
440      *
441      * @since 3.2
442      */
443     private static final String RETRY_HANDLER_EXCEPTIONS =
444             System.getProperty( "maven.wagon.http.retryHandler.nonRetryableClasses" );
445 
446     private static HttpRequestRetryHandler createRetryHandler()
447     {
448         switch ( RETRY_HANDLER_CLASS )
449         {
450             case "default":
451                 if ( StringUtils.isEmpty( RETRY_HANDLER_EXCEPTIONS ) )
452                 {
453                     return new DefaultHttpRequestRetryHandler(
454                             RETRY_HANDLER_COUNT, RETRY_HANDLER_REQUEST_SENT_ENABLED );
455                 }
456                 return new DefaultHttpRequestRetryHandler(
457                         RETRY_HANDLER_COUNT, RETRY_HANDLER_REQUEST_SENT_ENABLED, getNonRetryableExceptions() )
458                 {
459                 };
460             case "standard":
461                 return new StandardHttpRequestRetryHandler( RETRY_HANDLER_COUNT, RETRY_HANDLER_REQUEST_SENT_ENABLED );
462             default:
463                 try
464                 {
465                     final ClassLoader classLoader = AbstractHttpClientWagon.class.getClassLoader();
466                     return HttpRequestRetryHandler.class.cast( classLoader.loadClass( RETRY_HANDLER_CLASS )
467                                                                           .getConstructor().newInstance() );
468                 }
469                 catch ( final Exception e )
470                 {
471                     throw new IllegalArgumentException( e );
472                 }
473         }
474     }
475 
476     /**
477      * The type of the serviceUnavailableRetryStrategy, defaults to {@code none}.
478      * Values can be {@link default DefaultServiceUnavailableRetryStrategy},
479      * or {@link standard StandardServiceUnavailableRetryStrategy}, or
480      * a fully qualified name class with a no-arg or none to not use a ServiceUnavailableRetryStrategy.
481      */
482     private static final String SERVICE_UNAVAILABLE_RETRY_STRATEGY_CLASS =
483             System.getProperty( "maven.wagon.http.serviceUnavailableRetryStrategy.class", "none" );
484 
485     /**
486      * Interval in milliseconds between retries when using a serviceUnavailableRetryStrategy.
487      * <b>1000 by default</b>
488      */
489     private static final int SERVICE_UNAVAILABLE_RETRY_STRATEGY_RETRY_INTERVAL =
490         Integer.getInteger( "maven.wagon.http.serviceUnavailableRetryStrategy.retryInterval", 1000 );
491 
492     /**
493      * Maximum number of retries when using a serviceUnavailableRetryStrategy.
494      * <b>5 by default</b>
495      */
496     private static final int SERVICE_UNAVAILABLE_RETRY_STRATEGY_MAX_RETRIES =
497         Integer.getInteger( "maven.wagon.http.serviceUnavailableRetryStrategy.maxRetries", 5 );
498 
499     private static ServiceUnavailableRetryStrategy createServiceUnavailableRetryStrategy()
500     {
501         switch ( SERVICE_UNAVAILABLE_RETRY_STRATEGY_CLASS )
502         {
503             case "none": return null;
504             case "default":
505                 return new DefaultServiceUnavailableRetryStrategy(
506                     SERVICE_UNAVAILABLE_RETRY_STRATEGY_MAX_RETRIES, SERVICE_UNAVAILABLE_RETRY_STRATEGY_RETRY_INTERVAL );
507             case "standard":
508                 return new StandardServiceUnavailableRetryStrategy(
509                     SERVICE_UNAVAILABLE_RETRY_STRATEGY_MAX_RETRIES, SERVICE_UNAVAILABLE_RETRY_STRATEGY_RETRY_INTERVAL );
510             default:
511                 try
512                 {
513                     final ClassLoader classLoader = AbstractHttpClientWagon.class.getClassLoader();
514                     return ServiceUnavailableRetryStrategy.class.cast(
515                             classLoader.loadClass( SERVICE_UNAVAILABLE_RETRY_STRATEGY_CLASS )
516                                                                           .getConstructor().newInstance() );
517                 }
518                 catch ( final Exception e )
519                 {
520                     throw new IllegalArgumentException( e );
521                 }
522         }
523     }
524 
525     private static Registry<AuthSchemeProvider> createAuthSchemeRegistry()
526     {
527         return RegistryBuilder.<AuthSchemeProvider>create()
528             .register( AuthSchemes.BASIC, new BasicSchemeFactory( StandardCharsets.UTF_8 ) )
529             .register( AuthSchemes.DIGEST, new DigestSchemeFactory( StandardCharsets.UTF_8 ) )
530             .register( AuthSchemes.NTLM, new NTLMSchemeFactory() )
531             .build();
532     }
533 
534     private static Collection<Class<? extends IOException>> getNonRetryableExceptions()
535     {
536         final List<Class<? extends IOException>> exceptions = new ArrayList<>();
537         final ClassLoader loader = AbstractHttpClientWagon.class.getClassLoader();
538         for ( final String ex : RETRY_HANDLER_EXCEPTIONS.split( "," ) )
539         {
540             try
541             {
542                 exceptions.add( ( Class<? extends IOException> ) loader.loadClass( ex ) );
543             }
544             catch ( final ClassNotFoundException e )
545             {
546                 throw new IllegalArgumentException( e );
547             }
548         }
549         return exceptions;
550     }
551 
552     private static CloseableHttpClient httpClient = createClient();
553 
554     private static CloseableHttpClient createClient()
555     {
556         return HttpClientBuilder.create() //
557             .useSystemProperties() //
558             .disableConnectionState() //
559             .setConnectionManager( httpClientConnectionManager ) //
560             .setRetryHandler( createRetryHandler() )
561             .setServiceUnavailableRetryStrategy( createServiceUnavailableRetryStrategy() )
562             .setDefaultAuthSchemeRegistry( createAuthSchemeRegistry() )
563             .setRedirectStrategy( new WagonRedirectStrategy() )
564             .build();
565     }
566 
567     private CredentialsProvider credentialsProvider;
568 
569     private AuthCache authCache;
570 
571     private Closeable closeable;
572 
573     /**
574      * @plexus.configuration
575      * @deprecated Use httpConfiguration instead.
576      */
577     private Properties httpHeaders;
578 
579     /**
580      * @since 1.0-beta-6
581      */
582     private HttpConfiguration httpConfiguration;
583 
584     /**
585      * Basic auth scope overrides
586      * @since 2.8
587      */
588     private BasicAuthScope basicAuth;
589 
590     /**
591      * Proxy basic auth scope overrides
592      * @since 2.8
593      */
594     private BasicAuthScope proxyAuth;
595 
596     public void openConnectionInternal()
597     {
598         repository.setUrl( getURL( repository ) );
599 
600         credentialsProvider = new BasicCredentialsProvider();
601         authCache = new BasicAuthCache();
602 
603         if ( authenticationInfo != null )
604         {
605 
606             String username = authenticationInfo.getUserName();
607             String password = authenticationInfo.getPassword();
608 
609             if ( StringUtils.isNotEmpty( username ) && StringUtils.isNotEmpty( password ) )
610             {
611                 Credentials creds = new UsernamePasswordCredentials( username, password );
612 
613                 AuthScope targetScope = getBasicAuthScope().getScope( getRepository().getHost(),
614                                                                     getRepository().getPort() );
615                 credentialsProvider.setCredentials( targetScope, creds );
616             }
617         }
618 
619         ProxyInfo proxyInfo = getProxyInfo( getRepository().getProtocol(), getRepository().getHost() );
620         if ( proxyInfo != null )
621         {
622             String proxyUsername = proxyInfo.getUserName();
623             String proxyPassword = proxyInfo.getPassword();
624             String proxyHost = proxyInfo.getHost();
625             String proxyNtlmHost = proxyInfo.getNtlmHost();
626             String proxyNtlmDomain = proxyInfo.getNtlmDomain();
627             if ( proxyHost != null )
628             {
629                 if ( proxyUsername != null && proxyPassword != null )
630                 {
631                     Credentials creds;
632                     if ( proxyNtlmHost != null || proxyNtlmDomain != null )
633                     {
634                         creds = new NTCredentials( proxyUsername, proxyPassword, proxyNtlmHost, proxyNtlmDomain );
635                     }
636                     else
637                     {
638                         creds = new UsernamePasswordCredentials( proxyUsername, proxyPassword );
639                     }
640 
641                     AuthScope proxyScope = getProxyBasicAuthScope().getScope( proxyHost, proxyInfo.getPort() );
642                     credentialsProvider.setCredentials( proxyScope, creds );
643                 }
644             }
645         }
646     }
647 
648     public void closeConnection()
649     {
650         if ( !persistentPool )
651         {
652             httpClientConnectionManager.closeIdleConnections( 0, TimeUnit.MILLISECONDS );
653         }
654 
655         if ( authCache != null )
656         {
657             authCache.clear();
658             authCache = null;
659         }
660 
661         if ( credentialsProvider != null )
662         {
663             credentialsProvider.clear();
664             credentialsProvider = null;
665         }
666     }
667 
668     public static CloseableHttpClient getHttpClient()
669     {
670         return httpClient;
671     }
672 
673     public static void setPersistentPool( boolean persistent )
674     {
675         persistentPool = persistent;
676     }
677 
678     public static void setPoolingHttpClientConnectionManager(
679         PoolingHttpClientConnectionManager poolingHttpClientConnectionManager )
680     {
681         httpClientConnectionManager = poolingHttpClientConnectionManager;
682         httpClient = createClient();
683     }
684 
685     public void put( File source, String resourceName )
686         throws TransferFailedException, ResourceDoesNotExistException, AuthorizationException
687     {
688         Resource resource = new Resource( resourceName );
689 
690         firePutInitiated( resource, source );
691 
692         resource.setContentLength( source.length() );
693 
694         resource.setLastModified( source.lastModified() );
695 
696         put( null, resource, source );
697     }
698 
699     public void putFromStream( final InputStream stream, String destination, long contentLength, long lastModified )
700         throws TransferFailedException, ResourceDoesNotExistException, AuthorizationException
701     {
702         Resource resource = new Resource( destination );
703 
704         firePutInitiated( resource, null );
705 
706         resource.setContentLength( contentLength );
707 
708         resource.setLastModified( lastModified );
709 
710         put( stream, resource, null );
711     }
712 
713     private void put( final InputStream stream, Resource resource, File source )
714         throws TransferFailedException, AuthorizationException, ResourceDoesNotExistException
715     {
716         put( resource, source, new WagonHttpEntity( stream, resource, this, source ) );
717     }
718 
719     private void put( Resource resource, File source, HttpEntity httpEntity )
720         throws TransferFailedException, AuthorizationException, ResourceDoesNotExistException
721     {
722         put( resource, source, httpEntity, buildUrl( resource ) );
723     }
724 
725     /**
726      * Builds a complete URL string from the repository URL and the relative path of the resource passed.
727      *
728      * @param resource the resource to extract the relative path from.
729      * @return the complete URL
730      */
731     private String buildUrl( Resource resource )
732     {
733         return buildUrl( resource.getName() );
734     }
735 
736     /**
737      * Builds a complete URL string from the repository URL and the relative path of the resource passed.
738      *
739      * @param resourceName the resourcerelative path
740      * @return the complete URL
741      */
742     private String buildUrl( String resourceName )
743     {
744         return EncodingUtil.encodeURLToString( getRepository().getUrl(), resourceName );
745     }
746 
747     private void put( Resource resource, File source, HttpEntity httpEntity, String url )
748         throws TransferFailedException, AuthorizationException, ResourceDoesNotExistException
749     {
750         put( getInitialBackoffSeconds(), resource, source, httpEntity, url );
751     }
752 
753 
754     private void put( int wait, Resource resource, File source, HttpEntity httpEntity, String url )
755         throws TransferFailedException, AuthorizationException, ResourceDoesNotExistException
756     {
757 
758         //Parent directories need to be created before posting
759         try
760         {
761             mkdirs( PathUtils.dirname( resource.getName() ) );
762         }
763         catch ( HttpException he )
764         {
765             fireTransferError( resource, he, TransferEvent.REQUEST_PUT );
766         }
767         catch ( IOException e )
768         {
769             fireTransferError( resource, e, TransferEvent.REQUEST_PUT );
770         }
771 
772         // preemptive for put
773         // TODO: is it a good idea, though? 'Expect-continue' handshake would serve much better
774 
775         // FIXME Perform only when preemptive has been configured
776         Repository repo = getRepository();
777         HttpHost targetHost = new HttpHost( repo.getHost(), repo.getPort(), repo.getProtocol() );
778         AuthScope targetScope = getBasicAuthScope().getScope( targetHost );
779 
780         if ( credentialsProvider.getCredentials( targetScope ) != null )
781         {
782             BasicScheme targetAuth = new BasicScheme( StandardCharsets.UTF_8 );
783             authCache.put( targetHost, targetAuth );
784         }
785 
786         HttpPut putMethod = new HttpPut( url );
787 
788         firePutStarted( resource, source );
789 
790         try
791         {
792             putMethod.setEntity( httpEntity );
793 
794             CloseableHttpResponse response = execute( putMethod );
795             try
796             {
797                 fireTransferDebug( formatTransferDebugMessage( url, response.getStatusLine().getStatusCode(),
798                         response.getStatusLine().getReasonPhrase(), getProxyInfo() ) );
799                 int statusCode = response.getStatusLine().getStatusCode();
800 
801                 // Check that we didn't run out of retries.
802                 switch ( statusCode )
803                 {
804                     // Success Codes
805                     case HttpStatus.SC_OK: // 200
806                     case HttpStatus.SC_CREATED: // 201
807                     case HttpStatus.SC_ACCEPTED: // 202
808                     case HttpStatus.SC_NO_CONTENT:  // 204
809                         break;
810 
811                     // TODO Move 401/407 to AuthenticationException after WAGON-587
812                     case HttpStatus.SC_FORBIDDEN:
813                     case HttpStatus.SC_UNAUTHORIZED:
814                     case HttpStatus.SC_PROXY_AUTHENTICATION_REQUIRED:
815                         EntityUtils.consumeQuietly( response.getEntity() );
816                         fireSessionConnectionRefused();
817                         throw new AuthorizationException( formatAuthorizationMessage( url,
818                                 response.getStatusLine().getStatusCode(),
819                                 response.getStatusLine().getReasonPhrase(), getProxyInfo() ) );
820 
821                     case HttpStatus.SC_NOT_FOUND:
822                     case HttpStatus.SC_GONE:
823                         EntityUtils.consumeQuietly( response.getEntity() );
824                         throw new ResourceDoesNotExistException( formatResourceDoesNotExistMessage( url,
825                                 response.getStatusLine().getStatusCode(),
826                                 response.getStatusLine().getReasonPhrase(), getProxyInfo() ) );
827 
828                     case SC_TOO_MANY_REQUESTS:
829                         EntityUtils.consumeQuietly( response.getEntity() );
830                         put( backoff( wait, url ), resource, source, httpEntity, url );
831                         break;
832                     //add more entries here
833                     default:
834                         EntityUtils.consumeQuietly( response.getEntity() );
835                         TransferFailedException e = new TransferFailedException( formatTransferFailedMessage( url,
836                                 response.getStatusLine().getStatusCode(),
837                                 response.getStatusLine().getReasonPhrase(), getProxyInfo() ) );
838                         fireTransferError( resource, e, TransferEvent.REQUEST_PUT );
839                         throw e;
840                 }
841 
842                 firePutCompleted( resource, source );
843 
844                 EntityUtils.consume( response.getEntity() );
845             }
846             finally
847             {
848                 response.close();
849             }
850         }
851         catch ( IOException | HttpException | InterruptedException e )
852         {
853             fireTransferError( resource, e, TransferEvent.REQUEST_PUT );
854 
855             throw new TransferFailedException( formatTransferFailedMessage( url, getProxyInfo() ), e );
856         }
857 
858     }
859 
860     protected void mkdirs( String dirname )
861         throws HttpException, IOException
862     {
863         // nothing to do
864     }
865 
866     public boolean resourceExists( String resourceName )
867         throws TransferFailedException, AuthorizationException
868     {
869         return resourceExists( getInitialBackoffSeconds(), resourceName );
870     }
871 
872 
873     private boolean resourceExists( int wait, String resourceName )
874         throws TransferFailedException, AuthorizationException
875     {
876         String url = buildUrl( resourceName );
877         HttpHead headMethod = new HttpHead( url );
878         try
879         {
880             CloseableHttpResponse response = execute( headMethod );
881             try
882             {
883                 int statusCode = response.getStatusLine().getStatusCode();
884                 boolean result;
885                 switch ( statusCode )
886                 {
887                     case HttpStatus.SC_OK:
888                         result = true;
889                         break;
890                     case HttpStatus.SC_NOT_MODIFIED:
891                         result = true;
892                         break;
893 
894                     // TODO Move 401/407 to AuthenticationException after WAGON-587
895                     case HttpStatus.SC_FORBIDDEN:
896                     case HttpStatus.SC_UNAUTHORIZED:
897                     case HttpStatus.SC_PROXY_AUTHENTICATION_REQUIRED:
898                         throw new AuthorizationException( formatAuthorizationMessage( url,
899                                 response.getStatusLine().getStatusCode(),
900                                 response.getStatusLine().getReasonPhrase(), getProxyInfo() ) );
901 
902                     case HttpStatus.SC_NOT_FOUND:
903                     case HttpStatus.SC_GONE:
904                         result = false;
905                         break;
906 
907                     case SC_TOO_MANY_REQUESTS:
908                         return resourceExists( backoff( wait, resourceName ), resourceName );
909 
910                     //add more entries here
911                     default:
912                         throw new TransferFailedException( formatTransferFailedMessage( url,
913                                 response.getStatusLine().getStatusCode(),
914                                 response.getStatusLine().getReasonPhrase(), getProxyInfo() ) );
915                 }
916 
917                 return result;
918             }
919             finally
920             {
921                 response.close();
922             }
923         }
924         catch ( IOException | HttpException | InterruptedException e )
925         {
926             throw new TransferFailedException( formatTransferFailedMessage( url, getProxyInfo() ), e );
927         }
928 
929     }
930 
931     protected CloseableHttpResponse execute( HttpUriRequest httpMethod )
932         throws HttpException, IOException
933     {
934         setHeaders( httpMethod );
935         String userAgent = getUserAgent( httpMethod );
936         if ( userAgent != null )
937         {
938             httpMethod.setHeader( HTTP.USER_AGENT, userAgent );
939         }
940 
941         RequestConfig.Builder requestConfigBuilder = RequestConfig.custom();
942         // WAGON-273: default the cookie-policy to browser compatible
943         requestConfigBuilder.setCookieSpec( CookieSpecs.BROWSER_COMPATIBILITY );
944 
945         Repository repo = getRepository();
946         ProxyInfo proxyInfo = getProxyInfo( repo.getProtocol(), repo.getHost() );
947         if ( proxyInfo != null )
948         {
949             HttpHost proxy = new HttpHost( proxyInfo.getHost(), proxyInfo.getPort() );
950             requestConfigBuilder.setProxy( proxy );
951         }
952 
953         requestConfigBuilder.setConnectTimeout( getTimeout() );
954         requestConfigBuilder.setSocketTimeout( getReadTimeout() );
955         // We don't apply this to MKCOL because RFC 7231 says that this will not work without a body
956         // and our MKCOL requests don't have a body. They will logically behave like GET.
957         if ( httpMethod instanceof HttpPut )
958         {
959             requestConfigBuilder.setExpectContinueEnabled( true );
960         }
961 
962         HttpMethodConfiguration config =
963                 httpConfiguration == null ? null : httpConfiguration.getMethodConfiguration( httpMethod );
964         if ( config != null )
965         {
966             ConfigurationUtils.copyConfig( config, requestConfigBuilder );
967         }
968 
969         HttpClientContext localContext = HttpClientContext.create();
970         localContext.setCredentialsProvider( credentialsProvider );
971         localContext.setAuthCache( authCache );
972         localContext.setRequestConfig( requestConfigBuilder.build() );
973 
974         if ( config != null && config.isUsePreemptive() )
975         {
976             HttpHost targetHost = new HttpHost( repo.getHost(), repo.getPort(), repo.getProtocol() );
977             AuthScope targetScope = getBasicAuthScope().getScope( targetHost );
978 
979             if ( credentialsProvider.getCredentials( targetScope ) != null )
980             {
981                 BasicScheme targetAuth = new BasicScheme( StandardCharsets.UTF_8 );
982                 authCache.put( targetHost, targetAuth );
983             }
984         }
985 
986         if ( proxyInfo != null )
987         {
988             if ( proxyInfo.getHost() != null )
989             {
990                 HttpHost proxyHost = new HttpHost( proxyInfo.getHost(), proxyInfo.getPort() );
991                 AuthScope proxyScope = getProxyBasicAuthScope().getScope( proxyHost );
992 
993                 if ( credentialsProvider.getCredentials( proxyScope ) != null )
994                 {
995                     /* This is extremely ugly because we need to set challengeState to PROXY, but
996                      * the constructor is deprecated. Alternatively, we could subclass BasicScheme
997                      * to ProxyBasicScheme and set the state internally in the constructor.
998                      */
999                     BasicScheme proxyAuth = new BasicScheme( ChallengeState.PROXY );
1000                     authCache.put( proxyHost, proxyAuth );
1001                 }
1002             }
1003         }
1004 
1005         return httpClient.execute( httpMethod, localContext );
1006     }
1007 
1008     public void setHeaders( HttpUriRequest method )
1009     {
1010         HttpMethodConfiguration config =
1011             httpConfiguration == null ? null : httpConfiguration.getMethodConfiguration( method );
1012         if ( config == null || config.isUseDefaultHeaders() )
1013         {
1014             // TODO: merge with the other headers and have some better defaults, unify with lightweight headers
1015             method.addHeader(  "Cache-control", "no-cache" );
1016             method.addHeader( "Pragma", "no-cache" );
1017         }
1018 
1019         if ( httpHeaders != null )
1020         {
1021             for ( Map.Entry<Object, Object> entry : httpHeaders.entrySet() )
1022             {
1023                 method.setHeader( (String) entry.getKey(), (String) entry.getValue() );
1024             }
1025         }
1026 
1027         Header[] headers = config == null ? null : config.asRequestHeaders();
1028         if ( headers != null )
1029         {
1030             for ( Header header : headers )
1031             {
1032                 method.setHeader( header );
1033             }
1034         }
1035 
1036         Header userAgentHeader = method.getFirstHeader( HTTP.USER_AGENT );
1037         if ( userAgentHeader == null )
1038         {
1039             String userAgent = getUserAgent( method );
1040             if ( userAgent != null )
1041             {
1042                 method.setHeader( HTTP.USER_AGENT, userAgent );
1043             }
1044         }
1045     }
1046 
1047     protected String getUserAgent( HttpUriRequest method )
1048     {
1049         if ( httpHeaders != null )
1050         {
1051             String value = (String) httpHeaders.get( HTTP.USER_AGENT );
1052             if ( value != null )
1053             {
1054                 return value;
1055             }
1056         }
1057         HttpMethodConfiguration config =
1058             httpConfiguration == null ? null : httpConfiguration.getMethodConfiguration( method );
1059 
1060         if ( config != null )
1061         {
1062             return (String) config.getHeaders().get( HTTP.USER_AGENT );
1063         }
1064         return null;
1065     }
1066 
1067     /**
1068      * getUrl
1069      * Implementors can override this to remove unwanted parts of the url such as role-hints
1070      *
1071      * @param repository
1072      * @return
1073      */
1074     protected String getURL( Repository repository )
1075     {
1076         return repository.getUrl();
1077     }
1078 
1079     public HttpConfiguration getHttpConfiguration()
1080     {
1081         return httpConfiguration;
1082     }
1083 
1084     public void setHttpConfiguration( HttpConfiguration httpConfiguration )
1085     {
1086         this.httpConfiguration = httpConfiguration;
1087     }
1088 
1089     /**
1090      * Get the override values for standard HttpClient AuthScope
1091      *
1092      * @return the basicAuth
1093      */
1094     public BasicAuthScope getBasicAuthScope()
1095     {
1096         if ( basicAuth == null )
1097         {
1098             basicAuth = new BasicAuthScope();
1099         }
1100         return basicAuth;
1101     }
1102 
1103     /**
1104      * Set the override values for standard HttpClient AuthScope
1105      *
1106      * @param basicAuth the AuthScope to set
1107      */
1108     public void setBasicAuthScope( BasicAuthScope basicAuth )
1109     {
1110         this.basicAuth = basicAuth;
1111     }
1112 
1113     /**
1114      * Get the override values for proxy HttpClient AuthScope
1115      *
1116      * @return the proxyAuth
1117      */
1118     public BasicAuthScope getProxyBasicAuthScope()
1119     {
1120         if ( proxyAuth == null )
1121         {
1122             proxyAuth = new BasicAuthScope();
1123         }
1124         return proxyAuth;
1125     }
1126 
1127     /**
1128      * Set the override values for proxy HttpClient AuthScope
1129      *
1130      * @param proxyAuth the AuthScope to set
1131      */
1132     public void setProxyBasicAuthScope( BasicAuthScope proxyAuth )
1133     {
1134         this.proxyAuth = proxyAuth;
1135     }
1136 
1137     public void fillInputData( InputData inputData )
1138         throws TransferFailedException, ResourceDoesNotExistException, AuthorizationException
1139     {
1140         fillInputData( getInitialBackoffSeconds(), inputData );
1141     }
1142 
1143     private void fillInputData( int wait, InputData inputData )
1144         throws TransferFailedException, ResourceDoesNotExistException, AuthorizationException
1145     {
1146         Resource resource = inputData.getResource();
1147 
1148         String url = buildUrl( resource );
1149         HttpGet getMethod = new HttpGet( url );
1150         long timestamp = resource.getLastModified();
1151         if ( timestamp > 0 )
1152         {
1153             SimpleDateFormat fmt = new SimpleDateFormat( "EEE, dd-MMM-yy HH:mm:ss zzz", Locale.US );
1154             fmt.setTimeZone( GMT_TIME_ZONE );
1155             Header hdr = new BasicHeader( "If-Modified-Since", fmt.format( new Date( timestamp ) ) );
1156             fireTransferDebug( "sending ==> " + hdr + "(" + timestamp + ")" );
1157             getMethod.addHeader( hdr );
1158         }
1159 
1160         try
1161         {
1162             CloseableHttpResponse response = execute( getMethod );
1163             closeable = response;
1164 
1165             fireTransferDebug( formatTransferDebugMessage( url, response.getStatusLine().getStatusCode(),
1166                     response.getStatusLine().getReasonPhrase(), getProxyInfo() ) );
1167             int statusCode = response.getStatusLine().getStatusCode();
1168 
1169             switch ( statusCode )
1170             {
1171                 case HttpStatus.SC_OK:
1172                     break;
1173 
1174                 case HttpStatus.SC_NOT_MODIFIED:
1175                     // return, leaving last modified set to original value so getIfNewer should return unmodified
1176                     return;
1177 
1178                 // TODO Move 401/407 to AuthenticationException after WAGON-587
1179                 case HttpStatus.SC_FORBIDDEN:
1180                 case HttpStatus.SC_UNAUTHORIZED:
1181                 case HttpStatus.SC_PROXY_AUTHENTICATION_REQUIRED:
1182                     EntityUtils.consumeQuietly( response.getEntity() );
1183                     fireSessionConnectionRefused();
1184                     throw new AuthorizationException( formatAuthorizationMessage( url,
1185                             response.getStatusLine().getStatusCode(), response.getStatusLine().getReasonPhrase(),
1186                             getProxyInfo() ) );
1187 
1188                 case HttpStatus.SC_NOT_FOUND:
1189                 case HttpStatus.SC_GONE:
1190                     EntityUtils.consumeQuietly( response.getEntity() );
1191                     throw new ResourceDoesNotExistException( formatResourceDoesNotExistMessage( url,
1192                             response.getStatusLine().getStatusCode(),
1193                             response.getStatusLine().getReasonPhrase(), getProxyInfo() ) );
1194 
1195                 case SC_TOO_MANY_REQUESTS:
1196                     EntityUtils.consumeQuietly( response.getEntity() );
1197                     fillInputData( backoff( wait, url ), inputData );
1198                     break;
1199 
1200                 // add more entries here
1201                 default:
1202                     EntityUtils.consumeQuietly( response.getEntity() );
1203                     cleanupGetTransfer( resource );
1204                     TransferFailedException e = new TransferFailedException( formatTransferFailedMessage( url,
1205                             response.getStatusLine().getStatusCode(), response.getStatusLine().getReasonPhrase(),
1206                             getProxyInfo() ) );
1207                     fireTransferError( resource, e, TransferEvent.REQUEST_GET );
1208                     throw e;
1209             }
1210 
1211             Header contentLengthHeader = response.getFirstHeader( "Content-Length" );
1212 
1213             if ( contentLengthHeader != null )
1214             {
1215                 try
1216                 {
1217                     long contentLength = Long.parseLong( contentLengthHeader.getValue() );
1218 
1219                     resource.setContentLength( contentLength );
1220                 }
1221                 catch ( NumberFormatException e )
1222                 {
1223                     fireTransferDebug(
1224                         "error parsing content length header '" + contentLengthHeader.getValue() + "' " + e );
1225                 }
1226             }
1227 
1228             Header lastModifiedHeader = response.getFirstHeader( "Last-Modified" );
1229             if ( lastModifiedHeader != null )
1230             {
1231                 Date lastModified = DateUtils.parseDate( lastModifiedHeader.getValue() );
1232                 if ( lastModified != null )
1233                 {
1234                     resource.setLastModified( lastModified.getTime() );
1235                     fireTransferDebug( "last-modified = " + lastModifiedHeader.getValue() + " ("
1236                         + lastModified.getTime() + ")" );
1237                 }
1238             }
1239 
1240             HttpEntity entity = response.getEntity();
1241             if ( entity != null )
1242             {
1243                 inputData.setInputStream( entity.getContent() );
1244             }
1245         }
1246         catch ( IOException | HttpException | InterruptedException e )
1247         {
1248             fireTransferError( resource, e, TransferEvent.REQUEST_GET );
1249 
1250             throw new TransferFailedException( formatTransferFailedMessage( url, getProxyInfo() ), e );
1251         }
1252 
1253     }
1254 
1255     protected void cleanupGetTransfer( Resource resource )
1256     {
1257         if ( closeable != null )
1258         {
1259             try
1260             {
1261                 closeable.close();
1262             }
1263             catch ( IOException ignore )
1264             {
1265                 // ignore
1266             }
1267 
1268         }
1269     }
1270 
1271 
1272     @Override
1273     public void putFromStream( InputStream stream, String destination )
1274         throws TransferFailedException, ResourceDoesNotExistException, AuthorizationException
1275     {
1276         putFromStream( stream, destination, -1, -1 );
1277     }
1278 
1279     @Override
1280     protected void putFromStream( InputStream stream, Resource resource )
1281         throws TransferFailedException, AuthorizationException, ResourceDoesNotExistException
1282     {
1283         putFromStream( stream, resource.getName(), -1, -1 );
1284     }
1285 
1286     public Properties getHttpHeaders()
1287     {
1288         return httpHeaders;
1289     }
1290 
1291     public void setHttpHeaders( Properties httpHeaders )
1292     {
1293         this.httpHeaders = httpHeaders;
1294     }
1295 
1296     @Override
1297     public void fillOutputData( OutputData outputData )
1298         throws TransferFailedException
1299     {
1300         // no needed in this implementation but throw an Exception if used
1301         throw new IllegalStateException( "this wagon http client must not use fillOutputData" );
1302     }
1303 
1304     protected CredentialsProvider getCredentialsProvider()
1305     {
1306         return credentialsProvider;
1307     }
1308 
1309     protected AuthCache getAuthCache()
1310     {
1311         return authCache;
1312     }
1313 
1314     public int getInitialBackoffSeconds()
1315     {
1316         return initialBackoffSeconds;
1317     }
1318 
1319     public void setInitialBackoffSeconds( int initialBackoffSeconds )
1320     {
1321         this.initialBackoffSeconds = initialBackoffSeconds;
1322     }
1323 
1324     public static int getMaxBackoffWaitSeconds()
1325     {
1326         return MAX_BACKOFF_WAIT_SECONDS;
1327     }
1328 }