View Javadoc
1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *   http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing,
13   * software distributed under the License is distributed on an
14   * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15   * KIND, either express or implied.  See the License for the
16   * specific language governing permissions and limitations
17   * under the License.
18   */
19  package org.eclipse.aether.transport.http;
20  
21  import java.io.File;
22  import java.io.IOException;
23  import java.io.InputStream;
24  import java.io.InterruptedIOException;
25  import java.io.OutputStream;
26  import java.io.UncheckedIOException;
27  import java.net.InetAddress;
28  import java.net.URI;
29  import java.net.URISyntaxException;
30  import java.net.UnknownHostException;
31  import java.nio.charset.Charset;
32  import java.nio.file.Files;
33  import java.nio.file.StandardCopyOption;
34  import java.nio.file.attribute.FileTime;
35  import java.util.Collections;
36  import java.util.Date;
37  import java.util.HashSet;
38  import java.util.List;
39  import java.util.Map;
40  import java.util.Set;
41  import java.util.regex.Matcher;
42  import java.util.regex.Pattern;
43  
44  import org.apache.http.Header;
45  import org.apache.http.HttpEntity;
46  import org.apache.http.HttpEntityEnclosingRequest;
47  import org.apache.http.HttpHeaders;
48  import org.apache.http.HttpHost;
49  import org.apache.http.HttpResponse;
50  import org.apache.http.HttpStatus;
51  import org.apache.http.auth.AuthSchemeProvider;
52  import org.apache.http.auth.AuthScope;
53  import org.apache.http.client.CredentialsProvider;
54  import org.apache.http.client.HttpRequestRetryHandler;
55  import org.apache.http.client.HttpResponseException;
56  import org.apache.http.client.ServiceUnavailableRetryStrategy;
57  import org.apache.http.client.config.AuthSchemes;
58  import org.apache.http.client.config.RequestConfig;
59  import org.apache.http.client.methods.CloseableHttpResponse;
60  import org.apache.http.client.methods.HttpGet;
61  import org.apache.http.client.methods.HttpHead;
62  import org.apache.http.client.methods.HttpOptions;
63  import org.apache.http.client.methods.HttpPut;
64  import org.apache.http.client.methods.HttpUriRequest;
65  import org.apache.http.client.utils.DateUtils;
66  import org.apache.http.client.utils.URIUtils;
67  import org.apache.http.config.Registry;
68  import org.apache.http.config.RegistryBuilder;
69  import org.apache.http.config.SocketConfig;
70  import org.apache.http.entity.AbstractHttpEntity;
71  import org.apache.http.entity.ByteArrayEntity;
72  import org.apache.http.impl.NoConnectionReuseStrategy;
73  import org.apache.http.impl.auth.BasicScheme;
74  import org.apache.http.impl.auth.BasicSchemeFactory;
75  import org.apache.http.impl.auth.DigestSchemeFactory;
76  import org.apache.http.impl.auth.KerberosSchemeFactory;
77  import org.apache.http.impl.auth.NTLMSchemeFactory;
78  import org.apache.http.impl.auth.SPNegoSchemeFactory;
79  import org.apache.http.impl.client.CloseableHttpClient;
80  import org.apache.http.impl.client.DefaultHttpRequestRetryHandler;
81  import org.apache.http.impl.client.HttpClientBuilder;
82  import org.apache.http.impl.client.StandardHttpRequestRetryHandler;
83  import org.apache.http.protocol.HttpContext;
84  import org.apache.http.util.EntityUtils;
85  import org.eclipse.aether.ConfigurationProperties;
86  import org.eclipse.aether.RepositorySystemSession;
87  import org.eclipse.aether.repository.AuthenticationContext;
88  import org.eclipse.aether.repository.Proxy;
89  import org.eclipse.aether.repository.RemoteRepository;
90  import org.eclipse.aether.spi.connector.transport.AbstractTransporter;
91  import org.eclipse.aether.spi.connector.transport.GetTask;
92  import org.eclipse.aether.spi.connector.transport.PeekTask;
93  import org.eclipse.aether.spi.connector.transport.PutTask;
94  import org.eclipse.aether.spi.connector.transport.TransportTask;
95  import org.eclipse.aether.transfer.NoTransporterException;
96  import org.eclipse.aether.transfer.TransferCancelledException;
97  import org.eclipse.aether.util.ConfigUtils;
98  import org.eclipse.aether.util.FileUtils;
99  import org.slf4j.Logger;
100 import org.slf4j.LoggerFactory;
101 
102 import static java.util.Objects.requireNonNull;
103 
104 /**
105  * A transporter for HTTP/HTTPS.
106  */
107 final class HttpTransporter extends AbstractTransporter {
108 
109     static final String BIND_ADDRESS = "aether.connector.bind.address";
110 
111     static final String SUPPORT_WEBDAV = "aether.connector.http.supportWebDav";
112 
113     static final String PREEMPTIVE_PUT_AUTH = "aether.connector.http.preemptivePutAuth";
114 
115     static final String USE_SYSTEM_PROPERTIES = "aether.connector.http.useSystemProperties";
116 
117     static final String HTTP_RETRY_HANDLER_NAME = "aether.connector.http.retryHandler.name";
118 
119     private static final String HTTP_RETRY_HANDLER_NAME_STANDARD = "standard";
120 
121     private static final String HTTP_RETRY_HANDLER_NAME_DEFAULT = "default";
122 
123     static final String HTTP_RETRY_HANDLER_REQUEST_SENT_ENABLED =
124             "aether.connector.http.retryHandler.requestSentEnabled";
125 
126     private static final Pattern CONTENT_RANGE_PATTERN =
127             Pattern.compile("\\s*bytes\\s+([0-9]+)\\s*-\\s*([0-9]+)\\s*/.*");
128 
129     private static final Logger LOGGER = LoggerFactory.getLogger(HttpTransporter.class);
130 
131     private final Map<String, ChecksumExtractor> checksumExtractors;
132 
133     private final AuthenticationContext repoAuthContext;
134 
135     private final AuthenticationContext proxyAuthContext;
136 
137     private final URI baseUri;
138 
139     private final HttpHost server;
140 
141     private final HttpHost proxy;
142 
143     private final CloseableHttpClient client;
144 
145     private final Map<?, ?> headers;
146 
147     private final LocalState state;
148 
149     private final boolean preemptiveAuth;
150 
151     private final boolean preemptivePutAuth;
152 
153     private final boolean supportWebDav;
154 
155     @SuppressWarnings("checkstyle:methodlength")
156     HttpTransporter(
157             Map<String, ChecksumExtractor> checksumExtractors,
158             RemoteRepository repository,
159             RepositorySystemSession session)
160             throws NoTransporterException {
161         if (!"http".equalsIgnoreCase(repository.getProtocol()) && !"https".equalsIgnoreCase(repository.getProtocol())) {
162             throw new NoTransporterException(repository);
163         }
164         this.checksumExtractors = requireNonNull(checksumExtractors, "checksum extractors must not be null");
165         try {
166             this.baseUri = new URI(repository.getUrl()).parseServerAuthority();
167             if (baseUri.isOpaque()) {
168                 throw new URISyntaxException(repository.getUrl(), "URL must not be opaque");
169             }
170             this.server = URIUtils.extractHost(baseUri);
171             if (server == null) {
172                 throw new URISyntaxException(repository.getUrl(), "URL lacks host name");
173             }
174         } catch (URISyntaxException e) {
175             throw new NoTransporterException(repository, e.getMessage(), e);
176         }
177         this.proxy = toHost(repository.getProxy());
178 
179         this.repoAuthContext = AuthenticationContext.forRepository(session, repository);
180         this.proxyAuthContext = AuthenticationContext.forProxy(session, repository);
181 
182         String httpsSecurityMode = ConfigUtils.getString(
183                 session,
184                 ConfigurationProperties.HTTPS_SECURITY_MODE_DEFAULT,
185                 ConfigurationProperties.HTTPS_SECURITY_MODE + "." + repository.getId(),
186                 ConfigurationProperties.HTTPS_SECURITY_MODE);
187         final int connectionMaxTtlSeconds = ConfigUtils.getInteger(
188                 session,
189                 ConfigurationProperties.DEFAULT_HTTP_CONNECTION_MAX_TTL,
190                 ConfigurationProperties.HTTP_CONNECTION_MAX_TTL + "." + repository.getId(),
191                 ConfigurationProperties.HTTP_CONNECTION_MAX_TTL);
192         final int maxConnectionsPerRoute = ConfigUtils.getInteger(
193                 session,
194                 ConfigurationProperties.DEFAULT_HTTP_MAX_CONNECTIONS_PER_ROUTE,
195                 ConfigurationProperties.HTTP_MAX_CONNECTIONS_PER_ROUTE + "." + repository.getId(),
196                 ConfigurationProperties.HTTP_MAX_CONNECTIONS_PER_ROUTE);
197         this.state = new LocalState(
198                 session,
199                 repository,
200                 new ConnMgrConfig(
201                         session, repoAuthContext, httpsSecurityMode, connectionMaxTtlSeconds, maxConnectionsPerRoute));
202 
203         this.headers = ConfigUtils.getMap(
204                 session,
205                 Collections.emptyMap(),
206                 ConfigurationProperties.HTTP_HEADERS + "." + repository.getId(),
207                 ConfigurationProperties.HTTP_HEADERS);
208 
209         this.preemptiveAuth = ConfigUtils.getBoolean(
210                 session,
211                 ConfigurationProperties.DEFAULT_HTTP_PREEMPTIVE_AUTH,
212                 ConfigurationProperties.HTTP_PREEMPTIVE_AUTH + "." + repository.getId(),
213                 ConfigurationProperties.HTTP_PREEMPTIVE_AUTH);
214         this.preemptivePutAuth = // defaults to true: Wagon does same
215                 ConfigUtils.getBoolean(
216                         session, true, PREEMPTIVE_PUT_AUTH + "." + repository.getId(), PREEMPTIVE_PUT_AUTH);
217         this.supportWebDav = // defaults to false: who needs it will enable it
218                 ConfigUtils.getBoolean(session, false, SUPPORT_WEBDAV + "." + repository.getId(), SUPPORT_WEBDAV);
219         String credentialEncoding = ConfigUtils.getString(
220                 session,
221                 ConfigurationProperties.DEFAULT_HTTP_CREDENTIAL_ENCODING,
222                 ConfigurationProperties.HTTP_CREDENTIAL_ENCODING + "." + repository.getId(),
223                 ConfigurationProperties.HTTP_CREDENTIAL_ENCODING);
224         int connectTimeout = ConfigUtils.getInteger(
225                 session,
226                 ConfigurationProperties.DEFAULT_CONNECT_TIMEOUT,
227                 ConfigurationProperties.CONNECT_TIMEOUT + "." + repository.getId(),
228                 ConfigurationProperties.CONNECT_TIMEOUT);
229         int requestTimeout = ConfigUtils.getInteger(
230                 session,
231                 ConfigurationProperties.DEFAULT_REQUEST_TIMEOUT,
232                 ConfigurationProperties.REQUEST_TIMEOUT + "." + repository.getId(),
233                 ConfigurationProperties.REQUEST_TIMEOUT);
234         int retryCount = ConfigUtils.getInteger(
235                 session,
236                 ConfigurationProperties.DEFAULT_HTTP_RETRY_HANDLER_COUNT,
237                 ConfigurationProperties.HTTP_RETRY_HANDLER_COUNT + "." + repository.getId(),
238                 ConfigurationProperties.HTTP_RETRY_HANDLER_COUNT);
239         long retryInterval = ConfigUtils.getLong(
240                 session,
241                 ConfigurationProperties.DEFAULT_HTTP_RETRY_HANDLER_INTERVAL,
242                 ConfigurationProperties.HTTP_RETRY_HANDLER_INTERVAL + "." + repository.getId(),
243                 ConfigurationProperties.HTTP_RETRY_HANDLER_INTERVAL);
244         long retryIntervalMax = ConfigUtils.getLong(
245                 session,
246                 ConfigurationProperties.DEFAULT_HTTP_RETRY_HANDLER_INTERVAL_MAX,
247                 ConfigurationProperties.HTTP_RETRY_HANDLER_INTERVAL_MAX + "." + repository.getId(),
248                 ConfigurationProperties.HTTP_RETRY_HANDLER_INTERVAL_MAX);
249         String serviceUnavailableCodesString = ConfigUtils.getString(
250                 session,
251                 ConfigurationProperties.DEFAULT_HTTP_RETRY_HANDLER_SERVICE_UNAVAILABLE,
252                 ConfigurationProperties.HTTP_RETRY_HANDLER_SERVICE_UNAVAILABLE + "." + repository.getId(),
253                 ConfigurationProperties.HTTP_RETRY_HANDLER_SERVICE_UNAVAILABLE);
254         String retryHandlerName = ConfigUtils.getString(
255                 session,
256                 HTTP_RETRY_HANDLER_NAME_STANDARD,
257                 HTTP_RETRY_HANDLER_NAME + "." + repository.getId(),
258                 HTTP_RETRY_HANDLER_NAME);
259         boolean retryHandlerRequestSentEnabled = ConfigUtils.getBoolean(
260                 session,
261                 false,
262                 HTTP_RETRY_HANDLER_REQUEST_SENT_ENABLED + "." + repository.getId(),
263                 HTTP_RETRY_HANDLER_REQUEST_SENT_ENABLED);
264         String userAgent = ConfigUtils.getString(
265                 session, ConfigurationProperties.DEFAULT_USER_AGENT, ConfigurationProperties.USER_AGENT);
266 
267         Charset credentialsCharset = Charset.forName(credentialEncoding);
268         Registry<AuthSchemeProvider> authSchemeRegistry = RegistryBuilder.<AuthSchemeProvider>create()
269                 .register(AuthSchemes.BASIC, new BasicSchemeFactory(credentialsCharset))
270                 .register(AuthSchemes.DIGEST, new DigestSchemeFactory(credentialsCharset))
271                 .register(AuthSchemes.NTLM, new NTLMSchemeFactory())
272                 .register(AuthSchemes.SPNEGO, new SPNegoSchemeFactory())
273                 .register(AuthSchemes.KERBEROS, new KerberosSchemeFactory())
274                 .build();
275         SocketConfig socketConfig =
276                 SocketConfig.custom().setSoTimeout(requestTimeout).build();
277         RequestConfig requestConfig = RequestConfig.custom()
278                 .setConnectTimeout(connectTimeout)
279                 .setConnectionRequestTimeout(connectTimeout)
280                 .setLocalAddress(getBindAddress(session, repository))
281                 .setSocketTimeout(requestTimeout)
282                 .build();
283 
284         HttpRequestRetryHandler retryHandler;
285         if (HTTP_RETRY_HANDLER_NAME_STANDARD.equals(retryHandlerName)) {
286             retryHandler = new StandardHttpRequestRetryHandler(retryCount, retryHandlerRequestSentEnabled);
287         } else if (HTTP_RETRY_HANDLER_NAME_DEFAULT.equals(retryHandlerName)) {
288             retryHandler = new DefaultHttpRequestRetryHandler(retryCount, retryHandlerRequestSentEnabled);
289         } else {
290             throw new IllegalArgumentException(
291                     "Unsupported parameter " + HTTP_RETRY_HANDLER_NAME + " value: " + retryHandlerName);
292         }
293         Set<Integer> serviceUnavailableCodes = new HashSet<>();
294         try {
295             for (String code : ConfigUtils.parseCommaSeparatedUniqueNames(serviceUnavailableCodesString)) {
296                 serviceUnavailableCodes.add(Integer.parseInt(code));
297             }
298         } catch (NumberFormatException e) {
299             throw new IllegalArgumentException(
300                     "Illegal HTTP codes for " + ConfigurationProperties.HTTP_RETRY_HANDLER_SERVICE_UNAVAILABLE
301                             + " (list of integers): " + serviceUnavailableCodesString);
302         }
303         ServiceUnavailableRetryStrategy serviceUnavailableRetryStrategy = new ResolverServiceUnavailableRetryStrategy(
304                 retryCount, retryInterval, retryIntervalMax, serviceUnavailableCodes);
305 
306         HttpClientBuilder builder = HttpClientBuilder.create()
307                 .setUserAgent(userAgent)
308                 .setDefaultSocketConfig(socketConfig)
309                 .setDefaultRequestConfig(requestConfig)
310                 .setServiceUnavailableRetryStrategy(serviceUnavailableRetryStrategy)
311                 .setRetryHandler(retryHandler)
312                 .setDefaultAuthSchemeRegistry(authSchemeRegistry)
313                 .setConnectionManager(state.getConnectionManager())
314                 .setConnectionManagerShared(true)
315                 .setDefaultCredentialsProvider(toCredentialsProvider(server, repoAuthContext, proxy, proxyAuthContext))
316                 .setProxy(proxy);
317         final boolean useSystemProperties = ConfigUtils.getBoolean(
318                 session, false, USE_SYSTEM_PROPERTIES + "." + repository.getId(), USE_SYSTEM_PROPERTIES);
319         if (useSystemProperties) {
320             LOGGER.warn(
321                     "Transport used Apache HttpClient is instructed to use system properties: this may yield in unwanted side-effects!");
322             LOGGER.warn("Please use documented means to configure resolver transport.");
323             builder.useSystemProperties();
324         }
325 
326         final String expectContinue = ConfigUtils.getString(
327                 session,
328                 null,
329                 ConfigurationProperties.HTTP_EXPECT_CONTINUE + "." + repository.getId(),
330                 ConfigurationProperties.HTTP_EXPECT_CONTINUE);
331         if (expectContinue != null) {
332             state.setExpectContinue(Boolean.parseBoolean(expectContinue));
333         }
334 
335         final boolean reuseConnections = ConfigUtils.getBoolean(
336                 session,
337                 ConfigurationProperties.DEFAULT_HTTP_REUSE_CONNECTIONS,
338                 ConfigurationProperties.HTTP_REUSE_CONNECTIONS + "." + repository.getId(),
339                 ConfigurationProperties.HTTP_REUSE_CONNECTIONS);
340         if (!reuseConnections) {
341             builder.setConnectionReuseStrategy(NoConnectionReuseStrategy.INSTANCE);
342         }
343 
344         this.client = builder.build();
345     }
346 
347     /**
348      * Returns non-null {@link InetAddress} if set in configuration, {@code null} otherwise.
349      */
350     private InetAddress getBindAddress(RepositorySystemSession session, RemoteRepository repository) {
351         String bindAddress =
352                 ConfigUtils.getString(session, null, BIND_ADDRESS + "." + repository.getId(), BIND_ADDRESS);
353         if (bindAddress == null) {
354             return null;
355         }
356         try {
357             return InetAddress.getByName(bindAddress);
358         } catch (UnknownHostException uhe) {
359             throw new IllegalArgumentException(
360                     "Given bind address (" + bindAddress + ") cannot be resolved for remote repository " + repository,
361                     uhe);
362         }
363     }
364 
365     private static HttpHost toHost(Proxy proxy) {
366         HttpHost host = null;
367         if (proxy != null) {
368             host = new HttpHost(proxy.getHost(), proxy.getPort());
369         }
370         return host;
371     }
372 
373     private static CredentialsProvider toCredentialsProvider(
374             HttpHost server, AuthenticationContext serverAuthCtx, HttpHost proxy, AuthenticationContext proxyAuthCtx) {
375         CredentialsProvider provider = toCredentialsProvider(server.getHostName(), AuthScope.ANY_PORT, serverAuthCtx);
376         if (proxy != null) {
377             CredentialsProvider p = toCredentialsProvider(proxy.getHostName(), proxy.getPort(), proxyAuthCtx);
378             provider = new DemuxCredentialsProvider(provider, p, proxy);
379         }
380         return provider;
381     }
382 
383     private static CredentialsProvider toCredentialsProvider(String host, int port, AuthenticationContext ctx) {
384         DeferredCredentialsProvider provider = new DeferredCredentialsProvider();
385         if (ctx != null) {
386             AuthScope basicScope = new AuthScope(host, port);
387             provider.setCredentials(basicScope, new DeferredCredentialsProvider.BasicFactory(ctx));
388 
389             AuthScope ntlmScope = new AuthScope(host, port, AuthScope.ANY_REALM, "ntlm");
390             provider.setCredentials(ntlmScope, new DeferredCredentialsProvider.NtlmFactory(ctx));
391         }
392         return provider;
393     }
394 
395     LocalState getState() {
396         return state;
397     }
398 
399     private URI resolve(TransportTask task) {
400         return UriUtils.resolve(baseUri, task.getLocation());
401     }
402 
403     @Override
404     public int classify(Throwable error) {
405         if (error instanceof HttpResponseException
406                 && ((HttpResponseException) error).getStatusCode() == HttpStatus.SC_NOT_FOUND) {
407             return ERROR_NOT_FOUND;
408         }
409         return ERROR_OTHER;
410     }
411 
412     @Override
413     protected void implPeek(PeekTask task) throws Exception {
414         HttpHead request = commonHeaders(new HttpHead(resolve(task)));
415         execute(request, null);
416     }
417 
418     @Override
419     protected void implGet(GetTask task) throws Exception {
420         boolean resume = true;
421         boolean applyChecksumExtractors = true;
422 
423         EntityGetter getter = new EntityGetter(task);
424         HttpGet request = commonHeaders(new HttpGet(resolve(task)));
425         while (true) {
426             try {
427                 if (resume) {
428                     resume(request, task);
429                 }
430                 if (applyChecksumExtractors) {
431                     for (ChecksumExtractor checksumExtractor : checksumExtractors.values()) {
432                         checksumExtractor.prepareRequest(request);
433                     }
434                 }
435                 execute(request, getter);
436                 break;
437             } catch (HttpResponseException e) {
438                 if (resume
439                         && e.getStatusCode() == HttpStatus.SC_PRECONDITION_FAILED
440                         && request.containsHeader(HttpHeaders.RANGE)) {
441                     request = commonHeaders(new HttpGet(resolve(task)));
442                     resume = false;
443                     continue;
444                 }
445                 if (applyChecksumExtractors) {
446                     boolean retryWithoutExtractors = false;
447                     for (ChecksumExtractor checksumExtractor : checksumExtractors.values()) {
448                         if (checksumExtractor.retryWithoutExtractor(e)) {
449                             retryWithoutExtractors = true;
450                             break;
451                         }
452                     }
453                     if (retryWithoutExtractors) {
454                         request = commonHeaders(new HttpGet(resolve(task)));
455                         applyChecksumExtractors = false;
456                         continue;
457                     }
458                 }
459                 throw e;
460             }
461         }
462     }
463 
464     @Override
465     protected void implPut(PutTask task) throws Exception {
466         PutTaskEntity entity = new PutTaskEntity(task);
467         HttpPut request = commonHeaders(entity(new HttpPut(resolve(task)), entity));
468         try {
469             execute(request, null);
470         } catch (HttpResponseException e) {
471             if (e.getStatusCode() == HttpStatus.SC_EXPECTATION_FAILED && request.containsHeader(HttpHeaders.EXPECT)) {
472                 state.setExpectContinue(false);
473                 request = commonHeaders(entity(new HttpPut(request.getURI()), entity));
474                 execute(request, null);
475                 return;
476             }
477             throw e;
478         }
479     }
480 
481     private void execute(HttpUriRequest request, EntityGetter getter) throws Exception {
482         try {
483             SharingHttpContext context = new SharingHttpContext(state);
484             prepare(request, context);
485             try (CloseableHttpResponse response = client.execute(server, request, context)) {
486                 try {
487                     context.close();
488                     handleStatus(response);
489                     if (getter != null) {
490                         getter.handle(response);
491                     }
492                 } finally {
493                     EntityUtils.consumeQuietly(response.getEntity());
494                 }
495             }
496         } catch (IOException e) {
497             if (e.getCause() instanceof TransferCancelledException) {
498                 throw (Exception) e.getCause();
499             }
500             throw e;
501         }
502     }
503 
504     private void prepare(HttpUriRequest request, SharingHttpContext context) {
505         final boolean put = HttpPut.METHOD_NAME.equalsIgnoreCase(request.getMethod());
506         if (preemptiveAuth || (preemptivePutAuth && put)) {
507             context.getAuthCache().put(server, new BasicScheme());
508         }
509         if (supportWebDav) {
510             if (state.getWebDav() == null && (put || isPayloadPresent(request))) {
511                 HttpOptions req = commonHeaders(new HttpOptions(request.getURI()));
512                 try (CloseableHttpResponse response = client.execute(server, req, context)) {
513                     state.setWebDav(response.containsHeader(HttpHeaders.DAV));
514                     EntityUtils.consumeQuietly(response.getEntity());
515                 } catch (IOException e) {
516                     LOGGER.debug("Failed to prepare HTTP context", e);
517                 }
518             }
519             if (put && Boolean.TRUE.equals(state.getWebDav())) {
520                 mkdirs(request.getURI(), context);
521             }
522         }
523     }
524 
525     @SuppressWarnings("checkstyle:magicnumber")
526     private void mkdirs(URI uri, SharingHttpContext context) {
527         List<URI> dirs = UriUtils.getDirectories(baseUri, uri);
528         int index = 0;
529         for (; index < dirs.size(); index++) {
530             try (CloseableHttpResponse response =
531                     client.execute(server, commonHeaders(new HttpMkCol(dirs.get(index))), context)) {
532                 try {
533                     int status = response.getStatusLine().getStatusCode();
534                     if (status < 300 || status == HttpStatus.SC_METHOD_NOT_ALLOWED) {
535                         break;
536                     } else if (status == HttpStatus.SC_CONFLICT) {
537                         continue;
538                     }
539                     handleStatus(response);
540                 } finally {
541                     EntityUtils.consumeQuietly(response.getEntity());
542                 }
543             } catch (IOException e) {
544                 LOGGER.debug("Failed to create parent directory {}", dirs.get(index), e);
545                 return;
546             }
547         }
548         for (index--; index >= 0; index--) {
549             try (CloseableHttpResponse response =
550                     client.execute(server, commonHeaders(new HttpMkCol(dirs.get(index))), context)) {
551                 try {
552                     handleStatus(response);
553                 } finally {
554                     EntityUtils.consumeQuietly(response.getEntity());
555                 }
556             } catch (IOException e) {
557                 LOGGER.debug("Failed to create parent directory {}", dirs.get(index), e);
558                 return;
559             }
560         }
561     }
562 
563     private <T extends HttpEntityEnclosingRequest> T entity(T request, HttpEntity entity) {
564         request.setEntity(entity);
565         return request;
566     }
567 
568     private boolean isPayloadPresent(HttpUriRequest request) {
569         if (request instanceof HttpEntityEnclosingRequest) {
570             HttpEntity entity = ((HttpEntityEnclosingRequest) request).getEntity();
571             return entity != null && entity.getContentLength() != 0;
572         }
573         return false;
574     }
575 
576     private <T extends HttpUriRequest> T commonHeaders(T request) {
577         request.setHeader(HttpHeaders.CACHE_CONTROL, "no-cache, no-store");
578         request.setHeader(HttpHeaders.PRAGMA, "no-cache");
579 
580         if (state.isExpectContinue() && isPayloadPresent(request)) {
581             request.setHeader(HttpHeaders.EXPECT, "100-continue");
582         }
583 
584         for (Map.Entry<?, ?> entry : headers.entrySet()) {
585             if (!(entry.getKey() instanceof String)) {
586                 continue;
587             }
588             if (entry.getValue() instanceof String) {
589                 request.setHeader(entry.getKey().toString(), entry.getValue().toString());
590             } else {
591                 request.removeHeaders(entry.getKey().toString());
592             }
593         }
594 
595         if (!state.isExpectContinue()) {
596             request.removeHeaders(HttpHeaders.EXPECT);
597         }
598 
599         return request;
600     }
601 
602     @SuppressWarnings("checkstyle:magicnumber")
603     private <T extends HttpUriRequest> T resume(T request, GetTask task) {
604         long resumeOffset = task.getResumeOffset();
605         if (resumeOffset > 0L && task.getDataFile() != null) {
606             request.setHeader(HttpHeaders.RANGE, "bytes=" + resumeOffset + '-');
607             request.setHeader(
608                     HttpHeaders.IF_UNMODIFIED_SINCE,
609                     DateUtils.formatDate(new Date(task.getDataFile().lastModified() - 60L * 1000L)));
610             request.setHeader(HttpHeaders.ACCEPT_ENCODING, "identity");
611         }
612         return request;
613     }
614 
615     @SuppressWarnings("checkstyle:magicnumber")
616     private void handleStatus(CloseableHttpResponse response) throws HttpResponseException {
617         int status = response.getStatusLine().getStatusCode();
618         if (status >= 300) {
619             throw new HttpResponseException(status, response.getStatusLine().getReasonPhrase() + " (" + status + ")");
620         }
621     }
622 
623     @Override
624     protected void implClose() {
625         try {
626             client.close();
627         } catch (IOException e) {
628             throw new UncheckedIOException(e);
629         }
630         AuthenticationContext.close(repoAuthContext);
631         AuthenticationContext.close(proxyAuthContext);
632         state.close();
633     }
634 
635     private class EntityGetter {
636 
637         private final GetTask task;
638 
639         EntityGetter(GetTask task) {
640             this.task = task;
641         }
642 
643         public void handle(CloseableHttpResponse response) throws IOException, TransferCancelledException {
644             HttpEntity entity = response.getEntity();
645             if (entity == null) {
646                 entity = new ByteArrayEntity(new byte[0]);
647             }
648 
649             long offset = 0L, length = entity.getContentLength();
650             Header rangeHeader = response.getFirstHeader(HttpHeaders.CONTENT_RANGE);
651             String range = rangeHeader != null ? rangeHeader.getValue() : null;
652             if (range != null) {
653                 Matcher m = CONTENT_RANGE_PATTERN.matcher(range);
654                 if (!m.matches()) {
655                     throw new IOException("Invalid Content-Range header for partial download: " + range);
656                 }
657                 offset = Long.parseLong(m.group(1));
658                 length = Long.parseLong(m.group(2)) + 1L;
659                 if (offset < 0L || offset >= length || (offset > 0L && offset != task.getResumeOffset())) {
660                     throw new IOException("Invalid Content-Range header for partial download from offset "
661                             + task.getResumeOffset() + ": " + range);
662                 }
663             }
664 
665             final boolean resume = offset > 0L;
666             final File dataFile = task.getDataFile();
667             if (dataFile == null) {
668                 try (InputStream is = entity.getContent()) {
669                     utilGet(task, is, true, length, resume);
670                     extractChecksums(response);
671                 }
672             } else {
673                 try (FileUtils.CollocatedTempFile tempFile = FileUtils.newTempFile(dataFile.toPath())) {
674                     task.setDataFile(tempFile.getPath().toFile(), resume);
675                     if (resume && Files.isRegularFile(dataFile.toPath())) {
676                         try (InputStream inputStream = Files.newInputStream(dataFile.toPath())) {
677                             Files.copy(inputStream, tempFile.getPath(), StandardCopyOption.REPLACE_EXISTING);
678                         }
679                     }
680                     try (InputStream is = entity.getContent()) {
681                         utilGet(task, is, true, length, resume);
682                     }
683                     tempFile.move();
684                 } finally {
685                     task.setDataFile(dataFile);
686                 }
687             }
688             if (task.getDataFile() != null) {
689                 Header lastModifiedHeader =
690                         response.getFirstHeader(HttpHeaders.LAST_MODIFIED); // note: Wagon also does first not last
691                 if (lastModifiedHeader != null) {
692                     Date lastModified = DateUtils.parseDate(lastModifiedHeader.getValue());
693                     if (lastModified != null) {
694                         Files.setLastModifiedTime(
695                                 task.getDataFile().toPath(), FileTime.fromMillis(lastModified.getTime()));
696                     }
697                 }
698             }
699             extractChecksums(response);
700         }
701 
702         private void extractChecksums(CloseableHttpResponse response) {
703             for (Map.Entry<String, ChecksumExtractor> extractorEntry : checksumExtractors.entrySet()) {
704                 Map<String, String> checksums = extractorEntry.getValue().extractChecksums(response);
705                 if (checksums != null) {
706                     checksums.forEach(task::setChecksum);
707                     return;
708                 }
709             }
710         }
711     }
712 
713     private class PutTaskEntity extends AbstractHttpEntity {
714 
715         private final PutTask task;
716 
717         PutTaskEntity(PutTask task) {
718             this.task = task;
719         }
720 
721         @Override
722         public boolean isRepeatable() {
723             return true;
724         }
725 
726         @Override
727         public boolean isStreaming() {
728             return false;
729         }
730 
731         @Override
732         public long getContentLength() {
733             return task.getDataLength();
734         }
735 
736         @Override
737         public InputStream getContent() throws IOException {
738             return task.newInputStream();
739         }
740 
741         @Override
742         public void writeTo(OutputStream os) throws IOException {
743             try {
744                 utilPut(task, os, false);
745             } catch (TransferCancelledException e) {
746                 throw (IOException) new InterruptedIOException().initCause(e);
747             }
748         }
749     }
750 
751     private static class ResolverServiceUnavailableRetryStrategy implements ServiceUnavailableRetryStrategy {
752         private final int retryCount;
753 
754         private final long retryInterval;
755 
756         private final long retryIntervalMax;
757 
758         private final Set<Integer> serviceUnavailableHttpCodes;
759 
760         /**
761          * Ugly, but forced by HttpClient API {@link ServiceUnavailableRetryStrategy}: the calls for
762          * {@link #retryRequest(HttpResponse, int, HttpContext)} and {@link #getRetryInterval()} are done by same
763          * thread and are actually done from spot that are very close to each other (almost subsequent calls).
764          */
765         private static final ThreadLocal<Long> RETRY_INTERVAL_HOLDER = new ThreadLocal<>();
766 
767         private ResolverServiceUnavailableRetryStrategy(
768                 int retryCount, long retryInterval, long retryIntervalMax, Set<Integer> serviceUnavailableHttpCodes) {
769             if (retryCount < 0) {
770                 throw new IllegalArgumentException("retryCount must be >= 0");
771             }
772             if (retryInterval < 0L) {
773                 throw new IllegalArgumentException("retryInterval must be >= 0");
774             }
775             if (retryIntervalMax < 0L) {
776                 throw new IllegalArgumentException("retryIntervalMax must be >= 0");
777             }
778             this.retryCount = retryCount;
779             this.retryInterval = retryInterval;
780             this.retryIntervalMax = retryIntervalMax;
781             this.serviceUnavailableHttpCodes = requireNonNull(serviceUnavailableHttpCodes);
782         }
783 
784         @Override
785         public boolean retryRequest(HttpResponse response, int executionCount, HttpContext context) {
786             final boolean retry = executionCount <= retryCount
787                     && (serviceUnavailableHttpCodes.contains(
788                             response.getStatusLine().getStatusCode()));
789             if (retry) {
790                 Long retryInterval = retryInterval(response, executionCount, context);
791                 if (retryInterval != null) {
792                     RETRY_INTERVAL_HOLDER.set(retryInterval);
793                     return true;
794                 }
795             }
796             RETRY_INTERVAL_HOLDER.remove();
797             return false;
798         }
799 
800         /**
801          * Calculates retry interval in milliseconds. If {@link HttpHeaders#RETRY_AFTER} header present, it obeys it.
802          * Otherwise, it returns {@link this#retryInterval} long value multiplied with {@code executionCount} (starts
803          * from 1 and goes 2, 3,...).
804          *
805          * @return Long representing the retry interval as millis, or {@code null} if the request should be failed.
806          */
807         private Long retryInterval(HttpResponse httpResponse, int executionCount, HttpContext httpContext) {
808             Long result = null;
809             Header header = httpResponse.getFirstHeader(HttpHeaders.RETRY_AFTER);
810             if (header != null && header.getValue() != null) {
811                 String headerValue = header.getValue();
812                 if (headerValue.contains(":")) { // is date when to retry
813                     Date when = DateUtils.parseDate(headerValue); // presumably future
814                     if (when != null) {
815                         result = Math.max(when.getTime() - System.currentTimeMillis(), 0L);
816                     }
817                 } else {
818                     try {
819                         result = Long.parseLong(headerValue) * 1000L; // is in seconds
820                     } catch (NumberFormatException e) {
821                         // fall through
822                     }
823                 }
824             }
825             if (result == null) {
826                 result = executionCount * this.retryInterval;
827             }
828             if (result > retryIntervalMax) {
829                 return null;
830             }
831             return result;
832         }
833 
834         @Override
835         public long getRetryInterval() {
836             Long ri = RETRY_INTERVAL_HOLDER.get();
837             if (ri == null) {
838                 return 0L;
839             }
840             RETRY_INTERVAL_HOLDER.remove();
841             return ri;
842         }
843     }
844 }