1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.eclipse.aether.transport.http;
20
21 import java.io.File;
22 import java.io.IOException;
23 import java.io.InputStream;
24 import java.io.InterruptedIOException;
25 import java.io.OutputStream;
26 import java.io.UncheckedIOException;
27 import java.net.InetAddress;
28 import java.net.URI;
29 import java.net.URISyntaxException;
30 import java.net.UnknownHostException;
31 import java.nio.charset.Charset;
32 import java.nio.file.Files;
33 import java.nio.file.StandardCopyOption;
34 import java.nio.file.attribute.FileTime;
35 import java.util.Collections;
36 import java.util.Date;
37 import java.util.HashSet;
38 import java.util.List;
39 import java.util.Map;
40 import java.util.Set;
41 import java.util.regex.Matcher;
42 import java.util.regex.Pattern;
43
44 import org.apache.http.Header;
45 import org.apache.http.HttpEntity;
46 import org.apache.http.HttpEntityEnclosingRequest;
47 import org.apache.http.HttpHeaders;
48 import org.apache.http.HttpHost;
49 import org.apache.http.HttpResponse;
50 import org.apache.http.HttpStatus;
51 import org.apache.http.auth.AuthSchemeProvider;
52 import org.apache.http.auth.AuthScope;
53 import org.apache.http.client.CredentialsProvider;
54 import org.apache.http.client.HttpRequestRetryHandler;
55 import org.apache.http.client.HttpResponseException;
56 import org.apache.http.client.ServiceUnavailableRetryStrategy;
57 import org.apache.http.client.config.AuthSchemes;
58 import org.apache.http.client.config.RequestConfig;
59 import org.apache.http.client.methods.CloseableHttpResponse;
60 import org.apache.http.client.methods.HttpGet;
61 import org.apache.http.client.methods.HttpHead;
62 import org.apache.http.client.methods.HttpOptions;
63 import org.apache.http.client.methods.HttpPut;
64 import org.apache.http.client.methods.HttpUriRequest;
65 import org.apache.http.client.utils.DateUtils;
66 import org.apache.http.client.utils.URIUtils;
67 import org.apache.http.config.Registry;
68 import org.apache.http.config.RegistryBuilder;
69 import org.apache.http.config.SocketConfig;
70 import org.apache.http.entity.AbstractHttpEntity;
71 import org.apache.http.entity.ByteArrayEntity;
72 import org.apache.http.impl.NoConnectionReuseStrategy;
73 import org.apache.http.impl.auth.BasicScheme;
74 import org.apache.http.impl.auth.BasicSchemeFactory;
75 import org.apache.http.impl.auth.DigestSchemeFactory;
76 import org.apache.http.impl.auth.KerberosSchemeFactory;
77 import org.apache.http.impl.auth.NTLMSchemeFactory;
78 import org.apache.http.impl.auth.SPNegoSchemeFactory;
79 import org.apache.http.impl.client.CloseableHttpClient;
80 import org.apache.http.impl.client.DefaultHttpRequestRetryHandler;
81 import org.apache.http.impl.client.HttpClientBuilder;
82 import org.apache.http.impl.client.StandardHttpRequestRetryHandler;
83 import org.apache.http.protocol.HttpContext;
84 import org.apache.http.util.EntityUtils;
85 import org.eclipse.aether.ConfigurationProperties;
86 import org.eclipse.aether.RepositorySystemSession;
87 import org.eclipse.aether.repository.AuthenticationContext;
88 import org.eclipse.aether.repository.Proxy;
89 import org.eclipse.aether.repository.RemoteRepository;
90 import org.eclipse.aether.spi.connector.transport.AbstractTransporter;
91 import org.eclipse.aether.spi.connector.transport.GetTask;
92 import org.eclipse.aether.spi.connector.transport.PeekTask;
93 import org.eclipse.aether.spi.connector.transport.PutTask;
94 import org.eclipse.aether.spi.connector.transport.TransportTask;
95 import org.eclipse.aether.transfer.NoTransporterException;
96 import org.eclipse.aether.transfer.TransferCancelledException;
97 import org.eclipse.aether.util.ConfigUtils;
98 import org.eclipse.aether.util.FileUtils;
99 import org.slf4j.Logger;
100 import org.slf4j.LoggerFactory;
101
102 import static java.util.Objects.requireNonNull;
103
104
105
106
107 final class HttpTransporter extends AbstractTransporter {
108
109 static final String BIND_ADDRESS = "aether.connector.bind.address";
110
111 static final String SUPPORT_WEBDAV = "aether.connector.http.supportWebDav";
112
113 static final String PREEMPTIVE_PUT_AUTH = "aether.connector.http.preemptivePutAuth";
114
115 static final String USE_SYSTEM_PROPERTIES = "aether.connector.http.useSystemProperties";
116
117 static final String HTTP_RETRY_HANDLER_NAME = "aether.connector.http.retryHandler.name";
118
119 private static final String HTTP_RETRY_HANDLER_NAME_STANDARD = "standard";
120
121 private static final String HTTP_RETRY_HANDLER_NAME_DEFAULT = "default";
122
123 static final String HTTP_RETRY_HANDLER_REQUEST_SENT_ENABLED =
124 "aether.connector.http.retryHandler.requestSentEnabled";
125
126 private static final Pattern CONTENT_RANGE_PATTERN =
127 Pattern.compile("\\s*bytes\\s+([0-9]+)\\s*-\\s*([0-9]+)\\s*/.*");
128
129 private static final Logger LOGGER = LoggerFactory.getLogger(HttpTransporter.class);
130
131 private final Map<String, ChecksumExtractor> checksumExtractors;
132
133 private final AuthenticationContext repoAuthContext;
134
135 private final AuthenticationContext proxyAuthContext;
136
137 private final URI baseUri;
138
139 private final HttpHost server;
140
141 private final HttpHost proxy;
142
143 private final CloseableHttpClient client;
144
145 private final Map<?, ?> headers;
146
147 private final LocalState state;
148
149 private final boolean preemptiveAuth;
150
151 private final boolean preemptivePutAuth;
152
153 private final boolean supportWebDav;
154
155 @SuppressWarnings("checkstyle:methodlength")
156 HttpTransporter(
157 Map<String, ChecksumExtractor> checksumExtractors,
158 RemoteRepository repository,
159 RepositorySystemSession session)
160 throws NoTransporterException {
161 if (!"http".equalsIgnoreCase(repository.getProtocol()) && !"https".equalsIgnoreCase(repository.getProtocol())) {
162 throw new NoTransporterException(repository);
163 }
164 this.checksumExtractors = requireNonNull(checksumExtractors, "checksum extractors must not be null");
165 try {
166 this.baseUri = new URI(repository.getUrl()).parseServerAuthority();
167 if (baseUri.isOpaque()) {
168 throw new URISyntaxException(repository.getUrl(), "URL must not be opaque");
169 }
170 this.server = URIUtils.extractHost(baseUri);
171 if (server == null) {
172 throw new URISyntaxException(repository.getUrl(), "URL lacks host name");
173 }
174 } catch (URISyntaxException e) {
175 throw new NoTransporterException(repository, e.getMessage(), e);
176 }
177 this.proxy = toHost(repository.getProxy());
178
179 this.repoAuthContext = AuthenticationContext.forRepository(session, repository);
180 this.proxyAuthContext = AuthenticationContext.forProxy(session, repository);
181
182 String httpsSecurityMode = ConfigUtils.getString(
183 session,
184 ConfigurationProperties.HTTPS_SECURITY_MODE_DEFAULT,
185 ConfigurationProperties.HTTPS_SECURITY_MODE + "." + repository.getId(),
186 ConfigurationProperties.HTTPS_SECURITY_MODE);
187 final int connectionMaxTtlSeconds = ConfigUtils.getInteger(
188 session,
189 ConfigurationProperties.DEFAULT_HTTP_CONNECTION_MAX_TTL,
190 ConfigurationProperties.HTTP_CONNECTION_MAX_TTL + "." + repository.getId(),
191 ConfigurationProperties.HTTP_CONNECTION_MAX_TTL);
192 final int maxConnectionsPerRoute = ConfigUtils.getInteger(
193 session,
194 ConfigurationProperties.DEFAULT_HTTP_MAX_CONNECTIONS_PER_ROUTE,
195 ConfigurationProperties.HTTP_MAX_CONNECTIONS_PER_ROUTE + "." + repository.getId(),
196 ConfigurationProperties.HTTP_MAX_CONNECTIONS_PER_ROUTE);
197 this.state = new LocalState(
198 session,
199 repository,
200 new ConnMgrConfig(
201 session, repoAuthContext, httpsSecurityMode, connectionMaxTtlSeconds, maxConnectionsPerRoute));
202
203 this.headers = ConfigUtils.getMap(
204 session,
205 Collections.emptyMap(),
206 ConfigurationProperties.HTTP_HEADERS + "." + repository.getId(),
207 ConfigurationProperties.HTTP_HEADERS);
208
209 this.preemptiveAuth = ConfigUtils.getBoolean(
210 session,
211 ConfigurationProperties.DEFAULT_HTTP_PREEMPTIVE_AUTH,
212 ConfigurationProperties.HTTP_PREEMPTIVE_AUTH + "." + repository.getId(),
213 ConfigurationProperties.HTTP_PREEMPTIVE_AUTH);
214 this.preemptivePutAuth =
215 ConfigUtils.getBoolean(
216 session, true, PREEMPTIVE_PUT_AUTH + "." + repository.getId(), PREEMPTIVE_PUT_AUTH);
217 this.supportWebDav =
218 ConfigUtils.getBoolean(session, false, SUPPORT_WEBDAV + "." + repository.getId(), SUPPORT_WEBDAV);
219 String credentialEncoding = ConfigUtils.getString(
220 session,
221 ConfigurationProperties.DEFAULT_HTTP_CREDENTIAL_ENCODING,
222 ConfigurationProperties.HTTP_CREDENTIAL_ENCODING + "." + repository.getId(),
223 ConfigurationProperties.HTTP_CREDENTIAL_ENCODING);
224 int connectTimeout = ConfigUtils.getInteger(
225 session,
226 ConfigurationProperties.DEFAULT_CONNECT_TIMEOUT,
227 ConfigurationProperties.CONNECT_TIMEOUT + "." + repository.getId(),
228 ConfigurationProperties.CONNECT_TIMEOUT);
229 int requestTimeout = ConfigUtils.getInteger(
230 session,
231 ConfigurationProperties.DEFAULT_REQUEST_TIMEOUT,
232 ConfigurationProperties.REQUEST_TIMEOUT + "." + repository.getId(),
233 ConfigurationProperties.REQUEST_TIMEOUT);
234 int retryCount = ConfigUtils.getInteger(
235 session,
236 ConfigurationProperties.DEFAULT_HTTP_RETRY_HANDLER_COUNT,
237 ConfigurationProperties.HTTP_RETRY_HANDLER_COUNT + "." + repository.getId(),
238 ConfigurationProperties.HTTP_RETRY_HANDLER_COUNT);
239 long retryInterval = ConfigUtils.getLong(
240 session,
241 ConfigurationProperties.DEFAULT_HTTP_RETRY_HANDLER_INTERVAL,
242 ConfigurationProperties.HTTP_RETRY_HANDLER_INTERVAL + "." + repository.getId(),
243 ConfigurationProperties.HTTP_RETRY_HANDLER_INTERVAL);
244 long retryIntervalMax = ConfigUtils.getLong(
245 session,
246 ConfigurationProperties.DEFAULT_HTTP_RETRY_HANDLER_INTERVAL_MAX,
247 ConfigurationProperties.HTTP_RETRY_HANDLER_INTERVAL_MAX + "." + repository.getId(),
248 ConfigurationProperties.HTTP_RETRY_HANDLER_INTERVAL_MAX);
249 String serviceUnavailableCodesString = ConfigUtils.getString(
250 session,
251 ConfigurationProperties.DEFAULT_HTTP_RETRY_HANDLER_SERVICE_UNAVAILABLE,
252 ConfigurationProperties.HTTP_RETRY_HANDLER_SERVICE_UNAVAILABLE + "." + repository.getId(),
253 ConfigurationProperties.HTTP_RETRY_HANDLER_SERVICE_UNAVAILABLE);
254 String retryHandlerName = ConfigUtils.getString(
255 session,
256 HTTP_RETRY_HANDLER_NAME_STANDARD,
257 HTTP_RETRY_HANDLER_NAME + "." + repository.getId(),
258 HTTP_RETRY_HANDLER_NAME);
259 boolean retryHandlerRequestSentEnabled = ConfigUtils.getBoolean(
260 session,
261 false,
262 HTTP_RETRY_HANDLER_REQUEST_SENT_ENABLED + "." + repository.getId(),
263 HTTP_RETRY_HANDLER_REQUEST_SENT_ENABLED);
264 String userAgent = ConfigUtils.getString(
265 session, ConfigurationProperties.DEFAULT_USER_AGENT, ConfigurationProperties.USER_AGENT);
266
267 Charset credentialsCharset = Charset.forName(credentialEncoding);
268 Registry<AuthSchemeProvider> authSchemeRegistry = RegistryBuilder.<AuthSchemeProvider>create()
269 .register(AuthSchemes.BASIC, new BasicSchemeFactory(credentialsCharset))
270 .register(AuthSchemes.DIGEST, new DigestSchemeFactory(credentialsCharset))
271 .register(AuthSchemes.NTLM, new NTLMSchemeFactory())
272 .register(AuthSchemes.SPNEGO, new SPNegoSchemeFactory())
273 .register(AuthSchemes.KERBEROS, new KerberosSchemeFactory())
274 .build();
275 SocketConfig socketConfig =
276 SocketConfig.custom().setSoTimeout(requestTimeout).build();
277 RequestConfig requestConfig = RequestConfig.custom()
278 .setConnectTimeout(connectTimeout)
279 .setConnectionRequestTimeout(connectTimeout)
280 .setLocalAddress(getBindAddress(session, repository))
281 .setSocketTimeout(requestTimeout)
282 .build();
283
284 HttpRequestRetryHandler retryHandler;
285 if (HTTP_RETRY_HANDLER_NAME_STANDARD.equals(retryHandlerName)) {
286 retryHandler = new StandardHttpRequestRetryHandler(retryCount, retryHandlerRequestSentEnabled);
287 } else if (HTTP_RETRY_HANDLER_NAME_DEFAULT.equals(retryHandlerName)) {
288 retryHandler = new DefaultHttpRequestRetryHandler(retryCount, retryHandlerRequestSentEnabled);
289 } else {
290 throw new IllegalArgumentException(
291 "Unsupported parameter " + HTTP_RETRY_HANDLER_NAME + " value: " + retryHandlerName);
292 }
293 Set<Integer> serviceUnavailableCodes = new HashSet<>();
294 try {
295 for (String code : ConfigUtils.parseCommaSeparatedUniqueNames(serviceUnavailableCodesString)) {
296 serviceUnavailableCodes.add(Integer.parseInt(code));
297 }
298 } catch (NumberFormatException e) {
299 throw new IllegalArgumentException(
300 "Illegal HTTP codes for " + ConfigurationProperties.HTTP_RETRY_HANDLER_SERVICE_UNAVAILABLE
301 + " (list of integers): " + serviceUnavailableCodesString);
302 }
303 ServiceUnavailableRetryStrategy serviceUnavailableRetryStrategy = new ResolverServiceUnavailableRetryStrategy(
304 retryCount, retryInterval, retryIntervalMax, serviceUnavailableCodes);
305
306 HttpClientBuilder builder = HttpClientBuilder.create()
307 .setUserAgent(userAgent)
308 .setDefaultSocketConfig(socketConfig)
309 .setDefaultRequestConfig(requestConfig)
310 .setServiceUnavailableRetryStrategy(serviceUnavailableRetryStrategy)
311 .setRetryHandler(retryHandler)
312 .setDefaultAuthSchemeRegistry(authSchemeRegistry)
313 .setConnectionManager(state.getConnectionManager())
314 .setConnectionManagerShared(true)
315 .setDefaultCredentialsProvider(toCredentialsProvider(server, repoAuthContext, proxy, proxyAuthContext))
316 .setProxy(proxy);
317 final boolean useSystemProperties = ConfigUtils.getBoolean(
318 session, false, USE_SYSTEM_PROPERTIES + "." + repository.getId(), USE_SYSTEM_PROPERTIES);
319 if (useSystemProperties) {
320 LOGGER.warn(
321 "Transport used Apache HttpClient is instructed to use system properties: this may yield in unwanted side-effects!");
322 LOGGER.warn("Please use documented means to configure resolver transport.");
323 builder.useSystemProperties();
324 }
325
326 final String expectContinue = ConfigUtils.getString(
327 session,
328 null,
329 ConfigurationProperties.HTTP_EXPECT_CONTINUE + "." + repository.getId(),
330 ConfigurationProperties.HTTP_EXPECT_CONTINUE);
331 if (expectContinue != null) {
332 state.setExpectContinue(Boolean.parseBoolean(expectContinue));
333 }
334
335 final boolean reuseConnections = ConfigUtils.getBoolean(
336 session,
337 ConfigurationProperties.DEFAULT_HTTP_REUSE_CONNECTIONS,
338 ConfigurationProperties.HTTP_REUSE_CONNECTIONS + "." + repository.getId(),
339 ConfigurationProperties.HTTP_REUSE_CONNECTIONS);
340 if (!reuseConnections) {
341 builder.setConnectionReuseStrategy(NoConnectionReuseStrategy.INSTANCE);
342 }
343
344 this.client = builder.build();
345 }
346
347
348
349
350 private InetAddress getBindAddress(RepositorySystemSession session, RemoteRepository repository) {
351 String bindAddress =
352 ConfigUtils.getString(session, null, BIND_ADDRESS + "." + repository.getId(), BIND_ADDRESS);
353 if (bindAddress == null) {
354 return null;
355 }
356 try {
357 return InetAddress.getByName(bindAddress);
358 } catch (UnknownHostException uhe) {
359 throw new IllegalArgumentException(
360 "Given bind address (" + bindAddress + ") cannot be resolved for remote repository " + repository,
361 uhe);
362 }
363 }
364
365 private static HttpHost toHost(Proxy proxy) {
366 HttpHost host = null;
367 if (proxy != null) {
368 host = new HttpHost(proxy.getHost(), proxy.getPort());
369 }
370 return host;
371 }
372
373 private static CredentialsProvider toCredentialsProvider(
374 HttpHost server, AuthenticationContext serverAuthCtx, HttpHost proxy, AuthenticationContext proxyAuthCtx) {
375 CredentialsProvider provider = toCredentialsProvider(server.getHostName(), AuthScope.ANY_PORT, serverAuthCtx);
376 if (proxy != null) {
377 CredentialsProvider p = toCredentialsProvider(proxy.getHostName(), proxy.getPort(), proxyAuthCtx);
378 provider = new DemuxCredentialsProvider(provider, p, proxy);
379 }
380 return provider;
381 }
382
383 private static CredentialsProvider toCredentialsProvider(String host, int port, AuthenticationContext ctx) {
384 DeferredCredentialsProvider provider = new DeferredCredentialsProvider();
385 if (ctx != null) {
386 AuthScope basicScope = new AuthScope(host, port);
387 provider.setCredentials(basicScope, new DeferredCredentialsProvider.BasicFactory(ctx));
388
389 AuthScope ntlmScope = new AuthScope(host, port, AuthScope.ANY_REALM, "ntlm");
390 provider.setCredentials(ntlmScope, new DeferredCredentialsProvider.NtlmFactory(ctx));
391 }
392 return provider;
393 }
394
395 LocalState getState() {
396 return state;
397 }
398
399 private URI resolve(TransportTask task) {
400 return UriUtils.resolve(baseUri, task.getLocation());
401 }
402
403 @Override
404 public int classify(Throwable error) {
405 if (error instanceof HttpResponseException
406 && ((HttpResponseException) error).getStatusCode() == HttpStatus.SC_NOT_FOUND) {
407 return ERROR_NOT_FOUND;
408 }
409 return ERROR_OTHER;
410 }
411
412 @Override
413 protected void implPeek(PeekTask task) throws Exception {
414 HttpHead request = commonHeaders(new HttpHead(resolve(task)));
415 execute(request, null);
416 }
417
418 @Override
419 protected void implGet(GetTask task) throws Exception {
420 boolean resume = true;
421 boolean applyChecksumExtractors = true;
422
423 EntityGetter getter = new EntityGetter(task);
424 HttpGet request = commonHeaders(new HttpGet(resolve(task)));
425 while (true) {
426 try {
427 if (resume) {
428 resume(request, task);
429 }
430 if (applyChecksumExtractors) {
431 for (ChecksumExtractor checksumExtractor : checksumExtractors.values()) {
432 checksumExtractor.prepareRequest(request);
433 }
434 }
435 execute(request, getter);
436 break;
437 } catch (HttpResponseException e) {
438 if (resume
439 && e.getStatusCode() == HttpStatus.SC_PRECONDITION_FAILED
440 && request.containsHeader(HttpHeaders.RANGE)) {
441 request = commonHeaders(new HttpGet(resolve(task)));
442 resume = false;
443 continue;
444 }
445 if (applyChecksumExtractors) {
446 boolean retryWithoutExtractors = false;
447 for (ChecksumExtractor checksumExtractor : checksumExtractors.values()) {
448 if (checksumExtractor.retryWithoutExtractor(e)) {
449 retryWithoutExtractors = true;
450 break;
451 }
452 }
453 if (retryWithoutExtractors) {
454 request = commonHeaders(new HttpGet(resolve(task)));
455 applyChecksumExtractors = false;
456 continue;
457 }
458 }
459 throw e;
460 }
461 }
462 }
463
464 @Override
465 protected void implPut(PutTask task) throws Exception {
466 PutTaskEntity entity = new PutTaskEntity(task);
467 HttpPut request = commonHeaders(entity(new HttpPut(resolve(task)), entity));
468 try {
469 execute(request, null);
470 } catch (HttpResponseException e) {
471 if (e.getStatusCode() == HttpStatus.SC_EXPECTATION_FAILED && request.containsHeader(HttpHeaders.EXPECT)) {
472 state.setExpectContinue(false);
473 request = commonHeaders(entity(new HttpPut(request.getURI()), entity));
474 execute(request, null);
475 return;
476 }
477 throw e;
478 }
479 }
480
481 private void execute(HttpUriRequest request, EntityGetter getter) throws Exception {
482 try {
483 SharingHttpContext context = new SharingHttpContext(state);
484 prepare(request, context);
485 try (CloseableHttpResponse response = client.execute(server, request, context)) {
486 try {
487 context.close();
488 handleStatus(response);
489 if (getter != null) {
490 getter.handle(response);
491 }
492 } finally {
493 EntityUtils.consumeQuietly(response.getEntity());
494 }
495 }
496 } catch (IOException e) {
497 if (e.getCause() instanceof TransferCancelledException) {
498 throw (Exception) e.getCause();
499 }
500 throw e;
501 }
502 }
503
504 private void prepare(HttpUriRequest request, SharingHttpContext context) {
505 final boolean put = HttpPut.METHOD_NAME.equalsIgnoreCase(request.getMethod());
506 if (preemptiveAuth || (preemptivePutAuth && put)) {
507 context.getAuthCache().put(server, new BasicScheme());
508 }
509 if (supportWebDav) {
510 if (state.getWebDav() == null && (put || isPayloadPresent(request))) {
511 HttpOptions req = commonHeaders(new HttpOptions(request.getURI()));
512 try (CloseableHttpResponse response = client.execute(server, req, context)) {
513 state.setWebDav(response.containsHeader(HttpHeaders.DAV));
514 EntityUtils.consumeQuietly(response.getEntity());
515 } catch (IOException e) {
516 LOGGER.debug("Failed to prepare HTTP context", e);
517 }
518 }
519 if (put && Boolean.TRUE.equals(state.getWebDav())) {
520 mkdirs(request.getURI(), context);
521 }
522 }
523 }
524
525 @SuppressWarnings("checkstyle:magicnumber")
526 private void mkdirs(URI uri, SharingHttpContext context) {
527 List<URI> dirs = UriUtils.getDirectories(baseUri, uri);
528 int index = 0;
529 for (; index < dirs.size(); index++) {
530 try (CloseableHttpResponse response =
531 client.execute(server, commonHeaders(new HttpMkCol(dirs.get(index))), context)) {
532 try {
533 int status = response.getStatusLine().getStatusCode();
534 if (status < 300 || status == HttpStatus.SC_METHOD_NOT_ALLOWED) {
535 break;
536 } else if (status == HttpStatus.SC_CONFLICT) {
537 continue;
538 }
539 handleStatus(response);
540 } finally {
541 EntityUtils.consumeQuietly(response.getEntity());
542 }
543 } catch (IOException e) {
544 LOGGER.debug("Failed to create parent directory {}", dirs.get(index), e);
545 return;
546 }
547 }
548 for (index--; index >= 0; index--) {
549 try (CloseableHttpResponse response =
550 client.execute(server, commonHeaders(new HttpMkCol(dirs.get(index))), context)) {
551 try {
552 handleStatus(response);
553 } finally {
554 EntityUtils.consumeQuietly(response.getEntity());
555 }
556 } catch (IOException e) {
557 LOGGER.debug("Failed to create parent directory {}", dirs.get(index), e);
558 return;
559 }
560 }
561 }
562
563 private <T extends HttpEntityEnclosingRequest> T entity(T request, HttpEntity entity) {
564 request.setEntity(entity);
565 return request;
566 }
567
568 private boolean isPayloadPresent(HttpUriRequest request) {
569 if (request instanceof HttpEntityEnclosingRequest) {
570 HttpEntity entity = ((HttpEntityEnclosingRequest) request).getEntity();
571 return entity != null && entity.getContentLength() != 0;
572 }
573 return false;
574 }
575
576 private <T extends HttpUriRequest> T commonHeaders(T request) {
577 request.setHeader(HttpHeaders.CACHE_CONTROL, "no-cache, no-store");
578 request.setHeader(HttpHeaders.PRAGMA, "no-cache");
579
580 if (state.isExpectContinue() && isPayloadPresent(request)) {
581 request.setHeader(HttpHeaders.EXPECT, "100-continue");
582 }
583
584 for (Map.Entry<?, ?> entry : headers.entrySet()) {
585 if (!(entry.getKey() instanceof String)) {
586 continue;
587 }
588 if (entry.getValue() instanceof String) {
589 request.setHeader(entry.getKey().toString(), entry.getValue().toString());
590 } else {
591 request.removeHeaders(entry.getKey().toString());
592 }
593 }
594
595 if (!state.isExpectContinue()) {
596 request.removeHeaders(HttpHeaders.EXPECT);
597 }
598
599 return request;
600 }
601
602 @SuppressWarnings("checkstyle:magicnumber")
603 private <T extends HttpUriRequest> T resume(T request, GetTask task) {
604 long resumeOffset = task.getResumeOffset();
605 if (resumeOffset > 0L && task.getDataFile() != null) {
606 request.setHeader(HttpHeaders.RANGE, "bytes=" + resumeOffset + '-');
607 request.setHeader(
608 HttpHeaders.IF_UNMODIFIED_SINCE,
609 DateUtils.formatDate(new Date(task.getDataFile().lastModified() - 60L * 1000L)));
610 request.setHeader(HttpHeaders.ACCEPT_ENCODING, "identity");
611 }
612 return request;
613 }
614
615 @SuppressWarnings("checkstyle:magicnumber")
616 private void handleStatus(CloseableHttpResponse response) throws HttpResponseException {
617 int status = response.getStatusLine().getStatusCode();
618 if (status >= 300) {
619 throw new HttpResponseException(status, response.getStatusLine().getReasonPhrase() + " (" + status + ")");
620 }
621 }
622
623 @Override
624 protected void implClose() {
625 try {
626 client.close();
627 } catch (IOException e) {
628 throw new UncheckedIOException(e);
629 }
630 AuthenticationContext.close(repoAuthContext);
631 AuthenticationContext.close(proxyAuthContext);
632 state.close();
633 }
634
635 private class EntityGetter {
636
637 private final GetTask task;
638
639 EntityGetter(GetTask task) {
640 this.task = task;
641 }
642
643 public void handle(CloseableHttpResponse response) throws IOException, TransferCancelledException {
644 HttpEntity entity = response.getEntity();
645 if (entity == null) {
646 entity = new ByteArrayEntity(new byte[0]);
647 }
648
649 long offset = 0L, length = entity.getContentLength();
650 Header rangeHeader = response.getFirstHeader(HttpHeaders.CONTENT_RANGE);
651 String range = rangeHeader != null ? rangeHeader.getValue() : null;
652 if (range != null) {
653 Matcher m = CONTENT_RANGE_PATTERN.matcher(range);
654 if (!m.matches()) {
655 throw new IOException("Invalid Content-Range header for partial download: " + range);
656 }
657 offset = Long.parseLong(m.group(1));
658 length = Long.parseLong(m.group(2)) + 1L;
659 if (offset < 0L || offset >= length || (offset > 0L && offset != task.getResumeOffset())) {
660 throw new IOException("Invalid Content-Range header for partial download from offset "
661 + task.getResumeOffset() + ": " + range);
662 }
663 }
664
665 final boolean resume = offset > 0L;
666 final File dataFile = task.getDataFile();
667 if (dataFile == null) {
668 try (InputStream is = entity.getContent()) {
669 utilGet(task, is, true, length, resume);
670 extractChecksums(response);
671 }
672 } else {
673 try (FileUtils.CollocatedTempFile tempFile = FileUtils.newTempFile(dataFile.toPath())) {
674 task.setDataFile(tempFile.getPath().toFile(), resume);
675 if (resume && Files.isRegularFile(dataFile.toPath())) {
676 try (InputStream inputStream = Files.newInputStream(dataFile.toPath())) {
677 Files.copy(inputStream, tempFile.getPath(), StandardCopyOption.REPLACE_EXISTING);
678 }
679 }
680 try (InputStream is = entity.getContent()) {
681 utilGet(task, is, true, length, resume);
682 }
683 tempFile.move();
684 } finally {
685 task.setDataFile(dataFile);
686 }
687 }
688 if (task.getDataFile() != null) {
689 Header lastModifiedHeader =
690 response.getFirstHeader(HttpHeaders.LAST_MODIFIED);
691 if (lastModifiedHeader != null) {
692 Date lastModified = DateUtils.parseDate(lastModifiedHeader.getValue());
693 if (lastModified != null) {
694 Files.setLastModifiedTime(
695 task.getDataFile().toPath(), FileTime.fromMillis(lastModified.getTime()));
696 }
697 }
698 }
699 extractChecksums(response);
700 }
701
702 private void extractChecksums(CloseableHttpResponse response) {
703 for (Map.Entry<String, ChecksumExtractor> extractorEntry : checksumExtractors.entrySet()) {
704 Map<String, String> checksums = extractorEntry.getValue().extractChecksums(response);
705 if (checksums != null) {
706 checksums.forEach(task::setChecksum);
707 return;
708 }
709 }
710 }
711 }
712
713 private class PutTaskEntity extends AbstractHttpEntity {
714
715 private final PutTask task;
716
717 PutTaskEntity(PutTask task) {
718 this.task = task;
719 }
720
721 @Override
722 public boolean isRepeatable() {
723 return true;
724 }
725
726 @Override
727 public boolean isStreaming() {
728 return false;
729 }
730
731 @Override
732 public long getContentLength() {
733 return task.getDataLength();
734 }
735
736 @Override
737 public InputStream getContent() throws IOException {
738 return task.newInputStream();
739 }
740
741 @Override
742 public void writeTo(OutputStream os) throws IOException {
743 try {
744 utilPut(task, os, false);
745 } catch (TransferCancelledException e) {
746 throw (IOException) new InterruptedIOException().initCause(e);
747 }
748 }
749 }
750
751 private static class ResolverServiceUnavailableRetryStrategy implements ServiceUnavailableRetryStrategy {
752 private final int retryCount;
753
754 private final long retryInterval;
755
756 private final long retryIntervalMax;
757
758 private final Set<Integer> serviceUnavailableHttpCodes;
759
760
761
762
763
764
765 private static final ThreadLocal<Long> RETRY_INTERVAL_HOLDER = new ThreadLocal<>();
766
767 private ResolverServiceUnavailableRetryStrategy(
768 int retryCount, long retryInterval, long retryIntervalMax, Set<Integer> serviceUnavailableHttpCodes) {
769 if (retryCount < 0) {
770 throw new IllegalArgumentException("retryCount must be >= 0");
771 }
772 if (retryInterval < 0L) {
773 throw new IllegalArgumentException("retryInterval must be >= 0");
774 }
775 if (retryIntervalMax < 0L) {
776 throw new IllegalArgumentException("retryIntervalMax must be >= 0");
777 }
778 this.retryCount = retryCount;
779 this.retryInterval = retryInterval;
780 this.retryIntervalMax = retryIntervalMax;
781 this.serviceUnavailableHttpCodes = requireNonNull(serviceUnavailableHttpCodes);
782 }
783
784 @Override
785 public boolean retryRequest(HttpResponse response, int executionCount, HttpContext context) {
786 final boolean retry = executionCount <= retryCount
787 && (serviceUnavailableHttpCodes.contains(
788 response.getStatusLine().getStatusCode()));
789 if (retry) {
790 Long retryInterval = retryInterval(response, executionCount, context);
791 if (retryInterval != null) {
792 RETRY_INTERVAL_HOLDER.set(retryInterval);
793 return true;
794 }
795 }
796 RETRY_INTERVAL_HOLDER.remove();
797 return false;
798 }
799
800
801
802
803
804
805
806
807 private Long retryInterval(HttpResponse httpResponse, int executionCount, HttpContext httpContext) {
808 Long result = null;
809 Header header = httpResponse.getFirstHeader(HttpHeaders.RETRY_AFTER);
810 if (header != null && header.getValue() != null) {
811 String headerValue = header.getValue();
812 if (headerValue.contains(":")) {
813 Date when = DateUtils.parseDate(headerValue);
814 if (when != null) {
815 result = Math.max(when.getTime() - System.currentTimeMillis(), 0L);
816 }
817 } else {
818 try {
819 result = Long.parseLong(headerValue) * 1000L;
820 } catch (NumberFormatException e) {
821
822 }
823 }
824 }
825 if (result == null) {
826 result = executionCount * this.retryInterval;
827 }
828 if (result > retryIntervalMax) {
829 return null;
830 }
831 return result;
832 }
833
834 @Override
835 public long getRetryInterval() {
836 Long ri = RETRY_INTERVAL_HOLDER.get();
837 if (ri == null) {
838 return 0L;
839 }
840 RETRY_INTERVAL_HOLDER.remove();
841 return ri;
842 }
843 }
844 }