1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.eclipse.aether.transport.apache;
20
21 import java.io.File;
22 import java.io.IOException;
23 import java.io.InputStream;
24 import java.io.InterruptedIOException;
25 import java.io.OutputStream;
26 import java.io.UncheckedIOException;
27 import java.net.InetAddress;
28 import java.net.URI;
29 import java.net.URISyntaxException;
30 import java.net.UnknownHostException;
31 import java.nio.charset.Charset;
32 import java.nio.file.Files;
33 import java.nio.file.StandardCopyOption;
34 import java.nio.file.attribute.FileTime;
35 import java.util.Collections;
36 import java.util.Date;
37 import java.util.HashSet;
38 import java.util.List;
39 import java.util.Map;
40 import java.util.Set;
41 import java.util.function.Function;
42 import java.util.regex.Matcher;
43
44 import org.apache.http.Header;
45 import org.apache.http.HttpEntity;
46 import org.apache.http.HttpEntityEnclosingRequest;
47 import org.apache.http.HttpHeaders;
48 import org.apache.http.HttpHost;
49 import org.apache.http.HttpResponse;
50 import org.apache.http.HttpStatus;
51 import org.apache.http.auth.AuthSchemeProvider;
52 import org.apache.http.auth.AuthScope;
53 import org.apache.http.client.CredentialsProvider;
54 import org.apache.http.client.HttpRequestRetryHandler;
55 import org.apache.http.client.HttpResponseException;
56 import org.apache.http.client.ServiceUnavailableRetryStrategy;
57 import org.apache.http.client.config.AuthSchemes;
58 import org.apache.http.client.config.RequestConfig;
59 import org.apache.http.client.methods.CloseableHttpResponse;
60 import org.apache.http.client.methods.HttpGet;
61 import org.apache.http.client.methods.HttpHead;
62 import org.apache.http.client.methods.HttpOptions;
63 import org.apache.http.client.methods.HttpPut;
64 import org.apache.http.client.methods.HttpUriRequest;
65 import org.apache.http.client.utils.DateUtils;
66 import org.apache.http.client.utils.URIUtils;
67 import org.apache.http.config.Registry;
68 import org.apache.http.config.RegistryBuilder;
69 import org.apache.http.config.SocketConfig;
70 import org.apache.http.entity.AbstractHttpEntity;
71 import org.apache.http.entity.ByteArrayEntity;
72 import org.apache.http.impl.NoConnectionReuseStrategy;
73 import org.apache.http.impl.auth.BasicScheme;
74 import org.apache.http.impl.auth.BasicSchemeFactory;
75 import org.apache.http.impl.auth.DigestSchemeFactory;
76 import org.apache.http.impl.auth.KerberosSchemeFactory;
77 import org.apache.http.impl.auth.NTLMSchemeFactory;
78 import org.apache.http.impl.auth.SPNegoSchemeFactory;
79 import org.apache.http.impl.client.CloseableHttpClient;
80 import org.apache.http.impl.client.DefaultHttpRequestRetryHandler;
81 import org.apache.http.impl.client.HttpClientBuilder;
82 import org.apache.http.impl.client.StandardHttpRequestRetryHandler;
83 import org.apache.http.protocol.HttpContext;
84 import org.apache.http.util.EntityUtils;
85 import org.eclipse.aether.ConfigurationProperties;
86 import org.eclipse.aether.RepositorySystemSession;
87 import org.eclipse.aether.repository.AuthenticationContext;
88 import org.eclipse.aether.repository.Proxy;
89 import org.eclipse.aether.repository.RemoteRepository;
90 import org.eclipse.aether.spi.connector.transport.AbstractTransporter;
91 import org.eclipse.aether.spi.connector.transport.GetTask;
92 import org.eclipse.aether.spi.connector.transport.PeekTask;
93 import org.eclipse.aether.spi.connector.transport.PutTask;
94 import org.eclipse.aether.spi.connector.transport.TransportTask;
95 import org.eclipse.aether.spi.connector.transport.http.ChecksumExtractor;
96 import org.eclipse.aether.spi.connector.transport.http.HttpTransporter;
97 import org.eclipse.aether.spi.connector.transport.http.HttpTransporterException;
98 import org.eclipse.aether.transfer.NoTransporterException;
99 import org.eclipse.aether.transfer.TransferCancelledException;
100 import org.eclipse.aether.util.ConfigUtils;
101 import org.eclipse.aether.util.FileUtils;
102 import org.slf4j.Logger;
103 import org.slf4j.LoggerFactory;
104
105 import static java.util.Objects.requireNonNull;
106 import static org.eclipse.aether.spi.connector.transport.http.HttpConstants.CONTENT_RANGE_PATTERN;
107 import static org.eclipse.aether.transport.apache.ApacheTransporterConfigurationKeys.CONFIG_PROP_HTTP_RETRY_HANDLER_NAME;
108 import static org.eclipse.aether.transport.apache.ApacheTransporterConfigurationKeys.CONFIG_PROP_HTTP_RETRY_HANDLER_REQUEST_SENT_ENABLED;
109 import static org.eclipse.aether.transport.apache.ApacheTransporterConfigurationKeys.CONFIG_PROP_USE_SYSTEM_PROPERTIES;
110 import static org.eclipse.aether.transport.apache.ApacheTransporterConfigurationKeys.DEFAULT_HTTP_RETRY_HANDLER_REQUEST_SENT_ENABLED;
111 import static org.eclipse.aether.transport.apache.ApacheTransporterConfigurationKeys.DEFAULT_USE_SYSTEM_PROPERTIES;
112 import static org.eclipse.aether.transport.apache.ApacheTransporterConfigurationKeys.HTTP_RETRY_HANDLER_NAME_DEFAULT;
113 import static org.eclipse.aether.transport.apache.ApacheTransporterConfigurationKeys.HTTP_RETRY_HANDLER_NAME_STANDARD;
114
115
116
117
118 final class ApacheTransporter extends AbstractTransporter implements HttpTransporter {
119 private static final Logger LOGGER = LoggerFactory.getLogger(ApacheTransporter.class);
120
121 private final ChecksumExtractor checksumExtractor;
122
123 private final AuthenticationContext repoAuthContext;
124
125 private final AuthenticationContext proxyAuthContext;
126
127 private final URI baseUri;
128
129 private final HttpHost server;
130
131 private final HttpHost proxy;
132
133 private final CloseableHttpClient client;
134
135 private final Map<?, ?> headers;
136
137 private final LocalState state;
138
139 private final boolean preemptiveAuth;
140
141 private final boolean preemptivePutAuth;
142
143 private final boolean supportWebDav;
144
145 @SuppressWarnings("checkstyle:methodlength")
146 ApacheTransporter(RemoteRepository repository, RepositorySystemSession session, ChecksumExtractor checksumExtractor)
147 throws NoTransporterException {
148 if (!"http".equalsIgnoreCase(repository.getProtocol()) && !"https".equalsIgnoreCase(repository.getProtocol())) {
149 throw new NoTransporterException(repository);
150 }
151 this.checksumExtractor = checksumExtractor;
152 try {
153 this.baseUri = new URI(repository.getUrl()).parseServerAuthority();
154 if (baseUri.isOpaque()) {
155 throw new URISyntaxException(repository.getUrl(), "URL must not be opaque");
156 }
157 this.server = URIUtils.extractHost(baseUri);
158 if (server == null) {
159 throw new URISyntaxException(repository.getUrl(), "URL lacks host name");
160 }
161 } catch (URISyntaxException e) {
162 throw new NoTransporterException(repository, e.getMessage(), e);
163 }
164 this.proxy = toHost(repository.getProxy());
165
166 this.repoAuthContext = AuthenticationContext.forRepository(session, repository);
167 this.proxyAuthContext = AuthenticationContext.forProxy(session, repository);
168
169 String httpsSecurityMode = ConfigUtils.getString(
170 session,
171 ConfigurationProperties.HTTPS_SECURITY_MODE_DEFAULT,
172 ConfigurationProperties.HTTPS_SECURITY_MODE + "." + repository.getId(),
173 ConfigurationProperties.HTTPS_SECURITY_MODE);
174 final int connectionMaxTtlSeconds = ConfigUtils.getInteger(
175 session,
176 ConfigurationProperties.DEFAULT_HTTP_CONNECTION_MAX_TTL,
177 ConfigurationProperties.HTTP_CONNECTION_MAX_TTL + "." + repository.getId(),
178 ConfigurationProperties.HTTP_CONNECTION_MAX_TTL);
179 final int maxConnectionsPerRoute = ConfigUtils.getInteger(
180 session,
181 ConfigurationProperties.DEFAULT_HTTP_MAX_CONNECTIONS_PER_ROUTE,
182 ConfigurationProperties.HTTP_MAX_CONNECTIONS_PER_ROUTE + "." + repository.getId(),
183 ConfigurationProperties.HTTP_MAX_CONNECTIONS_PER_ROUTE);
184 this.state = new LocalState(
185 session,
186 repository,
187 new ConnMgrConfig(
188 session, repoAuthContext, httpsSecurityMode, connectionMaxTtlSeconds, maxConnectionsPerRoute));
189
190 this.headers = ConfigUtils.getMap(
191 session,
192 Collections.emptyMap(),
193 ConfigurationProperties.HTTP_HEADERS + "." + repository.getId(),
194 ConfigurationProperties.HTTP_HEADERS);
195
196 this.preemptiveAuth = ConfigUtils.getBoolean(
197 session,
198 ConfigurationProperties.DEFAULT_HTTP_PREEMPTIVE_AUTH,
199 ConfigurationProperties.HTTP_PREEMPTIVE_AUTH + "." + repository.getId(),
200 ConfigurationProperties.HTTP_PREEMPTIVE_AUTH);
201 this.preemptivePutAuth = ConfigUtils.getBoolean(
202 session,
203 ConfigurationProperties.DEFAULT_HTTP_PREEMPTIVE_PUT_AUTH,
204 ConfigurationProperties.HTTP_PREEMPTIVE_PUT_AUTH + "." + repository.getId(),
205 ConfigurationProperties.HTTP_PREEMPTIVE_PUT_AUTH);
206 this.supportWebDav = ConfigUtils.getBoolean(
207 session,
208 ConfigurationProperties.DEFAULT_HTTP_SUPPORT_WEBDAV,
209 ConfigurationProperties.HTTP_SUPPORT_WEBDAV + "." + repository.getId(),
210 ConfigurationProperties.HTTP_SUPPORT_WEBDAV);
211 String credentialEncoding = ConfigUtils.getString(
212 session,
213 ConfigurationProperties.DEFAULT_HTTP_CREDENTIAL_ENCODING,
214 ConfigurationProperties.HTTP_CREDENTIAL_ENCODING + "." + repository.getId(),
215 ConfigurationProperties.HTTP_CREDENTIAL_ENCODING);
216 int connectTimeout = ConfigUtils.getInteger(
217 session,
218 ConfigurationProperties.DEFAULT_CONNECT_TIMEOUT,
219 ConfigurationProperties.CONNECT_TIMEOUT + "." + repository.getId(),
220 ConfigurationProperties.CONNECT_TIMEOUT);
221 int requestTimeout = ConfigUtils.getInteger(
222 session,
223 ConfigurationProperties.DEFAULT_REQUEST_TIMEOUT,
224 ConfigurationProperties.REQUEST_TIMEOUT + "." + repository.getId(),
225 ConfigurationProperties.REQUEST_TIMEOUT);
226 int retryCount = ConfigUtils.getInteger(
227 session,
228 ConfigurationProperties.DEFAULT_HTTP_RETRY_HANDLER_COUNT,
229 ConfigurationProperties.HTTP_RETRY_HANDLER_COUNT + "." + repository.getId(),
230 ConfigurationProperties.HTTP_RETRY_HANDLER_COUNT);
231 long retryInterval = ConfigUtils.getLong(
232 session,
233 ConfigurationProperties.DEFAULT_HTTP_RETRY_HANDLER_INTERVAL,
234 ConfigurationProperties.HTTP_RETRY_HANDLER_INTERVAL + "." + repository.getId(),
235 ConfigurationProperties.HTTP_RETRY_HANDLER_INTERVAL);
236 long retryIntervalMax = ConfigUtils.getLong(
237 session,
238 ConfigurationProperties.DEFAULT_HTTP_RETRY_HANDLER_INTERVAL_MAX,
239 ConfigurationProperties.HTTP_RETRY_HANDLER_INTERVAL_MAX + "." + repository.getId(),
240 ConfigurationProperties.HTTP_RETRY_HANDLER_INTERVAL_MAX);
241 String serviceUnavailableCodesString = ConfigUtils.getString(
242 session,
243 ConfigurationProperties.DEFAULT_HTTP_RETRY_HANDLER_SERVICE_UNAVAILABLE,
244 ConfigurationProperties.HTTP_RETRY_HANDLER_SERVICE_UNAVAILABLE + "." + repository.getId(),
245 ConfigurationProperties.HTTP_RETRY_HANDLER_SERVICE_UNAVAILABLE);
246 String retryHandlerName = ConfigUtils.getString(
247 session,
248 HTTP_RETRY_HANDLER_NAME_STANDARD,
249 CONFIG_PROP_HTTP_RETRY_HANDLER_NAME + "." + repository.getId(),
250 CONFIG_PROP_HTTP_RETRY_HANDLER_NAME);
251 boolean retryHandlerRequestSentEnabled = ConfigUtils.getBoolean(
252 session,
253 DEFAULT_HTTP_RETRY_HANDLER_REQUEST_SENT_ENABLED,
254 CONFIG_PROP_HTTP_RETRY_HANDLER_REQUEST_SENT_ENABLED + "." + repository.getId(),
255 CONFIG_PROP_HTTP_RETRY_HANDLER_REQUEST_SENT_ENABLED);
256 String userAgent = ConfigUtils.getString(
257 session, ConfigurationProperties.DEFAULT_USER_AGENT, ConfigurationProperties.USER_AGENT);
258
259 Charset credentialsCharset = Charset.forName(credentialEncoding);
260 Registry<AuthSchemeProvider> authSchemeRegistry = RegistryBuilder.<AuthSchemeProvider>create()
261 .register(AuthSchemes.BASIC, new BasicSchemeFactory(credentialsCharset))
262 .register(AuthSchemes.DIGEST, new DigestSchemeFactory(credentialsCharset))
263 .register(AuthSchemes.NTLM, new NTLMSchemeFactory())
264 .register(AuthSchemes.SPNEGO, new SPNegoSchemeFactory())
265 .register(AuthSchemes.KERBEROS, new KerberosSchemeFactory())
266 .build();
267 SocketConfig socketConfig =
268 SocketConfig.custom().setSoTimeout(requestTimeout).build();
269 RequestConfig requestConfig = RequestConfig.custom()
270 .setConnectTimeout(connectTimeout)
271 .setConnectionRequestTimeout(connectTimeout)
272 .setLocalAddress(getHttpLocalAddress(session, repository))
273 .setSocketTimeout(requestTimeout)
274 .build();
275
276 HttpRequestRetryHandler retryHandler;
277 if (HTTP_RETRY_HANDLER_NAME_STANDARD.equals(retryHandlerName)) {
278 retryHandler = new StandardHttpRequestRetryHandler(retryCount, retryHandlerRequestSentEnabled);
279 } else if (HTTP_RETRY_HANDLER_NAME_DEFAULT.equals(retryHandlerName)) {
280 retryHandler = new DefaultHttpRequestRetryHandler(retryCount, retryHandlerRequestSentEnabled);
281 } else {
282 throw new IllegalArgumentException(
283 "Unsupported parameter " + CONFIG_PROP_HTTP_RETRY_HANDLER_NAME + " value: " + retryHandlerName);
284 }
285 Set<Integer> serviceUnavailableCodes = new HashSet<>();
286 try {
287 for (String code : ConfigUtils.parseCommaSeparatedUniqueNames(serviceUnavailableCodesString)) {
288 serviceUnavailableCodes.add(Integer.parseInt(code));
289 }
290 } catch (NumberFormatException e) {
291 throw new IllegalArgumentException(
292 "Illegal HTTP codes for " + ConfigurationProperties.HTTP_RETRY_HANDLER_SERVICE_UNAVAILABLE
293 + " (list of integers): " + serviceUnavailableCodesString);
294 }
295 ServiceUnavailableRetryStrategy serviceUnavailableRetryStrategy = new ResolverServiceUnavailableRetryStrategy(
296 retryCount, retryInterval, retryIntervalMax, serviceUnavailableCodes);
297
298 HttpClientBuilder builder = HttpClientBuilder.create()
299 .setUserAgent(userAgent)
300 .setDefaultSocketConfig(socketConfig)
301 .setDefaultRequestConfig(requestConfig)
302 .setServiceUnavailableRetryStrategy(serviceUnavailableRetryStrategy)
303 .setRetryHandler(retryHandler)
304 .setDefaultAuthSchemeRegistry(authSchemeRegistry)
305 .setConnectionManager(state.getConnectionManager())
306 .setConnectionManagerShared(true)
307 .setDefaultCredentialsProvider(toCredentialsProvider(server, repoAuthContext, proxy, proxyAuthContext))
308 .setProxy(proxy);
309 final boolean useSystemProperties = ConfigUtils.getBoolean(
310 session,
311 DEFAULT_USE_SYSTEM_PROPERTIES,
312 CONFIG_PROP_USE_SYSTEM_PROPERTIES + "." + repository.getId(),
313 CONFIG_PROP_USE_SYSTEM_PROPERTIES);
314 if (useSystemProperties) {
315 LOGGER.warn(
316 "Transport used Apache HttpClient is instructed to use system properties: this may yield in unwanted side-effects!");
317 LOGGER.warn("Please use documented means to configure resolver transport.");
318 builder.useSystemProperties();
319 }
320
321 final String expectContinue = ConfigUtils.getString(
322 session,
323 null,
324 ConfigurationProperties.HTTP_EXPECT_CONTINUE + "." + repository.getId(),
325 ConfigurationProperties.HTTP_EXPECT_CONTINUE);
326 if (expectContinue != null) {
327 state.setExpectContinue(Boolean.parseBoolean(expectContinue));
328 }
329
330 final boolean reuseConnections = ConfigUtils.getBoolean(
331 session,
332 ConfigurationProperties.DEFAULT_HTTP_REUSE_CONNECTIONS,
333 ConfigurationProperties.HTTP_REUSE_CONNECTIONS + "." + repository.getId(),
334 ConfigurationProperties.HTTP_REUSE_CONNECTIONS);
335 if (!reuseConnections) {
336 builder.setConnectionReuseStrategy(NoConnectionReuseStrategy.INSTANCE);
337 }
338
339 this.client = builder.build();
340 }
341
342
343
344
345 private InetAddress getHttpLocalAddress(RepositorySystemSession session, RemoteRepository repository) {
346 String bindAddress = ConfigUtils.getString(
347 session,
348 null,
349 ConfigurationProperties.HTTP_LOCAL_ADDRESS + "." + repository.getId(),
350 ConfigurationProperties.HTTP_LOCAL_ADDRESS);
351 if (bindAddress == null) {
352 return null;
353 }
354 try {
355 return InetAddress.getByName(bindAddress);
356 } catch (UnknownHostException uhe) {
357 throw new IllegalArgumentException(
358 "Given bind address (" + bindAddress + ") cannot be resolved for remote repository " + repository,
359 uhe);
360 }
361 }
362
363 private static HttpHost toHost(Proxy proxy) {
364 HttpHost host = null;
365 if (proxy != null) {
366 host = new HttpHost(proxy.getHost(), proxy.getPort());
367 }
368 return host;
369 }
370
371 private static CredentialsProvider toCredentialsProvider(
372 HttpHost server, AuthenticationContext serverAuthCtx, HttpHost proxy, AuthenticationContext proxyAuthCtx) {
373 CredentialsProvider provider = toCredentialsProvider(server.getHostName(), AuthScope.ANY_PORT, serverAuthCtx);
374 if (proxy != null) {
375 CredentialsProvider p = toCredentialsProvider(proxy.getHostName(), proxy.getPort(), proxyAuthCtx);
376 provider = new DemuxCredentialsProvider(provider, p, proxy);
377 }
378 return provider;
379 }
380
381 private static CredentialsProvider toCredentialsProvider(String host, int port, AuthenticationContext ctx) {
382 DeferredCredentialsProvider provider = new DeferredCredentialsProvider();
383 if (ctx != null) {
384 AuthScope basicScope = new AuthScope(host, port);
385 provider.setCredentials(basicScope, new DeferredCredentialsProvider.BasicFactory(ctx));
386
387 AuthScope ntlmScope = new AuthScope(host, port, AuthScope.ANY_REALM, "ntlm");
388 provider.setCredentials(ntlmScope, new DeferredCredentialsProvider.NtlmFactory(ctx));
389 }
390 return provider;
391 }
392
393 LocalState getState() {
394 return state;
395 }
396
397 private URI resolve(TransportTask task) {
398 return UriUtils.resolve(baseUri, task.getLocation());
399 }
400
401 @Override
402 public int classify(Throwable error) {
403 if (error instanceof HttpTransporterException
404 && ((HttpTransporterException) error).getStatusCode() == HttpStatus.SC_NOT_FOUND) {
405 return ERROR_NOT_FOUND;
406 }
407 return ERROR_OTHER;
408 }
409
410 @Override
411 protected void implPeek(PeekTask task) throws Exception {
412 HttpHead request = commonHeaders(new HttpHead(resolve(task)));
413 try {
414 execute(request, null);
415 } catch (HttpResponseException e) {
416 throw new HttpTransporterException(e.getStatusCode());
417 }
418 }
419
420 @Override
421 protected void implGet(GetTask task) throws Exception {
422 boolean resume = true;
423
424 EntityGetter getter = new EntityGetter(task);
425 HttpGet request = commonHeaders(new HttpGet(resolve(task)));
426 while (true) {
427 try {
428 if (resume) {
429 resume(request, task);
430 }
431 execute(request, getter);
432 break;
433 } catch (HttpResponseException e) {
434 if (resume
435 && e.getStatusCode() == HttpStatus.SC_PRECONDITION_FAILED
436 && request.containsHeader(HttpHeaders.RANGE)) {
437 request = commonHeaders(new HttpGet(resolve(task)));
438 resume = false;
439 continue;
440 }
441 throw new HttpTransporterException(e.getStatusCode());
442 }
443 }
444 }
445
446 @Override
447 protected void implPut(PutTask task) throws Exception {
448 PutTaskEntity entity = new PutTaskEntity(task);
449 HttpPut request = commonHeaders(entity(new HttpPut(resolve(task)), entity));
450 try {
451 execute(request, null);
452 } catch (HttpResponseException e) {
453 if (e.getStatusCode() == HttpStatus.SC_EXPECTATION_FAILED && request.containsHeader(HttpHeaders.EXPECT)) {
454 state.setExpectContinue(false);
455 request = commonHeaders(entity(new HttpPut(request.getURI()), entity));
456 execute(request, null);
457 return;
458 }
459 throw new HttpTransporterException(e.getStatusCode());
460 }
461 }
462
463 private void execute(HttpUriRequest request, EntityGetter getter) throws Exception {
464 try {
465 SharingHttpContext context = new SharingHttpContext(state);
466 prepare(request, context);
467 try (CloseableHttpResponse response = client.execute(server, request, context)) {
468 try {
469 context.close();
470 handleStatus(response);
471 if (getter != null) {
472 getter.handle(response);
473 }
474 } finally {
475 EntityUtils.consumeQuietly(response.getEntity());
476 }
477 }
478 } catch (IOException e) {
479 if (e.getCause() instanceof TransferCancelledException) {
480 throw (Exception) e.getCause();
481 }
482 throw e;
483 }
484 }
485
486 private void prepare(HttpUriRequest request, SharingHttpContext context) throws HttpTransporterException {
487 final boolean put = HttpPut.METHOD_NAME.equalsIgnoreCase(request.getMethod());
488 if (preemptiveAuth || (preemptivePutAuth && put)) {
489 context.getAuthCache().put(server, new BasicScheme());
490 }
491 if (supportWebDav) {
492 if (state.getWebDav() == null && (put || isPayloadPresent(request))) {
493 HttpOptions req = commonHeaders(new HttpOptions(request.getURI()));
494 try (CloseableHttpResponse response = client.execute(server, req, context)) {
495 state.setWebDav(response.containsHeader(HttpHeaders.DAV));
496 EntityUtils.consumeQuietly(response.getEntity());
497 } catch (IOException e) {
498 LOGGER.debug("Failed to prepare HTTP context", e);
499 }
500 }
501 if (put && Boolean.TRUE.equals(state.getWebDav())) {
502 mkdirs(request.getURI(), context);
503 }
504 }
505 }
506
507 @SuppressWarnings("checkstyle:magicnumber")
508 private void mkdirs(URI uri, SharingHttpContext context) throws HttpTransporterException {
509 List<URI> dirs = UriUtils.getDirectories(baseUri, uri);
510 int index = 0;
511 for (; index < dirs.size(); index++) {
512 try (CloseableHttpResponse response =
513 client.execute(server, commonHeaders(new HttpMkCol(dirs.get(index))), context)) {
514 try {
515 int status = response.getStatusLine().getStatusCode();
516 if (status < 300 || status == HttpStatus.SC_METHOD_NOT_ALLOWED) {
517 break;
518 } else if (status == HttpStatus.SC_CONFLICT) {
519 continue;
520 }
521 handleStatus(response);
522 } finally {
523 EntityUtils.consumeQuietly(response.getEntity());
524 }
525 } catch (IOException e) {
526 LOGGER.debug("Failed to create parent directory {}", dirs.get(index), e);
527 return;
528 }
529 }
530 for (index--; index >= 0; index--) {
531 try (CloseableHttpResponse response =
532 client.execute(server, commonHeaders(new HttpMkCol(dirs.get(index))), context)) {
533 try {
534 handleStatus(response);
535 } finally {
536 EntityUtils.consumeQuietly(response.getEntity());
537 }
538 } catch (IOException e) {
539 LOGGER.debug("Failed to create parent directory {}", dirs.get(index), e);
540 return;
541 }
542 }
543 }
544
545 private <T extends HttpEntityEnclosingRequest> T entity(T request, HttpEntity entity) {
546 request.setEntity(entity);
547 return request;
548 }
549
550 private boolean isPayloadPresent(HttpUriRequest request) {
551 if (request instanceof HttpEntityEnclosingRequest) {
552 HttpEntity entity = ((HttpEntityEnclosingRequest) request).getEntity();
553 return entity != null && entity.getContentLength() != 0;
554 }
555 return false;
556 }
557
558 private <T extends HttpUriRequest> T commonHeaders(T request) {
559 request.setHeader(HttpHeaders.CACHE_CONTROL, "no-cache, no-store");
560 request.setHeader(HttpHeaders.PRAGMA, "no-cache");
561
562 if (state.isExpectContinue() && isPayloadPresent(request)) {
563 request.setHeader(HttpHeaders.EXPECT, "100-continue");
564 }
565
566 for (Map.Entry<?, ?> entry : headers.entrySet()) {
567 if (!(entry.getKey() instanceof String)) {
568 continue;
569 }
570 if (entry.getValue() instanceof String) {
571 request.setHeader(entry.getKey().toString(), entry.getValue().toString());
572 } else {
573 request.removeHeaders(entry.getKey().toString());
574 }
575 }
576
577 if (!state.isExpectContinue()) {
578 request.removeHeaders(HttpHeaders.EXPECT);
579 }
580
581 return request;
582 }
583
584 @SuppressWarnings("checkstyle:magicnumber")
585 private <T extends HttpUriRequest> void resume(T request, GetTask task) {
586 long resumeOffset = task.getResumeOffset();
587 if (resumeOffset > 0L && task.getDataFile() != null) {
588 request.setHeader(HttpHeaders.RANGE, "bytes=" + resumeOffset + '-');
589 request.setHeader(
590 HttpHeaders.IF_UNMODIFIED_SINCE,
591 DateUtils.formatDate(new Date(task.getDataFile().lastModified() - 60L * 1000L)));
592 request.setHeader(HttpHeaders.ACCEPT_ENCODING, "identity");
593 }
594 }
595
596 @SuppressWarnings("checkstyle:magicnumber")
597 private void handleStatus(CloseableHttpResponse response) throws HttpResponseException {
598 int status = response.getStatusLine().getStatusCode();
599 if (status >= 300) {
600 throw new HttpResponseException(status, response.getStatusLine().getReasonPhrase() + " (" + status + ")");
601 }
602 }
603
604 @Override
605 protected void implClose() {
606 try {
607 client.close();
608 } catch (IOException e) {
609 throw new UncheckedIOException(e);
610 }
611 AuthenticationContext.close(repoAuthContext);
612 AuthenticationContext.close(proxyAuthContext);
613 state.close();
614 }
615
616 private class EntityGetter {
617
618 private final GetTask task;
619
620 EntityGetter(GetTask task) {
621 this.task = task;
622 }
623
624 public void handle(CloseableHttpResponse response) throws IOException, TransferCancelledException {
625 HttpEntity entity = response.getEntity();
626 if (entity == null) {
627 entity = new ByteArrayEntity(new byte[0]);
628 }
629
630 long offset = 0L, length = entity.getContentLength();
631 Header rangeHeader = response.getFirstHeader(HttpHeaders.CONTENT_RANGE);
632 String range = rangeHeader != null ? rangeHeader.getValue() : null;
633 if (range != null) {
634 Matcher m = CONTENT_RANGE_PATTERN.matcher(range);
635 if (!m.matches()) {
636 throw new IOException("Invalid Content-Range header for partial download: " + range);
637 }
638 offset = Long.parseLong(m.group(1));
639 length = Long.parseLong(m.group(2)) + 1L;
640 if (offset < 0L || offset >= length || (offset > 0L && offset != task.getResumeOffset())) {
641 throw new IOException("Invalid Content-Range header for partial download from offset "
642 + task.getResumeOffset() + ": " + range);
643 }
644 }
645
646 final boolean resume = offset > 0L;
647 final File dataFile = task.getDataFile();
648 if (dataFile == null) {
649 try (InputStream is = entity.getContent()) {
650 utilGet(task, is, true, length, resume);
651 extractChecksums(response);
652 }
653 } else {
654 try (FileUtils.CollocatedTempFile tempFile = FileUtils.newTempFile(dataFile.toPath())) {
655 task.setDataFile(tempFile.getPath().toFile(), resume);
656 if (resume && Files.isRegularFile(dataFile.toPath())) {
657 try (InputStream inputStream = Files.newInputStream(dataFile.toPath())) {
658 Files.copy(inputStream, tempFile.getPath(), StandardCopyOption.REPLACE_EXISTING);
659 }
660 }
661 try (InputStream is = entity.getContent()) {
662 utilGet(task, is, true, length, resume);
663 }
664 tempFile.move();
665 } finally {
666 task.setDataFile(dataFile);
667 }
668 }
669 if (task.getDataFile() != null) {
670 Header lastModifiedHeader =
671 response.getFirstHeader(HttpHeaders.LAST_MODIFIED);
672 if (lastModifiedHeader != null) {
673 Date lastModified = DateUtils.parseDate(lastModifiedHeader.getValue());
674 if (lastModified != null) {
675 Files.setLastModifiedTime(
676 task.getDataFile().toPath(), FileTime.fromMillis(lastModified.getTime()));
677 }
678 }
679 }
680 extractChecksums(response);
681 }
682
683 private void extractChecksums(CloseableHttpResponse response) {
684 Map<String, String> checksums = checksumExtractor.extractChecksums(headerGetter(response));
685 if (checksums != null && !checksums.isEmpty()) {
686 checksums.forEach(task::setChecksum);
687 }
688 }
689 }
690
691 private static Function<String, String> headerGetter(CloseableHttpResponse closeableHttpResponse) {
692 return s -> {
693 Header header = closeableHttpResponse.getFirstHeader(s);
694 return header != null ? header.getValue() : null;
695 };
696 }
697
698 private class PutTaskEntity extends AbstractHttpEntity {
699
700 private final PutTask task;
701
702 PutTaskEntity(PutTask task) {
703 this.task = task;
704 }
705
706 @Override
707 public boolean isRepeatable() {
708 return true;
709 }
710
711 @Override
712 public boolean isStreaming() {
713 return false;
714 }
715
716 @Override
717 public long getContentLength() {
718 return task.getDataLength();
719 }
720
721 @Override
722 public InputStream getContent() throws IOException {
723 return task.newInputStream();
724 }
725
726 @Override
727 public void writeTo(OutputStream os) throws IOException {
728 try {
729 utilPut(task, os, false);
730 } catch (TransferCancelledException e) {
731 throw (IOException) new InterruptedIOException().initCause(e);
732 }
733 }
734 }
735
736 private static class ResolverServiceUnavailableRetryStrategy implements ServiceUnavailableRetryStrategy {
737 private final int retryCount;
738
739 private final long retryInterval;
740
741 private final long retryIntervalMax;
742
743 private final Set<Integer> serviceUnavailableHttpCodes;
744
745
746
747
748
749
750 private static final ThreadLocal<Long> RETRY_INTERVAL_HOLDER = new ThreadLocal<>();
751
752 private ResolverServiceUnavailableRetryStrategy(
753 int retryCount, long retryInterval, long retryIntervalMax, Set<Integer> serviceUnavailableHttpCodes) {
754 if (retryCount < 0) {
755 throw new IllegalArgumentException("retryCount must be >= 0");
756 }
757 if (retryInterval < 0L) {
758 throw new IllegalArgumentException("retryInterval must be >= 0");
759 }
760 if (retryIntervalMax < 0L) {
761 throw new IllegalArgumentException("retryIntervalMax must be >= 0");
762 }
763 this.retryCount = retryCount;
764 this.retryInterval = retryInterval;
765 this.retryIntervalMax = retryIntervalMax;
766 this.serviceUnavailableHttpCodes = requireNonNull(serviceUnavailableHttpCodes);
767 }
768
769 @Override
770 public boolean retryRequest(HttpResponse response, int executionCount, HttpContext context) {
771 final boolean retry = executionCount <= retryCount
772 && (serviceUnavailableHttpCodes.contains(
773 response.getStatusLine().getStatusCode()));
774 if (retry) {
775 Long retryInterval = retryInterval(response, executionCount, context);
776 if (retryInterval != null) {
777 RETRY_INTERVAL_HOLDER.set(retryInterval);
778 return true;
779 }
780 }
781 RETRY_INTERVAL_HOLDER.remove();
782 return false;
783 }
784
785
786
787
788
789
790
791
792 private Long retryInterval(HttpResponse httpResponse, int executionCount, HttpContext httpContext) {
793 Long result = null;
794 Header header = httpResponse.getFirstHeader(HttpHeaders.RETRY_AFTER);
795 if (header != null && header.getValue() != null) {
796 String headerValue = header.getValue();
797 if (headerValue.contains(":")) {
798 Date when = DateUtils.parseDate(headerValue);
799 if (when != null) {
800 result = Math.max(when.getTime() - System.currentTimeMillis(), 0L);
801 }
802 } else {
803 try {
804 result = Long.parseLong(headerValue) * 1000L;
805 } catch (NumberFormatException e) {
806
807 }
808 }
809 }
810 if (result == null) {
811 result = executionCount * this.retryInterval;
812 }
813 if (result > retryIntervalMax) {
814 return null;
815 }
816 return result;
817 }
818
819 @Override
820 public long getRetryInterval() {
821 Long ri = RETRY_INTERVAL_HOLDER.get();
822 if (ri == null) {
823 return 0L;
824 }
825 RETRY_INTERVAL_HOLDER.remove();
826 return ri;
827 }
828 }
829 }