1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.eclipse.aether.transport.apache;
20
21 import java.io.File;
22 import java.io.IOException;
23 import java.io.InputStream;
24 import java.io.InterruptedIOException;
25 import java.io.OutputStream;
26 import java.io.UncheckedIOException;
27 import java.net.InetAddress;
28 import java.net.URI;
29 import java.net.URISyntaxException;
30 import java.net.UnknownHostException;
31 import java.nio.charset.Charset;
32 import java.nio.file.Files;
33 import java.nio.file.StandardCopyOption;
34 import java.nio.file.attribute.FileTime;
35 import java.util.Collections;
36 import java.util.Date;
37 import java.util.HashMap;
38 import java.util.HashSet;
39 import java.util.List;
40 import java.util.Map;
41 import java.util.Set;
42 import java.util.regex.Matcher;
43 import java.util.regex.Pattern;
44
45 import org.apache.http.Header;
46 import org.apache.http.HttpEntity;
47 import org.apache.http.HttpEntityEnclosingRequest;
48 import org.apache.http.HttpHeaders;
49 import org.apache.http.HttpHost;
50 import org.apache.http.HttpResponse;
51 import org.apache.http.HttpStatus;
52 import org.apache.http.auth.AuthSchemeProvider;
53 import org.apache.http.auth.AuthScope;
54 import org.apache.http.client.CredentialsProvider;
55 import org.apache.http.client.HttpRequestRetryHandler;
56 import org.apache.http.client.HttpResponseException;
57 import org.apache.http.client.ServiceUnavailableRetryStrategy;
58 import org.apache.http.client.config.AuthSchemes;
59 import org.apache.http.client.config.RequestConfig;
60 import org.apache.http.client.methods.CloseableHttpResponse;
61 import org.apache.http.client.methods.HttpGet;
62 import org.apache.http.client.methods.HttpHead;
63 import org.apache.http.client.methods.HttpOptions;
64 import org.apache.http.client.methods.HttpPut;
65 import org.apache.http.client.methods.HttpUriRequest;
66 import org.apache.http.client.utils.DateUtils;
67 import org.apache.http.client.utils.URIUtils;
68 import org.apache.http.config.Registry;
69 import org.apache.http.config.RegistryBuilder;
70 import org.apache.http.config.SocketConfig;
71 import org.apache.http.entity.AbstractHttpEntity;
72 import org.apache.http.entity.ByteArrayEntity;
73 import org.apache.http.impl.NoConnectionReuseStrategy;
74 import org.apache.http.impl.auth.BasicScheme;
75 import org.apache.http.impl.auth.BasicSchemeFactory;
76 import org.apache.http.impl.auth.DigestSchemeFactory;
77 import org.apache.http.impl.auth.KerberosSchemeFactory;
78 import org.apache.http.impl.auth.NTLMSchemeFactory;
79 import org.apache.http.impl.auth.SPNegoSchemeFactory;
80 import org.apache.http.impl.client.CloseableHttpClient;
81 import org.apache.http.impl.client.DefaultHttpRequestRetryHandler;
82 import org.apache.http.impl.client.HttpClientBuilder;
83 import org.apache.http.impl.client.StandardHttpRequestRetryHandler;
84 import org.apache.http.protocol.HttpContext;
85 import org.apache.http.util.EntityUtils;
86 import org.eclipse.aether.ConfigurationProperties;
87 import org.eclipse.aether.RepositorySystemSession;
88 import org.eclipse.aether.repository.AuthenticationContext;
89 import org.eclipse.aether.repository.Proxy;
90 import org.eclipse.aether.repository.RemoteRepository;
91 import org.eclipse.aether.spi.connector.transport.AbstractTransporter;
92 import org.eclipse.aether.spi.connector.transport.GetTask;
93 import org.eclipse.aether.spi.connector.transport.PeekTask;
94 import org.eclipse.aether.spi.connector.transport.PutTask;
95 import org.eclipse.aether.spi.connector.transport.TransportTask;
96 import org.eclipse.aether.spi.connector.transport.http.HttpTransporter;
97 import org.eclipse.aether.spi.connector.transport.http.HttpTransporterException;
98 import org.eclipse.aether.transfer.NoTransporterException;
99 import org.eclipse.aether.transfer.TransferCancelledException;
100 import org.eclipse.aether.util.ConfigUtils;
101 import org.eclipse.aether.util.FileUtils;
102 import org.slf4j.Logger;
103 import org.slf4j.LoggerFactory;
104
105 import static java.util.Objects.requireNonNull;
106 import static org.eclipse.aether.transport.apache.ApacheTransporterConfigurationKeys.CONFIG_PROP_HTTP_RETRY_HANDLER_NAME;
107 import static org.eclipse.aether.transport.apache.ApacheTransporterConfigurationKeys.CONFIG_PROP_HTTP_RETRY_HANDLER_REQUEST_SENT_ENABLED;
108 import static org.eclipse.aether.transport.apache.ApacheTransporterConfigurationKeys.CONFIG_PROP_USE_SYSTEM_PROPERTIES;
109 import static org.eclipse.aether.transport.apache.ApacheTransporterConfigurationKeys.DEFAULT_HTTP_RETRY_HANDLER_REQUEST_SENT_ENABLED;
110 import static org.eclipse.aether.transport.apache.ApacheTransporterConfigurationKeys.DEFAULT_USE_SYSTEM_PROPERTIES;
111 import static org.eclipse.aether.transport.apache.ApacheTransporterConfigurationKeys.HTTP_RETRY_HANDLER_NAME_DEFAULT;
112 import static org.eclipse.aether.transport.apache.ApacheTransporterConfigurationKeys.HTTP_RETRY_HANDLER_NAME_STANDARD;
113
114
115
116
117 final class ApacheTransporter extends AbstractTransporter implements HttpTransporter {
118 private static final Pattern CONTENT_RANGE_PATTERN =
119 Pattern.compile("\\s*bytes\\s+([0-9]+)\\s*-\\s*([0-9]+)\\s*/.*");
120
121 private static final Logger LOGGER = LoggerFactory.getLogger(ApacheTransporter.class);
122
123 private final AuthenticationContext repoAuthContext;
124
125 private final AuthenticationContext proxyAuthContext;
126
127 private final URI baseUri;
128
129 private final HttpHost server;
130
131 private final HttpHost proxy;
132
133 private final CloseableHttpClient client;
134
135 private final Map<?, ?> headers;
136
137 private final LocalState state;
138
139 private final boolean preemptiveAuth;
140
141 private final boolean preemptivePutAuth;
142
143 private final boolean supportWebDav;
144
145 @SuppressWarnings("checkstyle:methodlength")
146 ApacheTransporter(RemoteRepository repository, RepositorySystemSession session) throws NoTransporterException {
147 if (!"http".equalsIgnoreCase(repository.getProtocol()) && !"https".equalsIgnoreCase(repository.getProtocol())) {
148 throw new NoTransporterException(repository);
149 }
150 try {
151 this.baseUri = new URI(repository.getUrl()).parseServerAuthority();
152 if (baseUri.isOpaque()) {
153 throw new URISyntaxException(repository.getUrl(), "URL must not be opaque");
154 }
155 this.server = URIUtils.extractHost(baseUri);
156 if (server == null) {
157 throw new URISyntaxException(repository.getUrl(), "URL lacks host name");
158 }
159 } catch (URISyntaxException e) {
160 throw new NoTransporterException(repository, e.getMessage(), e);
161 }
162 this.proxy = toHost(repository.getProxy());
163
164 this.repoAuthContext = AuthenticationContext.forRepository(session, repository);
165 this.proxyAuthContext = AuthenticationContext.forProxy(session, repository);
166
167 String httpsSecurityMode = ConfigUtils.getString(
168 session,
169 ConfigurationProperties.HTTPS_SECURITY_MODE_DEFAULT,
170 ConfigurationProperties.HTTPS_SECURITY_MODE + "." + repository.getId(),
171 ConfigurationProperties.HTTPS_SECURITY_MODE);
172 final int connectionMaxTtlSeconds = ConfigUtils.getInteger(
173 session,
174 ConfigurationProperties.DEFAULT_HTTP_CONNECTION_MAX_TTL,
175 ConfigurationProperties.HTTP_CONNECTION_MAX_TTL + "." + repository.getId(),
176 ConfigurationProperties.HTTP_CONNECTION_MAX_TTL);
177 final int maxConnectionsPerRoute = ConfigUtils.getInteger(
178 session,
179 ConfigurationProperties.DEFAULT_HTTP_MAX_CONNECTIONS_PER_ROUTE,
180 ConfigurationProperties.HTTP_MAX_CONNECTIONS_PER_ROUTE + "." + repository.getId(),
181 ConfigurationProperties.HTTP_MAX_CONNECTIONS_PER_ROUTE);
182 this.state = new LocalState(
183 session,
184 repository,
185 new ConnMgrConfig(
186 session, repoAuthContext, httpsSecurityMode, connectionMaxTtlSeconds, maxConnectionsPerRoute));
187
188 this.headers = ConfigUtils.getMap(
189 session,
190 Collections.emptyMap(),
191 ConfigurationProperties.HTTP_HEADERS + "." + repository.getId(),
192 ConfigurationProperties.HTTP_HEADERS);
193
194 this.preemptiveAuth = ConfigUtils.getBoolean(
195 session,
196 ConfigurationProperties.DEFAULT_HTTP_PREEMPTIVE_AUTH,
197 ConfigurationProperties.HTTP_PREEMPTIVE_AUTH + "." + repository.getId(),
198 ConfigurationProperties.HTTP_PREEMPTIVE_AUTH);
199 this.preemptivePutAuth = ConfigUtils.getBoolean(
200 session,
201 ConfigurationProperties.DEFAULT_HTTP_PREEMPTIVE_PUT_AUTH,
202 ConfigurationProperties.HTTP_PREEMPTIVE_PUT_AUTH + "." + repository.getId(),
203 ConfigurationProperties.HTTP_PREEMPTIVE_PUT_AUTH);
204 this.supportWebDav = ConfigUtils.getBoolean(
205 session,
206 ConfigurationProperties.DEFAULT_HTTP_SUPPORT_WEBDAV,
207 ConfigurationProperties.HTTP_SUPPORT_WEBDAV + "." + repository.getId(),
208 ConfigurationProperties.HTTP_SUPPORT_WEBDAV);
209 String credentialEncoding = ConfigUtils.getString(
210 session,
211 ConfigurationProperties.DEFAULT_HTTP_CREDENTIAL_ENCODING,
212 ConfigurationProperties.HTTP_CREDENTIAL_ENCODING + "." + repository.getId(),
213 ConfigurationProperties.HTTP_CREDENTIAL_ENCODING);
214 int connectTimeout = ConfigUtils.getInteger(
215 session,
216 ConfigurationProperties.DEFAULT_CONNECT_TIMEOUT,
217 ConfigurationProperties.CONNECT_TIMEOUT + "." + repository.getId(),
218 ConfigurationProperties.CONNECT_TIMEOUT);
219 int requestTimeout = ConfigUtils.getInteger(
220 session,
221 ConfigurationProperties.DEFAULT_REQUEST_TIMEOUT,
222 ConfigurationProperties.REQUEST_TIMEOUT + "." + repository.getId(),
223 ConfigurationProperties.REQUEST_TIMEOUT);
224 int retryCount = ConfigUtils.getInteger(
225 session,
226 ConfigurationProperties.DEFAULT_HTTP_RETRY_HANDLER_COUNT,
227 ConfigurationProperties.HTTP_RETRY_HANDLER_COUNT + "." + repository.getId(),
228 ConfigurationProperties.HTTP_RETRY_HANDLER_COUNT);
229 long retryInterval = ConfigUtils.getLong(
230 session,
231 ConfigurationProperties.DEFAULT_HTTP_RETRY_HANDLER_INTERVAL,
232 ConfigurationProperties.HTTP_RETRY_HANDLER_INTERVAL + "." + repository.getId(),
233 ConfigurationProperties.HTTP_RETRY_HANDLER_INTERVAL);
234 long retryIntervalMax = ConfigUtils.getLong(
235 session,
236 ConfigurationProperties.DEFAULT_HTTP_RETRY_HANDLER_INTERVAL_MAX,
237 ConfigurationProperties.HTTP_RETRY_HANDLER_INTERVAL_MAX + "." + repository.getId(),
238 ConfigurationProperties.HTTP_RETRY_HANDLER_INTERVAL_MAX);
239 String serviceUnavailableCodesString = ConfigUtils.getString(
240 session,
241 ConfigurationProperties.DEFAULT_HTTP_RETRY_HANDLER_SERVICE_UNAVAILABLE,
242 ConfigurationProperties.HTTP_RETRY_HANDLER_SERVICE_UNAVAILABLE + "." + repository.getId(),
243 ConfigurationProperties.HTTP_RETRY_HANDLER_SERVICE_UNAVAILABLE);
244 String retryHandlerName = ConfigUtils.getString(
245 session,
246 HTTP_RETRY_HANDLER_NAME_STANDARD,
247 CONFIG_PROP_HTTP_RETRY_HANDLER_NAME + "." + repository.getId(),
248 CONFIG_PROP_HTTP_RETRY_HANDLER_NAME);
249 boolean retryHandlerRequestSentEnabled = ConfigUtils.getBoolean(
250 session,
251 DEFAULT_HTTP_RETRY_HANDLER_REQUEST_SENT_ENABLED,
252 CONFIG_PROP_HTTP_RETRY_HANDLER_REQUEST_SENT_ENABLED + "." + repository.getId(),
253 CONFIG_PROP_HTTP_RETRY_HANDLER_REQUEST_SENT_ENABLED);
254 String userAgent = ConfigUtils.getString(
255 session, ConfigurationProperties.DEFAULT_USER_AGENT, ConfigurationProperties.USER_AGENT);
256
257 Charset credentialsCharset = Charset.forName(credentialEncoding);
258 Registry<AuthSchemeProvider> authSchemeRegistry = RegistryBuilder.<AuthSchemeProvider>create()
259 .register(AuthSchemes.BASIC, new BasicSchemeFactory(credentialsCharset))
260 .register(AuthSchemes.DIGEST, new DigestSchemeFactory(credentialsCharset))
261 .register(AuthSchemes.NTLM, new NTLMSchemeFactory())
262 .register(AuthSchemes.SPNEGO, new SPNegoSchemeFactory())
263 .register(AuthSchemes.KERBEROS, new KerberosSchemeFactory())
264 .build();
265 SocketConfig socketConfig =
266 SocketConfig.custom().setSoTimeout(requestTimeout).build();
267 RequestConfig requestConfig = RequestConfig.custom()
268 .setConnectTimeout(connectTimeout)
269 .setConnectionRequestTimeout(connectTimeout)
270 .setLocalAddress(getHttpLocalAddress(session, repository))
271 .setSocketTimeout(requestTimeout)
272 .build();
273
274 HttpRequestRetryHandler retryHandler;
275 if (HTTP_RETRY_HANDLER_NAME_STANDARD.equals(retryHandlerName)) {
276 retryHandler = new StandardHttpRequestRetryHandler(retryCount, retryHandlerRequestSentEnabled);
277 } else if (HTTP_RETRY_HANDLER_NAME_DEFAULT.equals(retryHandlerName)) {
278 retryHandler = new DefaultHttpRequestRetryHandler(retryCount, retryHandlerRequestSentEnabled);
279 } else {
280 throw new IllegalArgumentException(
281 "Unsupported parameter " + CONFIG_PROP_HTTP_RETRY_HANDLER_NAME + " value: " + retryHandlerName);
282 }
283 Set<Integer> serviceUnavailableCodes = new HashSet<>();
284 try {
285 for (String code : ConfigUtils.parseCommaSeparatedUniqueNames(serviceUnavailableCodesString)) {
286 serviceUnavailableCodes.add(Integer.parseInt(code));
287 }
288 } catch (NumberFormatException e) {
289 throw new IllegalArgumentException(
290 "Illegal HTTP codes for " + ConfigurationProperties.HTTP_RETRY_HANDLER_SERVICE_UNAVAILABLE
291 + " (list of integers): " + serviceUnavailableCodesString);
292 }
293 ServiceUnavailableRetryStrategy serviceUnavailableRetryStrategy = new ResolverServiceUnavailableRetryStrategy(
294 retryCount, retryInterval, retryIntervalMax, serviceUnavailableCodes);
295
296 HttpClientBuilder builder = HttpClientBuilder.create()
297 .setUserAgent(userAgent)
298 .setDefaultSocketConfig(socketConfig)
299 .setDefaultRequestConfig(requestConfig)
300 .setServiceUnavailableRetryStrategy(serviceUnavailableRetryStrategy)
301 .setRetryHandler(retryHandler)
302 .setDefaultAuthSchemeRegistry(authSchemeRegistry)
303 .setConnectionManager(state.getConnectionManager())
304 .setConnectionManagerShared(true)
305 .setDefaultCredentialsProvider(toCredentialsProvider(server, repoAuthContext, proxy, proxyAuthContext))
306 .setProxy(proxy);
307 final boolean useSystemProperties = ConfigUtils.getBoolean(
308 session,
309 DEFAULT_USE_SYSTEM_PROPERTIES,
310 CONFIG_PROP_USE_SYSTEM_PROPERTIES + "." + repository.getId(),
311 CONFIG_PROP_USE_SYSTEM_PROPERTIES);
312 if (useSystemProperties) {
313 LOGGER.warn(
314 "Transport used Apache HttpClient is instructed to use system properties: this may yield in unwanted side-effects!");
315 LOGGER.warn("Please use documented means to configure resolver transport.");
316 builder.useSystemProperties();
317 }
318
319 final String expectContinue = ConfigUtils.getString(
320 session,
321 null,
322 ConfigurationProperties.HTTP_EXPECT_CONTINUE + "." + repository.getId(),
323 ConfigurationProperties.HTTP_EXPECT_CONTINUE);
324 if (expectContinue != null) {
325 state.setExpectContinue(Boolean.parseBoolean(expectContinue));
326 }
327
328 final boolean reuseConnections = ConfigUtils.getBoolean(
329 session,
330 ConfigurationProperties.DEFAULT_HTTP_REUSE_CONNECTIONS,
331 ConfigurationProperties.HTTP_REUSE_CONNECTIONS + "." + repository.getId(),
332 ConfigurationProperties.HTTP_REUSE_CONNECTIONS);
333 if (!reuseConnections) {
334 builder.setConnectionReuseStrategy(NoConnectionReuseStrategy.INSTANCE);
335 }
336
337 this.client = builder.build();
338 }
339
340
341
342
343 private InetAddress getHttpLocalAddress(RepositorySystemSession session, RemoteRepository repository) {
344 String bindAddress = ConfigUtils.getString(
345 session,
346 null,
347 ConfigurationProperties.HTTP_LOCAL_ADDRESS + "." + repository.getId(),
348 ConfigurationProperties.HTTP_LOCAL_ADDRESS);
349 if (bindAddress == null) {
350 return null;
351 }
352 try {
353 return InetAddress.getByName(bindAddress);
354 } catch (UnknownHostException uhe) {
355 throw new IllegalArgumentException(
356 "Given bind address (" + bindAddress + ") cannot be resolved for remote repository " + repository,
357 uhe);
358 }
359 }
360
361 private static HttpHost toHost(Proxy proxy) {
362 HttpHost host = null;
363 if (proxy != null) {
364 host = new HttpHost(proxy.getHost(), proxy.getPort());
365 }
366 return host;
367 }
368
369 private static CredentialsProvider toCredentialsProvider(
370 HttpHost server, AuthenticationContext serverAuthCtx, HttpHost proxy, AuthenticationContext proxyAuthCtx) {
371 CredentialsProvider provider = toCredentialsProvider(server.getHostName(), AuthScope.ANY_PORT, serverAuthCtx);
372 if (proxy != null) {
373 CredentialsProvider p = toCredentialsProvider(proxy.getHostName(), proxy.getPort(), proxyAuthCtx);
374 provider = new DemuxCredentialsProvider(provider, p, proxy);
375 }
376 return provider;
377 }
378
379 private static CredentialsProvider toCredentialsProvider(String host, int port, AuthenticationContext ctx) {
380 DeferredCredentialsProvider provider = new DeferredCredentialsProvider();
381 if (ctx != null) {
382 AuthScope basicScope = new AuthScope(host, port);
383 provider.setCredentials(basicScope, new DeferredCredentialsProvider.BasicFactory(ctx));
384
385 AuthScope ntlmScope = new AuthScope(host, port, AuthScope.ANY_REALM, "ntlm");
386 provider.setCredentials(ntlmScope, new DeferredCredentialsProvider.NtlmFactory(ctx));
387 }
388 return provider;
389 }
390
391 LocalState getState() {
392 return state;
393 }
394
395 private URI resolve(TransportTask task) {
396 return UriUtils.resolve(baseUri, task.getLocation());
397 }
398
399 @Override
400 public int classify(Throwable error) {
401 if (error instanceof HttpTransporterException
402 && ((HttpTransporterException) error).getStatusCode() == HttpStatus.SC_NOT_FOUND) {
403 return ERROR_NOT_FOUND;
404 }
405 return ERROR_OTHER;
406 }
407
408 @Override
409 protected void implPeek(PeekTask task) throws Exception {
410 HttpHead request = commonHeaders(new HttpHead(resolve(task)));
411 try {
412 execute(request, null);
413 } catch (HttpResponseException e) {
414 throw new HttpTransporterException(e.getStatusCode());
415 }
416 }
417
418 @Override
419 protected void implGet(GetTask task) throws Exception {
420 boolean resume = true;
421
422 EntityGetter getter = new EntityGetter(task);
423 HttpGet request = commonHeaders(new HttpGet(resolve(task)));
424 while (true) {
425 try {
426 if (resume) {
427 resume(request, task);
428 }
429 execute(request, getter);
430 break;
431 } catch (HttpResponseException e) {
432 if (resume
433 && e.getStatusCode() == HttpStatus.SC_PRECONDITION_FAILED
434 && request.containsHeader(HttpHeaders.RANGE)) {
435 request = commonHeaders(new HttpGet(resolve(task)));
436 resume = false;
437 continue;
438 }
439 throw new HttpTransporterException(e.getStatusCode());
440 }
441 }
442 }
443
444 @Override
445 protected void implPut(PutTask task) throws Exception {
446 PutTaskEntity entity = new PutTaskEntity(task);
447 HttpPut request = commonHeaders(entity(new HttpPut(resolve(task)), entity));
448 try {
449 execute(request, null);
450 } catch (HttpResponseException e) {
451 if (e.getStatusCode() == HttpStatus.SC_EXPECTATION_FAILED && request.containsHeader(HttpHeaders.EXPECT)) {
452 state.setExpectContinue(false);
453 request = commonHeaders(entity(new HttpPut(request.getURI()), entity));
454 execute(request, null);
455 return;
456 }
457 throw new HttpTransporterException(e.getStatusCode());
458 }
459 }
460
461 private void execute(HttpUriRequest request, EntityGetter getter) throws Exception {
462 try {
463 SharingHttpContext context = new SharingHttpContext(state);
464 prepare(request, context);
465 try (CloseableHttpResponse response = client.execute(server, request, context)) {
466 try {
467 context.close();
468 handleStatus(response);
469 if (getter != null) {
470 getter.handle(response);
471 }
472 } finally {
473 EntityUtils.consumeQuietly(response.getEntity());
474 }
475 }
476 } catch (IOException e) {
477 if (e.getCause() instanceof TransferCancelledException) {
478 throw (Exception) e.getCause();
479 }
480 throw e;
481 }
482 }
483
484 private void prepare(HttpUriRequest request, SharingHttpContext context) throws HttpTransporterException {
485 final boolean put = HttpPut.METHOD_NAME.equalsIgnoreCase(request.getMethod());
486 if (preemptiveAuth || (preemptivePutAuth && put)) {
487 context.getAuthCache().put(server, new BasicScheme());
488 }
489 if (supportWebDav) {
490 if (state.getWebDav() == null && (put || isPayloadPresent(request))) {
491 HttpOptions req = commonHeaders(new HttpOptions(request.getURI()));
492 try (CloseableHttpResponse response = client.execute(server, req, context)) {
493 state.setWebDav(response.containsHeader(HttpHeaders.DAV));
494 EntityUtils.consumeQuietly(response.getEntity());
495 } catch (IOException e) {
496 LOGGER.debug("Failed to prepare HTTP context", e);
497 }
498 }
499 if (put && Boolean.TRUE.equals(state.getWebDav())) {
500 mkdirs(request.getURI(), context);
501 }
502 }
503 }
504
505 @SuppressWarnings("checkstyle:magicnumber")
506 private void mkdirs(URI uri, SharingHttpContext context) throws HttpTransporterException {
507 List<URI> dirs = UriUtils.getDirectories(baseUri, uri);
508 int index = 0;
509 for (; index < dirs.size(); index++) {
510 try (CloseableHttpResponse response =
511 client.execute(server, commonHeaders(new HttpMkCol(dirs.get(index))), context)) {
512 try {
513 int status = response.getStatusLine().getStatusCode();
514 if (status < 300 || status == HttpStatus.SC_METHOD_NOT_ALLOWED) {
515 break;
516 } else if (status == HttpStatus.SC_CONFLICT) {
517 continue;
518 }
519 handleStatus(response);
520 } finally {
521 EntityUtils.consumeQuietly(response.getEntity());
522 }
523 } catch (IOException e) {
524 LOGGER.debug("Failed to create parent directory {}", dirs.get(index), e);
525 return;
526 }
527 }
528 for (index--; index >= 0; index--) {
529 try (CloseableHttpResponse response =
530 client.execute(server, commonHeaders(new HttpMkCol(dirs.get(index))), context)) {
531 try {
532 handleStatus(response);
533 } finally {
534 EntityUtils.consumeQuietly(response.getEntity());
535 }
536 } catch (IOException e) {
537 LOGGER.debug("Failed to create parent directory {}", dirs.get(index), e);
538 return;
539 }
540 }
541 }
542
543 private <T extends HttpEntityEnclosingRequest> T entity(T request, HttpEntity entity) {
544 request.setEntity(entity);
545 return request;
546 }
547
548 private boolean isPayloadPresent(HttpUriRequest request) {
549 if (request instanceof HttpEntityEnclosingRequest) {
550 HttpEntity entity = ((HttpEntityEnclosingRequest) request).getEntity();
551 return entity != null && entity.getContentLength() != 0;
552 }
553 return false;
554 }
555
556 private <T extends HttpUriRequest> T commonHeaders(T request) {
557 request.setHeader(HttpHeaders.CACHE_CONTROL, "no-cache, no-store");
558 request.setHeader(HttpHeaders.PRAGMA, "no-cache");
559
560 if (state.isExpectContinue() && isPayloadPresent(request)) {
561 request.setHeader(HttpHeaders.EXPECT, "100-continue");
562 }
563
564 for (Map.Entry<?, ?> entry : headers.entrySet()) {
565 if (!(entry.getKey() instanceof String)) {
566 continue;
567 }
568 if (entry.getValue() instanceof String) {
569 request.setHeader(entry.getKey().toString(), entry.getValue().toString());
570 } else {
571 request.removeHeaders(entry.getKey().toString());
572 }
573 }
574
575 if (!state.isExpectContinue()) {
576 request.removeHeaders(HttpHeaders.EXPECT);
577 }
578
579 return request;
580 }
581
582 @SuppressWarnings("checkstyle:magicnumber")
583 private <T extends HttpUriRequest> void resume(T request, GetTask task) {
584 long resumeOffset = task.getResumeOffset();
585 if (resumeOffset > 0L && task.getDataFile() != null) {
586 request.setHeader(HttpHeaders.RANGE, "bytes=" + resumeOffset + '-');
587 request.setHeader(
588 HttpHeaders.IF_UNMODIFIED_SINCE,
589 DateUtils.formatDate(new Date(task.getDataFile().lastModified() - 60L * 1000L)));
590 request.setHeader(HttpHeaders.ACCEPT_ENCODING, "identity");
591 }
592 }
593
594 @SuppressWarnings("checkstyle:magicnumber")
595 private void handleStatus(CloseableHttpResponse response) throws HttpResponseException {
596 int status = response.getStatusLine().getStatusCode();
597 if (status >= 300) {
598 throw new HttpResponseException(status, response.getStatusLine().getReasonPhrase() + " (" + status + ")");
599 }
600 }
601
602 @Override
603 protected void implClose() {
604 try {
605 client.close();
606 } catch (IOException e) {
607 throw new UncheckedIOException(e);
608 }
609 AuthenticationContext.close(repoAuthContext);
610 AuthenticationContext.close(proxyAuthContext);
611 state.close();
612 }
613
614 private class EntityGetter {
615
616 private final GetTask task;
617
618 EntityGetter(GetTask task) {
619 this.task = task;
620 }
621
622 public void handle(CloseableHttpResponse response) throws IOException, TransferCancelledException {
623 HttpEntity entity = response.getEntity();
624 if (entity == null) {
625 entity = new ByteArrayEntity(new byte[0]);
626 }
627
628 long offset = 0L, length = entity.getContentLength();
629 Header rangeHeader = response.getFirstHeader(HttpHeaders.CONTENT_RANGE);
630 String range = rangeHeader != null ? rangeHeader.getValue() : null;
631 if (range != null) {
632 Matcher m = CONTENT_RANGE_PATTERN.matcher(range);
633 if (!m.matches()) {
634 throw new IOException("Invalid Content-Range header for partial download: " + range);
635 }
636 offset = Long.parseLong(m.group(1));
637 length = Long.parseLong(m.group(2)) + 1L;
638 if (offset < 0L || offset >= length || (offset > 0L && offset != task.getResumeOffset())) {
639 throw new IOException("Invalid Content-Range header for partial download from offset "
640 + task.getResumeOffset() + ": " + range);
641 }
642 }
643
644 final boolean resume = offset > 0L;
645 final File dataFile = task.getDataFile();
646 if (dataFile == null) {
647 try (InputStream is = entity.getContent()) {
648 utilGet(task, is, true, length, resume);
649 extractChecksums(response);
650 }
651 } else {
652 try (FileUtils.CollocatedTempFile tempFile = FileUtils.newTempFile(dataFile.toPath())) {
653 task.setDataFile(tempFile.getPath().toFile(), resume);
654 if (resume && Files.isRegularFile(dataFile.toPath())) {
655 try (InputStream inputStream = Files.newInputStream(dataFile.toPath())) {
656 Files.copy(inputStream, tempFile.getPath(), StandardCopyOption.REPLACE_EXISTING);
657 }
658 }
659 try (InputStream is = entity.getContent()) {
660 utilGet(task, is, true, length, resume);
661 }
662 tempFile.move();
663 } finally {
664 task.setDataFile(dataFile);
665 }
666 }
667 if (task.getDataFile() != null) {
668 Header lastModifiedHeader =
669 response.getFirstHeader(HttpHeaders.LAST_MODIFIED);
670 if (lastModifiedHeader != null) {
671 Date lastModified = DateUtils.parseDate(lastModifiedHeader.getValue());
672 if (lastModified != null) {
673 Files.setLastModifiedTime(
674 task.getDataFile().toPath(), FileTime.fromMillis(lastModified.getTime()));
675 }
676 }
677 }
678 extractChecksums(response);
679 }
680
681 private void extractChecksums(CloseableHttpResponse response) {
682 Map<String, String> checksums = extractXChecksums(response);
683 if (checksums == null) {
684 checksums = extractNx2Checksums(response);
685 }
686 if (checksums != null) {
687 checksums.forEach(task::setChecksum);
688 }
689 }
690 }
691
692 private static Map<String, String> extractNx2Checksums(HttpResponse response) {
693
694 Header header = response.getFirstHeader(HttpHeaders.ETAG);
695 String etag = header != null ? header.getValue() : null;
696 if (etag != null) {
697 int start = etag.indexOf("SHA1{"), end = etag.indexOf("}", start + 5);
698 if (start >= 0 && end > start) {
699 return Collections.singletonMap("SHA-1", etag.substring(start + 5, end));
700 }
701 }
702 return null;
703 }
704
705 private static Map<String, String> extractXChecksums(HttpResponse response) {
706 String value;
707 HashMap<String, String> result = new HashMap<>();
708
709 value = extractXChecksum(response, "x-checksum-sha1");
710 if (value != null) {
711 result.put("SHA-1", value);
712 }
713
714 value = extractXChecksum(response, "x-checksum-md5");
715 if (value != null) {
716 result.put("MD5", value);
717 }
718 if (!result.isEmpty()) {
719 return result;
720 }
721
722 value = extractXChecksum(response, "x-goog-meta-checksum-sha1");
723 if (value != null) {
724 result.put("SHA-1", value);
725 }
726
727 value = extractXChecksum(response, "x-goog-meta-checksum-md5");
728 if (value != null) {
729 result.put("MD5", value);
730 }
731
732 return result.isEmpty() ? null : result;
733 }
734
735 private static String extractXChecksum(HttpResponse response, String name) {
736 Header header = response.getFirstHeader(name);
737 return header != null ? header.getValue() : null;
738 }
739
740 private class PutTaskEntity extends AbstractHttpEntity {
741
742 private final PutTask task;
743
744 PutTaskEntity(PutTask task) {
745 this.task = task;
746 }
747
748 @Override
749 public boolean isRepeatable() {
750 return true;
751 }
752
753 @Override
754 public boolean isStreaming() {
755 return false;
756 }
757
758 @Override
759 public long getContentLength() {
760 return task.getDataLength();
761 }
762
763 @Override
764 public InputStream getContent() throws IOException {
765 return task.newInputStream();
766 }
767
768 @Override
769 public void writeTo(OutputStream os) throws IOException {
770 try {
771 utilPut(task, os, false);
772 } catch (TransferCancelledException e) {
773 throw (IOException) new InterruptedIOException().initCause(e);
774 }
775 }
776 }
777
778 private static class ResolverServiceUnavailableRetryStrategy implements ServiceUnavailableRetryStrategy {
779 private final int retryCount;
780
781 private final long retryInterval;
782
783 private final long retryIntervalMax;
784
785 private final Set<Integer> serviceUnavailableHttpCodes;
786
787
788
789
790
791
792 private static final ThreadLocal<Long> RETRY_INTERVAL_HOLDER = new ThreadLocal<>();
793
794 private ResolverServiceUnavailableRetryStrategy(
795 int retryCount, long retryInterval, long retryIntervalMax, Set<Integer> serviceUnavailableHttpCodes) {
796 if (retryCount < 0) {
797 throw new IllegalArgumentException("retryCount must be >= 0");
798 }
799 if (retryInterval < 0L) {
800 throw new IllegalArgumentException("retryInterval must be >= 0");
801 }
802 if (retryIntervalMax < 0L) {
803 throw new IllegalArgumentException("retryIntervalMax must be >= 0");
804 }
805 this.retryCount = retryCount;
806 this.retryInterval = retryInterval;
807 this.retryIntervalMax = retryIntervalMax;
808 this.serviceUnavailableHttpCodes = requireNonNull(serviceUnavailableHttpCodes);
809 }
810
811 @Override
812 public boolean retryRequest(HttpResponse response, int executionCount, HttpContext context) {
813 final boolean retry = executionCount <= retryCount
814 && (serviceUnavailableHttpCodes.contains(
815 response.getStatusLine().getStatusCode()));
816 if (retry) {
817 Long retryInterval = retryInterval(response, executionCount, context);
818 if (retryInterval != null) {
819 RETRY_INTERVAL_HOLDER.set(retryInterval);
820 return true;
821 }
822 }
823 RETRY_INTERVAL_HOLDER.remove();
824 return false;
825 }
826
827
828
829
830
831
832
833
834 private Long retryInterval(HttpResponse httpResponse, int executionCount, HttpContext httpContext) {
835 Long result = null;
836 Header header = httpResponse.getFirstHeader(HttpHeaders.RETRY_AFTER);
837 if (header != null && header.getValue() != null) {
838 String headerValue = header.getValue();
839 if (headerValue.contains(":")) {
840 Date when = DateUtils.parseDate(headerValue);
841 if (when != null) {
842 result = Math.max(when.getTime() - System.currentTimeMillis(), 0L);
843 }
844 } else {
845 try {
846 result = Long.parseLong(headerValue) * 1000L;
847 } catch (NumberFormatException e) {
848
849 }
850 }
851 }
852 if (result == null) {
853 result = executionCount * this.retryInterval;
854 }
855 if (result > retryIntervalMax) {
856 return null;
857 }
858 return result;
859 }
860
861 @Override
862 public long getRetryInterval() {
863 Long ri = RETRY_INTERVAL_HOLDER.get();
864 if (ri == null) {
865 return 0L;
866 }
867 RETRY_INTERVAL_HOLDER.remove();
868 return ri;
869 }
870 }
871 }