1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.eclipse.aether.transport.http;
20
21 import java.io.File;
22 import java.io.IOException;
23 import java.io.InputStream;
24 import java.io.InterruptedIOException;
25 import java.io.OutputStream;
26 import java.io.UncheckedIOException;
27 import java.net.URI;
28 import java.net.URISyntaxException;
29 import java.nio.charset.Charset;
30 import java.nio.file.Files;
31 import java.nio.file.StandardCopyOption;
32 import java.util.Collections;
33 import java.util.Date;
34 import java.util.List;
35 import java.util.Map;
36 import java.util.regex.Matcher;
37 import java.util.regex.Pattern;
38
39 import org.apache.http.Header;
40 import org.apache.http.HttpEntity;
41 import org.apache.http.HttpEntityEnclosingRequest;
42 import org.apache.http.HttpHeaders;
43 import org.apache.http.HttpHost;
44 import org.apache.http.HttpStatus;
45 import org.apache.http.auth.AuthSchemeProvider;
46 import org.apache.http.auth.AuthScope;
47 import org.apache.http.client.CredentialsProvider;
48 import org.apache.http.client.HttpResponseException;
49 import org.apache.http.client.config.AuthSchemes;
50 import org.apache.http.client.config.RequestConfig;
51 import org.apache.http.client.methods.CloseableHttpResponse;
52 import org.apache.http.client.methods.HttpGet;
53 import org.apache.http.client.methods.HttpHead;
54 import org.apache.http.client.methods.HttpOptions;
55 import org.apache.http.client.methods.HttpPut;
56 import org.apache.http.client.methods.HttpUriRequest;
57 import org.apache.http.client.utils.DateUtils;
58 import org.apache.http.client.utils.URIUtils;
59 import org.apache.http.config.Registry;
60 import org.apache.http.config.RegistryBuilder;
61 import org.apache.http.config.SocketConfig;
62 import org.apache.http.entity.AbstractHttpEntity;
63 import org.apache.http.entity.ByteArrayEntity;
64 import org.apache.http.impl.auth.BasicScheme;
65 import org.apache.http.impl.auth.BasicSchemeFactory;
66 import org.apache.http.impl.auth.DigestSchemeFactory;
67 import org.apache.http.impl.auth.KerberosSchemeFactory;
68 import org.apache.http.impl.auth.NTLMSchemeFactory;
69 import org.apache.http.impl.auth.SPNegoSchemeFactory;
70 import org.apache.http.impl.client.CloseableHttpClient;
71 import org.apache.http.impl.client.DefaultHttpRequestRetryHandler;
72 import org.apache.http.impl.client.HttpClientBuilder;
73 import org.apache.http.util.EntityUtils;
74 import org.eclipse.aether.ConfigurationProperties;
75 import org.eclipse.aether.RepositorySystemSession;
76 import org.eclipse.aether.repository.AuthenticationContext;
77 import org.eclipse.aether.repository.Proxy;
78 import org.eclipse.aether.repository.RemoteRepository;
79 import org.eclipse.aether.spi.connector.transport.AbstractTransporter;
80 import org.eclipse.aether.spi.connector.transport.GetTask;
81 import org.eclipse.aether.spi.connector.transport.PeekTask;
82 import org.eclipse.aether.spi.connector.transport.PutTask;
83 import org.eclipse.aether.spi.connector.transport.TransportTask;
84 import org.eclipse.aether.transfer.NoTransporterException;
85 import org.eclipse.aether.transfer.TransferCancelledException;
86 import org.eclipse.aether.util.ConfigUtils;
87 import org.eclipse.aether.util.FileUtils;
88 import org.slf4j.Logger;
89 import org.slf4j.LoggerFactory;
90
91 import static java.util.Objects.requireNonNull;
92
93
94
95
96 final class HttpTransporter extends AbstractTransporter {
97
98 static final String SUPPORT_WEBDAV = "aether.connector.http.supportWebDav";
99
100 static final String PREEMPTIVE_PUT_AUTH = "aether.connector.http.preemptivePutAuth";
101
102 static final String USE_SYSTEM_PROPERTIES = "aether.connector.http.useSystemProperties";
103
104 private static final Pattern CONTENT_RANGE_PATTERN =
105 Pattern.compile("\\s*bytes\\s+([0-9]+)\\s*-\\s*([0-9]+)\\s*/.*");
106
107 private static final Logger LOGGER = LoggerFactory.getLogger(HttpTransporter.class);
108
109 private final Map<String, ChecksumExtractor> checksumExtractors;
110
111 private final AuthenticationContext repoAuthContext;
112
113 private final AuthenticationContext proxyAuthContext;
114
115 private final URI baseUri;
116
117 private final HttpHost server;
118
119 private final HttpHost proxy;
120
121 private final CloseableHttpClient client;
122
123 private final Map<?, ?> headers;
124
125 private final LocalState state;
126
127 private final boolean preemptiveAuth;
128
129 private final boolean preemptivePutAuth;
130
131 private final boolean supportWebDav;
132
133 HttpTransporter(
134 Map<String, ChecksumExtractor> checksumExtractors,
135 RemoteRepository repository,
136 RepositorySystemSession session)
137 throws NoTransporterException {
138 if (!"http".equalsIgnoreCase(repository.getProtocol()) && !"https".equalsIgnoreCase(repository.getProtocol())) {
139 throw new NoTransporterException(repository);
140 }
141 this.checksumExtractors = requireNonNull(checksumExtractors, "checksum extractors must not be null");
142 try {
143 this.baseUri = new URI(repository.getUrl()).parseServerAuthority();
144 if (baseUri.isOpaque()) {
145 throw new URISyntaxException(repository.getUrl(), "URL must not be opaque");
146 }
147 this.server = URIUtils.extractHost(baseUri);
148 if (server == null) {
149 throw new URISyntaxException(repository.getUrl(), "URL lacks host name");
150 }
151 } catch (URISyntaxException e) {
152 throw new NoTransporterException(repository, e.getMessage(), e);
153 }
154 this.proxy = toHost(repository.getProxy());
155
156 this.repoAuthContext = AuthenticationContext.forRepository(session, repository);
157 this.proxyAuthContext = AuthenticationContext.forProxy(session, repository);
158
159 String httpsSecurityMode = ConfigUtils.getString(
160 session,
161 ConfigurationProperties.HTTPS_SECURITY_MODE_DEFAULT,
162 ConfigurationProperties.HTTPS_SECURITY_MODE + "." + repository.getId(),
163 ConfigurationProperties.HTTPS_SECURITY_MODE);
164 this.state = new LocalState(session, repository, new SslConfig(session, repoAuthContext, httpsSecurityMode));
165
166 this.headers = ConfigUtils.getMap(
167 session,
168 Collections.emptyMap(),
169 ConfigurationProperties.HTTP_HEADERS + "." + repository.getId(),
170 ConfigurationProperties.HTTP_HEADERS);
171
172 this.preemptiveAuth = ConfigUtils.getBoolean(
173 session,
174 ConfigurationProperties.DEFAULT_HTTP_PREEMPTIVE_AUTH,
175 ConfigurationProperties.HTTP_PREEMPTIVE_AUTH + "." + repository.getId(),
176 ConfigurationProperties.HTTP_PREEMPTIVE_AUTH);
177 this.preemptivePutAuth =
178 ConfigUtils.getBoolean(
179 session, true, PREEMPTIVE_PUT_AUTH + "." + repository.getId(), PREEMPTIVE_PUT_AUTH);
180 this.supportWebDav =
181 ConfigUtils.getBoolean(session, false, SUPPORT_WEBDAV + "." + repository.getId(), SUPPORT_WEBDAV);
182 String credentialEncoding = ConfigUtils.getString(
183 session,
184 ConfigurationProperties.DEFAULT_HTTP_CREDENTIAL_ENCODING,
185 ConfigurationProperties.HTTP_CREDENTIAL_ENCODING + "." + repository.getId(),
186 ConfigurationProperties.HTTP_CREDENTIAL_ENCODING);
187 int connectTimeout = ConfigUtils.getInteger(
188 session,
189 ConfigurationProperties.DEFAULT_CONNECT_TIMEOUT,
190 ConfigurationProperties.CONNECT_TIMEOUT + "." + repository.getId(),
191 ConfigurationProperties.CONNECT_TIMEOUT);
192 int requestTimeout = ConfigUtils.getInteger(
193 session,
194 ConfigurationProperties.DEFAULT_REQUEST_TIMEOUT,
195 ConfigurationProperties.REQUEST_TIMEOUT + "." + repository.getId(),
196 ConfigurationProperties.REQUEST_TIMEOUT);
197 int retryCount = ConfigUtils.getInteger(
198 session,
199 ConfigurationProperties.DEFAULT_HTTP_RETRY_HANDLER_COUNT,
200 ConfigurationProperties.HTTP_RETRY_HANDLER_COUNT + "." + repository.getId(),
201 ConfigurationProperties.HTTP_RETRY_HANDLER_COUNT);
202 String userAgent = ConfigUtils.getString(
203 session, ConfigurationProperties.DEFAULT_USER_AGENT, ConfigurationProperties.USER_AGENT);
204
205 Charset credentialsCharset = Charset.forName(credentialEncoding);
206
207 Registry<AuthSchemeProvider> authSchemeRegistry = RegistryBuilder.<AuthSchemeProvider>create()
208 .register(AuthSchemes.BASIC, new BasicSchemeFactory(credentialsCharset))
209 .register(AuthSchemes.DIGEST, new DigestSchemeFactory(credentialsCharset))
210 .register(AuthSchemes.NTLM, new NTLMSchemeFactory())
211 .register(AuthSchemes.SPNEGO, new SPNegoSchemeFactory())
212 .register(AuthSchemes.KERBEROS, new KerberosSchemeFactory())
213 .build();
214
215 SocketConfig socketConfig =
216 SocketConfig.custom().setSoTimeout(requestTimeout).build();
217
218 RequestConfig requestConfig = RequestConfig.custom()
219 .setConnectTimeout(connectTimeout)
220 .setConnectionRequestTimeout(connectTimeout)
221 .setSocketTimeout(requestTimeout)
222 .build();
223
224 DefaultHttpRequestRetryHandler retryHandler = new DefaultHttpRequestRetryHandler(retryCount, false);
225
226 HttpClientBuilder builder = HttpClientBuilder.create()
227 .setUserAgent(userAgent)
228 .setDefaultSocketConfig(socketConfig)
229 .setDefaultRequestConfig(requestConfig)
230 .setRetryHandler(retryHandler)
231 .setDefaultAuthSchemeRegistry(authSchemeRegistry)
232 .setConnectionManager(state.getConnectionManager())
233 .setConnectionManagerShared(true)
234 .setDefaultCredentialsProvider(toCredentialsProvider(server, repoAuthContext, proxy, proxyAuthContext))
235 .setProxy(proxy);
236
237 final boolean useSystemProperties = ConfigUtils.getBoolean(
238 session, false, USE_SYSTEM_PROPERTIES + "." + repository.getId(), USE_SYSTEM_PROPERTIES);
239 if (useSystemProperties) {
240 LOGGER.warn(
241 "Transport used Apache HttpClient is instructed to use system properties: this may yield in unwanted side-effects!");
242 LOGGER.warn("Please use documented means to configure resolver transport.");
243 builder.useSystemProperties();
244 }
245
246 this.client = builder.build();
247 }
248
249 private static HttpHost toHost(Proxy proxy) {
250 HttpHost host = null;
251 if (proxy != null) {
252 host = new HttpHost(proxy.getHost(), proxy.getPort());
253 }
254 return host;
255 }
256
257 private static CredentialsProvider toCredentialsProvider(
258 HttpHost server, AuthenticationContext serverAuthCtx, HttpHost proxy, AuthenticationContext proxyAuthCtx) {
259 CredentialsProvider provider = toCredentialsProvider(server.getHostName(), AuthScope.ANY_PORT, serverAuthCtx);
260 if (proxy != null) {
261 CredentialsProvider p = toCredentialsProvider(proxy.getHostName(), proxy.getPort(), proxyAuthCtx);
262 provider = new DemuxCredentialsProvider(provider, p, proxy);
263 }
264 return provider;
265 }
266
267 private static CredentialsProvider toCredentialsProvider(String host, int port, AuthenticationContext ctx) {
268 DeferredCredentialsProvider provider = new DeferredCredentialsProvider();
269 if (ctx != null) {
270 AuthScope basicScope = new AuthScope(host, port);
271 provider.setCredentials(basicScope, new DeferredCredentialsProvider.BasicFactory(ctx));
272
273 AuthScope ntlmScope = new AuthScope(host, port, AuthScope.ANY_REALM, "ntlm");
274 provider.setCredentials(ntlmScope, new DeferredCredentialsProvider.NtlmFactory(ctx));
275 }
276 return provider;
277 }
278
279 LocalState getState() {
280 return state;
281 }
282
283 private URI resolve(TransportTask task) {
284 return UriUtils.resolve(baseUri, task.getLocation());
285 }
286
287 @Override
288 public int classify(Throwable error) {
289 if (error instanceof HttpResponseException
290 && ((HttpResponseException) error).getStatusCode() == HttpStatus.SC_NOT_FOUND) {
291 return ERROR_NOT_FOUND;
292 }
293 return ERROR_OTHER;
294 }
295
296 @Override
297 protected void implPeek(PeekTask task) throws Exception {
298 HttpHead request = commonHeaders(new HttpHead(resolve(task)));
299 execute(request, null);
300 }
301
302 @Override
303 protected void implGet(GetTask task) throws Exception {
304 boolean resume = true;
305 boolean applyChecksumExtractors = true;
306
307 EntityGetter getter = new EntityGetter(task);
308 HttpGet request = commonHeaders(new HttpGet(resolve(task)));
309 while (true) {
310 try {
311 if (resume) {
312 resume(request, task);
313 }
314 if (applyChecksumExtractors) {
315 for (ChecksumExtractor checksumExtractor : checksumExtractors.values()) {
316 checksumExtractor.prepareRequest(request);
317 }
318 }
319 execute(request, getter);
320 break;
321 } catch (HttpResponseException e) {
322 if (resume
323 && e.getStatusCode() == HttpStatus.SC_PRECONDITION_FAILED
324 && request.containsHeader(HttpHeaders.RANGE)) {
325 request = commonHeaders(new HttpGet(resolve(task)));
326 resume = false;
327 continue;
328 }
329 if (applyChecksumExtractors) {
330 boolean retryWithoutExtractors = false;
331 for (ChecksumExtractor checksumExtractor : checksumExtractors.values()) {
332 if (checksumExtractor.retryWithoutExtractor(e)) {
333 retryWithoutExtractors = true;
334 break;
335 }
336 }
337 if (retryWithoutExtractors) {
338 request = commonHeaders(new HttpGet(resolve(task)));
339 applyChecksumExtractors = false;
340 continue;
341 }
342 }
343 throw e;
344 }
345 }
346 }
347
348 @Override
349 protected void implPut(PutTask task) throws Exception {
350 PutTaskEntity entity = new PutTaskEntity(task);
351 HttpPut request = commonHeaders(entity(new HttpPut(resolve(task)), entity));
352 try {
353 execute(request, null);
354 } catch (HttpResponseException e) {
355 if (e.getStatusCode() == HttpStatus.SC_EXPECTATION_FAILED && request.containsHeader(HttpHeaders.EXPECT)) {
356 state.setExpectContinue(false);
357 request = commonHeaders(entity(new HttpPut(request.getURI()), entity));
358 execute(request, null);
359 return;
360 }
361 throw e;
362 }
363 }
364
365 private void execute(HttpUriRequest request, EntityGetter getter) throws Exception {
366 try {
367 SharingHttpContext context = new SharingHttpContext(state);
368 prepare(request, context);
369 try (CloseableHttpResponse response = client.execute(server, request, context)) {
370 try {
371 context.close();
372 handleStatus(response);
373 if (getter != null) {
374 getter.handle(response);
375 }
376 } finally {
377 EntityUtils.consumeQuietly(response.getEntity());
378 }
379 }
380 } catch (IOException e) {
381 if (e.getCause() instanceof TransferCancelledException) {
382 throw (Exception) e.getCause();
383 }
384 throw e;
385 }
386 }
387
388 private void prepare(HttpUriRequest request, SharingHttpContext context) {
389 final boolean put = HttpPut.METHOD_NAME.equalsIgnoreCase(request.getMethod());
390 if (preemptiveAuth || (preemptivePutAuth && put)) {
391 context.getAuthCache().put(server, new BasicScheme());
392 }
393 if (supportWebDav) {
394 if (state.getWebDav() == null && (put || isPayloadPresent(request))) {
395 HttpOptions req = commonHeaders(new HttpOptions(request.getURI()));
396 try (CloseableHttpResponse response = client.execute(server, req, context)) {
397 state.setWebDav(response.containsHeader(HttpHeaders.DAV));
398 EntityUtils.consumeQuietly(response.getEntity());
399 } catch (IOException e) {
400 LOGGER.debug("Failed to prepare HTTP context", e);
401 }
402 }
403 if (put && Boolean.TRUE.equals(state.getWebDav())) {
404 mkdirs(request.getURI(), context);
405 }
406 }
407 }
408
409 @SuppressWarnings("checkstyle:magicnumber")
410 private void mkdirs(URI uri, SharingHttpContext context) {
411 List<URI> dirs = UriUtils.getDirectories(baseUri, uri);
412 int index = 0;
413 for (; index < dirs.size(); index++) {
414 try (CloseableHttpResponse response =
415 client.execute(server, commonHeaders(new HttpMkCol(dirs.get(index))), context)) {
416 try {
417 int status = response.getStatusLine().getStatusCode();
418 if (status < 300 || status == HttpStatus.SC_METHOD_NOT_ALLOWED) {
419 break;
420 } else if (status == HttpStatus.SC_CONFLICT) {
421 continue;
422 }
423 handleStatus(response);
424 } finally {
425 EntityUtils.consumeQuietly(response.getEntity());
426 }
427 } catch (IOException e) {
428 LOGGER.debug("Failed to create parent directory {}", dirs.get(index), e);
429 return;
430 }
431 }
432 for (index--; index >= 0; index--) {
433 try (CloseableHttpResponse response =
434 client.execute(server, commonHeaders(new HttpMkCol(dirs.get(index))), context)) {
435 try {
436 handleStatus(response);
437 } finally {
438 EntityUtils.consumeQuietly(response.getEntity());
439 }
440 } catch (IOException e) {
441 LOGGER.debug("Failed to create parent directory {}", dirs.get(index), e);
442 return;
443 }
444 }
445 }
446
447 private <T extends HttpEntityEnclosingRequest> T entity(T request, HttpEntity entity) {
448 request.setEntity(entity);
449 return request;
450 }
451
452 private boolean isPayloadPresent(HttpUriRequest request) {
453 if (request instanceof HttpEntityEnclosingRequest) {
454 HttpEntity entity = ((HttpEntityEnclosingRequest) request).getEntity();
455 return entity != null && entity.getContentLength() != 0;
456 }
457 return false;
458 }
459
460 private <T extends HttpUriRequest> T commonHeaders(T request) {
461 request.setHeader(HttpHeaders.CACHE_CONTROL, "no-cache, no-store");
462 request.setHeader(HttpHeaders.PRAGMA, "no-cache");
463
464 if (state.isExpectContinue() && isPayloadPresent(request)) {
465 request.setHeader(HttpHeaders.EXPECT, "100-continue");
466 }
467
468 for (Map.Entry<?, ?> entry : headers.entrySet()) {
469 if (!(entry.getKey() instanceof String)) {
470 continue;
471 }
472 if (entry.getValue() instanceof String) {
473 request.setHeader(entry.getKey().toString(), entry.getValue().toString());
474 } else {
475 request.removeHeaders(entry.getKey().toString());
476 }
477 }
478
479 if (!state.isExpectContinue()) {
480 request.removeHeaders(HttpHeaders.EXPECT);
481 }
482
483 return request;
484 }
485
486 @SuppressWarnings("checkstyle:magicnumber")
487 private <T extends HttpUriRequest> T resume(T request, GetTask task) {
488 long resumeOffset = task.getResumeOffset();
489 if (resumeOffset > 0L && task.getDataFile() != null) {
490 request.setHeader(HttpHeaders.RANGE, "bytes=" + resumeOffset + '-');
491 request.setHeader(
492 HttpHeaders.IF_UNMODIFIED_SINCE,
493 DateUtils.formatDate(new Date(task.getDataFile().lastModified() - 60L * 1000L)));
494 request.setHeader(HttpHeaders.ACCEPT_ENCODING, "identity");
495 }
496 return request;
497 }
498
499 @SuppressWarnings("checkstyle:magicnumber")
500 private void handleStatus(CloseableHttpResponse response) throws HttpResponseException {
501 int status = response.getStatusLine().getStatusCode();
502 if (status >= 300) {
503 throw new HttpResponseException(status, response.getStatusLine().getReasonPhrase() + " (" + status + ")");
504 }
505 }
506
507 @Override
508 protected void implClose() {
509 try {
510 client.close();
511 } catch (IOException e) {
512 throw new UncheckedIOException(e);
513 }
514 AuthenticationContext.close(repoAuthContext);
515 AuthenticationContext.close(proxyAuthContext);
516 state.close();
517 }
518
519 private class EntityGetter {
520
521 private final GetTask task;
522
523 EntityGetter(GetTask task) {
524 this.task = task;
525 }
526
527 public void handle(CloseableHttpResponse response) throws IOException, TransferCancelledException {
528 HttpEntity entity = response.getEntity();
529 if (entity == null) {
530 entity = new ByteArrayEntity(new byte[0]);
531 }
532
533 long offset = 0L, length = entity.getContentLength();
534 Header rangeHeader = response.getFirstHeader(HttpHeaders.CONTENT_RANGE);
535 String range = rangeHeader != null ? rangeHeader.getValue() : null;
536 if (range != null) {
537 Matcher m = CONTENT_RANGE_PATTERN.matcher(range);
538 if (!m.matches()) {
539 throw new IOException("Invalid Content-Range header for partial download: " + range);
540 }
541 offset = Long.parseLong(m.group(1));
542 length = Long.parseLong(m.group(2)) + 1L;
543 if (offset < 0L || offset >= length || (offset > 0L && offset != task.getResumeOffset())) {
544 throw new IOException("Invalid Content-Range header for partial download from offset "
545 + task.getResumeOffset() + ": " + range);
546 }
547 }
548
549 final boolean resume = offset > 0L;
550 final File dataFile = task.getDataFile();
551 if (dataFile == null) {
552 try (InputStream is = entity.getContent()) {
553 utilGet(task, is, true, length, resume);
554 extractChecksums(response);
555 }
556 } else {
557 try (FileUtils.CollocatedTempFile tempFile = FileUtils.newTempFile(dataFile.toPath())) {
558 task.setDataFile(tempFile.getPath().toFile(), resume);
559 if (resume && Files.isRegularFile(dataFile.toPath())) {
560 try (InputStream inputStream = Files.newInputStream(dataFile.toPath())) {
561 Files.copy(inputStream, tempFile.getPath(), StandardCopyOption.REPLACE_EXISTING);
562 }
563 }
564 try (InputStream is = entity.getContent()) {
565 utilGet(task, is, true, length, resume);
566 }
567 tempFile.move();
568 } finally {
569 task.setDataFile(dataFile);
570 }
571 }
572 extractChecksums(response);
573 }
574
575 private void extractChecksums(CloseableHttpResponse response) {
576 for (Map.Entry<String, ChecksumExtractor> extractorEntry : checksumExtractors.entrySet()) {
577 Map<String, String> checksums = extractorEntry.getValue().extractChecksums(response);
578 if (checksums != null) {
579 checksums.forEach(task::setChecksum);
580 return;
581 }
582 }
583 }
584 }
585
586 private class PutTaskEntity extends AbstractHttpEntity {
587
588 private final PutTask task;
589
590 PutTaskEntity(PutTask task) {
591 this.task = task;
592 }
593
594 @Override
595 public boolean isRepeatable() {
596 return true;
597 }
598
599 @Override
600 public boolean isStreaming() {
601 return false;
602 }
603
604 @Override
605 public long getContentLength() {
606 return task.getDataLength();
607 }
608
609 @Override
610 public InputStream getContent() throws IOException {
611 return task.newInputStream();
612 }
613
614 @Override
615 public void writeTo(OutputStream os) throws IOException {
616 try {
617 utilPut(task, os, false);
618 } catch (TransferCancelledException e) {
619 throw (IOException) new InterruptedIOException().initCause(e);
620 }
621 }
622 }
623 }