1 package org.eclipse.aether.transport.http;
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22 import org.apache.http.Header;
23 import org.apache.http.HttpEntity;
24 import org.apache.http.HttpEntityEnclosingRequest;
25 import org.apache.http.HttpHeaders;
26 import org.apache.http.HttpHost;
27 import org.apache.http.HttpResponse;
28 import org.apache.http.HttpStatus;
29 import org.apache.http.auth.AuthSchemeProvider;
30 import org.apache.http.auth.AuthScope;
31 import org.apache.http.client.CredentialsProvider;
32 import org.apache.http.client.HttpResponseException;
33 import org.apache.http.client.config.AuthSchemes;
34 import org.apache.http.client.config.RequestConfig;
35 import org.apache.http.client.methods.HttpGet;
36 import org.apache.http.client.methods.HttpHead;
37 import org.apache.http.client.methods.HttpOptions;
38 import org.apache.http.client.methods.HttpPut;
39 import org.apache.http.client.methods.HttpUriRequest;
40 import org.apache.http.client.utils.DateUtils;
41 import org.apache.http.client.utils.URIUtils;
42 import org.apache.http.config.Registry;
43 import org.apache.http.config.RegistryBuilder;
44 import org.apache.http.config.SocketConfig;
45 import org.apache.http.entity.AbstractHttpEntity;
46 import org.apache.http.entity.ByteArrayEntity;
47 import org.apache.http.impl.auth.BasicSchemeFactory;
48 import org.apache.http.impl.auth.DigestSchemeFactory;
49 import org.apache.http.impl.auth.KerberosSchemeFactory;
50 import org.apache.http.impl.auth.NTLMSchemeFactory;
51 import org.apache.http.impl.auth.SPNegoSchemeFactory;
52 import org.apache.http.impl.client.CloseableHttpClient;
53 import org.apache.http.impl.client.HttpClientBuilder;
54 import org.apache.http.util.EntityUtils;
55 import org.eclipse.aether.ConfigurationProperties;
56 import org.eclipse.aether.RepositorySystemSession;
57 import org.eclipse.aether.repository.AuthenticationContext;
58 import org.eclipse.aether.repository.Proxy;
59 import org.eclipse.aether.repository.RemoteRepository;
60 import org.eclipse.aether.spi.connector.transport.AbstractTransporter;
61 import org.eclipse.aether.spi.connector.transport.GetTask;
62 import org.eclipse.aether.spi.connector.transport.PeekTask;
63 import org.eclipse.aether.spi.connector.transport.PutTask;
64 import org.eclipse.aether.spi.connector.transport.TransportTask;
65 import org.eclipse.aether.transfer.NoTransporterException;
66 import org.eclipse.aether.transfer.TransferCancelledException;
67 import org.eclipse.aether.util.ConfigUtils;
68 import org.slf4j.Logger;
69 import org.slf4j.LoggerFactory;
70
71 import java.io.IOException;
72 import java.io.InputStream;
73 import java.io.InterruptedIOException;
74 import java.io.OutputStream;
75 import java.io.UncheckedIOException;
76 import java.net.URI;
77 import java.net.URISyntaxException;
78 import java.nio.charset.Charset;
79 import java.util.Collections;
80 import java.util.Date;
81 import java.util.List;
82 import java.util.Map;
83 import java.util.regex.Matcher;
84 import java.util.regex.Pattern;
85
86 import static java.util.Objects.requireNonNull;
87
88
89
90
91 final class HttpTransporter
92 extends AbstractTransporter
93 {
94
95 private static final Pattern CONTENT_RANGE_PATTERN =
96 Pattern.compile( "\\s*bytes\\s+([0-9]+)\\s*-\\s*([0-9]+)\\s*/.*" );
97
98 private static final Logger LOGGER = LoggerFactory.getLogger( HttpTransporter.class );
99
100 private final Map<String, ChecksumExtractor> checksumExtractors;
101
102 private final AuthenticationContext repoAuthContext;
103
104 private final AuthenticationContext proxyAuthContext;
105
106 private final URI baseUri;
107
108 private final HttpHost server;
109
110 private final HttpHost proxy;
111
112 private final CloseableHttpClient client;
113
114 private final Map<?, ?> headers;
115
116 private final LocalState state;
117
118 HttpTransporter( Map<String, ChecksumExtractor> checksumExtractors,
119 RemoteRepository repository,
120 RepositorySystemSession session )
121 throws NoTransporterException
122 {
123 if ( !"http".equalsIgnoreCase( repository.getProtocol() )
124 && !"https".equalsIgnoreCase( repository.getProtocol() ) )
125 {
126 throw new NoTransporterException( repository );
127 }
128 this.checksumExtractors = requireNonNull( checksumExtractors, "checksum extractors must not be null" );
129 try
130 {
131 this.baseUri = new URI( repository.getUrl() ).parseServerAuthority();
132 if ( baseUri.isOpaque() )
133 {
134 throw new URISyntaxException( repository.getUrl(), "URL must not be opaque" );
135 }
136 this.server = URIUtils.extractHost( baseUri );
137 if ( server == null )
138 {
139 throw new URISyntaxException( repository.getUrl(), "URL lacks host name" );
140 }
141 }
142 catch ( URISyntaxException e )
143 {
144 throw new NoTransporterException( repository, e.getMessage(), e );
145 }
146 this.proxy = toHost( repository.getProxy() );
147
148 this.repoAuthContext = AuthenticationContext.forRepository( session, repository );
149 this.proxyAuthContext = AuthenticationContext.forProxy( session, repository );
150
151 this.state = new LocalState( session, repository, new SslConfig( session, repoAuthContext ) );
152
153 this.headers = ConfigUtils.getMap( session, Collections.emptyMap(),
154 ConfigurationProperties.HTTP_HEADERS + "." + repository.getId(),
155 ConfigurationProperties.HTTP_HEADERS );
156
157 String credentialEncoding = ConfigUtils.getString( session,
158 ConfigurationProperties.DEFAULT_HTTP_CREDENTIAL_ENCODING,
159 ConfigurationProperties.HTTP_CREDENTIAL_ENCODING + "." + repository.getId(),
160 ConfigurationProperties.HTTP_CREDENTIAL_ENCODING );
161 int connectTimeout = ConfigUtils.getInteger( session,
162 ConfigurationProperties.DEFAULT_CONNECT_TIMEOUT,
163 ConfigurationProperties.CONNECT_TIMEOUT + "." + repository.getId(),
164 ConfigurationProperties.CONNECT_TIMEOUT );
165 int requestTimeout = ConfigUtils.getInteger( session,
166 ConfigurationProperties.DEFAULT_REQUEST_TIMEOUT,
167 ConfigurationProperties.REQUEST_TIMEOUT + "." + repository.getId(),
168 ConfigurationProperties.REQUEST_TIMEOUT );
169 String userAgent = ConfigUtils.getString( session,
170 ConfigurationProperties.DEFAULT_USER_AGENT,
171 ConfigurationProperties.USER_AGENT );
172
173 Charset credentialsCharset = Charset.forName( credentialEncoding );
174
175 Registry<AuthSchemeProvider> authSchemeRegistry = RegistryBuilder.<AuthSchemeProvider>create()
176 .register( AuthSchemes.BASIC, new BasicSchemeFactory( credentialsCharset ) )
177 .register( AuthSchemes.DIGEST, new DigestSchemeFactory( credentialsCharset ) )
178 .register( AuthSchemes.NTLM, new NTLMSchemeFactory() )
179 .register( AuthSchemes.SPNEGO, new SPNegoSchemeFactory() )
180 .register( AuthSchemes.KERBEROS, new KerberosSchemeFactory() )
181 .build();
182
183 SocketConfig socketConfig = SocketConfig.custom()
184 .setSoTimeout( requestTimeout ).build();
185
186 RequestConfig requestConfig = RequestConfig.custom()
187 .setConnectTimeout( connectTimeout )
188 .setConnectionRequestTimeout( connectTimeout )
189 .setSocketTimeout( requestTimeout ).build();
190
191 this.client = HttpClientBuilder.create()
192 .setUserAgent( userAgent )
193 .setDefaultSocketConfig( socketConfig )
194 .setDefaultRequestConfig( requestConfig )
195 .setDefaultAuthSchemeRegistry( authSchemeRegistry )
196 .setConnectionManager( state.getConnectionManager() )
197 .setConnectionManagerShared( true )
198 .setDefaultCredentialsProvider(
199 toCredentialsProvider( server, repoAuthContext, proxy, proxyAuthContext )
200 )
201 .setProxy( proxy )
202 .build();
203 }
204
205 private static HttpHost toHost( Proxy proxy )
206 {
207 HttpHost host = null;
208 if ( proxy != null )
209 {
210 host = new HttpHost( proxy.getHost(), proxy.getPort() );
211 }
212 return host;
213 }
214
215 private static CredentialsProvider toCredentialsProvider( HttpHost server, AuthenticationContext serverAuthCtx,
216 HttpHost proxy, AuthenticationContext proxyAuthCtx )
217 {
218 CredentialsProvider provider = toCredentialsProvider( server.getHostName(), AuthScope.ANY_PORT, serverAuthCtx );
219 if ( proxy != null )
220 {
221 CredentialsProvider p = toCredentialsProvider( proxy.getHostName(), proxy.getPort(), proxyAuthCtx );
222 provider = new DemuxCredentialsProvider( provider, p, proxy );
223 }
224 return provider;
225 }
226
227 private static CredentialsProvider toCredentialsProvider( String host, int port, AuthenticationContext ctx )
228 {
229 DeferredCredentialsProvider provider = new DeferredCredentialsProvider();
230 if ( ctx != null )
231 {
232 AuthScope basicScope = new AuthScope( host, port );
233 provider.setCredentials( basicScope, new DeferredCredentialsProvider.BasicFactory( ctx ) );
234
235 AuthScope ntlmScope = new AuthScope( host, port, AuthScope.ANY_REALM, "ntlm" );
236 provider.setCredentials( ntlmScope, new DeferredCredentialsProvider.NtlmFactory( ctx ) );
237 }
238 return provider;
239 }
240
241 LocalState getState()
242 {
243 return state;
244 }
245
246 private URI resolve( TransportTask task )
247 {
248 return UriUtils.resolve( baseUri, task.getLocation() );
249 }
250
251 @Override
252 public int classify( Throwable error )
253 {
254 if ( error instanceof HttpResponseException
255 && ( (HttpResponseException) error ).getStatusCode() == HttpStatus.SC_NOT_FOUND )
256 {
257 return ERROR_NOT_FOUND;
258 }
259 return ERROR_OTHER;
260 }
261
262 @Override
263 protected void implPeek( PeekTask task )
264 throws Exception
265 {
266 HttpHead request = commonHeaders( new HttpHead( resolve( task ) ) );
267 execute( request, null );
268 }
269
270 @Override
271 protected void implGet( GetTask task )
272 throws Exception
273 {
274 boolean resume = true;
275 boolean applyChecksumExtractors = true;
276
277 EntityGetter getter = new EntityGetter( task );
278 HttpGet request = commonHeaders( new HttpGet( resolve( task ) ) );
279 while ( true )
280 {
281 try
282 {
283 if ( resume )
284 {
285 resume( request, task );
286 }
287 if ( applyChecksumExtractors )
288 {
289 for ( ChecksumExtractor checksumExtractor : checksumExtractors.values() )
290 {
291 checksumExtractor.prepareRequest( request );
292 }
293 }
294 execute( request, getter );
295 break;
296 }
297 catch ( HttpResponseException e )
298 {
299 if ( resume && e.getStatusCode() == HttpStatus.SC_PRECONDITION_FAILED
300 && request.containsHeader( HttpHeaders.RANGE ) )
301 {
302 request = commonHeaders( new HttpGet( resolve( task ) ) );
303 resume = false;
304 continue;
305 }
306 if ( applyChecksumExtractors )
307 {
308 boolean retryWithoutExtractors = false;
309 for ( ChecksumExtractor checksumExtractor : checksumExtractors.values() )
310 {
311 if ( checksumExtractor.retryWithoutExtractor( e ) )
312 {
313 retryWithoutExtractors = true;
314 break;
315 }
316 }
317 if ( retryWithoutExtractors )
318 {
319 request = commonHeaders( new HttpGet( resolve( task ) ) );
320 applyChecksumExtractors = false;
321 continue;
322 }
323 }
324 throw e;
325 }
326 }
327 }
328
329 @Override
330 protected void implPut( PutTask task )
331 throws Exception
332 {
333 PutTaskEntity entity = new PutTaskEntity( task );
334 HttpPut request = commonHeaders( entity( new HttpPut( resolve( task ) ), entity ) );
335 try
336 {
337 execute( request, null );
338 }
339 catch ( HttpResponseException e )
340 {
341 if ( e.getStatusCode() == HttpStatus.SC_EXPECTATION_FAILED && request.containsHeader( HttpHeaders.EXPECT ) )
342 {
343 state.setExpectContinue( false );
344 request = commonHeaders( entity( new HttpPut( request.getURI() ), entity ) );
345 execute( request, null );
346 return;
347 }
348 throw e;
349 }
350 }
351
352 private void execute( HttpUriRequest request, EntityGetter getter )
353 throws Exception
354 {
355 try
356 {
357 SharingHttpContext context = new SharingHttpContext( state );
358 prepare( request, context );
359 HttpResponse response = client.execute( server, request, context );
360 try
361 {
362 context.close();
363 handleStatus( response );
364 if ( getter != null )
365 {
366 getter.handle( response );
367 }
368 }
369 finally
370 {
371 EntityUtils.consumeQuietly( response.getEntity() );
372 }
373 }
374 catch ( IOException e )
375 {
376 if ( e.getCause() instanceof TransferCancelledException )
377 {
378 throw (Exception) e.getCause();
379 }
380 throw e;
381 }
382 }
383
384 private void prepare( HttpUriRequest request, SharingHttpContext context )
385 {
386 boolean put = HttpPut.METHOD_NAME.equalsIgnoreCase( request.getMethod() );
387 if ( state.getWebDav() == null && ( put || isPayloadPresent( request ) ) )
388 {
389 try
390 {
391 HttpOptions req = commonHeaders( new HttpOptions( request.getURI() ) );
392 HttpResponse response = client.execute( server, req, context );
393 state.setWebDav( isWebDav( response ) );
394 EntityUtils.consumeQuietly( response.getEntity() );
395 }
396 catch ( IOException e )
397 {
398 LOGGER.debug( "Failed to prepare HTTP context", e );
399 }
400 }
401 if ( put && Boolean.TRUE.equals( state.getWebDav() ) )
402 {
403 mkdirs( request.getURI(), context );
404 }
405 }
406
407 private boolean isWebDav( HttpResponse response )
408 {
409 return response.containsHeader( HttpHeaders.DAV );
410 }
411
412 @SuppressWarnings( "checkstyle:magicnumber" )
413 private void mkdirs( URI uri, SharingHttpContext context )
414 {
415 List<URI> dirs = UriUtils.getDirectories( baseUri, uri );
416 int index = 0;
417 for ( ; index < dirs.size(); index++ )
418 {
419 try
420 {
421 HttpResponse response =
422 client.execute( server, commonHeaders( new HttpMkCol( dirs.get( index ) ) ), context );
423 try
424 {
425 int status = response.getStatusLine().getStatusCode();
426 if ( status < 300 || status == HttpStatus.SC_METHOD_NOT_ALLOWED )
427 {
428 break;
429 }
430 else if ( status == HttpStatus.SC_CONFLICT )
431 {
432 continue;
433 }
434 handleStatus( response );
435 }
436 finally
437 {
438 EntityUtils.consumeQuietly( response.getEntity() );
439 }
440 }
441 catch ( IOException e )
442 {
443 LOGGER.debug( "Failed to create parent directory {}", dirs.get( index ), e );
444 return;
445 }
446 }
447 for ( index--; index >= 0; index-- )
448 {
449 try
450 {
451 HttpResponse response =
452 client.execute( server, commonHeaders( new HttpMkCol( dirs.get( index ) ) ), context );
453 try
454 {
455 handleStatus( response );
456 }
457 finally
458 {
459 EntityUtils.consumeQuietly( response.getEntity() );
460 }
461 }
462 catch ( IOException e )
463 {
464 LOGGER.debug( "Failed to create parent directory {}", dirs.get( index ), e );
465 return;
466 }
467 }
468 }
469
470 private <T extends HttpEntityEnclosingRequest> T entity( T request, HttpEntity entity )
471 {
472 request.setEntity( entity );
473 return request;
474 }
475
476 private boolean isPayloadPresent( HttpUriRequest request )
477 {
478 if ( request instanceof HttpEntityEnclosingRequest )
479 {
480 HttpEntity entity = ( (HttpEntityEnclosingRequest) request ).getEntity();
481 return entity != null && entity.getContentLength() != 0;
482 }
483 return false;
484 }
485
486 private <T extends HttpUriRequest> T commonHeaders( T request )
487 {
488 request.setHeader( HttpHeaders.CACHE_CONTROL, "no-cache, no-store" );
489 request.setHeader( HttpHeaders.PRAGMA, "no-cache" );
490
491 if ( state.isExpectContinue() && isPayloadPresent( request ) )
492 {
493 request.setHeader( HttpHeaders.EXPECT, "100-continue" );
494 }
495
496 for ( Map.Entry<?, ?> entry : headers.entrySet() )
497 {
498 if ( !( entry.getKey() instanceof String ) )
499 {
500 continue;
501 }
502 if ( entry.getValue() instanceof String )
503 {
504 request.setHeader( entry.getKey().toString(), entry.getValue().toString() );
505 }
506 else
507 {
508 request.removeHeaders( entry.getKey().toString() );
509 }
510 }
511
512 if ( !state.isExpectContinue() )
513 {
514 request.removeHeaders( HttpHeaders.EXPECT );
515 }
516
517 return request;
518 }
519
520 @SuppressWarnings( "checkstyle:magicnumber" )
521 private <T extends HttpUriRequest> T resume( T request, GetTask task )
522 {
523 long resumeOffset = task.getResumeOffset();
524 if ( resumeOffset > 0L && task.getDataFile() != null )
525 {
526 request.setHeader( HttpHeaders.RANGE, "bytes=" + resumeOffset + '-' );
527 request.setHeader( HttpHeaders.IF_UNMODIFIED_SINCE,
528 DateUtils.formatDate( new Date( task.getDataFile().lastModified() - 60L * 1000L ) ) );
529 request.setHeader( HttpHeaders.ACCEPT_ENCODING, "identity" );
530 }
531 return request;
532 }
533
534 @SuppressWarnings( "checkstyle:magicnumber" )
535 private void handleStatus( HttpResponse response )
536 throws HttpResponseException
537 {
538 int status = response.getStatusLine().getStatusCode();
539 if ( status >= 300 )
540 {
541 throw new HttpResponseException( status, response.getStatusLine().getReasonPhrase() + " (" + status + ")" );
542 }
543 }
544
545 @Override
546 protected void implClose()
547 {
548 try
549 {
550 client.close();
551 }
552 catch ( IOException e )
553 {
554 throw new UncheckedIOException( e );
555 }
556 AuthenticationContext.close( repoAuthContext );
557 AuthenticationContext.close( proxyAuthContext );
558 state.close();
559 }
560
561 private class EntityGetter
562 {
563
564 private final GetTask task;
565
566 EntityGetter( GetTask task )
567 {
568 this.task = task;
569 }
570
571 public void handle( HttpResponse response )
572 throws IOException, TransferCancelledException
573 {
574 HttpEntity entity = response.getEntity();
575 if ( entity == null )
576 {
577 entity = new ByteArrayEntity( new byte[0] );
578 }
579
580 long offset = 0L, length = entity.getContentLength();
581 Header rangeHeader = response.getFirstHeader( HttpHeaders.CONTENT_RANGE );
582 String range = rangeHeader != null ? rangeHeader.getValue() : null;
583 if ( range != null )
584 {
585 Matcher m = CONTENT_RANGE_PATTERN.matcher( range );
586 if ( !m.matches() )
587 {
588 throw new IOException( "Invalid Content-Range header for partial download: " + range );
589 }
590 offset = Long.parseLong( m.group( 1 ) );
591 length = Long.parseLong( m.group( 2 ) ) + 1L;
592 if ( offset < 0L || offset >= length || ( offset > 0L && offset != task.getResumeOffset() ) )
593 {
594 throw new IOException( "Invalid Content-Range header for partial download from offset "
595 + task.getResumeOffset() + ": " + range );
596 }
597 }
598
599 InputStream is = entity.getContent();
600 utilGet( task, is, true, length, offset > 0L );
601 extractChecksums( response );
602 }
603
604 private void extractChecksums( HttpResponse response )
605 {
606 for ( Map.Entry<String, ChecksumExtractor> extractorEntry : checksumExtractors.entrySet() )
607 {
608 Map<String, String> checksums = extractorEntry.getValue().extractChecksums( response );
609 if ( checksums != null )
610 {
611 checksums.forEach( task::setChecksum );
612 return;
613 }
614 }
615 }
616 }
617
618 private class PutTaskEntity
619 extends AbstractHttpEntity
620 {
621
622 private final PutTask task;
623
624 PutTaskEntity( PutTask task )
625 {
626 this.task = task;
627 }
628
629 @Override
630 public boolean isRepeatable()
631 {
632 return true;
633 }
634
635 @Override
636 public boolean isStreaming()
637 {
638 return false;
639 }
640
641 @Override
642 public long getContentLength()
643 {
644 return task.getDataLength();
645 }
646
647 @Override
648 public InputStream getContent()
649 throws IOException
650 {
651 return task.newInputStream();
652 }
653
654 @Override
655 public void writeTo( OutputStream os )
656 throws IOException
657 {
658 try
659 {
660 utilPut( task, os, false );
661 }
662 catch ( TransferCancelledException e )
663 {
664 throw (IOException) new InterruptedIOException().initCause( e );
665 }
666 }
667
668 }
669
670 }