View Javadoc
1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *   http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing,
13   * software distributed under the License is distributed on an
14   * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15   * KIND, either express or implied.  See the License for the
16   * specific language governing permissions and limitations
17   * under the License.
18   */
19  package org.eclipse.aether.transport.jetty;
20  
21  import javax.net.ssl.SSLContext;
22  
23  import java.io.File;
24  import java.io.IOException;
25  import java.io.InputStream;
26  import java.net.URI;
27  import java.net.URISyntaxException;
28  import java.nio.file.Files;
29  import java.nio.file.StandardCopyOption;
30  import java.util.Collections;
31  import java.util.HashMap;
32  import java.util.Map;
33  import java.util.concurrent.ExecutionException;
34  import java.util.concurrent.TimeUnit;
35  import java.util.regex.Matcher;
36  import java.util.regex.Pattern;
37  
38  import org.eclipse.aether.ConfigurationProperties;
39  import org.eclipse.aether.RepositorySystemSession;
40  import org.eclipse.aether.repository.AuthenticationContext;
41  import org.eclipse.aether.repository.RemoteRepository;
42  import org.eclipse.aether.spi.connector.transport.AbstractTransporter;
43  import org.eclipse.aether.spi.connector.transport.GetTask;
44  import org.eclipse.aether.spi.connector.transport.PeekTask;
45  import org.eclipse.aether.spi.connector.transport.PutTask;
46  import org.eclipse.aether.spi.connector.transport.TransportTask;
47  import org.eclipse.aether.transfer.NoTransporterException;
48  import org.eclipse.aether.util.ConfigUtils;
49  import org.eclipse.aether.util.FileUtils;
50  import org.eclipse.jetty.client.HttpClient;
51  import org.eclipse.jetty.client.HttpProxy;
52  import org.eclipse.jetty.client.api.Authentication;
53  import org.eclipse.jetty.client.api.Request;
54  import org.eclipse.jetty.client.api.Response;
55  import org.eclipse.jetty.client.dynamic.HttpClientTransportDynamic;
56  import org.eclipse.jetty.client.http.HttpClientConnectionFactory;
57  import org.eclipse.jetty.client.util.BasicAuthentication;
58  import org.eclipse.jetty.client.util.InputStreamResponseListener;
59  import org.eclipse.jetty.client.util.PathRequestContent;
60  import org.eclipse.jetty.http.HttpHeader;
61  import org.eclipse.jetty.http2.client.HTTP2Client;
62  import org.eclipse.jetty.http2.client.http.ClientConnectionFactoryOverHTTP2;
63  import org.eclipse.jetty.io.ClientConnector;
64  import org.eclipse.jetty.util.ssl.SslContextFactory;
65  
66  /**
67   * A transporter for HTTP/HTTPS.
68   *
69   * @since TBD
70   */
71  final class JettyTransporter extends AbstractTransporter {
72      private static final int MULTIPLE_CHOICES = 300;
73  
74      private static final int NOT_FOUND = 404;
75  
76      private static final int PRECONDITION_FAILED = 412;
77  
78      private static final long MODIFICATION_THRESHOLD = 60L * 1000L;
79  
80      private static final String ACCEPT_ENCODING = "Accept-Encoding";
81  
82      private static final String CONTENT_LENGTH = "Content-Length";
83  
84      private static final String CONTENT_RANGE = "Content-Range";
85  
86      private static final String IF_UNMODIFIED_SINCE = "If-Unmodified-Since";
87  
88      private static final String RANGE = "Range";
89  
90      private static final String USER_AGENT = "User-Agent";
91  
92      private static final Pattern CONTENT_RANGE_PATTERN =
93              Pattern.compile("\\s*bytes\\s+([0-9]+)\\s*-\\s*([0-9]+)\\s*/.*");
94  
95      private final URI baseUri;
96  
97      private final HttpClient client;
98  
99      private final int requestTimeout;
100 
101     private final Map<String, String> headers;
102 
103     JettyTransporter(RepositorySystemSession session, RemoteRepository repository) throws NoTransporterException {
104         try {
105             URI uri = new URI(repository.getUrl()).parseServerAuthority();
106             if (uri.isOpaque()) {
107                 throw new URISyntaxException(repository.getUrl(), "URL must not be opaque");
108             }
109             if (uri.getRawFragment() != null || uri.getRawQuery() != null) {
110                 throw new URISyntaxException(repository.getUrl(), "URL must not have fragment or query");
111             }
112             String path = uri.getPath();
113             if (path == null) {
114                 path = "/";
115             }
116             if (!path.startsWith("/")) {
117                 path = "/" + path;
118             }
119             if (!path.endsWith("/")) {
120                 path = path + "/";
121             }
122             this.baseUri = URI.create(uri.getScheme() + "://" + uri.getRawAuthority() + path);
123         } catch (URISyntaxException e) {
124             throw new NoTransporterException(repository, e.getMessage(), e);
125         }
126 
127         HashMap<String, String> headers = new HashMap<>();
128         String userAgent = ConfigUtils.getString(
129                 session, ConfigurationProperties.DEFAULT_USER_AGENT, ConfigurationProperties.USER_AGENT);
130         if (userAgent != null) {
131             headers.put(USER_AGENT, userAgent);
132         }
133         @SuppressWarnings("unchecked")
134         Map<Object, Object> configuredHeaders = (Map<Object, Object>) ConfigUtils.getMap(
135                 session,
136                 Collections.emptyMap(),
137                 ConfigurationProperties.HTTP_HEADERS + "." + repository.getId(),
138                 ConfigurationProperties.HTTP_HEADERS);
139         if (configuredHeaders != null) {
140             configuredHeaders.forEach((k, v) -> headers.put(String.valueOf(k), v != null ? String.valueOf(v) : null));
141         }
142 
143         this.headers = headers;
144 
145         this.requestTimeout = ConfigUtils.getInteger(
146                 session,
147                 ConfigurationProperties.DEFAULT_REQUEST_TIMEOUT,
148                 ConfigurationProperties.REQUEST_TIMEOUT + "." + repository.getId(),
149                 ConfigurationProperties.REQUEST_TIMEOUT);
150 
151         this.client = getOrCreateClient(session, repository);
152     }
153 
154     private URI resolve(TransportTask task) {
155         return baseUri.resolve(task.getLocation());
156     }
157 
158     @Override
159     public int classify(Throwable error) {
160         if (error instanceof JettyException && ((JettyException) error).getStatusCode() == NOT_FOUND) {
161             return ERROR_NOT_FOUND;
162         }
163         return ERROR_OTHER;
164     }
165 
166     @Override
167     protected void implPeek(PeekTask task) throws Exception {
168         Request request = client.newRequest(resolve(task))
169                 .timeout(requestTimeout, TimeUnit.MILLISECONDS)
170                 .method("HEAD");
171         request.headers(m -> headers.forEach(m::add));
172         Response response = request.send();
173         if (response.getStatus() >= MULTIPLE_CHOICES) {
174             throw new JettyException(response.getStatus());
175         }
176     }
177 
178     @Override
179     protected void implGet(GetTask task) throws Exception {
180         boolean resume = task.getResumeOffset() > 0L && task.getDataFile() != null;
181         Response response;
182         InputStreamResponseListener listener;
183 
184         while (true) {
185             Request request = client.newRequest(resolve(task))
186                     .timeout(requestTimeout, TimeUnit.MILLISECONDS)
187                     .method("GET");
188             request.headers(m -> headers.forEach(m::add));
189 
190             if (resume) {
191                 long resumeOffset = task.getResumeOffset();
192                 request.headers(h -> {
193                     h.add(RANGE, "bytes=" + resumeOffset + '-');
194                     h.addDateField(IF_UNMODIFIED_SINCE, task.getDataFile().lastModified() - MODIFICATION_THRESHOLD);
195                     h.remove(HttpHeader.ACCEPT_ENCODING);
196                     h.add(ACCEPT_ENCODING, "identity");
197                 });
198             }
199 
200             listener = new InputStreamResponseListener();
201             request.send(listener);
202             try {
203                 response = listener.get(requestTimeout, TimeUnit.MILLISECONDS);
204             } catch (ExecutionException e) {
205                 Throwable t = e.getCause();
206                 if (t instanceof Exception) {
207                     throw (Exception) t;
208                 } else {
209                     throw new RuntimeException(t);
210                 }
211             }
212             if (response.getStatus() >= MULTIPLE_CHOICES) {
213                 if (resume && response.getStatus() == PRECONDITION_FAILED) {
214                     resume = false;
215                     continue;
216                 }
217                 throw new JettyException(response.getStatus());
218             }
219             break;
220         }
221 
222         long offset = 0L, length = response.getHeaders().getLongField(CONTENT_LENGTH);
223         if (resume) {
224             String range = response.getHeaders().get(CONTENT_RANGE);
225             if (range != null) {
226                 Matcher m = CONTENT_RANGE_PATTERN.matcher(range);
227                 if (!m.matches()) {
228                     throw new IOException("Invalid Content-Range header for partial download: " + range);
229                 }
230                 offset = Long.parseLong(m.group(1));
231                 length = Long.parseLong(m.group(2)) + 1L;
232                 if (offset < 0L || offset >= length || (offset > 0L && offset != task.getResumeOffset())) {
233                     throw new IOException("Invalid Content-Range header for partial download from offset "
234                             + task.getResumeOffset() + ": " + range);
235                 }
236             }
237         }
238 
239         final boolean downloadResumed = offset > 0L;
240         final File dataFile = task.getDataFile();
241         if (dataFile == null) {
242             try (InputStream is = listener.getInputStream()) {
243                 utilGet(task, is, true, length, downloadResumed);
244             }
245         } else {
246             try (FileUtils.CollocatedTempFile tempFile = FileUtils.newTempFile(dataFile.toPath())) {
247                 task.setDataFile(tempFile.getPath().toFile(), downloadResumed);
248                 if (downloadResumed && Files.isRegularFile(dataFile.toPath())) {
249                     try (InputStream inputStream = Files.newInputStream(dataFile.toPath())) {
250                         Files.copy(inputStream, tempFile.getPath(), StandardCopyOption.REPLACE_EXISTING);
251                     }
252                 }
253                 try (InputStream is = listener.getInputStream()) {
254                     utilGet(task, is, true, length, downloadResumed);
255                 }
256                 tempFile.move();
257             } finally {
258                 task.setDataFile(dataFile);
259             }
260         }
261         Map<String, String> checksums = extractXChecksums(response);
262         if (checksums != null) {
263             checksums.forEach(task::setChecksum);
264             return;
265         }
266         checksums = extractNexus2Checksums(response);
267         if (checksums != null) {
268             checksums.forEach(task::setChecksum);
269         }
270     }
271 
272     private static Map<String, String> extractXChecksums(Response response) {
273         String value;
274         HashMap<String, String> result = new HashMap<>();
275         // Central style: x-checksum-sha1: c74edb60ca2a0b57ef88d9a7da28f591e3d4ce7b
276         value = response.getHeaders().get("x-checksum-sha1");
277         if (value != null) {
278             result.put("SHA-1", value);
279         }
280         // Central style: x-checksum-md5: 9ad0d8e3482767c122e85f83567b8ce6
281         value = response.getHeaders().get("x-checksum-md5");
282         if (value != null) {
283             result.put("MD5", value);
284         }
285         if (!result.isEmpty()) {
286             return result;
287         }
288         // Google style: x-goog-meta-checksum-sha1: c74edb60ca2a0b57ef88d9a7da28f591e3d4ce7b
289         value = response.getHeaders().get("x-goog-meta-checksum-sha1");
290         if (value != null) {
291             result.put("SHA-1", value);
292         }
293         // Central style: x-goog-meta-checksum-sha1: 9ad0d8e3482767c122e85f83567b8ce6
294         value = response.getHeaders().get("x-goog-meta-checksum-md5");
295         if (value != null) {
296             result.put("MD5", value);
297         }
298 
299         return result.isEmpty() ? null : result;
300     }
301 
302     private static Map<String, String> extractNexus2Checksums(Response response) {
303         // Nexus-style, ETag: "{SHA1{d40d68ba1f88d8e9b0040f175a6ff41928abd5e7}}"
304         String etag = response.getHeaders().get("ETag");
305         if (etag != null) {
306             int start = etag.indexOf("SHA1{"), end = etag.indexOf("}", start + 5);
307             if (start >= 0 && end > start) {
308                 return Collections.singletonMap("SHA-1", etag.substring(start + 5, end));
309             }
310         }
311         return null;
312     }
313 
314     @Override
315     protected void implPut(PutTask task) throws Exception {
316         Request request = client.newRequest(resolve(task)).method("PUT").timeout(requestTimeout, TimeUnit.MILLISECONDS);
317         request.headers(m -> headers.forEach(m::add));
318         try (FileUtils.TempFile tempFile = FileUtils.newTempFile()) {
319             utilPut(task, Files.newOutputStream(tempFile.getPath()), true);
320             request.body(new PathRequestContent(tempFile.getPath()));
321 
322             Response response;
323             try {
324                 response = request.send();
325             } catch (ExecutionException e) {
326                 Throwable t = e.getCause();
327                 if (t instanceof Exception) {
328                     throw (Exception) t;
329                 } else {
330                     throw new RuntimeException(t);
331                 }
332             }
333             if (response.getStatus() >= MULTIPLE_CHOICES) {
334                 throw new JettyException(response.getStatus());
335             }
336         }
337     }
338 
339     @Override
340     protected void implClose() {
341         // noop
342     }
343 
344     /**
345      * Visible for testing.
346      */
347     static final String JETTY_INSTANCE_KEY_PREFIX = JettyTransporterFactory.class.getName() + ".jetty.";
348 
349     @SuppressWarnings("checkstyle:methodlength")
350     private static HttpClient getOrCreateClient(RepositorySystemSession session, RemoteRepository repository)
351             throws NoTransporterException {
352 
353         final String instanceKey = JETTY_INSTANCE_KEY_PREFIX + repository.getId();
354 
355         try {
356             return (HttpClient) session.getData().computeIfAbsent(instanceKey, () -> {
357                 SSLContext sslContext = null;
358                 BasicAuthentication basicAuthentication = null;
359                 try {
360                     try (AuthenticationContext repoAuthContext =
361                             AuthenticationContext.forRepository(session, repository)) {
362                         if (repoAuthContext != null) {
363                             sslContext = repoAuthContext.get(AuthenticationContext.SSL_CONTEXT, SSLContext.class);
364 
365                             String username = repoAuthContext.get(AuthenticationContext.USERNAME);
366                             String password = repoAuthContext.get(AuthenticationContext.PASSWORD);
367 
368                             basicAuthentication = new BasicAuthentication(
369                                     URI.create(repository.getUrl()), Authentication.ANY_REALM, username, password);
370                         }
371                     }
372 
373                     if (sslContext == null) {
374                         sslContext = SSLContext.getDefault();
375                     }
376 
377                     int connectTimeout = ConfigUtils.getInteger(
378                             session,
379                             ConfigurationProperties.DEFAULT_CONNECT_TIMEOUT,
380                             ConfigurationProperties.CONNECT_TIMEOUT + "." + repository.getId(),
381                             ConfigurationProperties.CONNECT_TIMEOUT);
382 
383                     SslContextFactory.Client sslContextFactory = new SslContextFactory.Client();
384                     sslContextFactory.setSslContext(sslContext);
385 
386                     ClientConnector clientConnector = new ClientConnector();
387                     clientConnector.setSslContextFactory(sslContextFactory);
388 
389                     HTTP2Client http2Client = new HTTP2Client(clientConnector);
390                     ClientConnectionFactoryOverHTTP2.HTTP2 http2 =
391                             new ClientConnectionFactoryOverHTTP2.HTTP2(http2Client);
392 
393                     HttpClientTransportDynamic transport;
394                     if ("https".equalsIgnoreCase(repository.getProtocol())) {
395                         transport = new HttpClientTransportDynamic(
396                                 clientConnector, http2, HttpClientConnectionFactory.HTTP11); // HTTPS, prefer H2
397                     } else {
398                         transport = new HttpClientTransportDynamic(
399                                 clientConnector,
400                                 HttpClientConnectionFactory.HTTP11,
401                                 http2); // plaintext HTTP, H2 cannot be used
402                     }
403 
404                     HttpClient httpClient = new HttpClient(transport);
405                     httpClient.setConnectTimeout(connectTimeout);
406                     httpClient.setFollowRedirects(true);
407                     httpClient.setMaxRedirects(2);
408 
409                     httpClient.setUserAgentField(null); // we manage it
410 
411                     if (basicAuthentication != null) {
412                         httpClient.getAuthenticationStore().addAuthentication(basicAuthentication);
413                     }
414 
415                     if (repository.getProxy() != null) {
416                         HttpProxy proxy = new HttpProxy(
417                                 repository.getProxy().getHost(),
418                                 repository.getProxy().getPort());
419 
420                         httpClient.getProxyConfiguration().addProxy(proxy);
421                         try (AuthenticationContext proxyAuthContext =
422                                 AuthenticationContext.forProxy(session, repository)) {
423                             if (proxyAuthContext != null) {
424                                 String username = proxyAuthContext.get(AuthenticationContext.USERNAME);
425                                 String password = proxyAuthContext.get(AuthenticationContext.PASSWORD);
426 
427                                 BasicAuthentication proxyAuthentication = new BasicAuthentication(
428                                         proxy.getURI(), Authentication.ANY_REALM, username, password);
429 
430                                 httpClient.getAuthenticationStore().addAuthentication(proxyAuthentication);
431                             }
432                         }
433                     }
434                     httpClient.start();
435                     return httpClient;
436                 } catch (Exception e) {
437                     throw new WrapperEx(e);
438                 }
439             });
440         } catch (WrapperEx e) {
441             throw new NoTransporterException(repository, e.getCause());
442         }
443     }
444 
445     private static final class WrapperEx extends RuntimeException {
446         private WrapperEx(Throwable cause) {
447             super(cause);
448         }
449     }
450 }