View Javadoc
1   /*
2    * ====================================================================
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *   http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing,
14   * software distributed under the License is distributed on an
15   * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16   * KIND, either express or implied.  See the License for the
17   * specific language governing permissions and limitations
18   * under the License.
19   * ====================================================================
20   *
21   * This software consists of voluntary contributions made by many
22   * individuals on behalf of the Apache Software Foundation.  For more
23   * information on the Apache Software Foundation, please see
24   * <http://www.apache.org/>.
25   *
26   */
27  
28  package org.apache.hc.client5.testing.sync;
29  
30  import java.io.ByteArrayInputStream;
31  import java.io.ByteArrayOutputStream;
32  import java.io.IOException;
33  import java.io.OutputStream;
34  import java.nio.charset.StandardCharsets;
35  import java.util.ArrayList;
36  import java.util.Iterator;
37  import java.util.List;
38  import java.util.concurrent.CountDownLatch;
39  import java.util.concurrent.ExecutorService;
40  import java.util.concurrent.Executors;
41  import java.util.zip.Deflater;
42  import java.util.zip.GZIPOutputStream;
43  
44  import org.apache.hc.client5.http.classic.methods.HttpGet;
45  import org.apache.hc.client5.http.impl.classic.BasicHttpClientResponseHandler;
46  import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
47  import org.apache.hc.core5.http.ClassicHttpRequest;
48  import org.apache.hc.core5.http.ClassicHttpResponse;
49  import org.apache.hc.core5.http.HeaderElement;
50  import org.apache.hc.core5.http.HttpException;
51  import org.apache.hc.core5.http.HttpHost;
52  import org.apache.hc.core5.http.HttpStatus;
53  import org.apache.hc.core5.http.io.HttpRequestHandler;
54  import org.apache.hc.core5.http.io.entity.EntityUtils;
55  import org.apache.hc.core5.http.io.entity.InputStreamEntity;
56  import org.apache.hc.core5.http.io.entity.StringEntity;
57  import org.apache.hc.core5.http.message.MessageSupport;
58  import org.apache.hc.core5.http.protocol.HttpContext;
59  import org.junit.Assert;
60  import org.junit.Test;
61  
62  /**
63   * Test case for how Content Codings are processed. By default, we want to do the right thing and
64   * require no intervention from the user of HttpClient, but we still want to let clients do their
65   * own thing if they so wish.
66   */
67  public class TestContentCodings extends LocalServerTestBase {
68  
69      /**
70       * Test for when we don't get an entity back; e.g. for a 204 or 304 response; nothing blows
71       * up with the new behaviour.
72       *
73       * @throws Exception
74       *             if there was a problem
75       */
76      @Test
77      public void testResponseWithNoContent() throws Exception {
78          this.server.registerHandler("*", new HttpRequestHandler() {
79  
80              /**
81               * {@inheritDoc}
82               */
83              @Override
84              public void handle(
85                      final ClassicHttpRequest request,
86                      final ClassicHttpResponse response,
87                      final HttpContext context) throws HttpException, IOException {
88                  response.setCode(HttpStatus.SC_NO_CONTENT);
89              }
90          });
91  
92          final HttpHost target = start();
93  
94          final HttpGet request = new HttpGet("/some-resource");
95          final ClassicHttpResponse response = this.httpclient.execute(target, request);
96          Assert.assertEquals(HttpStatus.SC_NO_CONTENT, response.getCode());
97          Assert.assertNull(response.getEntity());
98      }
99  
100     /**
101      * Test for when we are handling content from a server that has correctly interpreted RFC2616
102      * to return RFC1950 streams for {@code deflate} content coding.
103      *
104      * @throws Exception
105      */
106     @Test
107     public void testDeflateSupportForServerReturningRfc1950Stream() throws Exception {
108         final String entityText = "Hello, this is some plain text coming back.";
109 
110         this.server.registerHandler("*", createDeflateEncodingRequestHandler(entityText, false));
111 
112         final HttpHost target = start();
113 
114         final HttpGet request = new HttpGet("/some-resource");
115         final ClassicHttpResponse response = this.httpclient.execute(target, request);
116         Assert.assertEquals("The entity text is correctly transported", entityText,
117                 EntityUtils.toString(response.getEntity()));
118     }
119 
120     /**
121      * Test for when we are handling content from a server that has incorrectly interpreted RFC2616
122      * to return RFC1951 streams for {@code deflate} content coding.
123      *
124      * @throws Exception
125      */
126     @Test
127     public void testDeflateSupportForServerReturningRfc1951Stream() throws Exception {
128         final String entityText = "Hello, this is some plain text coming back.";
129 
130         this.server.registerHandler("*", createDeflateEncodingRequestHandler(entityText, true));
131 
132         final HttpHost target = start();
133 
134         final HttpGet request = new HttpGet("/some-resource");
135         final ClassicHttpResponse response = this.httpclient.execute(target, request);
136         Assert.assertEquals("The entity text is correctly transported", entityText,
137                 EntityUtils.toString(response.getEntity()));
138     }
139 
140     /**
141      * Test for a server returning gzipped content.
142      *
143      * @throws Exception
144      */
145     @Test
146     public void testGzipSupport() throws Exception {
147         final String entityText = "Hello, this is some plain text coming back.";
148 
149         this.server.registerHandler("*", createGzipEncodingRequestHandler(entityText));
150 
151         final HttpHost target = start();
152 
153         final HttpGet request = new HttpGet("/some-resource");
154         final ClassicHttpResponse response = this.httpclient.execute(target, request);
155         Assert.assertEquals("The entity text is correctly transported", entityText,
156                 EntityUtils.toString(response.getEntity()));
157     }
158 
159     /**
160      * Try with a bunch of client threads, to check that it's thread-safe.
161      *
162      * @throws Exception
163      *             if there was a problem
164      */
165     @Test
166     public void testThreadSafetyOfContentCodings() throws Exception {
167         final String entityText = "Hello, this is some plain text coming back.";
168 
169         this.server.registerHandler("*", createGzipEncodingRequestHandler(entityText));
170 
171         /*
172          * Create a load of workers which will access the resource. Half will use the default
173          * gzip behaviour; half will require identity entity.
174          */
175         final int clients = 100;
176 
177         this.connManager.setMaxTotal(clients);
178 
179         final HttpHost target = start();
180 
181         final ExecutorService executor = Executors.newFixedThreadPool(clients);
182 
183         final CountDownLatch startGate = new CountDownLatch(1);
184         final CountDownLatch endGate = new CountDownLatch(clients);
185 
186         final List<WorkerTask> workers = new ArrayList<>();
187 
188         for (int i = 0; i < clients; ++i) {
189             workers.add(new WorkerTask(this.httpclient, target, i % 2 == 0, startGate, endGate));
190         }
191 
192         for (final WorkerTask workerTask : workers) {
193 
194             /* Set them all in motion, but they will block until we call startGate.countDown(). */
195             executor.execute(workerTask);
196         }
197 
198         startGate.countDown();
199 
200         /* Wait for the workers to complete. */
201         endGate.await();
202 
203         for (final WorkerTask workerTask : workers) {
204             if (workerTask.isFailed()) {
205                 Assert.fail("A worker failed");
206             }
207             Assert.assertEquals(entityText, workerTask.getText());
208         }
209     }
210 
211     @Test
212     public void testHttpEntityWriteToForGzip() throws Exception {
213         final String entityText = "Hello, this is some plain text coming back.";
214 
215         this.server.registerHandler("*", createGzipEncodingRequestHandler(entityText));
216 
217         final HttpHost target = start();
218 
219         final HttpGet request = new HttpGet("/some-resource");
220         final ClassicHttpResponse response = this.httpclient.execute(target, request);
221         final ByteArrayOutputStream out = new ByteArrayOutputStream();
222 
223         response.getEntity().writeTo(out);
224 
225         Assert.assertEquals(entityText, out.toString("utf-8"));
226     }
227 
228     @Test
229     public void testHttpEntityWriteToForDeflate() throws Exception {
230         final String entityText = "Hello, this is some plain text coming back.";
231 
232         this.server.registerHandler("*", createDeflateEncodingRequestHandler(entityText, true));
233 
234         final HttpHost target = start();
235 
236         final HttpGet request = new HttpGet("/some-resource");
237         final ClassicHttpResponse response = this.httpclient.execute(target, request);
238         final ByteArrayOutputStream out = new ByteArrayOutputStream();
239 
240         response.getEntity().writeTo(out);
241 
242         Assert.assertEquals(entityText, out.toString("utf-8"));
243     }
244 
245     @Test
246     public void gzipResponsesWorkWithBasicResponseHandler() throws Exception {
247         final String entityText = "Hello, this is some plain text coming back.";
248 
249         this.server.registerHandler("*", createGzipEncodingRequestHandler(entityText));
250 
251         final HttpHost target = start();
252 
253         final HttpGet request = new HttpGet("/some-resource");
254         final String response = this.httpclient.execute(target, request, new BasicHttpClientResponseHandler());
255         Assert.assertEquals("The entity text is correctly transported", entityText, response);
256     }
257 
258     @Test
259     public void deflateResponsesWorkWithBasicResponseHandler() throws Exception {
260         final String entityText = "Hello, this is some plain text coming back.";
261 
262         this.server.registerHandler("*", createDeflateEncodingRequestHandler(entityText, false));
263 
264         final HttpHost target = start();
265 
266         final HttpGet request = new HttpGet("/some-resource");
267         final String response = this.httpclient.execute(target, request, new BasicHttpClientResponseHandler());
268         Assert.assertEquals("The entity text is correctly transported", entityText, response);
269     }
270 
271     /**
272      * Creates a new {@link HttpRequestHandler} that will attempt to provide a deflate stream
273      * Content-Coding.
274      *
275      * @param entityText
276      *            the non-null String entity text to be returned by the server
277      * @param rfc1951
278      *            if true, then the stream returned will be a raw RFC1951 deflate stream, which
279      *            some servers return as a result of misinterpreting the HTTP 1.1 RFC. If false,
280      *            then it will return an RFC2616 compliant deflate encoded zlib stream.
281      * @return a non-null {@link HttpRequestHandler}
282      */
283     private HttpRequestHandler createDeflateEncodingRequestHandler(
284             final String entityText, final boolean rfc1951) {
285         return new HttpRequestHandler() {
286 
287             /**
288              * {@inheritDoc}
289              */
290             @Override
291             public void handle(
292                     final ClassicHttpRequest request,
293                     final ClassicHttpResponse response,
294                     final HttpContext context) throws HttpException, IOException {
295                 response.setEntity(new StringEntity(entityText));
296                 response.addHeader("Content-Type", "text/plain");
297                 final Iterator<HeaderElement> it = MessageSupport.iterate(request, "Accept-Encoding");
298                 while (it.hasNext()) {
299                     final HeaderElement element = it.next();
300                     if ("deflate".equalsIgnoreCase(element.getName())) {
301                         response.addHeader("Content-Encoding", "deflate");
302 
303                             /* Gack. DeflaterInputStream is Java 6. */
304                         // response.setEntity(new InputStreamEntity(new DeflaterInputStream(new
305                         // ByteArrayInputStream(
306                         // entityText.getBytes("utf-8"))), -1));
307                         final byte[] uncompressed = entityText.getBytes(StandardCharsets.UTF_8);
308                         final Deflater compressor = new Deflater(Deflater.DEFAULT_COMPRESSION, rfc1951);
309                         compressor.setInput(uncompressed);
310                         compressor.finish();
311                         final byte[] output = new byte[100];
312                         final int compressedLength = compressor.deflate(output);
313                         final byte[] compressed = new byte[compressedLength];
314                         System.arraycopy(output, 0, compressed, 0, compressedLength);
315                         response.setEntity(new InputStreamEntity(
316                                 new ByteArrayInputStream(compressed), compressedLength, null));
317                         return;
318                     }
319                 }
320             }
321         };
322     }
323 
324     /**
325      * Returns an {@link HttpRequestHandler} implementation that will attempt to provide a gzip
326      * Content-Encoding.
327      *
328      * @param entityText
329      *            the non-null String entity to be returned by the server
330      * @return a non-null {@link HttpRequestHandler}
331      */
332     private HttpRequestHandler createGzipEncodingRequestHandler(final String entityText) {
333         return new HttpRequestHandler() {
334 
335             /**
336              * {@inheritDoc}
337              */
338             @Override
339             public void handle(
340                     final ClassicHttpRequest request,
341                     final ClassicHttpResponse response,
342                     final HttpContext context) throws HttpException, IOException {
343                 response.setEntity(new StringEntity(entityText));
344                 response.addHeader("Content-Type", "text/plain");
345                 response.addHeader("Content-Type", "text/plain");
346                 final Iterator<HeaderElement> it = MessageSupport.iterate(request, "Accept-Encoding");
347                 while (it.hasNext()) {
348                     final HeaderElement element = it.next();
349                     if ("gzip".equalsIgnoreCase(element.getName())) {
350                         response.addHeader("Content-Encoding", "gzip");
351 
352                         /*
353                          * We have to do a bit more work with gzip versus deflate, since
354                          * Gzip doesn't appear to have an equivalent to DeflaterInputStream in
355                          * the JDK.
356                          *
357                          * UPDATE: DeflaterInputStream is Java 6 anyway, so we have to do a bit
358                          * of work there too!
359                          */
360                         final ByteArrayOutputStream bytes = new ByteArrayOutputStream();
361                         final OutputStream out = new GZIPOutputStream(bytes);
362 
363                         final ByteArrayInputStream uncompressed = new ByteArrayInputStream(
364                                 entityText.getBytes(StandardCharsets.UTF_8));
365 
366                         final byte[] buf = new byte[60];
367 
368                         int n;
369                         while ((n = uncompressed.read(buf)) != -1) {
370                             out.write(buf, 0, n);
371                         }
372 
373                         out.close();
374 
375                         final byte[] arr = bytes.toByteArray();
376                         response.setEntity(new InputStreamEntity(new ByteArrayInputStream(arr),
377                                 arr.length, null));
378 
379                         return;
380                     }
381                 }
382             }
383         };
384     }
385 
386     /**
387      * Sub-ordinate task passed off to a different thread to be executed.
388      *
389      * @author jabley
390      *
391      */
392     class WorkerTask implements Runnable {
393 
394         private final CloseableHttpClient client;
395         private final HttpHost target;
396         private final HttpGet request;
397         private final CountDownLatch startGate;
398         private final CountDownLatch endGate;
399 
400         private boolean failed;
401         private String text;
402 
403         WorkerTask(final CloseableHttpClient client, final HttpHost target, final boolean identity, final CountDownLatch startGate, final CountDownLatch endGate) {
404             this.client = client;
405             this.target = target;
406             this.request = new HttpGet("/some-resource");
407             if (identity) {
408                 request.addHeader("Accept-Encoding", "identity");
409             }
410             this.startGate = startGate;
411             this.endGate = endGate;
412         }
413 
414         /**
415          * Returns the text of the HTTP entity.
416          *
417          * @return a String - may be null.
418          */
419         public String getText() {
420             return this.text;
421         }
422 
423         /**
424          * {@inheritDoc}
425          */
426         @Override
427         public void run() {
428             try {
429                 startGate.await();
430                 try {
431                     final ClassicHttpResponse response = client.execute(target, request);
432                     text = EntityUtils.toString(response.getEntity());
433                 } catch (final Exception e) {
434                     failed = true;
435                 } finally {
436                     endGate.countDown();
437                 }
438             } catch (final InterruptedException ignore) {
439             }
440         }
441 
442         /**
443          * Returns true if this task failed, otherwise false.
444          *
445          * @return a flag
446          */
447         boolean isFailed() {
448             return this.failed;
449         }
450     }
451 }