View Javadoc
1   /*
2    * ====================================================================
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *   http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing,
14   * software distributed under the License is distributed on an
15   * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16   * KIND, either express or implied.  See the License for the
17   * specific language governing permissions and limitations
18   * under the License.
19   * ====================================================================
20   *
21   * This software consists of voluntary contributions made by many
22   * individuals on behalf of the Apache Software Foundation.  For more
23   * information on the Apache Software Foundation, please see
24   * <http://www.apache.org/>.
25   *
26   */
27  
28  package org.apache.hc.core5.http.io.entity;
29  
30  import java.io.IOException;
31  import java.io.InputStream;
32  import java.io.InputStreamReader;
33  import java.io.Reader;
34  import java.io.UnsupportedEncodingException;
35  import java.nio.charset.Charset;
36  import java.nio.charset.StandardCharsets;
37  import java.nio.charset.UnsupportedCharsetException;
38  import java.util.Collections;
39  import java.util.HashMap;
40  import java.util.List;
41  import java.util.Map;
42  
43  import org.apache.hc.core5.http.ContentType;
44  import org.apache.hc.core5.http.HttpEntity;
45  import org.apache.hc.core5.http.NameValuePair;
46  import org.apache.hc.core5.http.ParseException;
47  import org.apache.hc.core5.io.Closer;
48  import org.apache.hc.core5.net.WWWFormCodec;
49  import org.apache.hc.core5.util.Args;
50  import org.apache.hc.core5.util.ByteArrayBuffer;
51  import org.apache.hc.core5.util.CharArrayBuffer;
52  
53  /**
54   * Support methods for {@link HttpEntity}.
55   *
56   * @since 4.0
57   */
58  public final class EntityUtils {
59  
60      // TODO Consider using a sane value, but what is sane? 1 GB? 100 MB? 10 MB?
61      private static final int DEFAULT_ENTITY_RETURN_MAX_LENGTH = Integer.MAX_VALUE;
62      private static final Charset DEFAULT_CHARSET = StandardCharsets.ISO_8859_1;
63      private static final int DEFAULT_CHAR_BUFFER_SIZE = 1024;
64      private static final int DEFAULT_BYTE_BUFFER_SIZE = 4096;
65  
66      private EntityUtils() {
67          // NoOp
68      }
69  
70      /**
71       * Ensures that the entity content is fully consumed and the content stream, if exists,
72       * is closed. The process is done, <i>quietly</i> , without throwing any IOException.
73       *
74       * @param entity the entity to consume.
75       *
76       * @since 4.2
77       */
78      public static void consumeQuietly(final HttpEntity entity) {
79          try {
80            consume(entity);
81          } catch (final IOException ignore) {
82              // Ignore exception
83          }
84      }
85  
86      /**
87       * Ensures that the entity content is fully consumed and the content stream, if exists,
88       * is closed.
89       *
90       * @param entity the entity to consume.
91       * @throws IOException if an error occurs reading the input stream
92       *
93       * @since 4.1
94       */
95      public static void consume(final HttpEntity entity) throws IOException {
96          if (entity == null) {
97              return;
98          }
99          if (entity.isStreaming()) {
100             Closer.close(entity.getContent());
101         }
102     }
103 
104     /**
105      * Gets a usable content length value for the given candidate.
106      *
107      * @param contentLength an integer.
108      * @return The given content length or {@value #DEFAULT_BYTE_BUFFER_SIZE} if it is &lt 0.
109      */
110     private static int toContentLength(final int contentLength) {
111         return contentLength < 0 ? DEFAULT_BYTE_BUFFER_SIZE : contentLength;
112     }
113 
114     /**
115      * Reads the contents of an entity and return it as a byte array.
116      *
117      * @param entity the entity to read from=
118      * @return byte array containing the entity content. May be null if
119      *   {@link HttpEntity#getContent()} is null.
120      * @throws IOException if an error occurs reading the input stream
121      * @throws IllegalArgumentException if entity is null or if content length &gt; Integer.MAX_VALUE
122      */
123     public static byte[] toByteArray(final HttpEntity entity) throws IOException {
124         Args.notNull(entity, "HttpEntity");
125         final int contentLength = toContentLength((int) Args.checkContentLength(entity));
126         try (final InputStream inStream = entity.getContent()) {
127             if (inStream == null) {
128                 return null;
129             }
130             final ByteArrayBuffer buffer = new ByteArrayBuffer(contentLength);
131             final byte[] tmp = new byte[DEFAULT_BYTE_BUFFER_SIZE];
132             int l;
133             while ((l = inStream.read(tmp)) != -1) {
134                 buffer.append(tmp, 0, l);
135             }
136             return buffer.toByteArray();
137         }
138     }
139 
140     /**
141      * Reads the contents of an entity and return it as a byte array.
142      *
143      * @param entity the entity to read from=
144      * @return byte array containing the entity content. May be null if
145      *   {@link HttpEntity#getContent()} is null.
146      * @param maxResultLength
147      *            The maximum size of the String to return; use it to guard against unreasonable or malicious processing.
148      * @throws IOException if an error occurs reading the input stream
149      * @throws IllegalArgumentException if entity is null or if content length &gt; Integer.MAX_VALUE
150      */
151     public static byte[] toByteArray(final HttpEntity entity, final int maxResultLength) throws IOException {
152         Args.notNull(entity, "HttpEntity");
153         final int contentLength = toContentLength((int) Args.checkContentLength(entity));
154         try (final InputStream inStream = entity.getContent()) {
155             if (inStream == null) {
156                 return null;
157             }
158             final ByteArrayBuffer buffer = new ByteArrayBuffer(Math.min(maxResultLength, contentLength));
159             final byte[] tmp = new byte[DEFAULT_BYTE_BUFFER_SIZE];
160             int l;
161             while ((l = inStream.read(tmp)) != -1 && buffer.length() < maxResultLength) {
162                 buffer.append(tmp, 0, l);
163             }
164             buffer.setLength(Math.min(buffer.length(), maxResultLength));
165             return buffer.toByteArray();
166         }
167     }
168 
169     private static CharArrayBuffer toCharArrayBuffer(final InputStream inStream, final int contentLength,
170             final Charset charset, final int maxResultLength) throws IOException {
171         Args.notNull(inStream, "InputStream");
172         Args.positive(maxResultLength, "maxResultLength");
173         final Charset actualCharset = charset == null ? DEFAULT_CHARSET : charset;
174         final CharArrayBuffer buf = new CharArrayBuffer(
175                 Math.min(maxResultLength, contentLength > 0 ? contentLength : DEFAULT_CHAR_BUFFER_SIZE));
176         final Reader reader = new InputStreamReader(inStream, actualCharset);
177         final char[] tmp = new char[DEFAULT_CHAR_BUFFER_SIZE];
178         int chReadCount;
179         while ((chReadCount = reader.read(tmp)) != -1 && buf.length() < maxResultLength) {
180             buf.append(tmp, 0, chReadCount);
181         }
182         buf.setLength(Math.min(buf.length(), maxResultLength));
183         return buf;
184     }
185 
186     private static final Map<String, ContentType> CONTENT_TYPE_MAP;
187     static {
188         final ContentType[] contentTypes = {
189                 ContentType.APPLICATION_ATOM_XML,
190                 ContentType.APPLICATION_FORM_URLENCODED,
191                 ContentType.APPLICATION_JSON,
192                 ContentType.APPLICATION_SVG_XML,
193                 ContentType.APPLICATION_XHTML_XML,
194                 ContentType.APPLICATION_XML,
195                 ContentType.MULTIPART_FORM_DATA,
196                 ContentType.TEXT_HTML,
197                 ContentType.TEXT_PLAIN,
198                 ContentType.TEXT_XML };
199         final HashMap<String, ContentType> map = new HashMap<>();
200         for (final ContentType contentType: contentTypes) {
201             map.put(contentType.getMimeType(), contentType);
202         }
203         CONTENT_TYPE_MAP = Collections.unmodifiableMap(map);
204     }
205 
206     private static String toString(final HttpEntity entity, final ContentType contentType, final int maxResultLength)
207             throws IOException {
208         Args.notNull(entity, "HttpEntity");
209         final int contentLength = toContentLength((int) Args.checkContentLength(entity));
210         try (final InputStream inStream = entity.getContent()) {
211             if (inStream == null) {
212                 return null;
213             }
214             Charset charset = null;
215             if (contentType != null) {
216                 charset = contentType.getCharset();
217                 if (charset == null) {
218                     final ContentType defaultContentType = CONTENT_TYPE_MAP.get(contentType.getMimeType());
219                     charset = defaultContentType != null ? defaultContentType.getCharset() : null;
220                 }
221             }
222             return toCharArrayBuffer(inStream, contentLength, charset, maxResultLength).toString();
223         }
224     }
225 
226     /**
227      * Gets the entity content as a String, using the provided default character set
228      * if none is found in the entity.
229      * If defaultCharset is null, the default "ISO-8859-1" is used.
230      *
231      * @param entity must not be null
232      * @param defaultCharset character set to be applied if none found in the entity,
233      * or if the entity provided charset is invalid or not available.
234      * @return the entity content as a String. May be null if
235      *   {@link HttpEntity#getContent()} is null.
236      * @throws ParseException if header elements cannot be parsed
237      * @throws IllegalArgumentException if entity is null or if content length &gt; Integer.MAX_VALUE
238      * @throws IOException if an error occurs reading the input stream
239      * @throws java.nio.charset.UnsupportedCharsetException Thrown when the named entity's charset is not available in
240      * this instance of the Java virtual machine and no defaultCharset is provided.
241      */
242     public static String toString(
243             final HttpEntity entity, final Charset defaultCharset) throws IOException, ParseException {
244         return toString(entity, defaultCharset, DEFAULT_ENTITY_RETURN_MAX_LENGTH);
245     }
246 
247     /**
248      * Gets the entity content as a String, using the provided default character set
249      * if none is found in the entity.
250      * If defaultCharset is null, the default "ISO-8859-1" is used.
251      *
252      * @param entity must not be null
253      * @param defaultCharset character set to be applied if none found in the entity,
254      * or if the entity provided charset is invalid or not available.
255      * @param maxResultLength
256      *            The maximum size of the String to return; use it to guard against unreasonable or malicious processing.
257      * @return the entity content as a String. May be null if
258      *   {@link HttpEntity#getContent()} is null.
259      * @throws ParseException if header elements cannot be parsed
260      * @throws IllegalArgumentException if entity is null or if content length &gt; Integer.MAX_VALUE
261      * @throws IOException if an error occurs reading the input stream
262      * @throws java.nio.charset.UnsupportedCharsetException Thrown when the named entity's charset is not available in
263      * this instance of the Java virtual machine and no defaultCharset is provided.
264      */
265     public static String toString(
266             final HttpEntity entity, final Charset defaultCharset, final int maxResultLength) throws IOException, ParseException {
267         Args.notNull(entity, "HttpEntity");
268         ContentType contentType = null;
269         try {
270             contentType = ContentType.parse(entity.getContentType());
271         } catch (final UnsupportedCharsetException ex) {
272             if (defaultCharset == null) {
273                 throw new UnsupportedEncodingException(ex.getMessage());
274             }
275         }
276         if (contentType != null) {
277             if (contentType.getCharset() == null) {
278                 contentType = contentType.withCharset(defaultCharset);
279             }
280         } else {
281             contentType = ContentType.DEFAULT_TEXT.withCharset(defaultCharset);
282         }
283         return toString(entity, contentType, maxResultLength);
284     }
285 
286     /**
287      * Gets the entity content as a String, using the provided default character set
288      * if none is found in the entity.
289      * If defaultCharset is null, the default "ISO-8859-1" is used.
290      *
291      * @param entity must not be null
292      * @param defaultCharset character set to be applied if none found in the entity
293      * @return the entity content as a String. May be null if
294      *   {@link HttpEntity#getContent()} is null.
295      * @throws ParseException if header elements cannot be parsed
296      * @throws IllegalArgumentException if entity is null or if content length &gt; Integer.MAX_VALUE
297      * @throws IOException if an error occurs reading the input stream
298      * @throws java.nio.charset.UnsupportedCharsetException Thrown when the named charset is not available in
299      * this instance of the Java virtual machine
300      */
301     public static String toString(
302             final HttpEntity entity, final String defaultCharset) throws IOException, ParseException {
303         return toString(entity, defaultCharset, DEFAULT_ENTITY_RETURN_MAX_LENGTH);
304     }
305 
306     /**
307      * Gets the entity content as a String, using the provided default character set
308      * if none is found in the entity.
309      * If defaultCharset is null, the default "ISO-8859-1" is used.
310      *
311      * @param entity must not be null
312      * @param defaultCharset character set to be applied if none found in the entity
313      * @param maxResultLength
314      *            The maximum size of the String to return; use it to guard against unreasonable or malicious processing.
315      * @return the entity content as a String. May be null if
316      *   {@link HttpEntity#getContent()} is null.
317      * @throws ParseException if header elements cannot be parsed
318      * @throws IllegalArgumentException if entity is null or if content length &gt; Integer.MAX_VALUE
319      * @throws IOException if an error occurs reading the input stream
320      * @throws java.nio.charset.UnsupportedCharsetException Thrown when the named charset is not available in
321      * this instance of the Java virtual machine
322      */
323     public static String toString(
324             final HttpEntity entity, final String defaultCharset, final int maxResultLength) throws IOException, ParseException {
325         return toString(entity, defaultCharset != null ? Charset.forName(defaultCharset) : null, maxResultLength);
326     }
327 
328     /**
329      * Reads the contents of an entity and return it as a String.
330      * The content is converted using the character set from the entity (if any),
331      * failing that, "ISO-8859-1" is used.
332      *
333      * @param entity the entity to convert to a string; must not be null
334      * @return String containing the content.
335      * @throws ParseException if header elements cannot be parsed
336      * @throws IllegalArgumentException if entity is null or if content length &gt; Integer.MAX_VALUE
337      * @throws IOException if an error occurs reading the input stream
338      * @throws java.nio.charset.UnsupportedCharsetException Thrown when the named charset is not available in
339      * this instance of the Java virtual machine
340      */
341     public static String toString(final HttpEntity entity) throws IOException, ParseException {
342         return toString(entity, DEFAULT_ENTITY_RETURN_MAX_LENGTH);
343     }
344 
345     /**
346      * Reads the contents of an entity and return it as a String.
347      * The content is converted using the character set from the entity (if any),
348      * failing that, "ISO-8859-1" is used.
349      *
350      * @param entity the entity to convert to a string; must not be null
351      * @param maxResultLength
352      *            The maximum size of the String to return; use it to guard against unreasonable or malicious processing.
353      * @return String containing the content.
354      * @throws ParseException if header elements cannot be parsed
355      * @throws IllegalArgumentException if entity is null or if content length &gt; Integer.MAX_VALUE
356      * @throws IOException if an error occurs reading the input stream
357      * @throws java.nio.charset.UnsupportedCharsetException Thrown when the named charset is not available in
358      * this instance of the Java virtual machine
359      */
360     public static String toString(final HttpEntity entity, final int maxResultLength) throws IOException, ParseException {
361         Args.notNull(entity, "HttpEntity");
362         return toString(entity, ContentType.parse(entity.getContentType()), maxResultLength);
363     }
364 
365     /**
366      * Returns a list of {@link NameValuePair NameValuePairs} as parsed from an {@link HttpEntity}.
367      * The encoding is taken from the entity's Content-Encoding header.
368      * <p>
369      * This is typically used while parsing an HTTP POST.
370      * </p>
371      *
372      * @param entity
373      *            The entity to parse
374      * @return a list of {@link NameValuePair} as built from the URI's query portion.
375      * @throws IOException
376      *             If there was an exception getting the entity's data.
377      */
378     public static List<NameValuePair> parse(final HttpEntity entity) throws IOException {
379         return parse(entity, DEFAULT_ENTITY_RETURN_MAX_LENGTH);
380     }
381 
382     /**
383      * Returns a list of {@link NameValuePair NameValuePairs} as parsed from an {@link HttpEntity}.
384      * The encoding is taken from the entity's Content-Encoding header.
385      * <p>
386      * This is typically used while parsing an HTTP POST.
387      * </p>
388      *
389      * @param entity
390      *            The entity to parse
391      * @param maxStreamLength
392      *            The maximum size of the stream to read; use it to guard against unreasonable or malicious processing.
393      * @return a list of {@link NameValuePair} as built from the URI's query portion.
394      * @throws IOException
395      *             If there was an exception getting the entity's data.
396      */
397     public static List<NameValuePair> parse(final HttpEntity entity, final int maxStreamLength) throws IOException {
398         Args.notNull(entity, "HttpEntity");
399         final int contentLength = toContentLength((int) Args.checkContentLength(entity));
400         final ContentType contentType = ContentType.parse(entity.getContentType());
401         if (!ContentType.APPLICATION_FORM_URLENCODED.isSameMimeType(contentType)) {
402             return Collections.emptyList();
403         }
404         final Charset charset = contentType.getCharset(DEFAULT_CHARSET);
405         final CharArrayBuffer buf;
406         try (final InputStream inStream = entity.getContent()) {
407             if (inStream == null) {
408                 return Collections.emptyList();
409             }
410             buf = toCharArrayBuffer(inStream, contentLength, charset, maxStreamLength);
411 
412         }
413         if (buf.isEmpty()) {
414             return Collections.emptyList();
415         }
416         return WWWFormCodec.parse(buf, charset);
417     }
418 
419 }