1 /*
2 * ====================================================================
3 * Licensed to the Apache Software Foundation (ASF) under one
4 * or more contributor license agreements. See the NOTICE file
5 * distributed with this work for additional information
6 * regarding copyright ownership. The ASF licenses this file
7 * to you under the Apache License, Version 2.0 (the
8 * "License"); you may not use this file except in compliance
9 * with the License. You may obtain a copy of the License at
10 *
11 * http://www.apache.org/licenses/LICENSE-2.0
12 *
13 * Unless required by applicable law or agreed to in writing,
14 * software distributed under the License is distributed on an
15 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16 * KIND, either express or implied. See the License for the
17 * specific language governing permissions and limitations
18 * under the License.
19 * ====================================================================
20 *
21 * This software consists of voluntary contributions made by many
22 * individuals on behalf of the Apache Software Foundation. For more
23 * information on the Apache Software Foundation, please see
24 * <http://www.apache.org/>.
25 *
26 */
27
28 package org.apache.http.message;
29
30 import java.util.NoSuchElementException;
31
32 import org.apache.http.HeaderIterator;
33 import org.apache.http.ParseException;
34 import org.apache.http.TokenIterator;
35 import org.apache.http.util.Args;
36
37 /**
38 * Basic implementation of a {@link TokenIterator}.
39 * This implementation parses {@code #token} sequences as
40 * defined by RFC 2616, section 2.
41 * It extends that definition somewhat beyond US-ASCII.
42 *
43 * @since 4.0
44 */
45 public class BasicTokenIterator implements TokenIterator {
46
47 /** The HTTP separator characters. Defined in RFC 2616, section 2.2. */
48 // the order of the characters here is adjusted to put the
49 // most likely candidates at the beginning of the collection
50 public final static String HTTP_SEPARATORS = " ,;=()<>@:\\\"/[]?{}\t";
51
52
53 /** The iterator from which to obtain the next header. */
54 protected final HeaderIterator headerIt;
55
56 /**
57 * The value of the current header.
58 * This is the header value that includes {@link #currentToken}.
59 * Undefined if the iteration is over.
60 */
61 protected String currentHeader;
62
63 /**
64 * The token to be returned by the next call to {@link #nextToken()}.
65 * {@code null} if the iteration is over.
66 */
67 protected String currentToken;
68
69 /**
70 * The position after {@link #currentToken} in {@link #currentHeader}.
71 * Undefined if the iteration is over.
72 */
73 protected int searchPos;
74
75
76 /**
77 * Creates a new instance of {@link BasicTokenIterator}.
78 *
79 * @param headerIterator the iterator for the headers to tokenize
80 */
81 public BasicTokenIterator(final HeaderIterator headerIterator) {
82 super();
83 this.headerIt = Args.notNull(headerIterator, "Header iterator");
84 this.searchPos = findNext(-1);
85 }
86
87
88 // non-javadoc, see interface TokenIterator
89 @Override
90 public boolean hasNext() {
91 return (this.currentToken != null);
92 }
93
94
95 /**
96 * Obtains the next token from this iteration.
97 *
98 * @return the next token in this iteration
99 *
100 * @throws NoSuchElementException if the iteration is already over
101 * @throws ParseException if an invalid header value is encountered
102 */
103 @Override
104 public String nextToken()
105 throws NoSuchElementException, ParseException {
106
107 if (this.currentToken == null) {
108 throw new NoSuchElementException("Iteration already finished.");
109 }
110
111 final String result = this.currentToken;
112 // updates currentToken, may trigger ParseException:
113 this.searchPos = findNext(this.searchPos);
114
115 return result;
116 }
117
118
119 /**
120 * Returns the next token.
121 * Same as {@link #nextToken}, but with generic return type.
122 *
123 * @return the next token in this iteration
124 *
125 * @throws NoSuchElementException if there are no more tokens
126 * @throws ParseException if an invalid header value is encountered
127 */
128 @Override
129 public final Object next()
130 throws NoSuchElementException, ParseException {
131 return nextToken();
132 }
133
134
135 /**
136 * Removing tokens is not supported.
137 *
138 * @throws UnsupportedOperationException always
139 */
140 @Override
141 public final void remove()
142 throws UnsupportedOperationException {
143
144 throw new UnsupportedOperationException
145 ("Removing tokens is not supported.");
146 }
147
148
149 /**
150 * Determines the next token.
151 * If found, the token is stored in {@link #currentToken}.
152 * The return value indicates the position after the token
153 * in {@link #currentHeader}. If necessary, the next header
154 * will be obtained from {@link #headerIt}.
155 * If not found, {@link #currentToken} is set to {@code null}.
156 *
157 * @param pos the position in the current header at which to
158 * start the search, -1 to search in the first header
159 *
160 * @return the position after the found token in the current header, or
161 * negative if there was no next token
162 *
163 * @throws ParseException if an invalid header value is encountered
164 */
165 protected int findNext(final int pos) throws ParseException {
166 int from = pos;
167 if (from < 0) {
168 // called from the constructor, initialize the first header
169 if (!this.headerIt.hasNext()) {
170 return -1;
171 }
172 this.currentHeader = this.headerIt.nextHeader().getValue();
173 from = 0;
174 } else {
175 // called after a token, make sure there is a separator
176 from = findTokenSeparator(from);
177 }
178
179 final int start = findTokenStart(from);
180 if (start < 0) {
181 this.currentToken = null;
182 return -1; // nothing found
183 }
184
185 final int end = findTokenEnd(start);
186 this.currentToken = createToken(this.currentHeader, start, end);
187 return end;
188 }
189
190
191 /**
192 * Creates a new token to be returned.
193 * Called from {@link #findNext findNext} after the token is identified.
194 * The default implementation simply calls
195 * {@link java.lang.String#substring String.substring}.
196 * <p>
197 * If header values are significantly longer than tokens, and some
198 * tokens are permanently referenced by the application, there can
199 * be problems with garbage collection. A substring will hold a
200 * reference to the full characters of the original string and
201 * therefore occupies more memory than might be expected.
202 * To avoid this, override this method and create a new string
203 * instead of a substring.
204 * </p>
205 *
206 * @param value the full header value from which to create a token
207 * @param start the index of the first token character
208 * @param end the index after the last token character
209 *
210 * @return a string representing the token identified by the arguments
211 */
212 protected String createToken(final String value, final int start, final int end) {
213 return value.substring(start, end);
214 }
215
216
217 /**
218 * Determines the starting position of the next token.
219 * This method will iterate over headers if necessary.
220 *
221 * @param pos the position in the current header at which to
222 * start the search
223 *
224 * @return the position of the token start in the current header,
225 * negative if no token start could be found
226 */
227 protected int findTokenStart(final int pos) {
228 int from = Args.notNegative(pos, "Search position");
229 boolean found = false;
230 while (!found && (this.currentHeader != null)) {
231
232 final int to = this.currentHeader.length();
233 while (!found && (from < to)) {
234
235 final char ch = this.currentHeader.charAt(from);
236 if (isTokenSeparator(ch) || isWhitespace(ch)) {
237 // whitspace and token separators are skipped
238 from++;
239 } else if (isTokenChar(this.currentHeader.charAt(from))) {
240 // found the start of a token
241 found = true;
242 } else {
243 throw new ParseException
244 ("Invalid character before token (pos " + from +
245 "): " + this.currentHeader);
246 }
247 }
248 if (!found) {
249 if (this.headerIt.hasNext()) {
250 this.currentHeader = this.headerIt.nextHeader().getValue();
251 from = 0;
252 } else {
253 this.currentHeader = null;
254 }
255 }
256 } // while headers
257
258 return found ? from : -1;
259 }
260
261
262 /**
263 * Determines the position of the next token separator.
264 * Because of multi-header joining rules, the end of a
265 * header value is a token separator. This method does
266 * therefore not need to iterate over headers.
267 *
268 * @param pos the position in the current header at which to
269 * start the search
270 *
271 * @return the position of a token separator in the current header,
272 * or at the end
273 *
274 * @throws ParseException
275 * if a new token is found before a token separator.
276 * RFC 2616, section 2.1 explicitly requires a comma between
277 * tokens for {@code #}.
278 */
279 protected int findTokenSeparator(final int pos) {
280 int from = Args.notNegative(pos, "Search position");
281 boolean found = false;
282 final int to = this.currentHeader.length();
283 while (!found && (from < to)) {
284 final char ch = this.currentHeader.charAt(from);
285 if (isTokenSeparator(ch)) {
286 found = true;
287 } else if (isWhitespace(ch)) {
288 from++;
289 } else if (isTokenChar(ch)) {
290 throw new ParseException
291 ("Tokens without separator (pos " + from +
292 "): " + this.currentHeader);
293 } else {
294 throw new ParseException
295 ("Invalid character after token (pos " + from +
296 "): " + this.currentHeader);
297 }
298 }
299
300 return from;
301 }
302
303
304 /**
305 * Determines the ending position of the current token.
306 * This method will not leave the current header value,
307 * since the end of the header value is a token boundary.
308 *
309 * @param from the position of the first character of the token
310 *
311 * @return the position after the last character of the token.
312 * The behavior is undefined if {@code from} does not
313 * point to a token character in the current header value.
314 */
315 protected int findTokenEnd(final int from) {
316 Args.notNegative(from, "Search position");
317 final int to = this.currentHeader.length();
318 int end = from+1;
319 while ((end < to) && isTokenChar(this.currentHeader.charAt(end))) {
320 end++;
321 }
322
323 return end;
324 }
325
326
327 /**
328 * Checks whether a character is a token separator.
329 * RFC 2616, section 2.1 defines comma as the separator for
330 * {@code #token} sequences. The end of a header value will
331 * also separate tokens, but that is not a character check.
332 *
333 * @param ch the character to check
334 *
335 * @return {@code true} if the character is a token separator,
336 * {@code false} otherwise
337 */
338 protected boolean isTokenSeparator(final char ch) {
339 return (ch == ',');
340 }
341
342
343 /**
344 * Checks whether a character is a whitespace character.
345 * RFC 2616, section 2.2 defines space and horizontal tab as whitespace.
346 * The optional preceeding line break is irrelevant, since header
347 * continuation is handled transparently when parsing messages.
348 *
349 * @param ch the character to check
350 *
351 * @return {@code true} if the character is whitespace,
352 * {@code false} otherwise
353 */
354 protected boolean isWhitespace(final char ch) {
355
356 // we do not use Character.isWhitspace(ch) here, since that allows
357 // many control characters which are not whitespace as per RFC 2616
358 return ((ch == '\t') || Character.isSpaceChar(ch));
359 }
360
361
362 /**
363 * Checks whether a character is a valid token character.
364 * Whitespace, control characters, and HTTP separators are not
365 * valid token characters. The HTTP specification (RFC 2616, section 2.2)
366 * defines tokens only for the US-ASCII character set, this
367 * method extends the definition to other character sets.
368 *
369 * @param ch the character to check
370 *
371 * @return {@code true} if the character is a valid token start,
372 * {@code false} otherwise
373 */
374 protected boolean isTokenChar(final char ch) {
375
376 // common sense extension of ALPHA + DIGIT
377 if (Character.isLetterOrDigit(ch)) {
378 return true;
379 }
380
381 // common sense extension of CTL
382 if (Character.isISOControl(ch)) {
383 return false;
384 }
385
386 // no common sense extension for this
387 if (isHttpSeparator(ch)) {
388 return false;
389 }
390
391 // RFC 2616, section 2.2 defines a token character as
392 // "any CHAR except CTLs or separators". The controls
393 // and separators are included in the checks above.
394 // This will yield unexpected results for Unicode format characters.
395 // If that is a problem, overwrite isHttpSeparator(char) to filter
396 // out the false positives.
397 return true;
398 }
399
400
401 /**
402 * Checks whether a character is an HTTP separator.
403 * The implementation in this class checks only for the HTTP separators
404 * defined in RFC 2616, section 2.2. If you need to detect other
405 * separators beyond the US-ASCII character set, override this method.
406 *
407 * @param ch the character to check
408 *
409 * @return {@code true} if the character is an HTTP separator
410 */
411 protected boolean isHttpSeparator(final char ch) {
412 return (HTTP_SEPARATORS.indexOf(ch) >= 0);
413 }
414
415
416 } // class BasicTokenIterator
417