1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28 package org.apache.hc.core5.util;
29
30 import org.junit.Assert;
31 import org.junit.Before;
32 import org.junit.Test;
33
34 public class TestTokenizer {
35
36 private Tokenizer parser;
37
38 @Before
39 public void setUp() throws Exception {
40 parser = new Tokenizer();
41 }
42
43 private static CharArrayBuffer createBuffer(final String value) {
44 if (value == null) {
45 return null;
46 }
47 final CharArrayBuffer buffer = new CharArrayBuffer(value.length());
48 buffer.append(value);
49 return buffer;
50 }
51
52 @Test
53 public void testBasicTokenParsing() throws Exception {
54 final String s = " raw: \" some stuff \"";
55 final CharArrayBuffer raw = createBuffer(s);
56 final Tokenizer.Cursor cursor = new Tokenizer.Cursor(0, s.length());
57
58 parser.skipWhiteSpace(raw, cursor);
59
60 Assert.assertFalse(cursor.atEnd());
61 Assert.assertEquals(3, cursor.getPos());
62
63 final StringBuilder strbuf1 = new StringBuilder();
64 parser.copyContent(raw, cursor, Tokenizer.INIT_BITSET(':'), strbuf1);
65
66 Assert.assertFalse(cursor.atEnd());
67 Assert.assertEquals(6, cursor.getPos());
68 Assert.assertEquals("raw", strbuf1.toString());
69 Assert.assertEquals(':', raw.charAt(cursor.getPos()));
70 cursor.updatePos(cursor.getPos() + 1);
71
72 parser.skipWhiteSpace(raw, cursor);
73
74 Assert.assertFalse(cursor.atEnd());
75 Assert.assertEquals(8, cursor.getPos());
76
77 final StringBuilder strbuf2 = new StringBuilder();
78 parser.copyQuotedContent(raw, cursor, strbuf2);
79
80 Assert.assertTrue(cursor.atEnd());
81 Assert.assertEquals(" some stuff ", strbuf2.toString());
82
83 parser.copyQuotedContent(raw, cursor, strbuf2);
84 Assert.assertTrue(cursor.atEnd());
85
86 parser.skipWhiteSpace(raw, cursor);
87 Assert.assertTrue(cursor.atEnd());
88 }
89
90 @Test
91 public void testTokenParsingWithQuotedPairs() throws Exception {
92 final String s = "raw: \"\\\"some\\stuff\\\\\"";
93 final CharArrayBuffer raw = createBuffer(s);
94 final Tokenizer.Cursor cursor = new Tokenizer.Cursor(0, s.length());
95
96 parser.skipWhiteSpace(raw, cursor);
97
98 Assert.assertFalse(cursor.atEnd());
99 Assert.assertEquals(0, cursor.getPos());
100
101 final StringBuilder strbuf1 = new StringBuilder();
102 parser.copyContent(raw, cursor, Tokenizer.INIT_BITSET(':'), strbuf1);
103
104 Assert.assertFalse(cursor.atEnd());
105 Assert.assertEquals("raw", strbuf1.toString());
106 Assert.assertEquals(':', raw.charAt(cursor.getPos()));
107 cursor.updatePos(cursor.getPos() + 1);
108
109 parser.skipWhiteSpace(raw, cursor);
110
111 Assert.assertFalse(cursor.atEnd());
112
113 final StringBuilder strbuf2 = new StringBuilder();
114 parser.copyQuotedContent(raw, cursor, strbuf2);
115
116 Assert.assertTrue(cursor.atEnd());
117 Assert.assertEquals("\"some\\stuff\\", strbuf2.toString());
118 }
119
120 @Test
121 public void testTokenParsingIncompleteQuote() throws Exception {
122 final String s = "\"stuff and more stuff ";
123 final CharArrayBuffer raw = createBuffer(s);
124 final Tokenizer.Cursor cursor = new Tokenizer.Cursor(0, s.length());
125 final StringBuilder strbuf1 = new StringBuilder();
126 parser.copyQuotedContent(raw, cursor, strbuf1);
127 Assert.assertEquals("stuff and more stuff ", strbuf1.toString());
128 }
129
130 @Test
131 public void testTokenParsingTokensWithUnquotedBlanks() throws Exception {
132 final String s = " stuff and \tsome\tmore stuff ;";
133 final CharArrayBuffer raw = createBuffer(s);
134 final Tokenizer.Cursor cursor = new Tokenizer.Cursor(0, s.length());
135 final String result = parser.parseToken(raw, cursor, Tokenizer.INIT_BITSET(';'));
136 Assert.assertEquals("stuff and some more stuff", result);
137 }
138
139 @Test
140 public void testTokenParsingMixedValuesAndQuotedValues() throws Exception {
141 final String s = " stuff and \" some more \" \"stuff ;";
142 final CharArrayBuffer raw = createBuffer(s);
143 final Tokenizer.Cursor cursor = new Tokenizer.Cursor(0, s.length());
144 final String result = parser.parseValue(raw, cursor, Tokenizer.INIT_BITSET(';'));
145 Assert.assertEquals("stuff and some more stuff ;", result);
146 }
147
148 @Test
149 public void testTokenParsingMixedValuesAndQuotedValues2() throws Exception {
150 final String s = "stuff\"more\"stuff;";
151 final CharArrayBuffer raw = createBuffer(s);
152 final Tokenizer.Cursor cursor = new Tokenizer.Cursor(0, s.length());
153 final String result = parser.parseValue(raw, cursor, Tokenizer.INIT_BITSET(';'));
154 Assert.assertEquals("stuffmorestuff", result);
155 }
156
157 @Test
158 public void testTokenParsingEscapedQuotes() throws Exception {
159 final String s = "stuff\"\\\"more\\\"\"stuff;";
160 final CharArrayBuffer raw = createBuffer(s);
161 final Tokenizer.Cursor cursor = new Tokenizer.Cursor(0, s.length());
162 final String result = parser.parseValue(raw, cursor, Tokenizer.INIT_BITSET(';'));
163 Assert.assertEquals("stuff\"more\"stuff", result);
164 }
165
166 @Test
167 public void testTokenParsingEscapedDelimiter() throws Exception {
168 final String s = "stuff\"\\\"more\\\";\"stuff;";
169 final CharArrayBuffer raw = createBuffer(s);
170 final Tokenizer.Cursor cursor = new Tokenizer.Cursor(0, s.length());
171 final String result = parser.parseValue(raw, cursor, Tokenizer.INIT_BITSET(';'));
172 Assert.assertEquals("stuff\"more\";stuff", result);
173 }
174
175 @Test
176 public void testTokenParsingEscapedSlash() throws Exception {
177 final String s = "stuff\"\\\"more\\\";\\\\\"stuff;";
178 final CharArrayBuffer raw = createBuffer(s);
179 final Tokenizer.Cursor cursor = new Tokenizer.Cursor(0, s.length());
180 final String result = parser.parseValue(raw, cursor, Tokenizer.INIT_BITSET(';'));
181 Assert.assertEquals("stuff\"more\";\\stuff", result);
182 }
183
184 @Test
185 public void testTokenParsingSlashOutsideQuotes() throws Exception {
186 final String s = "stuff\\; more stuff;";
187 final CharArrayBuffer raw = createBuffer(s);
188 final Tokenizer.Cursor cursor = new Tokenizer.Cursor(0, s.length());
189 final String result = parser.parseValue(raw, cursor, Tokenizer.INIT_BITSET(';'));
190 Assert.assertEquals("stuff\\", result);
191 }
192 }