001    /**
002     * Licensed to the Apache Software Foundation (ASF) under one or more
003     * contributor license agreements.  See the NOTICE file distributed with
004     * this work for additional information regarding copyright ownership.
005     * The ASF licenses this file to You under the Apache License, Version 2.0
006     * (the "License"); you may not use this file except in compliance with
007     * the License.  You may obtain a copy of the License at
008     *
009     *     http://www.apache.org/licenses/LICENSE-2.0
010     *
011     * Unless required by applicable law or agreed to in writing, software
012     * distributed under the License is distributed on an "AS IS" BASIS,
013     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
014     * See the License for the specific language governing permissions and
015     * limitations under the License.
016     */
017    
018    package org.apache.geronimo.javamail.store.imap.connection;
019    
020    import java.io.ByteArrayOutputStream;
021    import java.io.UnsupportedEncodingException;
022    import java.util.ArrayList; 
023    import java.util.Date; 
024    import java.util.List; 
025    
026    import javax.mail.Flags;
027    import javax.mail.MessagingException;
028    import javax.mail.internet.InternetAddress;
029    import javax.mail.internet.MailDateFormat;
030    import javax.mail.internet.ParameterList;
031    
032    import org.apache.geronimo.javamail.util.ResponseFormatException; 
033    
034    /**
035     * @version $Rev: 753004 $ $Date: 2009-03-12 21:34:17 +0100 (Do, 12. Mär 2009) $
036     */
037    public class IMAPResponseTokenizer {
038        /*
039         * set up the decoding table.
040         */
041        protected static final byte[] decodingTable = new byte[256];
042    
043        protected static void initializeDecodingTable()
044        {
045            for (int i = 0; i < IMAPCommand.encodingTable.length; i++)
046            {
047                decodingTable[IMAPCommand.encodingTable[i]] = (byte)i;
048            }
049        }
050    
051    
052        static {
053            initializeDecodingTable();
054        }
055        
056        // a singleton formatter for header dates.
057        protected static MailDateFormat dateParser = new MailDateFormat();
058        
059        
060        public static class Token {
061            // Constant values from J2SE 1.4 API Docs (Constant values)
062            public static final int ATOM = -1;
063            public static final int QUOTEDSTRING = -2;
064            public static final int LITERAL = -3;
065            public static final int NUMERIC = -4;
066            public static final int EOF = -5;
067            public static final int NIL = -6;
068            // special single character markers     
069            public static final int CONTINUATION = '-';
070            public static final int UNTAGGED = '*';
071                
072            /**
073             * The type indicator.  This will be either a specific type, represented by 
074             * a negative number, or the actual character value. 
075             */
076            private int type;
077            /**
078             * The String value associated with this token.  All tokens have a String value, 
079             * except for the EOF and NIL tokens. 
080             */
081            private String value;
082    
083            public Token(int type, String value) {
084                this.type = type;
085                this.value = value;
086            }
087    
088            public int getType() {
089                return type;
090            }
091    
092            public String getValue() {
093                return value;
094            }
095    
096            public boolean isType(int type) {
097                return this.type == type;
098            }
099    
100            /**
101             * Return the token as an integer value.  If this can't convert, an exception is 
102             * thrown. 
103             * 
104             * @return The integer value of the token. 
105             * @exception ResponseFormatException
106             */
107            public int getInteger() throws MessagingException {
108                if (value != null) {
109                    try {
110                        return Integer.parseInt(value);
111                    } catch (NumberFormatException e) {
112                    }
113                }
114    
115                throw new ResponseFormatException("Number value expected in response; fount: " + value);
116            }
117    
118            /**
119             * Return the token as a long value.  If it can't convert, an exception is 
120             * thrown. 
121             * 
122             * @return The token as a long value. 
123             * @exception ResponseFormatException
124             */
125            public long getLong() throws MessagingException {
126                if (value != null) {
127                    try {
128                        return Long.parseLong(value);
129                    } catch (NumberFormatException e) {
130                    }
131                }
132                throw new ResponseFormatException("Number value expected in response; fount: " + value);
133            }
134            
135            /**
136             * Handy debugging toString() method for token. 
137             * 
138             * @return The string value of the token. 
139             */
140            public String toString() {
141                if (type == NIL) {
142                    return "NIL"; 
143                }
144                else if (type == EOF) {
145                    return "EOF";
146                }
147                
148                if (value == null) {
149                    return ""; 
150                }
151                return value; 
152            }
153        }
154    
155        public static final Token EOF = new Token(Token.EOF, null);
156        public static final Token NIL = new Token(Token.NIL, null);
157    
158        private static final String WHITE = " \t\n\r";
159        // The list of delimiter characters we process when    
160        // handling parsing of ATOMs.  
161        private static final String atomDelimiters = "(){}%*\"\\" + WHITE;
162        // this set of tokens is a slighly expanded set used for 
163        // specific response parsing.  When dealing with Body 
164        // section names, there are sub pieces to the name delimited 
165        // by "[", "]", ".", "<", ">" and SPACE, so reading these using 
166        // a superset of the ATOM processing makes for easier parsing. 
167        private static final String tokenDelimiters = "<>[].(){}%*\"\\" + WHITE;
168    
169        // the response data read from the connection
170        private byte[] response;
171        // current parsing position
172        private int pos;
173    
174        public IMAPResponseTokenizer(byte [] response) {
175            this.response = response;
176        }
177    
178        /**
179         * Get the remainder of the response as a string.
180         *
181         * @return A string representing the remainder of the response.
182         */
183        public String getRemainder() {
184            // make sure we're still in range
185            if (pos >= response.length) {
186                return "";
187            }
188    
189            return new String(response, pos, response.length - pos);
190        }
191        
192        
193        public Token next() throws MessagingException {
194            return next(false);
195        }
196    
197        public Token next(boolean nilAllowed) throws MessagingException {
198            return readToken(nilAllowed, false);
199        }
200    
201        public Token next(boolean nilAllowed, boolean expandedDelimiters) throws MessagingException {
202            return readToken(nilAllowed, expandedDelimiters);
203        }
204    
205        public Token peek() throws MessagingException {
206            return peek(false, false);
207        }
208    
209        public Token peek(boolean nilAllowed) throws MessagingException {
210            return peek(nilAllowed, false);
211        }
212    
213        public Token peek(boolean nilAllowed, boolean expandedDelimiters) throws MessagingException {
214            int start = pos;
215            try {
216                return readToken(nilAllowed, expandedDelimiters);
217            } finally {
218                pos = start;
219            }
220        }
221    
222        /**
223         * Read an ATOM token from the parsed response.
224         *
225         * @return A token containing the value of the atom token.
226         */
227        private Token readAtomicToken(String delimiters) {
228            // skip to next delimiter
229            int start = pos;
230            while (++pos < response.length) {
231                // break on the first non-atom character.
232                byte ch = response[pos];
233                if (delimiters.indexOf(response[pos]) != -1 || ch < 32 || ch >= 127) {
234                    break;
235                }
236            }
237            
238            // Numeric tokens we store as a different type.  
239            String value = new String(response, start, pos - start); 
240            try {
241                int intValue = Integer.parseInt(value); 
242                return new Token(Token.NUMERIC, value);
243            } catch (NumberFormatException e) {
244            }
245            return new Token(Token.ATOM, value);
246        }
247    
248        /**
249         * Read the next token from the response.
250         *
251         * @return The next token from the response.  White space is skipped, and comment
252         *         tokens are also skipped if indicated.
253         * @exception ResponseFormatException
254         */
255        private Token readToken(boolean nilAllowed, boolean expandedDelimiters) throws MessagingException {
256            String delimiters = expandedDelimiters ? tokenDelimiters : atomDelimiters; 
257            
258            if (pos >= response.length) {
259                return EOF;
260            } else {
261                byte ch = response[pos];
262                if (ch == '\"') {
263                    return readQuotedString();
264                // beginning of a length-specified literal?
265                } else if (ch == '{') {
266                    return readLiteral();
267                // white space, eat this and find a real token.
268                } else if (WHITE.indexOf(ch) != -1) {
269                    eatWhiteSpace();
270                    return readToken(nilAllowed, expandedDelimiters);
271                // either a CTL or special.  These characters have a self-defining token type.
272                } else if (ch < 32 || ch >= 127 || delimiters.indexOf(ch) != -1) {
273                    pos++;
274                    return new Token((int)ch, String.valueOf((char)ch));
275                } else {
276                    // start of an atom, parse it off.
277                    Token token = readAtomicToken(delimiters);
278                    // now, if we've been asked to look at NIL tokens, check to see if it is one,
279                    // and return that instead of the ATOM.
280                    if (nilAllowed) {
281                        if (token.getValue().equalsIgnoreCase("NIL")) {
282                            return NIL;
283                        }
284                    }
285                    return token;
286                }
287            }
288        }
289    
290        /**
291         * Read the next token from the response, returning it as a byte array value.
292         *
293         * @return The next token from the response.  White space is skipped, and comment
294         *         tokens are also skipped if indicated.
295         * @exception ResponseFormatException
296         */
297        private byte[] readData(boolean nilAllowed) throws MessagingException {
298            if (pos >= response.length) {
299                return null;
300            } else {
301                byte ch = response[pos];
302                if (ch == '\"') {
303                    return readQuotedStringData();
304                // beginning of a length-specified literal?
305                } else if (ch == '{') {
306                    return readLiteralData();
307                // white space, eat this and find a real token.
308                } else if (WHITE.indexOf(ch) != -1) {
309                    eatWhiteSpace();
310                    return readData(nilAllowed);
311                // either a CTL or special.  These characters have a self-defining token type.
312                } else if (ch < 32 || ch >= 127 || atomDelimiters.indexOf(ch) != -1) {
313                    throw new ResponseFormatException("Invalid string value: " + ch);
314                } else {
315                    // only process this if we're allowing NIL as an option.
316                    if (nilAllowed) {
317                        // start of an atom, parse it off.
318                        Token token = next(true);
319                        if (token.isType(Token.NIL)) {
320                            return null;
321                        }
322                        // invalid token type.
323                        throw new ResponseFormatException("Invalid string value: " + token.getValue());
324                    }
325                    // invalid token type.
326                    throw new ResponseFormatException("Invalid string value: " + ch);
327                }
328            }
329        }
330    
331        /**
332         * Extract a substring from the response string and apply any
333         * escaping/folding rules to the string.
334         *
335         * @param start  The starting offset in the response.
336         * @param end    The response end offset + 1.
337         *
338         * @return The processed string value.
339         * @exception ResponseFormatException
340         */
341        private byte[] getEscapedValue(int start, int end) throws MessagingException {
342            ByteArrayOutputStream value = new ByteArrayOutputStream();
343    
344            for (int i = start; i < end; i++) {
345                byte ch = response[i];
346                // is this an escape character?
347                if (ch == '\\') {
348                    i++;
349                    if (i == end) {
350                        throw new ResponseFormatException("Invalid escape character");
351                    }
352                    value.write(response[i]);
353                }
354                // line breaks are ignored, except for naked '\n' characters, which are consider
355                // parts of linear whitespace.
356                else if (ch == '\r') {
357                    // see if this is a CRLF sequence, and skip the second if it is.
358                    if (i < end - 1 && response[i + 1] == '\n') {
359                        i++;
360                    }
361                }
362                else {
363                    // just append the ch value.
364                    value.write(ch);
365                }
366            }
367            return value.toByteArray();
368        }
369    
370        /**
371         * Parse out a quoted string from the response, applying escaping
372         * rules to the value.
373         *
374         * @return The QUOTEDSTRING token with the value.
375         * @exception ResponseFormatException
376         */
377        private Token readQuotedString() throws MessagingException {
378    
379            String value = new String(readQuotedStringData());
380            return new Token(Token.QUOTEDSTRING, value);
381        }
382    
383        /**
384         * Parse out a quoted string from the response, applying escaping
385         * rules to the value.
386         *
387         * @return The byte array with the resulting string bytes.
388         * @exception ResponseFormatException
389         */
390        private byte[] readQuotedStringData() throws MessagingException {
391            int start = pos + 1;
392            boolean requiresEscaping = false;
393    
394            // skip to end of comment/string
395            while (++pos < response.length) {
396                byte ch = response[pos];
397                if (ch == '"') {
398                    byte[] value;
399                    if (requiresEscaping) {
400                        value = getEscapedValue(start, pos);
401                    }
402                    else {
403                        value = subarray(start, pos);
404                    }
405                    // step over the delimiter for all cases.
406                    pos++;
407                    return value;
408                }
409                else if (ch == '\\') {
410                    pos++;
411                    requiresEscaping = true;
412                }
413                // we need to process line breaks also
414                else if (ch == '\r') {
415                    requiresEscaping = true;
416                }
417            }
418    
419            throw new ResponseFormatException("Missing '\"'");
420        }
421    
422    
423        /**
424         * Parse out a literal string from the response, using the length
425         * encoded before the listeral.
426         *
427         * @return The LITERAL token with the value.
428         * @exception ResponseFormatException
429         */
430        protected Token readLiteral() throws MessagingException {
431            String value = new String(readLiteralData());
432            return new Token(Token.LITERAL, value);
433        }
434    
435    
436        /**
437         * Parse out a literal string from the response, using the length
438         * encoded before the listeral.
439         *
440         * @return The byte[] array with the value.
441         * @exception ResponseFormatException
442         */
443        protected byte[] readLiteralData() throws MessagingException {
444            int lengthStart = pos + 1;
445    
446            // see if we have a close marker.
447            int lengthEnd = indexOf("}\r\n", lengthStart);
448            if (lengthEnd == -1) {
449                throw new ResponseFormatException("Missing terminator on literal length");
450            }
451    
452            int count = 0;
453            try {
454                count = Integer.parseInt(substring(lengthStart, lengthEnd));
455            } catch (NumberFormatException e) {
456                throw new ResponseFormatException("Invalid literal length " + substring(lengthStart, lengthEnd));
457            }
458            
459            // step over the length
460            pos = lengthEnd + 3;
461    
462            // too long?
463            if (pos + count > response.length) {
464                throw new ResponseFormatException("Invalid literal length: " + count);
465            }
466    
467            byte[] value = subarray(pos, pos + count);
468            pos += count;
469            
470            return value;
471        }
472    
473    
474        /**
475         * Extract a substring from the response buffer.
476         *
477         * @param start  The starting offset.
478         * @param end    The end offset (+ 1).
479         *
480         * @return A String extracted from the buffer.
481         */
482        protected String substring(int start, int end ) {
483            return new String(response, start, end - start);
484        }
485    
486    
487        /**
488         * Extract a subarray from the response buffer.
489         *
490         * @param start  The starting offset.
491         * @param end    The end offset (+ 1).
492         *
493         * @return A byte array string extracted rom the buffer.
494         */
495        protected byte[] subarray(int start, int end ) {
496            byte[] result = new byte[end - start];
497            System.arraycopy(response, start, result, 0, end - start);
498            return result;
499        }
500    
501    
502        /**
503         * Test if the bytes in the response buffer match a given
504         * string value.
505         *
506         * @param position The compare position.
507         * @param needle   The needle string we're testing for.
508         *
509         * @return True if the bytes match the needle value, false for any
510         *         mismatch.
511         */
512        public boolean match(int position, String needle) {
513            int length = needle.length();
514    
515            if (response.length - position < length) {
516                return false;
517            }
518    
519            for (int i = 0; i < length; i++) {
520                if (response[position + i ] != needle.charAt(i)) {
521                    return false;
522                }
523            }
524            return true;
525        }
526    
527    
528        /**
529         * Search for a given string starting from the current position
530         * cursor.
531         *
532         * @param needle The search string.
533         *
534         * @return The index of a match (in absolute byte position in the
535         *         response buffer).
536         */
537        public int indexOf(String needle) {
538            return indexOf(needle, pos);
539        }
540    
541        /**
542         * Search for a string in the response buffer starting from the
543         * indicated position.
544         *
545         * @param needle   The search string.
546         * @param position The starting buffer position.
547         *
548         * @return The index of the match position.  Returns -1 for no match.
549         */
550        public int indexOf(String needle, int position) {
551            // get the last possible match position
552            int last = response.length - needle.length();
553            // no match possible
554            if (last < position) {
555                return -1;
556            }
557    
558            for (int i = position; i <= last; i++) {
559                if (match(i, needle)) {
560                    return i;
561                }
562            }
563            return -1;
564        }
565    
566    
567    
568        /**
569         * Skip white space in the token string.
570         */
571        private void eatWhiteSpace() {
572            // skip to end of whitespace
573            while (++pos < response.length
574                    && WHITE.indexOf(response[pos]) != -1)
575                ;
576        }
577        
578        
579        /**
580         * Ensure that the next token in the parsed response is a
581         * '(' character.
582         *
583         * @exception ResponseFormatException
584         */
585        public void checkLeftParen() throws MessagingException {
586            Token token = next();
587            if (token.getType() != '(') {
588                throw new ResponseFormatException("Missing '(' in response");
589            }
590        }
591    
592    
593        /**
594         * Ensure that the next token in the parsed response is a
595         * ')' character.
596         *
597         * @exception ResponseFormatException
598         */
599        public void checkRightParen() throws MessagingException {
600            Token token = next();
601            if (token.getType() != ')') {
602                throw new ResponseFormatException("Missing ')' in response");
603            }
604        }
605    
606    
607        /**
608         * Read a string-valued token from the response.  A string
609         * valued token can be either a quoted string, a literal value,
610         * or an atom.  Any other token type is an error.
611         *
612         * @return The string value of the source token.
613         * @exception ResponseFormatException
614         */
615        public String readString() throws MessagingException {
616            Token token = next(true);
617            int type = token.getType();
618            if (type == Token.NIL) {
619                return null;
620            }
621            if (type != Token.ATOM && type != Token.QUOTEDSTRING && type != Token.LITERAL && type != Token.NUMERIC) {
622                throw new ResponseFormatException("String token expected in response: " + token.getValue());
623            }
624            return token.getValue();
625        }
626        
627    
628        /**
629         * Read an encoded string-valued token from the response.  A string
630         * valued token can be either a quoted string, a literal value,
631         * or an atom.  Any other token type is an error.
632         *
633         * @return The string value of the source token.
634         * @exception ResponseFormatException
635         */
636        public String readEncodedString() throws MessagingException {
637            String value = readString(); 
638            return decode(value); 
639        }
640    
641    
642        /**
643         * Decode a Base 64 encoded string value.
644         * 
645         * @param original The original encoded string.
646         * 
647         * @return The decoded string. 
648         * @exception MessagingException
649         */
650        public String decode(String original) throws MessagingException {
651            StringBuffer result = new StringBuffer();
652    
653            for (int i = 0; i < original.length(); i++) {
654                char ch = original.charAt(i);
655    
656                if (ch == '&') {
657                    i = decode(original, i, result);
658                }
659                else {
660                    result.append(ch);
661                }
662            }
663    
664            return result.toString();
665        }
666    
667    
668        /**
669         * Decode a section of an encoded string value. 
670         * 
671         * @param original The original source string.
672         * @param index    The current working index.
673         * @param result   The StringBuffer used for the decoded result.
674         * 
675         * @return The new index for the decoding operation. 
676         * @exception MessagingException
677         */
678        public static int decode(String original, int index, StringBuffer result) throws MessagingException {
679            // look for the section terminator
680            int terminator = original.indexOf('-', index);
681    
682            // unmatched?
683            if (terminator == -1) {
684                throw new MessagingException("Invalid UTF-7 encoded string");
685            }
686    
687            // is this just an escaped "&"?
688            if (terminator == index + 1) {
689                // append and skip over this.
690                result.append('&');
691                return index + 2;
692            }
693    
694            // step over the starting char
695            index++;
696    
697            int chars = terminator - index;
698            int quads = chars / 4;
699            int residual = chars % 4;
700    
701            // buffer for decoded characters
702            byte[] buffer = new byte[4];
703            int bufferCount = 0;
704    
705            // process each of the full triplet pairs
706            for (int i = 0; i < quads; i++) {
707                byte b1 = decodingTable[original.charAt(index++) & 0xff];
708                byte b2 = decodingTable[original.charAt(index++) & 0xff];
709                byte b3 = decodingTable[original.charAt(index++) & 0xff];
710                byte b4 = decodingTable[original.charAt(index++) & 0xff];
711    
712                buffer[bufferCount++] = (byte)((b1 << 2) | (b2 >> 4));
713                buffer[bufferCount++] = (byte)((b2 << 4) | (b3 >> 2));
714                buffer[bufferCount++] = (byte)((b3 << 6) | b4);
715    
716                // we've written 3 bytes to the buffer, but we might have a residual from a previous
717                // iteration to deal with.
718                if (bufferCount == 4) {
719                    // two complete chars here
720                    b1 = buffer[0];
721                    b2 = buffer[1];
722                    result.append((char)((b1 << 8) + (b2 & 0xff)));
723                    b1 = buffer[2];
724                    b2 = buffer[3];
725                    result.append((char)((b1 << 8) + (b2 & 0xff)));
726                    bufferCount = 0;
727                }
728                else {
729                    // we need to save the 3rd byte for the next go around
730                    b1 = buffer[0];
731                    b2 = buffer[1];
732                    result.append((char)((b1 << 8) + (b2 & 0xff)));
733                    buffer[0] = buffer[2];
734                    bufferCount = 1;
735                }
736            }
737    
738            // properly encoded, we should have an even number of bytes left.
739    
740            switch (residual) {
741                // no residual...so we better not have an extra in the buffer
742                case 0:
743                    // this is invalid...we have an odd number of bytes so far,
744                    if (bufferCount == 1) {
745                        throw new MessagingException("Invalid UTF-7 encoded string");
746                    }
747                // one byte left.  This shouldn't be valid.  We need at least 2 bytes to
748                // encode one unprintable char.
749                case 1:
750                    throw new MessagingException("Invalid UTF-7 encoded string");
751    
752                // ok, we have two bytes left, which can only encode a single byte.  We must have
753                // a dangling unhandled char.
754                case 2:
755                {
756                    if (bufferCount != 1) {
757                        throw new MessagingException("Invalid UTF-7 encoded string");
758                    }
759                    byte b1 = decodingTable[original.charAt(index++) & 0xff];
760                    byte b2 = decodingTable[original.charAt(index++) & 0xff];
761                    buffer[bufferCount++] = (byte)((b1 << 2) | (b2 >> 4));
762    
763                    b1 = buffer[0];
764                    b2 = buffer[1];
765                    result.append((char)((b1 << 8) + (b2 & 0xff)));
766                    break;
767                }
768    
769                // we have 2 encoded chars.  In this situation, we can't have a leftover.
770                case 3:
771                {
772                    // this is invalid...we have an odd number of bytes so far,
773                    if (bufferCount == 1) {
774                        throw new MessagingException("Invalid UTF-7 encoded string");
775                    }
776                    byte b1 = decodingTable[original.charAt(index++) & 0xff];
777                    byte b2 = decodingTable[original.charAt(index++) & 0xff];
778                    byte b3 = decodingTable[original.charAt(index++) & 0xff];
779    
780                    buffer[bufferCount++] = (byte)((b1 << 2) | (b2 >> 4));
781                    buffer[bufferCount++] = (byte)((b2 << 4) | (b3 >> 2));
782    
783                    b1 = buffer[0];
784                    b2 = buffer[1];
785                    result.append((char)((b1 << 8) + (b2 & 0xff)));
786                    break;
787                }
788            }
789    
790            // return the new scan location
791            return terminator + 1;
792        }
793    
794        /**
795         * Read a string-valued token from the response, verifying this is an ATOM token.
796         *
797         * @return The string value of the source token.
798         * @exception ResponseFormatException
799         */
800        public String readAtom() throws MessagingException {
801            return readAtom(false); 
802        }
803        
804    
805        /**
806         * Read a string-valued token from the response, verifying this is an ATOM token.
807         *
808         * @return The string value of the source token.
809         * @exception ResponseFormatException
810         */
811        public String readAtom(boolean expandedDelimiters) throws MessagingException {
812            Token token = next(false, expandedDelimiters);
813            int type = token.getType();
814    
815            if (type != Token.ATOM) {
816                throw new ResponseFormatException("ATOM token expected in response: " + token.getValue());
817            }
818            return token.getValue();
819        }
820    
821    
822        /**
823         * Read a number-valued token from the response.  This must be an ATOM
824         * token.
825         *
826         * @return The integer value of the source token.
827         * @exception ResponseFormatException
828         */
829        public int readInteger() throws MessagingException {
830            Token token = next();
831            return token.getInteger(); 
832        }
833    
834    
835        /**
836         * Read a number-valued token from the response.  This must be an ATOM
837         * token.
838         *
839         * @return The long value of the source token.
840         * @exception ResponseFormatException
841         */
842        public int readLong() throws MessagingException {
843            Token token = next();
844            return token.getInteger(); 
845        }
846    
847    
848        /**
849         * Read a string-valued token from the response.  A string
850         * valued token can be either a quoted string, a literal value,
851         * or an atom.  Any other token type is an error.
852         *
853         * @return The string value of the source token.
854         * @exception ResponseFormatException
855         */
856        public String readStringOrNil() throws MessagingException {
857            // we need to recognize the NIL token.
858            Token token = next(true);
859            int type = token.getType();
860    
861            if (type != Token.ATOM && type != Token.QUOTEDSTRING && type != Token.LITERAL && type != Token.NIL) {
862                throw new ResponseFormatException("String token or NIL expected in response: " + token.getValue());
863            }
864            // this returns null if the token is the NIL token.
865            return token.getValue();
866        }
867    
868    
869        /**
870         * Read a quoted string-valued token from the response.
871         * Any other token type other than NIL is an error.
872         *
873         * @return The string value of the source token.
874         * @exception ResponseFormatException
875         */
876        protected String readQuotedStringOrNil() throws MessagingException {
877            // we need to recognize the NIL token.
878            Token token = next(true);
879            int type = token.getType();
880    
881            if (type != Token.QUOTEDSTRING && type != Token.NIL) {
882                throw new ResponseFormatException("String token or NIL expected in response");
883            }
884            // this returns null if the token is the NIL token.
885            return token.getValue();
886        }
887    
888    
889        /**
890         * Read a date from a response string.  This is expected to be in
891         * Internet Date format, but there's a lot of variation implemented
892         * out there.  If we're unable to format this correctly, we'll
893         * just return null.
894         *
895         * @return A Date object created from the source date.
896         */
897        public Date readDate() throws MessagingException {
898            String value = readString();
899    
900            try {
901                return dateParser.parse(value);
902            } catch (Exception e) {
903                // we're just skipping over this, so return null
904                return null;
905            }
906        }
907    
908    
909        /**
910         * Read a date from a response string.  This is expected to be in
911         * Internet Date format, but there's a lot of variation implemented
912         * out there.  If we're unable to format this correctly, we'll
913         * just return null.
914         *
915         * @return A Date object created from the source date.
916         */
917        public Date readDateOrNil() throws MessagingException {
918            String value = readStringOrNil();
919            // this might be optional
920            if (value == null) {
921                return null; 
922            }
923    
924            try {
925                return dateParser.parse(value);
926            } catch (Exception e) {
927                // we're just skipping over this, so return null
928                return null;
929            }
930        }
931    
932        /**
933         * Read an internet address from a Fetch response.  The
934         * addresses are returned as a set of string tokens in the
935         * order "personal list mailbox host".  Any of these tokens
936         * can be NIL.
937         *
938         * The address may also be the start of a group list, which
939         * is indicated by the host being NIL.  If we have found the
940         * start of a group, then we need to parse multiple elements
941         * until we find the group end marker (indicated by both the
942         * mailbox and the host being NIL), and create a group
943         * InternetAddress instance from this.
944         *
945         * @return An InternetAddress instance parsed from the
946         *         element.
947         * @exception ResponseFormatException
948         */
949        public InternetAddress readAddress() throws MessagingException {
950            // we recurse, expecting a null response back for sublists.  
951            if (peek().getType() != '(') {
952                return null; 
953            }
954            
955            // must start with a paren
956            checkLeftParen(); 
957    
958            // personal information
959            String personal = readStringOrNil();
960            // the domain routine information.
961            String routing = readStringOrNil();
962            // the target mailbox
963            String mailbox = readStringOrNil();
964            // and finally the host
965            String host = readStringOrNil();
966            // and validate the closing paren
967            checkRightParen();
968    
969            // if this is a real address, we need to compose
970            if (host != null) {
971                StringBuffer address = new StringBuffer();
972                if (routing != null) {
973                    address.append(routing);
974                    address.append(':');
975                }
976                address.append(mailbox);
977                address.append('@');
978                address.append(host);
979    
980                try {
981                    return new InternetAddress(address.toString(), personal);
982                } catch (UnsupportedEncodingException e) {
983                    throw new ResponseFormatException("Invalid Internet address format");
984                }
985            }
986            else {
987                // we're going to recurse on this.  If the mailbox is null (the group name), this is the group item
988                // terminator.
989                if (mailbox == null) {
990                    return null;
991                }
992    
993                StringBuffer groupAddress = new StringBuffer();
994    
995                groupAddress.append(mailbox);
996                groupAddress.append(':');
997                int count = 0;
998    
999                while (true) {
1000                    // now recurse until we hit the end of the list
1001                    InternetAddress member = readAddress();
1002                    if (member == null) {
1003                        groupAddress.append(';');
1004    
1005                        try {
1006                            return new InternetAddress(groupAddress.toString(), personal);
1007                        } catch (UnsupportedEncodingException e) {
1008                            throw new ResponseFormatException("Invalid Internet address format");
1009                        }
1010                    }
1011                    else {
1012                        if (count != 0) {
1013                            groupAddress.append(',');
1014                        }
1015                        groupAddress.append(member.toString());
1016                        count++;
1017                    }
1018                }
1019            }
1020        }
1021    
1022    
1023        /**
1024         * Parse out a list of addresses.  This list of addresses is
1025         * surrounded by parentheses, and each address is also
1026         * parenthized (SP?).
1027         *
1028         * @return An array of the parsed addresses.
1029         * @exception ResponseFormatException
1030         */
1031        public InternetAddress[] readAddressList() throws MessagingException {
1032            // must start with a paren, but can be NIL also.
1033            Token token = next(true);
1034            int type = token.getType();
1035    
1036            // either of these results in a null address.  The caller determines based on
1037            // context whether this was optional or not.
1038            if (type == Token.NIL) {
1039                return null;
1040            }
1041            // non-nil address and no paren.  This is a syntax error.
1042            else if (type != '(') {
1043                throw new ResponseFormatException("Missing '(' in response");
1044            }
1045    
1046            List addresses = new ArrayList();
1047    
1048            // we have a list, now parse it.
1049            while (notListEnd()) {
1050                // go read the next address.  If we had an address, add to the list.
1051                // an address ITEM cannot be NIL inside the parens. 
1052                InternetAddress address = readAddress();
1053                addresses.add(address);
1054            }
1055            // we need to skip over the peeked token.
1056            checkRightParen(); 
1057            return (InternetAddress[])addresses.toArray(new InternetAddress[addresses.size()]);
1058        }
1059    
1060    
1061        /**
1062         * Check to see if we're at the end of a parenthized list
1063         * without advancing the parsing pointer.  If we are at the
1064         * end, then this will step over the closing paren.
1065         *
1066         * @return True if the next token is a closing list paren, false otherwise.
1067         * @exception ResponseFormatException
1068         */
1069        public boolean checkListEnd() throws MessagingException {
1070            Token token = peek(true);
1071            if (token.getType() == ')') {
1072                // step over this token.
1073                next();
1074                return true;
1075            }
1076            return false;
1077        }
1078    
1079    
1080        /**
1081         * Reads a string item which can be encoded either as a single
1082         * string-valued token or a parenthized list of string tokens.
1083         *
1084         * @return A List containing all of the strings.
1085         * @exception ResponseFormatException
1086         */
1087        public List readStringList() throws MessagingException {
1088            Token token = peek(true);
1089    
1090            if (token.getType() == '(') {
1091                List list = new ArrayList();
1092    
1093                next();
1094    
1095                while (notListEnd()) {
1096                    String value = readString();
1097                    // this can be NIL, technically
1098                    if (value != null) {
1099                        list.add(value);
1100                    }
1101                }
1102                // step over the closing paren 
1103                next();
1104    
1105                return list;
1106            }
1107            else if (token != NIL) {
1108                List list = new ArrayList();
1109    
1110                // just a single string value.
1111                String value = readString();
1112                // this can be NIL, technically
1113                if (value != null) {
1114                    list.add(value);
1115                }
1116    
1117                return list;
1118            } else {
1119                next();
1120            }
1121            return null;
1122        }
1123    
1124    
1125        /**
1126         * Reads all remaining tokens and returns them as a list of strings. 
1127         * NIL values are not supported. 
1128         *
1129         * @return A List containing all of the strings.
1130         * @exception ResponseFormatException
1131         */
1132        public List readStrings() throws MessagingException {
1133            List list = new ArrayList();
1134            
1135            while (hasMore()) {
1136                String value = readString();
1137                list.add(value);
1138            }
1139            return list; 
1140        }
1141    
1142    
1143        /**
1144         * Skip over an extension item.  This may be either a string
1145         * token or a parenthized item (with potential nesting).
1146         *
1147         * At the point where this is called, we're looking for a closing
1148         * ')', but we know it is not that.  An EOF is an error, however,
1149         */
1150        public void skipExtensionItem() throws MessagingException {
1151            Token token = next();
1152            int type = token.getType();
1153    
1154            // list form?  Scan to find the correct list closure.
1155            if (type == '(') {
1156                skipNestedValue();
1157            }
1158            // found an EOF?  Big problem
1159            else if (type == Token.EOF) {
1160                throw new ResponseFormatException("Missing ')'");
1161            }
1162        }
1163    
1164        /**
1165         * Skip over a parenthized value that we're not interested in.
1166         * These lists may contain nested sublists, so we need to
1167         * handle the nesting properly.
1168         */
1169        public void skipNestedValue() throws MessagingException {
1170            Token token = next();
1171    
1172            while (true) {
1173                int type = token.getType();
1174                // list terminator?
1175                if (type == ')') {
1176                    return;
1177                }
1178                // unexpected end of the tokens.
1179                else if (type == Token.EOF) {
1180                    throw new ResponseFormatException("Missing ')'");
1181                }
1182                // encountered a nested list?
1183                else if (type == '(') {
1184                    // recurse and finish this list.
1185                    skipNestedValue();
1186                }
1187                // we're just skipping the token.
1188                token = next();
1189            }
1190        }
1191    
1192        /**
1193         * Get the next token and verify that it's of the expected type
1194         * for the context.
1195         *
1196         * @param type   The type of token we're expecting.
1197         */
1198        public void checkToken(int type) throws MessagingException {
1199            Token token = next();
1200            if (token.getType() != type) {
1201                throw new ResponseFormatException("Unexpected token: " + token.getValue());
1202            }
1203        }
1204    
1205    
1206        /**
1207         * Read the next token as binary data.  The next token can be a literal, a quoted string, or
1208         * the token NIL (which returns a null result).  Any other token throws a ResponseFormatException.
1209         *
1210         * @return A byte array representing the rest of the response data.
1211         */
1212        public byte[] readByteArray() throws MessagingException {
1213            return readData(true);
1214        }
1215        
1216        
1217        /**
1218         * Determine what type of token encoding needs to be 
1219         * used for a string value.
1220         * 
1221         * @param value  The string to test.
1222         * 
1223         * @return Either Token.ATOM, Token.QUOTEDSTRING, or 
1224         *         Token.LITERAL, depending on the characters contained
1225         *         in the value.
1226         */
1227        static public int getEncoding(byte[] value) {
1228            
1229            // a null string always needs to be represented as a quoted literal. 
1230            if (value.length == 0) {
1231                return Token.QUOTEDSTRING; 
1232            }
1233            
1234            for (int i = 0; i < value.length; i++) {
1235                int ch = value[i]; 
1236                // make sure the sign extension is eliminated 
1237                ch = ch & 0xff;
1238                // check first for any characters that would 
1239                // disqualify a quoted string 
1240                // NULL
1241                if (ch == 0x00) {
1242                    return Token.LITERAL; 
1243                }
1244                // non-7bit ASCII
1245                if (ch > 0x7F) {
1246                    return Token.LITERAL; 
1247                }
1248                // carriage return
1249                if (ch == '\r') {
1250                    return Token.LITERAL; 
1251                }
1252                // linefeed 
1253                if (ch == '\n') {
1254                    return Token.LITERAL; 
1255                }
1256                // now check for ATOM disqualifiers 
1257                if (atomDelimiters.indexOf(ch) != -1) {
1258                    return Token.QUOTEDSTRING; 
1259                }
1260                // CTL character.  We've already eliminated the high characters 
1261                if (ch < 0x20) {
1262                    return Token.QUOTEDSTRING; 
1263                }
1264            }
1265            // this can be an ATOM token 
1266            return Token.ATOM;
1267        }
1268        
1269        
1270        /**
1271         * Read a ContentType or ContentDisposition parameter 
1272         * list from an IMAP command response.
1273         * 
1274         * @return A ParameterList instance containing the parameters. 
1275         * @exception MessagingException
1276         */
1277        public ParameterList readParameterList() throws MessagingException {
1278            ParameterList params = new ParameterList(); 
1279            
1280            // read the tokens, taking NIL into account. 
1281            Token token = next(true, false); 
1282            
1283            // just return an empty list if this is NIL 
1284            if (token.isType(token.NIL)) {
1285                return params; 
1286            }
1287            
1288            // these are pairs of strings for each parameter value 
1289            while (notListEnd()) {
1290                String name = readString(); 
1291                String value = readString(); 
1292                params.set(name, value); 
1293            }
1294            // we need to consume the list terminator 
1295            checkRightParen(); 
1296            return params; 
1297        }
1298        
1299        
1300        /**
1301         * Test if we have more data in the response buffer.
1302         * 
1303         * @return true if there are more tokens to process.  false if 
1304         *         we've reached the end of the stream.
1305         */
1306        public boolean hasMore() throws MessagingException {
1307            // we need to eat any white space that might be in the stream.  
1308            eatWhiteSpace();
1309            return pos < response.length; 
1310        }
1311        
1312        
1313        /**
1314         * Tests if we've reached the end of a parenthetical
1315         * list in our parsing stream.
1316         * 
1317         * @return true if the next token will be a ')'.  false if the 
1318         *         next token is anything else.
1319         * @exception MessagingException
1320         */
1321        public boolean notListEnd() throws MessagingException {
1322            return peek().getType() != ')';
1323        }
1324        
1325        /**
1326         * Read a list of Flag values from an IMAP response, 
1327         * returning a Flags instance containing the appropriate 
1328         * pieces. 
1329         * 
1330         * @return A Flags instance with the flag values. 
1331         * @exception MessagingException
1332         */
1333        public Flags readFlagList() throws MessagingException {
1334            Flags flags = new Flags();
1335            
1336            // this should be a list here 
1337            checkLeftParen(); 
1338            
1339            // run through the flag list 
1340            while (notListEnd()) {
1341                // the flags are a bit of a pain.  The flag names include "\" in the name, which 
1342                // is not a character allowed in an atom.  This requires a bit of customized parsing 
1343                // to handle this. 
1344                Token token = next(); 
1345                // flags can be specified as just atom tokens, so allow this as a user flag. 
1346                if (token.isType(token.ATOM)) {
1347                    // append the atom as a raw name 
1348                    flags.add(token.getValue()); 
1349                }
1350                // all of the system flags start with a '\' followed by 
1351                // an atom.  They also can be extension flags.  IMAP has a special 
1352                // case of "\*" that we need to check for. 
1353                else if (token.isType('\\')) {
1354                    token = next(); 
1355                    // the next token is the real bit we need to process. 
1356                    if (token.isType('*')) {
1357                        // this indicates USER flags are allowed. 
1358                        flags.add(Flags.Flag.USER); 
1359                    }
1360                    // if this is an atom name, handle as a system flag 
1361                    else if (token.isType(Token.ATOM)) {
1362                        String name = token.getValue(); 
1363                        if (name.equalsIgnoreCase("Seen")) {
1364                            flags.add(Flags.Flag.SEEN);
1365                        }
1366                        else if (name.equalsIgnoreCase("RECENT")) {
1367                            flags.add(Flags.Flag.RECENT);
1368                        }
1369                        else if (name.equalsIgnoreCase("DELETED")) {
1370                            flags.add(Flags.Flag.DELETED);
1371                        }
1372                        else if (name.equalsIgnoreCase("ANSWERED")) {
1373                            flags.add(Flags.Flag.ANSWERED);
1374                        }
1375                        else if (name.equalsIgnoreCase("DRAFT")) {
1376                            flags.add(Flags.Flag.DRAFT);
1377                        }
1378                        else if (name.equalsIgnoreCase("FLAGGED")) {
1379                            flags.add(Flags.Flag.FLAGGED);
1380                        }
1381                        else {
1382                            // this is a server defined flag....just add the name with the 
1383                            // flag thingy prepended. 
1384                            flags.add("\\" + name); 
1385                        }
1386                    }
1387                    else {
1388                        throw new MessagingException("Invalid Flag: " + token.getValue()); 
1389                    }
1390                }
1391                else {
1392                    throw new MessagingException("Invalid Flag: " + token.getValue()); 
1393                }
1394            }
1395            
1396            // step over this for good practice. 
1397            checkRightParen(); 
1398            
1399            return flags; 
1400        }
1401        
1402        
1403        /**
1404         * Read a list of Flag values from an IMAP response, 
1405         * returning a Flags instance containing the appropriate 
1406         * pieces. 
1407         * 
1408         * @return A Flags instance with the flag values. 
1409         * @exception MessagingException
1410         */
1411        public List readSystemNameList() throws MessagingException {
1412            List flags = new ArrayList(); 
1413            
1414            // this should be a list here 
1415            checkLeftParen(); 
1416            
1417            // run through the flag list 
1418            while (notListEnd()) {
1419                // the flags are a bit of a pain.  The flag names include "\" in the name, which 
1420                // is not a character allowed in an atom.  This requires a bit of customized parsing 
1421                // to handle this. 
1422                Token token = next(); 
1423                // all of the system flags start with a '\' followed by 
1424                // an atom.  They also can be extension flags.  IMAP has a special 
1425                // case of "\*" that we need to check for. 
1426                if (token.isType('\\')) {
1427                    token = next(); 
1428                    // if this is an atom name, handle as a system flag 
1429                    if (token.isType(Token.ATOM)) {
1430                        // add the token value to the list WITH the 
1431                        // flag indicator included.  The attributes method returns 
1432                        // these flag indicators, so we need to include it. 
1433                        flags.add("\\" + token.getValue()); 
1434                    }
1435                    else {
1436                        throw new MessagingException("Invalid Flag: " + token.getValue()); 
1437                    }
1438                }
1439                else {
1440                    throw new MessagingException("Invalid Flag: " + token.getValue()); 
1441                }
1442            }
1443            
1444            // step over this for good practice. 
1445            checkRightParen(); 
1446            
1447            return flags; 
1448        }
1449    }
1450