/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/*
NOTE: if you change this file and need to regenerate the tokenizer,
remember to use JRE 1.4 when running jflex (before Lucene 3.0).
This grammar now uses constructs (eg :digit:) whose meaning can
vary according to the JRE used to run jflex. See
https://issues.apache.org/jira/browse/LUCENE-1126 for details
*/
using System;
using Token = Lucene.Net.Analysis.Token;
namespace Lucene.Net.Analysis.Standard
{
/// This class is a scanner generated by
/// JFlex 1.4.1
/// on 12/18/07 9:22 PM from the specification file
/// /Volumes/User/grantingersoll/projects/lucene/java/lucene-clean/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex
///
class StandardTokenizerImpl
{
/// This character denotes the end of file
public const int YYEOF = - 1;
/// initial size of the lookahead buffer
private const int ZZ_BUFFERSIZE = 16384;
/// lexical states
public const int YYINITIAL = 0;
/// Translates characters to character classes
private const System.String ZZ_CMAP_PACKED =
"\x0009\x0000\x0001\x0000\x0001\x000d\x0001\x0000\x0001\x0000"+
"\x0001\x000c\x0012\x0000\x0001\x0000\x0005\x0000\x0001\x0005"+
"\x0001\x0003\x0004\x0000\x0001\x0009\x0001\x0007\x0001\x0004"+
"\x0001\x0009\x000a\x0002\x0006\x0000\x0001\x0006\x001a\x000a"+
"\x0004\x0000\x0001\x0008\x0001\x0000\x001a\x000a\x002f\x0000"+
"\x0001\x000a\x000a\x0000\x0001\x000a\x0004\x0000\x0001\x000a"+
"\x0005\x0000\x0017\x000a\x0001\x0000\x001f\x000a\x0001\x0000"+
"\u0128\x000a\x0002\x0000\x0012\x000a\x001c\x0000\x005e\x000a"+
"\x0002\x0000\x0009\x000a\x0002\x0000\x0007\x000a\x000e\x0000"+
"\x0002\x000a\x000e\x0000\x0005\x000a\x0009\x0000\x0001\x000a"+
"\x008b\x0000\x0001\x000a\x000b\x0000\x0001\x000a\x0001\x0000"+
"\x0003\x000a\x0001\x0000\x0001\x000a\x0001\x0000\x0014\x000a"+
"\x0001\x0000\x002c\x000a\x0001\x0000\x0008\x000a\x0002\x0000"+
"\x001a\x000a\x000c\x0000\x0082\x000a\x000a\x0000\x0039\x000a"+
"\x0002\x0000\x0002\x000a\x0002\x0000\x0002\x000a\x0003\x0000"+
"\x0026\x000a\x0002\x0000\x0002\x000a\x0037\x0000\x0026\x000a"+
"\x0002\x0000\x0001\x000a\x0007\x0000\x0027\x000a\x0048\x0000"+
"\x001b\x000a\x0005\x0000\x0003\x000a\x002e\x0000\x001a\x000a"+
"\x0005\x0000\x000b\x000a\x0015\x0000\x000a\x0002\x0007\x0000"+
"\x0063\x000a\x0001\x0000\x0001\x000a\x000f\x0000\x0002\x000a"+
"\x0009\x0000\x000a\x0002\x0003\x000a\x0013\x0000\x0001\x000a"+
"\x0001\x0000\x001b\x000a\x0053\x0000\x0026\x000a\u015f\x0000"+
"\x0035\x000a\x0003\x0000\x0001\x000a\x0012\x0000\x0001\x000a"+
"\x0007\x0000\x000a\x000a\x0004\x0000\x000a\x0002\x0015\x0000"+
"\x0008\x000a\x0002\x0000\x0002\x000a\x0002\x0000\x0016\x000a"+
"\x0001\x0000\x0007\x000a\x0001\x0000\x0001\x000a\x0003\x0000"+
"\x0004\x000a\x0022\x0000\x0002\x000a\x0001\x0000\x0003\x000a"+
"\x0004\x0000\x000a\x0002\x0002\x000a\x0013\x0000\x0006\x000a"+
"\x0004\x0000\x0002\x000a\x0002\x0000\x0016\x000a\x0001\x0000"+
"\x0007\x000a\x0001\x0000\x0002\x000a\x0001\x0000\x0002\x000a"+
"\x0001\x0000\x0002\x000a\x001f\x0000\x0004\x000a\x0001\x0000"+
"\x0001\x000a\x0007\x0000\x000a\x0002\x0002\x0000\x0003\x000a"+
"\x0010\x0000\x0007\x000a\x0001\x0000\x0001\x000a\x0001\x0000"+
"\x0003\x000a\x0001\x0000\x0016\x000a\x0001\x0000\x0007\x000a"+
"\x0001\x0000\x0002\x000a\x0001\x0000\x0005\x000a\x0003\x0000"+
"\x0001\x000a\x0012\x0000\x0001\x000a\x000f\x0000\x0001\x000a"+
"\x0005\x0000\x000a\x0002\x0015\x0000\x0008\x000a\x0002\x0000"+
"\x0002\x000a\x0002\x0000\x0016\x000a\x0001\x0000\x0007\x000a"+
"\x0001\x0000\x0002\x000a\x0002\x0000\x0004\x000a\x0003\x0000"+
"\x0001\x000a\x001e\x0000\x0002\x000a\x0001\x0000\x0003\x000a"+
"\x0004\x0000\x000a\x0002\x0015\x0000\x0006\x000a\x0003\x0000"+
"\x0003\x000a\x0001\x0000\x0004\x000a\x0003\x0000\x0002\x000a"+
"\x0001\x0000\x0001\x000a\x0001\x0000\x0002\x000a\x0003\x0000"+
"\x0002\x000a\x0003\x0000\x0003\x000a\x0003\x0000\x0008\x000a"+
"\x0001\x0000\x0003\x000a\x002d\x0000\x0009\x0002\x0015\x0000"+
"\x0008\x000a\x0001\x0000\x0003\x000a\x0001\x0000\x0017\x000a"+
"\x0001\x0000\x000a\x000a\x0001\x0000\x0005\x000a\x0026\x0000"+
"\x0002\x000a\x0004\x0000\x000a\x0002\x0015\x0000\x0008\x000a"+
"\x0001\x0000\x0003\x000a\x0001\x0000\x0017\x000a\x0001\x0000"+
"\x000a\x000a\x0001\x0000\x0005\x000a\x0024\x0000\x0001\x000a"+
"\x0001\x0000\x0002\x000a\x0004\x0000\x000a\x0002\x0015\x0000"+
"\x0008\x000a\x0001\x0000\x0003\x000a\x0001\x0000\x0017\x000a"+
"\x0001\x0000\x0010\x000a\x0026\x0000\x0002\x000a\x0004\x0000"+
"\x000a\x0002\x0015\x0000\x0012\x000a\x0003\x0000\x0018\x000a"+
"\x0001\x0000\x0009\x000a\x0001\x0000\x0001\x000a\x0002\x0000"+
"\x0007\x000a\x0039\x0000\x0001\x0001\x0030\x000a\x0001\x0001"+
"\x0002\x000a\x000c\x0001\x0007\x000a\x0009\x0001\x000a\x0002"+
"\x0027\x0000\x0002\x000a\x0001\x0000\x0001\x000a\x0002\x0000"+
"\x0002\x000a\x0001\x0000\x0001\x000a\x0002\x0000\x0001\x000a"+
"\x0006\x0000\x0004\x000a\x0001\x0000\x0007\x000a\x0001\x0000"+
"\x0003\x000a\x0001\x0000\x0001\x000a\x0001\x0000\x0001\x000a"+
"\x0002\x0000\x0002\x000a\x0001\x0000\x0004\x000a\x0001\x0000"+
"\x0002\x000a\x0009\x0000\x0001\x000a\x0002\x0000\x0005\x000a"+
"\x0001\x0000\x0001\x000a\x0009\x0000\x000a\x0002\x0002\x0000"+
"\x0002\x000a\x0022\x0000\x0001\x000a\x001f\x0000\x000a\x0002"+
"\x0016\x0000\x0008\x000a\x0001\x0000\x0022\x000a\x001d\x0000"+
"\x0004\x000a\x0074\x0000\x0022\x000a\x0001\x0000\x0005\x000a"+
"\x0001\x0000\x0002\x000a\x0015\x0000\x000a\x0002\x0006\x0000"+
"\x0006\x000a\x004a\x0000\x0026\x000a\x000a\x0000\x0027\x000a"+
"\x0009\x0000\x005a\x000a\x0005\x0000\x0044\x000a\x0005\x0000"+
"\x0052\x000a\x0006\x0000\x0007\x000a\x0001\x0000\x003f\x000a"+
"\x0001\x0000\x0001\x000a\x0001\x0000\x0004\x000a\x0002\x0000"+
"\x0007\x000a\x0001\x0000\x0001\x000a\x0001\x0000\x0004\x000a"+
"\x0002\x0000\x0027\x000a\x0001\x0000\x0001\x000a\x0001\x0000"+
"\x0004\x000a\x0002\x0000\x001f\x000a\x0001\x0000\x0001\x000a"+
"\x0001\x0000\x0004\x000a\x0002\x0000\x0007\x000a\x0001\x0000"+
"\x0001\x000a\x0001\x0000\x0004\x000a\x0002\x0000\x0007\x000a"+
"\x0001\x0000\x0007\x000a\x0001\x0000\x0017\x000a\x0001\x0000"+
"\x001f\x000a\x0001\x0000\x0001\x000a\x0001\x0000\x0004\x000a"+
"\x0002\x0000\x0007\x000a\x0001\x0000\x0027\x000a\x0001\x0000"+
"\x0013\x000a\x000e\x0000\x0009\x0002\x002e\x0000\x0055\x000a"+
"\x000c\x0000\u026c\x000a\x0002\x0000\x0008\x000a\x000a\x0000"+
"\x001a\x000a\x0005\x0000\x004b\x000a\x0095\x0000\x0034\x000a"+
"\x002c\x0000\x000a\x0002\x0026\x0000\x000a\x0002\x0006\x0000"+
"\x0058\x000a\x0008\x0000\x0029\x000a\u0557\x0000\x009c\x000a"+
"\x0004\x0000\x005a\x000a\x0006\x0000\x0016\x000a\x0002\x0000"+
"\x0006\x000a\x0002\x0000\x0026\x000a\x0002\x0000\x0006\x000a"+
"\x0002\x0000\x0008\x000a\x0001\x0000\x0001\x000a\x0001\x0000"+
"\x0001\x000a\x0001\x0000\x0001\x000a\x0001\x0000\x001f\x000a"+
"\x0002\x0000\x0035\x000a\x0001\x0000\x0007\x000a\x0001\x0000"+
"\x0001\x000a\x0003\x0000\x0003\x000a\x0001\x0000\x0007\x000a"+
"\x0003\x0000\x0004\x000a\x0002\x0000\x0006\x000a\x0004\x0000"+
"\x000d\x000a\x0005\x0000\x0003\x000a\x0001\x0000\x0007\x000a"+
"\x0082\x0000\x0001\x000a\x0082\x0000\x0001\x000a\x0004\x0000"+
"\x0001\x000a\x0002\x0000\x000a\x000a\x0001\x0000\x0001\x000a"+
"\x0003\x0000\x0005\x000a\x0006\x0000\x0001\x000a\x0001\x0000"+
"\x0001\x000a\x0001\x0000\x0001\x000a\x0001\x0000\x0004\x000a"+
"\x0001\x0000\x0003\x000a\x0001\x0000\x0007\x000a\u0ecb\x0000"+
"\x0002\x000a\x002a\x0000\x0005\x000a\x000a\x0000\x0001\x000b"+
"\x0054\x000b\x0008\x000b\x0002\x000b\x0002\x000b\x005a\x000b"+
"\x0001\x000b\x0003\x000b\x0006\x000b\x0028\x000b\x0003\x000b"+
"\x0001\x0000\x005e\x000a\x0011\x0000\x0018\x000a\x0038\x0000"+
"\x0010\x000b\u0100\x0000\x0080\x000b\x0080\x0000\u19b6\x000b"+
"\x000a\x000b\x0040\x0000\u51a6\x000b\x005a\x000b\u048d\x000a"+
"\u0773\x0000\u2ba4\x000a\u215c\x0000\u012e\x000b\x00d2\x000b"+
"\x0007\x000a\x000c\x0000\x0005\x000a\x0005\x0000\x0001\x000a"+
"\x0001\x0000\x000a\x000a\x0001\x0000\x000d\x000a\x0001\x0000"+
"\x0005\x000a\x0001\x0000\x0001\x000a\x0001\x0000\x0002\x000a"+
"\x0001\x0000\x0002\x000a\x0001\x0000\x006c\x000a\x0021\x0000"+
"\u016b\x000a\x0012\x0000\x0040\x000a\x0002\x0000\x0036\x000a"+
"\x0028\x0000\x000c\x000a\x0074\x0000\x0003\x000a\x0001\x0000"+
"\x0001\x000a\x0001\x0000\x0087\x000a\x0013\x0000\x000a\x0002"+
"\x0007\x0000\x001a\x000a\x0006\x0000\x001a\x000a\x000a\x0000"+
"\x0001\x000b\x003a\x000b\x001f\x000a\x0003\x0000\x0006\x000a"+
"\x0002\x0000\x0006\x000a\x0002\x0000\x0006\x000a\x0002\x0000"+
"\x0003\x000a\x0023\x0000";
/// Translates characters to character classes
private static readonly char[] ZZ_CMAP = ZzUnpackCMap(ZZ_CMAP_PACKED);
/// Translates DFA states to action switch labels.
private static readonly int[] ZZ_ACTION = ZzUnpackAction();
private const System.String ZZ_ACTION_PACKED_0 =
"\x0001\x0000\x0001\x0001\x0003\x0002\x0001\x0003\x0001\x0001"+
"\x000b\x0000\x0001\x0002\x0003\x0004\x0002\x0000\x0001\x0005"+
"\x0001\x0000\x0001\x0005\x0003\x0004\x0006\x0005\x0001\x0006"+
"\x0001\x0004\x0002\x0007\x0001\x0008\x0001\x0000\x0001\x0008"+
"\x0003\x0000\x0002\x0008\x0001\x0009\x0001\x000a\x0001\x0004";
private static int[] ZzUnpackAction()
{
int[] result = new int[51];
int offset = 0;
offset = ZzUnpackAction(ZZ_ACTION_PACKED_0, offset, result);
return result;
}
private static int ZzUnpackAction(System.String packed, int offset, int[] result)
{
int i = 0; /* index in packed string */
int j = offset; /* index in unpacked array */
int l = packed.Length;
while (i < l)
{
int count = packed[i++];
int value_Renamed = packed[i++];
do
result[j++] = value_Renamed;
while (--count > 0);
}
return j;
}
/// Translates a state to a row index in the transition table
private static readonly int[] ZZ_ROWMAP = ZzUnpackRowMap();
private const System.String ZZ_ROWMAP_PACKED_0 =
"\x0000\x0000\x0000\x000e\x0000\x001c\x0000\x002a\x0000\x0038"+
"\x0000\x000e\x0000\x0046\x0000\x0054\x0000\x0062\x0000\x0070"+
"\x0000\x007e\x0000\x008c\x0000\x009a\x0000\x00a8\x0000\x00b6"+
"\x0000\x00c4\x0000\x00d2\x0000\x00e0\x0000\x00ee\x0000\x00fc"+
"\x0000\u010a\x0000\u0118\x0000\u0126\x0000\u0134\x0000\u0142"+
"\x0000\u0150\x0000\u015e\x0000\u016c\x0000\u017a\x0000\u0188"+
"\x0000\u0196\x0000\u01a4\x0000\u01b2\x0000\u01c0\x0000\u01ce"+
"\x0000\u01dc\x0000\u01ea\x0000\u01f8\x0000\x00d2\x0000\u0206"+
"\x0000\u0214\x0000\u0222\x0000\u0230\x0000\u023e\x0000\u024c"+
"\x0000\u025a\x0000\x0054\x0000\x008c\x0000\u0268\x0000\u0276"+
"\x0000\u0284";
private static int[] ZzUnpackRowMap()
{
int[] result = new int[51];
int offset = 0;
offset = ZzUnpackRowMap(ZZ_ROWMAP_PACKED_0, offset, result);
return result;
}
private static int ZzUnpackRowMap(System.String packed, int offset, int[] result)
{
int i = 0; /* index in packed string */
int j = offset; /* index in unpacked array */
int l = packed.Length;
while (i < l)
{
int high = packed[i++] << 16;
result[j++] = high | packed[i++];
}
return j;
}
/// The transition table of the DFA
private static readonly int[] ZZ_TRANS = ZzUnpackTrans();
private const System.String ZZ_TRANS_PACKED_0 =
"\x0001\x0002\x0001\x0003\x0001\x0004\x0007\x0002\x0001\x0005"+
"\x0001\x0006\x0001\x0007\x0001\x0002\x000f\x0000\x0002\x0003"+
"\x0001\x0000\x0001\x0008\x0001\x0000\x0001\x0009\x0002\x000a"+
"\x0001\x000b\x0001\x0003\x0004\x0000\x0001\x0003\x0001\x0004"+
"\x0001\x0000\x0001\x000c\x0001\x0000\x0001\x0009\x0002\x000d"+
"\x0001\x000e\x0001\x0004\x0004\x0000\x0001\x0003\x0001\x0004"+
"\x0001\x000f\x0001\x0010\x0001\x0011\x0001\x0012\x0002\x000a"+
"\x0001\x000b\x0001\x0013\x0010\x0000\x0001\x0002\x0001\x0000"+
"\x0001\x0014\x0001\x0015\x0007\x0000\x0001\x0016\x0004\x0000"+
"\x0002\x0017\x0007\x0000\x0001\x0017\x0004\x0000\x0001\x0018"+
"\x0001\x0019\x0007\x0000\x0001\x001a\x0005\x0000\x0001\x001b"+
"\x0007\x0000\x0001\x000b\x0004\x0000\x0001\x001c\x0001\x001d"+
"\x0007\x0000\x0001\x001e\x0004\x0000\x0001\x001f\x0001\x0020"+
"\x0007\x0000\x0001\x0021\x0004\x0000\x0001\x0022\x0001\x0023"+
"\x0007\x0000\x0001\x0024\x000d\x0000\x0001\x0025\x0004\x0000"+
"\x0001\x0014\x0001\x0015\x0007\x0000\x0001\x0026\x000d\x0000"+
"\x0001\x0027\x0004\x0000\x0002\x0017\x0007\x0000\x0001\x0028"+
"\x0004\x0000\x0001\x0003\x0001\x0004\x0001\x000f\x0001\x0008"+
"\x0001\x0011\x0001\x0012\x0002\x000a\x0001\x000b\x0001\x0013"+
"\x0004\x0000\x0002\x0014\x0001\x0000\x0001\x0029\x0001\x0000"+
"\x0001\x0009\x0002\x002a\x0001\x0000\x0001\x0014\x0004\x0000"+
"\x0001\x0014\x0001\x0015\x0001\x0000\x0001\x002b\x0001\x0000"+
"\x0001\x0009\x0002\x002c\x0001\x002d\x0001\x0015\x0004\x0000"+
"\x0001\x0014\x0001\x0015\x0001\x0000\x0001\x0029\x0001\x0000"+
"\x0001\x0009\x0002\x002a\x0001\x0000\x0001\x0016\x0004\x0000"+
"\x0002\x0017\x0001\x0000\x0001\x002e\x0002\x0000\x0001\x002e"+
"\x0002\x0000\x0001\x0017\x0004\x0000\x0002\x0018\x0001\x0000"+
"\x0001\x002a\x0001\x0000\x0001\x0009\x0002\x002a\x0001\x0000"+
"\x0001\x0018\x0004\x0000\x0001\x0018\x0001\x0019\x0001\x0000"+
"\x0001\x002c\x0001\x0000\x0001\x0009\x0002\x002c\x0001\x002d"+
"\x0001\x0019\x0004\x0000\x0001\x0018\x0001\x0019\x0001\x0000"+
"\x0001\x002a\x0001\x0000\x0001\x0009\x0002\x002a\x0001\x0000"+
"\x0001\x001a\x0005\x0000\x0001\x001b\x0001\x0000\x0001\x002d"+
"\x0002\x0000\x0003\x002d\x0001\x001b\x0004\x0000\x0002\x001c"+
"\x0001\x0000\x0001\x002f\x0001\x0000\x0001\x0009\x0002\x000a"+
"\x0001\x000b\x0001\x001c\x0004\x0000\x0001\x001c\x0001\x001d"+
"\x0001\x0000\x0001\x0030\x0001\x0000\x0001\x0009\x0002\x000d"+
"\x0001\x000e\x0001\x001d\x0004\x0000\x0001\x001c\x0001\x001d"+
"\x0001\x0000\x0001\x002f\x0001\x0000\x0001\x0009\x0002\x000a"+
"\x0001\x000b\x0001\x001e\x0004\x0000\x0002\x001f\x0001\x0000"+
"\x0001\x000a\x0001\x0000\x0001\x0009\x0002\x000a\x0001\x000b"+
"\x0001\x001f\x0004\x0000\x0001\x001f\x0001\x0020\x0001\x0000"+
"\x0001\x000d\x0001\x0000\x0001\x0009\x0002\x000d\x0001\x000e"+
"\x0001\x0020\x0004\x0000\x0001\x001f\x0001\x0020\x0001\x0000"+
"\x0001\x000a\x0001\x0000\x0001\x0009\x0002\x000a\x0001\x000b"+
"\x0001\x0021\x0004\x0000\x0002\x0022\x0001\x0000\x0001\x000b"+
"\x0002\x0000\x0003\x000b\x0001\x0022\x0004\x0000\x0001\x0022"+
"\x0001\x0023\x0001\x0000\x0001\x000e\x0002\x0000\x0003\x000e"+
"\x0001\x0023\x0004\x0000\x0001\x0022\x0001\x0023\x0001\x0000"+
"\x0001\x000b\x0002\x0000\x0003\x000b\x0001\x0024\x0006\x0000"+
"\x0001\x000f\x0006\x0000\x0001\x0025\x0004\x0000\x0001\x0014"+
"\x0001\x0015\x0001\x0000\x0001\x0031\x0001\x0000\x0001\x0009"+
"\x0002\x002a\x0001\x0000\x0001\x0016\x0004\x0000\x0002\x0017"+
"\x0001\x0000\x0001\x002e\x0002\x0000\x0001\x002e\x0002\x0000"+
"\x0001\x0028\x0004\x0000\x0002\x0014\x0007\x0000\x0001\x0014"+
"\x0004\x0000\x0002\x0018\x0007\x0000\x0001\x0018\x0004\x0000"+
"\x0002\x001c\x0007\x0000\x0001\x001c\x0004\x0000\x0002\x001f"+
"\x0007\x0000\x0001\x001f\x0004\x0000\x0002\x0022\x0007\x0000"+
"\x0001\x0022\x0004\x0000\x0002\x0032\x0007\x0000\x0001\x0032"+
"\x0004\x0000\x0002\x0014\x0007\x0000\x0001\x0033\x0004\x0000"+
"\x0002\x0032\x0001\x0000\x0001\x002e\x0002\x0000\x0001\x002e"+
"\x0002\x0000\x0001\x0032\x0004\x0000\x0002\x0014\x0001\x0000"+
"\x0001\x0031\x0001\x0000\x0001\x0009\x0002\x002a\x0001\x0000"+
"\x0001\x0014\x0003\x0000";
private static int[] ZzUnpackTrans()
{
int[] result = new int[658];
int offset = 0;
offset = ZzUnpackTrans(ZZ_TRANS_PACKED_0, offset, result);
return result;
}
private static int ZzUnpackTrans(System.String packed, int offset, int[] result)
{
int i = 0; /* index in packed string */
int j = offset; /* index in unpacked array */
int l = packed.Length;
while (i < l)
{
int count = packed[i++];
int value_Renamed = packed[i++];
value_Renamed--;
do
result[j++] = value_Renamed;
while (--count > 0);
}
return j;
}
/* error codes */
private const int ZZ_UNKNOWN_ERROR = 0;
private const int ZZ_NO_MATCH = 1;
private const int ZZ_PUSHBACK_2BIG = 2;
/* error messages for the codes above */
private static readonly System.String[] ZZ_ERROR_MSG = new System.String[]{"Unkown internal scanner error", "Error: could not match input", "Error: pushback value was too large"};
/// ZZ_ATTRIBUTE[aState] contains the attributes of state aState
private static readonly int[] ZZ_ATTRIBUTE = ZzUnpackAttribute();
private const System.String ZZ_ATTRIBUTE_PACKED_0 =
"\x0001\x0000\x0001\x0009\x0003\x0001\x0001\x0009\x0001\x0001"+
"\x000b\x0000\x0004\x0001\x0002\x0000\x0001\x0001\x0001\x0000"+
"\x000f\x0001\x0001\x0000\x0001\x0001\x0003\x0000\x0005\x0001";
private static int[] ZzUnpackAttribute()
{
int[] result = new int[51];
int offset = 0;
offset = ZzUnpackAttribute(ZZ_ATTRIBUTE_PACKED_0, offset, result);
return result;
}
private static int ZzUnpackAttribute(System.String packed, int offset, int[] result)
{
int i = 0; /* index in packed string */
int j = offset; /* index in unpacked array */
int l = packed.Length;
while (i < l)
{
int count = packed[i++];
int value_Renamed = packed[i++];
do
result[j++] = value_Renamed;
while (--count > 0);
}
return j;
}
/// the input device
private System.IO.TextReader zzReader;
/// the current state of the DFA
private int zzState;
/// the current lexical state
private int zzLexicalState = YYINITIAL;
/// this buffer contains the current text to be matched and is
/// the source of the yytext() string
///
private char[] zzBuffer = new char[ZZ_BUFFERSIZE];
/// the textposition at the last accepting state
private int zzMarkedPos;
/// the textposition at the last state to be included in yytext
private int zzPushbackPos;
/// the current text position in the buffer
private int zzCurrentPos;
/// startRead marks the beginning of the yytext() string in the buffer
private int zzStartRead;
/// endRead marks the last character in the buffer, that has been read
/// from input
///
private int zzEndRead;
/// number of newlines encountered up to the start of the matched text
private int yyline;
/// the number of characters up to the start of the matched text
private int yychar;
/// the number of characters from the last newline up to the start of the
/// matched text
///
private int yycolumn;
/// zzAtBOL == true <=> the scanner is currently at the beginning of a line
private bool zzAtBOL = true;
/// zzAtEOF == true <=> the scanner is at the EOF
private bool zzAtEOF;
/* user code: */
public const int ALPHANUM = StandardTokenizer.ALPHANUM;
public const int APOSTROPHE = StandardTokenizer.APOSTROPHE;
public const int ACRONYM = StandardTokenizer.ACRONYM;
public const int COMPANY = StandardTokenizer.COMPANY;
public const int EMAIL = StandardTokenizer.EMAIL;
public const int HOST = StandardTokenizer.HOST;
public const int NUM = StandardTokenizer.NUM;
public const int CJ = StandardTokenizer.CJ;
/// this solves a bug where HOSTs that end with '.' are identified
/// as ACRONYMs. It is deprecated and will be removed in the next
/// release.
///
public const int ACRONYM_DEP = StandardTokenizer.ACRONYM_DEP;
public static readonly System.String[] TOKEN_TYPES = StandardTokenizer.TOKEN_TYPES;
public int Yychar()
{
return yychar;
}
/// Fills Lucene token with the current token text.
internal void GetText(Token t)
{
t.SetTermBuffer(zzBuffer, zzStartRead, zzMarkedPos - zzStartRead);
}
/// Creates a new scanner
/// There is also a java.io.InputStream version of this constructor.
///
///
/// the java.io.Reader to read input from.
///
internal StandardTokenizerImpl(System.IO.TextReader in_Renamed)
{
this.zzReader = in_Renamed;
}
/// Creates a new scanner.
/// There is also java.io.Reader version of this constructor.
///
///
/// the java.io.Inputstream to read input from.
///
internal StandardTokenizerImpl(System.IO.Stream in_Renamed) : this(new System.IO.StreamReader(in_Renamed, System.Text.Encoding.Default))
{
}
/// Unpacks the compressed character translation table.
///
///
/// the packed character translation table
///
/// the unpacked character translation table
///
private static char[] ZzUnpackCMap(System.String packed)
{
char[] map = new char[0x10000];
int i = 0; /* index in packed string */
int j = 0; /* index in unpacked array */
while (i < 1154)
{
int count = packed[i++];
char value_Renamed = packed[i++];
do
map[j++] = value_Renamed;
while (--count > 0);
}
return map;
}
/// Refills the input buffer.
///
///
/// false
, iff there was new input.
///
///
/// if any I/O-Error occurs
///
private bool ZzRefill()
{
/* first: make room (if you can) */
if (zzStartRead > 0)
{
Array.Copy(zzBuffer, zzStartRead, zzBuffer, 0, zzEndRead - zzStartRead);
/* translate stored positions */
zzEndRead -= zzStartRead;
zzCurrentPos -= zzStartRead;
zzMarkedPos -= zzStartRead;
zzPushbackPos -= zzStartRead;
zzStartRead = 0;
}
/* is the buffer big enough? */
if (zzCurrentPos >= zzBuffer.Length)
{
/* if not: blow it up */
char[] newBuffer = new char[zzCurrentPos * 2];
Array.Copy(zzBuffer, 0, newBuffer, 0, zzBuffer.Length);
zzBuffer = newBuffer;
}
/* finally: fill the buffer with new input */
int numRead = zzReader.Read(zzBuffer, zzEndRead, zzBuffer.Length - zzEndRead);
if (numRead < 1)
{
return true;
}
else
{
zzEndRead += numRead;
return false;
}
}
/// Closes the input stream.
public void Yyclose()
{
zzAtEOF = true; /* indicate end of file */
zzEndRead = zzStartRead; /* invalidate buffer */
if (zzReader != null)
zzReader.Close();
}
/// Resets the scanner to read from a new input stream.
/// Does not close the old reader.
///
/// All internal variables are reset, the old input stream
/// cannot be reused (internal buffer is discarded and lost).
/// Lexical state is set to ZZ_INITIAL.
///
///
/// the new input stream
///
public void Yyreset(System.IO.TextReader reader)
{
zzReader = reader;
zzAtBOL = true;
zzAtEOF = false;
zzEndRead = zzStartRead = 0;
zzCurrentPos = zzMarkedPos = zzPushbackPos = 0;
yyline = yychar = yycolumn = 0;
zzLexicalState = YYINITIAL;
}
/// Returns the current lexical state.
public int Yystate()
{
return zzLexicalState;
}
/// Enters a new lexical state
///
///
/// the new lexical state
///
public void Yybegin(int newState)
{
zzLexicalState = newState;
}
/// Returns the text matched by the current regular expression.
public System.String Yytext()
{
return new System.String(zzBuffer, zzStartRead, zzMarkedPos - zzStartRead);
}
/// Returns the character at position pos from the
/// matched text.
///
/// It is equivalent to yytext().charAt(pos), but faster
///
///
/// the position of the character to fetch.
/// A value from 0 to yylength()-1.
///
///
/// the character at position pos
///
public char Yycharat(int pos)
{
return zzBuffer[zzStartRead + pos];
}
/// Returns the length of the matched text region.
public int Yylength()
{
return zzMarkedPos - zzStartRead;
}
/// Reports an error that occured while scanning.
///
/// In a wellformed scanner (no or only correct usage of
/// yypushback(int) and a match-all fallback rule) this method
/// will only be called with things that "Can't Possibly Happen".
/// If this method is called, something is seriously wrong
/// (e.g. a JFlex bug producing a faulty scanner etc.).
///
/// Usual syntax/scanner level error handling should be done
/// in error fallback rules.
///
///
/// the code of the errormessage to display
///
private void ZzScanError(int errorCode)
{
System.String message;
try
{
message = ZZ_ERROR_MSG[errorCode];
}
catch (System.IndexOutOfRangeException)
{
message = ZZ_ERROR_MSG[ZZ_UNKNOWN_ERROR];
}
throw new System.ApplicationException(message);
}
/// Pushes the specified amount of characters back into the input stream.
///
/// They will be read again by then next call of the scanning method
///
///
/// the number of characters to be read again.
/// This number must not be greater than yylength()!
///
public virtual void Yypushback(int number)
{
if (number > Yylength())
ZzScanError(ZZ_PUSHBACK_2BIG);
zzMarkedPos -= number;
}
/// Resumes scanning until the next regular expression is matched,
/// the end of input is encountered or an I/O-Error occurs.
///
///
/// the next token
///
/// if any I/O-Error occurs
///
public virtual int GetNextToken()
{
int zzInput;
int zzAction;
// cached fields:
int zzCurrentPosL;
int zzMarkedPosL;
int zzEndReadL = zzEndRead;
char[] zzBufferL = zzBuffer;
char[] zzCMapL = ZZ_CMAP;
int[] zzTransL = ZZ_TRANS;
int[] zzRowMapL = ZZ_ROWMAP;
int[] zzAttrL = ZZ_ATTRIBUTE;
while (true)
{
zzMarkedPosL = zzMarkedPos;
yychar += zzMarkedPosL - zzStartRead;
zzAction = - 1;
zzCurrentPosL = zzCurrentPos = zzStartRead = zzMarkedPosL;
zzState = zzLexicalState;
{
while (true)
{
if (zzCurrentPosL < zzEndReadL)
zzInput = zzBufferL[zzCurrentPosL++];
else if (zzAtEOF)
{
zzInput = YYEOF;
goto zzForAction_brk;
}
else
{
// store back cached positions
zzCurrentPos = zzCurrentPosL;
zzMarkedPos = zzMarkedPosL;
bool eof = ZzRefill();
// get translated positions and possibly new buffer
zzCurrentPosL = zzCurrentPos;
zzMarkedPosL = zzMarkedPos;
zzBufferL = zzBuffer;
zzEndReadL = zzEndRead;
if (eof)
{
zzInput = YYEOF;
goto zzForAction_brk;
}
else
{
zzInput = zzBufferL[zzCurrentPosL++];
}
}
int zzNext = zzTransL[zzRowMapL[zzState] + zzCMapL[zzInput]];
if (zzNext == - 1)
{
goto zzForAction_brk;
}
zzState = zzNext;
int zzAttributes = zzAttrL[zzState];
if ((zzAttributes & 1) == 1)
{
zzAction = zzState;
zzMarkedPosL = zzCurrentPosL;
if ((zzAttributes & 8) == 8)
{
goto zzForAction_brk;
}
}
}
}
zzForAction_brk: ;
// store back cached position
zzMarkedPos = zzMarkedPosL;
switch (zzAction < 0 ? zzAction : ZZ_ACTION[zzAction])
{
case 4:
{
return HOST;
}
case 11: break;
case 9:
{
return ACRONYM;
}
case 12: break;
case 8:
{
return ACRONYM_DEP;
}
case 13: break;
case 1:
{
/* ignore */
}
goto case 14;
case 14: break;
case 5:
{
return NUM;
}
case 15: break;
case 3:
{
return CJ;
}
case 16: break;
case 2:
{
return ALPHANUM;
}
case 17: break;
case 7:
{
return COMPANY;
}
case 18: break;
case 6:
{
return APOSTROPHE;
}
case 19: break;
case 10:
{
return EMAIL;
}
case 20: break;
default:
if (zzInput == YYEOF && zzStartRead == zzCurrentPos)
{
zzAtEOF = true;
return YYEOF;
}
else
{
ZzScanError(ZZ_NO_MATCH);
}
break;
}
}
}
}
}