Returns the next token in the stream, or null at EOS.
/// Removes 's from the end of words.
///
Removes dots from acronyms.
///
public override Lucene.Net.Analysis.Token Next()
{
Lucene.Net.Analysis.Token t = input.Next();
if (t == null)
return null;
System.String text = t.TermText();
System.String type = t.Type();
if (type == APOSTROPHE_TYPE && (text.EndsWith("'s") || text.EndsWith("'S")))
{
return new Lucene.Net.Analysis.Token(text.Substring(0, (text.Length - 2) - (0)), t.StartOffset(), t.EndOffset(), type);
}
else if (type == ACRONYM_TYPE)
{
// remove dots
System.Text.StringBuilder trimmed = new System.Text.StringBuilder();
for (int i = 0; i < text.Length; i++)
{
char c = text[i];
if (c != '.')
trimmed.Append(c);
}
return new Lucene.Net.Analysis.Token(trimmed.ToString(), t.StartOffset(), t.EndOffset(), type);
}
else
{
return t;
}
}
}
}