Returns the next token in the stream, or null at EOS.
/// Removes 's from the end of words.
///
Removes dots from acronyms.
///
public override Token Next(/* in */ Token reusableToken)
{
System.Diagnostics.Debug.Assert(reusableToken != null);
Token nextToken = input.Next(reusableToken);
if (nextToken == null)
return null;
char[] buffer = nextToken.TermBuffer();
int bufferLength = nextToken.TermLength();
System.String type = nextToken.Type();
if (type == APOSTROPHE_TYPE &&
bufferLength >= 2 &&
buffer[bufferLength - 2] == '\'' &&
(buffer[bufferLength - 1] == 's' || buffer[bufferLength - 1] == 'S'))
{
// Strip last 2 characters off
nextToken.SetTermLength(bufferLength - 2);
}
else if (type == ACRONYM_TYPE)
{
// remove dots
int upto = 0;
for (int i = 0; i < bufferLength; i++)
{
char c = buffer[i];
if (c != '.')
buffer[upto++] = c;
}
nextToken.SetTermLength(upto);
}
return nextToken;
}
}
}