A TokenStream enumerates the sequence of tokens, either from
/// fields of a document or from query text.
///
/// This is an abstract class. Concrete subclasses are:
///
/// - {@link Tokenizer}, a TokenStream
/// whose input is a Reader; and
///
- {@link TokenFilter}, a TokenStream
/// whose input is another TokenStream.
///
///
public abstract class TokenStream
{
///