// -----------------------------------------------------------------------
//
// TODO: Update copyright text.
//
// -----------------------------------------------------------------------
namespace Lucene.Net.Analysis
{
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using Lucene.Net.Util;
///
/// The abstract class which will perform an lexical analysis that will transform sequence of
/// characters into a sequence of tokens.
///
///
///
/// The abstract Tokenizer class in Lucene.Net is essentially a
/// that has an internal .
///
///
/// Subclasses must override and
/// must call
/// before setting attributes.
///
///
public abstract class Tokenizer : TokenStream
{
///
/// Initializes a new instance of the class.
///
protected Tokenizer()
{
}
///
/// Initializes a new instance of the class.
///
/// The reader.
protected Tokenizer(StreamReader reader)
{
this.Reader = CharReader.CastOrCreate(reader);
}
///
/// Initializes a new instance of the class.
///
/// The factory.
protected Tokenizer(AttributeFactory factory)
: base(factory)
{
}
///
/// Initializes a new instance of the class.
///
/// The factory.
/// The reader.
protected Tokenizer(AttributeFactory factory, StreamReader reader)
: base(factory)
{
this.Reader = CharReader.CastOrCreate(reader);
}
///
/// Gets or sets the reader.
///
/// The reader.
protected TextReader Reader { get; set; }
///
/// Resets the specified reader.
///
/// The reader.
public void Reset(StreamReader reader)
{
this.Reader = reader;
}
///
/// Corrects and returns the corrected offset.
///
///
///
/// If the of the Tokenizer is an instance of
/// then this method will call otherwise it will
/// return the value of the parameter .
///
///
/// The offset as seen in the output.
/// The corrected offset based on the input.
protected int CorrectOffset(int offset)
{
var charStream = this.Reader as CharStream;
if (charStream != null)
return charStream.CorrectOffset(offset);
return offset;
}
///
/// Releases unmanaged and - optionally - managed resources
///
/// true to release both managed and unmanaged resources; false to release only unmanaged resources.
protected override void Dispose(bool release)
{
//// LUCENE-2387: don't hold onto Reader after close, so
//// GC can reclaim
if (this.Reader != null)
{
this.Reader.Dispose();
this.Reader = null;
}
}
}
}