/* 
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 * 
 * http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

using System;
using Lucene.Net.Analysis.Tokenattributes;
using Lucene.Net.Support;
using Lucene.Net.Util;
using Payload = Lucene.Net.Index.Payload;
using TermPositions = Lucene.Net.Index.TermPositions;
using ArrayUtil = Lucene.Net.Util.ArrayUtil;
using Attribute = Lucene.Net.Util.Attribute;

namespace Lucene.Net.Analysis
{
	
	/// <summary>A Token is an occurrence of a term from the text of a field.  It consists of
	/// a term's text, the start and end offset of the term in the text of the field,
	/// and a type string.
	/// <p/>
	/// The start and end offsets permit applications to re-associate a token with
	/// its source text, e.g., to display highlighted query terms in a document
	/// browser, or to show matching text fragments in a <abbr
	/// title="KeyWord In Context">KWIC</abbr> display, etc.
	/// <p/>
	/// The type is a string, assigned by a lexical analyzer
	/// (a.k.a. tokenizer), naming the lexical or syntactic class that the token
	/// belongs to.  For example an end of sentence marker token might be implemented
	/// with type "eos".  The default token type is "word".  
	/// <p/>
	/// A Token can optionally have metadata (a.k.a. Payload) in the form of a variable
	/// length byte array. Use <see cref="TermPositions.PayloadLength" /> and 
	/// <see cref="TermPositions.GetPayload(byte[], int)" /> to retrieve the payloads from the index.
	/// </summary>
	/// <summary><br/><br/>
	/// </summary>
	/// <summary><p/><b>NOTE:</b> As of 2.9, Token implements all <see cref="IAttribute" /> interfaces
	/// that are part of core Lucene and can be found in the <see cref="Lucene.Net.Analysis.Tokenattributes"/> namespace.
	/// Even though it is not necessary to use Token anymore, with the new TokenStream API it can
	/// be used as convenience class that implements all <see cref="IAttribute" />s, which is especially useful
	/// to easily switch from the old to the new TokenStream API.
	/// <br/><br/>
	/// <p/>Tokenizers and TokenFilters should try to re-use a Token instance when
	/// possible for best performance, by implementing the
	/// <see cref="TokenStream.IncrementToken()" /> API.
	/// Failing that, to create a new Token you should first use
	/// one of the constructors that starts with null text.  To load
	/// the token from a char[] use <see cref="SetTermBuffer(char[], int, int)" />.
	/// To load from a String use <see cref="SetTermBuffer(String)" /> or <see cref="SetTermBuffer(String, int, int)" />.
	/// Alternatively you can get the Token's termBuffer by calling either <see cref="TermBuffer()" />,
	/// if you know that your text is shorter than the capacity of the termBuffer
	/// or <see cref="ResizeTermBuffer(int)" />, if there is any possibility
	/// that you may need to grow the buffer. Fill in the characters of your term into this
    /// buffer, with <see cref="string.ToCharArray(int, int)" /> if loading from a string,
	/// or with <see cref="Array.Copy(Array, long, Array, long, long)" />, and finally call <see cref="SetTermLength(int)" /> to
	/// set the length of the term text.  See <a target="_top"
	/// href="https://issues.apache.org/jira/browse/LUCENE-969">LUCENE-969</a>
	/// for details.<p/>
	/// <p/>Typical Token reuse patterns:
	/// <list type="bullet">
	/// <item> Copying text from a string (type is reset to <see cref="DEFAULT_TYPE" /> if not
	/// specified):<br/>
	/// <code>
	/// return reusableToken.reinit(string, startOffset, endOffset[, type]);
	/// </code>
	/// </item>
	/// <item> Copying some text from a string (type is reset to <see cref="DEFAULT_TYPE" />
	/// if not specified):<br/>
    /// <code>
	/// return reusableToken.reinit(string, 0, string.length(), startOffset, endOffset[, type]);
    /// </code>
	/// </item>
	/// <item> Copying text from char[] buffer (type is reset to <see cref="DEFAULT_TYPE" />
	/// if not specified):<br/>
    /// <code>
	/// return reusableToken.reinit(buffer, 0, buffer.length, startOffset, endOffset[, type]);
    /// </code>
	/// </item>
	/// <item> Copying some text from a char[] buffer (type is reset to
	/// <see cref="DEFAULT_TYPE" /> if not specified):<br/>
    /// <code>
	/// return reusableToken.reinit(buffer, start, end - start, startOffset, endOffset[, type]);
    /// </code>
	/// </item>
	/// <item> Copying from one one Token to another (type is reset to
	/// <see cref="DEFAULT_TYPE" /> if not specified):<br/>
    /// <code>
	/// return reusableToken.reinit(source.termBuffer(), 0, source.termLength(), source.startOffset(), source.endOffset()[, source.type()]);
    /// </code>
	/// </item>
	/// </list>
	/// A few things to note:
	/// <list type="bullet">
	/// <item>clear() initializes all of the fields to default values. This was changed in contrast to Lucene 2.4, but should affect no one.</item>
	/// <item>Because <c>TokenStreams</c> can be chained, one cannot assume that the <c>Token's</c> current type is correct.</item>
	/// <item>The startOffset and endOffset represent the start and offset in the
	/// source text, so be careful in adjusting them.</item>
	/// <item>When caching a reusable token, clone it. When injecting a cached token into a stream that can be reset, clone it again.</item>
	/// </list>
	/// <p/>
	/// </summary>
	/// <seealso cref="Lucene.Net.Index.Payload">
	/// </seealso>
	[Serializable]
	public class Token : Attribute, ITermAttribute, ITypeAttribute, IPositionIncrementAttribute, IFlagsAttribute, IOffsetAttribute, IPayloadAttribute
	{
		public const String DEFAULT_TYPE = "word";

		private const int MIN_BUFFER_SIZE = 10;

		private char[] termBuffer;
		private int termLength;
		private int startOffset, endOffset;
		private string type = DEFAULT_TYPE;
		private int flags;
		private Payload payload;
		private int positionIncrement = 1;
		
		/// <summary>Constructs a Token will null text. </summary>
		public Token()
		{
		}
		
		/// <summary>Constructs a Token with null text and start &amp; end
		/// offsets.
		/// </summary>
		/// <param name="start">start offset in the source text</param>
		/// <param name="end">end offset in the source text</param>
		public Token(int start, int end)
		{
			startOffset = start;
			endOffset = end;
		}
		
		/// <summary>Constructs a Token with null text and start &amp; end
		/// offsets plus the Token type.
		/// </summary>
		/// <param name="start">start offset in the source text</param>
		/// <param name="end">end offset in the source text</param>
		/// <param name="typ">the lexical type of this Token</param>
		public Token(int start, int end, String typ)
		{
			startOffset = start;
			endOffset = end;
			type = typ;
		}
		
		/// <summary> Constructs a Token with null text and start &amp; end
		/// offsets plus flags. NOTE: flags is EXPERIMENTAL.
		/// </summary>
		/// <param name="start">start offset in the source text</param>
		/// <param name="end">end offset in the source text</param>
		/// <param name="flags">The bits to set for this token</param>
		public Token(int start, int end, int flags)
		{
			startOffset = start;
			endOffset = end;
			this.flags = flags;
		}
		
		/// <summary>Constructs a Token with the given term text, and start
		/// &amp; end offsets.  The type defaults to "word."
		/// <b>NOTE:</b> for better indexing speed you should
		/// instead use the char[] termBuffer methods to set the
		/// term text.
		/// </summary>
		/// <param name="text">term text</param>
		/// <param name="start">start offset</param>
		/// <param name="end">end offset</param>
		public Token(String text, int start, int end)
		{
		    SetTermBuffer(text);
			startOffset = start;
			endOffset = end;
		}
		
		/// <summary>Constructs a Token with the given text, start and end
		/// offsets, &amp; type.  <b>NOTE:</b> for better indexing
		/// speed you should instead use the char[] termBuffer
		/// methods to set the term text.
		/// </summary>
		/// <param name="text">term text</param>
		/// <param name="start">start offset</param>
		/// <param name="end">end offset</param>
		/// <param name="typ">token type</param>
		public Token(System.String text, int start, int end, System.String typ)
		{
		    SetTermBuffer(text);
			startOffset = start;
			endOffset = end;
			type = typ;
		}
		
		/// <summary>  Constructs a Token with the given text, start and end
		/// offsets, &amp; type.  <b>NOTE:</b> for better indexing
		/// speed you should instead use the char[] termBuffer
		/// methods to set the term text.
		/// </summary>
		/// <param name="text"></param>
		/// <param name="start"></param>
		/// <param name="end"></param>
		/// <param name="flags">token type bits</param>
		public Token(System.String text, int start, int end, int flags)
		{
		    SetTermBuffer(text);
			startOffset = start;
			endOffset = end;
			this.flags = flags;
		}
		
		/// <summary>  Constructs a Token with the given term buffer (offset
		/// &amp; length), start and end
		/// offsets
		/// </summary>
		/// <param name="startTermBuffer"></param>
		/// <param name="termBufferOffset"></param>
		/// <param name="termBufferLength"></param>
		/// <param name="start"></param>
		/// <param name="end"></param>
		public Token(char[] startTermBuffer, int termBufferOffset, int termBufferLength, int start, int end)
		{
			SetTermBuffer(startTermBuffer, termBufferOffset, termBufferLength);
			startOffset = start;
			endOffset = end;
		}

	    /// <summary>Set the position increment.  This determines the position of this token
	    /// relative to the previous Token in a <see cref="TokenStream" />, used in phrase
	    /// searching.
	    /// 
	    /// <p/>The default value is one.
	    /// 
	    /// <p/>Some common uses for this are:<list>
	    /// 
	    /// <item>Set it to zero to put multiple terms in the same position.  This is
	    /// useful if, e.g., a word has multiple stems.  Searches for phrases
	    /// including either stem will match.  In this case, all but the first stem's
	    /// increment should be set to zero: the increment of the first instance
	    /// should be one.  Repeating a token with an increment of zero can also be
	    /// used to boost the scores of matches on that token.</item>
	    /// 
	    /// <item>Set it to values greater than one to inhibit exact phrase matches.
	    /// If, for example, one does not want phrases to match across removed stop
	    /// words, then one could build a stop word filter that removes stop words and
	    /// also sets the increment to the number of stop words removed before each
	    /// non-stop word.  Then exact phrase queries will only match when the terms
	    /// occur with no intervening stop words.</item>
	    /// 
	    /// </list>
	    /// </summary>
	    /// <value> the distance from the prior term </value>
	    /// <seealso cref="Lucene.Net.Index.TermPositions">
	    /// </seealso>
	    public virtual int PositionIncrement
	    {
	        set
	        {
	            if (value < 0)
	                throw new System.ArgumentException("Increment must be zero or greater: " + value);
	            this.positionIncrement = value;
	        }
	        get { return positionIncrement; }
	    }

	    /// <summary>Returns the Token's term text.
	    /// 
	    /// This method has a performance penalty
	    /// because the text is stored internally in a char[].  If
	    /// possible, use <see cref="TermBuffer()" /> and <see cref="TermLength()"/>
	    /// directly instead.  If you really need a
	    /// String, use this method, which is nothing more than
	    /// a convenience call to <b>new String(token.termBuffer(), 0, token.termLength())</b>
	    /// </summary>
	    public string Term
	    {
	        get
	        {
	            InitTermBuffer();
	            return new System.String(termBuffer, 0, termLength);
	        }
	    }

	    /// <summary>Copies the contents of buffer, starting at offset for
		/// length characters, into the termBuffer array.
		/// </summary>
		/// <param name="buffer">the buffer to copy</param>
		/// <param name="offset">the index in the buffer of the first character to copy</param>
		/// <param name="length">the number of characters to copy</param>
		public void  SetTermBuffer(char[] buffer, int offset, int length)
		{
			GrowTermBuffer(length);
			Array.Copy(buffer, offset, termBuffer, 0, length);
			termLength = length;
		}
		
		/// <summary>Copies the contents of buffer into the termBuffer array.</summary>
		/// <param name="buffer">the buffer to copy
		/// </param>
		public void  SetTermBuffer(System.String buffer)
		{
			int length = buffer.Length;
			GrowTermBuffer(length);
			TextSupport.GetCharsFromString(buffer, 0, length, termBuffer, 0);
			termLength = length;
		}
		
		/// <summary>Copies the contents of buffer, starting at offset and continuing
		/// for length characters, into the termBuffer array.
		/// </summary>
		/// <param name="buffer">the buffer to copy
		/// </param>
		/// <param name="offset">the index in the buffer of the first character to copy
		/// </param>
		/// <param name="length">the number of characters to copy
		/// </param>
		public void  SetTermBuffer(System.String buffer, int offset, int length)
		{
			System.Diagnostics.Debug.Assert(offset <= buffer.Length);
			System.Diagnostics.Debug.Assert(offset + length <= buffer.Length);
			GrowTermBuffer(length);
			TextSupport.GetCharsFromString(buffer, offset, offset + length, termBuffer, 0);
			termLength = length;
		}
		
		/// <summary>Returns the internal termBuffer character array which
		/// you can then directly alter.  If the array is too
		/// small for your token, use <see cref="ResizeTermBuffer(int)" />
		/// to increase it.  After
		/// altering the buffer be sure to call <see cref="SetTermLength" />
		/// to record the number of valid
		/// characters that were placed into the termBuffer. 
		/// </summary>
		public char[] TermBuffer()
		{
			InitTermBuffer();
			return termBuffer;
		}
		
		/// <summary>Grows the termBuffer to at least size newSize, preserving the
		/// existing content. Note: If the next operation is to change
		/// the contents of the term buffer use
		/// <see cref="SetTermBuffer(char[], int, int)" />,
		/// <see cref="SetTermBuffer(String)" />, or
		/// <see cref="SetTermBuffer(String, int, int)" />
		/// to optimally combine the resize with the setting of the termBuffer.
		/// </summary>
		/// <param name="newSize">minimum size of the new termBuffer
		/// </param>
		/// <returns> newly created termBuffer with length >= newSize
		/// </returns>
		public virtual char[] ResizeTermBuffer(int newSize)
		{
			if (termBuffer == null)
			{
                termBuffer = new char[ArrayUtil.GetNextSize(newSize < MIN_BUFFER_SIZE ? MIN_BUFFER_SIZE : newSize)];
			}
			else
			{
				if (termBuffer.Length < newSize)
				{
					// Not big enough; create a new array with slight
					// over allocation and preserve content
					var newCharBuffer = new char[ArrayUtil.GetNextSize(newSize)];
					Array.Copy(termBuffer, 0, newCharBuffer, 0, termBuffer.Length);
					termBuffer = newCharBuffer;
				}
			}
			return termBuffer;
		}
		
		/// <summary>Allocates a buffer char[] of at least newSize, without preserving the existing content.
		/// its always used in places that set the content 
		/// </summary>
		/// <param name="newSize">minimum size of the buffer
		/// </param>
		private void  GrowTermBuffer(int newSize)
		{
			if (termBuffer == null)
			{
				// The buffer is always at least MIN_BUFFER_SIZE    
				termBuffer = new char[ArrayUtil.GetNextSize(newSize < MIN_BUFFER_SIZE?MIN_BUFFER_SIZE:newSize)];
			}
			else
			{
				if (termBuffer.Length < newSize)
				{
					// Not big enough; create a new array with slight
					// over allocation:
					termBuffer = new char[ArrayUtil.GetNextSize(newSize)];
				}
			}
		}
		
		private void  InitTermBuffer()
		{
			if (termBuffer == null)
			{
                termBuffer = new char[ArrayUtil.GetNextSize(MIN_BUFFER_SIZE)];
                termLength = 0;
			}
		}
		
		/// <summary>Return number of valid characters (length of the term)
		/// in the termBuffer array. 
		/// </summary>
		public int TermLength()
		{
			InitTermBuffer();
			return termLength;
		}
		
		/// <summary>Set number of valid characters (length of the term) in
		/// the termBuffer array. Use this to truncate the termBuffer
		/// or to synchronize with external manipulation of the termBuffer.
		/// Note: to grow the size of the array,
		/// use <see cref="ResizeTermBuffer(int)" /> first.
		/// </summary>
		/// <param name="length">the truncated length
		/// </param>
		public void  SetTermLength(int length)
		{
			InitTermBuffer();
			if (length > termBuffer.Length)
				throw new System.ArgumentException("length " + length + " exceeds the size of the termBuffer (" + termBuffer.Length + ")");
			termLength = length;
		}

	    /// <summary>Gets or sets this Token's starting offset, the position of the first character
	    /// corresponding to this token in the source text.
	    /// Note that the difference between endOffset() and startOffset() may not be
	    /// equal to <see cref="TermLength"/>, as the term text may have been altered by a
	    /// stemmer or some other filter. 
	    /// </summary>
	    public virtual int StartOffset
	    {
	        get { return startOffset; }
	        set { this.startOffset = value; }
	    }

	    /// <summary>Gets or sets this Token's ending offset, one greater than the position of the
	    /// last character corresponding to this token in the source text. The length
	    /// of the token in the source text is (endOffset - startOffset). 
	    /// </summary>
	    public virtual int EndOffset
	    {
	        get { return endOffset; }
	        set { this.endOffset = value; }
	    }

	    /// <summary>Set the starting and ending offset.
		/// See StartOffset() and EndOffset()
		/// </summary>
		public virtual void  SetOffset(int startOffset, int endOffset)
		{
			this.startOffset = startOffset;
			this.endOffset = endOffset;
		}

	    /// <summary>Returns this Token's lexical type.  Defaults to "word". </summary>
	    public string Type
	    {
	        get { return type; }
	        set { this.type = value; }
	    }

	    /// <summary> EXPERIMENTAL:  While we think this is here to stay, we may want to change it to be a long.
	    /// <p/>
	    /// 
	    /// Get the bitset for any bits that have been set.  This is completely distinct from <see cref="Type()" />, although they do share similar purposes.
	    /// The flags can be used to encode information about the token for use by other <see cref="TokenFilter"/>s.
	    /// 
	    /// 
	    /// </summary>
	    /// <value> The bits </value>
	    public virtual int Flags
	    {
	        get { return flags; }
	        set { flags = value; }
	    }

	    /// <summary> Returns this Token's payload.</summary>
	    public virtual Payload Payload
	    {
	        get { return payload; }
	        set { payload = value; }
	    }

	    public override String ToString()
		{
			var sb = new System.Text.StringBuilder();
			sb.Append('(');
			InitTermBuffer();
			if (termBuffer == null)
				sb.Append("null");
			else
				sb.Append(termBuffer, 0, termLength);
			sb.Append(',').Append(startOffset).Append(',').Append(endOffset);
			if (!type.Equals("word"))
				sb.Append(",type=").Append(type);
			if (positionIncrement != 1)
				sb.Append(",posIncr=").Append(positionIncrement);
			sb.Append(')');
			return sb.ToString();
		}
		
		/// <summary>Resets the term text, payload, flags, and positionIncrement,
		/// startOffset, endOffset and token type to default.
		/// </summary>
		public override void  Clear()
		{
			payload = null;
			// Leave termBuffer to allow re-use
			termLength = 0;
			positionIncrement = 1;
			flags = 0;
			startOffset = endOffset = 0;
			type = DEFAULT_TYPE;
		}
		
		public override System.Object Clone()
		{
			var t = (Token) base.Clone();
			// Do a deep clone
			if (termBuffer != null)
			{
				t.termBuffer = new char[termBuffer.Length];
				termBuffer.CopyTo(t.termBuffer, 0);
			}
			if (payload != null)
			{
				t.payload = (Payload) payload.Clone();
			}
			return t;
		}
		
		/// <summary>Makes a clone, but replaces the term buffer &amp;
		/// start/end offset in the process.  This is more
		/// efficient than doing a full clone (and then calling
		/// setTermBuffer) because it saves a wasted copy of the old
		/// termBuffer. 
		/// </summary>
		public virtual Token Clone(char[] newTermBuffer, int newTermOffset, int newTermLength, int newStartOffset, int newEndOffset)
		{
			var t = new Token(newTermBuffer, newTermOffset, newTermLength, newStartOffset, newEndOffset)
			        	{positionIncrement = positionIncrement, flags = flags, type = type};
			if (payload != null)
				t.payload = (Payload) payload.Clone();
			return t;
		}
		
		public  override bool Equals(Object obj)
		{
			if (obj == this)
				return true;

			var other = obj as Token;
			if (other == null)
				return false;
			
			InitTermBuffer();
			other.InitTermBuffer();

			if (termLength == other.termLength && startOffset == other.startOffset && endOffset == other.endOffset &&
			    flags == other.flags && positionIncrement == other.positionIncrement && SubEqual(type, other.type) &&
			    SubEqual(payload, other.payload))
			{
				for (int i = 0; i < termLength; i++)
					if (termBuffer[i] != other.termBuffer[i])
						return false;
				return true;
			}
			return false;
		}
		
		private bool SubEqual(System.Object o1, System.Object o2)
		{
			if (o1 == null)
				return o2 == null;
			return o1.Equals(o2);
		}

		public override int GetHashCode()
		{
			InitTermBuffer();
			int code = termLength;
			code = code * 31 + startOffset;
			code = code * 31 + endOffset;
			code = code * 31 + flags;
			code = code * 31 + positionIncrement;
			code = code * 31 + type.GetHashCode();
			code = (payload == null?code:code * 31 + payload.GetHashCode());
			code = code * 31 + ArrayUtil.HashCode(termBuffer, 0, termLength);
			return code;
		}
		
		// like clear() but doesn't clear termBuffer/text
		private void  ClearNoTermBuffer()
		{
			payload = null;
			positionIncrement = 1;
			flags = 0;
			startOffset = endOffset = 0;
			type = DEFAULT_TYPE;
		}
		
		/// <summary>Shorthand for calling <see cref="Clear" />,
		/// <see cref="SetTermBuffer(char[], int, int)" />,
		/// <see cref="StartOffset" />,
		/// <see cref="EndOffset" />,
		/// <see cref="Type" />
		/// </summary>
		/// <returns> this Token instance 
		/// </returns>
		public virtual Token Reinit(char[] newTermBuffer, int newTermOffset, int newTermLength, int newStartOffset, int newEndOffset, System.String newType)
		{
			ClearNoTermBuffer();
			payload = null;
			positionIncrement = 1;
			SetTermBuffer(newTermBuffer, newTermOffset, newTermLength);
			startOffset = newStartOffset;
			endOffset = newEndOffset;
			type = newType;
			return this;
		}
		
		/// <summary>Shorthand for calling <see cref="Clear" />,
		/// <see cref="SetTermBuffer(char[], int, int)" />,
		/// <see cref="StartOffset" />,
		/// <see cref="EndOffset" />
		/// <see cref="Type" /> on Token.DEFAULT_TYPE
		/// </summary>
		/// <returns> this Token instance 
		/// </returns>
		public virtual Token Reinit(char[] newTermBuffer, int newTermOffset, int newTermLength, int newStartOffset, int newEndOffset)
		{
			ClearNoTermBuffer();
			SetTermBuffer(newTermBuffer, newTermOffset, newTermLength);
			startOffset = newStartOffset;
			endOffset = newEndOffset;
			type = DEFAULT_TYPE;
			return this;
		}
		
		/// <summary>Shorthand for calling <see cref="Clear" />,
		/// <see cref="SetTermBuffer(String)" />,
		/// <see cref="StartOffset" />,
		/// <see cref="EndOffset" />
		/// <see cref="Type" />
		/// </summary>
		/// <returns> this Token instance 
		/// </returns>
		public virtual Token Reinit(System.String newTerm, int newStartOffset, int newEndOffset, System.String newType)
		{
			ClearNoTermBuffer();
			SetTermBuffer(newTerm);
			startOffset = newStartOffset;
			endOffset = newEndOffset;
			type = newType;
			return this;
		}
		
		/// <summary>Shorthand for calling <see cref="Clear" />,
		/// <see cref="SetTermBuffer(String, int, int)" />,
		/// <see cref="StartOffset" />,
		/// <see cref="EndOffset" />
		/// <see cref="Type" />
		/// </summary>
		/// <returns> this Token instance 
		/// </returns>
		public virtual Token Reinit(System.String newTerm, int newTermOffset, int newTermLength, int newStartOffset, int newEndOffset, System.String newType)
		{
			ClearNoTermBuffer();
			SetTermBuffer(newTerm, newTermOffset, newTermLength);
			startOffset = newStartOffset;
			endOffset = newEndOffset;
			type = newType;
			return this;
		}
		
		/// <summary>Shorthand for calling <see cref="Clear" />,
		/// <see cref="SetTermBuffer(String)" />,
		/// <see cref="StartOffset" />,
		/// <see cref="EndOffset" />
		/// <see cref="Type" /> on Token.DEFAULT_TYPE
		/// </summary>
		/// <returns> this Token instance 
		/// </returns>
		public virtual Token Reinit(System.String newTerm, int newStartOffset, int newEndOffset)
		{
			ClearNoTermBuffer();
			SetTermBuffer(newTerm);
			startOffset = newStartOffset;
			endOffset = newEndOffset;
			type = DEFAULT_TYPE;
			return this;
		}
		
		/// <summary>Shorthand for calling <see cref="Clear" />,
		/// <see cref="SetTermBuffer(String, int, int)" />,
		/// <see cref="StartOffset" />,
		/// <see cref="EndOffset" />
		/// <see cref="Type" /> on Token.DEFAULT_TYPE
		/// </summary>
		/// <returns> this Token instance 
		/// </returns>
		public virtual Token Reinit(System.String newTerm, int newTermOffset, int newTermLength, int newStartOffset, int newEndOffset)
		{
			ClearNoTermBuffer();
			SetTermBuffer(newTerm, newTermOffset, newTermLength);
			startOffset = newStartOffset;
			endOffset = newEndOffset;
			type = DEFAULT_TYPE;
			return this;
		}
		
		/// <summary> Copy the prototype token's fields into this one. Note: Payloads are shared.</summary>
		/// <param name="prototype">
		/// </param>
		public virtual void  Reinit(Token prototype)
		{
			prototype.InitTermBuffer();
			SetTermBuffer(prototype.termBuffer, 0, prototype.termLength);
			positionIncrement = prototype.positionIncrement;
			flags = prototype.flags;
			startOffset = prototype.startOffset;
			endOffset = prototype.endOffset;
			type = prototype.type;
			payload = prototype.payload;
		}
		
		/// <summary> Copy the prototype token's fields into this one, with a different term. Note: Payloads are shared.</summary>
		/// <param name="prototype">
		/// </param>
		/// <param name="newTerm">
		/// </param>
		public virtual void  Reinit(Token prototype, System.String newTerm)
		{
			SetTermBuffer(newTerm);
			positionIncrement = prototype.positionIncrement;
			flags = prototype.flags;
			startOffset = prototype.startOffset;
			endOffset = prototype.endOffset;
			type = prototype.type;
			payload = prototype.payload;
		}
		
		/// <summary> Copy the prototype token's fields into this one, with a different term. Note: Payloads are shared.</summary>
		/// <param name="prototype">
		/// </param>
		/// <param name="newTermBuffer">
		/// </param>
		/// <param name="offset">
		/// </param>
		/// <param name="length">
		/// </param>
		public virtual void  Reinit(Token prototype, char[] newTermBuffer, int offset, int length)
		{
			SetTermBuffer(newTermBuffer, offset, length);
			positionIncrement = prototype.positionIncrement;
			flags = prototype.flags;
			startOffset = prototype.startOffset;
			endOffset = prototype.endOffset;
			type = prototype.type;
			payload = prototype.payload;
		}
		
		public override void  CopyTo(Attribute target)
		{
			if (target is Token)
			{
				var to = (Token) target;
				to.Reinit(this);
				// reinit shares the payload, so clone it:
				if (payload != null)
				{
					to.payload = (Payload) payload.Clone();
				}
			}
			else
			{
				InitTermBuffer();
				((ITermAttribute) target).SetTermBuffer(termBuffer, 0, termLength);
				((IOffsetAttribute) target).SetOffset(startOffset, endOffset);
				((IPositionIncrementAttribute) target).PositionIncrement = positionIncrement;
				((IPayloadAttribute) target).Payload = (payload == null)?null:(Payload) payload.Clone();
				((IFlagsAttribute) target).Flags = flags;
				((ITypeAttribute) target).Type = type;
			}
		}
       
        ///<summary>
        /// Convenience factory that returns <c>Token</c> as implementation for the basic
        /// attributes and return the default impl (with &quot;Impl&quot; appended) for all other
        /// attributes.
        /// @since 3.0
        /// </summary>
	    public static AttributeSource.AttributeFactory TOKEN_ATTRIBUTE_FACTORY =
	        new TokenAttributeFactory(AttributeSource.AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY);
  
        /// <summary>
        /// <b>Expert</b>: Creates an AttributeFactory returning {@link Token} as instance for the basic attributes
        /// and for all other attributes calls the given delegate factory.
        /// </summary>
        public class TokenAttributeFactory : AttributeSource.AttributeFactory
        {

            private readonly AttributeSource.AttributeFactory _delegateFactory;

            /// <summary>
            /// <b>Expert</b>: Creates an AttributeFactory returning {@link Token} as instance for the basic attributes
            /// and for all other attributes calls the given delegate factory.
            /// </summary>
            public TokenAttributeFactory(AttributeSource.AttributeFactory delegateFactory)
            {
                this._delegateFactory = delegateFactory;
            }

            public override Attribute CreateAttributeInstance<T>()
            {
                return typeof(T).IsAssignableFrom(typeof(Token))
                           ? new Token()
                           : _delegateFactory.CreateAttributeInstance<T>();
            }

            public override bool Equals(Object other)
            {
                if (this == other) return true;

            	var af = other as TokenAttributeFactory;
            	return af != null && _delegateFactory.Equals(af._delegateFactory);
            }

            public override int GetHashCode()
            {
                return _delegateFactory.GetHashCode() ^ 0x0a45aa31;
            }
        }
    }
}