/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ using System; using Lucene.Net.Documents; using Lucene.Net.Store; using Lucene.Net.Util; namespace Lucene.Net.Analysis { /// An Analyzer builds TokenStreams, which analyze text. It thus represents a /// policy for extracting index terms from text. ///

/// Typical implementations first build a Tokenizer, which breaks the stream of /// characters from the Reader into raw Tokens. One or more TokenFilters may /// then be applied to the output of the Tokenizer. ///

public abstract class Analyzer : IDisposable { /// Creates a TokenStream which tokenizes all the text in the provided /// Reader. Must be able to handle null field name for /// backward compatibility. /// public abstract TokenStream TokenStream(String fieldName, System.IO.TextReader reader); /// Creates a TokenStream that is allowed to be re-used /// from the previous time that the same thread called /// this method. Callers that do not need to use more /// than one TokenStream at the same time from this /// analyzer should use this method for better /// performance. /// public virtual TokenStream ReusableTokenStream(String fieldName, System.IO.TextReader reader) { return TokenStream(fieldName, reader); } private CloseableThreadLocal tokenStreams = new CloseableThreadLocal(); private bool isDisposed; /// Used by Analyzers that implement reusableTokenStream /// to retrieve previously saved TokenStreams for re-use /// by the same thread. /// protected internal virtual object PreviousTokenStream { get { if (tokenStreams == null) { throw new AlreadyClosedException("this Analyzer is closed"); } return tokenStreams.Get(); } set { if (tokenStreams == null) { throw new AlreadyClosedException("this Analyzer is closed"); } tokenStreams.Set(value); } } [Obsolete()] protected internal bool overridesTokenStreamMethod = false; /// This is only present to preserve /// back-compat of classes that subclass a core analyzer /// and override tokenStream but not reusableTokenStream /// /// /// Java uses Class<? extends Analyer> to constrain to /// only Types that inherit from Analyzer. C# does not have a generic type class, /// ie Type<t>. The method signature stays the same, and an exception may /// still be thrown, if the method doesn't exist. /// [Obsolete("This is only present to preserve back-compat of classes that subclass a core analyzer and override tokenStream but not reusableTokenStream ")] protected internal virtual void SetOverridesTokenStreamMethod() where TClass : Analyzer { try { System.Reflection.MethodInfo m = this.GetType().GetMethod("TokenStream", new[] { typeof(string), typeof(System.IO.TextReader) }); overridesTokenStreamMethod = m.DeclaringType != typeof(TClass); } catch (MethodAccessException) { // can't happen, as baseClass is subclass of Analyzer overridesTokenStreamMethod = false; } } /// Invoked before indexing a Fieldable instance if /// terms have already been added to that field. This allows custom /// analyzers to place an automatic position increment gap between /// Fieldable instances using the same field name. The default value /// position increment gap is 0. With a 0 position increment gap and /// the typical default token position increment of 1, all terms in a field, /// including across Fieldable instances, are in successive positions, allowing /// exact PhraseQuery matches, for instance, across Fieldable instance boundaries. /// /// /// Fieldable name being indexed. /// /// position increment gap, added to the next token emitted from /// public virtual int GetPositionIncrementGap(String fieldName) { return 0; } /// Just like , except for /// Token offsets instead. By default this returns 1 for /// tokenized fields and, as if the fields were joined /// with an extra space character, and 0 for un-tokenized /// fields. This method is only called if the field /// produced at least one token for indexing. /// /// /// the field just indexed /// /// offset gap, added to the next token emitted from /// public virtual int GetOffsetGap(IFieldable field) { return field.IsTokenized ? 1 : 0; } /// Frees persistent resources used by this Analyzer public void Close() { Dispose(); } public virtual void Dispose() { Dispose(true); } protected virtual void Dispose(bool disposing) { if (isDisposed) return; if (disposing) { if (tokenStreams != null) { tokenStreams.Close(); tokenStreams = null; } } isDisposed = true; } } }