/*
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*
*/
using System;
using System.IO;
using System.Text;
using System.Collections;
using Lucene.Net.Analysis;
namespace Lucene.Net.Analysis.Cn
{
///
/// An that tokenizes text with and
/// filters with
///
public class ChineseAnalyzer : Analyzer
{
public ChineseAnalyzer()
{
}
///
/// Creates a TokenStream which tokenizes all the text in the provided Reader.
///
/// A TokenStream build from a ChineseTokenizer filtered with ChineseFilter.
public override sealed TokenStream TokenStream(String fieldName, TextReader reader)
{
TokenStream result = new ChineseTokenizer(reader);
result = new ChineseFilter(result);
return result;
}
private class SavedStreams
{
protected internal Tokenizer source;
protected internal TokenStream result;
};
///
/// Returns a (possibly reused) which tokenizes all the text in the
/// provided .
///
///
/// A built from a
/// filtered with .
///
public override TokenStream ReusableTokenStream(String fieldName, TextReader reader)
{
/* tokenStream() is final, no back compat issue */
SavedStreams streams = (SavedStreams) PreviousTokenStream;
if (streams == null)
{
streams = new SavedStreams();
streams.source = new ChineseTokenizer(reader);
streams.result = new ChineseFilter(streams.source);
PreviousTokenStream = streams;
}
else
{
streams.source.Reset(reader);
}
return streams.result;
}
}
}