/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
using System;
using System.Collections.Generic;
using Lucene.Net.Analysis.Tokenattributes;
namespace Lucene.Net.Analysis.Fr
{
/*
* Removes elisions from a {@link TokenStream}. For example, "l'avion" (the plane) will be
* tokenized as "avion" (plane).
*
* Note that {@link StandardTokenizer} sees " ' " as a space, and cuts it out.
*
* @see Elision in Wikipedia
*/
public sealed class ElisionFilter : TokenFilter
{
private CharArraySet articles = null;
private ITermAttribute termAtt;
private static char[] apostrophes = { '\'', '’' };
public void SetArticles(ISet articles)
{
if (articles is CharArraySet)
this.articles = (CharArraySet)articles;
else
this.articles = new CharArraySet(articles, true);
}
/*
* Constructs an elision filter with standard stop words
*/
internal ElisionFilter(TokenStream input)
: this(input, new[] { "l", "m", "t", "qu", "n", "s", "j" })
{ }
/*
* Constructs an elision filter with a Set of stop words
*/
public ElisionFilter(TokenStream input, ISet articles)
: base(input)
{
SetArticles(articles);
termAtt = AddAttribute();
}
/*
* Constructs an elision filter with an array of stop words
*/
public ElisionFilter(TokenStream input, IEnumerable articles)
: base(input)
{
this.articles = new CharArraySet(articles, true);
termAtt = AddAttribute();
}
/*
* Increments the {@link TokenStream} with a {@link TermAttribute} without elisioned start
*/
public override sealed bool IncrementToken()
{
if (input.IncrementToken())
{
char[] termBuffer = termAtt.TermBuffer();
int termLength = termAtt.TermLength();
int minPoz = int.MaxValue;
for (int i = 0; i < apostrophes.Length; i++)
{
char apos = apostrophes[i];
// The equivalent of String.indexOf(ch)
for (int poz = 0; poz < termLength; poz++)
{
if (termBuffer[poz] == apos)
{
minPoz = Math.Min(poz, minPoz);
break;
}
}
}
// An apostrophe has been found. If the prefix is an article strip it off.
if (minPoz != int.MaxValue
&& articles.Contains(termAtt.TermBuffer(), 0, minPoz))
{
termAtt.SetTermBuffer(termAtt.TermBuffer(), minPoz + 1, termAtt.TermLength() - (minPoz + 1));
}
return true;
}
else
{
return false;
}
}
}
}