* Spell Checker class (Main class)
* (initially inspired by the David Spencer code).
*
Example Usage: * *
* SpellChecker spellchecker = new SpellChecker(spellIndexDirectory);
* // To index a field of a user index:
* spellchecker.indexDictionary(new LuceneDictionary(my_lucene_reader, a_field));
* // To index a file containing words:
* spellchecker.indexDictionary(new PlainTextDictionary(new File("myfile.txt")));
* String[] suggestions = spellchecker.suggestSimilar("misspelt", 5);
*
*
*
* @version 1.0
*/
public class SpellChecker {
/**
* Field name for each word in the ngram index.
*/
public static final String F_WORD = "word";
/**
* the spell index
*/
Directory spellIndex;
/**
* Boost value for start and end grams
*/
private float bStart = 2.0f;
private float bEnd = 1.0f;
private IndexSearcher searcher;
// minimum score for hits generated by the spell checker query
private float minScore = 0.5f;
private StringDistance sd;
/**
* Use the given directory as a spell checker index. The directory
* is created if it doesn't exist yet.
*
* @param spellIndex
* @throws IOException
*/
public SpellChecker(Directory spellIndex,StringDistance sd) throws IOException {
this.setSpellIndex(spellIndex);
this.setStringDistance(sd);
}
public SpellChecker(Directory spellIndex) throws IOException {
this(spellIndex, new LevensteinDistance());
}
/**
* Use a different index as the spell checker index or re-open
* the existing index if spellIndex is the same value
* as given in the constructor.
*
* @param spellIndex
* @throws IOException
*/
public void setSpellIndex(Directory spellIndex) throws IOException {
this.spellIndex = spellIndex;
if (!IndexReader.indexExists(spellIndex)) {
IndexWriter writer = new IndexWriter(spellIndex, null, true, IndexWriter.MaxFieldLength.UNLIMITED);
writer.close();
}
// close the old searcher, if there was one
if (searcher != null) {
searcher.close();
}
searcher = new IndexSearcher(this.spellIndex, true);
}
public void setStringDistance(StringDistance sd) {
this.sd = sd;
}
public StringDistance getStringDistance() {
return sd;
}
/**
* Sets the accuracy 0 < minScore < 1; default 0.5
*/
public void setAccuracy(float minScore) {
this.minScore = minScore;
}
/**
* Suggest similar words.
*
* As the Lucene similarity that is used to fetch the most relevant n-grammed terms * is not the same as the edit distance strategy used to calculate the best * matching spell-checked word from the hits that Lucene found, one usually has * to retrieve a couple of numSug's in order to get the true best match. * *
I.e. if numSug == 1, don't count on that suggestion being the best one. * Thus, you should set this value to at least 5 for a good suggestion. * * @param word the word you want a spell check done on * @param numSug the number of suggested words * @throws IOException * @return String[] */ public String[] suggestSimilar(String word, int numSug) throws IOException { return this.suggestSimilar(word, numSug, null, null, false); } /** * Suggest similar words (optionally restricted to a field of an index). * *
As the Lucene similarity that is used to fetch the most relevant n-grammed terms * is not the same as the edit distance strategy used to calculate the best * matching spell-checked word from the hits that Lucene found, one usually has * to retrieve a couple of numSug's in order to get the true best match. * *
I.e. if numSug == 1, don't count on that suggestion being the best one.
* Thus, you should set this value to at least 5 for a good suggestion.
*
* @param word the word you want a spell check done on
* @param numSug the number of suggested words
* @param ir the indexReader of the user index (can be null see field param)
* @param field the field of the user index: if field is not null, the suggested
* words are restricted to the words present in this field.
* @param morePopular return only the suggest words that are as frequent or more frequent than the searched word
* (only if restricted mode = (indexReader!=null and field!=null)
* @throws IOException
* @return String[] the sorted list of the suggest words with these 2 criteria:
* first criteria: the edit distance, second criteria (only if restricted mode): the popularity
* of the suggest words in the field of the user index
*/
public String[] suggestSimilar(String word, int numSug, IndexReader ir,
String field, boolean morePopular) throws IOException {
float min = this.minScore;
final int lengthWord = word.length();
final int freq = (ir != null && field != null) ? ir.docFreq(new Term(field, word)) : 0;
final int goalFreq = (morePopular && ir != null && field != null) ? freq : 0;
// if the word exists in the real index and we don't care for word frequency, return the word itself
if (!morePopular && freq > 0) {
return new String[] { word };
}
BooleanQuery query = new BooleanQuery();
String[] grams;
String key;
for (int ng = getMin(lengthWord); ng <= getMax(lengthWord); ng++) {
key = "gram" + ng; // form key
grams = formGrams(word, ng); // form word into ngrams (allow dups too)
if (grams.length == 0) {
continue; // hmm
}
if (bStart > 0) { // should we boost prefixes?
add(query, "start" + ng, grams[0], bStart); // matches start of word
}
if (bEnd > 0) { // should we boost suffixes
add(query, "end" + ng, grams[grams.length - 1], bEnd); // matches end of word
}
for (int i = 0; i < grams.length; i++) {
add(query, key, grams[i]);
}
}
int maxHits = 10 * numSug;
// System.out.println("Q: " + query);
ScoreDoc[] hits = searcher.search(query, null, maxHits).scoreDocs;
// System.out.println("HITS: " + hits.length());
SuggestWordQueue sugQueue = new SuggestWordQueue(numSug);
// go thru more than 'maxr' matches in case the distance filter triggers
int stop = Math.min(hits.length, maxHits);
SuggestWord sugWord = new SuggestWord();
for (int i = 0; i < stop; i++) {
sugWord.string = searcher.doc(hits[i].doc).get(F_WORD); // get orig word
// don't suggest a word for itself, that would be silly
if (sugWord.string.equals(word)) {
continue;
}
// edit distance
sugWord.score = sd.getDistance(word,sugWord.string);
if (sugWord.score < min) {
continue;
}
if (ir != null && field != null) { // use the user index
sugWord.freq = ir.docFreq(new Term(field, sugWord.string)); // freq in the index
// don't suggest a word that is not present in the field
if ((morePopular && goalFreq > sugWord.freq) || sugWord.freq < 1) {
continue;
}
}
sugQueue.insertWithOverflow(sugWord);
if (sugQueue.size() == numSug) {
// if queue full, maintain the minScore score
min = sugQueue.top().score;
}
sugWord = new SuggestWord();
}
// convert to array string
String[] list = new String[sugQueue.size()];
for (int i = sugQueue.size() - 1; i >= 0; i--) {
list[i] = sugQueue.pop().string;
}
return list;
}
/**
* Add a clause to a boolean query.
*/
private static void add(BooleanQuery q, String name, String value, float boost) {
Query tq = new TermQuery(new Term(name, value));
tq.setBoost(boost);
q.add(new BooleanClause(tq, BooleanClause.Occur.SHOULD));
}
/**
* Add a clause to a boolean query.
*/
private static void add(BooleanQuery q, String name, String value) {
q.add(new BooleanClause(new TermQuery(new Term(name, value)), BooleanClause.Occur.SHOULD));
}
/**
* Form all ngrams for a given word.
* @param text the word to parse
* @param ng the ngram length e.g. 3
* @return an array of all ngrams in the word and note that duplicates are not removed
*/
private static String[] formGrams(String text, int ng) {
int len = text.length();
String[] res = new String[len - ng + 1];
for (int i = 0; i < len - ng + 1; i++) {
res[i] = text.substring(i, i + ng);
}
return res;
}
/**
* Removes all terms from the spell check index.
* @throws IOException
*/
public void clearIndex() throws IOException {
IndexWriter writer = new IndexWriter(spellIndex, null, true, IndexWriter.MaxFieldLength.UNLIMITED);
writer.close();
//close the old searcher
searcher.close();
searcher = new IndexSearcher(this.spellIndex, true);
}
/**
* Check whether the word exists in the index.
* @param word
* @throws IOException
* @return true iff the word exists in the index
*/
public boolean exist(String word) throws IOException {
return searcher.docFreq(new Term(F_WORD, word)) > 0;
}
/**
* Indexes the data from the given {@link Dictionary}.
* @param dict Dictionary to index
* @param mergeFactor mergeFactor to use when indexing
* @param ramMB the max amount or memory in MB to use
* @throws IOException
*/
public void indexDictionary(Dictionary dict, int mergeFactor, int ramMB) throws IOException {
IndexWriter writer = new IndexWriter(spellIndex, new WhitespaceAnalyzer(), IndexWriter.MaxFieldLength.UNLIMITED);
writer.setMergeFactor(mergeFactor);
writer.setRAMBufferSizeMB(ramMB);
Iterator