View Javadoc
1   package org.apache.maven.index;
2   
3   /*
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *   http://www.apache.org/licenses/LICENSE-2.0    
13   *
14   * Unless required by applicable law or agreed to in writing,
15   * software distributed under the License is distributed on an
16   * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
17   * KIND, either express or implied.  See the License for the
18   * specific language governing permissions and limitations
19   * under the License.
20   */
21  
22  import javax.inject.Named;
23  import javax.inject.Singleton;
24  import java.io.IOException;
25  import java.io.StringReader;
26  import org.apache.lucene.analysis.TokenStream;
27  import org.apache.lucene.index.Term;
28  import org.apache.lucene.queryparser.classic.ParseException;
29  import org.apache.lucene.queryparser.classic.QueryParser;
30  import org.apache.lucene.queryparser.classic.QueryParser.Operator;
31  import org.apache.lucene.search.BooleanClause.Occur;
32  import org.apache.lucene.search.BooleanQuery;
33  import org.apache.lucene.search.PrefixQuery;
34  import org.apache.lucene.search.Query;
35  import org.apache.lucene.search.TermQuery;
36  import org.apache.lucene.search.WildcardQuery;
37  import org.apache.maven.index.context.NexusAnalyzer;
38  import org.apache.maven.index.creator.JarFileContentsIndexCreator;
39  import org.apache.maven.index.creator.MinimalArtifactInfoIndexCreator;
40  import org.apache.maven.index.expr.SearchExpression;
41  import org.apache.maven.index.expr.SearchTyped;
42  import org.slf4j.Logger;
43  import org.slf4j.LoggerFactory;
44  
45  /**
46   * A default {@link QueryCreator} constructs Lucene query for provided query text.
47   * <p>
48   * By default wildcards are created such as query text matches beginning of the field value or beginning of the
49   * class/package name segment for {@link ArtifactInfo#NAMES NAMES} field. But it can be controlled by using special
50   * markers:
51   * <ul>
52   * <li>* - any character</li>
53   * <li>'^' - beginning of the text</li>
54   * <li>'$' or '&lt;' or ' ' end of the text</li>
55   * </ul>
56   * For example:
57   * <ul>
58   * <li>junit - matches junit and junit-foo, but not foo-junit</li>
59   * <li>*junit - matches junit, junit-foo and foo-junit</li>
60   * <li>^junit$ - matches junit, but not junit-foo, nor foo-junit</li>
61   * </ul>
62   * 
63   * @author Eugene Kuleshov
64   */
65  @Singleton
66  @Named
67  public class DefaultQueryCreator
68      implements QueryCreator
69  {
70  
71      private final Logger logger = LoggerFactory.getLogger( getClass() );
72  
73      protected Logger getLogger()
74      {
75          return logger;
76      }
77  
78      // ==
79  
80      public IndexerField selectIndexerField( final Field field, final SearchType type )
81      {
82          IndexerField lastField = null;
83  
84          for ( IndexerField indexerField : field.getIndexerFields() )
85          {
86              lastField = indexerField;
87  
88              if ( type.matchesIndexerField( indexerField ) )
89              {
90                  return indexerField;
91              }
92          }
93  
94          return lastField;
95      }
96  
97      public Query constructQuery( final Field field, final SearchExpression expression )
98          throws ParseException
99      {
100         SearchType searchType = SearchType.SCORED;
101 
102         if ( expression instanceof SearchTyped )
103         {
104             searchType = ( (SearchTyped) expression ).getSearchType();
105         }
106 
107         return constructQuery( field, expression.getStringValue(), searchType );
108     }
109 
110     public Query constructQuery( final Field field, final String query, final SearchType type )
111         throws ParseException
112     {
113         if ( type == null )
114         {
115             throw new NullPointerException( "Cannot construct query with type of \"null\"!" );
116         }
117 
118         if ( field == null )
119         {
120             throw new NullPointerException( "Cannot construct query for field \"null\"!" );
121         }
122         else
123         {
124             return constructQuery( field, selectIndexerField( field, type ), query, type );
125         }
126     }
127 
128     @Deprecated
129     public Query constructQuery( String field, String query )
130     {
131         Query result = null;
132 
133         if ( MinimalArtifactInfoIndexCreator.FLD_GROUP_ID_KW.getKey().equals( field )
134             || MinimalArtifactInfoIndexCreator.FLD_ARTIFACT_ID_KW.getKey().equals( field )
135             || MinimalArtifactInfoIndexCreator.FLD_VERSION_KW.getKey().equals( field )
136             || JarFileContentsIndexCreator.FLD_CLASSNAMES_KW.getKey().equals( field ) )
137         {
138             // these are special untokenized fields, kept for use cases like TreeView is (exact matching).
139             result = legacyConstructQuery( field, query );
140         }
141         else
142         {
143             QueryParser qp = new QueryParser( field, new NexusAnalyzer() );
144 
145             // small cheap trick
146             // if a query is not "expert" (does not contain field:val kind of expression)
147             // but it contains star and/or punctuation chars, example: "common-log*"
148             if ( !query.contains( ":" ) )
149             {
150                 if ( query.contains( "*" ) && query.matches( ".*(\\.|-|_).*" ) )
151                 {
152                     query = query.toLowerCase().replaceAll( "\\*", "X" ).replaceAll( "\\.|-|_", " " ).replaceAll( "X",
153                                                                                                                   "*" );
154                 }
155             }
156 
157             try
158             {
159                 result = qp.parse( query );
160             }
161             catch ( ParseException e )
162             {
163                 getLogger().debug(
164                     "Query parsing with \"legacy\" method, we got ParseException from QueryParser: " + e.getMessage() );
165 
166                 result = legacyConstructQuery( field, query );
167             }
168         }
169 
170         if ( getLogger().isDebugEnabled() )
171         {
172             getLogger().debug( "Query parsed as: " + result.toString() );
173         }
174 
175         return result;
176     }
177 
178     // ==
179 
180     public Query constructQuery( final Field field, final IndexerField indexerField, final String query,
181                                  final SearchType type )
182         throws ParseException
183     {
184         if ( indexerField == null )
185         {
186             getLogger().warn( "Querying for field \"" + field.toString() + "\" without any indexer field was tried. "
187                 + "Please review your code, and consider adding this field to index!" );
188 
189             return null;
190         }
191         if ( !indexerField.isIndexed() )
192         {
193             getLogger().warn(
194                 "Querying for non-indexed field " + field.toString()
195                     + " was tried. Please review your code or consider adding this field to index!" );
196 
197             return null;
198         }
199 
200         if ( Field.NOT_PRESENT.equals( query ) )
201         {
202             return new WildcardQuery( new Term( indexerField.getKey(), "*" ) );
203         }
204 
205         if ( SearchType.EXACT.equals( type ) )
206         {
207             if ( indexerField.isKeyword() )
208             {
209                 // no tokenization should happen against the field!
210                 if ( query.contains( "*" ) || query.contains( "?" ) )
211                 {
212                     return new WildcardQuery( new Term( indexerField.getKey(), query ) );
213                 }
214                 else
215                 {
216                     // exactly what callee wants
217                     return new TermQuery( new Term( indexerField.getKey(), query ) );
218                 }
219             }
220             else if ( !indexerField.isKeyword() && indexerField.isStored() )
221             {
222                 // TODO: resolve this better! Decouple QueryCreator and IndexCreators!
223                 // This is a hack/workaround here
224                 if ( JarFileContentsIndexCreator.FLD_CLASSNAMES_KW.equals( indexerField ) )
225                 {
226                     if ( query.startsWith( "/" ) )
227                     {
228                         return new TermQuery( new Term( indexerField.getKey(), query.toLowerCase().replaceAll( "\\.",
229                             "/" ) ) );
230                     }
231                     else
232                     {
233                         return new TermQuery( new Term( indexerField.getKey(), "/"
234                             + query.toLowerCase().replaceAll( "\\.", "/" ) ) );
235                     }
236                 }
237                 else
238                 {
239                     getLogger().warn(
240                         type.toString()
241                             + " type of querying for non-keyword (but stored) field "
242                             + indexerField.getOntology().toString()
243                             + " was tried. Please review your code, or indexCreator involved, "
244                             + "since this type of querying of this field is currently unsupported." );
245 
246                     // will never succeed (unless we supply him "filter" too, but that would kill performance)
247                     // and is possible with stored fields only
248                     return null;
249                 }
250             }
251             else
252             {
253                 getLogger().warn(
254                     type.toString()
255                         + " type of querying for non-keyword (and not stored) field "
256                         + indexerField.getOntology().toString()
257                         + " was tried. Please review your code, or indexCreator involved, "
258                         + "since this type of querying of this field is impossible." );
259 
260                 // not a keyword indexerField, nor stored. No hope at all. Impossible even with "filtering"
261                 return null;
262             }
263         }
264         else if ( SearchType.SCORED.equals( type ) )
265         {
266             if ( JarFileContentsIndexCreator.FLD_CLASSNAMES.equals( indexerField ) )
267             {
268                 String qpQuery = query.toLowerCase().replaceAll( "\\.", " " ).replaceAll( "/", " " );
269                 // tokenization should happen against the field!
270                 QueryParser qp = new QueryParser( indexerField.getKey(), new NexusAnalyzer() );
271                 qp.setDefaultOperator( Operator.AND );
272                 return qp.parse( qpQuery );
273             }
274             else if ( indexerField.isKeyword() )
275             {
276                 // no tokenization should happen against the field!
277                 if ( query.contains( "*" ) || query.contains( "?" ) )
278                 {
279                     return new WildcardQuery( new Term( indexerField.getKey(), query ) );
280                 }
281                 else
282                 {
283                     BooleanQuery bq = new BooleanQuery();
284 
285                     Term t = new Term( indexerField.getKey(), query );
286 
287                     bq.add( new TermQuery( t ), Occur.SHOULD );
288 
289                     PrefixQuery pq = new PrefixQuery( t );
290                     pq.setBoost( 0.8f );
291 
292                     bq.add( pq, Occur.SHOULD );
293 
294                     return bq;
295                 }
296             }
297             else
298             {
299                 // to save "original" query
300                 String qpQuery = query;
301 
302                 // tokenization should happen against the field!
303                 QueryParser qp = new QueryParser( indexerField.getKey(), new NexusAnalyzer() );
304                 qp.setDefaultOperator( Operator.AND );
305 
306                 // small cheap trick
307                 // if a query is not "expert" (does not contain field:val kind of expression)
308                 // but it contains star and/or punctuation chars, example: "common-log*"
309                 // since Lucene does not support multi-terms WITH wildcards.
310                 // So, here, we "mimic" NexusAnalyzer (this should be fixed!)
311                 // but do this with PRESERVING original query!
312                 if ( qpQuery.matches( ".*(\\.|-|_|/).*" ) )
313                 {
314                     qpQuery =
315                         qpQuery.toLowerCase().replaceAll( "\\*", "X" ).replaceAll( "\\.|-|_|/", " " ).replaceAll( "X",
316                             "*" ).replaceAll( " \\* ", "" ).replaceAll( "^\\* ", "" ).replaceAll( " \\*$", "" );
317                 }
318 
319                 // "fix" it with trailing "*" if not there, but only if it not ends with a space
320                 if ( !qpQuery.endsWith( "*" ) && !qpQuery.endsWith( " " ) )
321                 {
322                     qpQuery += "*";
323                 }
324 
325                 try
326                 {
327                     // qpQuery = "\"" + qpQuery + "\"";
328 
329                     BooleanQuery q1 = new BooleanQuery();
330 
331                     q1.add( qp.parse( qpQuery ), Occur.SHOULD );
332 
333                     if ( qpQuery.contains( " " ) )
334                     {
335                         q1.add( qp.parse( "\"" + qpQuery + "\"" ), Occur.SHOULD );
336                     }
337 
338                     Query q2 = null;
339 
340                     int termCount = countTerms( indexerField, query );
341 
342                     // try with KW only if the processed query in qpQuery does not have spaces!
343                     if ( !query.contains( " " ) && termCount > 1 )
344                     {
345                         // get the KW field
346                         IndexerField keywordField = selectIndexerField( indexerField.getOntology(), SearchType.EXACT );
347 
348                         if ( keywordField.isKeyword() )
349                         {
350                             q2 = constructQuery( indexerField.getOntology(), keywordField, query, type );
351                         }
352                     }
353 
354                     if ( q2 == null )
355                     {
356                         return q1;
357                     }
358                     else
359                     {
360                         BooleanQuery bq = new BooleanQuery();
361 
362                         // trick with order
363                         bq.add( q2, Occur.SHOULD );
364                         bq.add( q1, Occur.SHOULD );
365 
366                         return bq;
367                     }
368                 }
369                 catch ( ParseException e )
370                 {
371                     // TODO: we are not falling back anymore to legacy!
372                     throw e;
373 
374                     // getLogger().debug(
375                     // "Query parsing with \"legacy\" method, we got ParseException from QueryParser: "
376                     // + e.getMessage() );
377                     //
378                     // return legacyConstructQuery( indexerField.getKey(), query );
379                 }
380             }
381         }
382         else
383         {
384             // what search type is this?
385             return null;
386         }
387     }
388 
389     public Query legacyConstructQuery( String field, String query )
390     {
391         if ( query == null || query.length() == 0 )
392         {
393             getLogger().info( "Empty or null query for field:" + field );
394 
395             return null;
396         }
397 
398         String q = query.toLowerCase();
399 
400         char h = query.charAt( 0 );
401 
402         if ( JarFileContentsIndexCreator.FLD_CLASSNAMES_KW.getKey().equals( field )
403             || JarFileContentsIndexCreator.FLD_CLASSNAMES.getKey().equals( field ) )
404         {
405             q = q.replaceAll( "\\.", "/" );
406 
407             if ( h == '^' )
408             {
409                 q = q.substring( 1 );
410 
411                 if ( q.charAt( 0 ) != '/' )
412                 {
413                     q = '/' + q;
414                 }
415             }
416             else if ( h != '*' )
417             {
418                 q = "*/" + q;
419             }
420         }
421         else
422         {
423             if ( h == '^' )
424             {
425                 q = q.substring( 1 );
426             }
427             else if ( h != '*' )
428             {
429                 q = "*" + q;
430             }
431         }
432 
433         int l = q.length() - 1;
434         char c = q.charAt( l );
435         if ( c == ' ' || c == '<' || c == '$' )
436         {
437             q = q.substring( 0, q.length() - 1 );
438         }
439         else if ( c != '*' )
440         {
441             q += "*";
442         }
443 
444         int n = q.indexOf( '*' );
445         if ( n == -1 )
446         {
447             return new TermQuery( new Term( field, q ) );
448         }
449         else if ( n > 0 && n == q.length() - 1 )
450         {
451             return new PrefixQuery( new Term( field, q.substring( 0, q.length() - 1 ) ) );
452         }
453 
454         return new WildcardQuery( new Term( field, q ) );
455     }
456 
457     // ==
458 
459     private NexusAnalyzer nexusAnalyzer = new NexusAnalyzer();
460 
461     protected int countTerms( final IndexerField indexerField, final String query )
462     {
463         try
464         {
465             TokenStream ts = nexusAnalyzer.tokenStream( indexerField.getKey(), new StringReader( query ) );
466             ts.reset();
467 
468             int result = 0;
469 
470             while ( ts.incrementToken() )
471             {
472                 result++;
473             }
474             
475             ts.end();
476             ts.close();
477 
478             return result;
479         }
480         catch ( IOException e )
481         {
482             // will not happen
483             return 1;
484         }
485     }
486 }