001 package org.apache.archiva.indexer.search;
002
003 /*
004 * Licensed to the Apache Software Foundation (ASF) under one
005 * or more contributor license agreements. See the NOTICE file
006 * distributed with this work for additional information
007 * regarding copyright ownership. The ASF licenses this file
008 * to you under the Apache License, Version 2.0 (the
009 * "License"); you may not use this file except in compliance
010 * with the License. You may obtain a copy of the License at
011 *
012 * http://www.apache.org/licenses/LICENSE-2.0
013 *
014 * Unless required by applicable law or agreed to in writing,
015 * software distributed under the License is distributed on an
016 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
017 * KIND, either express or implied. See the License for the
018 * specific language governing permissions and limitations
019 * under the License.
020 */
021
022 import org.apache.archiva.admin.model.RepositoryAdminException;
023 import org.apache.archiva.admin.model.beans.ManagedRepository;
024 import org.apache.archiva.admin.model.beans.ProxyConnector;
025 import org.apache.archiva.admin.model.managed.ManagedRepositoryAdmin;
026 import org.apache.archiva.admin.model.proxyconnector.ProxyConnectorAdmin;
027 import org.apache.archiva.common.plexusbridge.MavenIndexerUtils;
028 import org.apache.archiva.common.plexusbridge.PlexusSisuBridge;
029 import org.apache.archiva.common.plexusbridge.PlexusSisuBridgeException;
030 import org.apache.archiva.indexer.util.SearchUtil;
031 import org.apache.commons.lang.StringUtils;
032 import org.apache.lucene.search.BooleanClause.Occur;
033 import org.apache.lucene.search.BooleanQuery;
034 import org.apache.maven.index.ArtifactInfo;
035 import org.apache.maven.index.FlatSearchRequest;
036 import org.apache.maven.index.FlatSearchResponse;
037 import org.apache.maven.index.MAVEN;
038 import org.apache.maven.index.NexusIndexer;
039 import org.apache.maven.index.OSGI;
040 import org.apache.maven.index.context.IndexCreator;
041 import org.apache.maven.index.context.IndexingContext;
042 import org.apache.maven.index.expr.SourcedSearchExpression;
043 import org.apache.maven.index.expr.UserInputSearchExpression;
044 import org.slf4j.Logger;
045 import org.slf4j.LoggerFactory;
046 import org.springframework.stereotype.Service;
047
048 import javax.inject.Inject;
049 import java.io.IOException;
050 import java.util.ArrayList;
051 import java.util.Collection;
052 import java.util.Collections;
053 import java.util.HashSet;
054 import java.util.List;
055 import java.util.Map;
056 import java.util.Set;
057
058 /**
059 * RepositorySearch implementation which uses the Maven Indexer for searching.
060 */
061 @Service("repositorySearch#maven")
062 public class MavenRepositorySearch
063 implements RepositorySearch
064 {
065 private Logger log = LoggerFactory.getLogger( getClass() );
066
067 private NexusIndexer indexer;
068
069 private ManagedRepositoryAdmin managedRepositoryAdmin;
070
071 private ProxyConnectorAdmin proxyConnectorAdmin;
072
073 private MavenIndexerUtils mavenIndexerUtils;
074
075 protected MavenRepositorySearch()
076 {
077 // for test purpose
078 }
079
080 @Inject
081 public MavenRepositorySearch( PlexusSisuBridge plexusSisuBridge, ManagedRepositoryAdmin managedRepositoryAdmin,
082 MavenIndexerUtils mavenIndexerUtils, ProxyConnectorAdmin proxyConnectorAdmin )
083 throws PlexusSisuBridgeException
084 {
085 this.indexer = plexusSisuBridge.lookup( NexusIndexer.class );
086 this.managedRepositoryAdmin = managedRepositoryAdmin;
087 this.mavenIndexerUtils = mavenIndexerUtils;
088 this.proxyConnectorAdmin = proxyConnectorAdmin;
089 }
090
091 /**
092 * @see RepositorySearch#search(String, List, String, SearchResultLimits, List)
093 */
094 public SearchResults search( String principal, List<String> selectedRepos, String term, SearchResultLimits limits,
095 List<String> previousSearchTerms )
096 throws RepositorySearchException
097 {
098 List<String> indexingContextIds = addIndexingContexts( selectedRepos );
099
100 // since upgrade to nexus 2.0.0, query has changed from g:[QUERIED TERM]* to g:*[QUERIED TERM]*
101 // resulting to more wildcard searches so we need to increase max clause count
102 BooleanQuery.setMaxClauseCount( Integer.MAX_VALUE );
103 BooleanQuery q = new BooleanQuery();
104
105 if ( previousSearchTerms == null || previousSearchTerms.isEmpty() )
106 {
107 constructQuery( term, q );
108 }
109 else
110 {
111 for ( String previousTerm : previousSearchTerms )
112 {
113 BooleanQuery iQuery = new BooleanQuery();
114 constructQuery( previousTerm, iQuery );
115
116 q.add( iQuery, Occur.MUST );
117 }
118
119 BooleanQuery iQuery = new BooleanQuery();
120 constructQuery( term, iQuery );
121 q.add( iQuery, Occur.MUST );
122 }
123
124 // we retun only artifacts without classifier in quick search, olamy cannot find a way to say with this field empty
125 // FIXME cannot find a way currently to setup this in constructQuery !!!
126 return search( limits, q, indexingContextIds, NoClassifierArtifactInfoFilter.LIST, selectedRepos, false );
127
128 }
129
130 /**
131 * @see RepositorySearch#search(String, SearchFields, SearchResultLimits)
132 */
133 public SearchResults search( String principal, SearchFields searchFields, SearchResultLimits limits )
134 throws RepositorySearchException
135 {
136 if ( searchFields.getRepositories() == null )
137 {
138 throw new RepositorySearchException( "Repositories cannot be null." );
139 }
140
141 List<String> indexingContextIds = addIndexingContexts( searchFields.getRepositories() );
142
143 // if no index found in the specified ones return an empty search result instead of doing a search on all index
144 // olamy: IMHO doesn't make sense
145 if ( !searchFields.getRepositories().isEmpty() && ( indexingContextIds == null
146 || indexingContextIds.isEmpty() ) )
147 {
148 return new SearchResults();
149 }
150
151 BooleanQuery q = new BooleanQuery();
152 if ( StringUtils.isNotBlank( searchFields.getGroupId() ) )
153 {
154 q.add( indexer.constructQuery( MAVEN.GROUP_ID, new UserInputSearchExpression( searchFields.getGroupId() ) ),
155 Occur.MUST );
156 }
157
158 if ( StringUtils.isNotBlank( searchFields.getArtifactId() ) )
159 {
160 q.add( indexer.constructQuery( MAVEN.ARTIFACT_ID,
161 new UserInputSearchExpression( searchFields.getArtifactId() ) ),
162 Occur.MUST );
163 }
164
165 if ( StringUtils.isNotBlank( searchFields.getVersion() ) )
166 {
167 q.add( indexer.constructQuery( MAVEN.VERSION, new SourcedSearchExpression( searchFields.getVersion() ) ),
168 Occur.MUST );
169 }
170
171 if ( StringUtils.isNotBlank( searchFields.getPackaging() ) )
172 {
173 q.add(
174 indexer.constructQuery( MAVEN.PACKAGING, new UserInputSearchExpression( searchFields.getPackaging() ) ),
175 Occur.MUST );
176 }
177
178 if ( StringUtils.isNotBlank( searchFields.getClassName() ) )
179 {
180 q.add( indexer.constructQuery( MAVEN.CLASSNAMES,
181 new UserInputSearchExpression( searchFields.getClassName() ) ), Occur.MUST );
182 }
183
184 if ( StringUtils.isNotBlank( searchFields.getBundleSymbolicName() ) )
185 {
186 q.add( indexer.constructQuery( OSGI.SYMBOLIC_NAME,
187 new UserInputSearchExpression( searchFields.getBundleSymbolicName() ) ),
188 Occur.MUST );
189 }
190
191 if ( StringUtils.isNotBlank( searchFields.getBundleVersion() ) )
192 {
193 q.add( indexer.constructQuery( OSGI.VERSION,
194 new UserInputSearchExpression( searchFields.getBundleVersion() ) ),
195 Occur.MUST );
196 }
197
198 if ( StringUtils.isNotBlank( searchFields.getBundleExportPackage() ) )
199 {
200 q.add( indexer.constructQuery( OSGI.EXPORT_PACKAGE,
201 new UserInputSearchExpression( searchFields.getBundleExportPackage() ) ),
202 Occur.MUST );
203 }
204
205 if ( StringUtils.isNotBlank( searchFields.getBundleExportService() ) )
206 {
207 q.add( indexer.constructQuery( OSGI.EXPORT_SERVICE,
208 new UserInputSearchExpression( searchFields.getBundleExportService() ) ),
209 Occur.MUST );
210 }
211
212 if ( StringUtils.isNotBlank( searchFields.getBundleImportPackage() ) )
213 {
214 q.add( indexer.constructQuery( OSGI.IMPORT_PACKAGE,
215 new UserInputSearchExpression( searchFields.getBundleImportPackage() ) ),
216 Occur.MUST );
217 }
218
219 if ( StringUtils.isNotBlank( searchFields.getBundleName() ) )
220 {
221 q.add( indexer.constructQuery( OSGI.NAME, new UserInputSearchExpression( searchFields.getBundleName() ) ),
222 Occur.MUST );
223 }
224
225 if ( StringUtils.isNotBlank( searchFields.getBundleImportPackage() ) )
226 {
227 q.add( indexer.constructQuery( OSGI.IMPORT_PACKAGE,
228 new UserInputSearchExpression( searchFields.getBundleImportPackage() ) ),
229 Occur.MUST );
230 }
231
232 if ( StringUtils.isNotBlank( searchFields.getBundleRequireBundle() ) )
233 {
234 q.add( indexer.constructQuery( OSGI.REQUIRE_BUNDLE,
235 new UserInputSearchExpression( searchFields.getBundleRequireBundle() ) ),
236 Occur.MUST );
237 }
238
239 if ( StringUtils.isNotBlank( searchFields.getClassifier() ) )
240 {
241 q.add( indexer.constructQuery( MAVEN.CLASSIFIER,
242 new UserInputSearchExpression( searchFields.getClassifier() ) ),
243 Occur.MUST );
244 }
245
246 if ( q.getClauses() == null || q.getClauses().length <= 0 )
247 {
248 throw new RepositorySearchException( "No search fields set." );
249 }
250
251 return search( limits, q, indexingContextIds, Collections.<ArtifactInfoFilter>emptyList(),
252 searchFields.getRepositories(), searchFields.isIncludePomArtifacts() );
253 }
254
255 private SearchResults search( SearchResultLimits limits, BooleanQuery q, List<String> indexingContextIds,
256 List<? extends ArtifactInfoFilter> filters, List<String> selectedRepos,
257 boolean includePoms )
258 throws RepositorySearchException
259 {
260
261 try
262 {
263 FlatSearchRequest request = new FlatSearchRequest( q );
264 request.setContexts( getIndexingContexts( indexingContextIds ) );
265
266 FlatSearchResponse response = indexer.searchFlat( request );
267
268 if ( response == null || response.getTotalHits() == 0 )
269 {
270 SearchResults results = new SearchResults();
271 results.setLimits( limits );
272 return results;
273 }
274
275 return convertToSearchResults( response, limits, filters, selectedRepos, includePoms );
276 }
277 catch ( IOException e )
278 {
279 throw new RepositorySearchException( e.getMessage(), e );
280 }
281 catch ( RepositoryAdminException e )
282 {
283 throw new RepositorySearchException( e.getMessage(), e );
284 }
285
286 }
287
288 private List<IndexingContext> getIndexingContexts( List<String> ids )
289 {
290 List<IndexingContext> contexts = new ArrayList<IndexingContext>( ids.size() );
291
292 for ( String id : ids )
293 {
294 IndexingContext context = indexer.getIndexingContexts().get( id );
295 if ( context != null )
296 {
297 contexts.add( context );
298 }
299 else
300 {
301 log.warn( "context with id {} not exists", id );
302 }
303 }
304
305 return contexts;
306 }
307
308 private void constructQuery( String term, BooleanQuery q )
309 {
310 q.add( indexer.constructQuery( MAVEN.GROUP_ID, new UserInputSearchExpression( term ) ), Occur.SHOULD );
311 q.add( indexer.constructQuery( MAVEN.ARTIFACT_ID, new UserInputSearchExpression( term ) ), Occur.SHOULD );
312 q.add( indexer.constructQuery( MAVEN.VERSION, new UserInputSearchExpression( term ) ), Occur.SHOULD );
313 q.add( indexer.constructQuery( MAVEN.PACKAGING, new UserInputSearchExpression( term ) ), Occur.SHOULD );
314 q.add( indexer.constructQuery( MAVEN.CLASSNAMES, new UserInputSearchExpression( term ) ), Occur.SHOULD );
315
316 //Query query =
317 // new WildcardQuery( new Term( MAVEN.CLASSNAMES.getFieldName(), "*" ) );
318 //q.add( query, Occur.MUST_NOT );
319 // olamy IMHO we could set this option as at least one must match
320 //q.setMinimumNumberShouldMatch( 1 );
321 }
322
323
324 /**
325 * @param selectedRepos
326 * @return indexing contextId used
327 */
328 private List<String> addIndexingContexts( List<String> selectedRepos )
329 {
330 Set<String> indexingContextIds = new HashSet<String>();
331 for ( String repo : selectedRepos )
332 {
333 try
334 {
335 ManagedRepository repoConfig = managedRepositoryAdmin.getManagedRepository( repo );
336
337 if ( repoConfig != null )
338 {
339
340 IndexingContext context = managedRepositoryAdmin.createIndexContext( repoConfig );
341 if ( context.isSearchable() )
342 {
343 indexingContextIds.addAll( getRemoteIndexingContextIds( repo ) );
344 indexingContextIds.add( context.getId() );
345 }
346 else
347 {
348 log.warn( "indexingContext with id {} not searchable", repoConfig.getId() );
349 }
350
351 }
352 else
353 {
354 log.warn( "Repository '{}' not found in configuration.", repo );
355 }
356 }
357 catch ( RepositoryAdminException e )
358 {
359 log.warn( "RepositoryAdminException occured while accessing index of repository '{}' : {}", repo,
360 e.getMessage() );
361 continue;
362 }
363 }
364
365 return new ArrayList<String>( indexingContextIds );
366 }
367
368
369 public Set<String> getRemoteIndexingContextIds( String managedRepoId )
370 throws RepositoryAdminException
371 {
372 Set<String> ids = new HashSet<String>();
373
374 List<ProxyConnector> proxyConnectors = proxyConnectorAdmin.getProxyConnectorAsMap().get( managedRepoId );
375
376 if ( proxyConnectors == null || proxyConnectors.isEmpty() )
377 {
378 return ids;
379 }
380
381 for ( ProxyConnector proxyConnector : proxyConnectors )
382 {
383 String remoteId = "remote-" + proxyConnector.getTargetRepoId();
384 IndexingContext context = indexer.getIndexingContexts().get( remoteId );
385 if ( context != null && context.isSearchable() )
386 {
387 ids.add( remoteId );
388 }
389 }
390
391 return ids;
392 }
393
394 public Collection<String> getAllGroupIds( String principal, List<String> selectedRepos )
395 throws RepositorySearchException
396 {
397 List<IndexingContext> indexContexts = getIndexingContexts( selectedRepos );
398
399 if ( indexContexts == null || indexContexts.isEmpty() )
400 {
401 return Collections.emptyList();
402 }
403
404 try
405 {
406 Set<String> allGroupIds = new HashSet<String>();
407 for ( IndexingContext indexingContext : indexContexts )
408 {
409 allGroupIds.addAll( indexingContext.getAllGroups() );
410 }
411 return allGroupIds;
412 }
413 catch ( IOException e )
414 {
415 throw new RepositorySearchException( e.getMessage(), e );
416 }
417
418 }
419
420
421 protected List<? extends IndexCreator> getAllIndexCreators()
422 {
423 return mavenIndexerUtils.getAllIndexCreators();
424 }
425
426
427 private SearchResults convertToSearchResults( FlatSearchResponse response, SearchResultLimits limits,
428 List<? extends ArtifactInfoFilter> artifactInfoFilters,
429 List<String> selectedRepos, boolean includePoms )
430 throws RepositoryAdminException
431 {
432 SearchResults results = new SearchResults();
433 Set<ArtifactInfo> artifactInfos = response.getResults();
434
435 for ( ArtifactInfo artifactInfo : artifactInfos )
436 {
437 if ( StringUtils.equalsIgnoreCase( "pom", artifactInfo.fextension ) && !includePoms )
438 {
439 continue;
440 }
441 String id = SearchUtil.getHitId( artifactInfo.groupId, artifactInfo.artifactId, artifactInfo.classifier,
442 artifactInfo.packaging );
443 Map<String, SearchResultHit> hitsMap = results.getHitsMap();
444
445 if ( !applyArtifactInfoFilters( artifactInfo, artifactInfoFilters, hitsMap ) )
446 {
447 continue;
448 }
449
450 SearchResultHit hit = hitsMap.get( id );
451 if ( hit != null )
452 {
453 if ( !hit.getVersions().contains( artifactInfo.version ) )
454 {
455 hit.addVersion( artifactInfo.version );
456 }
457 }
458 else
459 {
460 hit = new SearchResultHit();
461 hit.setArtifactId( artifactInfo.artifactId );
462 hit.setGroupId( artifactInfo.groupId );
463 hit.setRepositoryId( artifactInfo.repository );
464 hit.addVersion( artifactInfo.version );
465 hit.setBundleExportPackage( artifactInfo.bundleExportPackage );
466 hit.setBundleExportService( artifactInfo.bundleExportService );
467 hit.setBundleSymbolicName( artifactInfo.bundleSymbolicName );
468 hit.setBundleVersion( artifactInfo.bundleVersion );
469 hit.setBundleDescription( artifactInfo.bundleDescription );
470 hit.setBundleDocUrl( artifactInfo.bundleDocUrl );
471 hit.setBundleRequireBundle( artifactInfo.bundleRequireBundle );
472 hit.setBundleImportPackage( artifactInfo.bundleImportPackage );
473 hit.setBundleLicense( artifactInfo.bundleLicense );
474 hit.setBundleName( artifactInfo.bundleName );
475 hit.setContext( artifactInfo.context );
476 hit.setGoals( artifactInfo.goals );
477 hit.setPrefix( artifactInfo.prefix );
478 hit.setPackaging( artifactInfo.packaging );
479 hit.setClassifier( artifactInfo.classifier );
480 hit.setFileExtension( artifactInfo.fextension );
481 hit.setUrl( getBaseUrl( artifactInfo, selectedRepos ) );
482 }
483
484 results.addHit( id, hit );
485 }
486
487 results.setTotalHits( response.getTotalHitsCount() );
488 results.setTotalHitsMapSize( results.getHitsMap().values().size() );
489 results.setReturnedHitsCount( response.getReturnedHitsCount() );
490 results.setLimits( limits );
491
492 if ( limits == null || limits.getSelectedPage() == SearchResultLimits.ALL_PAGES )
493 {
494 return results;
495 }
496 else
497 {
498 return paginate( results );
499 }
500 }
501
502 /**
503 * calculate baseUrl without the context and base Archiva Url
504 *
505 * @param artifactInfo
506 * @return
507 */
508 protected String getBaseUrl( ArtifactInfo artifactInfo, List<String> selectedRepos )
509 throws RepositoryAdminException
510 {
511 StringBuilder sb = new StringBuilder();
512 if ( StringUtils.startsWith( artifactInfo.context, "remote-" ) )
513 {
514 // it's a remote index result we search a managed which proxying this remote and on which
515 // current user has read karma
516 String managedRepoId =
517 getManagedRepoId( StringUtils.substringAfter( artifactInfo.context, "remote-" ), selectedRepos );
518 if ( managedRepoId != null )
519 {
520 sb.append( '/' ).append( managedRepoId );
521 artifactInfo.context = managedRepoId;
522 }
523 }
524 else
525 {
526 sb.append( '/' ).append( artifactInfo.context );
527 }
528
529 sb.append( '/' ).append( StringUtils.replaceChars( artifactInfo.groupId, '.', '/' ) );
530 sb.append( '/' ).append( artifactInfo.artifactId );
531 sb.append( '/' ).append( artifactInfo.version );
532 sb.append( '/' ).append( artifactInfo.artifactId );
533 sb.append( '-' ).append( artifactInfo.version );
534 if ( StringUtils.isNotBlank( artifactInfo.classifier ) )
535 {
536 sb.append( '-' ).append( artifactInfo.classifier );
537 }
538 // maven-plugin packaging is a jar
539 if ( StringUtils.equals( "maven-plugin", artifactInfo.packaging ) )
540 {
541 sb.append( "jar" );
542 }
543 else
544 {
545 sb.append( '.' ).append( artifactInfo.packaging );
546 }
547
548 return sb.toString();
549 }
550
551 /**
552 * return a managed repo for a remote result
553 *
554 * @param remoteRepo
555 * @param selectedRepos
556 * @return
557 * @throws RepositoryAdminException
558 */
559 private String getManagedRepoId( String remoteRepo, List<String> selectedRepos )
560 throws RepositoryAdminException
561 {
562 Map<String, List<ProxyConnector>> proxyConnectorMap = proxyConnectorAdmin.getProxyConnectorAsMap();
563 if ( proxyConnectorMap == null || proxyConnectorMap.isEmpty() )
564 {
565 return null;
566 }
567 if ( selectedRepos != null && !selectedRepos.isEmpty() )
568 {
569 for ( Map.Entry<String, List<ProxyConnector>> entry : proxyConnectorMap.entrySet() )
570 {
571 if ( selectedRepos.contains( entry.getKey() ) )
572 {
573 for ( ProxyConnector proxyConnector : entry.getValue() )
574 {
575 if ( StringUtils.equals( remoteRepo, proxyConnector.getTargetRepoId() ) )
576 {
577 return proxyConnector.getSourceRepoId();
578 }
579 }
580 }
581 }
582 }
583
584 // we don't find in search selected repos so return the first one
585 for ( Map.Entry<String, List<ProxyConnector>> entry : proxyConnectorMap.entrySet() )
586 {
587
588 for ( ProxyConnector proxyConnector : entry.getValue() )
589 {
590 if ( StringUtils.equals( remoteRepo, proxyConnector.getTargetRepoId() ) )
591 {
592 return proxyConnector.getSourceRepoId();
593 }
594 }
595
596 }
597 return null;
598 }
599
600 private boolean applyArtifactInfoFilters( ArtifactInfo artifactInfo,
601 List<? extends ArtifactInfoFilter> artifactInfoFilters,
602 Map<String, SearchResultHit> currentResult )
603 {
604 if ( artifactInfoFilters == null || artifactInfoFilters.isEmpty() )
605 {
606 return true;
607 }
608
609 for ( ArtifactInfoFilter filter : artifactInfoFilters )
610 {
611 if ( !filter.addArtifactInResult( artifactInfo, currentResult ) )
612 {
613 return false;
614 }
615 }
616 return true;
617 }
618
619 protected SearchResults paginate( SearchResults results )
620 {
621 SearchResultLimits limits = results.getLimits();
622 SearchResults paginated = new SearchResults();
623
624 int fetchCount = limits.getPageSize();
625 int offset = ( limits.getSelectedPage() * limits.getPageSize() );
626
627 if ( fetchCount > results.getTotalHits() )
628 {
629 fetchCount = results.getTotalHits();
630 }
631
632 // Goto offset.
633 if ( offset < results.getTotalHits() )
634 {
635 // only process if the offset is within the hit count.
636 for ( int i = 0; i < fetchCount; i++ )
637 {
638 // Stop fetching if we are past the total # of available hits.
639 if ( offset + i >= results.getHits().size() )
640 {
641 break;
642 }
643
644 SearchResultHit hit = results.getHits().get( ( offset + i ) );
645 if ( hit != null )
646 {
647 String id = SearchUtil.getHitId( hit.getGroupId(), hit.getArtifactId(), hit.getClassifier(),
648 hit.getPackaging() );
649 paginated.addHit( id, hit );
650 }
651 else
652 {
653 break;
654 }
655 }
656 }
657 paginated.setTotalHits( results.getTotalHits() );
658 paginated.setReturnedHitsCount( paginated.getHits().size() );
659 paginated.setTotalHitsMapSize( results.getTotalHitsMapSize() );
660 paginated.setLimits( limits );
661
662 return paginated;
663 }
664
665
666 }