Skip to content

Lazy init SMO backend and increase timeout #7587

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
mbien merged 1 commit into from
Jul 17, 2024
Merged

Lazy init SMO backend and increase timeout #7587

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion java/maven.indexer/nbproject/project.properties
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@

test.config.stableBTD.includes=**/*Test.class
is.autoload=true
javac.release=11
javac.release=17
javac.compilerargs=-Xlint -Xlint:-serial
release.external/indexer-core-7.1.4.jar=modules/ext/maven/indexer-core-7.1.4.jar
release.external/search-api-7.1.4.jar=modules/ext/maven/search-api-7.1.4.jar
Expand Down
11 changes: 2 additions & 9 deletions java/maven.indexer/src/org/netbeans/modules/maven/indexer/CompositeResult.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,21 +19,14 @@
package org.netbeans.modules.maven.indexer;

import java.util.List;
import java.util.stream.Collectors;
import org.netbeans.modules.maven.indexer.spi.ResultImplementation;

/**
* Merged view over multiple {@link ResultImplementation}s.
*
* @author mbien
*/
final class CompositeResult<T> implements ResultImplementation<T> {

private final List<ResultImplementation<T>> results;

public CompositeResult(List<ResultImplementation<T>> results) {
this.results = results;
}
record CompositeResult<T>(List<ResultImplementation<T>> results) implements ResultImplementation<T> {

public CompositeResult(ResultImplementation<T> first, ResultImplementation<T> second) {
this(List.of(first, second));
Expand Down Expand Up @@ -62,7 +55,7 @@ public List<T> getResults() {
.flatMap(r -> r.getResults().stream())
.sorted()
.distinct()
.collect(Collectors.toList());
.toList();
}

@Override
Expand Down
70 changes: 32 additions & 38 deletions java/maven.indexer/src/org/netbeans/modules/maven/indexer/NexusRepositoryIndexerImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,7 @@
import org.apache.maven.search.api.SearchRequest;
import org.apache.maven.search.api.request.FieldQuery;
import org.apache.maven.search.api.request.Paging;
import org.apache.maven.search.api.transport.Java11HttpClientTransport;
import org.apache.maven.search.backend.smo.SmoSearchBackend;
import org.apache.maven.search.backend.smo.SmoSearchBackendFactory;
import org.apache.maven.settings.Proxy;
Expand Down Expand Up @@ -181,8 +182,8 @@ public class NexusRepositoryIndexerImpl implements RepositoryIndexerImplementati
* For remote repo download and indexing tasks.
*/
private static final RequestProcessor RP_REMOTE = new RequestProcessor("maven-remote-indexing");
private final SmoSearchBackend smo = SmoSearchBackendFactory.createDefault();

private SmoSearchBackend smo = null;

@Override
public boolean handlesRepository(RepositoryInfo repo) {
Expand Down Expand Up @@ -360,7 +361,7 @@ private boolean loadIndexingContext(final RepositoryInfo info) throws IOExceptio
);
} else {
creators = List.of(
info.getId().equals(smo.getRepositoryId())
info.getId().equals(getSMO().getRepositoryId())
? new MinimalArtifactInfoRemoteIndexCreator()
: new MinimalArtifactInfoIndexCreator(),
new NotifyingIndexCreator()
Expand Down Expand Up @@ -417,14 +418,14 @@ private boolean loadIndexingContext(final RepositoryInfo info) throws IOExceptio

int MAX_MAX_CLAUSE = 1<<11; // conservative maximum for too general queries, like "c:*class*"

if (q instanceof BooleanQuery) {
List<BooleanClause> list = ((BooleanQuery)q).clauses();
if (q instanceof BooleanQuery booleanQuery) {
List<BooleanClause> list = booleanQuery.clauses();
if (list.size() == 1) {
Query q1 = list.get(0).getQuery();
if (q1 instanceof PrefixQuery && "u".equals(((PrefixQuery)q1).getPrefix().field())) {
if (q1 instanceof PrefixQuery pq && "u".equals(pq.getPrefix().field())) {
// increase for queries like "+u:org.netbeans.modules|*" to succeed
MAX_MAX_CLAUSE = 1<<16;
} else if (q1 instanceof TermQuery && "p".equals(((TermQuery) q1).getTerm().field())) {
} else if (q1 instanceof TermQuery tq && "p".equals(tq.getTerm().field())) {
// +p:nbm also produces several thousand hits
MAX_MAX_CLAUSE = 1<<16;
}
Expand Down Expand Up @@ -524,9 +525,7 @@ private void indexLoadedRepo(final RepositoryInfo repo, boolean updateLocal) thr
}
// MINDEXER-42: cannot use WagonHelper.getWagonResourceFetcher
Wagon wagon = embedder.lookup(Wagon.class, protocol);
if (wagon instanceof HttpWagon) { //#216401
HttpWagon httpwagon = (HttpWagon) wagon;
//#215343
if (wagon instanceof HttpWagon httpwagon) { //#215343
Properties p = new Properties();
p.setProperty("User-Agent", "netBeans/" + System.getProperty("netbeans.buildnumber"));
httpwagon.setHttpHeaders(p);
Expand Down Expand Up @@ -1210,7 +1209,7 @@ private ResultImplementation<NBVersionInfo> getVersions(final String groupId, fi
public ResultImplementation<NBVersionInfo> findVersionsByClass(final String className, List<RepositoryInfo> repos) {

Optional<RepositoryInfo> central = repos.stream()
.filter(repo -> repo.getId().equals(smo.getRepositoryId()))
.filter(repo -> repo.getId().equals(getSMO().getRepositoryId()))
.findFirst();

// remote index contains no class data -> use web service
Expand All @@ -1223,7 +1222,7 @@ public ResultImplementation<NBVersionInfo> findVersionsByClass(final String clas
org.apache.maven.search.api.MAVEN.FQ_CLASS_NAME
: org.apache.maven.search.api.MAVEN.CLASS_NAME, className));

return new CompositeResult<>(findVersionsByClass(className, otherRepos), new SMORequestResult(smo, request));
return new CompositeResult<>(findVersionsByClass(className, otherRepos), new SMORequestResult(getSMO(), request));
} else {
ResultImpl<NBVersionInfo> result = new ResultImpl<>((ResultImpl<NBVersionInfo> result1) -> {
findVersionsByClass(className, result1, result1.getSkipped(), false);
Expand Down Expand Up @@ -1379,7 +1378,7 @@ private static void convertToNBGroupInfo(Collection<NBVersionInfo> artifactInfos
public ResultImplementation<NBVersionInfo> findBySHA1(final String sha1, List<RepositoryInfo> repos) {

Optional<RepositoryInfo> central = repos.stream()
.filter(repo -> repo.getId().equals(smo.getRepositoryId()))
.filter(repo -> repo.getId().equals(getSMO().getRepositoryId()))
.findFirst();

// remote index contains no sh1 data -> use web service
Expand All @@ -1390,7 +1389,7 @@ public ResultImplementation<NBVersionInfo> findBySHA1(final String sha1, List<Re
SearchRequest request = new SearchRequest(new Paging(8),
FieldQuery.fieldQuery(org.apache.maven.search.api.MAVEN.SHA1, sha1));

return new CompositeResult<>(findBySHA1(sha1, otherRepos), new SMORequestResult(smo, request));
return new CompositeResult<>(findBySHA1(sha1, otherRepos), new SMORequestResult(getSMO(), request));
} else {
ResultImpl<NBVersionInfo> result = new ResultImpl<>((ResultImpl<NBVersionInfo> result1) -> {
findBySHA1(sha1, result1, result1.getSkipped(), false);
Expand Down Expand Up @@ -1744,30 +1743,22 @@ static NBVersionInfo convertToNBVersionInfo(ArtifactInfo ai) {
}

private static Query setBooleanRewrite (final Query q) {
if (q instanceof MultiTermQuery) {
((MultiTermQuery)q).setRewriteMethod(MultiTermQuery.CONSTANT_SCORE_REWRITE);
} else if (q instanceof BooleanQuery) {
for (BooleanClause c : ((BooleanQuery)q).clauses()) {
if (q instanceof MultiTermQuery multiTermQuery) {
multiTermQuery.setRewriteMethod(MultiTermQuery.CONSTANT_SCORE_REWRITE);
} else if (q instanceof BooleanQuery booleanQuery) {
for (BooleanClause c : booleanQuery.clauses()) {
setBooleanRewrite(c.getQuery());
}
}
return q;
}

private static class SkippedAction implements RepoAction {

private final ResultImpl<?> result;

private SkippedAction(ResultImpl<?> result) {
this.result = result;
}

private record SkippedAction(ResultImpl<?> result) implements RepoAction {
@Override
public void run(RepositoryInfo repo, IndexingContext context) throws IOException {
//indexing context is always null here..
result.addSkipped(repo);
}

}

private static class NoJavadocSourceFilter implements ArtifactInfoFilter {
Expand Down Expand Up @@ -1842,21 +1833,24 @@ private static void rebuildGroupCache(RepositoryInfo repo, IndexingContext conte
});
}

// somewhat based on maven-indexer impl (in WagonHelper) prior to removal in maven-indexer 7.0.0
private static class WagonFetcher implements ResourceFetcher {
private synchronized SmoSearchBackend getSMO() {
if (smo == null) {
smo = SmoSearchBackendFactory.create(
SmoSearchBackendFactory.DEFAULT_BACKEND_ID,
SmoSearchBackendFactory.DEFAULT_REPOSITORY_ID,
SmoSearchBackendFactory.DEFAULT_SMO_URI,
new Java11HttpClientTransport(SMORequestResult.REQUEST_TIMEOUT)
);
}
return smo;
}

private final TransferListener listener;
private final AuthenticationInfo authenticationInfo;
private final ProxyInfo proxyInfo;
private final Wagon wagon;
// somewhat based on maven-indexer impl (in WagonHelper) prior to removal in maven-indexer 7.0.0
private record WagonFetcher(Wagon wagon, TransferListener listener, AuthenticationInfo authenticationInfo, ProxyInfo proxyInfo) implements ResourceFetcher {

public WagonFetcher(Wagon wagon, TransferListener listener, AuthenticationInfo authenticationInfo, ProxyInfo proxyInfo) {
public WagonFetcher {
Objects.requireNonNull(wagon);
Objects.requireNonNull(listener);
this.wagon = wagon;
this.listener = listener;
this.authenticationInfo = authenticationInfo;
this.proxyInfo = proxyInfo;
}

@Override
Expand Down
4 changes: 2 additions & 2 deletions java/maven.indexer/src/org/netbeans/modules/maven/indexer/OnStop.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,8 +46,8 @@ public void run() {
}
}
for (RepositoryIndexerImplementation rii : Lookup.getDefault().lookupAll(RepositoryIndexerImplementation.class)) {
if (rii instanceof NexusRepositoryIndexerImpl) {
((NexusRepositoryIndexerImpl)rii).shutdownAll();
if (rii instanceof NexusRepositoryIndexerImpl impl) {
impl.shutdownAll();
}
}
}
Expand Down
16 changes: 10 additions & 6 deletions java/maven.indexer/src/org/netbeans/modules/maven/indexer/SMORequestResult.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,13 @@
package org.netbeans.modules.maven.indexer;

import java.io.IOException;
import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.stream.Collectors;
import org.apache.maven.search.api.MAVEN;
import org.apache.maven.search.api.SearchRequest;
import org.apache.maven.search.backend.smo.SmoSearchBackend;
Expand All @@ -43,7 +44,10 @@ final class SMORequestResult implements ResultImplementation<NBVersionInfo> {

// if we paginate too much we will get throttled
private static final int MAX_PAGES = 10;
private static final long TIMEOUT = 20_000;

// response time is fairly unpredictable
static final Duration REQUEST_TIMEOUT = Duration.ofSeconds(15);
private static final Duration SEARCH_TIMEOUT = Duration.ofSeconds(20);

private final SmoSearchBackend smo;
private SmoSearchResponse response;
Expand All @@ -58,7 +62,7 @@ public SMORequestResult(SmoSearchBackend smo, SearchRequest request) {
} catch (IOException ex) {
LOG.log(Level.INFO, "SMO "+request+" failed", ex);
this.response = null;
this.list = Collections.emptyList();
this.list = List.of();
}
}

Expand All @@ -76,7 +80,7 @@ public List<NBVersionInfo> getResults() {
null,
null,
rec.getValue(MAVEN.CLASSIFIER)))
.collect(Collectors.toList());
.toList();
}
return list;
}
Expand All @@ -87,9 +91,9 @@ public void waitForSkipped() {
return;
}
List<NBVersionInfo> fullList = new ArrayList<>(getResults());
long start = System.currentTimeMillis();
Instant timeout = Instant.now().plus(SEARCH_TIMEOUT);
int page = 0;
while (isPartial() && page++ < MAX_PAGES && (System.currentTimeMillis()-start) < TIMEOUT) {
while (isPartial() && page++ < MAX_PAGES && Instant.now().isBefore(timeout)) {
list = null;
try {
response = search(response.getSearchRequest().nextPage());
Expand Down
3 changes: 1 addition & 2 deletions java/maven.indexer/src/org/netbeans/modules/maven/indexer/api/PluginIndexManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
Expand Down Expand Up @@ -265,7 +264,7 @@ public static Set<String> getPluginsForGoalPrefix(String prefix) throws Exceptio
qf.setValue(prefix);
qf.setOccur(QueryField.OCCUR_MUST);
qf.setMatch(QueryField.MATCH_EXACT);
for (NBVersionInfo v : RepositoryQueries.findResult(Collections.singletonList(qf), null).getResults()) {
for (NBVersionInfo v : RepositoryQueries.findResult(List.of(qf), null).getResults()) {
result.add(v.getGroupId() + '|' + v.getArtifactId() + '|' + v.getVersion());
}
// This is more complete but much too slow:
Expand Down
8 changes: 2 additions & 6 deletions java/maven.indexer/src/org/netbeans/modules/maven/indexer/api/RepositoryQueries.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -225,12 +225,8 @@ private static Map<RepositoryIndexQueryProvider, List<RepositoryInfo>> getQueryP
for (RepositoryInfo repo : repos) {
for (RepositoryIndexQueryProvider idx : idxs) {
if(idx.handlesRepository(repo)) {
List<RepositoryInfo> mappedRepos = qp2Repo.get(idx);
if(mappedRepos == null) {
mappedRepos = new LinkedList<>();
qp2Repo.put(idx, mappedRepos);
}
mappedRepos.add(repo);
qp2Repo.computeIfAbsent(idx, k -> new LinkedList<>())
.add(repo);
break;
}
}
Expand Down