- set the chunksize to 100 to meet the max of the embedded solr
- re-enable sorting (the case where we switched it of should be away)
- enable recrawling on remote-solr
pull/278/head
sgaebel 6 years ago
parent 8f58c1dcfa
commit 8d2e7262d9

@ -140,9 +140,6 @@ public class IndexReIndexMonitor_p {
if (recrawlbt == null || recrawlbt.shutdownInProgress()) { if (recrawlbt == null || recrawlbt.shutdownInProgress()) {
prop.put("recrawljobrunning_simulationResult", 0); prop.put("recrawljobrunning_simulationResult", 0);
prop.put("recrawljobrunning_error", 0); prop.put("recrawljobrunning_error", 0);
if(!sb.index.fulltext().connectedLocalSolr()) {
prop.put("recrawljobrunning_error", 1); // Re-crawl works only with an embedded local Solr index
} else {
if (post.containsKey("recrawlnow")) { if (post.containsKey("recrawlnow")) {
sb.deployThread(RecrawlBusyThread.THREAD_NAME, "ReCrawl", "recrawl existing documents", null, sb.deployThread(RecrawlBusyThread.THREAD_NAME, "ReCrawl", "recrawl existing documents", null,
new RecrawlBusyThread(Switchboard.getSwitchboard(), recrawlQuery, inclerrdoc), 1000); new RecrawlBusyThread(Switchboard.getSwitchboard(), recrawlQuery, inclerrdoc), 1000);
@ -191,7 +188,6 @@ public class IndexReIndexMonitor_p {
prop.put("recrawljobrunning_simulationResult", 3); prop.put("recrawljobrunning_simulationResult", 3);
} }
} }
}
if(post.containsKey("recrawlDefaults")) { if(post.containsKey("recrawlDefaults")) {
recrawlQuery = RecrawlBusyThread.DEFAULT_QUERY; recrawlQuery = RecrawlBusyThread.DEFAULT_QUERY;

@ -72,7 +72,7 @@ public class RecrawlBusyThread extends AbstractBusyThread {
private boolean includefailed; private boolean includefailed;
private int chunkstart = 0; private int chunkstart = 0;
private final int chunksize; private final int chunksize = 100;
private final Switchboard sb; private final Switchboard sb;
/** buffer of urls to recrawl */ /** buffer of urls to recrawl */
@ -129,8 +129,7 @@ public class RecrawlBusyThread extends AbstractBusyThread {
this.urlstack = new HashSet<DigestURL>(); this.urlstack = new HashSet<DigestURL>();
// workaround to prevent solr exception on existing index (not fully reindexed) since intro of schema with docvalues // workaround to prevent solr exception on existing index (not fully reindexed) since intro of schema with docvalues
// org.apache.solr.core.SolrCore java.lang.IllegalStateException: unexpected docvalues type NONE for field 'load_date_dt' (expected=NUMERIC). Use UninvertingReader or index with docvalues. // org.apache.solr.core.SolrCore java.lang.IllegalStateException: unexpected docvalues type NONE for field 'load_date_dt' (expected=NUMERIC). Use UninvertingReader or index with docvalues.
this.solrSortBy = null; // CollectionSchema.load_date_dt.getSolrFieldName() + " asc"; solrSortBy = CollectionSchema.load_date_dt.getSolrFieldName() + " asc";
this.chunksize = sb.getConfigInt(SwitchboardConstants.CRAWLER_THREADS_ACTIVE_MAX, 200);
final SolrConnector solrConnector = this.sb.index.fulltext().getDefaultConnector(); final SolrConnector solrConnector = this.sb.index.fulltext().getDefaultConnector();
if (solrConnector != null && !solrConnector.isClosed()) { if (solrConnector != null && !solrConnector.isClosed()) {

Loading…
Cancel
Save