boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

fixes for problems with remote solr and non-activated webgraph index

Browse Source
pull/1/head
orbiter 12 years ago
parent 336f86394c
commit 6fb2811e68
8 changed files with 35 additions and 33 deletions
Show Stats Download Patch File Download Diff File

2
htroot/IndexDeletion_p.java
Unescape View File

@ -187,7 +187,7 @@ public class IndexDeletion_p {
} else { } else {
try { try {
defaultConnector.deleteByQuery(collection1Query); defaultConnector.deleteByQuery(collection1Query);
webgraphConnector.deleteByQuery(webgraphQuery); if (webgraphConnector != null) webgraphConnector.deleteByQuery(webgraphQuery);
sb.tables.recordAPICall(post, "IndexDeletion_p.html", WorkTables.TABLE_API_TYPE_DELETION, "deletion, docs older than " + timedelete_number + " " + timedelete_unit); sb.tables.recordAPICall(post, "IndexDeletion_p.html", WorkTables.TABLE_API_TYPE_DELETION, "deletion, docs older than " + timedelete_number + " " + timedelete_unit);
} catch (final IOException e) { } catch (final IOException e) {
} }

2
htroot/IndexFederated_p.java
Unescape View File

@ -84,7 +84,7 @@ public class IndexFederated_p {
} }
boolean webgraph = post.getBoolean(SwitchboardConstants.CORE_SERVICE_WEBGRAPH); boolean webgraph = post.getBoolean(SwitchboardConstants.CORE_SERVICE_WEBGRAPH);
sb.index.writeWebgraph(webgraph); sb.index.fulltext().writeWebgraph(webgraph);
env.setConfig(SwitchboardConstants.CORE_SERVICE_WEBGRAPH, webgraph); env.setConfig(SwitchboardConstants.CORE_SERVICE_WEBGRAPH, webgraph);
boolean jena = post.getBoolean(SwitchboardConstants.CORE_SERVICE_JENA); boolean jena = post.getBoolean(SwitchboardConstants.CORE_SERVICE_JENA);

2
source/net/yacy/cora/federate/opensearch/OpenSearchConnector.java
Unescape View File

@ -181,7 +181,7 @@ public class OpenSearchConnector {
if (sb == null) { if (sb == null) {
return false; return false;
} }
final SolrConnector connector = sb.index.fulltext().getWebgraphConnector(); final SolrConnector connector = sb.index.fulltext().writeToWebgraph() ? null : sb.index.fulltext().getWebgraphConnector();
// check if needed Solr fields are available (selected) // check if needed Solr fields are available (selected)
if (connector == null) { if (connector == null) {
ConcurrentLog.severe("OpenSearchConnector.Discover", "Error on connecting to embedded Solr webgraph index"); ConcurrentLog.severe("OpenSearchConnector.Discover", "Error on connecting to embedded Solr webgraph index");

4
source/net/yacy/search/Switchboard.java
Unescape View File

@ -489,7 +489,7 @@ public final class Switchboard extends serverSwitch {
this.index.connectUrlDb(this.useTailCache, this.exceed134217727); this.index.connectUrlDb(this.useTailCache, this.exceed134217727);
try {this.index.fulltext().connectLocalSolr();} catch (final IOException e) {ConcurrentLog.logException(e);} try {this.index.fulltext().connectLocalSolr();} catch (final IOException e) {ConcurrentLog.logException(e);}
} }
this.index.writeWebgraph(this.getConfigBool(SwitchboardConstants.CORE_SERVICE_WEBGRAPH, false)); this.index.fulltext().writeWebgraph(this.getConfigBool(SwitchboardConstants.CORE_SERVICE_WEBGRAPH, false));
// set up the solr interface // set up the solr interface
final String solrurls = getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_URL, "http://127.0.0.1:8983/solr"); final String solrurls = getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_URL, "http://127.0.0.1:8983/solr");
@ -1322,7 +1322,7 @@ public final class Switchboard extends serverSwitch {
this.index.fulltext().connectLocalSolr(); this.index.fulltext().connectLocalSolr();
this.index.connectUrlDb(this.useTailCache, this.exceed134217727); this.index.connectUrlDb(this.useTailCache, this.exceed134217727);
} }
this.index.writeWebgraph(this.getConfigBool(SwitchboardConstants.CORE_SERVICE_WEBGRAPH, false)); this.index.fulltext().writeWebgraph(this.getConfigBool(SwitchboardConstants.CORE_SERVICE_WEBGRAPH, false));
// set up the solr interface // set up the solr interface
final String solrurls = getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_URL, "http://127.0.0.1:8983/solr"); final String solrurls = getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_URL, "http://127.0.0.1:8983/solr");

2
source/net/yacy/search/index/DocumentIndex.java
Unescape View File

@ -77,7 +77,7 @@ public class DocumentIndex extends Segment {
false // exceed134217727 false // exceed134217727
); );
super.fulltext().connectLocalSolr(); super.fulltext().connectLocalSolr();
super.writeWebgraph(true); super.fulltext().writeWebgraph(true);
this.callback = callback; this.callback = callback;
this.queue = new LinkedBlockingQueue<DigestURI>(WorkflowProcessor.availableCPU * 300); this.queue = new LinkedBlockingQueue<DigestURI>(WorkflowProcessor.availableCPU * 300);
this.worker = new Worker[WorkflowProcessor.availableCPU]; this.worker = new Worker[WorkflowProcessor.availableCPU];

38
source/net/yacy/search/index/Fulltext.java
Unescape View File

@ -89,14 +89,15 @@ public final class Fulltext {
private static final String SOLR_OLD_PATH[] = new String[]{"solr_36"}; private static final String SOLR_OLD_PATH[] = new String[]{"solr_36"};
// class objects // class objects
private final File segmentPath; private final File segmentPath;
private Index urlIndexFile; private Index urlIndexFile;
private Export exportthread; // will have a export thread assigned if exporter is running private Export exportthread; // will have a export thread assigned if exporter is running
private String tablename; private String tablename;
private ArrayList<HostStat> statsDump; private ArrayList<HostStat> statsDump;
private InstanceMirror solrInstances; private InstanceMirror solrInstances;
private final CollectionConfiguration collectionConfiguration; private final CollectionConfiguration collectionConfiguration;
private final WebgraphConfiguration webgraphConfiguration; private final WebgraphConfiguration webgraphConfiguration;
private boolean writeWebgraph;
protected Fulltext(final File segmentPath, final CollectionConfiguration collectionConfiguration, final WebgraphConfiguration webgraphConfiguration) { protected Fulltext(final File segmentPath, final CollectionConfiguration collectionConfiguration, final WebgraphConfiguration webgraphConfiguration) {
this.segmentPath = segmentPath; this.segmentPath = segmentPath;
@ -107,6 +108,15 @@ public final class Fulltext {
this.solrInstances = new InstanceMirror(); this.solrInstances = new InstanceMirror();
this.collectionConfiguration = collectionConfiguration; this.collectionConfiguration = collectionConfiguration;
this.webgraphConfiguration = webgraphConfiguration; this.webgraphConfiguration = webgraphConfiguration;
this.writeWebgraph = false;
}
public void writeWebgraph(boolean check) {
this.writeWebgraph = check;
}
public boolean writeToWebgraph() {
return this.writeWebgraph;
} }
/** /**
@ -211,6 +221,7 @@ public final class Fulltext {
} }
public SolrConnector getWebgraphConnector() { public SolrConnector getWebgraphConnector() {
if (!this.writeWebgraph) return null;
synchronized (this.solrInstances) { synchronized (this.solrInstances) {
return this.solrInstances.getMirrorConnector(WebgraphSchema.CORE_NAME); return this.solrInstances.getMirrorConnector(WebgraphSchema.CORE_NAME);
} }
@ -276,7 +287,7 @@ public final class Fulltext {
* @return * @return
*/ */
public long webgraphSize() { public long webgraphSize() {
return this.getWebgraphConnector().getSize(); return this.writeWebgraph ? this.getWebgraphConnector().getSize() : 0;
} }
public void close() { public void close() {
@ -294,7 +305,7 @@ public final class Fulltext {
if (lastCommit + 10000 > t) return; if (lastCommit + 10000 > t) return;
lastCommit = t; lastCommit = t;
getDefaultConnector().commit(softCommit); getDefaultConnector().commit(softCommit);
getWebgraphConnector().commit(softCommit); if (this.writeWebgraph) getWebgraphConnector().commit(softCommit);
} }
public Date getLoadDate(final String urlHash) { public Date getLoadDate(final String urlHash) {
@ -393,6 +404,7 @@ public final class Fulltext {
} }
public void putEdges(final Collection<SolrInputDocument> edges) throws IOException { public void putEdges(final Collection<SolrInputDocument> edges) throws IOException {
if (!this.writeToWebgraph()) return;
if (edges == null || edges.size() == 0) return; if (edges == null || edges.size() == 0) return;
try { try {
this.getWebgraphConnector().add(edges); this.getWebgraphConnector().add(edges);
@ -443,7 +455,7 @@ public final class Fulltext {
// delete in solr // delete in solr
try {Fulltext.this.getDefaultConnector().deleteByQuery(collection1Query);} catch (final IOException e) {} try {Fulltext.this.getDefaultConnector().deleteByQuery(collection1Query);} catch (final IOException e) {}
try {Fulltext.this.getWebgraphConnector().deleteByQuery(webgraphQuery);} catch (final IOException e) {} if (this.writeWebgraph) try {Fulltext.this.getWebgraphConnector().deleteByQuery(webgraphQuery);} catch (final IOException e) {}
// delete in old metadata structure // delete in old metadata structure
if (Fulltext.this.urlIndexFile != null) { if (Fulltext.this.urlIndexFile != null) {
@ -493,7 +505,7 @@ public final class Fulltext {
// delete in solr // delete in solr
try {Fulltext.this.getDefaultConnector().deleteByQuery(collectionQuery);} catch (final IOException e) {} try {Fulltext.this.getDefaultConnector().deleteByQuery(collectionQuery);} catch (final IOException e) {}
try {Fulltext.this.getWebgraphConnector().deleteByQuery(webgraphQuery);} catch (final IOException e) {} if (this.writeWebgraph) try {Fulltext.this.getWebgraphConnector().deleteByQuery(webgraphQuery);} catch (final IOException e) {}
// finally remove the line with statistics // finally remove the line with statistics
if (Fulltext.this.statsDump != null) { if (Fulltext.this.statsDump != null) {
@ -546,7 +558,7 @@ public final class Fulltext {
if (deleteIDs == null || deleteIDs.size() == 0) return; if (deleteIDs == null || deleteIDs.size() == 0) return;
try { try {
this.getDefaultConnector().deleteByIds(deleteIDs); this.getDefaultConnector().deleteByIds(deleteIDs);
this.getWebgraphConnector().deleteByIds(deleteIDs); if (this.writeWebgraph) this.getWebgraphConnector().deleteByIds(deleteIDs);
} catch (final Throwable e) { } catch (final Throwable e) {
ConcurrentLog.logException(e); ConcurrentLog.logException(e);
} }
@ -563,7 +575,7 @@ public final class Fulltext {
try { try {
String id = ASCII.String(urlHash); String id = ASCII.String(urlHash);
this.getDefaultConnector().deleteById(id); this.getDefaultConnector().deleteById(id);
this.getWebgraphConnector().deleteById(id); if (this.writeWebgraph) this.getWebgraphConnector().deleteById(id);
} catch (final Throwable e) { } catch (final Throwable e) {
ConcurrentLog.logException(e); ConcurrentLog.logException(e);
} }
@ -709,7 +721,7 @@ public final class Fulltext {
public void optimize(final int size) { public void optimize(final int size) {
if (size < 1) return; if (size < 1) return;
getDefaultConnector().optimize(size); getDefaultConnector().optimize(size);
getWebgraphConnector().optimize(size); if (this.writeWebgraph) getWebgraphConnector().optimize(size);
} }
/** /**

17
source/net/yacy/search/index/Segment.java
Unescape View File

@ -74,7 +74,6 @@ import net.yacy.kelondro.rwi.ReferenceFactory;
import net.yacy.kelondro.util.Bitfield; import net.yacy.kelondro.util.Bitfield;
import net.yacy.kelondro.util.ByteBuffer; import net.yacy.kelondro.util.ByteBuffer;
import net.yacy.kelondro.util.ISO639; import net.yacy.kelondro.util.ISO639;
import net.yacy.kelondro.util.MemoryControl;
import net.yacy.kelondro.workflow.WorkflowProcessor; import net.yacy.kelondro.workflow.WorkflowProcessor;
import net.yacy.repository.LoaderDispatcher; import net.yacy.repository.LoaderDispatcher;
import net.yacy.search.StorageQueueEntry; import net.yacy.search.StorageQueueEntry;
@ -111,12 +110,11 @@ public class Segment {
public static final ReferenceFactory<CitationReference> citationReferenceFactory = new CitationReferenceFactory(); public static final ReferenceFactory<CitationReference> citationReferenceFactory = new CitationReferenceFactory();
public static final ByteOrder wordOrder = Base64Order.enhancedCoder; public static final ByteOrder wordOrder = Base64Order.enhancedCoder;
private final ConcurrentLog log; private final ConcurrentLog log;
private final File segmentPath; private final File segmentPath;
protected final Fulltext fulltext; protected final Fulltext fulltext;
protected IndexCell<WordReference> termIndex; protected IndexCell<WordReference> termIndex;
protected IndexCell<CitationReference> urlCitationIndex; protected IndexCell<CitationReference> urlCitationIndex;
protected boolean writeWebgraph;
private WorkflowProcessor<StorageQueueEntry> indexingPutDocumentProcessor; private WorkflowProcessor<StorageQueueEntry> indexingPutDocumentProcessor;
/** /**
@ -135,7 +133,6 @@ public class Segment {
this.fulltext = new Fulltext(segmentPath, collectionConfiguration, webgraphConfiguration); this.fulltext = new Fulltext(segmentPath, collectionConfiguration, webgraphConfiguration);
this.termIndex = null; this.termIndex = null;
this.urlCitationIndex = null; this.urlCitationIndex = null;
this.writeWebgraph = false;
this.indexingPutDocumentProcessor = new WorkflowProcessor<StorageQueueEntry>( this.indexingPutDocumentProcessor = new WorkflowProcessor<StorageQueueEntry>(
"putDocument", "putDocument",
@ -148,14 +145,6 @@ public class Segment {
1); 1);
} }
public void writeWebgraph(boolean check) {
this.writeWebgraph = check;
}
public boolean writeToWebgraph() {
return this.writeWebgraph;
}
public boolean connectedRWI() { public boolean connectedRWI() {
return this.termIndex != null; return this.termIndex != null;
} }
@ -317,7 +306,7 @@ public class Segment {
this.externalHosts = new RowHandleSet(6, Base64Order.enhancedCoder, 0); this.externalHosts = new RowHandleSet(6, Base64Order.enhancedCoder, 0);
this.internalIDs = new RowHandleSet(12, Base64Order.enhancedCoder, 0); this.internalIDs = new RowHandleSet(12, Base64Order.enhancedCoder, 0);
this.externalIDs = new RowHandleSet(12, Base64Order.enhancedCoder, 0); this.externalIDs = new RowHandleSet(12, Base64Order.enhancedCoder, 0);
if (writeToWebgraph()) { if (Segment.this.fulltext.writeToWebgraph()) {
// reqd the references from the webgraph // reqd the references from the webgraph
SolrConnector webgraph = Segment.this.fulltext.getWebgraphConnector(); SolrConnector webgraph = Segment.this.fulltext.getWebgraphConnector();
webgraph.commit(true); webgraph.commit(true);
@ -670,7 +659,7 @@ public class Segment {
// STORE TO SOLR // STORE TO SOLR
String error = null; String error = null;
this.putDocumentInQueue(vector); this.putDocumentInQueue(vector);
if (this.writeWebgraph) { if (this.fulltext.writeToWebgraph()) {
tryloop: for (int i = 0; i < 20; i++) { tryloop: for (int i = 0; i < 20; i++) {
try { try {
error = null; error = null;

1
source/net/yacy/search/schema/WebgraphConfiguration.java
Unescape View File

@ -279,6 +279,7 @@ public class WebgraphConfiguration extends SchemaConfiguration implements Serial
public int postprocessing(Segment segment) { public int postprocessing(Segment segment) {
if (!this.contains(WebgraphSchema.process_sxt)) return 0; if (!this.contains(WebgraphSchema.process_sxt)) return 0;
if (!segment.connectedCitation()) return 0; if (!segment.connectedCitation()) return 0;
if (!segment.fulltext().writeToWebgraph()) return 0;
SolrConnector connector = segment.fulltext().getWebgraphConnector(); SolrConnector connector = segment.fulltext().getWebgraphConnector();
// that means we must search for those entries. // that means we must search for those entries.
connector.commit(true); // make sure that we have latest information that can be found connector.commit(true); // make sure that we have latest information that can be found

Write Preview
Loading…
Cancel
Save