diff --git a/source/net/yacy/cora/federate/solr/connector/AbstractSolrConnector.java b/source/net/yacy/cora/federate/solr/connector/AbstractSolrConnector.java index 9dd21f295..abaa9ffe8 100644 --- a/source/net/yacy/cora/federate/solr/connector/AbstractSolrConnector.java +++ b/source/net/yacy/cora/federate/solr/connector/AbstractSolrConnector.java @@ -85,7 +85,7 @@ public abstract class AbstractSolrConnector implements SolrConnector { } protected final static int pagesize = 100; - protected static Metadata getMetadata(final Object doc) { + protected static LoadTimeURL getLoadTimeURL(final Object doc) { if (doc == null) return null; Object d = null; String url = null; @@ -110,7 +110,7 @@ public abstract class AbstractSolrConnector implements SolrConnector { long date = -1; if (d instanceof Long) date = ((Long) d).longValue(); if (d instanceof Date) date = ((Date) d).getTime(); - return new Metadata(url, date); + return new LoadTimeURL(url, date); } /** @@ -306,7 +306,7 @@ public abstract class AbstractSolrConnector implements SolrConnector { * @throws IOException */ @Override - public Metadata getMetadata(String id) throws IOException { + public LoadTimeURL getLoadTimeURL(String id) throws IOException { // construct raw query final SolrQuery params = new SolrQuery(); //params.setQuery(CollectionSchema.id.getSolrFieldName() + ":\"" + id + "\""); @@ -326,7 +326,7 @@ public abstract class AbstractSolrConnector implements SolrConnector { final SolrDocumentList sdl = getDocumentListByParams(params); if (sdl == null || sdl.getNumFound() <= 0) return null; SolrDocument doc = sdl.iterator().next(); - Metadata md = getMetadata(doc); + LoadTimeURL md = getLoadTimeURL(doc); return md; } diff --git a/source/net/yacy/cora/federate/solr/connector/ConcurrentUpdateSolrConnector.java b/source/net/yacy/cora/federate/solr/connector/ConcurrentUpdateSolrConnector.java index c8e2e62a7..dd6468635 100644 --- a/source/net/yacy/cora/federate/solr/connector/ConcurrentUpdateSolrConnector.java +++ b/source/net/yacy/cora/federate/solr/connector/ConcurrentUpdateSolrConnector.java @@ -72,7 +72,7 @@ public class ConcurrentUpdateSolrConnector implements SolrConnector { } private SolrConnector connector; - private ARC metadataCache; + private ARC metadataCache; private final ARH missCache; private final LinkedHashMap docBuffer; private CommitHandler processHandler; @@ -119,7 +119,7 @@ public class ConcurrentUpdateSolrConnector implements SolrConnector { } // move documents to metadata cache for (Map.Entry entry: this.docBuffer.entrySet()) { - updateCache(entry.getKey(), AbstractSolrConnector.getMetadata(entry.getValue())); + updateCache(entry.getKey(), AbstractSolrConnector.getLoadTimeURL(entry.getValue())); } this.docBuffer.clear(); } @@ -137,7 +137,7 @@ public class ConcurrentUpdateSolrConnector implements SolrConnector { this.missCache.clear(); } - private void updateCache(final String id, final Metadata md) { + private void updateCache(final String id, final LoadTimeURL md) { if (id == null) return; if (MemoryControl.shortStatus()) { this.metadataCache.clear(); @@ -243,9 +243,9 @@ public class ConcurrentUpdateSolrConnector implements SolrConnector { } @Override - public Metadata getMetadata(String id) throws IOException { + public LoadTimeURL getLoadTimeURL(String id) throws IOException { if (this.missCache.contains(id)) return null; - Metadata md = this.metadataCache.get(id); + LoadTimeURL md = this.metadataCache.get(id); if (md != null) { //System.out.println("*** metadata cache hit; metadataCache.size() = " + metadataCache.size()); //Thread.dumpStack(); @@ -255,9 +255,9 @@ public class ConcurrentUpdateSolrConnector implements SolrConnector { if (doc != null) { //System.out.println("*** docBuffer cache hit; docBuffer.size() = " + docBuffer.size()); //Thread.dumpStack(); - return AbstractSolrConnector.getMetadata(doc); + return AbstractSolrConnector.getLoadTimeURL(doc); } - md = this.connector.getMetadata(id); + md = this.connector.getLoadTimeURL(id); if (md == null) {this.missCache.add(id); return null;} updateCache(id, md); return md; @@ -272,7 +272,7 @@ public class ConcurrentUpdateSolrConnector implements SolrConnector { synchronized (this.docBuffer) {this.docBuffer.put(id, solrdoc);} } else { this.connector.add(solrdoc); - updateCache(id, AbstractSolrConnector.getMetadata(solrdoc)); + updateCache(id, AbstractSolrConnector.getLoadTimeURL(solrdoc)); } if (MemoryControl.shortStatus() || this.docBuffer.size() > this.updateCapacity) { commitDocBuffer(); @@ -312,7 +312,7 @@ public class ConcurrentUpdateSolrConnector implements SolrConnector { this.missCache.add(id); this.metadataCache.remove(id); } else { - updateCache(id, AbstractSolrConnector.getMetadata(solrdoc)); + updateCache(id, AbstractSolrConnector.getLoadTimeURL(solrdoc)); } return solrdoc; } @@ -329,7 +329,7 @@ public class ConcurrentUpdateSolrConnector implements SolrConnector { SolrDocumentList sdl = this.connector.getDocumentListByParams(params); for (SolrDocument doc: sdl) { String id = (String) doc.getFieldValue(CollectionSchema.id.getSolrFieldName()); - updateCache(id, AbstractSolrConnector.getMetadata(doc)); + updateCache(id, AbstractSolrConnector.getLoadTimeURL(doc)); } return sdl; } diff --git a/source/net/yacy/cora/federate/solr/connector/EmbeddedSolrConnector.java b/source/net/yacy/cora/federate/solr/connector/EmbeddedSolrConnector.java index 9d36801d4..661d498f3 100644 --- a/source/net/yacy/cora/federate/solr/connector/EmbeddedSolrConnector.java +++ b/source/net/yacy/cora/federate/solr/connector/EmbeddedSolrConnector.java @@ -405,7 +405,7 @@ public class EmbeddedSolrConnector extends SolrServerConnector implements SolrCo * @throws IOException */ @Override - public Metadata getMetadata(String id) { + public LoadTimeURL getLoadTimeURL(String id) { int responseCount = 0; DocListSearcher docListSearcher = null; try { @@ -417,7 +417,7 @@ public class EmbeddedSolrConnector extends SolrServerConnector implements SolrCo //for (int i = 0; i < responseCount; i++) { Document doc = searcher.doc(iterator.nextDoc(), AbstractSolrConnector.SOLR_ID_and_LOAD_DATE_FIELDS); if (doc == null) return null; - return AbstractSolrConnector.getMetadata(doc); + return AbstractSolrConnector.getLoadTimeURL(doc); //} } catch (Throwable e) { ConcurrentLog.logException(e); diff --git a/source/net/yacy/cora/federate/solr/connector/MirrorSolrConnector.java b/source/net/yacy/cora/federate/solr/connector/MirrorSolrConnector.java index 1820e4248..4220f50db 100644 --- a/source/net/yacy/cora/federate/solr/connector/MirrorSolrConnector.java +++ b/source/net/yacy/cora/federate/solr/connector/MirrorSolrConnector.java @@ -413,17 +413,17 @@ public class MirrorSolrConnector extends AbstractSolrConnector implements SolrCo } @Override - public Metadata getMetadata(String id) throws IOException { - if (this.solr0 != null && this.solr1 == null) return this.solr0.getMetadata(id); - if (this.solr0 == null && this.solr1 != null) return this.solr1.getMetadata(id); + public LoadTimeURL getLoadTimeURL(String id) throws IOException { + if (this.solr0 != null && this.solr1 == null) return this.solr0.getLoadTimeURL(id); + if (this.solr0 == null && this.solr1 != null) return this.solr1.getLoadTimeURL(id); if (this.solr0 == null && this.solr1 == null) return null; - Metadata md0 = this.solr0.getMetadata(id); - Metadata md1 = this.solr1.getMetadata(id); + LoadTimeURL md0 = this.solr0.getLoadTimeURL(id); + LoadTimeURL md1 = this.solr1.getLoadTimeURL(id); if (md0 == null) return md1; if (md1 == null) return md0; long date = Math.max(md0.date, md1.date); assert md0.url.equals(md1.url); - return new Metadata(md0.url, date); + return new LoadTimeURL(md0.url, date); } @Override diff --git a/source/net/yacy/cora/federate/solr/connector/SolrConnector.java b/source/net/yacy/cora/federate/solr/connector/SolrConnector.java index ccd774e93..040ddf53d 100644 --- a/source/net/yacy/cora/federate/solr/connector/SolrConnector.java +++ b/source/net/yacy/cora/federate/solr/connector/SolrConnector.java @@ -36,10 +36,10 @@ import org.apache.solr.common.params.ModifiableSolrParams; public interface SolrConnector extends Iterable /* Iterable of document IDs */ { - public static class Metadata { + public static class LoadTimeURL { public long date; public String url; - public Metadata(final String url, final long date) { + public LoadTimeURL(final String url, final long date) { this.url = url; this.date = date; } @@ -119,10 +119,10 @@ public interface SolrConnector extends Iterable /* Iterable of document /** * check if a given document, identified by url hash as document id exists * @param id the url hash and document id - * @return the metadata (url and load data) if any entry in solr exists, null otherwise + * @return the load time metadata (url and load data) if any entry in solr exists, null otherwise * @throws IOException */ - public Metadata getMetadata(final String id) throws IOException; + public LoadTimeURL getLoadTimeURL(final String id) throws IOException; /** * add a solr input document diff --git a/source/net/yacy/crawler/CrawlStacker.java b/source/net/yacy/crawler/CrawlStacker.java index c81a0dd07..1b71f1600 100644 --- a/source/net/yacy/crawler/CrawlStacker.java +++ b/source/net/yacy/crawler/CrawlStacker.java @@ -43,7 +43,7 @@ import net.yacy.cora.document.id.AnchorURL; import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.document.id.MultiProtocolURL; import net.yacy.cora.federate.solr.FailCategory; -import net.yacy.cora.federate.solr.connector.SolrConnector.Metadata; +import net.yacy.cora.federate.solr.connector.SolrConnector.LoadTimeURL; import net.yacy.cora.order.Base64Order; import net.yacy.cora.protocol.Domains; import net.yacy.cora.protocol.ftp.FTPClient; @@ -384,9 +384,9 @@ public final class CrawlStacker { return "double in: " + dbocc.name(); } String urlhash = ASCII.String(url.hash()); - Metadata oldEntry = null; + LoadTimeURL oldEntry = null; try { - oldEntry = this.indexSegment.fulltext().getDefaultConnector().getMetadata(urlhash); + oldEntry = this.indexSegment.fulltext().getDefaultConnector().getLoadTimeURL(urlhash); } catch (IOException e) { ConcurrentLog.logException(e); } diff --git a/source/net/yacy/search/Switchboard.java b/source/net/yacy/search/Switchboard.java index b4afb9065..2be7fbe4d 100644 --- a/source/net/yacy/search/Switchboard.java +++ b/source/net/yacy/search/Switchboard.java @@ -99,7 +99,7 @@ import net.yacy.cora.federate.solr.FailCategory; import net.yacy.cora.federate.solr.Ranking; import net.yacy.cora.federate.solr.SchemaConfiguration; import net.yacy.cora.federate.solr.connector.ShardSelection; -import net.yacy.cora.federate.solr.connector.SolrConnector.Metadata; +import net.yacy.cora.federate.solr.connector.SolrConnector.LoadTimeURL; import net.yacy.cora.federate.solr.instance.RemoteInstance; import net.yacy.cora.federate.yacy.CacheStrategy; import net.yacy.cora.order.Base64Order; @@ -1625,7 +1625,7 @@ public final class Switchboard extends serverSwitch { HarvestProcess hp = this.crawlQueues.exists(ASCII.getBytes(hash)); if (hp != null) return hp; try { - Metadata md = this.index.fulltext().getDefaultConnector().getMetadata(hash); + LoadTimeURL md = this.index.fulltext().getDefaultConnector().getLoadTimeURL(hash); if (md == null) return null; return HarvestProcess.LOADED; // todo: can also be in error } catch (IOException e) { diff --git a/source/net/yacy/search/index/Fulltext.java b/source/net/yacy/search/index/Fulltext.java index fc50684ab..cebdceeb0 100644 --- a/source/net/yacy/search/index/Fulltext.java +++ b/source/net/yacy/search/index/Fulltext.java @@ -477,7 +477,7 @@ public final class Fulltext { if (urlHash == null || this.getDefaultConnector() == null) return null; try { - SolrConnector.Metadata md = this.getDefaultConnector().getMetadata(urlHash); + SolrConnector.LoadTimeURL md = this.getDefaultConnector().getLoadTimeURL(urlHash); if (md == null) return null; return new DigestURL(md.url, ASCII.getBytes(urlHash)); } catch (final IOException e) { @@ -493,7 +493,7 @@ public final class Fulltext { public long getLoadTime(final String urlHash) { if (urlHash == null) return -1l; try { - SolrConnector.Metadata md = this.getDefaultConnector().getMetadata(urlHash); + SolrConnector.LoadTimeURL md = this.getDefaultConnector().getLoadTimeURL(urlHash); if (md == null) return -1l; return md.date; } catch (final Throwable e) { diff --git a/source/net/yacy/search/schema/CollectionConfiguration.java b/source/net/yacy/search/schema/CollectionConfiguration.java index c97e3fdd6..1bcdf097f 100644 --- a/source/net/yacy/search/schema/CollectionConfiguration.java +++ b/source/net/yacy/search/schema/CollectionConfiguration.java @@ -59,7 +59,7 @@ import net.yacy.cora.federate.solr.SchemaConfiguration; import net.yacy.cora.federate.solr.SchemaDeclaration; import net.yacy.cora.federate.solr.connector.AbstractSolrConnector; import net.yacy.cora.federate.solr.connector.SolrConnector; -import net.yacy.cora.federate.solr.connector.SolrConnector.Metadata; +import net.yacy.cora.federate.solr.connector.SolrConnector.LoadTimeURL; import net.yacy.cora.order.Base64Order; import net.yacy.cora.protocol.Domains; import net.yacy.cora.protocol.HeaderFramework; @@ -1583,7 +1583,7 @@ public class CollectionConfiguration extends SchemaConfiguration implements Seri for (Map.Entry entry: rm.entrySet()) { if (entry == null || entry.getValue() == null) continue; try { - Metadata md = connector.getMetadata(ASCII.String(entry.getKey())); + LoadTimeURL md = connector.getLoadTimeURL(ASCII.String(entry.getKey())); ConcurrentLog.info("CollectionConfiguration", "CR for " + md.url); ConcurrentLog.info("CollectionConfiguration", ">> " + entry.getValue().toString()); } catch (final IOException e) {