diff --git a/htroot/Supporter.java b/htroot/Supporter.java index 3e9083232..9409dcf43 100644 --- a/htroot/Supporter.java +++ b/htroot/Supporter.java @@ -33,8 +33,8 @@ import java.util.Iterator; import net.yacy.cora.date.GenericFormatter; import net.yacy.cora.document.UTF8; import net.yacy.cora.protocol.RequestHeader; -import net.yacy.cora.storage.DynamicScore; -import net.yacy.cora.storage.ScoreCluster; +import net.yacy.cora.storage.ConcurrentScoreMap; +import net.yacy.cora.storage.ScoreMap; import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.index.Row; import net.yacy.kelondro.index.Row.Entry; @@ -107,7 +107,7 @@ public class Supporter { accumulateVotes(sb, negativeHashes, positiveHashes, yacyNewsPool.INCOMING_DB); //accumulateVotes(negativeHashes, positiveHashes, yacyNewsPool.OUTGOING_DB); //accumulateVotes(negativeHashes, positiveHashes, yacyNewsPool.PUBLISHED_DB); - final DynamicScore ranking = new ScoreCluster(); // score cluster for url hashes + final ScoreMap ranking = new ConcurrentScoreMap(); // score cluster for url hashes final Row rowdef = new Row("String url-255, String title-120, String description-120, String refid-" + (GenericFormatter.PATTERN_SHORT_SECOND.length() + 12), NaturalOrder.naturalOrder); final HashMap Supporter = new HashMap(); // a mapping from an url hash to a kelondroRow.Entry with display properties accumulateSupporter(sb, Supporter, ranking, rowdef, negativeHashes, positiveHashes, yacyNewsPool.INCOMING_DB); @@ -200,7 +200,7 @@ public class Supporter { private static void accumulateSupporter( final Switchboard sb, - final HashMap Supporter, final DynamicScore ranking, final Row rowdef, + final HashMap Supporter, final ScoreMap ranking, final Row rowdef, final HashMap negativeHashes, final HashMap positiveHashes, final int dbtype) { final int maxCount = Math.min(1000, sb.peers.newsPool.size(dbtype)); yacyNewsDB.Record record; diff --git a/htroot/Surftips.java b/htroot/Surftips.java index 5af109d3a..f41c3b63b 100644 --- a/htroot/Surftips.java +++ b/htroot/Surftips.java @@ -33,8 +33,8 @@ import java.util.Iterator; import net.yacy.cora.date.GenericFormatter; import net.yacy.cora.document.UTF8; import net.yacy.cora.protocol.RequestHeader; -import net.yacy.cora.storage.DynamicScore; -import net.yacy.cora.storage.ScoreCluster; +import net.yacy.cora.storage.ConcurrentScoreMap; +import net.yacy.cora.storage.ScoreMap; import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.index.Row; import net.yacy.kelondro.index.Row.Entry; @@ -115,7 +115,7 @@ public class Surftips { accumulateVotes(sb , negativeHashes, positiveHashes, yacyNewsPool.INCOMING_DB); //accumulateVotes(negativeHashes, positiveHashes, yacyNewsPool.OUTGOING_DB); //accumulateVotes(negativeHashes, positiveHashes, yacyNewsPool.PUBLISHED_DB); - final DynamicScore ranking = new ScoreCluster(); // score cluster for url hashes + final ScoreMap ranking = new ConcurrentScoreMap(); // score cluster for url hashes final Row rowdef = new Row("String url-255, String title-120, String description-120, String refid-" + (GenericFormatter.PATTERN_SHORT_SECOND.length() + 12), NaturalOrder.naturalOrder); final HashMap surftips = new HashMap(); // a mapping from an url hash to a kelondroRow.Entry with display properties accumulateSurftips(sb, surftips, ranking, rowdef, negativeHashes, positiveHashes, yacyNewsPool.INCOMING_DB); @@ -209,7 +209,7 @@ public class Surftips { private static void accumulateSurftips( final Switchboard sb, - final HashMap surftips, final DynamicScore ranking, final Row rowdef, + final HashMap surftips, final ScoreMap ranking, final Row rowdef, final HashMap negativeHashes, final HashMap positiveHashes, final int dbtype) { final int maxCount = Math.min(1000, sb.peers.newsPool.size(dbtype)); yacyNewsDB.Record record; diff --git a/htroot/yacy/search.java b/htroot/yacy/search.java index 752de7452..a7c5e8bd3 100644 --- a/htroot/yacy/search.java +++ b/htroot/yacy/search.java @@ -40,7 +40,7 @@ import net.yacy.cora.document.UTF8; import net.yacy.cora.protocol.Domains; import net.yacy.cora.protocol.HeaderFramework; import net.yacy.cora.protocol.RequestHeader; -import net.yacy.cora.storage.StaticScore; +import net.yacy.cora.storage.ScoreMap; import net.yacy.cora.storage.WeakPriorityBlockingQueue; import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.data.word.WordReference; @@ -351,7 +351,7 @@ public final class search { // prepare reference hints final long timer = System.currentTimeMillis(); - StaticScore topicNavigator = theSearch.getTopicNavigator(5); + ScoreMap topicNavigator = theSearch.getTopicNavigator(5); final StringBuilder refstr = new StringBuilder(6000); Iterator navigatorIterator = topicNavigator.keys(false); int i = 0; diff --git a/htroot/yacysearchtrailer.java b/htroot/yacysearchtrailer.java index 7ede8adb7..42f2ccb05 100644 --- a/htroot/yacysearchtrailer.java +++ b/htroot/yacysearchtrailer.java @@ -27,7 +27,7 @@ import java.util.Iterator; import net.yacy.cora.protocol.RequestHeader; -import net.yacy.cora.storage.StaticScore; +import net.yacy.cora.storage.ScoreMap; import net.yacy.document.LibraryProvider; import net.yacy.kelondro.util.EventTracker; @@ -59,7 +59,7 @@ public class yacysearchtrailer { // compose search navigation // namespace navigators - StaticScore namespaceNavigator = theSearch.getNamespaceNavigator(); + ScoreMap namespaceNavigator = theSearch.getNamespaceNavigator(); String name; int count; Iterator navigatorIterator; @@ -86,7 +86,7 @@ public class yacysearchtrailer { } // host navigators - StaticScore hostNavigator = theSearch.getHostNavigator(); + ScoreMap hostNavigator = theSearch.getHostNavigator(); if (hostNavigator == null || hostNavigator.isEmpty()) { prop.put("nav-domains", 0); } else { @@ -110,7 +110,7 @@ public class yacysearchtrailer { } // author navigators - StaticScore authorNavigator = theSearch.getAuthorNavigator(); + ScoreMap authorNavigator = theSearch.getAuthorNavigator(); if (authorNavigator == null || authorNavigator.isEmpty()) { prop.put("nav-authors", 0); } else { @@ -136,7 +136,7 @@ public class yacysearchtrailer { } // topics navigator - StaticScore topicNavigator = theSearch.getTopicNavigator(MAX_TOPWORDS); + ScoreMap topicNavigator = theSearch.getTopicNavigator(MAX_TOPWORDS); if (topicNavigator == null || topicNavigator.isEmpty()) { prop.put("nav-topics", "0"); } else { diff --git a/source/de/anomic/crawler/ResultURLs.java b/source/de/anomic/crawler/ResultURLs.java index 8bf7ef714..46e11bbc4 100644 --- a/source/de/anomic/crawler/ResultURLs.java +++ b/source/de/anomic/crawler/ResultURLs.java @@ -33,8 +33,8 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import net.yacy.cora.document.UTF8; -import net.yacy.cora.storage.ScoreCluster; -import net.yacy.cora.storage.DynamicScore; +import net.yacy.cora.storage.ClusteredScoreMap; +import net.yacy.cora.storage.ScoreMap; import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.data.meta.URIMetadataRow; import net.yacy.kelondro.logging.Log; @@ -77,12 +77,12 @@ public final class ResultURLs { } private final static Map> resultStacks = new ConcurrentHashMap>(); // a mapping from urlHash to Entries - private final static Map> resultDomains = new ConcurrentHashMap>(); + private final static Map> resultDomains = new ConcurrentHashMap>(); static { for (EventOrigin origin: EventOrigin.values()) { resultStacks.put(origin, new LinkedHashMap()); - resultDomains.put(origin, new ScoreCluster()); + resultDomains.put(origin, new ClusteredScoreMap()); } } @@ -112,7 +112,7 @@ public final class ResultURLs { return; } try { - final DynamicScore domains = getDomains(stackType); + final ScoreMap domains = getDomains(stackType); if (domains != null) { domains.inc(e.metadata().url().getHost()); } @@ -129,7 +129,7 @@ public final class ResultURLs { } public static int getDomainListSize(final EventOrigin stack) { - final DynamicScore domains = getDomains(stack); + final ScoreMap domains = getDomains(stack); if (domains == null) return 0; return domains.size(); } @@ -185,7 +185,7 @@ public final class ResultURLs { private static Map getStack(final EventOrigin stack) { return resultStacks.get(stack); } - private static DynamicScore getDomains(final EventOrigin stack) { + private static ScoreMap getDomains(final EventOrigin stack) { return resultDomains.get(stack); } @@ -196,7 +196,7 @@ public final class ResultURLs { public static void clearStack(final EventOrigin stack) { final Map resultStack = getStack(stack); if (resultStack != null) resultStack.clear(); - final DynamicScore resultDomains = getDomains(stack); + final ScoreMap resultDomains = getDomains(stack); if (resultDomains != null) { // we do not clear this completely, just remove most of the less important entries resultDomains.shrinkToMaxSize(100); diff --git a/source/de/anomic/data/DidYouMean.java b/source/de/anomic/data/DidYouMean.java index c51b8b0d9..de39a2136 100644 --- a/source/de/anomic/data/DidYouMean.java +++ b/source/de/anomic/data/DidYouMean.java @@ -7,8 +7,8 @@ import java.util.SortedSet; import java.util.TreeSet; import java.util.concurrent.LinkedBlockingQueue; -import net.yacy.cora.storage.DynamicScore; -import net.yacy.cora.storage.ScoreCluster; +import net.yacy.cora.storage.ReversibleScoreMap; +import net.yacy.cora.storage.ClusteredScoreMap; import net.yacy.document.LibraryProvider; import net.yacy.kelondro.data.word.Word; import net.yacy.kelondro.data.word.WordReference; @@ -135,7 +135,7 @@ public class DidYouMean { return preSorted; } - final DynamicScore scored = new ScoreCluster(); + final ReversibleScoreMap scored = new ClusteredScoreMap(); for (final String s: preSorted) { if (System.currentTimeMillis() > timelimit) break; if (scored.size() >= 2 * preSortSelection) break; diff --git a/source/de/anomic/search/MetadataRepository.java b/source/de/anomic/search/MetadataRepository.java index 1ce97788d..0f59c66da 100644 --- a/source/de/anomic/search/MetadataRepository.java +++ b/source/de/anomic/search/MetadataRepository.java @@ -43,8 +43,8 @@ import de.anomic.crawler.CrawlStacker; import net.yacy.cora.document.MultiProtocolURI; import net.yacy.cora.document.UTF8; import net.yacy.cora.protocol.http.HTTPClient; -import net.yacy.cora.storage.DynamicScore; -import net.yacy.cora.storage.ScoreCluster; +import net.yacy.cora.storage.ConcurrentScoreMap; +import net.yacy.cora.storage.ScoreMap; import net.yacy.document.parser.html.CharacterCoding; import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.data.meta.URIMetadataRow; @@ -602,7 +602,7 @@ public final class MetadataRepository implements Iterable { Map map = domainSampleCollector(); // order elements by size - DynamicScore s = new ScoreCluster(); + ScoreMap s = new ConcurrentScoreMap(); for (Map.Entry e: map.entrySet()) { s.inc(e.getValue().urlhash, e.getValue().count); } diff --git a/source/de/anomic/search/RankingProcess.java b/source/de/anomic/search/RankingProcess.java index 3c4449708..367c83425 100644 --- a/source/de/anomic/search/RankingProcess.java +++ b/source/de/anomic/search/RankingProcess.java @@ -43,9 +43,9 @@ import java.util.concurrent.TimeUnit; import net.yacy.cora.document.MultiProtocolURI; import net.yacy.cora.document.UTF8; import net.yacy.cora.protocol.Scanner; -import net.yacy.cora.storage.DynamicScore; -import net.yacy.cora.storage.ScoreCluster; -import net.yacy.cora.storage.StaticScore; +import net.yacy.cora.storage.ConcurrentScoreMap; +import net.yacy.cora.storage.ClusteredScoreMap; +import net.yacy.cora.storage.ScoreMap; import net.yacy.cora.storage.WeakPriorityBlockingQueue; import net.yacy.cora.storage.WeakPriorityBlockingQueue.ReverseElement; import net.yacy.document.Condenser; @@ -84,11 +84,11 @@ public final class RankingProcess extends Thread { private final ConcurrentHashMap> doubleDomCache; // key = domhash (6 bytes); value = like stack //private final HandleSet handover; // key = urlhash; used for double-check of urls that had been handed over to search process - private final DynamicScore ref; // reference score computation for the commonSense heuristic - private final DynamicScore hostNavigator; // a counter for the appearance of the host hash + private final ScoreMap ref; // reference score computation for the commonSense heuristic + private final ScoreMap hostNavigator; // a counter for the appearance of the host hash private final Map hostResolver; // a mapping from a host hash (6 bytes) to the full url hash of one of these urls that have the host hash - private final DynamicScore authorNavigator; - private final DynamicScore namespaceNavigator; + private final ScoreMap authorNavigator; + private final ScoreMap namespaceNavigator; private final ReferenceOrder order; private final long startTime; @@ -112,11 +112,11 @@ public final class RankingProcess extends Thread { //this.misses = new HandleSet(URIMetadataRow.rowdef.primaryKeyLength, URIMetadataRow.rowdef.objectOrder, 0); this.flagcount = new int[32]; for (int i = 0; i < 32; i++) {this.flagcount[i] = 0;} - this.hostNavigator = new ScoreCluster(); + this.hostNavigator = new ConcurrentScoreMap(); this.hostResolver = new ConcurrentHashMap(); - this.authorNavigator = new ScoreCluster(); - this.namespaceNavigator = new ScoreCluster(); - this.ref = new ScoreCluster(); + this.authorNavigator = new ConcurrentScoreMap(); + this.namespaceNavigator = new ConcurrentScoreMap(); + this.ref = new ConcurrentScoreMap(); this.feeders = 1; this.startTime = System.currentTimeMillis(); } @@ -551,14 +551,14 @@ public final class RankingProcess extends Thread { return this.misses.size(); } - public StaticScore getNamespaceNavigator() { - if (!this.query.navigators.equals("all") && this.query.navigators.indexOf("namespace") < 0) return new ScoreCluster(); + public ScoreMap getNamespaceNavigator() { + if (!this.query.navigators.equals("all") && this.query.navigators.indexOf("namespace") < 0) return new ClusteredScoreMap(); if (this.namespaceNavigator.size() < 2) this.namespaceNavigator.clear(); // navigators with one entry are not useful return this.namespaceNavigator; } - public StaticScore getHostNavigator() { - ScoreCluster result = new ScoreCluster(); + public ScoreMap getHostNavigator() { + ScoreMap result = new ConcurrentScoreMap(); if (!this.query.navigators.equals("all") && this.query.navigators.indexOf("hosts") < 0) return result; final Iterator domhashs = this.hostNavigator.keys(false); @@ -586,10 +586,10 @@ public final class RankingProcess extends Thread { } }; - public StaticScore getTopicNavigator(int count) { + public ScoreMap getTopicNavigator(int count) { // create a list of words that had been computed by statistics over all // words that appeared in the url or the description of all urls - final ScoreCluster result = new ScoreCluster(); + final ScoreMap result = new ConcurrentScoreMap(); if (!this.query.navigators.equals("all") && this.query.navigators.indexOf("topics") < 0) return result; if (this.ref.size() < 2) this.ref.clear(); // navigators with one entry are not useful final Map counts = new HashMap(); @@ -643,10 +643,10 @@ public final class RankingProcess extends Thread { addTopic(descrcomps); } - public StaticScore getAuthorNavigator() { + public ScoreMap getAuthorNavigator() { // create a list of words that had been computed by statistics over all // words that appeared in the url or the description of all urls - if (!this.query.navigators.equals("all") && this.query.navigators.indexOf("authors") < 0) return new ScoreCluster(); + if (!this.query.navigators.equals("all") && this.query.navigators.indexOf("authors") < 0) return new ConcurrentScoreMap(); if (this.authorNavigator.size() < 2) this.authorNavigator.clear(); // navigators with one entry are not useful return this.authorNavigator; } diff --git a/source/de/anomic/search/ReferenceOrder.java b/source/de/anomic/search/ReferenceOrder.java index 2bb71ac88..233633697 100644 --- a/source/de/anomic/search/ReferenceOrder.java +++ b/source/de/anomic/search/ReferenceOrder.java @@ -34,8 +34,8 @@ import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.Semaphore; import net.yacy.cora.document.UTF8; -import net.yacy.cora.storage.DynamicScore; -import net.yacy.cora.storage.ScoreCluster; +import net.yacy.cora.storage.ReversibleScoreMap; +import net.yacy.cora.storage.ClusteredScoreMap; import net.yacy.document.Condenser; import net.yacy.document.LargeNumberCache; import net.yacy.kelondro.data.meta.DigestURI; @@ -54,7 +54,7 @@ public class ReferenceOrder { private int maxdomcount; private WordReferenceVars min, max; - private final DynamicScore doms; // collected for "authority" heuristic + private final ReversibleScoreMap doms; // collected for "authority" heuristic private final RankingProfile ranking; private final byte[] language; @@ -62,7 +62,7 @@ public class ReferenceOrder { this.min = null; this.max = null; this.ranking = profile; - this.doms = new ScoreCluster(); + this.doms = new ClusteredScoreMap(); this.maxdomcount = 0; this.language = language; } diff --git a/source/de/anomic/search/ResultFetcher.java b/source/de/anomic/search/ResultFetcher.java index 4915d9ae4..bac9b1ee0 100644 --- a/source/de/anomic/search/ResultFetcher.java +++ b/source/de/anomic/search/ResultFetcher.java @@ -32,7 +32,7 @@ import java.util.List; import net.yacy.cora.document.MultiProtocolURI; -import net.yacy.cora.storage.StaticScore; +import net.yacy.cora.storage.ScoreMap; import net.yacy.cora.storage.WeakPriorityBlockingQueue; import net.yacy.cora.storage.WeakPriorityBlockingQueue.ReverseElement; import net.yacy.document.Condenser; @@ -202,7 +202,7 @@ public class ResultFetcher { public long postRanking( final ResultEntry rentry, - final StaticScore topwords) { + final ScoreMap topwords) { long r = 0; diff --git a/source/de/anomic/search/SearchEvent.java b/source/de/anomic/search/SearchEvent.java index d94978818..e4e0b7bd3 100644 --- a/source/de/anomic/search/SearchEvent.java +++ b/source/de/anomic/search/SearchEvent.java @@ -36,7 +36,7 @@ import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; import net.yacy.cora.document.UTF8; -import net.yacy.cora.storage.StaticScore; +import net.yacy.cora.storage.ScoreMap; import net.yacy.document.LargeNumberCache; import net.yacy.kelondro.data.word.WordReference; import net.yacy.kelondro.logging.Log; @@ -311,20 +311,20 @@ public final class SearchEvent { return this.rankingProcess; } - public StaticScore getNamespaceNavigator() { + public ScoreMap getNamespaceNavigator() { return this.rankingProcess.getNamespaceNavigator(); } - public StaticScore getHostNavigator() { + public ScoreMap getHostNavigator() { return this.rankingProcess.getHostNavigator(); } - public StaticScore getTopicNavigator(int count) { + public ScoreMap getTopicNavigator(int count) { // returns a set of words that are computed as toplist return this.rankingProcess.getTopicNavigator(count); } - public StaticScore getAuthorNavigator() { + public ScoreMap getAuthorNavigator() { // returns a list of authors so far seen on result set return this.rankingProcess.getAuthorNavigator(); } diff --git a/source/de/anomic/yacy/dht/PeerSelection.java b/source/de/anomic/yacy/dht/PeerSelection.java index 7d8faaca3..819e6cd89 100755 --- a/source/de/anomic/yacy/dht/PeerSelection.java +++ b/source/de/anomic/yacy/dht/PeerSelection.java @@ -35,8 +35,8 @@ import java.util.SortedMap; import net.yacy.cora.date.AbstractFormatter; import net.yacy.cora.document.UTF8; -import net.yacy.cora.storage.DynamicScore; -import net.yacy.cora.storage.ScoreCluster; +import net.yacy.cora.storage.ConcurrentScoreMap; +import net.yacy.cora.storage.ScoreMap; import net.yacy.kelondro.data.word.Word; import net.yacy.kelondro.index.HandleSet; import net.yacy.kelondro.index.RowSpaceExceededException; @@ -410,7 +410,7 @@ public class PeerSelection { if (count > seedDB.sizeConnected()) count = seedDB.sizeConnected(); // fill a score object - final DynamicScore seedScore = new ScoreCluster(); + final ScoreMap seedScore = new ConcurrentScoreMap(); yacySeed ys; long absage; final Iterator s = seedDB.seedsConnected(true, false, null, (float) 0.0); diff --git a/source/net/yacy/cora/services/SearchHub.java b/source/net/yacy/cora/services/SearchHub.java index cbb9e636c..2eb8a0373 100644 --- a/source/net/yacy/cora/services/SearchHub.java +++ b/source/net/yacy/cora/services/SearchHub.java @@ -32,6 +32,7 @@ import java.util.concurrent.ConcurrentHashMap; import net.yacy.cora.document.RSSMessage; import net.yacy.cora.protocol.http.HTTPClient; +import net.yacy.cora.storage.ConcurrentScoreMap; import net.yacy.cora.storage.ScoreMap; public class SearchHub { @@ -100,7 +101,7 @@ public class SearchHub { * @return a score map of urls */ public ScoreMap getResults() { - ScoreMap scores = new ScoreMap(); + ScoreMap scores = new ConcurrentScoreMap(); int m = threads.size(); for (Map.Entry> entry: this.result.entrySet()) { int a = 0; diff --git a/source/net/yacy/cora/storage/ScoreCluster.java b/source/net/yacy/cora/storage/ClusteredScoreMap.java similarity index 91% rename from source/net/yacy/cora/storage/ScoreCluster.java rename to source/net/yacy/cora/storage/ClusteredScoreMap.java index 159a7f36a..df55283cc 100644 --- a/source/net/yacy/cora/storage/ScoreCluster.java +++ b/source/net/yacy/cora/storage/ClusteredScoreMap.java @@ -26,7 +26,6 @@ package net.yacy.cora.storage; import java.text.ParseException; import java.text.SimpleDateFormat; -import java.util.Comparator; import java.util.Iterator; import java.util.Locale; import java.util.Map; @@ -36,24 +35,15 @@ import java.util.TreeMap; import net.yacy.cora.document.UTF8; -public final class ScoreCluster implements DynamicScore { +public final class ClusteredScoreMap implements ReversibleScoreMap { protected final Map map; // a mapping from a reference to the cluster key protected final TreeMap pam; // a mapping from the cluster key to the reference private long gcount; private int encnt; - public ScoreCluster() { - this(null); - } - - public ScoreCluster(Comparator comparator) { - if (comparator == null) { - //map = new HashMap(); - map = new TreeMap(comparator); - } else { - map = new TreeMap(comparator); - } + public ClusteredScoreMap() { + map = new TreeMap(); pam = new TreeMap(); gcount = 0; encnt = 0; @@ -402,7 +392,7 @@ public final class ScoreCluster implements DynamicScore { } System.out.println("Test for Score: start"); - final ScoreCluster s = new ScoreCluster(); + final ClusteredScoreMap s = new ClusteredScoreMap(); long c = 0; // create cluster @@ -410,12 +400,10 @@ public final class ScoreCluster implements DynamicScore { final Random random = new Random(1234); int r; final int count = 20; - final int[] mem = new int[count]; - for (int x = 0; x < 100; x++) { + for (int x = 0; x < 100000; x++) { for (int i = 0; i < count; i++) { - r = random.nextInt(); - mem[i] = r; + r = Math.abs(random.nextInt(100)); s.inc("score#" + r, r); c += r; } @@ -423,14 +411,15 @@ public final class ScoreCluster implements DynamicScore { // delete some int p; for (int i = 0; i < (count / 2); i++) { - p = (int) (random.nextFloat() * count); - if (s.containsKey("score#" + mem[p])) { - System.out.println("delete score#" + mem[p]); - s.delete("score#" + mem[p]); - c -= mem[p]; + p = Math.abs(random.nextInt(1000)); + if (s.containsKey("score#" + p)) { + //System.out.println("delete score#" + s.get("score#" + p)); + c -= s.delete("score#" + p); } } } + + System.out.println("finished create. time = " + (System.currentTimeMillis() - time)); System.out.println("result:"); Iterator i = s.keys(true); @@ -438,7 +427,6 @@ public final class ScoreCluster implements DynamicScore { i = s.keys(false); while (i.hasNext()) System.out.println("down: " + i.next()); - System.out.println("finished create. time = " + (System.currentTimeMillis() - time)); System.out.println("total=" + s.totalCount() + ", elements=" + s.size() + ", redundant count=" + c); } } diff --git a/source/net/yacy/cora/storage/ConcurrentScoreMap.java b/source/net/yacy/cora/storage/ConcurrentScoreMap.java new file mode 100644 index 000000000..8ec89bb97 --- /dev/null +++ b/source/net/yacy/cora/storage/ConcurrentScoreMap.java @@ -0,0 +1,205 @@ +/** + * ConcurrentScoreMap + * Copyright 2010 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany + * First released 13.03.2011 at http://yacy.net + * + * $LastChangedDate: 2011-03-08 02:51:51 +0100 (Di, 08 Mrz 2011) $ + * $LastChangedRevision: 7567 $ + * $LastChangedBy: low012 $ + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program in the file lgpl21.txt + * If not, see . + */ + +package net.yacy.cora.storage; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; + + +public class ConcurrentScoreMap implements ScoreMap { + + protected final ConcurrentHashMap map; // a mapping from a reference to the cluster key + private long gcount; + + public ConcurrentScoreMap() { + map = new ConcurrentHashMap(); + gcount = 0; + } + + public synchronized void clear() { + map.clear(); + gcount = 0; + } + + /** + * shrink the cluster to a demanded size + * @param maxsize + */ + public void shrinkToMaxSize(int maxsize) { + if (this.map.size() <= maxsize) return; + int minScore = getMinScore(); + while (this.map.size() > maxsize) { + minScore++; + shrinkToMinScore(minScore); + } + } + + /** + * shrink the cluster in such a way that the smallest score is equal or greater than a given minScore + * @param minScore + */ + public void shrinkToMinScore(int minScore) { + Iterator> i = this.map.entrySet().iterator(); + Map.Entry entry; + while (i.hasNext()) { + entry = i.next(); + if (entry.getValue().intValue() < minScore) i.remove(); + } + } + + public long totalCount() { + return gcount; + } + + public int size() { + return map.size(); + } + + public boolean isEmpty() { + return map.isEmpty(); + } + + public void inc(final E obj) { + if (obj == null) return; + + // use atomic operations + this.map.putIfAbsent(obj, new AtomicLong(0)); + this.map.get(obj).incrementAndGet(); + + // increase overall counter + gcount++; + } + + public void dec(final E obj) { + if (obj == null) return; + + // use atomic operations + this.map.putIfAbsent(obj, new AtomicLong(0)); + this.map.get(obj).decrementAndGet(); + + // increase overall counter + gcount--; + } + + public void set(final E obj, final int newScore) { + if (obj == null) return; + + // use atomic operations + this.map.putIfAbsent(obj, new AtomicLong(0)); + this.map.get(obj).set(newScore); + + // increase overall counter + gcount += newScore; + } + + public void inc(final E obj, final int incrementScore) { + if (obj == null) return; + + // use atomic operations + this.map.putIfAbsent(obj, new AtomicLong(0)); + this.map.get(obj).addAndGet(incrementScore); + + // increase overall counter + gcount += incrementScore; + } + + public void dec(final E obj, final int decrementScore) { + inc(obj, -decrementScore); + } + + public int delete(final E obj) { + // deletes entry and returns previous score + if (obj == null) return 0; + final AtomicLong score = this.map.remove(obj); + if (score == null) return 0; + + // decrease overall counter + gcount -= score.intValue(); + return score.intValue(); + } + + public boolean containsKey(final E obj) { + return this.map.containsKey(obj); + } + + public int get(final E obj) { + if (obj == null) return 0; + final AtomicLong score = this.map.get(obj); + if (score == null) return 0; + return score.intValue(); + } + + private int getMinScore() { + if (this.map.isEmpty()) return -1; + int minScore = Integer.MAX_VALUE; + for (Map.Entry entry: this.map.entrySet()) if (entry.getValue().intValue() < minScore) { + minScore = entry.getValue().intValue(); + } + return minScore; + } + + @Override + public String toString() { + return map.toString(); + } + + public Iterator keys(boolean up) { + // re-organize entries + TreeMap> m = new TreeMap>(); + Set s; + IntScore is; + for (Map.Entry entry: this.map.entrySet()) { + is = new IntScore(entry.getValue().intValue()); + s = m.get(is); + if (s == null) { + s = new HashSet(); + s.add(entry.getKey()); + m.put(is, s); + } else { + s.add(entry.getKey()); + } + } + + // flatten result + List l = new ArrayList(m.size()); + for (Set f: m.values()) { + for (E e: f) l.add(e); + } + if (up) return l.iterator(); + + // optionally reverse list + List r = new ArrayList(l.size()); + for (int i = l.size() - 1; i >= 0; i--) r.add(l.get(i)); + return r.iterator(); + } + +} diff --git a/source/net/yacy/cora/storage/IntScore.java b/source/net/yacy/cora/storage/IntScore.java index f58e11998..43fe9251e 100644 --- a/source/net/yacy/cora/storage/IntScore.java +++ b/source/net/yacy/cora/storage/IntScore.java @@ -34,9 +34,6 @@ import java.util.Comparator; * rewriting the new key value to a map. */ public class IntScore implements Comparable, Comparator { - - public static IntScore ZERO = new IntScore(0); - public static IntScore ONE = new IntScore(1); private int value; diff --git a/source/net/yacy/cora/storage/OrderedScoreMap.java b/source/net/yacy/cora/storage/OrderedScoreMap.java new file mode 100644 index 000000000..61ccce30b --- /dev/null +++ b/source/net/yacy/cora/storage/OrderedScoreMap.java @@ -0,0 +1,280 @@ +/** + * ScoreMap + * Copyright 2010 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany + * First released 14.10.2010 at http://yacy.net + * + * $LastChangedDate$ + * $LastChangedRevision$ + * $LastChangedBy$ + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program in the file lgpl21.txt + * If not, see . + */ + +package net.yacy.cora.storage; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.SortedMap; +import java.util.TreeMap; +import java.util.TreeSet; + + +public class OrderedScoreMap implements ScoreMap { + + protected final Map map; // a mapping from a reference to the cluster key + private long gcount; + + public OrderedScoreMap(Comparator comparator) { + if (comparator == null) { + map = new HashMap(); + } else { + map = new TreeMap(comparator); + } + gcount = 0; + } + + public synchronized void clear() { + map.clear(); + gcount = 0; + } + + /** + * shrink the cluster to a demanded size + * @param maxsize + */ + public void shrinkToMaxSize(int maxsize) { + if (this.map.size() <= maxsize) return; + int minScore = getMinScore(); + while (this.map.size() > maxsize) { + minScore++; + shrinkToMinScore(minScore); + } + } + + /** + * shrink the cluster in such a way that the smallest score is equal or greater than a given minScore + * @param minScore + */ + public void shrinkToMinScore(int minScore) { + synchronized (this) { + Iterator> i = this.map.entrySet().iterator(); + Map.Entry entry; + while (i.hasNext()) { + entry = i.next(); + if (entry.getValue().intValue() < minScore) i.remove(); + } + } + } + + public synchronized long totalCount() { + return gcount; + } + + public synchronized int size() { + return map.size(); + } + + public synchronized boolean isEmpty() { + return map.isEmpty(); + } + + public void inc(final E obj) { + if (obj == null) return; + synchronized (this) { + IntScore score = this.map.get(obj); + if (score == null) { + this.map.put(obj, new IntScore(1)); + } else { + score.inc(); + } + } + // increase overall counter + gcount++; + } + + public void dec(final E obj) { + if (obj == null) return; + synchronized (this) { + IntScore score = this.map.get(obj); + if (score == null) { + this.map.put(obj, IntScore.valueOf(-1)); + } else { + score.dec(); + } + } + // increase overall counter + gcount--; + } + + public void set(final E obj, final int newScore) { + if (obj == null) return; + synchronized (this) { + IntScore score = this.map.get(obj); + if (score == null) { + this.map.put(obj, new IntScore(1)); + } else { + gcount -= score.intValue(); + score.set(newScore); + } + } + // increase overall counter + gcount += newScore; + } + + public void inc(final E obj, final int incrementScore) { + if (obj == null) return; + synchronized (this) { + IntScore score = this.map.get(obj); + if (score == null) { + this.map.put(obj, IntScore.valueOf(incrementScore)); + } else { + score.inc(incrementScore); + } + } + // increase overall counter + gcount += incrementScore; + } + + public void dec(final E obj, final int incrementScore) { + inc(obj, -incrementScore); + } + + public int delete(final E obj) { + // deletes entry and returns previous score + if (obj == null) return 0; + final IntScore score; + synchronized (this) { + score = map.remove(obj); + if (score == null) return 0; + } + + // decrease overall counter + gcount -= score.intValue(); + return score.intValue(); + } + + public synchronized boolean containsKey(final E obj) { + return map.containsKey(obj); + } + + public int get(final E obj) { + if (obj == null) return 0; + final IntScore score; + synchronized (this) { + score = map.get(obj); + } + if (score == null) return 0; + return score.intValue(); + } + + public SortedMap tailMap(E obj) { + if (this.map instanceof TreeMap) { + return ((TreeMap) this.map).tailMap(obj); + } + throw new UnsupportedOperationException("map must have comparator"); + } + + private int getMinScore() { + if (map.isEmpty()) return -1; + int minScore = Integer.MAX_VALUE; + synchronized (this) { + for (Map.Entry entry: this.map.entrySet()) if (entry.getValue().intValue() < minScore) { + minScore = entry.getValue().intValue(); + } + } + return minScore; + } + + /* + public int getMaxScore() { + if (map.isEmpty()) return -1; + int maxScore = Integer.MIN_VALUE; + synchronized (this) { + for (Map.Entry entry: this.map.entrySet()) if (entry.getValue().intValue() > maxScore) { + maxScore = entry.getValue().intValue(); + } + } + return maxScore; + } + + public E getMaxKey() { + if (map.isEmpty()) return null; + E maxObject = null; + int maxScore = Integer.MIN_VALUE; + synchronized (this) { + for (Map.Entry entry: this.map.entrySet()) if (entry.getValue().intValue() > maxScore) { + maxScore = entry.getValue().intValue(); + maxObject = entry.getKey(); + } + } + return maxObject; + } + + public E getMinKey() { + if (map.isEmpty()) return null; + E minObject = null; + int minScore = Integer.MAX_VALUE; + synchronized (this) { + for (Map.Entry entry: this.map.entrySet()) if (entry.getValue().intValue() < minScore) { + minScore = entry.getValue().intValue(); + minObject = entry.getKey(); + } + } + return minObject; + } + */ + + @Override + public String toString() { + return map.toString(); + } + + public Iterator keys(boolean up) { + synchronized (this) { + // re-organize entries + TreeMap> m = new TreeMap>(); + Set s; + for (Map.Entry entry: this.map.entrySet()) { + s = m.get(entry.getValue()); + if (s == null) { + s = this.map instanceof TreeMap ? new TreeSet(((TreeMap) this.map).comparator()) : new HashSet(); + s.add(entry.getKey()); + m.put(entry.getValue(), s); + } else { + s.add(entry.getKey()); + } + } + + // flatten result + List l = new ArrayList(this.map.size()); + for (Set f: m.values()) { + for (E e: f) l.add(e); + } + if (up) return l.iterator(); + + // optionally reverse list + List r = new ArrayList(l.size()); + for (int i = l.size() - 1; i >= 0; i--) r.add(l.get(i)); + return r.iterator(); + } + } + +} diff --git a/source/net/yacy/cora/storage/DynamicScore.java b/source/net/yacy/cora/storage/ReversibleScoreMap.java similarity index 77% rename from source/net/yacy/cora/storage/DynamicScore.java rename to source/net/yacy/cora/storage/ReversibleScoreMap.java index f8485b9a3..c84f4f655 100644 --- a/source/net/yacy/cora/storage/DynamicScore.java +++ b/source/net/yacy/cora/storage/ReversibleScoreMap.java @@ -1,5 +1,5 @@ /** - * DynamicScore + * ReversibleScoreMap * Copyright 2010 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany * First released 14.10.2010 at http://yacy.net * @@ -24,12 +24,10 @@ package net.yacy.cora.storage; -public interface DynamicScore extends StaticScore { - - public void inc(final E obj); - public void inc(final E obj, final int incrementScore); - - public void dec(final E obj); - public void dec(final E obj, final int incrementScore); - +public interface ReversibleScoreMap extends ScoreMap { + + public int getMaxScore(); + public int getMinScore(); + public E getMaxKey(); + public E getMinKey(); } diff --git a/source/net/yacy/cora/storage/ScoreMap.java b/source/net/yacy/cora/storage/ScoreMap.java index daaef0dad..9a87f1c8c 100644 --- a/source/net/yacy/cora/storage/ScoreMap.java +++ b/source/net/yacy/cora/storage/ScoreMap.java @@ -24,260 +24,47 @@ package net.yacy.cora.storage; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.HashMap; -import java.util.HashSet; import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.SortedMap; -import java.util.TreeMap; -import java.util.TreeSet; +public interface ScoreMap { -public class ScoreMap implements DynamicScore { - - protected final Map map; // a mapping from a reference to the cluster key - private long gcount; - - public ScoreMap() { - this(null); - } - - public ScoreMap(Comparator comparator) { - if (comparator == null) { - map = new HashMap(); - } else { - map = new TreeMap(comparator); - } - gcount = 0; - } - - public synchronized void clear() { - map.clear(); - gcount = 0; - } + public void clear(); /** * shrink the cluster to a demanded size * @param maxsize */ - public void shrinkToMaxSize(int maxsize) { - if (this.map.size() <= maxsize) return; - int minScore = getMinScore(); - while (this.map.size() > maxsize) { - minScore++; - shrinkToMinScore(minScore); - } - } + public void shrinkToMaxSize(int maxsize); /** * shrink the cluster in such a way that the smallest score is equal or greater than a given minScore * @param minScore */ - public void shrinkToMinScore(int minScore) { - synchronized (this) { - Iterator> i = this.map.entrySet().iterator(); - Map.Entry entry; - while (i.hasNext()) { - entry = i.next(); - if (entry.getValue().intValue() < minScore) i.remove(); - } - } - } - - public synchronized long totalCount() { - return gcount; - } - - public synchronized int size() { - return map.size(); - } - - public synchronized boolean isEmpty() { - return map.isEmpty(); - } - - public void inc(final E obj) { - if (obj == null) return; - synchronized (this) { - IntScore score = this.map.get(obj); - if (score == null) { - this.map.put(obj, IntScore.ONE); - } else { - score.inc(); - } - } - // increase overall counter - gcount++; - } + public void shrinkToMinScore(int minScore); - public void dec(final E obj) { - if (obj == null) return; - synchronized (this) { - IntScore score = this.map.get(obj); - if (score == null) { - this.map.put(obj, IntScore.valueOf(-1)); - } else { - score.dec(); - } - } - // increase overall counter - gcount--; - } + public long totalCount(); - public void set(final E obj, final int newScore) { - if (obj == null) return; - synchronized (this) { - IntScore score = this.map.get(obj); - if (score == null) { - this.map.put(obj, IntScore.ONE); - } else { - gcount -= score.intValue(); - score.set(newScore); - } - } - // increase overall counter - gcount += newScore; - } + public int size(); - public void inc(final E obj, final int incrementScore) { - if (obj == null) return; - synchronized (this) { - IntScore score = this.map.get(obj); - if (score == null) { - this.map.put(obj, IntScore.valueOf(incrementScore)); - } else { - score.inc(incrementScore); - } - } - // increase overall counter - gcount += incrementScore; - } + public boolean isEmpty(); - public void dec(final E obj, final int incrementScore) { - inc(obj, -incrementScore); - } + public void set(final E obj, final int newScore); - public int delete(final E obj) { - // deletes entry and returns previous score - if (obj == null) return 0; - final IntScore score; - synchronized (this) { - score = map.remove(obj); - if (score == null) return 0; - } + public int delete(final E obj); - // decrease overall counter - gcount -= score.intValue(); - return score.intValue(); - } - - public synchronized boolean containsKey(final E obj) { - return map.containsKey(obj); - } - - public int get(final E obj) { - if (obj == null) return 0; - final IntScore score; - synchronized (this) { - score = map.get(obj); - } - if (score == null) return 0; - return score.intValue(); - } + public boolean containsKey(final E obj); - public SortedMap tailMap(E obj) { - if (this.map instanceof TreeMap) { - return ((TreeMap) this.map).tailMap(obj); - } - throw new UnsupportedOperationException("map must have comparator"); - } + public int get(final E obj); + @Override + public String toString(); - public int getMaxScore() { - if (map.isEmpty()) return -1; - int maxScore = Integer.MIN_VALUE; - synchronized (this) { - for (Map.Entry entry: this.map.entrySet()) if (entry.getValue().intValue() > maxScore) { - maxScore = entry.getValue().intValue(); - } - } - return maxScore; - } - - public int getMinScore() { - if (map.isEmpty()) return -1; - int minScore = Integer.MAX_VALUE; - synchronized (this) { - for (Map.Entry entry: this.map.entrySet()) if (entry.getValue().intValue() < minScore) { - minScore = entry.getValue().intValue(); - } - } - return minScore; - } + public Iterator keys(final boolean up); - public E getMaxKey() { - if (map.isEmpty()) return null; - E maxObject = null; - int maxScore = Integer.MIN_VALUE; - synchronized (this) { - for (Map.Entry entry: this.map.entrySet()) if (entry.getValue().intValue() > maxScore) { - maxScore = entry.getValue().intValue(); - maxObject = entry.getKey(); - } - } - return maxObject; - } + public void inc(final E obj); + public void inc(final E obj, final int incrementScore); - public E getMinKey() { - if (map.isEmpty()) return null; - E minObject = null; - int minScore = Integer.MAX_VALUE; - synchronized (this) { - for (Map.Entry entry: this.map.entrySet()) if (entry.getValue().intValue() < minScore) { - minScore = entry.getValue().intValue(); - minObject = entry.getKey(); - } - } - return minObject; - } - - @Override - public String toString() { - return map.toString(); - } + public void dec(final E obj); + public void dec(final E obj, final int incrementScore); - public Iterator keys(boolean up) { - synchronized (this) { - // re-organize entries - TreeMap> m = new TreeMap>(); - Set s; - for (Map.Entry entry: this.map.entrySet()) { - s = m.get(entry.getValue()); - if (s == null) { - s = this.map instanceof TreeMap ? new TreeSet(((TreeMap) this.map).comparator()) : new HashSet(); - s.add(entry.getKey()); - m.put(entry.getValue(), s); - } else { - s.add(entry.getKey()); - } - } - - // flatten result - List l = new ArrayList(this.map.size()); - for (Set f: m.values()) { - for (E e: f) l.add(e); - } - if (up) return l.iterator(); - - // optionally reverse list - List r = new ArrayList(l.size()); - for (int i = l.size() - 1; i >= 0; i--) r.add(r.get(i)); - return r.iterator(); - } - } - } diff --git a/source/net/yacy/cora/storage/StaticScore.java b/source/net/yacy/cora/storage/StaticScore.java deleted file mode 100644 index 0bfe8fc7e..000000000 --- a/source/net/yacy/cora/storage/StaticScore.java +++ /dev/null @@ -1,72 +0,0 @@ -/** - * StaticScore - * Copyright 2010 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany - * First released 14.10.2010 at http://yacy.net - * - * $LastChangedDate$ - * $LastChangedRevision$ - * $LastChangedBy$ - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program in the file lgpl21.txt - * If not, see . - */ - -package net.yacy.cora.storage; - -import java.util.Iterator; - -public interface StaticScore { - - public void clear(); - - /** - * shrink the cluster to a demanded size - * @param maxsize - */ - public void shrinkToMaxSize(int maxsize); - - /** - * shrink the cluster in such a way that the smallest score is equal or greater than a given minScore - * @param minScore - */ - public void shrinkToMinScore(int minScore); - - public long totalCount(); - - public int size(); - - public boolean isEmpty(); - - public void set(final E obj, final int newScore); - - public int delete(final E obj); - - public boolean containsKey(final E obj); - - public int get(final E obj); - - public int getMaxScore(); - - public int getMinScore(); - - public E getMaxKey(); - - public E getMinKey(); - - @Override - public String toString(); - - public Iterator keys(final boolean up); - -} diff --git a/source/net/yacy/document/WordCache.java b/source/net/yacy/document/WordCache.java index 563c9ee11..2eec00aef 100644 --- a/source/net/yacy/document/WordCache.java +++ b/source/net/yacy/document/WordCache.java @@ -36,7 +36,7 @@ import java.util.TreeSet; import java.util.zip.GZIPInputStream; import net.yacy.cora.storage.IntScore; -import net.yacy.cora.storage.ScoreMap; +import net.yacy.cora.storage.OrderedScoreMap; import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.util.MemoryControl; @@ -49,7 +49,7 @@ public class WordCache { // common word cache private static final int commonWordsMaxSize = 100000; // maximum size of common word cache private static final int commonWordsMinLength = 5; // words must have that length at minimum - private ScoreMap commonWords = new ScoreMap(String.CASE_INSENSITIVE_ORDER); + private OrderedScoreMap commonWords = new OrderedScoreMap(String.CASE_INSENSITIVE_ORDER); // dictionaries private final File dictionaryPath; diff --git a/source/net/yacy/kelondro/blob/MapDataMining.java b/source/net/yacy/kelondro/blob/MapDataMining.java index 91bb0c09b..f5b4188c5 100644 --- a/source/net/yacy/kelondro/blob/MapDataMining.java +++ b/source/net/yacy/kelondro/blob/MapDataMining.java @@ -35,8 +35,9 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import net.yacy.cora.document.UTF8; -import net.yacy.cora.storage.ScoreCluster; -import net.yacy.cora.storage.StaticScore; +import net.yacy.cora.storage.ConcurrentScoreMap; +import net.yacy.cora.storage.ClusteredScoreMap; +import net.yacy.cora.storage.ScoreMap; import net.yacy.kelondro.index.RowSpaceExceededException; import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.order.ByteOrder; @@ -50,7 +51,7 @@ public class MapDataMining extends MapHeap { private final static Float FLOAT0 = Float.valueOf(0.0f); private final String[] sortfields, longaccfields, floataccfields; - private Map> sortClusterMap; // a String-kelondroMScoreCluster - relation + private Map> sortClusterMap; // a String-kelondroMScoreCluster - relation private Map accLong; // to store accumulations of Long cells private Map accFloat; // to store accumulations of Float cells @@ -71,12 +72,12 @@ public class MapDataMining extends MapHeap { this.longaccfields = longaccfields; this.floataccfields = floataccfields; - ScoreCluster[] cluster = null; + ScoreMap[] cluster = null; if (sortfields == null) sortClusterMap = null; else { - sortClusterMap = new ConcurrentHashMap>(); - cluster = new ScoreCluster[sortfields.length]; + sortClusterMap = new ConcurrentHashMap>(); + cluster = new ScoreMap[sortfields.length]; for (int i = 0; i < sortfields.length; i++) { - cluster[i] = new ScoreCluster(); + cluster[i] = new ConcurrentScoreMap(); } } @@ -121,7 +122,7 @@ public class MapDataMining extends MapHeap { if (sortfields != null && cluster != null) for (int i = 0; i < sortfields.length; i++) { cell = map.get(sortfields[i]); - if (cell != null) cluster[i].set(UTF8.String(mapnameb), ScoreCluster.object2score(cell)); + if (cell != null) cluster[i].set(UTF8.String(mapnameb), ClusteredScoreMap.object2score(cell)); } if (longaccfields != null && longaccumulator != null) for (int i = 0; i < longaccfields.length; i++) { @@ -156,9 +157,9 @@ public class MapDataMining extends MapHeap { public synchronized void clear() { super.clear(); if (sortfields == null) sortClusterMap = null; else { - sortClusterMap = new HashMap>(); + sortClusterMap = new HashMap>(); for (int i = 0; i < sortfields.length; i++) { - sortClusterMap.put(sortfields[i], new ScoreCluster()); + sortClusterMap.put(sortfields[i], new ConcurrentScoreMap()); } } @@ -242,12 +243,12 @@ public class MapDataMining extends MapHeap { private void updateSortCluster(final String key, final Map map) { Object cell; - StaticScore cluster; + ScoreMap cluster; for (int i = 0; i < sortfields.length; i++) { cell = map.get(sortfields[i]); if (cell != null) { cluster = sortClusterMap.get(sortfields[i]); - cluster.set(key, ScoreCluster.object2score(cell)); + cluster.set(key, ClusteredScoreMap.object2score(cell)); sortClusterMap.put(sortfields[i], cluster); } } @@ -280,7 +281,7 @@ public class MapDataMining extends MapHeap { private void deleteSortCluster(final String key) { if (key == null) return; - StaticScore cluster; + ScoreMap cluster; for (int i = 0; i < sortfields.length; i++) { cluster = sortClusterMap.get(sortfields[i]); cluster.delete(key); @@ -291,7 +292,7 @@ public class MapDataMining extends MapHeap { public synchronized Iterator keys(final boolean up, /* sorted by */ final String field) { // sorted iteration using the sortClusters if (sortClusterMap == null) return null; - final StaticScore cluster = sortClusterMap.get(field); + final ScoreMap cluster = sortClusterMap.get(field); if (cluster == null) return null; // sort field does not exist //System.out.println("DEBUG: cluster for field " + field + ": " + cluster.toString()); return new string2bytearrayIterator(cluster.keys(up)); diff --git a/source/net/yacy/yacy.java b/source/net/yacy/yacy.java index e35c443ef..578847bf9 100644 --- a/source/net/yacy/yacy.java +++ b/source/net/yacy/yacy.java @@ -51,8 +51,8 @@ import net.yacy.cora.document.MultiProtocolURI; import net.yacy.cora.document.UTF8; import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.protocol.http.HTTPClient; -import net.yacy.cora.storage.DynamicScore; -import net.yacy.cora.storage.ScoreCluster; +import net.yacy.cora.storage.OrderedScoreMap; +import net.yacy.cora.storage.ScoreMap; import net.yacy.gui.YaCyApp; import net.yacy.gui.framework.Browser; import net.yacy.kelondro.blob.MapDataMining; @@ -601,7 +601,7 @@ public final class yacy { final enumerateFiles ef = new enumerateFiles(new File(dbRoot, "WORDS"), true, false, true, true); File f; byte[] h; - final DynamicScore hs = new ScoreCluster(Base64Order.standardCoder); + final ScoreMap hs = new OrderedScoreMap(Base64Order.standardCoder); while (ef.hasMoreElements()) { f = ef.nextElement(); h = f.getName().substring(0, Word.commonHashLength).getBytes();