From 10d3627c90a01e8c3dc37318c32d24e0e6e779ce Mon Sep 17 00:00:00 2001 From: orbiter Date: Tue, 11 Oct 2005 07:06:33 +0000 Subject: [PATCH] changed word cache flush scheduling and removed possible locks git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@910 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- htroot/NetworkPicture.java | 2 +- htroot/PerformanceQueues_p.java | 1 + .../de/anomic/plasma/plasmaSearchEvent.java | 5 +++- .../de/anomic/plasma/plasmaSwitchboard.java | 10 ++++++-- .../anomic/plasma/plasmaWordIndexCache.java | 25 +++++++++++++------ 5 files changed, 31 insertions(+), 12 deletions(-) diff --git a/htroot/NetworkPicture.java b/htroot/NetworkPicture.java index 5d413aa4f..c4299ffa1 100644 --- a/htroot/NetworkPicture.java +++ b/htroot/NetworkPicture.java @@ -66,7 +66,7 @@ public class NetworkPicture { public static BufferedImage respond(httpHeader header, serverObjects post, serverSwitch env) { int width = 640; - int height = 420; + int height = 480; if (post != null) { width = post.getInt("width", 640); diff --git a/htroot/PerformanceQueues_p.java b/htroot/PerformanceQueues_p.java index 9dec459be..64d38e86f 100644 --- a/htroot/PerformanceQueues_p.java +++ b/htroot/PerformanceQueues_p.java @@ -173,6 +173,7 @@ public class PerformanceQueues_p { if ((post != null) && (post.containsKey("cacheSizeSubmit"))) { int wordCacheMaxLow = Integer.parseInt((String) post.get("wordCacheMaxLow", "8000")); int wordCacheMaxHigh = Integer.parseInt((String) post.get("wordCacheMaxHigh", "10000")); + if (wordCacheMaxLow > wordCacheMaxHigh) wordCacheMaxLow = wordCacheMaxHigh; switchboard.setConfig("wordCacheMaxLow", Integer.toString(wordCacheMaxLow)); switchboard.setConfig("wordCacheMaxHigh", Integer.toString(wordCacheMaxHigh)); switchboard.wordIndex.setMaxWords(wordCacheMaxLow, wordCacheMaxHigh); diff --git a/source/de/anomic/plasma/plasmaSearchEvent.java b/source/de/anomic/plasma/plasmaSearchEvent.java index eff1493ee..13e556cf3 100644 --- a/source/de/anomic/plasma/plasmaSearchEvent.java +++ b/source/de/anomic/plasma/plasmaSearchEvent.java @@ -42,11 +42,14 @@ package de.anomic.plasma; +import java.util.Iterator; public final class plasmaSearchEvent { + private plasmaSearchQuery query; - public plasmaSearchEvent() { + public plasmaSearchEvent(plasmaSearchQuery query) { + this.query = query; } } diff --git a/source/de/anomic/plasma/plasmaSwitchboard.java b/source/de/anomic/plasma/plasmaSwitchboard.java index aed15375f..06d651a83 100644 --- a/source/de/anomic/plasma/plasmaSwitchboard.java +++ b/source/de/anomic/plasma/plasmaSwitchboard.java @@ -306,8 +306,8 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser urlPool = new plasmaURLPool(plasmaPath, ramLURL, ramNURL, ramEURL); wordIndex = new plasmaWordIndex(plasmaPath, ramRWI, log); - int wordCacheMaxLow = Integer.parseInt((String) getConfig("wordCacheMaxLow", "8000")); - int wordCacheMaxHigh = Integer.parseInt((String) getConfig("wordCacheMaxHigh", "10000")); + int wordCacheMaxLow = (int) getConfigLong("wordCacheMaxLow", 8000); + int wordCacheMaxHigh = (int) getConfigLong("wordCacheMaxHigh", 10000); wordIndex.setMaxWords(wordCacheMaxLow, wordCacheMaxHigh); searchManager = new plasmaSearch(urlPool.loadedURL, wordIndex); @@ -698,6 +698,12 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser return false; } + if (wordIndex.wordCacheRAMSize() + 1000 > (int) getConfigLong("wordCacheMaxLow", 8000)) { + log.logFine("deQueue: word index ram cache too full (" + ((int) getConfigLong("wordCacheMaxLow", 8000) - wordIndex.wordCacheRAMSize()) + + " slots left); dismissed to omit ram flush lock"); + return false; + } + int stackCrawlQueueSize; if ((stackCrawlQueueSize = sbStackCrawlThread.size()) >= stackCrawlSlots) { log.logFine("deQueue: too many processes in stack crawl thread queue, dismissed to protect emergency case (" + diff --git a/source/de/anomic/plasma/plasmaWordIndexCache.java b/source/de/anomic/plasma/plasmaWordIndexCache.java index b31564ae8..d3daade4d 100644 --- a/source/de/anomic/plasma/plasmaWordIndexCache.java +++ b/source/de/anomic/plasma/plasmaWordIndexCache.java @@ -390,7 +390,7 @@ public final class plasmaWordIndexCache implements plasmaWordIndexInterface { } // now decide where to flush that container - if (container.size() <= assortmentCluster.clusterCapacity) { + //if (container.size() <= assortmentCluster.clusterCapacity) { // this fits into the assortments plasmaWordIndexEntryContainer feedback = assortmentCluster.storeTry(key, container); if (feedback == null) { @@ -399,10 +399,12 @@ public final class plasmaWordIndexCache implements plasmaWordIndexInterface { // *** should care about another option here *** return backend.addEntries(feedback, time, true); } + /* } else { // store to back-end; this should be a rare case return backend.addEntries(container, time, true); } + **/ } @@ -466,9 +468,10 @@ public final class plasmaWordIndexCache implements plasmaWordIndexInterface { return removed; } - public synchronized int addEntries(plasmaWordIndexEntryContainer container, long updateTime, boolean highPriority) { + public int addEntries(plasmaWordIndexEntryContainer container, long updateTime, boolean highPriority) { // this puts the entries into the cache, not into the assortment directly + int added = 0; // check cache space if (cache.size() > 0) try { // pause to get space in the cache (while it is flushed) @@ -485,14 +488,17 @@ public final class plasmaWordIndexCache implements plasmaWordIndexInterface { Thread.sleep(pausetime); } catch (InterruptedException e) {} - // stop flushing now for one moment - flushThread.pause(); //serverLog.logDebug("PLASMA INDEXING", "addEntryToIndexMem: cache.size=" + cache.size() + "; hashScore.size=" + hashScore.size()); - + // put new words into cache - int added = 0; String wordHash = container.wordHash(); - synchronized (cache) { + + synchronized (cache) { + + // stop flushing now for one moment + flushThread.pause(); + + // put container into cache plasmaWordIndexEntryContainer entries = (plasmaWordIndexEntryContainer) cache.get(wordHash); // null pointer exception? wordhash != null! must be cache==null if (entries == null) entries = new plasmaWordIndexEntryContainer(wordHash); added = entries.add(container); @@ -502,9 +508,12 @@ public final class plasmaWordIndexCache implements plasmaWordIndexInterface { hashDate.setScore(wordHash, intTime(updateTime)); } entries = null; + + // resume flushing + flushThread.proceed(); } //System.out.println("DEBUG: cache = " + cache.toString()); - flushThread.proceed(); + return added; }