diff --git a/htroot/PerformanceMemory_p.html b/htroot/PerformanceMemory_p.html index 654c6d715..61adc1a13 100644 --- a/htroot/PerformanceMemory_p.html +++ b/htroot/PerformanceMemory_p.html @@ -312,6 +312,30 @@ Increasing this cache may speed up crawling, but not much space is needed, so th

+

+

Write Cache Object Allocation:
+ + + +#{sizes}# + +#{/sizes}# + + + +#{alive}# + +#{/alive}# + + + +#{heap}# + +#{/heap}# + +
 
Chunk Sizes		#[chunk]#
now alive in write cache#[count]#
currently held in write buffer heap#[count]#
+

+ #%env/templates/footer.template%# diff --git a/htroot/PerformanceMemory_p.java b/htroot/PerformanceMemory_p.java index 87aa33a7b..824bead8a 100644 --- a/htroot/PerformanceMemory_p.java +++ b/htroot/PerformanceMemory_p.java @@ -43,6 +43,7 @@ //javac -classpath .:../classes PerformanceMemory_p.java //if the shell's current path is HTROOT +import java.util.Iterator; import java.util.Map; import java.io.File; @@ -52,6 +53,7 @@ import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; import de.anomic.server.serverFileUtils; import de.anomic.yacy.yacyCore; +import de.anomic.kelondro.kelondroObjectSpace; public class PerformanceMemory_p { @@ -196,12 +198,12 @@ public class PerformanceMemory_p { req = plasmaSwitchboard.robots.size(); chk = plasmaSwitchboard.robots.dbCacheChunkSize(); slt = plasmaSwitchboard.robots.dbCacheFillStatus(); - putprop(prop, env, "Robots", set); + putprop(prop, env, "Robots", set); req = sb.profiles.size(); chk = sb.profiles.dbCacheChunkSize(); slt = sb.profiles.dbCacheFillStatus(); - putprop(prop, env, "Profiles", set); + putprop(prop, env, "Profiles", set); prop.put("usedTotal", usedTotal / MB); prop.put("currTotal", currTotal / MB); @@ -214,6 +216,38 @@ public class PerformanceMemory_p { prop.put("Xmx", Xmx.substring(0, Xmx.length() - 1)); String Xms = env.getConfig("javastart_Xms", "Xms10m").substring(3); prop.put("Xms", Xms.substring(0, Xms.length() - 1)); + + // create statistics about write cache object space + int chunksizes = Math.max( + ((Integer) kelondroObjectSpace.statAlive().lastKey()).intValue(), + ((Integer) kelondroObjectSpace.statHeap().lastKey()).intValue() + ); + int[] statAlive = new int[chunksizes]; + int[] statHeap = new int[chunksizes]; + for (int i = 0; i < chunksizes; i++) { statAlive[i] = 0; statHeap[i] = 0; } + Map.Entry entry; + Iterator i = kelondroObjectSpace.statAlive().entrySet().iterator(); + while (i.hasNext()) { + entry = (Map.Entry) i.next(); + statAlive[((Integer) entry.getKey()).intValue() - 1] = ((Integer) entry.getValue()).intValue(); + } + i = kelondroObjectSpace.statHeap().entrySet().iterator(); + while (i.hasNext()) { + entry = (Map.Entry) i.next(); + statHeap[((Integer) entry.getKey()).intValue() - 1] = ((Integer) entry.getValue()).intValue(); + } + int c = 0; + for (int j = 0; j < chunksizes; j++) { + if ((statAlive[j] > 0) || (statHeap[j] > 0)) { + prop.put("sizes_" + c + "_chunk", Integer.toString(j + 1)); + prop.put("alive_" + c + "_count", Integer.toString(statAlive[j])); + prop.put("heap_" + c + "_count", Integer.toString(statHeap[j])); + c++; + } + } + prop.put("sizes", Integer.toString(c)); + prop.put("alive", Integer.toString(c)); + prop.put("heap" , Integer.toString(c)); // return rewrite values for templates return prop; diff --git a/source/de/anomic/data/messageBoard.java b/source/de/anomic/data/messageBoard.java index 09bc2af74..3ed3d358f 100644 --- a/source/de/anomic/data/messageBoard.java +++ b/source/de/anomic/data/messageBoard.java @@ -94,8 +94,8 @@ public class messageBoard { return database.cacheFillStatus(); } - public void close() throws IOException { - database.close(); + public void close() { + try {database.close();} catch (IOException e) {} } private static String dateString() { diff --git a/source/de/anomic/data/wikiBoard.java b/source/de/anomic/data/wikiBoard.java index da2e348c4..079bdc3e4 100644 --- a/source/de/anomic/data/wikiBoard.java +++ b/source/de/anomic/data/wikiBoard.java @@ -116,9 +116,9 @@ public class wikiBoard { return new int[]{a[0] + b[0], a[1] + b[1], a[2] + b[2], a[3] + b[3]}; } - public void close() throws IOException { - datbase.close(); - bkpbase.close(); + public void close() { + try {datbase.close();} catch (IOException e) {} + try {bkpbase.close();} catch (IOException e) {} } private static String dateString() { diff --git a/source/de/anomic/kelondro/kelondroBufferedIOChunks.java b/source/de/anomic/kelondro/kelondroBufferedIOChunks.java index 8514695a9..116054d3a 100644 --- a/source/de/anomic/kelondro/kelondroBufferedIOChunks.java +++ b/source/de/anomic/kelondro/kelondroBufferedIOChunks.java @@ -107,6 +107,8 @@ public final class kelondroBufferedIOChunks extends kelondroAbstractIOChunks imp public void write(long pos, byte[] b, int off, int len) throws IOException { assert (b.length >= off + len): "write pos=" + pos + ", b.length=" + b.length + ", b='" + new String(b) + "', off=" + off + ", len=" + len; + //if (len > 10) System.out.println("WRITE(" + name + ", " + pos + ", " + b.length + ", " + off + ", " + len + ")"); + // do the write into buffer byte[] bb = kelondroObjectSpace.alloc(len); System.arraycopy(b, off, bb, 0, len); diff --git a/source/de/anomic/kelondro/kelondroObjectSpace.java b/source/de/anomic/kelondro/kelondroObjectSpace.java index bca6f24e6..20f5fdf4e 100644 --- a/source/de/anomic/kelondro/kelondroObjectSpace.java +++ b/source/de/anomic/kelondro/kelondroObjectSpace.java @@ -43,18 +43,40 @@ package de.anomic.kelondro; import java.util.ArrayList; import java.util.HashMap; +import java.util.Iterator; +import java.util.TreeMap; +import java.util.Map; public class kelondroObjectSpace { private static final int minSize = 10; - private static final int maxSize = 4096; + private static final int maxSize = 256; - private static HashMap objects = new HashMap(); + private static HashMap objHeap = new HashMap(); + private static TreeMap aliveNow = new TreeMap(); + //private static TreeMap aliveMax = new TreeMap(); + + private static void incAlive(int size) { + final Integer s = new Integer(size); + synchronized (aliveNow) { + final Integer x = (Integer) aliveNow.get(s); + if (x == null) aliveNow.put(s, new Integer(1)); else aliveNow.put(s, new Integer(x.intValue() + 1)); + } + } + + private static void decAlive(int size) { + final Integer s = new Integer(size); + synchronized (aliveNow) { + final Integer x = (Integer) aliveNow.get(s); + if (x == null) aliveNow.put(s, new Integer(-1)); else aliveNow.put(s, new Integer(x.intValue() - 1)); + } + } public static byte[] alloc(int len) { if ((len < minSize) || (len > maxSize)) return new byte[len]; - synchronized (objects) { - ArrayList buf = (ArrayList) objects.get(new Integer(len)); + incAlive(len); + synchronized (objHeap) { + ArrayList buf = (ArrayList) objHeap.get(new Integer(len)); if ((buf == null) || (buf.size() == 0)) return new byte[len]; return (byte[]) buf.remove(buf.size() - 1); } @@ -65,13 +87,14 @@ public class kelondroObjectSpace { b = null; return; } - synchronized (objects) { + decAlive(b.length); + synchronized (objHeap) { final Integer i = new Integer(b.length); - ArrayList buf = (ArrayList) objects.get(i); + ArrayList buf = (ArrayList) objHeap.get(i); if (buf == null) { buf = new ArrayList(); buf.add(b); - objects.put(i, buf); + objHeap.put(i, buf); } else { buf.add(b); } @@ -79,4 +102,24 @@ public class kelondroObjectSpace { b = null; } + public static TreeMap statAlive() { + return aliveNow; + } + + public static TreeMap statHeap() { + // creates a statistic output of this object space + // the result is a mapping from Integer (chunk size) to Integer (number of counts) + // and shows how many Objects are held in this space for usage + TreeMap result = new TreeMap(); + synchronized (objHeap) { + Iterator i = objHeap.entrySet().iterator(); + Map.Entry entry; + while (i.hasNext()) { + entry = (Map.Entry) i.next(); + result.put(entry.getKey(), new Integer(((ArrayList) entry.getValue()).size())); + } + } + return result; + } + } diff --git a/source/de/anomic/kelondro/kelondroRecords.java b/source/de/anomic/kelondro/kelondroRecords.java index 85bc9b350..8e8b52dd6 100644 --- a/source/de/anomic/kelondro/kelondroRecords.java +++ b/source/de/anomic/kelondro/kelondroRecords.java @@ -419,7 +419,7 @@ public class kelondroRecords { return new File(filename); } - protected int cacheChunkSize(boolean cacheControl) { + protected final int cacheChunkSize(boolean cacheControl) { return this.headchunksize + element_in_cache + ((cacheControl) ? cache_control_entry : 0); } @@ -475,7 +475,7 @@ public class kelondroRecords { dispose(handle); } - public class Node { + public final class Node { // an Node holds all information of one row of data. This includes the key to the entry // which is stored as entry element at position 0 // an Node object can be created in two ways: @@ -732,12 +732,14 @@ public class kelondroRecords { // save head if (this.headChanged) { + //System.out.println("WRITEH(" + filename + ", " + seekpos(this.handle) + ", " + this.headChunk.length + ")"); entryFile.write(seekpos(this.handle), this.headChunk); update2Cache(cachePriority); } // save tail if ((this.tailChunk != null) && (this.tailChanged)) { + //System.out.println("WRITET(" + filename + ", " + (seekpos(this.handle) + headchunksize) + ", " + this.tailChunk.length + ")"); entryFile.write(seekpos(this.handle) + headchunksize, this.tailChunk); } } @@ -954,7 +956,7 @@ public class kelondroRecords { return this.COLWIDTHS[column]; } - private long seekpos(Handle handle) { + private final long seekpos(Handle handle) { assert (handle.index >= 0): "handle index too low: " + handle.index; assert (handle.index < USAGE.allCount()): "handle index too high:" + handle.index; return POS_NODES + ((long) recordsize * handle.index); @@ -1069,21 +1071,21 @@ public class kelondroRecords { return x; } - public static void NUL2bytes(byte[] b, int offset) { + public final static void NUL2bytes(byte[] b, int offset) { b[offset ] = (byte) (0XFF & (NUL >> 24)); b[offset + 1] = (byte) (0XFF & (NUL >> 16)); b[offset + 2] = (byte) (0XFF & (NUL >> 8)); b[offset + 3] = (byte) (0XFF & NUL); } - public static void int2bytes(long i, byte[] b, int offset) { + public final static void int2bytes(long i, byte[] b, int offset) { b[offset ] = (byte) (0XFF & (i >> 24)); b[offset + 1] = (byte) (0XFF & (i >> 16)); b[offset + 2] = (byte) (0XFF & (i >> 8)); b[offset + 3] = (byte) (0XFF & i); } - public static int bytes2int(byte[] b, int offset) { + public final static int bytes2int(byte[] b, int offset) { return ( ((b[offset ] & 0xff) << 24) | ((b[offset + 1] & 0xff) << 16) | diff --git a/source/de/anomic/plasma/plasmaCrawlNURL.java b/source/de/anomic/plasma/plasmaCrawlNURL.java index b40604238..b18268132 100644 --- a/source/de/anomic/plasma/plasmaCrawlNURL.java +++ b/source/de/anomic/plasma/plasmaCrawlNURL.java @@ -174,14 +174,12 @@ public class plasmaCrawlNURL extends plasmaURL { public void close() { coreStack.close(); - try { - limitStack.close(); - overhangStack.close(); - remoteStack.close(); - imageStack.close(); - movieStack.close(); - musicStack.close(); - } catch (IOException e) {} + limitStack.close(); + overhangStack.close(); + remoteStack.close(); + try {imageStack.close();} catch (IOException e) {} + try {movieStack.close();} catch (IOException e) {} + try {musicStack.close();} catch (IOException e) {} try { super.close(); } catch (IOException e) {} } diff --git a/source/de/anomic/plasma/plasmaHTCache.java b/source/de/anomic/plasma/plasmaHTCache.java index dfaf2284a..6c7765bbe 100644 --- a/source/de/anomic/plasma/plasmaHTCache.java +++ b/source/de/anomic/plasma/plasmaHTCache.java @@ -291,8 +291,8 @@ public final class plasmaHTCache { } } - public void close() throws IOException { - this.responseHeaderDB.close(); + public void close() { + try {this.responseHeaderDB.close();} catch (IOException e) {} } private String ageString(long date, File f) { diff --git a/source/de/anomic/plasma/plasmaSwitchboard.java b/source/de/anomic/plasma/plasmaSwitchboard.java index 1efca8f92..2b038ab45 100644 --- a/source/de/anomic/plasma/plasmaSwitchboard.java +++ b/source/de/anomic/plasma/plasmaSwitchboard.java @@ -737,30 +737,26 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser public void close() { log.logConfig("SWITCHBOARD SHUTDOWN STEP 1: sending termination signal to managed threads:"); terminateAllThreads(true); - log.logConfig("SWITCHBOARD SHUTDOWN STEP 2: sending termination signal to threaded indexing (stand by...)"); + log.logConfig("SWITCHBOARD SHUTDOWN STEP 2: sending termination signal to database manager"); + // closing all still running db importer jobs + plasmaDbImporter.close(); + indexDistribution.close(); + cacheLoader.close(); + wikiDB.close(); + userDB.close(); + messageDB.close(); + if (facilityDB != null) try {facilityDB.close();} catch (IOException e) {} + sbStackCrawlThread.close(); + urlPool.close(); + profiles.close(); + robots.close(); + parser.close(); + cacheManager.close(); + sbQueue.close(); + flushCitationReference(crg, "crg"); + log.logConfig("SWITCHBOARD SHUTDOWN STEP 3: sending termination signal to threaded indexing (stand by...)"); int waitingBoundSeconds = Integer.parseInt(getConfig("maxWaitingWordFlush", "120")); wordIndex.close(waitingBoundSeconds); - log.logConfig("SWITCHBOARD SHUTDOWN STEP 3: sending termination signal to database manager"); - try { - // closing all still running db importer jobs - plasmaDbImporter.close(); - - indexDistribution.close(); - cacheLoader.close(); - wikiDB.close(); - userDB.close(); - messageDB.close(); - if (facilityDB != null) facilityDB.close(); - sbStackCrawlThread.close(); - urlPool.close(); - profiles.close(); - robots.close(); - parser.close(); - cacheManager.close(); - sbQueue.close(); - //flushCitationReference(crl, "crl"); - flushCitationReference(crg, "crg"); - } catch (IOException e) {} log.logConfig("SWITCHBOARD SHUTDOWN TERMINATED"); } diff --git a/source/de/anomic/plasma/plasmaURLPool.java b/source/de/anomic/plasma/plasmaURLPool.java index 5885f892c..ed3d96465 100644 --- a/source/de/anomic/plasma/plasmaURLPool.java +++ b/source/de/anomic/plasma/plasmaURLPool.java @@ -82,9 +82,9 @@ public class plasmaURLPool { return null; } - public void close() throws IOException { - loadedURL.close(); + public void close() { + try {loadedURL.close();} catch (IOException e) {} noticeURL.close(); - errorURL.close(); + try {errorURL.close();} catch (IOException e) {} } } diff --git a/source/de/anomic/plasma/plasmaWordIndexAssortmentCluster.java b/source/de/anomic/plasma/plasmaWordIndexAssortmentCluster.java index 1707a8277..ced5c9e53 100644 --- a/source/de/anomic/plasma/plasmaWordIndexAssortmentCluster.java +++ b/source/de/anomic/plasma/plasmaWordIndexAssortmentCluster.java @@ -167,17 +167,16 @@ public final class plasmaWordIndexAssortmentCluster { int [] spaces = new int[testsize]; for (int i = testsize - 1; i >= 0; i--) spaces[i] = 0; int need = newContainer.size(); - int s = testsize - 1; - while (s >= 0) { - spaces[s] = (assortments[s].get(wordHash) == null) ? (s + 1) : 0; - need -= spaces[s]; + int selectedAssortment = testsize - 1; + while (selectedAssortment >= 0) { + spaces[selectedAssortment] = (assortments[selectedAssortment].get(wordHash) == null) ? (selectedAssortment + 1) : 0; + need -= spaces[selectedAssortment]; assert (need >= 0); if (need == 0) break; - s = (need < s) ? need : s - 1; + selectedAssortment = (need < selectedAssortment) ? need : selectedAssortment - 1; } if (need == 0) { // we found spaces so that we can put in the newContainer into these spaces - plasmaWordIndexEntryContainer c; Iterator i = newContainer.entries(); for (int j = testsize - 1; j >= 0; j--) { @@ -189,7 +188,6 @@ public final class plasmaWordIndexAssortmentCluster { } storeForced(wordHash, c); } - return null; } diff --git a/source/de/anomic/plasma/plasmaWordIndexCache.java b/source/de/anomic/plasma/plasmaWordIndexCache.java index feb30f300..1aa645af2 100644 --- a/source/de/anomic/plasma/plasmaWordIndexCache.java +++ b/source/de/anomic/plasma/plasmaWordIndexCache.java @@ -464,7 +464,7 @@ public final class plasmaWordIndexCache implements plasmaWordIndexInterface { entries = null; // force flush (sometimes) - if (System.currentTimeMillis() % 5 == 0) flushFromMem(); + if (System.currentTimeMillis() % 7 == 4) flushFromMem(); } return added; }