diff --git a/htroot/PerformanceMemory_p.java b/htroot/PerformanceMemory_p.java index 4a1b5942f..488584b8c 100644 --- a/htroot/PerformanceMemory_p.java +++ b/htroot/PerformanceMemory_p.java @@ -109,14 +109,14 @@ public class PerformanceMemory_p { prop.put("EcoList_" + c + "_tableIndexPath", ((p = filename.indexOf("DATA",0)) < 0) ? filename : filename.substring(p)); prop.putNum("EcoList_" + c + "_tableSize", mapx.get(Table.StatKeys.tableSize)); - assert mapx.get(Table.StatKeys.tableKeyMem) != null : mapx; String v = mapx.get(Table.StatKeys.tableKeyMem); mem = v == null ? 0 : Long.parseLong(v); totalmem += mem; prop.put("EcoList_" + c + "_tableKeyMem", Formatter.bytesToString(mem)); prop.put("EcoList_" + c + "_tableKeyChunkSize", mapx.get(Table.StatKeys.tableKeyChunkSize)); - mem = Long.parseLong(mapx.get(Table.StatKeys.tableValueMem)); + v = mapx.get(Table.StatKeys.tableValueMem); + mem = v == null ? 0 : Long.parseLong(v); totalmem += mem; prop.put("EcoList_" + c + "_tableValueMem", Formatter.bytesToString(mem)); prop.put("EcoList_" + c + "_tableValueChunkSize", mapx.get(Table.StatKeys.tableValueChunkSize)); diff --git a/source/net/yacy/kelondro/blob/MapHeap.java b/source/net/yacy/kelondro/blob/MapHeap.java index 6300a643d..fadc160b3 100644 --- a/source/net/yacy/kelondro/blob/MapHeap.java +++ b/source/net/yacy/kelondro/blob/MapHeap.java @@ -392,17 +392,19 @@ public class MapHeap implements Map> { this.firstKey = firstKey; this.secondKey = secondKey; final CloneableIterator i = MapHeap.this.blob.keys(up, firstKey); - this.iterator = (rotating) ? new RotateIterator(i, secondKey, MapHeap.this.blob.size()) : i; + this.iterator = rotating ? new RotateIterator(i, secondKey, MapHeap.this.blob.size()) : i; } @Override public byte[] next() { + assert this.iterator != null; + if (this.iterator == null) return null; return removeFillchar(this.iterator.next()); } @Override public boolean hasNext() { - return this.iterator.hasNext(); + return this.iterator != null && this.iterator.hasNext(); } @Override diff --git a/source/net/yacy/kelondro/index/HandleMap.java b/source/net/yacy/kelondro/index/HandleMap.java index 0747e415c..5cb8a33ee 100644 --- a/source/net/yacy/kelondro/index/HandleMap.java +++ b/source/net/yacy/kelondro/index/HandleMap.java @@ -110,7 +110,7 @@ public final class HandleMap implements Iterable { } private static final int spread(final int expectedspace) { - return Math.min(Runtime.getRuntime().availableProcessors(), Math.max(1, expectedspace / 3000)); + return Math.min(Runtime.getRuntime().availableProcessors(), Math.max(Runtime.getRuntime().availableProcessors(), expectedspace / 8000)); } public final int[] saturation() { diff --git a/source/net/yacy/kelondro/index/RAMIndexCluster.java b/source/net/yacy/kelondro/index/RAMIndexCluster.java index a70a00057..212dd36cd 100644 --- a/source/net/yacy/kelondro/index/RAMIndexCluster.java +++ b/source/net/yacy/kelondro/index/RAMIndexCluster.java @@ -51,7 +51,7 @@ public final class RAMIndexCluster implements Index, Iterable, Clonea this.cluster = new RAMIndex[clusterSize]; this.rowdef = rowdef; for (int i = 0; i < clusterSize; i++) { - this.cluster[i] = new RAMIndex(name + "." + i, rowdef, 0); + this.cluster[i] = null; // lazy initialization, the actual initialization is at accessArray() } } @@ -69,7 +69,7 @@ public final class RAMIndexCluster implements Index, Iterable, Clonea public RAMIndexCluster clone() { final RAMIndex[] a = new RAMIndex[this.cluster.length]; for (int i = 0; i < this.cluster.length; i++) { - a[i] = this.cluster[i].clone(); + a[i] = this.cluster[i] == null ? null : this.cluster[i].clone(); } return new RAMIndexCluster(this.name + ".clone", this.rowdef, a); } @@ -111,8 +111,11 @@ public final class RAMIndexCluster implements Index, Iterable, Clonea private final RAMIndex accessArray(final int i) { RAMIndex r = this.cluster[i]; if (r == null) synchronized (this.cluster) { - r = new RAMIndex(this.name + "." + i, this.rowdef, 0); - this.cluster[i] = r; + r = this.cluster[i]; + if (r == null) { + r = new RAMIndex(this.name + "." + i, this.rowdef, 0); + this.cluster[i] = r; + } } return r; } @@ -295,15 +298,13 @@ public final class RAMIndexCluster implements Index, Iterable, Clonea @SuppressWarnings("unchecked") public final CloneableIterator rows(final boolean up, final byte[] firstKey) { synchronized (this.cluster) { - final CloneableIterator[] col = new CloneableIterator[this.cluster.length]; - for (int i = 0; i < this.cluster.length; i++) { - if (this.cluster[i] == null) { - col[i] = null; - } else { - col[i] = this.cluster[i].rows(up, firstKey); + final List> col = new ArrayList>(this.cluster.length); + for (RAMIndex element : this.cluster) { + if (element != null) { + col.add(element.rows(up, firstKey)); } } - return StackIterator.stack(col); + return StackIterator.stack(col.toArray(new CloneableIterator[col.size()])); } } diff --git a/source/net/yacy/kelondro/order/StackIterator.java b/source/net/yacy/kelondro/order/StackIterator.java index d9cf0a902..dd197d69c 100644 --- a/source/net/yacy/kelondro/order/StackIterator.java +++ b/source/net/yacy/kelondro/order/StackIterator.java @@ -28,10 +28,10 @@ import net.yacy.cora.order.CloneableIterator; public class StackIterator implements CloneableIterator { - + private final CloneableIterator a, b; private E na, nb; - + public StackIterator( final CloneableIterator a, final CloneableIterator b) { @@ -42,57 +42,76 @@ public class StackIterator implements CloneableIterator { nextb(); } + @Override public StackIterator clone(final Object modifier) { - return new StackIterator(a.clone(modifier), b.clone(modifier)); + return new StackIterator(this.a.clone(modifier), this.b.clone(modifier)); } - + private void nexta() { try { - if ((a != null) && (a.hasNext())) na = a.next(); else na = null; + if ((this.a != null) && (this.a.hasNext())) this.na = this.a.next(); else this.na = null; } catch (final ConcurrentModificationException e) { - na = null; + this.na = null; } } private void nextb() { try { - if ((b != null) && (b.hasNext())) nb = b.next(); else nb = null; + if ((this.b != null) && (this.b.hasNext())) this.nb = this.b.next(); else this.nb = null; } catch (final ConcurrentModificationException e) { - nb = null; + this.nb = null; } } - + + @Override public boolean hasNext() { - return (na != null) || (nb != null); + return (this.na != null) || (this.nb != null); } - + + @Override public E next() { E s; - if (na == null) { - s = nb; + if (this.na == null) { + s = this.nb; nextb(); return s; } - if (nb == null) { - s = na; + if (this.nb == null) { + s = this.na; nexta(); return s; } // just stack the Objects - s = na; + s = this.na; nexta(); return s; } - + + @Override public void remove() { throw new java.lang.UnsupportedOperationException("merge does not support remove"); } - + @SuppressWarnings("unchecked") public static CloneableIterator stack(final CloneableIterator[] iterators) { // this extends the ability to combine two iterators // to the ability of combining a set of iterators - if (iterators == null) return null; - if (iterators.length == 0) return null; + if (iterators == null || iterators.length == 0) return new CloneableIterator() { + @Override + public boolean hasNext() { + return false; + } + @Override + public A next() { + return null; + } + @Override + public void remove() { + } + @Override + public CloneableIterator clone(Object modifier) { + return null; + } + }; if (iterators.length == 1) { return iterators[0]; } diff --git a/source/net/yacy/kelondro/util/StandardMemoryStrategy.java b/source/net/yacy/kelondro/util/StandardMemoryStrategy.java index 581a21d8d..7035e9486 100644 --- a/source/net/yacy/kelondro/util/StandardMemoryStrategy.java +++ b/source/net/yacy/kelondro/util/StandardMemoryStrategy.java @@ -38,7 +38,7 @@ public class StandardMemoryStrategy extends MemoryStrategy { private long prevTreshold = 0L; private int tresholdCount = 0; private boolean proper = true; - + public StandardMemoryStrategy() { name = "Standard Memory Strategy"; error= false; //since this is the standard implementation we assume always false here @@ -49,6 +49,7 @@ public class StandardMemoryStrategy extends MemoryStrategy { * @param last time which must be passed since lased gc * @param info additional info for log */ + @Override protected final synchronized boolean gc(final int last, final String info) { // thq assert last >= 10000; // too many forced GCs will cause bad execution performance final long elapsed = System.currentTimeMillis() - lastGC; @@ -58,8 +59,8 @@ public class StandardMemoryStrategy extends MemoryStrategy { System.gc(); lastGC = System.currentTimeMillis(); final long after = free(); - gcs[gcs_pos++] = after - before; - if (gcs_pos >= gcs.length) gcs_pos = 0; + this.gcs[this.gcs_pos++] = after - before; + if (this.gcs_pos >= this.gcs.length) this.gcs_pos = 0; if (log.isFine()) log.logInfo("[gc] before: " + Formatter.bytesToString(before) + ", after: " + Formatter.bytesToString(after) + @@ -80,7 +81,7 @@ public class StandardMemoryStrategy extends MemoryStrategy { protected final long getAverageGCFree() { long x = 0; int y = 0; - for (final long gc : gcs) + for (final long gc : this.gcs) if (gc != 0) { x += gc; y++; @@ -92,14 +93,16 @@ public class StandardMemoryStrategy extends MemoryStrategy { * memory that is free without increasing of total memory taken from os * @return bytes */ + @Override protected final long free() { - return runtime.freeMemory(); + return this.runtime.freeMemory(); } /** * memory that is available including increasing total memory up to maximum * @return bytes */ + @Override protected final long available() { return maxMemory() - total() + free(); } @@ -108,18 +111,20 @@ public class StandardMemoryStrategy extends MemoryStrategy { * maximum memory the Java virtual will allocate machine; may vary over time in some cases * @return bytes */ + @Override protected final long maxMemory() { - return runtime.maxMemory(); + return this.runtime.maxMemory(); } /** * currently allocated memory in the Java virtual machine; may vary over time * @return bytes */ + @Override protected final long total() { - return runtime.totalMemory(); + return this.runtime.totalMemory(); } /** @@ -141,6 +146,7 @@ public class StandardMemoryStrategy extends MemoryStrategy { * @param force specifies whether a GC should be run even in case former GCs didn't provide enough memory * @return whether enough memory could be freed (or is free) or not */ + @Override protected boolean request(final long size, final boolean force, boolean shortStatus) { if (size <= 0) return true; final boolean r = request0(size, force); @@ -181,42 +187,46 @@ public class StandardMemoryStrategy extends MemoryStrategy { * memory that is currently bound in objects * @return used bytes */ + @Override protected final long used() { return total() - free(); } + @Override protected boolean properState() { - return proper; + return this.proper; } + @Override protected void resetProperState() { - proper = true; - tresholdCount = 0; + this.proper = true; + this.tresholdCount = 0; } /** * set the memory to be available */ + @Override protected void setProperMbyte(final long mbyte) { - properMbyte = mbyte; - tresholdCount = 0; + this.properMbyte = mbyte; + this.tresholdCount = 0; } private void checkProper(final long available) { // disable proper state if memory is less than treshold - 4 times, maximum 11 minutes between each detection - if ((available >> 20) < properMbyte) { + if ((available >> 20) < this.properMbyte) { final long t = System.currentTimeMillis(); - if(prevTreshold + 11L /* minutes */ * 60000L > t) { - tresholdCount++; - if(tresholdCount > 3 /* occurencies - 1 */) proper = false; + if(this.prevTreshold + 11L /* minutes */ * 60000L > t) { + this.tresholdCount++; + if(this.tresholdCount > 3 /* occurencies - 1 */) this.proper = false; } - else tresholdCount = 1; + else this.tresholdCount = 1; - prevTreshold = t; + this.prevTreshold = t; - log.logInfo("checkProper: below treshold; tresholdCount: " + tresholdCount + "; proper: " + proper); + log.logInfo("checkProper: below treshold; tresholdCount: " + this.tresholdCount + "; proper: " + this.proper); } - else if (!proper && (available >> 20) > (properMbyte * 2L)) // we were wrong! + else if (!this.proper && (available >> 20) > (this.properMbyte * 2L)) // we were wrong! resetProperState(); }