boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

fixed handlemap spread factor and null iterator handling

Browse Source
pull/1/head
Michael Peter Christen 13 years ago
parent 00f2df1120
commit 10c9c17d51
6 changed files with 88 additions and 56 deletions
Show Stats Download Patch File Download Diff File

4
htroot/PerformanceMemory_p.java
Unescape View File

@ -109,14 +109,14 @@ public class PerformanceMemory_p {
prop.put("EcoList_" + c + "_tableIndexPath", ((p = filename.indexOf("DATA",0)) < 0) ? filename : filename.substring(p)); prop.put("EcoList_" + c + "_tableIndexPath", ((p = filename.indexOf("DATA",0)) < 0) ? filename : filename.substring(p));
prop.putNum("EcoList_" + c + "_tableSize", mapx.get(Table.StatKeys.tableSize)); prop.putNum("EcoList_" + c + "_tableSize", mapx.get(Table.StatKeys.tableSize));
assert mapx.get(Table.StatKeys.tableKeyMem) != null : mapx;
String v = mapx.get(Table.StatKeys.tableKeyMem); String v = mapx.get(Table.StatKeys.tableKeyMem);
mem = v == null ? 0 : Long.parseLong(v); mem = v == null ? 0 : Long.parseLong(v);
totalmem += mem; totalmem += mem;
prop.put("EcoList_" + c + "_tableKeyMem", Formatter.bytesToString(mem)); prop.put("EcoList_" + c + "_tableKeyMem", Formatter.bytesToString(mem));
prop.put("EcoList_" + c + "_tableKeyChunkSize", mapx.get(Table.StatKeys.tableKeyChunkSize)); prop.put("EcoList_" + c + "_tableKeyChunkSize", mapx.get(Table.StatKeys.tableKeyChunkSize));
mem = Long.parseLong(mapx.get(Table.StatKeys.tableValueMem)); v = mapx.get(Table.StatKeys.tableValueMem);
mem = v == null ? 0 : Long.parseLong(v);
totalmem += mem; totalmem += mem;
prop.put("EcoList_" + c + "_tableValueMem", Formatter.bytesToString(mem)); prop.put("EcoList_" + c + "_tableValueMem", Formatter.bytesToString(mem));
prop.put("EcoList_" + c + "_tableValueChunkSize", mapx.get(Table.StatKeys.tableValueChunkSize)); prop.put("EcoList_" + c + "_tableValueChunkSize", mapx.get(Table.StatKeys.tableValueChunkSize));

6
source/net/yacy/kelondro/blob/MapHeap.java
Unescape View File

@ -392,17 +392,19 @@ public class MapHeap implements Map<byte[], Map<String, String>> {
this.firstKey = firstKey; this.firstKey = firstKey;
this.secondKey = secondKey; this.secondKey = secondKey;
final CloneableIterator<byte[]> i = MapHeap.this.blob.keys(up, firstKey); final CloneableIterator<byte[]> i = MapHeap.this.blob.keys(up, firstKey);
this.iterator = (rotating) ? new RotateIterator<byte[]>(i, secondKey, MapHeap.this.blob.size()) : i; this.iterator = rotating ? new RotateIterator<byte[]>(i, secondKey, MapHeap.this.blob.size()) : i;
} }
@Override @Override
public byte[] next() { public byte[] next() {
assert this.iterator != null;
if (this.iterator == null) return null;
return removeFillchar(this.iterator.next()); return removeFillchar(this.iterator.next());
} }
@Override @Override
public boolean hasNext() { public boolean hasNext() {
return this.iterator.hasNext(); return this.iterator != null && this.iterator.hasNext();
} }
@Override @Override

2
source/net/yacy/kelondro/index/HandleMap.java
Unescape View File

@ -110,7 +110,7 @@ public final class HandleMap implements Iterable<Row.Entry> {
} }
private static final int spread(final int expectedspace) { private static final int spread(final int expectedspace) {
return Math.min(Runtime.getRuntime().availableProcessors(), Math.max(1, expectedspace / 3000)); return Math.min(Runtime.getRuntime().availableProcessors(), Math.max(Runtime.getRuntime().availableProcessors(), expectedspace / 8000));
} }
public final int[] saturation() { public final int[] saturation() {

23
source/net/yacy/kelondro/index/RAMIndexCluster.java
Unescape View File

@ -51,7 +51,7 @@ public final class RAMIndexCluster implements Index, Iterable<Row.Entry>, Clonea
this.cluster = new RAMIndex[clusterSize]; this.cluster = new RAMIndex[clusterSize];
this.rowdef = rowdef; this.rowdef = rowdef;
for (int i = 0; i < clusterSize; i++) { for (int i = 0; i < clusterSize; i++) {
this.cluster[i] = new RAMIndex(name + "." + i, rowdef, 0); this.cluster[i] = null; // lazy initialization, the actual initialization is at accessArray()
} }
} }
@ -69,7 +69,7 @@ public final class RAMIndexCluster implements Index, Iterable<Row.Entry>, Clonea
public RAMIndexCluster clone() { public RAMIndexCluster clone() {
final RAMIndex[] a = new RAMIndex[this.cluster.length]; final RAMIndex[] a = new RAMIndex[this.cluster.length];
for (int i = 0; i < this.cluster.length; i++) { for (int i = 0; i < this.cluster.length; i++) {
a[i] = this.cluster[i].clone(); a[i] = this.cluster[i] == null ? null : this.cluster[i].clone();
} }
return new RAMIndexCluster(this.name + ".clone", this.rowdef, a); return new RAMIndexCluster(this.name + ".clone", this.rowdef, a);
} }
@ -111,8 +111,11 @@ public final class RAMIndexCluster implements Index, Iterable<Row.Entry>, Clonea
private final RAMIndex accessArray(final int i) { private final RAMIndex accessArray(final int i) {
RAMIndex r = this.cluster[i]; RAMIndex r = this.cluster[i];
if (r == null) synchronized (this.cluster) { if (r == null) synchronized (this.cluster) {
r = new RAMIndex(this.name + "." + i, this.rowdef, 0); r = this.cluster[i];
this.cluster[i] = r; if (r == null) {
r = new RAMIndex(this.name + "." + i, this.rowdef, 0);
this.cluster[i] = r;
}
} }
return r; return r;
} }
@ -295,15 +298,13 @@ public final class RAMIndexCluster implements Index, Iterable<Row.Entry>, Clonea
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
public final CloneableIterator<Entry> rows(final boolean up, final byte[] firstKey) { public final CloneableIterator<Entry> rows(final boolean up, final byte[] firstKey) {
synchronized (this.cluster) { synchronized (this.cluster) {
final CloneableIterator<Entry>[] col = new CloneableIterator[this.cluster.length]; final List<CloneableIterator<Entry>> col = new ArrayList<CloneableIterator<Entry>>(this.cluster.length);
for (int i = 0; i < this.cluster.length; i++) { for (RAMIndex element : this.cluster) {
if (this.cluster[i] == null) { if (element != null) {
col[i] = null; col.add(element.rows(up, firstKey));
} else {
col[i] = this.cluster[i].rows(up, firstKey);
} }
} }
return StackIterator.stack(col); return StackIterator.stack(col.toArray(new CloneableIterator[col.size()]));
} }
} }

45
source/net/yacy/kelondro/order/StackIterator.java
Unescape View File

@ -42,47 +42,51 @@ public class StackIterator<E> implements CloneableIterator<E> {
nextb(); nextb();
} }
@Override
public StackIterator<E> clone(final Object modifier) { public StackIterator<E> clone(final Object modifier) {
return new StackIterator<E>(a.clone(modifier), b.clone(modifier)); return new StackIterator<E>(this.a.clone(modifier), this.b.clone(modifier));
} }
private void nexta() { private void nexta() {
try { try {
if ((a != null) && (a.hasNext())) na = a.next(); else na = null; if ((this.a != null) && (this.a.hasNext())) this.na = this.a.next(); else this.na = null;
} catch (final ConcurrentModificationException e) { } catch (final ConcurrentModificationException e) {
na = null; this.na = null;
} }
} }
private void nextb() { private void nextb() {
try { try {
if ((b != null) && (b.hasNext())) nb = b.next(); else nb = null; if ((this.b != null) && (this.b.hasNext())) this.nb = this.b.next(); else this.nb = null;
} catch (final ConcurrentModificationException e) { } catch (final ConcurrentModificationException e) {
nb = null; this.nb = null;
} }
} }
@Override
public boolean hasNext() { public boolean hasNext() {
return (na != null) || (nb != null); return (this.na != null) || (this.nb != null);
} }
@Override
public E next() { public E next() {
E s; E s;
if (na == null) { if (this.na == null) {
s = nb; s = this.nb;
nextb(); nextb();
return s; return s;
} }
if (nb == null) { if (this.nb == null) {
s = na; s = this.na;
nexta(); nexta();
return s; return s;
} }
// just stack the Objects // just stack the Objects
s = na; s = this.na;
nexta(); nexta();
return s; return s;
} }
@Override
public void remove() { public void remove() {
throw new java.lang.UnsupportedOperationException("merge does not support remove"); throw new java.lang.UnsupportedOperationException("merge does not support remove");
} }
@ -91,8 +95,23 @@ public class StackIterator<E> implements CloneableIterator<E> {
public static <A> CloneableIterator<A> stack(final CloneableIterator<A>[] iterators) { public static <A> CloneableIterator<A> stack(final CloneableIterator<A>[] iterators) {
// this extends the ability to combine two iterators // this extends the ability to combine two iterators
// to the ability of combining a set of iterators // to the ability of combining a set of iterators
if (iterators == null) return null; if (iterators == null || iterators.length == 0) return new CloneableIterator<A>() {
if (iterators.length == 0) return null; @Override
public boolean hasNext() {
return false;
}
@Override
public A next() {
return null;
}
@Override
public void remove() {
}
@Override
public CloneableIterator<A> clone(Object modifier) {
return null;
}
};
if (iterators.length == 1) { if (iterators.length == 1) {
return iterators[0]; return iterators[0];
} }

48
source/net/yacy/kelondro/util/StandardMemoryStrategy.java
Unescape View File

@ -49,6 +49,7 @@ public class StandardMemoryStrategy extends MemoryStrategy {
* @param last time which must be passed since lased gc * @param last time which must be passed since lased gc
* @param info additional info for log * @param info additional info for log
*/ */
@Override
protected final synchronized boolean gc(final int last, final String info) { // thq protected final synchronized boolean gc(final int last, final String info) { // thq
assert last >= 10000; // too many forced GCs will cause bad execution performance assert last >= 10000; // too many forced GCs will cause bad execution performance
final long elapsed = System.currentTimeMillis() - lastGC; final long elapsed = System.currentTimeMillis() - lastGC;
@ -58,8 +59,8 @@ public class StandardMemoryStrategy extends MemoryStrategy {
System.gc(); System.gc();
lastGC = System.currentTimeMillis(); lastGC = System.currentTimeMillis();
final long after = free(); final long after = free();
gcs[gcs_pos++] = after - before; this.gcs[this.gcs_pos++] = after - before;
if (gcs_pos >= gcs.length) gcs_pos = 0; if (this.gcs_pos >= this.gcs.length) this.gcs_pos = 0;
if (log.isFine()) log.logInfo("[gc] before: " + Formatter.bytesToString(before) + if (log.isFine()) log.logInfo("[gc] before: " + Formatter.bytesToString(before) +
", after: " + Formatter.bytesToString(after) + ", after: " + Formatter.bytesToString(after) +
@ -80,7 +81,7 @@ public class StandardMemoryStrategy extends MemoryStrategy {
protected final long getAverageGCFree() { protected final long getAverageGCFree() {
long x = 0; long x = 0;
int y = 0; int y = 0;
for (final long gc : gcs) for (final long gc : this.gcs)
if (gc != 0) { if (gc != 0) {
x += gc; x += gc;
y++; y++;
@ -92,14 +93,16 @@ public class StandardMemoryStrategy extends MemoryStrategy {
* memory that is free without increasing of total memory taken from os * memory that is free without increasing of total memory taken from os
* @return bytes * @return bytes
*/ */
@Override
protected final long free() { protected final long free() {
return runtime.freeMemory(); return this.runtime.freeMemory();
} }
/** /**
* memory that is available including increasing total memory up to maximum * memory that is available including increasing total memory up to maximum
* @return bytes * @return bytes
*/ */
@Override
protected final long available() { protected final long available() {
return maxMemory() - total() + free(); return maxMemory() - total() + free();
} }
@ -108,18 +111,20 @@ public class StandardMemoryStrategy extends MemoryStrategy {
* maximum memory the Java virtual will allocate machine; may vary over time in some cases * maximum memory the Java virtual will allocate machine; may vary over time in some cases
* @return bytes * @return bytes
*/ */
@Override
protected final long maxMemory() protected final long maxMemory()
{ {
return runtime.maxMemory(); return this.runtime.maxMemory();
} }
/** /**
* currently allocated memory in the Java virtual machine; may vary over time * currently allocated memory in the Java virtual machine; may vary over time
* @return bytes * @return bytes
*/ */
@Override
protected final long total() protected final long total()
{ {
return runtime.totalMemory(); return this.runtime.totalMemory();
} }
/** /**
@ -141,6 +146,7 @@ public class StandardMemoryStrategy extends MemoryStrategy {
* @param force specifies whether a GC should be run even in case former GCs didn't provide enough memory * @param force specifies whether a GC should be run even in case former GCs didn't provide enough memory
* @return whether enough memory could be freed (or is free) or not * @return whether enough memory could be freed (or is free) or not
*/ */
@Override
protected boolean request(final long size, final boolean force, boolean shortStatus) { protected boolean request(final long size, final boolean force, boolean shortStatus) {
if (size <= 0) return true; if (size <= 0) return true;
final boolean r = request0(size, force); final boolean r = request0(size, force);
@ -181,42 +187,46 @@ public class StandardMemoryStrategy extends MemoryStrategy {
* memory that is currently bound in objects * memory that is currently bound in objects
* @return used bytes * @return used bytes
*/ */
@Override
protected final long used() { protected final long used() {
return total() - free(); return total() - free();
} }
@Override
protected boolean properState() { protected boolean properState() {
return proper; return this.proper;
} }
@Override
protected void resetProperState() { protected void resetProperState() {
proper = true; this.proper = true;
tresholdCount = 0; this.tresholdCount = 0;
} }
/** /**
* set the memory to be available * set the memory to be available
*/ */
@Override
protected void setProperMbyte(final long mbyte) { protected void setProperMbyte(final long mbyte) {
properMbyte = mbyte; this.properMbyte = mbyte;
tresholdCount = 0; this.tresholdCount = 0;
} }
private void checkProper(final long available) { private void checkProper(final long available) {
// disable proper state if memory is less than treshold - 4 times, maximum 11 minutes between each detection // disable proper state if memory is less than treshold - 4 times, maximum 11 minutes between each detection
if ((available >> 20) < properMbyte) { if ((available >> 20) < this.properMbyte) {
final long t = System.currentTimeMillis(); final long t = System.currentTimeMillis();
if(prevTreshold + 11L /* minutes */ * 60000L > t) { if(this.prevTreshold + 11L /* minutes */ * 60000L > t) {
tresholdCount++; this.tresholdCount++;
if(tresholdCount > 3 /* occurencies - 1 */) proper = false; if(this.tresholdCount > 3 /* occurencies - 1 */) this.proper = false;
} }
else tresholdCount = 1; else this.tresholdCount = 1;
prevTreshold = t; this.prevTreshold = t;
log.logInfo("checkProper: below treshold; tresholdCount: " + tresholdCount + "; proper: " + proper); log.logInfo("checkProper: below treshold; tresholdCount: " + this.tresholdCount + "; proper: " + this.proper);
} }
else if (!proper && (available >> 20) > (properMbyte * 2L)) // we were wrong! else if (!this.proper && (available >> 20) > (this.properMbyte * 2L)) // we were wrong!
resetProperState(); resetProperState();
} }

Write Preview
Loading…
Cancel
Save