diff --git a/source/de/anomic/server/serverCore.java b/source/de/anomic/server/serverCore.java index 4b4e19917..a235a9e4b 100644 --- a/source/de/anomic/server/serverCore.java +++ b/source/de/anomic/server/serverCore.java @@ -726,8 +726,22 @@ public final class serverCore extends AbstractBusyThread implements BusyThread { parameter[0] = this.request.trim(); } } - //long commandStart = System.currentTimeMillis(); - Object result = commandMethod.invoke(this.commandObj, parameter); + + Object result = null; + try { + result = commandMethod.invoke(this.commandObj, parameter); + } catch (OutOfMemoryError e) { + // try again + terminateOldSessions(2000); + try { + result = commandMethod.invoke(this.commandObj, parameter); + } catch (OutOfMemoryError e2) { + // try again + Thread.sleep(1000); + result = commandMethod.invoke(this.commandObj, parameter); + } + } + //announceMoreExecTime(commandStart - System.currentTimeMillis()); // shall be negative! //this.log.logDebug("* session " + handle + " completed command '" + request + "'. time = " + (System.currentTimeMillis() - handle)); this.out.flush(); diff --git a/source/net/yacy/cora/protocol/http/HTTPClient.java b/source/net/yacy/cora/protocol/http/HTTPClient.java index ec8f04a0b..e812a3146 100644 --- a/source/net/yacy/cora/protocol/http/HTTPClient.java +++ b/source/net/yacy/cora/protocol/http/HTTPClient.java @@ -445,8 +445,12 @@ public class HTTPClient { // get the response body final HttpEntity httpEntity = httpResponse.getEntity(); if (httpEntity != null) { - if (getStatusCode() == 200 && httpEntity.getContentLength() < maxBytes) { + if (getStatusCode() == 200 && httpEntity.getContentLength() < maxBytes) { + try { content = EntityUtils.toByteArray(httpEntity); + } catch (OutOfMemoryError e) { + throw new IOException(e.toString()); + } } // Ensures that the entity content is fully consumed and the content stream, if exists, is closed. EntityUtils.consume(httpEntity); diff --git a/source/net/yacy/kelondro/blob/BLOB.java b/source/net/yacy/kelondro/blob/BLOB.java index b832338df..4ee9e80e3 100644 --- a/source/net/yacy/kelondro/blob/BLOB.java +++ b/source/net/yacy/kelondro/blob/BLOB.java @@ -189,8 +189,9 @@ public interface BLOB { * of the input * @param b * @return an array that is equal or smaller in size than b + * @throws RowSpaceExceededException */ - public byte[] rewrite(byte[] b); + public byte[] rewrite(byte[] b) throws RowSpaceExceededException; } diff --git a/source/net/yacy/kelondro/blob/HeapModifier.java b/source/net/yacy/kelondro/blob/HeapModifier.java index 0c87006f7..5090aca47 100644 --- a/source/net/yacy/kelondro/blob/HeapModifier.java +++ b/source/net/yacy/kelondro/blob/HeapModifier.java @@ -28,6 +28,7 @@ import java.io.File; import java.io.IOException; import java.util.SortedMap; +import net.yacy.kelondro.index.RowSpaceExceededException; import net.yacy.kelondro.io.CachedFileWriter; import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.order.ByteOrder; @@ -242,7 +243,7 @@ public class HeapModifier extends HeapReader implements BLOB { throw new UnsupportedOperationException(); } - public int reduce(byte[] key, final Reducer reducer) throws IOException { + public int reduce(byte[] key, final Reducer reducer) throws IOException, RowSpaceExceededException { key = normalizeKey(key); assert key.length == this.keylength; diff --git a/source/net/yacy/kelondro/index/HandleSet.java b/source/net/yacy/kelondro/index/HandleSet.java index cba51f813..5e75d967b 100644 --- a/source/net/yacy/kelondro/index/HandleSet.java +++ b/source/net/yacy/kelondro/index/HandleSet.java @@ -64,7 +64,7 @@ public final class HandleSet implements Iterable, Cloneable { this.index = index; } - public HandleSet(Row rowdef, byte[] b) { + public HandleSet(Row rowdef, byte[] b) throws RowSpaceExceededException { this.rowdef = rowdef; this.index = RowSet.importRowSet(b, this.rowdef); } diff --git a/source/net/yacy/kelondro/index/RowSet.java b/source/net/yacy/kelondro/index/RowSet.java index cdc7c0599..5fb7b44ee 100644 --- a/source/net/yacy/kelondro/index/RowSet.java +++ b/source/net/yacy/kelondro/index/RowSet.java @@ -32,6 +32,7 @@ import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.order.Base64Order; import net.yacy.kelondro.order.CloneableIterator; import net.yacy.kelondro.order.NaturalOrder; +import net.yacy.kelondro.util.MemoryControl; public class RowSet extends RowCollection implements Index, Iterable { @@ -68,7 +69,7 @@ public class RowSet extends RowCollection implements Index, Iterable assert rowdef.objectOrder != null; } - public final static RowSet importRowSet(final byte[] b, final Row rowdef) { + public final static RowSet importRowSet(final byte[] b, final Row rowdef) throws RowSpaceExceededException { assert b.length >= exportOverheadSize : "b.length = " + b.length; if (b.length < exportOverheadSize) return new RowSet(rowdef); final int size = (int) NaturalOrder.decodeLong(b, 0, 4); @@ -80,7 +81,13 @@ public class RowSet extends RowCollection implements Index, Iterable long alloc = ((long) size) * ((long) rowdef.objectsize); assert alloc <= Integer.MAX_VALUE : "alloc = " + alloc; assert alloc == b.length - exportOverheadSize; - final byte[] chunkcache = new byte[(int) alloc]; + MemoryControl.request((int) alloc, true); + final byte[] chunkcache; + try { + chunkcache = new byte[(int) alloc]; + } catch (OutOfMemoryError e) { + throw new RowSpaceExceededException((int) alloc, "importRowSet"); + } //assert b.length - exportOverheadSize == size * rowdef.objectsize : "b.length = " + b.length + ", size * rowdef.objectsize = " + size * rowdef.objectsize; if (b.length - exportOverheadSize != alloc) { Log.logSevere("RowSet", "exportOverheadSize wrong: b.length = " + b.length + ", size * rowdef.objectsize = " + size * rowdef.objectsize); diff --git a/source/net/yacy/kelondro/rwi/ReferenceContainerArray.java b/source/net/yacy/kelondro/rwi/ReferenceContainerArray.java index 5eb655ff4..9e0381ad5 100644 --- a/source/net/yacy/kelondro/rwi/ReferenceContainerArray.java +++ b/source/net/yacy/kelondro/rwi/ReferenceContainerArray.java @@ -291,7 +291,7 @@ public final class ReferenceContainerArray { this.wordHash = wordHash; } - public byte[] rewrite(byte[] b) { + public byte[] rewrite(byte[] b) throws RowSpaceExceededException { if (b == null) return null; ReferenceContainer c = rewriter.reduce(new ReferenceContainer(factory, this.wordHash, RowSet.importRowSet(b, payloadrow))); if (c == null) return null; diff --git a/source/net/yacy/kelondro/rwi/ReferenceIterator.java b/source/net/yacy/kelondro/rwi/ReferenceIterator.java index 4cdc2ce3f..0415f330b 100644 --- a/source/net/yacy/kelondro/rwi/ReferenceIterator.java +++ b/source/net/yacy/kelondro/rwi/ReferenceIterator.java @@ -34,6 +34,7 @@ import java.util.Map; import net.yacy.kelondro.blob.HeapReader; import net.yacy.kelondro.index.Row; import net.yacy.kelondro.index.RowSet; +import net.yacy.kelondro.index.RowSpaceExceededException; import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.order.CloneableIterator; @@ -67,7 +68,12 @@ public class ReferenceIterator implements Clon public ReferenceContainer next() { Map.Entry entry = blobs.next(); byte[] payload = entry.getValue(); - return new ReferenceContainer(factory, entry.getKey(), RowSet.importRowSet(payload, payloadrow)); + try { + return new ReferenceContainer(factory, entry.getKey(), RowSet.importRowSet(payload, payloadrow)); + } catch (RowSpaceExceededException e) { + Log.logSevere("ReferenceIterator", "lost entry '" + entry.getKey() + "' because of too low memory: " + e.toString()); + return null; + } } public void remove() {