boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

better handling of OOM situations

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6918 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 15 years ago
parent 56ff9d5fd4
commit b03caaa57a
30 changed files with 228 additions and 158 deletions
Show Stats Download Patch File Download Diff File

3
htroot/Table_API_p.java
Unescape View File

@ -24,6 +24,7 @@ import java.util.Map;
import java.util.TreeSet;
import net.yacy.kelondro.blob.Tables;
import net.yacy.kelondro.index.RowSpaceExceededException;
import net.yacy.kelondro.logging.Log;
import de.anomic.data.WorkTables;
@ -79,6 +80,8 @@ public class Table_API_p {
}
} catch (IOException e) {
Log.logException(e);
} catch (RowSpaceExceededException e) {
Log.logException(e);
}
}

7
htroot/Tables_p.java
Unescape View File

@ -26,6 +26,7 @@ import java.util.Map;
import java.util.regex.Pattern;
import net.yacy.kelondro.blob.Tables;
import net.yacy.kelondro.index.RowSpaceExceededException;
import net.yacy.kelondro.logging.Log;
import de.anomic.http.server.RequestHeader;
@ -146,6 +147,8 @@ public class Tables_p {
}
} catch (IOException e) {
Log.logException(e);
} catch (RowSpaceExceededException e) {
Log.logException(e);
}
} else if (post.containsKey("addrow")) try {
// get a new key
@ -153,6 +156,8 @@ public class Tables_p {
setEdit(sb, prop, table, pk, columns);
} catch (IOException e) {
Log.logException(e);
} catch (RowSpaceExceededException e) {
Log.logException(e);
} else {
prop.put("showtable", 1);
prop.put("showtable_table", table);
@ -209,7 +214,7 @@ public class Tables_p {
return prop;
}
private static void setEdit(final Switchboard sb, final serverObjects prop, final String table, final String pk, List<String> columns) throws IOException {
private static void setEdit(final Switchboard sb, final serverObjects prop, final String table, final String pk, List<String> columns) throws IOException, RowSpaceExceededException {
prop.put("showedit", 1);
prop.put("showedit_table", table);
prop.put("showedit_pk", pk);

3
source/de/anomic/crawler/CrawlProfile.java
Unescape View File

@ -208,6 +208,9 @@ public class CrawlProfile {
} catch (final IOException e) {
Log.logException(e);
return null;
} catch (RowSpaceExceededException e) {
Log.logException(e);
return null;
}
if (m == null) return null;
return new entry(m);

16
source/de/anomic/crawler/RobotsTxt.java
Unescape View File

@ -37,6 +37,7 @@ import java.util.concurrent.ConcurrentHashMap;
import net.yacy.cora.document.MultiProtocolURI;
import net.yacy.kelondro.blob.BEncodedHeap;
import net.yacy.kelondro.index.RowSpaceExceededException;
import net.yacy.kelondro.logging.Log;
import net.yacy.kelondro.util.ByteBuffer;
import net.yacy.kelondro.util.DateFormatter;
@ -82,7 +83,13 @@ public class RobotsTxt {
// this method will always return a non-null value
String urlHostPort = getHostPort(theURL);
RobotsEntry robotsTxt4Host = null;
Map<String, byte[]> record = this.robotsTable.get(this.robotsTable.encodedKey(urlHostPort));
Map<String, byte[]> record;
try {
record = this.robotsTable.get(this.robotsTable.encodedKey(urlHostPort));
} catch (RowSpaceExceededException e) {
Log.logException(e);
record = null;
}
if (record != null) robotsTxt4Host = new RobotsEntry(urlHostPort, record);
if (fetchOnlineIfNotAvailableOrNotFresh && (
@ -105,7 +112,12 @@ public class RobotsTxt {
// check the robots table again for all threads that come here because they waited for another one
// to complete a download
record = this.robotsTable.get(this.robotsTable.encodedKey(urlHostPort));
try {
record = this.robotsTable.get(this.robotsTable.encodedKey(urlHostPort));
} catch (RowSpaceExceededException e) {
Log.logException(e);
record = null;
}
if (record != null) robotsTxt4Host = new RobotsEntry(urlHostPort, record);
if (robotsTxt4Host != null &&
robotsTxt4Host.getLoadedDate() != null &&

5
source/de/anomic/data/BookmarkDate.java
Unescape View File

@ -35,6 +35,7 @@ import java.util.Map;
import de.anomic.data.bookmarksDB.Bookmark;
import net.yacy.kelondro.blob.MapHeap;
import net.yacy.kelondro.index.RowSpaceExceededException;
import net.yacy.kelondro.logging.Log;
import net.yacy.kelondro.order.NaturalOrder;
@ -57,8 +58,10 @@ public class BookmarkDate {
map = datesTable.get(date.getBytes());
} catch (final IOException e) {
map = null;
} catch (RowSpaceExceededException e) {
map = null;
}
if (map==null) return new Entry(date);
if (map == null) return new Entry(date);
return new Entry(date, map);
}

3
source/de/anomic/data/WorkTables.java
Unescape View File

@ -31,6 +31,7 @@ import java.io.IOException;
import java.util.Date;
import net.yacy.kelondro.blob.Tables;
import net.yacy.kelondro.index.RowSpaceExceededException;
import net.yacy.kelondro.logging.Log;
import net.yacy.kelondro.util.DateFormatter;
import de.anomic.server.serverObjects;
@ -66,6 +67,8 @@ public class WorkTables extends Tables {
);
} catch (IOException e) {
Log.logException(e);
} catch (RowSpaceExceededException e) {
Log.logException(e);
}
Log.logInfo("APICALL", apiurl);
}

4
source/de/anomic/data/blogBoard.java
Unescape View File

@ -142,6 +142,10 @@ public class blogBoard {
try {
record = base.get(normalized.substring(0, Math.min(normalized.length(), KEY_LENGTH)).getBytes());
} catch (final IOException e) {
Log.logException(e);
record = null;
} catch (RowSpaceExceededException e) {
Log.logException(e);
record = null;
}
return (record == null) ?

5
source/de/anomic/data/blogBoardComments.java
Unescape View File

@ -43,6 +43,7 @@ import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import net.yacy.kelondro.blob.MapHeap;
import net.yacy.kelondro.index.RowSpaceExceededException;
import net.yacy.kelondro.logging.Log;
import net.yacy.kelondro.order.Base64Order;
import net.yacy.kelondro.order.NaturalOrder;
@ -130,6 +131,10 @@ public class blogBoardComments {
try {
record = base.get(copyOfKey.getBytes());
} catch (final IOException e) {
Log.logException(e);
record = null;
} catch (RowSpaceExceededException e) {
Log.logException(e);
record = null;
}
return (record == null) ?

5
source/de/anomic/data/bookmarksDB.java
Unescape View File

@ -46,6 +46,7 @@ import java.util.regex.Pattern;
import net.yacy.kelondro.blob.MapHeap;
import net.yacy.kelondro.data.meta.DigestURI;
import net.yacy.kelondro.index.RowSpaceExceededException;
import net.yacy.kelondro.logging.Log;
import net.yacy.kelondro.order.NaturalOrder;
import net.yacy.kelondro.util.DateFormatter;
@ -332,6 +333,10 @@ public class bookmarksDB {
final Map<String, String> map = bookmarks.get(urlHash.getBytes());
return (map == null) ? null : new Bookmark(map);
} catch (final IOException e) {
Log.logException(e);
return null;
} catch (RowSpaceExceededException e) {
Log.logException(e);
return null;
}
}

5
source/de/anomic/data/messageBoard.java
Unescape View File

@ -33,6 +33,7 @@ import java.util.Map;
import java.util.TimeZone;
import net.yacy.kelondro.blob.MapHeap;
import net.yacy.kelondro.index.RowSpaceExceededException;
import net.yacy.kelondro.logging.Log;
import net.yacy.kelondro.order.Base64Order;
import net.yacy.kelondro.order.NaturalOrder;
@ -195,6 +196,10 @@ public class messageBoard {
try {
record = database.get(key.getBytes());
} catch (final IOException e) {
Log.logException(e);
return null;
} catch (RowSpaceExceededException e) {
Log.logException(e);
return null;
}
return new entry(key, record);

4
source/de/anomic/data/userDB.java
Unescape View File

@ -97,6 +97,10 @@ public final class userDB {
try {
record = userTable.get(userName.getBytes());
} catch (final IOException e) {
Log.logException(e);
return null;
} catch (RowSpaceExceededException e) {
Log.logException(e);
return null;
}
if (record == null) return null;

5
source/de/anomic/data/wiki/wikiBoard.java
Unescape View File

@ -36,6 +36,7 @@ import java.util.Map;
import java.util.TimeZone;
import net.yacy.kelondro.blob.MapHeap;
import net.yacy.kelondro.index.RowSpaceExceededException;
import net.yacy.kelondro.logging.Log;
import net.yacy.kelondro.order.Base64Order;
import net.yacy.kelondro.order.NaturalOrder;
@ -287,6 +288,10 @@ public class wikiBoard {
if (record == null) return newEntry(key, "anonymous", "127.0.0.1", "New Page", "".getBytes());
return new entry(key, record);
} catch (final IOException e) {
Log.logException(e);
return null;
} catch (RowSpaceExceededException e) {
Log.logException(e);
return null;
}
}

7
source/de/anomic/http/client/Cache.java
Unescape View File

@ -45,6 +45,7 @@ import net.yacy.kelondro.blob.Compressor;
import net.yacy.kelondro.blob.MapHeap;
import net.yacy.kelondro.data.meta.DigestURI;
import net.yacy.kelondro.data.word.Word;
import net.yacy.kelondro.index.RowSpaceExceededException;
import net.yacy.kelondro.logging.Log;
import net.yacy.kelondro.order.Base64Order;
@ -175,6 +176,9 @@ public final class Cache {
} catch (final IOException e) {
Log.logException(e);
return null;
} catch (RowSpaceExceededException e) {
Log.logException(e);
return null;
}
if (hdb == null) return null;
@ -203,6 +207,9 @@ public final class Cache {
} catch (IOException e) {
Log.logException(e);
return null;
} catch (RowSpaceExceededException e) {
Log.logException(e);
return null;
}
}

4
source/de/anomic/yacy/yacyNewsDB.java
Unescape View File

@ -260,7 +260,7 @@ public class yacyNewsDB {
private final Map<String, String> attributes; // elements of the news for a special category
public Record(final String newsString) {
protected Record(final String newsString) {
this.attributes = MapTools.string2map(newsString, ",");
if (attributes.toString().length() > attributesMaxLength) throw new IllegalArgumentException("attributes length (" + attributes.toString().length() + ") exceeds maximum (" + attributesMaxLength + ")");
this.category = (attributes.containsKey("cat")) ? attributes.get("cat") : "";
@ -272,7 +272,7 @@ public class yacyNewsDB {
removeStandards();
}
public Record(final yacySeed mySeed, final String category, final Map<String, String> attributes) {
protected Record(final yacySeed mySeed, final String category, final Map<String, String> attributes) {
if (category.length() > yacyNewsDB.categoryStringLength) throw new IllegalArgumentException("category length (" + category.length() + ") exceeds maximum (" + yacyNewsDB.categoryStringLength + ")");
if (attributes.toString().length() > attributesMaxLength) throw new IllegalArgumentException("attributes length (" + attributes.toString().length() + ") exceeds maximum (" + attributesMaxLength + ")");
this.attributes = attributes;

3
source/de/anomic/yacy/yacySeedDB.java
Unescape View File

@ -554,6 +554,9 @@ public final class yacySeedDB implements AlternativeDomainNames {
} catch (final IOException e) {
Log.logException(e);
return null;
} catch (RowSpaceExceededException e) {
Log.logException(e);
return null;
}
return new yacySeed(hash, entry);
}

57
source/net/yacy/kelondro/blob/ArrayStack.java
Unescape View File

@ -60,6 +60,7 @@ import net.yacy.kelondro.rwi.ReferenceFactory;
import net.yacy.kelondro.rwi.ReferenceIterator;
import net.yacy.kelondro.util.DateFormatter;
import net.yacy.kelondro.util.FileUtils;
import net.yacy.kelondro.util.LookAheadIterator;
import net.yacy.kelondro.util.NamePrefixThreadFactory;
@ -574,7 +575,7 @@ public class ArrayStack implements BLOB {
* @return
* @throws IOException
*/
public synchronized byte[] get(byte[] key) throws IOException {
public synchronized byte[] get(byte[] key) throws IOException, RowSpaceExceededException {
//blobItem bi = keeperOf(key);
//return (bi == null) ? null : bi.blob.get(key);
@ -595,65 +596,36 @@ public class ArrayStack implements BLOB {
* @throws IOException
*/
public Iterable<byte[]> getAll(byte[] key) throws IOException {
/*
byte[] b;
ArrayList<byte[]> l = new ArrayList<byte[]>(blobs.size());
for (blobItem bi: blobs) {
b = bi.blob.get(key);
if (b != null) l.add(b);
}
return l;
*/
return new BlobValues(key);
}
public class BlobValues implements Iterator<byte[]>, Iterable<byte[]> {
public class BlobValues extends LookAheadIterator<byte[]> {
private final Iterator<blobItem> bii;
private final byte[] key;
private byte[] next;
public BlobValues(byte[] key) {
this.bii = blobs.iterator();
this.key = key;
this.next = null;
next0();
}
private void next0() {
protected byte[] next0() {
while (this.bii.hasNext()) {
BLOB b = this.bii.next().blob;
if (b == null) continue;
try {
this.next = b.get(key);
if (this.next != null) return;
byte[] n = b.get(key);
if (n != null) return n;
} catch (IOException e) {
Log.logSevere("ArrayStack", "", e);
this.next = null;
return;
return null;
} catch (RowSpaceExceededException e) {
continue;
}
}
this.next = null;
}
public Iterator<byte[]> iterator() {
return this;
}
public boolean hasNext() {
return this.next != null;
}
public byte[] next() {
byte[] n = this.next;
next0();
return n;
}
public void remove() {
throw new UnsupportedOperationException("no remove in BlobValues");
}
return null;
}
}
/**
@ -692,7 +664,7 @@ public class ArrayStack implements BLOB {
* @throws IOException
* @throws RowSpaceExceededException
*/
public synchronized void put(byte[] key, byte[] b) throws IOException, RowSpaceExceededException {
public synchronized void put(byte[] key, byte[] b) throws IOException {
blobItem bi = (blobs.isEmpty()) ? null : blobs.get(blobs.size() - 1);
if (bi == null)
System.out.println("bi == null");
@ -714,8 +686,9 @@ public class ArrayStack implements BLOB {
* replace a BLOB entry with another which must be smaller or same size
* @param key the primary key
* @throws IOException
* @throws RowSpaceExceededException
*/
public synchronized int replace(byte[] key, Rewriter rewriter) throws IOException {
public synchronized int replace(byte[] key, Rewriter rewriter) throws IOException, RowSpaceExceededException {
int d = 0;
for (blobItem bi: blobs) {
d += bi.blob.replace(key, rewriter);
@ -1013,8 +986,6 @@ public class ArrayStack implements BLOB {
heap.close(true);
} catch (final IOException e) {
Log.logException(e);
} catch (RowSpaceExceededException e) {
Log.logException(e);
}
}

12
source/net/yacy/kelondro/blob/BEncodedHeap.java
Unescape View File

@ -118,7 +118,7 @@ public class BEncodedHeap implements Iterable<Map.Entry<byte[], Map<String, byte
public void put(
byte[] pk,
String key, byte[] value
) throws RowSpaceExceededException, IOException {
) throws IOException {
byte[] b = BEncoder.encodeMap(key, value);
this.table.put(pk, b);
this.columnames.add(key);
@ -127,7 +127,7 @@ public class BEncodedHeap implements Iterable<Map.Entry<byte[], Map<String, byte
byte[] pk,
String key0, byte[] value0,
String key1, byte[] value1
) throws RowSpaceExceededException, IOException {
) throws IOException {
byte[] b = BEncoder.encodeMap(
key0, value0,
key1, value1
@ -141,7 +141,7 @@ public class BEncodedHeap implements Iterable<Map.Entry<byte[], Map<String, byte
String key0, byte[] value0,
String key1, byte[] value1,
String key2, byte[] value2
) throws RowSpaceExceededException, IOException {
) throws IOException {
byte[] b = BEncoder.encodeMap(
key0, value0,
key1, value1,
@ -158,7 +158,7 @@ public class BEncodedHeap implements Iterable<Map.Entry<byte[], Map<String, byte
String key1, byte[] value1,
String key2, byte[] value2,
String key3, byte[] value3
) throws RowSpaceExceededException, IOException {
) throws IOException {
byte[] b = BEncoder.encodeMap(
key0, value0,
key1, value1,
@ -178,13 +178,13 @@ public class BEncodedHeap implements Iterable<Map.Entry<byte[], Map<String, byte
* @return the map if one found or NULL if no entry exists or the entry is corrupt
* @throws IOException
*/
public Map<String, byte[]> get(byte[] pk) throws IOException {
public Map<String, byte[]> get(byte[] pk) throws IOException, RowSpaceExceededException {
byte[] b = this.table.get(pk);
if (b == null) return null;
return b2m(b);
}
public byte[] getProp(byte[] pk, String key) throws IOException {
public byte[] getProp(byte[] pk, String key) throws IOException, RowSpaceExceededException {
byte[] b = this.table.get(pk);
if (b == null) return null;
Map<String, byte[]> map = b2m(b);

7
source/net/yacy/kelondro/blob/BLOB.java
Unescape View File

@ -103,7 +103,7 @@ public interface BLOB {
* @return
* @throws IOException
*/
public byte[] get(byte[] key) throws IOException;
public byte[] get(byte[] key) throws IOException, RowSpaceExceededException;
/**
* retrieve the size of the database
@ -128,7 +128,7 @@ public interface BLOB {
* @throws IOException
* @throws RowSpaceExceededException
*/
public void put(byte[] key, byte[] b) throws IOException, RowSpaceExceededException;
public void put(byte[] key, byte[] b) throws IOException;
/**
* replace an existing entry in the BLOB with a new entry
@ -142,8 +142,9 @@ public interface BLOB {
* @param b
* @return the number of bytes that the rewriter reduced the BLOB
* @throws IOException
* @throws RowSpaceExceededException
*/
public int replace(byte[] key, Rewriter rewriter) throws IOException;
public int replace(byte[] key, Rewriter rewriter) throws IOException, RowSpaceExceededException;
/**
* remove a BLOB

19
source/net/yacy/kelondro/blob/Compressor.java
Unescape View File

@ -103,9 +103,6 @@ public class Compressor implements BLOB {
while ((entry = writeQueue.take()) != poisonWorkerEntry) {
try {
Compressor.this.backend.put(entry.getKey().getBytes(), compress(entry.getValue()));
} catch (RowSpaceExceededException e) {
Log.logException(e);
buffer.put(entry.getKey(), entry.getValue());
} catch (IOException e) {
Log.logException(e);
buffer.put(entry.getKey(), entry.getValue());
@ -223,7 +220,7 @@ public class Compressor implements BLOB {
}
}
public synchronized byte[] get(byte[] key) throws IOException {
public synchronized byte[] get(byte[] key) throws IOException, RowSpaceExceededException {
// depending on the source of the result, we additionally do entry compression
// because if a document was read once, we think that it will not be retrieved another time again soon
byte[] b = buffer.remove(new String(key));
@ -266,10 +263,14 @@ public class Compressor implements BLOB {
public synchronized long length(byte[] key) throws IOException {
byte[] b = buffer.get(new String(key));
if (b != null) return b.length;
b = this.backend.get(key);
if (b == null) return 0;
b = decompress(b);
return (b == null) ? 0 : b.length;
try {
b = this.backend.get(key);
if (b == null) return 0;
b = decompress(b);
return (b == null) ? 0 : b.length;
} catch (RowSpaceExceededException e) {
throw new IOException(e.getMessage());
}
}
private int removeFromQueues(byte[] key) {
@ -350,7 +351,7 @@ public class Compressor implements BLOB {
}
}
public int replace(byte[] key, Rewriter rewriter) throws IOException {
public int replace(byte[] key, Rewriter rewriter) throws IOException, RowSpaceExceededException {
byte[] b = get(key);
if (b == null) return 0;
byte[] c = rewriter.rewrite(b);

55
source/net/yacy/kelondro/blob/Heap.java
Unescape View File

@ -137,15 +137,19 @@ public final class Heap extends HeapModifier implements BLOB {
* @throws IOException
* @throws RowSpaceExceededException
*/
private void add(byte[] key, final byte[] blob) throws IOException, RowSpaceExceededException {
private void add(byte[] key, final byte[] blob) throws IOException {
assert blob.length > 0;
if ((blob == null) || (blob.length == 0)) return;
final int pos = (int) file.length();
index.put(key, pos);
file.seek(pos);
file.writeInt(this.keylength + blob.length);
file.write(key);
file.write(blob, 0, blob.length);
try {
index.put(key, pos);
file.seek(pos);
file.writeInt(this.keylength + blob.length);
file.write(key);
file.write(blob, 0, blob.length);
} catch (RowSpaceExceededException e) {
throw new IOException(e.getMessage()); // should never occur;
}
}
/**
@ -154,7 +158,7 @@ public final class Heap extends HeapModifier implements BLOB {
* @throws IOException
* @throws RowSpaceExceededException
*/
private void flushBuffer() throws IOException, RowSpaceExceededException {
private void flushBuffer() throws IOException {
// check size of buffer
Iterator<Map.Entry<byte[], byte[]>> i = this.buffer.entrySet().iterator();
int l = 0;
@ -181,11 +185,17 @@ public final class Heap extends HeapModifier implements BLOB {
posBuffer = 0;
byte[] ba = new byte[l + (4 + this.keylength) * this.buffer.size()];
byte[] b;
while (i.hasNext()) {
TreeMap<byte[], byte[]> nextBuffer = new TreeMap<byte[], byte[]>(ordering);
flush: while (i.hasNext()) {
entry = i.next();
key = normalizeKey(entry.getKey());
blob = entry.getValue();
index.put(key, posFile);
try {
index.put(key, posFile);
} catch (RowSpaceExceededException e) {
nextBuffer.put(entry.getKey(), blob);
continue flush;
}
b = AbstractWriter.int2array(this.keylength + blob.length);
assert b.length == 4;
assert posBuffer + 4 < ba.length : "posBuffer = " + posBuffer + ", ba.length = " + ba.length;
@ -202,7 +212,7 @@ public final class Heap extends HeapModifier implements BLOB {
assert ba.length == posBuffer; // must fit exactly
this.file.seek(pos);
this.file.write(ba);
this.buffer.clear();
this.buffer = nextBuffer;
this.buffersize = 0;
}
@ -213,7 +223,7 @@ public final class Heap extends HeapModifier implements BLOB {
* @throws IOException
*/
@Override
public byte[] get(byte[] key) throws IOException {
public byte[] get(byte[] key) throws IOException, RowSpaceExceededException {
key = normalizeKey(key);
synchronized (this) {
@ -265,8 +275,6 @@ public final class Heap extends HeapModifier implements BLOB {
flushBuffer();
} catch (IOException e) {
Log.logException(e);
} catch (RowSpaceExceededException e) {
Log.logException(e);
}
}
this.buffer = null;
@ -294,9 +302,10 @@ public final class Heap extends HeapModifier implements BLOB {
* @param b
* @throws IOException
* @throws RowSpaceExceededException
* @throws RowSpaceExceededException
*/
@Override
public void put(byte[] key, final byte[] b) throws IOException, RowSpaceExceededException {
public void put(byte[] key, final byte[] b) throws IOException {
key = normalizeKey(key);
// we do not write records of length 0 into the BLOB
@ -308,7 +317,9 @@ public final class Heap extends HeapModifier implements BLOB {
this.remove(key);
// then look if we can use a free entry
if (putToGap(key, b)) return;
try {
if (putToGap(key, b)) return;
} catch (RowSpaceExceededException e) {} // too less space can be ignored, we have a second try
// if there is not enough space in the buffer, flush all
if (this.buffersize + b.length > buffermax) {
@ -446,11 +457,7 @@ public final class Heap extends HeapModifier implements BLOB {
*/
@Override
public synchronized CloneableIterator<byte[]> keys(final boolean up, final boolean rotating) throws IOException {
try {
this.flushBuffer();
} catch (RowSpaceExceededException e) {
Log.logException(e);
}
this.flushBuffer();
return super.keys(up, rotating);
}
@ -463,11 +470,7 @@ public final class Heap extends HeapModifier implements BLOB {
*/
@Override
public synchronized CloneableIterator<byte[]> keys(final boolean up, final byte[] firstKey) throws IOException {
try {
this.flushBuffer();
} catch (RowSpaceExceededException e) {
Log.logException(e);
}
this.flushBuffer();
return super.keys(up, firstKey);
}
@ -500,8 +503,6 @@ public final class Heap extends HeapModifier implements BLOB {
heap.close(true);
} catch (final IOException e) {
Log.logException(e);
} catch (RowSpaceExceededException e) {
Log.logException(e);
}
}

3
source/net/yacy/kelondro/blob/HeapModifier.java
Unescape View File

@ -28,7 +28,6 @@ import java.io.File;
import java.io.IOException;
import java.util.SortedMap;
import net.yacy.kelondro.index.RowSpaceExceededException;
import net.yacy.kelondro.io.CachedFileWriter;
import net.yacy.kelondro.logging.Log;
import net.yacy.kelondro.order.ByteOrder;
@ -235,7 +234,7 @@ public class HeapModifier extends HeapReader implements BLOB {
}
}
public void put(byte[] key, byte[] b) throws IOException, RowSpaceExceededException {
public void put(byte[] key, byte[] b) throws IOException {
throw new UnsupportedOperationException("put is not supported in BLOBHeapModifier");
}

8
source/net/yacy/kelondro/blob/HeapReader.java
Unescape View File

@ -344,7 +344,7 @@ public class HeapReader {
* @return the entry which key is the smallest in the heap
* @throws IOException
*/
protected byte[] first() throws IOException {
protected byte[] first() throws IOException, RowSpaceExceededException {
synchronized (this.index) {
byte[] key = index.smallestKey();
if (key == null) return null;
@ -372,7 +372,7 @@ public class HeapReader {
* @return the entry which key is the smallest in the heap
* @throws IOException
*/
protected byte[] last() throws IOException {
protected byte[] last() throws IOException, RowSpaceExceededException {
synchronized (this.index) {
byte[] key = index.largestKey();
if (key == null) return null;
@ -386,7 +386,7 @@ public class HeapReader {
* @return
* @throws IOException
*/
public byte[] get(byte[] key) throws IOException {
public byte[] get(byte[] key) throws IOException, RowSpaceExceededException {
key = normalizeKey(key);
synchronized (this.index) {
@ -398,7 +398,7 @@ public class HeapReader {
file.seek(pos);
final int len = file.readInt() - index.row().primaryKeyLength;
if (MemoryControl.available() < len * 2 + keepFreeMem) {
if (!MemoryControl.request(len * 2 + keepFreeMem, true)) return null; // not enough memory available for this blob
if (!MemoryControl.request(len * 2 + keepFreeMem, true)) throw new RowSpaceExceededException(len * 2 + keepFreeMem, "HeapReader.get()"); // not enough memory available for this blob
}
// read the key

29
source/net/yacy/kelondro/blob/MapDataMining.java
Unescape View File

@ -35,6 +35,7 @@ import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import net.yacy.kelondro.index.RowSpaceExceededException;
import net.yacy.kelondro.logging.Log;
import net.yacy.kelondro.order.ByteOrder;
import net.yacy.kelondro.order.CloneableIterator;
import net.yacy.kelondro.util.ScoreCluster;
@ -107,7 +108,12 @@ public class MapDataMining extends MapHeap {
Map<String, String> map;
while (it.hasNext()) {
mapnameb = it.next();
map = super.get(mapnameb);
try {
map = super.get(mapnameb);
} catch (RowSpaceExceededException e) {
Log.logWarning("MapDataMining", e.getMessage());
break;
}
if (map == null) break;
if (sortfields != null && cluster != null) for (int i = 0; i < sortfields.length; i++) {
@ -250,14 +256,20 @@ public class MapDataMining extends MapHeap {
// update elementCount
if ((sortfields != null) || (longaccfields != null) || (doubleaccfields != null)) {
final Map<String, String> map = super.get(key);
if (map != null) {
Map<String, String> map;
try {
map = super.get(key);
if (map != null) {
// update accumulators (subtract)
if ((longaccfields != null) || (doubleaccfields != null)) updateAcc(map, false);
// update accumulators (subtract)
if ((longaccfields != null) || (doubleaccfields != null)) updateAcc(map, false);
// remove from sortCluster
if (sortfields != null) deleteSortCluster(new String(key));
// remove from sortCluster
if (sortfields != null) deleteSortCluster(new String(key));
}
} catch (RowSpaceExceededException e) {
map = null;
Log.logException(e);
}
}
super.remove(key);
@ -383,6 +395,9 @@ public class MapDataMining extends MapHeap {
map = get(nextKey);
} catch (final IOException e) {
break;
} catch (RowSpaceExceededException e) {
Log.logException(e);
continue;
}
assert map != null;
if (map == null) continue; // circumvention of a modified exception

7
source/net/yacy/kelondro/blob/MapHeap.java
Unescape View File

@ -189,7 +189,7 @@ public class MapHeap {
* @return
* @throws IOException
*/
public Map<String, String> get(final byte[] key) throws IOException {
public Map<String, String> get(final byte[] key) throws IOException, RowSpaceExceededException {
if (key == null) return null;
return get(key, true);
}
@ -210,7 +210,7 @@ public class MapHeap {
return key;
}
protected Map<String, String> get(byte[] key, final boolean storeCache) throws IOException {
protected Map<String, String> get(byte[] key, final boolean storeCache) throws IOException, RowSpaceExceededException {
// load map from cache
assert key != null;
if (cache == null) return null; // case may appear during shutdown
@ -350,6 +350,9 @@ public class MapHeap {
} catch (final IOException e) {
finish = true;
return null;
} catch (final RowSpaceExceededException e) {
finish = true;
return null;
}
}

15
source/net/yacy/kelondro/blob/Stack.java
Unescape View File

@ -29,6 +29,7 @@ import java.io.IOException;
import java.util.Iterator;
import net.yacy.kelondro.index.RowSpaceExceededException;
import net.yacy.kelondro.logging.Log;
import net.yacy.kelondro.order.NaturalOrder;
@ -116,8 +117,9 @@ public class Stack {
* @return the object that belongs to the handle
* or null if no such element exists
* @throws IOException
* @throws RowSpaceExceededException
*/
public synchronized byte[] get(final long handle) throws IOException {
public synchronized byte[] get(final long handle) throws IOException, RowSpaceExceededException {
byte[] k = NaturalOrder.encodeLong(handle, 8);
byte[] b = this.stack.get(k);
if (b == null) return null;
@ -129,8 +131,9 @@ public class Stack {
* @param handle
* @return the removed element
* @throws IOException
* @throws RowSpaceExceededException
*/
public synchronized byte[] remove(final long handle) throws IOException {
public synchronized byte[] remove(final long handle) throws IOException, RowSpaceExceededException {
byte[] k = NaturalOrder.encodeLong(handle, 8);
byte[] b = this.stack.get(k);
if (b == null) return null;
@ -181,7 +184,13 @@ public class Stack {
private Entry po(final byte[] k, final boolean remove) throws IOException {
if (k == null) return null;
assert k.length == 8;
byte[] b = this.stack.get(k);
byte[] b;
try {
b = this.stack.get(k);
} catch (RowSpaceExceededException e) {
Log.logException(e);
b = null;
}
assert b != null;
if (b == null) return null;
if (remove) this.stack.remove(k);

6
source/net/yacy/kelondro/blob/Stacks.java
Unescape View File

@ -178,8 +178,9 @@ public class Stacks {
* @return the object that belongs to the handle
* or null if no such element exists
* @throws IOException
* @throws RowSpaceExceededException
*/
public byte[] get(final String stack, final long handle) throws IOException {
public byte[] get(final String stack, final long handle) throws IOException, RowSpaceExceededException {
Stack s = getStack(stack);
if (s == null) return null;
return s.get(handle);
@ -191,8 +192,9 @@ public class Stacks {
* @param handle
* @return the removed element
* @throws IOException
* @throws RowSpaceExceededException
*/
public byte[] remove(final String stack, final long handle) throws IOException {
public byte[] remove(final String stack, final long handle) throws IOException, RowSpaceExceededException {
Stack s = getStack(stack);
if (s == null) return null;
return s.remove(handle);

65
source/net/yacy/kelondro/blob/Tables.java
Unescape View File

@ -130,7 +130,7 @@ public class Tables {
return heap.size();
}
private byte[] ukey(String tablename) throws IOException {
private byte[] ukey(String tablename) throws IOException, RowSpaceExceededException {
Row row = select(system_table_pkcounter, tablename.getBytes());
if (row == null) {
// table counter entry in pkcounter table does not exist: make a new table entry
@ -166,15 +166,16 @@ public class Tables {
* @param map
* @throws RowSpaceExceededException
* @throws IOException
* @throws RowSpaceExceededException
*/
public byte[] insert(final String tablename, Map<String, byte[]> map) throws IOException {
public byte[] insert(final String tablename, Map<String, byte[]> map) throws IOException, RowSpaceExceededException {
byte[] uk = ukey(tablename);
insert(tablename, uk, map);
insert(system_table_pkcounter, tablename.getBytes(), system_table_pkcounter_counterName, uk);
return uk;
}
public byte[] insert(final String tablename, String key, byte[] value) throws IOException {
public byte[] insert(final String tablename, String key, byte[] value) throws IOException, RowSpaceExceededException {
byte[] uk = ukey(tablename);
insert(tablename, uk, key, value);
insert(system_table_pkcounter, tablename.getBytes(), system_table_pkcounter_counterName, uk);
@ -184,7 +185,7 @@ public class Tables {
public byte[] insert(final String tablename,
String key0, byte[] value0,
String key1, byte[] value1
) throws IOException {
) throws IOException, RowSpaceExceededException {
byte[] uk = ukey(tablename);
insert(tablename, uk,
key0, value0,
@ -198,7 +199,7 @@ public class Tables {
String key0, byte[] value0,
String key1, byte[] value1,
String key2, byte[] value2
) throws IOException {
) throws IOException, RowSpaceExceededException {
byte[] uk = ukey(tablename);
insert(tablename, uk,
key0, value0,
@ -214,7 +215,7 @@ public class Tables {
String key1, byte[] value1,
String key2, byte[] value2,
String key3, byte[] value3
) throws IOException {
) throws IOException, RowSpaceExceededException {
byte[] uk = ukey(tablename);
insert(tablename, uk,
key0, value0,
@ -230,11 +231,7 @@ public class Tables {
String key, byte[] value
) throws IOException {
BEncodedHeap heap = getHeap(table);
try {
heap.put(pk, key, value);
} catch (RowSpaceExceededException e) {
throw new IOException(e.getMessage());
}
heap.put(pk, key, value);
}
public void insert(final String table, byte[] pk,
@ -242,14 +239,10 @@ public class Tables {
String key1, byte[] value1
) throws IOException {
BEncodedHeap heap = getHeap(table);
try {
heap.put(pk,
key0, value0,
key1, value1
);
} catch (RowSpaceExceededException e) {
throw new IOException(e.getMessage());
}
heap.put(pk,
key0, value0,
key1, value1
);
}
public void insert(final String table, byte[] pk,
@ -258,15 +251,11 @@ public class Tables {
String key2, byte[] value2
) throws IOException {
BEncodedHeap heap = getHeap(table);
try {
heap.put(pk,
key0, value0,
key1, value1,
key2, value2
);
} catch (RowSpaceExceededException e) {
throw new IOException(e.getMessage());
}
heap.put(pk,
key0, value0,
key1, value1,
key2, value2
);
}
public void insert(final String table, byte[] pk,
@ -276,16 +265,12 @@ public class Tables {
String key3, byte[] value3
) throws IOException {
BEncodedHeap heap = getHeap(table);
try {
heap.put(pk,
key0, value0,
key1, value1,
key2, value2,
key3, value3
);
} catch (RowSpaceExceededException e) {
throw new IOException(e.getMessage());
}
heap.put(pk,
key0, value0,
key1, value1,
key2, value2,
key3, value3
);
}
public void insert(final String table, byte[] pk, Map<String, byte[]> map) throws IOException {
@ -306,11 +291,11 @@ public class Tables {
}
}
public byte[] createRow(String table) throws IOException {
public byte[] createRow(String table) throws IOException, RowSpaceExceededException {
return this.insert(table, new ConcurrentHashMap<String, byte[]>());
}
public Row select(final String table, byte[] pk) throws IOException {
public Row select(final String table, byte[] pk) throws IOException, RowSpaceExceededException {
BEncodedHeap heap = getHeap(table);
if (heap.has(pk)) return new Row(pk, heap.get(pk));
return null;

16
source/net/yacy/kelondro/rwi/IndexCell.java
Unescape View File

@ -248,13 +248,25 @@ public final class IndexCell<ReferenceType extends Reference> extends AbstractBu
*/
public int remove(byte[] termHash, HandleSet urlHashes) throws IOException {
int removed = this.ram.remove(termHash, urlHashes);
int reduced = this.array.replace(termHash, new RemoveRewriter<ReferenceType>(urlHashes));
int reduced;
try {
reduced = this.array.replace(termHash, new RemoveRewriter<ReferenceType>(urlHashes));
} catch (RowSpaceExceededException e) {
reduced = 0;
Log.logWarning("IndexCell", "not possible to remove urlHashes from a RWI because of too low memory. Remove was not applied. Please increase RAM assignment");
}
return removed + (reduced / this.array.rowdef().objectsize);
}
public boolean remove(byte[] termHash, byte[] urlHashBytes) throws IOException {
boolean removed = this.ram.remove(termHash, urlHashBytes);
int reduced = this.array.replace(termHash, new RemoveRewriter<ReferenceType>(urlHashBytes));
int reduced;
try {
reduced = this.array.replace(termHash, new RemoveRewriter<ReferenceType>(urlHashBytes));
} catch (RowSpaceExceededException e) {
reduced = 0;
Log.logWarning("IndexCell", "not possible to remove urlHashes from a RWI because of too low memory. Remove was not applied. Please increase RAM assignment");
}
return removed || (reduced > 0);
}

2
source/net/yacy/kelondro/rwi/ReferenceContainerArray.java
Unescape View File

@ -247,7 +247,7 @@ public final class ReferenceContainerArray<ReferenceType extends Reference> {
array.remove(termHash);
}
public int replace(final byte[] termHash, ContainerRewriter<ReferenceType> rewriter) throws IOException {
public int replace(final byte[] termHash, ContainerRewriter<ReferenceType> rewriter) throws IOException, RowSpaceExceededException {
return array.replace(termHash, new BLOBRewriter(termHash, rewriter));
}

6
source/net/yacy/kelondro/util/LookAheadIterator.java
Unescape View File

@ -35,13 +35,17 @@ import java.util.Iterator;
* latest return value of next0()
* To use this class just implement the next0() method
*/
public abstract class LookAheadIterator<A> implements Iterator<A> {
public abstract class LookAheadIterator<A> implements Iterator<A>, Iterable<A> {
private boolean fresh = true;
private A next = null;
public LookAheadIterator() {
}
public Iterator<A> iterator() {
return this;
}
/**
* the internal next-method

Write Preview
Loading…
Cancel
Save