From 078fcde0ddb072d7f1d20de78bf5d16096ba90a7 Mon Sep 17 00:00:00 2001 From: Michael Christen Date: Wed, 7 Dec 2011 01:02:23 +0100 Subject: [PATCH] bad initialization --- source/net/yacy/kelondro/blob/HeapReader.java | 169 +++++++++--------- 1 file changed, 88 insertions(+), 81 deletions(-) diff --git a/source/net/yacy/kelondro/blob/HeapReader.java b/source/net/yacy/kelondro/blob/HeapReader.java index 865ba8400..ce1af19a1 100644 --- a/source/net/yacy/kelondro/blob/HeapReader.java +++ b/source/net/yacy/kelondro/blob/HeapReader.java @@ -80,21 +80,21 @@ public class HeapReader { this.file = new CachedFileWriter(this.heapFile); // read or initialize the index - fingerprintFileIdx = null; - fingerprintFileGap = null; + this.fingerprintFileIdx = null; + this.fingerprintFileGap = null; if (initIndexReadDump()) { // verify that everything worked just fine // pick some elements of the index Iterator i = this.index.keys(true, null); int c = 3; - byte[] b, b1 = new byte[index.row().primaryKeyLength]; + byte[] b, b1 = new byte[this.index.row().primaryKeyLength]; long pos; boolean ok = true; while (i.hasNext() && c-- > 0) { b = i.next(); pos = this.index.get(b); - file.seek(pos + 4); - file.readFully(b1, 0, b1.length); + this.file.seek(pos + 4); + this.file.readFully(b1, 0, b1.length); if (!this.ordering.equal(b, b1)) { ok = false; break; @@ -122,7 +122,7 @@ public class HeapReader { } public long mem() { - return index.mem(); // don't add the memory for free here since then the asserts for memory management don't work + return this.index.mem(); // don't add the memory for free here since then the asserts for memory management don't work } public void trim() { @@ -181,7 +181,7 @@ public class HeapReader { // check saturation int[] saturation = this.index.saturation(); - Log.logInfo("HeapReader", "saturation of " + this.fingerprintFileIdx.getName() + ": keylength = " + saturation[0] + ", vallength = " + saturation[1] + ", possible saving: " + ((this.keylength - saturation[0] + 8 - saturation[1]) * index.size() / 1024 / 1024) + " MB"); + Log.logInfo("HeapReader", "saturation of " + this.fingerprintFileIdx.getName() + ": keylength = " + saturation[0] + ", vallength = " + saturation[1] + ", possible saving: " + ((this.keylength - saturation[0] + 8 - saturation[1]) * this.index.size() / 1024 / 1024) + " MB"); // read the gap file: try { @@ -239,31 +239,32 @@ public class HeapReader { private void initIndexReadFromHeap() throws IOException { // this initializes the this.index object by reading positions from the heap file - Log.logInfo("HeapReader", "generating index for " + heapFile.toString() + ", " + (file.length() / 1024 / 1024) + " MB. Please wait."); + Log.logInfo("HeapReader", "generating index for " + this.heapFile.toString() + ", " + (this.file.length() / 1024 / 1024) + " MB. Please wait."); this.free = new Gap(); - HandleMap.initDataConsumer indexready = HandleMap.asynchronusInitializer(this.name() + ".initializer", keylength, this.ordering, 8, Math.max(10, (int) (Runtime.getRuntime().freeMemory() / (10 * 1024 * 1024)))); - byte[] key = new byte[keylength]; + HandleMap.initDataConsumer indexready = HandleMap.asynchronusInitializer(this.name() + ".initializer", this.keylength, this.ordering, 8, Math.max(10, (int) (Runtime.getRuntime().freeMemory() / (10 * 1024 * 1024)))); + byte[] key = new byte[this.keylength]; int reclen; long seek = 0; + if (this.file.length() > 0) { loop: while (true) { // don't test available() here because this does not work for files > 2GB try { // go to seek position - file.seek(seek); + this.file.seek(seek); // read length of the following record without the length of the record size bytes - reclen = file.readInt(); + reclen = this.file.readInt(); //assert reclen > 0 : " reclen == 0 at seek pos " + seek; if (reclen == 0) { // very bad file inconsistency - Log.logSevere("HeapReader", "reclen == 0 at seek pos " + seek + " in file " + heapFile); + Log.logSevere("HeapReader", "reclen == 0 at seek pos " + seek + " in file " + this.heapFile); this.file.setLength(seek); // delete everything else at the remaining of the file :-( break loop; } // read key - file.readFully(key, 0, key.length); + this.file.readFully(key, 0, key.length); } catch (final IOException e) { // EOF reached @@ -273,23 +274,24 @@ public class HeapReader { // check if this record is empty if (key == null || key[0] == 0) { // it is an empty record, store to free list - if (reclen > 0) free.put(seek, reclen); + if (reclen > 0) this.free.put(seek, reclen); } else { if (this.ordering.wellformed(key)) { indexready.consume(key, seek); - key = new byte[keylength]; + key = new byte[this.keylength]; } else { // free the lost space - free.put(seek, reclen); - file.seek(seek + 4); + this.free.put(seek, reclen); + this.file.seek(seek + 4); Arrays.fill(key, (byte) 0); - file.write(key); // mark the place as empty record - Log.logWarning("HeapReader", "BLOB " + heapFile.getName() + ": skiped not wellformed key " + UTF8.String(key) + " at seek pos " + seek); + this.file.write(key); // mark the place as empty record + Log.logWarning("HeapReader", "BLOB " + this.heapFile.getName() + ": skiped not wellformed key " + UTF8.String(key) + " at seek pos " + seek); } } // new seek position seek += 4L + reclen; } + } indexready.finish(); // finish the index generation @@ -300,13 +302,13 @@ public class HeapReader { } catch (ExecutionException e) { Log.logException(e); } - Log.logInfo("HeapReader", "finished index generation for " + heapFile.toString() + ", " + index.size() + " entries, " + free.size() + " gaps."); + Log.logInfo("HeapReader", "finished index generation for " + this.heapFile.toString() + ", " + this.index.size() + " entries, " + this.free.size() + " gaps."); } private void mergeFreeEntries() throws IOException { // try to merge free entries - if (free.size() > 1) { + if (this.free.size() > 1) { int merged = 0; Map.Entry lastFree, nextFree; final Iterator> i = this.free.entrySet().iterator(); @@ -328,7 +330,7 @@ public class HeapReader { lastFree = nextFree; } } - Log.logInfo("HeapReader", "BLOB " + heapFile.toString() + ": merged " + merged + " free records"); + Log.logInfo("HeapReader", "BLOB " + this.heapFile.toString() + ": merged " + merged + " free records"); if (merged > 0) deleteFingerprint(); } } @@ -347,14 +349,14 @@ public class HeapReader { */ public int size() { if (this.index == null) return 0; - synchronized (index) { + synchronized (this.index) { return (this.index == null) ? 0 : this.index.size(); } } public boolean isEmpty() { if (this.index == null) return true; - synchronized (index) { + synchronized (this.index) { return this.index.isEmpty(); } } @@ -365,12 +367,12 @@ public class HeapReader { * @return true if the key exists, false otherwise */ public boolean containsKey(byte[] key) { - assert index != null; + assert this.index != null; key = normalizeKey(key); synchronized (this.index) { // check if the file index contains the key - return index.get(key) >= 0; + return this.index.get(key) >= 0; } } @@ -387,7 +389,7 @@ public class HeapReader { */ protected synchronized byte[] firstKey() throws IOException { synchronized (this.index) { - return index.smallestKey(); + return this.index.smallestKey(); } } @@ -400,7 +402,7 @@ public class HeapReader { */ protected byte[] first() throws IOException, RowSpaceExceededException { synchronized (this.index) { - byte[] key = index.smallestKey(); + byte[] key = this.index.smallestKey(); if (key == null) return null; return get(key); } @@ -415,7 +417,7 @@ public class HeapReader { */ protected byte[] lastKey() throws IOException { synchronized (this.index) { - return index.largestKey(); + return this.index.largestKey(); } } @@ -428,7 +430,7 @@ public class HeapReader { */ protected byte[] last() throws IOException, RowSpaceExceededException { synchronized (this.index) { - byte[] key = index.largestKey(); + byte[] key = this.index.largestKey(); if (key == null) return null; return get(key); } @@ -446,20 +448,20 @@ public class HeapReader { synchronized (this.index) { // check if the index contains the key - final long pos = index.get(key); + final long pos = this.index.get(key); if (pos < 0) return null; // access the file and read the container - file.seek(pos); - final int len = file.readInt() - index.row().primaryKeyLength; + this.file.seek(pos); + final int len = this.file.readInt() - this.index.row().primaryKeyLength; if (len < 0) { // database file may be corrupted and should be deleted :-(( - Log.logSevere("HeapReader", "file " + file.file() + " corrupted at " + pos + ": negative len. len = " + len + ", pk.len = " + index.row().primaryKeyLength); + Log.logSevere("HeapReader", "file " + this.file.file() + " corrupted at " + pos + ": negative len. len = " + len + ", pk.len = " + this.index.row().primaryKeyLength); // to get lazy over that problem (who wants to tell the user to stop operation and delete the file???) we work on like the entry does not exist - index.remove(key); + this.index.remove(key); return null; } - long memr = len + index.row().primaryKeyLength + 64; + long memr = len + this.index.row().primaryKeyLength + 64; if (MemoryControl.available() < memr) { if (!MemoryControl.request(memr, true)) throw new RowSpaceExceededException(memr, "HeapReader.get()/check"); // not enough memory available for this blob } @@ -467,17 +469,17 @@ public class HeapReader { // read the key byte[] keyf; try { - keyf = new byte[index.row().primaryKeyLength]; + keyf = new byte[this.index.row().primaryKeyLength]; } catch (OutOfMemoryError e) { - throw new RowSpaceExceededException(index.row().primaryKeyLength, "HeapReader.get()/keyf"); + throw new RowSpaceExceededException(this.index.row().primaryKeyLength, "HeapReader.get()/keyf"); } - file.readFully(keyf, 0, keyf.length); + this.file.readFully(keyf, 0, keyf.length); if (!this.ordering.equal(key, keyf)) { // verification of the indexed access failed. we must re-read the index - Log.logSevere("HeapReader", "indexed verification access failed for " + heapFile.toString()); + Log.logSevere("HeapReader", "indexed verification access failed for " + this.heapFile.toString()); // this is a severe operation, it should never happen. // remove entry from index because keeping that element in the index would not make sense - index.remove(key); + this.index.remove(key); // nothing to return return null; // but if the process ends in this state, it would completely fail @@ -492,7 +494,7 @@ public class HeapReader { } catch (OutOfMemoryError e) { throw new RowSpaceExceededException(len, "HeapReader.get()/blob"); } - file.readFully(blob, 0, blob.length); + this.file.readFully(blob, 0, blob.length); return blob; } @@ -512,12 +514,12 @@ public class HeapReader { protected boolean checkKey(byte[] key, final long pos) throws IOException { key = normalizeKey(key); - file.seek(pos); - file.readInt(); // skip the size value + this.file.seek(pos); + this.file.readInt(); // skip the size value // read the key - final byte[] keyf = new byte[index.row().primaryKeyLength]; - file.readFully(keyf, 0, keyf.length); + final byte[] keyf = new byte[this.index.row().primaryKeyLength]; + this.file.readFully(keyf, 0, keyf.length); return this.ordering.equal(key, keyf); } @@ -532,12 +534,12 @@ public class HeapReader { synchronized (this.index) { // check if the index contains the key - final long pos = index.get(key); + final long pos = this.index.get(key); if (pos < 0) return -1; // access the file and read the size of the container - file.seek(pos); - return file.readInt() - index.row().primaryKeyLength; + this.file.seek(pos); + return this.file.readInt() - this.index.row().primaryKeyLength; } } @@ -546,20 +548,20 @@ public class HeapReader { */ public void close(boolean writeIDX) { synchronized (this.index) { - if (file != null) + if (this.file != null) try { - file.close(); + this.file.close(); } catch (IOException e) { Log.logException(e); } - file = null; - if (writeIDX && index != null && free != null && (index.size() > 3 || free.size() > 3)) { + this.file = null; + if (writeIDX && this.index != null && this.free != null && (this.index.size() > 3 || this.free.size() > 3)) { // now we can create a dump of the index and the gap information // to speed up the next start try { String fingerprint = fingerprintFileHash(this.heapFile); if (fingerprint == null) { - Log.logSevere("HeapReader", "cannot write a dump for " + heapFile.getName()+ ": fingerprint is null"); + Log.logSevere("HeapReader", "cannot write a dump for " + this.heapFile.getName()+ ": fingerprint is null"); } else { File newFingerprintFileGap = HeapWriter.fingerprintGapFile(this.heapFile, fingerprint); if (this.fingerprintFileGap != null && @@ -568,12 +570,12 @@ public class HeapReader { Log.logInfo("HeapReader", "using existing gap dump instead of writing a new one: " + this.fingerprintFileGap.getName()); } else { long start = System.currentTimeMillis(); - free.dump(newFingerprintFileGap); - Log.logInfo("HeapReader", "wrote a dump for the " + this.free.size() + " gap entries of " + heapFile.getName()+ " in " + (System.currentTimeMillis() - start) + " milliseconds."); + this.free.dump(newFingerprintFileGap); + Log.logInfo("HeapReader", "wrote a dump for the " + this.free.size() + " gap entries of " + this.heapFile.getName()+ " in " + (System.currentTimeMillis() - start) + " milliseconds."); } } - free.clear(); - free = null; + this.free.clear(); + this.free = null; if (fingerprint != null) { File newFingerprintFileIdx = HeapWriter.fingerprintIndexFile(this.heapFile, fingerprint); if (this.fingerprintFileIdx != null && @@ -582,20 +584,20 @@ public class HeapReader { Log.logInfo("HeapReader", "using existing idx dump instead of writing a new one: " + this.fingerprintFileIdx.getName()); } else { long start = System.currentTimeMillis(); - index.dump(newFingerprintFileIdx); - Log.logInfo("HeapReader", "wrote a dump for the " + this.index.size() + " index entries of " + heapFile.getName()+ " in " + (System.currentTimeMillis() - start) + " milliseconds."); + this.index.dump(newFingerprintFileIdx); + Log.logInfo("HeapReader", "wrote a dump for the " + this.index.size() + " index entries of " + this.heapFile.getName()+ " in " + (System.currentTimeMillis() - start) + " milliseconds."); } } - index.close(); - index = null; + this.index.close(); + this.index = null; } catch (IOException e) { Log.logException(e); } } - if (free != null) free.clear(); - free = null; - if (index != null) index.close(); - index = null; + if (this.free != null) this.free.clear(); + this.free = null; + if (this.index != null) this.index.close(); + this.index = null; } } @@ -672,18 +674,20 @@ public class HeapReader { this.blobFile = blobFile; } + @Override public CloneableIterator> clone(Object modifier) { // if the entries iterator is cloned, close the file! - if (is != null) try { is.close(); } catch (final IOException e) {} - is = null; + if (this.is != null) try { this.is.close(); } catch (final IOException e) {} + this.is = null; try { - return new entries(blobFile, keylen); + return new entries(this.blobFile, this.keylen); } catch (IOException e) { Log.logException(e); return null; } } + @Override public Map.Entry next0() { try { byte b; @@ -692,14 +696,14 @@ public class HeapReader { byte[] key; final int keylen1 = this.keylen - 1; while (true) { - len = is.readInt(); + len = this.is.readInt(); if (len == 0) continue; // rare, but possible: zero length record (takes 4 bytes) - b = is.readByte(); // read a single by te to check for empty record + b = this.is.readByte(); // read a single by te to check for empty record if (b == 0) { // this is empty // read some more bytes to consume the empty record if (len > 1) { - if (len - 1 != is.skipBytes(len - 1)) { // all that is remaining + if (len - 1 != this.is.skipBytes(len - 1)) { // all that is remaining Log.logWarning("HeapReader", "problem skiping " + + len + " bytes in " + this.blobFile.getName()); return null; } @@ -709,13 +713,13 @@ public class HeapReader { // we are now ahead of remaining this.keylen - 1 bytes of the key key = new byte[this.keylen]; key[0] = b; // the first entry that we know already - if (is.read(key, 1, keylen1) < keylen1) return null; // read remaining key bytes + if (this.is.read(key, 1, keylen1) < keylen1) return null; // read remaining key bytes // so far we have read this.keylen - 1 + 1 = this.keylen bytes. // there must be a remaining number of len - this.keylen bytes left for the BLOB if (len < this.keylen) return null; // a strange case that can only happen in case of corrupted data try { payload = new byte[len - this.keylen]; // the remaining record entries - if (is.read(payload) < payload.length) return null; + if (this.is.read(payload) < payload.length) return null; return new entry(key, payload); } catch (OutOfMemoryError e) { // the allocation of memory for the payload may fail @@ -731,8 +735,8 @@ public class HeapReader { } public void close() { - if (is != null) try { is.close(); } catch (final IOException e) {} - is = null; + if (this.is != null) try { this.is.close(); } catch (final IOException e) {} + this.is = null; } @Override @@ -750,17 +754,20 @@ public class HeapReader { this.b = b; } + @Override public byte[] getKey() { - return s; + return this.s; } + @Override public byte[] getValue() { - return b; + return this.b; } + @Override public byte[] setValue(byte[] value) { - byte[] b1 = b; - b = value; + byte[] b1 = this.b; + this.b = value; return b1; } }