From 0f276dd63fae69c54280e56e31ca262308ff9a38 Mon Sep 17 00:00:00 2001 From: orbiter Date: Tue, 24 Aug 2010 12:36:56 +0000 Subject: [PATCH] - MapHeap now implements Map> - refactoring of method names to comply with Map method names git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7072 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- source/de/anomic/crawler/CrawlProfile.java | 14 +-- source/de/anomic/data/BookmarkDate.java | 4 +- source/de/anomic/data/blogBoard.java | 6 +- source/de/anomic/data/blogBoardComments.java | 4 +- source/de/anomic/data/bookmarksDB.java | 4 +- source/de/anomic/data/messageBoard.java | 4 +- source/de/anomic/data/userDB.java | 8 +- source/de/anomic/data/wiki/wikiBoard.java | 4 +- source/de/anomic/http/client/Cache.java | 44 ++++--- source/de/anomic/yacy/yacySeedDB.java | 56 ++++----- source/net/yacy/kelondro/blob/ArrayStack.java | 40 ++++--- .../net/yacy/kelondro/blob/BEncodedHeap.java | 11 +- source/net/yacy/kelondro/blob/BLOB.java | 7 +- source/net/yacy/kelondro/blob/Compressor.java | 30 +++-- source/net/yacy/kelondro/blob/Heap.java | 40 +++---- .../net/yacy/kelondro/blob/HeapModifier.java | 4 +- source/net/yacy/kelondro/blob/HeapReader.java | 14 ++- .../net/yacy/kelondro/blob/MapDataMining.java | 10 +- source/net/yacy/kelondro/blob/MapHeap.java | 111 +++++++++++++++--- source/net/yacy/kelondro/blob/Stack.java | 8 +- .../kelondro/rwi/ReferenceContainerArray.java | 4 +- 21 files changed, 271 insertions(+), 156 deletions(-) diff --git a/source/de/anomic/crawler/CrawlProfile.java b/source/de/anomic/crawler/CrawlProfile.java index 4fb53ccc5..cccc1b65d 100644 --- a/source/de/anomic/crawler/CrawlProfile.java +++ b/source/de/anomic/crawler/CrawlProfile.java @@ -136,7 +136,7 @@ public class CrawlProfile { public void removeEntry(final byte[] handle) { try { - profileTable.remove(handle); + profileTable.delete(handle); } catch (final IOException e) { Log.logException(e); } @@ -145,11 +145,11 @@ public class CrawlProfile { public entry newEntry(final Map mem) { final entry ne = new entry(mem); try { - profileTable.put(ne.handle().getBytes(), ne.map()); + profileTable.insert(ne.handle().getBytes(), ne.map()); } catch (final Exception e) { clear(); try { - profileTable.put(ne.handle().getBytes(), ne.map()); + profileTable.insert(ne.handle().getBytes(), ne.map()); } catch (final Exception ee) { Log.logException(e); System.exit(0); @@ -182,11 +182,11 @@ public class CrawlProfile { xsstopw, xdstopw, xpstopw, cacheStrategy); try { - profileTable.put(ne.handle().getBytes(), ne.map()); + profileTable.insert(ne.handle().getBytes(), ne.map()); } catch (final Exception e) { clear(); try { - profileTable.put(ne.handle().getBytes(), ne.map()); + profileTable.insert(ne.handle().getBytes(), ne.map()); } catch (final Exception ee) { Log.logException(e); System.exit(0); @@ -196,7 +196,7 @@ public class CrawlProfile { } public boolean hasEntry(final String handle) { - return profileTable.has(handle.getBytes()); + return profileTable.containsKey(handle.getBytes()); } public entry getEntry(final String handle) { @@ -218,7 +218,7 @@ public class CrawlProfile { public void changeEntry(final entry e, final String propName, final String newValue) throws IOException, RowSpaceExceededException { e.mem.put(propName, newValue); assert e.handle() != null; - profileTable.put(e.handle().getBytes(), e.mem); + profileTable.insert(e.handle().getBytes(), e.mem); } public long getRecrawlDate(final long oldTimeMinutes) { diff --git a/source/de/anomic/data/BookmarkDate.java b/source/de/anomic/data/BookmarkDate.java index c13940209..bdcbbb4d7 100644 --- a/source/de/anomic/data/BookmarkDate.java +++ b/source/de/anomic/data/BookmarkDate.java @@ -144,13 +144,13 @@ public class BookmarkDate { public void setDatesTable() { if (this.size() >0) { try { - datesTable.put(getDateString().getBytes(), mem); + datesTable.insert(getDateString().getBytes(), mem); } catch (Exception e) { Log.logException(e); } } else { try { - datesTable.remove(getDateString().getBytes()); + datesTable.delete(getDateString().getBytes()); } catch (IOException e) { Log.logException(e); } diff --git a/source/de/anomic/data/blogBoard.java b/source/de/anomic/data/blogBoard.java index e476584c4..8d4b62a94 100644 --- a/source/de/anomic/data/blogBoard.java +++ b/source/de/anomic/data/blogBoard.java @@ -81,7 +81,7 @@ public class blogBoard { * @return true if the database contains the element, else false */ public boolean contains(final String key) { - return database.has(key.getBytes()); + return database.containsKey(key.getBytes()); } public void close() { @@ -122,7 +122,7 @@ public class blogBoard { public String writeBlogEntry(final BlogEntry page) { String ret = null; try { - database.put(page.key.getBytes(), page.record); + database.insert(page.key.getBytes(), page.record); ret = page.key; } catch (IOException ex) { Log.logException(ex); @@ -240,7 +240,7 @@ public class blogBoard { public void deleteBlogEntry(final String key) { try { - database.remove(normalize(key).getBytes()); + database.delete(normalize(key).getBytes()); } catch (final IOException e) { } } diff --git a/source/de/anomic/data/blogBoardComments.java b/source/de/anomic/data/blogBoardComments.java index 37515f4d2..c00931966 100644 --- a/source/de/anomic/data/blogBoardComments.java +++ b/source/de/anomic/data/blogBoardComments.java @@ -112,7 +112,7 @@ public class blogBoardComments { public String write(final CommentEntry page) { // writes a new page and returns key try { - database.put(page.key.getBytes(), page.record); + database.insert(page.key.getBytes(), page.record); return page.key; } catch (final Exception e) { Log.logException(e); @@ -231,7 +231,7 @@ public class blogBoardComments { public void delete(final String key) { try { - database.remove(normalize(key).getBytes()); + database.delete(normalize(key).getBytes()); } catch (final IOException e) { } } diff --git a/source/de/anomic/data/bookmarksDB.java b/source/de/anomic/data/bookmarksDB.java index 385dfa927..41ed704f0 100644 --- a/source/de/anomic/data/bookmarksDB.java +++ b/source/de/anomic/data/bookmarksDB.java @@ -131,7 +131,7 @@ public class bookmarksDB { // adding a bookmark to the bookmarksDB public void saveBookmark(final Bookmark bookmark){ try { - bookmarks.put(bookmark.getUrlHash().getBytes(), bookmark.entry); + bookmarks.insert(bookmark.getUrlHash().getBytes(), bookmark.entry); } catch (final Exception e) { Log.logException(e); } @@ -171,7 +171,7 @@ public class bookmarksDB { Bookmark b; try { b = getBookmark(urlHash); - bookmarks.remove(urlHash.getBytes()); + bookmarks.delete(urlHash.getBytes()); } catch (final IOException e) { b = null; } diff --git a/source/de/anomic/data/messageBoard.java b/source/de/anomic/data/messageBoard.java index 45f53d872..178f27070 100644 --- a/source/de/anomic/data/messageBoard.java +++ b/source/de/anomic/data/messageBoard.java @@ -183,7 +183,7 @@ public class messageBoard { public String write(final entry message) { // writes a message and returns key try { - database.put(message.key.getBytes(), message.record); + database.insert(message.key.getBytes(), message.record); return message.key; } catch (final Exception e) { Log.logException(e); @@ -207,7 +207,7 @@ public class messageBoard { public void remove(final String key) { try { - database.remove(key.getBytes()); + database.delete(key.getBytes()); } catch (final IOException e) { } } diff --git a/source/de/anomic/data/userDB.java b/source/de/anomic/data/userDB.java index 008726a5a..337755740 100644 --- a/source/de/anomic/data/userDB.java +++ b/source/de/anomic/data/userDB.java @@ -85,7 +85,7 @@ public final class userDB { public void removeEntry(final String hostName) { try { - userTable.remove(hostName.toLowerCase().getBytes()); + userTable.delete(hostName.toLowerCase().getBytes()); } catch (final IOException e) {} } @@ -114,7 +114,7 @@ public final class userDB { public String addEntry(final Entry entry) { try { - userTable.put(entry.userName.getBytes(), entry.mem); + userTable.insert(entry.userName.getBytes(), entry.mem); return entry.userName; } catch (final Exception e) { Log.logException(e); @@ -488,7 +488,7 @@ public final class userDB { } try { - userDB.this.userTable.put(getUserName().getBytes(), this.mem); + userDB.this.userTable.insert(getUserName().getBytes(), this.mem); } catch(final Exception e){ Log.logException(e); } @@ -505,7 +505,7 @@ public final class userDB { public void setProperty(final String propName, final String newValue) throws IOException, RowSpaceExceededException { this.mem.put(propName, newValue); - userDB.this.userTable.put(getUserName().getBytes(), this.mem); + userDB.this.userTable.insert(getUserName().getBytes(), this.mem); } public String getProperty(final String propName, final String defaultValue) { diff --git a/source/de/anomic/data/wiki/wikiBoard.java b/source/de/anomic/data/wiki/wikiBoard.java index e4ac69e2c..a14e302c6 100644 --- a/source/de/anomic/data/wiki/wikiBoard.java +++ b/source/de/anomic/data/wiki/wikiBoard.java @@ -266,9 +266,9 @@ public class wikiBoard { //System.out.println("key = " + page.key); //System.out.println("oldDate = " + oldDate); //System.out.println("record = " + oldEntry.record.toString()); - bkpbase.put((page.key + dateString(oldDate)).getBytes(), oldEntry.record); + bkpbase.insert((page.key + dateString(oldDate)).getBytes(), oldEntry.record); // write the new page - datbase.put(page.key.getBytes(), page.record); + datbase.insert(page.key.getBytes(), page.record); return page.key; } catch (final Exception e) { Log.logException(e); diff --git a/source/de/anomic/http/client/Cache.java b/source/de/anomic/http/client/Cache.java index 2059a5d1f..949fc375a 100644 --- a/source/de/anomic/http/client/Cache.java +++ b/source/de/anomic/http/client/Cache.java @@ -56,7 +56,7 @@ public final class Cache { private static final String RESPONSE_HEADER_DB_NAME = "responseHeader.heap"; private static final String FILE_DB_NAME = "file.array"; - private static MapHeap responseHeaderDB = null; + private static Map> responseHeaderDB = null; private static Compressor fileDB = null; private static ArrayStack fileDBunbuffered = null; @@ -105,7 +105,9 @@ public final class Cache { * close the databases */ public static void close() { - responseHeaderDB.close(); + if (responseHeaderDB instanceof MapHeap) { + ((MapHeap) responseHeaderDB).close(); + } fileDB.close(true); } @@ -119,14 +121,18 @@ public final class Cache { hm.putAll(responseHeader); hm.put("@@URL", url.toNormalform(true, false)); try { - responseHeaderDB.put(url.hash(), hm); + if (responseHeaderDB instanceof MapHeap) { + ((MapHeap) responseHeaderDB).insert(url.hash(), hm); + } else { + responseHeaderDB.put(url.hash(), hm); + } } catch (Exception e) { throw new IOException("Cache.store: cannot write to headerDB: " + e.getMessage()); } // store the file try { - fileDB.put(url.hash(), file); + fileDB.insert(url.hash(), file); } catch (UnsupportedEncodingException e) { throw new IOException("Cache.store: cannot write to fileDB (1): " + e.getMessage()); } catch (IOException e) { @@ -142,18 +148,22 @@ public final class Cache { */ public static boolean has(final DigestURI url) { boolean headerExists; - headerExists = responseHeaderDB.has(url.hash()); - boolean fileExists = fileDB.has(url.hash()); + headerExists = responseHeaderDB.containsKey(url.hash()); + boolean fileExists = fileDB.containsKey(url.hash()); if (headerExists && fileExists) return true; if (!headerExists && !fileExists) return false; // if not both is there then we do a clean-up if (headerExists) try { log.logWarning("header but not content of url " + url.toString() + " in cache; cleaned up"); - responseHeaderDB.remove(url.hash()); + if (responseHeaderDB instanceof MapHeap) { + ((MapHeap) responseHeaderDB).delete(url.hash()); + } else { + responseHeaderDB.remove(url.hash()); + } } catch (IOException e) {} if (fileExists) try { log.logWarning("content but not header of url " + url.toString() + " in cache; cleaned up"); - fileDB.remove(url.hash()); + fileDB.delete(url.hash()); } catch (IOException e) {} return false; } @@ -171,15 +181,7 @@ public final class Cache { // loading data from database Map hdb; - try { - hdb = responseHeaderDB.get(url.hash()); - } catch (final IOException e) { - Log.logException(e); - return null; - } catch (RowSpaceExceededException e) { - Log.logException(e); - return null; - } + hdb = responseHeaderDB.get(url.hash()); if (hdb == null) return null; return new ResponseHeader(null, hdb); @@ -219,7 +221,11 @@ public final class Cache { * @throws IOException */ public static void delete(final DigestURI url) throws IOException { - responseHeaderDB.remove(url.hash()); - fileDB.remove(url.hash()); + if (responseHeaderDB instanceof MapHeap) { + ((MapHeap) responseHeaderDB).delete(url.hash()); + } else { + responseHeaderDB.remove(url.hash()); + } + fileDB.delete(url.hash()); } } diff --git a/source/de/anomic/yacy/yacySeedDB.java b/source/de/anomic/yacy/yacySeedDB.java index 5a320cb83..cd73336f5 100644 --- a/source/de/anomic/yacy/yacySeedDB.java +++ b/source/de/anomic/yacy/yacySeedDB.java @@ -269,9 +269,9 @@ public final class yacySeedDB implements AlternativeDomainNames { if (seedActiveDB.isEmpty() && seedPassiveDB.isEmpty() && seedPotentialDB.isEmpty()) return; // avoid that the own seed is initialized too early if (this.mySeed == null) initMySeed(); try { - seedActiveDB.remove(mySeed.hash.getBytes()); - seedPassiveDB.remove(mySeed.hash.getBytes()); - seedPotentialDB.remove(mySeed.hash.getBytes()); + seedActiveDB.delete(mySeed.hash.getBytes()); + seedPassiveDB.delete(mySeed.hash.getBytes()); + seedPotentialDB.delete(mySeed.hash.getBytes()); } catch (final IOException e) { Log.logWarning("yacySeedDB", "could not remove hash ("+ e.getClass() +"): "+ e.getMessage()); } } @@ -470,10 +470,10 @@ public final class yacySeedDB implements AlternativeDomainNames { nameLookupCache.put(seed.getName(), seed.hash); final Map seedPropMap = seed.getMap(); synchronized (seedPropMap) { - seedActiveDB.put(seed.hash.getBytes(), seedPropMap); + seedActiveDB.insert(seed.hash.getBytes(), seedPropMap); } - seedPassiveDB.remove(seed.hash.getBytes()); - seedPotentialDB.remove(seed.hash.getBytes()); + seedPassiveDB.delete(seed.hash.getBytes()); + seedPotentialDB.delete(seed.hash.getBytes()); } catch (final Exception e) { yacyCore.log.logSevere("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e); resetActiveTable(); @@ -484,14 +484,14 @@ public final class yacySeedDB implements AlternativeDomainNames { if (seed.isProper(false) != null) return; try { nameLookupCache.remove(seed.getName()); - seedActiveDB.remove(seed.hash.getBytes()); - seedPotentialDB.remove(seed.hash.getBytes()); + seedActiveDB.delete(seed.hash.getBytes()); + seedPotentialDB.delete(seed.hash.getBytes()); } catch (final Exception e) { Log.logWarning("yacySeedDB", "could not remove hash ("+ e.getClass() +"): "+ e.getMessage()); } //seed.put(yacySeed.LASTSEEN, yacyCore.shortFormatter.format(new Date(yacyCore.universalTime()))); try { final Map seedPropMap = seed.getMap(); synchronized (seedPropMap) { - seedPassiveDB.put(seed.hash.getBytes(), seedPropMap); + seedPassiveDB.insert(seed.hash.getBytes(), seedPropMap); } } catch (final Exception e) { yacyCore.log.logSevere("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e); @@ -503,14 +503,14 @@ public final class yacySeedDB implements AlternativeDomainNames { if (seed.isProper(false) != null) return; try { nameLookupCache.remove(seed.getName()); - seedActiveDB.remove(seed.hash.getBytes()); - seedPassiveDB.remove(seed.hash.getBytes()); + seedActiveDB.delete(seed.hash.getBytes()); + seedPassiveDB.delete(seed.hash.getBytes()); } catch (final Exception e) { Log.logWarning("yacySeedDB", "could not remove hash ("+ e.getClass() +"): "+ e.getMessage()); } //seed.put(yacySeed.LASTSEEN, yacyCore.shortFormatter.format(new Date(yacyCore.universalTime()))); try { final Map seedPropMap = seed.getMap(); synchronized (seedPropMap) { - seedPotentialDB.put(seed.hash.getBytes(), seedPropMap); + seedPotentialDB.insert(seed.hash.getBytes(), seedPropMap); } } catch (final Exception e) { yacyCore.log.logSevere("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e); @@ -521,27 +521,27 @@ public final class yacySeedDB implements AlternativeDomainNames { public synchronized void removeDisconnected(final String peerHash) { if(peerHash == null) return; try { - seedPassiveDB.remove(peerHash.getBytes()); + seedPassiveDB.delete(peerHash.getBytes()); } catch (final IOException e) { Log.logWarning("yacySeedDB", "could not remove hash ("+ e.getClass() +"): "+ e.getMessage()); } } public synchronized void removePotential(final String peerHash) { if(peerHash == null) return; try { - seedPotentialDB.remove(peerHash.getBytes()); + seedPotentialDB.delete(peerHash.getBytes()); } catch (final IOException e) { Log.logWarning("yacySeedDB", "could not remove hash ("+ e.getClass() +"): "+ e.getMessage()); } } public boolean hasConnected(final byte[] hash) { - return seedActiveDB.has(hash); + return seedActiveDB.containsKey(hash); } public boolean hasDisconnected(final byte[] hash) { - return seedPassiveDB.has(hash); + return seedPassiveDB.containsKey(hash); } public boolean hasPotential(final byte[] hash) { - return seedPotentialDB.has(hash); + return seedPotentialDB.containsKey(hash); } private yacySeed get(final String hash, final MapDataMining database) { @@ -589,13 +589,13 @@ public final class yacySeedDB implements AlternativeDomainNames { } yacySeed s = get(hash, seedActiveDB); - if (s != null) try { seedActiveDB.put(hash.getBytes(), seed.getMap()); return;} catch (final Exception e) {Log.logException(e);} + if (s != null) try { seedActiveDB.insert(hash.getBytes(), seed.getMap()); return;} catch (final Exception e) {Log.logException(e);} s = get(hash, seedPassiveDB); - if (s != null) try { seedPassiveDB.put(hash.getBytes(), seed.getMap()); return;} catch (final Exception e) {Log.logException(e);} + if (s != null) try { seedPassiveDB.insert(hash.getBytes(), seed.getMap()); return;} catch (final Exception e) {Log.logException(e);} s = get(hash, seedPotentialDB); - if (s != null) try { seedPotentialDB.put(hash.getBytes(), seed.getMap()); return;} catch (final Exception e) {Log.logException(e);} + if (s != null) try { seedPotentialDB.insert(hash.getBytes(), seed.getMap()); return;} catch (final Exception e) {Log.logException(e);} } public yacySeed lookupByName(String peerName) { @@ -695,7 +695,7 @@ public final class yacySeedDB implements AlternativeDomainNames { } // delete bad peers final Iterator i = badPeerHashes.iterator(); - while (i.hasNext()) try {seedActiveDB.remove(i.next());} catch (final IOException e1) {Log.logException(e1);} + while (i.hasNext()) try {seedActiveDB.delete(i.next());} catch (final IOException e1) {Log.logException(e1);} badPeerHashes.clear(); } @@ -729,7 +729,7 @@ public final class yacySeedDB implements AlternativeDomainNames { } // delete bad peers final Iterator i = badPeerHashes.iterator(); - while (i.hasNext()) try {seedActiveDB.remove(i.next());} catch (final IOException e1) {Log.logException(e1);} + while (i.hasNext()) try {seedActiveDB.delete(i.next());} catch (final IOException e1) {Log.logException(e1);} badPeerHashes.clear(); } @@ -1061,15 +1061,9 @@ public final class yacySeedDB implements AlternativeDomainNames { Log.logException(e); // emergency reset yacyCore.log.logSevere("seed-db emergency reset", e); - try { - database.clear(); - nextSeed = null; - return null; - } catch (final IOException e1) { - // no recovery possible - Log.logException(e1); - System.exit(-1); - } + database.clear(); + nextSeed = null; + return null; } return seed; } diff --git a/source/net/yacy/kelondro/blob/ArrayStack.java b/source/net/yacy/kelondro/blob/ArrayStack.java index 017377735..3e25ad7ac 100755 --- a/source/net/yacy/kelondro/blob/ArrayStack.java +++ b/source/net/yacy/kelondro/blob/ArrayStack.java @@ -521,7 +521,7 @@ public class ArrayStack implements BLOB { * @return * @throws IOException */ - public synchronized boolean has(byte[] key) { + public synchronized boolean containsKey(byte[] key) { blobItem bi = keeperOf(key); return bi != null; //for (blobItem bi: blobs) if (bi.blob.has(key)) return true; @@ -541,7 +541,7 @@ public class ArrayStack implements BLOB { try { cs.submit(new Callable() { public blobItem call() { - if (bi.blob.has(key)) return bi; + if (bi.blob.containsKey(key)) return bi; return null; } }); @@ -549,7 +549,7 @@ public class ArrayStack implements BLOB { } catch (final RejectedExecutionException e) { // the executor is either shutting down or the blocking queue is full // execute the search direct here without concurrency - if (bi.blob.has(key)) return bi; + if (bi.blob.containsKey(key)) return bi; } } @@ -596,6 +596,18 @@ public class ArrayStack implements BLOB { } + public byte[] get(Object key) { + if (!(key instanceof byte[])) return null; + try { + return get((byte[]) key); + } catch (IOException e) { + Log.logException(e); + } catch (RowSpaceExceededException e) { + Log.logException(e); + } + return null; + } + /** * get all BLOBs in the array. * this is useful when it is not clear if an entry is unique in all BLOBs in this array. @@ -671,7 +683,7 @@ public class ArrayStack implements BLOB { * @throws IOException * @throws RowSpaceExceededException */ - public synchronized void put(byte[] key, byte[] b) throws IOException { + public synchronized void insert(byte[] key, byte[] b) throws IOException { blobItem bi = (blobs.isEmpty()) ? null : blobs.get(blobs.size() - 1); /* if (bi == null) @@ -687,7 +699,7 @@ public class ArrayStack implements BLOB { blobs.add(bi); } assert bi.blob instanceof Heap; - bi.blob.put(key, b); + bi.blob.insert(key, b); executeLimits(); } @@ -724,9 +736,9 @@ public class ArrayStack implements BLOB { * @param key the primary key * @throws IOException */ - public synchronized void remove(byte[] key) throws IOException { + public synchronized void delete(byte[] key) throws IOException { long m = this.mem(); - for (blobItem bi: blobs) bi.blob.remove(key); + for (blobItem bi: blobs) bi.blob.delete(key); assert this.mem() <= m : "m = " + m + ", mem() = " + mem(); } @@ -996,18 +1008,18 @@ public class ArrayStack implements BLOB { try { //f.delete(); final ArrayStack heap = new ArrayStack(f, "test", 12, NaturalOrder.naturalOrder, 512 * 1024, false); - heap.put("aaaaaaaaaaaa".getBytes(), "eins zwei drei".getBytes()); - heap.put("aaaaaaaaaaab".getBytes(), "vier fuenf sechs".getBytes()); - heap.put("aaaaaaaaaaac".getBytes(), "sieben acht neun".getBytes()); - heap.put("aaaaaaaaaaad".getBytes(), "zehn elf zwoelf".getBytes()); + heap.insert("aaaaaaaaaaaa".getBytes(), "eins zwei drei".getBytes()); + heap.insert("aaaaaaaaaaab".getBytes(), "vier fuenf sechs".getBytes()); + heap.insert("aaaaaaaaaaac".getBytes(), "sieben acht neun".getBytes()); + heap.insert("aaaaaaaaaaad".getBytes(), "zehn elf zwoelf".getBytes()); // iterate over keys Iterator i = heap.keys(true, false); while (i.hasNext()) { System.out.println("key_b: " + new String(i.next())); } - heap.remove("aaaaaaaaaaab".getBytes()); - heap.remove("aaaaaaaaaaac".getBytes()); - heap.put("aaaaaaaaaaaX".getBytes(), "WXYZ".getBytes()); + heap.delete("aaaaaaaaaaab".getBytes()); + heap.delete("aaaaaaaaaaac".getBytes()); + heap.insert("aaaaaaaaaaaX".getBytes(), "WXYZ".getBytes()); heap.close(true); } catch (final IOException e) { Log.logException(e); diff --git a/source/net/yacy/kelondro/blob/BEncodedHeap.java b/source/net/yacy/kelondro/blob/BEncodedHeap.java index cd7b514b4..10872b2ef 100644 --- a/source/net/yacy/kelondro/blob/BEncodedHeap.java +++ b/source/net/yacy/kelondro/blob/BEncodedHeap.java @@ -178,7 +178,7 @@ public class BEncodedHeap implements Map>, Iterable< * @return true if the row exists */ public boolean containsKey(byte[] pk) { - return this.table.has(pk); + return this.table.containsKey(pk); } /** @@ -260,13 +260,13 @@ public class BEncodedHeap implements Map>, Iterable< */ public void insert(byte[] pk, Map map) throws RowSpaceExceededException, IOException { byte[] b = BEncoder.encode(BEncoder.transcode(map)); - this.table.put(pk, b); + this.table.insert(pk, b); this.columnames.addAll(map.keySet()); } public void insert(byte[] pk, String key, byte[] value) throws IOException { byte[] b = BEncoder.encodeMap(key, value); - this.table.put(pk, b); + this.table.insert(pk, b); this.columnames.add(key); } @@ -279,7 +279,7 @@ public class BEncodedHeap implements Map>, Iterable< try { Map entry = this.get(pk); byte[] b = BEncoder.encode(BEncoder.transcode(map)); - this.table.put(pk, b); + this.table.insert(pk, b); this.columnames.addAll(map.keySet()); return entry; } catch (IOException e) { @@ -297,7 +297,7 @@ public class BEncodedHeap implements Map>, Iterable< * @throws IOException */ public void delete(byte[] pk) throws IOException { - this.table.remove(pk); + this.table.delete(pk); } /** @@ -339,7 +339,6 @@ public class BEncodedHeap implements Map>, Iterable< Log.logException(e); } catch (IOException e) { Log.logException(e); - e.printStackTrace(); } } } diff --git a/source/net/yacy/kelondro/blob/BLOB.java b/source/net/yacy/kelondro/blob/BLOB.java index 03e088bee..b832338df 100644 --- a/source/net/yacy/kelondro/blob/BLOB.java +++ b/source/net/yacy/kelondro/blob/BLOB.java @@ -107,7 +107,7 @@ public interface BLOB { * @return * @throws IOException */ - public boolean has(byte[] key); + public boolean containsKey(byte[] key); /** * retrieve the whole BLOB from the table @@ -116,6 +116,7 @@ public interface BLOB { * @throws IOException */ public byte[] get(byte[] key) throws IOException, RowSpaceExceededException; + public byte[] get(Object key); /** * retrieve the size of the database @@ -140,7 +141,7 @@ public interface BLOB { * @throws IOException * @throws RowSpaceExceededException */ - public void put(byte[] key, byte[] b) throws IOException; + public void insert(byte[] key, byte[] b) throws IOException; /** * replace an existing entry in the BLOB with a new entry @@ -173,7 +174,7 @@ public interface BLOB { * @param key the primary key * @throws IOException */ - public void remove(byte[] key) throws IOException; + public void delete(byte[] key) throws IOException; /** * close the BLOB table diff --git a/source/net/yacy/kelondro/blob/Compressor.java b/source/net/yacy/kelondro/blob/Compressor.java index 836b395cb..dab126d93 100644 --- a/source/net/yacy/kelondro/blob/Compressor.java +++ b/source/net/yacy/kelondro/blob/Compressor.java @@ -110,7 +110,7 @@ public class Compressor implements BLOB { try { while ((entry = writeQueue.take()) != poisonWorkerEntry) { try { - Compressor.this.backend.put(entry.getKey().getBytes(), compress(entry.getValue())); + Compressor.this.backend.insert(entry.getKey().getBytes(), compress(entry.getValue())); } catch (IOException e) { Log.logException(e); buffer.put(entry.getKey(), entry.getValue()); @@ -250,9 +250,21 @@ public class Compressor implements BLOB { return decompress(b); } - public synchronized boolean has(byte[] key) { + public byte[] get(Object key) { + if (!(key instanceof byte[])) return null; + try { + return get((byte[]) key); + } catch (IOException e) { + Log.logException(e); + } catch (RowSpaceExceededException e) { + Log.logException(e); + } + return null; + } + + public synchronized boolean containsKey(byte[] key) { return - this.buffer.containsKey(new String(key)) || this.backend.has(key); + this.buffer.containsKey(new String(key)) || this.backend.containsKey(key); } public int keylength() { @@ -287,10 +299,10 @@ public class Compressor implements BLOB { return 0; } - public void put(byte[] key, byte[] b) throws IOException { + public void insert(byte[] key, byte[] b) throws IOException { // first ensure that the files do not exist anywhere - remove(key); + delete(key); // check if the buffer is full or could be full after this write if (this.bufferlength + b.length * 2 > this.maxbufferlength) synchronized (this) { @@ -312,8 +324,8 @@ public class Compressor implements BLOB { } } - public synchronized void remove(byte[] key) throws IOException { - this.backend.remove(key); + public synchronized void delete(byte[] key) throws IOException { + this.backend.delete(key); long rx = removeFromQueues(key); if (rx > 0) this.bufferlength -= rx; } @@ -366,7 +378,7 @@ public class Compressor implements BLOB { int reduction = c.length - b.length; assert reduction >= 0; if (reduction == 0) return 0; - this.put(key, c); + this.insert(key, c); return reduction; } @@ -377,7 +389,7 @@ public class Compressor implements BLOB { int reduction = c.length - b.length; assert reduction >= 0; if (reduction == 0) return 0; - this.put(key, c); + this.insert(key, c); return reduction; } diff --git a/source/net/yacy/kelondro/blob/Heap.java b/source/net/yacy/kelondro/blob/Heap.java index 7243da643..97696e90e 100755 --- a/source/net/yacy/kelondro/blob/Heap.java +++ b/source/net/yacy/kelondro/blob/Heap.java @@ -121,7 +121,7 @@ public final class Heap extends HeapModifier implements BLOB { * @return true if the key exists, false otherwise */ @Override - public boolean has(byte[] key) { + public boolean containsKey(byte[] key) { assert index != null; key = normalizeKey(key); synchronized (this) { @@ -130,7 +130,7 @@ public final class Heap extends HeapModifier implements BLOB { if (buffer == null) { if (this.buffer.containsKey(key)) return true; } - return super.has(key); + return super.containsKey(key); } } @@ -322,7 +322,7 @@ public final class Heap extends HeapModifier implements BLOB { * @throws RowSpaceExceededException */ @Override - public void put(byte[] key, final byte[] b) throws IOException { + public void insert(byte[] key, final byte[] b) throws IOException { key = normalizeKey(key); // we do not write records of length 0 into the BLOB @@ -331,7 +331,7 @@ public final class Heap extends HeapModifier implements BLOB { synchronized (this) { // first remove the old entry (removes from buffer and file) // TODO: this can be enhanced! - this.remove(key); + this.delete(key); // then look if we can use a free entry try { @@ -456,7 +456,7 @@ public final class Heap extends HeapModifier implements BLOB { * @throws IOException */ @Override - public void remove(byte[] key) throws IOException { + public void delete(byte[] key) throws IOException { key = normalizeKey(key); synchronized (this) { @@ -470,7 +470,7 @@ public final class Heap extends HeapModifier implements BLOB { } } - super.remove(key); + super.delete(key); } } @@ -510,10 +510,10 @@ public final class Heap extends HeapModifier implements BLOB { try { //f.delete(); final Heap heap = new Heap(f, 12, NaturalOrder.naturalOrder, 1024 * 512); - heap.put("aaaaaaaaaaaa".getBytes(), "eins zwei drei".getBytes()); - heap.put("aaaaaaaaaaab".getBytes(), "vier fuenf sechs".getBytes()); - heap.put("aaaaaaaaaaac".getBytes(), "sieben acht neun".getBytes()); - heap.put("aaaaaaaaaaad".getBytes(), "zehn elf zwoelf".getBytes()); + heap.insert("aaaaaaaaaaaa".getBytes(), "eins zwei drei".getBytes()); + heap.insert("aaaaaaaaaaab".getBytes(), "vier fuenf sechs".getBytes()); + heap.insert("aaaaaaaaaaac".getBytes(), "sieben acht neun".getBytes()); + heap.insert("aaaaaaaaaaad".getBytes(), "zehn elf zwoelf".getBytes()); // iterate over keys Iterator i = heap.index.keys(true, null); while (i.hasNext()) { @@ -523,9 +523,9 @@ public final class Heap extends HeapModifier implements BLOB { while (i.hasNext()) { System.out.println("key_b: " + new String(i.next())); } - heap.remove("aaaaaaaaaaab".getBytes()); - heap.remove("aaaaaaaaaaac".getBytes()); - heap.put("aaaaaaaaaaaX".getBytes(), "WXYZ".getBytes()); + heap.delete("aaaaaaaaaaab".getBytes()); + heap.delete("aaaaaaaaaaac".getBytes()); + heap.insert("aaaaaaaaaaaX".getBytes(), "WXYZ".getBytes()); heap.close(true); } catch (final IOException e) { Log.logException(e); @@ -543,13 +543,13 @@ public final class Heap extends HeapModifier implements BLOB { try { //f.delete(); final MapHeap heap = new MapHeap(f, 12, NaturalOrder.naturalOrder, 1024 * 512, 500, '_'); - heap.put("aaaaaaaaaaaa".getBytes(), map("aaaaaaaaaaaa", "eins zwei drei")); - heap.put("aaaaaaaaaaab".getBytes(), map("aaaaaaaaaaab", "vier fuenf sechs")); - heap.put("aaaaaaaaaaac".getBytes(), map("aaaaaaaaaaac", "sieben acht neun")); - heap.put("aaaaaaaaaaad".getBytes(), map("aaaaaaaaaaad", "zehn elf zwoelf")); - heap.remove("aaaaaaaaaaab".getBytes()); - heap.remove("aaaaaaaaaaac".getBytes()); - heap.put("aaaaaaaaaaaX".getBytes(), map("aaaaaaaaaaad", "WXYZ")); + heap.insert("aaaaaaaaaaaa".getBytes(), map("aaaaaaaaaaaa", "eins zwei drei")); + heap.insert("aaaaaaaaaaab".getBytes(), map("aaaaaaaaaaab", "vier fuenf sechs")); + heap.insert("aaaaaaaaaaac".getBytes(), map("aaaaaaaaaaac", "sieben acht neun")); + heap.insert("aaaaaaaaaaad".getBytes(), map("aaaaaaaaaaad", "zehn elf zwoelf")); + heap.delete("aaaaaaaaaaab".getBytes()); + heap.delete("aaaaaaaaaaac".getBytes()); + heap.insert("aaaaaaaaaaaX".getBytes(), map("aaaaaaaaaaad", "WXYZ")); heap.close(); } catch (final IOException e) { Log.logException(e); diff --git a/source/net/yacy/kelondro/blob/HeapModifier.java b/source/net/yacy/kelondro/blob/HeapModifier.java index 799e70032..0c87006f7 100644 --- a/source/net/yacy/kelondro/blob/HeapModifier.java +++ b/source/net/yacy/kelondro/blob/HeapModifier.java @@ -93,7 +93,7 @@ public class HeapModifier extends HeapReader implements BLOB { * @param key the primary key * @throws IOException */ - public void remove(byte[] key) throws IOException { + public void delete(byte[] key) throws IOException { key = normalizeKey(key); // pre-check before synchronization @@ -234,7 +234,7 @@ public class HeapModifier extends HeapReader implements BLOB { } } - public void put(byte[] key, byte[] b) throws IOException { + public void insert(byte[] key, byte[] b) throws IOException { throw new UnsupportedOperationException("put is not supported in BLOBHeapModifier"); } diff --git a/source/net/yacy/kelondro/blob/HeapReader.java b/source/net/yacy/kelondro/blob/HeapReader.java index 371d57519..33e317840 100644 --- a/source/net/yacy/kelondro/blob/HeapReader.java +++ b/source/net/yacy/kelondro/blob/HeapReader.java @@ -317,7 +317,7 @@ public class HeapReader { * @param key * @return true if the key exists, false otherwise */ - public boolean has(byte[] key) { + public boolean containsKey(byte[] key) { assert index != null; key = normalizeKey(key); @@ -427,6 +427,18 @@ public class HeapReader { return blob; } } + + public byte[] get(Object key) { + if (!(key instanceof byte[])) return null; + try { + return get((byte[]) key); + } catch (IOException e) { + Log.logException(e); + } catch (RowSpaceExceededException e) { + Log.logException(e); + } + return null; + } protected boolean checkKey(byte[] key, final long pos) throws IOException { key = normalizeKey(key); diff --git a/source/net/yacy/kelondro/blob/MapDataMining.java b/source/net/yacy/kelondro/blob/MapDataMining.java index e186d0fd2..71c95c71a 100644 --- a/source/net/yacy/kelondro/blob/MapDataMining.java +++ b/source/net/yacy/kelondro/blob/MapDataMining.java @@ -150,7 +150,7 @@ public class MapDataMining extends MapHeap { } @Override - public synchronized void clear() throws IOException { + public synchronized void clear() { super.clear(); if (sortfields == null) sortClusterMap = null; else { sortClusterMap = new HashMap>(); @@ -178,7 +178,7 @@ public class MapDataMining extends MapHeap { } @Override - public synchronized void put(final byte[] key, final Map newMap) throws IOException, RowSpaceExceededException { + public synchronized void insert(final byte[] key, final Map newMap) throws IOException, RowSpaceExceededException { assert (key != null); assert (key.length > 0); assert (newMap != null); @@ -195,7 +195,7 @@ public class MapDataMining extends MapHeap { updateAcc(newMap, true); } - super.put(key, newMap); + super.insert(key, newMap); // update sortCluster if (sortClusterMap != null) updateSortCluster(new String(key), newMap); @@ -251,7 +251,7 @@ public class MapDataMining extends MapHeap { } @Override - public synchronized void remove(final byte[] key) throws IOException { + public synchronized void delete(final byte[] key) throws IOException { if (key == null) return; // update elementCount @@ -272,7 +272,7 @@ public class MapDataMining extends MapHeap { Log.logException(e); } } - super.remove(key); + super.delete(key); } private void deleteSortCluster(final String key) { diff --git a/source/net/yacy/kelondro/blob/MapHeap.java b/source/net/yacy/kelondro/blob/MapHeap.java index 2fea3a1df..426a13c01 100644 --- a/source/net/yacy/kelondro/blob/MapHeap.java +++ b/source/net/yacy/kelondro/blob/MapHeap.java @@ -32,9 +32,12 @@ import java.io.ByteArrayInputStream; import java.io.File; import java.io.IOException; import java.io.InputStreamReader; +import java.util.Collection; import java.util.HashMap; import java.util.Iterator; import java.util.Map; +import java.util.Set; +import java.util.TreeSet; import java.util.concurrent.ConcurrentHashMap; import net.yacy.cora.storage.ARC; @@ -51,7 +54,7 @@ import net.yacy.kelondro.util.kelondroException; -public class MapHeap { +public class MapHeap implements Map> { private BLOB blob; private ARC> cache; @@ -82,8 +85,12 @@ public class MapHeap { * clears the content of the database * @throws IOException */ - public synchronized void clear() throws IOException { - this.blob.clear(); + public synchronized void clear() { + try { + this.blob.clear(); + } catch (IOException e) { + Log.logException(e); + } this.cache.clear(); } @@ -129,7 +136,7 @@ public class MapHeap { * @throws IOException * @throws RowSpaceExceededException */ - public void put(byte[] key, final Map newMap) throws IOException, RowSpaceExceededException { + public void insert(byte[] key, final Map newMap) throws IOException, RowSpaceExceededException { assert key != null; assert key.length > 0; assert newMap != null; @@ -139,19 +146,32 @@ public class MapHeap { byte[] sb = s.getBytes(); synchronized (this) { // write entry - if (blob != null) blob.put(key, sb); + if (blob != null) blob.insert(key, sb); // write map to cache if (cache != null) cache.put(new String(key), newMap); } } + + public Map put(byte[] key, final Map newMap) { + Map v = null; + try { + v = this.get(key); + insert(key, newMap); + } catch (IOException e) { + Log.logException(e); + } catch (RowSpaceExceededException e) { + Log.logException(e); + } + return v; + } /** * remove a Map * @param key the primary key * @throws IOException */ - public void remove(byte[] key) throws IOException { + public void delete(byte[] key) throws IOException { // update elementCount if (key == null) return; key = normalizeKey(key); @@ -161,10 +181,21 @@ public class MapHeap { cache.remove(new String(key)); // remove from file - blob.remove(key); + blob.delete(key); } } + public Map remove(Object key) { + Map v = null; + try { + v = this.get(key); + this.delete((byte[]) key); + } catch (IOException e) { + Log.logException(e); + } + return v; + } + /** * check if a specific key is in the database * @param key the primary key @@ -172,13 +203,14 @@ public class MapHeap { * @throws IOException */ - public boolean has(byte[] key) { - assert key != null; + public boolean containsKey(Object k) { + if (!(k instanceof byte[])) return false; + assert k != null; if (cache == null) return false; // case may appear during shutdown - key = normalizeKey(key); + byte[] key = normalizeKey((byte[]) k); boolean h; synchronized (this) { - h = this.cache.containsKey(new String(key)) || this.blob.has(key); + h = this.cache.containsKey(new String(key)) || this.blob.containsKey(key); } return h; } @@ -193,6 +225,18 @@ public class MapHeap { if (key == null) return null; return get(key, true); } + + public Map get(final Object key) { + if (key == null) return null; + try { + return get((byte[]) key); + } catch (IOException e) { + Log.logException(e); + } catch (RowSpaceExceededException e) { + Log.logException(e); + } + return null; + } private byte[] normalizeKey(byte[] key) { if (blob == null || key == null) return key; @@ -360,7 +404,43 @@ public class MapHeap { throw new UnsupportedOperationException(); } } // class mapIterator - + + public void putAll(Map> map) { + for (Map.Entry> me: map.entrySet()) { + try { + this.insert(me.getKey(), me.getValue()); + } catch (RowSpaceExceededException e) { + Log.logException(e); + } catch (IOException e) { + Log.logException(e); + } + } + } + + public Set keySet() { + TreeSet set = new TreeSet(this.blob.ordering()); + try { + Iterator i = this.blob.keys(true, false); + while (i.hasNext()) set.add(i.next()); + } catch (IOException e) {} + return set; + } + + public Collection> values() { + // this method shall not be used because it is not appropriate for this kind of data + throw new UnsupportedOperationException(); + } + + public Set>> entrySet() { + // this method shall not be used because it is not appropriate for this kind of data + throw new UnsupportedOperationException(); + } + + public boolean containsValue(Object value) { + // this method shall not be used because it is not appropriate for this kind of data + throw new UnsupportedOperationException(); + } + public static void main(String[] args) { // test the class File f = new File("maptest"); @@ -370,9 +450,9 @@ public class MapHeap { MapHeap map = new MapHeap(f, 12, NaturalOrder.naturalOrder, 1024 * 1024, 1024, '_'); // put some values into the map Map m = new HashMap(); - m.put("k", "000"); map.put("123".getBytes(), m); - m.put("k", "111"); map.put("456".getBytes(), m); - m.put("k", "222"); map.put("789".getBytes(), m); + m.put("k", "000"); map.insert("123".getBytes(), m); + m.put("k", "111"); map.insert("456".getBytes(), m); + m.put("k", "222"); map.insert("789".getBytes(), m); // iterate over keys Iterator i = map.keys(true, false); while (i.hasNext()) { @@ -386,5 +466,4 @@ public class MapHeap { Log.logException(e); } } - } diff --git a/source/net/yacy/kelondro/blob/Stack.java b/source/net/yacy/kelondro/blob/Stack.java index d8373eed6..8a1d1227b 100644 --- a/source/net/yacy/kelondro/blob/Stack.java +++ b/source/net/yacy/kelondro/blob/Stack.java @@ -94,7 +94,7 @@ public class Stack { */ public synchronized long push(final byte[] b) throws IOException, RowSpaceExceededException { long handle = nextHandle(); - this.stack.put(NaturalOrder.encodeLong(handle, 8), b); + this.stack.insert(NaturalOrder.encodeLong(handle, 8), b); return handle; } @@ -108,7 +108,7 @@ public class Stack { * @throws RowSpaceExceededException */ protected synchronized void push(final Entry e) throws IOException, RowSpaceExceededException { - this.stack.put(NaturalOrder.encodeLong(e.h, 8), e.b); + this.stack.insert(NaturalOrder.encodeLong(e.h, 8), e.b); } /** @@ -137,7 +137,7 @@ public class Stack { byte[] k = NaturalOrder.encodeLong(handle, 8); byte[] b = this.stack.get(k); if (b == null) return null; - this.stack.remove(k); + this.stack.delete(k); return b; } @@ -193,7 +193,7 @@ public class Stack { } assert b != null; if (b == null) return null; - if (remove) this.stack.remove(k); + if (remove) this.stack.delete(k); return new Entry(k, b); } diff --git a/source/net/yacy/kelondro/rwi/ReferenceContainerArray.java b/source/net/yacy/kelondro/rwi/ReferenceContainerArray.java index 04ea9a2cd..ffccd66c9 100644 --- a/source/net/yacy/kelondro/rwi/ReferenceContainerArray.java +++ b/source/net/yacy/kelondro/rwi/ReferenceContainerArray.java @@ -197,7 +197,7 @@ public final class ReferenceContainerArray { * @throws IOException */ public boolean has(final byte[] termHash) { - return this.array.has(termHash); + return this.array.containsKey(termHash); } /** @@ -251,7 +251,7 @@ public final class ReferenceContainerArray { */ public void delete(final byte[] termHash) throws IOException { // returns the index that had been deleted - array.remove(termHash); + array.delete(termHash); } public int reduce(final byte[] termHash, ContainerReducer reducer) throws IOException, RowSpaceExceededException {