From 8ce936bcddb576727d6920518986411c3d95318e Mon Sep 17 00:00:00 2001
From: orbiter <orbiter@6c8d7289-2bf4-0310-a012-ef5d649a1542>
Date: Thu, 21 Jan 2010 22:06:03 +0000
Subject: [PATCH] added an api recording function: it shall be possible to
 record all operations on YaCy in a database that should make it possible 1)
 to re-create a setting on fresh peers 2) to transmit a setting from one peer
 to another 3) to re-create crawl starts after a complete deletion of the
 index This functionality will also support 4) scheduled re-crawls (new
 implementation) To implement this, a new database structure has been crated
 that stores maps into blob heaps. to encode maps the b-encoding technique was
 used (this is the same encoding that torrent files use) - added a b-encoder -
 enhanced the b-decoder - added a b-encoded map heap data structure - added a
 table organisation based on b-encoded heaps - added a servlet to maintain
 such tables (see Tables_p.html) - integrated the servlet into the Advanced
 Settings menu - added an api recording based on the new tables

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6606 6c8d7289-2bf4-0310-a012-ef5d649a1542
---
 htroot/Blacklist_p.java                       |   8 +-
 htroot/ConfigBasic.java                       |   5 +-
 htroot/ConfigLanguage_p.java                  |   4 +
 htroot/ConfigNetwork_p.java                   |   3 +
 htroot/ConfigPortal.java                      |   6 +-
 htroot/Crawler_p.java                         |  18 +-
 htroot/Settings_p.html                        |   5 +
 htroot/Tables_p.html                          |  86 +++++
 htroot/Tables_p.java                          | 194 ++++++++++
 htroot/env/templates/header.template          |   2 -
 .../anomic/crawler/retrieval/FTPLoader.java   |   5 +-
 source/de/anomic/net/ftpc.java                |   4 +-
 source/de/anomic/search/Switchboard.java      |  32 +-
 source/de/anomic/server/serverSwitch.java     |  12 +-
 source/de/anomic/yacy/yacyClient.java         |   2 +-
 .../net/yacy/kelondro/blob/BEncodedHeap.java  | 176 +++++++--
 .../yacy/kelondro/blob/BEncodedHeapArray.java | 333 ++++++++++++++++++
 source/net/yacy/kelondro/blob/Heap.java       |  42 +--
 .../net/yacy/kelondro/blob/HeapModifier.java  |  10 +-
 source/net/yacy/kelondro/blob/HeapReader.java |  63 +++-
 source/net/yacy/kelondro/blob/HeapWriter.java |   6 +-
 .../net/yacy/kelondro/blob/MapDataMining.java |  96 ++---
 source/net/yacy/kelondro/util/BDecoder.java   |  64 ++++
 source/net/yacy/kelondro/util/BEncoder.java   |  70 +++-
 source/net/yacy/yacy.java                     |   4 +-
 25 files changed, 1089 insertions(+), 161 deletions(-)
 create mode 100644 htroot/Tables_p.html
 create mode 100644 htroot/Tables_p.java
 create mode 100644 source/net/yacy/kelondro/blob/BEncodedHeapArray.java

diff --git a/htroot/Blacklist_p.java b/htroot/Blacklist_p.java
index e9f73a00d..0e1718e49 100644
--- a/htroot/Blacklist_p.java
+++ b/htroot/Blacklist_p.java
@@ -250,8 +250,12 @@ public class Blacklist_p {
                  * =========================================================== */
 
                 blacklistToUse = post.get("currentBlacklist", "").trim();
-
-                final String temp = addBlacklistEntry(blacklistToUse, post.get("newEntry", "").trim(), header, supportedBlacklistTypes);
+                String blentry = post.get("newEntry", "").trim();
+                
+                // store this call as api call
+                listManager.switchboard.recordAPICall(post, "Blacklist_p.html", "blacklist", "add to blacklist: " + blentry);
+                
+                final String temp = addBlacklistEntry(blacklistToUse, blentry, header, supportedBlacklistTypes);
                 if (temp != null) {
                     prop.put("LOCATION", temp);
                     return prop;
diff --git a/htroot/ConfigBasic.java b/htroot/ConfigBasic.java
index 95714bf51..054247c0d 100644
--- a/htroot/ConfigBasic.java
+++ b/htroot/ConfigBasic.java
@@ -70,7 +70,10 @@ public class ConfigBasic {
             return prop;
         }
         
-        // starting a peer ping
+        // store this call as api call
+        if (post != null && post.containsKey("set")) {
+            sb.recordAPICall(post, "ConfigBasic.html", "configuration", "basic settings");
+        }
         
         //boolean doPeerPing = false;
         if ((sb.peers.mySeed().isVirgin()) || (sb.peers.mySeed().isJunior())) {
diff --git a/htroot/ConfigLanguage_p.java b/htroot/ConfigLanguage_p.java
index efcedf281..277897012 100644
--- a/htroot/ConfigLanguage_p.java
+++ b/htroot/ConfigLanguage_p.java
@@ -46,6 +46,7 @@ import de.anomic.data.translator;
 import de.anomic.http.client.Client;
 import de.anomic.http.server.HeaderFramework;
 import de.anomic.http.server.RequestHeader;
+import de.anomic.search.Switchboard;
 import de.anomic.server.serverObjects;
 import de.anomic.server.serverSwitch;
 import java.util.Collections;
@@ -72,6 +73,9 @@ public class ConfigLanguage_p {
         if (post != null){
             String selectedLanguage = post.get("language");
 
+            // store this call as api call
+            ((Switchboard) env).recordAPICall(post, "ConfigLanguage.html", "configuration", "language settings: " + selectedLanguage);
+            
             //change language
             if(post.containsKey("use_button") && selectedLanguage != null){
                 /* Only change language if filename is contained in list of filesnames
diff --git a/htroot/ConfigNetwork_p.java b/htroot/ConfigNetwork_p.java
index d7c4ab758..ef0d1c2c2 100644
--- a/htroot/ConfigNetwork_p.java
+++ b/htroot/ConfigNetwork_p.java
@@ -54,6 +54,9 @@ public class ConfigNetwork_p {
         
         if (post != null) {
             
+            // store this call as api call
+            sb.recordAPICall(post, "ConfigNetwork.html", "configuration", "network settings");
+            
             if (post.containsKey("changeNetwork")) {
                 final String networkDefinition = post.get("networkDefinition", "defaults/yacy.network.freeworld.unit");
                 if (networkDefinition.equals(sb.getConfig("network.unit.definition", ""))) {
diff --git a/htroot/ConfigPortal.java b/htroot/ConfigPortal.java
index f57872082..4206419fd 100644
--- a/htroot/ConfigPortal.java
+++ b/htroot/ConfigPortal.java
@@ -57,7 +57,11 @@ public class ConfigPortal {
                 }
             }
             if (post.containsKey("searchpage_set")) {
-                sb.setConfig(SwitchboardConstants.GREETING, post.get(SwitchboardConstants.GREETING, ""));
+                String newGreeting = post.get(SwitchboardConstants.GREETING, "");
+                // store this call as api call
+                sb.recordAPICall(post, "ConfigPortal.html", "appearance", "new portal design. greeting: " + newGreeting);
+                
+                sb.setConfig(SwitchboardConstants.GREETING, newGreeting);
                 sb.setConfig(SwitchboardConstants.GREETING_HOMEPAGE, post.get(SwitchboardConstants.GREETING_HOMEPAGE, ""));
                 sb.setConfig(SwitchboardConstants.GREETING_LARGE_IMAGE, post.get(SwitchboardConstants.GREETING_LARGE_IMAGE, ""));
                 sb.setConfig(SwitchboardConstants.GREETING_SMALL_IMAGE, post.get(SwitchboardConstants.GREETING_SMALL_IMAGE, ""));
diff --git a/htroot/Crawler_p.java b/htroot/Crawler_p.java
index 070150d54..4dadbc9c4 100644
--- a/htroot/Crawler_p.java
+++ b/htroot/Crawler_p.java
@@ -130,16 +130,8 @@ public class Crawler_p {
                 if (sb.peers == null) {
                     prop.put("info", "3");
                 } else {
-                    // log a GET url for this crawl start for possible use in cronjobs
-                    Log.logInfo("CRAWLSTART-URL", "http://localhost:" + sb.getConfig("port", "8080") + "/Crawler_p.html?" + post.toString());
-                    
-                    // set new properties
-                    final boolean fullDomain = post.get("range", "wide").equals("domain"); // special property in simple crawl start
-                    final boolean subPath    = post.get("range", "wide").equals("subpath"); // special property in simple crawl start
-                    
                     String crawlingStart = post.get("crawlingURL","").trim(); // the crawljob start url
-
-                    // adding the prefix http:// if necessary
+                    // add the prefix http:// if necessary
                     int pos = crawlingStart.indexOf("://");
                     if (pos == -1) crawlingStart = "http://" + crawlingStart;
 
@@ -147,6 +139,14 @@ public class Crawler_p {
                     DigestURI crawlingStartURL = null;
                     try {crawlingStartURL = new DigestURI(crawlingStart, null);} catch (final MalformedURLException e1) {}
                     crawlingStart = (crawlingStartURL == null) ? null : crawlingStartURL.toNormalform(true, true);
+                   
+                    // store this call as api call
+                    sb.recordAPICall(post, "Crawler_p.html", "crawler", "crawl start for " + crawlingStartURL.getHost());
+                    
+                    // set new properties
+                    final boolean fullDomain = post.get("range", "wide").equals("domain"); // special property in simple crawl start
+                    final boolean subPath    = post.get("range", "wide").equals("subpath"); // special property in simple crawl start
+                    
                     
                     // set the crawling filter
                     String newcrawlingMustMatch = post.get("mustmatch", CrawlProfile.MATCH_ALL);
diff --git a/htroot/Settings_p.html b/htroot/Settings_p.html
index a10298134..6b29782f4 100644
--- a/htroot/Settings_p.html
+++ b/htroot/Settings_p.html
@@ -18,6 +18,11 @@
       <li><a href="PerformanceMemory_p.html">Performance Settings for Memory</a></li>
       <li><a href="PerformanceSearch_p.html">Performance Settings of Search Sequence</a></li>
     </ul>
+    <ul class="settingsMenu">
+      <li><a href="Tables_p.html">Viewer and administration for database tables</a></li>
+      <li><a href="News.html">Viewer for Peer-News</a></li>
+      <li><a href="CookieMonitorIncoming_p.html">Viewer for Cookies in Proxy</a></li>
+    </ul>
     <ul class="settingsMenu">
       <li><a href="?page=ServerAccess">Server Access Settings</a></li>
       <li><a href="?page=ProxyAccess">Proxy Access Settings</a></li>
diff --git a/htroot/Tables_p.html b/htroot/Tables_p.html
new file mode 100644
index 000000000..2f497b39c
--- /dev/null
+++ b/htroot/Tables_p.html
@@ -0,0 +1,86 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+  <head>
+    <title>YaCy '#[clientname]#': Table Viewer</title>
+    #(showtable)#::
+    <link rel="alternate" type="application/xml" title="Tables" href="Tables.rss?table=#[table]#" />
+    #(/showtable)#
+    #%env/templates/metas.template%#
+  </head>
+  <body id="Tables">
+    #%env/templates/header.template%#
+    #%env/templates/submenuConfig.template%#
+    
+    <h2>Table Administration</h2>
+    <form action="Tables_p.html" method="get">
+      <fieldset><legend>Table Selection</legend>
+      <dl>
+        <dt class="TableCellDark">Select Table:
+          <select name="table" size="1">
+            #{tables}#
+            <option value="#[name]#" #(selected)#::selected="selected"#(/selected)#>#[name]#</option>
+            #{/tables}#
+          </select></dt>
+        <dd>
+          <input type="submit" name="edittable" value="Show Table" />
+        </dd>
+      </dl>
+    </fieldset>
+    </form>
+    #(showtable)#::
+    <form action="Tables_p.html" method="post" enctype="multipart/form-data" accept-charset="UTF-8">
+      <fieldset>
+      <legend><label for="table">Table Editor: showing table '#[table]#'</label></legend>
+      <table border="0" cellpadding="2" cellspacing="1">
+        <tr class="TableHeader" valign="bottom">
+          <td>&nbsp;</td>
+          <td>PK</td>
+          #{columns}#
+          <td>#[header]#</td>
+          #{/columns}#
+        </tr>
+        #{list}#
+        <tr class="TableCell#(dark)#Light::Dark::Summary#(/dark)#">
+          <td align="left"><input type="checkbox" name="mark_#[pk]#" /></td>
+          <td>#[pk]#</td>
+          #{columns}#
+          <td align="left">#[cell]#</td>
+          #{/columns}#
+        </tr>
+        #{/list}#
+      </table>
+      <p>
+      <input type="hidden" name="table" value="#[table]#" />
+      <input type="submit" name="editrow" value="Edit Selected Row" />
+      <input type="submit" name="addrow" value="Add a new Row" />&nbsp;&nbsp;&nbsp;&nbsp;
+      <input type="submit" name="deleterows" value="Delete Selected Rows" />
+      <input type="submit" name="deletetable" value="Delete Table" />
+      </p>
+      </fieldset>
+    </form>
+    #(/showtable)#
+    #(showedit)#::
+    <form action="Tables_p.html" method="post" enctype="multipart/form-data" accept-charset="UTF-8">
+      <fieldset><legend>Row Editor</legend>
+      <dl>
+        <dt class="TableCellLight">Primary Key</dt>
+        <dd>#[pk]#</dd>
+        #{list}#
+        <dt class="TableCellLight">#[key]#</dt>
+        <dd>
+          <input type="text" name="col_#[key]#" value="#[value]#" size="80"/>
+        </dd>
+        #{/list}#
+        <dt class="TableCellDark"></dt>
+        <dd>
+          <input type="hidden" name="table" value="#[table]#" />
+          <input type="hidden" name="pk" value="#[pk]#" />
+          <input type="submit" name="commitrow" value="Commit" />
+        </dd>
+      </dl>
+    </fieldset>
+    </form>
+    #(/showedit)#
+    #%env/templates/footer.template%#
+  </body>
+</html>
diff --git a/htroot/Tables_p.java b/htroot/Tables_p.java
new file mode 100644
index 000000000..a2bb6e489
--- /dev/null
+++ b/htroot/Tables_p.java
@@ -0,0 +1,194 @@
+// News.java
+// -----------------------
+// part of YaCy
+// (C) by Michael Peter Christen; mc@yacy.net
+// first published on http://www.anomic.de
+// Frankfurt, Germany, 2005
+// last major change: 29.07.2005
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program; if not, write to the Free Software
+// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+// You must compile this file with
+// javac -classpath .:../classes Network.java
+// if the shell's current path is HTROOT
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import net.yacy.kelondro.index.RowSpaceExceededException;
+import net.yacy.kelondro.logging.Log;
+
+import de.anomic.http.server.RequestHeader;
+import de.anomic.search.Switchboard;
+import de.anomic.server.serverObjects;
+import de.anomic.server.serverSwitch;
+
+public class Tables_p {
+    
+    public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
+        final Switchboard sb = (Switchboard) env;
+        final serverObjects prop = new serverObjects();
+        String table = (post == null) ? null : post.get("table", null);
+        if (table != null && !sb.tables.hasHeap(table)) table = null;
+
+        // show table selection
+        int count = 0;
+        Iterator<String> ti = sb.tables.tables();
+        String tablename;
+        while (ti.hasNext()) {
+            tablename = ti.next();
+            prop.put("tables_" + count + "_name", tablename);
+            prop.put("tables_" + count + "_selected", (table != null && table.equals(tablename)) ? 1 : 0);
+            count++;
+        }
+        prop.put("tables", count);
+        
+        List<String> columns = null;
+        if (table != null) try {
+            columns = sb.tables.columns(table);
+        } catch (IOException e) {
+            Log.logException(e);
+        }
+        
+        // apply deletion requests
+        if (post != null && post.get("deletetable", "").length() > 0) {
+            try {
+                sb.tables.clear(table);
+            } catch (IOException e) {
+                Log.logException(e);
+            }
+        }
+        
+        if (post != null && post.get("deleterows", "").length() > 0) {
+            try {
+                for (Map.Entry<String, String> entry: post.entrySet()) {
+                    if (entry.getKey().startsWith("mark_") && entry.getValue().equals("on")) {
+                        sb.tables.delete(table, entry.getKey().substring(5).getBytes());
+                    }
+                }
+            } catch (IOException e) {
+                Log.logException(e);
+            }
+        }
+        
+        if (post != null && post.get("commitrow", "").length() > 0) {
+            String pk = post.get("pk");
+            Map<String, byte[]> map = new HashMap<String, byte[]>();
+            for (Map.Entry<String, String> entry: post.entrySet()) {
+                if (entry.getKey().startsWith("col_")) {
+                    map.put(entry.getKey().substring(4), entry.getValue().getBytes());
+                }
+            }
+            try {
+                sb.tables.insert(table, pk.getBytes(), map);
+            } catch (IOException e) {
+                Log.logException(e);
+            } catch (RowSpaceExceededException e) {
+                Log.logException(e);
+            }
+        }
+        
+        // generate table
+        prop.put("showtable", 0);
+        prop.put("showedit", 0);
+        
+        if (table != null && !post.containsKey("editrow") && !post.containsKey("addrow")) try {
+            prop.put("showtable", 1);
+            prop.put("showtable_table", table);
+            
+            // insert the columns
+            
+            for (int i = 0; i < columns.size(); i++) {
+                prop.putHTML("showtable_columns_" + i + "_header", columns.get(i));
+            }
+            prop.put("showtable_columns", columns.size());
+            
+            // insert all rows
+            final int maxCount = Math.min(1000, sb.tables.size(table));
+            final Iterator<Map.Entry<byte[], Map<String, byte[]>>> mapIterator = sb.tables.iterator(table);
+            Map.Entry<byte[], Map<String, byte[]>> record;
+            Map<String, byte[]> map;
+            byte[] pk;
+            count = 0;
+            boolean dark = true;
+            byte[] cell;
+            while ((mapIterator.hasNext()) && (count < maxCount)) {
+                record = mapIterator.next();
+                if (record == null) continue;
+                pk = record.getKey();
+                map = record.getValue();
+                prop.put("showtable_list_" + count + "_dark", ((dark) ? 1 : 0) ); dark=!dark;
+                prop.put("showtable_list_" + count + "_pk",  new String(pk));
+                for (int i = 0; i < columns.size(); i++) {
+                    cell = map.get(columns.get(i));
+                    prop.putHTML("showtable_list_" + count + "_columns_" + i + "_cell", cell == null ? "" : new String(cell));
+                }
+                prop.put("showtable_list_" + count + "_columns", columns.size());
+                
+                count++;
+            }
+            prop.put("showtable_list", count);
+        } catch (IOException e) {}
+        
+        if (post != null && table != null && post.containsKey("editrow")) try {
+            // check if we can find a key
+            String pk = null;
+            for (Map.Entry<String, String> entry: post.entrySet()) {
+                if (entry.getKey().startsWith("mark_") && entry.getValue().equals("on")) {
+                    pk = entry.getKey().substring(5);
+                    break;
+                }
+            }
+            if (pk != null && sb.tables.has(table, pk.getBytes())) {
+                setEdit(sb, prop, table, pk, columns);
+            }
+        } catch (IOException e) {}
+        
+        if (post != null && table != null && post.containsKey("addrow")) try {
+            // get a new key
+            String pk = new String(sb.tables.insert(table, new HashMap<String, byte[]>()));
+            setEdit(sb, prop, table, pk, columns);
+        } catch (IOException e) {
+            Log.logException(e);
+        } catch (RowSpaceExceededException e) {
+            Log.logException(e);
+        }
+        
+        // adding the peer address
+        prop.put("address", sb.peers.mySeed().getPublicAddress());
+        
+        // return rewrite properties
+        return prop;
+    }
+    
+    private static void setEdit(final Switchboard sb, final serverObjects prop, final String table, final String pk, List<String> columns) throws IOException {
+        prop.put("showedit", 1);
+        prop.put("showedit_table", table);
+        prop.put("showedit_pk", pk);
+        Map<String, byte[]> map = sb.tables.select(table, pk.getBytes());
+        int count = 0;
+        byte[] cell;
+        for (String col: columns) {
+            cell = map.get(col);
+            prop.put("showedit_list_" + count + "_key", col);
+            prop.put("showedit_list_" + count + "_value", cell == null ? "" : new String(cell));
+            count++;
+        }
+        prop.put("showedit_list", count);
+    }
+}
diff --git a/htroot/env/templates/header.template b/htroot/env/templates/header.template
index b0eee8b56..6564c8869 100644
--- a/htroot/env/templates/header.template
+++ b/htroot/env/templates/header.template
@@ -82,8 +82,6 @@
       <li><a href="/WatchWebStructure_p.html?host=auto&amp;depth=2&amp;time=1000" class="MenuItemLink lock">Web Visualization</a></li>
       <li><a href="/AccessTracker_p.html" class="MenuItemLink lock">Access Tracker</a></li>
       <li><a href="/ViewLog_p.html" class="MenuItemLink lock">Server Log</a></li>
-      <li><a href="/News.html" accesskey="n" class="MenuItemLink">News Monitor</a></li>
-      <li><a href="/CookieMonitorIncoming_p.html" class="MenuItemLink lock">Cookies in Proxy</a></li>
     </ul>
   </li>
   <li class="menugroup" id="menugroupPeerControl">
diff --git a/source/de/anomic/crawler/retrieval/FTPLoader.java b/source/de/anomic/crawler/retrieval/FTPLoader.java
index 3f3dcd3d6..1b7adc4ca 100644
--- a/source/de/anomic/crawler/retrieval/FTPLoader.java
+++ b/source/de/anomic/crawler/retrieval/FTPLoader.java
@@ -110,7 +110,10 @@ public class FTPLoader {
                 if (file.length() == 0) {
                     // directory -> get list of files
                     RequestHeader requestHeader = new RequestHeader();
-                    if (request.referrerhash() != null) requestHeader.put(RequestHeader.REFERER, sb.getURL(Segments.Process.LOCALCRAWLING, request.referrerhash()).toNormalform(true, false));
+                    if (request.referrerhash() != null) {
+                        DigestURI u = sb.getURL(Segments.Process.LOCALCRAWLING, request.referrerhash());
+                        if (u != null) requestHeader.put(RequestHeader.REFERER, u.toNormalform(true, false));
+                    }
                     
                     byte[] dirList = generateDirlist(ftpClient, request, path);
 
diff --git a/source/de/anomic/net/ftpc.java b/source/de/anomic/net/ftpc.java
index df65810c2..ba8eca043 100644
--- a/source/de/anomic/net/ftpc.java
+++ b/source/de/anomic/net/ftpc.java
@@ -550,7 +550,7 @@ public class ftpc {
             }
             printElements(l);
         } catch (final IOException e) {
-            errPrintln("Error: remote list not available");
+            errPrintln("Error: remote list not available (1): " + e.getMessage());
         }
         return true;
     }
@@ -1378,7 +1378,7 @@ public class ftpc {
             }
             printElements(l);
         } catch (final IOException e) {
-            errPrintln("Error: remote list not available");
+            errPrintln("Error: remote list not available (2): " + e.getMessage());
         }
         return true;
     }
diff --git a/source/de/anomic/search/Switchboard.java b/source/de/anomic/search/Switchboard.java
index 2e91214d5..5e38114e4 100644
--- a/source/de/anomic/search/Switchboard.java
+++ b/source/de/anomic/search/Switchboard.java
@@ -70,9 +70,11 @@ import net.yacy.document.content.RSSMessage;
 import net.yacy.document.content.SurrogateReader;
 import net.yacy.document.parser.html.ImageEntry;
 import net.yacy.document.parser.xml.RSSFeed;
+import net.yacy.kelondro.blob.BEncodedHeapArray;
 import net.yacy.kelondro.data.meta.DigestURI;
 import net.yacy.kelondro.data.meta.URIMetadataRow;
 import net.yacy.kelondro.data.word.Word;
+import net.yacy.kelondro.index.RowSpaceExceededException;
 import net.yacy.kelondro.logging.Log;
 import net.yacy.kelondro.order.Base64Order;
 import net.yacy.kelondro.order.Digest;
@@ -129,6 +131,7 @@ import de.anomic.http.server.ResponseHeader;
 import de.anomic.http.server.RobotsTxtConfig;
 import de.anomic.net.UPnP;
 import de.anomic.search.blockrank.CRDistribution;
+import de.anomic.server.serverObjects;
 import de.anomic.server.serverSwitch;
 import de.anomic.server.serverCore;
 import de.anomic.tools.crypt;
@@ -214,6 +217,7 @@ public final class Switchboard extends serverSwitch {
     public  Dispatcher                     dhtDispatcher;
     public  List<String>                   trail;
     public  yacySeedDB                     peers;
+    public  BEncodedHeapArray              tables;
     
     public WorkflowProcessor<indexingQueueEntry> indexingDocumentProcessor;
     public WorkflowProcessor<indexingQueueEntry> indexingCondensementProcessor;
@@ -234,14 +238,13 @@ public final class Switchboard extends serverSwitch {
     
     private static Switchboard sb = null;
     
-    public Switchboard(final File rootPath, final String initPath, final String configPath, final boolean applyPro) throws IOException {
-        super(rootPath, initPath, configPath, applyPro);
+    public Switchboard(final File rootPath, final String initPath, final String configPath) throws IOException {
+        super(rootPath, initPath, configPath);
         MemoryTracker.startSystemProfiling();
         sb=this;
         
         // set loglevel and log
         setLog(new Log("PLASMA"));
-        if (applyPro) this.log.logInfo("This is the pro-version of YaCy");
         
         // UPnP port mapping
         if (getConfigBool(SwitchboardConstants.UPNP_ENABLED, false))
@@ -274,6 +277,9 @@ public final class Switchboard extends serverSwitch {
         this.dictionariesPath = getConfigPath(SwitchboardConstants.DICTIONARY_SOURCE_PATH, SwitchboardConstants.DICTIONARY_SOURCE_PATH_DEFAULT);
         this.log.logConfig("Dictionaries Path:" + this.dictionariesPath.toString());
         
+        // init tables
+        this.tables = new BEncodedHeapArray(this.workPath, 12);
+        
         // init libraries
         this.log.logConfig("initializing libraries");
         LibraryProvider.initialize(this.dictionariesPath);
@@ -1100,6 +1106,7 @@ public final class Switchboard extends serverSwitch {
         indexSegments.close();
         peers.close();
         Cache.close();
+        tables.close();
         UPnP.deletePortMapping();
         Tray.removeTray();
         log.logConfig("SWITCHBOARD SHUTDOWN TERMINATED");
@@ -2091,6 +2098,25 @@ public final class Switchboard extends serverSwitch {
         yacyCore.log.logInfo("BOOTSTRAP: " + (peers.sizeConnected() - sc) + " new seeds while bootstraping.");
     }
     
+    
+    public void recordAPICall(final serverObjects post, final String servletName, String type, String comment) {
+        String apiurl = /*"http://localhost:" + getConfig("port", "8080") +*/ "/" + servletName + "?" + post.toString();
+        try {
+            sb.tables.insert(
+                    "api",
+                    "type", type.getBytes(),
+                    "comment", comment.getBytes(),
+                    "date", DateFormatter.formatShortMilliSecond(new Date()).getBytes(),
+                    "url", apiurl.getBytes()
+                    );
+        } catch (RowSpaceExceededException e2) {
+            Log.logException(e2);
+        } catch (IOException e2) {
+            Log.logException(e2);
+        }
+        Log.logInfo("APICALL", apiurl);
+    }
+    
     public void checkInterruption() throws InterruptedException {
         final Thread curThread = Thread.currentThread();
         if ((curThread instanceof WorkflowThread) && ((WorkflowThread)curThread).shutdownInProgress()) throw new InterruptedException("Shutdown in progress ...");
diff --git a/source/de/anomic/server/serverSwitch.java b/source/de/anomic/server/serverSwitch.java
index 319daa9f8..099f2761e 100644
--- a/source/de/anomic/server/serverSwitch.java
+++ b/source/de/anomic/server/serverSwitch.java
@@ -55,7 +55,7 @@ public class serverSwitch {
     private   final TreeMap<String, serverSwitchAction>  switchActions;
     private   final serverAccessTracker                  accessTracker;
     
-    public serverSwitch(final File rootPath, final String initPath, final String configPath, final boolean applyPro) {
+    public serverSwitch(final File rootPath, final String initPath, final String configPath) {
         // we initialize the switchboard with a property file,
         // but maintain these properties then later in a new 'config' file
         // to reset all changed configs, the config file must
@@ -79,15 +79,7 @@ public class serverSwitch {
         // if 'pro'-version is selected, overload standard settings with 'pro'-settings
         Iterator<String> i;
         String prop;
-    	if (applyPro) {
-        	i = new HashMap<String, String>(initProps).keySet().iterator(); // clone the map to avoid concurrent modification exceptions
-        	while (i.hasNext()) {
-        		prop = i.next();
-        		if (prop.endsWith("__pro")) {
-        			initProps.put(prop.substring(0, prop.length() - 5), initProps.get(prop));
-        		}
-        	}
-        }
+        
         // delete the 'pro' init settings
         i = initProps.keySet().iterator();
         while (i.hasNext()) {
diff --git a/source/de/anomic/yacy/yacyClient.java b/source/de/anomic/yacy/yacyClient.java
index 67d4d26b2..8d44b4cf6 100644
--- a/source/de/anomic/yacy/yacyClient.java
+++ b/source/de/anomic/yacy/yacyClient.java
@@ -1092,7 +1092,7 @@ public final class yacyClient {
         if(args.length > 1) {
         System.out.println("yacyClient Test");
         try {
-            final Switchboard sb = new Switchboard(new File(args[0]), "httpProxy.init", "DATA/SETTINGS/yacy.conf", false);
+            final Switchboard sb = new Switchboard(new File(args[0]), "httpProxy.init", "DATA/SETTINGS/yacy.conf");
             /*final yacyCore core =*/ new yacyCore(sb);
             sb.loadSeedLists();
             final yacySeed target = sb.peers.getConnected(args[1]);
diff --git a/source/net/yacy/kelondro/blob/BEncodedHeap.java b/source/net/yacy/kelondro/blob/BEncodedHeap.java
index f27110c2a..644f4b98a 100644
--- a/source/net/yacy/kelondro/blob/BEncodedHeap.java
+++ b/source/net/yacy/kelondro/blob/BEncodedHeap.java
@@ -26,8 +26,11 @@ package net.yacy.kelondro.blob;
 
 import java.io.File;
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.Iterator;
+import java.util.LinkedHashSet;
+import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 
@@ -47,6 +50,7 @@ import net.yacy.kelondro.util.BDecoder.BObject;
 public class BEncodedHeap implements Iterable<Map.Entry<byte[], Map<String, byte[]>>> {
 
     private Heap table;
+    private LinkedHashSet<String> columnames;
     
     /**
      * produce or open a properties table
@@ -62,6 +66,7 @@ public class BEncodedHeap implements Iterable<Map.Entry<byte[], Map<String, byte
             final ByteOrder ordering,
             int buffermax) throws IOException {
         this.table = new Heap(location, keylength, ordering, buffermax);
+        this.columnames = new LinkedHashSet<String>();
     }
     
     /**
@@ -73,12 +78,25 @@ public class BEncodedHeap implements Iterable<Map.Entry<byte[], Map<String, byte
             final File location,
             final int keylength) throws IOException {
         this.table = new Heap(location, keylength, NaturalOrder.naturalOrder, 100);
+        this.columnames = new LinkedHashSet<String>();  
+    }
+    
+    public File getFile() {
+        return this.table.heapFile;
+    }
+    
+    public int size() {
+        return this.table.size();
     }
     
     public void close() {
         this.table.close();
     }
     
+    public void clear() throws IOException {
+        this.table.clear();
+    }
+    
     /**
      * insert a map into the table
      * @param key
@@ -86,10 +104,67 @@ public class BEncodedHeap implements Iterable<Map.Entry<byte[], Map<String, byte
      * @throws RowSpaceExceededException
      * @throws IOException
      */
-    public void put(byte[] key, Map<String, byte[]> map) throws RowSpaceExceededException, IOException {
+    public void put(byte[] pk, Map<String, byte[]> map) throws RowSpaceExceededException, IOException {
         byte[] b = BEncoder.encode(BEncoder.transcode(map));
-        System.out.println(new String(b));
-        this.table.put(key, b);
+        this.table.put(pk, b);
+        this.columnames.addAll(map.keySet());
+    }
+    
+    public void put(
+            byte[] pk,
+            String key, byte[] value
+            ) throws RowSpaceExceededException, IOException {
+        byte[] b = BEncoder.encodeMap(key, value);
+        this.table.put(pk, b);
+        this.columnames.add(key);
+    }
+    public void put(
+            byte[] pk,
+            String key0, byte[] value0,
+            String key1, byte[] value1
+            ) throws RowSpaceExceededException, IOException {
+        byte[] b = BEncoder.encodeMap(
+                key0, value0,
+                key1, value1
+                );
+        this.table.put(pk, b);
+        this.columnames.add(key0);
+        this.columnames.add(key1);
+    }
+    public void put(
+            byte[] pk,
+            String key0, byte[] value0,
+            String key1, byte[] value1,
+            String key2, byte[] value2
+            ) throws RowSpaceExceededException, IOException {
+        byte[] b = BEncoder.encodeMap(
+                key0, value0,
+                key1, value1,
+                key2, value2
+                );
+        this.table.put(pk, b);
+        this.columnames.add(key0);
+        this.columnames.add(key1);
+        this.columnames.add(key2);
+    }
+    public void put(
+            byte[] pk,
+            String key0, byte[] value0,
+            String key1, byte[] value1,
+            String key2, byte[] value2,
+            String key3, byte[] value3
+            ) throws RowSpaceExceededException, IOException {
+        byte[] b = BEncoder.encodeMap(
+                key0, value0,
+                key1, value1,
+                key2, value2,
+                key3, value3
+                );
+        this.table.put(pk, b);
+        this.columnames.add(key0);
+        this.columnames.add(key1);
+        this.columnames.add(key2);
+        this.columnames.add(key3);
     }
     
     /**
@@ -98,15 +173,22 @@ public class BEncodedHeap implements Iterable<Map.Entry<byte[], Map<String, byte
      * @return the map if one found or NULL if no entry exists or the entry is corrupt
      * @throws IOException
      */
-    public Map<String, byte[]> get(byte[] key) throws IOException {
-        byte[] b = this.table.get(key);
+    public Map<String, byte[]> get(byte[] pk) throws IOException {
+        byte[] b = this.table.get(pk);
         if (b == null) return null;
         return b2m(b);
     }
+
+    public byte[] getProp(byte[] pk, String key) throws IOException {
+        byte[] b = this.table.get(pk);
+        if (b == null) return null;
+        Map<String, byte[]> map = b2m(b);
+        return map.get(key);
+    }
     
-    static Map<String, byte[]> b2m(byte[] b) {
+    private static Map<String, byte[]> b2m(byte[] b) {
         if (b == null) return null;
-        System.out.println("b = " + new String(b));
+        //System.out.println("b = " + new String(b));
         BDecoder decoder = new BDecoder(b);
         BObject bobj = decoder.parse();
         if (bobj.getType() != BDecoder.BType.dictionary) return null;
@@ -124,8 +206,8 @@ public class BEncodedHeap implements Iterable<Map.Entry<byte[], Map<String, byte
      * @param key
      * @throws IOException
      */
-    public void delete(byte[] key) throws IOException {
-        this.table.remove(key);
+    public void delete(byte[] pk) throws IOException {
+        this.table.remove(pk);
     }
     
     /**
@@ -133,8 +215,8 @@ public class BEncodedHeap implements Iterable<Map.Entry<byte[], Map<String, byte
      * @param key
      * @return true if the row exists
      */
-    public boolean has(byte[] key) {
-        return this.table.has(key);
+    public boolean has(byte[] pk) {
+        return this.table.has(pk);
     }
     
     /**
@@ -226,31 +308,57 @@ public class BEncodedHeap implements Iterable<Map.Entry<byte[], Map<String, byte
         }
     }
     
+    public List<String> columns() {
+        if (this.columnames.size() == 0) {
+            for (Map.Entry<byte[], Map<String, byte[]>> row: this) {
+                this.columnames.addAll(row.getValue().keySet());
+            }
+        }
+        List<String> l = new ArrayList<String>();
+        l.addAll(this.columnames);
+        return l;
+    }
+    
     public static void main(String[] args) {
-        // test the class
-        File f = new File(new File("maptest").getAbsolutePath());
-        //System.out.println(f.getAbsolutePath());
-        //System.out.println(f.getParent());
-        if (f.exists()) FileUtils.deletedelete(f);
-        try {
-            BEncodedHeap map = new BEncodedHeap(f, 4);
-            // put some values into the map
-            Map<String, byte[]> m = new HashMap<String, byte[]>();
-            m.put("k", "000".getBytes()); map.put("123".getBytes(), m);
-            m.put("k", "111".getBytes()); map.put("456".getBytes(), m);
-            m.put("k", "222".getBytes()); map.put("789".getBytes(), m);
-            // iterate over keys
-            Iterator<Map.Entry<byte[], Map<String, byte[]>>> i = map.iterator();
-            while (i.hasNext()) {
-                Map.Entry<byte[], Map<String, byte[]>> entry = i.next();
-                System.out.println(new String(entry.getKey(), "UTF-8") + ": " + entry.getValue());
+        if (args.length == 0) {
+            // test the class
+            File f = new File(new File("maptest").getAbsolutePath());
+            //System.out.println(f.getAbsolutePath());
+            //System.out.println(f.getParent());
+            if (f.exists()) FileUtils.deletedelete(f);
+            try {
+                BEncodedHeap map = new BEncodedHeap(f, 4);
+                // put some values into the map
+                Map<String, byte[]> m = new HashMap<String, byte[]>();
+                m.put("k", "000".getBytes()); map.put("123".getBytes(), m);
+                m.put("k", "111".getBytes()); map.put("456".getBytes(), m);
+                m.put("k", "222".getBytes()); map.put("789".getBytes(), m);
+                // iterate over keys
+                Iterator<Map.Entry<byte[], Map<String, byte[]>>> i = map.iterator();
+                while (i.hasNext()) {
+                    Map.Entry<byte[], Map<String, byte[]>> entry = i.next();
+                    System.out.println(new String(entry.getKey(), "UTF-8") + ": " + entry.getValue());
+                }
+                // clean up
+                map.close();
+            } catch (IOException e) {
+                Log.logException(e);
+            } catch (RowSpaceExceededException e) {
+                Log.logException(e);
+            }
+        } else {
+            File f = new File(args[0]);
+            try {
+                BEncodedHeap map = new BEncodedHeap(f, 12);
+                Iterator<Map.Entry<byte[], Map<String, byte[]>>> i = map.iterator();
+                while (i.hasNext()) {
+                    Map.Entry<byte[], Map<String, byte[]>> entry = i.next();
+                    System.out.println(new String(entry.getKey(), "UTF-8") + ": " + entry.getValue());
+                }
+                map.close();
+            } catch (IOException e) {
+                Log.logException(e);
             }
-            // clean up
-            map.close();
-        } catch (IOException e) {
-            Log.logException(e);
-        } catch (RowSpaceExceededException e) {
-            Log.logException(e);
         }
     }
 }
diff --git a/source/net/yacy/kelondro/blob/BEncodedHeapArray.java b/source/net/yacy/kelondro/blob/BEncodedHeapArray.java
new file mode 100644
index 000000000..7c6513607
--- /dev/null
+++ b/source/net/yacy/kelondro/blob/BEncodedHeapArray.java
@@ -0,0 +1,333 @@
+// BEncodedHeapArray.java
+// (C) 2010 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
+// first published 14.01.2010 on http://yacy.net
+//
+// This is a part of YaCy, a peer-to-peer based web search engine
+//
+// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
+// $LastChangedRevision: 6539 $
+// $LastChangedBy: low012 $
+//
+// LICENSE
+// 
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program; if not, write to the Free Software
+// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+
+package net.yacy.kelondro.blob;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+import net.yacy.kelondro.index.RowSpaceExceededException;
+import net.yacy.kelondro.logging.Log;
+import net.yacy.kelondro.util.FileUtils;
+
+
+public class BEncodedHeapArray {
+ 
+    private static final String suffix = ".bheap";
+    private static final String system_table_pkcounter = "pkcounter";
+    private static final String system_table_pkcounter_counterName = "pk";
+    
+    private File location;
+    private ConcurrentHashMap<String, BEncodedHeap> tables;
+    private int keymaxlen;
+    
+    public BEncodedHeapArray(final File location, final int keymaxlen) {
+        this.location = location;
+        this.keymaxlen = keymaxlen;
+        this.tables = new ConcurrentHashMap<String, BEncodedHeap>();
+        String[] files = this.location.list();
+        String tablename;
+        File file;
+        for (String f: files) {
+            if (f.endsWith(suffix)) {
+                file = new File(this.location, f);
+                if (file.length() == 0) {
+                    file.delete();
+                    continue;
+                }
+                tablename = f.substring(0, f.length() - suffix.length());
+                try {
+                    getHeap(tablename);
+                } catch (IOException e) {
+                }
+            }
+        }
+    }
+    
+    public Iterator<String> tables() {
+        return this.tables.keySet().iterator();
+    }
+    
+    public void close(final String tablename) {
+        final BEncodedHeap heap = this.tables.remove(tablename);
+        if (heap == null) return;
+        heap.close();
+    }
+
+    public void close() {
+        for (BEncodedHeap heap: this.tables.values()) heap.close();
+        this.tables.clear();
+    }
+
+    public void clear(final String tablename) throws IOException {
+        BEncodedHeap heap = getHeap(tablename);
+        if (heap == null) return;
+        heap.clear();
+        File f = heap.getFile();
+        heap.close();
+        heap = null;
+        this.tables.remove(tablename);
+        FileUtils.deletedelete(f);
+    }
+    
+    public boolean hasHeap(final String tablename) {
+        try {
+            return getHeap(tablename) != null;
+        } catch (IOException e) {
+            return false;
+        }
+    }
+    
+    private BEncodedHeap getHeap(final String tablename) throws IOException {
+        final String table = tablename + suffix;
+        BEncodedHeap heap = this.tables.get(tablename);
+        if (heap != null) return heap;
+        
+        // open a new heap and register it in the tables
+        final File heapf = new File(this.location, table);
+        heap = new BEncodedHeap(heapf, this.keymaxlen);
+        this.tables.put(tablename, heap);
+        return heap;
+    }
+
+    public int size(String table) throws IOException {
+        BEncodedHeap heap = getHeap(table);
+        return heap.size();
+    }
+    
+    private byte[] ukey(String tablename) throws IOException {
+        byte[] pk = select(system_table_pkcounter, tablename.getBytes(), system_table_pkcounter_counterName);
+        int pki;
+        if (pk == null) {
+            pki = size(tablename);
+        } else {
+            pki = Integer.parseInt(new String(pk)) + 1;
+        }
+        while (true) {
+            pk = int2key(pki).getBytes();
+            if (!has(tablename, pk)) break;
+            pki++;
+        }
+        return pk;
+    }
+    
+    private String int2key(int i) {
+        StringBuilder sb = new StringBuilder(this.keymaxlen);
+        String is = Integer.toString(i);
+        for (int j = 0; j < this.keymaxlen - is.length(); j++) sb.append('0');
+        sb.append(is);
+        return sb.toString();
+    }
+    
+    /**
+     * insert a map into a table using a new unique key
+     * @param tablename
+     * @param map
+     * @throws RowSpaceExceededException
+     * @throws IOException
+     */
+    public byte[] insert(final String tablename, Map<String, byte[]> map) throws RowSpaceExceededException, IOException {
+        byte[] uk = ukey(tablename);
+        insert(tablename, uk, map);
+        insert(system_table_pkcounter, tablename.getBytes(), system_table_pkcounter_counterName, uk);
+        return uk;
+    }
+
+    public byte[] insert(final String tablename, String key, byte[] value) throws RowSpaceExceededException, IOException {
+        byte[] uk = ukey(tablename);
+        insert(tablename, uk, key, value);
+        insert(system_table_pkcounter, tablename.getBytes(), system_table_pkcounter_counterName, uk);
+        return uk;
+    }
+    
+    public byte[] insert(final String tablename,
+            String key0, byte[] value0,
+            String key1, byte[] value1
+            ) throws RowSpaceExceededException, IOException {
+        byte[] uk = ukey(tablename);
+        insert(tablename, uk,
+                key0, value0,
+                key1, value1
+                );
+        insert(system_table_pkcounter, tablename.getBytes(), system_table_pkcounter_counterName, uk);
+        return uk;
+    }
+    
+    public byte[] insert(final String tablename,
+            String key0, byte[] value0,
+            String key1, byte[] value1,
+            String key2, byte[] value2
+            ) throws RowSpaceExceededException, IOException {
+        byte[] uk = ukey(tablename);
+        insert(tablename, uk,
+                key0, value0,
+                key1, value1,
+                key2, value2
+                );
+        insert(system_table_pkcounter, tablename.getBytes(), system_table_pkcounter_counterName, uk);
+        return uk;
+    }
+    
+    public byte[] insert(final String tablename,
+            String key0, byte[] value0,
+            String key1, byte[] value1,
+            String key2, byte[] value2,
+            String key3, byte[] value3
+            ) throws RowSpaceExceededException, IOException {
+        byte[] uk = ukey(tablename);
+        insert(tablename, uk,
+                key0, value0,
+                key1, value1,
+                key2, value2,
+                key3, value3
+                );
+        insert(system_table_pkcounter, tablename.getBytes(), system_table_pkcounter_counterName, uk);
+        return uk;
+    }
+
+    public void insert(final String table, byte[] pk,
+            String key, byte[] value
+            ) throws RowSpaceExceededException, IOException {
+        BEncodedHeap heap = getHeap(table);
+        heap.put(pk, key, value);
+    }
+
+    public void insert(final String table, byte[] pk,
+            String key0, byte[] value0,
+            String key1, byte[] value1
+            ) throws RowSpaceExceededException, IOException {
+        BEncodedHeap heap = getHeap(table);
+        heap.put(pk,
+                key0, value0,
+                key1, value1
+                );
+    }
+    
+    public void insert(final String table, byte[] pk,
+            String key0, byte[] value0,
+            String key1, byte[] value1,
+            String key2, byte[] value2
+            ) throws RowSpaceExceededException, IOException {
+        BEncodedHeap heap = getHeap(table);
+        heap.put(pk,
+                key0, value0,
+                key1, value1,
+                key2, value2
+                );
+    }
+    
+    public void insert(final String table, byte[] pk,
+            String key0, byte[] value0,
+            String key1, byte[] value1,
+            String key2, byte[] value2,
+            String key3, byte[] value3
+            ) throws RowSpaceExceededException, IOException {
+        BEncodedHeap heap = getHeap(table);
+        heap.put(pk,
+                key0, value0,
+                key1, value1,
+                key2, value2,
+                key3, value3
+                );
+    }
+
+    public void insert(final String table, byte[] pk, Map<String, byte[]> map) throws RowSpaceExceededException, IOException {
+        BEncodedHeap heap = getHeap(table);
+        heap.put(pk, map);
+    }
+
+    public Map<String, byte[]> select(final String table, byte[] pk) throws IOException {
+        BEncodedHeap heap = getHeap(table);
+        return heap.get(pk);
+    }
+
+    public byte[] select(final String table, byte[] pk, String key) throws IOException {
+        BEncodedHeap heap = getHeap(table);
+        return heap.getProp(pk, key);
+    }
+
+    public void delete(final String table, byte[] pk) throws IOException {
+        BEncodedHeap heap = getHeap(table);
+        heap.delete(pk);
+    }
+
+    public boolean has(String table, byte[] key) throws IOException {
+        BEncodedHeap heap = getHeap(table);
+        return heap.has(key);
+    }
+
+    public Iterator<byte[]> keys(String table) throws IOException {
+        BEncodedHeap heap = getHeap(table);
+        return heap.keys();
+    }
+
+    public Iterator<Map.Entry<byte[], Map<String, byte[]>>> iterator(String table) throws IOException {
+        BEncodedHeap heap = getHeap(table);
+        return heap.iterator();
+    }
+    
+    public List<String> columns(String table) throws IOException {
+        BEncodedHeap heap = getHeap(table);
+        return heap.columns();
+    }
+    
+    public static void main(String[] args) {
+        // test the class
+        File f = new File(new File("maptest").getAbsolutePath());
+        // System.out.println(f.getAbsolutePath());
+        // System.out.println(f.getParent());
+        try {
+            BEncodedHeapArray map = new BEncodedHeapArray(f.getParentFile(), 4);
+            // put some values into the map
+            Map<String, byte[]> m = new HashMap<String, byte[]>();
+            m.put("k", "000".getBytes());
+            map.insert("testdao", "123".getBytes(), m);
+            m.put("k", "111".getBytes());
+            map.insert("testdao", "456".getBytes(), m);
+            m.put("k", "222".getBytes());
+            map.insert("testdao", "789".getBytes(), m);
+            // iterate over keys
+            Iterator<Map.Entry<byte[], Map<String, byte[]>>> i = map.iterator("testdao");
+            while (i.hasNext()) {
+                Map.Entry<byte[], Map<String, byte[]>> entry = i.next();
+                System.out.println(new String(entry.getKey(), "UTF-8") + ": "
+                        + entry.getValue());
+            }
+            // clean up
+            map.close();
+        } catch (IOException e) {
+            Log.logException(e);
+        } catch (RowSpaceExceededException e) {
+            Log.logException(e);
+        }
+    }
+}
diff --git a/source/net/yacy/kelondro/blob/Heap.java b/source/net/yacy/kelondro/blob/Heap.java
index b85d1d1cc..3dc3c35e0 100755
--- a/source/net/yacy/kelondro/blob/Heap.java
+++ b/source/net/yacy/kelondro/blob/Heap.java
@@ -119,15 +119,14 @@ public final class Heap extends HeapModifier implements BLOB {
      * @return true if the key exists, false otherwise
      */
     @Override
-    public synchronized boolean has(final byte[] key) {
+    public synchronized boolean has(byte[] key) {
         assert index != null;
-        assert this.keylength == key.length : this.keylength + "!=" + key.length;
-        
+        key = normalizeKey(key);
         // check the buffer
         if (this.buffer.containsKey(new String(key))) return true;
         return super.has(key);
     }
-
+    
     /**
      * add a BLOB to the heap: this adds the blob always to the end of the file
      * @param key
@@ -135,19 +134,15 @@ public final class Heap extends HeapModifier implements BLOB {
      * @throws IOException
      * @throws RowSpaceExceededException 
      */
-    private void add(final byte[] key, final byte[] blob) throws IOException, RowSpaceExceededException {
+    private void add(byte[] key, final byte[] blob) throws IOException, RowSpaceExceededException {
         assert blob.length > 0;
-        assert key.length == this.keylength;
-        assert this.keylength == key.length : this.keylength + "!=" + key.length;
         if ((blob == null) || (blob.length == 0)) return;
         final int pos = (int) file.length();
+        key = normalizeKey(key);
         index.put(key, pos);
         file.seek(pos);
         file.writeInt(this.keylength + blob.length);
         file.write(key);
-        if (this.keylength > key.length) {
-            for (int i = 0; i < this.keylength - key.length; i++) file.write(HeapWriter.ZERO);
-        }
         file.write(blob, 0, blob.length);
     }
     
@@ -171,8 +166,7 @@ public final class Heap extends HeapModifier implements BLOB {
         byte[] key, blob;
         while (i.hasNext()) {
             entry = i.next();
-            key = entry.getKey().getBytes();
-            assert key.length == this.keylength : "key.length = " + key.length + ", this.keylength = " + this.keylength;
+            key = normalizeKey(entry.getKey().getBytes());
             blob = entry.getValue();
             posBuffer += 4 + this.keylength + blob.length;
         }
@@ -187,8 +181,7 @@ public final class Heap extends HeapModifier implements BLOB {
         byte[] b;
         while (i.hasNext()) {
             entry = i.next();
-            key = entry.getKey().getBytes();
-            assert key.length == this.keylength : "key.length = " + key.length + ", this.keylength = " + this.keylength;
+            key = normalizeKey(entry.getKey().getBytes());
             blob = entry.getValue();
             index.put(key, posFile);
             b = AbstractWriter.int2array(this.keylength + blob.length);
@@ -218,8 +211,8 @@ public final class Heap extends HeapModifier implements BLOB {
      * @throws IOException
      */
     @Override
-    public synchronized byte[] get(final byte[] key) throws IOException {
-        assert this.keylength == key.length : this.keylength + "!=" + key.length;
+    public synchronized byte[] get(byte[] key) throws IOException {
+        key = normalizeKey(key);
         
         // check the buffer
         byte[] blob = this.buffer.get(new String(key));
@@ -235,9 +228,8 @@ public final class Heap extends HeapModifier implements BLOB {
      * @throws IOException
      */
     @Override
-    public synchronized long length(final byte[] key) throws IOException {
-        assert this.keylength == key.length : this.keylength + "!=" + key.length;
-        
+    public synchronized long length(byte[] key) throws IOException {
+        key = normalizeKey(key);
         // check the buffer
         byte[] blob = this.buffer.get(new String(key));
         if (blob != null) return blob.length;
@@ -297,8 +289,8 @@ public final class Heap extends HeapModifier implements BLOB {
      * @throws RowSpaceExceededException 
      */
     @Override
-    public synchronized void put(final byte[] key, final byte[] b) throws IOException, RowSpaceExceededException {
-        assert this.keylength == key.length : this.keylength + "!=" + key.length;
+    public synchronized void put(byte[] key, final byte[] b) throws IOException, RowSpaceExceededException {
+        key = normalizeKey(key);
         
         // we do not write records of length 0 into the BLOB
         if (b.length == 0) return;
@@ -329,8 +321,8 @@ public final class Heap extends HeapModifier implements BLOB {
         this.buffersize += b.length;
     }
     
-    private boolean putToGap(final byte[] key, final byte[] b) throws IOException, RowSpaceExceededException {
-        assert this.keylength == key.length : this.keylength + "!=" + key.length;
+    private boolean putToGap(byte[] key, final byte[] b) throws IOException, RowSpaceExceededException {
+        key = normalizeKey(key);
         
         // we do not write records of length 0 into the BLOB
         if (b.length == 0) return true;
@@ -423,8 +415,8 @@ public final class Heap extends HeapModifier implements BLOB {
      * @throws IOException
      */
     @Override
-    public synchronized void remove(final byte[] key) throws IOException {
-        assert this.keylength == key.length : this.keylength + "!=" + key.length;
+    public synchronized void remove(byte[] key) throws IOException {
+        key = normalizeKey(key);
         
         // check the buffer
         byte[] blob = this.buffer.remove(new String(key));
diff --git a/source/net/yacy/kelondro/blob/HeapModifier.java b/source/net/yacy/kelondro/blob/HeapModifier.java
index 6b673c792..a4d73fa03 100644
--- a/source/net/yacy/kelondro/blob/HeapModifier.java
+++ b/source/net/yacy/kelondro/blob/HeapModifier.java
@@ -94,8 +94,8 @@ public class HeapModifier extends HeapReader implements BLOB {
      * @param key  the primary key
      * @throws IOException
      */
-    public synchronized void remove(final byte[] key) throws IOException {
-        assert index.row().primaryKeyLength == key.length : index.row().primaryKeyLength + "!=" + key.length;
+    public synchronized void remove(byte[] key) throws IOException {
+        key = normalizeKey(key);
         
         // check if the index contains the key
         final long seek = index.get(key);
@@ -234,14 +234,14 @@ public class HeapModifier extends HeapReader implements BLOB {
 	}
 
 	public synchronized int replace(byte[] key, Rewriter rewriter) throws IOException {
-	    assert index.row().primaryKeyLength == key.length : index.row().primaryKeyLength + "!=" + key.length;
-        
+	    key = normalizeKey(key);
+	    
 	    // check if the index contains the key
         final long pos = index.get(key);
         if (pos < 0) return 0;
         
         // check consistency of the index
-        assert (checkKey(key, pos)) : "key compare failed; key = " + new String(key) + ", seek = " + pos;
+        assert checkKey(key, pos) : "key compare failed; key = " + new String(key) + ", seek = " + pos;
         
         // access the file and read the container
         file.seek(pos);
diff --git a/source/net/yacy/kelondro/blob/HeapReader.java b/source/net/yacy/kelondro/blob/HeapReader.java
index dbaeb5394..38ba321f5 100644
--- a/source/net/yacy/kelondro/blob/HeapReader.java
+++ b/source/net/yacy/kelondro/blob/HeapReader.java
@@ -110,6 +110,21 @@ public class HeapReader {
         this.file.close();
         // the file will be opened again automatically when the next access to it comes.
     }
+
+    protected byte[] normalizeKey(byte[] key) {
+        return normalizeKey(key, this.keylength);
+    }
+    
+    protected static byte[] normalizeKey(byte[] key, int keylength) {
+        if (key.length == keylength) return key;
+        byte[] k = new byte[keylength];
+        if (key.length < keylength) {
+            System.arraycopy(key, 0, k, 0, key.length);
+        } else {
+            System.arraycopy(key, 0, k, 0, keylength);
+        }
+        return k;
+    }
     
     private boolean initIndexReadDump() {
         // look for an index dump and read it if it exist
@@ -278,9 +293,9 @@ public class HeapReader {
      * @param key
      * @return true if the key exists, false otherwise
      */
-    public synchronized boolean has(final byte[] key) {
+    public synchronized boolean has(byte[] key) {
         assert index != null;
-        assert index.row().primaryKeyLength == key.length : index.row().primaryKeyLength + "!=" + key.length;
+        key = normalizeKey(key);
         
         // check if the file index contains the key
         return index.get(key) >= 0;
@@ -344,8 +359,8 @@ public class HeapReader {
      * @return
      * @throws IOException
      */
-    public synchronized byte[] get(final byte[] key) throws IOException {
-        assert index.row().primaryKeyLength == key.length : index.row().primaryKeyLength + "!=" + key.length;
+    public synchronized byte[] get(byte[] key) throws IOException {
+        key = normalizeKey(key);
        
         // check if the index contains the key
         final long pos = index.get(key);
@@ -377,8 +392,9 @@ public class HeapReader {
         return blob;
     }
     
-    protected boolean checkKey(final byte[] key, final long pos) throws IOException {
-    	file.seek(pos);
+    protected boolean checkKey(byte[] key, final long pos) throws IOException {
+        key = normalizeKey(key);
+        file.seek(pos);
         file.readInt(); // skip the size value
         
         // read the key
@@ -394,7 +410,7 @@ public class HeapReader {
      * @throws IOException
      */
     public synchronized long length(byte[] key) throws IOException {
-        assert index.row().primaryKeyLength == key.length : index.row().primaryKeyLength + "!=" + key.length;
+        key = normalizeKey(key);
         
         // check if the index contains the key
         final long pos = index.get(key);
@@ -533,13 +549,23 @@ public class HeapReader {
 
         private Map.Entry<byte[], byte[]> next0() {
             try {
+                byte b;
+                byte[] payload;
+                final byte[] key = new byte[this.keylen];
+                final int keylen1 = this.keylen - 1;
                 while (true) {
                     int len = is.readInt();
-                    byte[] key = new byte[this.keylen];
-                    if (is.read(key) < key.length) return null;
-                    byte[] payload = new byte[len - this.keylen];
+                    if (len == 0) continue;
+                    b = is.readByte(); // check for empty record
+                    if (b == 0) {
+                        // read some more bytes to consume the empty record
+                        is.skip(len - 1);
+                        continue;
+                    }
+                    key[0] = b;
+                    if (is.read(key, 1, keylen1) < keylen1) return null;
+                    payload = new byte[len - this.keylen];
                     if (is.read(payload) < payload.length) return null;
-                    if (key[0] == 0) continue; // this is an empty gap
                     return new entry(key, payload);
                 }
             } catch (final IOException e) {
@@ -596,4 +622,19 @@ public class HeapReader {
         }
     }
     
+    public static void main(final String args[]) {
+        File f = new File(args[0]);
+        try {
+            entries hr = new HeapReader.entries(f, 12);
+            Map.Entry<byte[], byte[]> entry;
+            while (hr.hasNext()) {
+                entry = hr.next();
+                System.out.println(new String(entry.getKey()) + ":" + new String(entry.getValue()));
+            }
+        } catch (IOException e) {
+            Log.logException(e);
+        }
+        
+    }
+    
 }
diff --git a/source/net/yacy/kelondro/blob/HeapWriter.java b/source/net/yacy/kelondro/blob/HeapWriter.java
index feb25f6d2..abf64ce83 100644
--- a/source/net/yacy/kelondro/blob/HeapWriter.java
+++ b/source/net/yacy/kelondro/blob/HeapWriter.java
@@ -95,11 +95,11 @@ public final class HeapWriter {
      * @throws RowSpaceExceededException 
      * @throws RowSpaceExceededException 
      */
-    public synchronized void add(final byte[] key, final byte[] blob) throws IOException, RowSpaceExceededException {
+    public synchronized void add(byte[] key, final byte[] blob) throws IOException, RowSpaceExceededException {
         //System.out.println("HeapWriter.add: " + new String(key));
         assert blob.length > 0;
-        assert key.length == this.keylength;
-        assert index.row().primaryKeyLength == key.length : index.row().primaryKeyLength + "!=" + key.length;
+        key = HeapReader.normalizeKey(key, this.keylength);
+        assert index.row().primaryKeyLength == this.keylength : index.row().primaryKeyLength + "!=" + key.length;
         assert index.get(key) < 0 : "index.get(key) = " + index.get(key) + ", index.size() = " + index.size() + ", file.length() = " + this.heapFileTMP.length() +  ", key = " + new String(key); // must not occur before
         if ((blob == null) || (blob.length == 0)) return;
         index.putUnique(key, this.seek);
diff --git a/source/net/yacy/kelondro/blob/MapDataMining.java b/source/net/yacy/kelondro/blob/MapDataMining.java
index 36201cc54..6514c14f7 100644
--- a/source/net/yacy/kelondro/blob/MapDataMining.java
+++ b/source/net/yacy/kelondro/blob/MapDataMining.java
@@ -43,10 +43,14 @@ import net.yacy.kelondro.util.ScoreCluster;
 
 
 public class MapDataMining extends MapHeap {
+    
+    private final static Long LONG0 = Long.valueOf(0);
+    private final static Double DOUBLE0 = Double.valueOf(0.0);
 
     private final String[] sortfields, longaccfields, doubleaccfields;
     private HashMap<String, ScoreCluster<String>> sortClusterMap; // a String-kelondroMScoreCluster - relation
-    private HashMap<String, Object> accMap; // to store accumulations of specific fields
+    private HashMap<String, Long>   accLong; // to store accumulations of Long cells
+    private HashMap<String, Double> accDouble; // to store accumulations of Double cells
     
 	@SuppressWarnings("unchecked")
 	public MapDataMining(final File heapFile,
@@ -77,21 +81,22 @@ public class MapDataMining extends MapHeap {
 
         Long[] longaccumulator = null;
         Double[] doubleaccumulator = null;
-        if ((longaccfields == null) && (doubleaccfields == null)) {
-        	accMap = null;
+        if (longaccfields == null) {
+        	accLong = null;
         } else {
-            accMap = new HashMap<String, Object>();
-            if (longaccfields != null) {
-                longaccumulator = new Long[longaccfields.length];
-                for (int i = 0; i < longaccfields.length; i++) {
-                    longaccumulator[i] = Long.valueOf(0);   
-                }
+            accLong = new HashMap<String, Long>();
+            longaccumulator = new Long[longaccfields.length];
+            for (int i = 0; i < longaccfields.length; i++) {
+                longaccumulator[i] = LONG0;   
             }
-            if (doubleaccfields != null) {
-                doubleaccumulator = new Double[doubleaccfields.length];
-                for (int i = 0; i < doubleaccfields.length; i++) {
-                    doubleaccumulator[i] = Double.valueOf(0);   
-                }
+        }
+        if (doubleaccfields == null) {
+            accDouble = null;
+        } else {
+            accDouble = new HashMap<String, Double>();
+            doubleaccumulator = new Double[doubleaccfields.length];
+            for (int i = 0; i < doubleaccfields.length; i++) {
+                doubleaccumulator[i] = DOUBLE0;   
             }
         }
 
@@ -99,7 +104,7 @@ public class MapDataMining extends MapHeap {
         if ((sortfields != null) || (longaccfields != null) || (doubleaccfields != null)) try {
             final CloneableIterator<byte[]> it = super.keys(true, false);
             String mapname;
-            Object cell;
+            String cell;
             long valuel;
             double valued;
             Map<String, String> map;
@@ -117,8 +122,7 @@ public class MapDataMining extends MapHeap {
                     cell = map.get(longaccfields[i]);
                     valuel = 0;
                     if (cell != null) try {
-                        if (cell instanceof Long)   valuel = ((Long) cell).longValue();
-                        if (cell instanceof String) valuel = Long.parseLong((String) cell);
+                        valuel = Long.parseLong(cell);
                         longaccumulator[i] = Long.valueOf(longaccumulator[i].longValue() + valuel);
                     } catch (final NumberFormatException e) {}
                 }
@@ -127,8 +131,7 @@ public class MapDataMining extends MapHeap {
                     cell = map.get(doubleaccfields[i]);
                     valued = 0d;
                     if (cell != null) try {
-                        if (cell instanceof Double) valued = ((Double) cell).doubleValue();
-                        if (cell instanceof String) valued = Double.parseDouble((String) cell);
+                        valued = Double.parseDouble(cell);
                         doubleaccumulator[i] = new Double(doubleaccumulator[i].doubleValue() + valued);
                     } catch (final NumberFormatException e) {}
                 }
@@ -151,8 +154,8 @@ public class MapDataMining extends MapHeap {
         if (sortfields != null && cluster != null) for (int i = 0; i < sortfields.length; i++) sortClusterMap.put(sortfields[i], cluster[i]);
 
         // fill acc map
-        if (longaccfields != null && longaccumulator != null) for (int i = 0; i < longaccfields.length; i++) accMap.put(longaccfields[i], longaccumulator[i]);
-        if (doubleaccfields != null && doubleaccumulator != null) for (int i = 0; i < doubleaccfields.length; i++) accMap.put(doubleaccfields[i], doubleaccumulator[i]);
+        if (longaccfields != null && longaccumulator != null) for (int i = 0; i < longaccfields.length; i++) accLong.put(longaccfields[i], longaccumulator[i]);
+        if (doubleaccfields != null && doubleaccumulator != null) for (int i = 0; i < doubleaccfields.length; i++) accDouble.put(doubleaccfields[i], doubleaccumulator[i]);
     }
 
     @Override
@@ -165,19 +168,20 @@ public class MapDataMining extends MapHeap {
             }
         }
         
-        if ((longaccfields == null) && (doubleaccfields == null)) {
-        	accMap = null;
+        if (longaccfields == null) {
+            accLong = null;
         } else {
-        	accMap = new HashMap<String, Object>();
-        	if (longaccfields != null) {
-                for (int i = 0; i < longaccfields.length; i++) {
-            		accMap.put(longaccfields[i], Long.valueOf(0));
-            	}
+            accLong = new HashMap<String, Long>();
+            for (int i = 0; i < longaccfields.length; i++) {
+                accLong.put(longaccfields[i], LONG0);
             }
-        	if (doubleaccfields != null) {
-                for (int i = 0; i < doubleaccfields.length; i++) {
-            		accMap.put(doubleaccfields[i], new Double(0));
-            	}
+        }
+        if (doubleaccfields == null) {
+            accDouble = null;
+        } else {
+            accDouble = new HashMap<String, Double>();
+            for (int i = 0; i < doubleaccfields.length; i++) {
+                accDouble.put(doubleaccfields[i], DOUBLE0);
             }
         }
     }
@@ -190,6 +194,9 @@ public class MapDataMining extends MapHeap {
         
         super.put(key, newMap);
         
+        // update sortCluster
+        if (sortClusterMap != null) updateSortCluster(key, newMap);
+
         // update elementCount
         if ((longaccfields != null) || (doubleaccfields != null)) {
             final Map<String, String> oldMap = super.get(key, false);
@@ -197,13 +204,10 @@ public class MapDataMining extends MapHeap {
                 // element exists, update acc
                 if ((longaccfields != null) || (doubleaccfields != null)) updateAcc(oldMap, false);
             }
-        }
         
-        // update sortCluster
-        if (sortClusterMap != null) updateSortCluster(key, newMap);
-
-        // update accumulators with new values (add)
-        if ((longaccfields != null) || (doubleaccfields != null)) updateAcc(newMap, true);
+            // update accumulators with new values (add)
+            updateAcc(newMap, true);
+        }
     }
     
     private void updateAcc(final Map<String, String> map, final boolean add) {
@@ -217,11 +221,11 @@ public class MapDataMining extends MapHeap {
             if (value != null) {
                 try {
                     valuel = Long.parseLong(value);
-                    longaccumulator = (Long) accMap.get(longaccfields[i]);
+                    longaccumulator = accLong.get(longaccfields[i]);
                     if (add) {
-                        accMap.put(longaccfields[i], Long.valueOf(longaccumulator.longValue() + valuel));
+                        accLong.put(longaccfields[i], Long.valueOf(longaccumulator.longValue() + valuel));
                     } else {
-                        accMap.put(longaccfields[i], Long.valueOf(longaccumulator.longValue() - valuel));
+                        accLong.put(longaccfields[i], Long.valueOf(longaccumulator.longValue() - valuel));
                     }
                 } catch (final NumberFormatException e) {}
             }
@@ -231,11 +235,11 @@ public class MapDataMining extends MapHeap {
             if (value != null) {
                 try {
                     valued = Double.parseDouble(value);
-                    doubleaccumulator = (Double) accMap.get(doubleaccfields[i]);
+                    doubleaccumulator = accDouble.get(doubleaccfields[i]);
                     if (add) {
-                        accMap.put(doubleaccfields[i], new Double(doubleaccumulator.doubleValue() + valued));
+                        accDouble.put(doubleaccfields[i], Double.valueOf(doubleaccumulator.doubleValue() + valued));
                     } else {
-                        accMap.put(doubleaccfields[i], new Double(doubleaccumulator.doubleValue() - valued));
+                        accDouble.put(doubleaccfields[i], Double.valueOf(doubleaccumulator.doubleValue() - valued));
                     }
                 } catch (final NumberFormatException e) {}
             }
@@ -330,13 +334,13 @@ public class MapDataMining extends MapHeap {
     }
     
     public synchronized long getLongAcc(final String field) {
-        final Long accumulator = (Long) accMap.get(field);
+        final Long accumulator = accLong.get(field);
         if (accumulator == null) return -1;
         return accumulator.longValue();
     }
     
     public synchronized double getDoubleAcc(final String field) {
-        final Double accumulator = (Double) accMap.get(field);
+        final Double accumulator = accDouble.get(field);
         if (accumulator == null) return -1;
         return accumulator.doubleValue();
     }
diff --git a/source/net/yacy/kelondro/util/BDecoder.java b/source/net/yacy/kelondro/util/BDecoder.java
index 96ae8610c..4eaea6dfc 100644
--- a/source/net/yacy/kelondro/util/BDecoder.java
+++ b/source/net/yacy/kelondro/util/BDecoder.java
@@ -109,6 +109,16 @@ public class BDecoder {
             os.write(_p);
             os.write(this.b);
         }
+        public static void toStream(OutputStream os, byte[] b) throws IOException {
+            os.write(Integer.toString(b.length).getBytes());
+            os.write(_p);
+            os.write(b);
+        }
+        public static void toStream(OutputStream os, String s) throws IOException {
+            os.write(Integer.toString(s.length()).getBytes());
+            os.write(_p);
+            os.write(s.getBytes());
+        }
     }
     
     public static class BListObject extends BDfltObject implements BObject {
@@ -164,6 +174,60 @@ public class BDecoder {
             }
             os.write(_e);
         }
+        public static void toStream(
+                OutputStream os,
+                String key, byte[] value
+                ) throws IOException {
+            os.write(_d);
+            BStringObject.toStream(os, key);
+            BStringObject.toStream(os, value);
+            os.write(_e);
+        }
+        public static void toStream(
+                OutputStream os,
+                String key0, byte[] value0,
+                String key1, byte[] value1
+                ) throws IOException {
+            os.write(_d);
+            BStringObject.toStream(os, key0);
+            BStringObject.toStream(os, value0);
+            BStringObject.toStream(os, key1);
+            BStringObject.toStream(os, value1);
+            os.write(_e);
+        }
+        public static void toStream(
+                OutputStream os,
+                String key0, byte[] value0,
+                String key1, byte[] value1,
+                String key2, byte[] value2
+                ) throws IOException {
+            os.write(_d);
+            BStringObject.toStream(os, key0);
+            BStringObject.toStream(os, value0);
+            BStringObject.toStream(os, key1);
+            BStringObject.toStream(os, value1);
+            BStringObject.toStream(os, key2);
+            BStringObject.toStream(os, value2);
+            os.write(_e);
+        }
+        public static void toStream(
+                OutputStream os,
+                String key0, byte[] value0,
+                String key1, byte[] value1,
+                String key2, byte[] value2,
+                String key3, byte[] value3
+                ) throws IOException {
+            os.write(_d);
+            BStringObject.toStream(os, key0);
+            BStringObject.toStream(os, value0);
+            BStringObject.toStream(os, key1);
+            BStringObject.toStream(os, value1);
+            BStringObject.toStream(os, key2);
+            BStringObject.toStream(os, value2);
+            BStringObject.toStream(os, key3);
+            BStringObject.toStream(os, value3);
+            os.write(_e);
+        }
     }
     
     public static class BIntegerObject extends BDfltObject implements BObject {
diff --git a/source/net/yacy/kelondro/util/BEncoder.java b/source/net/yacy/kelondro/util/BEncoder.java
index 14a07c96c..6c04590b9 100644
--- a/source/net/yacy/kelondro/util/BEncoder.java
+++ b/source/net/yacy/kelondro/util/BEncoder.java
@@ -68,8 +68,74 @@ public class BEncoder {
             dict.toStream(baos);
             baos.close();
             return baos.toByteArray();
-        } catch (IOException e) {
-        }
+        } catch (IOException e) {}
+        return null;
+    }
+    
+    public static byte[] encodeMap(String key, byte[] value) {
+        ByteArrayOutputStream baos = new ByteArrayOutputStream();
+        try {
+            BDecoder.BDictionaryObject.toStream(baos, key, value);
+            baos.close();
+            return baos.toByteArray();
+        } catch (IOException e) {}
+        return null;
+    }
+    
+    public static byte[] encodeMap(
+            String key0, byte[] value0,
+            String key1, byte[] value1
+            ) {
+        ByteArrayOutputStream baos = new ByteArrayOutputStream();
+        try {
+            BDecoder.BDictionaryObject.toStream(
+                    baos,
+                    key0, value0,
+                    key1, value1
+                    );
+            baos.close();
+            return baos.toByteArray();
+        } catch (IOException e) {}
+        return null;
+    }
+    
+    public static byte[] encodeMap(
+            String key0, byte[] value0,
+            String key1, byte[] value1,
+            String key2, byte[] value2
+            ) {
+        ByteArrayOutputStream baos = new ByteArrayOutputStream();
+        try {
+            BDecoder.BDictionaryObject.toStream(
+                    baos,
+                    key0, value0,
+                    key1, value1,
+                    key2, value2
+                    );
+            baos.close();
+            return baos.toByteArray();
+        } catch (IOException e) {}
+        return null;
+    }
+    
+    public static byte[] encodeMap(
+            String key0, byte[] value0,
+            String key1, byte[] value1,
+            String key2, byte[] value2,
+            String key3, byte[] value3
+            ) {
+        ByteArrayOutputStream baos = new ByteArrayOutputStream();
+        try {
+            BDecoder.BDictionaryObject.toStream(
+                    baos,
+                    key0, value0,
+                    key1, value1,
+                    key2, value2,
+                    key3, value3
+                    );
+            baos.close();
+            return baos.toByteArray();
+        } catch (IOException e) {}
         return null;
     }
     
diff --git a/source/net/yacy/yacy.java b/source/net/yacy/yacy.java
index 0b6c27214..b67c2dbcd 100644
--- a/source/net/yacy/yacy.java
+++ b/source/net/yacy/yacy.java
@@ -124,7 +124,6 @@ public final class yacy {
     // static objects
     public static final String vString = yacyBuildProperties.getVersion();
     public static double version = 0.1;
-    public static boolean pro = false;
     
     public static final String vDATE   = yacyBuildProperties.getBuildDate();
     public static final String copyright = "[ YaCy v" + vString + ", build " + vDATE + " by Michael Christen / www.yacy.net ]";
@@ -211,7 +210,6 @@ public final class yacy {
                 Log.logSevere("STARTUP", "WARNING: the file " + f + " can not be created!");
             f.deleteOnExit();
             
-            pro = new File(homePath, "libx").exists();
             final String oldconf = "DATA/SETTINGS/httpProxy.conf".replace("/", File.separator);
             final String newconf = "DATA/SETTINGS/yacy.conf".replace("/", File.separator);
             final File oldconffile = new File(homePath, oldconf);
@@ -220,7 +218,7 @@ public final class yacy {
                 if(!oldconffile.renameTo(newconfFile))
                     Log.logSevere("STARTUP", "WARNING: the file " + oldconffile + " can not be renamed to "+ newconfFile +"!");
             }
-            sb = new Switchboard(homePath, "defaults/yacy.init".replace("/", File.separator), newconf, pro);
+            sb = new Switchboard(homePath, "defaults/yacy.init".replace("/", File.separator), newconf);
             //sbSync.V(); // signal that the sb reference was set
             
             // save information about available memory at startup time