diff --git a/htroot/NetworkHistory.java b/htroot/NetworkHistory.java index b51727cee..cd2c98fc9 100644 --- a/htroot/NetworkHistory.java +++ b/htroot/NetworkHistory.java @@ -21,9 +21,9 @@ import java.io.IOException; import java.text.ParseException; import java.util.ArrayList; -import java.util.Collection; import java.util.Date; import java.util.HashMap; +import java.util.Iterator; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; @@ -33,7 +33,6 @@ import net.yacy.cora.date.GenericFormatter; import net.yacy.cora.document.encoding.ASCII; import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.util.ConcurrentLog; -import net.yacy.kelondro.blob.Tables; import net.yacy.kelondro.blob.Tables.Row; import net.yacy.search.Switchboard; import net.yacy.server.serverObjects; @@ -70,16 +69,13 @@ public class NetworkHistory { long now = System.currentTimeMillis(); long timelimit = now - maxtime * 3600000L; try { - // BEncodedHeap statTable = sb.tables.getHeap("stats"); - // Iterator i = statTable.keys(false, false); - Collection rowi = Tables.orderByPK(sb.tables.iterator("stats"), 10000, true); // this is bad, fix the statTable.keys(false, false) method + Iterator rowi = sb.tables.iterator("stats", false); Map statrow; - for (Row row: rowi) { + while (rowi.hasNext()) { + Row row = rowi.next(); String d = ASCII.String(row.getPK()); Date date = GenericFormatter.SHORT_MINUTE_FORMATTER.parse(d); - //System.out.println(date); if (date.getTime() < timelimit) break; - //Map row = statTable.get(pk); statrow = new HashMap<>(); for (String key: columns) { byte[] x = row.get(key); diff --git a/htroot/Table_YMark_p.java b/htroot/Table_YMark_p.java index da3e8eadb..a66294e02 100644 --- a/htroot/Table_YMark_p.java +++ b/htroot/Table_YMark_p.java @@ -243,7 +243,7 @@ public class Table_YMark_p { final String tagsString = YMarkUtil.cleanTagsString(post.get(YMarkEntry.BOOKMARK.TAGS.key())); mapIterator = sb.tables.bookmarks.getBookmarksByTag(bmk_user, tagsString); } else { - mapIterator = Tables.orderByPK(sb.tables.iterator(table, matcher), maxcount, false).iterator(); + mapIterator = sb.tables.iterator(table, matcher, true); } Tables.Row row; diff --git a/htroot/Tables_p.java b/htroot/Tables_p.java index 6673f03ea..50738e5fc 100644 --- a/htroot/Tables_p.java +++ b/htroot/Tables_p.java @@ -166,14 +166,12 @@ public class Tables_p { } count = 0; try { - final Iterator plainIterator = sb.tables.iterator(table, matcher); - final Iterator mapIterator = Tables.orderByPK(plainIterator, maxcount, reverse).iterator(); - Tables.Row row; + final Iterator plainIterator = sb.tables.iterator(table, matcher, !reverse); boolean dark = true; - byte[] cell; prop.putXML("showtable_" + count + "_table", table); // only used in XML - while (mapIterator.hasNext() && count < maxcount) { - row = mapIterator.next(); + byte[] cell; + while (plainIterator.hasNext() && count < maxcount) { + Tables.Row row = plainIterator.next(); if (row == null) continue; // write table content diff --git a/htroot/api/table_p.java b/htroot/api/table_p.java index f0360a90f..3044fa5e9 100644 --- a/htroot/api/table_p.java +++ b/htroot/api/table_p.java @@ -154,8 +154,7 @@ public class table_p { } int count = 0; try { - final Iterator plainIterator = sb.tables.iterator(table, matcher); - final Iterator mapIterator = Tables.orderByPK(plainIterator, maxcount, false).iterator(); + final Iterator mapIterator = sb.tables.iterator(table, matcher, true); Tables.Row trow; boolean dark = true; String cellName, cellValue; diff --git a/source/net/yacy/kelondro/blob/BEncodedHeap.java b/source/net/yacy/kelondro/blob/BEncodedHeap.java index 96d4f55cd..65a141bca 100644 --- a/source/net/yacy/kelondro/blob/BEncodedHeap.java +++ b/source/net/yacy/kelondro/blob/BEncodedHeap.java @@ -564,7 +564,7 @@ public class BEncodedHeap implements MapStore { } /** - * iteratore all keys of the table + * iterate all keys of the table * * @param up * @param rotating diff --git a/source/net/yacy/kelondro/blob/Tables.java b/source/net/yacy/kelondro/blob/Tables.java index 1718bca0b..7dc399db8 100644 --- a/source/net/yacy/kelondro/blob/Tables.java +++ b/source/net/yacy/kelondro/blob/Tables.java @@ -437,33 +437,44 @@ public class Tables implements Iterable { final BEncodedHeap heap = getHeap(table); return heap.keys(); } + + public Iterator keys(final String table, final boolean up, final boolean rotating) throws IOException { + final BEncodedHeap heap = getHeap(table); + return heap.keys(up, rotating); + } public Iterator iterator(final String table) throws IOException { - return new RowIterator(table); + return new HeapRowIterator(table); } public Iterator iterator(final String table, final String whereColumn, final byte[] whereValue) throws IOException { - return new RowIterator(table, whereColumn, whereValue); + return new HeapRowIterator(table, whereColumn, whereValue); } public Iterator iterator(final String table, final String whereColumn, final Pattern wherePattern) throws IOException { - return new RowIterator(table, whereColumn, wherePattern); + return new HeapRowIterator(table, whereColumn, wherePattern); } public Iterator iterator(final String table, final Pattern wherePattern) throws IOException { - return new RowIterator(table, wherePattern); + return new HeapRowIterator(table, wherePattern); } - public static Collection orderByPK(final Iterator rowIterator, int maxcount, boolean reverse) { - final TreeMap sortTree = new TreeMap(); - Row row; - while ((maxcount < 0 || maxcount-- > 0) && rowIterator.hasNext()) { - row = rowIterator.next(); - sortTree.put(UTF8.String(row.pk), row); - } - return reverse ? sortTree.descendingMap().values() : sortTree.values(); + public Iterator iterator(final String table, final boolean up) throws IOException { + return new OrderedRowIterator(table, up); + } + + public Iterator iterator(final String table, final String whereColumn, final byte[] whereValue, final boolean up) throws IOException { + return new OrderedRowIterator(table, whereColumn, whereValue, up); + } + + public Iterator iterator(final String table, final String whereColumn, final Pattern wherePattern, final boolean up) throws IOException { + return new OrderedRowIterator(table, whereColumn, wherePattern, up); } + public Iterator iterator(final String table, final Pattern wherePattern, final boolean up) throws IOException { + return new OrderedRowIterator(table, wherePattern, up); + } + public static Collection orderBy(final Iterator rowIterator, int maxcount, final String sortColumn) { final TreeMap sortTree = new TreeMap(); Row row; @@ -485,7 +496,7 @@ public class Tables implements Iterable { return heap.columns(); } - public class RowIterator extends LookAheadIterator implements Iterator { + public class HeapRowIterator extends LookAheadIterator implements Iterator { private final String whereColumn; private final byte[] whereValue; @@ -497,7 +508,7 @@ public class Tables implements Iterable { * @param table * @throws IOException */ - public RowIterator(final String table) throws IOException { + public HeapRowIterator(final String table) throws IOException { this.whereColumn = null; this.whereValue = null; this.wherePattern = null; @@ -513,7 +524,7 @@ public class Tables implements Iterable { * @param whereValue * @throws IOException */ - public RowIterator(final String table, final String whereColumn, final byte[] whereValue) throws IOException { + public HeapRowIterator(final String table, final String whereColumn, final byte[] whereValue) throws IOException { assert whereColumn != null || whereValue == null; this.whereColumn = whereColumn; this.whereValue = whereValue; @@ -530,7 +541,7 @@ public class Tables implements Iterable { * @param wherePattern * @throws IOException */ - public RowIterator(final String table, final String whereColumn, final Pattern wherePattern) throws IOException { + public HeapRowIterator(final String table, final String whereColumn, final Pattern wherePattern) throws IOException { this.whereColumn = whereColumn; this.whereValue = null; this.wherePattern = wherePattern == null || wherePattern.toString().isEmpty() ? null : wherePattern; @@ -545,7 +556,7 @@ public class Tables implements Iterable { * @param pattern * @throws IOException */ - public RowIterator(final String table, final Pattern pattern) throws IOException { + public HeapRowIterator(final String table, final Pattern pattern) throws IOException { this.whereColumn = null; this.whereValue = null; this.wherePattern = pattern == null || pattern.toString().isEmpty() ? null : pattern; @@ -577,9 +588,116 @@ public class Tables implements Iterable { } return null; } - } + public class OrderedRowIterator extends LookAheadIterator implements Iterator { + + private final String whereColumn; + private final byte[] whereValue; + private final Pattern wherePattern; + private final Iterator i; + private final BEncodedHeap heap; + + /** + * iterator that iterates all elements in the given table + * @param table + * @param up + * @throws IOException + */ + public OrderedRowIterator(final String table, final boolean up) throws IOException { + this.whereColumn = null; + this.whereValue = null; + this.wherePattern = null; + this.heap = getHeap(table); + this.i = heap.keys(up, false); + } + + /** + * iterator that iterates all elements in the given table + * where a given column is equal to a given value + * @param table + * @param whereColumn + * @param whereValue + * @param up + * @throws IOException + */ + public OrderedRowIterator(final String table, final String whereColumn, final byte[] whereValue, final boolean up) throws IOException { + assert whereColumn != null || whereValue == null; + this.whereColumn = whereColumn; + this.whereValue = whereValue; + this.wherePattern = null; + this.heap = getHeap(table); + this.i = heap.keys(up, false); + } + + /** + * iterator that iterates all elements in the given table + * where a given column matches with a given value + * @param table + * @param whereColumn + * @param wherePattern + * @param up + * @throws IOException + */ + public OrderedRowIterator(final String table, final String whereColumn, final Pattern wherePattern, final boolean up) throws IOException { + this.whereColumn = whereColumn; + this.whereValue = null; + this.wherePattern = wherePattern == null || wherePattern.toString().isEmpty() ? null : wherePattern; + this.heap = getHeap(table); + this.i = heap.keys(up, false); + } + + /** + * iterator that iterates all elements in the given table + * where a given column matches with a given value + * @param table + * @param wherePattern + * @param up + * @throws IOException + */ + public OrderedRowIterator(final String table, final Pattern wherePattern, final boolean up) throws IOException { + this.whereColumn = null; + this.whereValue = null; + this.wherePattern = wherePattern == null || wherePattern.toString().isEmpty() ? null : wherePattern; + this.heap = getHeap(table); + this.i = heap.keys(up, false); + } + + @Override + protected Row next0() { + if (this.i == null) return null; + Row r; + while (this.i.hasNext()) { + byte[] pk = this.i.next(); + Map map; + try { + map = this.heap.get(pk); + if (map == null) continue; + r = new Row(pk, map); + if (this.whereValue != null) { + if (ByteBuffer.equals(r.get(this.whereColumn), this.whereValue)) return r; + } else if (this.wherePattern != null) { + if (this.whereColumn == null) { + // shall match any column + for (final byte[] b: r.values()) { + if (this.wherePattern.matcher(UTF8.String(b)).matches()) return r; + } + } else { + // must match the given column + if (this.wherePattern.matcher(UTF8.String(r.get(this.whereColumn))).matches()) return r; + } + } else { + return r; + } + } catch (IOException | SpaceExceededException e) { + continue; + } + } + return null; + } + + } + public static class Data extends LinkedHashMap { private static final long serialVersionUID = 978426054043749337L; diff --git a/source/net/yacy/kelondro/index/RowSet.java b/source/net/yacy/kelondro/index/RowSet.java index 0beda9791..c2bc4a80f 100644 --- a/source/net/yacy/kelondro/index/RowSet.java +++ b/source/net/yacy/kelondro/index/RowSet.java @@ -360,6 +360,7 @@ public class RowSet extends RowCollection implements Index, Iterable, @Override public final synchronized CloneableIterator keys(final boolean up, final byte[] firstKey) { + this.sort(); return new keyIterator(up, firstKey); } @@ -378,10 +379,10 @@ public class RowSet extends RowCollection implements Index, Iterable, this.first = firstKey; this.bound = RowSet.this.sortBound; if (this.first == null) { - this.p = 0; + this.p = up ? 0 : this.bound - 1; } else { assert this.first.length == RowSet.this.rowdef.primaryKeyLength : "first.length = " + this.first.length + ", rowdef.primaryKeyLength = " + RowSet.this.rowdef.primaryKeyLength; - this.p = binaryPosition(this.first, 0); // check this to find bug in DHT selection enumeration + this.p = up ? binaryPosition(this.first, 0) : this.bound - 1; // check this to find bug in DHT selection enumeration } }