From f4ae8082c346cb2e8c29e8442c5e08194add2ba2 Mon Sep 17 00:00:00 2001 From: orbiter Date: Sun, 15 Jun 2008 23:25:57 +0000 Subject: [PATCH] - better error analysis for ooRange Exception in kelondroBase64Ordering - quadcore support for kelondroRowSet array ordering git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4932 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- .../anomic/kelondro/kelondroBase64Order.java | 12 ++- .../kelondro/kelondroRowCollection.java | 75 +++++++++++++++++-- 2 files changed, 78 insertions(+), 9 deletions(-) diff --git a/source/de/anomic/kelondro/kelondroBase64Order.java b/source/de/anomic/kelondro/kelondroBase64Order.java index 09d63abba..da45826d8 100644 --- a/source/de/anomic/kelondro/kelondroBase64Order.java +++ b/source/de/anomic/kelondro/kelondroBase64Order.java @@ -355,11 +355,15 @@ public class kelondroBase64Order extends kelondroAbstractOrder implement assert boffset < b.length; assert boffset + Math.min(bl, compiledPivot.length) - 1 >= 0; assert boffset + Math.min(bl, compiledPivot.length) - 1 < b.length; + byte bb; while ((i < compiledPivot.length) && (i < bl)) { acc = compiledPivot[i]; assert boffset + i >= 0; assert boffset + i < b.length; - bcc = ahpla[b[boffset + i]]; + bb = b[boffset + i]; + assert bb >= 0; + assert bb < 128; + bcc = ahpla[bb]; assert (bcc >= 0) : "bcc = " + bcc + ", b = " + serverLog.arrayList(b, boffset, bl) + "/" + new String(b, boffset, bl) + ", boffset = 0x" + Integer.toHexString(boffset) + ", i = " + i + "\n" + serverLog.table(b, 16, boffset); if (acc > bcc) return 1; if (acc < bcc) return -1; @@ -376,8 +380,12 @@ public class kelondroBase64Order extends kelondroAbstractOrder implement public final byte[] compilePivot(byte[] a, int aoffset, int alength) { assert (aoffset + alength <= a.length) : "a.length = " + a.length + ", aoffset = " + aoffset + ", alength = " + alength; byte[] cp = new byte[Math.min(alength, a.length - aoffset)]; + byte aa; for (int i = cp.length - 1; i >= 0; i--) { - cp[i] = ahpla[a[aoffset + i]]; + aa = a[aoffset + i]; + assert aa >= 0; + assert aa < 128; + cp[i] = ahpla[aa]; assert cp[i] != -1; } return cp; diff --git a/source/de/anomic/kelondro/kelondroRowCollection.java b/source/de/anomic/kelondro/kelondroRowCollection.java index 36c4f4b1b..ead506a1e 100644 --- a/source/de/anomic/kelondro/kelondroRowCollection.java +++ b/source/de/anomic/kelondro/kelondroRowCollection.java @@ -52,12 +52,15 @@ public class kelondroRowCollection { static final Integer dummy = new Integer(0); public static ExecutorService sortingthreadexecutor = null; + public static ExecutorService partitionthreadexecutor = null; static { if (serverProcessor.useCPU > 1) { sortingthreadexecutor = Executors.newCachedThreadPool(new NamePrefixThreadFactory("sorting")); + partitionthreadexecutor = Executors.newCachedThreadPool(new NamePrefixThreadFactory("partition")); } else { sortingthreadexecutor = null; + partitionthreadexecutor = null; } } @@ -503,13 +506,55 @@ public class kelondroRowCollection { if ((sortingthreadexecutor != null) && (!sortingthreadexecutor.isShutdown()) && (serverProcessor.useCPU > 1) && - (this.chunkcount > 80)) { + (this.chunkcount > 8000)) { // sort this using multi-threading + Future part0 = partitionthreadexecutor.submit(new partitionthread(this, 0, p, 0)); + Future part1 = partitionthreadexecutor.submit(new partitionthread(this, p, this.chunkcount, p)); + try { + int p0 = part0.get().intValue(); + Future sort0 = sortingthreadexecutor.submit(new qsortthread(this, 0, p0, 0)); + Future sort1 = sortingthreadexecutor.submit(new qsortthread(this, p0, p, p0)); + int p1 = part1.get().intValue(); + Future sort2 = sortingthreadexecutor.submit(new qsortthread(this, p, p1, p)); + Future sort3 = sortingthreadexecutor.submit(new qsortthread(this, p1, this.chunkcount, p1)); + sort0.get(); + sort1.get(); + sort2.get(); + sort3.get(); + } catch (InterruptedException e) { + e.printStackTrace(); + } catch (ExecutionException e) { + e.printStackTrace(); + } + } else { + qsort(0, p, 0, swapspace); + qsort(p + 1, this.chunkcount, 0, swapspace); + } + this.sortBound = this.chunkcount; + //assert this.isSorted(); + } + + public synchronized final void sort2() { + assert (this.rowdef.objectOrder != null); + if (this.sortBound == this.chunkcount) return; // this is already sorted + if (this.chunkcount < isortlimit) { + isort(0, this.chunkcount, new byte[this.rowdef.objectsize]); + this.sortBound = this.chunkcount; + assert this.isSorted(); + return; + } + byte[] swapspace = new byte[this.rowdef.objectsize]; + int p = partition(0, this.chunkcount, this.sortBound, swapspace); + if ((sortingthreadexecutor != null) && + (!sortingthreadexecutor.isShutdown()) && + (serverProcessor.useCPU > 1) && + (this.chunkcount > 4000)) { + // sort this using multi-threading Future part = sortingthreadexecutor.submit(new qsortthread(this, 0, p, 0)); //CompletionService sortingthreadcompletion = new ExecutorCompletionService(sortingthreadexecutor); //Future part = sortingthreadcompletion.submit(new qsortthread(this, 0, p, 0)); - qsort(p + 1, this.chunkcount, 0, swapspace); - try { + qsort(p + 1, this.chunkcount, 0, swapspace); + try { part.get(); } catch (InterruptedException e) { e.printStackTrace(); @@ -517,8 +562,8 @@ public class kelondroRowCollection { e.printStackTrace(); } } else { - qsort(0, p, 0, swapspace); - qsort(p + 1, this.chunkcount, 0, swapspace); + qsort(0, p, 0, swapspace); + qsort(p + 1, this.chunkcount, 0, swapspace); } this.sortBound = this.chunkcount; //assert this.isSorted(); @@ -554,6 +599,22 @@ public class kelondroRowCollection { qsort(p + 1, R, 0, swapspace); } + public static class partitionthread implements Callable { + kelondroRowCollection rc; + int L, R, S; + + public partitionthread(kelondroRowCollection rc, int L, int R, int S) { + this.rc = rc; + this.L = L; + this.R = R; + this.S = S; + } + + public Integer call() throws Exception { + return new Integer(rc.partition(L, R, S, new byte[rc.rowdef.objectsize])); + } + } + private final int partition(int L, int R, int S, byte[] swapspace) { // L is the first element in the sequence // R is the right bound of the sequence, and outside of the sequence @@ -584,7 +645,7 @@ public class kelondroRowCollection { if (p <= q) { oldpivot = pivot; pivot = swap(p, q, pivot, swapspace); - if (pivot != oldpivot) compiledPivot = null; // must be computed again + if (pivot != oldpivot && compiledPivot != null) compiledPivot = null; // must be computed again p++; q--; } @@ -891,7 +952,7 @@ public class kelondroRowCollection { System.out.println("kelondroRowCollection test with size = " + testsize); a = new kelondroRowCollection(r, testsize); - long t0 = System.currentTimeMillis(); + long t0 = System.nanoTime(); random = new Random(0); for (int i = 0; i < testsize; i++) a.add(randomHash().getBytes()); random = new Random(0);