boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

- better implementation of search query properties

Browse Source 
- basic protection against start-up problems when database files are corrupted
- auto-delete of not-critical databases during startup when load error occurs
- on-the-fly reset option for all database tables
- automatic on-the-fly reset for seed tables during enumeration exceptions

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@3547 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 18 years ago
parent 0c8ff6729f
commit 40c14a4f0e
51 changed files with 449 additions and 234 deletions
Show Stats Download Patch File Download Diff File

7
htroot/DetailedSearch.java
Unescape View File

@ -58,6 +58,7 @@ import java.util.Map;
import de.anomic.http.httpHeader;
import de.anomic.kelondro.kelondroMSetTools;
import de.anomic.net.URL;
import de.anomic.plasma.plasmaCondenser;
import de.anomic.plasma.plasmaSearchPreOrder;
import de.anomic.plasma.plasmaSearchQuery;
import de.anomic.plasma.plasmaSearchRankingProfile;
@ -239,7 +240,7 @@ public class DetailedSearch {
}
// do the search
plasmaSearchQuery thisSearch = new plasmaSearchQuery(query[0], query[1], wdist, "", plasmaSearchQuery.CONTENTDOM_TEXT, count, searchtime, urlmask,
plasmaSearchQuery thisSearch = new plasmaSearchQuery(querystring, plasmaCondenser.words2hashes(query[0]), plasmaCondenser.words2hashes(query[1]), wdist, "", plasmaSearchQuery.CONTENTDOM_TEXT, count, searchtime, urlmask,
((global) && (yacyonline) && (!(env.getConfig("last-search","").equals(querystring)))) ? plasmaSearchQuery.SEARCHDOM_GLOBALDHT : plasmaSearchQuery.SEARCHDOM_LOCAL,
"", 20, plasmaSearchQuery.catchall_constraint);
plasmaSearchRankingProfile localRanking = new plasmaSearchRankingProfile("local", post.toString());
@ -278,12 +279,12 @@ public class DetailedSearch {
prop.put("type_results_" + i + "_ybr", plasmaSearchPreOrder.ybr(result.getUrlentry().hash()));
prop.put("type_results_" + i + "_size", Long.toString(result.getUrlentry().size()));
try {
prop.put("type_results_" + i + "_words", URLEncoder.encode(results.getQuery().queryWords.toString(),"UTF-8"));
prop.put("type_results_" + i + "_words", URLEncoder.encode(query[0].toString(),"UTF-8"));
} catch (UnsupportedEncodingException e) {}
prop.put("type_results_" + i + "_former", results.getFormerSearch());
prop.put("type_results_" + i + "_rankingprops", result.getUrlentry().word().toPropertyForm() + ", domLengthEstimated=" + plasmaURL.domLengthEstimation(result.getUrlhash()) +
((plasmaURL.probablyRootURL(result.getUrlhash())) ? ", probablyRootURL" : "") +
(((wordURL = plasmaURL.probablyWordURL(result.getUrlhash(), results.getQuery().words(""))) != null) ? ", probablyWordURL=" + wordURL.toNormalform() : ""));
(((wordURL = plasmaURL.probablyWordURL(result.getUrlhash(), query[0])) != null) ? ", probablyWordURL=" + wordURL.toNormalform() : ""));
// adding snippet if available
if (result.hasSnippet()) {
prop.put("type_results_" + i + "_snippet", 1);

8
htroot/yacy/search.java
Unescape View File

@ -50,10 +50,10 @@
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
import de.anomic.http.httpHeader;
import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroBitfield;
import de.anomic.index.indexContainer;
import de.anomic.net.natLib;
@ -116,7 +116,7 @@ public final class search {
// tell all threads to do nothing for a specific time
sb.intermissionAllThreads(2 * duetime);
Set abstractSet = ((abstracts.length() == 0) || (abstracts.equals("auto"))) ? null : plasmaSearchQuery.hashes2Set(abstracts);
TreeSet abstractSet = ((abstracts.length() == 0) || (abstracts.equals("auto"))) ? null : plasmaSearchQuery.hashes2Set(abstracts);
// store accessing peer
if (yacyCore.seedDB == null) {
@ -126,7 +126,7 @@ public final class search {
}
// prepare search
final Set keyhashes = plasmaSearchQuery.hashes2Set(query);
final TreeSet keyhashes = plasmaSearchQuery.hashes2Set(query);
final long timestamp = System.currentTimeMillis();
serverObjects prop = new serverObjects();
@ -255,7 +255,7 @@ public final class search {
searchProfile.put("time", trackerHandle);
sb.remoteSearches.add(searchProfile);
TreeSet handles = (TreeSet) sb.remoteSearchTracker.get(client);
if (handles == null) handles = new TreeSet();
if (handles == null) handles = new TreeSet(kelondroBase64Order.enhancedCoder);
handles.add(trackerHandle);
sb.remoteSearchTracker.put(client, handles);

11
htroot/yacysearch.java
Unescape View File

@ -52,6 +52,7 @@ import java.net.MalformedURLException;
import java.net.URLEncoder;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Set;
import java.util.regex.PatternSyntaxException;
import java.util.TreeSet;
@ -254,9 +255,11 @@ public class yacysearch {
final boolean globalsearch = (global) && (yacyonline) && (!samesearch);
// do the search
Set querywords = query[0];
plasmaSearchQuery thisSearch = new plasmaSearchQuery(
query[0],
query[1],
querystring,
plasmaCondenser.words2hashes(query[0]),
plasmaCondenser.words2hashes(query[1]),
maxDistance,
prefermask,
contentdomCode,
@ -314,12 +317,12 @@ public class yacysearch {
prop.put("type_results_" + i + "_ybr", plasmaSearchPreOrder.ybr(result.getUrlentry().hash()));
prop.put("type_results_" + i + "_size", Long.toString(result.getUrlentry().size()));
try {
prop.put("type_results_" + i + "_words", URLEncoder.encode(results.getQuery().queryWords.toString(),"UTF-8"));
prop.put("type_results_" + i + "_words", URLEncoder.encode(querywords.toString(),"UTF-8"));
} catch (UnsupportedEncodingException e) {}
prop.put("type_results_" + i + "_former", results.getFormerSearch());
prop.put("type_results_" + i + "_rankingprops", result.getUrlentry().word().toPropertyForm() + ", domLengthEstimated=" + plasmaURL.domLengthEstimation(result.getUrlhash()) +
((plasmaURL.probablyRootURL(result.getUrlhash())) ? ", probablyRootURL" : "") +
(((wordURL = plasmaURL.probablyWordURL(result.getUrlhash(), results.getQuery().words(""))) != null) ? ", probablyWordURL=" + wordURL.toNormalform() : ""));
(((wordURL = plasmaURL.probablyWordURL(result.getUrlhash(), query[0])) != null) ? ", probablyWordURL=" + wordURL.toNormalform() : ""));
// adding snippet if available
if (result.hasSnippet()) {
prop.put("type_results_" + i + "_snippet", 1);

9
source/dbtest.java
Unescape View File

@ -188,11 +188,11 @@ public class dbtest {
}
if (dbe.equals("kelondroFlexTable")) {
File tablepath = new File(tablename).getParentFile();
table = new kelondroFlexTable(tablepath, new File(tablename).getName(), preload, testRow);
table = new kelondroFlexTable(tablepath, new File(tablename).getName(), preload, testRow, true);
}
if (dbe.equals("kelondroFlexSplitTable")) {
File tablepath = new File(tablename).getParentFile();
table = new kelondroFlexSplitTable(tablepath, new File(tablename).getName(), preload, testRow);
table = new kelondroFlexSplitTable(tablepath, new File(tablename).getName(), preload, testRow, true);
}
if (dbe.equals("mysql")) {
table = new dbTable("mysql", testRow);
@ -654,6 +654,11 @@ final class dbTable implements kelondroIndex {
public final int[] cacheNodeStatus() {
return new int[]{0,0,0,0,0,0,0,0,0,0};
}
public void reset() {
// TODO Auto-generated method stub
}
}

2
source/de/anomic/data/blogBoard.java
Unescape View File

@ -84,7 +84,7 @@ public class blogBoard {
public blogBoard(File actpath, long preloadTime) {
new File(actpath.getParent()).mkdir();
if (datbase == null) {
datbase = new kelondroMapObjects(kelondroDyn.open(actpath, true, true, preloadTime, keyLength, recordSize, '_', true, false), 500);
datbase = new kelondroMapObjects(new kelondroDyn(actpath, true, true, preloadTime, keyLength, recordSize, '_', true, false, false), 500);
}
}

2
source/de/anomic/data/blogBoardComments.java
Unescape View File

@ -84,7 +84,7 @@ public class blogBoardComments {
public blogBoardComments(File actpath, long preloadTime) {
new File(actpath.getParent()).mkdir();
if (datbase == null) {
datbase = new kelondroMapObjects(kelondroDyn.open(actpath, true, true, preloadTime, keyLength, recordSize, '_', false, false), 500);
datbase = new kelondroMapObjects(new kelondroDyn(actpath, true, true, preloadTime, keyLength, recordSize, '_', false, false, false), 500);
}
}

6
source/de/anomic/data/bookmarksDB.java
Unescape View File

@ -136,17 +136,17 @@ public class bookmarksDB {
bookmarkCache=new HashMap();
bookmarksFile.getParentFile().mkdirs();
//this.bookmarksTable = new kelondroMap(kelondroDyn.open(bookmarksFile, bufferkb * 1024, preloadTime, 12, 256, '_', true, false));
this.bookmarksTable = new kelondroObjects(kelondroDyn.open(bookmarksFile, true, true, preloadTime, 12, 256, '_', true, false), 1000);
this.bookmarksTable = new kelondroObjects(new kelondroDyn(bookmarksFile, true, true, preloadTime, 12, 256, '_', true, false, false), 1000);
// tags
tagsFile.getParentFile().mkdirs();
boolean tagsFileExisted = tagsFile.exists();
this.tagsTable = new kelondroMapObjects(kelondroDyn.open(tagsFile, true, true, preloadTime, 12, 256, '_', true, false), 500);
this.tagsTable = new kelondroMapObjects(new kelondroDyn(tagsFile, true, true, preloadTime, 12, 256, '_', true, false, false), 500);
if (!tagsFileExisted) rebuildTags();
// dates
boolean datesExisted = datesFile.exists();
this.datesTable = new kelondroMapObjects(kelondroDyn.open(datesFile, true, true, preloadTime, 20, 256, '_', true, false), 500);
this.datesTable = new kelondroMapObjects(new kelondroDyn(datesFile, true, true, preloadTime, 20, 256, '_', true, false, false), 500);
if (!datesExisted) rebuildDates();
}

2
source/de/anomic/data/messageBoard.java
Unescape View File

@ -70,7 +70,7 @@ public class messageBoard {
public messageBoard(File path, long preloadTime) {
new File(path.getParent()).mkdir();
if (database == null) {
database = new kelondroMapObjects(kelondroDyn.open(path, true, true, preloadTime, categoryLength + dateFormat.length() + 2, recordSize, '_', true, false), 500);
database = new kelondroMapObjects(new kelondroDyn(path, true, true, preloadTime, categoryLength + dateFormat.length() + 2, recordSize, '_', true, false, false), 500);
}
sn = 0;
}

4
source/de/anomic/data/userDB.java
Unescape View File

@ -77,7 +77,7 @@ public final class userDB {
this.userTableFile = userTableFile;
this.preloadTime = preloadTime;
userTableFile.getParentFile().mkdirs();
this.userTable = new kelondroMapObjects(kelondroDyn.open(userTableFile, true, true, preloadTime, 128, 256, '_', true, false), 10);
this.userTable = new kelondroMapObjects(new kelondroDyn(userTableFile, true, true, preloadTime, 128, 256, '_', true, false, false), 10);
}
void resetDatabase() {
@ -85,7 +85,7 @@ public final class userDB {
if (userTable != null) userTable.close();
if (!(userTableFile.delete())) throw new RuntimeException("cannot delete user database");
userTableFile.getParentFile().mkdirs();
userTable = new kelondroMapObjects(kelondroDyn.open(userTableFile, true, true, preloadTime, 256, 512, '_', true, false), 10);
userTable = new kelondroMapObjects(new kelondroDyn(userTableFile, true, true, preloadTime, 256, 512, '_', true, false, false), 10);
}
public void close() {

4
source/de/anomic/data/wikiBoard.java
Unescape View File

@ -71,11 +71,11 @@ public class wikiBoard {
public wikiBoard(File actpath, File bkppath, long preloadTime) {
new File(actpath.getParent()).mkdirs();
if (datbase == null) {
datbase = new kelondroMapObjects(kelondroDyn.open(actpath, true, true, preloadTime, keyLength, recordSize, '_', true, false), 500);
datbase = new kelondroMapObjects(new kelondroDyn(actpath, true, true, preloadTime, keyLength, recordSize, '_', true, false, false), 500);
}
new File(bkppath.getParent()).mkdirs();
if (bkpbase == null) {
bkpbase = new kelondroMapObjects(kelondroDyn.open(bkppath, true, true, preloadTime, keyLength + dateFormat.length(), recordSize, '_', true, false), 500);
bkpbase = new kelondroMapObjects(new kelondroDyn(bkppath, true, true, preloadTime, keyLength + dateFormat.length(), recordSize, '_', true, false, false), 500);
}
}

4
source/de/anomic/kelondro/kelondroBufferedIOChunks.java
Unescape View File

@ -70,6 +70,10 @@ public final class kelondroBufferedIOChunks extends kelondroAbstractIOChunks imp
this.lastCommit = System.currentTimeMillis();
}
public kelondroRA getRA() {
return this.ra;
}
public long length() throws IOException {
return ra.length();
}

24
source/de/anomic/kelondro/kelondroCache.java
Unescape View File

@ -62,10 +62,18 @@ public class kelondroCache implements kelondroIndex {
private kelondroRow keyrow;
private int readHit, readMiss, writeUnique, writeDouble, cacheDelete, cacheFlush;
private int hasnotHit, hasnotMiss, hasnotUnique, hasnotDouble, hasnotDelete, hasnotFlush;
private boolean read, write;
public kelondroCache(kelondroIndex backupIndex, boolean read, boolean write) throws IOException {
public kelondroCache(kelondroIndex backupIndex, boolean read, boolean write) {
assert write == false;
this.index = backupIndex;
this.read = read;
this.write = write;
init();
objectTracker.put(backupIndex.filename(), this);
}
private void init() {
this.keyrow = new kelondroRow(new kelondroColumn[]{index.row().column(index.row().primaryKey)}, index.row().objectOrder, index.row().primaryKey);
this.readHitCache = (read) ? new kelondroRowSet(index.row(), 0) : null;
this.readMissCache = (read) ? new kelondroRowSet(this.keyrow, 0) : null;
@ -83,15 +91,10 @@ public class kelondroCache implements kelondroIndex {
this.hasnotDouble = 0;
this.hasnotDelete = 0;
this.hasnotFlush = 0;
objectTracker.put(backupIndex.filename(), this);
}
public final int cacheObjectChunkSize() {
try {
return index.row().objectsize();
} catch (IOException e) {
return 0;
}
return index.row().objectsize();
}
public int writeBufferSize() {
@ -626,7 +629,7 @@ public class kelondroCache implements kelondroIndex {
return entry;
}
public synchronized kelondroRow row() throws IOException {
public synchronized kelondroRow row() {
return index.row();
}
@ -643,4 +646,9 @@ public class kelondroCache implements kelondroIndex {
return index.filename();
}
public void reset() throws IOException {
this.index.reset();
init();
}
}

8
source/de/anomic/kelondro/kelondroCollectionIndex.java
Unescape View File

@ -123,8 +123,8 @@ public class kelondroCollectionIndex {
boolean ramIndexGeneration = false;
boolean fileIndexGeneration = !(new File(path, filenameStub + ".index").exists());
if (ramIndexGeneration) index = new kelondroRowSet(indexRow(keyLength, indexOrder), 0);
if (fileIndexGeneration) index = new kelondroFlexTable(path, filenameStub + ".index", preloadTime, indexRow(keyLength, indexOrder));
if (fileIndexGeneration) index = new kelondroFlexTable(path, filenameStub + ".index", preloadTime, indexRow(keyLength, indexOrder), true);
// open array files
this.arrays = new HashMap(); // all entries will be dynamically created with getArray()
if (((fileIndexGeneration) || (ramIndexGeneration))) {
@ -199,7 +199,7 @@ public class kelondroCollectionIndex {
private kelondroIndex openIndexFile(File path, String filenameStub, kelondroOrder indexOrder,
long preloadTime, int loadfactor, kelondroRow rowdef) throws IOException {
// open/create index table
kelondroIndex theindex = new kelondroCache(new kelondroFlexTable(path, filenameStub + ".index", preloadTime, indexRow(keylength, indexOrder)), true, false);
kelondroIndex theindex = new kelondroCache(new kelondroFlexTable(path, filenameStub + ".index", preloadTime, indexRow(keylength, indexOrder), true), true, false);
// save/check property file for this array
File propfile = propertyFile(path, filenameStub, loadfactor, rowdef.objectsize());
@ -1063,7 +1063,7 @@ public class kelondroCollectionIndex {
// printout of index
collectionIndex.close();
kelondroFlexTable index = new kelondroFlexTable(path, filenameStub + ".index", preloadTime, kelondroCollectionIndex.indexRow(9, kelondroNaturalOrder.naturalOrder));
kelondroFlexTable index = new kelondroFlexTable(path, filenameStub + ".index", preloadTime, kelondroCollectionIndex.indexRow(9, kelondroNaturalOrder.naturalOrder), true);
index.print();
index.close();
} catch (IOException e) {

68
source/de/anomic/kelondro/kelondroDyn.java
Unescape View File

@ -58,8 +58,6 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.util.Iterator;
import de.anomic.server.logging.serverLog;
public class kelondroDyn {
private static final int counterlen = 8;
@ -72,50 +70,58 @@ public class kelondroDyn {
private kelondroObjectBuffer buffer;
private kelondroRow rowdef;
public kelondroDyn(File file, boolean useNodeCache, boolean useObjectCache, long preloadTime, int key, int nodesize, char fillChar, boolean usetree, boolean writebuffer) throws IOException {
this(file, useNodeCache, useObjectCache, preloadTime, key, nodesize, fillChar, new kelondroNaturalOrder(true), usetree, writebuffer);
public kelondroDyn(File file, boolean useNodeCache, boolean useObjectCache, long preloadTime, int key, int nodesize, char fillChar, boolean usetree, boolean writebuffer, boolean resetOnFail) {
this(file, useNodeCache, useObjectCache, preloadTime, key, nodesize, fillChar, new kelondroNaturalOrder(true), usetree, writebuffer, resetOnFail);
}
public kelondroDyn(File file, boolean useNodeCache, boolean useObjectCache, long preloadTime, int key,
int nodesize, char fillChar, kelondroOrder objectOrder, boolean usetree, boolean writebuffer) throws IOException {
int nodesize, char fillChar, kelondroOrder objectOrder, boolean usetree, boolean writebuffer, boolean resetOnFail) {
// creates or opens a dynamic tree
rowdef = new kelondroRow("byte[] key-" + (key + counterlen) + ", byte[] node-" + nodesize, objectOrder, 0);
kelondroIndex fbi;
if (usetree) {
kelondroTree tree = new kelondroTree(file, useNodeCache, preloadTime, rowdef, 1, 8);
this.index = (useObjectCache) ? (kelondroIndex) new kelondroCache(tree, true, writebuffer) : (kelondroIndex) tree;
try {
fbi = new kelondroTree(file, useNodeCache, preloadTime, rowdef, 1, 8);
} catch (IOException e) {
e.printStackTrace();
if (resetOnFail) {
file.delete();
try {
fbi = new kelondroTree(file, useNodeCache, -1, rowdef, 1, 8);
} catch (IOException e1) {
e1.printStackTrace();
throw new kelondroException(e.getMessage());
}
} else {
throw new kelondroException(e.getMessage());
}
}
} else {
kelondroFlexTable table = new kelondroFlexTable(file.getParentFile(), file.getName(), 10000, rowdef);
this.index = (useObjectCache) ? (kelondroIndex) new kelondroCache(table, true, writebuffer) : (kelondroIndex) table;
fbi = new kelondroFlexTable(file.getParentFile(), file.getName(), 10000, rowdef, resetOnFail);
}
this.keylen = index.row().width(0) - counterlen;
this.reclen = index.row().width(1);
this.index = (useObjectCache) ? (kelondroIndex) new kelondroCache(fbi, true, writebuffer) : fbi;
this.keylen = key;
this.reclen = nodesize;
this.fillChar = fillChar;
//this.segmentCount = 0;
//if (!(tree.fileExisted)) writeSegmentCount();
buffer = new kelondroObjectBuffer(file.toString());
}
public static final kelondroDyn open(File file, boolean useNodeCache, boolean useObjectCache, long preloadTime, int key, int nodesize, char fillChar, boolean usetree, boolean writebuffer) {
return open(file, useNodeCache, useObjectCache, preloadTime, key, nodesize, fillChar, new kelondroNaturalOrder(true), usetree, writebuffer);
public static final kelondroDyn open(File file, boolean useNodeCache, boolean useObjectCache, long preloadTime, int key, int nodesize, char fillChar, boolean usetree, boolean writebuffer, boolean resetOnFail) {
return open(file, useNodeCache, useObjectCache, preloadTime, key, nodesize, fillChar, new kelondroNaturalOrder(true), usetree, writebuffer, resetOnFail);
}
public static final kelondroDyn open(File file, boolean useNodeCache, boolean useObjectCache, long preloadTime, int key,
int nodesize, char fillChar, kelondroOrder objectOrder, boolean usetree, boolean writebuffer) {
// opens new or existing file; in case that any error occur the file is deleted again and it is tried to create the file again
// if that fails, the method returns null
try {
return new kelondroDyn(file, useNodeCache, useObjectCache, preloadTime, key, nodesize, fillChar, objectOrder, usetree, writebuffer);
} catch (IOException e) {
file.delete();
try {
return new kelondroDyn(file, useNodeCache, useObjectCache, preloadTime, key, nodesize, fillChar, objectOrder, usetree, writebuffer);
} catch (IOException ee) {
serverLog.logSevere("kelondroDyn", "cannot open or create file " + file.toString());
e.printStackTrace();
ee.printStackTrace();
return null;
}
}
int nodesize, char fillChar, kelondroOrder objectOrder, boolean usetree, boolean writebuffer, boolean resetOnFail) {
return new kelondroDyn(file, useNodeCache, useObjectCache, preloadTime, key, nodesize, fillChar, objectOrder, usetree, writebuffer, resetOnFail);
}
public void reset() throws IOException {
String name = this.index.filename();
this.index.reset();
this.buffer = new kelondroObjectBuffer(name);
}
/*
@ -478,7 +484,7 @@ public class kelondroDyn {
if (args.length == 1) {
// open a db and list keys
try {
kelondroDyn kd = new kelondroDyn(new File(args[0]), true, true, 0, 4 ,100, '_', false, false);
kelondroDyn kd = new kelondroDyn(new File(args[0]), true, true, 0, 4 ,100, '_', false, false, true);
System.out.println(kd.sizeDyn() + " elements in DB");
Iterator i = kd.dynKeys(true, false);
while (i.hasNext())
@ -495,7 +501,7 @@ public class kelondroDyn {
File f = new File(args[3]);
kelondroDyn kd;
try {
kd = new kelondroDyn(db, true, true, 0, 80, 200, '_', false, false);
kd = new kelondroDyn(db, true, true, 0, 80, 200, '_', false, false, true);
if (writeFile)
kd.readFile(key, f);
else

8
source/de/anomic/kelondro/kelondroDynTree.java
Unescape View File

@ -68,7 +68,7 @@ public class kelondroDynTree {
private Hashtable buffer, cache;
private long cycleBuffer;
public kelondroDynTree(File file, long preloadTime, int keylength, int nodesize, kelondroRow rowdef, char fillChar) throws IOException {
public kelondroDynTree(File file, long preloadTime, int keylength, int nodesize, kelondroRow rowdef, char fillChar, boolean resetOnFail) throws IOException {
// creates or opens a DynTree
this.file = file;
this.preloadTime = preloadTime;
@ -77,7 +77,7 @@ public class kelondroDynTree {
this.cache = new Hashtable();
//this.cycleCache = Long.MIN_VALUE;
this.cycleBuffer = Long.MIN_VALUE;
this.table = new kelondroDyn(file, true, true, preloadTime, keylength, nodesize, fillChar, true, false);
this.table = new kelondroDyn(file, true, true, preloadTime, keylength, nodesize, fillChar, true, false, resetOnFail);
this.treeRAHandles = new Hashtable();
}
@ -296,10 +296,10 @@ public class kelondroDynTree {
System.out.println("start");
File file = new File("D:\\bin\\testDyn.db");
if (file.exists()) {
kelondroDynTree dt = new kelondroDynTree(file, 0, 16, 512, new kelondroRow("byte[] a-10, byte[] b-20, byte[] c-30", kelondroNaturalOrder.naturalOrder, 0), '_');
kelondroDynTree dt = new kelondroDynTree(file, 0, 16, 512, new kelondroRow("byte[] a-10, byte[] b-20, byte[] c-30", kelondroNaturalOrder.naturalOrder, 0), '_', true);
System.out.println("opened: table keylength=" + dt.table.row().width(0) + ", sectorsize=" + dt.table.row().width(1) + ", " + dt.table.sizeDyn() + " entries.");
} else {
kelondroDynTree dt = new kelondroDynTree(file, 0, 16, 512, new kelondroRow("byte[] a-10, byte[] b-20, byte[] c-30", kelondroNaturalOrder.naturalOrder, 0), '_');
kelondroDynTree dt = new kelondroDynTree(file, 0, 16, 512, new kelondroRow("byte[] a-10, byte[] b-20, byte[] c-30", kelondroNaturalOrder.naturalOrder, 0), '_', true);
String name;
kelondroTree t;
kelondroRow.Entry line;

25
source/de/anomic/kelondro/kelondroFlexSplitTable.java
Unescape View File

@ -46,10 +46,14 @@ public class kelondroFlexSplitTable implements kelondroIndex {
private File path;
private String tablename;
public kelondroFlexSplitTable(File path, String tablename, long preloadTime, kelondroRow rowdef) throws IOException {
public kelondroFlexSplitTable(File path, String tablename, long preloadTime, kelondroRow rowdef, boolean resetOnFail) throws IOException {
this.path = path;
this.tablename = tablename;
this.rowdef = rowdef;
init(preloadTime, resetOnFail);
}
public void init(long preloadTime, boolean resetOnFail) throws IOException {
// initialized tables map
this.tables = new HashMap();
@ -96,11 +100,22 @@ public class kelondroFlexSplitTable implements kelondroIndex {
// open next biggest table
t.remove(maxf);
date = maxf.substring(tablename.length() + 1);
table = new kelondroCache(new kelondroFlexTable(path, maxf, preloadTime, rowdef), true, false);
table = new kelondroCache(new kelondroFlexTable(path, maxf, preloadTime, rowdef, resetOnFail), true, false);
tables.put(date, table);
}
}
public void reset() throws IOException {
this.close();
String[] l = path.list();
for (int i = 0; i < l.length; i++) {
if (l[i].startsWith(tablename)) {
kelondroFlexTable.delete(path, l[i]);
}
}
init(-1, true);
}
public String filename() {
return new File(path, tablename).toString();
}
@ -160,7 +175,7 @@ public class kelondroFlexSplitTable implements kelondroIndex {
}
}
public kelondroRow row() throws IOException {
public kelondroRow row() {
return this.rowdef;
}
@ -198,7 +213,7 @@ public class kelondroFlexSplitTable implements kelondroIndex {
kelondroIndex table = (kelondroIndex) tables.get(suffix);
if (table == null) {
// make new table
table = new kelondroFlexTable(path, tablename + "." + suffix, -1, rowdef);
table = new kelondroFlexTable(path, tablename + "." + suffix, -1, rowdef, true);
tables.put(suffix, table);
}
table.put(row);
@ -229,7 +244,7 @@ public class kelondroFlexSplitTable implements kelondroIndex {
kelondroIndex table = (kelondroIndex) tables.get(suffix);
if (table == null) {
// make new table
table = new kelondroFlexTable(path, tablename + "." + suffix, -1, rowdef);
table = new kelondroFlexTable(path, tablename + "." + suffix, -1, rowdef, true);
tables.put(suffix, table);
}
table.addUnique(row, entryDate);

30
source/de/anomic/kelondro/kelondroFlexTable.java
Unescape View File

@ -47,17 +47,19 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr
protected kelondroBytesIntMap ROindex, RWindex;
private boolean RAMIndex;
public kelondroFlexTable(File path, String tablename, long preloadTime, kelondroRow rowdef) throws IOException {
public kelondroFlexTable(File path, String tablename, long preloadTime, kelondroRow rowdef, boolean resetOnFail) {
// the buffersize applies to a possible load of the ram-index
// if the ram is not sufficient, a tree file is generated
// if, and only if a tree file exists, the preload time is applied
super(path, tablename, rowdef);
super(path, tablename, rowdef, resetOnFail);
try {
long neededRAM = (long) ((super.row().column(0).cellwidth() + 4) * super.size() * kelondroRowCollection.growfactor);
File newpath = new File(path, tablename);
File indexfile = new File(newpath, "col.000.index");
kelondroIndex ki = null;
String description = new String(this.col[0].getDescription());
String description = "";
description = new String(this.col[0].getDescription());
int p = description.indexOf(';', 4);
long stt = (p > 0) ? Long.parseLong(description.substring(4, p)) : 0;
System.out.println("*** Last Startup time: " + stt + " milliseconds");
@ -107,6 +109,26 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr
// assign index to wrapper
description = "stt=" + Long.toString(System.currentTimeMillis() - start) + ";";
super.col[0].setDescription(description.getBytes());
} catch (IOException e) {
if (resetOnFail) {
RAMIndex = true;
ROindex = null;
try {
RWindex = new kelondroBytesIntMap(new kelondroRowSet(new kelondroRow(new kelondroColumn[]{super.row().column(0), new kelondroColumn("int c-4 {b256}")}, super.rowdef.objectOrder, super.rowdef.primaryKey), 100));
} catch (IOException e1) {
throw new kelondroException(e1.getMessage());
}
} else {
throw new kelondroException(e.getMessage());
}
}
}
public void reset() throws IOException {
super.reset();
RAMIndex = true;
ROindex = null;
RWindex = new kelondroBytesIntMap(new kelondroRowSet(new kelondroRow(new kelondroColumn[]{super.row().column(0), new kelondroColumn("int c-4 {b256}")}, super.rowdef.objectOrder, super.rowdef.primaryKey), 100));
}
public static int staticSize(File path, String tablename) {
@ -402,7 +424,7 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr
String name = args[1];
kelondroRow row = new kelondroRow("Cardinal key-4 {b256}, byte[] x-64", kelondroNaturalOrder.naturalOrder, 0);
try {
kelondroFlexTable t = new kelondroFlexTable(f, name, 0, row);
kelondroFlexTable t = new kelondroFlexTable(f, name, 0, row, true);
kelondroRow.Entry entry = row.newEntry();
entry.setCol(0, System.currentTimeMillis());
entry.setCol(1, "dummy".getBytes());

65
source/de/anomic/kelondro/kelondroFlexWidthArray.java
Unescape View File

@ -34,17 +34,52 @@ import java.util.Map;
import java.util.TreeMap;
import de.anomic.server.serverFileUtils;
import de.anomic.server.logging.serverLog;
public class kelondroFlexWidthArray implements kelondroArray {
protected kelondroFixedWidthArray[] col;
protected kelondroRow rowdef;
protected File path;
protected String tablename;
protected String filename;
public kelondroFlexWidthArray(File path, String tablename, kelondroRow rowdef) throws IOException {
this.rowdef = rowdef;
public kelondroFlexWidthArray(File path, String tablename, kelondroRow rowdef, boolean resetOnFail) {
this.path = path;
this.rowdef = rowdef;
this.tablename = tablename;
try {
init();
} catch (IOException e) {
if (resetOnFail) {
serverLog.logSevere("kelondroFlexWidthArray", "IOException during initialization of " + new File(path, tablename).toString() + ": reset");
delete(path, tablename);
try {
init();
} catch (IOException e1) {
e1.printStackTrace();
throw new kelondroException("IOException during initialization of " + new File(path, tablename).toString() + ": cannot reset: " + e1.getMessage());
}
} else {
throw new kelondroException("IOException during initialization of " + new File(path, tablename).toString() + ": not allowed to reset: " + e.getMessage());
}
} catch (kelondroException e) {
if (resetOnFail) {
serverLog.logSevere("kelondroFlexWidthArray", "kelondroException during initialization of " + new File(path, tablename).toString() + ": reset");
delete(path, tablename);
try {
init();
} catch (IOException e1) {
e1.printStackTrace();
throw new kelondroException("kelondroException during initialization of " + new File(path, tablename).toString() + ": cannot reset: " + e1.getMessage());
}
} else {
throw new kelondroException("kelondroException during initialization of " + new File(path, tablename).toString() + ": not allowed to reset: " + e.getMessage());
}
}
}
public void init() throws IOException {
// initialize columns
col = new kelondroFixedWidthArray[rowdef.columns()];
@ -145,20 +180,10 @@ public class kelondroFlexWidthArray implements kelondroArray {
tabledir.delete();
}
public static kelondroFlexWidthArray open(File path, String tablename, kelondroRow rowdef) {
try {
return new kelondroFlexWidthArray(path, tablename, rowdef);
} catch (IOException e) {
kelondroFlexWidthArray.delete(path, tablename);
try {
return new kelondroFlexWidthArray(path, tablename, rowdef);
} catch (IOException ee) {
e.printStackTrace();
ee.printStackTrace();
System.exit(-1);
return null;
}
}
public void reset() throws IOException {
this.close();
delete(path, tablename);
this.init();
}
public void close() {
@ -352,7 +377,7 @@ public class kelondroFlexWidthArray implements kelondroArray {
try {
System.out.println("erster Test");
kelondroFlexWidthArray.delete(f, testname);
kelondroFlexWidthArray k = kelondroFlexWidthArray.open(f, "flextest", rowdef);
kelondroFlexWidthArray k = new kelondroFlexWidthArray(f, "flextest", rowdef, true);
k.add(k.row().newEntry(new byte[][]{"a".getBytes(), "xxxx".getBytes()}));
k.add(k.row().newEntry(new byte[][]{"b".getBytes(), "xxxx".getBytes()}));
k.remove(0, false);
@ -375,19 +400,19 @@ public class kelondroFlexWidthArray implements kelondroArray {
//k = kelondroFlexWidthArray.open(f, "flextest", rowdef);
for (int i = 1; i <= 20; i = i * 2) {
System.out.println("LOOP: " + i);
k = kelondroFlexWidthArray.open(f, "flextest", rowdef);
k = new kelondroFlexWidthArray(f, "flextest", rowdef, true);
for (int j = 0; j < i*2; j++) {
k.add(k.row().newEntry(new byte[][]{(Integer.toString(i) + "-" + Integer.toString(j)).getBytes(), "xxxx".getBytes()}));
}
k.close();
k = kelondroFlexWidthArray.open(f, "flextest", rowdef);
k = new kelondroFlexWidthArray(f, "flextest", rowdef, true);
for (int j = 0; j < i; j++) {
k.remove(i*2 - j - 1, true);
}
k.close();
}
k.resolveMarkedRemoved();
k = kelondroFlexWidthArray.open(f, "flextest", rowdef);
k = new kelondroFlexWidthArray(f, "flextest", rowdef, true);
k.print();
k.col[0].print(true);
k.close();

3
source/de/anomic/kelondro/kelondroIOChunks.java
Unescape View File

@ -48,6 +48,9 @@ public interface kelondroIOChunks {
// logging support
public String name();
// reference handling
public kelondroRA getRA();
// pseudo-native methods:
public long length() throws IOException;
public int read(long pos, byte[] b, int off, int len) throws IOException;

3
source/de/anomic/kelondro/kelondroIndex.java
Unescape View File

@ -59,7 +59,7 @@ public interface kelondroIndex {
public String filename(); // returns a unique identified for this index; can be a real or artificial file name
public int size() throws IOException;
public kelondroProfile profile();
public kelondroRow row() throws IOException;
public kelondroRow row();
public boolean has(byte[] key) throws IOException; // use this only if there is no get in case that has returns true
public kelondroRow.Entry get(byte[] key) throws IOException;
public kelondroRow.Entry put(kelondroRow.Entry row) throws IOException;
@ -72,4 +72,5 @@ public interface kelondroIndex {
public kelondroRow.Entry removeOne() throws IOException;
public kelondroCloneableIterator rows(boolean up, byte[] firstKey) throws IOException;
public void close();
public void reset() throws IOException;
}

54
source/de/anomic/kelondro/kelondroMSetTools.java
Unescape View File

@ -245,7 +245,59 @@ public class kelondroMSetTools {
return result;
}
// now the same for set-set
public static boolean anymatch(TreeSet set1, TreeSet set2) {
// comparators must be equal
if ((set1 == null) || (set2 == null)) return false;
if (set1.comparator() != set2.comparator()) return false;
if ((set1.size() == 0) || (set2.size() == 0)) return false;
// decide which method to use
int high = ((set1.size() > set2.size()) ? set1.size() : set2.size());
int low = ((set1.size() > set2.size()) ? set2.size() : set1.size());
int stepsEnum = 10 * (high + low - 1);
int stepsTest = 12 * log2a(high) * low;
// start most efficient method
if (stepsEnum > stepsTest) {
if (set1.size() < set2.size()) return anymatchByTest(set1, set2);
return anymatchByTest(set2, set1);
}
return anymatchByEnumeration(set1, set2);
}
private static boolean anymatchByTest(TreeSet small, TreeSet large) {
Iterator mi = small.iterator();
Object o;
while (mi.hasNext()) {
o = mi.next();
if (large.contains(o)) return true;
}
return false;
}
private static boolean anymatchByEnumeration(TreeSet set1, TreeSet set2) {
// implement pairvise enumeration
Comparator comp = set1.comparator();
Iterator mi = set1.iterator();
Iterator si = set2.iterator();
int c;
if ((mi.hasNext()) && (si.hasNext())) {
Object mobj = mi.next();
Object sobj = si.next();
while (true) {
c = compare(mobj, sobj, comp);
if (c < 0) {
if (mi.hasNext()) mobj = mi.next(); else break;
} else if (c > 0) {
if (si.hasNext()) sobj = si.next(); else break;
} else {
return true;
}
}
}
return false;
}
// ------------------------------------------------------------------------------------------------
// exclude

24
source/de/anomic/kelondro/kelondroMapObjects.java
Unescape View File

@ -140,6 +140,30 @@ public class kelondroMapObjects extends kelondroObjects {
if (longaccfields != null) for (int i = 0; i < longaccfields.length; i++) accMap.put(longaccfields[i], longaccumulator[i]);
if (doubleaccfields != null) for (int i = 0; i < doubleaccfields.length; i++) accMap.put(doubleaccfields[i], doubleaccumulator[i]);
}
public void reset() throws IOException {
super.reset();
if (sortfields == null) sortClusterMap = null; else {
sortClusterMap = new HashMap();
for (int i = 0; i < sortfields.length; i++) {
sortClusterMap.put(sortfields[i], new kelondroMScoreCluster());
}
}
if (longaccfields == null) accMap = null; else {
accMap = new HashMap();
for (int i = 0; i < longaccfields.length; i++) {
accMap.put(longaccfields[i], new Long(0));
}
}
if (doubleaccfields == null) accMap = null; else {
accMap = new HashMap();
for (int i = 0; i < doubleaccfields.length; i++) {
accMap.put(doubleaccfields[i], new Double(0));
}
}
}
public synchronized void set(String key, Map newMap) throws IOException {
assert (key != null);

12
source/de/anomic/kelondro/kelondroMapTable.java
Unescape View File

@ -63,26 +63,26 @@ public class kelondroMapTable {
public void declareMaps(
String tablename, int keysize, int nodesize, int cacheslots,
char fillChar) throws IOException {
declareMaps(tablename, keysize, nodesize, cacheslots, null, null, null, fillChar);
char fillChar, boolean resetOnFail) throws IOException {
declareMaps(tablename, keysize, nodesize, cacheslots, null, null, null, fillChar, resetOnFail);
}
public void declareMaps(
String tablename, int keysize, int nodesize, int cacheslots,
String[] sortfields, String[] longaccfields, String[] doubleaccfields, char fillChar) throws IOException {
declareMaps(tablename, keysize, nodesize, cacheslots, sortfields, longaccfields, doubleaccfields, fillChar, 0);
String[] sortfields, String[] longaccfields, String[] doubleaccfields, char fillChar, boolean resetOnFail) throws IOException {
declareMaps(tablename, keysize, nodesize, cacheslots, sortfields, longaccfields, doubleaccfields, fillChar, 0, resetOnFail);
}
public void declareMaps(
String tablename, int keysize, int nodesize, int cacheslots,
String[] sortfields, String[] longaccfields, String[] doubleaccfields, char fillChar,
long preloadTime) throws IOException {
long preloadTime, boolean resetOnFail) throws IOException {
if (mTables.containsKey(tablename)) throw new RuntimeException("kelondroTables.declareMap: table '" + tablename + "' declared twice.");
if (tTables.containsKey(tablename)) throw new RuntimeException("kelondroTables.declareMap: table '" + tablename + "' declared already in other context.");
File tablefile = new File(tablesPath, "table." + tablename + ".mdb");
kelondroDyn dyn;
if (!(tablefile.exists())) tablefile.getParentFile().mkdirs();
dyn = new kelondroDyn(tablefile, true, true, preloadTime, keysize, nodesize, fillChar, true, false);
dyn = new kelondroDyn(tablefile, true, true, preloadTime, keysize, nodesize, fillChar, true, false, resetOnFail);
kelondroMapObjects map = new kelondroMapObjects(dyn, cacheslots, sortfields, longaccfields, doubleaccfields, null, null);
mTables.put(tablename, map);
}

6
source/de/anomic/kelondro/kelondroObjects.java
Unescape View File

@ -47,6 +47,12 @@ public class kelondroObjects {
this.startup = System.currentTimeMillis();
this.cachesize = cachesize;
}
public void reset() throws IOException {
this.dyn.reset();
this.cache = new HashMap();
this.cacheScore = new kelondroMScoreCluster();
}
public int keySize() {
return dyn.row().width(0);

1
source/de/anomic/kelondro/kelondroRA.java
Unescape View File

@ -96,5 +96,4 @@ public interface kelondroRA {
public void writeArray(byte[] b) throws IOException;
public byte[] readArray() throws IOException;
}

4
source/de/anomic/kelondro/kelondroRAIOChunks.java
Unescape View File

@ -51,6 +51,10 @@ public final class kelondroRAIOChunks extends kelondroAbstractIOChunks implement
this.name = name;
this.ra = ra;
}
public kelondroRA getRA() {
return this.ra;
}
public synchronized long length() throws IOException {
return ra.length();

41
source/de/anomic/kelondro/kelondroRecords.java
Unescape View File

@ -191,6 +191,10 @@ public class kelondroRecords {
if ((rest != 0) || (calculated_used != this.USEDC + this.FREEC)) {
theLogger.log(Level.WARNING, "USEDC inconsistency at startup: calculated_used = " + calculated_used + ", USEDC = " + this.USEDC + ", FREEC = " + this.FREEC + ", recordsize = " + recordsize + ", file = " + filename);
this.USEDC = calculated_used - this.FREEC;
if (this.USEDC < 0) {
theLogger.log(Level.WARNING, "USEDC inconsistency at startup: cannot recover " + filename);
throw new kelondroException("cannot recover inconsistency in " + filename);
}
writeused(true);
}
} catch (IOException e) {
@ -409,6 +413,11 @@ public class kelondroRecords {
// txtProps: number of text properties
this.fileExisted = file.exists(); // can be used by extending class to track if this class created the file
this.OHBYTEC = ohbytec;
this.OHHANDLEC = ohhandlec;
this.ROW = rowdef; // create row
this.TXTPROPW = txtPropWidth;
if (file.exists()) {
// opens an existing tree
this.filename = file.getCanonicalPath();
@ -422,7 +431,7 @@ public class kelondroRecords {
kelondroRA raf = new kelondroFileRA(this.filename);
// kelondroRA raf = new kelondroBufferedRA(new kelondroFileRA(this.filename), 1024, 100);
// kelondroRA raf = new kelondroNIOFileRA(this.filename, false, 10000);
initNewFile(raf, ohbytec, ohhandlec, rowdef, FHandles, txtProps, txtPropWidth, useNodeCache);
initNewFile(raf, FHandles, txtProps);
}
assignRowdef(rowdef);
if (fileExisted) {
@ -446,8 +455,13 @@ public class kelondroRecords {
// this always creates a new file
this.fileExisted = false;
this.filename = filename;
this.OHBYTEC = ohbytec;
this.OHHANDLEC = ohhandlec;
this.ROW = rowdef; // create row
this.TXTPROPW = txtPropWidth;
try {
initNewFile(ra, ohbytec, ohhandlec, rowdef, FHandles, txtProps, txtPropWidth, useCache);
initNewFile(ra, FHandles, txtProps);
} catch (IOException e) {
logFailure("cannot create / " + e.getMessage());
if (exitOnFail) System.exit(-1);
@ -458,8 +472,16 @@ public class kelondroRecords {
if (useCache) recordTracker.put(this.filename, this);
}
private void initNewFile(kelondroRA ra, short ohbytec, short ohhandlec,
kelondroRow rowdef, int FHandles, int txtProps, int txtPropWidth, boolean useCache) throws IOException {
public void reset() throws IOException {
kelondroRA ra = this.entryFile.getRA();
File f = new File(ra.name());
this.entryFile.close();
f.delete();
ra = new kelondroFileRA(f);
initNewFile(ra, this.HANDLES.length, this.TXTPROPS.length);
}
private void initNewFile(kelondroRA ra, int FHandles, int txtProps) throws IOException {
// create new Chunked IO
if (useWriteBuffer) {
@ -468,11 +490,9 @@ public class kelondroRecords {
this.entryFile = new kelondroRAIOChunks(ra, ra.name());
}
// create row
ROW = rowdef;
// store dynamic run-time data
this.overhead = ohbytec + 4 * ohhandlec;
this.overhead = this.OHBYTEC + 4 * this.OHHANDLEC;
this.recordsize = this.overhead + ROW.objectsize();
this.headchunksize = overhead + ROW.width(0);
this.tailchunksize = this.recordsize - this.headchunksize;
@ -481,18 +501,15 @@ public class kelondroRecords {
// store dynamic run-time seek pointers
POS_HANDLES = POS_COLWIDTHS + ROW.columns() * 4;
POS_TXTPROPS = POS_HANDLES + FHandles * 4;
POS_NODES = POS_TXTPROPS + txtProps * txtPropWidth;
POS_NODES = POS_TXTPROPS + txtProps * this.TXTPROPW;
//System.out.println("*** DEBUG: POS_NODES = " + POS_NODES + " for " + filename);
// store dynamic back-up variables
USAGE = new usageControl(true);
OHBYTEC = ohbytec;
OHHANDLEC = ohhandlec;
HANDLES = new Handle[FHandles];
for (int i = 0; i < FHandles; i++) HANDLES[i] = new Handle(NUL);
TXTPROPS = new byte[txtProps][];
for (int i = 0; i < txtProps; i++) TXTPROPS[i] = new byte[0];
TXTPROPW = txtPropWidth;
// write data to file
entryFile.writeByte(POS_MAGIC, 4); // magic marker for this file type
@ -510,7 +527,7 @@ public class kelondroRecords {
entryFile.writeLong(POS_OFFSET, POS_NODES);
entryFile.writeInt(POS_INTPROPC, FHandles);
entryFile.writeInt(POS_TXTPROPC, txtProps);
entryFile.writeInt(POS_TXTPROPW, txtPropWidth);
entryFile.writeInt(POS_TXTPROPW, this.TXTPROPW);
// write configuration arrays
for (int i = 0; i < this.ROW.columns(); i++) {

3
source/de/anomic/kelondro/kelondroRow.java
Unescape View File

@ -34,6 +34,7 @@ import java.util.Map;
import java.util.StringTokenizer;
import de.anomic.server.serverByteBuffer;
import de.anomic.server.logging.serverLog;
public class kelondroRow {
@ -148,7 +149,7 @@ public class kelondroRow {
public Entry newEntry(byte[] rowinstance) {
if (rowinstance == null) return null;
//assert (rowinstance[0] != 0);
assert (this.objectOrder.wellformed(rowinstance, 0, row[0].cellwidth()));
assert (this.objectOrder.wellformed(rowinstance, 0, row[0].cellwidth())) : "rowinstance[0] = " + serverLog.arrayList(rowinstance, 0, row[0].cellwidth());
if (!(this.objectOrder.wellformed(rowinstance, 0, row[0].cellwidth()))) return null;
return new Entry(rowinstance);
}

10
source/de/anomic/kelondro/kelondroRowCollection.java
Unescape View File

@ -70,7 +70,7 @@ public class kelondroRowCollection {
this.lastTimeRead = System.currentTimeMillis();
this.lastTimeWrote = System.currentTimeMillis();
}
public kelondroRowCollection(kelondroRow rowdef, int objectCount, byte[] cache, int sortBound) {
this.rowdef = rowdef;
this.chunkcache = cache;
@ -112,7 +112,13 @@ public class kelondroRowCollection {
}
this.chunkcache = exportedCollection.getColBytes(exp_collection);
}
public void reset() {
this.chunkcache = new byte[0];
this.chunkcount = 0;
this.sortBound = 0;
}
private static final kelondroRow exportMeasureRow = exportRow(0 /* no relevance */);
protected static final int sizeOfExportedCollectionRows(kelondroRow.Entry exportedCollectionRowEnvironment, int columnInEnvironment) {

8
source/de/anomic/kelondro/kelondroRowSet.java
Unescape View File

@ -55,7 +55,12 @@ public class kelondroRowSet extends kelondroRowCollection implements kelondroInd
super(rowdef, exportedCollectionRowEnvironment, columnInEnvironment);
this.profile = new kelondroProfile();
}
public void reset() {
super.reset();
this.profile = new kelondroProfile();
}
public synchronized boolean has(byte[] key) throws IOException {
return (get(key) != null);
}
@ -379,4 +384,5 @@ public class kelondroRowSet extends kelondroRowCollection implements kelondroInd
public String filename() {
return null;
}
}

6
source/de/anomic/kelondro/kelondroSplittedTree.java
Unescape View File

@ -87,6 +87,12 @@ public class kelondroSplittedTree implements kelondroIndex {
ff = forkfactor;
}
public void reset() throws IOException {
for (int i = 0; i < ktfs.length; i++) {
ktfs[i].reset();
}
}
public void close() {
for (int i = 0; i < ktfs.length; i++) ktfs[i].close();
}

5
source/de/anomic/kelondro/kelondroTree.java
Unescape View File

@ -156,6 +156,11 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
super.setLogger(log);
}
public void reset() throws IOException {
super.reset();
setHandle(root, null);
}
private void commitNode(Node n) throws IOException {
Handle left = n.getOHHandle(leftchild);
Handle right = n.getOHHandle(rightchild);

6
source/de/anomic/plasma/plasmaCondenser.java
Unescape View File

@ -269,7 +269,7 @@ public final class plasmaCondenser {
}
public static final Set words2hashSet(String[] words) {
TreeSet hashes = new TreeSet();
TreeSet hashes = new TreeSet(kelondroBase64Order.enhancedCoder);
for (int i = 0; i < words.length; i++) hashes.add(word2hash(words[i]));
return hashes;
}
@ -280,9 +280,9 @@ public final class plasmaCondenser {
return new String(sb);
}
public static final Set words2hashes(Set words) {
public static final TreeSet words2hashes(Set words) {
Iterator i = words.iterator();
TreeSet hashes = new TreeSet();
TreeSet hashes = new TreeSet(kelondroBase64Order.enhancedCoder);
while (i.hasNext()) hashes.add(word2hash((String) i.next()));
return hashes;
}

7
source/de/anomic/plasma/plasmaCrawlBalancer.java
Unescape View File

@ -122,12 +122,7 @@ public class plasmaCrawlBalancer {
private void openFileIndex() {
cacheStacksPath.mkdirs();
try {
urlFileIndex = new kelondroCache(new kelondroFlexTable(cacheStacksPath, stackname + indexSuffix, -1, plasmaCrawlEntry.rowdef), true, false);
} catch (IOException e) {
e.printStackTrace();
System.exit(-1);
}
urlFileIndex = new kelondroCache(new kelondroFlexTable(cacheStacksPath, stackname + indexSuffix, -1, plasmaCrawlEntry.rowdef, true), true, false);
}
private void resetFileIndex() {

2
source/de/anomic/plasma/plasmaCrawlLURL.java
Unescape View File

@ -96,7 +96,7 @@ public final class plasmaCrawlLURL {
super();
try {
urlIndexFile = new kelondroFlexSplitTable(new File(indexPath, "PUBLIC/TEXT"), "urls", preloadTime, indexURLEntry.rowdef);
urlIndexFile = new kelondroFlexSplitTable(new File(indexPath, "PUBLIC/TEXT"), "urls", preloadTime, indexURLEntry.rowdef, false);
} catch (IOException e) {
e.printStackTrace();
System.exit(-1);

4
source/de/anomic/plasma/plasmaCrawlProfile.java
Unescape View File

@ -68,7 +68,7 @@ public class plasmaCrawlProfile {
this.profileTableFile = file;
this.preloadTime = preloadTime;
profileTableFile.getParentFile().mkdirs();
kelondroDyn dyn = kelondroDyn.open(profileTableFile, true, true, preloadTime, crawlProfileHandleLength, 2000, '#', true, false);
kelondroDyn dyn = new kelondroDyn(profileTableFile, true, true, preloadTime, crawlProfileHandleLength, 2000, '#', true, false, true);
profileTable = new kelondroMapObjects(dyn, 500);
domsCache = new HashMap();
}
@ -78,7 +78,7 @@ public class plasmaCrawlProfile {
if (profileTable != null) profileTable.close();
if (!(profileTableFile.delete())) throw new RuntimeException("cannot delete crawl profile database");
profileTableFile.getParentFile().mkdirs();
kelondroDyn dyn = kelondroDyn.open(profileTableFile, true, true, preloadTime, crawlProfileHandleLength, 2000, '#', true, false);
kelondroDyn dyn = new kelondroDyn(profileTableFile, true, true, preloadTime, crawlProfileHandleLength, 2000, '#', true, false, true);
profileTable = new kelondroMapObjects(dyn, 500);
}

4
source/de/anomic/plasma/plasmaCrawlRobotsTxt.java
Unescape View File

@ -71,7 +71,7 @@ public class plasmaCrawlRobotsTxt {
this.robotsTableFile = robotsTableFile;
this.preloadTime = preloadTime;
robotsTableFile.getParentFile().mkdirs();
robotsTable = new kelondroMapObjects(kelondroDyn.open(robotsTableFile, true, true, preloadTime, 256, 512, '_', true, false), 100);
robotsTable = new kelondroMapObjects(new kelondroDyn(robotsTableFile, true, true, preloadTime, 256, 512, '_', true, false, true), 100);
}
private void resetDatabase() {
@ -79,7 +79,7 @@ public class plasmaCrawlRobotsTxt {
if (robotsTable != null) robotsTable.close();
if (!(robotsTableFile.delete())) throw new RuntimeException("cannot delete robots.txt database");
robotsTableFile.getParentFile().mkdirs();
robotsTable = new kelondroMapObjects(kelondroDyn.open(robotsTableFile, true, true, preloadTime, 256, 512, '_', true, false), 100);
robotsTable = new kelondroMapObjects(new kelondroDyn(robotsTableFile, true, true, preloadTime, 256, 512, '_', true, false, true), 100);
}
public void close() {

11
source/de/anomic/plasma/plasmaCrawlStacker.java
Unescape View File

@ -520,13 +520,13 @@ public final class plasmaCrawlStacker {
String newCacheName = "urlNoticeStacker7.db";
cacheStacksPath.mkdirs();
try {
this.urlEntryCache = new kelondroCache(new kelondroFlexTable(cacheStacksPath, newCacheName, preloadTime, plasmaCrawlEntry.rowdef), true, false);
this.urlEntryCache = new kelondroCache(new kelondroFlexTable(cacheStacksPath, newCacheName, preloadTime, plasmaCrawlEntry.rowdef, true), true, false);
} catch (Exception e) {
e.printStackTrace();
// kill DB and try again
kelondroFlexTable.delete(cacheStacksPath, newCacheName);
try {
this.urlEntryCache = new kelondroCache(new kelondroFlexTable(cacheStacksPath, newCacheName, preloadTime, plasmaCrawlEntry.rowdef), true, false);
this.urlEntryCache = new kelondroCache(new kelondroFlexTable(cacheStacksPath, newCacheName, preloadTime, plasmaCrawlEntry.rowdef, true), true, false);
} catch (Exception ee) {
ee.printStackTrace();
System.exit(-1);
@ -536,12 +536,7 @@ public final class plasmaCrawlStacker {
if (this.dbtype == QUEUE_DB_TYPE_TREE) {
File cacheFile = new File(cacheStacksPath, "urlNoticeStacker.db");
cacheFile.getParentFile().mkdirs();
try {
this.urlEntryCache = new kelondroCache(kelondroTree.open(cacheFile, true, preloadTime, plasmaCrawlEntry.rowdef), true, true);
} catch (IOException e) {
e.printStackTrace();
System.exit(-1);
}
this.urlEntryCache = new kelondroCache(kelondroTree.open(cacheFile, true, preloadTime, plasmaCrawlEntry.rowdef), true, true);
}
}

7
source/de/anomic/plasma/plasmaCrawlZURL.java
Unescape View File

@ -58,12 +58,7 @@ public class plasmaCrawlZURL {
public plasmaCrawlZURL(File cachePath, String tablename) {
cachePath.mkdirs();
try {
urlIndexFile = new kelondroFlexTable(cachePath, tablename, -1, rowdef);
} catch (IOException e) {
e.printStackTrace();
System.exit(-1);
}
urlIndexFile = new kelondroFlexTable(cachePath, tablename, -1, rowdef, true);
}
public int size() {

7
source/de/anomic/plasma/plasmaHTCache.java
Unescape View File

@ -197,12 +197,7 @@ public final class plasmaHTCache {
private void openResponseHeaderDB(long preloadTime) {
// open the response header database
File dbfile = new File(this.cachePath, "responseHeader.db");
try {
this.responseHeaderDB = new kelondroMapObjects(new kelondroDyn(dbfile, true, true, preloadTime, yacySeedDB.commonHashLength, 150, '#', true, false), 500);
} catch (IOException e) {
this.log.logSevere("the request header database could not be opened: " + e.getMessage());
System.exit(0);
}
this.responseHeaderDB = new kelondroMapObjects(new kelondroDyn(dbfile, true, true, preloadTime, yacySeedDB.commonHashLength, 150, '#', true, false, true), 500);
}
private void deleteOldHTCache(File directory) {

2
source/de/anomic/plasma/plasmaRankingCRProcess.java
Unescape View File

@ -261,7 +261,7 @@ public class plasmaRankingCRProcess {
kelondroCollectionIndex newseq = null;
if (newdb) {
File path = to_file.getParentFile(); // path to storage place
newacc = new kelondroFlexTable(path, CRG_accname, -1, CRG_accrow);
newacc = new kelondroFlexTable(path, CRG_accname, -1, CRG_accrow, false);
newseq = new kelondroCollectionIndex(path, CRG_seqname, 12, kelondroBase64Order.enhancedCoder, -1, 2, 9, CRG_colrow);
} else {
if (!(to_file.exists())) {

3
source/de/anomic/plasma/plasmaRankingRCIEvaluation.java
Unescape View File

@ -55,6 +55,7 @@ import java.util.TreeSet;
import de.anomic.plasma.plasmaURL;
import de.anomic.kelondro.kelondroAttrSeq;
import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.net.URL;
import de.anomic.server.serverCodings;
import de.anomic.server.serverFileUtils;
@ -155,7 +156,7 @@ public class plasmaRankingRCIEvaluation {
public static TreeSet[] genRankingTable(kelondroAttrSeq rci, int[] partition) {
TreeSet[] ranked = new TreeSet[partition.length];
for (int i = 0; i < partition.length; i++) ranked[i] = new TreeSet();
for (int i = 0; i < partition.length; i++) ranked[i] = new TreeSet(kelondroBase64Order.enhancedCoder);
Iterator i = rci.keys();
String key;
kelondroAttrSeq.Entry entry;

21
source/de/anomic/plasma/plasmaSearchEvent.java
Unescape View File

@ -125,7 +125,7 @@ public final class plasmaSearchEvent extends Thread implements Runnable {
// generate statistics about search: query, time, etc
HashMap r = new HashMap();
r.put("queryhashes", query.queryHashes);
r.put("querywords", query.queryWords);
r.put("querystring", query.queryString);
r.put("querycount", new Integer(query.wantedResults));
r.put("querytime", new Long(query.maximumTime));
r.put("resultcount", new Integer(this.searchcount));
@ -398,8 +398,7 @@ public final class plasmaSearchEvent extends Thread implements Runnable {
Long preranking;
Object[] preorderEntry;
indexURLEntry.Components comp;
String pagetitle, pageurl, pageauthor, exclw;
Iterator excli;
String pagetitle, pageurl, pageauthor;
int minEntries = profileLocal.getTargetCount(plasmaSearchTimingProfile.PROCESS_POSTSORT);
try {
ordering: while (preorder.hasNext()) {
@ -417,13 +416,9 @@ public final class plasmaSearchEvent extends Thread implements Runnable {
pageauthor = comp.author().toLowerCase();
// check exclusion
excli = query.excludeWords.iterator();
while (excli.hasNext()) {
exclw = (String) excli.next();
if ((pagetitle.indexOf(exclw) >= 0) ||
(pageurl.indexOf(exclw) >= 0) ||
(pageauthor.indexOf(exclw) >= 0)) continue ordering;
}
if (plasmaSearchQuery.matches(pagetitle, query.excludeHashes)) continue ordering;
if (plasmaSearchQuery.matches(pageurl, query.excludeHashes)) continue ordering;
if (plasmaSearchQuery.matches(pageauthor, query.excludeHashes)) continue ordering;
// check constraints
if ((!(query.constraint.equals(plasmaSearchQuery.catchall_constraint))) &&
@ -494,7 +489,7 @@ public final class plasmaSearchEvent extends Thread implements Runnable {
// this must be called after search results had been computed
// it is wise to call this within a separate thread because
// this method waits until all threads are finished
serverLog.logFine("PLASMA", "STARTED FLUSHING GLOBAL SEARCH RESULTS FOR SEARCH " + query.queryWords);
serverLog.logFine("PLASMA", "STARTED FLUSHING GLOBAL SEARCH RESULTS FOR SEARCH " + query.queryString);
int remaining = 0;
if (primarySearchThreads == null) return;
@ -511,13 +506,13 @@ public final class plasmaSearchEvent extends Thread implements Runnable {
if (System.currentTimeMillis() - starttime > 90000) {
yacySearch.interruptAlive(primarySearchThreads);
if (secondarySearchThreads != null) yacySearch.interruptAlive(secondarySearchThreads);
log.logFine("SEARCH FLUSH: " + remaining + " PEERS STILL BUSY; ABANDONED; SEARCH WAS " + query.queryWords);
log.logFine("SEARCH FLUSH: " + remaining + " PEERS STILL BUSY; ABANDONED; SEARCH WAS " + query.queryString);
break;
}
//log.logFine("FINISHED FLUSH RESULTS PROCESS for query " + query.hashes(","));
}
serverLog.logFine("PLASMA", "FINISHED FLUSHING " + rcContainerFlushCount + " GLOBAL SEARCH RESULTS FOR SEARCH " + query.queryWords);
serverLog.logFine("PLASMA", "FINISHED FLUSHING " + rcContainerFlushCount + " GLOBAL SEARCH RESULTS FOR SEARCH " + query.queryString);
// finally delete the temporary index
rcContainers = null;

6
source/de/anomic/plasma/plasmaSearchPreOrder.java
Unescape View File

@ -48,6 +48,7 @@ import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
import java.util.TreeMap;
import java.util.TreeSet;
import de.anomic.index.indexContainer;
import de.anomic.index.indexRWIEntry;
@ -98,6 +99,7 @@ public final class plasmaSearchPreOrder {
// second pass: normalize entries and get ranking
i = container.entries();
this.pageAcc = new TreeMap();
TreeSet searchWords = plasmaSearchQuery.cleanQuery(query.queryString)[0];
for (int j = 0; j < count; j++) {
iEntry = (indexRWIEntry) i.next();
if (iEntry.urlHash().length() != container.row().width(container.row().primaryKey())) continue;
@ -108,7 +110,7 @@ public final class plasmaSearchPreOrder {
if ((query.contentdom == plasmaSearchQuery.CONTENTDOM_IMAGE) && (!(iEntry.flags().get(plasmaCondenser.flag_cat_hasimage)))) continue;
if ((query.contentdom == plasmaSearchQuery.CONTENTDOM_APP ) && (!(iEntry.flags().get(plasmaCondenser.flag_cat_hasapp )))) continue;
}
pageAcc.put(serverCodings.encodeHex(Long.MAX_VALUE - this.ranking.preRanking(iEntry.generateNormalized(this.entryMin, this.entryMax), query.words("")), 16) + iEntry.urlHash(), iEntry);
pageAcc.put(serverCodings.encodeHex(Long.MAX_VALUE - this.ranking.preRanking(iEntry.generateNormalized(this.entryMin, this.entryMax), searchWords), 16) + iEntry.urlHash(), iEntry);
}
this.filteredCount = pageAcc.size();
}
@ -127,7 +129,7 @@ public final class plasmaSearchPreOrder {
indexRWIEntry iEntry;
String hashpart;
boolean isWordRootURL;
String querywords = query.words("");
TreeSet querywords = plasmaSearchQuery.cleanQuery(query.queryString())[0];
while (i.hasNext()) {
if (pageAcc.size() <= query.wantedResults) break;
entry = (Map.Entry) i.next();

69
source/de/anomic/plasma/plasmaSearchQuery.java
Unescape View File

@ -42,13 +42,14 @@
package de.anomic.plasma;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Set;
import java.util.TreeSet;
import de.anomic.htmlFilter.htmlFilterAbstractScraper;
import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroBitfield;
import de.anomic.kelondro.kelondroMSetTools;
import de.anomic.kelondro.kelondroNaturalOrder;
import de.anomic.server.serverCharBuffer;
import de.anomic.yacy.yacySeedDB;
@ -70,8 +71,8 @@ public final class plasmaSearchQuery {
public static final kelondroBitfield empty_constraint = new kelondroBitfield(4, "AAAAAA");
public static final kelondroBitfield catchall_constraint = new kelondroBitfield(4, "______");
public Set queryWords, queryHashes;
public Set excludeWords;
public String queryString;
public TreeSet queryHashes, excludeHashes;
public int wantedResults;
public String prefer;
public int contentdom;
@ -83,13 +84,13 @@ public final class plasmaSearchQuery {
public int maxDistance;
public kelondroBitfield constraint;
public plasmaSearchQuery(Set queryWords, Set excludeWords, int maxDistance, String prefer, int contentdom,
public plasmaSearchQuery(String queryString, TreeSet queryHashes, TreeSet excludeHashes, int maxDistance, String prefer, int contentdom,
int wantedResults, long maximumTime, String urlMask,
int domType, String domGroupName, int domMaxTargets,
kelondroBitfield constraint) {
this.queryWords = queryWords;
this.queryHashes = plasmaCondenser.words2hashes(queryWords);
this.excludeWords = excludeWords;
this.queryString = queryString;
this.queryHashes = queryHashes;
this.excludeHashes = excludeHashes;
this.maxDistance = maxDistance;
this.prefer = prefer;
this.contentdom = contentdom;
@ -102,11 +103,11 @@ public final class plasmaSearchQuery {
this.constraint = constraint;
}
public plasmaSearchQuery(Set queryHashes, int maxDistance, String prefer, int contentdom,
public plasmaSearchQuery(TreeSet queryHashes, int maxDistance, String prefer, int contentdom,
int wantedResults, long maximumTime, String urlMask,
kelondroBitfield constraint) {
this.queryWords = null;
this.excludeWords = null;
this.queryString = null;
this.excludeHashes = new TreeSet(kelondroBase64Order.enhancedCoder);;
this.maxDistance = maxDistance;
this.prefer = prefer;
this.contentdom = contentdom;
@ -129,9 +130,9 @@ public final class plasmaSearchQuery {
return CONTENTDOM_TEXT;
}
public static Set hashes2Set(String query) {
if (query == null) return new HashSet();
final HashSet keyhashes = new HashSet(query.length() / yacySeedDB.commonHashLength);
public static TreeSet hashes2Set(String query) {
if (query == null) return new TreeSet(kelondroBase64Order.enhancedCoder);
final TreeSet keyhashes = new TreeSet(kelondroBase64Order.enhancedCoder);
for (int i = 0; i < (query.length() / yacySeedDB.commonHashLength); i++) {
keyhashes.add(query.substring(i * yacySeedDB.commonHashLength, (i + 1) * yacySeedDB.commonHashLength));
}
@ -145,25 +146,32 @@ public final class plasmaSearchQuery {
return new String(sb);
}
public static TreeSet[] cleanQuery(String words) {
public static final boolean matches(String text, TreeSet keyhashes) {
// returns true if any of the word hashes in keyhashes appear in the String text
// to do this, all words in the string must be recognized and transcoded to word hashes
TreeSet wordhashes = plasmaCondenser.words2hashes(plasmaCondenser.getWords(text).keySet());
return kelondroMSetTools.anymatch(wordhashes, keyhashes);
}
public static TreeSet[] cleanQuery(String querystring) {
// returns two sets: a query set and a exclude set
if ((words == null) || (words.length() == 0)) return new TreeSet[]{new TreeSet(), new TreeSet()};
if ((querystring == null) || (querystring.length() == 0)) return new TreeSet[]{new TreeSet(kelondroBase64Order.enhancedCoder), new TreeSet(kelondroBase64Order.enhancedCoder)};
// convert Umlaute
words = htmlFilterAbstractScraper.convertUmlaute(new serverCharBuffer(words.toCharArray())).toString();
querystring = htmlFilterAbstractScraper.convertUmlaute(new serverCharBuffer(querystring.toCharArray())).toString();
// remove funny symbols
final String seps = "'.,:/&";
words = words.toLowerCase().trim();
querystring = querystring.toLowerCase().trim();
int c;
for (int i = 0; i < seps.length(); i++) {
while ((c = words.indexOf(seps.charAt(i))) >= 0) { words = words.substring(0, c) + (((c + 1) < words.length()) ? (" " + words.substring(c + 1)) : ""); }
while ((c = querystring.indexOf(seps.charAt(i))) >= 0) { querystring = querystring.substring(0, c) + (((c + 1) < querystring.length()) ? (" " + querystring.substring(c + 1)) : ""); }
}
// the string is clean now, but we must generate a set out of it
final TreeSet query = new TreeSet(kelondroNaturalOrder.naturalOrder);
final TreeSet exclude = new TreeSet(kelondroNaturalOrder.naturalOrder);
final String[] a = words.split(" ");
final String[] a = querystring.split(" ");
for (int i = 0; i < a.length; i++) {
if (a[i].startsWith("-")) {
exclude.add(a[i].substring(1));
@ -179,29 +187,18 @@ public final class plasmaSearchQuery {
}
public int size() {
return queryHashes.size();
return queryHashes.size();
}
public String words(String separator) {
if (queryWords == null) return "";
StringBuffer result = new StringBuffer(8 * queryWords.size());
Iterator i = queryWords.iterator();
if (i.hasNext()) result.append((String) i.next());
while (i.hasNext()) {
result.append(separator);
result.append((String) i.next());
}
return result.toString();
public String queryString() {
return this.queryString;
}
public void filterOut(Set blueList) {
// filter out words that appear in this set
Iterator it = queryWords.iterator();
String word;
while (it.hasNext()) {
word = (String) it.next();
if (blueList.contains(word)) it.remove();
}
// this is applied to the queryHashes
TreeSet blues = plasmaCondenser.words2hashes(blueList);
kelondroMSetTools.excludeDestructive(queryHashes, blues);
}
public static String anonymizedQueryHashes(Set hashes) {

5
source/de/anomic/plasma/plasmaSearchRankingProfile.java
Unescape View File

@ -45,6 +45,7 @@ import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
import de.anomic.index.indexRWIEntry;
import de.anomic.plasma.plasmaURL;
@ -248,7 +249,7 @@ public class plasmaSearchRankingProfile {
return new String(ext);
}
public long preRanking(indexRWIEntry normalizedEntry, String searchedWord) {
public long preRanking(indexRWIEntry normalizedEntry, TreeSet searchedWords) {
// the normalizedEntry must be a normalized indexEntry
long ranking = 0;
ranking += (256 - plasmaURL.domLengthNormalized(normalizedEntry.urlHash())) << coeff_domlength;
@ -280,7 +281,7 @@ public class plasmaSearchRankingProfile {
ranking += (flags.get(plasmaCondenser.flag_cat_hasapp)) ? 256 << coeff_cathasapp : 0;
ranking += (plasmaURL.probablyRootURL(normalizedEntry.urlHash())) ? 16 << coeff_urllength : 0;
ranking += (plasmaURL.probablyWordURL(normalizedEntry.urlHash(), searchedWord) != null) ? 256 << coeff_appurl : 0;
ranking += (plasmaURL.probablyWordURL(normalizedEntry.urlHash(), searchedWords) != null) ? 256 << coeff_appurl : 0;
/*
if (indexURL.probablyWordURL(normalizedEntry.urlHash(), searchedWord))

6
source/de/anomic/plasma/plasmaSwitchboard.java
Unescape View File

@ -2709,7 +2709,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
results.setQuery(query);
// log
log.logInfo("INIT WORD SEARCH: " + query.queryWords + ":" + query.queryHashes + " - " + query.wantedResults + " links, " + (query.maximumTime / 1000) + " seconds");
log.logInfo("INIT WORD SEARCH: " + query.queryString + ":" + query.queryHashes + " - " + query.wantedResults + " links, " + (query.maximumTime / 1000) + " seconds");
long timestamp = System.currentTimeMillis();
// start a presearch, which makes only sense if we idle afterwards.
@ -2747,7 +2747,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
String host, hash, address;
yacySeed seed;
boolean includeSnippets = false;
results.setFormerSearch(query.words(" "));
results.setFormerSearch(query.queryString());
long targetTime = timestamp + query.maximumTime;
if (targetTime < System.currentTimeMillis()) targetTime = System.currentTimeMillis() + 1000;
while ((acc.hasMoreElements()) && (i < query.wantedResults) && (System.currentTimeMillis() < targetTime)) {
@ -2838,7 +2838,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
}
// log
log.logInfo("EXIT WORD SEARCH: " + query.queryWords + " - " +
log.logInfo("EXIT WORD SEARCH: " + query.queryString + " - " +
results.getTotalcount() + " links found, " +
results.getFilteredcount() + " links filtered, " +
results.getOrderedcount() + " links ordered, " +

33
source/de/anomic/plasma/plasmaURL.java
Unescape View File

@ -32,6 +32,7 @@ import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.TreeMap;
import java.util.TreeSet;
import de.anomic.index.indexContainer;
import de.anomic.index.indexRWIEntry;
@ -489,25 +490,31 @@ public class plasmaURL {
private static String[] testTLDs = new String[] { "com", "net", "org", "uk", "fr", "de", "es", "it" };
public static final URL probablyWordURL(String urlHash, String word) {
if ((word == null) || (word.length() == 0))return null;
String pattern = urlHash.substring(6, 11);
for (int i = 0; i < testTLDs.length; i++) {
if (pattern.equals(protocolHostPort("http", "www." + word.toLowerCase() + "." + testTLDs[i], 80)))
try {
return new URL("http://www." + word.toLowerCase() + "." + testTLDs[i]);
} catch (MalformedURLException e) {
return null;
}
public static final URL probablyWordURL(String urlHash, TreeSet words) {
Iterator wi = words.iterator();
String word;
while (wi.hasNext()) {
word = (String) wi.next();
if ((word == null) || (word.length() == 0)) continue;
String pattern = urlHash.substring(6, 11);
for (int i = 0; i < testTLDs.length; i++) {
if (pattern.equals(protocolHostPort("http", "www." + word.toLowerCase() + "." + testTLDs[i], 80)))
try {
return new URL("http://www." + word.toLowerCase() + "." + testTLDs[i]);
} catch (MalformedURLException e) {
return null;
}
}
}
return null;
}
public static final boolean isWordRootURL(String givenURLHash, String word) {
public static final boolean isWordRootURL(String givenURLHash, TreeSet words) {
if (!(probablyRootURL(givenURLHash))) return false;
URL wordURL = probablyWordURL(givenURLHash, word);
URL wordURL = probablyWordURL(givenURLHash, words);
if (wordURL == null) return false;
return urlHash(wordURL).equals(givenURLHash);
if (urlHash(wordURL).equals(givenURLHash)) return true;
return false;
}
public static final int domLengthEstimation(String urlHash) {

2
source/de/anomic/plasma/plasmaWordConnotation.java
Unescape View File

@ -60,7 +60,7 @@ public class plasmaWordConnotation {
private kelondroDynTree refDB;
public plasmaWordConnotation(File refDBfile, long preloadTime, char fillChar) throws IOException {
refDB = new kelondroDynTree(refDBfile, preloadTime, wordlength, nodesize, new kelondroRow("byte[] word-" + wordlength + ", Cardinal count-" + countlength, kelondroNaturalOrder.naturalOrder, 0), fillChar);
refDB = new kelondroDynTree(refDBfile, preloadTime, wordlength, nodesize, new kelondroRow("byte[] word-" + wordlength + ", Cardinal count-" + countlength, kelondroNaturalOrder.naturalOrder, 0), fillChar, true);
}
private void addSingleRef(String word, String reference) throws IOException {

19
source/de/anomic/yacy/yacySeedDB.java
Unescape View File

@ -192,11 +192,11 @@ public final class yacySeedDB {
initializeHandlerMethod = null;
}
try {
return new kelondroMapObjects(kelondroDyn.open(seedDBFile, true, true, preloadTime / 3, commonHashLength, 480, '#', false, false), 500, sortFields, longaccFields, doubleaccFields, initializeHandlerMethod, this);
return new kelondroMapObjects(kelondroDyn.open(seedDBFile, true, true, preloadTime / 3, commonHashLength, 480, '#', false, false, true), 500, sortFields, longaccFields, doubleaccFields, initializeHandlerMethod, this);
} catch (Exception e) {
seedDBFile.delete();
// try again
return new kelondroMapObjects(kelondroDyn.open(seedDBFile, true, true, preloadTime / 3, commonHashLength, 480, '#', false, false), 500, sortFields, longaccFields, doubleaccFields, initializeHandlerMethod, this);
return new kelondroMapObjects(kelondroDyn.open(seedDBFile, true, true, preloadTime / 3, commonHashLength, 480, '#', false, false, true), 500, sortFields, longaccFields, doubleaccFields, initializeHandlerMethod, this);
}
}
@ -888,10 +888,23 @@ public final class yacySeedDB {
public Object nextElement() {
yacySeed seed = nextSeed;
while (true) {
try {while (true) {
nextSeed = internalNext();
if (nextSeed == null) break;
if (nextSeed.getVersion() >= this.minVersion) break;
}} catch (kelondroException e) {
e.printStackTrace();
// eergency reset
yacyCore.log.logSevere("seed-db emergency reset", e);
try {
database.reset();
nextSeed = null;
return null;
} catch (IOException e1) {
// no recovery possible
e1.printStackTrace();
System.exit(-1);
}
}
return seed;
}

2
source/yacy.java
Unescape View File

@ -1071,7 +1071,7 @@ public final class yacy {
String[] dbFileNames = {"seed.new.db","seed.old.db","seed.pot.db"};
for (int i=0; i < dbFileNames.length; i++) {
File dbFile = new File(yacyDBPath,dbFileNames[i]);
kelondroMapObjects db = new kelondroMapObjects(new kelondroDyn(dbFile, true, true, 3000, yacySeedDB.commonHashLength, 480, '#', true, false), 500, yacySeedDB.sortFields, yacySeedDB.longaccFields, yacySeedDB.doubleaccFields, null, null);
kelondroMapObjects db = new kelondroMapObjects(new kelondroDyn(dbFile, true, true, 3000, yacySeedDB.commonHashLength, 480, '#', true, false, true), 500, yacySeedDB.sortFields, yacySeedDB.longaccFields, yacySeedDB.doubleaccFields, null, null);
kelondroMapObjects.mapIterator it;
it = db.maps(true, false);

Write Preview
Loading…
Cancel
Save