boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

some performance hacks (CPU only, not IO)

Browse Source 
this will cause better computation speed for single- and multi-core;
there are enhancements that will speed up old and slow machines as well
as multi-core CPUs. Indexing of surrogates has been speed up
from 4000 PPM to over 20000 PPM on a simple dual core office computer.
Since the enhancements are mostly in core routines, the hack should also
speed up search performance.

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6276 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 16 years ago
parent 92407009b2
commit 1a9cfd8718
14 changed files with 191 additions and 119 deletions
Show Stats Download Patch File Download Diff File

29
htroot/PerformanceSearch_p.java
Unescape View File

@ -24,6 +24,7 @@
// along with this program; if not, write to the Free Software // along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
import java.util.ArrayList;
import java.util.Date; import java.util.Date;
import java.util.Iterator; import java.util.Iterator;
@ -39,25 +40,27 @@ public class PerformanceSearch_p {
// return variable that accumulates replacements // return variable that accumulates replacements
final serverObjects prop = new serverObjects(); final serverObjects prop = new serverObjects();
final Iterator<serverProfiling.Event> events = serverProfiling.history("SEARCH"); final ArrayList<serverProfiling.Event> events = serverProfiling.history("SEARCH");
int c = 0; int c = 0;
serverProfiling.Event event; serverProfiling.Event event;
ProfilingGraph.searchEvent search; ProfilingGraph.searchEvent search;
long lastt = 0; long lastt = 0;
while (events.hasNext()) { if (events != null) synchronized (events) {
event = events.next(); Iterator<serverProfiling.Event> i = events.iterator();
search = (ProfilingGraph.searchEvent) event.payload; while (i.hasNext()) {
prop.put("table_" + c + "_query", search.queryID); event = i.next();
prop.put("table_" + c + "_event", search.processName); search = (ProfilingGraph.searchEvent) event.payload;
prop.putNum("table_" + c + "_count", search.resultCount); prop.put("table_" + c + "_query", search.queryID);
prop.putNum("table_" + c + "_delta", event.time - lastt); prop.put("table_" + c + "_event", search.processName);
prop.put("table_" + c + "_time", (new Date(event.time)).toString()); prop.putNum("table_" + c + "_count", search.resultCount);
prop.putNum("table_" + c + "_duration", search.duration); prop.putNum("table_" + c + "_delta", event.time - lastt);
c++; prop.put("table_" + c + "_time", (new Date(event.time)).toString());
lastt = event.time; prop.putNum("table_" + c + "_duration", search.duration);
c++;
lastt = event.time;
}
} }
prop.put("table", c); prop.put("table", c);
return prop; return prop;
} }
} }

8
source/de/anomic/document/Condenser.java
Unescape View File

@ -87,7 +87,7 @@ public final class Condenser {
private final static int numlength = 5; private final static int numlength = 5;
//private Properties analysis; //private Properties analysis;
private TreeMap<String, Word> words; // a string (the words) to (indexWord) - relation private Map<String, Word> words; // a string (the words) to (indexWord) - relation
private final int wordminsize; private final int wordminsize;
private final int wordcut; private final int wordcut;
@ -108,7 +108,7 @@ public final class Condenser {
// added media words are flagged with the appropriate media flag // added media words are flagged with the appropriate media flag
this.wordminsize = 3; this.wordminsize = 3;
this.wordcut = 2; this.wordcut = 2;
this.words = new TreeMap<String, Word>(); this.words = new HashMap<String, Word>();
this.RESULT_FLAGS = new Bitfield(4); this.RESULT_FLAGS = new Bitfield(4);
// construct flag set for document // construct flag set for document
@ -260,7 +260,7 @@ public final class Condenser {
// subtracts the given stopwords from the word list // subtracts the given stopwords from the word list
// the word list shrinkes. This returns the number of shrinked words // the word list shrinkes. This returns the number of shrinked words
final int oldsize = words.size(); final int oldsize = words.size();
words = SetTools.excludeConstructive(words, stopwords); SetTools.excludeDestructive(words, stopwords);
return oldsize - words.size(); return oldsize - words.size();
} }
@ -738,7 +738,7 @@ public final class Condenser {
} }
static StringBuilder readSentence(final Reader reader, final boolean pre) throws IOException { static StringBuilder readSentence(final Reader reader, final boolean pre) throws IOException {
final StringBuilder s = new StringBuilder(40); final StringBuilder s = new StringBuilder(80);
int nextChar; int nextChar;
char c, lc = ' '; // starting with ' ' as last character prevents that the result string starts with a ' ' char c, lc = ' '; // starting with ' ' as last character prevents that the result string starts with a ' '

2
source/de/anomic/document/Word.java
Unescape View File

@ -41,7 +41,7 @@ import de.anomic.yacy.yacySeedDB;
public class Word { public class Word {
public static final int hashCacheSize = (int) (MemoryControl.available() / 10000L); public static final int hashCacheSize = Math.max(2000, Math.min(100000, (int) (MemoryControl.available() / 20000L)));
private static final SimpleARC<String, byte[]> hashCache = new SimpleARC<String, byte[]>(hashCacheSize); private static final SimpleARC<String, byte[]> hashCache = new SimpleARC<String, byte[]>(hashCacheSize);
// object carries statistics for words and sentences // object carries statistics for words and sentences

2
source/de/anomic/kelondro/index/ObjectIndexCache.java
Unescape View File

@ -54,7 +54,7 @@ public class ObjectIndexCache implements ObjectIndex {
public synchronized void reset(final int initialspace) { public synchronized void reset(final int initialspace) {
this.index0 = null; // first flush RAM to make room this.index0 = null; // first flush RAM to make room
this.index0 = new RowSet(rowdef, initialspace); this.index0 = new RowSet(rowdef, initialspace);
this.index1 = null; // to show that this is the initialization phase this.index1 = null; // to show that this is the initialization phase
} }

36
source/de/anomic/kelondro/index/RowCollection.java
Unescape View File

@ -49,7 +49,7 @@ import de.anomic.yacy.logging.Log;
public class RowCollection implements Iterable<Row.Entry> { public class RowCollection implements Iterable<Row.Entry> {
public static final double growfactor = 1.4; public static final long growfactor100 = 140L;
private static final int isortlimit = 20; private static final int isortlimit = 20;
static final Integer dummy = 0; static final Integer dummy = 0;
@ -142,8 +142,10 @@ public class RowCollection implements Iterable<Row.Entry> {
return (int) (time / day) - 10957; return (int) (time / day) - 10957;
} }
private static Column exportColumn0, exportColumn1, exportColumn2, exportColumn3, exportColumn4;
private static Row exportRow(final int chunkcachelength) { private static Row exportRow(final int chunkcachelength) {
// find out the size of this collection /*
return new Row( return new Row(
"int size-4 {b256}," + "int size-4 {b256}," +
"short lastread-2 {b256}," + // as daysSince2000 "short lastread-2 {b256}," + // as daysSince2000
@ -153,6 +155,23 @@ public class RowCollection implements Iterable<Row.Entry> {
"byte[] collection-" + chunkcachelength, "byte[] collection-" + chunkcachelength,
NaturalOrder.naturalOrder NaturalOrder.naturalOrder
); );
*/
if (exportColumn0 == null) exportColumn0 = new Column("int size-4 {b256}");
if (exportColumn1 == null) exportColumn1 = new Column("short lastread-2 {b256}");
if (exportColumn2 == null) exportColumn2 = new Column("short lastwrote-2 {b256}");
if (exportColumn3 == null) exportColumn3 = new Column("byte[] orderkey-2");
if (exportColumn4 == null) exportColumn4 = new Column("int orderbound-4 {b256}");
/*
* because of a strange bug these objects cannot be initialized as normal
* static final. If I try that, they are not initialized and are assigned null. why?
*/
return new Row(new Column[]{
exportColumn0, exportColumn1, exportColumn2, exportColumn3, exportColumn4,
new Column("byte[] collection-" + chunkcachelength)
},
NaturalOrder.naturalOrder
);
} }
public static final int exportOverheadSize = 14; public static final int exportOverheadSize = 14;
@ -183,9 +202,12 @@ public class RowCollection implements Iterable<Row.Entry> {
} }
protected final void ensureSize(final int elements) { protected final void ensureSize(final int elements) {
final int needed = elements * rowdef.objectsize; assert elements > 0 : "elements = " + elements;
final long needed = elements * rowdef.objectsize;
if (chunkcache.length >= needed) return; if (chunkcache.length >= needed) return;
byte[] newChunkcache = new byte[(int) (needed * growfactor)]; // increase space assert needed > 0 : "needed = " + needed;
assert needed * growfactor100 / 100L > 0 : "elements = " + elements + ", new = " + (needed * growfactor100 / 100L);
byte[] newChunkcache = new byte[(int) (needed * growfactor100 / 100L)]; // increase space
System.arraycopy(chunkcache, 0, newChunkcache, 0, chunkcache.length); System.arraycopy(chunkcache, 0, newChunkcache, 0, chunkcache.length);
chunkcache = newChunkcache; chunkcache = newChunkcache;
} }
@ -193,17 +215,17 @@ public class RowCollection implements Iterable<Row.Entry> {
/** /**
* compute the needed memory in case of a cache extension. That is, if the cache is full and must * compute the needed memory in case of a cache extension. That is, if the cache is full and must
* be copied into a new cache which is larger. In such a case the Collection needs more than the double size * be copied into a new cache which is larger. In such a case the Collection needs more than the double size
* than is necessary to store the data. This method coputes the extra memory that is needed to perform this task. * than is necessary to store the data. This method computes the extra memory that is needed to perform this task.
* @return * @return
*/ */
public final long memoryNeededForGrow() { public final long memoryNeededForGrow() {
return (long) ((((long) (chunkcount + 1)) * ((long) rowdef.objectsize)) * growfactor); return (long) ((((long) (chunkcount + 1)) * ((long) rowdef.objectsize)) * growfactor100 / 100L);
} }
public synchronized void trim(final boolean plusGrowFactor) { public synchronized void trim(final boolean plusGrowFactor) {
if (chunkcache.length == 0) return; if (chunkcache.length == 0) return;
int needed = chunkcount * rowdef.objectsize; int needed = chunkcount * rowdef.objectsize;
if (plusGrowFactor) needed = (int) (needed * growfactor); if (plusGrowFactor) needed = (int) (((long) needed) * growfactor100 / 100L);
if (needed >= chunkcache.length) if (needed >= chunkcache.length)
return; // in case that the growfactor causes that the cache would return; // in case that the growfactor causes that the cache would
// grow instead of shrink, simply ignore the growfactor // grow instead of shrink, simply ignore the growfactor

57
source/de/anomic/kelondro/table/SplitTable.java
Unescape View File

@ -114,14 +114,6 @@ public class SplitTable implements ObjectIndex {
public void init() { public void init() {
current = null; current = null;
// init the thread pool for the keeperOf executor service
this.executor = new ThreadPoolExecutor(
Runtime.getRuntime().availableProcessors() + 1,
Runtime.getRuntime().availableProcessors() + 1, 10,
TimeUnit.SECONDS,
new LinkedBlockingQueue<Runnable>(),
new NamePrefixThreadFactory(prefix));
// initialized tables map // initialized tables map
this.tables = new HashMap<String, ObjectIndex>(); this.tables = new HashMap<String, ObjectIndex>();
if (!(path.exists())) path.mkdirs(); if (!(path.exists())) path.mkdirs();
@ -199,6 +191,16 @@ public class SplitTable implements ObjectIndex {
tables.put(maxf, table); tables.put(maxf, table);
} }
} }
// init the thread pool for the keeperOf executor service
this.executor = new ThreadPoolExecutor(
Math.max(tables.size(), Runtime.getRuntime().availableProcessors()) + 1,
Math.max(tables.size(), Runtime.getRuntime().availableProcessors()) + 1, 10,
TimeUnit.SECONDS,
new LinkedBlockingQueue<Runnable>(),
new NamePrefixThreadFactory(prefix));
} }
public void clear() throws IOException { public void clear() throws IOException {
@ -304,20 +306,46 @@ public class SplitTable implements ObjectIndex {
keeper.put(row); keeper.put(row);
} }
private static final class ReadyCheck {
private boolean r;
public ReadyCheck() {
this.r = false;
}
public void isReady() {
this.r = true;
}
public boolean check() {
return this.r;
}
}
public synchronized ObjectIndex keeperOf(final byte[] key) { public synchronized ObjectIndex keeperOf(final byte[] key) {
if (tables.size() < 2) {
// no concurrency if not needed
for (final ObjectIndex table : tables.values()) {
if (table.has(key)) return table;
}
return null;
}
// because the index is stored only in one table, // because the index is stored only in one table,
// and the index is completely in RAM, a concurrency will create // and the index is completely in RAM, a concurrency will create
// not concurrent File accesses // not concurrent File accesses
//long start = System.currentTimeMillis();
// start a concurrent query to database tables // start a concurrent query to database tables
final CompletionService<ObjectIndex> cs = new ExecutorCompletionService<ObjectIndex>(executor); final CompletionService<ObjectIndex> cs = new ExecutorCompletionService<ObjectIndex>(executor);
int accepted = 0; int accepted = 0;
final ReadyCheck ready = new ReadyCheck();
for (final ObjectIndex table : tables.values()) { for (final ObjectIndex table : tables.values()) {
if (ready.check()) break; // found already a table
try { try {
cs.submit(new Callable<ObjectIndex>() { cs.submit(new Callable<ObjectIndex>() {
public ObjectIndex call() { public ObjectIndex call() {
if (table.has(key)) return table; if (table.has(key)) {
ready.isReady();
return table;
}
return dummyIndex; return dummyIndex;
} }
}); });
@ -333,15 +361,11 @@ public class SplitTable implements ObjectIndex {
try { try {
for (int i = 0; i < accepted; i++) { for (int i = 0; i < accepted; i++) {
final Future<ObjectIndex> f = cs.take(); final Future<ObjectIndex> f = cs.take();
//hash(System.out.println("**********accepted = " + accepted + ", i =" + i);
if (f == null) continue; if (f == null) continue;
final ObjectIndex index = f.get(); final ObjectIndex index = f.get();
if (index != dummyIndex) { if (index == dummyIndex) continue;
//System.out.println("*DEBUG SplitTable success.time = " + (System.currentTimeMillis() - start) + " ms"); return index;
return index;
}
} }
//System.out.println("*DEBUG SplitTable fail.time = " + (System.currentTimeMillis() - start) + " ms");
return null; return null;
} catch (final InterruptedException e) { } catch (final InterruptedException e) {
Thread.currentThread().interrupt(); Thread.currentThread().interrupt();
@ -349,7 +373,6 @@ public class SplitTable implements ObjectIndex {
e.printStackTrace(); e.printStackTrace();
throw new RuntimeException(e.getCause()); throw new RuntimeException(e.getCause());
} }
//System.out.println("*DEBUG SplitTable fail.time = " + (System.currentTimeMillis() - start) + " ms");
return null; return null;
} }

6
source/de/anomic/kelondro/table/Table.java
Unescape View File

@ -266,9 +266,9 @@ public class Table implements ObjectIndex {
final HashMap<String, String> map = new HashMap<String, String>(); final HashMap<String, String> map = new HashMap<String, String>();
map.put("tableSize", Integer.toString(index.size())); map.put("tableSize", Integer.toString(index.size()));
map.put("tableKeyChunkSize", Integer.toString(index.row().objectsize)); map.put("tableKeyChunkSize", Integer.toString(index.row().objectsize));
map.put("tableKeyMem", Integer.toString((int) (index.row().objectsize * index.size() * RowCollection.growfactor))); map.put("tableKeyMem", Integer.toString((int) (((long) index.row().objectsize) * ((long) index.size()) * RowCollection.growfactor100 / 100L)));
map.put("tableValueChunkSize", (table == null) ? "0" : Integer.toString(table.row().objectsize)); map.put("tableValueChunkSize", (table == null) ? "0" : Integer.toString(table.row().objectsize));
map.put("tableValueMem", (table == null) ? "0" : Integer.toString((int) (table.row().objectsize * table.size() * RowCollection.growfactor))); map.put("tableValueMem", (table == null) ? "0" : Integer.toString((int) (((long) index.row().objectsize) * ((long) index.size()) * RowCollection.growfactor100 / 100L)));
return map; return map;
} }
@ -277,7 +277,7 @@ public class Table implements ObjectIndex {
} }
public static int staticRAMIndexNeed(final File f, final Row rowdef) { public static int staticRAMIndexNeed(final File f, final Row rowdef) {
return (int) ((rowdef.primaryKeyLength + 4) * tableSize(f, rowdef.objectsize) * RowCollection.growfactor); return (int) (((long)(rowdef.primaryKeyLength + 4)) * ((long) tableSize(f, rowdef.objectsize)) * RowCollection.growfactor100 / 100L);
} }
public synchronized void addUnique(final Entry row) throws IOException { public synchronized void addUnique(final Entry row) throws IOException {

6
source/de/anomic/kelondro/text/IndexCell.java
Unescape View File

@ -52,7 +52,7 @@ import de.anomic.server.serverProfiling;
public final class IndexCell<ReferenceType extends Reference> extends AbstractBufferedIndex<ReferenceType> implements BufferedIndex<ReferenceType> { public final class IndexCell<ReferenceType extends Reference> extends AbstractBufferedIndex<ReferenceType> implements BufferedIndex<ReferenceType> {
private static final long cleanupCycle = 10000; private static final long cleanupCycle = 60000;
// class variables // class variables
private final ReferenceContainerArray<ReferenceType> array; private final ReferenceContainerArray<ReferenceType> array;
@ -101,7 +101,7 @@ public final class IndexCell<ReferenceType extends Reference> extends AbstractBu
*/ */
public void add(ReferenceContainer<ReferenceType> newEntries) throws IOException { public void add(ReferenceContainer<ReferenceType> newEntries) throws IOException {
this.ram.add(newEntries); this.ram.add(newEntries);
if (this.ram.size() % 100 == 0) { if (this.ram.size() % 1000 == 0 || this.lastCleanup + cleanupCycle < System.currentTimeMillis()) {
serverProfiling.update("wordcache", Long.valueOf(this.ram.size()), true); serverProfiling.update("wordcache", Long.valueOf(this.ram.size()), true);
cleanCache(); cleanCache();
} }
@ -109,7 +109,7 @@ public final class IndexCell<ReferenceType extends Reference> extends AbstractBu
public void add(byte[] termHash, ReferenceType entry) throws IOException { public void add(byte[] termHash, ReferenceType entry) throws IOException {
this.ram.add(termHash, entry); this.ram.add(termHash, entry);
if (this.ram.size() % 100 == 0) { if (this.ram.size() % 1000 == 0 || this.lastCleanup + cleanupCycle < System.currentTimeMillis()) {
serverProfiling.update("wordcache", Long.valueOf(this.ram.size()), true); serverProfiling.update("wordcache", Long.valueOf(this.ram.size()), true);
cleanCache(); cleanCache();
} }

56
source/de/anomic/kelondro/text/ReferenceContainerCache.java
Unescape View File

@ -28,11 +28,11 @@ package de.anomic.kelondro.text;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.util.Arrays;
import java.util.Comparator;
import java.util.Iterator; import java.util.Iterator;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.Set;
import java.util.SortedMap;
import java.util.TreeMap;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import de.anomic.kelondro.blob.HeapReader; import de.anomic.kelondro.blob.HeapReader;
@ -48,8 +48,15 @@ import de.anomic.yacy.logging.Log;
public final class ReferenceContainerCache<ReferenceType extends Reference> extends AbstractIndex<ReferenceType> implements Index<ReferenceType>, IndexReader<ReferenceType>, Iterable<ReferenceContainer<ReferenceType>> { public final class ReferenceContainerCache<ReferenceType extends Reference> extends AbstractIndex<ReferenceType> implements Index<ReferenceType>, IndexReader<ReferenceType>, Iterable<ReferenceContainer<ReferenceType>> {
public class ContainerOrder implements Comparator<ReferenceContainer<ReferenceType>> {
public int compare(ReferenceContainer<ReferenceType> arg0, ReferenceContainer<ReferenceType> arg1) {
return termOrder.compare(arg0.getTermHash(), arg1.getTermHash());
}
}
private final Row payloadrow; private final Row payloadrow;
private final ByteOrder termOrder; protected final ByteOrder termOrder;
private final ContainerOrder containerOrder;
protected Map<ByteArray, ReferenceContainer<ReferenceType>> cache; protected Map<ByteArray, ReferenceContainer<ReferenceType>> cache;
/** /**
@ -63,6 +70,7 @@ public final class ReferenceContainerCache<ReferenceType extends Reference> exte
super(factory); super(factory);
this.payloadrow = payloadrow; this.payloadrow = payloadrow;
this.termOrder = termOrder; this.termOrder = termOrder;
this.containerOrder = new ContainerOrder();
this.cache = null; this.cache = null;
} }
@ -102,17 +110,15 @@ public final class ReferenceContainerCache<ReferenceType extends Reference> exte
final long startTime = System.currentTimeMillis(); final long startTime = System.currentTimeMillis();
// sort the map // sort the map
SortedMap<byte[], ReferenceContainer<ReferenceType>> cachecopy = sortedClone(); ReferenceContainer<ReferenceType>[] cachecopy = sortedClone();
// write wCache // write wCache
long wordcount = 0, urlcount = 0; long wordcount = 0, urlcount = 0;
byte[] wordHash = null, lwh; byte[] wordHash = null, lwh;
ReferenceContainer<ReferenceType> container; for (final ReferenceContainer<ReferenceType> container: cachecopy) {
for (final Map.Entry<byte[], ReferenceContainer<ReferenceType>> entry: cachecopy.entrySet()) {
// get entries // get entries
lwh = wordHash; lwh = wordHash;
wordHash = entry.getKey(); wordHash = container.getTermHash();
container = entry.getValue();
// check consistency: entries must be ordered // check consistency: entries must be ordered
assert (lwh == null || this.ordering().compare(wordHash, lwh) > 0); assert (lwh == null || this.ordering().compare(wordHash, lwh) > 0);
@ -139,6 +145,19 @@ public final class ReferenceContainerCache<ReferenceType extends Reference> exte
} }
} }
@SuppressWarnings("unchecked")
public ReferenceContainer<ReferenceType>[] sortedClone() {
ReferenceContainer<ReferenceType>[] cachecopy = new ReferenceContainer[cache.size()];
synchronized (cache) {
int p = 0;
for (final Map.Entry<ByteArray, ReferenceContainer<ReferenceType>> entry: cache.entrySet()) {
cachecopy[p++] = entry.getValue();
}
}
Arrays.sort(cachecopy, this.containerOrder);
return cachecopy;
}
/*
public SortedMap<byte[], ReferenceContainer<ReferenceType>> sortedClone() { public SortedMap<byte[], ReferenceContainer<ReferenceType>> sortedClone() {
SortedMap<byte[], ReferenceContainer<ReferenceType>> cachecopy; SortedMap<byte[], ReferenceContainer<ReferenceType>> cachecopy;
synchronized (cache) { synchronized (cache) {
@ -149,7 +168,7 @@ public final class ReferenceContainerCache<ReferenceType extends Reference> exte
} }
return cachecopy; return cachecopy;
} }
*/
public int size() { public int size() {
return (this.cache == null) ? 0 : this.cache.size(); return (this.cache == null) ? 0 : this.cache.size();
} }
@ -252,14 +271,15 @@ public final class ReferenceContainerCache<ReferenceType extends Reference> exte
// plus the mentioned features // plus the mentioned features
private final boolean rot; private final boolean rot;
private Iterator<ReferenceContainer<ReferenceType>> iterator; private ReferenceContainer<ReferenceType>[] cachecopy;
private int p;
private byte[] latestTermHash; private byte[] latestTermHash;
public heapCacheIterator(byte[] startWordHash, final boolean rot) { public heapCacheIterator(byte[] startWordHash, final boolean rot) {
this.rot = rot; this.rot = rot;
if (startWordHash != null && startWordHash.length == 0) startWordHash = null; if (startWordHash != null && startWordHash.length == 0) startWordHash = null;
SortedMap<byte[], ReferenceContainer<ReferenceType>> cachecopy = sortedClone(); this.cachecopy = sortedClone();
this.iterator = (startWordHash == null) ? cachecopy.values().iterator() : cachecopy.tailMap(startWordHash).values().iterator(); this.p = 0;
this.latestTermHash = null; this.latestTermHash = null;
// The collection's iterator will return the values in the order that their corresponding keys appear in the tree. // The collection's iterator will return the values in the order that their corresponding keys appear in the tree.
} }
@ -270,12 +290,12 @@ public final class ReferenceContainerCache<ReferenceType extends Reference> exte
public boolean hasNext() { public boolean hasNext() {
if (rot) return true; if (rot) return true;
return iterator.hasNext(); return this.p < this.cachecopy.length;
} }
public ReferenceContainer<ReferenceType> next() { public ReferenceContainer<ReferenceType> next() {
if (iterator.hasNext()) { if (this.p < this.cachecopy.length) {
ReferenceContainer<ReferenceType> c = iterator.next(); ReferenceContainer<ReferenceType> c = this.cachecopy[this.p++];
this.latestTermHash = c.getTermHash(); this.latestTermHash = c.getTermHash();
return c.topLevelClone(); return c.topLevelClone();
} }
@ -283,14 +303,14 @@ public final class ReferenceContainerCache<ReferenceType extends Reference> exte
if (!rot) { if (!rot) {
return null; return null;
} }
iterator = cache.values().iterator(); p = 0;
ReferenceContainer<ReferenceType> c = iterator.next(); ReferenceContainer<ReferenceType> c = this.cachecopy[this.p++];
this.latestTermHash = c.getTermHash(); this.latestTermHash = c.getTermHash();
return c.topLevelClone(); return c.topLevelClone();
} }
public void remove() { public void remove() {
iterator.remove(); System.arraycopy(this.cachecopy, this.p, this.cachecopy, this.p - 1, this.cachecopy.length - p);
cache.remove(new ByteArray(this.latestTermHash)); cache.remove(new ByteArray(this.latestTermHash));
} }

11
source/de/anomic/kelondro/text/ReferenceOrder.java
Unescape View File

@ -43,11 +43,12 @@ import de.anomic.search.RankingProcess;
import de.anomic.yacy.yacyURL; import de.anomic.yacy.yacyURL;
public class ReferenceOrder { public class ReferenceOrder {
private WordReferenceVars min, max;
private final RankingProfile ranking; protected int maxdomcount;
private final ScoreCluster<String> doms; // collected for "authority" heuristic protected WordReferenceVars min, max;
private int maxdomcount; protected final ScoreCluster<String> doms; // collected for "authority" heuristic
private String language; private final RankingProfile ranking;
private String language;
public ReferenceOrder(final RankingProfile profile, String language) { public ReferenceOrder(final RankingProfile profile, String language) {
this.min = null; this.min = null;

28
source/de/anomic/kelondro/util/SetTools.java
Unescape View File

@ -38,7 +38,6 @@ import java.util.Map;
import java.util.Set; import java.util.Set;
import java.util.TreeMap; import java.util.TreeMap;
import java.util.TreeSet; import java.util.TreeSet;
import java.util.Map.Entry;
public class SetTools { public class SetTools {
@ -294,17 +293,18 @@ public class SetTools {
// ------------------------------------------------------------------------------------------------ // ------------------------------------------------------------------------------------------------
// exclude // exclude
public static <A, B> TreeMap<A, B> excludeConstructive(final TreeMap<A, B> map, final TreeSet<A> set) { /*
// comparators must be equal public static <A, B> TreeMap<A, B> excludeConstructive(final TreeMap<A, B> map, final Set<A> set) {
if (map == null) return null; if (map == null) return null;
if (set == null) return map; if (set == null) return map;
if ((map.size() == 0) || (set.size() == 0)) return map; if ((map.size() == 0) || (set.size() == 0)) return map;
if (map.comparator() != set.comparator()) return excludeConstructiveByTestMapInSet(map, set); assert !(set instanceof TreeSet) || map.comparator() == ((TreeSet<A>) set).comparator();
// if (map.comparator() != set.comparator()) return excludeConstructiveByTestMapInSet(map, set);
return excludeConstructiveByTestMapInSet(map, set); return excludeConstructiveByTestMapInSet(map, set);
// return excludeConstructiveByEnumeration(map, set); // return excludeConstructiveByEnumeration(map, set);
} }
private static <A, B> TreeMap<A, B> excludeConstructiveByTestMapInSet(final TreeMap<A, B> map, final TreeSet<A> set) { private static <A, B> TreeMap<A, B> excludeConstructiveByTestMapInSet(final TreeMap<A, B> map, final Set<A> set) {
final TreeMap<A, B> result = new TreeMap<A, B>(map.comparator()); final TreeMap<A, B> result = new TreeMap<A, B>(map.comparator());
A o; A o;
for (Entry<A, B> entry: map.entrySet()) { for (Entry<A, B> entry: map.entrySet()) {
@ -313,12 +313,13 @@ public class SetTools {
} }
return result; return result;
} }
*/
public static <A, B> void excludeDestructive(final TreeMap<A, B> map, final TreeSet<A> set) { public static <A, B> void excludeDestructive(final Map<A, B> map, final Set<A> set) {
// comparators must be equal // comparators must be equal
if (map == null) return; if (map == null) return;
if (set == null) return; if (set == null) return;
if (map.comparator() != set.comparator()) return; assert !(map instanceof TreeMap && set instanceof TreeSet) || ((TreeMap<A, B>) map).comparator() == ((TreeSet<A>) set).comparator();
if ((map.size() == 0) || (set.size() == 0)) return; if ((map.size() == 0) || (set.size() == 0)) return;
if (map.size() < set.size()) if (map.size() < set.size())
@ -327,22 +328,21 @@ public class SetTools {
excludeDestructiveByTestSetInMap(map, set); excludeDestructiveByTestSetInMap(map, set);
} }
private static <A, B> void excludeDestructiveByTestMapInSet(final TreeMap<A, B> map, final TreeSet<A> set) { private static <A, B> void excludeDestructiveByTestMapInSet(final Map<A, B> map, final Set<A> set) {
final Iterator<A> mi = map.keySet().iterator(); final Iterator<A> mi = map.keySet().iterator();
while (mi.hasNext()) if (set.contains(mi.next())) mi.remove(); while (mi.hasNext()) if (set.contains(mi.next())) mi.remove();
} }
private static <A, B> void excludeDestructiveByTestSetInMap(final TreeMap<A, B> map, final TreeSet<A> set) { private static <A, B> void excludeDestructiveByTestSetInMap(final Map<A, B> map, final Set<A> set) {
final Iterator<A> si = set.iterator(); final Iterator<A> si = set.iterator();
while (si.hasNext()) map.remove(si.next()); while (si.hasNext()) map.remove(si.next());
} }
// and the same again with set-set // and the same again with set-set
public static <A> void excludeDestructive(final TreeSet<A> set1, final TreeSet<A> set2) { public static <A> void excludeDestructive(final Set<A> set1, final Set<A> set2) {
// comparators must be equal
if (set1 == null) return; if (set1 == null) return;
if (set2 == null) return; if (set2 == null) return;
if (set1.comparator() != set2.comparator()) return; assert !(set1 instanceof TreeSet && set2 instanceof TreeSet) || ((TreeSet<A>) set1).comparator() == ((TreeSet<A>) set2).comparator();
if ((set1.size() == 0) || (set2.size() == 0)) return; if ((set1.size() == 0) || (set2.size() == 0)) return;
if (set1.size() < set2.size()) if (set1.size() < set2.size())
@ -351,12 +351,12 @@ public class SetTools {
excludeDestructiveByTestLargeInSmall(set1, set2); excludeDestructiveByTestLargeInSmall(set1, set2);
} }
private static <A> void excludeDestructiveByTestSmallInLarge(final TreeSet<A> small, final TreeSet<A> large) { private static <A> void excludeDestructiveByTestSmallInLarge(final Set<A> small, final Set<A> large) {
final Iterator<A> mi = small.iterator(); final Iterator<A> mi = small.iterator();
while (mi.hasNext()) if (large.contains(mi.next())) mi.remove(); while (mi.hasNext()) if (large.contains(mi.next())) mi.remove();
} }
private static <A> void excludeDestructiveByTestLargeInSmall(final TreeSet<A> large, final TreeSet<A> small) { private static <A> void excludeDestructiveByTestLargeInSmall(final Set<A> large, final Set<A> small) {
final Iterator<A> si = small.iterator(); final Iterator<A> si = small.iterator();
while (si.hasNext()) large.remove(si.next()); while (si.hasNext()) large.remove(si.next());
} }

2
source/de/anomic/search/Switchboard.java
Unescape View File

@ -1208,6 +1208,8 @@ public final class Switchboard extends serverAbstractSwitch implements serverSwi
Response response; Response response;
while ((surrogate = reader.take()) != DCEntry.poison) { while ((surrogate = reader.take()) != DCEntry.poison) {
// check if url is in accepted domain // check if url is in accepted domain
assert surrogate != null;
assert crawlStacker != null;
final String urlRejectReason = crawlStacker.urlInAcceptedDomain(surrogate.url()); final String urlRejectReason = crawlStacker.urlInAcceptedDomain(surrogate.url());
if (urlRejectReason != null) { if (urlRejectReason != null) {
if (this.log.isFine()) this.log.logInfo("Rejected URL '" + surrogate.url() + "': " + urlRejectReason); if (this.log.isFine()) this.log.logInfo("Rejected URL '" + surrogate.url() + "': " + urlRejectReason);

30
source/de/anomic/server/serverProfiling.java
Unescape View File

@ -26,16 +26,16 @@
package de.anomic.server; package de.anomic.server;
import java.util.ArrayList;
import java.util.Iterator; import java.util.Iterator;
import java.util.Map; import java.util.Map;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentLinkedQueue;
import de.anomic.kelondro.util.MemoryControl; import de.anomic.kelondro.util.MemoryControl;
public class serverProfiling extends Thread { public class serverProfiling extends Thread {
private static final Map<String, ConcurrentLinkedQueue<Event>> historyMaps = new ConcurrentHashMap<String, ConcurrentLinkedQueue<Event>>(); private static final Map<String, ArrayList<Event>> historyMaps = new ConcurrentHashMap<String, ArrayList<Event>>();
private static final Map<String, Long> eventAccess = new ConcurrentHashMap<String, Long>(); // value: last time when this was accessed private static final Map<String, Long> eventAccess = new ConcurrentHashMap<String, Long>(); // value: last time when this was accessed
private static serverProfiling systemProfiler = null; private static serverProfiling systemProfiler = null;
@ -84,26 +84,26 @@ public class serverProfiling extends Thread {
return; // protect against too heavy load return; // protect against too heavy load
} }
} }
ConcurrentLinkedQueue<Event> history = historyMaps.get(eventName); ArrayList<Event> history = historyMaps.get(eventName);
if (history != null) { if (history != null) synchronized (history) {
// update entry // update entry
history.add(new Event(eventPayload)); history.add(new Event(eventPayload));
// clean up too old entries // clean up too old entries
int tp = history.size() - 30000; int tp = history.size() - 30000;
while (tp-- > 0) history.poll(); while (tp-- > 0) history.remove(0);
if (history.size() % 10 == 0) { // reduce number of System.currentTimeMillis() calls if (history.size() % 10 == 0) { // reduce number of System.currentTimeMillis() calls
Event e; Event e;
final long now = System.currentTimeMillis(); final long now = System.currentTimeMillis();
while (history.size() > 0) { while (history.size() > 0) {
e = history.peek(); e = history.get(0);
if (now - e.time < 600000) break; if (now - e.time < 600000) break;
history.poll(); history.remove(0);
} }
} }
} else { } else {
history = new ConcurrentLinkedQueue<Event>(); history = new ArrayList<Event>(100);
// update entry // update entry
history.add(new Event(eventPayload)); history.add(new Event(eventPayload));
@ -113,16 +113,20 @@ public class serverProfiling extends Thread {
} }
} }
public static Iterator<Event> history(final String eventName) { public static ArrayList<Event> history(final String eventName) {
return (historyMaps.containsKey(eventName) ? (historyMaps.get(eventName)) : new ConcurrentLinkedQueue<Event>()).iterator(); return historyMaps.get(eventName);
} }
public static int countEvents(final String eventName, long time) { public static int countEvents(final String eventName, long time) {
Iterator<Event> i = history(eventName); ArrayList<Event> event = history(eventName);
if (event == null) return 0;
long now = System.currentTimeMillis(); long now = System.currentTimeMillis();
int count = 0; int count = 0;
while (i.hasNext()) { synchronized (event) {
if (now - i.next().time < time) count++; Iterator<Event> i = event.iterator();
while (i.hasNext()) {
if (now - i.next().time < time) count++;
}
} }
return count; return count;
} }

37
source/de/anomic/ymage/ProfilingGraph.java
Unescape View File

@ -26,8 +26,8 @@
package de.anomic.ymage; package de.anomic.ymage;
import java.util.ArrayList;
import java.util.ConcurrentModificationException; import java.util.ConcurrentModificationException;
import java.util.Iterator;
import de.anomic.server.serverProfiling; import de.anomic.server.serverProfiling;
import de.anomic.server.serverProfiling.Event; import de.anomic.server.serverProfiling.Event;
@ -37,13 +37,14 @@ public class ProfilingGraph {
private static ymageChart bufferChart = null; private static ymageChart bufferChart = null;
public static long maxPayload(final String eventname, final long min) { public static long maxPayload(final String eventname, final long min) {
final Iterator<Event> i = serverProfiling.history(eventname); final ArrayList<Event> list = serverProfiling.history(eventname);
serverProfiling.Event event; if (list == null) return min;
long max = min, l; long max = min, l;
while (i.hasNext()) { synchronized (list) {
event = i.next(); for (serverProfiling.Event event: list) {
l = ((Long) event.payload).longValue(); l = ((Long) event.payload).longValue();
if (l > max) max = l; if (l > max) max = l;
}
} }
return max; return max;
} }
@ -79,7 +80,6 @@ public class ProfilingGraph {
final long now = System.currentTimeMillis(); final long now = System.currentTimeMillis();
long bytes; long bytes;
int x0, x1, y0, y1, ppm, words; int x0, x1, y0, y1, ppm, words;
serverProfiling.Event event;
try { try {
// draw urls // draw urls
/* /*
@ -98,10 +98,9 @@ public class ProfilingGraph {
} }
*/ */
// draw memory // draw memory
Iterator<Event> i = serverProfiling.history("memory"); ArrayList<Event> events = serverProfiling.history("memory");
x0 = 1; y0 = 0; x0 = 1; y0 = 0;
while (i.hasNext()) { if (events != null) synchronized (events) {for (serverProfiling.Event event: events) {
event = i.next();
time = event.time - now; time = event.time - now;
bytes = ((Long) event.payload).longValue(); bytes = ((Long) event.payload).longValue();
x1 = (int) (time/1000); x1 = (int) (time/1000);
@ -111,13 +110,12 @@ public class ProfilingGraph {
chart.setColor("0000FF"); chart.setColor("0000FF");
if (x0 < 0) chart.chartLine(ymageChart.DIMENSION_BOTTOM, ymageChart.DIMENSION_RIGHT, x0, y0, x1, y1); if (x0 < 0) chart.chartLine(ymageChart.DIMENSION_BOTTOM, ymageChart.DIMENSION_RIGHT, x0, y0, x1, y1);
x0 = x1; y0 = y1; x0 = x1; y0 = y1;
} }}
// draw wordcache // draw wordcache
i = serverProfiling.history("wordcache"); events = serverProfiling.history("wordcache");
x0 = 1; y0 = 0; x0 = 1; y0 = 0;
while (i.hasNext()) { if (events != null) synchronized (events) {for (serverProfiling.Event event: events) {
event = i.next();
time = event.time - now; time = event.time - now;
words = (int) ((Long) event.payload).longValue(); words = (int) ((Long) event.payload).longValue();
x1 = (int) (time/1000); x1 = (int) (time/1000);
@ -127,13 +125,12 @@ public class ProfilingGraph {
chart.setColor("008800"); chart.setColor("008800");
if (x0 < 0) chart.chartLine(ymageChart.DIMENSION_BOTTOM, ymageChart.DIMENSION_LEFT, x0, y0, x1, y1); if (x0 < 0) chart.chartLine(ymageChart.DIMENSION_BOTTOM, ymageChart.DIMENSION_LEFT, x0, y0, x1, y1);
x0 = x1; y0 = y1; x0 = x1; y0 = y1;
} }}
// draw ppm // draw ppm
i = serverProfiling.history("ppm"); events = serverProfiling.history("ppm");
x0 = 1; y0 = 0; x0 = 1; y0 = 0;
while (i.hasNext()) { if (events != null) synchronized (events) {for (serverProfiling.Event event: events) {
event = i.next();
time = event.time - now; time = event.time - now;
ppm = (int) ((Long) event.payload).longValue(); ppm = (int) ((Long) event.payload).longValue();
x1 = (int) (time/1000); x1 = (int) (time/1000);
@ -143,7 +140,7 @@ public class ProfilingGraph {
chart.setColor("AA2222"); chart.setColor("AA2222");
chart.chartDot(ymageChart.DIMENSION_BOTTOM, ymageChart.DIMENSION_ANOT0, x1, y1, 2, ppm + " PPM", 0); chart.chartDot(ymageChart.DIMENSION_BOTTOM, ymageChart.DIMENSION_ANOT0, x1, y1, 2, ppm + " PPM", 0);
x0 = x1; y0 = y1; x0 = x1; y0 = y1;
} }}
bufferChart = chart; bufferChart = chart;
} catch (final ConcurrentModificationException cme) { } catch (final ConcurrentModificationException cme) {

Write Preview
Loading…
Cancel
Save