boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

more memory leak fixing hacks

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5599 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 16 years ago
parent 8aafbe234b
commit 411f2212f2
20 changed files with 102 additions and 94 deletions
Show Stats Download Patch File Download Diff File

9
source/de/anomic/crawler/ImporterManager.java
Unescape View File

@ -6,11 +6,14 @@ import de.anomic.kelondro.util.Log;
public class ImporterManager { public class ImporterManager {
public final Vector<Importer> finishedJobs = new Vector<Importer>(); public final Vector<Importer> finishedJobs;
public final ThreadGroup runningJobs = new ThreadGroup("ImporterThreads"); public final ThreadGroup runningJobs;
public int currMaxJobNr = 0; public int currMaxJobNr;
public ImporterManager() { public ImporterManager() {
this.finishedJobs = new Vector<Importer>();
this.runningJobs = new ThreadGroup("ImporterThreads");
this.currMaxJobNr = 0;
} }
public int generateUniqueJobID() { public int generateUniqueJobID() {

22
source/de/anomic/crawler/ResourceObserver.java
Unescape View File

@ -35,7 +35,7 @@ import de.anomic.tools.diskUsage;
public final class ResourceObserver { public final class ResourceObserver {
// Unknown for now // Unknown for now
//private final static long MIN_FREE_MEMORY = 0; //private static final static long MIN_FREE_MEMORY = 0;
// We are called with the cleanup job every five minutes; // We are called with the cleanup job every five minutes;
// the disk usage should be checked with every run // the disk usage should be checked with every run
private static final int CHECK_DISK_USAGE_FREQ = 1; private static final int CHECK_DISK_USAGE_FREQ = 1;
@ -47,7 +47,7 @@ public final class ResourceObserver {
private static final int MEDIUM = 1; private static final int MEDIUM = 1;
private static final int HIGH = 2; private static final int HIGH = 2;
private final Log log = new Log("RESOURCE OBSERVER"); private static final Log log = new Log("RESOURCE OBSERVER");
private final plasmaSwitchboard sb; private final plasmaSwitchboard sb;
private int checkDiskUsageCount; private int checkDiskUsageCount;
@ -62,7 +62,7 @@ public final class ResourceObserver {
*/ */
public ResourceObserver(final plasmaSwitchboard sb) { public ResourceObserver(final plasmaSwitchboard sb) {
this.sb = sb; this.sb = sb;
this.log.logInfo("initializing the resource observer"); log.logInfo("initializing the resource observer");
final ArrayList<String> pathsToCheck = new ArrayList<String>(); final ArrayList<String> pathsToCheck = new ArrayList<String>();
// FIXME whats about the secondary path??? // FIXME whats about the secondary path???
@ -84,7 +84,7 @@ public final class ResourceObserver {
diskUsage.init(pathsToCheck); diskUsage.init(pathsToCheck);
if (!diskUsage.isUsable()) if (!diskUsage.isUsable())
this.log.logWarning("Disk usage returned: " + diskUsage.getErrorMessage()); log.logWarning("Disk usage returned: " + diskUsage.getErrorMessage());
checkDiskUsageCount = 0; checkDiskUsageCount = 0;
checkMemoryUsageCount = 0; checkMemoryUsageCount = 0;
@ -113,24 +113,24 @@ public final class ResourceObserver {
if (tmpDisksFree < HIGH || tmpMemoryFree < HIGH) { if (tmpDisksFree < HIGH || tmpMemoryFree < HIGH) {
if (!sb.crawlJobIsPaused(plasmaSwitchboardConstants.CRAWLJOB_LOCAL_CRAWL)) { if (!sb.crawlJobIsPaused(plasmaSwitchboardConstants.CRAWLJOB_LOCAL_CRAWL)) {
this.log.logInfo("pausing local crawls"); log.logInfo("pausing local crawls");
sb.pauseCrawlJob(plasmaSwitchboardConstants.CRAWLJOB_LOCAL_CRAWL); sb.pauseCrawlJob(plasmaSwitchboardConstants.CRAWLJOB_LOCAL_CRAWL);
} }
if (!sb.crawlJobIsPaused(plasmaSwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL)) { if (!sb.crawlJobIsPaused(plasmaSwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL)) {
this.log.logInfo("pausing remote triggered crawls"); log.logInfo("pausing remote triggered crawls");
sb.pauseCrawlJob(plasmaSwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL); sb.pauseCrawlJob(plasmaSwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL);
} }
if (tmpDisksFree == LOW && sb.getConfigBool(plasmaSwitchboardConstants.INDEX_RECEIVE_ALLOW, false)) { if (tmpDisksFree == LOW && sb.getConfigBool(plasmaSwitchboardConstants.INDEX_RECEIVE_ALLOW, false)) {
this.log.logInfo("disabling index receive"); log.logInfo("disabling index receive");
sb.setConfig(plasmaSwitchboardConstants.INDEX_RECEIVE_ALLOW, false); sb.setConfig(plasmaSwitchboardConstants.INDEX_RECEIVE_ALLOW, false);
sb.webIndex.seedDB.mySeed().setFlagAcceptRemoteIndex(false); sb.webIndex.seedDB.mySeed().setFlagAcceptRemoteIndex(false);
} }
} }
else { else {
if (diskUsage.isUsable()) if (diskUsage.isUsable())
this.log.logInfo("run completed; everything in order"); log.logInfo("run completed; everything in order");
else else
this.log.logInfo("The observer is out of order: " + diskUsage.getErrorMessage()); log.logInfo("The observer is out of order: " + diskUsage.getErrorMessage());
} }
} }
@ -173,9 +173,9 @@ public final class ResourceObserver {
long[] val; long[] val;
for (final Map.Entry<String, long[]> entry: usage.entrySet()) { for (final Map.Entry<String, long[]> entry: usage.entrySet()) {
val = entry.getValue(); val = entry.getValue();
this.log.logInfo("df of Volume " + entry.getKey() + ": " + (val[1] / 1024 / 1024) + " MB"); log.logInfo("df of Volume " + entry.getKey() + ": " + (val[1] / 1024 / 1024) + " MB");
if (val[1] < getMinFreeDiskSpace()) { if (val[1] < getMinFreeDiskSpace()) {
this.log.logWarning("Volume " + entry.getKey() + ": free space (" + (val[1] / 1024 / 1024) + " MB) is too low (< " + (getMinFreeDiskSpace() / 1024 / 1024) + " MB)"); log.logWarning("Volume " + entry.getKey() + ": free space (" + (val[1] / 1024 / 1024) + " MB) is too low (< " + (getMinFreeDiskSpace() / 1024 / 1024) + " MB)");
ret = MEDIUM; ret = MEDIUM;
} }
if (val[1] < Math.min(getMinFreeDiskSpace() / 5L, 100L)) { if (val[1] < Math.min(getMinFreeDiskSpace() / 5L, 100L)) {

20
source/de/anomic/crawler/ZURL.java
Unescape View File

@ -57,8 +57,8 @@ public class ZURL {
0); 0);
// the class object // the class object
ObjectIndex urlIndex = null; private final ObjectIndex urlIndex;
private final LinkedList<String> stack = new LinkedList<String>(); // strings: url private final LinkedList<String> stack;
public ZURL(final File cachePath, final String tablename, final boolean startWithEmptyFile) { public ZURL(final File cachePath, final String tablename, final boolean startWithEmptyFile) {
// creates a new ZURL in a file // creates a new ZURL in a file
@ -69,13 +69,15 @@ public class ZURL {
if (f.isDirectory()) FlexWidthArray.delete(cachePath, tablename); else f.delete(); if (f.isDirectory()) FlexWidthArray.delete(cachePath, tablename); else f.delete();
} }
} }
urlIndex = new EcoTable(f, rowdef, EcoTable.tailCacheDenyUsage, EcoFSBufferSize, 0); this.urlIndex = new EcoTable(f, rowdef, EcoTable.tailCacheDenyUsage, EcoFSBufferSize, 0);
//urlIndex = new kelondroFlexTable(cachePath, tablename, -1, rowdef, 0, true); //urlIndex = new kelondroFlexTable(cachePath, tablename, -1, rowdef, 0, true);
this.stack = new LinkedList<String>();
} }
public ZURL() { public ZURL() {
// creates a new ZUR in RAM // creates a new ZUR in RAM
urlIndex = new RowSet(rowdef, 0); this.urlIndex = new RowSet(rowdef, 0);
this.stack = new LinkedList<String>();
} }
public int size() { public int size() {
@ -83,15 +85,13 @@ public class ZURL {
} }
public void clear() throws IOException { public void clear() throws IOException {
urlIndex.clear(); if (urlIndex != null) urlIndex.clear();
stack.clear(); if (stack != null) stack.clear();
} }
public void close() { public void close() {
if (urlIndex != null) { try {this.clear();} catch (IOException e) {}
urlIndex.close(); if (urlIndex != null) urlIndex.close();
urlIndex = null;
}
} }
public synchronized Entry newEntry( public synchronized Entry newEntry(

16
source/de/anomic/data/SitemapParser.java
Unescape View File

@ -114,7 +114,7 @@ public class SitemapParser extends DefaultHandler {
/** /**
* the logger * the logger
*/ */
private final Log logger = new Log("SITEMAP"); private static final Log logger = new Log("SITEMAP");
/** /**
* The location of the sitemap file * The location of the sitemap file
@ -158,7 +158,7 @@ public class SitemapParser extends DefaultHandler {
try { try {
res = client.GET(siteMapURL.toString()); res = client.GET(siteMapURL.toString());
if (res.getStatusCode() != 200) { if (res.getStatusCode() != 200) {
this.logger.logWarning("Unable to download the sitemap file " + this.siteMapURL + logger.logWarning("Unable to download the sitemap file " + this.siteMapURL +
"\nServer returned status: " + res.getStatusLine()); "\nServer returned status: " + res.getStatusLine());
return; return;
} }
@ -171,13 +171,13 @@ public class SitemapParser extends DefaultHandler {
InputStream contentStream = res.getDataAsStream(); InputStream contentStream = res.getDataAsStream();
if ((contentMimeType != null) && if ((contentMimeType != null) &&
(contentMimeType.equals("application/x-gzip") || contentMimeType.equals("application/gzip"))) { (contentMimeType.equals("application/x-gzip") || contentMimeType.equals("application/gzip"))) {
if (this.logger.isFine()) this.logger.logFine("Sitemap file has mimetype " + contentMimeType); if (logger.isFine()) logger.logFine("Sitemap file has mimetype " + contentMimeType);
contentStream = new GZIPInputStream(contentStream); contentStream = new GZIPInputStream(contentStream);
} }
final httpdByteCountInputStream counterStream = new httpdByteCountInputStream(contentStream, null); final httpdByteCountInputStream counterStream = new httpdByteCountInputStream(contentStream, null);
// parse it // parse it
this.logger.logInfo("Start parsing sitemap file " + this.siteMapURL + "\n\tMimeType: " + contentMimeType + logger.logInfo("Start parsing sitemap file " + this.siteMapURL + "\n\tMimeType: " + contentMimeType +
"\n\tLength: " + this.contentLength); "\n\tLength: " + this.contentLength);
final SAXParser saxParser = SAXParserFactory.newInstance().newSAXParser(); final SAXParser saxParser = SAXParserFactory.newInstance().newSAXParser();
saxParser.parse(counterStream, this); saxParser.parse(counterStream, this);
@ -186,7 +186,7 @@ public class SitemapParser extends DefaultHandler {
res.closeStream(); res.closeStream();
} }
} catch (final Exception e) { } catch (final Exception e) {
this.logger.logWarning("Unable to parse sitemap file " + this.siteMapURL, e); logger.logWarning("Unable to parse sitemap file " + this.siteMapURL, e);
} finally { } finally {
if (res != null) { if (res != null) {
// release connection // release connection
@ -283,7 +283,7 @@ public class SitemapParser extends DefaultHandler {
0, 0,
0 0
)); ));
this.logger.logInfo("New URL '" + this.nextURL + "' added for crawling."); logger.logInfo("New URL '" + this.nextURL + "' added for crawling.");
this.urlCounter++; this.urlCounter++;
} }
} }
@ -293,7 +293,7 @@ public class SitemapParser extends DefaultHandler {
// TODO: we need to decode the URL here // TODO: we need to decode the URL here
this.nextURL = (new String(buf, offset, len)).trim(); this.nextURL = (new String(buf, offset, len)).trim();
if (!this.nextURL.startsWith("http") && !this.nextURL.startsWith("https")) { if (!this.nextURL.startsWith("http") && !this.nextURL.startsWith("https")) {
this.logger.logInfo("The url '" + this.nextURL + "' has a wrong format. Ignore it."); logger.logInfo("The url '" + this.nextURL + "' has a wrong format. Ignore it.");
this.nextURL = null; this.nextURL = null;
} }
} else if (this.currentElement.equalsIgnoreCase(SITEMAP_URL_LASTMOD)) { } else if (this.currentElement.equalsIgnoreCase(SITEMAP_URL_LASTMOD)) {
@ -301,7 +301,7 @@ public class SitemapParser extends DefaultHandler {
try { try {
this.lastMod = DateFormatter.parseISO8601(dateStr); this.lastMod = DateFormatter.parseISO8601(dateStr);
} catch (final ParseException e) { } catch (final ParseException e) {
this.logger.logInfo("Unable to parse datestring '" + dateStr + "'"); logger.logInfo("Unable to parse datestring '" + dateStr + "'");
} }
} }
} }

4
source/de/anomic/data/diff.java
Unescape View File

@ -38,8 +38,8 @@ import de.anomic.htmlFilter.htmlFilterCharacterCoding;
public class diff { public class diff {
private final ArrayList <Part> parts = new ArrayList<Part>(); private final ArrayList <Part> parts = new ArrayList<Part>();
final Object[] o; private final Object[] o;
final Object[] n; private final Object[] n;
/** /**
* @param o the original <code>String</code> * @param o the original <code>String</code>

7
source/de/anomic/htmlFilter/htmlFilterContentScraper.java
Unescape View File

@ -85,7 +85,7 @@ public class htmlFilterContentScraper extends htmlFilterAbstractScraper implemen
//private String headline; //private String headline;
private List<String>[] headlines; private List<String>[] headlines;
private serverCharBuffer content; private serverCharBuffer content;
private final EventListenerList htmlFilterEventListeners = new EventListenerList(); private final EventListenerList htmlFilterEventListeners;
/** /**
* {@link yacyURL} to the favicon that belongs to the document * {@link yacyURL} to the favicon that belongs to the document
@ -110,6 +110,7 @@ public class htmlFilterContentScraper extends htmlFilterAbstractScraper implemen
this.headlines = new ArrayList[4]; this.headlines = new ArrayList[4];
for (int i = 0; i < 4; i++) headlines[i] = new ArrayList<String>(); for (int i = 0; i < 4; i++) headlines[i] = new ArrayList<String>();
this.content = new serverCharBuffer(1024); this.content = new serverCharBuffer(1024);
this.htmlFilterEventListeners = new EventListenerList();
} }
public final static boolean punctuation(final char c) { public final static boolean punctuation(final char c) {
@ -463,7 +464,7 @@ public class htmlFilterContentScraper extends htmlFilterAbstractScraper implemen
} }
} }
void fireScrapeTag0(final String tagname, final Properties tagopts) { private void fireScrapeTag0(final String tagname, final Properties tagopts) {
final Object[] listeners = this.htmlFilterEventListeners.getListenerList(); final Object[] listeners = this.htmlFilterEventListeners.getListenerList();
for (int i=0; i<listeners.length; i+=2) { for (int i=0; i<listeners.length; i+=2) {
if (listeners[i]==htmlFilterEventListener.class) { if (listeners[i]==htmlFilterEventListener.class) {
@ -472,7 +473,7 @@ public class htmlFilterContentScraper extends htmlFilterAbstractScraper implemen
} }
} }
void fireScrapeTag1(final String tagname, final Properties tagopts, final char[] text) { private void fireScrapeTag1(final String tagname, final Properties tagopts, final char[] text) {
final Object[] listeners = this.htmlFilterEventListeners.getListenerList(); final Object[] listeners = this.htmlFilterEventListeners.getListenerList();
for (int i=0; i<listeners.length; i+=2) { for (int i=0; i<listeners.length; i+=2) {
if (listeners[i]==htmlFilterEventListener.class) { if (listeners[i]==htmlFilterEventListener.class) {

2
source/de/anomic/htmlFilter/htmlFilterInputStream.java
Unescape View File

@ -66,7 +66,7 @@ public class htmlFilterInputStream extends InputStream implements htmlFilterEven
this.bufferedIn.mark((int) preBufferSize); this.bufferedIn.mark((int) preBufferSize);
final htmlFilterContentScraper scraper = new htmlFilterContentScraper(rooturl); final htmlFilterContentScraper scraper = new htmlFilterContentScraper(rooturl);
scraper.registerHtmlFilterEventListener(this); //scraper.registerHtmlFilterEventListener(this);
this.reader = new InputStreamReader(this,inputStreamCharset); this.reader = new InputStreamReader(this,inputStreamCharset);
this.writer = new htmlFilterWriter(null,null,scraper,transformer,passbyIfBinarySuspect); this.writer = new htmlFilterWriter(null,null,scraper,transformer,passbyIfBinarySuspect);

8
source/de/anomic/icap/icapd.java
Unescape View File

@ -69,7 +69,7 @@ public class icapd implements serverHandler, Cloneable {
private int keepAliveRequestCount = 0; private int keepAliveRequestCount = 0;
// needed for logging // needed for logging
private final Log log = new Log("ICAPD"); private static final Log log = new Log("ICAPD");
private static plasmaSwitchboard sb = null; private static plasmaSwitchboard sb = null;
private static String virtualHost = null; private static String virtualHost = null;
@ -311,7 +311,7 @@ public class icapd implements serverHandler, Cloneable {
httpRequestHeader.parseRequestLine(httpRequestLine,httpReqProps,virtualHost); httpRequestHeader.parseRequestLine(httpRequestLine,httpReqProps,virtualHost);
if (!httpReqProps.getProperty(httpRequestHeader.CONNECTION_PROP_METHOD).equals(httpRequestHeader.METHOD_GET)) { if (!httpReqProps.getProperty(httpRequestHeader.CONNECTION_PROP_METHOD).equals(httpRequestHeader.METHOD_GET)) {
this.log.logInfo("Wrong http request method for indexing:" + log.logInfo("Wrong http request method for indexing:" +
"\nRequest Method: " + httpReqProps.getProperty(httpRequestHeader.CONNECTION_PROP_METHOD) + "\nRequest Method: " + httpReqProps.getProperty(httpRequestHeader.CONNECTION_PROP_METHOD) +
"\nRequest Line: " + httpRequestLine); "\nRequest Line: " + httpRequestLine);
reader.close(); reader.close();
@ -345,7 +345,7 @@ public class icapd implements serverHandler, Cloneable {
final Object[] httpRespStatus = httpResponseHeader.parseResponseLine(httpRespStatusLine); final Object[] httpRespStatus = httpResponseHeader.parseResponseLine(httpRespStatusLine);
if (!(httpRespStatus[1].equals(Integer.valueOf(200)) || httpRespStatus[1].equals(Integer.valueOf(203)))) { if (!(httpRespStatus[1].equals(Integer.valueOf(200)) || httpRespStatus[1].equals(Integer.valueOf(203)))) {
this.log.logInfo("Wrong status code for indexing:" + log.logInfo("Wrong status code for indexing:" +
"\nStatus Code: " + httpRespStatus[1] + "\nStatus Code: " + httpRespStatus[1] +
"\nRequest Line: " + httpRequestLine + "\nRequest Line: " + httpRequestLine +
"\nResponse Line: " + httpRespStatusLine); "\nResponse Line: " + httpRespStatusLine);
@ -365,7 +365,7 @@ public class icapd implements serverHandler, Cloneable {
} }
if (!plasmaParser.supportedContent(plasmaParser.PARSER_MODE_ICAP, httpRequestURL, httpResHeader.mime())) { if (!plasmaParser.supportedContent(plasmaParser.PARSER_MODE_ICAP, httpRequestURL, httpResHeader.mime())) {
this.log.logInfo("Wrong mimeType or fileExtension for indexing:" + log.logInfo("Wrong mimeType or fileExtension for indexing:" +
"\nMimeType: " + httpResHeader.mime() + "\nMimeType: " + httpResHeader.mime() +
"\nRequest Line:" + httpRequestLine); "\nRequest Line:" + httpRequestLine);
return ; return ;

2
source/de/anomic/kelondro/index/BinSearch.java
Unescape View File

@ -35,7 +35,7 @@ public class BinSearch {
private final byte[] chunks; private final byte[] chunks;
private final int chunksize; private final int chunksize;
private final int count; private final int count;
private final ByteOrder objectOrder = new NaturalOrder(true); private static final ByteOrder objectOrder = new NaturalOrder(true);
public BinSearch(final byte[] chunks, final int chunksize) { public BinSearch(final byte[] chunks, final int chunksize) {
this.chunks = chunks; this.chunks = chunks;

6
source/de/anomic/kelondro/index/BytesIntMap.java
Unescape View File

@ -143,6 +143,7 @@ public class BytesIntMap {
this.l = l; this.l = l;
} }
} }
private static final entry poisonEntry = new entry(new byte[0], 0);
/** /**
* this method creates a concurrent thread that can take entries that are used to initialize the map * this method creates a concurrent thread that can take entries that are used to initialize the map
@ -165,7 +166,6 @@ public class BytesIntMap {
public static class initDataConsumer implements Callable<BytesIntMap> { public static class initDataConsumer implements Callable<BytesIntMap> {
private BlockingQueue<entry> cache; private BlockingQueue<entry> cache;
private final entry poison = new entry(new byte[0], 0);
private BytesIntMap map; private BytesIntMap map;
private Future<BytesIntMap> result; private Future<BytesIntMap> result;
private boolean sortAtEnd; private boolean sortAtEnd;
@ -200,7 +200,7 @@ public class BytesIntMap {
public void finish(boolean sortAtEnd) { public void finish(boolean sortAtEnd) {
this.sortAtEnd = sortAtEnd; this.sortAtEnd = sortAtEnd;
try { try {
cache.put(poison); cache.put(poisonEntry);
} catch (InterruptedException e) { } catch (InterruptedException e) {
e.printStackTrace(); e.printStackTrace();
} }
@ -221,7 +221,7 @@ public class BytesIntMap {
public BytesIntMap call() throws IOException { public BytesIntMap call() throws IOException {
try { try {
entry c; entry c;
while ((c = cache.take()) != poison) { while ((c = cache.take()) != poisonEntry) {
map.addi(c.key, c.l); map.addi(c.key, c.l);
} }
} catch (InterruptedException e) { } catch (InterruptedException e) {

6
source/de/anomic/kelondro/index/BytesLongMap.java
Unescape View File

@ -214,11 +214,11 @@ public class BytesLongMap {
this.l = l; this.l = l;
} }
} }
private static final entry poisonEntry = new entry(new byte[0], 0);
public static class initDataConsumer implements Callable<BytesLongMap> { public static class initDataConsumer implements Callable<BytesLongMap> {
private BlockingQueue<entry> cache; private BlockingQueue<entry> cache;
private final entry poison = new entry(new byte[0], 0);
private BytesLongMap map; private BytesLongMap map;
private Future<BytesLongMap> result; private Future<BytesLongMap> result;
@ -250,7 +250,7 @@ public class BytesLongMap {
*/ */
public void finish() { public void finish() {
try { try {
cache.put(poison); cache.put(poisonEntry);
} catch (InterruptedException e) { } catch (InterruptedException e) {
e.printStackTrace(); e.printStackTrace();
} }
@ -271,7 +271,7 @@ public class BytesLongMap {
public BytesLongMap call() throws IOException { public BytesLongMap call() throws IOException {
try { try {
entry c; entry c;
while ((c = cache.take()) != poison) { while ((c = cache.take()) != poisonEntry) {
map.addl(c.key, c.l); map.addl(c.key, c.l);
} }
} catch (InterruptedException e) { } catch (InterruptedException e) {

4
source/de/anomic/kelondro/table/SQLTable.java
Unescape View File

@ -67,7 +67,7 @@ public class SQLTable implements ObjectIndex {
private static final String db_pwd_str = "yacy"; private static final String db_pwd_str = "yacy";
private Connection theDBConnection = null; private Connection theDBConnection = null;
private final ByteOrder order = new NaturalOrder(true); private static final ByteOrder order = new NaturalOrder(true);
private final Row rowdef; private final Row rowdef;
public SQLTable(final String dbType, final Row rowdef) throws Exception { public SQLTable(final String dbType, final Row rowdef) throws Exception {
@ -274,7 +274,7 @@ public class SQLTable implements ObjectIndex {
} }
public ByteOrder order() { public ByteOrder order() {
return this.order; return order;
} }
public int primarykey() { public int primarykey() {

42
source/de/anomic/plasma/plasmaParser.java
Unescape View File

@ -173,10 +173,10 @@ public final class plasmaParser {
loadAvailableParserList(); loadAvailableParserList();
} }
private final Log theLogger = new Log("PARSER"); private static final Log theLogger = new Log("PARSER");
public Log getLogger() { public Log getLogger() {
return this.theLogger; return theLogger;
} }
public static HashMap<String, plasmaParserConfig> getParserConfigList() { public static HashMap<String, plasmaParserConfig> getParserConfigList() {
@ -539,13 +539,13 @@ public final class plasmaParser {
throws InterruptedException, ParserException { throws InterruptedException, ParserException {
ByteArrayInputStream byteIn = null; ByteArrayInputStream byteIn = null;
try { try {
if (this.theLogger.isFine()) if (theLogger.isFine())
this.theLogger.logFine("Parsing '" + location + "' from byte-array"); theLogger.logFine("Parsing '" + location + "' from byte-array");
// testing if the resource is not empty // testing if the resource is not empty
if (sourceArray == null || sourceArray.length == 0) { if (sourceArray == null || sourceArray.length == 0) {
final String errorMsg = "No resource content available (1) " + ((sourceArray == null) ? "source == null" : "source.length() == 0"); final String errorMsg = "No resource content available (1) " + ((sourceArray == null) ? "source == null" : "source.length() == 0");
this.theLogger.logInfo("Unable to parse '" + location + "'. " + errorMsg); theLogger.logInfo("Unable to parse '" + location + "'. " + errorMsg);
throw new ParserException(errorMsg,location, errorMsg); throw new ParserException(errorMsg,location, errorMsg);
} }
@ -561,7 +561,7 @@ public final class plasmaParser {
if (e instanceof ParserException) throw (ParserException) e; if (e instanceof ParserException) throw (ParserException) e;
// log unexpected error // log unexpected error
this.theLogger.logSevere("Unexpected exception in parseSource from byte-array: " + e.getMessage(), e); theLogger.logSevere("Unexpected exception in parseSource from byte-array: " + e.getMessage(), e);
throw new ParserException("Unexpected exception while parsing " + location,location, e); throw new ParserException("Unexpected exception while parsing " + location,location, e);
} finally { } finally {
if (byteIn != null) try { byteIn.close(); } catch (final Exception ex){/* ignore this */} if (byteIn != null) try { byteIn.close(); } catch (final Exception ex){/* ignore this */}
@ -573,13 +573,13 @@ public final class plasmaParser {
BufferedInputStream sourceStream = null; BufferedInputStream sourceStream = null;
try { try {
if (this.theLogger.isFine()) if (theLogger.isFine())
this.theLogger.logFine("Parsing '" + location + "' from file"); theLogger.logFine("Parsing '" + location + "' from file");
// testing if the resource is not empty // testing if the resource is not empty
if (!(sourceFile.exists() && sourceFile.canRead() && sourceFile.length() > 0)) { if (!(sourceFile.exists() && sourceFile.canRead() && sourceFile.length() > 0)) {
final String errorMsg = sourceFile.exists() ? "Empty resource file." : "No resource content available (2)."; final String errorMsg = sourceFile.exists() ? "Empty resource file." : "No resource content available (2).";
this.theLogger.logInfo("Unable to parse '" + location + "'. " + errorMsg); theLogger.logInfo("Unable to parse '" + location + "'. " + errorMsg);
throw new ParserException(errorMsg,location, "document has no content"); throw new ParserException(errorMsg,location, "document has no content");
} }
@ -595,7 +595,7 @@ public final class plasmaParser {
if (e instanceof ParserException) throw (ParserException) e; if (e instanceof ParserException) throw (ParserException) e;
// log unexpected error // log unexpected error
this.theLogger.logSevere("Unexpected exception in parseSource from File: " + e.getMessage(), e); theLogger.logSevere("Unexpected exception in parseSource from File: " + e.getMessage(), e);
throw new ParserException("Unexpected exception while parsing " + location,location, e); throw new ParserException("Unexpected exception while parsing " + location,location, e);
} finally { } finally {
if (sourceStream != null) try { sourceStream.close(); } catch (final Exception ex){/* ignore this */} if (sourceStream != null) try { sourceStream.close(); } catch (final Exception ex){/* ignore this */}
@ -617,8 +617,8 @@ public final class plasmaParser {
Parser theParser = null; Parser theParser = null;
String mimeType = null; String mimeType = null;
try { try {
if (this.theLogger.isFine()) if (theLogger.isFine())
this.theLogger.logFine("Parsing '" + location + "' from stream"); theLogger.logFine("Parsing '" + location + "' from stream");
// getting the mimetype of the document // getting the mimetype of the document
mimeType = normalizeMimeType(theMimeType); mimeType = normalizeMimeType(theMimeType);
@ -633,12 +633,12 @@ public final class plasmaParser {
// testing if parsing is supported for this resource // testing if parsing is supported for this resource
if (!plasmaParser.supportedContent(location,mimeType)) { if (!plasmaParser.supportedContent(location,mimeType)) {
final String errorMsg = "No parser available to parse mimetype '" + mimeType + "'"; final String errorMsg = "No parser available to parse mimetype '" + mimeType + "'";
this.theLogger.logInfo("Unable to parse '" + location + "'. " + errorMsg); theLogger.logInfo("Unable to parse '" + location + "'. " + errorMsg);
throw new ParserException(errorMsg,location, "wrong mime type or wrong extension"); throw new ParserException(errorMsg,location, "wrong mime type or wrong extension");
} }
if (this.theLogger.isFine()) if (theLogger.isFine())
this.theLogger.logInfo("Parsing " + location + " with mimeType '" + mimeType + theLogger.logInfo("Parsing " + location + " with mimeType '" + mimeType +
"' and file extension '" + fileExt + "'."); "' and file extension '" + fileExt + "'.");
// getting the correct parser for the given mimeType // getting the correct parser for the given mimeType
@ -655,21 +655,21 @@ public final class plasmaParser {
doc = parseHtml(location, mimeType, documentCharset, sourceStream); doc = parseHtml(location, mimeType, documentCharset, sourceStream);
} else { } else {
final String errorMsg = "No parser available to parse mimetype '" + mimeType + "'"; final String errorMsg = "No parser available to parse mimetype '" + mimeType + "'";
this.theLogger.logInfo("Unable to parse '" + location + "'. " + errorMsg); theLogger.logInfo("Unable to parse '" + location + "'. " + errorMsg);
throw new ParserException(errorMsg,location, "wrong mime type or wrong extension"); throw new ParserException(errorMsg,location, "wrong mime type or wrong extension");
} }
// check result // check result
if (doc == null) { if (doc == null) {
final String errorMsg = "Unexpected error. Parser returned null."; final String errorMsg = "Unexpected error. Parser returned null.";
this.theLogger.logInfo("Unable to parse '" + location + "'. " + errorMsg); theLogger.logInfo("Unable to parse '" + location + "'. " + errorMsg);
throw new ParserException(errorMsg,location); throw new ParserException(errorMsg,location);
} }
return doc; return doc;
} catch (final UnsupportedEncodingException e) { } catch (final UnsupportedEncodingException e) {
final String errorMsg = "unsupported charset encoding: " + e.getMessage(); final String errorMsg = "unsupported charset encoding: " + e.getMessage();
this.theLogger.logSevere("Unable to parse '" + location + "'. " + errorMsg, e); theLogger.logSevere("Unable to parse '" + location + "'. " + errorMsg, e);
throw new ParserException(errorMsg,location, errorMsg); throw new ParserException(errorMsg,location, errorMsg);
} catch (final Exception e) { } catch (final Exception e) {
// Interrupted- and Parser-Exceptions should pass through // Interrupted- and Parser-Exceptions should pass through
@ -678,7 +678,7 @@ public final class plasmaParser {
// log unexpected error // log unexpected error
final String errorMsg = "Unexpected exception. " + e.getMessage(); final String errorMsg = "Unexpected exception. " + e.getMessage();
this.theLogger.logSevere("Unable to parse '" + location + "'. " + errorMsg, e); theLogger.logSevere("Unable to parse '" + location + "'. " + errorMsg, e);
throw new ParserException(errorMsg,location,e); throw new ParserException(errorMsg,location,e);
} finally { } finally {
@ -700,7 +700,7 @@ public final class plasmaParser {
} }
if (!documentCharset.equalsIgnoreCase(charset)) { if (!documentCharset.equalsIgnoreCase(charset)) {
this.theLogger.logInfo("Charset transformation needed from '" + documentCharset + "' to '" + charset + "' for URL = " + location.toNormalform(true, true)); theLogger.logInfo("Charset transformation needed from '" + documentCharset + "' to '" + charset + "' for URL = " + location.toNormalform(true, true));
} }
// parsing the content // parsing the content
@ -713,7 +713,7 @@ public final class plasmaParser {
//hfos.close(); //hfos.close();
if (writer.binarySuspect()) { if (writer.binarySuspect()) {
final String errorMsg = "Binary data found in resource"; final String errorMsg = "Binary data found in resource";
this.theLogger.logSevere("Unable to parse '" + location + "'. " + errorMsg); theLogger.logSevere("Unable to parse '" + location + "'. " + errorMsg);
throw new ParserException(errorMsg,location); throw new ParserException(errorMsg,location);
} }
return transformScraper(location, mimeType, documentCharset, scraper); return transformScraper(location, mimeType, documentCharset, scraper);

2
source/de/anomic/plasma/plasmaWordIndex.java
Unescape View File

@ -77,6 +77,7 @@ public final class plasmaWordIndex implements indexRI {
public static final int wCacheMaxChunk = 800; // maximum number of references for each urlhash public static final int wCacheMaxChunk = 800; // maximum number of references for each urlhash
public static final int lowcachedivisor = 900; public static final int lowcachedivisor = 900;
public static final int maxCollectionPartition = 7; // should be 7 public static final int maxCollectionPartition = 7; // should be 7
private static final ByteOrder indexOrder = Base64Order.enhancedCoder;
public static final String CRAWL_PROFILE_PROXY = "proxy"; public static final String CRAWL_PROFILE_PROXY = "proxy";
@ -95,7 +96,6 @@ public final class plasmaWordIndex implements indexRI {
public static final long CRAWL_PROFILE_SNIPPET_GLOBAL_MEDIA_RECRAWL_CYCLE = 60L * 24L * 30L; public static final long CRAWL_PROFILE_SNIPPET_GLOBAL_MEDIA_RECRAWL_CYCLE = 60L * 24L * 30L;
private final ByteOrder indexOrder = Base64Order.enhancedCoder;
private final indexRAMRI indexCache; private final indexRAMRI indexCache;
private final indexCollectionRI collections; // new database structure to replace AssortmentCluster and FileCluster private final indexCollectionRI collections; // new database structure to replace AssortmentCluster and FileCluster
private final Log log; private final Log log;

7
source/de/anomic/server/serverCore.java
Unescape View File

@ -91,6 +91,8 @@ public final class serverCore extends serverAbstractBusyThread implements server
public static final Class<?>[] stringType = {"".getClass()}; // set up some reflection public static final Class<?>[] stringType = {"".getClass()}; // set up some reflection
public static final long startupTime = System.currentTimeMillis(); public static final long startupTime = System.currentTimeMillis();
public static final ThreadGroup sessionThreadGroup = new ThreadGroup("sessionThreadGroup"); public static final ThreadGroup sessionThreadGroup = new ThreadGroup("sessionThreadGroup");
private static final HashMap<String, Object> commandObjMethodCache = new HashMap<String, Object>(5);
/** /**
* will be increased with each session and is used to return a hash code * will be increased with each session and is used to return a hash code
*/ */
@ -465,7 +467,6 @@ public final class serverCore extends serverAbstractBusyThread implements server
private long start; // startup time private long start; // startup time
private serverHandler commandObj; private serverHandler commandObj;
private final HashMap<String, Object> commandObjMethodCache = new HashMap<String, Object>(5);
private String request; // current command line private String request; // current command line
private int commandCounter; // for logging: number of commands in this session private int commandCounter; // for logging: number of commands in this session
@ -747,11 +748,11 @@ public final class serverCore extends serverAbstractBusyThread implements server
this.controlSocket.setSoTimeout(this.socketTimeout); this.controlSocket.setSoTimeout(this.socketTimeout);
// exec command and return value // exec command and return value
Object commandMethod = this.commandObjMethodCache.get(reqProtocol + "_" + reqCmd); Object commandMethod = commandObjMethodCache.get(reqProtocol + "_" + reqCmd);
if (commandMethod == null) { if (commandMethod == null) {
try { try {
commandMethod = this.commandObj.getClass().getMethod(reqCmd, stringType); commandMethod = this.commandObj.getClass().getMethod(reqCmd, stringType);
this.commandObjMethodCache.put(reqProtocol + "_" + reqCmd, commandMethod); commandObjMethodCache.put(reqProtocol + "_" + reqCmd, commandMethod);
} catch (final NoSuchMethodException noMethod) { } catch (final NoSuchMethodException noMethod) {
commandMethod = this.commandObj.getClass().getMethod("UNKNOWN", stringType); commandMethod = this.commandObj.getClass().getMethod("UNKNOWN", stringType);
stringParameter[0] = this.request.trim(); stringParameter[0] = this.request.trim();

14
source/de/anomic/urlRedirector/urlRedirectord.java
Unescape View File

@ -24,7 +24,7 @@ public class urlRedirectord implements serverHandler, Cloneable {
private serverCore.Session session; private serverCore.Session session;
private static plasmaSwitchboard sb = null; private static plasmaSwitchboard sb = null;
private final Log theLogger = new Log("URL-REDIRECTOR"); private static final Log theLogger = new Log("URL-REDIRECTOR");
private static CrawlProfile.entry profile = null; private static CrawlProfile.entry profile = null;
private String nextURL; private String nextURL;
@ -150,7 +150,7 @@ public class urlRedirectord implements serverHandler, Cloneable {
final int pos = line.indexOf(" "); final int pos = line.indexOf(" ");
if (pos != -1) { if (pos != -1) {
final String newDepth = line.substring(pos).trim(); final String newDepth = line.substring(pos).trim();
this.theLogger.logFine("Changing crawling depth to '" + newDepth + "'."); theLogger.logFine("Changing crawling depth to '" + newDepth + "'.");
sb.webIndex.profilesActiveCrawls.changeEntry(profile, "generalDepth",newDepth); sb.webIndex.profilesActiveCrawls.changeEntry(profile, "generalDepth",newDepth);
} }
outputWriter.print("\r\n"); outputWriter.print("\r\n");
@ -159,7 +159,7 @@ public class urlRedirectord implements serverHandler, Cloneable {
final int pos = line.indexOf(" "); final int pos = line.indexOf(" ");
if (pos != -1) { if (pos != -1) {
final String newValue = line.substring(pos).trim(); final String newValue = line.substring(pos).trim();
this.theLogger.logFine("Changing crawl dynamic setting to '" + newValue + "'"); theLogger.logFine("Changing crawl dynamic setting to '" + newValue + "'");
sb.webIndex.profilesActiveCrawls.changeEntry(profile, "crawlingQ",newValue); sb.webIndex.profilesActiveCrawls.changeEntry(profile, "crawlingQ",newValue);
} }
outputWriter.print("\r\n"); outputWriter.print("\r\n");
@ -172,7 +172,7 @@ public class urlRedirectord implements serverHandler, Cloneable {
final int pos = line.indexOf(" "); final int pos = line.indexOf(" ");
this.nextURL = (pos != -1) ? line.substring(0,pos):line; this.nextURL = (pos != -1) ? line.substring(0,pos):line;
this.theLogger.logFine("Receiving request " + line); theLogger.logFine("Receiving request " + line);
outputWriter.print("\r\n"); outputWriter.print("\r\n");
outputWriter.flush(); outputWriter.flush();
@ -216,18 +216,18 @@ public class urlRedirectord implements serverHandler, Cloneable {
} }
if (reasonString != null) { if (reasonString != null) {
this.theLogger.logFine("URL " + nextURL + " rejected. Reason: " + reasonString); theLogger.logFine("URL " + nextURL + " rejected. Reason: " + reasonString);
} }
nextURL = null; nextURL = null;
} }
} }
this.theLogger.logFine("Connection terminated"); theLogger.logFine("Connection terminated");
// Terminating connection // Terminating connection
return serverCore.TERMINATE_CONNECTION; return serverCore.TERMINATE_CONNECTION;
} catch (final Exception e) { } catch (final Exception e) {
this.theLogger.logSevere("Unexpected Error: " + e.getMessage(),e); theLogger.logSevere("Unexpected Error: " + e.getMessage(),e);
return serverCore.TERMINATE_CONNECTION; return serverCore.TERMINATE_CONNECTION;
} }
} }

3
source/de/anomic/xml/opensearchdescriptionReader.java
Unescape View File

@ -97,7 +97,7 @@ public class opensearchdescriptionReader extends DefaultHandler {
private Item channel; private Item channel;
private final StringBuilder buffer; private final StringBuilder buffer;
private boolean parsingChannel; private boolean parsingChannel;
private final String imageURL = null; private final String imageURL;
private final ArrayList<String> itemsGUID; // a list of GUIDs, so the items can be retrieved by a specific order private final ArrayList<String> itemsGUID; // a list of GUIDs, so the items can be retrieved by a specific order
private final HashMap<String, Item> items; // a guid:Item map private final HashMap<String, Item> items; // a guid:Item map
@ -108,6 +108,7 @@ public class opensearchdescriptionReader extends DefaultHandler {
buffer = new StringBuilder(); buffer = new StringBuilder();
channel = null; channel = null;
parsingChannel = false; parsingChannel = false;
imageURL = null;
} }
public opensearchdescriptionReader(final String path) { public opensearchdescriptionReader(final String path) {

6
source/de/anomic/yacy/logging/ConsoleOutErrHandler.java
Unescape View File

@ -39,10 +39,12 @@ public final class ConsoleOutErrHandler extends Handler {
private boolean ignoreCtrlChr = false; private boolean ignoreCtrlChr = false;
private Level splitLevel = Level.WARNING; private Level splitLevel = Level.WARNING;
private final Handler stdOutHandler = new ConsoleOutHandler(); private final Handler stdOutHandler;
private final Handler stdErrHandler = new ConsoleHandler(); private final Handler stdErrHandler;
public ConsoleOutErrHandler() { public ConsoleOutErrHandler() {
this.stdOutHandler = new ConsoleOutHandler();
this.stdErrHandler = new ConsoleHandler();
this.stdOutHandler.setLevel(Level.FINEST); this.stdOutHandler.setLevel(Level.FINEST);
this.stdErrHandler.setLevel(Level.WARNING); this.stdErrHandler.setLevel(Level.WARNING);
configure(); configure();

2
source/de/anomic/yacy/yacyURL.java
Unescape View File

@ -47,7 +47,7 @@ public class yacyURL implements Serializable {
*/ */
private static final long serialVersionUID = -1173233022912141884L; private static final long serialVersionUID = -1173233022912141884L;
public static final int TLD_any_zone_filter = 255; // from TLD zones can be filtered during search; this is the catch-all filter public static final int TLD_any_zone_filter = 255; // from TLD zones can be filtered during search; this is the catch-all filter
private final static Pattern backPathPattern = Pattern.compile("(/[^/]+(?<!/\\.{1,2})/)[.]{2}(?=/|$)|/\\.(?=/)|/(?=/)"); private static final Pattern backPathPattern = Pattern.compile("(/[^/]+(?<!/\\.{1,2})/)[.]{2}(?=/|$)|/\\.(?=/)|/(?=/)");
// class variables // class variables
private String protocol, host, userInfo, path, quest, ref, hash; private String protocol, host, userInfo, path, quest, ref, hash;

2
source/de/anomic/ymage/AnimGifEncoder.java
Unescape View File

@ -267,7 +267,7 @@ public class AnimGifEncoder {
* This hashtable uses the same hash mechanism as the LZH compressor: a * This hashtable uses the same hash mechanism as the LZH compressor: a
* double hash without chaining. * double hash without chaining.
*/ */
static private final int CHSIZE = 1023; private final static int CHSIZE = 1023;
/// The color hashtable's COLOR table (int rcolors) /// The color hashtable's COLOR table (int rcolors)
private int[] m_ccolor_ar; private int[] m_ccolor_ar;

Write Preview
Loading…
Cancel
Save