refactoring + parser bugfixes

pull/1/head
Michael Peter Christen 13 years ago
parent 6c4f8fdc44
commit ba6aaabc51

@ -28,8 +28,8 @@ import java.util.Date;
import java.util.Iterator; import java.util.Iterator;
import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.protocol.RequestHeader;
import net.yacy.kelondro.util.EventTracker;
import net.yacy.peers.graphics.ProfilingGraph; import net.yacy.peers.graphics.ProfilingGraph;
import net.yacy.search.EventTracker;
import de.anomic.server.serverObjects; import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch; import de.anomic.server.serverSwitch;

@ -37,13 +37,13 @@ import net.yacy.cora.protocol.Domains;
import net.yacy.cora.protocol.HeaderFramework; import net.yacy.cora.protocol.HeaderFramework;
import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.protocol.RequestHeader;
import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.logging.Log;
import net.yacy.kelondro.util.EventTracker;
import net.yacy.peers.Seed; import net.yacy.peers.Seed;
import net.yacy.peers.Protocol; import net.yacy.peers.Protocol;
import net.yacy.peers.Network; import net.yacy.peers.Network;
import net.yacy.peers.dht.PeerSelection; import net.yacy.peers.dht.PeerSelection;
import net.yacy.peers.graphics.ProfilingGraph; import net.yacy.peers.graphics.ProfilingGraph;
import net.yacy.peers.operation.yacyVersion; import net.yacy.peers.operation.yacyVersion;
import net.yacy.search.EventTracker;
import net.yacy.search.Switchboard; import net.yacy.search.Switchboard;
import de.anomic.server.serverCore; import de.anomic.server.serverCore;
import de.anomic.server.serverObjects; import de.anomic.server.serverObjects;

@ -55,7 +55,6 @@ import net.yacy.kelondro.index.HandleSet;
import net.yacy.kelondro.order.Bitfield; import net.yacy.kelondro.order.Bitfield;
import net.yacy.kelondro.rwi.ReferenceContainer; import net.yacy.kelondro.rwi.ReferenceContainer;
import net.yacy.kelondro.util.ByteBuffer; import net.yacy.kelondro.util.ByteBuffer;
import net.yacy.kelondro.util.EventTracker;
import net.yacy.kelondro.util.ISO639; import net.yacy.kelondro.util.ISO639;
import net.yacy.kelondro.util.MemoryControl; import net.yacy.kelondro.util.MemoryControl;
import net.yacy.peers.EventChannel; import net.yacy.peers.EventChannel;
@ -63,6 +62,7 @@ import net.yacy.peers.Network;
import net.yacy.peers.Protocol; import net.yacy.peers.Protocol;
import net.yacy.peers.Seed; import net.yacy.peers.Seed;
import net.yacy.peers.graphics.ProfilingGraph; import net.yacy.peers.graphics.ProfilingGraph;
import net.yacy.search.EventTracker;
import net.yacy.search.Switchboard; import net.yacy.search.Switchboard;
import net.yacy.search.SwitchboardConstants; import net.yacy.search.SwitchboardConstants;
import net.yacy.search.index.Segment; import net.yacy.search.index.Segment;

@ -61,7 +61,6 @@ import net.yacy.kelondro.data.word.Word;
import net.yacy.kelondro.index.HandleSet; import net.yacy.kelondro.index.HandleSet;
import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.logging.Log;
import net.yacy.kelondro.order.Bitfield; import net.yacy.kelondro.order.Bitfield;
import net.yacy.kelondro.util.EventTracker;
import net.yacy.kelondro.util.Formatter; import net.yacy.kelondro.util.Formatter;
import net.yacy.kelondro.util.ISO639; import net.yacy.kelondro.util.ISO639;
import net.yacy.kelondro.util.MemoryControl; import net.yacy.kelondro.util.MemoryControl;
@ -69,6 +68,7 @@ import net.yacy.kelondro.util.SetTools;
import net.yacy.peers.EventChannel; import net.yacy.peers.EventChannel;
import net.yacy.peers.NewsPool; import net.yacy.peers.NewsPool;
import net.yacy.peers.graphics.ProfilingGraph; import net.yacy.peers.graphics.ProfilingGraph;
import net.yacy.search.EventTracker;
import net.yacy.search.Switchboard; import net.yacy.search.Switchboard;
import net.yacy.search.SwitchboardConstants; import net.yacy.search.SwitchboardConstants;
import net.yacy.search.index.Segment; import net.yacy.search.index.Segment;

@ -36,11 +36,11 @@ import net.yacy.cora.protocol.RequestHeader;
import net.yacy.cora.protocol.RequestHeader.FileType; import net.yacy.cora.protocol.RequestHeader.FileType;
import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.data.meta.DigestURI;
import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.logging.Log;
import net.yacy.kelondro.util.EventTracker;
import net.yacy.kelondro.util.Formatter; import net.yacy.kelondro.util.Formatter;
import net.yacy.peers.NewsPool; import net.yacy.peers.NewsPool;
import net.yacy.peers.Seed; import net.yacy.peers.Seed;
import net.yacy.peers.graphics.ProfilingGraph; import net.yacy.peers.graphics.ProfilingGraph;
import net.yacy.search.EventTracker;
import net.yacy.search.Switchboard; import net.yacy.search.Switchboard;
import net.yacy.search.SwitchboardConstants; import net.yacy.search.SwitchboardConstants;
import net.yacy.search.query.QueryParams; import net.yacy.search.query.QueryParams;

@ -32,9 +32,9 @@ import net.yacy.cora.protocol.RequestHeader;
import net.yacy.cora.sorting.ScoreMap; import net.yacy.cora.sorting.ScoreMap;
import net.yacy.document.Autotagging; import net.yacy.document.Autotagging;
import net.yacy.document.LibraryProvider; import net.yacy.document.LibraryProvider;
import net.yacy.kelondro.util.EventTracker;
import net.yacy.kelondro.util.Formatter; import net.yacy.kelondro.util.Formatter;
import net.yacy.peers.graphics.ProfilingGraph; import net.yacy.peers.graphics.ProfilingGraph;
import net.yacy.search.EventTracker;
import net.yacy.search.query.QueryParams; import net.yacy.search.query.QueryParams;
import net.yacy.search.query.SearchEvent; import net.yacy.search.query.SearchEvent;
import net.yacy.search.query.SearchEventCache; import net.yacy.search.query.SearchEventCache;

@ -261,7 +261,7 @@ public class RobotsTxt {
// methods that had been in robotsParser.java: // methods that had been in robotsParser.java:
private static final int DOWNLOAD_ACCESS_RESTRICTED = 0; private static final int DOWNLOAD_ACCESS_RESTRICTED = 0;
private static final int DOWNLOAD_ROBOTS_TXT = 1; static final int DOWNLOAD_ROBOTS_TXT = 1;
private static final int DOWNLOAD_ETAG = 2; private static final int DOWNLOAD_ETAG = 2;
private static final int DOWNLOAD_MODDATE = 3; private static final int DOWNLOAD_MODDATE = 3;
@ -287,7 +287,7 @@ public class RobotsTxt {
return port; return port;
} }
private static Object[] downloadRobotsTxt(final MultiProtocolURI robotsURL, int redirectionCount, final RobotsTxtEntry entry) throws Exception { static Object[] downloadRobotsTxt(final MultiProtocolURI robotsURL, int redirectionCount, final RobotsTxtEntry entry) throws Exception {
if (robotsURL == null || !robotsURL.getProtocol().startsWith("http")) return null; if (robotsURL == null || !robotsURL.getProtocol().startsWith("http")) return null;
if (redirectionCount < 0) return new Object[]{Boolean.FALSE,null,null}; if (redirectionCount < 0) return new Object[]{Boolean.FALSE,null,null};
@ -308,7 +308,7 @@ public class RobotsTxt {
// adding referer // adding referer
reqHeaders.put(RequestHeader.REFERER, (MultiProtocolURI.newURL(robotsURL,"/")).toNormalform(true, true)); reqHeaders.put(RequestHeader.REFERER, (MultiProtocolURI.newURL(robotsURL,"/")).toNormalform(true, true));
reqHeaders.put(RequestHeader.ACCEPT, HTTPLoader.DEFAULT_ACCEPT); reqHeaders.put(HeaderFramework.ACCEPT, HTTPLoader.DEFAULT_ACCEPT);
if (entry != null) { if (entry != null) {
oldEtag = entry.getETag(); oldEtag = entry.getETag();
reqHeaders = new RequestHeader(); reqHeaders = new RequestHeader();

@ -35,11 +35,15 @@ import java.io.BufferedReader;
import java.io.ByteArrayInputStream; import java.io.ByteArrayInputStream;
import java.io.IOException; import java.io.IOException;
import java.io.InputStreamReader; import java.io.InputStreamReader;
import java.net.MalformedURLException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashSet;
import java.util.Set; import java.util.Set;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import net.yacy.cora.document.MultiProtocolURI;
import net.yacy.cora.document.UTF8; import net.yacy.cora.document.UTF8;
import net.yacy.kelondro.data.meta.DigestURI;
/* /*
* A class for Parsing robots.txt files. * A class for Parsing robots.txt files.
@ -96,6 +100,35 @@ public final class RobotsTxtParser {
} }
} }
public static RobotsTxtParser getRobots(String homepage) {
DigestURI theURL;
try {
theURL = new DigestURI(homepage);
} catch (MalformedURLException e1) {
return null;
}
final String urlHostPort = RobotsTxt.getHostPort(theURL);
MultiProtocolURI robotsURL = null;
try {
robotsURL = new MultiProtocolURI("http://" + urlHostPort + "/robots.txt");
} catch (final MalformedURLException e) {
return null;
}
Object[] result;
try {
result = RobotsTxt.downloadRobotsTxt(robotsURL, 0, null);
} catch (Exception e) {
return null;
}
final byte[] robotsTxt = (byte[]) result[RobotsTxt.DOWNLOAD_ROBOTS_TXT];
RobotsTxtParser parserResult = new RobotsTxtParser(new HashSet<String>(), robotsTxt);
return parserResult;
}
private void parse(final BufferedReader reader) { private void parse(final BufferedReader reader) {
final ArrayList<String> deny4AllAgents = new ArrayList<String>(); final ArrayList<String> deny4AllAgents = new ArrayList<String>();
final ArrayList<String> deny4ThisAgents = new ArrayList<String>(); final ArrayList<String> deny4ThisAgents = new ArrayList<String>();

@ -417,7 +417,7 @@ public class SolrScheme extends ConfigurationSet {
if (isEmpty() || contains(Field.images_alt_txt.name())) addSolr(solrdoc, Field.images_alt_txt, imgalts); if (isEmpty() || contains(Field.images_alt_txt.name())) addSolr(solrdoc, Field.images_alt_txt, imgalts);
// style sheets // style sheets
if (isEmpty() || contains("css_txt")) { if (isEmpty() || contains(Field.css_tag_txt.name())) {
final Map<MultiProtocolURI, String> csss = html.getCSS(); final Map<MultiProtocolURI, String> csss = html.getCSS();
final String[] css_tag = new String[csss.size()]; final String[] css_tag = new String[csss.size()];
final String[] css_url = new String[csss.size()]; final String[] css_url = new String[csss.size()];

@ -84,6 +84,7 @@ public class ContentScraper extends AbstractScraper implements Scraper {
link(TagType.singleton), link(TagType.singleton),
embed(TagType.singleton), //added by [MN] embed(TagType.singleton), //added by [MN]
param(TagType.singleton), //added by [MN] param(TagType.singleton), //added by [MN]
iframe(TagType.singleton), // scraped as singleton to get such iframes that have no closing tag
a(TagType.pair), a(TagType.pair),
h1(TagType.pair), h1(TagType.pair),
@ -97,7 +98,6 @@ public class ContentScraper extends AbstractScraper implements Scraper {
strong(TagType.pair), strong(TagType.pair),
i(TagType.pair), i(TagType.pair),
li(TagType.pair), li(TagType.pair),
iframe(TagType.pair),
script(TagType.pair); script(TagType.pair);
public TagType type; public TagType type;
@ -351,7 +351,7 @@ public class ContentScraper extends AbstractScraper implements Scraper {
final String content = tagopts.getProperty("content", EMPTY_STRING); final String content = tagopts.getProperty("content", EMPTY_STRING);
if (name.length() > 0) { if (name.length() > 0) {
this.metas.put(name.toLowerCase(), CharacterCoding.html2unicode(content)); this.metas.put(name.toLowerCase(), CharacterCoding.html2unicode(content));
if (name.equals("generator")) { if (name.toLowerCase().equals("generator")) {
this.evaluationScores.match(Element.metagenerator, content); this.evaluationScores.match(Element.metagenerator, content);
} }
} else { } else {
@ -420,6 +420,12 @@ public class ContentScraper extends AbstractScraper implements Scraper {
tagopts.put("value", url.toNormalform(true, false)); tagopts.put("value", url.toNormalform(true, false));
mergeAnchors(url, tagopts /* with property "name" */); mergeAnchors(url, tagopts /* with property "name" */);
} }
} else if (tagname.equalsIgnoreCase("iframe")) {
final MultiProtocolURI src = absolutePath(tagopts.getProperty("src", EMPTY_STRING));
tagopts.put("src", src.toNormalform(true, false));
mergeAnchors(src, tagopts /* with property "name" */);
this.iframes.add(src);
this.evaluationScores.match(Element.iframepath, src.toNormalform(true, false));
} }
// fire event // fire event
@ -482,12 +488,6 @@ public class ContentScraper extends AbstractScraper implements Scraper {
} else if ((tagname.equalsIgnoreCase("li")) && (text.length < 1024)) { } else if ((tagname.equalsIgnoreCase("li")) && (text.length < 1024)) {
h = recursiveParse(text); h = recursiveParse(text);
if (h.length() > 0) this.li.add(h); if (h.length() > 0) this.li.add(h);
} else if (tagname.equalsIgnoreCase("iframe")) {
final MultiProtocolURI src = absolutePath(tagopts.getProperty("src", EMPTY_STRING));
tagopts.put("src", src.toNormalform(true, false));
mergeAnchors(src, tagopts /* with property "name" */);
this.iframes.add(src);
this.evaluationScores.match(Element.iframepath, src.toNormalform(true, false));
} else if (tagname.equalsIgnoreCase("script")) { } else if (tagname.equalsIgnoreCase("script")) {
final String src = tagopts.getProperty("src", EMPTY_STRING); final String src = tagopts.getProperty("src", EMPTY_STRING);
if (src.length() > 0) { if (src.length() > 0) {

@ -84,6 +84,11 @@ public class Evaluation {
this.subject = subject; this.subject = subject;
this.pattern = pattern; this.pattern = pattern;
} }
@Override
public String toString() {
return this.subject + ":" + this.pattern.toString();
}
} }
private static class Model { private static class Model {
@ -153,11 +158,18 @@ public class Evaluation {
final List<Attribute> patterns = this.elementMatcher.get(element); final List<Attribute> patterns = this.elementMatcher.get(element);
if (patterns == null) return subjects; if (patterns == null) return subjects;
for (final Attribute attribute: patterns) { for (final Attribute attribute: patterns) {
if (attribute.pattern.matcher(content).matches()) subjects.inc(attribute.subject); if (attribute.pattern.matcher(content).matches()) {
subjects.inc(attribute.subject);
}
} }
return subjects; return subjects;
} }
@Override
public String toString() {
return this.modelName + ":" + this.elementMatcher.toString();
}
} }
private final Map<String, ClusteredScoreMap<String>> modelMap; // a map from model names to attribute scores private final Map<String, ClusteredScoreMap<String>> modelMap; // a map from model names to attribute scores
@ -166,6 +178,11 @@ public class Evaluation {
this.modelMap = new HashMap<String, ClusteredScoreMap<String>>(); this.modelMap = new HashMap<String, ClusteredScoreMap<String>>();
} }
@Override
public String toString() {
return this.modelMap.toString();
}
/** /**
* produce all model names * produce all model names
* @return a set of model names * @return a set of model names
@ -221,4 +238,15 @@ public class Evaluation {
} }
} }
public static void main(String[] args) {
String t =
"// [CDATA[\n" +
"var gaJsH xost = ((\"https:\" == document.location.protocol) ? \"https://ssl.\" : \"http://www.\");\n" +
"document.write(unescape(\"%3Cscript xsrc='\" + gaJ sHost + \"google-analytics.com/ga.js'\n" +
"type='text/javascript'%3E%3C/script%3E\"));\"\n" +
"\"// ]]\"";
Pattern p = Pattern.compile("(?s).*gaJsHost.*|(?s).*_gat._anonymizeIp.*");
if (p.matcher(t).matches()) System.out.println("1");
}
} }

@ -70,8 +70,6 @@ public final class TransformerWriter extends Writer {
private boolean inSingleQuote; private boolean inSingleQuote;
private boolean inDoubleQuote; private boolean inDoubleQuote;
private boolean inComment; private boolean inComment;
private boolean inScript;
private boolean inStyle;
private boolean binaryUnsuspect; private boolean binaryUnsuspect;
private final boolean passbyIfBinarySuspect; private final boolean passbyIfBinarySuspect;
@ -103,8 +101,6 @@ public final class TransformerWriter extends Writer {
this.inSingleQuote = false; this.inSingleQuote = false;
this.inDoubleQuote = false; this.inDoubleQuote = false;
this.inComment = false; this.inComment = false;
this.inScript = false;
this.inStyle = false;
this.binaryUnsuspect = true; this.binaryUnsuspect = true;
this.passbyIfBinarySuspect = passbyIfBinarySuspect; this.passbyIfBinarySuspect = passbyIfBinarySuspect;
@ -191,7 +187,7 @@ public final class TransformerWriter extends Writer {
} }
private char[] filterTag(final String tag, final boolean opening, final char[] content, final char quotechar) { private char[] filterTag(final String tag, final boolean opening, final char[] content, final char quotechar) {
//System.out.println("FILTER1: filterTag=" + ((this.filterTag == null) ? "null" : this.filterTag) + ", tag=" + tag + ", opening=" + ((opening) ? "true" : "false") + ", content=" + new String(content)); // debug //System.out.println("filterTag: filterTag=" + ((this.filterTag == null) ? "null" : this.filterTag) + ", tag=" + tag + ", opening=" + ((opening) ? "true" : "false") + ", content=" + new String(content)); // debug
// distinguish the following cases: // distinguish the following cases:
// - (1) not collecting data for a tag and getting no tag (not opener and not close) // - (1) not collecting data for a tag and getting no tag (not opener and not close)
// - (2) not collecting data for a tag and getting a tag opener // - (2) not collecting data for a tag and getting a tag opener
@ -224,7 +220,7 @@ public final class TransformerWriter extends Writer {
} }
// we are collection tag text for the tag 'filterTag' -> case (4) - (7) // we are collection tag text for the tag 'filterTag' -> case (4) - (7)
if (tag == null) { if (tag == null || tag.equals("!")) {
// case (4): getting no tag, go on collecting content // case (4): getting no tag, go on collecting content
if (this.scraper != null) this.scraper.scrapeText(content, this.filterTag); if (this.scraper != null) this.scraper.scrapeText(content, this.filterTag);
if (this.transformer != null) { if (this.transformer != null) {
@ -324,7 +320,7 @@ public final class TransformerWriter extends Writer {
private char[] filterSentence(final char[] in, final char quotechar) { private char[] filterSentence(final char[] in, final char quotechar) {
if (in.length == 0) return in; if (in.length == 0) return in;
//System.out.println("FILTER0: " + new String(in)); // debug //System.out.println("filterSentence, quotechar = \"" + quotechar + "\": " + new String(in)); // debug
// scan the string and parse structure // scan the string and parse structure
if (in.length > 2 && in[0] == lb) { if (in.length > 2 && in[0] == lb) {
@ -367,7 +363,7 @@ public final class TransformerWriter extends Writer {
@Override @Override
public void write(final int c) throws IOException { public void write(final int c) throws IOException {
// System.out.println((char) c); //System.out.println((char) c);
if ((this.binaryUnsuspect) && (binaryHint((char)c))) { if ((this.binaryUnsuspect) && (binaryHint((char)c))) {
this.binaryUnsuspect = false; this.binaryUnsuspect = false;
if (this.passbyIfBinarySuspect) close(); if (this.passbyIfBinarySuspect) close();
@ -412,41 +408,6 @@ public final class TransformerWriter extends Writer {
// this.buffer = new serverByteBuffer(); // this.buffer = new serverByteBuffer();
this.buffer.reset(); this.buffer.reset();
} }
} else if (this.inScript) {
this.buffer.append(c);
final int bufferLength = this.buffer.length();
if ((c == rb) && (bufferLength > 14) &&
(this.buffer.charAt(bufferLength - 9) == lb) &&
(this.buffer.charAt(bufferLength - 8) == '/') &&
(this.buffer.charAt(bufferLength - 7) == 's') &&
(this.buffer.charAt(bufferLength - 6) == 'c') &&
(this.buffer.charAt(bufferLength - 5) == 'r') &&
(this.buffer.charAt(bufferLength - 4) == 'i') &&
(this.buffer.charAt(bufferLength - 3) == 'p') &&
(this.buffer.charAt(bufferLength - 2) == 't')) {
// script is at end
this.inScript = false;
if (this.out != null) this.out.write(this.buffer.getChars());
// this.buffer = new serverByteBuffer();
this.buffer.reset();
}
} else if (this.inStyle) {
this.buffer.append(c);
final int bufferLength = this.buffer.length();
if ((c == rb) && (bufferLength > 13) &&
(this.buffer.charAt(bufferLength - 8) == lb) &&
(this.buffer.charAt(bufferLength - 7) == '/') &&
(this.buffer.charAt(bufferLength - 6) == 's') &&
(this.buffer.charAt(bufferLength - 5) == 't') &&
(this.buffer.charAt(bufferLength - 4) == 'y') &&
(this.buffer.charAt(bufferLength - 3) == 'l') &&
(this.buffer.charAt(bufferLength - 2) == 'e')) {
// style is at end
this.inStyle = false;
if (this.out != null) this.out.write(this.buffer.getChars());
// this.buffer = new serverByteBuffer();
this.buffer.reset();
}
} else { } else {
if (this.buffer.length() == 0) { if (this.buffer.length() == 0) {
if (c == rb) { if (c == rb) {
@ -464,25 +425,6 @@ public final class TransformerWriter extends Writer {
// this is the start of a comment // this is the start of a comment
this.inComment = true; this.inComment = true;
this.buffer.append(c); this.buffer.append(c);
} else if ((this.buffer.length() >= 6) &&
(this.buffer.charAt(1) == 's') &&
(this.buffer.charAt(2) == 'c') &&
(this.buffer.charAt(3) == 'r') &&
(this.buffer.charAt(4) == 'i') &&
(this.buffer.charAt(5) == 'p') &&
(c == 't')) {
// this is the start of a javascript
this.inScript = true;
this.buffer.append(c);
} else if ((this.buffer.length() >= 5) &&
(this.buffer.charAt(1) == 's') &&
(this.buffer.charAt(2) == 't') &&
(this.buffer.charAt(3) == 'y') &&
(this.buffer.charAt(4) == 'l') &&
(c == 'e')) {
// this is the start of a css-style
this.inStyle = true;
this.buffer.append(c);
} else if (c == rb) { } else if (c == rb) {
this.buffer.append(c); this.buffer.append(c);
// the tag ends here. after filtering: pass on // the tag ends here. after filtering: pass on

@ -54,6 +54,14 @@ public class Stack {
this.lastHandle = 0; this.lastHandle = 0;
} }
/**
* clear the stack content
* @throws IOException
*/
public void clear() throws IOException {
this.stack.clear();
}
/** /**
* create a new time handle. In case that the method is called * create a new time handle. In case that the method is called
* within a single millisecond twice, a new handle is created using * within a single millisecond twice, a new handle is created using
@ -63,8 +71,8 @@ public class Stack {
*/ */
private long nextHandle() { private long nextHandle() {
long h = System.currentTimeMillis(); long h = System.currentTimeMillis();
if (h <= this.lastHandle) h = lastHandle + 1; if (h <= this.lastHandle) h = this.lastHandle + 1;
lastHandle = h; this.lastHandle = h;
return h; return h;
} }
@ -227,7 +235,7 @@ public class Stack {
* @return the handle * @return the handle
*/ */
public long handle() { public long handle() {
return h; return this.h;
} }
/** /**
@ -235,7 +243,7 @@ public class Stack {
* @return the blob * @return the blob
*/ */
public byte[] blob() { public byte[] blob() {
return b; return this.b;
} }
} }

@ -42,8 +42,8 @@ import net.yacy.kelondro.index.HandleSet;
import net.yacy.kelondro.index.RowSpaceExceededException; import net.yacy.kelondro.index.RowSpaceExceededException;
import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.logging.Log;
import net.yacy.kelondro.order.MergeIterator; import net.yacy.kelondro.order.MergeIterator;
import net.yacy.kelondro.util.EventTracker;
import net.yacy.kelondro.util.MemoryControl; import net.yacy.kelondro.util.MemoryControl;
import net.yacy.search.EventTracker;
/* /*

@ -90,13 +90,13 @@ import net.yacy.kelondro.rwi.Reference;
import net.yacy.kelondro.rwi.ReferenceContainer; import net.yacy.kelondro.rwi.ReferenceContainer;
import net.yacy.kelondro.rwi.ReferenceContainerCache; import net.yacy.kelondro.rwi.ReferenceContainerCache;
import net.yacy.kelondro.util.ByteBuffer; import net.yacy.kelondro.util.ByteBuffer;
import net.yacy.kelondro.util.EventTracker;
import net.yacy.kelondro.util.FileUtils; import net.yacy.kelondro.util.FileUtils;
import net.yacy.peers.graphics.ProfilingGraph; import net.yacy.peers.graphics.ProfilingGraph;
import net.yacy.peers.graphics.WebStructureGraph; import net.yacy.peers.graphics.WebStructureGraph;
import net.yacy.peers.graphics.WebStructureGraph.HostReference; import net.yacy.peers.graphics.WebStructureGraph.HostReference;
import net.yacy.peers.operation.yacyVersion; import net.yacy.peers.operation.yacyVersion;
import net.yacy.repository.Blacklist; import net.yacy.repository.Blacklist;
import net.yacy.search.EventTracker;
import net.yacy.search.Switchboard; import net.yacy.search.Switchboard;
import net.yacy.search.SwitchboardConstants; import net.yacy.search.SwitchboardConstants;
import net.yacy.search.index.Segment; import net.yacy.search.index.Segment;

@ -29,8 +29,8 @@ package net.yacy.peers.graphics;
import java.util.ConcurrentModificationException; import java.util.ConcurrentModificationException;
import java.util.Iterator; import java.util.Iterator;
import net.yacy.kelondro.util.EventTracker; import net.yacy.search.EventTracker;
import net.yacy.kelondro.util.EventTracker.Event; import net.yacy.search.EventTracker.Event;
import net.yacy.search.query.SearchEvent; import net.yacy.search.query.SearchEvent;
import net.yacy.visualization.ChartPlotter; import net.yacy.visualization.ChartPlotter;
import net.yacy.visualization.RasterPlotter; import net.yacy.visualization.RasterPlotter;

@ -24,7 +24,7 @@
// along with this program; if not, write to the Free Software // along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package net.yacy.kelondro.util; package net.yacy.search;
import java.util.Iterator; import java.util.Iterator;
import java.util.Map; import java.util.Map;

@ -24,9 +24,10 @@
// along with this program; if not, write to the Free Software // along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package net.yacy.kelondro.util; package net.yacy.search;
import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.logging.Log;
import net.yacy.kelondro.util.MemoryControl;
public class MemoryTracker extends Thread { public class MemoryTracker extends Thread {

@ -116,10 +116,8 @@ import net.yacy.kelondro.order.Base64Order;
import net.yacy.kelondro.order.Digest; import net.yacy.kelondro.order.Digest;
import net.yacy.kelondro.order.NaturalOrder; import net.yacy.kelondro.order.NaturalOrder;
import net.yacy.kelondro.rwi.ReferenceContainer; import net.yacy.kelondro.rwi.ReferenceContainer;
import net.yacy.kelondro.util.EventTracker;
import net.yacy.kelondro.util.FileUtils; import net.yacy.kelondro.util.FileUtils;
import net.yacy.kelondro.util.MemoryControl; import net.yacy.kelondro.util.MemoryControl;
import net.yacy.kelondro.util.MemoryTracker;
import net.yacy.kelondro.util.OS; import net.yacy.kelondro.util.OS;
import net.yacy.kelondro.util.SetTools; import net.yacy.kelondro.util.SetTools;
import net.yacy.kelondro.workflow.BusyThread; import net.yacy.kelondro.workflow.BusyThread;

@ -62,8 +62,8 @@ import net.yacy.kelondro.index.RowSpaceExceededException;
import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.logging.Log;
import net.yacy.kelondro.rwi.ReferenceContainer; import net.yacy.kelondro.rwi.ReferenceContainer;
import net.yacy.kelondro.rwi.TermSearch; import net.yacy.kelondro.rwi.TermSearch;
import net.yacy.kelondro.util.EventTracker;
import net.yacy.peers.graphics.ProfilingGraph; import net.yacy.peers.graphics.ProfilingGraph;
import net.yacy.search.EventTracker;
import net.yacy.search.Switchboard; import net.yacy.search.Switchboard;
import net.yacy.search.index.Segment; import net.yacy.search.index.Segment;
import net.yacy.search.ranking.ReferenceOrder; import net.yacy.search.ranking.ReferenceOrder;

@ -47,7 +47,6 @@ import net.yacy.kelondro.data.word.WordReferenceFactory;
import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.logging.Log;
import net.yacy.kelondro.order.Base64Order; import net.yacy.kelondro.order.Base64Order;
import net.yacy.kelondro.rwi.ReferenceContainer; import net.yacy.kelondro.rwi.ReferenceContainer;
import net.yacy.kelondro.util.EventTracker;
import net.yacy.kelondro.util.MemoryControl; import net.yacy.kelondro.util.MemoryControl;
import net.yacy.kelondro.util.SetTools; import net.yacy.kelondro.util.SetTools;
import net.yacy.peers.RemoteSearch; import net.yacy.peers.RemoteSearch;
@ -55,6 +54,7 @@ import net.yacy.peers.SeedDB;
import net.yacy.peers.dht.FlatWordPartitionScheme; import net.yacy.peers.dht.FlatWordPartitionScheme;
import net.yacy.peers.graphics.ProfilingGraph; import net.yacy.peers.graphics.ProfilingGraph;
import net.yacy.repository.LoaderDispatcher; import net.yacy.repository.LoaderDispatcher;
import net.yacy.search.EventTracker;
import net.yacy.search.Switchboard; import net.yacy.search.Switchboard;
import net.yacy.search.query.SnippetProcess.Worker; import net.yacy.search.query.SnippetProcess.Worker;
import net.yacy.search.ranking.ReferenceOrder; import net.yacy.search.ranking.ReferenceOrder;

@ -47,11 +47,11 @@ import net.yacy.kelondro.data.word.Word;
import net.yacy.kelondro.index.HandleSet; import net.yacy.kelondro.index.HandleSet;
import net.yacy.kelondro.index.RowSpaceExceededException; import net.yacy.kelondro.index.RowSpaceExceededException;
import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.logging.Log;
import net.yacy.kelondro.util.EventTracker;
import net.yacy.kelondro.util.MemoryControl; import net.yacy.kelondro.util.MemoryControl;
import net.yacy.peers.SeedDB; import net.yacy.peers.SeedDB;
import net.yacy.peers.graphics.ProfilingGraph; import net.yacy.peers.graphics.ProfilingGraph;
import net.yacy.repository.LoaderDispatcher; import net.yacy.repository.LoaderDispatcher;
import net.yacy.search.EventTracker;
import net.yacy.search.Switchboard; import net.yacy.search.Switchboard;
import net.yacy.search.snippet.MediaSnippet; import net.yacy.search.snippet.MediaSnippet;
import net.yacy.search.snippet.ResultEntry; import net.yacy.search.snippet.ResultEntry;

Loading…
Cancel
Save