diff --git a/htroot/gsa/searchresult.java b/htroot/gsa/searchresult.java index f58afdc0a..24b143659 100644 --- a/htroot/gsa/searchresult.java +++ b/htroot/gsa/searchresult.java @@ -30,6 +30,7 @@ import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.services.federated.solr.GSAResponseWriter; import net.yacy.kelondro.logging.Log; import net.yacy.search.Switchboard; +import net.yacy.search.query.AccessTracker; import net.yacy.search.query.SnippetProcess; import net.yacy.search.solr.EmbeddedSolrConnector; @@ -38,6 +39,7 @@ import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.util.FastWriter; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; +import org.apache.solr.search.DocSlice; import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; @@ -98,6 +100,7 @@ public class searchresult { //post.put(, post.remove("site"));//required, example: col1|col2 //post.put(, post.remove("client"));//required, example: myfrontend //post.put(, post.remove("output"));//required, example: xml,xml_no_dtd + String q = post.get(CommonParams.Q, ""); post.put(CommonParams.ROWS, post.remove("num")); post.put(CommonParams.ROWS, Math.min(post.getInt("num", 10), (authenticated) ? 5000 : 100)); post.remove("num"); @@ -152,6 +155,11 @@ public class searchresult { try {ow.close();} catch (IOException e1) {} } + // log result + Object rv = response.getValues().get("response"); + if (rv != null && rv instanceof DocSlice) { + AccessTracker.addToDump(q, Integer.toString(((DocSlice) rv).matches())); + } return null; } } \ No newline at end of file diff --git a/htroot/solr/select.java b/htroot/solr/select.java index 074af46a7..f1e16690c 100644 --- a/htroot/solr/select.java +++ b/htroot/solr/select.java @@ -35,6 +35,7 @@ import net.yacy.cora.services.federated.solr.OpensearchResponseWriter; import net.yacy.kelondro.logging.Log; import net.yacy.search.Switchboard; import net.yacy.search.SwitchboardConstants; +import net.yacy.search.query.AccessTracker; import net.yacy.search.query.SnippetProcess; import net.yacy.search.solr.EmbeddedSolrConnector; import net.yacy.search.solr.SolrServlet; @@ -48,6 +49,7 @@ import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.QueryResponseWriter; import org.apache.solr.response.SolrQueryResponse; import org.apache.solr.response.XSLTResponseWriter; +import org.apache.solr.search.DocSlice; import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; @@ -132,6 +134,7 @@ public class select { // rename post fields according to result style if (!post.containsKey(CommonParams.Q)) post.put(CommonParams.Q, post.remove("query")); // sru patch + String q = post.get(CommonParams.Q, ""); if (!post.containsKey(CommonParams.START)) post.put(CommonParams.START, post.remove("startRecord")); // sru patch post.put(CommonParams.ROWS, Math.min(post.getInt(CommonParams.ROWS, post.getInt("maximumRecords", 10)), (authenticated) ? 5000 : 100)); post.remove("maximumRecords"); @@ -183,6 +186,11 @@ public class select { try {ow.close();} catch (IOException e1) {} } + // log result + Object rv = response.getValues().get("response"); + if (rv != null && rv instanceof DocSlice) { + AccessTracker.addToDump(q, Integer.toString(((DocSlice) rv).matches())); + } return null; } } diff --git a/source/net/yacy/search/Switchboard.java b/source/net/yacy/search/Switchboard.java index cc1628632..98f904e0f 100644 --- a/source/net/yacy/search/Switchboard.java +++ b/source/net/yacy/search/Switchboard.java @@ -291,6 +291,7 @@ public final class Switchboard extends serverSwitch // set loglevel and log setLog(new Log("SWITCHBOARD")); + AccessTracker.setDumpFile(new File("DATA/LOG/queries.log")); // set default peer name Seed.ANON_PREFIX = getConfig("peernameprefix", "_anon"); @@ -1589,7 +1590,7 @@ public final class Switchboard extends serverSwitch Cache.close(); this.tables.close(); Domains.close(); - AccessTracker.dumpLog(new File("DATA/LOG/queries.log")); + AccessTracker.dumpLog(); Switchboard.urlBlacklist.close(); UPnP.deletePortMapping(); this.tray.remove(); diff --git a/source/net/yacy/search/query/AccessTracker.java b/source/net/yacy/search/query/AccessTracker.java index 80b54b91a..286451d9f 100644 --- a/source/net/yacy/search/query/AccessTracker.java +++ b/source/net/yacy/search/query/AccessTracker.java @@ -42,15 +42,23 @@ import net.yacy.kelondro.util.MemoryControl; public class AccessTracker { + private final static long DUMP_PERIOD = 60000L; + public static final int minSize = 100; public static final int maxSize = 1000; public static final int maxAge = 24 * 60 * 60 * 1000; public enum Location {local, remote} - private static LinkedList localSearches = new LinkedList(); - private static LinkedList remoteSearches = new LinkedList(); - private static ArrayList log = new ArrayList(); + private static final LinkedList localSearches = new LinkedList(); + private static final LinkedList remoteSearches = new LinkedList(); + private static final ArrayList log = new ArrayList(); + private static long lastLogDump = System.currentTimeMillis(); + private static File dumpFile = null; + + public static void setDumpFile(File f) { + dumpFile = f; + } public static void add(final Location location, final QueryParams query) { if (location == Location.local) synchronized (localSearches) {add(localSearches, query);} @@ -100,36 +108,62 @@ public class AccessTracker { } private static void addToDump(final QueryParams query) { - //if (query.resultcount == 0) return; if (query.queryString == null || query.queryString.isEmpty()) return; + addToDump(query.queryString, Integer.toString(query.resultcount), new Date(query.starttime)); + } + + public static void addToDump(String querystring, String resultcount) { + addToDump(querystring, resultcount, new Date()); + if (lastLogDump + DUMP_PERIOD < System.currentTimeMillis()) { + lastLogDump = System.currentTimeMillis(); + dumpLog(); + } + } + + private static void addToDump(String querystring, String resultcount, Date d) { + //if (query.resultcount == 0) return; + if (querystring == null || querystring.isEmpty()) return; final StringBuilder sb = new StringBuilder(40); - sb.append(GenericFormatter.SHORT_SECOND_FORMATTER.format(new Date(query.starttime))); + sb.append(GenericFormatter.SHORT_SECOND_FORMATTER.format(d)); sb.append(' '); - sb.append(Integer.toString(query.resultcount)); + sb.append(resultcount); sb.append(' '); - sb.append(query.queryString); - log.add(sb.toString()); + sb.append(querystring); + synchronized (log) { + log.add(sb.toString()); + } } - public static void dumpLog(final File file) { + public static void dumpLog() { while (!localSearches.isEmpty()) { addToDump(localSearches.removeFirst()); } - RandomAccessFile raf = null; - try { - raf = new RandomAccessFile(file, "rw"); - raf.seek(raf.length()); - for (final String s: log) { - raf.write(UTF8.getBytes(s)); - raf.writeByte(10); + Thread t = new Thread() { + @Override + public void run() { + ArrayList logCopy = new ArrayList(); + synchronized (log) { + logCopy.addAll(log); + log.clear(); + } + RandomAccessFile raf = null; + try { + raf = new RandomAccessFile(dumpFile, "rw"); + raf.seek(raf.length()); + for (final String s: logCopy) { + raf.write(UTF8.getBytes(s)); + raf.writeByte(10); + } + logCopy.clear(); + } catch (final FileNotFoundException e) { + Log.logException(e); + } catch (final IOException e) { + Log.logException(e); + } finally { + if (raf != null) try {raf.close();} catch (IOException e) {} + } } - log.clear(); - } catch (final FileNotFoundException e) { - Log.logException(e); - } catch (final IOException e) { - Log.logException(e); - } finally { - if (raf != null) try {raf.close();} catch (IOException e) {} - } + }; + t.start(); } }