From 4eba88f2ff1466d9fad1e52f52c792eb360e665b Mon Sep 17 00:00:00 2001 From: luccioman Date: Thu, 24 Aug 2017 18:47:18 +0200 Subject: [PATCH] Removed some unnecessary uses of java.lang.reflect api. This improves code browsing and readability, making search by references or call hierarchy IDE features more accurate. --- htroot/ConfigBasic.java | 9 +- htroot/api/ymarks/import_ymark.java | 18 +- .../ContentControlFilterUpdateThread.java | 3 +- .../contentcontrol/SMWListSyncThread.java | 3 +- .../kelondro/workflow/InstantBusyThread.java | 175 +++++------- .../kelondro/workflow/OneTimeBusyThread.java | 80 ++++++ .../net/yacy/peers/OnePeerPingBusyThread.java | 53 ++++ source/net/yacy/search/Switchboard.java | 265 ++++++++++++------ .../net/yacy/search/SwitchboardConstants.java | 21 -- source/net/yacy/search/query/SearchEvent.java | 6 + 10 files changed, 402 insertions(+), 231 deletions(-) create mode 100644 source/net/yacy/kelondro/workflow/OneTimeBusyThread.java create mode 100644 source/net/yacy/peers/OnePeerPingBusyThread.java diff --git a/htroot/ConfigBasic.java b/htroot/ConfigBasic.java index 2334ea074..91ea5abde 100644 --- a/htroot/ConfigBasic.java +++ b/htroot/ConfigBasic.java @@ -35,12 +35,11 @@ import java.util.List; import java.util.regex.Pattern; import net.yacy.cora.protocol.Domains; -import net.yacy.cora.protocol.HeaderFramework; import net.yacy.cora.protocol.RequestHeader; import net.yacy.data.Translator; import net.yacy.data.WorkTables; import net.yacy.http.YaCyHttpServer; -import net.yacy.kelondro.workflow.InstantBusyThread; +import net.yacy.peers.OnePeerPingBusyThread; import net.yacy.peers.Seed; import net.yacy.search.Switchboard; import net.yacy.search.SwitchboardConstants; @@ -48,8 +47,8 @@ import net.yacy.server.serverObjects; import net.yacy.server.serverSwitch; import net.yacy.server.http.HTTPDFileHandler; import net.yacy.utils.translation.TranslatorXliff; -import net.yacy.utils.upnp.UPnPMappingType; import net.yacy.utils.upnp.UPnP; +import net.yacy.utils.upnp.UPnPMappingType; public class ConfigBasic { @@ -58,7 +57,7 @@ public class ConfigBasic { private static final int NEXTSTEP_PEERNAME = 2; private static final int NEXTSTEP_PEERPORT = 3; private static final int NEXTSTEP_RECONNECT = 4; - + public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) throws FileNotFoundException, IOException { // return variable that accumulates replacements @@ -81,7 +80,7 @@ public class ConfigBasic { //boolean doPeerPing = false; if ((sb.peers.mySeed().isVirgin()) || (sb.peers.mySeed().isJunior())) { - InstantBusyThread.oneTimeJob(sb.yc, "peerPing", 0); + new OnePeerPingBusyThread(sb.yc).start(); //doPeerPing = true; } diff --git a/htroot/api/ymarks/import_ymark.java b/htroot/api/ymarks/import_ymark.java index 5fa57b101..5344539f9 100644 --- a/htroot/api/ymarks/import_ymark.java +++ b/htroot/api/ymarks/import_ymark.java @@ -33,7 +33,7 @@ import net.yacy.data.ymark.YMarkUtil; import net.yacy.data.ymark.YMarkXBELImporter; import net.yacy.document.Parser.Failure; import net.yacy.kelondro.blob.Tables; -import net.yacy.kelondro.workflow.InstantBusyThread; +import net.yacy.kelondro.workflow.OneTimeBusyThread; import net.yacy.search.Switchboard; import net.yacy.server.serverObjects; import net.yacy.server.serverSwitch; @@ -71,7 +71,7 @@ public class import_ymark { empty = true; } YMarkAutoTagger autoTagger = new YMarkAutoTagger(autoTaggingQueue, sb.loader, sb.tables.bookmarks, bmk_user, merge); - InstantBusyThread.oneTimeJob(autoTagger, 0); + OneTimeBusyThread.startFromRunnable(autoTagger, 0); } if(isAdmin && post.containsKey("table") && post.get("table").length() > 0) { @@ -109,8 +109,8 @@ public class import_ymark { MonitoredReader reader = new MonitoredReader(new InputStreamReader(stream, StandardCharsets.UTF_8), 1024*16, bytes.length); if(post.get("importer").equals("html") && reader != null) { final YMarkHTMLImporter htmlImporter = new YMarkHTMLImporter(reader, queueSize, root); - InstantBusyThread.oneTimeJob(htmlImporter, 0); - InstantBusyThread.oneTimeJob(htmlImporter.getConsumer(sb, bmk_user, autoTaggingQueue, autotag, empty, indexing, medialink), 0); + OneTimeBusyThread.startFromRunnable(htmlImporter, 0); + OneTimeBusyThread.startFromRunnable(htmlImporter.getConsumer(sb, bmk_user, autoTaggingQueue, autotag, empty, indexing, medialink), 0); prop.put("status", "1"); } else if(post.get("importer").equals("xbel") && reader != null) { final YMarkXBELImporter xbelImporter; @@ -123,13 +123,13 @@ public class import_ymark { prop.put("status", "0"); return prop; } - InstantBusyThread.oneTimeJob(xbelImporter, 0); - InstantBusyThread.oneTimeJob(xbelImporter.getConsumer(sb, bmk_user, autoTaggingQueue, autotag, empty, indexing, medialink), 0); + OneTimeBusyThread.startFromRunnable(xbelImporter, 0); + OneTimeBusyThread.startFromRunnable(xbelImporter.getConsumer(sb, bmk_user, autoTaggingQueue, autotag, empty, indexing, medialink), 0); prop.put("status", "1"); } else if(post.get("importer").equals("json") && reader != null) { YMarkJSONImporter jsonImporter; jsonImporter = new YMarkJSONImporter(reader, queueSize, root); - InstantBusyThread.oneTimeJob(jsonImporter, 0); + OneTimeBusyThread.startFromRunnable(jsonImporter, 0); while ((bmk = jsonImporter.take()) != YMarkEntry.POISON) { putBookmark(sb, bmk_user, bmk, autoTaggingQueue, autotag, empty, indexing, medialink); } @@ -207,8 +207,8 @@ public class import_ymark { mreader.addChangeListener(sb.tables.bookmarks.getProgressListener("DMOZImporter")); DMOZImporter.setDepth(6); - InstantBusyThread.oneTimeJob(DMOZImporter, 0); - InstantBusyThread.oneTimeJob(DMOZImporter.getConsumer(sb, bmk_user, autoTaggingQueue, autotag, empty, indexing, medialink), 0); + OneTimeBusyThread.startFromRunnable(DMOZImporter, 0); + OneTimeBusyThread.startFromRunnable(DMOZImporter.getConsumer(sb, bmk_user, autoTaggingQueue, autotag, empty, indexing, medialink), 0); prop.put("status", "1"); } catch (final Exception e) { diff --git a/source/net/yacy/contentcontrol/ContentControlFilterUpdateThread.java b/source/net/yacy/contentcontrol/ContentControlFilterUpdateThread.java index 8f82b8e42..9a3d12e2f 100644 --- a/source/net/yacy/contentcontrol/ContentControlFilterUpdateThread.java +++ b/source/net/yacy/contentcontrol/ContentControlFilterUpdateThread.java @@ -8,7 +8,7 @@ import net.yacy.kelondro.blob.Tables.Row; import net.yacy.repository.FilterEngine; import net.yacy.search.Switchboard; -public class ContentControlFilterUpdateThread { +public class ContentControlFilterUpdateThread implements Runnable { private final Switchboard sb; @@ -22,6 +22,7 @@ public class ContentControlFilterUpdateThread { } + @Override public final void run() { if (!this.locked) { diff --git a/source/net/yacy/contentcontrol/SMWListSyncThread.java b/source/net/yacy/contentcontrol/SMWListSyncThread.java index 805aa6279..80cf8a0d3 100644 --- a/source/net/yacy/contentcontrol/SMWListSyncThread.java +++ b/source/net/yacy/contentcontrol/SMWListSyncThread.java @@ -13,7 +13,7 @@ import net.yacy.cora.util.CommonPattern; import net.yacy.cora.util.ConcurrentLog; import net.yacy.search.Switchboard; -public class SMWListSyncThread { +public class SMWListSyncThread implements Runnable { private final Switchboard sb; private Boolean locked = false; @@ -55,6 +55,7 @@ public class SMWListSyncThread { return ret; } + @Override public final void run() { if (!this.locked) { diff --git a/source/net/yacy/kelondro/workflow/InstantBusyThread.java b/source/net/yacy/kelondro/workflow/InstantBusyThread.java index e8d62ab23..ca091c173 100644 --- a/source/net/yacy/kelondro/workflow/InstantBusyThread.java +++ b/source/net/yacy/kelondro/workflow/InstantBusyThread.java @@ -24,77 +24,36 @@ package net.yacy.kelondro.workflow; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; import java.util.TreeMap; import net.yacy.cora.util.ConcurrentLog; -public final class InstantBusyThread extends AbstractBusyThread implements BusyThread { +public abstract class InstantBusyThread extends AbstractBusyThread implements BusyThread { - private Method jobExecMethod, jobCountMethod, freememExecMethod; - private final Object environment; private final Long handle; private static final TreeMap jobs = new TreeMap(); - public InstantBusyThread( - final Object env, - final String jobExec, - final String jobCount, - final String freemem, - final long idleSleep, - final long busySleep) { + /** + * @param idleSleep defines min idle sleep time that can be set via setIdleSleep() + * @param busySleep defines min busy sleep time that can be set via setBusySleep() + */ + public InstantBusyThread(final long idleSleep, final long busySleep) { + this("InstantBusyThread.job", idleSleep, busySleep); + } + + /** + * @param jobName the job name used to monitor the thread + * @param idleSleep defines min idle sleep time that can be set via setIdleSleep() + * @param busySleep defines min busy sleep time that can be set via setBusySleep() + */ + public InstantBusyThread(final String jobName, final long idleSleep, final long busySleep) { super(idleSleep, busySleep); - - // jobExec is the name of a method of the object 'env' that executes the one-step-run - // jobCount is the name of a method that returns the size of the job - // freemem is the name of a method that tries to free memory and returns void - final Class theClass = (env instanceof Class) ? (Class) env : env.getClass(); - try { - this.jobExecMethod = theClass.getMethod(jobExec); - } catch (final NoSuchMethodException e) { - throw new RuntimeException("serverInstantThread, wrong declaration of jobExec: " + e.getMessage()); - } - try { - if (jobCount == null) - this.jobCountMethod = null; - else - this.jobCountMethod = theClass.getMethod(jobCount); - - } catch (final NoSuchMethodException e) { - throw new RuntimeException("serverInstantThread, wrong declaration of jobCount: " + e.getMessage()); - } - try { - if (freemem == null) - this.freememExecMethod = null; - else - this.freememExecMethod = theClass.getMethod(freemem); - - } catch (final NoSuchMethodException e) { - throw new RuntimeException("serverInstantThread, wrong declaration of freemem: " + e.getMessage()); - } - this.environment = (env instanceof Class) ? null : env; - setName("BusyThread " + theClass.getName() + "." + jobExec); + setName("BusyThread " + jobName); this.handle = Long.valueOf(System.currentTimeMillis() + getName().hashCode()); } - - @Override - public int getJobCount() { - if (this.jobCountMethod == null) return Integer.MAX_VALUE; - try { - final Object result = this.jobCountMethod.invoke(this.environment); - return (result instanceof Integer) ? ((Integer) result).intValue() : -1; - } catch (final IllegalAccessException e) { - return -1; - } catch (final IllegalArgumentException e) { - return -1; - } catch (final InvocationTargetException e) { - ConcurrentLog.severe("BUSYTHREAD", "invocation serverInstantThread of thread '" + getName() + "': " + e.getMessage(), e); - return -1; - } - } + @Override public boolean job() throws Exception { @@ -102,25 +61,13 @@ public final class InstantBusyThread extends AbstractBusyThread implements BusyT synchronized(jobs) {jobs.put(this.handle, getName());} boolean jobHasDoneSomething = false; try { - final Object result = this.jobExecMethod.invoke(this.environment); - if (result == null) jobHasDoneSomething = true; - else if (result instanceof Boolean) jobHasDoneSomething = ((Boolean) result).booleanValue(); - } catch (final IllegalAccessException e) { - ConcurrentLog.severe("BUSYTHREAD", "Internal Error in serverInstantThread.job: " + e.getMessage()); - ConcurrentLog.severe("BUSYTHREAD", "shutting down thread '" + getName() + "'"); - terminate(false); + jobHasDoneSomething = jobImpl(); } catch (final IllegalArgumentException e) { - ConcurrentLog.severe("BUSYTHREAD", "Internal Error in serverInstantThread.job: " + e.getMessage()); + ConcurrentLog.severe("BUSYTHREAD", "Internal Error in InstantBusyThread.job: " + e.getMessage()); ConcurrentLog.severe("BUSYTHREAD", "shutting down thread '" + getName() + "'"); terminate(false); - } catch (final InvocationTargetException e) { - final String targetException = e.getTargetException().getMessage(); - ConcurrentLog.logException(e); - ConcurrentLog.logException(e.getCause()); - ConcurrentLog.logException(e.getTargetException()); - ConcurrentLog.severe("BUSYTHREAD", "Runtime Error in serverInstantThread.job, thread '" + getName() + "': " + e.getMessage() + "; target exception: " + targetException, e.getTargetException()); } catch (final OutOfMemoryError e) { - ConcurrentLog.severe("BUSYTHREAD", "OutOfMemory Error in serverInstantThread.job, thread '" + getName() + "': " + e.getMessage()); + ConcurrentLog.severe("BUSYTHREAD", "OutOfMemory Error in InstantBusyThread.job, thread '" + getName() + "': " + e.getMessage()); ConcurrentLog.logException(e); freemem(); } catch (final Exception e) { @@ -130,46 +77,36 @@ public final class InstantBusyThread extends AbstractBusyThread implements BusyT synchronized(jobs) {jobs.remove(this.handle);} return jobHasDoneSomething; } + + /** + * The job's main logic implementation + * @return true if it has done something, false if it is idle and does not expect to work on more for a longer time + * @throws Exception when an unexpected error occurred + */ + public abstract boolean jobImpl() throws Exception; @Override public void freemem() { - if (this.freememExecMethod == null) return; try { - this.freememExecMethod.invoke(this.environment); - } catch (final IllegalAccessException e) { - ConcurrentLog.severe("BUSYTHREAD", "Internal Error in serverInstantThread.freemem: " + e.getMessage()); - ConcurrentLog.severe("BUSYTHREAD", "shutting down thread '" + getName() + "'"); - terminate(false); - } catch (final IllegalArgumentException e) { - ConcurrentLog.severe("BUSYTHREAD", "Internal Error in serverInstantThread.freemem: " + e.getMessage()); - ConcurrentLog.severe("BUSYTHREAD", "shutting down thread '" + getName() + "'"); - terminate(false); - } catch (final InvocationTargetException e) { - final String targetException = e.getTargetException().getMessage(); - if (targetException.indexOf("heap space",0) > 0) ConcurrentLog.logException(e.getTargetException()); - ConcurrentLog.severe("BUSYTHREAD", "Runtime Error in serverInstantThread.freemem, thread '" + getName() + "': " + e.getMessage() + "; target exception: " + targetException, e.getTargetException()); - ConcurrentLog.logException(e.getTargetException()); + freememImpl(); } catch (final OutOfMemoryError e) { - ConcurrentLog.severe("BUSYTHREAD", "OutOfMemory Error in serverInstantThread.freemem, thread '" + getName() + "': " + e.getMessage()); + ConcurrentLog.severe("BUSYTHREAD", "OutOfMemory Error in InstantBusyThread.freemem, thread '" + getName() + "': " + e.getMessage()); ConcurrentLog.logException(e); } } + + @Override + public int getJobCount() { + return Integer.MAX_VALUE; + } + + /** + * Called when an outOfMemoryCycle is performed. + */ + public void freememImpl() { + // Do nothing in this implementation, please override + } - public static BusyThread oneTimeJob(final Object env, final String jobExec, final long startupDelay) { - // start the job and execute it once as background process - final BusyThread thread = new InstantBusyThread(env, jobExec, null, null, Long.MIN_VALUE, Long.MIN_VALUE); - thread.setStartupSleep(startupDelay); - thread.setIdleSleep(-1); - thread.setBusySleep(-1); - thread.setMemPreReqisite(0); - thread.setLoadPreReqisite(Double.MAX_VALUE); // this is called during initialization phase and some code parts depend on it; therefore we cannot set a prerequisite that prevents the start of that thread - thread.start(); - return thread; - } - - public static WorkflowThread oneTimeJob(final Runnable thread, final long startupDelay) { - return oneTimeJob(thread, "run", startupDelay); - } @Override public void open() { @@ -180,5 +117,35 @@ public final class InstantBusyThread extends AbstractBusyThread implements BusyT public synchronized void close() { // Not implemented in this thread } + + /** + * Construct an InstantBusyThread instance from a runnable task. + * + * @param task + * the task to run as a job + * @param idleSleep + * defines min idle sleep time that can be set via setIdleSleep() + * @param busySleep + * defines min busy sleep time that can be set via setBusySleep() + * @return a InstantBusyThread instance + * @throws IllegalArgumentException + * when the task is null + */ + public static InstantBusyThread createFromRunnable(final Runnable task, final long idleSleep, + final long busySleep) { + if (task == null) { + throw new IllegalArgumentException("Runnable task must not be null"); + } + InstantBusyThread busyThread = new InstantBusyThread(task.getClass().getName() + ".run", idleSleep, busySleep) { + + @Override + public boolean jobImpl() throws Exception { + task.run(); + return true; + } + + }; + return busyThread; + } } diff --git a/source/net/yacy/kelondro/workflow/OneTimeBusyThread.java b/source/net/yacy/kelondro/workflow/OneTimeBusyThread.java new file mode 100644 index 000000000..d7bbf832e --- /dev/null +++ b/source/net/yacy/kelondro/workflow/OneTimeBusyThread.java @@ -0,0 +1,80 @@ +// OneTimeBusyThread.java +// --------------------------- +// Copyright 2017 by luccioman; https://github.com/luccioman +// +// This is a part of YaCy, a peer-to-peer based web search engine +// +// LICENSE +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +package net.yacy.kelondro.workflow; + +/** + * A busy thread to run a job only once + */ +public abstract class OneTimeBusyThread extends InstantBusyThread { + + /** + * Construct an instance able to run a job once, after a given delay + * @param jobName the job name used to monitor the thread + * @param startupDelay the delay in milliseconds to wait before starting the job + */ + public OneTimeBusyThread(final String jobName, final long startupDelay) { + super(jobName, Long.MIN_VALUE, Long.MIN_VALUE); + this.setStartupSleep(startupDelay); + this.setIdleSleep(-1); + this.setBusySleep(-1); + this.setMemPreReqisite(0); + this.setLoadPreReqisite( + Double.MAX_VALUE); /* + * this is called during initialization phase and some code parts depend on it; + * therefore we cannot set a prerequisite that prevents the start of that thread + */ + } + + /** + * Construct an instance able to run a job once and immediately + * @param jobName the job name used to monitor the thread + */ + public OneTimeBusyThread(final String jobName) { + this(jobName, 0); + } + + /** + * Construct and start a OneTimeBusyThread instance from a runnable task. + * @param task the task to run once + * @param startupDelay the delay in milliseconds to wait before starting the job + * @return a OneTimeBusyThread instance + * @throws IllegalArgumentException when the task is null + */ + public static OneTimeBusyThread startFromRunnable(final Runnable task, final long startupDelay) { + if(task == null) { + throw new IllegalArgumentException("Runnable task must not be null"); + } + OneTimeBusyThread busyThread = new OneTimeBusyThread(task.getClass().getName() + ".run", startupDelay) { + + @Override + public boolean jobImpl() throws Exception { + task.run(); + return true; + } + + }; + busyThread.start(); + return busyThread; + } + +} diff --git a/source/net/yacy/peers/OnePeerPingBusyThread.java b/source/net/yacy/peers/OnePeerPingBusyThread.java new file mode 100644 index 000000000..347feb638 --- /dev/null +++ b/source/net/yacy/peers/OnePeerPingBusyThread.java @@ -0,0 +1,53 @@ +// OnePeerPingBusyThread.java +// --------------------------- +// Copyright 2017 by luccioman; https://github.com/luccioman +// +// This is a part of YaCy, a peer-to-peer based web search engine +// +// LICENSE +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +package net.yacy.peers; + +import net.yacy.kelondro.workflow.OneTimeBusyThread; + +/** + * A busy thread running peer ping only once. + */ +public class OnePeerPingBusyThread extends OneTimeBusyThread { + + /** Access to the peer to peer network */ + final net.yacy.peers.Network peersNetwork; + + /** + * @param peersNetwork a net.yacy.peers.Network instance + * @throws IllegalArgumentException when peersNetwork is null + */ + public OnePeerPingBusyThread(final net.yacy.peers.Network peersNetwork) { + super("Network.peerPing"); + if(peersNetwork == null) { + throw new IllegalArgumentException("peersNetwork parameter must not be null"); + } + this.peersNetwork = peersNetwork; + } + + @Override + public boolean jobImpl() throws Exception { + this.peersNetwork.peerPing(); + return true; + } + +} \ No newline at end of file diff --git a/source/net/yacy/search/Switchboard.java b/source/net/yacy/search/Switchboard.java index cd7e4850e..3406071f2 100644 --- a/source/net/yacy/search/Switchboard.java +++ b/source/net/yacy/search/Switchboard.java @@ -193,6 +193,7 @@ import net.yacy.kelondro.util.OS; import net.yacy.kelondro.util.SetTools; import net.yacy.kelondro.workflow.BusyThread; import net.yacy.kelondro.workflow.InstantBusyThread; +import net.yacy.kelondro.workflow.OneTimeBusyThread; import net.yacy.kelondro.workflow.WorkflowProcessor; import net.yacy.kelondro.workflow.WorkflowThread; import net.yacy.peers.DHTSelection; @@ -337,7 +338,14 @@ public final class Switchboard extends serverSwitch { // UPnP port mapping if ( getConfigBool(SwitchboardConstants.UPNP_ENABLED, false) ) { - InstantBusyThread.oneTimeJob(UPnP.class, "addPortMappings", 0); + new OneTimeBusyThread("UPnP.addPortMappings") { + + @Override + public boolean jobImpl() throws Exception { + UPnP.addPortMappings(); + return true; + } + }.start(); } // init TrayIcon if possible @@ -611,7 +619,14 @@ public final class Switchboard extends serverSwitch { // start yacy core this.log.config("Starting YaCy Protocol Core"); this.yc = new Network(this); - InstantBusyThread.oneTimeJob(this, "loadSeedLists", 0); + new OneTimeBusyThread("Switchboard.loadSeedLists") { + + @Override + public boolean jobImpl() throws Exception { + loadSeedLists(); + return true; + } + }.start(); //final long startedSeedListAquisition = System.currentTimeMillis(); // init a DHT transmission dispatcher @@ -941,7 +956,17 @@ public final class Switchboard extends serverSwitch { // initializing the resourceObserver this.observer = new ResourceObserver(this); - InstantBusyThread.oneTimeJob(this.observer, "resourceObserverJob", 0); + + final ResourceObserver resourceObserver = this.observer; + new OneTimeBusyThread("ResourceObserver.resourceObserverJob") { + + @Override + public boolean jobImpl() throws Exception { + resourceObserver.resourceObserverJob(); + return true; + } + }.start(); + // initializing the stackCrawlThread this.crawlStacker = @@ -1032,101 +1057,142 @@ public final class Switchboard extends serverSwitch { // deploy busy threads this.log.config("Starting Threads"); MemoryControl.gc(10000, "plasmaSwitchboard, help for profiler"); // help for profiler - thq - + deployThread( SwitchboardConstants.CLEANUP, "Cleanup", "cleaning process", null, - new InstantBusyThread( - this, - SwitchboardConstants.CLEANUP_METHOD_START, - SwitchboardConstants.CLEANUP_METHOD_JOBCOUNT, - SwitchboardConstants.CLEANUP_METHOD_FREEMEM, - 30000, - 10000), + new InstantBusyThread("Switchboard.cleanupJob", 30000, 10000) { + + @Override + public boolean jobImpl() throws Exception { + return cleanupJob(); + } + + @Override + public int getJobCount() { + return cleanupJobSize(); + } + + @Override + public void freememImpl() { + } + + }, 60000); // all 10 minutes, wait 1 minute until first run + deployThread( SwitchboardConstants.SCHEDULER, "Scheduler", "starts scheduled processes from the API Processing table", null, - new InstantBusyThread( - this, - SwitchboardConstants.SCHEDULER_METHOD_START, - SwitchboardConstants.SCHEDULER_METHOD_JOBCOUNT, - SwitchboardConstants.SCHEDULER_METHOD_FREEMEM, - 30000, - 10000), + new InstantBusyThread("Switchboard.schedulerJob", 30000, 10000) { + @Override + public boolean jobImpl() throws Exception { + return schedulerJob(); + } + + @Override + public int getJobCount() { + return schedulerJobSize(); + } + + @Override + public void freememImpl() { + } + }, 60000); // all 10 minutes, wait 1 minute until first run + deployThread( SwitchboardConstants.SURROGATES, "Surrogates", "A thread that polls the SURROGATES path and puts all Documents in one surroagte file into the indexing queue.", null, - new InstantBusyThread( - this, - SwitchboardConstants.SURROGATES_METHOD_START, - SwitchboardConstants.SURROGATES_METHOD_JOBCOUNT, - SwitchboardConstants.SURROGATES_METHOD_FREEMEM, - 20000, - 0), + new InstantBusyThread("Switchboard.surrogateProcess", 20000, 0) { + @Override + public boolean jobImpl() throws Exception { + return surrogateProcess(); + } + + @Override + public int getJobCount() { + return surrogateQueueSize(); + } + + @Override + public void freememImpl() { + surrogateFreeMem(); + } + }, 10000); this.initRemoteCrawler(this.getConfigBool(SwitchboardConstants.CRAWLJOB_REMOTE, false)); this.initAutocrawl(this.getConfigBool(SwitchboardConstants.AUTOCRAWL, false)); + final CrawlQueues crawlQueue = this.crawlQueues; deployThread( SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL, "Local Crawl", "thread that performes a single crawl step from the local crawl queue", "/IndexCreateQueues_p.html?stack=LOCAL", - new InstantBusyThread( - this.crawlQueues, - SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL_METHOD_START, - SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL_METHOD_JOBCOUNT, - SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL_METHOD_FREEMEM, - 0, - 0), + new InstantBusyThread("CrawlQueues.coreCrawlJob", 0, 0) { + @Override + public boolean jobImpl() throws Exception { + return crawlQueue.coreCrawlJob(); + } + + @Override + public int getJobCount() { + return crawlQueue.coreCrawlJobSize(); + } + + @Override + public void freememImpl() { + crawlQueue.freemem(); + } + }, 10000); + + final Network net = this.yc; deployThread( SwitchboardConstants.SEED_UPLOAD, "Seed-List Upload", "task that a principal peer performes to generate and upload a seed-list to a ftp account", null, - new InstantBusyThread( - this.yc, - SwitchboardConstants.SEED_UPLOAD_METHOD_START, - SwitchboardConstants.SEED_UPLOAD_METHOD_JOBCOUNT, - SwitchboardConstants.SEED_UPLOAD_METHOD_FREEMEM, - 600000, - 300000), + new InstantBusyThread("Network.publishSeedList", 600000, 300000) { + @Override + public boolean jobImpl() throws Exception { + net.publishSeedList(); + return true; + } + }, 180000); + deployThread( SwitchboardConstants.PEER_PING, "YaCy Core", "this is the p2p-control and peer-ping task", null, - new InstantBusyThread( - this.yc, - SwitchboardConstants.PEER_PING_METHOD_START, - SwitchboardConstants.PEER_PING_METHOD_JOBCOUNT, - SwitchboardConstants.PEER_PING_METHOD_FREEMEM, - 30000, - 30000), + new InstantBusyThread("Network.peerPing", 30000, 30000) { + @Override + public boolean jobImpl() throws Exception { + net.peerPing(); + return true; + } + }, 10000); deployThread( SwitchboardConstants.INDEX_DIST, "DHT Distribution", "selection, transfer and deletion of index entries that are not searched on your peer, but on others", null, - new InstantBusyThread( - this, - SwitchboardConstants.INDEX_DIST_METHOD_START, - SwitchboardConstants.INDEX_DIST_METHOD_JOBCOUNT, - SwitchboardConstants.INDEX_DIST_METHOD_FREEMEM, - 10000, - 1000), + new InstantBusyThread("Switchboard.dhtTransferJob", 10000, 1000) { + @Override + public boolean jobImpl() throws Exception { + return dhtTransferJob(); + } + }, 60000, Long.parseLong(getConfig(SwitchboardConstants.INDEX_DIST_IDLESLEEP, "5000")), Long.parseLong(getConfig(SwitchboardConstants.INDEX_DIST_BUSYSLEEP, "0")), @@ -1138,28 +1204,20 @@ public final class Switchboard extends serverSwitch { "720_ccimport", "Content Control Import", "this is the content control import thread", - null, - new InstantBusyThread( - new SMWListSyncThread(this, sb.getConfig("contentcontrol.bookmarklist", "contentcontrol"), "Category:Content Source", "/?Url/?Filter/?Category/?Modification date", sb.getConfigBool( - "contentcontrol.smwimport.purgelistoninit", false)), - "run", - SwitchboardConstants.PEER_PING_METHOD_JOBCOUNT, - SwitchboardConstants.PEER_PING_METHOD_FREEMEM, - 3000, - 3000), + null, + InstantBusyThread.createFromRunnable( + new SMWListSyncThread(this, sb.getConfig("contentcontrol.bookmarklist", "contentcontrol"), + "Category:Content Source", "/?Url/?Filter/?Category/?Modification date", + sb.getConfigBool("contentcontrol.smwimport.purgelistoninit", false)), + 3000, 3000), 2000); + deployThread( "730_ccfilter", "Content Control Filter", "this is the content control filter update thread", null, - new InstantBusyThread( - new ContentControlFilterUpdateThread(this), - "run", - SwitchboardConstants.PEER_PING_METHOD_JOBCOUNT, - SwitchboardConstants.PEER_PING_METHOD_FREEMEM, - 3000, - 3000), + InstantBusyThread.createFromRunnable(new ContentControlFilterUpdateThread(this), 3000, 3000), 2000); // set network-specific performance attributes @@ -1513,6 +1571,8 @@ public final class Switchboard extends serverSwitch { this.peers.mySeed().setFlagAcceptRemoteCrawl(activate); if (activate) { this.crawlQueues.initRemoteCrawlQueues(); + + final CrawlQueues queues = this.crawlQueues; BusyThread rct = getThread(SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL); if (rct == null) { @@ -1521,13 +1581,19 @@ public final class Switchboard extends serverSwitch { "Remote Crawl Job", "thread that performes a single crawl/indexing step triggered by a remote peer", "/IndexCreateQueues_p.html?stack=REMOTE", - new InstantBusyThread( - this.crawlQueues, - SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL_METHOD_START, - SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL_METHOD_JOBCOUNT, - SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL_METHOD_FREEMEM, - 0, - 0), + new InstantBusyThread("CrawlQueues.remoteTriggeredCrawlJob", 0, 0) { + + @Override + public boolean jobImpl() throws Exception { + return queues.remoteTriggeredCrawlJob(); + } + + @Override + public int getJobCount() { + return queues.remoteTriggeredCrawlJobSize(); + } + + }, 10000); rct = getThread(SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL); } @@ -1541,13 +1607,12 @@ public final class Switchboard extends serverSwitch { "Remote Crawl URL Loader", "thread that loads remote crawl lists from other peers", null, - new InstantBusyThread( - this.crawlQueues, - SwitchboardConstants.CRAWLJOB_REMOTE_CRAWL_LOADER_METHOD_START, - SwitchboardConstants.CRAWLJOB_REMOTE_CRAWL_LOADER_METHOD_JOBCOUNT, - SwitchboardConstants.CRAWLJOB_REMOTE_CRAWL_LOADER_METHOD_FREEMEM, - 10000, - 10000), + new InstantBusyThread("CrawlQueues.remoteCrawlLoaderJob", 10000, 10000) { + @Override + public boolean jobImpl() throws Exception { + return queues.remoteCrawlLoaderJob(); + } + }, 10000); rcl = getThread(SwitchboardConstants.CRAWLJOB_REMOTE_CRAWL_LOADER); @@ -1569,18 +1634,19 @@ public final class Switchboard extends serverSwitch { if (activate) { BusyThread acr = getThread(SwitchboardConstants.CRAWLJOB_AUTOCRAWL); if (acr == null) { + final CrawlQueues queues = this.crawlQueues; + deployThread( SwitchboardConstants.CRAWLJOB_AUTOCRAWL, "Autocrawl", "Thread that selects and automatically adds crawling jobs to the local queue", null, - new InstantBusyThread( - this.crawlQueues, - SwitchboardConstants.CRAWLJOB_AUTOCRAWL_METHOD_START, - SwitchboardConstants.CRAWLJOB_AUTOCRAWL_METHOD_JOBCOUNT, - SwitchboardConstants.CRAWLJOB_AUTOCRAWL_METHOD_FREEMEM, - 10000, - 10000), + new InstantBusyThread("CrawlQueues.autocrawlJob", 10000, 10000) { + @Override + public boolean jobImpl() throws Exception { + return queues.autocrawlJob(); + } + }, 10000); acr = getThread(SwitchboardConstants.CRAWLJOB_AUTOCRAWL); @@ -2832,6 +2898,11 @@ public final class Switchboard extends serverSwitch { } } + /** + * Parse a response to produce a new document to add to the index. + * Important : this method is called using reflection as a Workflow process and must therefore remain public. + * @param in an indexing workflow entry containing a response to parse + */ public IndexingQueueEntry parseDocument(final IndexingQueueEntry in) { in.queueEntry.updateStatus(Response.QUEUE_STATE_PARSING); Document[] documents = null; @@ -3009,6 +3080,10 @@ public final class Switchboard extends serverSwitch { return documents; } + /** + * Important : this method is called using reflection as a Workflow process and must therefore remain public. + * @param in an indexing workflow entry containing a response and the related parsed document(s) + */ public IndexingQueueEntry condenseDocument(final IndexingQueueEntry in) { in.queueEntry.updateStatus(Response.QUEUE_STATE_CONDENSING); CrawlProfile profile = in.queueEntry.profile(); @@ -3083,6 +3158,11 @@ public final class Switchboard extends serverSwitch { return new IndexingQueueEntry(in.queueEntry, in.documents, condenser); } + /** + * Perform web structure analysis on parsed documents and update the web structure graph. + * Important : this method is called using reflection as a Workflow process and must therefore remain public. + * @param in an indexing workflow entry containing parsed document(s) + */ public IndexingQueueEntry webStructureAnalysis(final IndexingQueueEntry in) { in.queueEntry.updateStatus(Response.QUEUE_STATE_STRUCTUREANALYSIS); for (Document document : in.documents) { @@ -3096,6 +3176,11 @@ public final class Switchboard extends serverSwitch { return in; } + /** + * Store a new entry to the local index. + * Important : this method is called using reflection as a Workflow process and must therefore remain public. + * @param in an indexing workflow entry containing parsed document(s) and a condenser instance + */ public void storeDocumentIndex(final IndexingQueueEntry in) { in.queueEntry.updateStatus(Response.QUEUE_STATE_INDEXSTORAGE); // the condenser may be null in case that an indexing is not wanted (there may be a no-indexing flag in the file) diff --git a/source/net/yacy/search/SwitchboardConstants.java b/source/net/yacy/search/SwitchboardConstants.java index af90841bd..013796b74 100644 --- a/source/net/yacy/search/SwitchboardConstants.java +++ b/source/net/yacy/search/SwitchboardConstants.java @@ -87,9 +87,6 @@ public final class SwitchboardConstants { * connected to the YaCy-network

*/ public static final String PEER_PING = "30_peerping"; - public static final String PEER_PING_METHOD_START = "peerPing"; - public static final String PEER_PING_METHOD_JOBCOUNT = null; - public static final String PEER_PING_METHOD_FREEMEM = null; public static final String PEER_PING_IDLESLEEP = "30_peerping_idlesleep"; public static final String PEER_PING_BUSYSLEEP = "30_peerping_busysleep"; // 40_peerseedcycle @@ -98,9 +95,6 @@ public final class SwitchboardConstants { *

Name of the seed upload thread, providing the so-called seed-lists needed during bootstrapping

*/ public static final String SEED_UPLOAD = "40_peerseedcycle"; - public static final String SEED_UPLOAD_METHOD_START = "publishSeedList"; - public static final String SEED_UPLOAD_METHOD_JOBCOUNT = null; - public static final String SEED_UPLOAD_METHOD_FREEMEM = null; public static final String SEED_UPLOAD_IDLESLEEP = "40_peerseedcycle_idlesleep"; public static final String SEED_UPLOAD_BUSYSLEEP = "40_peerseedcycle_busysleep"; // 50_localcrawl @@ -111,9 +105,6 @@ public final class SwitchboardConstants { * */ public static final String CRAWLJOB_LOCAL_CRAWL = "50_localcrawl"; - public static final String CRAWLJOB_LOCAL_CRAWL_METHOD_START = "coreCrawlJob"; - public static final String CRAWLJOB_LOCAL_CRAWL_METHOD_JOBCOUNT = "coreCrawlJobSize"; - public static final String CRAWLJOB_LOCAL_CRAWL_METHOD_FREEMEM = "freemem"; public static final String CRAWLJOB_LOCAL_CRAWL_IDLESLEEP = "50_localcrawl_idlesleep"; public static final String CRAWLJOB_LOCAL_CRAWL_BUSYSLEEP = "50_localcrawl_busysleep"; public static final String CRAWLJOB_LOCAL_CRAWL_LOADPREREQ = "50_localcrawl_loadprereq"; @@ -148,9 +139,6 @@ public final class SwitchboardConstants { *

Name of the remote triggered crawl thread, responsible for processing a remote crawl received from another peer

*/ public static final String CRAWLJOB_REMOTE_TRIGGERED_CRAWL = "62_remotetriggeredcrawl"; - public static final String CRAWLJOB_REMOTE_TRIGGERED_CRAWL_METHOD_START = "remoteTriggeredCrawlJob"; - public static final String CRAWLJOB_REMOTE_TRIGGERED_CRAWL_METHOD_JOBCOUNT = "remoteTriggeredCrawlJobSize"; - public static final String CRAWLJOB_REMOTE_TRIGGERED_CRAWL_METHOD_FREEMEM = null; public static final String CRAWLJOB_REMOTE_TRIGGERED_CRAWL_IDLESLEEP = "62_remotetriggeredcrawl_idlesleep"; public static final String CRAWLJOB_REMOTE_TRIGGERED_CRAWL_BUSYSLEEP = "62_remotetriggeredcrawl_busysleep"; // 70_surrogates @@ -163,18 +151,12 @@ public final class SwitchboardConstants { public static final String SURROGATES_LOADPREREQ = "70_surrogates_loadprereq"; public static final String SURROGATES_IDLESLEEP = "70_surrogates_idlesleep"; public static final String SURROGATES_BUSYSLEEP = "70_surrogates_busysleep"; - public static final String SURROGATES_METHOD_START = "surrogateProcess"; - public static final String SURROGATES_METHOD_JOBCOUNT = "surrogateQueueSize"; - public static final String SURROGATES_METHOD_FREEMEM = "surrogateFreeMem"; // 85_scheduler /** *

public static final String SCHEDULER = "85_scheduler"

*

The cleanup thread which is responsible for the start of scheduled processes from the API table

*/ public static final String SCHEDULER = "85_scheduler"; - public static final String SCHEDULER_METHOD_START = "schedulerJob"; - public static final String SCHEDULER_METHOD_JOBCOUNT = "schedulerJobSize"; - public static final String SCHEDULER_METHOD_FREEMEM = null; public static final String SCHEDULER_IDLESLEEP = "85_scheduler_idlesleep"; public static final String SCHEDULER_BUSYSLEEP = "85_scheduler_busysleep"; // 90_cleanup @@ -183,9 +165,6 @@ public final class SwitchboardConstants { *

The cleanup thread which is responsible for pendant cleanup-jobs, news/ranking distribution, etc.

*/ public static final String CLEANUP = "90_cleanup"; - public static final String CLEANUP_METHOD_START = "cleanupJob"; - public static final String CLEANUP_METHOD_JOBCOUNT = "cleanupJobSize"; - public static final String CLEANUP_METHOD_FREEMEM = null; public static final String CLEANUP_IDLESLEEP = "90_cleanup_idlesleep"; public static final String CLEANUP_BUSYSLEEP = "90_cleanup_busysleep"; /** diff --git a/source/net/yacy/search/query/SearchEvent.java b/source/net/yacy/search/query/SearchEvent.java index c56fd2b89..c2b44aa5b 100644 --- a/source/net/yacy/search/query/SearchEvent.java +++ b/source/net/yacy/search/query/SearchEvent.java @@ -521,6 +521,9 @@ public final class SearchEvent { SearchEventCache.put(this.query.id(false), this); } + /** + * A concurrent task to perform the current search query on the local RWI. + */ private class RWIProcess extends Thread { final Thread waitForThread; @@ -530,6 +533,9 @@ public final class SearchEvent { this.waitForThread = waitForThread; } + /** + * Query the local RWI and feed the search event with the obtained results. + */ @Override public void run() {