diff --git a/htroot/PerformanceQueues_p.java b/htroot/PerformanceQueues_p.java index 64d38e86f..4423fe647 100644 --- a/htroot/PerformanceQueues_p.java +++ b/htroot/PerformanceQueues_p.java @@ -224,6 +224,25 @@ public class PerformanceQueues_p { switchboard.setConfig("httpdMaxActiveSessions",maxActive); switchboard.setConfig("httpdMaxIdleSessions",maxIdle); switchboard.setConfig("httpdMinIdleSessions",minIdle); + + /* + * Configuring the crawlStacker pool + */ + GenericObjectPool.Config stackerPoolConfig = switchboard.sbStackCrawlThread.getPoolConfig(); + maxActive = Integer.parseInt(post.get("CrawlStacker Session Pool_maxActive","10")); + maxIdle = Integer.parseInt(post.get("CrawlStacker Session Pool_maxIdle","10")); + minIdle = Integer.parseInt(post.get("CrawlStacker Session Pool_minIdle","5")); + + stackerPoolConfig.minIdle = (minIdle > maxIdle) ? maxIdle/2 : minIdle; + stackerPoolConfig.maxIdle = (maxIdle > maxActive) ? maxActive/2 : maxIdle; + stackerPoolConfig.maxActive = maxActive; + + switchboard.sbStackCrawlThread.setPoolConfig(stackerPoolConfig); + + // storing the new values into configfile + switchboard.setConfig("stacker.MaxActiveThreads",maxActive); + switchboard.setConfig("stacker.MaxIdleThreads",maxIdle); + switchboard.setConfig("stacker.MinIdleThreads",minIdle); } if ((post != null) && (post.containsKey("proxyControlSubmit"))) { @@ -266,9 +285,14 @@ public class PerformanceQueues_p { prop.put("pool_1_name","httpd Session Pool"); prop.put("pool_1_maxActive",httpdPoolConfig.maxActive); prop.put("pool_1_maxIdle",httpdPoolConfig.maxIdle); - prop.put("pool_1_minIdle",httpdPoolConfig.minIdle); - prop.put("pool",2); + prop.put("pool_1_minIdle",httpdPoolConfig.minIdle); + GenericObjectPool.Config stackerPoolConfig = switchboard.sbStackCrawlThread.getPoolConfig(); + prop.put("pool_2_name","CrawlStacker Session Pool"); + prop.put("pool_2_maxActive",stackerPoolConfig.maxActive); + prop.put("pool_2_maxIdle",stackerPoolConfig.maxIdle); + prop.put("pool_2_minIdle",stackerPoolConfig.minIdle); + prop.put("pool",3); // return rewrite values for templates return prop; diff --git a/source/de/anomic/plasma/plasmaCrawlStacker.java b/source/de/anomic/plasma/plasmaCrawlStacker.java index d02469f66..6df5cc1eb 100644 --- a/source/de/anomic/plasma/plasmaCrawlStacker.java +++ b/source/de/anomic/plasma/plasmaCrawlStacker.java @@ -72,6 +72,7 @@ import de.anomic.yacy.yacyCore; public final class plasmaCrawlStacker { final WorkerPool theWorkerPool; + private GenericObjectPool.Config theWorkerPoolConfig = null; final ThreadGroup theWorkerThreadGroup = new ThreadGroup("stackCrawlThreadGroup"); final serverLog log = new serverLog("STACKCRAWL"); final plasmaSwitchboard sb; @@ -85,10 +86,40 @@ public final class plasmaCrawlStacker { this.log.logInfo(this.queue.size() + " entries in the stackCrawl queue."); this.log.logInfo("STACKCRAWL thread initialized."); - this.theWorkerPool = new WorkerPool(new WorkterFactory(this.theWorkerThreadGroup)); + // configuring the thread pool + // implementation of session thread pool + this.theWorkerPoolConfig = new GenericObjectPool.Config(); + + // The maximum number of active connections that can be allocated from pool at the same time, + // 0 for no limit + this.theWorkerPoolConfig.maxActive = Integer.parseInt(sb.getConfig("stacker.MaxActiveThreads","50")); + + // The maximum number of idle connections connections in the pool + // 0 = no limit. + this.theWorkerPoolConfig.maxIdle = Integer.parseInt(sb.getConfig("stacker.MaxIdleThreads","10")); + this.theWorkerPoolConfig.minIdle = Integer.parseInt(sb.getConfig("stacker.MinIdleThreads","5")); + + // block undefinitely + this.theWorkerPoolConfig.maxWait = -1; + + // Action to take in case of an exhausted DBCP statement pool + // 0 = fail, 1 = block, 2= grow + this.theWorkerPoolConfig.whenExhaustedAction = GenericObjectPool.WHEN_EXHAUSTED_BLOCK; + this.theWorkerPoolConfig.minEvictableIdleTimeMillis = 30000; + + // creating worker pool + this.theWorkerPool = new WorkerPool(new WorkterFactory(this.theWorkerThreadGroup),this.theWorkerPoolConfig); } + public GenericObjectPool.Config getPoolConfig() { + return this.theWorkerPoolConfig; + } + + public void setPoolConfig(GenericObjectPool.Config newConfig) { + this.theWorkerPool.setConfig(newConfig); + } + public void close() { try { this.log.logFine("Shutdown. Terminationg worker threads."); diff --git a/yacy.init b/yacy.init index b3cb485d6..eb4393485 100644 --- a/yacy.init +++ b/yacy.init @@ -559,6 +559,11 @@ crawler.MaxActiveThreads = 10 crawler.MaxIdleThreads = 7 crawler.MinIdleThreads = 5 +# maximum number of crawl-stacker threads +stacker.MaxActiveThreads = 50 +stacker.MaxIdleThreads = 10 +stacker.MinIdleThreads = 5 + # maximum size of indexing queue indexer.slots = 100