From ca8b100f96c272f2853a0c129eb78ca276ca3d45 Mon Sep 17 00:00:00 2001 From: Michael Peter Christen Date: Mon, 10 Feb 2014 12:59:30 +0100 Subject: [PATCH] run the cleanup process even when load is high, do postprocessing even if load > 1 (but < 2) but only if there is enough memory (now: 0.5 GB RAM available). The memory amount of the postprocessing is the cause that systems block because they run into a frequent-GC chain which almost locks the peer. If running with enough memory, the postprocessing is fast and not damaging to the system. Because the required RAM of 0.5 GB is never available in default setting, the postprocessing will not run if the peer is not reconfigured to use more memory. --- defaults/yacy.init | 6 +++--- source/net/yacy/search/Switchboard.java | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/defaults/yacy.init b/defaults/yacy.init index ce109aab7..f3c75cfee 100644 --- a/defaults/yacy.init +++ b/defaults/yacy.init @@ -632,11 +632,11 @@ collection=user 720_ccimport_idlesleep=100 720_ccimport_busysleep=1000 720_ccimport_memprereq=1048576 -720_ccimport_loadprereq=9.0 +720_ccimport_loadprereq=8.0 730_ccfilter_idlesleep=100 730_ccfilter_busysleep=1000 730_ccfilter_memprereq=1048576 -730_ccfilter_loadprereq=9.0 +730_ccfilter_loadprereq=8.0 80_searchresult_idlesleep=10000 80_searchresult_busysleep=200 80_searchresult_memprereq=0 @@ -644,7 +644,7 @@ collection=user 90_cleanup_idlesleep=300000 90_cleanup_busysleep=300000 90_cleanup_memprereq=0 -90_cleanup_loadprereq=3.0 +90_cleanup_loadprereq=16.0 # additional attributes: # performanceIO is a percent-value. a value of 10 means, that 10% of the busysleep time diff --git a/source/net/yacy/search/Switchboard.java b/source/net/yacy/search/Switchboard.java index 09a6abb30..3f4c57686 100644 --- a/source/net/yacy/search/Switchboard.java +++ b/source/net/yacy/search/Switchboard.java @@ -2309,7 +2309,7 @@ public final class Switchboard extends serverSwitch { Fulltext fulltext = index.fulltext(); CollectionConfiguration collection1Configuration = fulltext.getDefaultConfiguration(); WebgraphConfiguration webgraphConfiguration = fulltext.getWebgraphConfiguration(); - if (!this.crawlJobIsPaused(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL) && MemoryControl.request(256000000L, false) && Memory.load() < 1.0f) { + if (!this.crawlJobIsPaused(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL) && MemoryControl.available() > 512L * 1024L * 1024L && Memory.load() < 2.0f) { // we optimize first because that is useful for postprocessing int proccount = 0;