diff --git a/defaults/yacy.init b/defaults/yacy.init index 6157e8051..b24df970c 100644 --- a/defaults/yacy.init +++ b/defaults/yacy.init @@ -210,6 +210,12 @@ clientTimeout = 10000 # a limit on the number of concurrent connections httpdMaxBusySessions = 200 +# maximum number of simultaneously open outgoing HTTP connections in the general pool (net.yacy.cora.protocol.http.HTTPClient) +http.outgoing.pool.general.maxTotal = 200 + +# maximum number of simultaneously open outgoing HTTP connections in the remote Solr pool (net.yacy.cora.federate.solr.instance.RemoteInstance) +http.outgoing.pool.remoteSolr.maxTotal = 100 + # default root path for the file server # may be overridden by the htdocs parameter # users shall be encouraged to use the htdocs path for individual content, diff --git a/htroot/PerformanceQueues_p.html b/htroot/PerformanceQueues_p.html index 2f32ffb64..501fd98bc 100644 --- a/htroot/PerformanceQueues_p.html +++ b/htroot/PerformanceQueues_p.html @@ -160,7 +160,7 @@ -
+
Thread Pool Settings: @@ -185,6 +185,50 @@ + + +
Outgoing connections pools settings : +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Connection PoolTotal maximumCurrent statistics
ActiveIdlePending
General#[pool.general.leased]##[pool.general.available]##[pool.general.pending]#
Remote Solr servers#[pool.remoteSolr.leased]##[pool.remoteSolr.available]##[pool.remoteSolr.pending]#
+ + Changes take effect immediately +
+
+ + #%env/templates/footer.template%# diff --git a/htroot/PerformanceQueues_p.java b/htroot/PerformanceQueues_p.java index c2fe3b778..0771c5f15 100644 --- a/htroot/PerformanceQueues_p.java +++ b/htroot/PerformanceQueues_p.java @@ -28,8 +28,12 @@ import java.io.File; import java.util.Iterator; import java.util.Map; +import org.apache.http.pool.PoolStats; + +import net.yacy.cora.federate.solr.instance.RemoteInstance; import net.yacy.cora.protocol.ConnectionInfo; import net.yacy.cora.protocol.RequestHeader; +import net.yacy.cora.protocol.http.HTTPClient; import net.yacy.data.TransactionManager; import net.yacy.kelondro.data.word.WordReference; import net.yacy.kelondro.rwi.IndexCell; @@ -307,6 +311,25 @@ public class PerformanceQueues_p { sb.setConfig("httpdMaxBusySessions",maxBusy); } + + if ((post != null) && (post.containsKey("connectionPoolConfig"))) { + + /* Configure the general outgoing HTTP connection pool */ + int maxTotal = post.getInt(SwitchboardConstants.HTTP_OUTGOING_POOL_GENERAL_MAX_TOTAL, + SwitchboardConstants.HTTP_OUTGOING_POOL_GENERAL_MAX_TOTAL_DEFAULT); + if (maxTotal > 0) { + sb.setConfig(SwitchboardConstants.HTTP_OUTGOING_POOL_GENERAL_MAX_TOTAL, maxTotal); + HTTPClient.initPoolMaxConnections(HTTPClient.CONNECTION_MANAGER, maxTotal); + } + + /* Configure the remote Solr outgoing HTTP connection pool */ + maxTotal = post.getInt(SwitchboardConstants.HTTP_OUTGOING_POOL_REMOTE_SOLR_MAX_TOTAL, + SwitchboardConstants.HTTP_OUTGOING_POOL_REMOTE_SOLR_MAX_TOTAL_DEFAULT); + if (maxTotal > 0) { + sb.setConfig(SwitchboardConstants.HTTP_OUTGOING_POOL_REMOTE_SOLR_MAX_TOTAL, maxTotal); + RemoteInstance.initPoolMaxConnections(RemoteInstance.CONNECTION_MANAGER, maxTotal); + } + } if ((post != null) && (post.containsKey("onlineCautionSubmit"))) { sb.setConfig(SwitchboardConstants.PROXY_ONLINE_CAUTION_DELAY, Integer.toString(post.getInt("crawlPauseProxy", 30000))); @@ -344,6 +367,24 @@ public class PerformanceQueues_p { prop.put("pool", "3"); + /* Connection pools settings */ + prop.put(SwitchboardConstants.HTTP_OUTGOING_POOL_GENERAL_MAX_TOTAL, + sb.getConfigInt(SwitchboardConstants.HTTP_OUTGOING_POOL_GENERAL_MAX_TOTAL, + SwitchboardConstants.HTTP_OUTGOING_POOL_GENERAL_MAX_TOTAL_DEFAULT)); + prop.put(SwitchboardConstants.HTTP_OUTGOING_POOL_REMOTE_SOLR_MAX_TOTAL, + sb.getConfigInt(SwitchboardConstants.HTTP_OUTGOING_POOL_REMOTE_SOLR_MAX_TOTAL, + SwitchboardConstants.HTTP_OUTGOING_POOL_REMOTE_SOLR_MAX_TOTAL_DEFAULT)); + /* Connection pools stats */ + PoolStats stats = HTTPClient.CONNECTION_MANAGER.getTotalStats(); + prop.put("pool.general.leased", stats.getLeased()); + prop.put("pool.general.available", stats.getAvailable()); + prop.put("pool.general.pending", stats.getPending()); + + stats = RemoteInstance.CONNECTION_MANAGER.getTotalStats(); + prop.put("pool.remoteSolr.leased", stats.getLeased()); + prop.put("pool.remoteSolr.available", stats.getAvailable()); + prop.put("pool.remoteSolr.pending", stats.getPending()); + /* Remote searches max loads settings */ prop.put("remoteSearchRWIMaxLoad", sb.getConfigFloat(SwitchboardConstants.REMOTESEARCH_MAXLOAD_RWI, SwitchboardConstants.REMOTESEARCH_MAXLOAD_RWI_DEFAULT)); diff --git a/htroot/PerformanceQueues_p.xml b/htroot/PerformanceQueues_p.xml index d2a06e9ce..d68bbf6c5 100644 --- a/htroot/PerformanceQueues_p.xml +++ b/htroot/PerformanceQueues_p.xml @@ -44,4 +44,20 @@ #[numActive]# #{/pool}# + + + General + #[http.outgoing.pool.general.maxTotal]# + #[pool.general.leased]# + #[pool.general.available]# + #[pool.general.pending]# + + + Remote Solr servers + #[http.outgoing.pool.remoteSolr.maxTotal]# + #[pool.remoteSolr.leased]# + #[pool.remoteSolr.available]# + #[pool.remoteSolr.pending]# + + \ No newline at end of file diff --git a/source/net/yacy/cora/federate/solr/instance/RemoteInstance.java b/source/net/yacy/cora/federate/solr/instance/RemoteInstance.java index 1a97d9aae..7121d879c 100644 --- a/source/net/yacy/cora/federate/solr/instance/RemoteInstance.java +++ b/source/net/yacy/cora/federate/solr/instance/RemoteInstance.java @@ -65,6 +65,7 @@ import net.yacy.cora.document.id.MultiProtocolURL; import net.yacy.cora.protocol.HeaderFramework; import net.yacy.cora.util.CommonPattern; import net.yacy.cora.util.ConcurrentLog; +import net.yacy.cora.util.Memory; import net.yacy.kelondro.util.MemoryControl; import net.yacy.search.schema.CollectionSchema; import net.yacy.search.schema.WebgraphSchema; @@ -75,8 +76,8 @@ import net.yacy.search.schema.WebgraphSchema; @SuppressWarnings("deprecation") public class RemoteInstance implements SolrInstance { - /** The connection manager used to handle the common HTTP connections pool. */ - private static final org.apache.http.impl.conn.PoolingClientConnectionManager CONNECTION_MANAGER = buildConnectionManager(); + /** The connection manager holding the HTTP connections pool shared between remote Solr clients. */ + public static final org.apache.http.impl.conn.PoolingClientConnectionManager CONNECTION_MANAGER = buildConnectionManager(); /** A custom scheme registry allowing https connections to servers using self-signed certificate */ private static final SchemeRegistry SCHEME_REGISTRY = buildTrustSelfSignedSchemeRegistry(); @@ -205,6 +206,29 @@ public class RemoteInstance implements SolrInstance { if (this.defaultServer == null) throw new IOException("cannot connect to url " + url + " and connect core " + defaultCoreName); } + /** + * Initialize the maximum connections for the given pool + * + * @param pool + * a pooling connection manager. Must not be null. + * @param maxConnections. + * The new maximum connections values. Must be greater than 0. + * @throws IllegalArgumentException + * when pool is null or when maxConnections is lower than 1 + */ + public static void initPoolMaxConnections(final org.apache.http.impl.conn.PoolingClientConnectionManager pool, int maxConnections) { + if (pool == null) { + throw new IllegalArgumentException("pool parameter must not be null"); + } + if (maxConnections <= 0) { + throw new IllegalArgumentException("maxConnections parameter must be greater than zero"); + } + pool.setMaxTotal(maxConnections); + + /* max connections per host */ + pool.setDefaultMaxPerRoute((int) (2 * Memory.cores())); + } + /** * @return a connection manager with a HTTP connection pool */ @@ -212,12 +236,8 @@ public class RemoteInstance implements SolrInstance { /* Important note : use of deprecated Apache classes is required because SolrJ still use them internally (see HttpClientUtil). * Upgrade only when Solr implementation will become compatible */ - org.apache.http.impl.conn.PoolingClientConnectionManager cm; - - cm = new org.apache.http.impl.conn.PoolingClientConnectionManager(); // try also: ThreadSafeClientConnManager - - cm.setMaxTotal(100); - cm.setDefaultMaxPerRoute(100); + final org.apache.http.impl.conn.PoolingClientConnectionManager cm = new org.apache.http.impl.conn.PoolingClientConnectionManager(); + initPoolMaxConnections(cm, 100); return cm; } diff --git a/source/net/yacy/cora/protocol/http/HTTPClient.java b/source/net/yacy/cora/protocol/http/HTTPClient.java index eb1837787..8bce7f5c0 100644 --- a/source/net/yacy/cora/protocol/http/HTTPClient.java +++ b/source/net/yacy/cora/protocol/http/HTTPClient.java @@ -118,9 +118,14 @@ import net.yacy.kelondro.util.NamePrefixThreadFactory; public class HTTPClient { private final static int default_timeout = 6000; + /** Maximum number of simultaneously open outgoing HTTP connections in the pool */ private final static int maxcon = 200; private static IdleConnectionMonitorThread connectionMonitor = null; private final static RequestConfig dfltReqConf = initRequestConfig(); + + /** The connection manager holding the configured connection pool for this client */ + public static final PoolingHttpClientConnectionManager CONNECTION_MANAGER = initPoolingConnectionManager(); + private final static HttpClientBuilder clientBuilder = initClientBuilder(); private final RequestConfig.Builder reqConfBuilder; private Set> headers = null; @@ -171,7 +176,7 @@ public class HTTPClient { private static HttpClientBuilder initClientBuilder() { final HttpClientBuilder builder = HttpClientBuilder.create(); - builder.setConnectionManager(initPoolingConnectionManager()); + builder.setConnectionManager(CONNECTION_MANAGER); builder.setDefaultRequestConfig(dfltReqConf); // UserAgent @@ -210,15 +215,8 @@ public class HTTPClient { if (ip == null) throw new UnknownHostException(host0); return new InetAddress[]{ip}; }}); - // how much connections do we need? - default: 20 - pooling.setMaxTotal(maxcon); - // for statistics same value should also be set here - ConnectionInfo.setMaxcount(maxcon); - // connections per host (2 default) - pooling.setDefaultMaxPerRoute((int) (2 * Memory.cores())); - // Increase max connections for localhost - final HttpHost localhost = new HttpHost(Domains.LOCALHOST); - pooling.setMaxPerRoute(new HttpRoute(localhost), maxcon); + initPoolMaxConnections(pooling, maxcon); + pooling.setValidateAfterInactivity(default_timeout); // on init set to default 5000ms final SocketConfig socketConfig = SocketConfig.custom() // Defines whether the socket can be bound even though a previous connection is still in a timeout state. @@ -237,6 +235,35 @@ public class HTTPClient { return pooling; } + + /** + * Initialize the maximum connections for the given pool + * + * @param pool + * a pooling connection manager. Must not be null. + * @param maxConnections. + * The new maximum connections values. Must be greater than 0. + * @throws IllegalArgumentException + * when pool is null or when maxConnections is lower than 1 + */ + public static void initPoolMaxConnections(final PoolingHttpClientConnectionManager pool, int maxConnections) { + if (pool == null) { + throw new IllegalArgumentException("pool parameter must not be null"); + } + if (maxConnections <= 0) { + throw new IllegalArgumentException("maxConnections parameter must be greater than zero"); + } + pool.setMaxTotal(maxConnections); + // for statistics same value should also be set here + ConnectionInfo.setMaxcount(maxConnections); + + // connections per host (2 default) + pool.setDefaultMaxPerRoute((int) (2 * Memory.cores())); + + // Increase max connections for localhost + final HttpHost localhost = new HttpHost(Domains.LOCALHOST); + pool.setMaxPerRoute(new HttpRoute(localhost), maxConnections); + } /** * This method should be called just before shutdown diff --git a/source/net/yacy/search/Switchboard.java b/source/net/yacy/search/Switchboard.java index 46fc9c37e..c847b1632 100644 --- a/source/net/yacy/search/Switchboard.java +++ b/source/net/yacy/search/Switchboard.java @@ -470,6 +470,9 @@ public final class Switchboard extends serverSwitch { // set a high maximum cache size to current size; this is adopted later automatically final int wordCacheMaxCount = (int) getConfigLong(SwitchboardConstants.WORDCACHE_MAX_COUNT, 20000); setConfig(SwitchboardConstants.WORDCACHE_MAX_COUNT, Integer.toString(wordCacheMaxCount)); + + /* Init outgoing connections pools with user defined settings */ + initOutgoingConnectionPools(); // load the network definition try { @@ -1270,6 +1273,29 @@ public final class Switchboard extends serverSwitch { this.log.config("Finished Switchboard Initialization"); } + /** + * Initialize outgoing connections pools with user defined settings + */ + private void initOutgoingConnectionPools() { + int generalPoolMaxTotal = getConfigInt(SwitchboardConstants.HTTP_OUTGOING_POOL_GENERAL_MAX_TOTAL, + SwitchboardConstants.HTTP_OUTGOING_POOL_GENERAL_MAX_TOTAL_DEFAULT); + if (generalPoolMaxTotal <= 0) { + /* Fix eventually wrong value from the config file */ + generalPoolMaxTotal = SwitchboardConstants.HTTP_OUTGOING_POOL_GENERAL_MAX_TOTAL_DEFAULT; + setConfig(SwitchboardConstants.HTTP_OUTGOING_POOL_GENERAL_MAX_TOTAL, generalPoolMaxTotal); + } + HTTPClient.initPoolMaxConnections(HTTPClient.CONNECTION_MANAGER, generalPoolMaxTotal); + + int remoteSolrPoolMaxTotal = getConfigInt(SwitchboardConstants.HTTP_OUTGOING_POOL_REMOTE_SOLR_MAX_TOTAL, + SwitchboardConstants.HTTP_OUTGOING_POOL_REMOTE_SOLR_MAX_TOTAL_DEFAULT); + if (remoteSolrPoolMaxTotal <= 0) { + /* Fix eventually wrong value from the config file */ + remoteSolrPoolMaxTotal = SwitchboardConstants.HTTP_OUTGOING_POOL_REMOTE_SOLR_MAX_TOTAL_DEFAULT; + setConfig(SwitchboardConstants.HTTP_OUTGOING_POOL_REMOTE_SOLR_MAX_TOTAL, remoteSolrPoolMaxTotal); + } + RemoteInstance.initPoolMaxConnections(RemoteInstance.CONNECTION_MANAGER, remoteSolrPoolMaxTotal); + } + final String getSysinfo() { return getConfig(SwitchboardConstants.NETWORK_NAME, "") + (isRobinsonMode() ? "-" : "/") + getConfig(SwitchboardConstants.NETWORK_DOMAIN, "global"); } diff --git a/source/net/yacy/search/SwitchboardConstants.java b/source/net/yacy/search/SwitchboardConstants.java index 98792d9b2..b1145057e 100644 --- a/source/net/yacy/search/SwitchboardConstants.java +++ b/source/net/yacy/search/SwitchboardConstants.java @@ -490,6 +490,19 @@ public final class SwitchboardConstants { /** Default setting value controlling whether HTTP responses should be compressed */ public static final boolean SERVER_RESPONSE_COMPRESS_GZIP_DEFAULT = true; + + + /** Key of the setting controlling the maximum number of simultaneously open outgoing HTTP connections in the general pool (net.yacy.cora.protocol.http.HTTPClient) */ + public static final String HTTP_OUTGOING_POOL_GENERAL_MAX_TOTAL = "http.outgoing.pool.general.maxTotal"; + + /** Default setting value controlling the maximum number of simultaneously open outgoing HTTP connections in the general pool */ + public static final int HTTP_OUTGOING_POOL_GENERAL_MAX_TOTAL_DEFAULT = 200; + + /** Key of the setting controlling the maximum number of simultaneously open outgoing HTTP connections in the remote Solr pool (net.yacy.cora.federate.solr.instance.RemoteInstance) */ + public static final String HTTP_OUTGOING_POOL_REMOTE_SOLR_MAX_TOTAL = "http.outgoing.pool.remoteSolr.maxTotal"; + + /** Default setting value controlling the maximum number of simultaneously open outgoing HTTP connections in the remote Solr pool */ + public static final int HTTP_OUTGOING_POOL_REMOTE_SOLR_MAX_TOTAL_DEFAULT = 100; /*