boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

- disabled redirects in proxy (so client sees real path)

Browse Source 
- added connection stats (only connections currently in use)
- remove "old" connections (closed or idle for some time)
- synchronized shared parts of proxyHandler


git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4682 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
danielr 17 years ago
parent 8fe39ebd74
commit 959f448e5f
8 changed files with 287 additions and 115 deletions
Show Stats Download Patch File Download Diff File

3
htroot/Connections_p.html
Unescape View File

@ -36,8 +36,7 @@
</table> </table>
<h3>Outgoing Connections</h3> <h3>Outgoing Connections</h3>
<p><strong>Details currently not available!</strong></p> <p>Showing #[clientActive]# pooled outgoing connections used as:</p>
<p>Showing #[clientActive]# active outgoing connections:</p>
<table border="0" cellpadding="2" cellspacing="1"> <table border="0" cellpadding="2" cellspacing="1">
<tr class="TableHeader" valign="bottom"> <tr class="TableHeader" valign="bottom">
<td>Protocol</td> <td>Protocol</td>

3
htroot/Connections_p.java
Unescape View File

@ -50,6 +50,7 @@ import java.io.UnsupportedEncodingException;
import java.net.InetAddress; import java.net.InetAddress;
import java.net.URLEncoder; import java.net.URLEncoder;
import java.util.Properties; import java.util.Properties;
import java.util.Set;
import de.anomic.http.HttpConnectionInfo; import de.anomic.http.HttpConnectionInfo;
import de.anomic.http.JakartaCommonsHttpClient; import de.anomic.http.JakartaCommonsHttpClient;
@ -229,7 +230,7 @@ public final class Connections_p {
prop.putNum("numActivePending", numActivePending); prop.putNum("numActivePending", numActivePending);
// client sessions // client sessions
HttpConnectionInfo[] a = JakartaCommonsHttpClient.allConnections(); Set<HttpConnectionInfo> a = HttpConnectionInfo.getAllConnections();
// TODO sorting // TODO sorting
// Arrays.sort(a, httpc.connectionTimeComparatorInstance); // Arrays.sort(a, httpc.connectionTimeComparatorInstance);
int c = 0; int c = 0;

111
source/de/anomic/http/HttpConnectionInfo.java
Unescape View File

@ -26,7 +26,9 @@
package de.anomic.http; package de.anomic.http;
import org.apache.commons.httpclient.HttpConnection; import java.util.Collections;
import java.util.HashSet;
import java.util.Set;
/** /**
* Information about a connection * Information about a connection
@ -35,23 +37,18 @@ import org.apache.commons.httpclient.HttpConnection;
* @since 07.04.2008 * @since 07.04.2008
*/ */
public class HttpConnectionInfo { public class HttpConnectionInfo {
/**
* a list of all current connections
*/
private final static Set<HttpConnectionInfo> allConnections = Collections
.synchronizedSet(new HashSet<HttpConnectionInfo>());
private final String protocol; private final String protocol;
private final String targetHost; private final String targetHost;
private final String command; private final String command;
private final int id; private final int id;
private final long initTime; private final long initTime;
/**
* constructor using org.apache.commons.httpclient.HttpConnection
*
* @param connection
*/
public HttpConnectionInfo(final HttpConnection connection) {
this(connection.getProtocol().toString(), ((connection.getPort() == 80) ? connection.getHost()
: connection.getHost() + ":" + connection.getPort()), "unknown command", connection.hashCode(),
System.currentTimeMillis());
}
/** /**
* constructor setting all data * constructor setting all data
* *
@ -111,4 +108,94 @@ public class HttpConnectionInfo {
public int getID() { public int getID() {
return id; return id;
} }
/**
* @return the allConnections
*/
public static Set<HttpConnectionInfo> getAllConnections() {
return allConnections;
}
/**
* add a connection to the list of all current connections
*
* @param conInfo
*/
public static void addConnection(final HttpConnectionInfo conInfo) {
allConnections.add(conInfo);
}
/**
* remove a connection from the list of all current connections
*
* @param conInfo
*/
public static void removeConnection(final HttpConnectionInfo conInfo) {
allConnections.remove(conInfo);
}
/**
* connections with same id {@link equals()} another
*
* @param id
*/
public static void removeConnection(final int id) {
removeConnection(new HttpConnectionInfo(null, null, null, id, 0));
}
/* (non-Javadoc)
* @see java.lang.Object#toString()
*/
@Override
public String toString() {
StringBuilder string = new StringBuilder(50);
string.append("ID ");
string.append(getID());
string.append(", ");
string.append(getProtocol());
string.append("://");
string.append(getTargetHost());
string.append(" ");
string.append(getCommand());
string.append(", since ");
string.append(getLifetime());
string.append(" ms");
return string.toString();
}
/*
* (non-Javadoc)
*
* @see java.lang.Object#hashCode()
*/
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + id;
return result;
}
/*
* (non-Javadoc)
*
* @see java.lang.Object#equals(java.lang.Object)
*/
@Override
public boolean equals(final Object obj) {
if (this == obj) {
return true;
}
if (obj == null) {
return false;
}
if (getClass() != obj.getClass()) {
return false;
}
final HttpConnectionInfo other = (HttpConnectionInfo) obj;
if (id != other.id) {
return false;
}
return true;
}
} }

111
source/de/anomic/http/JakartaCommonsHttpClient.java
Unescape View File

@ -41,6 +41,7 @@ import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpException; import org.apache.commons.httpclient.HttpException;
import org.apache.commons.httpclient.HttpMethod; import org.apache.commons.httpclient.HttpMethod;
import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager; import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager;
import org.apache.commons.httpclient.URIException;
import org.apache.commons.httpclient.methods.GetMethod; import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.methods.HeadMethod; import org.apache.commons.httpclient.methods.HeadMethod;
import org.apache.commons.httpclient.methods.InputStreamRequestEntity; import org.apache.commons.httpclient.methods.InputStreamRequestEntity;
@ -77,14 +78,18 @@ public class JakartaCommonsHttpClient {
* set options for client * set options for client
*/ */
// set user-agent // set user-agent
yacyVersion thisversion = yacyVersion.thisVersion(); final yacyVersion thisversion = yacyVersion.thisVersion();
apacheHttpClient.getParams().setParameter(HttpMethodParams.USER_AGENT, apacheHttpClient.getParams().setParameter(
HttpMethodParams.USER_AGENT,
"yacy/" + ((thisversion == null) ? "0.0" : thisversion.releaseNr) + "yacy/" + ((thisversion == null) ? "0.0" : thisversion.releaseNr) +
" (www.yacy.net; " + " (www.yacy.net; " +
de.anomic.http.HttpClient.getSystemOST() + ") " + de.anomic.http.HttpClient.getSystemOST() + ") " +
getCurrentUserAgent().replace(';', ':')); // last ; must be before location (this is parsed) getCurrentUserAgent().replace(';', ':')); // last ; must be
// before location
// (this is parsed)
// only one retry // only one retry
apacheHttpClient.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, new DefaultHttpMethodRetryHandler(1, false)); apacheHttpClient.getParams().setParameter(HttpMethodParams.RETRY_HANDLER,
new DefaultHttpMethodRetryHandler(1, false));
/** /**
* set options for connection manager * set options for connection manager
@ -96,29 +101,37 @@ public class JakartaCommonsHttpClient {
// TODO should this be configurable? // TODO should this be configurable?
// accept self-signed or untrusted certificates // accept self-signed or untrusted certificates
Protocol.registerProtocol("https", new Protocol("https", (ProtocolSocketFactory)new EasySSLProtocolSocketFactory(), 443)); Protocol.registerProtocol("https", new Protocol("https",
(ProtocolSocketFactory) new EasySSLProtocolSocketFactory(), 443));
/** /**
* set network timeout properties. * set network timeout properties. see: http://java.sun.com/j2se/1.5.0/docs/guide/net/properties.html These
* see: http://java.sun.com/j2se/1.5.0/docs/guide/net/properties.html * properties specify the default connect and read timeout (resp.) for the protocol handler used by
* These properties specify the default connect and read timeout (resp.) * java.net.URLConnection. the java.net.URLConnection is also used by JakartaCommons HttpClient, see
* for the protocol handler used by java.net.URLConnection.
* the java.net.URLConnection is also used by JakartaCommons HttpClient, see
* http://hc.apache.org/httpclient-3.x/apidocs/org/apache/commons/httpclient/util/HttpURLConnection.html * http://hc.apache.org/httpclient-3.x/apidocs/org/apache/commons/httpclient/util/HttpURLConnection.html
*/ */
// specify the timeout, in milliseconds, to establish the connection to the host. // specify the timeout, in milliseconds, to establish the connection to the host.
// For HTTP connections, it is the timeout when establishing the connection to the HTTP server. // For HTTP connections, it is the timeout when establishing the connection to the HTTP server.
System.setProperty("sun.net.client.defaultConnectTimeout","10000"); System.setProperty("sun.net.client.defaultConnectTimeout", "10000");
// specify the response timeout, in milliseconds, when reading from an input stream // specify the response timeout, in milliseconds, when reading from an input stream
// after a connection is established with a resource // after a connection is established with a resource
System.setProperty("sun.net.client.defaultReadTimeout","60000"); System.setProperty("sun.net.client.defaultReadTimeout", "60000");
} }
/**
* every x milliseconds do a cleanup (close old connections)
*/
private final static int cleanupIntervall = 60000;
/**
* time the last cleanup was started
*/
private static long lastCleanup = 0;
private final Map<HttpMethod, InputStream> openStreams = new HashMap<HttpMethod, InputStream>(); private final Map<HttpMethod, InputStream> openStreams = new HashMap<HttpMethod, InputStream>();
private Header[] headers = new Header[0]; private Header[] headers = new Header[0];
private httpRemoteProxyConfig proxyConfig = null; private httpRemoteProxyConfig proxyConfig = null;
private boolean followRedirects = true;
/** /**
* constructs a new Client with given parameters * constructs a new Client with given parameters
@ -163,6 +176,15 @@ public class JakartaCommonsHttpClient {
apacheHttpClient.getParams().setIntParameter(HttpMethodParams.SO_TIMEOUT, timeout); apacheHttpClient.getParams().setIntParameter(HttpMethodParams.SO_TIMEOUT, timeout);
} }
/**
* should redirects automatically be followed?
*
* @param follow
*/
public void setFollowRedirects(final boolean follow) {
followRedirects = follow;
}
/* /*
* (non-Javadoc) * (non-Javadoc)
* *
@ -242,7 +264,8 @@ public class JakartaCommonsHttpClient {
if (file.isFile() && file.canRead()) { if (file.isFile() && file.canRead()) {
// read file // read file
final ByteArrayOutputStream fileData = new ByteArrayOutputStream(); final ByteArrayOutputStream fileData = new ByteArrayOutputStream();
serverFileUtils.copyToStream(new BufferedInputStream(new FileInputStream(file)), new BufferedOutputStream(fileData)); serverFileUtils.copyToStream(new BufferedInputStream(new FileInputStream(file)),
new BufferedOutputStream(fileData));
value = fileData.toByteArray(); value = fileData.toByteArray();
} }
} }
@ -368,39 +391,66 @@ public class JakartaCommonsHttpClient {
*/ */
private JakartaCommonsHttpResponse execute(final HttpMethod method) throws IOException, HttpException { private JakartaCommonsHttpResponse execute(final HttpMethod method) throws IOException, HttpException {
assert method != null : "precondition violated: method != null"; assert method != null : "precondition violated: method != null";
method.setFollowRedirects(followRedirects);
// set header // set header
for (final Header header : headers) { for (final Header header : headers) {
method.setRequestHeader(header); method.setRequestHeader(header);
} }
// set proxy // set proxy
final httpRemoteProxyConfig proxyConfig = getProxyConfig(method.getURI().getHost()); final httpRemoteProxyConfig proxyConfig = getProxyConfig(method.getURI().getHost());
addProxyAuth(method, proxyConfig); addProxyAuth(method, proxyConfig);
final HostConfiguration hostConfig = getProxyHostConfig(proxyConfig); final HostConfiguration hostConfig = getProxyHostConfig(proxyConfig);
// statistics
HttpConnectionInfo.addConnection(generateConInfo(method));
// execute (send request) // execute (send request)
if (hostConfig == null) { if (hostConfig == null) {
apacheHttpClient.executeMethod(method); apacheHttpClient.executeMethod(method);
} else { } else {
apacheHttpClient.executeMethod(hostConfig, method); apacheHttpClient.executeMethod(hostConfig, method);
} }
// return response // return response
return new JakartaCommonsHttpResponse(method); return new JakartaCommonsHttpResponse(method);
} }
/**
* @param method
* @return
*/
private HttpConnectionInfo generateConInfo(final HttpMethod method) {
int port = 80;
String host = null;
String protocol = null;
try {
port = method.getURI().getPort();
host = method.getURI().getHost();
protocol = method.getURI().getScheme();
} catch (final URIException e) {
// should not happen, because method is already executed
}
return new HttpConnectionInfo(protocol, (port == 80) ? host : host + ":" + port, method.getName() + " " +
method.getPath() + "?" + method.getQueryString(), method.hashCode(), System.currentTimeMillis());
}
/** /**
* if necessary adds a header for proxy-authentication * if necessary adds a header for proxy-authentication
* *
* @param method * @param method
* @param proxyConfig * @param proxyConfig
*/ */
private void addProxyAuth(HttpMethod method, httpRemoteProxyConfig proxyConfig) { private void addProxyAuth(final HttpMethod method, final httpRemoteProxyConfig proxyConfig) {
if(proxyConfig != null && proxyConfig.useProxy()) { if (proxyConfig != null && proxyConfig.useProxy()) {
final String remoteProxyUser = proxyConfig.getProxyUser(); final String remoteProxyUser = proxyConfig.getProxyUser();
if (remoteProxyUser != null && remoteProxyUser.length() > 0) { if (remoteProxyUser != null && remoteProxyUser.length() > 0) {
if (remoteProxyUser.contains(":")) { if (remoteProxyUser.contains(":")) {
serverLog.logWarning("HTTPC", "Proxy authentication contains invalid characters, trying anyway"); serverLog.logWarning("HTTPC", "Proxy authentication contains invalid characters, trying anyway");
} }
final String remoteProxyPwd = proxyConfig.getProxyPwd(); final String remoteProxyPwd = proxyConfig.getProxyPwd();
final String credentials = kelondroBase64Order.standardCoder.encodeString(remoteProxyUser.replace(":", "") + final String credentials = kelondroBase64Order.standardCoder.encodeString(remoteProxyUser.replace(":",
"") +
":" + remoteProxyPwd); ":" + remoteProxyPwd);
method.setRequestHeader(httpHeader.PROXY_AUTHORIZATION, "Basic " + credentials); method.setRequestHeader(httpHeader.PROXY_AUTHORIZATION, "Basic " + credentials);
} }
@ -432,7 +482,7 @@ public class JakartaCommonsHttpClient {
// generate http-configuration // generate http-configuration
if (proxyConfig != null && proxyConfig.useProxy()) { if (proxyConfig != null && proxyConfig.useProxy()) {
// new config based on client (default) // new config based on client (default)
HostConfiguration hostConfig = new HostConfiguration(apacheHttpClient.getHostConfiguration()); final HostConfiguration hostConfig = new HostConfiguration(apacheHttpClient.getHostConfiguration());
// add proxy // add proxy
hostConfig.setProxy(proxyConfig.getProxyHost(), proxyConfig.getProxyPort()); hostConfig.setProxy(proxyConfig.getProxyHost(), proxyConfig.getProxyPort());
return hostConfig; return hostConfig;
@ -447,9 +497,10 @@ public class JakartaCommonsHttpClient {
* @param date The Date-Object to be converted. * @param date The Date-Object to be converted.
* @return String with the date. * @return String with the date.
*/ */
public static String date2String(Date date) { public static String date2String(final Date date) {
if (date == null) if (date == null) {
return ""; return "";
}
return DateUtil.formatDate(date); return DateUtil.formatDate(date);
} }
@ -524,20 +575,24 @@ public class JakartaCommonsHttpClient {
} }
/** /**
* a list of all connections (not yet implemented) * number of active connections
* *
* @return * @return
*/ */
public static HttpConnectionInfo[] allConnections() { public static int connectionCount() {
return new HttpConnectionInfo[0]; return conManager.getConnectionsInPool();
} }
/** /**
* number of active connections * remove unused connections
*
* @return
*/ */
public static int connectionCount() { public static void cleanup() {
return conManager.getConnectionsInPool(); // do it only once a while
final long now = System.currentTimeMillis();
if(now - lastCleanup > cleanupIntervall) {
lastCleanup = now;
conManager.closeIdleConnections(120000);
conManager.deleteClosedConnections();
}
} }
} }

14
source/de/anomic/http/JakartaCommonsHttpResponse.java
Unescape View File

@ -88,19 +88,19 @@ public class JakartaCommonsHttpResponse {
* @throws IOException * @throws IOException
*/ */
public byte[] getData() throws IOException { public byte[] getData() throws IOException {
if (this.responseBody == null) { if (responseBody == null) {
InputStream instream = null; InputStream instream = null;
try { try {
instream = getDataAsStream(); instream = getDataAsStream();
if (instream != null) { if (instream != null) {
ByteArrayOutputStream outstream = new ByteArrayOutputStream(); final ByteArrayOutputStream outstream = new ByteArrayOutputStream();
byte[] buffer = new byte[4096]; final byte[] buffer = new byte[4096];
int len; int len;
while ((len = instream.read(buffer)) > 0) { while ((len = instream.read(buffer)) > 0) {
outstream.write(buffer, 0, len); outstream.write(buffer, 0, len);
} }
outstream.close(); outstream.close();
this.responseBody = outstream.toByteArray(); responseBody = outstream.toByteArray();
} }
} finally { } finally {
if (instream != null) { if (instream != null) {
@ -108,7 +108,7 @@ public class JakartaCommonsHttpResponse {
} }
} }
} }
return this.responseBody; return responseBody;
} }
/** /**
@ -118,7 +118,7 @@ public class JakartaCommonsHttpResponse {
*/ */
public InputStream getDataAsStream() throws IOException { public InputStream getDataAsStream() throws IOException {
InputStream inStream = method.getResponseBodyAsStream(); InputStream inStream = method.getResponseBodyAsStream();
if(getResponseHeader().gzip()) { if (getResponseHeader().gzip()) {
inStream = new GZIPInputStream(inStream); inStream = new GZIPInputStream(inStream);
} }
// count bytes for overall http-statistics // count bytes for overall http-statistics
@ -132,6 +132,8 @@ public class JakartaCommonsHttpResponse {
*/ */
public void closeStream() { public void closeStream() {
method.releaseConnection(); method.releaseConnection();
// statistics
HttpConnectionInfo.removeConnection(method.hashCode());
} }
/* /*

131
source/de/anomic/http/httpdProxyHandler.java
Unescape View File

@ -89,6 +89,7 @@ import java.util.Arrays;
import java.util.Date; import java.util.Date;
import java.util.HashSet; import java.util.HashSet;
import java.util.Iterator; import java.util.Iterator;
import java.util.Map;
import java.util.Properties; import java.util.Properties;
import java.util.logging.FileHandler; import java.util.logging.FileHandler;
import java.util.logging.Level; import java.util.logging.Level;
@ -96,6 +97,7 @@ import java.util.logging.LogManager;
import java.util.logging.Logger; import java.util.logging.Logger;
import java.util.zip.GZIPOutputStream; import java.util.zip.GZIPOutputStream;
import de.anomic.data.htmlTools;
import de.anomic.htmlFilter.htmlFilterContentTransformer; import de.anomic.htmlFilter.htmlFilterContentTransformer;
import de.anomic.htmlFilter.htmlFilterTransformer; import de.anomic.htmlFilter.htmlFilterTransformer;
import de.anomic.htmlFilter.htmlFilterWriter; import de.anomic.htmlFilter.htmlFilterWriter;
@ -238,9 +240,9 @@ public final class httpdProxyHandler {
private static final serverLog proxyLog = new serverLog("PROXY.access"); private static final serverLog proxyLog = new serverLog("PROXY.access");
/** /**
* Reusable {@link StringBuffer} for logging * Reusable {@link StringBuilder} for logging
*/ */
private static final StringBuffer logMessage = new StringBuffer(); private static final StringBuilder logMessage = new StringBuilder();
/** /**
* Reusable {@link StringBuffer} to generate the useragent string * Reusable {@link StringBuffer} to generate the useragent string
@ -278,9 +280,11 @@ public final class httpdProxyHandler {
*/ */
if (requestHeader.containsKey(httpHeader.COOKIE)) { if (requestHeader.containsKey(httpHeader.COOKIE)) {
Object[] entry = new Object[]{new Date(), clienthost, requestHeader.getMultiple(httpHeader.COOKIE)}; Object[] entry = new Object[]{new Date(), clienthost, requestHeader.getMultiple(httpHeader.COOKIE)};
synchronized(switchboard.outgoingCookies) {
switchboard.outgoingCookies.put(targethost, entry); switchboard.outgoingCookies.put(targethost, entry);
} }
} }
}
public static void handleIncomingCookies(httpHeader respondHeader, String serverhost, String targetclient) { public static void handleIncomingCookies(httpHeader respondHeader, String serverhost, String targetclient) {
/* /*
@ -300,9 +304,11 @@ public final class httpdProxyHandler {
*/ */
if (respondHeader.containsKey(httpHeader.SET_COOKIE)) { if (respondHeader.containsKey(httpHeader.SET_COOKIE)) {
Object[] entry = new Object[]{new Date(), targetclient, respondHeader.getMultiple(httpHeader.SET_COOKIE)}; Object[] entry = new Object[]{new Date(), targetclient, respondHeader.getMultiple(httpHeader.SET_COOKIE)};
synchronized(switchboard.incomingCookies) {
switchboard.incomingCookies.put(serverhost, entry); switchboard.incomingCookies.put(serverhost, entry);
} }
} }
}
/** /**
* @param conProp a collection of properties about the connection, like URL * @param conProp a collection of properties about the connection, like URL
@ -484,7 +490,6 @@ public final class httpdProxyHandler {
private static void fulfillRequestFromWeb(Properties conProp, yacyURL url,String ext, httpHeader requestHeader, httpHeader cachedResponseHeader, File cacheFile, OutputStream respond) { private static void fulfillRequestFromWeb(Properties conProp, yacyURL url,String ext, httpHeader requestHeader, httpHeader cachedResponseHeader, File cacheFile, OutputStream respond) {
GZIPOutputStream gzippedOut = null; GZIPOutputStream gzippedOut = null;
httpChunkedOutputStream chunkedOut = null;
Writer hfos = null; Writer hfos = null;
JakartaCommonsHttpResponse res = null; JakartaCommonsHttpResponse res = null;
@ -520,6 +525,7 @@ public final class httpdProxyHandler {
// setup HTTP-client // setup HTTP-client
final JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(timeout, requestHeader, null); final JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(timeout, requestHeader, null);
client.setFollowRedirects(false);
final String connectHost = hostPart(host, port, yAddress); final String connectHost = hostPart(host, port, yAddress);
final String getUrl = "http://"+ connectHost + remotePath; final String getUrl = "http://"+ connectHost + remotePath;
@ -530,31 +536,19 @@ public final class httpdProxyHandler {
res = client.GET(getUrl); res = client.GET(getUrl);
conProp.put(httpHeader.CONNECTION_PROP_CLIENT_REQUEST_HEADER,requestHeader); conProp.put(httpHeader.CONNECTION_PROP_CLIENT_REQUEST_HEADER,requestHeader);
// determine if it's an internal error of the httpc
final httpHeader responseHeader = res.getResponseHeader(); final httpHeader responseHeader = res.getResponseHeader();
// determine if it's an internal error of the httpc
if (responseHeader.size() == 0) { if (responseHeader.size() == 0) {
throw new Exception(res.getStatusLine()); throw new Exception(res.getStatusLine());
} }
// if the content length is not set we have to use chunked transfer encoding final httpChunkedOutputStream chunkedOut = setTransferEncoding(conProp, responseHeader, res.getStatusCode(), respond);
long contentLength = responseHeader.contentLength();
if (contentLength < 0) {
// according to http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html
// a 204,304 message must not contain a message body.
// Therefore we need to set the content-length to 0.
if (res.getStatusCode() == 204 ||
res.getStatusCode() == 304) {
responseHeader.put(httpHeader.CONTENT_LENGTH,"0");
} else {
if (httpVer.equals(httpHeader.HTTP_VERSION_0_9) || httpVer.equals(httpHeader.HTTP_VERSION_1_0)) {
conProp.setProperty(httpHeader.CONNECTION_PROP_PERSISTENT,"close");
} else {
chunkedOut = new httpChunkedOutputStream(respond);
}
responseHeader.remove(httpHeader.CONTENT_LENGTH);
}
}
// if (((String)requestHeader.get(httpHeader.ACCEPT_ENCODING,"")).indexOf("gzip") != -1) {
// zipped = new GZIPOutputStream((chunked != null) ? chunked : respond);
// res.responseHeader.put(httpHeader.CONTENT_ENCODING, "gzip");
// res.responseHeader.remove(httpHeader.CONTENT_LENGTH);
// }
// the cache does either not exist or is (supposed to be) stale // the cache does either not exist or is (supposed to be) stale
long sizeBeforeDelete = -1; long sizeBeforeDelete = -1;
@ -582,6 +576,7 @@ public final class httpdProxyHandler {
); );
// handle file types and make (possibly transforming) output stream // handle file types and make (possibly transforming) output stream
final OutputStream outStream = (gzippedOut != null) ? gzippedOut : ((chunkedOut != null)? chunkedOut : respond);
if ( if (
(!transformer.isIdentityTransformer()) && (!transformer.isIdentityTransformer()) &&
(plasmaParser.supportedHTMLContent(url,responseHeader.mime())) (plasmaParser.supportedHTMLContent(url,responseHeader.mime()))
@ -590,11 +585,11 @@ public final class httpdProxyHandler {
theLogger.logFine("create transformer for URL " + url); theLogger.logFine("create transformer for URL " + url);
//hfos = new htmlFilterOutputStream((gzippedOut != null) ? gzippedOut : ((chunkedOut != null)? chunkedOut : respond), null, transformer, (ext.length() == 0)); //hfos = new htmlFilterOutputStream((gzippedOut != null) ? gzippedOut : ((chunkedOut != null)? chunkedOut : respond), null, transformer, (ext.length() == 0));
final String charSet = httpHeader.getCharSet(responseHeader); final String charSet = httpHeader.getCharSet(responseHeader);
hfos = new htmlFilterWriter((gzippedOut != null) ? gzippedOut : ((chunkedOut != null)? chunkedOut : respond),charSet, null, transformer, (ext.length() == 0)); hfos = new htmlFilterWriter(outStream,charSet, null, transformer, (ext.length() == 0));
} else { } else {
// simply pass through without parsing // simply pass through without parsing
theLogger.logFine("create passthrough for URL " + url + ", extension '" + ext + "', mime-type '" + responseHeader.mime() + "'"); theLogger.logFine("create passthrough for URL " + url + ", extension '" + ext + "', mime-type '" + responseHeader.mime() + "'");
hfos = new OutputStreamWriter((gzippedOut != null) ? gzippedOut : ((chunkedOut != null)? chunkedOut : respond), httpHeader.getCharSet(res.getResponseHeader())); hfos = new OutputStreamWriter(outStream, httpHeader.getCharSet(res.getResponseHeader()));
} }
// handle incoming cookies // handle incoming cookies
@ -630,6 +625,7 @@ public final class httpdProxyHandler {
*/ */
((storeHTCache) || (isSupportedContent)) ((storeHTCache) || (isSupportedContent))
) { ) {
final long contentLength = responseHeader.contentLength();
// we write a new cache entry // we write a new cache entry
if ((contentLength > 0) && (contentLength < 1048576)) // if the length is known and < 1 MB if ((contentLength > 0) && (contentLength < 1048576)) // if the length is known and < 1 MB
{ {
@ -735,6 +731,49 @@ public final class httpdProxyHandler {
} }
} }
/**
* determines in which form the response should be send and sets header accordingly
* if the content length is not set we need to use chunked content encoding
* Implemented:
* if !content-length
* switch httpVer
* case 0.9:
* case 1.0:
* close connection after transfer
* break;
* default:
* new ChunkedStream around respond
* end if
*
* @param conProp
* @param responseHeader
* @param statusCode
* @param respond
* @return
*/
private static httpChunkedOutputStream setTransferEncoding(Properties conProp, final httpHeader responseHeader,
int statusCode, OutputStream respond) {
final String httpVer = conProp.getProperty(httpHeader.CONNECTION_PROP_HTTP_VER);
httpChunkedOutputStream chunkedOut = null;
// gzipped response is ungzipped an therefor the length is unknown
if (responseHeader.gzip() || responseHeader.contentLength() < 0) {
// according to http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html
// a 204,304 message must not contain a message body.
// Therefore we need to set the content-length to 0.
if (statusCode == 204 || statusCode == 304) {
responseHeader.put(httpHeader.CONTENT_LENGTH, "0");
} else {
if (httpVer.equals(httpHeader.HTTP_VERSION_0_9) || httpVer.equals(httpHeader.HTTP_VERSION_1_0)) {
forceConnectionClose(conProp);
} else {
chunkedOut = new httpChunkedOutputStream(respond);
}
responseHeader.remove(httpHeader.CONTENT_LENGTH);
}
}
return chunkedOut;
}
private static void fulfillRequestFromCache( private static void fulfillRequestFromCache(
Properties conProp, Properties conProp,
yacyURL url, yacyURL url,
@ -798,12 +837,13 @@ public final class httpdProxyHandler {
String charSet = httpHeader.getCharSet(cachedResponseHeader); String charSet = httpHeader.getCharSet(cachedResponseHeader);
// make a transformer // make a transformer
final OutputStream outStream = (gzippedOut != null) ? gzippedOut : ((chunkedOut != null)? chunkedOut : respond);
if (( !transformer.isIdentityTransformer()) && if (( !transformer.isIdentityTransformer()) &&
(ext == null || !plasmaParser.supportedHTMLFileExtContains(url)) && (ext == null || !plasmaParser.supportedHTMLFileExtContains(url)) &&
(plasmaParser.HTMLParsableMimeTypesContains(cachedResponseHeader.mime()))) { (plasmaParser.HTMLParsableMimeTypesContains(cachedResponseHeader.mime()))) {
hfos = new htmlFilterWriter((chunkedOut != null) ? chunkedOut : respond, charSet, null, transformer, (ext.length() == 0)); hfos = new htmlFilterWriter(outStream, charSet, null, transformer, (ext.length() == 0));
} else { } else {
hfos = (gzippedOut != null) ? gzippedOut : ((chunkedOut != null)? chunkedOut : respond); hfos = outStream;
} }
// send also the complete body now from the cache // send also the complete body now from the cache
@ -907,8 +947,6 @@ public final class httpdProxyHandler {
String args = conProp.getProperty(httpHeader.CONNECTION_PROP_ARGS); String args = conProp.getProperty(httpHeader.CONNECTION_PROP_ARGS);
String httpVer = conProp.getProperty(httpHeader.CONNECTION_PROP_HTTP_VER); String httpVer = conProp.getProperty(httpHeader.CONNECTION_PROP_HTTP_VER);
switchboard.proxyLastAccess = System.currentTimeMillis();
int port, pos; int port, pos;
if ((pos = host.indexOf(":")) < 0) { if ((pos = host.indexOf(":")) < 0) {
port = 80; port = 80;
@ -958,6 +996,7 @@ public final class httpdProxyHandler {
// setup HTTP-client // setup HTTP-client
JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(timeout, requestHeader, null); JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(timeout, requestHeader, null);
client.setFollowRedirects(false);
// generate request-url // generate request-url
final String connectHost = hostPart(host, port, yAddress); final String connectHost = hostPart(host, port, yAddress);
@ -1060,6 +1099,7 @@ public final class httpdProxyHandler {
// setup HTTP-client // setup HTTP-client
JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(timeout, requestHeader, null); JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(timeout, requestHeader, null);
client.setFollowRedirects(false);
final String connectHost = hostPart(host, port, yAddress); final String connectHost = hostPart(host, port, yAddress);
client.setProxy(getProxyConfig(connectHost)); client.setProxy(getProxyConfig(connectHost));
@ -1084,31 +1124,13 @@ public final class httpdProxyHandler {
// sending the request // sending the request
res = client.POST(getUrl, body); res = client.POST(getUrl, body);
// determine if it's an internal error of the httpc
final httpHeader responseHeader = res.getResponseHeader(); final httpHeader responseHeader = res.getResponseHeader();
// determine if it's an internal error of the httpc
if (responseHeader.size() == 0) { if (responseHeader.size() == 0) {
throw new Exception(res.getStatusLine()); throw new Exception(res.getStatusLine());
} }
// if the content length is not set we need to use chunked content encoding final httpChunkedOutputStream chunked = setTransferEncoding(conProp, responseHeader, res.getStatusCode(), respond);
long contentLength = responseHeader.contentLength();
httpChunkedOutputStream chunked = null;
if (contentLength <= 0) {
// according to http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html
// a 204,304 message must not contain a message body.
// Therefore we need to set the content-length to 0.
if (res.getStatusCode() == 204 ||
res.getStatusCode() == 304) {
responseHeader.put(httpHeader.CONTENT_LENGTH,"0");
} else {
if (httpVer.equals("HTTP/0.9") || httpVer.equals("HTTP/1.0")) {
forceConnectionClose(conProp);
} else {
chunked = new httpChunkedOutputStream(respond);
}
responseHeader.remove(httpHeader.CONTENT_LENGTH);
}
}
prepareResponseHeader(responseHeader, res.getHttpVer()); prepareResponseHeader(responseHeader, res.getHttpVer());
@ -1175,6 +1197,7 @@ public final class httpdProxyHandler {
// TODO gzip again? set "correct" encoding? // TODO gzip again? set "correct" encoding?
if(responseHeader.gzip()) { if(responseHeader.gzip()) {
responseHeader.remove(httpHeader.CONTENT_ENCODING); responseHeader.remove(httpHeader.CONTENT_ENCODING);
responseHeader.remove(httpHeader.CONTENT_LENGTH); // remove gziped length
} }
} }
@ -1240,6 +1263,7 @@ public final class httpdProxyHandler {
(proxyConfig.useProxy4SSL()) (proxyConfig.useProxy4SSL())
) { ) {
JakartaCommonsHttpClient remoteProxy = new JakartaCommonsHttpClient(timeout, requestHeader, proxyConfig); JakartaCommonsHttpClient remoteProxy = new JakartaCommonsHttpClient(timeout, requestHeader, proxyConfig);
remoteProxy.setFollowRedirects(false); // should not be needed, but safe is safe
JakartaCommonsHttpResponse response = null; JakartaCommonsHttpResponse response = null;
try { try {
@ -1485,6 +1509,7 @@ public final class httpdProxyHandler {
(exceptionMsg.indexOf("server has closed connection") >= 0) (exceptionMsg.indexOf("server has closed connection") >= 0)
)) { )) {
errorMessage = exceptionMsg; errorMessage = exceptionMsg;
e.printStackTrace();
} else { } else {
errorMessage = "Unexpected Error. " + e.getClass().getName() + ": " + e.getMessage(); errorMessage = "Unexpected Error. " + e.getClass().getName() + ": " + e.getMessage();
unknownError = true; unknownError = true;
@ -1501,7 +1526,7 @@ public final class httpdProxyHandler {
} }
} else { } else {
if (unknownError) { if (unknownError) {
theLogger.logWarning("Error while processing request '" + theLogger.logSevere("Unknown Error while processing request '" +
conProp.getProperty(httpHeader.CONNECTION_PROP_REQUESTLINE,"unknown") + "':" + conProp.getProperty(httpHeader.CONNECTION_PROP_REQUESTLINE,"unknown") + "':" +
"\n" + Thread.currentThread().getName() + "\n" + Thread.currentThread().getName() +
"\n" + errorMessage,e); "\n" + errorMessage,e);
@ -1615,7 +1640,7 @@ public final class httpdProxyHandler {
return detailedErrorMsgMap; return detailedErrorMsgMap;
} }
private static String generateUserAgent(httpHeader requestHeaders) { private static synchronized String generateUserAgent(httpHeader requestHeaders) {
userAgentStr.setLength(0); userAgentStr.setLength(0);
String browserUserAgent = (String) requestHeaders.get(httpHeader.USER_AGENT, proxyUserAgent); String browserUserAgent = (String) requestHeaders.get(httpHeader.USER_AGENT, proxyUserAgent);
@ -1658,7 +1683,7 @@ public final class httpdProxyHandler {
* e.g.<br> * e.g.<br>
* <code>1117528623.857 178 192.168.1.201 TCP_MISS/200 1069 GET http://www.yacy.de/ - DIRECT/81.169.145.74 text/html</code> * <code>1117528623.857 178 192.168.1.201 TCP_MISS/200 1069 GET http://www.yacy.de/ - DIRECT/81.169.145.74 text/html</code>
*/ */
private final static void logProxyAccess(Properties conProp) { private final static synchronized void logProxyAccess(Properties conProp) {
if (!doAccessLogging) return; if (!doAccessLogging) return;
@ -1737,13 +1762,13 @@ public final class httpdProxyHandler {
logMessage.append(mime); logMessage.append(mime);
// sending the logging message to the logger // sending the logging message to the logger
proxyLog.logFine(new String(logMessage)); proxyLog.logFine(logMessage.toString());
} }
/** /**
* @param remoteProxyConfig the remoteProxyConfig to set * @param remoteProxyConfig the remoteProxyConfig to set
*/ */
public static void setRemoteProxyConfig(httpRemoteProxyConfig remoteProxyConfig) { public static synchronized void setRemoteProxyConfig(httpRemoteProxyConfig remoteProxyConfig) {
httpdProxyHandler.remoteProxyConfig = remoteProxyConfig; httpdProxyHandler.remoteProxyConfig = remoteProxyConfig;
} }

5
source/de/anomic/plasma/plasmaSwitchboard.java
Unescape View File

@ -220,7 +220,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<plasmaSwitchbo
public HashMap<String, Object[]> outgoingCookies, incomingCookies; public HashMap<String, Object[]> outgoingCookies, incomingCookies;
public kelondroMapTable facilityDB; public kelondroMapTable facilityDB;
public plasmaParser parser; public plasmaParser parser;
public long proxyLastAccess, localSearchLastAccess, remoteSearchLastAccess; public volatile long proxyLastAccess, localSearchLastAccess, remoteSearchLastAccess;
public yacyCore yc; public yacyCore yc;
public userDB userDB; public userDB userDB;
public bookmarksDB bookmarksDB; public bookmarksDB bookmarksDB;
@ -1919,6 +1919,9 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<plasmaSwitchbo
try { try {
boolean hasDoneSomething = false; boolean hasDoneSomething = false;
// close unused connections
JakartaCommonsHttpClient.cleanup();
// do transmission of CR-files // do transmission of CR-files
checkInterruption(); checkInterruption();
int count = rankingOwnDistribution.size() / 100; int count = rankingOwnDistribution.size() / 100;

2
source/de/anomic/server/serverFileUtils.java
Unescape View File

@ -557,7 +557,7 @@ public final class serverFileUtils {
/** /**
* copies the input stream to all writers (byte per byte) * copies the input stream to all writers (byte per byte)
* @param data * @param data
* @param writers * @param writer
* @param charSet * @param charSet
* @return * @return
* @throws IOException * @throws IOException

Write Preview
Loading…
Cancel
Save