boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

Replace deprecated getIP with getIPs in Protocol transferURL() and

Browse Source 
getProfile().
Remember used ip for error handling and departInterface
pull/149/head
reger 8 years ago
parent ed36b47bec
commit 15d78b1064
1 changed files with 98 additions and 77 deletions
Show Stats Download Patch File Download Diff File

175
source/net/yacy/peers/Protocol.java
Unescape View File

@ -1611,7 +1611,8 @@ public final class Protocol {
String result = in.get("result"); String result = in.get("result");
if ( result == null ) { if ( result == null ) {
String errorCause = "no result from transferRWI"; String errorCause = "no result from transferRWI";
seeds.peerActions.peerDeparture(targetSeed, errorCause); // disconnect unavailable peer String usedIP = in.get(Seed.IP);
seeds.peerActions.interfaceDeparture(targetSeed, usedIP); // disconnect unavailable peer
return errorCause; return errorCause;
} }
@ -1647,7 +1648,8 @@ public final class Protocol {
result = in.get("result"); result = in.get("result");
if ( result == null ) { if ( result == null ) {
String errorCause = "no result from transferURL"; String errorCause = "no result from transferURL";
seeds.peerActions.peerDeparture(targetSeed, errorCause); // disconnect unavailable peer String usedIP = in.get(Seed.IP);
seeds.peerActions.interfaceDeparture(targetSeed, usedIP); // disconnect unavailable peer ip
return errorCause; return errorCause;
} }
@ -1738,6 +1740,7 @@ public final class Protocol {
final Map<String, String> result = FileUtils.table(v); final Map<String, String> result = FileUtils.table(v);
// return the transfered index data in bytes (for debugging only) // return the transfered index data in bytes (for debugging only)
result.put("indexPayloadSize", Integer.toString(entrypost.length())); result.put("indexPayloadSize", Integer.toString(entrypost.length()));
result.put(Seed.IP, ip); // add used ip to result for error handling (in case no "result" key was received)
return result; return result;
} catch (final Exception e ) { } catch (final Exception e ) {
Network.log.info("yacyClient.transferRWI to " + address + " error: " + e.getMessage()); Network.log.info("yacyClient.transferRWI to " + address + " error: " + e.getMessage());
@ -1748,6 +1751,17 @@ public final class Protocol {
return null; return null;
} }
/**
* Transfer URL entries to remote peer
*
* @param targetSeed
* @param uhs
* @param urlRefs
* @param segment
* @param gzipBody
* @param timeout
* @return remote peer response
*/
private static Map<String, String> transferURL( private static Map<String, String> transferURL(
final Seed targetSeed, final Seed targetSeed,
final String[] uhs, final String[] uhs,
@ -1756,97 +1770,104 @@ public final class Protocol {
boolean gzipBody, boolean gzipBody,
final int timeout) { final int timeout) {
// this post a message to the remote message board // this post a message to the remote message board
String ip = targetSeed.getIP(); for (String ip : targetSeed.getIPs()) {
final String address = targetSeed.getPublicAddress(ip); final String address = targetSeed.getPublicAddress(ip);
if ( address == null ) { if ( address == null ) {
return null; return null;
} }
// prepare post values // prepare post values
final String salt = crypt.randomSalt(); final String salt = crypt.randomSalt();
final Map<String, ContentBody> parts = final Map<String, ContentBody> parts =
basicRequestParts(Switchboard.getSwitchboard(), targetSeed.hash, salt); basicRequestParts(Switchboard.getSwitchboard(), targetSeed.hash, salt);
// enabling gzip compression for post request body // enabling gzip compression for post request body
if ( gzipBody && (targetSeed.getVersion() < yacyVersion.YACY_SUPPORTS_GZIP_POST_REQUESTS_CHUNKED) ) { if ( gzipBody && (targetSeed.getVersion() < yacyVersion.YACY_SUPPORTS_GZIP_POST_REQUESTS_CHUNKED) ) {
gzipBody = false; gzipBody = false;
} }
// extract the urlCache from the result; this is io-intensive; // extract the urlCache from the result; this is io-intensive;
// other transmissions should not be started as long as this is running // other transmissions should not be started as long as this is running
byte[] key; byte[] key;
URIMetadataNode url; URIMetadataNode url;
String resource; String resource;
int urlc = 0; int urlc = 0;
int urlPayloadSize = 0; int urlPayloadSize = 0;
metadataRetrievalRunning.incrementAndGet(); metadataRetrievalRunning.incrementAndGet();
for (int i = 0; i < uhs.length; i++) { for (int i = 0; i < uhs.length; i++) {
key = ASCII.getBytes(uhs[i]); key = ASCII.getBytes(uhs[i]);
if (urlRefs.has(key)) { if (urlRefs.has(key)) {
url = segment.fulltext().getMetadata(key); url = segment.fulltext().getMetadata(key);
if (url == null) { if (url == null) {
if (Network.log.isFine()) Network.log.fine("DEBUG transferIndex: requested url hash '" + uhs[i] + "'"); if (Network.log.isFine()) Network.log.fine("DEBUG transferIndex: requested url hash '" + uhs[i] + "'");
continue; continue;
} }
resource = url.toString(); resource = url.toString();
//System.out.println("*** DEBUG resource = " + resource); //System.out.println("*** DEBUG resource = " + resource);
if ( resource != null && resource.indexOf(0) == -1 ) { if ( resource != null && resource.indexOf(0) == -1 ) {
parts.put("url" + urlc, UTF8.StringBody(resource)); parts.put("url" + urlc, UTF8.StringBody(resource));
urlPayloadSize += resource.length(); urlPayloadSize += resource.length();
urlc++; urlc++;
} }
} }
} }
metadataRetrievalRunning.decrementAndGet(); metadataRetrievalRunning.decrementAndGet();
try { try {
parts.put("urlc", UTF8.StringBody(Integer.toString(urlc))); parts.put("urlc", UTF8.StringBody(Integer.toString(urlc)));
// final byte[] content = HTTPConnector.getConnector(MultiProtocolURI.yacybotUserAgent).post(new MultiProtocolURI("http://" + address + "/yacy/transferURL.html"), timeout, targetSeed.getHexHash() + ".yacyh", parts, gzipBody); final HTTPClient httpClient = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent, timeout);
final HTTPClient httpClient = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent, timeout); final byte[] content =
final byte[] content = httpClient.POSTbytes(
httpClient.POSTbytes( new MultiProtocolURL("http://" + address + "/yacy/transferURL.html"),
new MultiProtocolURL("http://" + address + "/yacy/transferURL.html"), targetSeed.getHexHash() + ".yacyh",
targetSeed.getHexHash() + ".yacyh", parts,
parts, gzipBody, true);
gzipBody, true); final Iterator<String> v = FileUtils.strings(content);
final Iterator<String> v = FileUtils.strings(content);
final Map<String, String> result = FileUtils.table(v); final Map<String, String> result = FileUtils.table(v);
// return the transfered url data in bytes (for debugging only) // return the transfered url data in bytes (for debugging only)
result.put("urlPayloadSize", Integer.toString(urlPayloadSize)); result.put("urlPayloadSize", Integer.toString(urlPayloadSize));
return result; result.put(Seed.IP, ip); // add used ip to result for error handling (in case no "result" key was received)
} catch (final Exception e ) { return result;
Network.log.warn("yacyClient.transferURL to " + address + " error: " + e.getMessage()); } catch (final Exception e ) {
return null; Network.log.warn("yacyClient.transferURL to " + address + " error: " + e.getMessage());
}
} }
return null;
} }
/**
* Receive remote peers profile data
*
* @param targetSeed
* @return profile or null
*/
public static Map<String, String> getProfile(final Seed targetSeed) { public static Map<String, String> getProfile(final Seed targetSeed) {
// ReferenceContainerCache<HostReference> ref = loadIDXHosts(targetSeed);
// this post a message to the remote message board // this post a message to the remote message board
final String salt = crypt.randomSalt(); final String salt = crypt.randomSalt();
String address = targetSeed.getPublicAddress(targetSeed.getIP()); for (String ip : targetSeed.getIPs()) {
if ( address == null ) { String address = targetSeed.getPublicAddress(ip);
address = "localhost:8090"; if ( address == null ) {
} break;
try { }
final Map<String, ContentBody> parts = try {
basicRequestParts(Switchboard.getSwitchboard(), targetSeed.hash, salt); final Map<String, ContentBody> parts =
// final byte[] content = HTTPConnector.getConnector(MultiProtocolURI.yacybotUserAgent).post(new MultiProtocolURI("http://" + address + "/yacy/profile.html"), 5000, targetSeed.getHexHash() + ".yacyh", parts); basicRequestParts(Switchboard.getSwitchboard(), targetSeed.hash, salt);
final HTTPClient httpclient = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent, 15000); final HTTPClient httpclient = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent, 15000);
final byte[] content = final byte[] content =
httpclient.POSTbytes( httpclient.POSTbytes(
new MultiProtocolURL("http://" + address + "/yacy/profile.html"), new MultiProtocolURL("http://" + address + "/yacy/profile.html"),
targetSeed.getHexHash() + ".yacyh", targetSeed.getHexHash() + ".yacyh",
parts, parts,
false, true); false, true);
return FileUtils.table(content); return FileUtils.table(content);
} catch (final Exception e ) { } catch (final Exception e ) {
Network.log.warn("yacyClient.getProfile error:" + e.getMessage()); Network.log.warn("yacyClient.getProfile error:" + e.getMessage());
return null; }
} }
return null;
} }
public static ReferenceContainerCache<HostReference> loadIDXHosts(final Seed target) { public static ReferenceContainerCache<HostReference> loadIDXHosts(final Seed target) {

Write Preview
Loading…
Cancel
Save