diff --git a/source/net/yacy/cora/services/federated/yacy/Peer.java b/source/net/yacy/cora/services/federated/yacy/Peer.java new file mode 100644 index 000000000..4f13ff774 --- /dev/null +++ b/source/net/yacy/cora/services/federated/yacy/Peer.java @@ -0,0 +1,136 @@ +package net.yacy.cora.services.federated.yacy; + +/** + * Peer + * Copyright 2012 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany + * First released 21.09.2012 at http://yacy.net + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program in the file lgpl21.txt + * If not, see . + */ + +import java.io.Serializable; +import java.util.HashMap; + +import net.yacy.cora.document.ASCII; +import net.yacy.cora.order.Base64Order; + +/** + * new implemenentation of "Seed" objects: Peers. + * A Peer contains the attributes that a peer wants to show in public. + * This Class is simply a representation of the XML Schema that is used in the Network.xml interface. + */ +public class Peer extends HashMap implements Comparable, Serializable { + + private static final long serialVersionUID = -3279480981385980050L; + + public enum Schema implements Serializable { + hash,fullname,version,ppm,qph,uptime,links, + words,rurls,lastseen,sendWords,receivedWords, + sendURLs,receivedURLs,type,direct,acceptcrawl,dhtreceive, + nodestate,location,seedurl,age,seeds,connects,address,useragent; + } + + private long time; + + public Peer() { + super(); + this.time = System.currentTimeMillis(); + } + + /** + * Get the number of minutes that the peer which returns the peer list knows when the peer was last seen. + * This value is only a relative to the moment when another peer was asked for fresh information. + * To compute an absolute value for last-seen, use the lastseenTime() method. + * @return time in minutes + */ + public int lastseen() { + String x = this.get(Schema.lastseen); + if (x == null) return Integer.MAX_VALUE; + try { + return Integer.parseInt(x); + } catch (NumberFormatException e) { + return Integer.MAX_VALUE; + } + } + + /** + * get the absolute time when this peer was seen in the network + * @return time in milliseconds + */ + public long lastseenTime() { + return time - lastseen() * 60000; + } + + /** + * get the version number of the peer + * @return + */ + public float version() { + String x = this.get(Schema.version); if (x == null) return 0.0f; + int p = x.indexOf('/'); if (p < 0) return 0.0f; + x = x.substring(0, p); + try { + return Float.parseFloat(x); + } catch (NumberFormatException e) { + return 0.0f; + } + } + + /** + * check if the peer supports the solr interface + * @return true if the peer supports the solr interface + */ + public boolean supportsSolr() { + return version() >= 1.041f; + } + + /** + * compare the peer to another peer. + * The comparisment is done using the base 64 order on the peer hash. + */ + @Override + public int compareTo(Peer o) { + String h0 = this.get(Schema.hash); + String h1 = o.get(Schema.hash); + return Base64Order.enhancedCoder.compare(ASCII.getBytes(h0), ASCII.getBytes(h1)); + } + + /** + * get the hash code of the peer. + * The hash code is a number that has the same order as the peer order. + */ + public int hashCode() { + String h = this.get(Schema.hash); + return (int) (Base64Order.enhancedCoder.cardinal(h) >> 32); + } + + /** + * check if two peers are equal: + * two peers are equal if they have the same hash. + */ + public boolean equals(Object o) { + if (!(o instanceof Peer)) return false; + String h0 = this.get(Schema.hash); + String h1 = ((Peer) o).get(Schema.hash); + return h0.equals(h1); + } + + public static void main(String[] args) { + String h = "____________"; + long l = Base64Order.enhancedCoder.cardinal(h); + System.out.println("l = " + l + ", h = " + ((int) (l >> 32))); + System.out.println("l-maxlong = " + (l - Long.MAX_VALUE) + ", (h-maxint) = " + (((int) (l >> 32)) - Integer.MAX_VALUE)); + } +} diff --git a/source/net/yacy/cora/services/federated/yacy/Peers.java b/source/net/yacy/cora/services/federated/yacy/Peers.java new file mode 100644 index 000000000..f4513cc1c --- /dev/null +++ b/source/net/yacy/cora/services/federated/yacy/Peers.java @@ -0,0 +1,176 @@ +/** + * Peers + * Copyright 2012 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany + * First released 21.09.2012 at http://yacy.net + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program in the file lgpl21.txt + * If not, see . + */ + +package net.yacy.cora.services.federated.yacy; + +import java.io.IOException; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Random; +import java.util.Set; +import java.util.TreeMap; + +import org.apache.log4j.Logger; + +import net.yacy.cora.document.ASCII; +import net.yacy.cora.order.Base64Order; +import net.yacy.cora.protocol.http.HTTPClient; +import net.yacy.cora.services.federated.yacy.api.Network; + +public class Peers extends TreeMap implements Serializable { + + public final static String[] bootstrapPeers = new String[]{ + "search.yacy.net", "yacy.dyndns.org:8000", "yacy-websuche.mxchange.org:8090", + "sokrates.homeunix.net:6070", "sokrates.homeunix.net:9090", + "141.52.175.27:8080", "62.75.214.113:8080", "141.52.175.30:8080"}; + + private final static Logger log = Logger.getLogger(Peers.class); + private static final long serialVersionUID = -2939656606305545080L; + private long lastBootstrap; + + + public Peers() { + super(Base64Order.enhancedCoder); + this.lastBootstrap = 0; + } + + /** + * refresh() gets a new network list from one random remote peer once every + * minute. This method will load a remote list not more then every one minute + * and if it does, it is done concurrently. Therefore this method can be called + * every time when a process needs specific remote peers. + */ + public void refresh() { + if (System.currentTimeMillis() - this.lastBootstrap < 60000) return; + lastBootstrap = System.currentTimeMillis(); + new Thread() { + public void run() { + String[] peers = bootstrapList(select(false, false)); + bootstrap(peers, 1); + } + }.start(); + } + + /** + * this method must be called once to bootstrap a list of network peers. + * To do this, a default list of peers must be given. + * @param peers a list of known peers + * @param selection number of peers which are taken from the given list of peers for bootstraping + */ + public void bootstrap(final String[] peers, int selection) { + int loops = 0; + while (this.size() == 0 || loops++ == 0) { + if (selection > peers.length) selection = peers.length; + Set s = new HashSet(); + Random r = new Random(System.currentTimeMillis()); + while (s.size() < selection) s.add(r.nextInt(peers.length)); + List t = new ArrayList(); + for (Integer pn: s) { + final String bp = peers[pn.intValue()]; + Thread t0 = new Thread() { + public void run() { + Peers ps; + try { + ps = Network.getNetwork(bp); + int c0 = Peers.this.size(); + for (Peer p: ps.values()) Peers.this.add(p); + int c1 = Peers.this.size(); + log.info("bootstrap with peer " + bp + ": added " + (c1 - c0) + " peers"); + } catch (IOException e) { + log.info("bootstrap with peer " + bp + ": FAILED - " + e.getMessage()); + } + } + }; + t0.start(); + t.add(t0); + } + for (Thread t0: t) try {t0.join(10000);} catch (InterruptedException e) {} + } + lastBootstrap = System.currentTimeMillis(); + log.info("bootstrap finished: " + this.size() + " peers"); + } + + /** + * add a new peer to the list of peers + * @param peer + */ + public synchronized void add(Peer peer) { + String hash = peer.get(Peer.Schema.hash); + if (hash == null) return; + Peer p = this.put(ASCII.getBytes(hash), peer); + if (p == null) return; + if (p.lastseenTime() < peer.lastseenTime()) this.put(ASCII.getBytes(hash), p); + } + + /** + * get a peer using the peer hash + * @param hash + * @return + */ + public synchronized Peer get(String hash) { + return super.get(ASCII.getBytes(hash)); + } + + /** + * select a list of peers according to special needs. The require parameters are combined as conjunction + * @param requireNode must be true to select only peers which are node peers + * @param requireSolr must be true to select only peers which support the solr interface + * @return + */ + public synchronized List select(final boolean requireNode, final boolean requireSolr) { + List l = new ArrayList(); + for (Peer p: this.values()) { + if (requireNode && !p.get(Peer.Schema.nodestate).equals("1")) continue; + if (requireSolr && !p.supportsSolr()) continue; + l.add(p); + } + return l; + } + + /** + * convenient method to produce a list of bootstrap peer addresses from given peer lists + * @param peers + * @return + */ + public static synchronized String[] bootstrapList(List peers) { + List l = new ArrayList(); + for (Peer p: peers) l.add(p.get(Peer.Schema.address)); + return l.toArray(new String[l.size()]); + } + + public static void main(String[] args) { + Peers peers = new Peers(); + peers.bootstrap(Peers.bootstrapPeers, 4); + //Peers peers = Network.getNetwork("sokrates.homeunix.net:9090"); + /* + for (Peer p: peers.values()) { + log.info(p.get(Peer.Schema.fullname) + " - " + p.get(Peer.Schema.address)); + } + */ + List nodes = peers.select(false, true); + for (Peer p: nodes) { + log.info(p.get(Peer.Schema.fullname) + " - " + p.get(Peer.Schema.address)); + } + try {HTTPClient.closeConnectionManager();} catch (InterruptedException e) {} + } + +} diff --git a/source/net/yacy/cora/services/federated/yacy/api/Network.java b/source/net/yacy/cora/services/federated/yacy/api/Network.java new file mode 100644 index 000000000..5e4daaf4a --- /dev/null +++ b/source/net/yacy/cora/services/federated/yacy/api/Network.java @@ -0,0 +1,97 @@ +/** + * Peers + * Copyright 2012 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany + * First released 21.09.2012 at http://yacy.net + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program in the file lgpl21.txt + * If not, see . + */ + +package net.yacy.cora.services.federated.yacy.api; + +import java.io.ByteArrayInputStream; +import java.io.IOException; + +import javax.xml.parsers.DocumentBuilderFactory; + +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; + +import net.yacy.cora.protocol.ClientIdentification; +import net.yacy.cora.protocol.http.HTTPClient; +import net.yacy.cora.services.federated.yacy.Peer; +import net.yacy.cora.services.federated.yacy.Peers; + +/** + * discover all peers in the network when only one peer is known. + * this works only for a limited number of peers, not more than some thousands + */ +public class Network { + + /** + * get the list of peers from one peer + * @param address + * @return a network as list of peers + * @throws IOException + */ + public static Peers getNetwork(final String address) throws IOException { + Peers peers = new Peers(); + final HTTPClient httpclient = new HTTPClient(ClientIdentification.getUserAgent(), 15000); + final byte[] content = httpclient.GETbytes("http://" + address + "/Network.xml?page=1&maxCount=1000&ip="); + ByteArrayInputStream bais = new ByteArrayInputStream(content); + Document doc = null; + try { + doc = DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(bais); + } catch (Throwable e) { + throw new IOException(e.getMessage()); + } + bais.close(); + doc.getDocumentElement().normalize(); + NodeList objects = doc.getElementsByTagName("peer"); + + for (int i = 0; i < objects.getLength(); i++) { + Node object = objects.item(i); + if (object.getNodeType() == Node.ELEMENT_NODE) { + Element element = (Element) object; + Peer peer = new Peer(); + for (Peer.Schema attr: Peer.Schema.values()) { + peer.put(attr, getAttr(attr.name(), element)); + } + peers.add(peer); + //log.info(peer.toString()); + } + } + return peers; + } + + private static String getAttr(String attr, Element eElement) { + NodeList nl0 = eElement.getElementsByTagName(attr); + if (nl0 == null) return ""; + Node n0 = nl0.item(0); + if (n0 == null) return ""; + NodeList nl1 = n0.getChildNodes(); + if (nl1 == null) return ""; + Node n1 = nl1.item(0); + if (n1 == null) return ""; + return n1.getNodeValue(); + } + + public static void main(String[] args) { + //getNetwork("search.yacy.net"); + try {getNetwork("sokrates.homeunix.net:9090");} catch (IOException e1) {} + try {HTTPClient.closeConnectionManager();} catch (InterruptedException e) {} + } +}