git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4736 6c8d7289-2bf4-0310-a012-ef5d649a1542pull/1/head
orbiter
17 years ago
parent
b9a2a2d287
commit
724bbdf9b2
@ -1,12 +1,27 @@
|
|||||||
touch /tmp/yp$$.dump
|
touch /tmp/yp$$.dump
|
||||||
|
echo "cleaning search buffer"
|
||||||
curl -s --user $1:$2 http://localhost:8080/Ranking_p.html > /dev/null
|
curl -s --user $1:$2 http://localhost:8080/Ranking_p.html > /dev/null
|
||||||
|
echo "start search for $4"
|
||||||
curl -s -o /tmp/yp$$.query "http://localhost:8080/yacysearch.rss?maximumRecords=10&resource=local&query=$4" &
|
curl -s -o /tmp/yp$$.query "http://localhost:8080/yacysearch.rss?maximumRecords=10&resource=local&query=$4" &
|
||||||
|
curl -s --user $1:$2 http://localhost:8080/Status.html > /dev/null &
|
||||||
|
curl -s --user $1:$2 http://localhost:8080/Status.html > /dev/null &
|
||||||
|
curl -s --user $1:$2 http://localhost:8080/Status.html > /dev/null &
|
||||||
|
curl -s --user $1:$2 http://localhost:8080/Status.html > /dev/null &
|
||||||
|
curl -s --user $1:$2 http://localhost:8080/Status.html > /dev/null &
|
||||||
|
echo "search started"
|
||||||
curl -s --user $1:$2 "http://localhost:8080/Threaddump_p.html?sleep=0&createThreaddump=&plain=true" >> /tmp/yp$$.dump
|
curl -s --user $1:$2 "http://localhost:8080/Threaddump_p.html?sleep=0&createThreaddump=&plain=true" >> /tmp/yp$$.dump
|
||||||
|
echo "thread dump 1 ready"
|
||||||
curl -s --user $1:$2 "http://localhost:8080/Threaddump_p.html?sleep=$3&createThreaddump=&plain=true" >> /tmp/yp$$.dump
|
curl -s --user $1:$2 "http://localhost:8080/Threaddump_p.html?sleep=$3&createThreaddump=&plain=true" >> /tmp/yp$$.dump
|
||||||
|
echo "thread dump 2 ready"
|
||||||
curl -s --user $1:$2 "http://localhost:8080/Threaddump_p.html?sleep=$3&createThreaddump=&plain=true" >> /tmp/yp$$.dump
|
curl -s --user $1:$2 "http://localhost:8080/Threaddump_p.html?sleep=$3&createThreaddump=&plain=true" >> /tmp/yp$$.dump
|
||||||
|
echo "thread dump 3 ready"
|
||||||
curl -s --user $1:$2 "http://localhost:8080/Threaddump_p.html?sleep=$3&createThreaddump=&plain=true" >> /tmp/yp$$.dump
|
curl -s --user $1:$2 "http://localhost:8080/Threaddump_p.html?sleep=$3&createThreaddump=&plain=true" >> /tmp/yp$$.dump
|
||||||
|
echo "thread dump 4 ready"
|
||||||
curl -s --user $1:$2 "http://localhost:8080/Threaddump_p.html?sleep=$3&createThreaddump=&plain=true" >> /tmp/yp$$.dump
|
curl -s --user $1:$2 "http://localhost:8080/Threaddump_p.html?sleep=$3&createThreaddump=&plain=true" >> /tmp/yp$$.dump
|
||||||
|
echo "thread dump 5 ready"
|
||||||
curl -s --user $1:$2 "http://localhost:8080/Threaddump_p.html?sleep=$3&createThreaddump=&plain=true" >> /tmp/yp$$.dump
|
curl -s --user $1:$2 "http://localhost:8080/Threaddump_p.html?sleep=$3&createThreaddump=&plain=true" >> /tmp/yp$$.dump
|
||||||
|
echo "thread dump 6 ready"
|
||||||
curl -s --user $1:$2 "http://localhost:8080/Threaddump_p.html?sleep=$3&createThreaddump=&plain=true" >> /tmp/yp$$.dump
|
curl -s --user $1:$2 "http://localhost:8080/Threaddump_p.html?sleep=$3&createThreaddump=&plain=true" >> /tmp/yp$$.dump
|
||||||
|
echo "thread dump 7 ready"
|
||||||
cat /tmp/yp$$.dump
|
cat /tmp/yp$$.dump
|
||||||
rm -f /tmp/yp$$.dump
|
rm -f /tmp/yp$$.dump
|
||||||
|
|||||||
@ -0,0 +1,142 @@
|
|||||||
|
// RSSFeed.java
|
||||||
|
// (C) 2007 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
|
||||||
|
// first published 24.04.2008 on http://yacy.net
|
||||||
|
//
|
||||||
|
// This is a part of YaCy, a peer-to-peer based web search engine
|
||||||
|
//
|
||||||
|
// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
|
||||||
|
// $LastChangedRevision: 1986 $
|
||||||
|
// $LastChangedBy: orbiter $
|
||||||
|
//
|
||||||
|
// LICENSE
|
||||||
|
//
|
||||||
|
// This program is free software; you can redistribute it and/or modify
|
||||||
|
// it under the terms of the GNU General Public License as published by
|
||||||
|
// the Free Software Foundation; either version 2 of the License, or
|
||||||
|
// (at your option) any later version.
|
||||||
|
//
|
||||||
|
// This program is distributed in the hope that it will be useful,
|
||||||
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
// GNU General Public License for more details.
|
||||||
|
//
|
||||||
|
// You should have received a copy of the GNU General Public License
|
||||||
|
// along with this program; if not, write to the Free Software
|
||||||
|
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||||
|
|
||||||
|
package de.anomic.xml;
|
||||||
|
|
||||||
|
import java.util.Iterator;
|
||||||
|
import java.util.concurrent.ConcurrentHashMap;
|
||||||
|
import java.util.concurrent.ConcurrentLinkedQueue;
|
||||||
|
|
||||||
|
public class RSSFeed implements Iterable<RSSMessage> {
|
||||||
|
|
||||||
|
// class variables
|
||||||
|
private RSSMessage channel;
|
||||||
|
private String imageURL;
|
||||||
|
private ConcurrentLinkedQueue<String> messageQueue; // a list of GUIDs, so the items can be retrieved by a specific order
|
||||||
|
private ConcurrentHashMap<String, RSSMessage> messages; // a guid:Item map
|
||||||
|
private int maxsize;
|
||||||
|
|
||||||
|
public RSSFeed() {
|
||||||
|
messageQueue = new ConcurrentLinkedQueue<String>();
|
||||||
|
messages = new ConcurrentHashMap<String, RSSMessage>();
|
||||||
|
channel = null;
|
||||||
|
maxsize = Integer.MAX_VALUE;
|
||||||
|
}
|
||||||
|
|
||||||
|
public RSSFeed(int maxsize) {
|
||||||
|
this();
|
||||||
|
this.maxsize = maxsize;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setMaxsize(int maxsize) {
|
||||||
|
this.maxsize = maxsize;
|
||||||
|
while (messageQueue.size() > this.maxsize) pollMessage();
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setChannel(RSSMessage channelItem) {
|
||||||
|
this.channel = channelItem;
|
||||||
|
}
|
||||||
|
|
||||||
|
public RSSMessage getChannel() {
|
||||||
|
return channel;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setImage(String imageURL) {
|
||||||
|
this.imageURL = imageURL;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getImage() {
|
||||||
|
return this.imageURL;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void addMessage(RSSMessage item) {
|
||||||
|
String guid = item.getGuid();
|
||||||
|
messageQueue.add(guid);
|
||||||
|
messages.put(guid, item);
|
||||||
|
while (messageQueue.size() > this.maxsize) pollMessage();
|
||||||
|
}
|
||||||
|
|
||||||
|
public RSSMessage getMessage(String guid) {
|
||||||
|
// retrieve item by guid
|
||||||
|
return messages.get(guid);
|
||||||
|
}
|
||||||
|
|
||||||
|
public int size() {
|
||||||
|
return messages.size();
|
||||||
|
}
|
||||||
|
|
||||||
|
public Iterator<RSSMessage> iterator() {
|
||||||
|
return new messageIterator();
|
||||||
|
}
|
||||||
|
|
||||||
|
public RSSMessage pollMessage() {
|
||||||
|
// retrieve and delete item
|
||||||
|
if (messageQueue.size() == 0) return null;
|
||||||
|
String nextGUID = messageQueue.poll();
|
||||||
|
if (nextGUID == null) return null;
|
||||||
|
return messages.remove(nextGUID);
|
||||||
|
}
|
||||||
|
|
||||||
|
public class messageIterator implements Iterator<RSSMessage>{
|
||||||
|
|
||||||
|
Iterator<String> GUIDiterator;
|
||||||
|
String lastGUID;
|
||||||
|
|
||||||
|
public messageIterator() {
|
||||||
|
GUIDiterator = messageQueue.iterator();
|
||||||
|
lastGUID = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
public boolean hasNext() {
|
||||||
|
return GUIDiterator.hasNext();
|
||||||
|
}
|
||||||
|
|
||||||
|
public RSSMessage next() {
|
||||||
|
lastGUID = GUIDiterator.next();
|
||||||
|
if (lastGUID == null) return null;
|
||||||
|
return messages.get(lastGUID);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void remove() {
|
||||||
|
if (lastGUID == null) return;
|
||||||
|
GUIDiterator.remove();
|
||||||
|
messages.remove(lastGUID);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* the following static channels object is used to organize a storage array for RSS feeds
|
||||||
|
*/
|
||||||
|
private static final ConcurrentHashMap<String, RSSFeed> channels = new ConcurrentHashMap<String, RSSFeed>();
|
||||||
|
|
||||||
|
public static RSSFeed channels(String channelName) {
|
||||||
|
RSSFeed feed = channels.get(channelName);
|
||||||
|
if (feed != null) return feed;
|
||||||
|
feed = new RSSFeed();
|
||||||
|
channels.put(channelName, feed);
|
||||||
|
return feed;
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -0,0 +1,117 @@
|
|||||||
|
// RSSMessage.java
|
||||||
|
// (C) 2007 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
|
||||||
|
// first published 16.07.2007 on http://yacy.net
|
||||||
|
//
|
||||||
|
// This is a part of YaCy, a peer-to-peer based web search engine
|
||||||
|
//
|
||||||
|
// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
|
||||||
|
// $LastChangedRevision: 1986 $
|
||||||
|
// $LastChangedBy: orbiter $
|
||||||
|
//
|
||||||
|
// LICENSE
|
||||||
|
//
|
||||||
|
// This program is free software; you can redistribute it and/or modify
|
||||||
|
// it under the terms of the GNU General Public License as published by
|
||||||
|
// the Free Software Foundation; either version 2 of the License, or
|
||||||
|
// (at your option) any later version.
|
||||||
|
//
|
||||||
|
// This program is distributed in the hope that it will be useful,
|
||||||
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
// GNU General Public License for more details.
|
||||||
|
//
|
||||||
|
// You should have received a copy of the GNU General Public License
|
||||||
|
// along with this program; if not, write to the Free Software
|
||||||
|
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||||
|
|
||||||
|
|
||||||
|
package de.anomic.xml;
|
||||||
|
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.HashSet;
|
||||||
|
|
||||||
|
public class RSSMessage {
|
||||||
|
|
||||||
|
// statics for item generation and automatic categorization
|
||||||
|
private static int guidcount = 0;
|
||||||
|
private static final String[] tagsDef = new String[] {
|
||||||
|
"author", //
|
||||||
|
"copyright", //
|
||||||
|
"category", //
|
||||||
|
"title", //
|
||||||
|
"link", //
|
||||||
|
"referrer", //
|
||||||
|
"language", //
|
||||||
|
"description", //
|
||||||
|
"creator", //
|
||||||
|
"pubDate", //
|
||||||
|
"guid", //
|
||||||
|
"docs" //
|
||||||
|
};
|
||||||
|
|
||||||
|
public static final HashSet<String> tags = new HashSet<String>();
|
||||||
|
static {
|
||||||
|
for (int i = 0; i < tagsDef.length; i++) {
|
||||||
|
tags.add(tagsDef[i]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private HashMap<String, String> map;
|
||||||
|
|
||||||
|
public RSSMessage() {
|
||||||
|
this.map = new HashMap<String, String>();
|
||||||
|
this.map.put("guid", Long.toHexString(System.currentTimeMillis()) + ":" + guidcount++);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setValue(String name, String value) {
|
||||||
|
map.put(name, value);
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getAuthor() {
|
||||||
|
return (String) map.get("author");
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getCopyright() {
|
||||||
|
return (String) map.get("copyright");
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getCategory() {
|
||||||
|
return (String) map.get("category");
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getTitle() {
|
||||||
|
return (String) map.get("title");
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getLink() {
|
||||||
|
return (String) map.get("link");
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getReferrer() {
|
||||||
|
return (String) map.get("referrer");
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getLanguage() {
|
||||||
|
return (String) map.get("language");
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getDescription() {
|
||||||
|
return (String) map.get("description");
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getCreator() {
|
||||||
|
return (String) map.get("creator");
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getPubDate() {
|
||||||
|
return (String) map.get("pubDate");
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getGuid() {
|
||||||
|
return (String) map.get("guid");
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getDocs() {
|
||||||
|
return (String) map.get("docs");
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -0,0 +1,165 @@
|
|||||||
|
// rssReader.java
|
||||||
|
// (C) 2007 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
|
||||||
|
// first published 16.07.2007 on http://yacy.net
|
||||||
|
//
|
||||||
|
// This is a part of YaCy, a peer-to-peer based web search engine
|
||||||
|
//
|
||||||
|
// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
|
||||||
|
// $LastChangedRevision: 1986 $
|
||||||
|
// $LastChangedBy: orbiter $
|
||||||
|
//
|
||||||
|
// LICENSE
|
||||||
|
//
|
||||||
|
// This program is free software; you can redistribute it and/or modify
|
||||||
|
// it under the terms of the GNU General Public License as published by
|
||||||
|
// the Free Software Foundation; either version 2 of the License, or
|
||||||
|
// (at your option) any later version.
|
||||||
|
//
|
||||||
|
// This program is distributed in the hope that it will be useful,
|
||||||
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
// GNU General Public License for more details.
|
||||||
|
//
|
||||||
|
// You should have received a copy of the GNU General Public License
|
||||||
|
// along with this program; if not, write to the Free Software
|
||||||
|
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||||
|
|
||||||
|
package de.anomic.xml;
|
||||||
|
|
||||||
|
import java.io.ByteArrayInputStream;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.io.InputStream;
|
||||||
|
|
||||||
|
import javax.xml.parsers.SAXParser;
|
||||||
|
import javax.xml.parsers.SAXParserFactory;
|
||||||
|
|
||||||
|
import org.xml.sax.Attributes;
|
||||||
|
import org.xml.sax.SAXException;
|
||||||
|
import org.xml.sax.helpers.DefaultHandler;
|
||||||
|
|
||||||
|
import de.anomic.server.serverByteBuffer;
|
||||||
|
import de.anomic.server.logging.serverLog;
|
||||||
|
|
||||||
|
public class RSSReader extends DefaultHandler {
|
||||||
|
|
||||||
|
// class variables
|
||||||
|
private RSSMessage item;
|
||||||
|
private StringBuffer buffer;
|
||||||
|
private boolean parsingChannel, parsingImage, parsingItem;
|
||||||
|
private RSSFeed theChannel;
|
||||||
|
|
||||||
|
public RSSReader() {
|
||||||
|
theChannel = new RSSFeed();
|
||||||
|
buffer = new StringBuffer();
|
||||||
|
item = null;
|
||||||
|
parsingChannel = false;
|
||||||
|
parsingImage = false;
|
||||||
|
parsingItem = false;
|
||||||
|
}
|
||||||
|
|
||||||
|
public RSSReader(String path) {
|
||||||
|
this();
|
||||||
|
try {
|
||||||
|
SAXParserFactory factory = SAXParserFactory.newInstance();
|
||||||
|
SAXParser saxParser = factory.newSAXParser();
|
||||||
|
saxParser.parse(path, this);
|
||||||
|
} catch (Exception e) {
|
||||||
|
e.printStackTrace();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public RSSReader(InputStream stream) {
|
||||||
|
this();
|
||||||
|
try {
|
||||||
|
SAXParserFactory factory = SAXParserFactory.newInstance();
|
||||||
|
SAXParser saxParser = factory.newSAXParser();
|
||||||
|
saxParser.parse(stream, this);
|
||||||
|
} catch (Exception e) {
|
||||||
|
e.printStackTrace();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public static RSSReader parse(byte[] a) {
|
||||||
|
|
||||||
|
// check integrity of array
|
||||||
|
if ((a == null) || (a.length == 0)) {
|
||||||
|
serverLog.logWarning("rssReader", "response=null");
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
if (a.length < 100) {
|
||||||
|
serverLog.logWarning("rssReader", "response=" + new String(a));
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
if (!serverByteBuffer.equals(a, "<?xml".getBytes())) {
|
||||||
|
serverLog.logWarning("rssReader", "response does not contain valid xml");
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
String end = new String(a, a.length - 10, 10);
|
||||||
|
if (end.indexOf("rss") < 0) {
|
||||||
|
serverLog.logWarning("rssReader", "response incomplete");
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
// make input stream
|
||||||
|
ByteArrayInputStream bais = new ByteArrayInputStream(a);
|
||||||
|
|
||||||
|
// parse stream
|
||||||
|
RSSReader reader = null;
|
||||||
|
try {
|
||||||
|
reader = new RSSReader(bais);
|
||||||
|
} catch (Exception e) {
|
||||||
|
serverLog.logWarning("rssReader", "parse exception: " + e);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
try { bais.close(); } catch (IOException e) {}
|
||||||
|
return reader;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void startElement(String uri, String name, String tag, Attributes atts) throws SAXException {
|
||||||
|
if ("channel".equals(tag)) {
|
||||||
|
item = new RSSMessage();
|
||||||
|
parsingChannel = true;
|
||||||
|
} else if ("item".equals(tag)) {
|
||||||
|
item = new RSSMessage();
|
||||||
|
parsingItem = true;
|
||||||
|
} else if ("image".equals(tag)) {
|
||||||
|
parsingImage = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void endElement(String uri, String name, String tag) {
|
||||||
|
if (tag == null) return;
|
||||||
|
if ("channel".equals(tag)) {
|
||||||
|
parsingChannel = false;
|
||||||
|
theChannel.setChannel(item);
|
||||||
|
} else if ("item".equals(tag)) {
|
||||||
|
theChannel.addMessage(item);
|
||||||
|
parsingItem = false;
|
||||||
|
} else if ("image".equals(tag)) {
|
||||||
|
parsingImage = false;
|
||||||
|
} else if ((parsingImage) && (parsingChannel)) {
|
||||||
|
String value = buffer.toString().trim();
|
||||||
|
buffer.setLength(0);
|
||||||
|
if ("url".equals(tag)) theChannel.setImage(value);
|
||||||
|
} else if (parsingItem) {
|
||||||
|
String value = buffer.toString().trim();
|
||||||
|
buffer.setLength(0);
|
||||||
|
if (RSSMessage.tags.contains(tag)) item.setValue(tag, value);
|
||||||
|
} else if (parsingChannel) {
|
||||||
|
String value = buffer.toString().trim();
|
||||||
|
buffer.setLength(0);
|
||||||
|
if (RSSMessage.tags.contains(tag)) item.setValue(tag, value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void characters(char ch[], int start, int length) {
|
||||||
|
if (parsingItem || parsingChannel) {
|
||||||
|
buffer.append(ch, start, length);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public RSSFeed getFeed() {
|
||||||
|
return theChannel;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
Loading…
Reference in new issue