diff --git a/htroot/gsa/searchresult.java b/htroot/gsa/searchresult.java index ec7afb048..e1a150cc8 100644 --- a/htroot/gsa/searchresult.java +++ b/htroot/gsa/searchresult.java @@ -116,7 +116,7 @@ public class searchresult { post.put("defType", "edismax"); post.put(CommonParams.Q, solrQ.toString()); post.put(CommonParams.ROWS, post.remove("num")); - post.put(CommonParams.ROWS, Math.min(post.getInt(CommonParams.ROWS, 10), (authenticated) ? 5000 : 100)); + post.put(CommonParams.ROWS, Math.min(post.getInt(CommonParams.ROWS, 10), (authenticated) ? 100000000 : 100)); // set ranking if (post.containsKey("sort")) { diff --git a/source/net/yacy/document/Document.java b/source/net/yacy/document/Document.java index 3f2339422..6fada35c9 100644 --- a/source/net/yacy/document/Document.java +++ b/source/net/yacy/document/Document.java @@ -94,6 +94,7 @@ public class Document { private final double lon, lat; private final Object parserObject; // the source object that was used to create the Document private final Map> generic_facets; // a map from vocabulary names to the set of tags for that vocabulary which apply for this document + private final Date date; public Document(final DigestURI location, final String mimeType, final String charset, final Object parserObject, @@ -107,7 +108,8 @@ public class Document { final Map anchors, final Map rss, final Map images, - final boolean indexingDenied) { + final boolean indexingDenied, + final Date date) { this.source = location; this.mimeType = (mimeType == null) ? "application/octet-stream" : mimeType; this.charset = charset; @@ -143,6 +145,7 @@ public class Document { this.indexingDenied = indexingDenied; this.text = text == null ? "" : text; this.generic_facets = new HashMap>(); + this.date = date == null ? new Date() : date; } public Object getParserObject() { @@ -451,6 +454,10 @@ dc_rights return this.emaillinks; } + public Date getDate() { + return this.date; + } + public double lon() { return this.lon; } @@ -783,6 +790,7 @@ dc_rights final Map rss = new HashMap(); final Map images = new HashMap(); double lon = 0.0d, lat = 0.0d; + Date date = new Date(); for (final Document doc: docs) { @@ -821,6 +829,7 @@ dc_rights rss.putAll(doc.getRSS()); ContentScraper.addAllImages(images, doc.getImages()); if (doc.lon() != 0.0 && doc.lat() != 0.0) { lon = doc.lon(); lat = doc.lat(); } + if (doc.date.before(date)) date = doc.date; } // clean up parser data @@ -852,7 +861,8 @@ dc_rights anchors, rss, images, - false); + false, + date); } public static Map getHyperlinks(final Document[] documents) { diff --git a/source/net/yacy/document/content/DCEntry.java b/source/net/yacy/document/content/DCEntry.java index b7253eac4..adb822bb2 100644 --- a/source/net/yacy/document/content/DCEntry.java +++ b/source/net/yacy/document/content/DCEntry.java @@ -100,7 +100,9 @@ public class DCEntry extends MultiMapSolrParams { */ public Date getDate() { String d = this.get("docdatetime"); + if (d == null) d = this.get("date"); if (d == null) d = this.get("dc:date"); + if (d == null) d = this.get("last-modified"); if (d == null) return null; if (d.isEmpty()) return null; try { @@ -286,7 +288,8 @@ public class DCEntry extends MultiMapSolrParams { null, null, null, - false); + false, + getDate()); } public void writeXML(OutputStreamWriter os) throws IOException { diff --git a/source/net/yacy/document/parser/audioTagParser.java b/source/net/yacy/document/parser/audioTagParser.java index aa5f3131d..82044c2cc 100644 --- a/source/net/yacy/document/parser/audioTagParser.java +++ b/source/net/yacy/document/parser/audioTagParser.java @@ -30,6 +30,7 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; +import java.util.Date; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -171,7 +172,8 @@ public class audioTagParser extends AbstractParser implements Parser { null, null, null, - false) + false, + new Date()) }; return docs; } catch (final Exception e) { @@ -193,7 +195,8 @@ public class audioTagParser extends AbstractParser implements Parser { null, null, null, - false + false, + new Date() )}; } finally { try { diff --git a/source/net/yacy/document/parser/csvParser.java b/source/net/yacy/document/parser/csvParser.java index c1bb6e0fa..6d50cbc93 100644 --- a/source/net/yacy/document/parser/csvParser.java +++ b/source/net/yacy/document/parser/csvParser.java @@ -30,6 +30,7 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.io.UnsupportedEncodingException; import java.util.ArrayList; +import java.util.Date; import java.util.List; import net.yacy.document.AbstractParser; @@ -77,7 +78,8 @@ public class csvParser extends AbstractParser implements Parser { null, null, null, - false)}; + false, + new Date())}; } private static String concatRow(String[] columns) { diff --git a/source/net/yacy/document/parser/docParser.java b/source/net/yacy/document/parser/docParser.java index 3e3c84dc1..3a402a1c0 100644 --- a/source/net/yacy/document/parser/docParser.java +++ b/source/net/yacy/document/parser/docParser.java @@ -28,6 +28,7 @@ package net.yacy.document.parser; import java.io.InputStream; +import java.util.Date; import net.yacy.document.AbstractParser; import net.yacy.document.Document; @@ -103,7 +104,8 @@ public class docParser extends AbstractParser implements Parser { null, null, null, - false)}; + false, + new Date())}; return docs; } diff --git a/source/net/yacy/document/parser/genericParser.java b/source/net/yacy/document/parser/genericParser.java index 359746844..8fa1d85ff 100644 --- a/source/net/yacy/document/parser/genericParser.java +++ b/source/net/yacy/document/parser/genericParser.java @@ -25,6 +25,7 @@ package net.yacy.document.parser; import java.io.InputStream; +import java.util.Date; import net.yacy.cora.document.MultiProtocolURI; import net.yacy.document.AbstractParser; @@ -65,7 +66,8 @@ public class genericParser extends AbstractParser implements Parser { null, null, null, - false)}; + false, + new Date())}; return docs; } } diff --git a/source/net/yacy/document/parser/html/ContentScraper.java b/source/net/yacy/document/parser/html/ContentScraper.java index 8cea42843..1e9dd363c 100644 --- a/source/net/yacy/document/parser/html/ContentScraper.java +++ b/source/net/yacy/document/parser/html/ContentScraper.java @@ -31,7 +31,9 @@ import java.io.IOException; import java.io.Writer; import java.net.MalformedURLException; import java.nio.charset.Charset; +import java.text.ParseException; import java.util.ArrayList; +import java.util.Date; import java.util.HashSet; import java.util.Iterator; import java.util.LinkedHashSet; @@ -45,6 +47,7 @@ import java.util.regex.Pattern; import javax.swing.event.EventListenerList; +import net.yacy.cora.date.ISO8601Formatter; import net.yacy.cora.document.MultiProtocolURI; import net.yacy.cora.sorting.ClusteredScoreMap; import net.yacy.cora.storage.SizeLimitedMap; @@ -848,6 +851,28 @@ public class ContentScraper extends AbstractScraper implements Scraper { if (s.toLowerCase().startsWith("url=")) return s.substring(4).trim(); return EMPTY_STRING; } + + public Date getDate() { + String content; + + // + content = this.metas.get("date"); + if (content != null) try {return ISO8601Formatter.FORMATTER.parse(content);} catch (ParseException e) {} + + // + content = this.metas.get("dc.date"); + if (content != null) try {return ISO8601Formatter.FORMATTER.parse(content);} catch (ParseException e) {} + + // + content = this.metas.get("dc:date"); + if (content != null) try {return ISO8601Formatter.FORMATTER.parse(content);} catch (ParseException e) {} + + // + content = this.metas.get("last-modified"); + if (content != null) try {return ISO8601Formatter.FORMATTER.parse(content);} catch (ParseException e) {} + + return new Date(); + } // parse location // diff --git a/source/net/yacy/document/parser/htmlParser.java b/source/net/yacy/document/parser/htmlParser.java index 14bd3eeab..2b5c05f93 100644 --- a/source/net/yacy/document/parser/htmlParser.java +++ b/source/net/yacy/document/parser/htmlParser.java @@ -141,7 +141,8 @@ public class htmlParser extends AbstractParser implements Parser { scraper.getAnchors(), scraper.getRSS(), scraper.getImages(), - scraper.indexingDenied()); + scraper.indexingDenied(), + scraper.getDate()); ppd.setFavicon(scraper.getFavicon()); return ppd; diff --git a/source/net/yacy/document/parser/images/genericImageParser.java b/source/net/yacy/document/parser/images/genericImageParser.java index f8915e9a8..39bf0df8d 100644 --- a/source/net/yacy/document/parser/images/genericImageParser.java +++ b/source/net/yacy/document/parser/images/genericImageParser.java @@ -35,6 +35,7 @@ import java.io.IOException; import java.io.InputStream; import java.net.MalformedURLException; import java.util.ArrayList; +import java.util.Date; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -221,7 +222,8 @@ public class genericImageParser extends AbstractParser implements Parser { anchors, // anchors null, images, - false)}; // images + false, + new Date())}; // images } @Override diff --git a/source/net/yacy/document/parser/mmParser.java b/source/net/yacy/document/parser/mmParser.java index f92738a58..773e0a71f 100644 --- a/source/net/yacy/document/parser/mmParser.java +++ b/source/net/yacy/document/parser/mmParser.java @@ -27,6 +27,7 @@ package net.yacy.document.parser; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; +import java.util.Date; import java.util.List; import javax.xml.parsers.ParserConfigurationException; @@ -116,7 +117,8 @@ public class mmParser extends AbstractParser implements Parser { null, null, null, - false)}; + false, + new Date())}; } private class FreeMindHandler extends DefaultHandler { diff --git a/source/net/yacy/document/parser/odtParser.java b/source/net/yacy/document/parser/odtParser.java index ccf5a3e44..19dcab095 100644 --- a/source/net/yacy/document/parser/odtParser.java +++ b/source/net/yacy/document/parser/odtParser.java @@ -30,6 +30,7 @@ package net.yacy.document.parser; import java.io.File; import java.io.InputStream; import java.util.ArrayList; +import java.util.Date; import java.util.Enumeration; import java.util.HashSet; import java.util.List; @@ -197,7 +198,9 @@ public class odtParser extends AbstractParser implements Parser { null, null, null, - false)}; + false, + new Date() + )}; return docs; } catch (final Exception e) { if (e instanceof InterruptedException) throw (InterruptedException) e; diff --git a/source/net/yacy/document/parser/ooxmlParser.java b/source/net/yacy/document/parser/ooxmlParser.java index 5f7ca2a21..0f473956d 100644 --- a/source/net/yacy/document/parser/ooxmlParser.java +++ b/source/net/yacy/document/parser/ooxmlParser.java @@ -30,6 +30,7 @@ package net.yacy.document.parser; import java.io.File; import java.io.InputStream; import java.util.ArrayList; +import java.util.Date; import java.util.Enumeration; import java.util.HashSet; import java.util.List; @@ -182,7 +183,8 @@ public class ooxmlParser extends AbstractParser implements Parser { null, null, null, - false)}; + false, + new Date())}; return docs; } catch (final Exception e) { if (e instanceof InterruptedException) throw (InterruptedException) e; diff --git a/source/net/yacy/document/parser/pdfParser.java b/source/net/yacy/document/parser/pdfParser.java index a364f0461..dbf6d1991 100644 --- a/source/net/yacy/document/parser/pdfParser.java +++ b/source/net/yacy/document/parser/pdfParser.java @@ -32,6 +32,7 @@ import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; +import java.util.Date; import org.apache.pdfbox.cos.COSName; import org.apache.pdfbox.exceptions.CryptographyException; @@ -125,6 +126,7 @@ public class pdfParser extends AbstractParser implements Parser { // extracting some metadata PDDocumentInformation info = pdfDoc.getDocumentInformation(); String docTitle = null, docSubject = null, docAuthor = null, docPublisher = null, docKeywordStr = null; + Date docDate = new Date(); if (info != null) { docTitle = info.getTitle(); docSubject = info.getSubject(); @@ -132,10 +134,9 @@ public class pdfParser extends AbstractParser implements Parser { docPublisher = info.getProducer(); if (docPublisher == null || docPublisher.isEmpty()) docPublisher = info.getCreator(); docKeywordStr = info.getKeywords(); + try {if (info.getModificationDate() != null) docDate = info.getModificationDate().getTime();} catch (IOException e) {} // unused: // info.getTrapped()); - // info.getCreationDate()); - // info.getModificationDate(); } info = null; @@ -218,7 +219,8 @@ public class pdfParser extends AbstractParser implements Parser { null, null, null, - false)}; + false, + docDate)}; } @SuppressWarnings("static-access") diff --git a/source/net/yacy/document/parser/pptParser.java b/source/net/yacy/document/parser/pptParser.java index c868823fc..59c770627 100644 --- a/source/net/yacy/document/parser/pptParser.java +++ b/source/net/yacy/document/parser/pptParser.java @@ -29,6 +29,7 @@ package net.yacy.document.parser; import java.io.BufferedInputStream; import java.io.InputStream; +import java.util.Date; import net.yacy.cora.util.ConcurrentLog; import net.yacy.document.AbstractParser; @@ -99,7 +100,8 @@ public class pptParser extends AbstractParser implements Parser { null, null, null, - false)}; + false, + new Date())}; return docs; } catch (final Exception e) { if (e instanceof InterruptedException) throw (InterruptedException) e; diff --git a/source/net/yacy/document/parser/psParser.java b/source/net/yacy/document/parser/psParser.java index 208853959..f112927e2 100644 --- a/source/net/yacy/document/parser/psParser.java +++ b/source/net/yacy/document/parser/psParser.java @@ -34,6 +34,7 @@ import java.io.FileReader; import java.io.FileWriter; import java.io.InputStream; import java.io.InputStreamReader; +import java.util.Date; import net.yacy.document.AbstractParser; import net.yacy.document.Document; @@ -115,7 +116,8 @@ public class psParser extends AbstractParser implements Parser { null, // anchors null, // rss null, // images - false)}; // indexingdenied + false, // indexingdenied + new Date())}; return docs; } catch (final Exception e) { diff --git a/source/net/yacy/document/parser/rdfParser.java b/source/net/yacy/document/parser/rdfParser.java index 69e0ab344..d14dacc37 100644 --- a/source/net/yacy/document/parser/rdfParser.java +++ b/source/net/yacy/document/parser/rdfParser.java @@ -27,6 +27,7 @@ package net.yacy.document.parser; import java.io.InputStream; import java.util.ArrayList; +import java.util.Date; import java.util.List; import net.yacy.document.AbstractParser; @@ -59,7 +60,7 @@ public class rdfParser extends AbstractParser implements Parser { String all = "rdfdatasource"; doc = new Document(url, mimeType, charset, null, null, null, singleList(""), "", - "", null, new ArrayList(0), 0, 0, all, null, null, null, false); + "", null, new ArrayList(0), 0, 0, all, null, null, null, false, new Date()); docs.add(doc); diff --git a/source/net/yacy/document/parser/rdfa/impl/RDFaParser.java b/source/net/yacy/document/parser/rdfa/impl/RDFaParser.java index ff011c1ea..0820cd84c 100644 --- a/source/net/yacy/document/parser/rdfa/impl/RDFaParser.java +++ b/source/net/yacy/document/parser/rdfa/impl/RDFaParser.java @@ -13,6 +13,7 @@ import java.io.Reader; import java.net.MalformedURLException; import java.net.URL; import java.util.ArrayList; +import java.util.Date; import java.util.HashSet; import java.util.Set; @@ -80,7 +81,7 @@ public class RDFaParser extends AbstractParser implements Parser { } Document doc = new Document(url, mimeType, charset, null, null, null, singleList(""), "", - "", null, new ArrayList(0), 0, 0, null, null, null, null, false); + "", null, new ArrayList(0), 0, 0, null, null, null, null, false, new Date()); try { if (allTriples.length > 0) @@ -139,7 +140,7 @@ public class RDFaParser extends AbstractParser implements Parser { } Document doc = new Document(url, mimeType, charset, null, null, null, singleList(""), "", - "", null, new ArrayList(0), 0, 0, all, null, null, null, false); + "", null, new ArrayList(0), 0, 0, all, null, null, null, false, new Date()); return doc; } diff --git a/source/net/yacy/document/parser/rssParser.java b/source/net/yacy/document/parser/rssParser.java index 809b2d2e8..af8cb186e 100644 --- a/source/net/yacy/document/parser/rssParser.java +++ b/source/net/yacy/document/parser/rssParser.java @@ -102,7 +102,8 @@ public class rssParser extends AbstractParser implements Parser { anchors, null, new HashMap(), - false); + false, + item.getPubDate()); docs.add(doc); } catch (final MalformedURLException e) { continue; diff --git a/source/net/yacy/document/parser/rtfParser.java b/source/net/yacy/document/parser/rtfParser.java index 22844ec48..1a1495aa7 100644 --- a/source/net/yacy/document/parser/rtfParser.java +++ b/source/net/yacy/document/parser/rtfParser.java @@ -28,6 +28,7 @@ package net.yacy.document.parser; import java.io.InputStream; +import java.util.Date; import javax.swing.text.DefaultStyledDocument; import javax.swing.text.rtf.RTFEditorKit; @@ -84,7 +85,8 @@ public class rtfParser extends AbstractParser implements Parser { null, null, null, - false)}; + false, + new Date())}; } catch (final Exception e) { if (e instanceof InterruptedException) throw (InterruptedException) e; if (e instanceof Parser.Failure) throw (Parser.Failure) e; diff --git a/source/net/yacy/document/parser/sevenzipParser.java b/source/net/yacy/document/parser/sevenzipParser.java index 97754e2f6..b7c7464d0 100644 --- a/source/net/yacy/document/parser/sevenzipParser.java +++ b/source/net/yacy/document/parser/sevenzipParser.java @@ -32,6 +32,7 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.util.Date; import net.yacy.cora.util.ConcurrentLog; import net.yacy.document.AbstractParser; @@ -72,7 +73,8 @@ public class sevenzipParser extends AbstractParser implements Parser { null, null, null, - false); + false, + new Date()); Handler archive; AbstractParser.log.fine("opening 7zip archive..."); try { diff --git a/source/net/yacy/document/parser/sidAudioParser.java b/source/net/yacy/document/parser/sidAudioParser.java index e1f13c8f0..aa3cf643d 100644 --- a/source/net/yacy/document/parser/sidAudioParser.java +++ b/source/net/yacy/document/parser/sidAudioParser.java @@ -27,6 +27,7 @@ package net.yacy.document.parser; import java.io.IOException; import java.io.InputStream; import java.nio.charset.Charset; +import java.util.Date; import java.util.HashMap; import java.util.Map; @@ -95,7 +96,8 @@ public class sidAudioParser extends AbstractParser implements Parser { null, null, null, - false)}; + false, + new Date())}; } throw new Parser.Failure("Unable to parse SID file, file does seems to be incomplete (len = " + available + ").", location); } catch (final IOException ex) { diff --git a/source/net/yacy/document/parser/sitemapParser.java b/source/net/yacy/document/parser/sitemapParser.java index e0c080b07..ce42595fa 100644 --- a/source/net/yacy/document/parser/sitemapParser.java +++ b/source/net/yacy/document/parser/sitemapParser.java @@ -96,7 +96,8 @@ public class sitemapParser extends AbstractParser implements Parser { null, null, new HashMap(), - false); + false, + new Date()); docs.add(doc); } catch (final MalformedURLException e) { continue; diff --git a/source/net/yacy/document/parser/swfParser.java b/source/net/yacy/document/parser/swfParser.java index b42a43b89..311420a8a 100644 --- a/source/net/yacy/document/parser/swfParser.java +++ b/source/net/yacy/document/parser/swfParser.java @@ -30,6 +30,7 @@ package net.yacy.document.parser; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; +import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -125,7 +126,8 @@ public class swfParser extends AbstractParser implements Parser { anchors, // a map of extracted anchors null, null, - false)}; // a treeset of image URLs + false, + new Date())}; // a treeset of image URLs } catch (final Exception e) { if (e instanceof InterruptedException) throw (InterruptedException) e; diff --git a/source/net/yacy/document/parser/torrentParser.java b/source/net/yacy/document/parser/torrentParser.java index 5c61d5672..4dcf33d4d 100644 --- a/source/net/yacy/document/parser/torrentParser.java +++ b/source/net/yacy/document/parser/torrentParser.java @@ -28,6 +28,7 @@ import java.io.ByteArrayInputStream; import java.io.File; import java.io.IOException; import java.io.InputStream; +import java.util.Date; import java.util.List; import java.util.Map; @@ -110,7 +111,8 @@ public class torrentParser extends AbstractParser implements Parser { null, null, null, - false)}; + false, + new Date())}; } public static void main(String[] args) { diff --git a/source/net/yacy/document/parser/vcfParser.java b/source/net/yacy/document/parser/vcfParser.java index 867c6ea25..25726314c 100644 --- a/source/net/yacy/document/parser/vcfParser.java +++ b/source/net/yacy/document/parser/vcfParser.java @@ -33,6 +33,7 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.net.MalformedURLException; import java.util.ArrayList; +import java.util.Date; import java.util.HashMap; import java.util.Iterator; import java.util.LinkedList; @@ -227,7 +228,8 @@ public class vcfParser extends AbstractParser implements Parser { anchors, // a map of extracted anchors null, null, // a treeset of image URLs - false)}; + false, + new Date())}; } catch (final Exception e) { if (e instanceof InterruptedException) throw (InterruptedException) e; if (e instanceof Parser.Failure) throw (Parser.Failure) e; diff --git a/source/net/yacy/document/parser/vsdParser.java b/source/net/yacy/document/parser/vsdParser.java index 9eb458456..6ea7b3885 100644 --- a/source/net/yacy/document/parser/vsdParser.java +++ b/source/net/yacy/document/parser/vsdParser.java @@ -29,6 +29,7 @@ package net.yacy.document.parser; import java.io.InputStream; import java.util.ArrayList; +import java.util.Date; import java.util.List; import net.yacy.cora.util.ConcurrentLog; @@ -115,7 +116,8 @@ public class vsdParser extends AbstractParser implements Parser { null, // a map of extracted anchors null, null, // a treeset of image URLs - false)}; + false, + new Date())}; } catch (final Exception e) { if (e instanceof InterruptedException) throw (InterruptedException) e; diff --git a/source/net/yacy/document/parser/xlsParser.java b/source/net/yacy/document/parser/xlsParser.java index 41c2a4244..850f4287a 100644 --- a/source/net/yacy/document/parser/xlsParser.java +++ b/source/net/yacy/document/parser/xlsParser.java @@ -28,6 +28,7 @@ package net.yacy.document.parser; import java.io.InputStream; +import java.util.Date; import net.yacy.cora.util.ConcurrentLog; import net.yacy.document.AbstractParser; @@ -129,7 +130,8 @@ public class xlsParser extends AbstractParser implements Parser { null, null, null, - false)}; + false, + new Date())}; } catch (final Exception e) { if (e instanceof InterruptedException) throw (InterruptedException) e; diff --git a/source/net/yacy/search/Switchboard.java b/source/net/yacy/search/Switchboard.java index 566ba979b..2fe0affdf 100644 --- a/source/net/yacy/search/Switchboard.java +++ b/source/net/yacy/search/Switchboard.java @@ -2885,8 +2885,8 @@ public final class Switchboard extends serverSwitch { while (t-- > 0 && this.index.exists(ids).size() > 0) { try {Thread.sleep(100);} catch (final InterruptedException e) {} ConcurrentLog.fine("Switchboard", "STACKURL: waiting for deletion, t=" + t); - if (t == 20) this.index.fulltext().commit(true); - if (t == 40) this.index.fulltext().commit(false); + //if (t == 20) this.index.fulltext().commit(true); + if (t == 1) this.index.fulltext().commit(false); } // special handling of ftp protocol diff --git a/source/net/yacy/search/schema/CollectionConfiguration.java b/source/net/yacy/search/schema/CollectionConfiguration.java index 46295b3bb..107c04686 100644 --- a/source/net/yacy/search/schema/CollectionConfiguration.java +++ b/source/net/yacy/search/schema/CollectionConfiguration.java @@ -441,7 +441,11 @@ public class CollectionConfiguration extends SchemaConfiguration implements Seri add(doc, CollectionSchema.author, author); } if (allAttr || contains(CollectionSchema.content_type)) add(doc, CollectionSchema.content_type, new String[]{document.dc_format()}); - if (allAttr || contains(CollectionSchema.last_modified)) add(doc, CollectionSchema.last_modified, responseHeader == null ? new Date() : responseHeader.lastModified()); + if (allAttr || contains(CollectionSchema.last_modified)) { + Date lastModified = responseHeader == null ? new Date() : responseHeader.lastModified(); + if (document.getDate().before(lastModified)) lastModified = document.getDate(); + add(doc, CollectionSchema.last_modified, lastModified); + } if (allAttr || contains(CollectionSchema.keywords)) add(doc, CollectionSchema.keywords, document.dc_subject(' ')); if (allAttr || contains(CollectionSchema.synonyms_sxt)) { List synonyms = condenser.synonyms(); diff --git a/source/net/yacy/visualization/AnimationPlotter.java b/source/net/yacy/visualization/AnimationPlotter.java new file mode 100644 index 000000000..4cf21c023 --- /dev/null +++ b/source/net/yacy/visualization/AnimationPlotter.java @@ -0,0 +1,97 @@ +/** + * AnimationPlotter + * Copyright 2013 by Michael Christen + * First released 9.9.2010 at http://yacy.net + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program in the file lgpl21.txt + * If not, see . + */ + +package net.yacy.visualization; + +import java.awt.image.BufferedImage; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import javax.imageio.ImageIO; +import javax.swing.ImageIcon; +import javax.swing.JFrame; +import javax.swing.JLabel; + +public class AnimationPlotter { + + public static class Frame { + BufferedImage image; + int delayMillis; + public Frame(BufferedImage image, int delayMillis) { + this.image = image; + this.delayMillis = delayMillis; + } + } + + private final List frames; + + + public AnimationPlotter() { + this.frames = new ArrayList(); + } + + public void addFrame(final BufferedImage image, final int delayMillis) { + this.frames.add(new Frame(image, delayMillis)); + } + + + public void save(final File path, final String filestub, final String type) throws IOException { + assert path.isDirectory(); + for (int i = 0; i < this.frames.size(); i++) { + Frame frame = this.frames.get(i); + File file = new File(path, filestub + "_" + intformat(i) + '.' + type); + final FileOutputStream fos = new FileOutputStream(file); + ImageIO.write(frame.image, type, fos); + fos.close(); + } + } + + private String intformat(final int i) { + String n = Integer.toString(i); + while (n.length() < 6) n = '0' + n; + return n; + } + + /** + * show the images as stream of JFrame on desktop + */ + public void show() { + JFrame f = new JFrame(); + f.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE); + f.setVisible(true); + JLabel label = null; + while (true) { + for (int i = 0; i < this.frames.size(); i++) { + Frame frame = this.frames.get(i); + if (label == null) { + label = new JLabel(new ImageIcon(frame.image)); + f.getContentPane().add(label); + f.pack(); + } else { + label.getGraphics().drawImage(frame.image,0,0, label); + } + try {Thread.sleep(frame.delayMillis);} catch (InterruptedException e) {} + } + } + } +} diff --git a/source/net/yacy/visualization/HexGridPlotter.java b/source/net/yacy/visualization/HexGridPlotter.java index 274c34b21..cc808708b 100644 --- a/source/net/yacy/visualization/HexGridPlotter.java +++ b/source/net/yacy/visualization/HexGridPlotter.java @@ -25,10 +25,8 @@ package net.yacy.visualization; import java.io.File; -import java.io.FileOutputStream; import java.io.IOException; - -import javax.imageio.ImageIO; +import net.yacy.cora.util.ConcurrentLog; /* * hexagonal grid plotter @@ -151,23 +149,21 @@ public class HexGridPlotter extends RasterPlotter { } } - public static void main(final String[] args) { - // go into headless awt mode - System.setProperty("java.awt.headless", "true"); - - final HexGridPlotter picture = new HexGridPlotter(640, 480, DrawMode.MODE_SUB, "FFFFFF", 18); - picture.drawGrid("555555"); - picture.setColor(Long.parseLong("33ff33", 16)); + private static HexGridPlotter testImage0(int width, int height, String bgcolor, String gridcolor, String dotcolor) { + final HexGridPlotter picture = new HexGridPlotter(width, height, bgcolor.equals("000000") ? DrawMode.MODE_ADD : DrawMode.MODE_SUB, bgcolor, 18); + picture.drawGrid(gridcolor); + long dotcolori = Long.parseLong(dotcolor, 16); + picture.setColor(dotcolori); picture.gridDot(0, 0, 5, true, 100); picture.gridPrint(0, 0, 5, "", "0,0", -1); for (int i = 1; i < picture.gridHeight() -1; i++) { - picture.setColor(Long.parseLong("33ff33", 16));picture.gridDot(0, i, 3, true, 100); - picture.setColor(Long.parseLong("334433", 16));picture.gridPrint(0, i, 3, "", "0," + i, -1); + picture.setColor(dotcolori);picture.gridDot(0, i, 3, true, 100); + picture.setColor(dotcolori);picture.gridPrint(0, i, 3, "", "0," + i, -1); } for (int i = 1; i < picture.gridWidth() -1; i++) { - picture.setColor(Long.parseLong("33ff33", 16));picture.gridDot(i, 0, 3, true, 100); - picture.setColor(Long.parseLong("334433", 16));picture.gridPrint315(i, 0, 3, i + ",0"); + picture.setColor(dotcolori);picture.gridDot(i, 0, 3, true, 100); + picture.setColor(dotcolori);picture.gridPrint315(i, 0, 3, i + ",0"); } - picture.setColor(Long.parseLong("33ff33", 16)); + picture.setColor(dotcolori); picture.gridDot(0, picture.gheight - 1, 5, true, 100); picture.gridPrint(0, picture.gheight - 1, 5, "0, grid.gheight - 1", "", -1); picture.gridDot(picture.gwidth - 1, 0, 5, true, 100); picture.gridPrint(picture.gwidth - 1, 0, 5, "", "grid.gwidth - 1, 0", -1); picture.gridDot(picture.gwidth - 1, picture.gheight - 1, 5, true, 100); picture.gridPrint(picture.gwidth - 1, picture.gheight - 1, 5, "grid.gwidth - 1, grid.gheight - 1", "", 1); @@ -191,13 +187,23 @@ public class HexGridPlotter extends RasterPlotter { picture.gridPrint(picture.gwidth / 2, picture.gheight / 2, 5, "HOME PEER", "ANON_23", 0); //grid.gridLine(grid.gwidth - 2, grid.gheight - 2, grid.gwidth / 2, grid.gheight / 2); picture.gridLine(picture.gwidth / 2, picture.gheight / 2, picture.gwidth - 1, picture.gheight - 4); + return picture; + } + + public static void main(final String[] args) { + // go into headless awt mode + //System.setProperty("java.awt.headless", "true"); + AnimationPlotter animation = new AnimationPlotter(); + for (int i = 640; i < 700; i++) { + animation.addFrame(testImage0(i, 480, "000000", "555555", "33ff33").getImage(), 10); + } + animation.show(); + HexGridPlotter picture = testImage0(640, 480, "FFFFFF", "555555", "33ff33"); final File file = new File("/Users/admin/Desktop/testimage.png"); - try { - final FileOutputStream fos = new FileOutputStream(file); - ImageIO.write(picture.getImage(), "png", fos); - fos.close(); - } catch (final IOException e) {} + try {picture.save(file, "png");} catch (final IOException e) {} + ConcurrentLog.shutdown(); + if (!System.getProperty("java.awt.headless", "false").equals("true")) picture.show(); } } diff --git a/source/net/yacy/visualization/RasterPlotter.java b/source/net/yacy/visualization/RasterPlotter.java index e4689d6f7..106f86c06 100644 --- a/source/net/yacy/visualization/RasterPlotter.java +++ b/source/net/yacy/visualization/RasterPlotter.java @@ -54,6 +54,9 @@ import java.util.zip.Deflater; import java.util.zip.DeflaterOutputStream; import javax.imageio.ImageIO; +import javax.swing.ImageIcon; +import javax.swing.JFrame; +import javax.swing.JLabel; import net.yacy.cora.util.ByteBuffer; import net.yacy.cora.util.ConcurrentLog; @@ -92,7 +95,11 @@ public class RasterPlotter { private DrawMode defaultMode; private byte[] frame; - public RasterPlotter(final int width, final int height, final long backgroundColor) { + public RasterPlotter(final int width, final int height, final DrawMode drawMode, final String backgroundColor) { + this(width, height, drawMode, Long.parseLong(backgroundColor, 16)); + } + + public RasterPlotter(final int width, final int height, final DrawMode drawMode, final long backgroundColor) { this.cc = new int[3]; this.width = width; this.height = height; @@ -100,14 +107,6 @@ public class RasterPlotter { this.defaultColR = 0xFF; this.defaultColG = 0xFF; this.defaultColB = 0xFF; - } - - public RasterPlotter(final int width, final int height, final DrawMode drawMode, final String backgroundColor) { - this(width, height, drawMode, Long.parseLong(backgroundColor, 16)); - } - - public RasterPlotter(final int width, final int height, final DrawMode drawMode, final long backgroundColor) { - this(width, height, backgroundColor); this.defaultMode = drawMode; try { // we need our own frame buffer to get a very, very fast transformation to png because we can omit the PixedGrabber, which is up to 800 times slower @@ -941,6 +940,30 @@ public class RasterPlotter { } } + /** + * save the image to a file + * @param file the storage file + * @param type the file type, may be i.e. 'png' or 'gif' + * @throws IOException + */ + public void save(File file, String type) throws IOException { + final FileOutputStream fos = new FileOutputStream(file); + ImageIO.write(this.image, type, fos); + fos.close(); + } + + /** + * show the image as JFrame on desktop + */ + public void show() { + JLabel label = new JLabel(new ImageIcon(this.image)); + JFrame f = new JFrame(); + f.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE); + f.getContentPane().add(label); + f.pack(); + f.setVisible(true); + } + /* * The following code was transformed from a library, coded by J. David Eisenberg, version 1.5, 19 Oct 2003 (C) LGPL * This code was very strongly transformed into the following very short method for an ultra-fast png generation. diff --git a/test/de/anomic/document/ParserTest.java b/test/de/anomic/document/ParserTest.java index dd7755f87..66b5891b1 100644 --- a/test/de/anomic/document/ParserTest.java +++ b/test/de/anomic/document/ParserTest.java @@ -53,7 +53,7 @@ public class ParserTest { assertThat(str.toString(), containsString("In München steht ein Hofbräuhaus, dort gibt es Bier in Maßkrügen")); assertThat(doc.dc_title(), containsString(testFile[2])); assertThat(doc.dc_creator(), containsString(testFile[3])); - assertThat(doc.dc_description(), containsString(testFile[4])); + assertThat(doc.dc_description()[0], containsString(testFile[4])); } } catch (final InterruptedException ex) {} } @@ -87,7 +87,7 @@ public class ParserTest { assertThat(str.toString(), containsString("In München steht ein Hofbräuhaus, dort gibt es Bier in Maßkrügen")); // assertThat(doc.dc_title(), containsString(testFile[2])); assertThat(doc.dc_creator(), containsString(testFile[3])); - assertThat(doc.dc_description(), containsString(testFile[4])); + assertThat(doc.dc_description()[0], containsString(testFile[4])); } } catch (final InterruptedException ex) {} } @@ -119,7 +119,7 @@ public class ParserTest { assertThat(str.toString(), containsString("In München steht ein Hofbräuhaus, dort gibt es Bier in Maßkrügen")); assertThat(doc.dc_title(), containsString(testFile[2])); assertThat(doc.dc_creator(), containsString(testFile[3])); - assertThat(doc.dc_description(), containsString(testFile[4])); + assertThat(doc.dc_description()[0], containsString(testFile[4])); } } catch (final InterruptedException ex) {} } @@ -151,7 +151,7 @@ public class ParserTest { assertThat(str.toString(), containsString("In München steht ein Hofbräuhaus, dort gibt es Bier in Maßkrügen")); assertThat(doc.dc_title(), containsString(testFile[2])); assertThat(doc.dc_creator(), containsString(testFile[3])); - assertThat(doc.dc_description(), containsString(testFile[4])); + assertThat(doc.dc_description()[0], containsString(testFile[4])); } } catch (final InterruptedException ex) {} }