diff --git a/.classpath b/.classpath index 139752929..9fbf35629 100644 --- a/.classpath +++ b/.classpath @@ -1,97 +1,97 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/build.xml b/build.xml index d5ed8631e..f20012f69 100644 --- a/build.xml +++ b/build.xml @@ -172,21 +172,21 @@ - + - + - - + + - - + + @@ -201,39 +201,39 @@ - + - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + - + - - - - + + + + - + diff --git a/defaults/yacy.init b/defaults/yacy.init index 9eab81b97..a238d2ed4 100644 --- a/defaults/yacy.init +++ b/defaults/yacy.init @@ -103,6 +103,7 @@ filesize.max.other = 8589934591 # In case of privately managed networks, this configuration must be changed BEFORE it is released # to the members of the separated network peers. network.unit.definition = defaults/yacy.network.freeworld.unit +#network.unit.definition = defaults/yacy.network.webportal.unit #network.unit.definition = defaults/yacy.network.intranet.unit # distinguish intranet/internet IPs: diff --git a/htroot/IndexExport_p.html b/htroot/IndexExport_p.html index aa03afedf..527190049 100644 --- a/htroot/IndexExport_p.html +++ b/htroot/IndexExport_p.html @@ -39,19 +39,21 @@
Only Domain:
Plain Text List (domains only)
HTML (domains as URLs, no title)
+
Only Text:
+
Fulltext of Search Index Text
 
-
+
:: -
Export to file #[exportfile]# is running .. #[urlcount]# URLs so far
:: +
Export to file #[exportfile]# is running .. #[urlcount]# Documents so far
:: #(/lurlexport)# #(lurlexportfinished)#:: -
Finished export of #[urlcount]# URLs to file #[exportfile]#
+
Finished export of #[urlcount]# Documents to file #[exportfile]#
Import this file by moving it to DATA/SURROGATES/in
:: #(/lurlexportfinished)# diff --git a/htroot/IndexExport_p.java b/htroot/IndexExport_p.java index 7a7a5e3bc..4dc8cfb93 100644 --- a/htroot/IndexExport_p.java +++ b/htroot/IndexExport_p.java @@ -93,27 +93,28 @@ public class IndexExport_p { if (post.containsKey("lurlexport")) { // parse format - int format = 0; + Fulltext.ExportFormat format = Fulltext.ExportFormat.text; final String fname = post.get("format", "url-text"); final boolean dom = fname.startsWith("dom"); // if dom== false complete urls are exported, otherwise only the domain - if (fname.endsWith("text")) format = 0; - if (fname.endsWith("html")) format = 1; - if (fname.endsWith("rss")) format = 2; - if (fname.endsWith("solr")) format = 3; + final boolean text = fname.startsWith("text"); + if (fname.endsWith("text")) format = Fulltext.ExportFormat.text; + if (fname.endsWith("html")) format = Fulltext.ExportFormat.html; + if (fname.endsWith("rss")) format = Fulltext.ExportFormat.rss; + if (fname.endsWith("solr")) format = Fulltext.ExportFormat.solr; // extend export file name String s = post.get("exportfile", ""); if (s.indexOf('.',0) < 0) { - if (format == 0) s = s + ".txt"; - if (format == 1) s = s + ".html"; - if (format == 2 ) s = s + "_rss.xml"; - if (format == 3) s = s + "_full.xml"; + if (format == Fulltext.ExportFormat.text) s = s + ".txt"; + if (format == Fulltext.ExportFormat.html) s = s + ".html"; + if (format == Fulltext.ExportFormat.rss ) s = s + "_rss.xml"; + if (format == Fulltext.ExportFormat.solr) s = s + "_full.xml"; } final File f = new File(s); f.getParentFile().mkdirs(); final String filter = post.get("exportfilter", ".*"); final String query = post.get("exportquery", "*:*"); - final Fulltext.Export running = segment.fulltext().export(f, filter, query, format, dom); + final Fulltext.Export running = segment.fulltext().export(f, filter, query, format, dom, text); prop.put("lurlexport_exportfile", s); prop.put("lurlexport_urlcount", running.count()); diff --git a/lib/fontbox-1.8.9.License b/lib/fontbox-1.8.10.License similarity index 100% rename from lib/fontbox-1.8.9.License rename to lib/fontbox-1.8.10.License diff --git a/lib/fontbox-1.8.9.jar b/lib/fontbox-1.8.10.jar similarity index 74% rename from lib/fontbox-1.8.9.jar rename to lib/fontbox-1.8.10.jar index 05062f156..3284950c8 100644 Binary files a/lib/fontbox-1.8.9.jar and b/lib/fontbox-1.8.10.jar differ diff --git a/lib/httpclient-4.4.1.License b/lib/httpclient-4.5.License similarity index 100% rename from lib/httpclient-4.4.1.License rename to lib/httpclient-4.5.License diff --git a/lib/httpclient-4.4.1.jar b/lib/httpclient-4.5.jar similarity index 77% rename from lib/httpclient-4.4.1.jar rename to lib/httpclient-4.5.jar index b80d37967..970c9891c 100644 Binary files a/lib/httpclient-4.4.1.jar and b/lib/httpclient-4.5.jar differ diff --git a/lib/httpmime-4.4.1.License b/lib/httpmime-4.5.License similarity index 100% rename from lib/httpmime-4.4.1.License rename to lib/httpmime-4.5.License diff --git a/lib/httpmime-4.4.1.jar b/lib/httpmime-4.5.jar similarity index 65% rename from lib/httpmime-4.4.1.jar rename to lib/httpmime-4.5.jar index e748cbde7..b631ceb4e 100644 Binary files a/lib/httpmime-4.4.1.jar and b/lib/httpmime-4.5.jar differ diff --git a/lib/icu4j-55_1.jar b/lib/icu4j-55_1.jar new file mode 100644 index 000000000..e281a87a4 Binary files /dev/null and b/lib/icu4j-55_1.jar differ diff --git a/lib/icu4j-core.jar b/lib/icu4j-core.jar deleted file mode 100644 index b62189c04..000000000 Binary files a/lib/icu4j-core.jar and /dev/null differ diff --git a/lib/jcl-over-slf4j-1.7.12.jar b/lib/jcl-over-slf4j-1.7.12.jar new file mode 100644 index 000000000..2030a7037 Binary files /dev/null and b/lib/jcl-over-slf4j-1.7.12.jar differ diff --git a/lib/jcl-over-slf4j-1.7.9.jar b/lib/jcl-over-slf4j-1.7.9.jar deleted file mode 100644 index 682a46b1f..000000000 Binary files a/lib/jcl-over-slf4j-1.7.9.jar and /dev/null differ diff --git a/lib/jempbox-1.8.9.License b/lib/jempbox-1.8.10.License similarity index 100% rename from lib/jempbox-1.8.9.License rename to lib/jempbox-1.8.10.License diff --git a/lib/jempbox-1.8.9.jar b/lib/jempbox-1.8.10.jar similarity index 85% rename from lib/jempbox-1.8.9.jar rename to lib/jempbox-1.8.10.jar index 29c35d073..48cc63375 100644 Binary files a/lib/jempbox-1.8.9.jar and b/lib/jempbox-1.8.10.jar differ diff --git a/lib/jsch-0.1.52.jar b/lib/jsch-0.1.52.jar deleted file mode 100644 index 9e50e6742..000000000 Binary files a/lib/jsch-0.1.52.jar and /dev/null differ diff --git a/lib/jsch-0.1.52.License b/lib/jsch-0.1.53.License similarity index 94% rename from lib/jsch-0.1.52.License rename to lib/jsch-0.1.53.License index 2cf7a501a..303096bf3 100644 --- a/lib/jsch-0.1.52.License +++ b/lib/jsch-0.1.53.License @@ -1,30 +1,30 @@ -JSch 0.0.* was released under the GNU LGPL license. Later, we have switched -over to a BSD-style license. - ------------------------------------------------------------------------------- -Copyright (c) 2002,2003,2004 Atsuhiko Yamanaka, JCraft,Inc. -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - 1. Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in - the documentation and/or other materials provided with the distribution. - - 3. The names of the authors may not be used to endorse or promote products - derived from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, -INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND -FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL JCRAFT, -INC. OR ANY CONTRIBUTORS TO THIS SOFTWARE BE LIABLE FOR ANY DIRECT, INDIRECT, -INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, -OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, -EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +JSch 0.0.* was released under the GNU LGPL license. Later, we have switched +over to a BSD-style license. + +------------------------------------------------------------------------------ +Copyright (c) 2002-2015 Atsuhiko Yamanaka, JCraft,Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the distribution. + + 3. The names of the authors may not be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, +INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND +FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL JCRAFT, +INC. OR ANY CONTRIBUTORS TO THIS SOFTWARE BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, +OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, +EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/lib/jsch-0.1.53.jar b/lib/jsch-0.1.53.jar new file mode 100644 index 000000000..2e4c69d6a Binary files /dev/null and b/lib/jsch-0.1.53.jar differ diff --git a/lib/log4j-over-slf4j-1.7.12.jar b/lib/log4j-over-slf4j-1.7.12.jar new file mode 100644 index 000000000..fd3f48c81 Binary files /dev/null and b/lib/log4j-over-slf4j-1.7.12.jar differ diff --git a/lib/log4j-over-slf4j-1.7.9.jar b/lib/log4j-over-slf4j-1.7.9.jar deleted file mode 100644 index c8289369d..000000000 Binary files a/lib/log4j-over-slf4j-1.7.9.jar and /dev/null differ diff --git a/lib/lucene-analyzers-common-5.2.0.jar b/lib/lucene-analyzers-common-5.2.1.jar similarity index 94% rename from lib/lucene-analyzers-common-5.2.0.jar rename to lib/lucene-analyzers-common-5.2.1.jar index 9cc923559..aaa26a135 100644 Binary files a/lib/lucene-analyzers-common-5.2.0.jar and b/lib/lucene-analyzers-common-5.2.1.jar differ diff --git a/lib/lucene-analyzers-phonetic-5.2.0.jar b/lib/lucene-analyzers-phonetic-5.2.1.jar similarity index 89% rename from lib/lucene-analyzers-phonetic-5.2.0.jar rename to lib/lucene-analyzers-phonetic-5.2.1.jar index 3287b9d37..838b8daf2 100644 Binary files a/lib/lucene-analyzers-phonetic-5.2.0.jar and b/lib/lucene-analyzers-phonetic-5.2.1.jar differ diff --git a/lib/lucene-backward-codecs-5.2.0.jar b/lib/lucene-backward-codecs-5.2.1.jar similarity index 92% rename from lib/lucene-backward-codecs-5.2.0.jar rename to lib/lucene-backward-codecs-5.2.1.jar index 3176e662e..bbdfff8b2 100644 Binary files a/lib/lucene-backward-codecs-5.2.0.jar and b/lib/lucene-backward-codecs-5.2.1.jar differ diff --git a/lib/lucene-classification-5.2.0.jar b/lib/lucene-classification-5.2.1.jar similarity index 92% rename from lib/lucene-classification-5.2.0.jar rename to lib/lucene-classification-5.2.1.jar index 290be7a74..9901b5a97 100644 Binary files a/lib/lucene-classification-5.2.0.jar and b/lib/lucene-classification-5.2.1.jar differ diff --git a/lib/lucene-codecs-5.2.0.jar b/lib/lucene-codecs-5.2.1.jar similarity index 93% rename from lib/lucene-codecs-5.2.0.jar rename to lib/lucene-codecs-5.2.1.jar index e5faf7922..be7e95778 100644 Binary files a/lib/lucene-codecs-5.2.0.jar and b/lib/lucene-codecs-5.2.1.jar differ diff --git a/lib/lucene-core-5.2.0.jar b/lib/lucene-core-5.2.1.jar similarity index 88% rename from lib/lucene-core-5.2.0.jar rename to lib/lucene-core-5.2.1.jar index 6b192d886..18b887f79 100644 Binary files a/lib/lucene-core-5.2.0.jar and b/lib/lucene-core-5.2.1.jar differ diff --git a/lib/lucene-facet-5.2.0.jar b/lib/lucene-facet-5.2.1.jar similarity index 91% rename from lib/lucene-facet-5.2.0.jar rename to lib/lucene-facet-5.2.1.jar index 587afa2b1..fb96a545d 100644 Binary files a/lib/lucene-facet-5.2.0.jar and b/lib/lucene-facet-5.2.1.jar differ diff --git a/lib/lucene-grouping-5.2.0.jar b/lib/lucene-grouping-5.2.1.jar similarity index 90% rename from lib/lucene-grouping-5.2.0.jar rename to lib/lucene-grouping-5.2.1.jar index 977019d0a..6d7b46dd1 100644 Binary files a/lib/lucene-grouping-5.2.0.jar and b/lib/lucene-grouping-5.2.1.jar differ diff --git a/lib/lucene-highlighter-5.2.0.jar b/lib/lucene-highlighter-5.2.1.jar similarity index 91% rename from lib/lucene-highlighter-5.2.0.jar rename to lib/lucene-highlighter-5.2.1.jar index d20d5d6c9..e81ecbd7c 100644 Binary files a/lib/lucene-highlighter-5.2.0.jar and b/lib/lucene-highlighter-5.2.1.jar differ diff --git a/lib/lucene-join-5.2.0.jar b/lib/lucene-join-5.2.1.jar similarity index 90% rename from lib/lucene-join-5.2.0.jar rename to lib/lucene-join-5.2.1.jar index 08fec00ac..570275134 100644 Binary files a/lib/lucene-join-5.2.0.jar and b/lib/lucene-join-5.2.1.jar differ diff --git a/lib/lucene-memory-5.2.0.jar b/lib/lucene-memory-5.2.1.jar similarity index 91% rename from lib/lucene-memory-5.2.0.jar rename to lib/lucene-memory-5.2.1.jar index 6bc7dc8a9..7c4eb8ec2 100644 Binary files a/lib/lucene-memory-5.2.0.jar and b/lib/lucene-memory-5.2.1.jar differ diff --git a/lib/lucene-misc-5.2.0.jar b/lib/lucene-misc-5.2.1.jar similarity index 91% rename from lib/lucene-misc-5.2.0.jar rename to lib/lucene-misc-5.2.1.jar index 69dc8474f..ce6eeba16 100644 Binary files a/lib/lucene-misc-5.2.0.jar and b/lib/lucene-misc-5.2.1.jar differ diff --git a/lib/lucene-queries-5.2.0.jar b/lib/lucene-queries-5.2.1.jar similarity index 87% rename from lib/lucene-queries-5.2.0.jar rename to lib/lucene-queries-5.2.1.jar index 145acb91e..01742fa90 100644 Binary files a/lib/lucene-queries-5.2.0.jar and b/lib/lucene-queries-5.2.1.jar differ diff --git a/lib/lucene-queryparser-5.2.0.jar b/lib/lucene-queryparser-5.2.1.jar similarity index 90% rename from lib/lucene-queryparser-5.2.0.jar rename to lib/lucene-queryparser-5.2.1.jar index bc8926528..ee7599359 100644 Binary files a/lib/lucene-queryparser-5.2.0.jar and b/lib/lucene-queryparser-5.2.1.jar differ diff --git a/lib/lucene-spatial-5.2.0.jar b/lib/lucene-spatial-5.2.1.jar similarity index 91% rename from lib/lucene-spatial-5.2.0.jar rename to lib/lucene-spatial-5.2.1.jar index d8478e77c..51b36233f 100644 Binary files a/lib/lucene-spatial-5.2.0.jar and b/lib/lucene-spatial-5.2.1.jar differ diff --git a/lib/lucene-suggest-5.2.0.jar b/lib/lucene-suggest-5.2.1.jar similarity index 92% rename from lib/lucene-suggest-5.2.0.jar rename to lib/lucene-suggest-5.2.1.jar index c78858bdc..e30c5d41d 100644 Binary files a/lib/lucene-suggest-5.2.0.jar and b/lib/lucene-suggest-5.2.1.jar differ diff --git a/lib/pdfbox-1.8.9.License b/lib/pdfbox-1.8.10.License similarity index 100% rename from lib/pdfbox-1.8.9.License rename to lib/pdfbox-1.8.10.License diff --git a/lib/pdfbox-1.8.9.jar b/lib/pdfbox-1.8.10.jar similarity index 86% rename from lib/pdfbox-1.8.9.jar rename to lib/pdfbox-1.8.10.jar index 9b82e73ee..87bb9a704 100644 Binary files a/lib/pdfbox-1.8.9.jar and b/lib/pdfbox-1.8.10.jar differ diff --git a/lib/slf4j-api-1.7.12.jar b/lib/slf4j-api-1.7.12.jar new file mode 100644 index 000000000..e357ddc7f Binary files /dev/null and b/lib/slf4j-api-1.7.12.jar differ diff --git a/lib/slf4j-api-1.7.9.jar b/lib/slf4j-api-1.7.9.jar deleted file mode 100644 index 3d7274426..000000000 Binary files a/lib/slf4j-api-1.7.9.jar and /dev/null differ diff --git a/lib/slf4j-jdk14-1.7.12.jar b/lib/slf4j-jdk14-1.7.12.jar new file mode 100644 index 000000000..cf078f810 Binary files /dev/null and b/lib/slf4j-jdk14-1.7.12.jar differ diff --git a/lib/slf4j-jdk14-1.7.9.jar b/lib/slf4j-jdk14-1.7.9.jar deleted file mode 100644 index 750272056..000000000 Binary files a/lib/slf4j-jdk14-1.7.9.jar and /dev/null differ diff --git a/lib/solr-core-5.2.0.jar b/lib/solr-core-5.2.1.jar similarity index 91% rename from lib/solr-core-5.2.0.jar rename to lib/solr-core-5.2.1.jar index 3a01c46b9..cd2844a21 100644 Binary files a/lib/solr-core-5.2.0.jar and b/lib/solr-core-5.2.1.jar differ diff --git a/lib/solr-solrj-5.2.0.jar b/lib/solr-solrj-5.2.1.jar similarity index 91% rename from lib/solr-solrj-5.2.0.jar rename to lib/solr-solrj-5.2.1.jar index e2a4e7855..1e33bd0e8 100644 Binary files a/lib/solr-solrj-5.2.0.jar and b/lib/solr-solrj-5.2.1.jar differ diff --git a/lib/weupnp-0.1.2.jar b/lib/weupnp-0.1.2.jar deleted file mode 100644 index 43daf9ff2..000000000 Binary files a/lib/weupnp-0.1.2.jar and /dev/null differ diff --git a/lib/weupnp-0.1.3.jar b/lib/weupnp-0.1.3.jar new file mode 100644 index 000000000..b728e879d Binary files /dev/null and b/lib/weupnp-0.1.3.jar differ diff --git a/nbproject/project.xml b/nbproject/project.xml index 122c4d389..05be69edb 100644 --- a/nbproject/project.xml +++ b/nbproject/project.xml @@ -83,7 +83,7 @@ source htroot - lib/J7Zip-modified.jar;lib/apache-mime4j-0.6.jar;lib/bcmail-jdk15-1.46.jar;lib/bcprov-jdk15-1.46.jar;lib/chardet.jar;lib/commons-codec-1.10.jar;lib/commons-compress-1.9.jar;lib/commons-fileupload-1.3.1.jar;lib/commons-io-2.4.jar;lib/commons-jxpath-1.3.jar;lib/commons-lang-2.6.jar;lib/commons-logging-1.2.jar;lib/fontbox-1.8.9.jar;lib/geronimo-stax-api_1.0_spec-1.0.1.jar;lib/guava-18.0.jar;lib/htmllexer.jar;lib/httpclient-4.4.1.jar;lib/httpcore-4.4.1.jar;lib/httpmime-4.4.1.jar;lib/icu4j-core.jar;lib/jakarta-oro-2.0.8.jar;lib/jaudiotagger-2.0.4-20111207.115108-15.jar;lib/javax.servlet-api-3.1.0.jar;lib/jcifs-1.3.17.jar;lib/jcl-over-slf4j-1.7.9.jar;lib/jempbox-1.8.9.jar;lib/jetty-client-9.2.11.v20150529.jar;lib/jetty-continuation-9.2.11.v20150529.jar;lib/jetty-deploy-9.2.11.v20150529.jar;lib/jetty-http-9.2.11.v20150529.jar;lib/jetty-io-9.2.11.v20150529.jar;lib/jetty-jmx-9.2.11.v20150529.jar;lib/jetty-proxy-9.2.11.v20150529.jar;lib/jetty-security-9.2.11.v20150529.jar;lib/jetty-server-9.2.11.v20150529.jar;lib/jetty-servlet-9.2.11.v20150529.jar;lib/jetty-servlets-9.2.11.v20150529.jar;lib/jetty-util-9.2.11.v20150529.jar;lib/jetty-webapp-9.2.11.v20150529.jar;lib/jetty-xml-9.2.11.v20150529.jar;lib/jsch-0.1.52.jar;lib/json-simple-1.1.1.jar;lib/jsoup-1.8.2.jar;lib/log4j-over-slf4j-1.7.9.jar;lib/lucene-analyzers-common-5.2.0.jar;lib/lucene-analyzers-phonetic-5.2.0.jar;lib/lucene-backward-codecs-5.2.0.jar;lib/lucene-classification-5.2.0.jar;lib/lucene-codecs-5.2.0.jar;lib/lucene-core-5.2.0.jar;lib/lucene-facet-5.2.0.jar;lib/lucene-grouping-5.2.0.jar;lib/lucene-highlighter-5.2.0.jar;lib/lucene-join-5.2.0.jar;lib/lucene-memory-5.2.0.jar;lib/lucene-misc-5.2.0.jar;lib/lucene-queries-5.2.0.jar;lib/lucene-queryparser-5.2.0.jar;lib/lucene-spatial-5.2.0.jar;lib/lucene-suggest-5.2.0.jar;lib/metadata-extractor-2.8.1.jar;lib/noggit-0.6.jar;lib/org.restlet.jar;lib/pdfbox-1.8.9.jar;lib/poi-3.12-20150511.jar;lib/poi-scratchpad-3.12-20150511.jar;lib/slf4j-api-1.7.9.jar;lib/slf4j-jdk14-1.7.9.jar;lib/solr-core-5.2.0.jar;lib/solr-solrj-5.2.0.jar;lib/spatial4j-0.4.1.jar;lib/stax2-api-3.1.4.jar;lib/webcat-0.1-swf.jar;lib/weupnp-0.1.2.jar;lib/woodstox-core-asl-4.4.1.jar;lib/wstx-asl-3.2.9.jar;lib/xercesImpl.jar;lib/xml-apis.jar;lib/xmpcore-5.1.2.jar;lib/zookeeper-3.4.6.jar + lib/J7Zip-modified.jar;lib/apache-mime4j-0.6.jar;lib/bcmail-jdk15-1.46.jar;lib/bcprov-jdk15-1.46.jar;lib/chardet.jar;lib/commons-codec-1.10.jar;lib/commons-compress-1.9.jar;lib/commons-fileupload-1.3.1.jar;lib/commons-io-2.4.jar;lib/commons-jxpath-1.3.jar;lib/commons-lang-2.6.jar;lib/commons-logging-1.2.jar;lib/fontbox-1.8.10.jar;lib/geronimo-stax-api_1.0_spec-1.0.1.jar;lib/guava-18.0.jar;lib/htmllexer.jar;lib/httpclient-4.5.jar;lib/httpcore-4.4.1.jar;lib/httpmime-4.5.jar;lib/icu4j-55_1.jar;lib/jakarta-oro-2.0.8.jar;lib/jaudiotagger-2.0.4-20111207.115108-15.jar;lib/javax.servlet-api-3.1.0.jar;lib/jcifs-1.3.17.jar;lib/jcl-over-slf4j-1.7.12.jar;lib/jempbox-1.8.10.jar;lib/jetty-client-9.2.11.v20150529.jar;lib/jetty-continuation-9.2.11.v20150529.jar;lib/jetty-deploy-9.2.11.v20150529.jar;lib/jetty-http-9.2.11.v20150529.jar;lib/jetty-io-9.2.11.v20150529.jar;lib/jetty-jmx-9.2.11.v20150529.jar;lib/jetty-proxy-9.2.11.v20150529.jar;lib/jetty-security-9.2.11.v20150529.jar;lib/jetty-server-9.2.11.v20150529.jar;lib/jetty-servlet-9.2.11.v20150529.jar;lib/jetty-servlets-9.2.11.v20150529.jar;lib/jetty-util-9.2.11.v20150529.jar;lib/jetty-webapp-9.2.11.v20150529.jar;lib/jetty-xml-9.2.11.v20150529.jar;lib/jsch-0.1.53.jar;lib/json-simple-1.1.1.jar;lib/jsoup-1.8.2.jar;lib/log4j-over-slf4j-1.7.12.jar;lib/lucene-analyzers-common-5.2.1.jar;lib/lucene-analyzers-phonetic-5.2.1.jar;lib/lucene-backward-codecs-5.2.1.jar;lib/lucene-classification-5.2.1.jar;lib/lucene-codecs-5.2.1.jar;lib/lucene-core-5.2.1.jar;lib/lucene-facet-5.2.1.jar;lib/lucene-grouping-5.2.1.jar;lib/lucene-highlighter-5.2.1.jar;lib/lucene-join-5.2.1.jar;lib/lucene-memory-5.2.1.jar;lib/lucene-misc-5.2.1.jar;lib/lucene-queries-5.2.1.jar;lib/lucene-queryparser-5.2.1.jar;lib/lucene-spatial-5.2.1.jar;lib/lucene-suggest-5.2.1.jar;lib/metadata-extractor-2.8.1.jar;lib/noggit-0.6.jar;lib/org.restlet.jar;lib/pdfbox-1.8.10.jar;lib/poi-3.12-20150511.jar;lib/poi-scratchpad-3.12-20150511.jar;lib/slf4j-api-1.7.12.jar;lib/slf4j-jdk14-1.7.12.jar;lib/solr-core-5.2.1.jar;lib/solr-solrj-5.2.1.jar;lib/spatial4j-0.4.1.jar;lib/stax2-api-3.1.4.jar;lib/webcat-0.1-swf.jar;lib/weupnp-0.1.3.jar;lib/woodstox-core-asl-4.4.1.jar;lib/wstx-asl-3.2.9.jar;lib/xercesImpl.jar;lib/xml-apis.jar;lib/xmpcore-5.1.2.jar;lib/zookeeper-3.4.6.jar lib/yacycore.jar 1.7 diff --git a/pom.xml b/pom.xml index 07e035b38..3fc7bdc1b 100644 --- a/pom.xml +++ b/pom.xml @@ -366,7 +366,7 @@ org.apache.pdfbox fontbox - 1.8.9 + 1.8.10 org.apache.geronimo.specs @@ -386,7 +386,7 @@ org.apache.httpcomponents httpclient - 4.4 + 4.5 org.apache.httpcomponents @@ -396,12 +396,12 @@ org.apache.httpcomponents httpmime - 4.4 + 4.5 com.ibm.icu icu4j - 4.8.1.1 + 55.1 org @@ -421,17 +421,17 @@ org.slf4j jcl-over-slf4j - 1.7.9 + 1.7.12 org.apache.pdfbox jempbox - 1.8.9 + 1.8.10 com.jcraft jsch - 0.1.52 + 0.1.53 com.googlecode.json-simple @@ -443,66 +443,16 @@ jsoup 1.8.2 - - log4j - log4j - 1.2.17 - org.slf4j log4j-over-slf4j - 1.7.9 - - - org.apache.lucene - lucene-analyzers-common - ${solr.version} - - - org.apache.lucene - lucene-analyzers-phonetic - ${solr.version} + 1.7.12 org.apache.lucene lucene-core ${solr.version} - - org.apache.lucene - lucene-grouping - ${solr.version} - - - org.apache.lucene - lucene-highlighter - ${solr.version} - - - org.apache.lucene - lucene-memory - ${solr.version} - - - org.apache.lucene - lucene-misc - ${solr.version} - - - org.apache.lucene - lucene-queries - ${solr.version} - - - org.apache.lucene - lucene-spatial - ${solr.version} - - - org.apache.lucene - lucene-suggest - ${solr.version} - com.drewnoakes metadata-extractor @@ -516,7 +466,7 @@ org.apache.pdfbox pdfbox - 1.8.9 + 1.8.10 org.apache.poi @@ -536,12 +486,12 @@ org.slf4j slf4j-api - 1.7.9 + 1.7.12 org.slf4j slf4j-jdk14 - 1.7.9 + 1.7.12 org.apache.solr @@ -552,16 +502,6 @@ org.apache.solr solr-core ${solr.version} - - - slf4j-log4j12 - org.slf4j - - - javax.servlet - org.eclipse.jetty.orbit - - org.eclipse.jetty @@ -631,7 +571,7 @@ org.bitlet weupnp - 0.1.2 + 0.1.3 org.codehaus.woodstox diff --git a/source/net/yacy/cora/document/id/MultiProtocolURL.java b/source/net/yacy/cora/document/id/MultiProtocolURL.java index c8af66661..6df62a416 100644 --- a/source/net/yacy/cora/document/id/MultiProtocolURL.java +++ b/source/net/yacy/cora/document/id/MultiProtocolURL.java @@ -843,14 +843,12 @@ public class MultiProtocolURL implements Serializable, Comparable q) { - return ""; // TODO: last . after ? (file.ext?param=one.txt) - } else { - return fileName.substring(p + 1, q).toLowerCase(); - } } + // check last dot in query part + if (p > q) { + return ""; // TODO: last . after ? (file.ext?param=one.txt) + } + return fileName.substring(p + 1, q).toLowerCase(); } public String getPath() { diff --git a/source/net/yacy/cora/federate/solr/connector/EmbeddedSolrConnector.java b/source/net/yacy/cora/federate/solr/connector/EmbeddedSolrConnector.java index 875d00f69..f1c1fb63d 100644 --- a/source/net/yacy/cora/federate/solr/connector/EmbeddedSolrConnector.java +++ b/source/net/yacy/cora/federate/solr/connector/EmbeddedSolrConnector.java @@ -205,7 +205,8 @@ public class EmbeddedSolrConnector extends SolrServerConnector implements SolrCo String fq = req.getParams().get(CommonParams.FQ); String sort = req.getParams().get(CommonParams.SORT); String threadname = Thread.currentThread().getName(); - if (q != null) Thread.currentThread().setName("solr query: q = " + q + (fq == null ? "" : ", fq = " + fq) + (sort == null ? "" : ", sort = " + sort)); // for debugging in Threaddump + if (q != null) Thread.currentThread().setName("solr query: q=" + q + (fq == null ? "" : "&fq = " + fq) + (sort == null ? "" : "&sort = " + sort)); // for debugging in Threaddump + ConcurrentLog.info("EmbeddedSolrConnector.query", "QUERY: q=" + q + (fq == null ? "" : "&" + fq.toString()) + (sort == null ? "" : "&sort = " + sort) + " SolrQueryRequest=" + req.toString()); SolrQueryResponse rsp = new SolrQueryResponse(); NamedList responseHeader = new SimpleOrderedMap(); @@ -334,8 +335,8 @@ public class EmbeddedSolrConnector extends SolrServerConnector implements SolrCo if (q != null) { StringBuilder fqa = new StringBuilder(); if (fq != null) for (String f: fq) fqa.append("fq=").append(f).append(' '); - Thread.currentThread().setName("solr query: q = " + q + (fq == null ? "" : ", " + fqa.toString()) + (fl == null ? "" : ", fl=" + fl)); - //System.out.println("solr query: q = " + q + (fq == null ? "" : ", " + fqa.toString()) + (fl == null ? "" : ", fl=" + fl)); + Thread.currentThread().setName("solr query: q=" + q + (fq == null ? "" : "&" + fqa.toString()) + (fl == null ? "" : "&fl=" + fl)); + ConcurrentLog.info("EmbeddedSolrConnector.getResponseByParams", "QUERY: q=" + q + (fq == null ? "" : "&" + fqa.toString()) + (fl == null ? "" : "&fl=" + fl) + " PARAMS: " + params.toString()); } QueryResponse rsp; try { diff --git a/source/net/yacy/crawler/data/Cache.java b/source/net/yacy/crawler/data/Cache.java index 4d6ec6592..ba571760a 100644 --- a/source/net/yacy/crawler/data/Cache.java +++ b/source/net/yacy/crawler/data/Cache.java @@ -86,6 +86,15 @@ public final class Cache { responseHeaderDB = new MapHeap(dbfile, Word.commonHashLength, Base64Order.enhancedCoder, 2048, 100, ' '); } catch (final IOException e) { ConcurrentLog.logException(e); + // try a healing + if (dbfile.exists()) { + dbfile.delete(); + try { + responseHeaderDB = new MapHeap(dbfile, Word.commonHashLength, Base64Order.enhancedCoder, 2048, 100, ' '); + } catch (final IOException ee) { + ConcurrentLog.logException(e); + } + } } // open the cache file try { @@ -94,8 +103,19 @@ public final class Cache { fileDB = new Compressor(fileDBunbuffered, 6 * 1024 * 1024); } catch (final IOException e) { ConcurrentLog.logException(e); + // try a healing + if (cachePath.exists()) { + cachePath.delete(); + try { + fileDBunbuffered = new ArrayStack(new File(cachePath, FILE_DB_NAME), prefix, Base64Order.enhancedCoder, 12, 1024 * 1024 * 2, false, true); + fileDBunbuffered.setMaxSize(maxCacheSize); + fileDB = new Compressor(fileDBunbuffered, 6 * 1024 * 1024); + } catch (final IOException ee) { + ConcurrentLog.logException(e); + } + } } - ConcurrentLog.info("Cache", "initialized cache database responseHeaderDB.size() = " + responseHeaderDB.size() + ", fileDB.size() = " + fileDB.size()); + ConcurrentLog.info("Cache", "initialized cache database responseHeaderDB.size() = " + (responseHeaderDB == null ? "NULL" : responseHeaderDB.size()) + ", fileDB.size() = " + (fileDB == null ? "NULL" : fileDB.size())); // clean up the responseHeaderDB which cannot be cleaned the same way as the cache files. // We do this as a concurrent job only once after start-up silently diff --git a/source/net/yacy/http/AbstractRemoteHandler.java b/source/net/yacy/http/AbstractRemoteHandler.java index 8929a5e36..a9fa89b99 100644 --- a/source/net/yacy/http/AbstractRemoteHandler.java +++ b/source/net/yacy/http/AbstractRemoteHandler.java @@ -64,15 +64,28 @@ abstract public class AbstractRemoteHandler extends ConnectHandler implements Ha new Thread() { @Override public void run() { - final InetAddress localInetAddress = Domains.myPublicLocalIP(); - if (localInetAddress != null) { - if (!localVirtualHostNames.contains(localInetAddress.getHostName())) { - localVirtualHostNames.add(localInetAddress.getHostName()); - localVirtualHostNames.add(localInetAddress.getHostAddress()); // same as getServer().getURI().getHost() + for (InetAddress localInetAddress : Domains.myPublicIPv4()) { + if (localInetAddress != null) { + if (!localVirtualHostNames.contains(localInetAddress.getHostName())) { + localVirtualHostNames.add(localInetAddress.getHostName()); + localVirtualHostNames.add(localInetAddress.getHostAddress()); // same as getServer().getURI().getHost() + } + + if (!localVirtualHostNames.contains(localInetAddress.getCanonicalHostName())) { + localVirtualHostNames.add(localInetAddress.getCanonicalHostName()); + } } + } + for (InetAddress localInetAddress : Domains.myPublicIPv6()) { + if (localInetAddress != null) { + if (!localVirtualHostNames.contains(localInetAddress.getHostName())) { + localVirtualHostNames.add(localInetAddress.getHostName()); + localVirtualHostNames.add(localInetAddress.getHostAddress()); // same as getServer().getURI().getHost() + } - if (!localVirtualHostNames.contains(localInetAddress.getCanonicalHostName())) { - localVirtualHostNames.add(localInetAddress.getCanonicalHostName()); + if (!localVirtualHostNames.contains(localInetAddress.getCanonicalHostName())) { + localVirtualHostNames.add(localInetAddress.getCanonicalHostName()); + } } } if (sb.peers != null) { diff --git a/source/net/yacy/peers/Transmission.java b/source/net/yacy/peers/Transmission.java index 631f3e463..255e0a9b5 100644 --- a/source/net/yacy/peers/Transmission.java +++ b/source/net/yacy/peers/Transmission.java @@ -83,13 +83,11 @@ public class Transmission { * to multiple peers and to ensure that all entries in the indexContainers * have a reference in the urls * - a set of yacy seeds which will shrink as the containers are transmitted to them - * - a counter that gives the number of sucessful and unsuccessful transmissions so far */ private final Seed dhtTarget; private final ReferenceContainerCache containers; private final HandleSet references; private final HandleSet badReferences; - private int hit, miss; /** * generate a new dispatcher target. such a target is defined with a primary target and @@ -103,8 +101,6 @@ public class Transmission { this.containers = new ReferenceContainerCache(Segment.wordReferenceFactory, Segment.wordOrder, Word.commonHashLength); this.references = new RowHandleSet(WordReferenceRow.urlEntryRow.primaryKeyLength, WordReferenceRow.urlEntryRow.objectOrder, 0); this.badReferences = new RowHandleSet(WordReferenceRow.urlEntryRow.primaryKeyLength, WordReferenceRow.urlEntryRow.objectOrder, 0); - this.hit = 0; - this.miss = 0; } /* @@ -211,28 +207,11 @@ public class Transmission { return this.dhtTarget; } - /** - * return the number of successful transmissions - * @return - */ - public int hit() { - return this.hit; - } - - /** - * return the number of unsuccessful transmissions - * @return - */ - public int miss() { - return this.miss; - } - public boolean transmit() { // transferring selected words to remote peer if (this.dhtTarget == Transmission.this.seeds.mySeed() || this.dhtTarget.hash.equals(Transmission.this.seeds.mySeed().hash)) { // target is my own peer. This is easy. Just restore the indexContainer restore(); - this.hit++; Transmission.this.log.info("Transfer of chunk to myself-target"); return true; } @@ -256,13 +235,11 @@ public class Transmission { // if the peer has set a pause time and we are in flush mode (index transfer) // then we pause for a while now Transmission.this.log.info("Transfer finished of chunk to target " + this.dhtTarget.hash + "/" + this.dhtTarget.getName()); - this.hit++; return true; } Transmission.this.log.info( "Index transfer to peer " + this.dhtTarget.getName() + ":" + this.dhtTarget.hash + " failed: " + error); - this.miss++; // write information that peer does not receive index transmissions Transmission.this.log.info("Transfer failed of chunk to target " + this.dhtTarget.hash + "/" + this.dhtTarget.getName() + ": " + error); // get possibly newer target Info diff --git a/source/net/yacy/search/Switchboard.java b/source/net/yacy/search/Switchboard.java index e46a3e9dc..5a8480935 100644 --- a/source/net/yacy/search/Switchboard.java +++ b/source/net/yacy/search/Switchboard.java @@ -315,7 +315,7 @@ public final class Switchboard extends serverSwitch { // set loglevel and log setLog(new ConcurrentLog("SWITCHBOARD")); - AccessTracker.setDumpFile(new File("DATA/LOG/queries.log")); + AccessTracker.setDumpFile(new File(dataPath, "DATA/LOG/queries.log")); // set default peer name Seed.ANON_PREFIX = getConfig("peernameprefix", "_anon"); @@ -359,7 +359,7 @@ public final class Switchboard extends serverSwitch { this.workPath = getDataPath(SwitchboardConstants.WORK_PATH, SwitchboardConstants.WORK_PATH_DEFAULT); this.workPath.mkdirs(); // if default work files exist, copy them (don't overwrite existing!) - File defaultWorkPath = new File("defaults/data/work"); + File defaultWorkPath = new File(appPath, "defaults/data/work"); if (defaultWorkPath.list() != null) { for (String fs : defaultWorkPath.list()) { File wf = new File(this.workPath, fs); @@ -730,7 +730,7 @@ public final class Switchboard extends serverSwitch { // copy opensearch heuristic config (if not exist) final File osdConfig = new File(getDataPath(), "DATA/SETTINGS/heuristicopensearch.conf"); if (!osdConfig.exists()) { - final File osdDefaultConfig = new File("defaults/heuristicopensearch.conf"); + final File osdDefaultConfig = new File(appPath, "defaults/heuristicopensearch.conf"); this.log.info("heuristic.opensearch list Path = " + osdDefaultConfig.getAbsolutePath()); try { Files.copy(osdDefaultConfig, osdConfig); @@ -781,7 +781,7 @@ public final class Switchboard extends serverSwitch { } // init html parser evaluation scheme - File parserPropertiesPath = new File("defaults/"); + File parserPropertiesPath = new File(appPath, "defaults/"); String[] settingsList = parserPropertiesPath.list(); for ( final String l : settingsList ) { if ( l.startsWith("parser.") && l.endsWith(".properties") ) { @@ -865,7 +865,7 @@ public final class Switchboard extends serverSwitch { // load oai tables final Map oaiFriends = OAIListFriendsLoader.loadListFriendsSources( - new File("defaults/oaiListFriendsSource.xml"), + new File(appPath, "defaults/oaiListFriendsSource.xml"), getDataPath()); OAIListFriendsLoader.init(this.loader, oaiFriends, ClientIdentification.yacyInternetCrawlerAgent); this.crawlQueues = new CrawlQueues(this, this.queuesRoot); @@ -2413,7 +2413,7 @@ public final class Switchboard extends serverSwitch { final Properties profile = new Properties(); FileInputStream fileIn = null; try { - fileIn = new FileInputStream(new File("DATA/SETTINGS/profile.txt")); + fileIn = new FileInputStream(new File(this.dataPath, "DATA/SETTINGS/profile.txt")); profile.load(fileIn); } catch (final IOException e ) { } finally { diff --git a/source/net/yacy/search/index/Fulltext.java b/source/net/yacy/search/index/Fulltext.java index 6428a58bf..1b460d0a1 100644 --- a/source/net/yacy/search/index/Fulltext.java +++ b/source/net/yacy/search/index/Fulltext.java @@ -618,13 +618,17 @@ public final class Fulltext { } } + public static enum ExportFormat { + text, html, rss, solr; + } + // export methods - public Export export(final File f, final String filter, final String query, final int format, final boolean dom) { + public Export export(final File f, final String filter, final String query, final ExportFormat format, final boolean dom, final boolean text) { if ((this.exportthread != null) && (this.exportthread.isAlive())) { ConcurrentLog.warn("LURL-EXPORT", "cannot start another export thread, already one running"); return this.exportthread; } - this.exportthread = new Export(f, filter, query, format, dom); + this.exportthread = new Export(f, filter, query, format, dom, text); this.exportthread.start(); return this.exportthread; } @@ -638,10 +642,10 @@ public final class Fulltext { private final Pattern pattern; private int count; private String failure, query; - private final int format; - private final boolean dom; + private final ExportFormat format; + private final boolean dom, text; - private Export(final File f, final String filter, final String query, final int format, boolean dom) { + private Export(final File f, final String filter, final String query, final ExportFormat format, final boolean dom, final boolean text) { // format: 0=text, 1=html, 2=rss/xml this.f = f; this.pattern = filter == null ? null : Pattern.compile(filter); @@ -650,6 +654,7 @@ public final class Fulltext { this.failure = null; this.format = format; this.dom = dom; + this.text = text; //if ((dom) && (format == 2)) dom = false; } @@ -658,13 +663,13 @@ public final class Fulltext { try { final File parentf = this.f.getParentFile(); if (parentf != null) parentf.mkdirs(); - OutputStream os = new FileOutputStream(this.format == 3 ? new File(this.f.getAbsolutePath() + ".gz") : this.f); - if (this.format == 3) os = new GZIPOutputStream(os, 65536){{def.setLevel(Deflater.BEST_COMPRESSION);}}; + OutputStream os = new FileOutputStream(this.format == ExportFormat.solr ? new File(this.f.getAbsolutePath() + ".gz") : this.f); + if (this.format == ExportFormat.solr) os = new GZIPOutputStream(os, 65536){{def.setLevel(Deflater.BEST_COMPRESSION);}}; final PrintWriter pw = new PrintWriter(new BufferedOutputStream(os)); - if (this.format == 1) { + if (this.format == ExportFormat.html) { pw.println(""); } - if (this.format == 2) { + if (this.format == ExportFormat.rss) { pw.println(""); pw.println(""); pw.println(""); @@ -673,7 +678,7 @@ public final class Fulltext { pw.println(""); pw.println("http://yacy.net"); } - if (this.format == 3) { + if (this.format == ExportFormat.solr) { pw.println(""); pw.println(""); pw.println(""); @@ -683,12 +688,25 @@ public final class Fulltext { ReversibleScoreMap stats = scores.get(CollectionSchema.host_s.getSolrFieldName()); for (final String host: stats) { if (this.pattern != null && !this.pattern.matcher(host).matches()) continue; - if (this.format == 0) pw.println(host); - if (this.format == 1) pw.println("" + host + "
"); + if (this.format == ExportFormat.text) pw.println(host); + if (this.format == ExportFormat.html) pw.println("" + host + "
"); this.count++; } } else { - if (this.format < 3) { + if (this.format == ExportFormat.solr || (this.text && this.format == ExportFormat.text)) { + BlockingQueue docs = Fulltext.this.getDefaultConnector().concurrentDocumentsByQuery(this.query + " AND " + CollectionSchema.httpstatus_i.getSolrFieldName() + ":200", null, 0, 100000000, Long.MAX_VALUE, 100, 1, true); + SolrDocument doc; + while ((doc = docs.take()) != AbstractSolrConnector.POISON_DOCUMENT) { + String url = getStringFrom(doc.getFieldValue(CollectionSchema.sku.getSolrFieldName())); + if (this.pattern != null && !this.pattern.matcher(url).matches()) continue; + CRIgnoreWriter sw = new CRIgnoreWriter(); + if (this.text) sw.write((String) doc.getFieldValue(CollectionSchema.text_t.getSolrFieldName())); else EnhancedXMLResponseWriter.writeDoc(sw, doc); + sw.close(); + String d = sw.toString(); + pw.println(d); + this.count++; + } + } else { BlockingQueue docs = Fulltext.this.getDefaultConnector().concurrentDocumentsByQuery(this.query + " AND " + CollectionSchema.httpstatus_i.getSolrFieldName() + ":200", null, 0, 100000000, Long.MAX_VALUE, 100, 1, true, CollectionSchema.id.getSolrFieldName(), CollectionSchema.sku.getSolrFieldName(), CollectionSchema.title.getSolrFieldName(), CollectionSchema.author.getSolrFieldName(), CollectionSchema.description_txt.getSolrFieldName(), CollectionSchema.size_i.getSolrFieldName(), CollectionSchema.last_modified.getSolrFieldName()); @@ -705,13 +723,13 @@ public final class Fulltext { size = (Integer) doc.getFieldValue(CollectionSchema.size_i.getSolrFieldName()); date = (Date) doc.getFieldValue(CollectionSchema.last_modified.getSolrFieldName()); if (this.pattern != null && !this.pattern.matcher(url).matches()) continue; - if (this.format == 0) { + if (this.format == ExportFormat.text) { pw.println(url); } - if (this.format == 1) { + if (this.format == ExportFormat.html) { if (title != null) pw.println("" + CharacterCoding.unicode2xml(title, true) + ""); } - if (this.format == 2) { + if (this.format == ExportFormat.rss) { pw.println(""); if (title != null) pw.println("" + CharacterCoding.unicode2xml(title, true) + ""); pw.println("" + MultiProtocolURL.escape(url) + ""); @@ -724,29 +742,16 @@ public final class Fulltext { } this.count++; } - } else { - BlockingQueue docs = Fulltext.this.getDefaultConnector().concurrentDocumentsByQuery(this.query + " AND " + CollectionSchema.httpstatus_i.getSolrFieldName() + ":200", null, 0, 100000000, Long.MAX_VALUE, 100, 1, true); - SolrDocument doc; - while ((doc = docs.take()) != AbstractSolrConnector.POISON_DOCUMENT) { - String url = getStringFrom(doc.getFieldValue(CollectionSchema.sku.getSolrFieldName())); - if (this.pattern != null && !this.pattern.matcher(url).matches()) continue; - CRIgnoreWriter sw = new CRIgnoreWriter(); - EnhancedXMLResponseWriter.writeDoc(sw, doc); - sw.close(); - String d = sw.toString(); - pw.println(d); - this.count++; - } } } - if (this.format == 1) { + if (this.format == ExportFormat.html) { pw.println(""); } - if (this.format == 2) { + if (this.format == ExportFormat.rss) { pw.println(""); pw.println(""); } - if (this.format == 3) { + if (this.format == ExportFormat.solr) { pw.println(""); pw.println(""); } diff --git a/source/net/yacy/yacy.java b/source/net/yacy/yacy.java index b823366c3..4726fde80 100644 --- a/source/net/yacy/yacy.java +++ b/source/net/yacy/yacy.java @@ -598,8 +598,6 @@ public final class yacy { public static void main(String args[]) { try { - System.setProperty("jsse.enableSNIExtension", "false"); // fix a ssl problem in Java 1.7, see http://teknosrc.com/javax-net-ssl-sslprotocolexception-handshake-alert-unrecognized_name-solved/ - // check assertion status //ClassLoader.getSystemClassLoader().setDefaultAssertionStatus(true); boolean assertionenabled = false; diff --git a/startYACY.bat b/startYACY.bat index 205b6b1f5..eaef4c6da 100644 --- a/startYACY.bat +++ b/startYACY.bat @@ -18,7 +18,9 @@ if exist DATA\SETTINGS\httpProxy.conf GoTo :RENAMEINDEX if exist DATA\SETTINGS\yacy.conf GoTo :GETSTARTOPTS :STARTJAVA -set javacmd=%javacmd% -Djava.awt.headless=true -Dsolr.directoryFactory=solr.MMapDirectoryFactory -Dfile.encoding=UTF-8 +set javacmd=%javacmd% -Djava.awt.headless=true -Dsolr.directoryFactory=solr.MMapDirectoryFactory -Dfile.encoding=UTF-8 -Djsse.enableSNIExtension=false +Rem -Djsse.enableSNIExtension=false fix a ssl problem in Java 1.7, see http://teknosrc.com/javax-net-ssl-sslprotocolexception-handshake-alert-unrecognized_name-solved/ + Rem Starting YaCy Echo Generated classpath:%CLASSPATH% Echo JRE Parameters:%javacmd% diff --git a/startYACY.sh b/startYACY.sh index bacf6c649..7598d7618 100755 --- a/startYACY.sh +++ b/startYACY.sh @@ -6,7 +6,9 @@ PIDFILE="yacy.pid" OS="`uname`" #get javastart args -JAVA_ARGS="-server -Djava.awt.headless=true -Dfile.encoding=UTF-8"; +JAVA_ARGS="-server -Djava.awt.headless=true -Dfile.encoding=UTF-8 -Djsse.enableSNIExtension=false"; +#rem -Djsse.enableSNIExtension=false fix a ssl problem in Java 1.7, see http://teknosrc.com/javax-net-ssl-sslprotocolexception-handshake-alert-unrecognized_name-solved/ + #JAVA_ARGS="-verbose:gc -XX:+PrintGCTimeStamps -XX:+PrintGCDetails $JAVA_ARGS"; #check if OS is Sun Solaris or one of the OpenSolaris distributions and use different version of id if necessary diff --git a/startYACY_debug.bat b/startYACY_debug.bat index 28bafe8be..94986af98 100644 --- a/startYACY_debug.bat +++ b/startYACY_debug.bat @@ -17,7 +17,9 @@ if exist DATA\SETTINGS\httpProxy.conf GoTo :RENAMEINDEX if exist DATA\SETTINGS\yacy.conf GoTo :GETSTARTOPTS :STARTJAVA -set javacmd=%javacmd% -XX:-UseGCOverheadLimit -Djava.awt.headless=true -Dfile.encoding=UTF-8 +set javacmd=%javacmd% -XX:-UseGCOverheadLimit -Djava.awt.headless=true -Dfile.encoding=UTF-8 -Djsse.enableSNIExtension=false +Rem -Djsse.enableSNIExtension=false fix a ssl problem in Java 1.7, see http://teknosrc.com/javax-net-ssl-sslprotocolexception-handshake-alert-unrecognized_name-solved/ + Rem Starting YaCy Echo Generated classpath:%CLASSPATH% Echo JRE Parameters:%javacmd%