- added a tag cloud to search results (using the topics)

- some refactoring of score classes
- added default package for new classes add_ymark and delete_ymark

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7251 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 15 years ago
parent 7aca763ca8
commit ca738ac924

@ -1,17 +1,18 @@
<?xml version="1.0" encoding="UTF-8"?> <?xml version="1.0" encoding="UTF-8"?>
<classpath> <classpath>
<classpathentry excluding="env/|htdocsdefault/|proxymsg/|yacy/|env/|yacy/user/|yacy/user/|yacy/ui/|processing/domaingraph/applet/|processing/domaingraph/|api/|api/bookmarks/posts/|api/bookmarks/|api/util/|api/bookmarks/xbel/|api/bookmarks/tags/" kind="src" path="htroot"/> <classpathentry excluding="env/|htdocsdefault/|proxymsg/|yacy/|env/|yacy/user/|yacy/user/|yacy/ui/|processing/domaingraph/applet/|processing/domaingraph/|api/|api/bookmarks/posts/|api/bookmarks/|api/util/|api/bookmarks/xbel/|api/bookmarks/tags/|api/ymarks/" kind="src" path="htroot"/>
<classpathentry kind="src" path="test"/> <classpathentry kind="src" path="test"/>
<classpathentry excluding="user/|user/|ui/" kind="src" path="htroot/yacy"/> <classpathentry excluding="user/|user/|ui/" kind="src" path="htroot/yacy"/>
<classpathentry kind="src" path="htroot/env"/> <classpathentry kind="src" path="htroot/env"/>
<classpathentry kind="src" path="source"/> <classpathentry kind="src" path="source"/>
<classpathentry kind="src" path="htroot/yacy/ui"/> <classpathentry kind="src" path="htroot/yacy/ui"/>
<classpathentry excluding="bookmarks/posts/|bookmarks/|util/|bookmarks/xbel/|bookmarks/tags/" kind="src" path="htroot/api"/> <classpathentry excluding="bookmarks/posts/|bookmarks/|util/|bookmarks/xbel/|bookmarks/tags/|ymarks/" kind="src" path="htroot/api"/>
<classpathentry kind="src" path="htroot/api/bookmarks/posts"/> <classpathentry kind="src" path="htroot/api/bookmarks/posts"/>
<classpathentry excluding="posts/|xbel/|tags/" kind="src" path="htroot/api/bookmarks"/> <classpathentry excluding="posts/|xbel/|tags/" kind="src" path="htroot/api/bookmarks"/>
<classpathentry kind="src" path="htroot/api/util"/> <classpathentry kind="src" path="htroot/api/util"/>
<classpathentry kind="src" path="htroot/api/bookmarks/xbel"/> <classpathentry kind="src" path="htroot/api/bookmarks/xbel"/>
<classpathentry kind="src" path="htroot/api/bookmarks/tags"/> <classpathentry kind="src" path="htroot/api/bookmarks/tags"/>
<classpathentry kind="src" path="htroot/api/ymarks"/>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/> <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
<classpathentry kind="lib" path="lib/commons-logging-1.1.1.jar"/> <classpathentry kind="lib" path="lib/commons-logging-1.1.1.jar"/>
<classpathentry kind="lib" path="lib/commons-io-1.4.jar"/> <classpathentry kind="lib" path="lib/commons-io-1.4.jar"/>

@ -113,7 +113,7 @@ public class Supporter {
//accumulateSupporter(Supporter, ranking, rowdef, negativeHashes, positiveHashes, yacyNewsPool.PUBLISHED_DB); //accumulateSupporter(Supporter, ranking, rowdef, negativeHashes, positiveHashes, yacyNewsPool.PUBLISHED_DB);
// read out surftipp array and create property entries // read out surftipp array and create property entries
final Iterator<String> k = ranking.scores(false); final Iterator<String> k = ranking.keys(false);
int i = 0; int i = 0;
Row.Entry row; Row.Entry row;
String url, urlhash, refid, title, description; String url, urlhash, refid, title, description;
@ -150,7 +150,7 @@ public class Supporter {
prop.put("supporter_results_" + i + "_url", url); prop.put("supporter_results_" + i + "_url", url);
prop.put("supporter_results_" + i + "_urlname", nxTools.shortenURLString(url, 60)); prop.put("supporter_results_" + i + "_urlname", nxTools.shortenURLString(url, 60));
prop.put("supporter_results_" + i + "_urlhash", urlhash); prop.put("supporter_results_" + i + "_urlhash", urlhash);
prop.putHTML("supporter_results_" + i + "_title", (showScore) ? ("(" + ranking.getScore(urlhash) + ") " + title) : title); prop.putHTML("supporter_results_" + i + "_title", (showScore) ? ("(" + ranking.get(urlhash) + ") " + title) : title);
prop.putHTML("supporter_results_" + i + "_description", description); prop.putHTML("supporter_results_" + i + "_description", description);
i++; i++;
@ -265,9 +265,9 @@ public class Supporter {
} }
// consider double-entries // consider double-entries
if (Supporter.containsKey(urlhash)) { if (Supporter.containsKey(urlhash)) {
ranking.addScore(urlhash, score); ranking.inc(urlhash, score);
} else { } else {
ranking.setScore(urlhash, score); ranking.set(urlhash, score);
Supporter.put(urlhash, entry); Supporter.put(urlhash, entry);
} }
} }

@ -121,7 +121,7 @@ public class Surftips {
//accumulateSurftips(surftips, ranking, rowdef, negativeHashes, positiveHashes, yacyNewsPool.PUBLISHED_DB); //accumulateSurftips(surftips, ranking, rowdef, negativeHashes, positiveHashes, yacyNewsPool.PUBLISHED_DB);
// read out surftipp array and create property entries // read out surftipp array and create property entries
final Iterator<String> k = ranking.scores(false); final Iterator<String> k = ranking.keys(false);
int i = 0; int i = 0;
Row.Entry row; Row.Entry row;
String url, urlhash, refid, title, description; String url, urlhash, refid, title, description;
@ -159,7 +159,7 @@ public class Surftips {
prop.putXML("surftips_results_" + i + "_url", url); prop.putXML("surftips_results_" + i + "_url", url);
prop.putXML("surftips_results_" + i + "_urlname", nxTools.shortenURLString(url, 60)); prop.putXML("surftips_results_" + i + "_urlname", nxTools.shortenURLString(url, 60));
prop.putXML("surftips_results_" + i + "_urlhash", urlhash); prop.putXML("surftips_results_" + i + "_urlhash", urlhash);
prop.putXML("surftips_results_" + i + "_title", (showScore) ? ("(" + ranking.getScore(urlhash) + ") " + title) : title); prop.putXML("surftips_results_" + i + "_title", (showScore) ? ("(" + ranking.get(urlhash) + ") " + title) : title);
prop.putHTML("surftips_results_" + i + "_description", description); prop.putHTML("surftips_results_" + i + "_description", description);
i++; i++;
@ -327,9 +327,9 @@ public class Surftips {
} }
// consider double-entries // consider double-entries
if (surftips.containsKey(urlhash)) { if (surftips.containsKey(urlhash)) {
ranking.addScore(urlhash, score); ranking.inc(urlhash, score);
} else { } else {
ranking.setScore(urlhash, score); ranking.set(urlhash, score);
surftips.put(urlhash, entry); surftips.put(urlhash, entry);
} }
} }

@ -0,0 +1,8 @@
/*
* jQuery TagCloud 0.5.0
* Copyright (c) 2008 Ron Valstar
* Dual licensed under the MIT and GPL licenses:
* http://www.opensource.org/licenses/mit-license.php
* http://www.gnu.org/licenses/gpl.html
*/
(function(F){var C;var A={};var G={};var L=2.399963;F.tagcloud={id:"TagCloud",version:"0.5.0",defaults:{height:null,type:"cloud",sizemax:20,sizemin:10,colormax:"00F",colormin:"B4D2FF",seed:null,power:0.5}};F.fn.extend({tagcloud:function(M){C=F.extend({},F.tagcloud.defaults,M);if(C.seed===null){C.seed=Math.ceil(Math.random()*45309714203)}switch(C.type){case"sphere":case"cloud":A={position:"relative"};G={position:"absolute",display:"block"};break;case"list":A={height:"auto"};G={position:"static",display:"inline"};break}B.setSeed(C.seed+123456);return this.each(function(f,a){var q=F(a);var R=q.find(">li");var S=R.length;var Z=q.width();var l=C.height===null?(0.004*Z*S):C.height;q.css({width:Z,height:l,listStyle:"none",margin:0,padding:0});q.css(A);var e=-2147483647;var r=2147483648;var T=-1;for(var d=0;d<S;d++){var p=F(R[d]);var n=p.attr("value")==-1?T++:p.attr("value");if(n>e){e=n}if(n<r){r=n}T=n}var b=e-r;var X=new Array();for(var d=0;d<S;d++){X[d]=d}for(var d,V,c=X.length;c;d=parseInt(B.rand(0,1000)/1000*c),V=X[--c],X[c]=X[d],X[d]=V){}T=-1;for(var d=0;d<S;d++){var p=F(R[d]);var n=p.attr("value")==-1?T++:p.attr("value");T=n;var g=((S-d-1)/(S-1));var g=(n-r)/b;var m=C.sizemin+g*(C.sizemax-C.sizemin);var N=D(C.colormin,C.colormax,g);p.css({fontSize:m,position:"absolute",color:"#"+N,margin:0,padding:0}).children().css({color:"#"+N});var Y=p.width();var h=p.height();var Q={};if(C.type!="list"){if(C.type=="cloud"){var s=B.rand(0,Z-Y);var W=X[d]*(l/S)-h/2}else{var P=Math.pow(d/S,C.power);var U=(d+Math.PI/2)*L;var s=Z/2-Y/2+0.5*Z*P*Math.sin(U);var W=l/2-h/2+0.5*l*P*Math.cos(U)}Q.left=s;Q.top=W}for(var O in G){Q[O]=G[O]}p.css(Q)}})}});var B=new function(){this.seed=23145678901;this.A=48271;this.M=2147483647;this.Q=this.M/this.A;this.R=this.M%this.A;this.oneOverM=1/this.M};B.setSeed=function(M){this.seed=M};B.next=function(){var M=this.seed/this.Q;var N=this.seed%this.Q;var O=this.A*N-this.R*M;this.seed=O+(O>0?0:this.M);return(this.seed*this.oneOverM)};B.rand=function(N,M){return Math.floor((M-N+1)*this.next()+N)};function I(M){return M.toString(16)}function K(M){return parseInt(M,16)}function H(Q){var M=Q.length==3;var O=[];for(var N=0;N<3;N++){var P=Q.substring(N*(M?1:2),(N+1)*(M?1:2));O.push(K(M?P+P:P))}return O}function J(M){var O="";for(var N=0;N<3;N++){var P=I(M[N]);if(P.length==1){P="0"+P}O+=P}return O}function D(R,S,P){var Q=H(R);var M=H(S);var O=[];for(var N=0;N<3;N++){O.push(Q[N]+Math.floor(P*(M[N]-Q[N])))}return J(O)}function E(M){if(window.console&&window.console.log){if(typeof (M)=="string"){window.console.log(M)}else{for(var N in M){window.console.log(N+": "+M[N])}}}}F.fn.TagCloud=F.fn.Tagcloud=F.fn.tagcloud})(jQuery);

@ -0,0 +1,8 @@
/*
* jQuery TinySort 1.0.1
* Copyright (c) 2008 Ron Valstar
* Dual licensed under the MIT and GPL licenses:
* http://www.opensource.org/licenses/mit-license.php
* http://www.gnu.org/licenses/gpl.html
*/
(function(C){C.tinysort={id:"TinySort",version:"1.0.1",defaults:{order:"asc",attr:"",place:"start",returns:false}};C.fn.extend({tinysort:function(I,J){if(I&&typeof (I)!="string"){J=I;I=null}var F=C.extend({},C.tinysort.defaults,J);var P={};this.each(function(T){var V=(!I||I=="")?C(this):C(this).find(I);var U=F.order=="rand"?""+Math.random():(F.attr==""?V.text():V.attr(F.attr));var S=C(this).parent();if(!P[S]){P[S]={s:[],n:[]}}if(V.length>0){P[S].s.push({s:U,e:C(this),n:T})}else{P[S].n.push({e:C(this),n:T})}});for(var H in P){var E=P[H];E.s.sort(function K(U,T){var S=U.s.toLowerCase?U.s.toLowerCase():U.s;var V=T.s.toLowerCase?T.s.toLowerCase():T.s;if(D(U.s)&&D(T.s)){S=parseFloat(U.s);V=parseFloat(T.s)}return(F.order=="asc"?1:-1)*(S<V?-1:(S>V?1:0))})}var M=[];for(var H in P){var E=P[H];var N=[];var G=C(this).length;switch(F.place){case"first":C.each(E.s,function(S,T){G=Math.min(G,T.n)});break;case"org":C.each(E.s,function(S,T){N.push(T.n)});break;case"end":G=E.n.length;break;default:G=0}var Q=[0,0];for(var L=0;L<C(this).length;L++){var O=L>=G&&L<G+E.s.length;if(A(N,L)){O=true}var R=(O?E.s:E.n)[Q[O?0:1]].e;R.parent().append(R);if(O||!F.returns){M.push(R.get(0))}Q[O?0:1]++}}return this.setArray(M)}});function D(E){return(parseFloat(E)+"")==E}function A(F,G){var E=false;C.each(F,function(I,H){if(!E){E=H==G}});return E}function B(E){if(window.console&&window.console.log){if(typeof (E)=="string"){window.console.log(E)}else{for(var F in E){window.console.log(F+": "+E[F])}}}}C.fn.TinySort=C.fn.Tinysort=C.fn.tsort=C.fn.tinysort})(jQuery);

@ -13,6 +13,8 @@
<script type="text/javascript">hs.outlineType = 'rounded-white';</script> <script type="text/javascript">hs.outlineType = 'rounded-white';</script>
<script src="/yacy/ui/js/jquery-1.3.2.min.js" type="text/javascript"></script> <script src="/yacy/ui/js/jquery-1.3.2.min.js" type="text/javascript"></script>
<script src="/yacy/ui/js/jquery-ui-1.7.2.min.js" type="text/javascript"></script> <script src="/yacy/ui/js/jquery-ui-1.7.2.min.js" type="text/javascript"></script>
<script src="/yacy/ui/js/jquery.tagcloud.min.js" type="text/javascript"></script>
<script src="/yacy/ui/js/jquery.tinysort.min.js" type="text/javascript"></script>
<link media="screen" type="text/css" href="/yacy/ui/css/themes/start/ui.base.css" rel="stylesheet" /> <link media="screen" type="text/css" href="/yacy/ui/css/themes/start/ui.base.css" rel="stylesheet" />
<link media="screen" type="text/css" href="/yacy/ui/css/themes/start/ui.theme.css" rel="stylesheet" /> <link media="screen" type="text/css" href="/yacy/ui/css/themes/start/ui.theme.css" rel="stylesheet" />
<link media="screen" type="text/css" href="/yacy/ui/css/autocomplete.css" rel="stylesheet" /> <link media="screen" type="text/css" href="/yacy/ui/css/autocomplete.css" rel="stylesheet" />
@ -94,8 +96,7 @@ $(function() {
$("#sidebar2").accordion({}); $("#sidebar2").accordion({});
$("#sidebar3").accordion({}); $("#sidebar3").accordion({});
$("#sidebar3").accordion('activate', false); $("#sidebar3").accordion('activate', false);
$("#sidebar4").accordion({}); $("#sidebar4").tagcloud({seed:0,sizemin:10,sizemax:20,height:60}).find("li").tsort();
$("#sidebar4").accordion('activate', false);
$("#sidebarAbout").accordion({}); $("#sidebarAbout").accordion({});
$("#search").focus(); $("#search").focus();
}); });

@ -8,6 +8,14 @@
</div> </div>
#(/cat-location)# #(/cat-location)#
#(nav-topics)#::
<div style="float: right; margin-top:5px; width: 220px; height: 60px">
<div><ul id="sidebar4" style="padding-left: 0px;">#{element}#
<li value="#[count]#">#[url]#</li>
#{/element}#</ul></div>
</div>
#(/nav-topics)#
#(nav-domains)#:: #(nav-domains)#::
<div id="sidebar1" style="float: right; margin-top:5px; width: 220px;"> <div id="sidebar1" style="float: right; margin-top:5px; width: 220px;">
<h3 style="padding-left:25px;">Domain Navigator</h3> <h3 style="padding-left:25px;">Domain Navigator</h3>
@ -35,15 +43,6 @@
</div> </div>
#(/nav-authors)# #(/nav-authors)#
#(nav-topics)#::
<div id="sidebar4" style="float: right; margin-top:5px; width: 220px;">
<h3 style="padding-left:25px;">Topic Navigator</h3>
<div><ul style="padding-left: 0px;">#{element}#
<li>#[url]#</li>
#{/element}#</ul></div>
</div>
#(/nav-topics)#
#(nav-about)#:: #(nav-about)#::
<div id="sidebarAbout" style="float: right; margin-top:5px; width: 220px;"> <div id="sidebarAbout" style="float: right; margin-top:5px; width: 220px;">
<h3 style="padding-left:25px;">#[headline]#</h3> <h3 style="padding-left:25px;">#[headline]#</h3>

@ -130,7 +130,7 @@ public class yacysearchtrailer {
} }
// topics navigator // topics navigator
List<Navigator.Item> topicNavigator = theSearch.getTopicNavigator(10); List<Navigator.Item> topicNavigator = theSearch.getTopicNavigator(30);
if (topicNavigator == null || topicNavigator.isEmpty()) { if (topicNavigator == null || topicNavigator.isEmpty()) {
topicNavigator = new ArrayList<Navigator.Item>(); topicNavigator = new ArrayList<Navigator.Item>();
prop.put("nav-topics", "0"); prop.put("nav-topics", "0");
@ -144,7 +144,9 @@ public class yacysearchtrailer {
if (/*(theQuery == null) ||*/ (theQuery.queryString == null)) break; if (/*(theQuery == null) ||*/ (theQuery.queryString == null)) break;
if (e != null && e.name != null) { if (e != null && e.name != null) {
prop.putHTML("nav-topics_element_" + i + "_name", e.name); prop.putHTML("nav-topics_element_" + i + "_name", e.name);
prop.put("nav-topics_element_" + i + "_url", "<a href=\"" + QueryParams.navurl("html", 0, display, theQuery, theQuery.queryStringForUrl() + "+" + e.name, theQuery.urlMask.toString(), theQuery.navigators) + "\">" + e.name + " (" + e.count + ")</a>"); prop.put("nav-topics_element_" + i + "_url",
"<a href=\"" + QueryParams.navurl("html", 0, display, theQuery, theQuery.queryStringForUrl() + "+" + e.name, theQuery.urlMask.toString(), theQuery.navigators) + "\">" + e.name + "</a>");
//+"<a href=\"" + QueryParams.navurl("html", 0, display, theQuery, theQuery.queryStringForUrl() + "+-" + e.name, theQuery.urlMask.toString(), theQuery.navigators) + "\">-</a>")*/;
prop.putJSON("nav-topics_element_" + i + "_url-json", QueryParams.navurl("json", 0, display, theQuery, theQuery.queryStringForUrl() + "+" + e.name, theQuery.urlMask.toString(), theQuery.navigators)); prop.putJSON("nav-topics_element_" + i + "_url-json", QueryParams.navurl("json", 0, display, theQuery, theQuery.queryStringForUrl() + "+" + e.name, theQuery.urlMask.toString(), theQuery.navigators));
prop.put("nav-topics_element_" + i + "_count", e.count); prop.put("nav-topics_element_" + i + "_count", e.count);
prop.put("nav-topics_element_" + i + "_modifier", e.name); prop.put("nav-topics_element_" + i + "_modifier", e.name);

@ -85,7 +85,7 @@ public final class ResultURLs {
try { try {
final DynamicScore<String> domains = getDomains(stackType); final DynamicScore<String> domains = getDomains(stackType);
if (domains != null) { if (domains != null) {
domains.incScore(e.metadata().url().getHost()); domains.inc(e.metadata().url().getHost());
} }
} catch (final Exception ex) { } catch (final Exception ex) {
System.out.println("INTERNAL ERROR in newEntry/3: " + ex.toString()); System.out.println("INTERNAL ERROR in newEntry/3: " + ex.toString());
@ -117,7 +117,7 @@ public final class ResultURLs {
*/ */
public Iterator<String> domains(final EventOrigin stack) { public Iterator<String> domains(final EventOrigin stack) {
assert getDomains(stack) != null : "getDomains(" + stack + ") = null"; assert getDomains(stack) != null : "getDomains(" + stack + ") = null";
return getDomains(stack).scores(false); return getDomains(stack).keys(false);
} }
public int deleteDomain(final EventOrigin stack, String host, String hosthash) { public int deleteDomain(final EventOrigin stack, String host, String hosthash) {
@ -132,7 +132,7 @@ public final class ResultURLs {
if (urlhash == null || urlhash.substring(6).equals(hosthash)) i.remove(); if (urlhash == null || urlhash.substring(6).equals(hosthash)) i.remove();
} }
assert getDomains(stack) != null : "getDomains(" + stack + ") = null"; assert getDomains(stack) != null : "getDomains(" + stack + ") = null";
return getDomains(stack).deleteScore(host); return getDomains(stack).delete(host);
} }
/** /**
@ -144,7 +144,7 @@ public final class ResultURLs {
public int domainCount(final EventOrigin stack, String domain) { public int domainCount(final EventOrigin stack, String domain) {
assert domain != null : "domain = null"; assert domain != null : "domain = null";
assert getDomains(stack) != null : "getDomains(" + stack + ") = null"; assert getDomains(stack) != null : "getDomains(" + stack + ") = null";
return getDomains(stack).getScore(domain); return getDomains(stack).get(domain);
} }
/** /**

@ -132,13 +132,13 @@ public class DidYouMean {
for (final String s: preSorted) { for (final String s: preSorted) {
if (System.currentTimeMillis() > timelimit) break; if (System.currentTimeMillis() > timelimit) break;
if (scored.size() >= 2 * preSortSelection) break; if (scored.size() >= 2 * preSortSelection) break;
scored.addScore(s, index.count(Word.word2hash(s))); scored.inc(s, index.count(Word.word2hash(s)));
} }
SortedSet<String> countSorted = Collections.synchronizedSortedSet(new TreeSet<String>(new indexSizeComparator())); SortedSet<String> countSorted = Collections.synchronizedSortedSet(new TreeSet<String>(new indexSizeComparator()));
int wc = index.count(Word.word2hash(this.word)); // all counts must be greater than this int wc = index.count(Word.word2hash(this.word)); // all counts must be greater than this
while (scored.size() > 0 && countSorted.size() < preSortSelection) { while (scored.size() > 0 && countSorted.size() < preSortSelection) {
String s = scored.getMaxObject(); String s = scored.getMaxKey();
int score = scored.deleteScore(s); int score = scored.delete(s);
if (s.length() >= MinimumOutputWordLength && score > wc) countSorted.add(s); if (s.length() >= MinimumOutputWordLength && score > wc) countSorted.add(s);
if (System.currentTimeMillis() > timelimit) break; if (System.currentTimeMillis() > timelimit) break;
} }

@ -78,7 +78,7 @@ public class DidYouMeanLibrary {
if (word == null) return; if (word == null) return;
word = word.trim().toLowerCase(); word = word.trim().toLowerCase();
if (word.length() < commonWordsMinLength) return; if (word.length() < commonWordsMinLength) return;
commonWords.incScore(word); commonWords.inc(word);
if (commonWords.size() >= commonWordsMaxSize) { if (commonWords.size() >= commonWordsMaxSize) {
commonWords.shrinkToMaxSize(commonWordsMaxSize / 2); commonWords.shrinkToMaxSize(commonWordsMaxSize / 2);
} }

@ -610,11 +610,11 @@ public final class MetadataRepository implements Iterable<byte[]> {
// order elements by size // order elements by size
DynamicScore<String> s = new ScoreCluster<String>(); DynamicScore<String> s = new ScoreCluster<String>();
for (Map.Entry<String, hashStat> e: map.entrySet()) { for (Map.Entry<String, hashStat> e: map.entrySet()) {
s.addScore(e.getValue().urlhash, e.getValue().count); s.inc(e.getValue().urlhash, e.getValue().count);
} }
// fetch urls from the database to determine the host in clear text // fetch urls from the database to determine the host in clear text
Iterator<String> j = s.scores(false); // iterate urlhash-examples in reverse order (biggest first) Iterator<String> j = s.keys(false); // iterate urlhash-examples in reverse order (biggest first)
URIMetadataRow urlref; URIMetadataRow urlref;
String urlhash; String urlhash;
count += 10; // make some more to prevent that we have to do this again after deletions too soon. count += 10; // make some more to prevent that we have to do this again after deletions too soon.
@ -630,7 +630,7 @@ public final class MetadataRepository implements Iterable<byte[]> {
if (statsDump == null) return new ArrayList<hostStat>().iterator(); // some other operation has destroyed the object if (statsDump == null) return new ArrayList<hostStat>().iterator(); // some other operation has destroyed the object
comps = urlref.metadata(); comps = urlref.metadata();
url = comps.url(); url = comps.url();
statsDump.add(new hostStat(url.getHost(), url.getPort(), urlhash.substring(6), s.getScore(urlhash))); statsDump.add(new hostStat(url.getHost(), url.getPort(), urlhash.substring(6), s.get(urlhash)));
count--; count--;
if (count == 0) break; if (count == 0) break;
} }

@ -620,7 +620,7 @@ public final class RankingProcess extends Thread {
// create a list of words that had been computed by statistics over all // create a list of words that had been computed by statistics over all
// words that appeared in the url or the description of all urls // words that appeared in the url or the description of all urls
if (!this.query.navigators.equals("all") && this.query.navigators.indexOf("topics") < 0) return new ArrayList<Navigator.Item>(0); if (!this.query.navigators.equals("all") && this.query.navigators.indexOf("topics") < 0) return new ArrayList<Navigator.Item>(0);
List<Navigator.Item> result = this.ref.entries(10); List<Navigator.Item> result = this.ref.entries(count);
if (result.size() < 2) result.clear(); // navigators with one entry are not useful if (result.size() < 2) result.clear(); // navigators with one entry are not useful
return result; return result;
} }

@ -166,7 +166,7 @@ public class ReferenceOrder {
final Iterator<Map.Entry<String, Integer>> di = doms0.entrySet().iterator(); final Iterator<Map.Entry<String, Integer>> di = doms0.entrySet().iterator();
while (di.hasNext()) { while (di.hasNext()) {
entry = di.next(); entry = di.next();
doms.addScore(entry.getKey(), (entry.getValue()).intValue()); doms.inc(entry.getKey(), (entry.getValue()).intValue());
} }
if (!doms.isEmpty()) maxdomcount = doms.getMaxScore(); if (!doms.isEmpty()) maxdomcount = doms.getMaxScore();
} catch (InterruptedException e) { } catch (InterruptedException e) {
@ -184,7 +184,7 @@ public class ReferenceOrder {
} }
public int authority(final byte[] urlHash) { public int authority(final byte[] urlHash) {
return (doms.getScore(new String(urlHash, 6, 6)) << 8) / (1 + this.maxdomcount); return (doms.get(new String(urlHash, 6, 6)) << 8) / (1 + this.maxdomcount);
} }
/** /**

@ -72,7 +72,7 @@ public class PeerSelection {
if (seed == null || seed.hash == null) continue; if (seed == null || seed.hash == null) continue;
if (!seed.getFlagAcceptRemoteIndex()) continue; // probably a robinson peer if (!seed.getFlagAcceptRemoteIndex()) continue; // probably a robinson peer
if (Log.isFine("PLASMA")) Log.logFine("PLASMA", "selectPeers/DHTorder: " + seed.hash + ":" + seed.getName() + "/ score " + c); if (Log.isFine("PLASMA")) Log.logFine("PLASMA", "selectPeers/DHTorder: " + seed.hash + ":" + seed.getName() + "/ score " + c);
ranking.addScore(seed.hash, 2 * c); ranking.inc(seed.hash, 2 * c);
regularSeeds.put(seed.hash, seed); regularSeeds.put(seed.hash, seed);
c--; c--;
} }
@ -351,13 +351,13 @@ public class PeerSelection {
ys = s.next(); ys = s.next();
if ((ys != null) && (ys.get(yacySeed.LASTSEEN, "").length() > 10)) try { if ((ys != null) && (ys.get(yacySeed.LASTSEEN, "").length() > 10)) try {
absage = Math.abs(System.currentTimeMillis() + DateFormatter.dayMillis - ys.getLastSeenUTC()); absage = Math.abs(System.currentTimeMillis() + DateFormatter.dayMillis - ys.getLastSeenUTC());
seedScore.addScore(ys.hash, (int) absage); // the higher absage, the older is the peer seedScore.inc(ys.hash, (int) absage); // the higher absage, the older is the peer
} catch (final Exception e) {} } catch (final Exception e) {}
} }
// result is now in the score object; create a result vector // result is now in the score object; create a result vector
final Map<String, yacySeed> result = new HashMap<String, yacySeed>(); final Map<String, yacySeed> result = new HashMap<String, yacySeed>();
final Iterator<String> it = seedScore.scores(up); final Iterator<String> it = seedScore.keys(up);
int c = 0; int c = 0;
while ((c < count) && (it.hasNext())) { while ((c < count) && (it.hasNext())) {
c++; c++;

@ -206,7 +206,7 @@ public class yacySearch extends Thread {
if (!seed.getFlagAcceptRemoteIndex()) continue; // probably a robinson peer if (!seed.getFlagAcceptRemoteIndex()) continue; // probably a robinson peer
score = (int) Math.round(Math.random() * ((c / 3) + 3)); score = (int) Math.round(Math.random() * ((c / 3) + 3));
if (Log.isFine("PLASMA")) Log.logFine("PLASMA", "selectPeers/RWIcount: " + seed.hash + ":" + seed.getName() + ", RWIcount=" + seed.getWordCount() + ", score " + score); if (Log.isFine("PLASMA")) Log.logFine("PLASMA", "selectPeers/RWIcount: " + seed.hash + ":" + seed.getName() + ", RWIcount=" + seed.getWordCount() + ", score " + score);
ranking.addScore(seed.hash, score); ranking.inc(seed.hash, score);
regularSeeds.put(seed.hash, seed); regularSeeds.put(seed.hash, seed);
c--; c--;
} }
@ -221,12 +221,12 @@ public class yacySearch extends Thread {
String specialized = seed.getPeerTags().toString(); String specialized = seed.getPeerTags().toString();
if (!specialized.equals("[*]")) Log.logInfo("PLASMA", "selectPeers/PeerTags: " + seed.hash + ":" + seed.getName() + ", is specialized peer for " + specialized); if (!specialized.equals("[*]")) Log.logInfo("PLASMA", "selectPeers/PeerTags: " + seed.hash + ":" + seed.getName() + ", is specialized peer for " + specialized);
regularSeeds.remove(seed.hash); regularSeeds.remove(seed.hash);
ranking.deleteScore(seed.hash); ranking.delete(seed.hash);
matchingSeeds.put(seed.hash, seed); matchingSeeds.put(seed.hash, seed);
} else if (seed.getFlagAcceptRemoteIndex() && seed.getAge() < 1) { // the 'workshop feature' } else if (seed.getFlagAcceptRemoteIndex() && seed.getAge() < 1) { // the 'workshop feature'
Log.logInfo("PLASMA", "selectPeers/Age: " + seed.hash + ":" + seed.getName() + ", is newbie, age = " + seed.getAge()); Log.logInfo("PLASMA", "selectPeers/Age: " + seed.hash + ":" + seed.getName() + ", is newbie, age = " + seed.getAge());
regularSeeds.remove(seed.hash); regularSeeds.remove(seed.hash);
ranking.deleteScore(seed.hash); ranking.delete(seed.hash);
matchingSeeds.put(seed.hash, seed); matchingSeeds.put(seed.hash, seed);
} }
} }
@ -235,7 +235,7 @@ public class yacySearch extends Thread {
seedcount = Math.min(ranking.size(), seedcount); seedcount = Math.min(ranking.size(), seedcount);
final yacySeed[] result = new yacySeed[seedcount + matchingSeeds.size()]; final yacySeed[] result = new yacySeed[seedcount + matchingSeeds.size()];
c = 0; c = 0;
Iterator<String> iters = ranking.scores(false); // higher are better Iterator<String> iters = ranking.keys(false); // higher are better
while (iters.hasNext() && c < seedcount) { while (iters.hasNext() && c < seedcount) {
seed = regularSeeds.get(iters.next()); seed = regularSeeds.get(iters.next());
seed.selectscore = c; seed.selectscore = c;

@ -22,10 +22,10 @@ package net.yacy.cora.storage;
public interface DynamicScore<E> extends StaticScore<E> { public interface DynamicScore<E> extends StaticScore<E> {
public void incScore(final E obj); public void inc(final E obj);
public void inc(final E obj, final int incrementScore);
public void decScore(final E obj); public void dec(final E obj);
public void dec(final E obj, final int incrementScore);
public void addScore(final E obj, final int incrementScore);
} }

@ -81,7 +81,8 @@ public class IntScore implements Comparable<IntScore>, Comparator<IntScore> {
} }
public int hashCode() { public int hashCode() {
return (int) (this.value ^ (this.value >>> 32)); return this.value;
// return (int) (this.value ^ (this.value >>> 32)); // hash code for long values
} }
public int compareTo(IntScore o) { public int compareTo(IntScore o) {

@ -186,15 +186,15 @@ public final class ScoreCluster<E> implements DynamicScore<E> {
return map.isEmpty(); return map.isEmpty();
} }
public synchronized void incScore(final E obj) { public synchronized void inc(final E obj) {
addScore(obj, 1); inc(obj, 1);
} }
public synchronized void decScore(final E obj) { public synchronized void dec(final E obj) {
addScore(obj, -1); inc(obj, -1);
} }
public void setScore(final E obj, final int newScore) { public void set(final E obj, final int newScore) {
if (obj == null) return; if (obj == null) return;
synchronized (this) { synchronized (this) {
Long usk = map.remove(obj); // get unique score key, old entry is not needed any more Long usk = map.remove(obj); // get unique score key, old entry is not needed any more
@ -228,7 +228,7 @@ public final class ScoreCluster<E> implements DynamicScore<E> {
gcount += newScore; gcount += newScore;
} }
public void addScore(final E obj, final int incrementScore) { public void inc(final E obj, final int incrementScore) {
if (obj == null) return; if (obj == null) return;
synchronized (this) { synchronized (this) {
Long usk = map.remove(obj); // get unique score key, old entry is not needed any more Long usk = map.remove(obj); // get unique score key, old entry is not needed any more
@ -263,7 +263,11 @@ public final class ScoreCluster<E> implements DynamicScore<E> {
gcount += incrementScore; gcount += incrementScore;
} }
public int deleteScore(final E obj) { public void dec(final E obj, final int incrementScore) {
inc(obj, -incrementScore);
}
public int delete(final E obj) {
// deletes entry and returns previous score // deletes entry and returns previous score
if (obj == null) return 0; if (obj == null) return 0;
final Long usk; final Long usk;
@ -284,11 +288,11 @@ public final class ScoreCluster<E> implements DynamicScore<E> {
return oldScore; return oldScore;
} }
public synchronized boolean existsScore(final E obj) { public synchronized boolean containsKey(final E obj) {
return map.containsKey(obj); return map.containsKey(obj);
} }
public int getScore(final E obj) { public int get(final E obj) {
if (obj == null) return 0; if (obj == null) return 0;
final Long cs; final Long cs;
synchronized (this) { synchronized (this) {
@ -308,12 +312,12 @@ public final class ScoreCluster<E> implements DynamicScore<E> {
return (int) ((pam.firstKey().longValue() & 0xFFFFFFFF00000000L) >> 32); return (int) ((pam.firstKey().longValue() & 0xFFFFFFFF00000000L) >> 32);
} }
public synchronized E getMaxObject() { public synchronized E getMaxKey() {
if (map.isEmpty()) return null; if (map.isEmpty()) return null;
return pam.get(pam.lastKey()); return pam.get(pam.lastKey());
} }
public synchronized E getMinObject() { public synchronized E getMinKey() {
if (map.isEmpty()) return null; if (map.isEmpty()) return null;
return pam.get(pam.firstKey()); return pam.get(pam.firstKey());
} }
@ -322,7 +326,7 @@ public final class ScoreCluster<E> implements DynamicScore<E> {
return map + " / " + pam; return map + " / " + pam;
} }
public synchronized Iterator<E> scores(final boolean up) { public synchronized Iterator<E> keys(final boolean up) {
if (up) return new simpleScoreIterator<E>(); if (up) return new simpleScoreIterator<E>();
return new reverseScoreIterator<E>(); return new reverseScoreIterator<E>();
} }
@ -405,7 +409,7 @@ public final class ScoreCluster<E> implements DynamicScore<E> {
for (int i = 0; i < count; i++) { for (int i = 0; i < count; i++) {
r = random.nextInt(); r = random.nextInt();
mem[i] = r; mem[i] = r;
s.addScore("score#" + r, r); s.inc("score#" + r, r);
c += r; c += r;
} }
@ -413,18 +417,18 @@ public final class ScoreCluster<E> implements DynamicScore<E> {
int p; int p;
for (int i = 0; i < (count / 2); i++) { for (int i = 0; i < (count / 2); i++) {
p = (int) (random.nextFloat() * count); p = (int) (random.nextFloat() * count);
if (s.existsScore("score#" + mem[p])) { if (s.containsKey("score#" + mem[p])) {
System.out.println("delete score#" + mem[p]); System.out.println("delete score#" + mem[p]);
s.deleteScore("score#" + mem[p]); s.delete("score#" + mem[p]);
c -= mem[p]; c -= mem[p];
} }
} }
} }
System.out.println("result:"); System.out.println("result:");
Iterator<String> i = s.scores(true); Iterator<String> i = s.keys(true);
while (i.hasNext()) System.out.println("up: " + i.next()); while (i.hasNext()) System.out.println("up: " + i.next());
i = s.scores(false); i = s.keys(false);
while (i.hasNext()) System.out.println("down: " + i.next()); while (i.hasNext()) System.out.println("down: " + i.next());
System.out.println("finished create. time = " + (System.currentTimeMillis() - time)); System.out.println("finished create. time = " + (System.currentTimeMillis() - time));

@ -95,7 +95,7 @@ public class ScoreMap<E> implements DynamicScore<E> {
return map.isEmpty(); return map.isEmpty();
} }
public void incScore(final E obj) { public void inc(final E obj) {
if (obj == null) return; if (obj == null) return;
synchronized (this) { synchronized (this) {
IntScore score = this.map.get(obj); IntScore score = this.map.get(obj);
@ -109,7 +109,7 @@ public class ScoreMap<E> implements DynamicScore<E> {
gcount++; gcount++;
} }
public void decScore(final E obj) { public void dec(final E obj) {
if (obj == null) return; if (obj == null) return;
synchronized (this) { synchronized (this) {
IntScore score = this.map.get(obj); IntScore score = this.map.get(obj);
@ -123,7 +123,7 @@ public class ScoreMap<E> implements DynamicScore<E> {
gcount--; gcount--;
} }
public void setScore(final E obj, final int newScore) { public void set(final E obj, final int newScore) {
if (obj == null) return; if (obj == null) return;
synchronized (this) { synchronized (this) {
IntScore score = this.map.get(obj); IntScore score = this.map.get(obj);
@ -138,7 +138,7 @@ public class ScoreMap<E> implements DynamicScore<E> {
gcount += newScore; gcount += newScore;
} }
public void addScore(final E obj, final int incrementScore) { public void inc(final E obj, final int incrementScore) {
if (obj == null) return; if (obj == null) return;
synchronized (this) { synchronized (this) {
IntScore score = this.map.get(obj); IntScore score = this.map.get(obj);
@ -152,7 +152,11 @@ public class ScoreMap<E> implements DynamicScore<E> {
gcount += incrementScore; gcount += incrementScore;
} }
public int deleteScore(final E obj) { public void dec(final E obj, final int incrementScore) {
inc(obj, -incrementScore);
}
public int delete(final E obj) {
// deletes entry and returns previous score // deletes entry and returns previous score
if (obj == null) return 0; if (obj == null) return 0;
final IntScore score; final IntScore score;
@ -166,11 +170,11 @@ public class ScoreMap<E> implements DynamicScore<E> {
return score.intValue(); return score.intValue();
} }
public synchronized boolean existsScore(final E obj) { public synchronized boolean containsKey(final E obj) {
return map.containsKey(obj); return map.containsKey(obj);
} }
public int getScore(final E obj) { public int get(final E obj) {
if (obj == null) return 0; if (obj == null) return 0;
final IntScore score; final IntScore score;
synchronized (this) { synchronized (this) {
@ -202,7 +206,7 @@ public class ScoreMap<E> implements DynamicScore<E> {
return minScore; return minScore;
} }
public E getMaxObject() { public E getMaxKey() {
if (map.isEmpty()) return null; if (map.isEmpty()) return null;
E maxObject = null; E maxObject = null;
int maxScore = Integer.MIN_VALUE; int maxScore = Integer.MIN_VALUE;
@ -215,7 +219,7 @@ public class ScoreMap<E> implements DynamicScore<E> {
return maxObject; return maxObject;
} }
public E getMinObject() { public E getMinKey() {
if (map.isEmpty()) return null; if (map.isEmpty()) return null;
E minObject = null; E minObject = null;
int minScore = Integer.MAX_VALUE; int minScore = Integer.MAX_VALUE;
@ -232,7 +236,7 @@ public class ScoreMap<E> implements DynamicScore<E> {
return map.toString(); return map.toString();
} }
public Iterator<E> scores(boolean up) { public Iterator<E> keys(boolean up) {
synchronized (this) { synchronized (this) {
// re-organize entries // re-organize entries
TreeMap<IntScore, Set<E>> m = new TreeMap<IntScore, Set<E>>(); TreeMap<IntScore, Set<E>> m = new TreeMap<IntScore, Set<E>>();

@ -44,24 +44,24 @@ public interface StaticScore<E> {
public boolean isEmpty(); public boolean isEmpty();
public void setScore(final E obj, final int newScore); public void set(final E obj, final int newScore);
public int deleteScore(final E obj); public int delete(final E obj);
public boolean existsScore(final E obj); public boolean containsKey(final E obj);
public int getScore(final E obj); public int get(final E obj);
public int getMaxScore(); public int getMaxScore();
public int getMinScore(); public int getMinScore();
public E getMaxObject(); public E getMaxKey();
public E getMinObject(); public E getMinKey();
public String toString(); public String toString();
public Iterator<E> scores(final boolean up); public Iterator<E> keys(final boolean up);
} }

@ -120,7 +120,7 @@ public class MapDataMining extends MapHeap {
if (sortfields != null && cluster != null) for (int i = 0; i < sortfields.length; i++) { if (sortfields != null && cluster != null) for (int i = 0; i < sortfields.length; i++) {
cell = map.get(sortfields[i]); cell = map.get(sortfields[i]);
if (cell != null) cluster[i].setScore(new String(mapnameb), ScoreCluster.object2score(cell)); if (cell != null) cluster[i].set(new String(mapnameb), ScoreCluster.object2score(cell));
} }
if (longaccfields != null && longaccumulator != null) for (int i = 0; i < longaccfields.length; i++) { if (longaccfields != null && longaccumulator != null) for (int i = 0; i < longaccfields.length; i++) {
@ -246,7 +246,7 @@ public class MapDataMining extends MapHeap {
cell = map.get(sortfields[i]); cell = map.get(sortfields[i]);
if (cell != null) { if (cell != null) {
cluster = sortClusterMap.get(sortfields[i]); cluster = sortClusterMap.get(sortfields[i]);
cluster.setScore(key, ScoreCluster.object2score(cell)); cluster.set(key, ScoreCluster.object2score(cell));
sortClusterMap.put(sortfields[i], cluster); sortClusterMap.put(sortfields[i], cluster);
} }
} }
@ -282,7 +282,7 @@ public class MapDataMining extends MapHeap {
StaticScore<String> cluster; StaticScore<String> cluster;
for (int i = 0; i < sortfields.length; i++) { for (int i = 0; i < sortfields.length; i++) {
cluster = sortClusterMap.get(sortfields[i]); cluster = sortClusterMap.get(sortfields[i]);
cluster.deleteScore(key); cluster.delete(key);
sortClusterMap.put(sortfields[i], cluster); sortClusterMap.put(sortfields[i], cluster);
} }
} }
@ -293,7 +293,7 @@ public class MapDataMining extends MapHeap {
final StaticScore<String> cluster = sortClusterMap.get(field); final StaticScore<String> cluster = sortClusterMap.get(field);
if (cluster == null) return null; // sort field does not exist if (cluster == null) return null; // sort field does not exist
//System.out.println("DEBUG: cluster for field " + field + ": " + cluster.toString()); //System.out.println("DEBUG: cluster for field " + field + ": " + cluster.toString());
return new string2bytearrayIterator(cluster.scores(up)); return new string2bytearrayIterator(cluster.keys(up));
} }
public static class string2bytearrayIterator implements Iterator<byte[]> { public static class string2bytearrayIterator implements Iterator<byte[]> {

@ -611,18 +611,18 @@ public final class yacy {
while (ef.hasMoreElements()) { while (ef.hasMoreElements()) {
f = ef.nextElement(); f = ef.nextElement();
h = f.getName().substring(0, Word.commonHashLength).getBytes(); h = f.getName().substring(0, Word.commonHashLength).getBytes();
hs.addScore(h, (int) f.length()); hs.inc(h, (int) f.length());
} }
// list the hashes in reverse order // list the hashes in reverse order
Log.logInfo("GEN-WORDSTAT", "listing words in reverse size order..."); Log.logInfo("GEN-WORDSTAT", "listing words in reverse size order...");
String w; String w;
final Iterator<byte[]> i = hs.scores(false); final Iterator<byte[]> i = hs.keys(false);
while (i.hasNext()) { while (i.hasNext()) {
h = i.next(); h = i.next();
w = words.get(h); w = words.get(h);
if (w == null) System.out.print("# " + h); else System.out.print(w); if (w == null) System.out.print("# " + h); else System.out.print(w);
System.out.println(" - " + hs.getScore(h)); System.out.println(" - " + hs.get(h));
} }
// finished // finished

Loading…
Cancel
Save