boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

- fixed some data types in new search stack

Browse Source 
- added image domain presentation to image preview
- added new search page to menu
- added automatic re-search when an old search profile is requested and a crawl is ongoing,
  to fetch newly crawled entries

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4501 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 17 years ago
parent 81687b6bd5
commit a7abee3578
12 changed files with 54 additions and 43 deletions
Show Stats Download Patch File Download Diff File

3
htroot/env/templates/header.template vendored
Unescape View File

@ -5,7 +5,8 @@
<li class="menugroup" id="menugroupGlobalIndex"> <li class="menugroup" id="menugroupGlobalIndex">
<h3>Search&nbsp;&amp;&nbsp;Browse</h3> <h3>Search&nbsp;&amp;&nbsp;Browse</h3>
<ul class="menu"> <ul class="menu">
<li><a href="/index.html?display=1" accesskey="s" class="MenuItemLink">Search Page</a></li> <li><a href="/index.html?display=1" accesskey="s" class="MenuItemLink">(old) Search Page</a></li>
<li><a href="/yacy/user/ysearch.html?display=1" accesskey="s" class="MenuItemLink">(new) Search Page</a></li>
<li><a href="/Ranking_p.html" class="MenuItemLink lock">Ranking Config</a></li> <li><a href="/Ranking_p.html" class="MenuItemLink lock">Ranking Config</a></li>
<li><a href="/Surftips.html?display=1" class="MenuItemLink">Surftips</a></li> <li><a href="/Surftips.html?display=1" class="MenuItemLink">Surftips</a></li>
<li><a href="/Wiki.html?display=1" class="MenuItemLink">Publication</a></li> <li><a href="/Wiki.html?display=1" class="MenuItemLink">Publication</a></li>

2
htroot/yacy/user/ysearch.java
Unescape View File

@ -166,7 +166,7 @@ public class ysearch {
int contentdomCode = plasmaSearchQuery.contentdomParser(post.get("contentdom", "text")); int contentdomCode = plasmaSearchQuery.contentdomParser(post.get("contentdom", "text"));
// patch until better search profiles are available // patch until better search profiles are available
if ((contentdomCode != plasmaSearchQuery.CONTENTDOM_TEXT) && (itemsPerPage <= 30)) itemsPerPage = 30; if ((contentdomCode != plasmaSearchQuery.CONTENTDOM_TEXT) && (itemsPerPage <= 32)) itemsPerPage = 32;
serverObjects prop = new serverObjects(); serverObjects prop = new serverObjects();
if (post.get("cat", "href").equals("href")) { if (post.get("cat", "href").equals("href")) {

4
htroot/yacy/user/ysearchitem.html
Unescape View File

@ -12,8 +12,8 @@
<a href="#[href]#" class="thumblink" onclick="return hs.expand(this)"> <a href="#[href]#" class="thumblink" onclick="return hs.expand(this)">
<img src="/ViewImage.png?maxwidth=96&maxheight=96&code=#[code]#" alt="#[name]#"> <img src="/ViewImage.png?maxwidth=96&maxheight=96&code=#[code]#" alt="#[name]#">
</a> </a>
<div class="highslide-caption"><a href="#[href]#">#[name]#</a></div> <div class="highslide-caption"><a href="#[href]#">#[name]#<br \><a href="#[source]#">#[sourcedom]#</a></a></div>
<div class="snippet"><a href="#[href]#">#[name]##[attr]#</a></div> <div class="TableCellDark"><a href="#[href]#">#[name]#</a><br \>#[attr]#</div>
</div> </div>
#{/items}# #{/items}#
:: ::

4
htroot/yacy/user/ysearchitem.java
Unescape View File

@ -235,7 +235,9 @@ public class ysearchitem {
prop.putHTML("content_items_0_href", ms.href.toNormalform(true, false)); prop.putHTML("content_items_0_href", ms.href.toNormalform(true, false));
prop.put("content_items_0_code", sb.licensedURLs.aquireLicense(ms.href)); prop.put("content_items_0_code", sb.licensedURLs.aquireLicense(ms.href));
prop.putHTML("content_items_0_name", shorten(ms.name, namelength)); prop.putHTML("content_items_0_name", shorten(ms.name, namelength));
prop.put("content_items_0_attr", (ms.attr.equals("-1 x -1")) ? "" : " (" + ms.attr + ")"); // attributes, here: original size of image prop.put("content_items_0_attr", (ms.attr.equals("-1 x -1")) ? "" : "(" + ms.attr + ")"); // attributes, here: original size of image
prop.put("content_items_0_source", ms.source.toNormalform(true, false));
prop.put("content_items_0_sourcedom", ms.source.getHost());
prop.put("content_items", 1); prop.put("content_items", 1);
} }
return prop; return prop;

2
htroot/yacysearch.java
Unescape View File

@ -167,7 +167,7 @@ public class yacysearch {
int contentdomCode = plasmaSearchQuery.contentdomParser(post.get("contentdom", "text")); int contentdomCode = plasmaSearchQuery.contentdomParser(post.get("contentdom", "text"));
// patch until better search profiles are available // patch until better search profiles are available
if ((contentdomCode != plasmaSearchQuery.CONTENTDOM_TEXT) && (itemsPerPage <= 30)) itemsPerPage = 30; if ((contentdomCode != plasmaSearchQuery.CONTENTDOM_TEXT) && (itemsPerPage <= 32)) itemsPerPage = 32;
serverObjects prop = new serverObjects(); serverObjects prop = new serverObjects();
if (post.get("cat", "href").equals("href")) { if (post.get("cat", "href").equals("href")) {

4
htroot/yacysearchitem.html
Unescape View File

@ -25,8 +25,8 @@
<a href="#[href]#" class="thumblink" onclick="return hs.expand(this)"> <a href="#[href]#" class="thumblink" onclick="return hs.expand(this)">
<img src="/ViewImage.png?maxwidth=96&maxheight=96&code=#[code]#" alt="#[name]#"> <img src="/ViewImage.png?maxwidth=96&maxheight=96&code=#[code]#" alt="#[name]#">
</a> </a>
<div class="highslide-caption"><a href="#[href]#">#[name]#</a></div> <div class="highslide-caption"><a href="#[href]#">#[name]#<br \><a href="#[source]#">#[sourcedom]#</a></a></div>
<div class="TableCellDark"><a href="#[href]#">#[name]##[attr]#</a></div> <div class="TableCellDark"><a href="#[href]#">#[name]#</a><br \>#[attr]#</div>
</div> </div>
#{/items}# #{/items}#
:: ::

4
htroot/yacysearchitem.java
Unescape View File

@ -236,7 +236,9 @@ public class yacysearchitem {
prop.putHTML("content_items_0_href", ms.href.toNormalform(true, false)); prop.putHTML("content_items_0_href", ms.href.toNormalform(true, false));
prop.put("content_items_0_code", sb.licensedURLs.aquireLicense(ms.href)); prop.put("content_items_0_code", sb.licensedURLs.aquireLicense(ms.href));
prop.putHTML("content_items_0_name", shorten(ms.name, namelength)); prop.putHTML("content_items_0_name", shorten(ms.name, namelength));
prop.put("content_items_0_attr", (ms.attr.equals("-1 x -1")) ? "" : " (" + ms.attr + ")"); // attributes, here: original size of image prop.put("content_items_0_attr", (ms.attr.equals("-1 x -1")) ? "" : "(" + ms.attr + ")"); // attributes, here: original size of image
prop.put("content_items_0_source", ms.source.toNormalform(true, false));
prop.put("content_items_0_sourcedom", ms.source.getHost());
prop.put("content_items", 1); prop.put("content_items", 1);
} }
return prop; return prop;

25
source/de/anomic/kelondro/kelondroSortStack.java
Unescape View File

@ -57,18 +57,17 @@ public class kelondroSortStack<E> {
push(se.element, se.weight); push(se.element, se.weight);
} }
public synchronized void push(E element, long weight) { public synchronized void push(E element, Long weight) {
if (exists(element)) return; if (exists(element)) return;
// manipulate weight in such a way that it has no conflicts // manipulate weight in such a way that it has no conflicts
Long w = new Long(weight); while (this.onstack.containsKey(weight)) weight = new Long(weight.longValue() + 1);
while (this.onstack.containsKey(w)) w = new Long(w.longValue() + 1);
// put the element on the stack // put the element on the stack
this.onstack.put(w, element); this.onstack.put(weight, element);
// register it for double-check // register it for double-check
this.instack.add(element.hashCode()); this.instack.add(new Integer(element.hashCode()));
// check maximum size of the stack an remove elements if the stack gets too large // check maximum size of the stack an remove elements if the stack gets too large
if (this.maxsize <= 0) return; if (this.maxsize <= 0) return;
@ -82,7 +81,7 @@ public class kelondroSortStack<E> {
if (this.onstack.size() == 0) return null; if (this.onstack.size() == 0) return null;
Long w = this.onstack.firstKey(); Long w = this.onstack.firstKey();
E element = this.onstack.get(w); E element = this.onstack.get(w);
return new stackElement(element, w.longValue()); return new stackElement(element, w);
} }
public synchronized stackElement pop() { public synchronized stackElement pop() {
@ -92,18 +91,18 @@ public class kelondroSortStack<E> {
if (this.onstack.size() == 0) return null; if (this.onstack.size() == 0) return null;
Long w = this.onstack.firstKey(); Long w = this.onstack.firstKey();
E element = this.onstack.remove(w); E element = this.onstack.remove(w);
stackElement se = new stackElement(element, w.longValue()); stackElement se = new stackElement(element, w);
return se; return se;
} }
public boolean exists(E element) { public boolean exists(E element) {
// uses the hashCode of the element to find out of the element had been on the list or the stack // uses the hashCode of the element to find out of the element had been on the list or the stack
return this.instack.contains(element.hashCode()); return this.instack.contains(new Integer(element.hashCode()));
} }
public boolean exists(int hashcode) { public boolean exists(int hashcode) {
// uses the hashCode of the element to find out of the element had been on the list or the stack // uses the hashCode of the element to find out of the element had been on the list or the stack
return this.instack.contains(hashcode); return this.instack.contains(new Integer(hashcode));
} }
public stackElement get(int hashcode) { public stackElement get(int hashcode) {
@ -111,7 +110,7 @@ public class kelondroSortStack<E> {
Map.Entry<Long, E> entry; Map.Entry<Long, E> entry;
while (i.hasNext()) { while (i.hasNext()) {
entry = i.next(); entry = i.next();
if (entry.getValue().hashCode() == hashcode) return new stackElement(entry.getValue(), entry.getKey().longValue()); if (entry.getValue().hashCode() == hashcode) return new stackElement(entry.getValue(), entry.getKey());
} }
return null; return null;
} }
@ -123,7 +122,7 @@ public class kelondroSortStack<E> {
while (i.hasNext()) { while (i.hasNext()) {
entry = i.next(); entry = i.next();
if (entry.getValue().hashCode() == hashcode) { if (entry.getValue().hashCode() == hashcode) {
se = new stackElement(entry.getValue(), entry.getKey().longValue()); se = new stackElement(entry.getValue(), entry.getKey());
this.onstack.remove(se.weight); this.onstack.remove(se.weight);
return se; return se;
} }
@ -137,9 +136,9 @@ public class kelondroSortStack<E> {
} }
public class stackElement { public class stackElement {
public long weight; public Long weight;
public E element; public E element;
public stackElement(E element, long weight) { public stackElement(E element, Long weight) {
this.element = element; this.element = element;
this.weight = weight; this.weight = weight;
} }

8
source/de/anomic/kelondro/kelondroSortStore.java
Unescape View File

@ -50,7 +50,7 @@ public class kelondroSortStore<E> extends kelondroSortStack<E> {
return this.offstack.size(); return this.offstack.size();
} }
public synchronized void push(E element, long weight) { public synchronized void push(E element, Long weight) {
super.push(element, weight); super.push(element, weight);
if (this.maxsize <= 0) return; if (this.maxsize <= 0) return;
while ((this.onstack.size() > 0) && (super.onstack.size() + this.offstack.size() > this.maxsize)) { while ((this.onstack.size() > 0) && (super.onstack.size() + this.offstack.size() > this.maxsize)) {
@ -79,7 +79,7 @@ public class kelondroSortStore<E> extends kelondroSortStack<E> {
while (position >= this.offstack.size()) { while (position >= this.offstack.size()) {
Long w = this.onstack.firstKey(); Long w = this.onstack.firstKey();
E element = this.onstack.remove(w); E element = this.onstack.remove(w);
stackElement se = new stackElement(element, w.longValue()); stackElement se = new stackElement(element, w);
this.offstack.add(se); this.offstack.add(se);
} }
return this.offstack.get(position); return this.offstack.get(position);
@ -94,7 +94,7 @@ public class kelondroSortStore<E> extends kelondroSortStack<E> {
while (this.onstack.size() > 0) { while (this.onstack.size() > 0) {
Long w = this.onstack.firstKey(); Long w = this.onstack.firstKey();
E element = this.onstack.remove(w); E element = this.onstack.remove(w);
stackElement se = new stackElement(element, w.longValue()); stackElement se = new stackElement(element, w);
this.offstack.add(se); this.offstack.add(se);
} }
return this.offstack; return this.offstack;
@ -103,7 +103,7 @@ public class kelondroSortStore<E> extends kelondroSortStack<E> {
while (this.onstack.size() < count) { while (this.onstack.size() < count) {
Long w = this.onstack.firstKey(); Long w = this.onstack.firstKey();
E element = this.onstack.remove(w); E element = this.onstack.remove(w);
stackElement se = new stackElement(element, w.longValue()); stackElement se = new stackElement(element, w);
this.offstack.add(se); this.offstack.add(se);
} }
return this.offstack; return this.offstack;

14
source/de/anomic/plasma/plasmaSearchEvent.java
Unescape View File

@ -216,7 +216,7 @@ public final class plasmaSearchEvent {
snippetComputationAllTime += resultEntry.snippetComputationTime; snippetComputationAllTime += resultEntry.snippetComputationTime;
// place the result to the result vector // place the result to the result vector
result.push(resultEntry, rankedCache.getOrder().cardinal(resultEntry.word())); result.push(resultEntry, new Long(rankedCache.getOrder().cardinal(resultEntry.word())));
// add references // add references
synchronized (rankedCache) { synchronized (rankedCache) {
@ -464,6 +464,14 @@ public final class plasmaSearchEvent {
event.query = query; event.query = query;
} }
// if a local crawl is ongoing, do another local search to enrich the current results with more
// entries that can possibly come out of the running crawl
if (plasmaSwitchboard.getSwitchboard().crawlQueues.noticeURL.size() > 0) {
synchronized (event.rankedCache) {
event.rankedCache.execQuery();
}
}
// if worker threads had been alive, but did not succeed, start them again to fetch missing links // if worker threads had been alive, but did not succeed, start them again to fetch missing links
if ((query.onlineSnippetFetch) && if ((query.onlineSnippetFetch) &&
(!event.anyWorkerAlive()) && (!event.anyWorkerAlive()) &&
@ -536,7 +544,7 @@ public final class plasmaSearchEvent {
// place the result to the result vector // place the result to the result vector
if (!result.exists(resultEntry)) { if (!result.exists(resultEntry)) {
result.push(resultEntry, rankedCache.getOrder().cardinal(resultEntry.word())); result.push(resultEntry, new Long(rankedCache.getOrder().cardinal(resultEntry.word())));
rankedCache.addReferences(resultEntry); rankedCache.addReferences(resultEntry);
} }
//System.out.println("DEBUG SNIPPET_LOADING: thread " + id + " got " + resultEntry.url()); //System.out.println("DEBUG SNIPPET_LOADING: thread " + id + " got " + resultEntry.url());
@ -620,7 +628,7 @@ public final class plasmaSearchEvent {
if (imagemedia != null) { if (imagemedia != null) {
for (int j = 0; j < imagemedia.size(); j++) { for (int j = 0; j < imagemedia.size(); j++) {
ms = imagemedia.get(j); ms = imagemedia.get(j);
images.push(ms, ms.ranking); images.push(ms, new Long(ms.ranking));
} }
} }
} }

12
source/de/anomic/plasma/plasmaSearchRankingProcess.java
Unescape View File

@ -154,16 +154,16 @@ public final class plasmaSearchRankingProcess {
// load url // load url
if (sortorder == 0) { if (sortorder == 0) {
this.stack.push(ientry, ientry.urlHash().hashCode()); this.stack.push(ientry, new Long(ientry.urlHash().hashCode()));
this.urlhashes.put(ientry.urlHash(), ientry.urlHash().hashCode()); this.urlhashes.put(ientry.urlHash(), new Integer(ientry.urlHash().hashCode()));
} else { } else {
uentry = wordIndex.loadedURL.load(ientry.urlHash(), ientry, 0); uentry = wordIndex.loadedURL.load(ientry.urlHash(), ientry, 0);
if (uentry == null) { if (uentry == null) {
this.misses.add(ientry.urlHash()); this.misses.add(ientry.urlHash());
} else { } else {
u = uentry.comp().url().toNormalform(false, true); u = uentry.comp().url().toNormalform(false, true);
this.stack.push(ientry, u.hashCode()); this.stack.push(ientry, new Long(u.hashCode()));
this.urlhashes.put(ientry.urlHash(), u.hashCode()); this.urlhashes.put(ientry.urlHash(), new Integer(u.hashCode()));
} }
} }
@ -306,7 +306,7 @@ public final class plasmaSearchRankingProcess {
continue; continue;
} }
o = m.top(); o = m.top();
if (o.weight < bestEntry.weight) { if (o.weight.longValue() < bestEntry.weight.longValue()) {
bestEntry = o; bestEntry = o;
} }
} }
@ -322,7 +322,7 @@ public final class plasmaSearchRankingProcess {
// returns from the current RWI list the best URL entry and removed this entry from the list // returns from the current RWI list the best URL entry and removed this entry from the list
while ((stack.size() > 0) || (size() > 0)) { while ((stack.size() > 0) || (size() > 0)) {
kelondroSortStack<indexRWIVarEntry>.stackElement obrwi = bestRWI(skipDoubleDom); kelondroSortStack<indexRWIVarEntry>.stackElement obrwi = bestRWI(skipDoubleDom);
indexURLEntry u = wordIndex.loadedURL.load(obrwi.element.urlHash(), obrwi.element, obrwi.weight); indexURLEntry u = wordIndex.loadedURL.load(obrwi.element.urlHash(), obrwi.element, obrwi.weight.longValue());
if (u != null) { if (u != null) {
indexURLEntry.Components comp = u.comp(); indexURLEntry.Components comp = u.comp();
if (comp.url() != null) this.handover.put(u.hash(), comp.url().toNormalform(true, false)); // remember that we handed over this url if (comp.url() != null) this.handover.put(u.hash(), comp.url().toNormalform(true, false)); // remember that we handed over this url

15
source/de/anomic/plasma/plasmaSnippetCache.java
Unescape View File

@ -230,12 +230,13 @@ public class plasmaSnippetCache {
public static class MediaSnippet { public static class MediaSnippet {
public int type; public int type;
public yacyURL href; public yacyURL href, source;
public String name, attr; public String name, attr;
public int ranking; public int ranking;
public MediaSnippet(int type, yacyURL href, String name, String attr, int ranking) { public MediaSnippet(int type, yacyURL href, String name, String attr, int ranking, yacyURL source) {
this.type = type; this.type = type;
this.href = href; this.href = href;
this.source = source; // the web page where the media resource appeared
this.name = name; this.name = name;
this.attr = attr; this.attr = attr;
this.ranking = ranking; // the smaller the better! small values should be shown first this.ranking = ranking; // the smaller the better! small values should be shown first
@ -682,12 +683,12 @@ public class plasmaSnippetCache {
desc = entry.getValue(); desc = entry.getValue();
s = removeAppearanceHashes(url.toNormalform(false, false), queryhashes); s = removeAppearanceHashes(url.toNormalform(false, false), queryhashes);
if (s.size() == 0) { if (s.size() == 0) {
result.add(new MediaSnippet(mediatype, url, desc, null, 0)); result.add(new MediaSnippet(mediatype, url, desc, null, 0, document.dc_source()));
continue; continue;
} }
s = removeAppearanceHashes(desc, s); s = removeAppearanceHashes(desc, s);
if (s.size() == 0) { if (s.size() == 0) {
result.add(new MediaSnippet(mediatype, url, desc, null, 0)); result.add(new MediaSnippet(mediatype, url, desc, null, 0, document.dc_source()));
continue; continue;
} }
} }
@ -712,15 +713,13 @@ public class plasmaSnippetCache {
s = removeAppearanceHashes(url.toNormalform(false, false), queryhashes); s = removeAppearanceHashes(url.toNormalform(false, false), queryhashes);
if (s.size() == 0) { if (s.size() == 0) {
int ranking = ientry.hashCode(); int ranking = ientry.hashCode();
System.out.println(ranking); result.add(new MediaSnippet(plasmaSearchQuery.CONTENTDOM_IMAGE, url, desc, ientry.width() + " x " + ientry.height(), ranking, document.dc_source()));
result.add(new MediaSnippet(plasmaSearchQuery.CONTENTDOM_IMAGE, url, desc, ientry.width() + " x " + ientry.height(), ranking));
continue; continue;
} }
s = removeAppearanceHashes(desc, s); s = removeAppearanceHashes(desc, s);
if (s.size() == 0) { if (s.size() == 0) {
int ranking = ientry.hashCode(); int ranking = ientry.hashCode();
System.out.println(ranking); result.add(new MediaSnippet(plasmaSearchQuery.CONTENTDOM_IMAGE, url, desc, ientry.width() + " x " + ientry.height(), ranking, document.dc_source()));
result.add(new MediaSnippet(plasmaSearchQuery.CONTENTDOM_IMAGE, url, desc, ientry.width() + " x " + ientry.height(), ranking));
continue; continue;
} }
} }

Write Preview
Loading…
Cancel
Save