boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

removed bookmark tags file, tags are now stored only in RAM

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6638 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 15 years ago
parent ada0ce9de3
commit d77782a8d5
3 changed files with 66 additions and 172 deletions
Show Stats Download Patch File Download Diff File

3
source/de/anomic/data/BookmarkHelper.java
Unescape View File

@ -155,8 +155,6 @@ public class BookmarkHelper {
importCount++; importCount++;
} }
db.flushTagCache();
return importCount; return importCount;
} }
@ -248,7 +246,6 @@ public class BookmarkHelper {
importCount += parseXMLimport(db, children.item(i), importPublic); importCount += parseXMLimport(db, children.item(i), importPublic);
} }
} }
db.flushTagCache();
return importCount; return importCount;
} }

218
source/de/anomic/data/bookmarksDB.java
Unescape View File

@ -33,7 +33,6 @@ import java.io.IOException;
import java.io.InputStreamReader; import java.io.InputStreamReader;
import java.io.Serializable; import java.io.Serializable;
import java.net.MalformedURLException; import java.net.MalformedURLException;
import java.util.ArrayList;
import java.util.Comparator; import java.util.Comparator;
import java.util.Date; import java.util.Date;
import java.util.HashMap; import java.util.HashMap;
@ -48,7 +47,6 @@ import java.util.regex.Pattern;
import net.yacy.kelondro.blob.MapHeap; import net.yacy.kelondro.blob.MapHeap;
import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.data.meta.DigestURI;
import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.logging.Log;
import net.yacy.kelondro.order.CloneableIterator;
import net.yacy.kelondro.order.NaturalOrder; import net.yacy.kelondro.order.NaturalOrder;
import net.yacy.kelondro.util.DateFormatter; import net.yacy.kelondro.util.DateFormatter;
import net.yacy.kelondro.util.kelondroException; import net.yacy.kelondro.util.kelondroException;
@ -74,11 +72,10 @@ public class bookmarksDB {
final static String SLEEP_TIME = "3600000"; // default sleepTime: check for recrawls every hour final static String SLEEP_TIME = "3600000"; // default sleepTime: check for recrawls every hour
// bookmarks // bookmarks
MapHeap bookmarksTable; // kelondroMap bookmarksTable; MapHeap bookmarks;
// tags // tags
MapHeap tagsTable; ConcurrentHashMap<String, Tag> tags;
ConcurrentHashMap<String, Tag> tagCache;
// autoReCrawl // autoReCrawl
private final BusyThread autoReCrawl; private final BusyThread autoReCrawl;
@ -89,20 +86,35 @@ public class bookmarksDB {
// bookmarksDB's class constructor // bookmarksDB's class constructor
// ------------------------------------ // ------------------------------------
public bookmarksDB(final File bookmarksFile, final File tagsFile, final File datesFile) throws IOException { public bookmarksDB(final File bookmarksFile, final File datesFile) throws IOException {
// bookmarks // bookmarks
tagCache = new ConcurrentHashMap<String, Tag>();
bookmarksFile.getParentFile().mkdirs(); bookmarksFile.getParentFile().mkdirs();
//this.bookmarksTable = new kelondroMap(kelondroDyn.open(bookmarksFile, bufferkb * 1024, preloadTime, 12, 256, '_', true, false)); //this.bookmarksTable = new kelondroMap(kelondroDyn.open(bookmarksFile, bufferkb * 1024, preloadTime, 12, 256, '_', true, false));
//this.bookmarksTable = new MapView(BLOBTree.toHeap(bookmarksFile, true, true, 12, 256, '_', NaturalOrder.naturalOrder, bookmarksFileNew), 1000, '_'); //this.bookmarksTable = new MapView(BLOBTree.toHeap(bookmarksFile, true, true, 12, 256, '_', NaturalOrder.naturalOrder, bookmarksFileNew), 1000, '_');
this.bookmarksTable = new MapHeap(bookmarksFile, 12, NaturalOrder.naturalOrder, 1024 * 64, 1000, '_'); this.bookmarks = new MapHeap(bookmarksFile, 12, NaturalOrder.naturalOrder, 1024 * 64, 1000, '_');
// tags // tags
tagsFile.getParentFile().mkdirs(); tags = new ConcurrentHashMap<String, Tag>();
final boolean tagsFileExisted = tagsFile.exists(); Log.logInfo("BOOKMARKS", "started init of tags from bookmarks.db...");
//this.tagsTable = new MapView(BLOBTree.toHeap(tagsFile, true, true, 12, 256, '_', NaturalOrder.naturalOrder, tagsFileNew), 500, '_'); final Iterator<Bookmark> it = bookmarkIterator(true);
this.tagsTable = new MapHeap(tagsFile, 12, NaturalOrder.naturalOrder, 1024 * 64, 500, '_'); Bookmark bookmark;
if (!tagsFileExisted) rebuildTags(); Tag tag;
String[] tags;
while(it.hasNext()){
bookmark=it.next();
tags = BookmarkHelper.cleanTagsString(bookmark.getTagsString() + bookmark.getFoldersString()).split(",");
tag = null;
for (int i = 0; i < tags.length; i++) {
tag = getTag(BookmarkHelper.tagHash(tags[i]));
if (tag == null) {
tag = new Tag(tags[i]);
}
tag.addUrl(bookmark.getUrlHash());
saveTag(tag);
}
}
Log.logInfo("BOOKMARKS", "finished init " + this.tags.size() + " tags using your "+bookmarks.size()+" bookmarks.");
// dates // dates
final boolean datesExisted = datesFile.exists(); final boolean datesExisted = datesFile.exists();
@ -125,9 +137,8 @@ public class bookmarksDB {
// ----------------------------------------------------- // -----------------------------------------------------
public void close(){ public void close(){
bookmarksTable.close(); bookmarks.close();
flushTagCache(); tags.clear();
tagsTable.close();
dates.close(); dates.close();
} }
@ -332,13 +343,13 @@ public class bookmarksDB {
// returning the number of bookmarks // returning the number of bookmarks
public int bookmarksSize(){ public int bookmarksSize(){
return bookmarksTable.size(); return bookmarks.size();
} }
// adding a bookmark to the bookmarksDB // adding a bookmark to the bookmarksDB
public void saveBookmark(final Bookmark bookmark){ public void saveBookmark(final Bookmark bookmark){
try { try {
bookmarksTable.put(bookmark.getUrlHash(), bookmark.entry); bookmarks.put(bookmark.getUrlHash(), bookmark.entry);
} catch (final Exception e) { } catch (final Exception e) {
Log.logException(e); Log.logException(e);
} }
@ -351,7 +362,7 @@ public class bookmarksDB {
public Bookmark getBookmark(final String urlHash){ public Bookmark getBookmark(final String urlHash){
try { try {
final Map<String, String> map = bookmarksTable.get(urlHash); final Map<String, String> map = bookmarks.get(urlHash);
if (map == null) return null; if (map == null) return null;
return new Bookmark(map); return new Bookmark(map);
} catch (final IOException e) { } catch (final IOException e) {
@ -375,7 +386,7 @@ public class bookmarksDB {
Bookmark b; Bookmark b;
try { try {
b = getBookmark(urlHash); b = getBookmark(urlHash);
bookmarksTable.remove(urlHash); bookmarks.remove(urlHash);
} catch (final IOException e) { } catch (final IOException e) {
b = null; b = null;
} }
@ -432,13 +443,7 @@ public class bookmarksDB {
// returning the number of tags // returning the number of tags
public int tagsSize(){ public int tagsSize(){
return tagSize(false); return this.tags.size();
}
public int tagSize(final boolean flushed){
if(flushed)
flushTagCache();
return tagsTable.size();
} }
/** /**
@ -446,19 +451,7 @@ public class bookmarksDB {
* @param hash an object of type String, containing a tagHash * @param hash an object of type String, containing a tagHash
*/ */
private Tag loadTag(final String hash){ private Tag loadTag(final String hash){
Map<String, String> map; return this.tags.get(hash);
Tag ret=null;
try {
map = tagsTable.get(hash);
} catch (final Exception e) {
Log.logException(e);
return null;
}
if(map!=null){
ret=new Tag(hash, map);
tagCache.put(hash, ret);
}
return ret;
} }
/** /**
@ -466,11 +459,12 @@ public class bookmarksDB {
* @param hash an object of type String, containing a tagHash * @param hash an object of type String, containing a tagHash
*/ */
public Tag getTag(final String hash){ public Tag getTag(final String hash){
if(tagCache.containsKey(hash)){ if(tags.containsKey(hash)){
return tagCache.get(hash); return tags.get(hash);
} }
return loadTag(hash); //null if it does not exists return loadTag(hash); //null if it does not exists
} }
/** /**
* store a Tag in tagsTable or remove an empty tag * store a Tag in tagsTable or remove an empty tag
* @param tag an object of type Tag to be stored/removed * @param tag an object of type Tag to be stored/removed
@ -478,35 +472,20 @@ public class bookmarksDB {
public void storeTag(final Tag tag){ public void storeTag(final Tag tag){
if (tag == null) return; if (tag == null) return;
if (tag.size() > 0) { if (tag.size() > 0) {
try { this.tags.put(tag.getTagHash(), tag);
bookmarksDB.this.tagsTable.put(tag.getTagHash(), tag.getMap());
} catch (Exception e) {
Log.logException(e);
}
} else { } else {
try { this.tags.remove(tag.getTagHash());
bookmarksDB.this.tagsTable.remove(tag.getTagHash());
} catch (IOException e) {
Log.logException(e);
}
} }
} }
/** /**
* save a Tag in tagCache; see also flushTagCache(), addTag(), loadTag() * save a Tag in tagCache; see also flushTagCache(), addTag(), loadTag()
* @param tag an object of type Tag to be saved in tagCache * @param tag an object of type Tag to be saved in tagCache
*/ */
public void saveTag(final Tag tag) { public void saveTag(final Tag tag) {
if (tag != null) { if (tag != null) {
tagCache.put(tag.getTagHash(), tag); tags.put(tag.getTagHash(), tag);
}
} }
public void flushTagCache() {
final Iterator<String> it=tagCache.keySet().iterator();
while(it.hasNext()){
storeTag(tagCache.get(it.next()));
}
tagCache = new ConcurrentHashMap<String, Tag>();
} }
public String addTag(final Tag tag) { // TODO: is addTag() really needed - check storeTag() and saveTag() public String addTag(final Tag tag) { // TODO: is addTag() really needed - check storeTag() and saveTag()
@ -517,21 +496,11 @@ public class bookmarksDB {
} }
public void removeTag(final String hash) { public void removeTag(final String hash) {
try { tags.remove(hash);
if(tagCache.containsKey(hash)){
tagCache.remove(hash);
}
tagsTable.remove(hash);
} catch (final IOException e) {}
} }
public Iterator<Tag> tagIterator(final boolean up) { public Iterator<Tag> tagIterator(final boolean up) {
try { return this.tags.values().iterator();
return new tagIterator(up);
} catch (final IOException e) {
Log.logException(e);
return new HashSet<Tag>().iterator();
}
} }
public Iterator<Tag> getTagIterator(final boolean priv) { public Iterator<Tag> getTagIterator(final boolean priv) {
@ -564,11 +533,7 @@ public class bookmarksDB {
} }
return set.iterator(); return set.iterator();
} }
/*
public Iterator<Tag> getTagIterator(String tagName, boolean priv){
return getTagIterator(tagName, priv, SORT_ALPHA);
}
*/
public Iterator<Tag> getTagIterator(final String tagName, final boolean priv, final int comp){ public Iterator<Tag> getTagIterator(final String tagName, final boolean priv, final int comp){
final TreeSet<Tag> set=new TreeSet<Tag>((comp == SORT_SIZE) ? tagSizeComparator : tagComparator); final TreeSet<Tag> set=new TreeSet<Tag>((comp == SORT_SIZE) ? tagSizeComparator : tagComparator);
Iterator<String> it=null; Iterator<String> it=null;
@ -603,29 +568,6 @@ public class bookmarksDB {
return set.iterator(); return set.iterator();
} }
// rebuilds the tagsDB from the bookmarksDB
public void rebuildTags(){
Log.logInfo("BOOKMARKS", "rebuilding tags.db from bookmarks.db...");
final Iterator<Bookmark> it = bookmarkIterator(true);
Bookmark bookmark;
Tag tag;
String[] tags;
while(it.hasNext()){
bookmark=it.next();
tags = BookmarkHelper.cleanTagsString(bookmark.getTagsString() + bookmark.getFoldersString()).split(",");
tag=null;
for(int i=0;i<tags.length;i++){
tag=getTag(BookmarkHelper.tagHash(tags[i]));
if(tag==null){
tag=new Tag(tags[i]);
}
tag.addUrl(bookmark.getUrlHash());
saveTag(tag);
}
}
flushTagCache();
Log.logInfo("BOOKMARKS", "Rebuilt "+tagsTable.size()+" tags using your "+bookmarksTable.size()+" bookmarks.");
}
// ------------------------------------- // -------------------------------------
// bookmarksDB's experimental functions // bookmarksDB's experimental functions
@ -762,16 +704,15 @@ public class bookmarksDB {
public static final String BOOKMARK_OWNER="bookmarkOwner"; public static final String BOOKMARK_OWNER="bookmarkOwner";
public static final String BOOKMARK_IS_FEED="bookmarkIsFeed"; public static final String BOOKMARK_IS_FEED="bookmarkIsFeed";
private String urlHash; private String urlHash;
private Set<String> tags; private Set<String> tagNames;
private long timestamp; private long timestamp;
Map<String, String> entry; Map<String, String> entry;
public Bookmark(final String urlHash, final Map<String, String> map) { public Bookmark(final String urlHash, final Map<String, String> map) {
this.entry = map; this.entry = map;
this.urlHash=urlHash; this.urlHash=urlHash;
tags=new TreeSet<String>(String.CASE_INSENSITIVE_ORDER); tagNames = new TreeSet<String>(String.CASE_INSENSITIVE_ORDER);
if(map.containsKey(BOOKMARK_TAGS)) if (map.containsKey(BOOKMARK_TAGS)) tagNames.addAll(listManager.string2set(map.get(BOOKMARK_TAGS)));
tags.addAll(listManager.string2set(map.get(BOOKMARK_TAGS)));
loadTimestamp(); loadTimestamp();
} }
@ -787,7 +728,7 @@ public class bookmarksDB {
} }
entry.put(BOOKMARK_URL, url); entry.put(BOOKMARK_URL, url);
this.timestamp=System.currentTimeMillis(); this.timestamp=System.currentTimeMillis();
tags=new HashSet<String>(); tagNames=new HashSet<String>();
final Bookmark oldBm=getBookmark(this.urlHash); final Bookmark oldBm=getBookmark(this.urlHash);
if(oldBm!=null && oldBm.entry.containsKey(BOOKMARK_TIMESTAMP)){ if(oldBm!=null && oldBm.entry.containsKey(BOOKMARK_TIMESTAMP)){
entry.put(BOOKMARK_TIMESTAMP, oldBm.entry.get(BOOKMARK_TIMESTAMP)); //preserve timestamp on edit entry.put(BOOKMARK_TIMESTAMP, oldBm.entry.get(BOOKMARK_TIMESTAMP)); //preserve timestamp on edit
@ -805,7 +746,7 @@ public class bookmarksDB {
entry = new HashMap<String, String>(); entry = new HashMap<String, String>();
this.urlHash=urlHash; this.urlHash=urlHash;
entry.put(BOOKMARK_URL, url.toNormalform(false, true)); entry.put(BOOKMARK_URL, url.toNormalform(false, true));
tags=new HashSet<String>(); tagNames=new HashSet<String>();
timestamp=System.currentTimeMillis(); timestamp=System.currentTimeMillis();
} }
@ -813,7 +754,7 @@ public class bookmarksDB {
entry = new HashMap<String, String>(); entry = new HashMap<String, String>();
this.urlHash=urlHash; this.urlHash=urlHash;
entry.put(BOOKMARK_URL, url); entry.put(BOOKMARK_URL, url);
tags=new HashSet<String>(); tagNames=new HashSet<String>();
timestamp=System.currentTimeMillis(); timestamp=System.currentTimeMillis();
} }
@ -822,7 +763,7 @@ public class bookmarksDB {
} }
Map<String, String> toMap() { Map<String, String> toMap() {
entry.put(BOOKMARK_TAGS, listManager.collection2string(tags)); entry.put(BOOKMARK_TAGS, listManager.collection2string(tagNames));
entry.put(BOOKMARK_TIMESTAMP, String.valueOf(this.timestamp)); entry.put(BOOKMARK_TIMESTAMP, String.valueOf(this.timestamp));
return entry; return entry;
} }
@ -841,7 +782,7 @@ public class bookmarksDB {
} }
public Set<String> getTags() { public Set<String> getTags() {
return tags; return tagNames;
} }
public String getTagsString() { public String getTagsString() {
@ -927,8 +868,8 @@ public class bookmarksDB {
} }
public void addTag(final String tagName){ public void addTag(final String tagName){
tags.add(tagName); tagNames.add(tagName);
setTags(tags); setTags(tagNames);
saveBookmark(this); saveBookmark(this);
} }
@ -942,13 +883,13 @@ public class bookmarksDB {
/** /**
* set the Tags of the bookmark * set the Tags of the bookmark
* @param tags ArrayList with the tagnames * @param tagNames ArrayList with the tagnames
* @param local sets, whether the updated tags should be stored to tagsDB * @param local sets, whether the updated tags should be stored to tagsDB
*/ */
public void setTags(final Set<String> tags2, final boolean local){ public void setTags(final Set<String> tags2, final boolean local){
tags = tags2; // TODO: check if this is safe tagNames = tags2; // TODO: check if this is safe
// tags.addAll(tags2); // in order for renameTag() to work I had to change this form 'add' to 'set' // tags.addAll(tags2); // in order for renameTag() to work I had to change this form 'add' to 'set'
final Iterator<String> it=tags.iterator(); final Iterator<String> it=tagNames.iterator();
while(it.hasNext()){ while(it.hasNext()){
final String tagName=it.next(); final String tagName=it.next();
Tag tag=getTag(BookmarkHelper.tagHash(tagName)); Tag tag=getTag(BookmarkHelper.tagHash(tagName));
@ -971,52 +912,7 @@ public class bookmarksDB {
this.timestamp=ts; this.timestamp=ts;
} }
} }
/**
* Subclass of bookmarksDB, which provides the tagIterator object-type
*/
public class tagIterator implements Iterator<Tag> {
CloneableIterator<byte[]> tagIter;
//bookmarksDB.Tag nextEntry;
public tagIterator(final boolean up) throws IOException {
flushTagCache(); //XXX: This costs performace :-((
this.tagIter = bookmarksDB.this.tagsTable.keys(up, false);
//this.nextEntry = null;
}
public boolean hasNext() {
try {
return this.tagIter.hasNext();
} catch (final Exception e) {
Log.logException(e);
return false;
}
}
public Tag next() {
try {
byte[] b = this.tagIter.next();
String s = new String(b);
//System.out.println("### DEBUG tagIterator - " + s);
Tag t = getTag(s);
return t;
} catch (final Exception e) {
Log.logException(e);
return null;
}
}
public void remove() {
// if (this.nextEntry != null) {
// try {
// final String tagHash = this.nextEntry.getTagHash();
// if (tagHash != null) removeTag(tagHash);
// } catch (final kelondroException e) {
// //resetDatabase();
// }
// }
}
}
/** /**
* Subclass of bookmarksDB, which provides the bookmarkIterator object-type * Subclass of bookmarksDB, which provides the bookmarkIterator object-type
@ -1026,7 +922,7 @@ public class bookmarksDB {
//bookmarksDB.Bookmark nextEntry; //bookmarksDB.Bookmark nextEntry;
public bookmarkIterator(final boolean up) throws IOException { public bookmarkIterator(final boolean up) throws IOException {
//flushBookmarkCache(); //XXX: this will cost performance //flushBookmarkCache(); //XXX: this will cost performance
this.bookmarkIter = bookmarksDB.this.bookmarksTable.keys(up, false); this.bookmarkIter = bookmarksDB.this.bookmarks.keys(up, false);
//this.nextEntry = null; //this.nextEntry = null;
} }

3
source/de/anomic/search/Switchboard.java
Unescape View File

@ -909,7 +909,8 @@ public final class Switchboard extends serverSwitch {
final File bookmarksFile = new File(workPath, "bookmarks.heap"); final File bookmarksFile = new File(workPath, "bookmarks.heap");
final File tagsFile = new File(workPath, "bookmarkTags.heap"); final File tagsFile = new File(workPath, "bookmarkTags.heap");
final File datesFile = new File(workPath, "bookmarkDates.heap"); final File datesFile = new File(workPath, "bookmarkDates.heap");
this.bookmarksDB = new bookmarksDB(bookmarksFile, tagsFile, datesFile); tagsFile.delete();
this.bookmarksDB = new bookmarksDB(bookmarksFile, datesFile);
this.log.logConfig("Loaded Bookmarks DB from files "+ bookmarksFile.getName()+ ", "+tagsFile.getName()); this.log.logConfig("Loaded Bookmarks DB from files "+ bookmarksFile.getName()+ ", "+tagsFile.getName());
this.log.logConfig(this.bookmarksDB.tagsSize()+" Tag, "+this.bookmarksDB.bookmarksSize()+" Bookmarks"); this.log.logConfig(this.bookmarksDB.tagsSize()+" Tag, "+this.bookmarksDB.bookmarksSize()+" Bookmarks");
} }

Write Preview
Loading…
Cancel
Save