From f6d2ac7fb2bb7937592000aede43c02df2e37007 Mon Sep 17 00:00:00 2001 From: zzz Date: Wed, 24 Aug 2011 14:25:58 +0000 Subject: [PATCH] * Blockfile DB: Add reverse lookup table; bump DB rev to 2 --- .../client/naming/BlockfileNamingService.java | 283 +++++++++++++++++- .../net/i2p/client/naming/NamingService.java | 12 +- 2 files changed, 280 insertions(+), 15 deletions(-) diff --git a/core/java/src/net/i2p/client/naming/BlockfileNamingService.java b/core/java/src/net/i2p/client/naming/BlockfileNamingService.java index b0efc768fe..aa440929fb 100644 --- a/core/java/src/net/i2p/client/naming/BlockfileNamingService.java +++ b/core/java/src/net/i2p/client/naming/BlockfileNamingService.java @@ -36,6 +36,7 @@ import net.i2p.util.SecureFileOutputStream; import net.metanotion.io.RAIFile; import net.metanotion.io.Serializer; import net.metanotion.io.block.BlockFile; +import net.metanotion.io.data.IntBytes; import net.metanotion.io.data.UTF8StringBytes; import net.metanotion.util.skiplist.SkipIterator; import net.metanotion.util.skiplist.SkipList; @@ -49,11 +50,21 @@ import net.metanotion.util.skiplist.SkipList; * * "%%__INFO__%%" is the master database skiplist, containing one entry: * "info": a Properties, serialized with DataHelper functions: - * "version": "1" + * "version": "2" * "created": Java long time (ms) + * "upgraded": Java long time (ms) (as of database version 2) * "lists": Comma-separated list of host databases, to be * searched in-order for lookups * + * "%%__REVERSE__%%" is the reverse lookup skiplist + * (as of database version 2): + * The skiplist keys are Integers, the first 4 bytes of the hash of the dest. + * The skiplist values are Properties. + * There may be multiple entries in the properties, each one is a reverse mapping, + * as there may be more than one hostname for a given destination, + * or there could be collisions with the same first 4 bytes of the hash. + * Each property key is a hostname. + * Each property value is the empty string. * * For each host database, there is a skiplist containing * the hosts for that database. @@ -82,22 +93,26 @@ public class BlockfileNamingService extends DummyNamingService { private final List _invalid; private volatile boolean _isClosed; private final boolean _readOnly; + private boolean _needsUpgrade; private static final Serializer _infoSerializer = new PropertiesSerializer(); private static final Serializer _stringSerializer = new UTF8StringBytes(); private static final Serializer _destSerializer = new DestEntrySerializer(); + private static final Serializer _hashIndexSerializer = new IntBytes(); private static final String HOSTS_DB = "hostsdb.blockfile"; private static final String FALLBACK_LIST = "hosts.txt"; private static final String PROP_FORCE = "i2p.naming.blockfile.writeInAppContext"; private static final String INFO_SKIPLIST = "%%__INFO__%%"; + private static final String REVERSE_SKIPLIST = "%%__REVERSE__%%"; private static final String PROP_INFO = "info"; private static final String PROP_VERSION = "version"; private static final String PROP_LISTS = "lists"; private static final String PROP_CREATED = "created"; - private static final String PROP_MODIFIED = "modified"; - private static final String VERSION = "1"; + private static final String PROP_UPGRADED = "upgraded"; + private static final String VERSION = "2"; + private static final String OLD_VERSION = "1"; private static final String PROP_ADDED = "a"; private static final String PROP_SOURCE = "s"; @@ -173,6 +188,8 @@ public class BlockfileNamingService extends DummyNamingService { _bf = bf; _raf = raf; _readOnly = readOnly; + if (_needsUpgrade) + upgrade(); _context.addShutdownTask(new Shutdown()); } @@ -193,6 +210,7 @@ public class BlockfileNamingService extends DummyNamingService { HostsTxtNamingService.DEFAULT_HOSTS_FILE); info.setProperty(PROP_LISTS, list); hdr.put(PROP_INFO, info); + rv.makeIndex(REVERSE_SKIPLIST, _hashIndexSerializer, _infoSerializer); int total = 0; for (String hostsfile : getFilenames(list)) { @@ -221,6 +239,7 @@ public class BlockfileNamingService extends DummyNamingService { Destination d = lookupBase64(b64); if (d != null) { addEntry(rv, hostsfile, key, d, sourceMsg); + addReverseEntry(rv, key, d, _log); count++; } else { _log.logAlways(Log.WARN, "Unable to import entry for " + key + @@ -261,9 +280,6 @@ public class BlockfileNamingService extends DummyNamingService { Properties info = (Properties) hdr.get(PROP_INFO); if (info == null) throw new IOException("No header info"); - String version = info.getProperty(PROP_VERSION); - if (!VERSION.equals(version)) - throw new IOException("Bad db version: " + version); String list = info.getProperty(PROP_LISTS); if (list == null) @@ -275,14 +291,26 @@ public class BlockfileNamingService extends DummyNamingService { createdOn = Long.parseLong(created); } catch (NumberFormatException nfe) {} } - if (_log.shouldLog(Log.INFO)) - _log.info("Found database version " + version + " created " + (new Date(createdOn)).toString() + - " containing lists: " + list); + + String version = info.getProperty(PROP_VERSION); + _needsUpgrade = needsUpgrade(bf, version); + if (_needsUpgrade) { + if (_log.shouldLog(Log.WARN)) + _log.warn("Upgrading from database version " + version + " to " + VERSION + + " created " + (new Date(createdOn)).toString() + + " containing lists: " + list); + } else { + if (_log.shouldLog(Log.INFO)) + _log.info("Found database version " + version + + " created " + (new Date(createdOn)).toString() + + " containing lists: " + list); + } List skiplists = getFilenames(list); if (skiplists.isEmpty()) skiplists.add(FALLBACK_LIST); _lists.addAll(skiplists); + if (_log.shouldLog(Log.INFO)) _log.info("DB init took " + DataHelper.formatDuration(_context.clock().now() - start)); return bf; @@ -292,6 +320,65 @@ public class BlockfileNamingService extends DummyNamingService { } } + /** + * @return true if needs an upgrade + * @throws IOE on bad version + * @since 0.8.9 + */ + private boolean needsUpgrade(BlockFile bf, String version) throws IOException { + if (VERSION.equals(version)) + return false; + if (!OLD_VERSION.equals(version)) + throw new IOException("Bad db version: " + version); + if (!bf.file.canWrite()) { + if (_log.shouldLog(Log.WARN)) + _log.warn("Not upgrading read-only database version " + version); + return false; + } + return true; + } + + /** + * Blockfile must be writable of course. + * @return true if upgraded successfully + * @since 0.8.9 + */ + private boolean upgrade() { + try { + // shouldn't ever be there... + SkipList rev = _bf.getIndex(REVERSE_SKIPLIST, _hashIndexSerializer, _infoSerializer); + if (rev == null) + rev = _bf.makeIndex(REVERSE_SKIPLIST, _hashIndexSerializer, _infoSerializer); + Map entries = getEntries(); + long start = System.currentTimeMillis(); + int i = 0; + for (Map.Entry entry : entries.entrySet()) { + addReverseEntry(entry.getKey(), entry.getValue()); + i++; + } + if (_log.shouldLog(Log.WARN)) + _log.warn("Created reverse index with " + i + " entries"); + SkipList hdr = _bf.getIndex(INFO_SKIPLIST, _stringSerializer, _infoSerializer); + if (hdr == null) + throw new IOException("No db header"); + Properties info = (Properties) hdr.get(PROP_INFO); + if (info == null) + throw new IOException("No header info"); + info.setProperty(PROP_VERSION, VERSION); + info.setProperty(PROP_UPGRADED, Long.toString(_context.clock().now())); + hdr.put(PROP_INFO, info); + if (_log.shouldLog(Log.WARN)) + _log.warn("Upgraded to version " + VERSION + " in " + + DataHelper.formatDuration(System.currentTimeMillis() - start)); + return true; + } catch (IOException ioe) { + _log.error("Error upgrading DB", ioe); + } catch (RuntimeException e) { + _log.error("Error upgrading DB", e); + } + return false; + } + /** * Caller must synchronize * @return entry or null, or throws ioe @@ -382,6 +469,136 @@ public class BlockfileNamingService extends DummyNamingService { return sl.remove(key); } + ///// Reverse index methods + + /** + * Caller must synchronize. + * @return null without exception on error (logs only) + * @since 0.8.9 + */ + private String getReverseEntry(Destination dest) { + return getReverseEntry(dest.calculateHash()); + } + + /** + * Caller must synchronize. + * Returns null without exception on error (logs only). + * Returns without logging if no reverse skiplist (version 1). + * + * @return the first one found if more than one + * @since 0.8.9 + */ + private String getReverseEntry(Hash hash) { + try { + SkipList rev = _bf.getIndex(REVERSE_SKIPLIST, _hashIndexSerializer, _infoSerializer); + if (rev == null) + return null; + Integer idx = getReverseKey(hash); + //_log.info("Get reverse " + idx + ' ' + hash); + Properties props = (Properties) rev.get(idx); + if (props == null) + return null; + for (Object okey : props.keySet()) { + String key = (String) okey; + // now do the forward lookup to verify (using the cache) + Destination d = lookup(key); + if (d != null && d.calculateHash().equals(hash)) + return key; + } + } catch (IOException ioe) { + _log.error("DB get reverse error", ioe); + } catch (RuntimeException e) { + _log.error("DB get reverse error", e); + } + return null; + } + + /** + * Caller must synchronize. + * Fails without exception on error (logs only) + * @since 0.8.9 + */ + private void addReverseEntry(String key, Destination dest) { + addReverseEntry(_bf, key, dest, _log); + } + + /** + * Caller must synchronize. + * Fails without exception on error (logs only). + * Returns without logging if no reverse skiplist (version 1). + * + * We store one or more hostnames for a given hash. + * The skiplist key is a signed Integer, the first 4 bytes of the dest hash. + * For convenience (since we have a serializer already) we use + * a Properties as the value, with a null string as the value for each hostname property. + * We could in the future use the property value for something. + * @since 0.8.9 + */ + private static void addReverseEntry(BlockFile bf, String key, Destination dest, Log log) { + //log.info("Add reverse " + key); + try { + SkipList rev = bf.getIndex(REVERSE_SKIPLIST, _hashIndexSerializer, _infoSerializer); + if (rev == null) + return; + Integer idx = getReverseKey(dest); + Properties props = (Properties) rev.get(idx); + if (props != null) { + if (props.getProperty(key) != null) + return; + } else { + props = new Properties(); + } + props.put(key, ""); + rev.put(idx, props); + } catch (IOException ioe) { + log.error("DB add reverse error", ioe); + } catch (RuntimeException e) { + log.error("DB add reverse error", e); + } + } + + /** + * Caller must synchronize. + * Fails without exception on error (logs only) + * @since 0.8.9 + */ + private void removeReverseEntry(String key, Destination dest) { + //_log.info("Remove reverse " + key); + try { + SkipList rev = _bf.getIndex(REVERSE_SKIPLIST, _hashIndexSerializer, _infoSerializer); + if (rev == null) + return; + Integer idx = getReverseKey(dest); + Properties props = (Properties) rev.get(idx); + if (props == null || props.remove(key) == null) + return; + if (props.isEmpty()) + rev.remove(idx); + else + rev.put(idx, props); + } catch (IOException ioe) { + _log.error("DB remove reverse error", ioe); + } catch (RuntimeException e) { + _log.error("DB remove reverse error", e); + } + } + + /** + * @since 0.8.9 + */ + private static Integer getReverseKey(Destination dest) { + return getReverseKey(dest.calculateHash()); + } + + /** + * @since 0.8.9 + */ + private static Integer getReverseKey(Hash hash) { + byte[] hashBytes = hash.getData(); + int i = (int) DataHelper.fromLong(hashBytes, 0, 4); + return Integer.valueOf(i); + } + ////////// Start NamingService API /* @@ -483,8 +700,10 @@ public class BlockfileNamingService extends DummyNamingService { if (changed && checkExisting) return false; addEntry(sl, key, d, props); - if (changed) + if (changed) { removeCache(hostname); + addReverseEntry(key, d); + } for (NamingServiceListener nsl : _listeners) { if (changed) nsl.entryChanged(this, hostname, d, options); @@ -527,9 +746,15 @@ public class BlockfileNamingService extends DummyNamingService { SkipList sl = _bf.getIndex(listname, _stringSerializer, _destSerializer); if (sl == null) return false; - boolean rv = removeEntry(sl, key) != null; + Object removed = removeEntry(sl, key); + boolean rv = removed != null; if (rv) { removeCache(hostname); + try { + removeReverseEntry(key, ((DestEntry)removed).dest); + } catch (ClassCastException cce) { + _log.error("DB reverse remove error", cce); + } for (NamingServiceListener nsl : _listeners) { nsl.entryRemoved(this, key); } @@ -642,6 +867,27 @@ public class BlockfileNamingService extends DummyNamingService { } } + /** + * @param options ignored + * @since 0.8.9 + */ + @Override + public String reverseLookup(Destination d, Properties options) { + return reverseLookup(d.calculateHash()); + } + + /** + * @since 0.8.9 + */ + @Override + public String reverseLookup(Hash h) { + synchronized(_bf) { + if (_isClosed) + return null; + return getReverseEntry(h); + } + } + /** * @param options If non-null and contains the key "list", return the * size of that list (default "hosts.txt", NOT all lists) @@ -940,16 +1186,27 @@ public class BlockfileNamingService extends DummyNamingService { System.out.println("Testing with " + names.size() + " hostnames"); int found = 0; int notfound = 0; + int rfound = 0; + int rnotfound = 0; long start = System.currentTimeMillis(); for (String name : names) { Destination dest = bns.lookup(name); - if (dest != null) + if (dest != null) { found++; - else + String reverse = bns.reverseLookup(dest); + if (reverse != null) + rfound++; + else + rnotfound++; + } else { notfound++; + } } System.out.println("BFNS took " + DataHelper.formatDuration(System.currentTimeMillis() - start)); System.out.println("found " + found + " notfound " + notfound); + System.out.println("reverse found " + rfound + " notfound " + rnotfound); + + //if (true) return; System.out.println("Removing all " + names.size() + " hostnames"); found = 0; diff --git a/core/java/src/net/i2p/client/naming/NamingService.java b/core/java/src/net/i2p/client/naming/NamingService.java index aa5ecb4bf8..9b4a33d727 100644 --- a/core/java/src/net/i2p/client/naming/NamingService.java +++ b/core/java/src/net/i2p/client/naming/NamingService.java @@ -61,7 +61,8 @@ public abstract class NamingService { } /** - * Reverse look up a destination + * Reverse lookup a destination + * @param dest non-null * @return a host name for this Destination, or null * if none is known. It is safe for subclasses to always return * null if no reverse lookup is possible. @@ -70,7 +71,13 @@ public abstract class NamingService { return reverseLookup(dest, null); } - /** @deprecated unused */ + /** + * Reverse lookup a hash + * @param h non-null + * @return a host name for this hash, or null + * if none is known. It is safe for subclasses to always return + * null if no reverse lookup is possible. + */ public String reverseLookup(Hash h) { return null; } /** @@ -370,6 +377,7 @@ public abstract class NamingService { /** * Same as reverseLookup(dest) but with options + * @param d non-null * @param options NamingService-specific, can be null * @return host name or null * @since 0.8.7