echelon/i2p.i2p
1
0
Fork 0
forked from I2P_Developers/i2p.i2p
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

* Blockfile DB: Add reverse lookup table; bump DB rev to 2

Browse Source
This commit is contained in:
zzz
2011-08-24 14:25:58 +00:00
parent f22865acb1
commit f6d2ac7fb2
2 changed files with 280 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

279
core/java/src/net/i2p/client/naming/BlockfileNamingService.java
View File

@ -36,6 +36,7 @@ import net.i2p.util.SecureFileOutputStream;
import net.metanotion.io.RAIFile;
import net.metanotion.io.Serializer;
import net.metanotion.io.block.BlockFile;
import net.metanotion.io.data.IntBytes;
import net.metanotion.io.data.UTF8StringBytes;
import net.metanotion.util.skiplist.SkipIterator;
import net.metanotion.util.skiplist.SkipList;
@ -49,11 +50,21 @@ import net.metanotion.util.skiplist.SkipList;
*
* "%%__INFO__%%" is the master database skiplist, containing one entry:
* "info": a Properties, serialized with DataHelper functions:
* "version": "1"
* "version": "2"
* "created": Java long time (ms)
* "upgraded": Java long time (ms) (as of database version 2)
* "lists": Comma-separated list of host databases, to be
* searched in-order for lookups
*
* "%%__REVERSE__%%" is the reverse lookup skiplist
* (as of database version 2):
* The skiplist keys are Integers, the first 4 bytes of the hash of the dest.
* The skiplist values are Properties.
* There may be multiple entries in the properties, each one is a reverse mapping,
* as there may be more than one hostname for a given destination,
* or there could be collisions with the same first 4 bytes of the hash.
* Each property key is a hostname.
* Each property value is the empty string.
*
* For each host database, there is a skiplist containing
* the hosts for that database.
@ -82,22 +93,26 @@ public class BlockfileNamingService extends DummyNamingService {
private final List<InvalidEntry> _invalid;
private volatile boolean _isClosed;
private final boolean _readOnly;
private boolean _needsUpgrade;
private static final Serializer _infoSerializer = new PropertiesSerializer();
private static final Serializer _stringSerializer = new UTF8StringBytes();
private static final Serializer _destSerializer = new DestEntrySerializer();
private static final Serializer _hashIndexSerializer = new IntBytes();
private static final String HOSTS_DB = "hostsdb.blockfile";
private static final String FALLBACK_LIST = "hosts.txt";
private static final String PROP_FORCE = "i2p.naming.blockfile.writeInAppContext";
private static final String INFO_SKIPLIST = "%%__INFO__%%";
private static final String REVERSE_SKIPLIST = "%%__REVERSE__%%";
private static final String PROP_INFO = "info";
private static final String PROP_VERSION = "version";
private static final String PROP_LISTS = "lists";
private static final String PROP_CREATED = "created";
private static final String PROP_MODIFIED = "modified";
private static final String VERSION = "1";
private static final String PROP_UPGRADED = "upgraded";
private static final String VERSION = "2";
private static final String OLD_VERSION = "1";
private static final String PROP_ADDED = "a";
private static final String PROP_SOURCE = "s";
@ -173,6 +188,8 @@ public class BlockfileNamingService extends DummyNamingService {
_bf = bf;
_raf = raf;
_readOnly = readOnly;
if (_needsUpgrade)
upgrade();
_context.addShutdownTask(new Shutdown());
}
@ -193,6 +210,7 @@ public class BlockfileNamingService extends DummyNamingService {
HostsTxtNamingService.DEFAULT_HOSTS_FILE);
info.setProperty(PROP_LISTS, list);
hdr.put(PROP_INFO, info);
rv.makeIndex(REVERSE_SKIPLIST, _hashIndexSerializer, _infoSerializer);
int total = 0;
for (String hostsfile : getFilenames(list)) {
@ -221,6 +239,7 @@ public class BlockfileNamingService extends DummyNamingService {
Destination d = lookupBase64(b64);
if (d != null) {
addEntry(rv, hostsfile, key, d, sourceMsg);
addReverseEntry(rv, key, d, _log);
count++;
} else {
_log.logAlways(Log.WARN, "Unable to import entry for " + key +
@ -261,9 +280,6 @@ public class BlockfileNamingService extends DummyNamingService {
Properties info = (Properties) hdr.get(PROP_INFO);
if (info == null)
throw new IOException("No header info");
String version = info.getProperty(PROP_VERSION);
if (!VERSION.equals(version))
throw new IOException("Bad db version: " + version);
String list = info.getProperty(PROP_LISTS);
if (list == null)
@ -275,14 +291,26 @@ public class BlockfileNamingService extends DummyNamingService {
createdOn = Long.parseLong(created);
} catch (NumberFormatException nfe) {}
}
if (_log.shouldLog(Log.INFO))
_log.info("Found database version " + version + " created " + (new Date(createdOn)).toString() +
String version = info.getProperty(PROP_VERSION);
_needsUpgrade = needsUpgrade(bf, version);
if (_needsUpgrade) {
if (_log.shouldLog(Log.WARN))
_log.warn("Upgrading from database version " + version + " to " + VERSION +
" created " + (new Date(createdOn)).toString() +
" containing lists: " + list);
} else {
if (_log.shouldLog(Log.INFO))
_log.info("Found database version " + version +
" created " + (new Date(createdOn)).toString() +
" containing lists: " + list);
}
List<String> skiplists = getFilenames(list);
if (skiplists.isEmpty())
skiplists.add(FALLBACK_LIST);
_lists.addAll(skiplists);
if (_log.shouldLog(Log.INFO))
_log.info("DB init took " + DataHelper.formatDuration(_context.clock().now() - start));
return bf;
@ -292,6 +320,65 @@ public class BlockfileNamingService extends DummyNamingService {
}
}
/**
* @return true if needs an upgrade
* @throws IOE on bad version
* @since 0.8.9
*/
private boolean needsUpgrade(BlockFile bf, String version) throws IOException {
if (VERSION.equals(version))
return false;
if (!OLD_VERSION.equals(version))
throw new IOException("Bad db version: " + version);
if (!bf.file.canWrite()) {
if (_log.shouldLog(Log.WARN))
_log.warn("Not upgrading read-only database version " + version);
return false;
}
return true;
}
/**
* Blockfile must be writable of course.
* @return true if upgraded successfully
* @since 0.8.9
*/
private boolean upgrade() {
try {
// shouldn't ever be there...
SkipList rev = _bf.getIndex(REVERSE_SKIPLIST, _hashIndexSerializer, _infoSerializer);
if (rev == null)
rev = _bf.makeIndex(REVERSE_SKIPLIST, _hashIndexSerializer, _infoSerializer);
Map<String, Destination> entries = getEntries();
long start = System.currentTimeMillis();
int i = 0;
for (Map.Entry<String, Destination> entry : entries.entrySet()) {
addReverseEntry(entry.getKey(), entry.getValue());
i++;
}
if (_log.shouldLog(Log.WARN))
_log.warn("Created reverse index with " + i + " entries");
SkipList hdr = _bf.getIndex(INFO_SKIPLIST, _stringSerializer, _infoSerializer);
if (hdr == null)
throw new IOException("No db header");
Properties info = (Properties) hdr.get(PROP_INFO);
if (info == null)
throw new IOException("No header info");
info.setProperty(PROP_VERSION, VERSION);
info.setProperty(PROP_UPGRADED, Long.toString(_context.clock().now()));
hdr.put(PROP_INFO, info);
if (_log.shouldLog(Log.WARN))
_log.warn("Upgraded to version " + VERSION + " in " +
DataHelper.formatDuration(System.currentTimeMillis() - start));
return true;
} catch (IOException ioe) {
_log.error("Error upgrading DB", ioe);
} catch (RuntimeException e) {
_log.error("Error upgrading DB", e);
}
return false;
}
/**
* Caller must synchronize
* @return entry or null, or throws ioe
@ -382,6 +469,136 @@ public class BlockfileNamingService extends DummyNamingService {
return sl.remove(key);
}
///// Reverse index methods
/**
* Caller must synchronize.
* @return null without exception on error (logs only)
* @since 0.8.9
*/
private String getReverseEntry(Destination dest) {
return getReverseEntry(dest.calculateHash());
}
/**
* Caller must synchronize.
* Returns null without exception on error (logs only).
* Returns without logging if no reverse skiplist (version 1).
*
* @return the first one found if more than one
* @since 0.8.9
*/
private String getReverseEntry(Hash hash) {
try {
SkipList rev = _bf.getIndex(REVERSE_SKIPLIST, _hashIndexSerializer, _infoSerializer);
if (rev == null)
return null;
Integer idx = getReverseKey(hash);
//_log.info("Get reverse " + idx + ' ' + hash);
Properties props = (Properties) rev.get(idx);
if (props == null)
return null;
for (Object okey : props.keySet()) {
String key = (String) okey;
// now do the forward lookup to verify (using the cache)
Destination d = lookup(key);
if (d != null && d.calculateHash().equals(hash))
return key;
}
} catch (IOException ioe) {
_log.error("DB get reverse error", ioe);
} catch (RuntimeException e) {
_log.error("DB get reverse error", e);
}
return null;
}
/**
* Caller must synchronize.
* Fails without exception on error (logs only)
* @since 0.8.9
*/
private void addReverseEntry(String key, Destination dest) {
addReverseEntry(_bf, key, dest, _log);
}
/**
* Caller must synchronize.
* Fails without exception on error (logs only).
* Returns without logging if no reverse skiplist (version 1).
*
* We store one or more hostnames for a given hash.
* The skiplist key is a signed Integer, the first 4 bytes of the dest hash.
* For convenience (since we have a serializer already) we use
* a Properties as the value, with a null string as the value for each hostname property.
* We could in the future use the property value for something.
* @since 0.8.9
*/
private static void addReverseEntry(BlockFile bf, String key, Destination dest, Log log) {
//log.info("Add reverse " + key);
try {
SkipList rev = bf.getIndex(REVERSE_SKIPLIST, _hashIndexSerializer, _infoSerializer);
if (rev == null)
return;
Integer idx = getReverseKey(dest);
Properties props = (Properties) rev.get(idx);
if (props != null) {
if (props.getProperty(key) != null)
return;
} else {
props = new Properties();
}
props.put(key, "");
rev.put(idx, props);
} catch (IOException ioe) {
log.error("DB add reverse error", ioe);
} catch (RuntimeException e) {
log.error("DB add reverse error", e);
}
}
/**
* Caller must synchronize.
* Fails without exception on error (logs only)
* @since 0.8.9
*/
private void removeReverseEntry(String key, Destination dest) {
//_log.info("Remove reverse " + key);
try {
SkipList rev = _bf.getIndex(REVERSE_SKIPLIST, _hashIndexSerializer, _infoSerializer);
if (rev == null)
return;
Integer idx = getReverseKey(dest);
Properties props = (Properties) rev.get(idx);
if (props == null || props.remove(key) == null)
return;
if (props.isEmpty())
rev.remove(idx);
else
rev.put(idx, props);
} catch (IOException ioe) {
_log.error("DB remove reverse error", ioe);
} catch (RuntimeException e) {
_log.error("DB remove reverse error", e);
}
}
/**
* @since 0.8.9
*/
private static Integer getReverseKey(Destination dest) {
return getReverseKey(dest.calculateHash());
}
/**
* @since 0.8.9
*/
private static Integer getReverseKey(Hash hash) {
byte[] hashBytes = hash.getData();
int i = (int) DataHelper.fromLong(hashBytes, 0, 4);
return Integer.valueOf(i);
}
////////// Start NamingService API
/*
@ -483,8 +700,10 @@ public class BlockfileNamingService extends DummyNamingService {
if (changed && checkExisting)
return false;
addEntry(sl, key, d, props);
if (changed)
if (changed) {
removeCache(hostname);
addReverseEntry(key, d);
}
for (NamingServiceListener nsl : _listeners) {
if (changed)
nsl.entryChanged(this, hostname, d, options);
@ -527,9 +746,15 @@ public class BlockfileNamingService extends DummyNamingService {
SkipList sl = _bf.getIndex(listname, _stringSerializer, _destSerializer);
if (sl == null)
return false;
boolean rv = removeEntry(sl, key) != null;
Object removed = removeEntry(sl, key);
boolean rv = removed != null;
if (rv) {
removeCache(hostname);
try {
removeReverseEntry(key, ((DestEntry)removed).dest);
} catch (ClassCastException cce) {
_log.error("DB reverse remove error", cce);
}
for (NamingServiceListener nsl : _listeners) {
nsl.entryRemoved(this, key);
}
@ -642,6 +867,27 @@ public class BlockfileNamingService extends DummyNamingService {
}
}
/**
* @param options ignored
* @since 0.8.9
*/
@Override
public String reverseLookup(Destination d, Properties options) {
return reverseLookup(d.calculateHash());
}
/**
* @since 0.8.9
*/
@Override
public String reverseLookup(Hash h) {
synchronized(_bf) {
if (_isClosed)
return null;
return getReverseEntry(h);
}
}
/**
* @param options If non-null and contains the key "list", return the
* size of that list (default "hosts.txt", NOT all lists)
@ -940,16 +1186,27 @@ public class BlockfileNamingService extends DummyNamingService {
System.out.println("Testing with " + names.size() + " hostnames");
int found = 0;
int notfound = 0;
int rfound = 0;
int rnotfound = 0;
long start = System.currentTimeMillis();
for (String name : names) {
Destination dest = bns.lookup(name);
if (dest != null)
if (dest != null) {
found++;
String reverse = bns.reverseLookup(dest);
if (reverse != null)
rfound++;
else
rnotfound++;
} else {
notfound++;
}
}
System.out.println("BFNS took " + DataHelper.formatDuration(System.currentTimeMillis() - start));
System.out.println("found " + found + " notfound " + notfound);
System.out.println("reverse found " + rfound + " notfound " + rnotfound);
//if (true) return;
System.out.println("Removing all " + names.size() + " hostnames");
found = 0;

10
core/java/src/net/i2p/client/naming/NamingService.java
View File

@ -62,6 +62,7 @@ public abstract class NamingService {
/**
* Reverse lookup a destination
* @param dest non-null
* @return a host name for this Destination, or <code>null</code>
* if none is known. It is safe for subclasses to always return
* <code>null</code> if no reverse lookup is possible.
@ -70,7 +71,13 @@ public abstract class NamingService {
return reverseLookup(dest, null);
}
/** @deprecated unused */
/**
* Reverse lookup a hash
* @param h non-null
* @return a host name for this hash, or <code>null</code>
* if none is known. It is safe for subclasses to always return
* <code>null</code> if no reverse lookup is possible.
*/
public String reverseLookup(Hash h) { return null; }
/**
@ -370,6 +377,7 @@ public abstract class NamingService {
/**
* Same as reverseLookup(dest) but with options
* @param d non-null
* @param options NamingService-specific, can be null
* @return host name or null
* @since 0.8.7