I2P_Developers/i2p.itoopie
3
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

* Tunnel IVValidator: Increase size of bloom filter

Browse Source 
for high-bw routers (>= 512KBps share bw) to reduce
      false positive rate. Adds 2MB heap for >= 512KBps routers
      and 6MB for >= 1536KBps.
This commit is contained in:
zzz
2009-10-02 03:14:16 +00:00
parent 1cd646afe2
commit 9931112387
5 changed files with 67 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
core/java/src/net/i2p/util/DecayingBloomFilter.java
View File

@ -37,6 +37,7 @@ public class DecayingBloomFilter {
private String _name;
private static final int DEFAULT_M = 23;
private static final int DEFAULT_K = 11;
private static final boolean ALWAYS_MISS = false;
/** noop for DHS */
@ -56,16 +57,24 @@ public class DecayingBloomFilter {
/** @param name just for logging / debugging / stats */
public DecayingBloomFilter(I2PAppContext context, int durationMs, int entryBytes, String name) {
// this is instantiated in four different places, they may have different
// requirements, but for now use this as a gross method of memory reduction.
// m == 23 => 1MB each BloomSHA1 (4 pairs = 8MB total)
this(context, durationMs, entryBytes, name, context.getProperty("router.decayingBloomFilterM", DEFAULT_M));
}
/** @param m filter size exponent */
public DecayingBloomFilter(I2PAppContext context, int durationMs, int entryBytes, String name, int m) {
_context = context;
_log = context.logManager().getLog(DecayingBloomFilter.class);
_entryBytes = entryBytes;
_name = name;
// this is instantiated in four different places, they may have different
// requirements, but for now use this as a gross method of memory reduction.
// m == 23 => 1MB each BloomSHA1 (4 pairs = 8MB total)
int m = context.getProperty("router.decayingBloomFilterM", DEFAULT_M);
_current = new BloomSHA1(m, 11); //new BloomSHA1(23, 11);
_previous = new BloomSHA1(m, 11); //new BloomSHA1(23, 11);
int k = DEFAULT_K;
// max is (23,11) or (26,10); see KeySelector for details
if (m > DEFAULT_M)
k--;
_current = new BloomSHA1(m, k);
_previous = new BloomSHA1(m, k);
_durationMs = durationMs;
int numExtenders = (32+ (entryBytes-1))/entryBytes - 1;
if (numExtenders < 0)
@ -83,7 +92,7 @@ public class DecayingBloomFilter {
_keepDecaying = true;
SimpleTimer.getInstance().addEvent(_decayEvent, _durationMs);
if (_log.shouldLog(Log.WARN))
_log.warn("New DBF " + name + " m = " + m + " entryBytes = " + entryBytes +
_log.warn("New DBF " + name + " m = " + m + " k = " + k + " entryBytes = " + entryBytes +
" numExtenders = " + numExtenders + " cycle (s) = " + (durationMs / 1000));
// try to get a handle on memory usage vs. false positives
context.statManager().createRateStat("router.decayingBloomFilter." + name + ".size",
@ -260,12 +269,25 @@ public class DecayingBloomFilter {
}
/**
* This filter is used only for participants and OBEPs, not
* IBGWs, so depending on your assumptions of avg. tunnel length,
* the performance is somewhat better than the gross share BW
* would indicate.
*
* Following stats for m=23, k=11:
* Theoretical false positive rate for 16 KBps: 1.17E-21
* Theoretical false positive rate for 24 KBps: 9.81E-20
* Theoretical false positive rate for 32 KBps: 2.24E-18
* Theoretical false positive rate for 256 KBps: 7.45E-9
* Theoretical false positive rate for 512 KBps: 5.32E-6
* Theoretical false positive rate for 1024 KBps: 1.48E-3
* Then it gets bad: 1280 .67%; 1536 2.0%; 1792 4.4%; 2048 8.2%.
*
* Following stats for m=24, k=10:
* 1280 4.5E-5; 1792 5.6E-4; 2048 0.14%
*
* Following stats for m=25, k=10:
* 1792 2.4E-6; 4096 0.14%
*/
public static void main(String args[]) {
int kbps = 256;

9
core/java/src/org/xlattice/crypto/filters/BloomSHA1.java
View File

@ -47,6 +47,8 @@ public class BloomSHA1 {
protected final int filterBits;
protected final int filterWords;
/* (24,11) too big - see KeySelector
public static void main(String args[]) {
BloomSHA1 b = new BloomSHA1(24, 11);
for (int i = 0; i < 100; i++) {
@ -55,14 +57,17 @@ public class BloomSHA1 {
b.insert(v);
}
}
*/
/**
* Creates a filter with 2^m bits and k 'hash functions', where
* each hash function is portion of the 160-bit SHA1 hash.
* @param m determines number of bits in filter, defaults to 20
* @param k number of hash functions, defaults to 8
* @param m determines number of bits in filter
* @param k number of hash functionsx
*
* See KeySelector for important restriction on max m and k
*/
public BloomSHA1( int m, int k) {
// XXX need to devise more reasonable set of checks

11
core/java/src/org/xlattice/crypto/filters/KeySelector.java
View File

@ -51,6 +51,13 @@ public class KeySelector {
* @param k number of 'hash functions'
* @param bitOffset array of k bit offsets (offset of flag bit in word)
* @param wordOffset array of k word offsets (offset of word flag is in)
*
* Note that if k and m are too big, the GenericWordSelector blows up -
* The max for 32-byte keys is m=23 and k=11.
* The precise restriction appears to be:
* ((5k + (k-1)(m-5)) / 8) + 2 < keySizeInBytes
*
* It isn't clear how to fix this.
*/
public KeySelector (int m, int k, int[] bitOffset, int [] wordOffset) {
//if ( (m < 2) || (m > 20)|| (k < 1)
@ -121,7 +128,7 @@ public class KeySelector {
}
/**
* Extracts the k word offsets from a key. Suitable for general
* values of m and k.
* values of m and k. See above for formula for max m and k.
*/
public class GenericWordSelector implements WordSelector {
/** Extract the k offsets into the word offset array */
@ -176,6 +183,8 @@ public class KeySelector {
// bits from third byte
bitsToGet -= 8;
if (bitsToGet > 0) {
// AIOOBE here if m and k too big (23,11 is the max)
// for a 32-byte key - see above
wordOffset[j] |=
((0xff & b[curByte + 2]) >> (8 - bitsToGet))
<< (stride - bitsToGet) ;

20
router/java/src/net/i2p/router/tunnel/BloomFilterIVValidator.java
View File

@ -25,14 +25,21 @@ public class BloomFilterIVValidator implements IVValidator {
*/
private static final int HALFLIFE_MS = 10*60*1000;
private static final int MIN_SHARE_KBPS_TO_USE_BLOOM = 64;
private static final int MIN_SHARE_KBPS_FOR_BIG_BLOOM = 512;
private static final int MIN_SHARE_KBPS_FOR_HUGE_BLOOM = 1536;
public BloomFilterIVValidator(RouterContext ctx, int KBps) {
_context = ctx;
// Select the filter based on share bandwidth.
// Note that at rates approaching 1MB, we need to do something else,
// as the Bloom filter false positive rates approach 0.1%. FIXME
if (getShareBandwidth(ctx) < MIN_SHARE_KBPS_TO_USE_BLOOM)
// Note that at rates above 512KB, we increase the filter size
// to keep acceptable false positive rates.
// See DBF, BloomSHA1, and KeySelector for details.
if (KBps < MIN_SHARE_KBPS_TO_USE_BLOOM)
_filter = new DecayingHashSet(ctx, HALFLIFE_MS, 16, "TunnelIVV"); // appx. 4MB max
else if (KBps >= MIN_SHARE_KBPS_FOR_HUGE_BLOOM)
_filter = new DecayingBloomFilter(ctx, HALFLIFE_MS, 16, "TunnelIVV", 25); // 8MB fixed
else if (KBps >= MIN_SHARE_KBPS_FOR_BIG_BLOOM)
_filter = new DecayingBloomFilter(ctx, HALFLIFE_MS, 16, "TunnelIVV", 24); // 4MB fixed
else
_filter = new DecayingBloomFilter(ctx, HALFLIFE_MS, 16, "TunnelIVV"); // 2MB fixed
ctx.statManager().createRateStat("tunnel.duplicateIV", "Note that a duplicate IV was received", "Tunnels",
@ -48,11 +55,4 @@ public class BloomFilterIVValidator implements IVValidator {
return !dup; // return true if it is OK, false if it isn't
}
public void destroy() { _filter.stopDecaying(); }
private static int getShareBandwidth(RouterContext ctx) {
int irateKBps = ctx.bandwidthLimiter().getInboundKBytesPerSecond();
int orateKBps = ctx.bandwidthLimiter().getOutboundKBytesPerSecond();
double pct = ctx.router().getSharePercentage();
return (int) (pct * Math.min(irateKBps, orateKBps));
}
}

13
router/java/src/net/i2p/router/tunnel/TunnelDispatcher.java
View File

@ -680,9 +680,18 @@ public class TunnelDispatcher implements Service {
******/
public void startup() {
// NB: 256 == assume max rate (size adjusted to handle 256 messages per second)
_validator = new BloomFilterIVValidator(_context, 256);
// Note that we only use the validator for participants and OBEPs, not IBGWs, so
// this BW estimate will be high by about 33% assuming 2-hop tunnels average
_validator = new BloomFilterIVValidator(_context, getShareBandwidth(_context));
}
private static int getShareBandwidth(RouterContext ctx) {
int irateKBps = ctx.bandwidthLimiter().getInboundKBytesPerSecond();
int orateKBps = ctx.bandwidthLimiter().getOutboundKBytesPerSecond();
double pct = ctx.router().getSharePercentage();
return (int) (pct * Math.min(irateKBps, orateKBps));
}
public void shutdown() {
if (_validator != null)
_validator.destroy();