* Tunnel IVValidator: Increase size of bloom filter

for high-bw routers (>= 512KBps share bw) to reduce
      false positive rate. Adds 2MB heap for >= 512KBps routers
      and 6MB for >= 1536KBps.

core/java/src/net/i2p/util/DecayingBloomFilter.java

@@ -37,6 +37,7 @@ public class DecayingBloomFilter {
     private String _name;
     
     private static final int DEFAULT_M = 23;
+    private static final int DEFAULT_K = 11;
     private static final boolean ALWAYS_MISS = false;
    
     /** noop for DHS */
@@ -56,16 +57,24 @@ public class DecayingBloomFilter {
 
     /** @param name just for logging / debugging / stats */
     public DecayingBloomFilter(I2PAppContext context, int durationMs, int entryBytes, String name) {
+        // this is instantiated in four different places, they may have different
+        // requirements, but for now use this as a gross method of memory reduction.
+        // m == 23 => 1MB each BloomSHA1 (4 pairs = 8MB total)
+        this(context, durationMs, entryBytes, name, context.getProperty("router.decayingBloomFilterM", DEFAULT_M));
+    }
+
+    /** @param m filter size exponent */
+    public DecayingBloomFilter(I2PAppContext context, int durationMs, int entryBytes, String name, int m) {
         _context = context;
         _log = context.logManager().getLog(DecayingBloomFilter.class);
         _entryBytes = entryBytes;
         _name = name;
-        // this is instantiated in four different places, they may have different
+        int k = DEFAULT_K;
-        // requirements, but for now use this as a gross method of memory reduction.
+        // max is (23,11) or (26,10); see KeySelector for details
-        // m == 23 => 1MB each BloomSHA1 (4 pairs = 8MB total)
+        if (m > DEFAULT_M)
-        int m = context.getProperty("router.decayingBloomFilterM", DEFAULT_M);
+            k--;
-        _current = new BloomSHA1(m, 11); //new BloomSHA1(23, 11);
+        _current = new BloomSHA1(m, k);
-        _previous = new BloomSHA1(m, 11); //new BloomSHA1(23, 11);
+        _previous = new BloomSHA1(m, k);
         _durationMs = durationMs;
         int numExtenders = (32+ (entryBytes-1))/entryBytes - 1;
         if (numExtenders < 0)
@@ -83,7 +92,7 @@ public class DecayingBloomFilter {
         _keepDecaying = true;
         SimpleTimer.getInstance().addEvent(_decayEvent, _durationMs);
         if (_log.shouldLog(Log.WARN))
-           _log.warn("New DBF " + name + " m = " + m + " entryBytes = " + entryBytes +
+           _log.warn("New DBF " + name + " m = " + m + " k = " + k + " entryBytes = " + entryBytes +
                      " numExtenders = " + numExtenders + " cycle (s) = " + (durationMs / 1000));
         // try to get a handle on memory usage vs. false positives
         context.statManager().createRateStat("router.decayingBloomFilter." + name + ".size",
@@ -260,12 +269,25 @@ public class DecayingBloomFilter {
     }
     
     /**
+     *  This filter is used only for participants and OBEPs, not
+     *  IBGWs, so depending on your assumptions of avg. tunnel length,
+     *  the performance is somewhat better than the gross share BW
+     *  would indicate.
+     *
+     *  Following stats for m=23, k=11:
      *  Theoretical false positive rate for   16 KBps: 1.17E-21
      *  Theoretical false positive rate for   24 KBps: 9.81E-20
      *  Theoretical false positive rate for   32 KBps: 2.24E-18
      *  Theoretical false positive rate for  256 KBps: 7.45E-9
      *  Theoretical false positive rate for  512 KBps: 5.32E-6
      *  Theoretical false positive rate for 1024 KBps: 1.48E-3
+     *  Then it gets bad: 1280 .67%; 1536 2.0%; 1792 4.4%; 2048 8.2%.
+     *
+     *  Following stats for m=24, k=10:
+     *  1280 4.5E-5; 1792 5.6E-4; 2048 0.14%
+     *
+     *  Following stats for m=25, k=10:
+     *  1792 2.4E-6; 4096 0.14%
      */
     public static void main(String args[]) {
         int kbps = 256;

core/java/src/org/xlattice/crypto/filters/BloomSHA1.java

@@ -47,6 +47,8 @@ public class BloomSHA1 {
     protected final int filterBits;
     protected final int filterWords;
     
+/* (24,11) too big - see KeySelector
+
     public static void main(String args[]) {
         BloomSHA1 b = new BloomSHA1(24, 11);
         for (int i = 0; i < 100; i++) {
@@ -55,14 +57,17 @@ public class BloomSHA1 {
             b.insert(v);
         }
     }
+*/
     
     
     /**
      * Creates a filter with 2^m bits and k 'hash functions', where
      * each hash function is portion of the 160-bit SHA1 hash.   
 
-     * @param m determines number of bits in filter, defaults to 20
+     * @param m determines number of bits in filter
-     * @param k number of hash functions, defaults to 8
+     * @param k number of hash functionsx
+     *
+     * See KeySelector for important restriction on max m and k
      */
     public BloomSHA1( int m, int k) {
         // XXX need to devise more reasonable set of checks

core/java/src/org/xlattice/crypto/filters/KeySelector.java

@@ -51,6 +51,13 @@ public class KeySelector {
      * @param k    number of 'hash functions'
      * @param bitOffset array of k bit offsets (offset of flag bit in word)
      * @param wordOffset array of k word offsets (offset of word flag is in)
+     *
+     * Note that if k and m are too big, the GenericWordSelector blows up -
+     * The max for 32-byte keys is m=23 and k=11.
+     * The precise restriction appears to be:
+     * ((5k + (k-1)(m-5)) / 8) + 2 < keySizeInBytes
+     *
+     * It isn't clear how to fix this.
      */
     public KeySelector (int m, int k, int[] bitOffset, int [] wordOffset) {
         //if ( (m < 2) || (m > 20)|| (k < 1) 
@@ -121,7 +128,7 @@ public class KeySelector {
     }
     /** 
      * Extracts the k word offsets from a key.  Suitable for general
-     * values of m and k.
+     * values of m and k. See above for formula for max m and k.
      */
     public class GenericWordSelector implements WordSelector {
         /** Extract the k offsets into the word offset array */
@@ -176,6 +183,8 @@ public class KeySelector {
                         // bits from third byte
                         bitsToGet -= 8;
                         if (bitsToGet > 0) {
+                            // AIOOBE here if m and k too big (23,11 is the max)
+                            // for a 32-byte key - see above
                             wordOffset[j] |= 
                                 ((0xff & b[curByte + 2]) >> (8 - bitsToGet))
                                                     << (stride - bitsToGet) ;

router/java/src/net/i2p/router/tunnel/BloomFilterIVValidator.java

@@ -25,14 +25,21 @@ public class BloomFilterIVValidator implements IVValidator {
      */
     private static final int HALFLIFE_MS = 10*60*1000;
     private static final int MIN_SHARE_KBPS_TO_USE_BLOOM = 64;
+    private static final int MIN_SHARE_KBPS_FOR_BIG_BLOOM = 512;
+    private static final int MIN_SHARE_KBPS_FOR_HUGE_BLOOM = 1536;
 
     public BloomFilterIVValidator(RouterContext ctx, int KBps) {
         _context = ctx;
         // Select the filter based on share bandwidth.
-        // Note that at rates approaching 1MB, we need to do something else,
+        // Note that at rates above 512KB, we increase the filter size
-        // as the Bloom filter false positive rates approach 0.1%. FIXME
+        // to keep acceptable false positive rates.
-        if (getShareBandwidth(ctx) < MIN_SHARE_KBPS_TO_USE_BLOOM)
+        // See DBF, BloomSHA1, and KeySelector for details.
+        if (KBps < MIN_SHARE_KBPS_TO_USE_BLOOM)
             _filter = new DecayingHashSet(ctx, HALFLIFE_MS, 16, "TunnelIVV"); // appx. 4MB max
+        else if (KBps >= MIN_SHARE_KBPS_FOR_HUGE_BLOOM)
+            _filter = new DecayingBloomFilter(ctx, HALFLIFE_MS, 16, "TunnelIVV", 25);  // 8MB fixed
+        else if (KBps >= MIN_SHARE_KBPS_FOR_BIG_BLOOM)
+            _filter = new DecayingBloomFilter(ctx, HALFLIFE_MS, 16, "TunnelIVV", 24);  // 4MB fixed
         else
             _filter = new DecayingBloomFilter(ctx, HALFLIFE_MS, 16, "TunnelIVV");  // 2MB fixed
         ctx.statManager().createRateStat("tunnel.duplicateIV", "Note that a duplicate IV was received", "Tunnels", 
@@ -48,11 +55,4 @@ public class BloomFilterIVValidator implements IVValidator {
         return !dup; // return true if it is OK, false if it isn't
     }
     public void destroy() { _filter.stopDecaying(); }
-
-    private static int getShareBandwidth(RouterContext ctx) {
-        int irateKBps = ctx.bandwidthLimiter().getInboundKBytesPerSecond();
-        int orateKBps = ctx.bandwidthLimiter().getOutboundKBytesPerSecond();
-        double pct = ctx.router().getSharePercentage();
-        return (int) (pct * Math.min(irateKBps, orateKBps));
-    }
 }

router/java/src/net/i2p/router/tunnel/TunnelDispatcher.java

@@ -680,9 +680,18 @@ public class TunnelDispatcher implements Service {
 ******/
 
     public void startup() {
-        // NB: 256 == assume max rate (size adjusted to handle 256 messages per second)
+        // Note that we only use the validator for participants and OBEPs, not IBGWs, so
-        _validator = new BloomFilterIVValidator(_context, 256);
+        // this BW estimate will be high by about 33% assuming 2-hop tunnels average
+        _validator = new BloomFilterIVValidator(_context, getShareBandwidth(_context));
     }
+
+    private static int getShareBandwidth(RouterContext ctx) {
+        int irateKBps = ctx.bandwidthLimiter().getInboundKBytesPerSecond();
+        int orateKBps = ctx.bandwidthLimiter().getOutboundKBytesPerSecond();
+        double pct = ctx.router().getSharePercentage();
+        return (int) (pct * Math.min(irateKBps, orateKBps));
+    }
+
     public void shutdown() {
         if (_validator != null)
             _validator.destroy();