* track searches more carefully

* detect situations where we may be inadvertantly flooding the netDb
and log them as CRIT with a stacktrace, as well as publish the count
of those events in the netDb
* detect potential netDb DoS situations by checking to see if we have
received more than 20 netDb lookups in 10 seconds, and if so,
probabalistically drop subsequent netDb messages (P=1-(10/numReceived)).
This is also published in the netDb.

router/java/src/net/i2p/data/i2np/DatabaseLookupMessage.java

@@ -36,11 +36,41 @@ public class DatabaseLookupMessage extends I2NPMessageImpl {
     private TunnelId _replyTunnel;
     private Set _dontIncludePeers;
     
+    private static volatile long _currentLookupPeriod;
+    private static volatile int _currentLookupCount;
+    // if we try to send over 20 netDb lookups in 10 seconds, we're acting up
+    private static final long LOOKUP_THROTTLE_PERIOD = 10*1000;
+    private static final long LOOKUP_THROTTLE_MAX = 20;
+    
     public DatabaseLookupMessage(I2PAppContext context) {
         super(context);
         setSearchKey(null);
         setFrom(null);
         setDontIncludePeers(null);
+        
+        context.statManager().createRateStat("router.throttleNetDbDoSSend", "How many netDb lookup messages we are sending during a period with a DoS detected", "Throttle", new long[] { 60*1000, 10*60*1000, 60*60*1000, 24*60*60*1000 });
+    }
+    
+    private static boolean detectDoS(I2PAppContext context) {
+        // now lets check for DoS
+        long now = context.clock().now();
+        if (_currentLookupPeriod + LOOKUP_THROTTLE_PERIOD > now) {
+            // same period, check for DoS
+            _currentLookupCount++;
+            if (_currentLookupCount >= LOOKUP_THROTTLE_MAX) {
+                context.statManager().addRateData("router.throttleNetDbDoSSend", _currentLookupCount, 0);
+                return true;
+            } else {
+                // no DoS, at least, not yet
+                return false;
+            }
+        } else {
+            // on to the next period, reset counter, no DoS
+            // (no, I'm not worried about concurrency here)
+            _currentLookupPeriod = now;
+            _currentLookupCount = 1;
+            return true;
+        }
     }
     
     /**
@@ -110,6 +140,15 @@ public class DatabaseLookupMessage extends I2NPMessageImpl {
         if (_key == null) throw new I2NPMessageException("Key being searched for not specified");
         if (_fromHash == null) throw new I2NPMessageException("From address not specified");
         
+        // we do this in the writeMessage so we know that we have all the data
+        boolean isDoS = detectDoS(_context);
+        if (isDoS) {
+            _log.log(Log.CRIT, "Are we flooding the network with NetDb lookup messages for " 
+                     + _key.toBase64() + " (reply through " + _fromHash + " / " + _replyTunnel + ")",
+                     new Exception("Flood cause"));
+        }
+
+        
         ByteArrayOutputStream os = new ByteArrayOutputStream(32);
         try {
             _key.writeBytes(os);

router/java/src/net/i2p/router/RouterContext.java

@@ -94,7 +94,8 @@ public class RouterContext extends I2PAppContext {
         _statPublisher = new StatisticsManager(this);
         _shitlist = new Shitlist(this);
         _messageValidator = new MessageValidator(this);
-        _throttle = new RouterThrottleImpl(this);
+        //_throttle = new RouterThrottleImpl(this);
+        _throttle = new RouterDoSThrottle(this);
         _isFailingCalc = new IsFailingCalculator(this);
         _integrationCalc = new IntegrationCalculator(this);
         _speedCalc = new SpeedCalculator(this);

router/java/src/net/i2p/router/RouterDoSThrottle.java

@@ -0,0 +1,57 @@
+package net.i2p.router;
+
+import net.i2p.data.Hash;
+import net.i2p.data.i2np.TunnelCreateMessage;
+import net.i2p.stat.Rate;
+import net.i2p.stat.RateStat;
+import net.i2p.util.Log;
+
+/**
+ * Minor extention of the router throttle to handle some DoS events and 
+ * throttle accordingly.
+ *
+ */
+class RouterDoSThrottle extends RouterThrottleImpl {
+    public RouterDoSThrottle(RouterContext context) {
+        super(context);
+        context.statManager().createRateStat("router.throttleNetDbDoS", "How many netDb lookup messages have we received so far during a period with a DoS detected", "Throttle", new long[] { 60*1000, 10*60*1000, 60*60*1000, 24*60*60*1000 });
+    }
+    
+    private volatile long _currentLookupPeriod;
+    private volatile int _currentLookupCount;
+    // if we receive over 20 netDb lookups in 10 seconds, someone is acting up
+    private static final long LOOKUP_THROTTLE_PERIOD = 10*1000;
+    private static final long LOOKUP_THROTTLE_MAX = 20;
+    
+    public boolean acceptNetDbLookupRequest(Hash key) { 
+        // if we were going to refuse it anyway, drop it
+        boolean shouldAccept = super.acceptNetDbLookupRequest(key);
+        if (!shouldAccept) return false;
+        
+        // now lets check for DoS
+        long now = getContext().clock().now();
+        if (_currentLookupPeriod + LOOKUP_THROTTLE_PERIOD > now) {
+            // same period, check for DoS
+            _currentLookupCount++;
+            if (_currentLookupCount >= LOOKUP_THROTTLE_MAX) {
+                getContext().statManager().addRateData("router.throttleNetDbDoS", _currentLookupCount, 0);
+                int rand = getContext().random().nextInt(_currentLookupCount);
+                if (rand > LOOKUP_THROTTLE_MAX) {
+                    return false;
+                } else {
+                    return true;
+                }
+            } else {
+                // no DoS, at least, not yet
+                return true;
+            }
+        } else {
+            // on to the next period, reset counter, no DoS
+            // (no, I'm not worried about concurrency here)
+            _currentLookupPeriod = now;
+            _currentLookupCount = 1;
+            return true;
+        }
+    }
+
+}

router/java/src/net/i2p/router/RouterThrottleImpl.java

@@ -129,4 +129,6 @@ class RouterThrottleImpl implements RouterThrottle {
                        + " tunnels with lag of " + lag + " and " + throttleEvents + " throttle events)");
         return true;
     }
+    
+    protected RouterContext getContext() { return _context; }
 }

router/java/src/net/i2p/router/StatisticsManager.java

@@ -125,7 +125,10 @@ public class StatisticsManager implements Service {
             includeRate("netDb.storeSent", stats, new long[] { 5*60*1000, 60*60*1000 });
             includeRate("netDb.successPeers", stats, new long[] { 60*60*1000 });
             includeRate("netDb.failedPeers", stats, new long[] { 60*60*1000 });
-            includeRate("netDb.searchCount", stats, new long[] { 3*60*60*1000});
+            includeRate("router.throttleNetDbDoSSend", stats, new long[] { 10*60*1000, 60*60*1000, 24*60*60*1000 });
+            includeRate("router.throttleNetDbDoS", stats, new long[] { 10*60*1000, 60*60*1000 });
+            //includeRate("netDb.searchCount", stats, new long[] { 3*60*60*1000});
+            //includeRate("netDb.searchMessageCount", stats, new long[] { 5*60*1000, 10*60*1000, 60*60*1000 });
             //includeRate("inNetMessage.timeToDiscard", stats, new long[] { 5*60*1000, 10*60*1000, 60*60*1000 });
             //includeRate("outNetMessage.timeToDiscard", stats, new long[] { 5*60*1000, 10*60*1000, 60*60*1000 });
             includeRate("router.throttleNetworkCause", stats, new long[] { 10*60*1000, 60*60*1000 });