2005-02-16 jrandom

* Added some error handling when the number of session tags exceeds the
      realistic capacity, dropping a random chunk of received tag sets and
      conducting some minor analysis of the remaining ones.  This is a part
      of a pretty serious error condition, and logs as CRIT (if/when people
      see "TOO MANY SESSION TAGS!", please let me know the full log line it
      puts in the wrapper.log or /logs.jsp)
    * Update the addressbook to only write to the published hosts location
      if the addressbook's config contains "should_publish=true" (by default,
      it contains "should_publish=false")

apps/addressbook/java/src/addressbook/Daemon.java

@@ -65,7 +65,8 @@ public class Daemon {
             master.merge((AddressBook) iter.next(), log);
         }
         master.write(new File(routerLocation));
-        master.write(published);
+        if (published != null)
+            master.write(published);
         subscriptions.write();
     }
 
@@ -82,7 +83,9 @@ public class Daemon {
                 .get("master_addressbook"));
         File routerFile = new File(home, (String) settings
                 .get("router_addressbook"));
-        File published = new File(home, (String) settings
+        File published = null;
+        if ("true".equals(settings.get("should_publish"))) 
+            published = new File(home, (String) settings
                 .get("published_addressbook"));
         File subscriptionFile = new File(home, (String) settings
                 .get("subscriptions"));
@@ -131,6 +134,7 @@ public class Daemon {
         defaultSettings.put("master_addressbook", "../userhosts.txt");
         defaultSettings.put("router_addressbook", "../hosts.txt");
         defaultSettings.put("published_addressbook", "../eepsite/docroot/hosts.txt");
+        defaultSettings.put("should_publish", "false");
         defaultSettings.put("log", "log.txt");
         defaultSettings.put("subscriptions", "subscriptions.txt");
         defaultSettings.put("etags", "etags");

core/java/src/net/i2p/crypto/TransientSessionKeyManager.java

@@ -34,8 +34,10 @@ import net.i2p.util.Log;
  */
 class TransientSessionKeyManager extends SessionKeyManager {
     private Log _log;
-    private Map _outboundSessions; // PublicKey --> OutboundSession
+    /** Map allowing us to go from the targeted PublicKey to the OutboundSession used */
-    private Map _inboundTagSets; // SessionTag --> TagSet
+    private Map _outboundSessions;
+    /** Map allowing us to go from a SessionTag to the containing TagSet */
+    private Map _inboundTagSets;
     protected I2PAppContext _context;
 
     /** 
@@ -46,7 +48,7 @@ class TransientSessionKeyManager extends SessionKeyManager {
      */
     public final static long SESSION_TAG_DURATION_MS = 10 * 60 * 1000;
     /**
-     * Keep unused inbound session tags around for up to 15 minutes (5 minutes longer than
+     * Keep unused inbound session tags around for up to 12 minutes (2 minutes longer than
      * session tags are used on the outbound side so that no reasonable network lag 
      * can cause failed decrypts)
      *
@@ -213,10 +215,11 @@ class TransientSessionKeyManager extends SessionKeyManager {
         sess.setCurrentKey(key);
         TagSet set = new TagSet(sessionTags, key, _context.clock().now());
         sess.addTags(set);
-        if (_log.shouldLog(Log.DEBUG))
+        if (_log.shouldLog(Log.DEBUG)) {
             _log.debug("Tags delivered to set " + set + " on session " + sess);
-        if (sessionTags.size() > 0)
+            if (sessionTags.size() > 0)
-            _log.debug("Tags delivered: " + sessionTags.size() + " total = " + sess.availableTags());
+                _log.debug("Tags delivered: " + sessionTags.size() + " total = " + sess.availableTags());
+        }
     }
 
     /**
@@ -234,6 +237,7 @@ class TransientSessionKeyManager extends SessionKeyManager {
      *
      */
     public void tagsReceived(SessionKey key, Set sessionTags) {
+        int overage = 0;
         TagSet tagSet = new TagSet(sessionTags, key, _context.clock().now());
         for (Iterator iter = sessionTags.iterator(); iter.hasNext();) {
             SessionTag tag = (SessionTag) iter.next();
@@ -241,18 +245,66 @@ class TransientSessionKeyManager extends SessionKeyManager {
                 _log.debug("Receiving tag " + tag + " for key " + key);
             synchronized (_inboundTagSets) {
                 _inboundTagSets.put(tag, tagSet);
+                overage = _inboundTagSets.size() - MAX_INBOUND_SESSION_TAGS;
             }
         }
-        synchronized (_inboundTagSets) {
+        if (overage > 0)
-            // todo: make this limit the tags by sessionKey and actually enforce the limit!
+            clearExcess(overage);
-            int overage = _inboundTagSets.size() - MAX_INBOUND_SESSION_TAGS;
-            if (overage > 0) {
-                if (_log.shouldLog(Log.ERROR))
-                    _log.error("TOO MANY SESSION TAGS! " + (_inboundTagSets.size()));
-            }
-        }
 
-        if (sessionTags.size() <= 0) _log.debug("Received 0 tags for key " + key);
+        if ( (sessionTags.size() <= 0) && (_log.shouldLog(Log.DEBUG)) )
+            _log.debug("Received 0 tags for key " + key);
+    }
+    
+    /**
+     * remove a bunch of arbitrarily selected tags, then drop all of
+     * the associated tag sets.  this is very time consuming - iterating
+     * across the entire _inboundTagSets map, but it should be very rare,
+     * and the stats we can gather can hopefully reduce the frequency of
+     * using too many session tags in the future
+     *
+     */
+    private void clearExcess(int overage) {
+        long now = _context.clock().now();
+        int old = 0;
+        int large = 0;
+        int absurd = 0;
+        int recent = 0;
+        int tags = 0;
+        int toRemove = overage * 2;
+        List removed = new ArrayList(toRemove);
+        synchronized (_inboundTagSets) {
+            for (Iterator iter = _inboundTagSets.values().iterator(); iter.hasNext(); ) {
+                TagSet set = (TagSet)iter.next();
+                int size = set.getTags().size();
+                if (size > 1000)
+                    absurd++;
+                if (size > 100)
+                    large++;
+                if (now - set.getDate() > SESSION_LIFETIME_MAX_MS)
+                    old++;
+                else if (now - set.getDate() < 1*60*1000)
+                    recent++;
+
+                if ((removed.size() < (toRemove)) || (now - set.getDate() > SESSION_LIFETIME_MAX_MS))
+                    removed.add(set);
+            }
+            for (int i = 0; i < removed.size(); i++) {
+                TagSet cur = (TagSet)removed.get(i);
+                for (Iterator iter = cur.getTags().iterator(); iter.hasNext(); ) {
+                    SessionTag tag = (SessionTag)iter.next();
+                    _inboundTagSets.remove(tag);
+                    tags++;
+                }
+            }
+        }
+        if (_log.shouldLog(Log.CRIT))
+            _log.log(Log.CRIT, "TOO MANY SESSION TAGS!  removing " + removed 
+                     + " tag sets arbitrarily, with " + tags + " tags,"
+                     + "where there are " + old + " long lasting sessions, "
+                     + recent + " ones created in the last minute, and "
+                     + large + " sessions with more than 100 tags (and "
+                     + absurd + " with more than 1000!), leaving a total of "
+                     + _inboundTagSets.size() + " tags behind");
     }
 
     /**
@@ -562,6 +614,7 @@ class TransientSessionKeyManager extends SessionKeyManager {
             _date = date;
         }
 
+        /** when the tag set was created */
         public long getDate() {
             return _date;
         }
@@ -570,6 +623,7 @@ class TransientSessionKeyManager extends SessionKeyManager {
             _date = when;
         }
 
+        /** tags still available */
         public Set getTags() {
             return _sessionTags;
         }

core/java/src/net/i2p/data/DataHelper.java

@@ -758,7 +758,7 @@ public class DataHelper {
         int rv = 0;
         if (b != null) {
             for (int i = 0; i < b.length && i < 8; i++)
-                rv += b[i];
+                rv += (b[i] << i);
         }
         return rv;
     }

history.txt

@@ -1,4 +1,15 @@
-$Id: history.txt,v 1.141 2005/02/10 21:44:49 smeghead Exp $
+$Id: history.txt,v 1.142 2005/02/16 17:23:47 jrandom Exp $
+
+2005-02-16  jrandom
+    * Added some error handling when the number of session tags exceeds the
+      realistic capacity, dropping a random chunk of received tag sets and
+      conducting some minor analysis of the remaining ones.  This is a part
+      of a pretty serious error condition, and logs as CRIT (if/when people 
+      see "TOO MANY SESSION TAGS!", please let me know the full log line it
+      puts in the wrapper.log or /logs.jsp)
+    * Update the addressbook to only write to the published hosts location
+      if the addressbook's config contains "should_publish=true" (by default,
+      it contains "should_publish=false")
 
 2005-02-16  jrandom
     * (Merged the 0.5-pre branch back into CVS HEAD)
@@ -47,7 +58,7 @@ $Id: history.txt,v 1.141 2005/02/10 21:44:49 smeghead Exp $
     * Substantial memory optimizations within the router and the SDK to reduce
       GC churn.  Client apps and the streaming libs have not been tuned, 
       however.
-    * More bugfixes thank you can shake a stick at.
+    * More bugfixes than you can shake a stick at.
 
 2005-02-13  jrandom
     * Updated jbigi source to handle 64bit CPUs.  The bundled jbigi.jar still 

router/java/src/net/i2p/router/RouterVersion.java

@@ -15,9 +15,9 @@ import net.i2p.CoreVersion;
  *
  */
 public class RouterVersion {
-    public final static String ID = "$Revision: 1.137.2.12 $ $Date: 2005/02/16 13:59:59 $";
+    public final static String ID = "$Revision: 1.139 $ $Date: 2005/02/16 17:23:55 $";
     public final static String VERSION = "0.5-pre";
-    public final static long BUILD = 12;
+    public final static long BUILD = 13;
     public static void main(String args[]) {
         System.out.println("I2P Router version: " + VERSION);
         System.out.println("Router ID: " + RouterVersion.ID);