2005-07-04 jrandom

* Within the tunnel, use xor(IV, msg[0:16]) as the flag to detect dups,
      rather than the IV by itself, preventing an attack that would let
      colluding internal adversaries tag a message to determine that they are
      in the same tunnel.  Thanks dvorak for the catch!
    * Drop long inactive profiles on startup and shutdown
    * /configstats.jsp: web interface to pick what stats to log
    * Deliver more session tags to account for wider window sizes
    * Cache some intermediate values in our HMACSHA256 and BC's HMAC
    * Track the client send rate (stream.sendBps and client.sendBpsRaw)
    * UrlLauncher: adjust the browser selection order
    * I2PAppContext: hooks for dummy HMACSHA256 and a weak PRNG
    * StreamSinkClient: add support for sending an unlimited amount of data
    * Migrate the tests out of the default build jars

2005-06-22  Comwiz
    * Migrate the core tests to junit

apps/addressbook/java/src/addressbook/AddressBook.java

@@ -87,6 +87,7 @@ public class AddressBook {
         this.location = subscription.getLocation();
 
         try {
+//            EepGet get = new EepGet(I2PAppContext.getGlobalContext(), true, )
             URL url = new URL(subscription.getLocation());
             HttpURLConnection connection = (HttpURLConnection) url
                     .openConnection();

apps/i2ptunnel/java/src/net/i2p/i2ptunnel/I2PTunnelClientBase.java

@@ -110,7 +110,7 @@ public abstract class I2PTunnelClientBase extends I2PTunnelTask implements Runna
                 }
             }
             if (sockMgr == null) {
-                _log.log(Log.CRIT, "Unable to create socket manager");
+                _log.log(Log.CRIT, "Unable to create socket manager (our own? " + ownDest + ")");
                 try { Thread.sleep(10*1000); } catch (InterruptedException ie) {}
             }
         }

apps/ministreaming/java/src/net/i2p/client/streaming/StreamSinkClient.java

@@ -84,7 +84,7 @@ public class StreamSinkClient {
             Random rand = new Random();
             OutputStream out = sock.getOutputStream();
             long beforeSending = System.currentTimeMillis();
-            for (int i = 0; i < _sendSize; i+= 32) {
+            for (int i = 0; (_sendSize < 0) || (i < _sendSize); i+= 32) {
                 rand.nextBytes(buf);
                 out.write(buf);
                 if (_log.shouldLog(Log.DEBUG))
@@ -117,7 +117,7 @@ public class StreamSinkClient {
     /**
      * Fire up the client.  <code>Usage: StreamSinkClient [i2cpHost i2cpPort] sendSizeKB writeDelayMs serverDestFile</code> <br />
      * <ul>
-     *  <li><b>sendSizeKB</b>: how many KB to send</li>
+     *  <li><b>sendSizeKB</b>: how many KB to send, or -1 for unlimited</li>
      *  <li><b>writeDelayMs</b>: how long to wait between each .write (0 for no delay)</li>
      *  <li><b>serverDestFile</b>: file containing the StreamSinkServer's binary Destination</li>
      * </ul>

apps/routerconsole/java/src/net/i2p/router/web/ConfigStatsHandler.java

@@ -0,0 +1,94 @@
+package net.i2p.router.web;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.StringTokenizer;
+
+import net.i2p.stat.StatManager;
+
+/**
+ * Handler to deal with form submissions from the stats config form and act
+ * upon the values.
+ *
+ */
+public class ConfigStatsHandler extends FormHandler {
+    private String _filename;
+    private List _stats;
+    private boolean _explicitFilter;
+    private String _explicitFilterValue;
+    
+    public ConfigStatsHandler() {
+        super();
+        _stats = new ArrayList();
+    }
+    
+    protected void processForm() {
+        saveChanges();
+    }
+    
+    public void setFilename(String filename) {
+        _filename = (filename != null ? filename.trim() : null);
+    }
+    
+    public void setStatList(String stat) { 
+        if (stat != null) {
+            if (stat.indexOf(',') != -1) {
+                StringTokenizer tok = new StringTokenizer(stat, ",");
+                while (tok.hasMoreTokens()) {
+                    String cur = tok.nextToken().trim();
+                    if ( (cur.length() > 0) && (!_stats.contains(cur)) )
+                        _stats.add(cur);
+                }
+            } else {
+                stat = stat.trim();
+                if ( (stat.length() > 0) && (!_stats.contains(stat)) )
+                    _stats.add(stat);
+            }
+        }
+    }
+    public void setStatList(String stats[]) {
+        if (stats != null) {
+            for (int i = 0; i < stats.length; i++) {
+                String cur = stats[i].trim();
+                if ( (cur.length() > 0) && (!_stats.contains(cur)) )
+                    _stats.add(cur);
+            }
+        }
+    }
+    
+    public void setExplicitFilter(String foo) { _explicitFilter = true; }
+    public void setExplicitFilterValue(String filter) { _explicitFilterValue = filter; }
+    
+    /**
+     * The user made changes to the config and wants to save them, so
+     * lets go ahead and do so.
+     *
+     */
+    private void saveChanges() {
+        if (_filename == null)
+            _filename = StatManager.DEFAULT_STAT_FILE;
+        _context.router().setConfigSetting(StatManager.PROP_STAT_FILE, _filename);
+        
+        if (_explicitFilter) {
+            _stats.clear();
+            setStatList(_explicitFilterValue);
+        }
+        
+        StringBuffer stats = new StringBuffer();
+        for (int i = 0; i < _stats.size(); i++) {
+            stats.append((String)_stats.get(i));
+            if (i + 1 < _stats.size())
+                stats.append(',');
+        }
+            
+        _context.router().setConfigSetting(StatManager.PROP_STAT_FILTER, stats.toString());
+        boolean ok = _context.router().saveConfig();
+        if (ok) 
+            addFormNotice("Stat filter and location updated successfully to: " + stats.toString());
+        else
+            addFormError("Failed to update the stat filter and location");
+    }
+    
+}

apps/routerconsole/java/src/net/i2p/router/web/ConfigStatsHelper.java

@@ -0,0 +1,125 @@
+package net.i2p.router.web;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.StringTokenizer;
+import net.i2p.stat.RateStat;
+import net.i2p.stat.FrequencyStat;
+import net.i2p.router.RouterContext;
+import net.i2p.util.Log;
+
+public class ConfigStatsHelper {
+    private RouterContext _context;
+    private Log _log;
+    private String _filter;
+    private Set _filters;
+    /** list of names of stats which are remaining, ordered by nested groups */
+    private List _stats;
+    private String _currentStatName;
+    private String _currentStatDescription;
+    private String _currentGroup;
+    /** true if the current stat is the first in the group */
+    private boolean _currentIsFirstInGroup;
+    /** true if the stat is being logged */
+    private boolean _currentIsLogged;
+    
+    /**
+     * Configure this bean to query a particular router context
+     *
+     * @param contextId begging few characters of the routerHash, or null to pick
+     *                  the first one we come across.
+     */
+    public void setContextId(String contextId) {
+        try {
+            _context = ContextHelper.getContext(contextId);
+            _log = _context.logManager().getLog(ConfigStatsHelper.class);
+        } catch (Throwable t) {
+            t.printStackTrace();
+        }
+        
+        _stats = new ArrayList();
+        Map groups = _context.statManager().getStatsByGroup();
+        for (Iterator iter = groups.values().iterator(); iter.hasNext(); ) {
+            Set stats = (Set)iter.next();
+            for (Iterator statIter = stats.iterator(); statIter.hasNext(); ) 
+                _stats.add(statIter.next());
+        }
+        _filter = _context.statManager().getStatFilter(); 
+        if (_filter == null)
+            _filter = "";
+        
+        _filters = new HashSet();
+        StringTokenizer tok = new StringTokenizer(_filter, ",");
+        while (tok.hasMoreTokens())
+            _filters.add(tok.nextToken().trim());
+    }
+
+    public ConfigStatsHelper() {}
+    
+    public String getFilename() { return _context.statManager().getStatFile(); }
+    
+    /** 
+     * move the cursor to the next known stat, returning true if a valid
+     * stat is available.
+     *
+     * @return true if a valid stat is available, otherwise false
+     */
+    public boolean hasMoreStats() {
+        if (_stats.size() <= 0)
+            return false;
+        _currentStatName = (String)_stats.remove(0);
+        RateStat rs = _context.statManager().getRate(_currentStatName);
+        if (rs != null) {
+            _currentStatDescription = rs.getDescription();
+            if (_currentGroup == null)
+                _currentIsFirstInGroup = true;
+            else if (!rs.getGroupName().equals(_currentGroup))
+                _currentIsFirstInGroup = true;
+            else
+                _currentIsFirstInGroup = false;
+            _currentGroup = rs.getGroupName();
+        } else {
+            FrequencyStat fs = _context.statManager().getFrequency(_currentStatName);
+            if (fs != null) {
+                _currentStatDescription = fs.getDescription();
+                if (_currentGroup == null)
+                    _currentIsFirstInGroup = true;
+                else if (!fs.getGroupName().equals(_currentGroup))
+                    _currentIsFirstInGroup = true;
+                else
+                    _currentIsFirstInGroup = false;
+                _currentGroup = fs.getGroupName();
+            } else {
+                if (_log.shouldLog(Log.ERROR))
+                    _log.error("Stat does not exist?!  [" + _currentStatName + "]");
+                return false;
+            }
+        }
+        
+        if (_filters.contains("*") || _filters.contains(_currentStatName))
+            _currentIsLogged = true;
+        else
+            _currentIsLogged = false;
+        return true;
+    }
+    
+    /** Is the current stat the first in the group? */
+    public boolean groupRequired() {
+        if (_currentIsFirstInGroup) {
+            _currentIsFirstInGroup = false;
+            return true;
+        } else {
+            return false;
+        }
+    }
+    /** What group is the current stat in */
+    public String getCurrentGroupName() { return _currentGroup; }
+    public String getCurrentStatName() { return _currentStatName; }
+    public String getCurrentStatDescription() { return _currentStatDescription; }
+    public boolean getCurrentIsLogged() { return _currentIsLogged; }
+    public String getExplicitFilter() { return _filter; }
+}

apps/routerconsole/jsp/confignav.jsp

@@ -8,5 +8,7 @@
  %>Tunnels | <% } else { %><a href="configtunnels.jsp">Tunnels</a> | <% }
  if (request.getRequestURI().indexOf("configlogging.jsp") != -1) {
  %>Logging | <% } else { %><a href="configlogging.jsp">Logging</a> | <% }
+ if (request.getRequestURI().indexOf("configstats.jsp") != -1) {
+ %>Stats | <% } else { %><a href="configstats.jsp">Stats</a> | <% }
  if (request.getRequestURI().indexOf("configadvanced.jsp") != -1) {
  %>Advanced<% } else { %><a href="configadvanced.jsp">Advanced</a><% } %></h4>

apps/routerconsole/jsp/configstats.jsp

@@ -0,0 +1,83 @@
+<%@page contentType="text/html"%>
+<%@page pageEncoding="UTF-8"%>
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
+
+<html><head>
+<title>I2P Router Console - config stats</title>
+<link rel="stylesheet" href="default.css" type="text/css" />
+<script language="JavaScript">
+function toggleAll(category) {
+  //alert("toggle all for " + category);
+  var shouldCheck = false;
+  var shouldCheckDetermined = false;
+  var elements = document.statsForm.elements;
+  for (var i = 0; i < elements.length; i++) {
+    //alert("cur element: " + i);
+    var curElement = elements.item(i);
+    //alert("cur elem: " + curElement);
+    var curName = curElement.name;
+    //alert("cur name: " + curName);
+    if (curName == 'statList') {
+      if (shouldCheckDetermined == false) {
+        shouldCheckDetermined = true;
+        shouldCheck = !curElement.checked;
+      }
+      if (shouldCheck)
+        curElement.checked = true;
+      else
+        curElement.checked = false;
+    }
+  }
+}
+</script>
+</head><body>
+<%@include file="nav.jsp" %>
+<%@include file="summary.jsp" %>
+
+<div class="main" id="main">
+ <%@include file="confignav.jsp" %>
+ 
+ <jsp:useBean class="net.i2p.router.web.ConfigStatsHandler" id="formhandler" scope="request" />
+ <jsp:setProperty name="formhandler" property="contextId" value="<%=(String)session.getAttribute("i2p.contextId")%>" />
+ <jsp:setProperty name="formhandler" property="*" />
+ <font color="red"><jsp:getProperty name="formhandler" property="errors" /></font>
+ <i><jsp:getProperty name="formhandler" property="notices" /></i>
+
+ <jsp:useBean class="net.i2p.router.web.ConfigStatsHelper" id="statshelper" scope="request" />
+ <jsp:setProperty name="statshelper" property="contextId" value="<%=(String)session.getAttribute("i2p.contextId")%>" />
+ 
+ <form id="statsForm" name="statsForm" action="configstats.jsp" method="POST">
+ <% String prev = System.getProperty("net.i2p.router.web.ConfigStatsHandler.nonce");
+    if (prev != null) System.setProperty("net.i2p.router.web.ConfigStatsHandler.noncePrev", prev);
+    System.setProperty("net.i2p.router.web.ConfigStatsHandler.nonce", new java.util.Random().nextLong()+""); %>
+ <input type="hidden" name="action" value="foo" />
+ <input type="hidden" name="nonce" value="<%=System.getProperty("net.i2p.router.web.ConfigStatsHandler.nonce")%>" />
+ Stat file: <input type="text" name="filename" value="<%=statshelper.getFilename()%>" /><br />
+ Filter: (<a href="javascript:toggleAll('*')">toggle all</a>)<br />
+ <table>
+ <% while (statshelper.hasMoreStats()) {
+      while (statshelper.groupRequired()) { %>
+ <tr><td valign="top" align="left" colspan="2">
+     <b><%=statshelper.getCurrentGroupName()%></b>
+     <!--(<a href="javascript:toggleAll('<%=statshelper.getCurrentGroupName()%>')">toggle all</a>)-->
+     </td></tr><%
+     } // end iterating over required groups for the current stat %>
+ <tr><td valign="top" align="left">
+     <input type="checkbox" name="statList" value="<%=statshelper.getCurrentStatName()%>" <% 
+     if (statshelper.getCurrentIsLogged()) { %>checked="true" <% } %>/></td>
+     <td valign="top" align="left"><b><%=statshelper.getCurrentStatName()%>:</b><br />
+     <%=statshelper.getCurrentStatDescription()%></td></tr><%
+    } // end iterating over all stats %>
+ <tr><td colspan="2"><hr /></td></tr>
+ <tr><td><input type="checkbox" name="explicitFilter" /></td>
+     <td>Advanced filter: 
+     <input type="text" name="explicitFilterValue" value="<%=statshelper.getExplicitFilter()%>" size="40" /></td></tr>
+ <tr><td colspan="2"><hr /></td></tr>
+ <tr><td><input type="submit" name="shouldsave" value="Save changes" /> </td>
+     <td><input type="reset" value="Cancel" /></td></tr>
+ </form>
+ </table>
+</div>
+
+</body>
+</html>

apps/streaming/java/src/net/i2p/client/streaming/Connection.java

@@ -72,8 +72,8 @@ public class Connection {
     private long _lifetimeDupMessageSent;
     private long _lifetimeDupMessageReceived;
     
-    public static final long MAX_RESEND_DELAY = 20*1000;
+    public static final long MAX_RESEND_DELAY = 10*1000;
-    public static final long MIN_RESEND_DELAY = 10*1000;
+    public static final long MIN_RESEND_DELAY = 3*1000;
 
     /** wait up to 5 minutes after disconnection so we can ack/close packets */
     public static int DISCONNECT_TIMEOUT = 5*60*1000;

apps/streaming/java/src/net/i2p/client/streaming/MessageOutputStream.java

@@ -38,6 +38,10 @@ public class MessageOutputStream extends OutputStream {
      * size
      */
     private volatile int _nextBufferSize;
+    // rate calc helpers
+    private long _sendPeriodBeginTime;
+    private long _sendPeriodBytes;
+    private int _sendBps;
     
     public MessageOutputStream(I2PAppContext ctx, DataReceiver receiver) {
         this(ctx, receiver, Packet.MAX_PAYLOAD_SIZE);
@@ -55,6 +59,10 @@ public class MessageOutputStream extends OutputStream {
         _writeTimeout = -1;
         _passiveFlushDelay = 500;
         _nextBufferSize = -1;
+        _sendPeriodBeginTime = ctx.clock().now();
+        _sendPeriodBytes = 0;
+        _sendBps = 0;
+        _context.statManager().createRateStat("stream.sendBps", "How fast we pump data through the stream", "Stream", new long[] { 60*1000, 5*60*1000, 60*60*1000 });
         _flusher = new Flusher();
         if (_log.shouldLog(Log.DEBUG))
             _log.debug("MessageOutputStream created");
@@ -137,6 +145,21 @@ public class MessageOutputStream extends OutputStream {
         if ( (elapsed > 10*1000) && (_log.shouldLog(Log.DEBUG)) )
             _log.debug("wtf, took " + elapsed + "ms to write to the stream?", new Exception("foo"));
         throwAnyError();
+        updateBps(len);
+    }
+    
+    private void updateBps(int len) {
+        long now = _context.clock().now();
+        int periods = (int)Math.floor((now - _sendPeriodBeginTime) / 1000d);
+        if (periods > 0) {
+            // first term decays on slow transmission
+            _sendBps = (int)(((float)0.9f*((float)_sendBps/(float)periods)) + ((float)0.1f*((float)_sendPeriodBytes/(float)periods)));
+            _sendPeriodBytes = len;
+            _sendPeriodBeginTime = now;
+            _context.statManager().addRateData("stream.sendBps", _sendBps, 0);
+        } else {
+            _sendPeriodBytes += len;
+        }
     }
     
     public void write(int b) throws IOException {

apps/streaming/java/src/net/i2p/client/streaming/PacketHandler.java

@@ -119,6 +119,19 @@ public class PacketHandler {
     }
     
     private void receiveKnownCon(Connection con, Packet packet) {
+        if (packet.isFlagSet(Packet.FLAG_ECHO)) {
+            if (packet.getSendStreamId() != null) {
+                receivePing(packet);
+            } else if (packet.getReceiveStreamId() != null) {
+                receivePong(packet);
+            } else {
+                if (_log.shouldLog(Log.WARN))
+                    _log.warn("Echo packet received with no stream IDs: " + packet);
+            }
+            packet.releasePayload();
+            return;
+        } 
+        
         // the packet is pointed at a stream ID we're receiving on
         if (isValidMatch(con.getSendStreamId(), packet.getReceiveStreamId())) {
             // the packet's receive stream ID also matches what we expect
@@ -163,8 +176,19 @@ public class PacketHandler {
                 } else {
                     if (!con.getResetSent()) {
                         // someone is sending us a packet on the wrong stream 
-                        if (_log.shouldLog(Log.WARN))
+                        if (_log.shouldLog(Log.ERROR)) {
-                            _log.warn("Received a packet on the wrong stream: " + packet + " connection: " + con);
+                            Set cons = _manager.listConnections();
+                            StringBuffer buf = new StringBuffer(512);
+                            buf.append("Received a packet on the wrong stream: ");
+                            buf.append(packet);
+                            buf.append(" connection: ");
+                            buf.append(con);
+                            for (Iterator iter = cons.iterator(); iter.hasNext();) {
+                                Connection cur = (Connection)iter.next();
+                                buf.append(" ").append(cur);
+                            }
+                            _log.error(buf.toString(), new Exception("Wrong stream"));
+                        }
                     }
                     packet.releasePayload();
                 }

apps/susimail/src/src/i2p/susi/webmail/Mail.java

@@ -19,7 +19,7 @@
  *  along with this program; if not, write to the Free Software
  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  *  
- * $Revision: 1.1 $
+ * $Revision: 1.2 $
  */
 package i2p.susi.webmail;
 

apps/susimail/src/src/i2p/susi/webmail/MailCache.java

@@ -19,7 +19,7 @@
  *  along with this program; if not, write to the Free Software
  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  *  
- *  $Revision: 1.1 $
+ *  $Revision: 1.2 $
  */
 package i2p.susi.webmail;
 

apps/systray/java/src/net/i2p/apps/systray/UrlLauncher.java

@@ -92,11 +92,7 @@ public class UrlLauncher {
 
             } else {
 
-                if (_shellCommand.executeSilentAndWaitTimed("konqueror " + url, 5))
+                // fall through
-                    return true;
-
-                if (_shellCommand.executeSilentAndWaitTimed("galeon " + url, 5))
-                    return true;
             }
 
             if (_shellCommand.executeSilentAndWaitTimed("opera -newpage " + url, 5))
@@ -111,11 +107,18 @@ public class UrlLauncher {
             if (_shellCommand.executeSilentAndWaitTimed("netscape " + url, 5))
                 return true;
 
+            if (_shellCommand.executeSilentAndWaitTimed("konqueror " + url, 5))
+                return true;
+
+            if (_shellCommand.executeSilentAndWaitTimed("galeon " + url, 5))
+                return true;
+            
             if (_shellCommand.executeSilentAndWaitTimed("links " + url, 5))
                 return true;
 
             if (_shellCommand.executeSilentAndWaitTimed("lynx " + url, 5))
                 return true;
+            
         }
         return false;
     }

core/java/build.xml

@@ -8,7 +8,12 @@
     <target name="compile">
         <mkdir dir="./build" />
         <mkdir dir="./build/obj" />
-        <javac srcdir="./src:./test" debug="true" source="1.3" target="1.3" deprecation="on" destdir="./build/obj" classpath="lib/junit.jar" />
+        <javac srcdir="./src" debug="true" source="1.3" target="1.3" deprecation="on" destdir="./build/obj" />
+    </target>
+    <target name="compileTest">
+        <mkdir dir="./build" />
+        <mkdir dir="./build/obj" />
+        <javac srcdir="./src:./test" debug="true" source="1.3" target="1.3" deprecation="on" destdir="./build/obj" classpath="./lib/junit.jar" />
     </target>
     <target name="jar" depends="compile">
         <jar destfile="./build/i2p.jar" basedir="./build/obj" includes="**/*.class" />
@@ -18,12 +23,13 @@
         <mkdir dir="./build/javadoc" />
         <javadoc sourcepath="./src:./test" destdir="./build/javadoc" packagenames="*" use="true" splitindex="true" windowtitle="I2P SDK" />
     </target>
-    <target name="test" depends="clean, jar">
+    <target name="test" depends="clean, compileTest">
         <junit printsummary="on" fork="yes">
             <classpath>
                 <pathelement path="${classpath}" />
-                <pathelement location="./build/i2p.jar" />
+                <pathelement location="./build/obj" />
                 <pathelement location="./lib/junit.jar" />
+                <pathelement location="../../installer/lib/jbigi/jbigi.jar" />
                 <pathelement path="${ant.home}/lib/clover.jar"/>
             </classpath>
             <batchtest>

core/java/src/net/i2p/I2PAppContext.java

@@ -10,6 +10,8 @@ import net.i2p.crypto.CryptixAESEngine;
 import net.i2p.crypto.DSAEngine;
 import net.i2p.crypto.DummyDSAEngine;
 import net.i2p.crypto.DummyElGamalEngine;
+import net.i2p.crypto.DummyHMACSHA256Generator;
+import net.i2p.crypto.DummyPooledRandomSource;
 import net.i2p.crypto.ElGamalAESEngine;
 import net.i2p.crypto.ElGamalEngine;
 import net.i2p.crypto.HMACSHA256Generator;
@@ -328,8 +330,12 @@ public class I2PAppContext {
     }
     private void initializeHMAC() {
         synchronized (this) {
-            if (_hmac == null)
+            if (_hmac == null) {
+                if ("true".equals(getProperty("i2p.fakeHMAC", "false")))
+                    _hmac = new DummyHMACSHA256Generator(this);
+                else
                     _hmac= new HMACSHA256Generator(this);
+            }
             _hmacInitialized = true;
         }
     }
@@ -432,8 +438,12 @@ public class I2PAppContext {
     }
     private void initializeRandom() {
         synchronized (this) {
-            if (_random == null)
+            if (_random == null) {
+                if ("true".equals(getProperty("i2p.weakPRNG", "false")))
+                    _random = new DummyPooledRandomSource(this);
+                else
                     _random = new PooledRandomSource(this);
+            }
             _randomInitialized = true;
         }
     }

core/java/src/net/i2p/client/I2CPMessageProducer.java

@@ -39,9 +39,14 @@ import net.i2p.util.Log;
 class I2CPMessageProducer {
     private final static Log _log = new Log(I2CPMessageProducer.class);
     private I2PAppContext _context;
+    private int _sendBps;
+    private long _sendPeriodBytes;
+    private long _sendPeriodBeginTime;
 
     public I2CPMessageProducer(I2PAppContext context) {
         _context = context;
+        _sendBps = 0;
+        context.statManager().createRateStat("client.sendBpsRaw", "How fast we pump out I2CP data messages", "ClientMessages", new long[] { 60*1000, 5*60*1000, 10*60*1000, 60*60*1000 });
     }
     
     /** 
@@ -94,8 +99,30 @@ class I2CPMessageProducer {
         Payload data = createPayload(dest, payload, tag, key, tags, newKey);
         msg.setPayload(data);
         session.sendMessage(msg);
+        updateBps(payload.length);
     }
 
+    private void updateBps(int len) {
+        long now = _context.clock().now();
+        float period = ((float)now-_sendPeriodBeginTime)/1000f;
+        if (period >= 1f) {
+            // first term decays on slow transmission
+            _sendBps = (int)(((float)0.9f * (float)_sendBps) + ((float)0.1f*((float)_sendPeriodBytes)/period));
+            _sendPeriodBytes = len;
+            _sendPeriodBeginTime = now;
+            _context.statManager().addRateData("client.sendBpsRaw", _sendBps, 0);
+        } else {
+            _sendPeriodBytes += len;
+        }
+    }
+    
+    /** 
+     * Should we include the I2CP end to end crypto (which is in addition to any
+     * garlic crypto added by the router)
+     *
+     */
+    static final boolean END_TO_END_CRYPTO = true;
+    
     /**
      * Create a new signed payload and send it off to the destination
      *
@@ -106,6 +133,10 @@ class I2CPMessageProducer {
         if (payload == null) throw new I2PSessionException("No payload specified");
 
         Payload data = new Payload();
+        if (!END_TO_END_CRYPTO) {
+            data.setEncryptedData(payload);
+            return data;
+        }
         // no padding at this level
         // the garlic may pad, and the tunnels may pad, and the transports may pad
         int size = payload.length;

core/java/src/net/i2p/client/I2PSessionImpl2.java

@@ -131,15 +131,15 @@ class I2PSessionImpl2 extends I2PSessionImpl {
         long availTimeLeft = _context.sessionKeyManager().getAvailableTimeLeft(dest.getPublicKey(), key);
         
         if ( (tagsSent == null) || (tagsSent.size() <= 0) ) {
-            if (oldTags < 10) {
+            if (oldTags < NUM_TAGS) {
                 sentTags = createNewTags(NUM_TAGS);
                 if (_log.shouldLog(Log.DEBUG))
-                    _log.debug("** sendBestEffort only had " + oldTags + " with " + availTimeLeft + ", adding " + NUM_TAGS);
+                    _log.debug("** sendBestEffort only had " + oldTags + " with " + availTimeLeft + ", adding " + NUM_TAGS + ": " + sentTags);
             } else if (availTimeLeft < 2 * 60 * 1000) {
-                // if we have > 10 tags, but they expire in under 2 minutes, we want more
+                // if we have > 50 tags, but they expire in under 2 minutes, we want more
                 sentTags = createNewTags(NUM_TAGS);
                 if (_log.shouldLog(Log.DEBUG)) 
-                    _log.debug(getPrefix() + "Tags expiring in " + availTimeLeft + ", adding " + NUM_TAGS + " new ones");
+                    _log.debug(getPrefix() + "Tags expiring in " + availTimeLeft + ", adding " + NUM_TAGS + " new ones: " + sentTags);
                 //_log.error("** sendBestEffort available time left " + availTimeLeft);
             } else {
                 if (_log.shouldLog(Log.DEBUG))

core/java/src/net/i2p/client/MessagePayloadMessageHandler.java

@@ -55,6 +55,11 @@ class MessagePayloadMessageHandler extends HandlerImpl {
      */
     private Payload decryptPayload(MessagePayloadMessage msg, I2PSessionImpl session) throws DataFormatException {
         Payload payload = msg.getPayload();
+        if (!I2CPMessageProducer.END_TO_END_CRYPTO) {
+            payload.setUnencryptedData(payload.getEncryptedData());
+            return payload;
+        }
+            
         byte[] data = _context.elGamalAESEngine().decrypt(payload.getEncryptedData(), session.getDecryptionKey());
         if (data == null) {
             if (_log.shouldLog(Log.WARN))

core/java/src/net/i2p/crypto/AESEngine.java

@@ -159,7 +159,6 @@ public class AESEngine {
         System.arraycopy(payload, inIndex, rv, outIndex, rv.length - outIndex);
     }
     
-    
     public static void main(String args[]) {
         I2PAppContext ctx = new I2PAppContext();
         SessionKey key = ctx.keyGenerator().generateSessionKey();

core/java/src/net/i2p/crypto/DummyHMACSHA256Generator.java

@@ -0,0 +1,52 @@
+package net.i2p.crypto;
+
+import java.util.Arrays;
+import java.util.ArrayList;
+import java.util.List;
+import net.i2p.I2PAppContext;
+import net.i2p.data.DataHelper;
+import net.i2p.data.Hash;
+import net.i2p.data.SessionKey;
+
+/**
+ * Calculate the HMAC-SHA256 of a key+message.  All the good stuff occurs
+ * in {@link org.bouncycastle.crypto.macs.HMac} and 
+ * {@link org.bouncycastle.crypto.digests.SHA256Digest}.
+ *
+ */
+public class DummyHMACSHA256Generator extends HMACSHA256Generator {
+    private I2PAppContext _context;
+    public DummyHMACSHA256Generator(I2PAppContext context) {
+        super(context);
+        _context = context;
+    }
+    
+    public static HMACSHA256Generator getInstance() {
+        return I2PAppContext.getGlobalContext().hmac();
+    }
+    
+    /**
+     * Calculate the HMAC of the data with the given key
+     */
+    public Hash calculate(SessionKey key, byte data[]) {
+        if ((key == null) || (key.getData() == null) || (data == null))
+            throw new NullPointerException("Null arguments for HMAC");
+        return calculate(key, data, 0, data.length);
+    }
+    
+    /**
+     * Calculate the HMAC of the data with the given key
+     */
+    public Hash calculate(SessionKey key, byte data[], int offset, int length) {
+        if ((key == null) || (key.getData() == null) || (data == null))
+            throw new NullPointerException("Null arguments for HMAC");
+        
+        byte rv[] = new byte[Hash.HASH_LENGTH];
+        System.arraycopy(key.getData(), 0, rv, 0, Hash.HASH_LENGTH);
+        if (Hash.HASH_LENGTH >= length)
+            DataHelper.xor(data, offset, rv, 0, rv, 0, length);
+        else
+            DataHelper.xor(data, offset, rv, 0, rv, 0, Hash.HASH_LENGTH);
+        return new Hash(rv);
+    }
+}

core/java/src/net/i2p/crypto/DummyPooledRandomSource.java

@@ -0,0 +1,94 @@
+package net.i2p.crypto;
+
+import java.util.Random;
+import net.i2p.I2PAppContext;
+import net.i2p.util.PooledRandomSource;
+import net.i2p.util.RandomSource;
+
+/**
+ *
+ */
+public class DummyPooledRandomSource extends PooledRandomSource {
+    public DummyPooledRandomSource(I2PAppContext context) {
+        super(context);
+        _pool = new RandomSource[POOL_SIZE];
+        for (int i = 0; i < POOL_SIZE; i++) {
+            _pool[i] = new DummyRandomSource(context);
+            _pool[i].nextBoolean();
+        }
+        _nextPool = 0;
+    }
+    
+    private class DummyRandomSource extends RandomSource {
+        private Random _prng;
+        public DummyRandomSource(I2PAppContext context) {
+            super(context);
+            // when we replace to have hooks for fortuna (etc), replace with
+            // a factory (or just a factory method)
+            _prng = new Random();
+        }
+
+        /**
+         * According to the java docs (http://java.sun.com/j2se/1.4.1/docs/api/java/util/Random.html#nextInt(int))
+         * nextInt(n) should return a number between 0 and n (including 0 and excluding n).  However, their pseudocode,
+         * as well as sun's, kaffe's, and classpath's implementation INCLUDES NEGATIVE VALUES.
+         * WTF.  Ok, so we're going to have it return between 0 and n (including 0, excluding n), since 
+         * thats what it has been used for.
+         *
+         */
+        public int nextInt(int n) {
+            if (n == 0) return 0;
+            int val = _prng.nextInt(n);
+            if (val < 0) val = 0 - val;
+            if (val >= n) val = val % n;
+            return val;
+        }
+
+        /**
+         * Like the modified nextInt, nextLong(n) returns a random number from 0 through n,
+         * including 0, excluding n.
+         */
+        public long nextLong(long n) {
+            long v = _prng.nextLong();
+            if (v < 0) v = 0 - v;
+            if (v >= n) v = v % n;
+            return v;
+        }
+
+        /**
+         * override as synchronized, for those JVMs that don't always pull via
+         * nextBytes (cough ibm)
+         */
+        public boolean nextBoolean() { return _prng.nextBoolean(); }
+        /**
+         * override as synchronized, for those JVMs that don't always pull via
+         * nextBytes (cough ibm)
+         */
+        public void nextBytes(byte buf[]) { _prng.nextBytes(buf); }
+        /**
+         * override as synchronized, for those JVMs that don't always pull via
+         * nextBytes (cough ibm)
+         */
+        public double nextDouble() { return _prng.nextDouble(); }
+        /**
+         * override as synchronized, for those JVMs that don't always pull via
+         * nextBytes (cough ibm)
+         */
+        public float nextFloat() { return _prng.nextFloat(); }
+        /**
+         * override as synchronized, for those JVMs that don't always pull via
+         * nextBytes (cough ibm)
+         */
+        public double nextGaussian() { return _prng.nextGaussian(); }
+        /**
+         * override as synchronized, for those JVMs that don't always pull via
+         * nextBytes (cough ibm)
+         */
+        public int nextInt() { return _prng.nextInt(); }
+        /**
+         * override as synchronized, for those JVMs that don't always pull via
+         * nextBytes (cough ibm)
+         */
+        public long nextLong() { return _prng.nextLong(); }
+    }
+}

core/java/src/net/i2p/crypto/ElGamalAESEngine.java

@@ -16,6 +16,7 @@ import java.util.List;
 import java.util.Set;
 
 import net.i2p.I2PAppContext;
+import net.i2p.data.Base64;
 import net.i2p.data.DataFormatException;
 import net.i2p.data.DataHelper;
 import net.i2p.data.Hash;
@@ -52,7 +53,7 @@ public class ElGamalAESEngine {
         _context.statManager().createFrequencyStat("crypto.elGamalAES.decryptExistingSession",
                                                    "how frequently we decrypt with an existing ElGamal/AES+SessionTag session?",
                                                    "Encryption", new long[] { 60 * 1000l, 60 * 60 * 1000l, 24 * 60 * 60 * 1000l});
-        _context.statManager().createFrequencyStat("crypto.elGamalAES.decryptFail",
+        _context.statManager().createFrequencyStat("crypto.elGamalAES.decryptFailed",
                                                    "how frequently we fail to decrypt with ElGamal/AES+SessionTag?", "Encryption",
                                                    new long[] { 60 * 60 * 1000l, 24 * 60 * 60 * 1000l});
     }
@@ -82,20 +83,35 @@ public class ElGamalAESEngine {
         Set foundTags = new HashSet();
         byte decrypted[] = null;
         if (key != null) {
-            if (_log.shouldLog(Log.DEBUG)) _log.debug("Key is known for tag " + st);
+            //if (_log.shouldLog(Log.DEBUG)) _log.debug("Key is known for tag " + st);
             usedKey.setData(key.getData());
+            long id = _context.random().nextLong();
+            if (_log.shouldLog(Log.DEBUG))
+                _log.debug(id + ": Decrypting existing session encrypted with tag: " + st.toString() + ": key: " + key.toBase64() + ": " + data.length + " bytes: " + Base64.encode(data, 0, 64));
+            
             decrypted = decryptExistingSession(data, key, targetPrivateKey, foundTags, usedKey, foundKey);
-            if (decrypted != null)
+            if (decrypted != null) {
                 _context.statManager().updateFrequency("crypto.elGamalAES.decryptExistingSession");
-            else
+                if ( (foundTags.size() > 0) && (_log.shouldLog(Log.WARN)) )
+                    _log.warn(id + ": ElG/AES decrypt success with " + st + ": found tags: " + foundTags);
+            } else {
                 _context.statManager().updateFrequency("crypto.elGamalAES.decryptFailed");
+                if (_log.shouldLog(Log.ERROR)) {
+                    _log.error(id + ": ElG decrypt fail: known tag [" + st + "], failed decrypt");
+                }
+            }
         } else {
             if (_log.shouldLog(Log.DEBUG)) _log.debug("Key is NOT known for tag " + st);
             decrypted = decryptNewSession(data, targetPrivateKey, foundTags, usedKey, foundKey);
-            if (decrypted != null)
+            if (decrypted != null) {
                 _context.statManager().updateFrequency("crypto.elGamalAES.decryptNewSession");
-            else
+                if ( (foundTags.size() > 0) && (_log.shouldLog(Log.WARN)) )
+                    _log.warn("ElG decrypt success: found tags: " + foundTags);
+            } else {
                 _context.statManager().updateFrequency("crypto.elGamalAES.decryptFailed");
+                if (_log.shouldLog(Log.ERROR))
+                    _log.error("ElG decrypt fail: unknown tag: " + st);
+            }
         }
 
         if ((key == null) && (decrypted == null)) {
@@ -104,10 +120,12 @@ public class ElGamalAESEngine {
 
         if (foundTags.size() > 0) {
             if (foundKey.getData() != null) {
-                if (_log.shouldLog(Log.DEBUG)) _log.debug("Found key: " + foundKey);
+                if (_log.shouldLog(Log.DEBUG)) 
+                    _log.debug("Found key: " + foundKey.toBase64() + " tags: " + foundTags);
                 _context.sessionKeyManager().tagsReceived(foundKey, foundTags);
             } else {
-                if (_log.shouldLog(Log.DEBUG)) _log.debug("Used key: " + usedKey);
+                if (_log.shouldLog(Log.DEBUG)) 
+                    _log.debug("Used key: " + usedKey.toBase64() + " tags: " + foundTags);
                 _context.sessionKeyManager().tagsReceived(usedKey, foundTags);
             }
         }
@@ -131,10 +149,10 @@ public class ElGamalAESEngine {
     byte[] decryptNewSession(byte data[], PrivateKey targetPrivateKey, Set foundTags, SessionKey usedKey,
                                     SessionKey foundKey) throws DataFormatException {
         if (data == null) {
-            if (_log.shouldLog(Log.WARN)) _log.warn("Data is null, unable to decrypt new session");
+            //if (_log.shouldLog(Log.WARN)) _log.warn("Data is null, unable to decrypt new session");
             return null;
         } else if (data.length < 514) {
-            if (_log.shouldLog(Log.WARN)) _log.warn("Data length is too small (" + data.length + ")");
+            //if (_log.shouldLog(Log.WARN)) _log.warn("Data length is too small (" + data.length + ")");
             return null;
         }
         byte elgEncr[] = new byte[514];
@@ -145,8 +163,8 @@ public class ElGamalAESEngine {
         }
         byte elgDecr[] = _context.elGamalEngine().decrypt(elgEncr, targetPrivateKey);
         if (elgDecr == null) {
-            if (_log.shouldLog(Log.WARN))
+            //if (_log.shouldLog(Log.WARN))
-                _log.warn("decrypt returned null", new Exception("decrypt failed"));
+             //   _log.warn("decrypt returned null", new Exception("decrypt failed"));
             return null;
         }
 
@@ -172,9 +190,9 @@ public class ElGamalAESEngine {
 
         byte aesDecr[] = decryptAESBlock(data, 514, data.length-514, usedKey, iv, null, foundTags, foundKey);
 
-        if (_log.shouldLog(Log.DEBUG))
+        //if (_log.shouldLog(Log.DEBUG))
-            _log.debug("Decrypt with a NEW session successfull: # tags read = " + foundTags.size(),
+        //    _log.debug("Decrypt with a NEW session successfull: # tags read = " + foundTags.size(),
-                       new Exception("Decrypted by"));
+        //               new Exception("Decrypted by"));
         return aesDecr;
     }
 
@@ -211,14 +229,23 @@ public class ElGamalAESEngine {
         byte decrypted[] = decryptAESBlock(data, 32, data.length-32, key, iv, preIV, foundTags, foundKey);
         if (decrypted == null) {
             // it begins with a valid session tag, but thats just a coincidence.
-            if (_log.shouldLog(Log.DEBUG))
+            //if (_log.shouldLog(Log.DEBUG))
-                _log.debug("Decrypt with a non session tag, but tags read: " + foundTags.size());
+            //    _log.debug("Decrypt with a non session tag, but tags read: " + foundTags.size());
-            return decryptNewSession(data, targetPrivateKey, foundTags, usedKey, foundKey);
+            if (_log.shouldLog(Log.WARN))
+                _log.warn("Decrypting looks negative... existing key fails with existing tag, lets try as a new one");
+            byte rv[] = decryptNewSession(data, targetPrivateKey, foundTags, usedKey, foundKey);
+            if (_log.shouldLog(Log.WARN)) {
+                if (rv == null)
+                    _log.warn("Decrypting failed with a known existing tag as either an existing message or a new session");
+                else
+                    _log.warn("Decrypting suceeded as a new session, even though it used an existing tag!");
+            }
+            return rv;
         }
         // existing session decrypted successfully!
-        if (_log.shouldLog(Log.DEBUG))
+        //if (_log.shouldLog(Log.DEBUG))
-            _log.debug("Decrypt with an EXISTING session tag successfull, # tags read: " + foundTags.size(),
+        //    _log.debug("Decrypt with an EXISTING session tag successfull, # tags read: " + foundTags.size(),
-                       new Exception("Decrypted by"));
+        //               new Exception("Decrypted by"));
         return decrypted;
     }
 
@@ -261,7 +288,7 @@ public class ElGamalAESEngine {
             long numTags = DataHelper.fromLong(decrypted, cur, 2);
             cur += 2;
             //_log.debug("# tags: " + numTags);
-            if ((numTags < 0) || (numTags > 65535)) throw new Exception("Invalid number of session tags");
+            if ((numTags < 0) || (numTags > 200)) throw new Exception("Invalid number of session tags");
             if (numTags * SessionTag.BYTE_LENGTH > decrypted.length - 2) {
                 throw new Exception("# tags: " + numTags + " is too many for " + (decrypted.length - 2));
             }
@@ -333,7 +360,10 @@ public class ElGamalAESEngine {
         if (_log.shouldLog(Log.INFO))
             _log.info("Current tag is NOT null, encrypting as existing session", new Exception("encrypt existing"));
         _context.statManager().updateFrequency("crypto.elGamalAES.encryptExistingSession");
-        return encryptExistingSession(data, target, key, tagsForDelivery, currentTag, newKey, paddedSize);
+        byte rv[] = encryptExistingSession(data, target, key, tagsForDelivery, currentTag, newKey, paddedSize);
+        if (_log.shouldLog(Log.DEBUG))
+            _log.debug("Existing session encrypted with tag: " + currentTag.toString() + ": " + rv.length + " bytes and key: " + key.toBase64() + ": " + Base64.encode(rv, 0, 64));
+        return rv;
     }
 
     /**
@@ -396,7 +426,7 @@ public class ElGamalAESEngine {
         if (elgEncr.length < 514) {
             byte elg[] = new byte[514];
             int diff = elg.length - elgEncr.length;
-            if (_log.shouldLog(Log.DEBUG)) _log.debug("Difference in size: " + diff);
+            //if (_log.shouldLog(Log.DEBUG)) _log.debug("Difference in size: " + diff);
             System.arraycopy(elgEncr, 0, elg, diff, elgEncr.length);
             elgEncr = elg;
         }
@@ -415,8 +445,8 @@ public class ElGamalAESEngine {
         System.arraycopy(aesEncr, 0, rv, elgEncr.length, aesEncr.length);
         //_log.debug("Return length: " + rv.length);
         long finish = _context.clock().now();
-        if (_log.shouldLog(Log.DEBUG))
+        //if (_log.shouldLog(Log.DEBUG))
-            _log.debug("after the elgEngine.encrypt took a total of " + (finish - after) + "ms");
+        //    _log.debug("after the elgEngine.encrypt took a total of " + (finish - after) + "ms");
         return rv;
     }
 
@@ -493,7 +523,7 @@ public class ElGamalAESEngine {
             System.arraycopy(tag.getData(), 0, aesData, cur, SessionTag.BYTE_LENGTH);
             cur += SessionTag.BYTE_LENGTH;
         }
-        //_log.debug("# tags created, registered, and written: " + tags.size());
+        //_log.debug("# tags created, registered, and written: " + tagsForDelivery.size());
         DataHelper.toLong(aesData, cur, 4, data.length);
         cur += 4;
         //_log.debug("data length: " + data.length);
@@ -519,8 +549,8 @@ public class ElGamalAESEngine {
         System.arraycopy(padding, 0, aesData, cur, padding.length);
         cur += padding.length;
 
-        //Hash h = _context.sha().calculateHash(aesUnencr);
+        //Hash h = _context.sha().calculateHash(data);
-        //_log.debug("Hash of entire aes block before encryption: (len=" + aesUnencr.length + ")\n" + DataHelper.toString(h.getData(), 32));
+        //_log.debug("Hash of entire aes block before encryption: (len=" + data.length + ")\n" + DataHelper.toString(h.getData(), 32));
         _context.aes().encrypt(aesData, prefixBytes, aesData, prefixBytes, key, iv, aesData.length - prefixBytes);
         //_log.debug("Encrypted length: " + aesEncr.length);
         //return aesEncr;

core/java/src/net/i2p/crypto/HMACSHA256Generator.java

@@ -19,10 +19,15 @@ import org.bouncycastle.crypto.macs.HMac;
  */
 public class HMACSHA256Generator {
     private I2PAppContext _context;
+    /** set of available HMAC instances for calculate */
     private List _available;
+    /** set of available byte[] buffers for verify */
+    private List _availableTmp;
+    
     public HMACSHA256Generator(I2PAppContext context) {
         _context = context;
         _available = new ArrayList(32);
+        _availableTmp = new ArrayList(32);
     }
     
     public static HMACSHA256Generator getInstance() {
@@ -54,6 +59,34 @@ public class HMACSHA256Generator {
         return new Hash(rv);
     }
     
+    /**
+     * Verify the MAC inline, reducing some unnecessary memory churn.
+     *
+     * @param key session key to verify the MAC with
+     * @param curData MAC to verify
+     * @param curOffset index into curData to MAC
+     * @param curLength how much data in curData do we want to run the HMAC over
+     * @param origMAC what do we expect the MAC of curData to equal
+     * @param origMACOffset index into origMAC
+     * @param origMACLength how much of the MAC do we want to verify
+     */
+    public boolean verify(SessionKey key, byte curData[], int curOffset, int curLength, byte origMAC[], int origMACOffset, int origMACLength) {
+        if ((key == null) || (key.getData() == null) || (curData == null))
+            throw new NullPointerException("Null arguments for HMAC");
+        
+        HMac mac = acquire();
+        mac.init(key.getData());
+        mac.update(curData, curOffset, curLength);
+        byte rv[] = acquireTmp();
+        //byte rv[] = new byte[Hash.HASH_LENGTH];
+        mac.doFinal(rv, 0);
+        release(mac);
+        
+        boolean eq = DataHelper.eq(rv, 0, origMAC, origMACOffset, origMACLength);
+        releaseTmp(rv);
+        return eq;
+    }
+    
     private HMac acquire() {
         synchronized (_available) {
             if (_available.size() > 0)
@@ -67,4 +100,24 @@ public class HMACSHA256Generator {
                 _available.add(mac);
         }
     }
+
+    // temp buffers for verify(..)
+    private byte[] acquireTmp() {
+        byte rv[] = null;
+        synchronized (_availableTmp) {
+            if (_availableTmp.size() > 0)
+                rv = (byte[])_availableTmp.remove(0);
+        }
+        if (rv != null)
+            Arrays.fill(rv, (byte)0x0);
+        else
+            rv = new byte[Hash.HASH_LENGTH];
+        return rv;
+    }
+    private void releaseTmp(byte tmp[]) {
+        synchronized (_availableTmp) {
+            if (_availableTmp.size() < 64)
+                _availableTmp.add((Object)tmp);
+        }
+    }
 }

core/java/src/net/i2p/crypto/TransientSessionKeyManager.java

@@ -174,7 +174,7 @@ class TransientSessionKeyManager extends SessionKeyManager {
         if (sess.getCurrentKey().equals(key)) {
             SessionTag nxt = sess.consumeNext();
             if (_log.shouldLog(Log.DEBUG))
-                _log.debug("Tag consumed: " + nxt);
+                _log.debug("Tag consumed: " + nxt + " with key: " + key.toBase64());
             return nxt;
         }
         if (_log.shouldLog(Log.DEBUG))
@@ -228,10 +228,10 @@ class TransientSessionKeyManager extends SessionKeyManager {
         sess.setCurrentKey(key);
         TagSet set = new TagSet(sessionTags, key, _context.clock().now());
         sess.addTags(set);
-        if (_log.shouldLog(Log.DEBUG)) {
+        if (_log.shouldLog(Log.WARN)) {
-            _log.debug("Tags delivered to set " + set + " on session " + sess);
+            //_log.debug("Tags delivered to set " + set + " on session " + sess);
             if (sessionTags.size() > 0)
-                _log.debug("Tags delivered: " + sessionTags.size() + " total = " + sess.availableTags());
+                _log.warn("Tags delivered: " + sessionTags.size() + " for key: " + key.toBase64() + ": " + sessionTags);
         }
     }
 
@@ -255,10 +255,27 @@ class TransientSessionKeyManager extends SessionKeyManager {
         for (Iterator iter = sessionTags.iterator(); iter.hasNext();) {
             SessionTag tag = (SessionTag) iter.next();
             if (_log.shouldLog(Log.DEBUG))
-                _log.debug("Receiving tag " + tag + " for key " + key);
+                _log.debug("Receiving tag " + tag + " for key " + key.toBase64() + " / " + key.toString() + ": tagSet: " + tagSet);
             synchronized (_inboundTagSets) {
-                _inboundTagSets.put(tag, tagSet);
+                Object old = _inboundTagSets.put(tag, tagSet);
                 overage = _inboundTagSets.size() - MAX_INBOUND_SESSION_TAGS;
+                if (old != null) {
+                    TagSet oldTS = (TagSet)old;
+                    if (!oldTS.getAssociatedKey().equals(tagSet.getAssociatedKey())) {
+                        if (_log.shouldLog(Log.ERROR)) {
+                            _log.error("Multiple tags matching!  tag: " + tag.toString() + " matches for new tagSet: " + tagSet + " and old tagSet: " + old);
+                            _log.error("Earlier tag set creation: " + old + ": key=" + oldTS.getAssociatedKey().toBase64(), oldTS.getCreatedBy());
+                            _log.error("Current tag set creation: " + tagSet + ": key=" + tagSet.getAssociatedKey().toBase64(), tagSet.getCreatedBy());
+                        }
+                    } else {
+                        if (_log.shouldLog(Log.DEBUG)) {
+                            //tagSet.getTags().addAll(oldTS.getTags());
+                            _log.debug("Multiple tags matching, but equal keys (probably just a retransmission). tag: " + tag.toString() + " matches for new tagSet: " + tagSet + " and old tagSet: " + old);
+                            _log.debug("Earlier tag set creation: " + old + ": key=" + oldTS.getAssociatedKey().toBase64(), oldTS.getCreatedBy());
+                            _log.debug("Current tag set creation: " + tagSet + ": key=" + tagSet.getAssociatedKey().toBase64(), tagSet.getCreatedBy());
+                        }
+                    }
+                }
             }
         }
         
@@ -267,6 +284,7 @@ class TransientSessionKeyManager extends SessionKeyManager {
 
         if ( (sessionTags.size() <= 0) && (_log.shouldLog(Log.DEBUG)) )
             _log.debug("Received 0 tags for key " + key);
+        if (false) aggressiveExpire();
     }
     
     /**
@@ -329,6 +347,7 @@ class TransientSessionKeyManager extends SessionKeyManager {
      *
      */
     public SessionKey consumeTag(SessionTag tag) {
+        if (false) aggressiveExpire();
         synchronized (_inboundTagSets) {
             TagSet tagSet = (TagSet) _inboundTagSets.remove(tag);
             if (tagSet == null) {
@@ -340,7 +359,7 @@ class TransientSessionKeyManager extends SessionKeyManager {
 
             SessionKey key = tagSet.getAssociatedKey();
             if (_log.shouldLog(Log.DEBUG))
-                _log.debug("Consuming tag " + tag + " for sessionKey " + key);
+                _log.debug("Consuming tag " + tag.toString() + " for sessionKey " + key.toBase64() + " / " + key.toString() + " on tagSet: " + tagSet);
             return key;
         }
     }
@@ -378,19 +397,36 @@ class TransientSessionKeyManager extends SessionKeyManager {
         int removed = 0;
         int remaining = 0;
         long now = _context.clock().now();
+        StringBuffer buf = null;
+        StringBuffer bufSummary = null;
+        if (_log.shouldLog(Log.WARN)) {
+            buf = new StringBuffer(128);
+            buf.append("Expiring inbound: ");
+            bufSummary = new StringBuffer(1024);
+        }
         synchronized (_inboundTagSets) {
             for (Iterator iter = _inboundTagSets.keySet().iterator(); iter.hasNext();) {
                 SessionTag tag = (SessionTag) iter.next();
                 TagSet ts = (TagSet) _inboundTagSets.get(tag);
-                if (ts.getDate() < now - SESSION_LIFETIME_MAX_MS) {
+                long age = now - ts.getDate();
+                if (age > SESSION_LIFETIME_MAX_MS) {
+                //if (ts.getDate() < now - SESSION_LIFETIME_MAX_MS) {
                     iter.remove();
                     removed++;
+                    if (buf != null)
+                        buf.append(tag.toString()).append(" @ age ").append(DataHelper.formatDuration(age));
+                } else if (false && (bufSummary != null) ) {
+                    bufSummary.append("\nTagSet: " + ts.toString() + ", key: " + ts.getAssociatedKey().toBase64()+"/" + ts.getAssociatedKey().toString() 
+                                      + ": tag: " + tag.toString());
                 }
             }
             remaining = _inboundTagSets.size();
         }
         _context.statManager().addRateData("crypto.sessionTagsRemaining", remaining, 0);
-
+        if ( (buf != null) && (removed > 0) )
+            _log.warn(buf.toString());
+        if (bufSummary != null)
+            _log.warn("Cleaning up with remaining: " + bufSummary.toString());
 
         //_log.warn("Expiring tags: [" + tagsToDrop + "]");
 
@@ -606,6 +642,7 @@ class TransientSessionKeyManager extends SessionKeyManager {
         private Set _sessionTags;
         private SessionKey _key;
         private long _date;
+        private Exception _createdBy;
 
         public TagSet(Set tags, SessionKey key, long date) {
             if (key == null) throw new IllegalArgumentException("Missing key");
@@ -613,6 +650,10 @@ class TransientSessionKeyManager extends SessionKeyManager {
             _sessionTags = tags;
             _key = key;
             _date = date;
+            if (true) 
+                _createdBy = new Exception("Created by: key=" + _key.toBase64() + " on " 
+                                           + new Date(I2PAppContext.getGlobalContext().clock().now()) 
+                                           + " via " + Thread.currentThread().getName());
         }
 
         /** when the tag set was created */
@@ -653,6 +694,8 @@ class TransientSessionKeyManager extends SessionKeyManager {
             return first;
         }
         
+        public Exception getCreatedBy() { return _createdBy; }
+
         public int hashCode() {
             long rv = 0;
             if (_key != null) rv = rv * 7 + _key.hashCode();

core/java/src/net/i2p/data/Base64.java

@@ -84,6 +84,17 @@ public class Base64 {
                                             (byte) 'w', (byte) 'x', (byte) 'y', (byte) 'z', (byte) '0', (byte) '1',
                                             (byte) '2', (byte) '3', (byte) '4', (byte) '5', (byte) '6', (byte) '7',
                                             (byte) '8', (byte) '9', (byte) '+', (byte) '/'};
+    private final static byte[] ALPHABET_ALT = { (byte) 'A', (byte) 'B', (byte) 'C', (byte) 'D', (byte) 'E', (byte) 'F',
+                                            (byte) 'G', (byte) 'H', (byte) 'I', (byte) 'J', (byte) 'K', (byte) 'L',
+                                            (byte) 'M', (byte) 'N', (byte) 'O', (byte) 'P', (byte) 'Q', (byte) 'R',
+                                            (byte) 'S', (byte) 'T', (byte) 'U', (byte) 'V', (byte) 'W', (byte) 'X',
+                                            (byte) 'Y', (byte) 'Z', (byte) 'a', (byte) 'b', (byte) 'c', (byte) 'd',
+                                            (byte) 'e', (byte) 'f', (byte) 'g', (byte) 'h', (byte) 'i', (byte) 'j',
+                                            (byte) 'k', (byte) 'l', (byte) 'm', (byte) 'n', (byte) 'o', (byte) 'p',
+                                            (byte) 'q', (byte) 'r', (byte) 's', (byte) 't', (byte) 'u', (byte) 'v',
+                                            (byte) 'w', (byte) 'x', (byte) 'y', (byte) 'z', (byte) '0', (byte) '1',
+                                            (byte) '2', (byte) '3', (byte) '4', (byte) '5', (byte) '6', (byte) '7',
+                                            (byte) '8', (byte) '9', (byte) '-', (byte) '~'};
 
     /** 
      * Translates a Base64 value to either its 6-bit reconstruction value
@@ -131,6 +142,7 @@ public class Base64 {
     }
 
     public static void main(String[] args) {
+        test();
         if (args.length == 0) {
             help();
             return;
@@ -312,6 +324,49 @@ public class Base64 {
         } // end switch
     } // end encode3to4
 
+    private static void encode3to4(byte[] source, int srcOffset, int numSigBytes, StringBuffer buf, byte alpha[]) {
+        //           1         2         3  
+        // 01234567890123456789012345678901 Bit position
+        // --------000000001111111122222222 Array position from threeBytes
+        // --------|    ||    ||    ||    | Six bit groups to index ALPHABET
+        //          >>18  >>12  >> 6  >> 0  Right shift necessary
+        //                0x3f  0x3f  0x3f  Additional AND
+
+        // Create buffer with zero-padding if there are only one or two
+        // significant bytes passed in the array.
+        // We have to shift left 24 in order to flush out the 1's that appear
+        // when Java treats a value as negative that is cast from a byte to an int.
+        int inBuff = (numSigBytes > 0 ? ((source[srcOffset] << 24) >>> 8) : 0)
+                     | (numSigBytes > 1 ? ((source[srcOffset + 1] << 24) >>> 16) : 0)
+                     | (numSigBytes > 2 ? ((source[srcOffset + 2] << 24) >>> 24) : 0);
+
+        switch (numSigBytes) {
+        case 3:
+            buf.append((char)alpha[(inBuff >>> 18)]);
+            buf.append((char)alpha[(inBuff >>> 12) & 0x3f]);
+            buf.append((char)alpha[(inBuff >>> 6) & 0x3f]);
+            buf.append((char)alpha[(inBuff) & 0x3f]);
+            return;
+
+        case 2:
+            buf.append((char)alpha[(inBuff >>> 18)]);
+            buf.append((char)alpha[(inBuff >>> 12) & 0x3f]);
+            buf.append((char)alpha[(inBuff >>> 6) & 0x3f]);
+            buf.append((char)EQUALS_SIGN);
+            return;
+
+        case 1:
+            buf.append((char)alpha[(inBuff >>> 18)]);
+            buf.append((char)alpha[(inBuff >>> 12) & 0x3f]);
+            buf.append((char)EQUALS_SIGN);
+            buf.append((char)EQUALS_SIGN);
+            return;
+
+        default:
+            return;
+        } // end switch
+    } // end encode3to4
+
     /**
      * Encodes a byte array into Base64 notation.
      * Equivalen to calling
@@ -331,14 +386,12 @@ public class Base64 {
     private static String safeEncode(byte[] source, int off, int len, boolean useStandardAlphabet) {
         if (len + off > source.length)
             throw new ArrayIndexOutOfBoundsException("Trying to encode too much!  source.len=" + source.length + " off=" + off + " len=" + len);
-        String encoded = encodeBytes(source, off, len, false);
+        StringBuffer buf = new StringBuffer(len * 4 / 3);
-        if (useStandardAlphabet) {
+        if (useStandardAlphabet)
-            // noop
+            encodeBytes(source, off, len, false, buf, ALPHABET);
-        } else {
+        else
-            encoded = encoded.replace('/', '~');
+            encodeBytes(source, off, len, false, buf, ALPHABET_ALT);
-            encoded = encoded.replace('+', '-');
+        return buf.toString();
-        }
-        return encoded;
     }
 
     /**
@@ -381,6 +434,12 @@ public class Base64 {
         return encodeBytes(source, off, len, true);
     } // end encodeBytes
 
+    private static String encodeBytes(byte[] source, int off, int len, boolean breakLines) {
+        StringBuffer buf = new StringBuffer( (len*4)/3 );
+        encodeBytes(source, off, len, breakLines, buf, ALPHABET);
+        return buf.toString();
+    }
+        
     /**
      * Encodes a byte array into Base64 notation.
      *
@@ -390,32 +449,36 @@ public class Base64 {
      * @param breakLines Break lines at 80 characters or less.
      * @since 1.4
      */
-    private static String encodeBytes(byte[] source, int off, int len, boolean breakLines) {
+    private static void encodeBytes(byte[] source, int off, int len, boolean breakLines, StringBuffer out, byte alpha[]) {
         int len43 = len * 4 / 3;
-        byte[] outBuff = new byte[(len43) // Main 4:3
+        //byte[] outBuff = new byte[(len43) // Main 4:3
-                                  + ((len % 3) > 0 ? 4 : 0) // Account for padding
+        //                          + ((len % 3) > 0 ? 4 : 0) // Account for padding
-                                  + (breakLines ? (len43 / MAX_LINE_LENGTH) : 0)]; // New lines      
+        //                          + (breakLines ? (len43 / MAX_LINE_LENGTH) : 0)]; // New lines      
         int d = 0;
         int e = 0;
         int len2 = len - 2;
         int lineLength = 0;
         for (; d < len2; d += 3, e += 4) {
-            encode3to4(source, d + off, 3, outBuff, e);
+            //encode3to4(source, d + off, 3, outBuff, e);
+            encode3to4(source, d + off, 3, out, alpha);
 
             lineLength += 4;
             if (breakLines && lineLength == MAX_LINE_LENGTH) {
-                outBuff[e + 4] = NEW_LINE;
+                //outBuff[e + 4] = NEW_LINE;
+                out.append('\n');
                 e++;
                 lineLength = 0;
             } // end if: end of line
         } // en dfor: each piece of array
 
         if (d < len) {
-            encode3to4(source, d + off, len - d, outBuff, e);
+            //encode3to4(source, d + off, len - d, outBuff, e);
+            encode3to4(source, d + off, len - d, out, alpha);
             e += 4;
         } // end if: some padding needed
 
-        return new String(outBuff, 0, e);
+        //out.append(new String(outBuff, 0, e));
+        //return new String(outBuff, 0, e);
     } // end encodeBytes
 
     /**

core/java/src/net/i2p/data/ByteArray.java

@@ -56,19 +56,20 @@ public class ByteArray implements Serializable, Comparable {
     public final boolean equals(Object o) {
         if (o == null) return false;
         if (o instanceof ByteArray) {
-            return compare(getData(), ((ByteArray) o).getData());
+            ByteArray ba = (ByteArray)o;
+            return compare(getData(), _offset, _valid, ba.getData(), ba.getOffset(), ba.getValid());
         }
 
         try {
             byte val[] = (byte[]) o;
-            return compare(getData(), val);
+            return compare(getData(), _offset, _valid, val, 0, val.length);
         } catch (Throwable t) {
             return false;
         }
     }
 
-    private static final boolean compare(byte[] lhs, byte[] rhs) {
+    private static final boolean compare(byte[] lhs, int loff, int llen, byte[] rhs, int roff, int rlen) {
-        return DataHelper.eq(lhs, rhs);
+        return (llen == rlen) && DataHelper.eq(lhs, loff, rhs, roff, llen);
     }
     
     public final int compareTo(Object obj) {

core/java/src/net/i2p/data/DataHelper.java

@@ -721,7 +721,7 @@ public class DataHelper {
     public static int hashCode(byte b[]) {
         int rv = 0;
         if (b != null) {
-            for (int i = 0; i < b.length && i < 8; i++)
+            for (int i = 0; i < b.length && i < 32; i++)
                 rv += (b[i] << i);
         }
         return rv;

core/java/src/net/i2p/data/SessionKey.java

@@ -79,6 +79,7 @@ public class SessionKey extends DataStructureImpl {
     }
 
     public String toString() { 
+        if (true) return super.toString(); 
         StringBuffer buf = new StringBuffer(64);
         buf.append("[SessionKey: ");
         if (_data == null) {

core/java/src/net/i2p/stat/BufferedStatLog.java

@@ -30,11 +30,13 @@ public class BufferedStatLog implements StatLog {
     private BufferedWriter _out;
     private String _outFile;
     
+    private static final int BUFFER_SIZE = 1024;
+    
     public BufferedStatLog(I2PAppContext ctx) {
         _context = ctx;
         _log = ctx.logManager().getLog(BufferedStatLog.class);
-        _events = new StatEvent[1000];
+        _events = new StatEvent[BUFFER_SIZE];
-        for (int i = 0; i < 1000; i++)
+        for (int i = 0; i < BUFFER_SIZE; i++)
             _events[i] = new StatEvent();
         _eventNext = 0;
         _lastWrite = _events.length-1;
@@ -73,7 +75,7 @@ public class BufferedStatLog implements StatLog {
     }
     
     private void updateFilters() {
-        String val = _context.getProperty("stat.logFilters");
+        String val = _context.getProperty(StatManager.PROP_STAT_FILTER);
         if (val != null) {
             if ( (_lastFilters != null) && (_lastFilters.equals(val)) ) {
                 // noop
@@ -90,9 +92,9 @@ public class BufferedStatLog implements StatLog {
             synchronized (_statFilters) { _statFilters.clear(); }
         }
         
-        String filename = _context.getProperty("stat.logFile");
+        String filename = _context.getProperty(StatManager.PROP_STAT_FILE);
         if (filename == null)
-            filename = "stats.log";
+            filename = StatManager.DEFAULT_STAT_FILE;
         if ( (_outFile != null) && (_outFile.equals(filename)) ) {
             // noop
         } else {

core/java/src/net/i2p/stat/StatManager.java

@@ -29,6 +29,10 @@ public class StatManager {
     private Map _rateStats;
     private StatLog _statLog;
 
+    public static final String PROP_STAT_FILTER = "stat.logFilters";
+    public static final String PROP_STAT_FILE = "stat.logFile";
+    public static final String DEFAULT_STAT_FILE = "stats.log";
+    
     /**
      * The stat manager should only be constructed and accessed through the 
      * application context.  This constructor should only be used by the 
@@ -139,7 +143,7 @@ public class StatManager {
         return _frequencyStats.containsKey(statName);
     }
 
-    /** Group name (String) to a Set of stat names */
+    /** Group name (String) to a Set of stat names, ordered alphabetically */
     public Map getStatsByGroup() {
         Map groups = new TreeMap();
         for (Iterator iter = _frequencyStats.values().iterator(); iter.hasNext();) {
@@ -156,4 +160,7 @@ public class StatManager {
         }
         return groups;
     }
+
+    public String getStatFilter() { return _context.getProperty(PROP_STAT_FILTER); }
+    public String getStatFile() { return _context.getProperty(PROP_STAT_FILE, DEFAULT_STAT_FILE); }
 }

core/java/src/net/i2p/util/Clock.java

@@ -59,6 +59,7 @@ public class Clock implements Timestamper.UpdateListener {
      *
      */
     public void setOffset(long offsetMs, boolean force) {
+        if (false) return;
         long delta = offsetMs - _offset;
         if (!force) {
             if ((offsetMs > MAX_OFFSET) || (offsetMs < 0 - MAX_OFFSET)) {

core/java/src/net/i2p/util/DecayingBloomFilter.java

@@ -29,6 +29,8 @@ public class DecayingBloomFilter {
     private boolean _keepDecaying;
     private DecayEvent _decayEvent;
     
+    private static final boolean ALWAYS_MISS = false;
+   
     /**
      * Create a bloom filter that will decay its entries over time.  
      *
@@ -41,8 +43,8 @@ public class DecayingBloomFilter {
         _context = context;
         _log = context.logManager().getLog(DecayingBloomFilter.class);
         _entryBytes = entryBytes;
-        _current = new BloomSHA1(23, 11);
+        _current = new BloomSHA1(23, 11); //new BloomSHA1(23, 11);
-        _previous = new BloomSHA1(23, 11);
+        _previous = new BloomSHA1(23, 11); //new BloomSHA1(23, 11);
         _durationMs = durationMs;
         int numExtenders = (32+ (entryBytes-1))/entryBytes - 1;
         if (numExtenders < 0)
@@ -78,6 +80,7 @@ public class DecayingBloomFilter {
      *
      */
     public boolean add(byte entry[]) {
+        if (ALWAYS_MISS) return false;
         if (entry == null) 
             throw new IllegalArgumentException("Null entry");
         if (entry.length != _entryBytes) 
@@ -95,6 +98,7 @@ public class DecayingBloomFilter {
      *
      */
     public boolean add(long entry) {
+        if (ALWAYS_MISS) return false;
         synchronized (this) {
             if (_entryBytes <= 7)
                 entry &= _longToEntryMask; 
@@ -114,6 +118,7 @@ public class DecayingBloomFilter {
      *
      */
     public boolean isKnown(long entry) {
+        if (ALWAYS_MISS) return false;
         synchronized (this) {
             if (_entryBytes <= 7)
                 entry &= _longToEntryMask; 

core/java/src/net/i2p/util/PooledRandomSource.java

@@ -17,10 +17,10 @@ import net.i2p.crypto.EntropyHarvester;
  */
 public class PooledRandomSource extends RandomSource {
     private Log _log;
-    private RandomSource _pool[];
+    protected RandomSource _pool[];
-    private volatile int _nextPool;
+    protected volatile int _nextPool;
 
-    private static final int POOL_SIZE = 16;
+    public static final int POOL_SIZE = 16;
     
     public PooledRandomSource(I2PAppContext context) {
         super(context);
@@ -34,7 +34,7 @@ public class PooledRandomSource extends RandomSource {
     }
 
     private final RandomSource pickPRNG() {
-        int i = _nextPool;
+        int i = _nextPool % POOL_SIZE;
         _nextPool = (++_nextPool) % POOL_SIZE;
         return _pool[i];
     }
@@ -140,4 +140,17 @@ public class PooledRandomSource extends RandomSource {
         RandomSource prng = pickPRNG();
         return prng.harvester();
     }
+    
+    public static void main(String args[]) {
+        PooledRandomSource prng = new PooledRandomSource(I2PAppContext.getGlobalContext());
+        int size = 8*1024*1024;
+        try {
+            java.io.FileOutputStream out = new java.io.FileOutputStream("random.file");
+            for (int i = 0; i < size; i++) {
+                out.write(prng.nextInt());
+            }
+            out.close();
+        } catch (Exception e) { e.printStackTrace(); }
+        System.out.println("Written to random.file");
+    }    
 }

core/java/src/org/bouncycastle/crypto/macs/HMac.java

@@ -31,13 +31,17 @@ import org.bouncycastle.crypto.Digest;
 import org.bouncycastle.crypto.Mac;
 //import org.bouncycastle.crypto.params.KeyParameter;
 import java.util.Arrays;
+import java.util.ArrayList;
 
 /**
  * HMAC implementation based on RFC2104
  *
  * H(K XOR opad, H(K XOR ipad, text))
  *
- * modified by jrandom to use the session key byte array directly
+ * modified by jrandom to use the session key byte array directly and to cache
+ * a frequently used buffer (called on doFinal).  changes released into the public
+ * domain in 2005.
+ *
  */
 public class HMac
 implements Mac
@@ -137,11 +141,13 @@ implements Mac
         byte[] out,
         int outOff)
     {
-        byte[] tmp = new byte[digestSize];
+        byte[] tmp = acquireTmp();
+        //byte[] tmp = new byte[digestSize];
         digest.doFinal(tmp, 0);
 
         digest.update(outputPad, 0, outputPad.length);
         digest.update(tmp, 0, tmp.length);
+        releaseTmp(tmp);
 
         int     len = digest.doFinal(out, outOff);
 
@@ -150,6 +156,26 @@ implements Mac
         return len;
     }
     
+    private static ArrayList _tmpBuf = new ArrayList();
+    private static byte[] acquireTmp() {
+        byte rv[] = null;
+        synchronized (_tmpBuf) {
+            if (_tmpBuf.size() > 0)
+                rv = (byte[])_tmpBuf.remove(0);
+        }
+        if (rv != null)
+            Arrays.fill(rv, (byte)0x0);
+        else
+            rv = new byte[32]; // hard coded against SHA256 (should be digestSize)
+        return rv;
+    }
+    private static void releaseTmp(byte buf[]) {
+        synchronized (_tmpBuf) {
+            if (_tmpBuf.size() < 100) 
+                _tmpBuf.add((Object)buf);
+        }
+    }
+
     /**
      * Reset the mac generator.
      */

core/java/src/org/xlattice/crypto/filters/BloomSHA1.java

@@ -47,6 +47,16 @@ public class BloomSHA1 {
     protected final int filterBits;
     protected final int filterWords;
     
+    public static void main(String args[]) {
+        BloomSHA1 b = new BloomSHA1(24, 11);
+        for (int i = 0; i < 100; i++) {
+            byte v[] = new byte[32];
+            v[0] = (byte)i;
+            b.insert(v);
+        }
+    }
+    
+    
     /**
      * Creates a filter with 2^m bits and k 'hash functions', where
      * each hash function is portion of the 160-bit SHA1 hash.   

core/java/test/net/i2p/crypto/HMACSHA256Bench.java

@@ -32,12 +32,19 @@ package net.i2p.crypto;
 import net.i2p.I2PAppContext;
 import net.i2p.data.Hash;
 import net.i2p.data.SessionKey;
+import java.util.Properties;
 
 public class HMACSHA256Bench {
         public static void main(String args[]) {
-        I2PAppContext ctx = I2PAppContext.getGlobalContext();
+            runTest(new I2PAppContext());
+            Properties props = new Properties();
+            props.setProperty("i2p.fakeHMAC", "true");
+            runTest(new I2PAppContext(props));
+
+        }
+        private static void runTest(I2PAppContext ctx) {
         SessionKey key = ctx.keyGenerator().generateSessionKey();
-		Hash asdfs = HMACSHA256Generator.getInstance().calculate(key, "qwerty".getBytes());
+		Hash asdfs = ctx.hmac().calculate(key, "qwerty".getBytes());
 			
 		int times = 100000;
 		long shorttime = 0;

history.txt

@@ -1,4 +1,22 @@
-$Id: history.txt,v 1.205 2005/05/01 21:35:16 jrandom Exp $
+$Id: history.txt,v 1.206 2005/05/25 16:32:38 duck Exp $
+
+2005-07-04  jrandom
+    * Within the tunnel, use xor(IV, msg[0:16]) as the flag to detect dups, 
+      rather than the IV by itself, preventing an attack that would let 
+      colluding internal adversaries tag a message to determine that they are
+      in the same tunnel.  Thanks dvorak for the catch!
+    * Drop long inactive profiles on startup and shutdown
+    * /configstats.jsp: web interface to pick what stats to log
+    * Deliver more session tags to account for wider window sizes
+    * Cache some intermediate values in our HMACSHA256 and BC's HMAC
+    * Track the client send rate (stream.sendBps and client.sendBpsRaw)
+    * UrlLauncher: adjust the browser selection order
+    * I2PAppContext: hooks for dummy HMACSHA256 and a weak PRNG
+    * StreamSinkClient: add support for sending an unlimited amount of data
+    * Migrate the tests out of the default build jars
+
+2005-06-22  Comwiz
+    * Migrate the core tests to junit
 
 2005-05-25  duck
     * Fixed PRNG bug (bugzilla #107)

router/java/src/net/i2p/data/i2np/I2NPMessage.java

@@ -20,6 +20,7 @@ import net.i2p.data.DataStructure;
  */
 public interface I2NPMessage extends DataStructure {
     final long MAX_ID_VALUE = (1l<<32l)-1l;
+    final int MAX_SIZE = 64*1024; // insane
     
     /**
      * Read the body into the data structures, after the initial type byte, using

router/java/src/net/i2p/data/i2np/I2NPMessageImpl.java

@@ -66,7 +66,7 @@ public abstract class I2NPMessageImpl extends DataStructureImpl implements I2NPM
             //Hash h = new Hash();
             //h.readBytes(in);
             if (buffer.length < size) {
-                if (size > 64*1024) throw new I2NPMessageException("size=" + size);
+                if (size > MAX_SIZE) throw new I2NPMessageException("size=" + size);
                 buffer = new byte[size];
             }
             

router/java/src/net/i2p/router/InNetMessagePool.java

@@ -217,7 +217,8 @@ public class InNetMessagePool implements Service {
                         if (_log.shouldLog(Log.WARN))
                             _log.warn("Message expiring on " 
                                       + (messageBody != null ? (messageBody.getMessageExpiration()+"") : " [unknown]")
-                                      + " was not handled by a HandlerJobBuilder - DROPPING: " + messageBody);
+                                      + " was not handled by a HandlerJobBuilder - DROPPING: " + messageBody, 
+                                      new Exception("f00!"));
                         _context.statManager().addRateData("inNetPool.dropped", 1, 0);
                     }
                 } else {

router/java/src/net/i2p/router/JobQueue.java

@@ -401,7 +401,7 @@ public class JobQueue {
             try {
                 while (_alive) {
                     long now = _context.clock().now();
-                    long timeToWait = 0;
+                    long timeToWait = -1;
                     ArrayList toAdd = null;
                     synchronized (_jobLock) {
                         for (int i = 0; i < _timedJobs.size(); i++) {
@@ -434,9 +434,11 @@ public class JobQueue {
                                 _readyJobs.add(toAdd.get(i));
                             _jobLock.notifyAll();
                         } else {
-                            if (timeToWait < 10)
+                            if (timeToWait < 0)
+                                timeToWait = 30*1000;
+                            else if (timeToWait < 10)
                                 timeToWait = 10;
-                            if (timeToWait > 10*1000)
+                            else if (timeToWait > 10*1000)
                                 timeToWait = 10*1000;
                             if (_log.shouldLog(Log.DEBUG))
                                 _log.debug("Waiting " + timeToWait + " before rechecking the timed queue");

router/java/src/net/i2p/router/MessageHistory.java

@@ -206,6 +206,19 @@ public class MessageHistory {
         addEntry(getPrefix() + "tunnel dispatched: " + info);
     }
     
+    public void tunnelDispatched(long messageId, long tunnelId, String type) {
+        if (!_doLog) return;
+        addEntry(getPrefix() + "message " + messageId + " on tunnel " + tunnelId + " as " + type);
+    }
+    
+    public void tunnelDispatched(long messageId, long tunnelId, long toTunnel, Hash toPeer, String type) {
+        if (!_doLog) return;
+        if (toPeer != null)
+            addEntry(getPrefix() + "message " + messageId + " on tunnel " + tunnelId + " / " + toTunnel + " to " + toPeer.toBase64() + " as " + type);
+        else
+            addEntry(getPrefix() + "message " + messageId + " on tunnel " + tunnelId + " / " + toTunnel + " as " + type);
+    }
+    
     /**
      * The local router has detected a failure in the given tunnel
      *

router/java/src/net/i2p/router/Router.java

@@ -414,14 +414,12 @@ public class Router {
             _cal.set(Calendar.SECOND, 0);
             _cal.set(Calendar.MILLISECOND, 0);
             long then = _cal.getTime().getTime();
-            _log.debug("Time till midnight: " + (then-now) + "ms");
+            long howLong = then - now;
-            if (then - now <= 60*1000) {
+            if (howLong < 0) // hi kaffe
-                // everyone wave at kaffe.
+                howLong = 24*60*60*1000 + howLong;
-                // "Hi Kaffe"
+            if (_log.shouldLog(Log.DEBUG))
-                return 60*1000;
+                _log.debug("Time till midnight: " + howLong + "ms");
-            } else {
+            return howLong;
-                return then - now;
-            }
         }
     }
     
@@ -838,13 +836,15 @@ public class Router {
         FileOutputStream fos = null;
         try {
             fos = new FileOutputStream(_configFilename);
-            TreeSet ordered = new TreeSet(_config.keySet());
             StringBuffer buf = new StringBuffer(8*1024);
+            synchronized (_config) {
+                TreeSet ordered = new TreeSet(_config.keySet());
                 for (Iterator iter = ordered.iterator() ; iter.hasNext(); ) {
                     String key = (String)iter.next();
                     String val = _config.getProperty(key);
                     buf.append(key).append('=').append(val).append('\n');
                 }
+            }
             fos.write(buf.toString().getBytes());
         } catch (IOException ioe) {
             if (_log.shouldLog(Log.ERROR))

router/java/src/net/i2p/router/RouterVersion.java

@@ -15,9 +15,9 @@ import net.i2p.CoreVersion;
  *
  */
 public class RouterVersion {
-    public final static String ID = "$Revision: 1.196 $ $Date: 2005/05/01 21:35:16 $";
+    public final static String ID = "$Revision: 1.197 $ $Date: 2005/05/25 16:32:38 $";
     public final static String VERSION = "0.5.0.7";
-    public final static long BUILD = 8;
+    public final static long BUILD = 9;
     public static void main(String args[]) {
         System.out.println("I2P Router version: " + VERSION);
         System.out.println("Router ID: " + RouterVersion.ID);

router/java/src/net/i2p/router/client/ClientListenerRunner.java

@@ -61,11 +61,21 @@ public class ClientListenerRunner implements Runnable {
         int curDelay = 0;
         while (_running) {
             try {
-                _log.info("Starting up listening for connections on port " + _port);
+                if (_bindAllInterfaces) {
-                if (_bindAllInterfaces)
+                    if (_log.shouldLog(Log.INFO))
+                        _log.info("Listening on port " + _port + " on all interfaces");
                     _socket = new ServerSocket(_port);
-                else
+                } else {
-                    _socket = new ServerSocket(_port, 0, InetAddress.getByName("127.0.0.1"));
+                    String listenInterface = _context.getProperty(ClientManagerFacadeImpl.PROP_CLIENT_HOST, 
+                                                                  ClientManagerFacadeImpl.DEFAULT_HOST);
+                    if (_log.shouldLog(Log.INFO))
+                        _log.info("Listening on port " + _port + " of the specific interface: " + listenInterface);
+                    _socket = new ServerSocket(_port, 0, InetAddress.getByName(listenInterface));
+                }
+                
+                
+                if (_log.shouldLog(Log.DEBUG))
+                    _log.debug("ServerSocket created, before accept: " + _socket);
                 
                 curDelay = 0;
                 while (_running) {

router/java/src/net/i2p/router/client/ClientManagerFacadeImpl.java

@@ -37,6 +37,8 @@ public class ClientManagerFacadeImpl extends ClientManagerFacade {
     private RouterContext _context;
     public final static String PROP_CLIENT_PORT = "i2cp.port";
     public final static int DEFAULT_PORT = 7654;
+    public final static String PROP_CLIENT_HOST = "i2cp.hostname";
+    public final static String DEFAULT_HOST = "127.0.0.1";
     
     public ClientManagerFacadeImpl(RouterContext context) {
         _context = context;

router/java/src/net/i2p/router/peermanager/ProfilePersistenceHelper.java

@@ -28,6 +28,14 @@ class ProfilePersistenceHelper {
     public final static String DEFAULT_PEER_PROFILE_DIR = "peerProfiles";
     private final static String NL = System.getProperty("line.separator");
     
+    /**
+     * If we haven't been able to get a message through to the peer in 3 days,
+     * drop the profile.  They may reappear, but if they do, their config may
+     * have changed (etc).
+     *
+     */
+    public static final long EXPIRE_AGE = 3*24*60*60*1000;
+    
     private File _profileDir = null;
     private Hash _us;
     
@@ -46,6 +54,9 @@ class ProfilePersistenceHelper {
     
     /** write out the data from the profile to the stream */
     public void writeProfile(PeerProfile profile) {
+        if (isExpired(profile.getLastSendSuccessful()))
+            return;
+        
         File f = pickFile(profile);
         long before = _context.clock().now();
         OutputStream fos = null;
@@ -159,6 +170,12 @@ class ProfilePersistenceHelper {
             rv.add(files[i]);
         return rv;
     }
+    
+    private boolean isExpired(long lastSentToSuccessfully) {
+        long timeSince = _context.clock().now() - lastSentToSuccessfully;
+        return (timeSince > EXPIRE_AGE);
+    }
+    
     public PeerProfile readProfile(File file) {
         Hash peer = getHash(file.getName());
         try {
@@ -171,6 +188,15 @@ class ProfilePersistenceHelper {
             
             loadProps(props, file);
             
+            long lastSentToSuccessfully = getLong(props, "lastSentToSuccessfully");
+            if (isExpired(lastSentToSuccessfully)) {
+                if (_log.shouldLog(Log.WARN))
+                    _log.warn("Dropping old profile for " + file.getName() + 
+                              ", since we haven't heard from them in a long time");
+                file.delete();
+                return null;
+            }
+            
             profile.setReliabilityBonus(getLong(props, "reliabilityBonus"));
             profile.setIntegrationBonus(getLong(props, "integrationBonus"));
             profile.setSpeedBonus(getLong(props, "speedBonus"));

router/java/src/net/i2p/router/transport/tcp/ConnectionBuilder.java

@@ -756,8 +756,14 @@ public class ConnectionBuilder {
                 _remoteAddress = tcpAddr;
                 _created = true;
             } catch (IOException ioe) {
+                Hash peer = _target.getIdentity().calculateHash();
+                String peerName = null;
+                if (peer == null)
+                    peerName = "unknown";
+                else
+                    peerName = peer.toBase64().substring(0,6);
                 fail("Error contacting " 
-                     + _target.getIdentity().calculateHash().toBase64().substring(0,6) 
+                     + peerName
                      + " on " + tcpAddr.toString() + ": " + ioe.getMessage());
                 return;
             }

router/java/src/net/i2p/router/transport/tcp/TCPTransport.java

@@ -136,6 +136,8 @@ public class TCPTransport extends TransportImpl {
     }
     
     public TransportBid bid(RouterInfo toAddress, long dataSize) {
+        if (false) return null;
+        
         RouterAddress addr = toAddress.getTargetAddress(STYLE);
         
         if ( (_myAddress != null) && (_myAddress.equals(addr)) ) 
@@ -306,8 +308,13 @@ public class TCPTransport extends TransportImpl {
     }
     
     void connectionClosed(TCPConnection con) {
+        Hash remotePeer = null;
+        if (con == null) return;
+        RouterIdentity peer = con.getRemoteRouterIdentity();
+        if (peer == null) return;
+        remotePeer = peer.getHash();
         synchronized (_connectionLock) {
-            TCPConnection cur = (TCPConnection)_connectionsByIdent.remove(con.getRemoteRouterIdentity().getHash());
+            TCPConnection cur = (TCPConnection)_connectionsByIdent.remove(remotePeer);
             if ( (cur != null) && (cur != con) )
                 _connectionsByIdent.put(cur.getRemoteRouterIdentity().getHash(), cur);
             cur = (TCPConnection)_connectionsByAddress.remove(con.getRemoteAddress().toString());

router/java/src/net/i2p/router/transport/udp/ACKBitfield.java

@@ -0,0 +1,16 @@
+package net.i2p.router.transport.udp;
+
+/**
+ * Generic means of SACK/NACK transmission for partially or fully 
+ * received messages
+ */
+public interface ACKBitfield {
+    /** what message is this partially ACKing? */
+    public long getMessageId(); 
+    /** how many fragments are covered in this bitfield? */
+    public int fragmentCount();
+    /** has the given fragment been received? */
+    public boolean received(int fragmentNum);
+    /** has the entire message been received completely? */
+    public boolean receivedComplete();
+}

router/java/src/net/i2p/router/transport/udp/ACKSender.java

@@ -68,8 +68,9 @@ public class ACKSender implements Runnable {
                 synchronized (_peersToACK) {
                     for (int i = 0; i < _peersToACK.size(); i++) {
                         PeerState cur = (PeerState)_peersToACK.get(i);
-                        long delta = cur.getWantedACKSendSince() + ACK_FREQUENCY - now;
+                        long wanted = cur.getWantedACKSendSince();
-                        if ( (delta < 0) || (cur.unsentACKThresholdReached()) ) {
+                        long delta = wanted + ACK_FREQUENCY - now;
+                        if ( ( (wanted > 0) && (delta < 0) ) || (cur.unsentACKThresholdReached()) ) {
                             _peersToACK.remove(i);
                             peer = cur;
                             break;
@@ -90,28 +91,33 @@ public class ACKSender implements Runnable {
             if (peer != null) {
                 long lastSend = peer.getLastACKSend();
                 long wanted = peer.getWantedACKSendSince();
-                List acks = peer.retrieveACKs();
+                List ackBitfields = peer.retrieveACKBitfields(_transport.getPartialACKSource());
-                if ( (acks != null) && (acks.size() > 0) ) {
+                
-                    _context.statManager().addRateData("udp.sendACKCount", acks.size(), 0);
+                if (wanted < 0)
+                    _log.error("wtf, why are we acking something they dont want?  remaining=" + remaining + ", peer=" + peer + ", bitfields=" + ackBitfields);
+                
+                if ( (ackBitfields != null) && (ackBitfields.size() > 0) ) {
+                    _context.statManager().addRateData("udp.sendACKCount", ackBitfields.size(), 0);
                     _context.statManager().addRateData("udp.sendACKRemaining", remaining, 0);
                     now = _context.clock().now();
                     _context.statManager().addRateData("udp.ackFrequency", now-lastSend, now-wanted);
-                    _context.statManager().getStatLog().addData(peer.getRemoteHostString(), "udp.peer.sendACKCount", acks.size(), 0);
+                    _context.statManager().getStatLog().addData(peer.getRemoteHostId().toString(), "udp.peer.sendACKCount", ackBitfields.size(), 0);
-                    UDPPacket ack = _builder.buildACK(peer, acks);
+                    UDPPacket ack = _builder.buildACK(peer, ackBitfields);
                     ack.markType(1);
                     if (_log.shouldLog(Log.INFO))
-                        _log.info("Sending ACK for " + acks);
+                        _log.info("Sending ACK for " + ackBitfields);
                     _transport.send(ack);
                     
-                    if (wanted == peer.getWantedACKSendSince()) {
+                    if ( (wanted > 0) && (wanted <= peer.getWantedACKSendSince()) ) {
-                        // still packets left to be ACKed, since wanted time
+                        // still full packets left to be ACKed, since wanted time
-                        // is reset by retrieveACKs when all of the IDs are
+                        // is reset by retrieveACKBitfields when all of the IDs are
                         // removed
+                        if (_log.shouldLog(Log.WARN))
+                            _log.warn("Rerequesting ACK for peer " + peer);
                         ackPeer(peer);
                     }
                 }
             }
         }
     }
-    
 }

router/java/src/net/i2p/router/transport/udp/EstablishmentManager.java

@@ -26,9 +26,9 @@ public class EstablishmentManager {
     private Log _log;
     private UDPTransport _transport;
     private PacketBuilder _builder;
-    /** map of host+port (String) to InboundEstablishState */
+    /** map of RemoteHostId to InboundEstablishState */
     private Map _inboundStates;
-    /** map of host+port (String) to OutboundEstablishState */
+    /** map of RemoteHostId to OutboundEstablishState */
     private Map _outboundStates;
     private boolean _alive;
     private Object _activityLock;
@@ -63,7 +63,7 @@ public class EstablishmentManager {
      * Grab the active establishing state
      */
     InboundEstablishState getInboundState(InetAddress fromHost, int fromPort) {
-        String from = PeerState.calculateRemoteHostString(fromHost.getAddress(), fromPort);
+        RemoteHostId from = new RemoteHostId(fromHost.getAddress(), fromPort);
         synchronized (_inboundStates) {
             InboundEstablishState state = (InboundEstablishState)_inboundStates.get(from);
             if ( (state == null) && (_log.shouldLog(Log.DEBUG)) )
@@ -73,7 +73,7 @@ public class EstablishmentManager {
     }
     
     OutboundEstablishState getOutboundState(InetAddress fromHost, int fromPort) {
-        String from = PeerState.calculateRemoteHostString(fromHost.getAddress(), fromPort);
+        RemoteHostId from = new RemoteHostId(fromHost.getAddress(), fromPort);
         synchronized (_outboundStates) {
             OutboundEstablishState state = (OutboundEstablishState)_outboundStates.get(from);
             if ( (state == null) && (_log.shouldLog(Log.DEBUG)) )
@@ -97,7 +97,7 @@ public class EstablishmentManager {
         UDPAddress addr = new UDPAddress(ra);
         InetAddress remAddr = addr.getHostAddress();
         int port = addr.getPort();
-        String to = PeerState.calculateRemoteHostString(remAddr.getAddress(), port);
+        RemoteHostId to = new RemoteHostId(remAddr.getAddress(), port);
         
         if (_log.shouldLog(Log.DEBUG))
             _log.debug("Add outobund establish state to: " + to);
@@ -120,7 +120,7 @@ public class EstablishmentManager {
      * Got a SessionRequest (initiates an inbound establishment)
      *
      */
-    void receiveSessionRequest(String from, InetAddress host, int port, UDPPacketReader reader) {
+    void receiveSessionRequest(RemoteHostId from, InetAddress host, int port, UDPPacketReader reader) {
         InboundEstablishState state = null;
         synchronized (_inboundStates) {
             state = (InboundEstablishState)_inboundStates.get(from);
@@ -132,7 +132,7 @@ public class EstablishmentManager {
         state.receiveSessionRequest(reader.getSessionRequestReader());
         
         if (_log.shouldLog(Log.DEBUG))
-            _log.debug("Receive session request from: " + state.getRemoteHostInfo());
+            _log.debug("Receive session request from: " + state.getRemoteHostId().toString());
         
         notifyActivity();
     }
@@ -141,7 +141,7 @@ public class EstablishmentManager {
      * got a SessionConfirmed (should only happen as part of an inbound 
      * establishment) 
      */
-    void receiveSessionConfirmed(String from, UDPPacketReader reader) {
+    void receiveSessionConfirmed(RemoteHostId from, UDPPacketReader reader) {
         InboundEstablishState state = null;
         synchronized (_inboundStates) {
             state = (InboundEstablishState)_inboundStates.get(from);
@@ -150,7 +150,7 @@ public class EstablishmentManager {
             state.receiveSessionConfirmed(reader.getSessionConfirmedReader());
             notifyActivity();
             if (_log.shouldLog(Log.DEBUG))
-                _log.debug("Receive session confirmed from: " + state.getRemoteHostInfo());
+                _log.debug("Receive session confirmed from: " + state.getRemoteHostId().toString());
         }
     }
     
@@ -158,7 +158,7 @@ public class EstablishmentManager {
      * Got a SessionCreated (in response to our outbound SessionRequest)
      *
      */
-    void receiveSessionCreated(String from, UDPPacketReader reader) {
+    void receiveSessionCreated(RemoteHostId from, UDPPacketReader reader) {
         OutboundEstablishState state = null;
         synchronized (_outboundStates) {
             state = (OutboundEstablishState)_outboundStates.get(from);
@@ -167,7 +167,7 @@ public class EstablishmentManager {
             state.receiveSessionCreated(reader.getSessionCreatedReader());
             notifyActivity();
             if (_log.shouldLog(Log.DEBUG))
-                _log.debug("Receive session created from: " + state.getRemoteHostInfo());
+                _log.debug("Receive session created from: " + state.getRemoteHostId().toString());
         }
     }
 
@@ -179,7 +179,7 @@ public class EstablishmentManager {
     PeerState receiveData(OutboundEstablishState state) {
         state.dataReceived();
         synchronized (_outboundStates) {
-            _outboundStates.remove(state.getRemoteHostInfo());
+            _outboundStates.remove(state.getRemoteHostId());
         }
         if (_log.shouldLog(Log.INFO))
             _log.info("Outbound established completely!  yay");
@@ -206,7 +206,7 @@ public class EstablishmentManager {
      */
     private void handleCompletelyEstablished(InboundEstablishState state) {
         if (_log.shouldLog(Log.DEBUG))
-            _log.debug("Handle completely established (inbound): " + state.getRemoteHostInfo());
+            _log.debug("Handle completely established (inbound): " + state.getRemoteHostId().toString());
         long now = _context.clock().now();
         RouterIdentity remote = state.getConfirmedIdentity();
         PeerState peer = new PeerState(_context);
@@ -236,7 +236,7 @@ public class EstablishmentManager {
      */
     private PeerState handleCompletelyEstablished(OutboundEstablishState state) {
         if (_log.shouldLog(Log.DEBUG))
-            _log.debug("Handle completely established (outbound): " + state.getRemoteHostInfo());
+            _log.debug("Handle completely established (outbound): " + state.getRemoteHostId().toString());
         long now = _context.clock().now();
         RouterIdentity remote = state.getRemoteIdentity();
         PeerState peer = new PeerState(_context);
@@ -284,7 +284,7 @@ public class EstablishmentManager {
             state.setSentRelayTag(0);
         
         if (_log.shouldLog(Log.DEBUG))
-            _log.debug("Send created to: " + state.getRemoteHostInfo());
+            _log.debug("Send created to: " + state.getRemoteHostId().toString());
         
         state.generateSessionKey();
         _transport.send(_builder.buildSessionCreatedPacket(state, _transport.getExternalPort(), _transport.getIntroKey()));
@@ -297,7 +297,7 @@ public class EstablishmentManager {
         long now = _context.clock().now();
         state.prepareSessionRequest();
         if (_log.shouldLog(Log.DEBUG))
-            _log.debug("Send request to: " + state.getRemoteHostInfo());
+            _log.debug("Send request to: " + state.getRemoteHostId().toString());
         _transport.send(_builder.buildSessionRequestPacket(state));
         state.requestSent();
     }
@@ -317,7 +317,7 @@ public class EstablishmentManager {
         UDPPacket packets[] = _builder.buildSessionConfirmedPackets(state, _context.router().getRouterInfo().getIdentity());
         
         if (_log.shouldLog(Log.DEBUG))
-            _log.debug("Send confirm to: " + state.getRemoteHostInfo());
+            _log.debug("Send confirm to: " + state.getRemoteHostId().toString());
         
         for (int i = 0; i < packets.length; i++)
             _transport.send(packets[i]);

router/java/src/net/i2p/router/transport/udp/InboundEstablishState.java

@@ -52,7 +52,7 @@ public class InboundEstablishState {
     private long _lastReceive;
     private long _lastSend;
     private long _nextSend;
-    private String _remoteHostInfo;
+    private RemoteHostId _remoteHostId;
     private int _currentState;
     
     /** nothin known yet */
@@ -71,7 +71,7 @@ public class InboundEstablishState {
         _log = ctx.logManager().getLog(InboundEstablishState.class);
         _aliceIP = remoteHost.getAddress();
         _alicePort = remotePort;
-        _remoteHostInfo = PeerState.calculateRemoteHostString(_aliceIP, _alicePort);
+        _remoteHostId = new RemoteHostId(_aliceIP, _alicePort);
         _bobPort = localPort;
         _keyBuilder = null;
         _verificationAttempted = false;
@@ -190,8 +190,8 @@ public class InboundEstablishState {
     public synchronized long getNextSendTime() { return _nextSend; }
     public synchronized void setNextSendTime(long when) { _nextSend = when; }
 
-    /** host+port, uniquely identifies an attempt */
+    /** RemoteHostId, uniquely identifies an attempt */
-    public String getRemoteHostInfo() { return _remoteHostInfo; }
+    public RemoteHostId getRemoteHostId() { return _remoteHostId; }
 
     public synchronized void receiveSessionConfirmed(UDPPacketReader.SessionConfirmedReader conf) {
         if (_receivedIdentity == null)

router/java/src/net/i2p/router/transport/udp/InboundMessageFragments.java

@@ -2,9 +2,11 @@ package net.i2p.router.transport.udp;
 
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 
+import net.i2p.data.Hash;
 import net.i2p.router.RouterContext;
 import net.i2p.util.DecayingBloomFilter;
 import net.i2p.util.I2PThread;
@@ -23,7 +25,7 @@ import net.i2p.util.Log;
  * a scheduled event)
  *
  */
-public class InboundMessageFragments {
+public class InboundMessageFragments implements UDPTransport.PartialACKSource {
     private RouterContext _context;
     private Log _log;
     /** Map of peer (Hash) to a Map of messageId (Long) to InboundMessageState objects */
@@ -87,6 +89,7 @@ public class InboundMessageFragments {
         receiveACKs(from, data);
         long afterACKs = _context.clock().now();
         
+        from.packetReceived();
         _context.statManager().addRateData("udp.receiveMessagePeriod", afterMsgs-beforeMsgs, afterACKs-beforeMsgs);
         _context.statManager().addRateData("udp.receiveACKPeriod", afterACKs-afterMsgs, afterACKs-beforeMsgs);
     }
@@ -100,7 +103,7 @@ public class InboundMessageFragments {
     private void receiveMessages(PeerState from, UDPPacketReader.DataReader data) {
         int fragments = data.readFragmentCount();
         if (fragments <= 0) return;
-        synchronized (_inboundMessages) {
+        synchronized (_inboundMessages) { // XXX: CHOKE POINT (to what extent?)
             Map messages = (Map)_inboundMessages.get(from.getRemotePeer());
             if (messages == null) {
                 messages = new HashMap(fragments);
@@ -158,6 +161,12 @@ public class InboundMessageFragments {
                     if (_log.shouldLog(Log.WARN))
                         _log.warn("Message expired while only being partially read: " + state);
                     state.releaseResources();
+                } else {
+                    // not expired but not yet complete... lets queue up a partial ACK
+                    if (_log.shouldLog(Log.DEBUG))
+                        _log.debug("Queueing up a partial ACK for peer: " + from + " for " + state);
+                    from.messagePartiallyReceived();
+                    _ackSender.ackPeer(from);
                 }
                 
                 if (!fragmentOK)
@@ -172,7 +181,7 @@ public class InboundMessageFragments {
             long acks[] = data.readACKs();
             if (acks != null) {
                 _context.statManager().addRateData("udp.receivedACKs", acks.length, 0);
-                _context.statManager().getStatLog().addData(from.getRemoteHostString(), "udp.peer.receiveACKCount", acks.length, 0);
+                //_context.statManager().getStatLog().addData(from.getRemoteHostId().toString(), "udp.peer.receiveACKCount", acks.length, 0);
 
                 for (int i = 0; i < acks.length; i++) {
                     if (_log.shouldLog(Log.INFO))
@@ -183,9 +192,40 @@ public class InboundMessageFragments {
                 _log.error("Received ACKs with no acks?! " + data);
             }
         }
+        if (data.readACKBitfieldsIncluded()) {
+            ACKBitfield bitfields[] = data.readACKBitfields();
+            if (bitfields != null) {
+                //_context.statManager().getStatLog().addData(from.getRemoteHostId().toString(), "udp.peer.receivePartialACKCount", bitfields.length, 0);
+
+                for (int i = 0; i < bitfields.length; i++) {
+                    if (_log.shouldLog(Log.INFO))
+                        _log.info("Partial ACK received: " + bitfields[i]);
+                    _outbound.acked(bitfields[i], from.getRemotePeer());
+                }
+            }
+        }
         if (data.readECN())
             from.ECNReceived();
         else
             from.dataReceived();
     }
+    
+    public void fetchPartialACKs(Hash fromPeer, List ackBitfields) {
+        synchronized (_inboundMessages) {
+            Map messages = (Map)_inboundMessages.get(fromPeer);
+            if (messages == null)
+                return;
+            for (Iterator iter = messages.values().iterator(); iter.hasNext(); ) {
+                InboundMessageState state = (InboundMessageState)iter.next();
+                if (state.isExpired()) {
+                    iter.remove();
+                } else {
+                    if (!state.isComplete())
+                        ackBitfields.add(state.createACKBitfield());
+                }
+            }
+            if (messages.size() <= 0)
+                _inboundMessages.remove(fromPeer);
+        }
+    }
 }

router/java/src/net/i2p/router/transport/udp/InboundMessageState.java

@@ -27,9 +27,9 @@ public class InboundMessageState {
     private long _receiveBegin;
     private int _completeSize;
     
-    /** expire after 30s */
+    /** expire after 10s */
     private static final long MAX_RECEIVE_TIME = 10*1000;
-    private static final int MAX_FRAGMENTS = 32;
+    private static final int MAX_FRAGMENTS = 64;
     
     private static final ByteCache _fragmentCache = ByteCache.getInstance(64, 2048);
     
@@ -49,7 +49,7 @@ public class InboundMessageState {
      *
      * @return true if the data was ok, false if it was corrupt
      */
-    public synchronized boolean receiveFragment(UDPPacketReader.DataReader data, int dataFragment) {
+    public boolean receiveFragment(UDPPacketReader.DataReader data, int dataFragment) {
         int fragmentNum = data.readMessageFragmentNum(dataFragment);
         if ( (fragmentNum < 0) || (fragmentNum > _fragments.length)) {
             StringBuffer buf = new StringBuffer(1024);
@@ -72,14 +72,14 @@ public class InboundMessageState {
         return true;
     }
     
-    public synchronized boolean isComplete() {
+    public boolean isComplete() {
         if (_lastFragment < 0) return false;
         for (int i = 0; i <= _lastFragment; i++)
             if (_fragments[i] == null)
                 return false;
         return true;
     }
-    public synchronized boolean isExpired() { 
+    public boolean isExpired() { 
         return _context.clock().now() > _receiveBegin + MAX_RECEIVE_TIME;
     }
     public long getLifetime() {
@@ -87,7 +87,7 @@ public class InboundMessageState {
     }
     public Hash getFrom() { return _from; }
     public long getMessageId() { return _messageId; }
-    public synchronized int getCompleteSize() {
+    public int getCompleteSize() {
         if (_completeSize < 0) {
             int size = 0;
             for (int i = 0; i <= _lastFragment; i++)
@@ -96,6 +96,46 @@ public class InboundMessageState {
         }
         return _completeSize;
     }
+    public ACKBitfield createACKBitfield() {
+        return new PartialBitfield(_messageId, _fragments);
+    }
+    
+    private static final class PartialBitfield implements ACKBitfield {
+        private long _bitfieldMessageId;
+        private boolean _fragmentsReceived[];
+        
+        public PartialBitfield(long messageId, Object data[]) {
+            _bitfieldMessageId = messageId;
+            for (int i = data.length - 1; i >= 0; i--) {
+                if (data[i] != null) {
+                    if (_fragmentsReceived == null)
+                        _fragmentsReceived = new boolean[i+1];
+                    _fragmentsReceived[i] = true;
+                }
+            }
+            if (_fragmentsReceived == null)
+                _fragmentsReceived = new boolean[0];
+        }
+        public int fragmentCount() { return _fragmentsReceived.length; }
+        public long getMessageId() { return _bitfieldMessageId; }
+        public boolean received(int fragmentNum) { 
+            if ( (fragmentNum < 0) || (fragmentNum >= _fragmentsReceived.length) )
+                return false;
+            return _fragmentsReceived[fragmentNum];
+        }
+        public boolean receivedComplete() { return false; }
+        
+        public String toString() { 
+            StringBuffer buf = new StringBuffer(64);
+            buf.append("Partial ACK of ");
+            buf.append(_bitfieldMessageId);
+            buf.append(" with ACKs for: ");
+            for (int i = 0; i < _fragmentsReceived.length; i++)
+                if (_fragmentsReceived[i])
+                    buf.append(i).append(" ");
+            return buf.toString();
+        }
+    }
     
     public void releaseResources() {
         if (_fragments != null)

router/java/src/net/i2p/router/transport/udp/MessageReceiver.java

@@ -10,6 +10,7 @@ import net.i2p.data.i2np.I2NPMessage;
 import net.i2p.data.i2np.I2NPMessageImpl;
 import net.i2p.data.i2np.I2NPMessageException;
 import net.i2p.router.RouterContext;
+import net.i2p.util.ByteCache;
 import net.i2p.util.I2PThread;
 import net.i2p.util.Log;
 
@@ -25,12 +26,14 @@ public class MessageReceiver implements Runnable {
     /** list of messages (InboundMessageState) fully received but not interpreted yet */
     private List _completeMessages;
     private boolean _alive;
+    private ByteCache _cache;
     
     public MessageReceiver(RouterContext ctx, UDPTransport transport) {
         _context = ctx;
         _log = ctx.logManager().getLog(MessageReceiver.class);
         _transport = transport;
         _completeMessages = new ArrayList(16);
+        _cache = ByteCache.getInstance(64, I2NPMessage.MAX_SIZE);
         _alive = true;
     }
 
@@ -70,8 +73,7 @@ public class MessageReceiver implements Runnable {
             if (message != null) {
                 int size = message.getCompleteSize();
                 if (_log.shouldLog(Log.INFO))
-                    _log.info("Full message received (" + message.getMessageId() + ") after " + message.getLifetime() 
+                    _log.info("Full message received (" + message.getMessageId() + ") after " + message.getLifetime());
-                              + "... todo: parse and plop it onto InNetMessagePool");
                 I2NPMessage msg = readMessage(message);
                 if (msg != null)
                     _transport.messageReceived(msg, null, message.getFrom(), message.getLifetime(), size);
@@ -81,21 +83,24 @@ public class MessageReceiver implements Runnable {
     }
     
     private I2NPMessage readMessage(InboundMessageState state) {
+        ByteArray buf = _cache.acquire();
         try {
-            byte buf[] = new byte[state.getCompleteSize()];
+            //byte buf[] = new byte[state.getCompleteSize()];
             ByteArray fragments[] = state.getFragments();
             int numFragments = state.getFragmentCount();
             int off = 0;
             for (int i = 0; i < numFragments; i++) {
-                System.arraycopy(fragments[i].getData(), 0, buf, off, fragments[i].getValid());
+                System.arraycopy(fragments[i].getData(), 0, buf.getData(), off, fragments[i].getValid());
                 if (_log.shouldLog(Log.DEBUG))
                     _log.debug("Raw fragment[" + i + "] for " + state.getMessageId() + ": " 
                                + Base64.encode(fragments[i].getData(), 0, fragments[i].getValid()));
                 off += fragments[i].getValid();
             }
+            if (off != state.getCompleteSize())
+                _log.error("Hmm, offset of the fragments = " + off + " while the state says " + state.getCompleteSize());
             if (_log.shouldLog(Log.DEBUG))
-                _log.debug("Raw byte array for " + state.getMessageId() + ": " + Base64.encode(buf));
+                _log.debug("Raw byte array for " + state.getMessageId() + ": " + Base64.encode(buf.getData(), 0, state.getCompleteSize()));
-            I2NPMessage m = I2NPMessageImpl.fromRawByteArray(_context, buf, 0, buf.length);
+            I2NPMessage m = I2NPMessageImpl.fromRawByteArray(_context, buf.getData(), 0, state.getCompleteSize());
             m.setUniqueId(state.getMessageId());
             return m;
         } catch (I2NPMessageException ime) {
@@ -107,6 +112,7 @@ public class MessageReceiver implements Runnable {
             return null;
         } finally {
             state.releaseResources();
+            _cache.release(buf);
         }
     }
 }

router/java/src/net/i2p/router/transport/udp/OutboundEstablishState.java

@@ -51,7 +51,7 @@ public class OutboundEstablishState {
     private long _lastReceive;
     private long _lastSend;
     private long _nextSend;
-    private String _remoteHostInfo;
+    private RemoteHostId _remoteHostId;
     private RouterIdentity _remotePeer;
     private SessionKey _introKey;
     private List _queuedMessages;
@@ -74,7 +74,7 @@ public class OutboundEstablishState {
         _log = ctx.logManager().getLog(OutboundEstablishState.class);
         _bobIP = remoteHost.getAddress();
         _bobPort = remotePort;
-        _remoteHostInfo = PeerState.calculateRemoteHostString(_bobIP, _bobPort);
+        _remoteHostId = new RemoteHostId(_bobIP, _bobPort);
         _remotePeer = remotePeer;
         _introKey = introKey;
         _keyBuilder = null;
@@ -172,7 +172,7 @@ public class OutboundEstablishState {
             return true;
         } else {
             if (_log.shouldLog(Log.WARN))
-                _log.warn("Session created failed validation, clearing state");
+                _log.warn("Session created failed validation, clearing state for " + _remoteHostId.toString());
             _receivedY = null;
             _aliceIP = null;
             _receivedRelayTag = 0;
@@ -244,7 +244,8 @@ public class OutboundEstablishState {
         DataHelper.toLong(signed, off, 4, _receivedRelayTag);
         off += 4;
         DataHelper.toLong(signed, off, 4, _receivedSignedOnTime);
-        if (_log.shouldLog(Log.DEBUG)) {
+        boolean valid = _context.dsa().verifySignature(_receivedSignature, signed, _remotePeer.getSigningPublicKey());
+        if (!valid || _log.shouldLog(Log.DEBUG)) {
             StringBuffer buf = new StringBuffer(128);
             buf.append("Signed sessionCreated:");
             buf.append(" AliceIP: ").append(Base64.encode(_aliceIP));
@@ -254,9 +255,12 @@ public class OutboundEstablishState {
             buf.append(" RelayTag: ").append(_receivedRelayTag);
             buf.append(" SignedOn: ").append(_receivedSignedOnTime);
             buf.append(" signature: ").append(Base64.encode(_receivedSignature.getData()));
+            if (valid)
                 _log.debug(buf.toString());
+            else if (_log.shouldLog(Log.WARN))
+                _log.warn("INVALID: " + buf.toString());
         }
-        return _context.dsa().verifySignature(_receivedSignature, signed, _remotePeer.getSigningPublicKey());
+        return valid;
     }
     
     public synchronized SessionKey getCipherKey() { return _sessionKey; }
@@ -331,8 +335,8 @@ public class OutboundEstablishState {
             _log.debug("Explicit nextSend=" + (_nextSend-_context.clock().now()), new Exception("Set by"));
     }
 
-    /** host+port, uniquely identifies an attempt */
+    /** uniquely identifies an attempt */
-    public String getRemoteHostInfo() { return _remoteHostInfo; }
+    public RemoteHostId getRemoteHostId() { return _remoteHostId; }
 
     /** we have received a real data packet, so we're done establishing */
     public synchronized void dataReceived() {

router/java/src/net/i2p/router/transport/udp/OutboundMessageFragments.java

@@ -13,7 +13,7 @@ import net.i2p.util.Log;
  * Coordinate the outbound fragments and select the next one to be built.
  * This pool contains messages we are actively trying to send, essentially 
  * doing a round robin across each message to send one fragment, as implemented
- * in {@link #getNextPacket()}.  This also honors per-peer throttling, taking 
+ * in {@link #getNextVolley()}.  This also honors per-peer throttling, taking 
  * note of each peer's allocations.  If a message has each of its fragments
  * sent more than a certain number of times, it is failed out.  In addition, 
  * this instance also receives notification of message ACKs from the 
@@ -57,6 +57,9 @@ public class OutboundMessageFragments {
         _context.statManager().createRateStat("udp.sendAggressiveFailed", "How many volleys was a packet sent before we gave up", "udp", new long[] { 60*1000, 10*60*1000, 60*60*1000, 24*60*60*1000 });
         _context.statManager().createRateStat("udp.outboundActiveCount", "How many messages are in the active pool when a new one is added", "udp", new long[] { 60*1000, 10*60*1000, 60*60*1000, 24*60*60*1000 });
         _context.statManager().createRateStat("udp.sendRejected", "What volley are we on when the peer was throttled (time == message lifetime)", "udp", new long[] { 60*1000, 10*60*1000, 60*60*1000, 24*60*60*1000 });
+        _context.statManager().createRateStat("udp.partialACKReceived", "How many fragments were partially ACKed (time == message lifetime)", "udp", new long[] { 60*1000, 10*60*1000, 60*60*1000, 24*60*60*1000 });
+        _context.statManager().createRateStat("udp.sendSparse", "How many fragments were partially ACKed and hence not resent (time == message lifetime)", "udp", new long[] { 60*1000, 10*60*1000, 60*60*1000, 24*60*60*1000 });
+        _context.statManager().createRateStat("udp.activeDelay", "How often we wait blocking on the active queue", "udp", new long[] { 60*1000, 10*60*1000, 60*60*1000, 24*60*60*1000 });
     }
     
     public void startup() { _alive = true; }
@@ -74,19 +77,27 @@ public class OutboundMessageFragments {
      * @return true if more messages are allowed
      */
     public boolean waitForMoreAllowed() {
+        // test without choking.  
+        // perhaps this should check the lifetime of the first activeMessage?
+        if (false) return true; 
+        
+        long start = _context.clock().now();
+        int numActive = 0;
         while (_alive) {
             finishMessages();
             try {
                 synchronized (_activeMessages) {
+                    numActive = _activeMessages.size();
                     if (!_alive)
                         return false;
-                    else if (_activeMessages.size() < MAX_ACTIVE)
+                    else if (numActive < MAX_ACTIVE)
                         return true;
                     else if (_allowExcess)
                         return true;
                     else
                         _activeMessages.wait(1000);
                 }
+                _context.statManager().addRateData("udp.activeDelay", numActive, _context.clock().now() - start);
             } catch (InterruptedException ie) {}
         }
         return false;
@@ -208,6 +219,7 @@ public class OutboundMessageFragments {
             long now = _context.clock().now();
             long nextSend = -1;
             finishMessages();
+            try {
                 synchronized (_activeMessages) {
                     for (int i = 0; i < _activeMessages.size(); i++) {
                         int cur = (i + _nextPacketMessage) % _activeMessages.size();
@@ -243,10 +255,8 @@ public class OutboundMessageFragments {
 
                     if (state == null) {
                         if (nextSend <= 0) {
-                        try {
                             _activeMessages.notifyAll();
-                            _activeMessages.wait();
+                            _activeMessages.wait(1000);
-                        } catch (InterruptedException ie) {}
                         } else {
                             // none of the packets were eligible for sending
                             long delay = nextSend - now;
@@ -256,15 +266,14 @@ public class OutboundMessageFragments {
                                 delay = 1000;
                             _allowExcess = true;
                             _activeMessages.notifyAll();
-                        try {
                             _activeMessages.wait(delay);
-                        } catch (InterruptedException ie) {}
                         }
                     } else {
                         _activeMessages.notifyAll();
                     }
                     _allowExcess = false;
                 } // end of the synchronized block
+            } catch (InterruptedException ie) {}
         } // end of the while (alive && !found)
 
         return preparePackets(state, peer);
@@ -290,10 +299,17 @@ public class OutboundMessageFragments {
                               + " for message " + state.getMessageId() + ": " + state);
 
                 if (state.getPushCount() > 0) {
-                    peer.messageRetransmitted();
+                    int fragments = state.getFragmentCount();
+                    int toSend = 0;
+                    for (int i = 0; i < fragments; i++) {
+                        if (state.needsSending(i))
+                            toSend++;
+                    }
+
+                    peer.messageRetransmitted(toSend);
                     if (_log.shouldLog(Log.WARN))
                         _log.warn("Retransmitting " + state + " to " + peer);
-                    _context.statManager().addRateData("udp.sendVolleyTime", state.getLifetime(), state.getFragmentCount());
+                    _context.statManager().addRateData("udp.sendVolleyTime", state.getLifetime(), toSend);
                 }
 
                 state.push();
@@ -326,13 +342,19 @@ public class OutboundMessageFragments {
             if (fragments < 0)
                 return null;
             
-            if (_log.shouldLog(Log.INFO))
+            int sparseCount = 0;
-                _log.info("Building packet for " + state + " to " + peer);
             UDPPacket rv[] = new UDPPacket[fragments]; //sparse
             for (int i = 0; i < fragments; i++) {
                 if (state.needsSending(i))
                     rv[i] = _builder.buildPacket(state, i, peer);
+                else
+                    sparseCount++;
             }
+            if (sparseCount > 0)
+                _context.statManager().addRateData("udp.sendSparse", sparseCount, state.getLifetime());
+            if (_log.shouldLog(Log.INFO))
+                _log.info("Building packet for " + state + " to " + peer + " with sparse count: " + sparseCount);
+            peer.packetsTransmitted(fragments - sparseCount);
             return rv;
         } else {
             // !alive
@@ -413,51 +435,84 @@ public class OutboundMessageFragments {
         }
     }
     
-    /**
+    public void acked(ACKBitfield bitfield, Hash ackedBy) {
-     * Receive a set of fragment ACKs for a given messageId from the 
+        if (bitfield.receivedComplete()) {
-     * specified peer
+            acked(bitfield.getMessageId(), ackedBy);
-     *
+            return;
-     */
+        }
-    public void acked(long messageId, int ackedFragments[], Hash ackedBy) {
+        
-        if (_log.shouldLog(Log.INFO))
-            _log.info("Received partial ack of " + messageId + " by " + ackedBy.toBase64());
         OutboundMessageState state = null;
+        boolean isComplete = false;
         synchronized (_activeMessages) {
             // linear search, since its tiny
             for (int i = 0; i < _activeMessages.size(); i++) {
                 state = (OutboundMessageState)_activeMessages.get(i);
-                if (state.getMessage().getMessageId() == messageId) {
+                if (state.getMessageId() == bitfield.getMessageId()) {
-                    Hash expectedBy = state.getMessage().getTarget().getIdentity().calculateHash();
+                    OutNetMessage msg = state.getMessage();
+                    if (msg != null) {
+                        Hash expectedBy = msg.getTarget().getIdentity().getHash();
                         if (!expectedBy.equals(ackedBy)) {
+                            state = null;
+                            _activeMessages.notifyAll();
                             return;
-                    } else {
+                        }
-                        state.acked(ackedFragments);
+                    }
-                        if (state.isComplete()) {
+                    isComplete = state.acked(bitfield);
+                    if (isComplete) {
+                        // either the message was a short circuit after establishment,
+                        // or it was received from who we sent it to.  yay!
                         _activeMessages.remove(i);
                         if (i < _nextPacketMessage) {
                             _nextPacketMessage--; 
                             if (_nextPacketMessage < 0)
                                 _nextPacketMessage = 0;
                         }
-                            _activeMessages.notifyAll();
                     }
                     break;
+                } else {
+                    state = null;
                 }
             }
-            }
+            _activeMessages.notifyAll();
         }
         
-        if ( (state != null) && (state.isComplete()) ) {
+        if (state != null) {
+            int numSends = state.getMaxSends();
+                        
+            int bits = bitfield.fragmentCount();
+            int numACKed = 0;
+            for (int i = 0; i < bits; i++)
+                if (bitfield.received(i))
+                    numACKed++;
+            
+            _context.statManager().addRateData("udp.partialACKReceived", numACKed, state.getLifetime());
+            
             if (_log.shouldLog(Log.INFO))
-                _log.info("Received ack of " + messageId + " by " + ackedBy.toBase64() 
+                _log.info("Received partial ack of " + state.getMessageId() + " by " + ackedBy.toBase64() 
-                          + " after " + state.getLifetime());
+                          + " after " + state.getLifetime() + " and " + numSends + " sends: " + bitfield + ": completely removed? " 
+                          + isComplete + ": " + state);
+            
+            if (isComplete) {
                 _context.statManager().addRateData("udp.sendConfirmTime", state.getLifetime(), state.getLifetime());
                 _context.statManager().addRateData("udp.sendConfirmFragments", state.getFragmentCount(), state.getLifetime());
+                _context.statManager().addRateData("udp.sendConfirmVolley", numSends, state.getFragmentCount());
                 _transport.succeeded(state.getMessage());
+                
+                if (state.getPeer() != null) {
+                    // this adjusts the rtt/rto/window/etc
+                    state.getPeer().messageACKed(state.getFragmentCount()*state.getFragmentSize(), state.getLifetime(), 0);
                     if (state.getPeer().getSendWindowBytesRemaining() > 0)
                         _throttle.unchoke(state.getPeer().getRemotePeer());
+                }
+
                 state.releaseResources();
             }
+            return;
+        } else {
+            if (_log.shouldLog(Log.WARN))
+                _log.warn("Received an ACK for a message not pending: " + bitfield);
+            return;
+        }
     }
     
     public interface ActiveThrottle {

router/java/src/net/i2p/router/transport/udp/OutboundMessageState.java

@@ -44,7 +44,7 @@ public class OutboundMessageState {
         _nextSendFragment = 0;
     }
     
-    public synchronized boolean initialize(OutNetMessage msg) {
+    public boolean initialize(OutNetMessage msg) {
         try {
             initialize(msg, msg.getMessage(), null);
             return true;
@@ -68,7 +68,7 @@ public class OutboundMessageState {
         }
     }
     
-    private synchronized void initialize(OutNetMessage m, I2NPMessage msg, PeerState peer) {
+    private void initialize(OutNetMessage m, I2NPMessage msg, PeerState peer) {
         _message = m;
         _peer = peer;
         if (_messageBuf != null) {
@@ -93,10 +93,10 @@ public class OutboundMessageState {
             _log.debug("Raw byte array for " + _messageId + ": " + Base64.encode(_messageBuf.getData(), 0, len));
     }
     
-    public synchronized void releaseResources() { 
+    public void releaseResources() { 
         if (_messageBuf != null)
             _cache.release(_messageBuf);
-        _messageBuf = null;
+        //_messageBuf = null;
     }
     
     public OutNetMessage getMessage() { return _message; }
@@ -107,23 +107,26 @@ public class OutboundMessageState {
         return _expiration < _context.clock().now(); 
     }
     public boolean isComplete() {
-        if (_fragmentSends == null) return false;
+        short sends[] = _fragmentSends;
-        for (int i = 0; i < _fragmentSends.length; i++)
+        if (sends == null) return false;
-            if (_fragmentSends[i] >= 0)
+        for (int i = 0; i < sends.length; i++)
+            if (sends[i] >= 0)
                 return false;
         // nothing else pending ack
         return true;
     }
-    public synchronized int getUnackedSize() {
+    public int getUnackedSize() {
+        short fragmentSends[] = _fragmentSends;
+        ByteArray messageBuf = _messageBuf;
         int rv = 0;
-        if ( (_messageBuf != null) && (_fragmentSends != null) ) {
+        if ( (messageBuf != null) && (fragmentSends != null) ) {
-            int totalSize = _messageBuf.getValid();
+            int totalSize = messageBuf.getValid();
             int lastSize = totalSize % _fragmentSize;
             if (lastSize == 0)
                 lastSize = _fragmentSize;
-            for (int i = 0; i < _fragmentSends.length; i++) {
+            for (int i = 0; i < fragmentSends.length; i++) {
-                if (_fragmentSends[i] >= (short)0) {
+                if (fragmentSends[i] >= (short)0) {
-                    if (i + 1 == _fragmentSends.length)
+                    if (i + 1 == fragmentSends.length)
                         rv += lastSize;
                     else
                         rv += _fragmentSize;
@@ -132,23 +135,45 @@ public class OutboundMessageState {
         }
         return rv;
     }
-    public synchronized boolean needsSending(int fragment) {
+    public boolean needsSending(int fragment) {
-        if ( (_fragmentSends == null) || (fragment >= _fragmentSends.length) || (fragment < 0) )
+        
+        short sends[] = _fragmentSends;
+        if ( (sends == null) || (fragment >= sends.length) || (fragment < 0) )
             return false;
-        return (_fragmentSends[fragment] >= (short)0);
+        return (sends[fragment] >= (short)0);
     }
     public long getLifetime() { return _context.clock().now() - _startedOn; }
     
     /**
-     * Ack all the fragments in the ack list
+     * Ack all the fragments in the ack list.  As a side effect, if there are
+     * still unacked fragments, the 'next send' time will be updated under the
+     * assumption that that all of the packets within a volley would reach the
+     * peer within that ack frequency (2-400ms).
+     *
+     * @return true if the message was completely ACKed
      */
-    public void acked(int ackedFragments[]) {
+    public boolean acked(ACKBitfield bitfield) {
         // stupid brute force, but the cardinality should be trivial
-        for (int i = 0; i < ackedFragments.length; i++) {
+        short sends[] = _fragmentSends;
-            if ( (ackedFragments[i] < 0) || (ackedFragments[i] >= _fragmentSends.length) )
+        if (sends != null)
-                continue;
+            for (int i = 0; i < bitfield.fragmentCount(); i++)
-            _fragmentSends[ackedFragments[i]] = -1;
+                if (bitfield.received(i))
+                    sends[i] = (short)-1;
+        
+        boolean rv = isComplete();
+        if (!rv && false) { // don't do the fast retransmit... lets give it time to get ACKed
+            long nextTime = _context.clock().now() + Math.max(_peer.getRTT(), ACKSender.ACK_FREQUENCY);
+            //_nextSendTime = Math.max(now, _startedOn+PeerState.MIN_RTO);
+            if (_nextSendTime <= 0)
+                _nextSendTime = nextTime;
+            else
+                _nextSendTime = Math.min(_nextSendTime, nextTime);
+            
+            //if (now + 100 > _nextSendTime)
+            //    _nextSendTime = now + 100;
+            //_nextSendTime = now;
         }
+        return rv;
     }
     
     public long getNextSendTime() { return _nextSendTime; }
@@ -156,7 +181,7 @@ public class OutboundMessageState {
     public int getMaxSends() { return _maxSends; }
     public int getPushCount() { return _pushCount; }
     /** note that we have pushed the message fragments */
-    public synchronized void push() { 
+    public void push() { 
         _pushCount++; 
         if (_pushCount > _maxSends)
             _maxSends = (short)_pushCount;
@@ -172,7 +197,7 @@ public class OutboundMessageState {
      * fragmentSize bytes per fragment.
      *
      */
-    public synchronized void fragment(int fragmentSize) {
+    public void fragment(int fragmentSize) {
         int totalSize = _messageBuf.getValid();
         int numFragments = totalSize / fragmentSize;
         if (numFragments * fragmentSize != totalSize)
@@ -198,7 +223,7 @@ public class OutboundMessageState {
     public int getFragmentSize() { return _fragmentSize; }
     /** should we continue sending this fragment? */
     public boolean shouldSend(int fragmentNum) { return _fragmentSends[fragmentNum] >= (short)0; }
-    public synchronized int fragmentSize(int fragmentNum) {
+    public int fragmentSize(int fragmentNum) {
         if (_messageBuf == null) return -1;
         if (fragmentNum + 1 == _fragmentSends.length)
             return _messageBuf.getValid() % _fragmentSize;
@@ -206,81 +231,6 @@ public class OutboundMessageState {
             return _fragmentSize;
     }
 
-    public void incrementCurrentFragment() {
-        int cur = _nextSendFragment;
-        _fragmentSends[cur]++;
-        _maxSends = _fragmentSends[cur];
-        _nextSendFragment++;
-        if (_nextSendFragment >= _fragmentSends.length) {
-            _nextSendFragment = 0;
-            _pushCount++;
-        }
-    }
-    
-    /**
-     * Pick a fragment that we still need to send.  Current implementation 
-     * picks the fragment which has been sent the least (randomly choosing 
-     * among equals), incrementing the # sends of the winner in the process.
-     *
-     * @return fragment index, or -1 if all of the fragments were acked
-     */
-    public int pickNextFragment() {
-        if (true) {
-            return _nextSendFragment;
-        }
-        short minValue = -1;
-        int minIndex = -1;
-        int startOffset = _context.random().nextInt(_fragmentSends.length);
-        for (int i = 0; i < _fragmentSends.length; i++) {
-            int cur = (i + startOffset) % _fragmentSends.length;
-            if (_fragmentSends[cur] < (short)0)
-                continue;
-            else if ( (minValue < (short)0) || (_fragmentSends[cur] < minValue) ) {
-                minValue = _fragmentSends[cur];
-                minIndex = cur;
-            }
-        }
-        if (minIndex >= 0) {
-            _fragmentSends[minIndex]++;
-            if (_fragmentSends[minIndex] > _maxSends)
-                _maxSends = _fragmentSends[minIndex];
-        }
-        
-        // if all fragments have now been sent an equal number of times,
-        // lets give pause for an ACK
-        boolean endOfVolley = true;
-        for (int i = 0; i < _fragmentSends.length; i++) {
-            if (_fragmentSends[i] < (short)0)
-                continue;
-            if (_fragmentSends[i] != (short)_pushCount+1) {
-                endOfVolley = false;
-                break;
-            }
-        }
-        if (endOfVolley) {
-            _pushCount++;
-        }
-        
-        if (_log.shouldLog(Log.DEBUG)) {
-            StringBuffer buf = new StringBuffer(64);
-            buf.append("Next fragment is ").append(minIndex);
-            if (minIndex >= 0) {
-                buf.append(" (#sends: ").append(_fragmentSends[minIndex]-1);
-                buf.append(" #fragments: ").append(_fragmentSends.length);
-                buf.append(")");
-            }
-            _log.debug(buf.toString());
-        }
-        return minIndex;
-    }
-
-    public boolean justBeganVolley() {
-        if (_fragmentSends.length == 1)
-            return true;
-        else
-            return _nextSendFragment == 1;
-    }
-    
     /**
      * Write a part of the the message onto the specified buffer.
      *
@@ -289,7 +239,7 @@ public class OutboundMessageState {
      * @param fragmentNum fragment to write (0 indexed)
      * @return bytesWritten
      */
-    public synchronized int writeFragment(byte out[], int outOffset, int fragmentNum) {
+    public int writeFragment(byte out[], int outOffset, int fragmentNum) {
         int start = _fragmentSize * fragmentNum;
         int end = start + _fragmentSize;
         if (_messageBuf == null) return -1;
@@ -303,15 +253,23 @@ public class OutboundMessageState {
         return toSend;
     }
     
-    public synchronized String toString() {
+    public String toString() {
+        short sends[] = _fragmentSends;
+        ByteArray messageBuf = _messageBuf;
         StringBuffer buf = new StringBuffer(64);
         buf.append("Message ").append(_messageId);
-        if (_fragmentSends != null)
+        if (sends != null)
-            buf.append(" with ").append(_fragmentSends.length).append(" fragments");
+            buf.append(" with ").append(sends.length).append(" fragments");
-        if (_messageBuf != null)
+        if (messageBuf != null)
-            buf.append(" of size ").append(_messageBuf.getValid());
+            buf.append(" of size ").append(messageBuf.getValid());
         buf.append(" volleys: ").append(_maxSends);
         buf.append(" lifetime: ").append(getLifetime());
+        if (sends != null) {
+            buf.append(" pending fragments: ");
+            for (int i = 0; i < sends.length; i++)
+                if (sends[i] >= 0)
+                    buf.append(i).append(' ');
+        }
         return buf.toString();
     }
 }

router/java/src/net/i2p/router/transport/udp/PacketBuilder.java

@@ -60,15 +60,16 @@ public class PacketBuilder {
         DataHelper.toLong(data, off, 4, state.getMessageId());
         off += 4;
         
-        data[off] |= fragment << 3;
+        data[off] |= fragment << 1;
         if (fragment == state.getFragmentCount() - 1)
-            data[off] |= 1 << 2; // isLast
+            data[off] |= 1; // isLast
         off++;
         
         int size = state.fragmentSize(fragment);
         if (size < 0)
             return null;
         DataHelper.toLong(data, off, 2, size);
+        data[off] &= (byte)3F; // 2 highest bits are reserved
         off += 2;
         
         size = state.writeFragment(data, off, fragment);
@@ -81,12 +82,18 @@ public class PacketBuilder {
         if ( (off % 16) != 0)
             off += 16 - (off % 16);
         packet.getPacket().setLength(off);
+        packet.setPacketDataLength(off);
         authenticate(packet, peer.getCurrentCipherKey(), peer.getCurrentMACKey());
         setTo(packet, peer.getRemoteIPAddress(), peer.getRemotePort());
         return packet;
     }
     
-    public UDPPacket buildACK(PeerState peer, List ackedMessageIds) {
+    private static final int ACK_PRIORITY = 1;
+    
+    /**
+     * @param ackBitfields list of ACKBitfield instances to either fully or partially ACK
+     */
+    public UDPPacket buildACK(PeerState peer, List ackBitfields) {
         UDPPacket packet = UDPPacket.acquire(_context);
         
         byte data[] = packet.getPacket().getData();
@@ -101,18 +108,56 @@ public class PacketBuilder {
         DataHelper.toLong(data, off, 4, now);
         off += 4;
         
+        int fullACKCount = 0;
+        int partialACKCount = 0;
+        for (int i = 0; i < ackBitfields.size(); i++) {
+            if (((ACKBitfield)ackBitfields.get(i)).receivedComplete())
+                fullACKCount++;
+            else
+                partialACKCount++;
+        }
         // ok, now for the body...
+        if (fullACKCount > 0)
             data[off] |= UDPPacket.DATA_FLAG_EXPLICIT_ACK;
+        if (partialACKCount > 0)
+            data[off] |= UDPPacket.DATA_FLAG_ACK_BITFIELDS;
         // add ECN if (peer.getSomethingOrOther())
         off++;
         
-        DataHelper.toLong(data, off, 1, ackedMessageIds.size());
+        if (fullACKCount > 0) {
+            DataHelper.toLong(data, off, 1, fullACKCount);
             off++;
-        for (int i = 0; i < ackedMessageIds.size(); i++) {
+            for (int i = 0; i < ackBitfields.size(); i++) {
-            Long id = (Long)ackedMessageIds.get(i);
+                ACKBitfield bf = (ACKBitfield)ackBitfields.get(i);
-            DataHelper.toLong(data, off, 4, id.longValue());
+                if (bf.receivedComplete()) {
+                    DataHelper.toLong(data, off, 4, bf.getMessageId());
                     off += 4;
                 }
+            }
+        }
+        
+        if (partialACKCount > 0) {
+            DataHelper.toLong(data, off, 1, partialACKCount);
+            off++;
+            for (int i = 0; i < ackBitfields.size(); i++) {
+                ACKBitfield bitfield = (ACKBitfield)ackBitfields.get(i);
+                if (bitfield.receivedComplete()) continue;
+                DataHelper.toLong(data, off, 4, bitfield.getMessageId());
+                off += 4;
+                int bits = bitfield.fragmentCount();
+                int size = (bits / 7) + 1;
+                for (int curByte = 0; curByte < size; curByte++) {
+                    if (curByte + 1 < size)
+                        data[off] |= (byte)(1 << 7);
+                    
+                    for (int curBit = 0; curBit < 7; curBit++) {
+                        if (bitfield.received(curBit + 7*curByte))
+                            data[off] |= (byte)(1 << curBit);
+                    }
+                    off++;
+                }
+            }
+        }
         
         DataHelper.toLong(data, off, 1, 0); // no fragments in this message
         off++;
@@ -123,6 +168,7 @@ public class PacketBuilder {
         if ( (off % 16) != 0)
             off += 16 - (off % 16);
         packet.getPacket().setLength(off);
+        packet.setPacketDataLength(off);
         authenticate(packet, peer.getCurrentCipherKey(), peer.getCurrentMACKey());
         setTo(packet, peer.getRemoteIPAddress(), peer.getRemotePort());
         return packet;
@@ -142,12 +188,13 @@ public class PacketBuilder {
      */
     public UDPPacket buildSessionCreatedPacket(InboundEstablishState state, int externalPort, SessionKey ourIntroKey) {
         UDPPacket packet = UDPPacket.acquire(_context);
+        
         InetAddress to = null;
         try {
             to = InetAddress.getByAddress(state.getSentIP());
         } catch (UnknownHostException uhe) {
             if (_log.shouldLog(Log.ERROR))
-                _log.error("How did we think this was a valid IP?  " + state.getRemoteHostInfo());
+                _log.error("How did we think this was a valid IP?  " + state.getRemoteHostId().toString());
             return null;
         }
 
@@ -214,6 +261,7 @@ public class PacketBuilder {
         if ( (off % 16) != 0)
             off += 16 - (off % 16);
         packet.getPacket().setLength(off);
+        packet.setPacketDataLength(off);
         authenticate(packet, ourIntroKey, ourIntroKey, iv);
         setTo(packet, to, state.getSentPort());
         _ivCache.release(iv);
@@ -239,7 +287,7 @@ public class PacketBuilder {
             to = InetAddress.getByAddress(state.getSentIP());
         } catch (UnknownHostException uhe) {
             if (_log.shouldLog(Log.ERROR))
-                _log.error("How did we think this was a valid IP?  " + state.getRemoteHostInfo());
+                _log.error("How did we think this was a valid IP?  " + state.getRemoteHostId().toString());
             return null;
         }
         
@@ -272,6 +320,7 @@ public class PacketBuilder {
         if ( (off % 16) != 0)
             off += 16 - (off % 16);
         packet.getPacket().setLength(off);
+        packet.setPacketDataLength(off);
         authenticate(packet, state.getIntroKey(), state.getIntroKey());
         setTo(packet, to, state.getSentPort());
         return packet;
@@ -315,7 +364,7 @@ public class PacketBuilder {
             to = InetAddress.getByAddress(state.getSentIP());
         } catch (UnknownHostException uhe) {
             if (_log.shouldLog(Log.ERROR))
-                _log.error("How did we think this was a valid IP?  " + state.getRemoteHostInfo());
+                _log.error("How did we think this was a valid IP?  " + state.getRemoteHostId().toString());
             return null;
         }
         
@@ -366,6 +415,7 @@ public class PacketBuilder {
             
             System.arraycopy(state.getSentSignature().getData(), 0, data, off, Signature.SIGNATURE_BYTES);
             packet.getPacket().setLength(off + Signature.SIGNATURE_BYTES);
+            packet.setPacketDataLength(off + Signature.SIGNATURE_BYTES);
             authenticate(packet, state.getCipherKey(), state.getMACKey());
         } else {
             // nothing more to add beyond the identity fragment, though we can
@@ -375,6 +425,7 @@ public class PacketBuilder {
             if ( (off % 16) != 0)
                 off += 16 - (off % 16);
             packet.getPacket().setLength(off);
+            packet.setPacketDataLength(off);
             authenticate(packet, state.getIntroKey(), state.getIntroKey());
         } 
         
@@ -417,7 +468,7 @@ public class PacketBuilder {
      */
     private void authenticate(UDPPacket packet, SessionKey cipherKey, SessionKey macKey, ByteArray iv) {
         int encryptOffset = packet.getPacket().getOffset() + UDPPacket.IV_SIZE + UDPPacket.MAC_SIZE;
-        int encryptSize = packet.getPacket().getLength() - UDPPacket.IV_SIZE - UDPPacket.MAC_SIZE - packet.getPacket().getOffset();
+        int encryptSize = packet.getPacketDataLength()/*packet.getPacket().getLength()*/ - UDPPacket.IV_SIZE - UDPPacket.MAC_SIZE - packet.getPacket().getOffset();
         byte data[] = packet.getPacket().getData();
         _context.aes().encrypt(data, encryptOffset, data, encryptOffset, cipherKey, iv.getData(), encryptSize);
         
@@ -434,7 +485,7 @@ public class PacketBuilder {
         Hash hmac = _context.hmac().calculate(macKey, data, hmacOff, hmacLen);
         
         if (_log.shouldLog(Log.DEBUG))
-            _log.debug("Authenticating " + packet.getPacket().getLength() +
+            _log.debug("Authenticating " + packet.getPacketDataLength() + // packet.getPacket().getLength() +
                        "\nIV: " + Base64.encode(iv.getData()) +
                        "\nraw mac: " + hmac.toBase64() +
                        "\nMAC key: " + macKey.toBase64());

router/java/src/net/i2p/router/transport/udp/PacketHandler.java

@@ -41,6 +41,7 @@ public class PacketHandler {
         _context.statManager().createRateStat("udp.handleTime", "How long it takes to handle a received packet after its been pulled off the queue", "udp", new long[] { 10*60*1000, 60*60*1000 });
         _context.statManager().createRateStat("udp.queueTime", "How long after a packet is received can we begin handling it", "udp", new long[] { 10*60*1000, 60*60*1000 });
         _context.statManager().createRateStat("udp.receivePacketSkew", "How long ago after the packet was sent did we receive it", "udp", new long[] { 10*60*1000, 60*60*1000 });
+        _context.statManager().createRateStat("udp.droppedInvalid", "How old the packet we dropped due to invalidity was", "udp", new long[] { 10*60*1000, 60*60*1000 });
     }
     
     public void startup() { 
@@ -65,11 +66,17 @@ public class PacketHandler {
         public void run() {
             while (_keepReading) {
                 UDPPacket packet = _endpoint.receive();
+                if (packet == null) continue; // keepReading is probably false...
                 if (_log.shouldLog(Log.DEBUG))
                     _log.debug("Received the packet " + packet);
                 long queueTime = packet.getLifetime();
                 long handleStart = _context.clock().now();
+                try {
                     handlePacket(_reader, packet);
+                } catch (Exception e) {
+                    if (_log.shouldLog(Log.ERROR))
+                        _log.error("Crazy error handling a packet: " + packet, e);
+                }
                 long handleTime = _context.clock().now() - handleStart;
                 _context.statManager().addRateData("udp.handleTime", handleTime, packet.getLifetime());
                 _context.statManager().addRateData("udp.queueTime", queueTime, packet.getLifetime());
@@ -151,6 +158,7 @@ public class PacketHandler {
                     } else {
                         if (_log.shouldLog(Log.WARN))
                             _log.warn("Validation with existing con failed, and validation as reestablish failed too.  DROP");
+                        _context.statManager().addRateData("udp.droppedInvalid", packet.getLifetime(), packet.getExpiration());
                     }
                     return;
                 }
@@ -169,6 +177,7 @@ public class PacketHandler {
         if (!isValid) {
             if (_log.shouldLog(Log.WARN))
                 _log.warn("Invalid introduction packet received: " + packet, new Exception("path"));
+            _context.statManager().addRateData("udp.droppedInvalid", packet.getLifetime(), packet.getExpiration());
             return;
         } else {
             if (_log.shouldLog(Log.INFO))
@@ -214,6 +223,8 @@ public class PacketHandler {
             // ok, we couldn't handle it with the established stuff, so fall back
             // on earlier state packets
             receivePacket(reader, packet);
+        } else {
+            _context.statManager().addRateData("udp.droppedInvalid", packet.getLifetime(), packet.getExpiration());
         }
     }
 
@@ -272,16 +283,18 @@ public class PacketHandler {
         if (skew > GRACE_PERIOD) {
             if (_log.shouldLog(Log.WARN))
                 _log.warn("Packet too far in the future: " + new Date(sendOn/1000) + ": " + packet);
+            _context.statManager().addRateData("udp.droppedInvalid", packet.getLifetime(), packet.getExpiration());
             return;
         } else if (skew < 0 - GRACE_PERIOD) {
             if (_log.shouldLog(Log.WARN))
                 _log.warn("Packet too far in the past: " + new Date(sendOn/1000) + ": " + packet);
+            _context.statManager().addRateData("udp.droppedInvalid", packet.getLifetime(), packet.getExpiration());
             return;
         }
         
         if (state != null) {
             if (_log.shouldLog(Log.DEBUG))
-                _log.debug("Received packet from " + state.getRemoteHostString() + " with skew " + skew);
+                _log.debug("Received packet from " + state.getRemoteHostId().toString() + " with skew " + skew);
             state.adjustClockSkew((short)skew);
         }
         
@@ -289,7 +302,7 @@ public class PacketHandler {
         
         InetAddress fromHost = packet.getPacket().getAddress();
         int fromPort = packet.getPacket().getPort();
-        String from = PeerState.calculateRemoteHostString(fromHost.getAddress(), fromPort);
+        RemoteHostId from = new RemoteHostId(fromHost.getAddress(), fromPort);
         
         switch (reader.readPayloadType()) {
             case UDPPacket.PAYLOAD_TYPE_SESSION_REQUEST:
@@ -311,6 +324,7 @@ public class PacketHandler {
             default:
                 if (_log.shouldLog(Log.WARN))
                     _log.warn("Unknown payload type: " + reader.readPayloadType());
+                _context.statManager().addRateData("udp.droppedInvalid", packet.getLifetime(), packet.getExpiration());
                 return;
         }
     }

router/java/src/net/i2p/router/transport/udp/PeerState.java

@@ -103,8 +103,8 @@ public class PeerState {
     private transient InetAddress _remoteIPAddress;
     /** what port is the peer sending and receiving packets on? */
     private int _remotePort;
-    /** cached remoteIP + port, used to find the peerState by remote info */
+    /** cached RemoteHostId, used to find the peerState by remote info */
-    private String _remoteHostString;
+    private RemoteHostId _remoteHostId;
     /** if we need to contact them, do we need to talk to an introducer? */
     private boolean _remoteRequiresIntroduction;
     /** 
@@ -129,8 +129,20 @@ public class PeerState {
     /** current retransmission timeout */
     private volatile int _rto;
     
+    /** how many packets will be considered within the retransmission rate calculation */
+    static final long RETRANSMISSION_PERIOD_WIDTH = 100;
+    
     private long _messagesReceived;
     private long _messagesSent;
+    private long _packetsTransmitted;
+    /** how many packets were retransmitted within the last RETRANSMISSION_PERIOD_WIDTH packets */
+    private long _packetsRetransmitted;
+    private int _packetRetransmissionRate;
+    /** what was the $packetsTransmitted when the current RETRANSMISSION_PERIOD_WIDTH began */
+    private long _retransmissionPeriodStart;
+    /** how many dup packets were received within the last RETRANSMISSION_PERIOD_WIDTH packets */
+    private long _packetsReceivedDuplicate;
+    private long _packetsReceived;
     
     private static final int DEFAULT_SEND_WINDOW_BYTES = 8*1024;
     private static final int MINIMUM_WINDOW_BYTES = DEFAULT_SEND_WINDOW_BYTES;
@@ -141,8 +153,8 @@ public class PeerState {
      * packets.
      */
     private static final int DEFAULT_MTU = 576;
-    private static final int MIN_RTO = 600;
+    private static final int MIN_RTO = 500 + ACKSender.ACK_FREQUENCY;
-    private static final int MAX_RTO = 5000;
+    private static final int MAX_RTO = 2000; // 5000;
     
     public PeerState(I2PAppContext ctx) {
         _context = ctx;
@@ -181,11 +193,20 @@ public class PeerState {
         _lastACKSend = -1;
         _rtt = 1000;
         _rttDeviation = _rtt;
-        _rto = 6000;
+        _rto = MAX_RTO;
         _messagesReceived = 0;
         _messagesSent = 0;
+        _packetsTransmitted = 0;
+        _packetsRetransmitted = 0;
+        _packetRetransmissionRate = 0;
+        _retransmissionPeriodStart = 0;
+        _packetsReceived = 0;
+        _packetsReceivedDuplicate = 0;
         _context.statManager().createRateStat("udp.congestionOccurred", "How large the cwin was when congestion occurred (duration == sendBps)", "udp", new long[] { 60*60*1000, 24*60*60*1000 });
         _context.statManager().createRateStat("udp.congestedRTO", "retransmission timeout after congestion (duration == rtt dev)", "udp", new long[] { 60*60*1000, 24*60*60*1000 });
+        _context.statManager().createRateStat("udp.sendACKPartial", "Number of partial ACKs sent (duration == number of full ACKs in that ack packet)", "udp", new long[] { 60*60*1000, 24*60*60*1000 });
+        _context.statManager().createRateStat("udp.sendBps", "How fast we are transmitting when a packet is acked", "udp", new long[] { 10*60*1000, 60*60*1000, 24*60*60*1000 });
+        _context.statManager().createRateStat("udp.receiveBps", "How fast we are receiving when a packet is fully received (at most one per second)", "udp", new long[] { 10*60*1000, 60*60*1000, 24*60*60*1000 });
     }
     
     /**
@@ -377,7 +398,7 @@ public class PeerState {
         _remoteIP = ip;
         _remoteIPAddress = null;
         _remotePort = port; 
-        _remoteHostString = calculateRemoteHostString(ip, port);
+        _remoteHostId = new RemoteHostId(ip, port);
     }
     /** if we need to contact them, do we need to talk to an introducer? */
     public void setRemoteRequiresIntroduction(boolean required) { _remoteRequiresIntroduction = required; }
@@ -403,6 +424,14 @@ public class PeerState {
     public void messageFullyReceived(Long messageId, int bytes) {
         if (bytes > 0)
             _receiveBytes += bytes;
+        else {
+            if (_retransmissionPeriodStart + RETRANSMISSION_PERIOD_WIDTH < _packetsReceived) {
+                _packetsReceivedDuplicate++;
+            } else {
+                _retransmissionPeriodStart = _packetsReceived;
+                _packetsReceivedDuplicate = 1;
+            }
+        }
         
         long now = _context.clock().now();
         long duration = now - _receivePeriodBegin;
@@ -410,6 +439,7 @@ public class PeerState {
             _receiveBps = (int)(0.9f*(float)_receiveBps + 0.1f*((float)_receiveBytes * (1000f/(float)duration)));
             _receiveBytes = 0;
             _receivePeriodBegin = now;
+           _context.statManager().addRateData("udp.receiveBps", _receiveBps, 0);
         }
         
         synchronized (_currentACKs) {
@@ -421,6 +451,11 @@ public class PeerState {
         _messagesReceived++;
     }
     
+    public void messagePartiallyReceived() {
+        if (_wantACKSendSince <= 0)
+            _wantACKSendSince = _context.clock().now();
+    }
+    
     /** 
      * either they told us to back off, or we had to resend to get 
      * the data through.  
@@ -445,25 +480,65 @@ public class PeerState {
         return true;
     }
     
-    /** pull off the ACKs (Long) to send to the peer */
+    /** 
-    public List retrieveACKs() {
+     * grab a list of ACKBitfield instances, some of which may fully 
-        List rv = null;
+     * ACK a message while others may only partially ACK a message.  
-        int threshold = countMaxACKs();
+     * the values returned are limited in size so that they will fit within
+     * the peer's current MTU as an ACK - as such, not all messages may be
+     * ACKed with this call.  Be sure to check getWantedACKSendSince() which
+     * will be unchanged if there are ACKs remaining.
+     *
+     */
+    public List retrieveACKBitfields(UDPTransport.PartialACKSource partialACKSource) {
+        List rv = new ArrayList(16);
+        int bytesRemaining = countMaxACKData();
         synchronized (_currentACKs) {
-            if (_currentACKs.size() < threshold) {
+            while ( (bytesRemaining >= 4) && (_currentACKs.size() > 0) ) {
-                rv = new ArrayList(_currentACKs);
+                rv.add(new FullACKBitfield((Long)_currentACKs.remove(0)));
-                _currentACKs.clear();
+                bytesRemaining -= 4;
+            }
+            if (_currentACKs.size() <= 0)
                 _wantACKSendSince = -1;
+        }
+            
+        int partialIncluded = 0;
+        if ( (bytesRemaining > 4) && (partialACKSource != null) ) {
+            // ok, there's room to *try* to fit in some partial ACKs, so
+            // we should try to find some packets to partially ACK 
+            // (preferably the ones which have the most received fragments)
+            List partial = new ArrayList();
+            partialACKSource.fetchPartialACKs(_remotePeer, partial);
+            // we may not be able to use them all, but lets try...
+            for (int i = 0; (bytesRemaining > 4) && (i < partial.size()); i++) {
+                ACKBitfield bitfield = (ACKBitfield)partial.get(i);
+                int bytes = (bitfield.fragmentCount() / 7) + 1;
+                if (bytesRemaining > bytes + 4) { // msgId + bitfields
+                    rv.add(bitfield);
+                    bytesRemaining -= bytes + 4;
+                    partialIncluded++;
                 } else {
-                rv = new ArrayList(threshold);
+                    // continue on to another partial, in case there's a 
-                for (int i = 0; i < threshold; i++)
+                    // smaller one that will fit
-                    rv.add(_currentACKs.remove(0));
                 }
             }
+        }
+
+        _context.statManager().addRateData("udp.sendACKPartial", partialIncluded, rv.size() - partialIncluded);
         _lastACKSend = _context.clock().now();
         return rv;
     }
     
+    /** represent a full ACK of a message */
+    private class FullACKBitfield implements ACKBitfield {
+        private long _msgId;
+        public FullACKBitfield(Long id) { _msgId = id.longValue(); }
+        public int fragmentCount() { return 0; }
+        public long getMessageId() { return _msgId; }
+        public boolean received(int fragmentNum) { return true; }
+        public boolean receivedComplete() { return true; }
+        public String toString() { return "Full ACK of " + _msgId; }
+    }
+        
     /** we sent a message which was ACKed containing the given # of bytes */
     public void messageACKed(int bytesACKed, long lifetime, int numSends) {
         _consecutiveFailedSends = 0;
@@ -489,27 +564,52 @@ public class PeerState {
             recalculateTimeouts(lifetime);
         else
             _log.warn("acked after numSends=" + numSends + " w/ lifetime=" + lifetime + " and size=" + bytesACKed);
+        
+        _context.statManager().addRateData("udp.sendBps", _sendBps, lifetime);
     }
 
     /** adjust the tcp-esque timeouts */
     private void recalculateTimeouts(long lifetime) {
         _rttDeviation = _rttDeviation + (int)(0.25d*(Math.abs(lifetime-_rtt)-_rttDeviation));
-        _rtt = (int)((float)_rtt*(0.9f) + (0.1f)*(float)lifetime);
+        
+        // the faster we are going, the slower we want to reduce the rtt
+        float scale = 0.1f;
+        if (_sendBps > 0)
+            scale = ((float)lifetime) / (float)((float)lifetime + (float)_sendBps);
+        if (scale < 0.001f) scale = 0.001f;
+        
+        _rtt = (int)(((float)_rtt)*(1.0f-scale) + (scale)*(float)lifetime);
         _rto = _rtt + (_rttDeviation<<2);
         if (_log.shouldLog(Log.DEBUG))
             _log.debug("Recalculating timeouts w/ lifetime=" + lifetime + ": rtt=" + _rtt
                        + " rttDev=" + _rttDeviation + " rto=" + _rto);
-        if (_rto < MIN_RTO)
+        if (_rto < minRTO())
-            _rto = MIN_RTO;
+            _rto = minRTO();
         if (_rto > MAX_RTO)
             _rto = MAX_RTO;
     }
+    
     /** we are resending a packet, so lets jack up the rto */
-    public void messageRetransmitted() { 
+    public void messageRetransmitted(int packets) { 
+        if (_retransmissionPeriodStart + RETRANSMISSION_PERIOD_WIDTH < _packetsTransmitted) {
+            _packetsRetransmitted += packets;
+        } else {
+            _packetRetransmissionRate = (int)((float)(0.9f*_packetRetransmissionRate) + (float)(0.1f*_packetsRetransmitted));
+            _retransmissionPeriodStart = _packetsTransmitted;
+            _packetsRetransmitted = packets;
+        }
         congestionOccurred();
         _context.statManager().addRateData("udp.congestedRTO", _rto, _rttDeviation);
         //_rto *= 2; 
     }
+    public void packetsTransmitted(int packets) { 
+        _packetsTransmitted += packets; 
+        if (_retransmissionPeriodStart + RETRANSMISSION_PERIOD_WIDTH > _packetsTransmitted) {
+            _packetRetransmissionRate = (int)((float)(0.9f*_packetRetransmissionRate) + (float)(0.1f*_packetsRetransmitted));
+            _retransmissionPeriodStart = _packetsTransmitted;
+            _packetsRetransmitted = 0;
+        }
+    }
     /** how long does it usually take to get a message ACKed? */
     public int getRTT() { return _rtt; }
     /** how soon should we retransmit an unacked packet? */
@@ -519,6 +619,13 @@ public class PeerState {
     
     public long getMessagesSent() { return _messagesSent; }
     public long getMessagesReceived() { return _messagesReceived; }
+    public long getPacketsTransmitted() { return _packetsTransmitted; }
+    public long getPacketsRetransmitted() { return _packetsRetransmitted; }
+    /** avg number of packets retransmitted for every 100 packets */
+    public long getPacketRetransmissionRate() { return _packetRetransmissionRate; }
+    public long getPacketsReceived() { return _packetsReceived; }
+    public long getPacketsReceivedDuplicate() { return _packetsReceivedDuplicate; }
+    public void packetReceived() { _packetsReceived++; }
     
     /** 
      * we received a backoff request, so cut our send window
@@ -538,13 +645,13 @@ public class PeerState {
     public void setLastACKSend(long when) { _lastACKSend = when; }
     public long getWantedACKSendSince() { return _wantACKSendSince; }
     public boolean unsentACKThresholdReached() {
-        int threshold = countMaxACKs();
+        int threshold = countMaxACKData() / 4;
         synchronized (_currentACKs) {
             return _currentACKs.size() >= threshold;
         }
     }
-    private int countMaxACKs() {
+    private int countMaxACKData() {
-        return (_mtu 
+        return _mtu 
                 - OutboundMessageFragments.IP_HEADER_SIZE
                 - OutboundMessageFragments.UDP_HEADER_SIZE
                 - UDPPacket.IV_SIZE 
@@ -553,25 +660,19 @@ public class PeerState {
                 - 4 // timestamp
                 - 1 // data flag
                 - 1 // # ACKs
-                - 16 // padding safety
+                - 16; // padding safety
-               ) / 4;
     }
 
-    public String getRemoteHostString() { return _remoteHostString; }
+    private int minRTO() {
-
+        if (_packetRetransmissionRate < 10)
-    public static String calculateRemoteHostString(byte ip[], int port) {
+            return MIN_RTO;
-        StringBuffer buf = new StringBuffer(ip.length * 4 + 5);
+        else if (_packetRetransmissionRate < 50)
-        for (int i = 0; i < ip.length; i++)
+            return 2*MIN_RTO;
-            buf.append(ip[i]&0xFF).append('.');
+        else
-        buf.append(port);
+            return MAX_RTO;
-        return buf.toString();
     }
     
-    public static String calculateRemoteHostString(UDPPacket packet) {
+    public RemoteHostId getRemoteHostId() { return _remoteHostId; }
-        InetAddress remAddr = packet.getPacket().getAddress();
-        int remPort = packet.getPacket().getPort();
-        return calculateRemoteHostString(remAddr.getAddress(), remPort);
-    }
     
     public int hashCode() {
         if (_remotePeer != null) 
@@ -594,7 +695,7 @@ public class PeerState {
     
     public String toString() {
         StringBuffer buf = new StringBuffer(64);
-        buf.append(_remoteHostString);
+        buf.append(_remoteHostId.toString());
         if (_remotePeer != null)
             buf.append(" ").append(_remotePeer.toBase64().substring(0,6));
         return buf.toString();

router/java/src/net/i2p/router/transport/udp/RemoteHostId.java

@@ -0,0 +1,46 @@
+package net.i2p.router.transport.udp;
+
+import net.i2p.data.DataHelper;
+
+/**
+ * Unique ID for a peer - its IP + port, all bundled into a tidy obj.
+ * Aint it cute?
+ *
+ */
+final class RemoteHostId {
+    private byte _ip[];
+    private int _port;
+    
+    public RemoteHostId(byte ip[], int port) {
+        _ip = ip;
+        _port = port;
+    }
+    
+    public byte[] getIP() { return _ip; }
+    public int getPort() { return _port; }
+    
+    public int hashCode() {
+        int rv = 0;
+        for (int i = 0; i < _ip.length; i++)
+            rv += _ip[i] << i;
+        rv += _port;
+        return rv;
+    }
+    
+    public boolean equals(Object obj) {
+        if (obj == null) 
+            throw new NullPointerException("obj is null");
+        if (!(obj instanceof RemoteHostId)) 
+            throw new ClassCastException("obj is a " + obj.getClass().getName());
+        RemoteHostId id = (RemoteHostId)obj;
+        return (_port == id.getPort()) && DataHelper.eq(_ip, id.getIP());
+    }
+    
+    public String toString() {
+        StringBuffer buf = new StringBuffer(_ip.length + 5);
+        for (int i = 0; i < _ip.length; i++)
+            buf.append(_ip[i]&0xFF).append('.');
+        buf.append(_port);
+        return buf.toString();
+    }
+}

router/java/src/net/i2p/router/transport/udp/TimedWeightedPriorityMessageQueue.java

@@ -2,6 +2,7 @@ package net.i2p.router.transport.udp;
 
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
@@ -74,7 +75,7 @@ public class TimedWeightedPriorityMessageQueue implements MessageQueue, Outbound
         _alive = true;
         _nextLock = this;
         _nextQueue = 0;
-        _chokedPeers = new HashSet(16);
+        _chokedPeers = Collections.synchronizedSet(new HashSet(16));
         _listener = lsnr;
         _context.statManager().createRateStat("udp.timeToEntrance", "Message lifetime until it reaches the UDP system", "udp", new long[] { 60*1000, 10*60*1000, 60*60*1000 });
         _context.statManager().createRateStat("udp.messageQueueSize", "How many messages are on the current class queue at removal", "udp", new long[] { 60*1000, 10*60*1000, 60*60*1000 });
@@ -123,10 +124,9 @@ public class TimedWeightedPriorityMessageQueue implements MessageQueue, Outbound
                     for (int j = 0; j < _queue[currentQueue].size(); j++) {
                         OutNetMessage msg = (OutNetMessage)_queue[currentQueue].get(j);
                         Hash to = msg.getTarget().getIdentity().getHash();
-                        synchronized (_nextLock) { // yikes!
                         if (_chokedPeers.contains(to))
                             continue;
-                        }
+                        
                         // not choked, lets push it to active
                         _queue[currentQueue].remove(j);
                         
@@ -189,23 +189,21 @@ public class TimedWeightedPriorityMessageQueue implements MessageQueue, Outbound
     
     public void choke(Hash peer) {
         if (true) return;
-        synchronized (_nextLock) {
         _chokedPeers.add(peer);
+        synchronized (_nextLock) {
             _nextLock.notifyAll();
         }
     }
     public void unchoke(Hash peer) {
         if (true) return;
-        synchronized (_nextLock) {
         _chokedPeers.remove(peer);
+        synchronized (_nextLock) {
             _nextLock.notifyAll();
         }
     }
     public boolean isChoked(Hash peer) {
-        synchronized (_nextLock) {
         return _chokedPeers.contains(peer);
     }
-    }
     
     private int pickQueue(OutNetMessage message) {
         int target = message.getPriority();

router/java/src/net/i2p/router/transport/udp/UDPEndpoint.java

@@ -26,6 +26,8 @@ public class UDPEndpoint {
     }
     
     public void startup() {
+        if (_log.shouldLog(Log.DEBUG))
+            _log.debug("Starting up the UDP endpoint");
         shutdown();
         try {
             DatagramSocket socket = new DatagramSocket(_listenPort);

router/java/src/net/i2p/router/transport/udp/UDPEndpointTest.java

@@ -3,7 +3,12 @@ package net.i2p.router.transport.udp;
 import java.net.InetAddress;
 import java.net.SocketException;
 import java.net.UnknownHostException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Properties;
 
+import net.i2p.data.ByteArray;
 import net.i2p.router.RouterContext;
 import net.i2p.util.I2PThread;
 import net.i2p.util.Log;
@@ -16,20 +21,22 @@ public class UDPEndpointTest {
     private Log _log;
     private UDPEndpoint _endpoints[];
     private boolean _beginTest;
+    private List _sentNotReceived;
     
     public UDPEndpointTest(RouterContext ctx) {
         _context = ctx;
         _log = ctx.logManager().getLog(UDPEndpointTest.class);
+        _sentNotReceived = Collections.synchronizedList(new ArrayList(1000));
     }
     
     public void runTest(int numPeers) {
-        RouterContext ctx = new RouterContext(null);
+        _log.debug("Run test("+numPeers+")");
         try {
             _endpoints = new UDPEndpoint[numPeers];
-            int base = 2000 + ctx.random().nextInt(10000);
+            int base = 2000 + _context.random().nextInt(10000);
             for (int i = 0; i < numPeers; i++) {
                 _log.debug("Building " + i);
-                UDPEndpoint endpoint = new UDPEndpoint(ctx, base + i);
+                UDPEndpoint endpoint = new UDPEndpoint(_context, base + i);
                 _endpoints[i] = endpoint;
                 endpoint.startup();
                 I2PThread read = new I2PThread(new TestRead(endpoint), "Test read " + i);
@@ -62,11 +69,20 @@ public class UDPEndpointTest {
             int received = 0;
             while (true) {
                 UDPPacket packet = _endpoint.receive();
+                ByteArray ba = new ByteArray(packet.getPacket().getData(), 0, packet.getPacket().getLength());
+                boolean removed = _sentNotReceived.remove(ba);
+                int outstanding = _sentNotReceived.size();
+                if (!removed) {
+                    _log.error("Received a packet that we weren't expecting: " + packet);
+                } else {
+                    _log.debug("Received an expected packet (" + received + ") with outstanding: " + outstanding);
                     received++;
-                if (received == 10000) {
-                    long time = System.currentTimeMillis() - start;
-                    _log.debug("Received 10000 in " + time);
                 }
+                if ((received % 10000) == 0) {
+                    long time = System.currentTimeMillis() - start;
+                    _log.debug("Received "+received+" in " + time);
+                }
+                packet.release();
             }
         }
     }
@@ -78,8 +94,9 @@ public class UDPEndpointTest {
         }
         public void run() {
             while (!_beginTest) {
-                try { Thread.sleep(1000); } catch (InterruptedException ie) {}
+                try { Thread.sleep(2000); } catch (InterruptedException ie) {}
             }
+                try { Thread.sleep(2000); } catch (InterruptedException ie) {}
             _log.debug("Beginning to write");
             for (int curPacket = 0; curPacket < 10000; curPacket++) {
                 byte data[] = new byte[1024];
@@ -91,11 +108,16 @@ public class UDPEndpointTest {
                     curPeer = 0;
                 short priority = 1;
                 long expiration = -1;
-                try {
                 UDPPacket packet = UDPPacket.acquire(_context);
+                try {
                     packet.initialize(priority, expiration, InetAddress.getLocalHost(), _endpoints[curPeer].getListenPort());
                     packet.writeData(data, 0, 1024);
+                    packet.setPacketDataLength(1024);
+                    int outstanding = _sentNotReceived.size() + 1;
+                    _sentNotReceived.add(new ByteArray(data, 0, 1024));
+                    _log.debug("Sending packet " + curPacket + " with outstanding " + outstanding);
                     _endpoint.send(packet);
+                    //try { Thread.sleep(10); } catch (InterruptedException ie) {}
                 } catch (UnknownHostException uhe) {
                     _log.error("foo!", uhe);
                 }
@@ -103,11 +125,18 @@ public class UDPEndpointTest {
                 //    _log.debug("Sent to " + _endpoints[curPeer].getListenPort() + " from " + _endpoint.getListenPort());
                 //}
             }
+            try { Thread.sleep(10*1000); } catch (InterruptedException e) {}
+            System.exit(0);
         }
     }
     
     public static void main(String args[]) {
-        UDPEndpointTest test = new UDPEndpointTest(new RouterContext(null));
+        try { System.out.println("Current dir: " + new java.io.File(".").getCanonicalPath()); } catch (Exception e) {}
+        new java.io.File("udpEndpointTest.stats").delete();
+        Properties props = new Properties();
+        props.setProperty("stat.logFile", "udpEndpointTest.stats");
+        props.setProperty("stat.logFilters", "*");
+        UDPEndpointTest test = new UDPEndpointTest(new RouterContext(null, props));
         test.runTest(2);
     }
 }

router/java/src/net/i2p/router/transport/udp/UDPFlooder.java

@@ -31,13 +31,15 @@ class UDPFlooder implements Runnable {
     
     public void addPeer(PeerState peer) {
         synchronized (_peers) {
+            if (!_peers.contains(peer))
                 _peers.add(peer);
             _peers.notifyAll();
         }
     }
     public void removePeer(PeerState peer) {
         synchronized (_peers) {
-            _peers.remove(peer);
+            while (_peers.remove(peer))
+                ;// loops until its empty
             _peers.notifyAll();
         }
     }

router/java/src/net/i2p/router/transport/udp/UDPPacket.java

@@ -25,13 +25,17 @@ import net.i2p.util.Log;
 public class UDPPacket {
     private I2PAppContext _context;
     private static Log _log;
-    private DatagramPacket _packet;
+    private volatile DatagramPacket _packet;
-    private short _priority;
+    private volatile int _packetDataLength;
-    private long _initializeTime;
+    private volatile short _priority;
-    private long _expiration;
+    private volatile long _initializeTime;
-    private byte[] _data;
+    private volatile long _expiration;
-    private ByteArray _dataBuf;
+    private volatile byte[] _data;
-    private int _markedType;
+    private volatile ByteArray _dataBuf;
+    private volatile int _markedType;
+    private volatile boolean _released;
+    private volatile Exception _releasedBy;
+    private volatile Exception _acquiredBy;
   
     private static final List _packetCache;
     static {
@@ -39,9 +43,9 @@ public class UDPPacket {
         _log = I2PAppContext.getGlobalContext().logManager().getLog(UDPPacket.class);
     }
     
-    private static final boolean CACHE = false;
+    private static final boolean CACHE = false; // TODO: support caching to cut churn down a /lot/
       
-    private static final int MAX_PACKET_SIZE = 2048;
+    static final int MAX_PACKET_SIZE = 2048;
     public static final int IV_SIZE = 16;
     public static final int MAC_SIZE = 16;
     
@@ -55,13 +59,14 @@ public class UDPPacket {
     
     // various flag fields for use in the data packets
     public static final byte DATA_FLAG_EXPLICIT_ACK = (byte)(1 << 7);
-    public static final byte DATA_FLAG_EXPLICIT_NACK = (1 << 6);
+    public static final byte DATA_FLAG_ACK_BITFIELDS = (1 << 6);
-    public static final byte DATA_FLAG_NUMACKS = (1 << 5);
     public static final byte DATA_FLAG_ECN = (1 << 4);
     public static final byte DATA_FLAG_WANT_ACKS = (1 << 3);
     public static final byte DATA_FLAG_WANT_REPLY = (1 << 2);
     public static final byte DATA_FLAG_EXTENDED = (1 << 1);
     
+    public static final byte BITFIELD_CONTINUATION = (byte)(1 << 7);
+    
     private static final int MAX_VALIDATE_SIZE = MAX_PACKET_SIZE;
     private static final ByteCache _validateCache = ByteCache.getInstance(16, MAX_VALIDATE_SIZE);
     private static final ByteCache _ivCache = ByteCache.getInstance(16, IV_SIZE);
@@ -72,34 +77,46 @@ public class UDPPacket {
         _dataBuf = _dataCache.acquire();
         _data = _dataBuf.getData(); 
         _packet = new DatagramPacket(_data, MAX_PACKET_SIZE);
+        _packetDataLength = 0;
         _initializeTime = _context.clock().now();
         _markedType = -1;
     }
     
-    public void initialize(short priority, long expiration, InetAddress host, int port) {
+    public void initialize(int priority, long expiration, InetAddress host, int port) {
-        _priority = priority;
+        _priority = (short)priority;
         _expiration = expiration;
         resetBegin();
         Arrays.fill(_data, (byte)0x00);
-        _packet.setLength(0);
+        //_packet.setLength(0);
         _packet.setAddress(host);
         _packet.setPort(port);
+        _released = false;
+        _releasedBy = null;
     }
     
     public void writeData(byte src[], int offset, int len) { 
+        verifyNotReleased();
         System.arraycopy(src, offset, _data, 0, len);
         _packet.setLength(len);
+        setPacketDataLength(len);
         resetBegin();
     }
-    public DatagramPacket getPacket() { return _packet; }
+    public DatagramPacket getPacket() { verifyNotReleased(); return _packet; }
-    public short getPriority() { return _priority; }
+    public short getPriority() { verifyNotReleased(); return _priority; }
-    public long getExpiration() { return _expiration; }
+    public long getExpiration() { verifyNotReleased(); return _expiration; }
-    public long getBegin() { return _initializeTime; }
+    public long getBegin() { verifyNotReleased(); return _initializeTime; }
-    public long getLifetime() { return _context.clock().now() - _initializeTime; }
+    public long getLifetime() { verifyNotReleased(); return _context.clock().now() - _initializeTime; }
+    public int getPacketDataLength() { return _packetDataLength; }
+    public void setPacketDataLength(int bytes) { _packetDataLength = bytes; }
     public void resetBegin() { _initializeTime = _context.clock().now(); }
     /** flag this packet as a particular type for accounting purposes */
-    public void markType(int type) { _markedType = type; }
+    public void markType(int type) { verifyNotReleased(); _markedType = type; }
-    public int getMarkedType() { return _markedType; }
+    /** 
+     * flag this packet as a particular type for accounting purposes, with
+     * 1 implying the packet is an ACK, otherwise it is a data packet
+     *
+     */
+    public int getMarkedType() { verifyNotReleased(); return _markedType; }
     
     /**
      * Validate the packet against the MAC specified, returning true if the
@@ -107,13 +124,14 @@ public class UDPPacket {
      *
      */
     public boolean validate(SessionKey macKey) {
+        verifyNotReleased(); 
         boolean eq = false;
         ByteArray buf = _validateCache.acquire();
         
         // validate by comparing _data[0:15] and
         // HMAC(payload + IV + payloadLength, macKey)
         
-        int payloadLength = _packet.getLength() - MAC_SIZE - IV_SIZE;
+        int payloadLength = _packetDataLength /*_packet.getLength()*/ - MAC_SIZE - IV_SIZE;
         if (payloadLength > 0) {
             int off = 0;
             System.arraycopy(_data, _packet.getOffset() + MAC_SIZE + IV_SIZE, buf.getData(), off, payloadLength);
@@ -123,6 +141,8 @@ public class UDPPacket {
             DataHelper.toLong(buf.getData(), off, 2, payloadLength);
             off += 2;
 
+            eq = _context.hmac().verify(macKey, buf.getData(), 0, off, _data, _packet.getOffset(), MAC_SIZE);
+            /*
             Hash hmac = _context.hmac().calculate(macKey, buf.getData(), 0, off);
 
             if (_log.shouldLog(Log.DEBUG)) {
@@ -139,6 +159,7 @@ public class UDPPacket {
                 _log.debug(str.toString());
             }
             eq = DataHelper.eq(hmac.getData(), 0, _data, _packet.getOffset(), MAC_SIZE);
+             */
         } else {
             if (_log.shouldLog(Log.WARN))
                 _log.warn("Payload length is " + payloadLength);
@@ -154,47 +175,83 @@ public class UDPPacket {
      * 
      */
     public void decrypt(SessionKey cipherKey) {
+        verifyNotReleased(); 
         ByteArray iv = _ivCache.acquire();
         System.arraycopy(_data, MAC_SIZE, iv.getData(), 0, IV_SIZE);
-        _context.aes().decrypt(_data, _packet.getOffset() + MAC_SIZE + IV_SIZE, _data, _packet.getOffset() + MAC_SIZE + IV_SIZE, cipherKey, iv.getData(), _packet.getLength() - MAC_SIZE - IV_SIZE);
+        int len = _packetDataLength; // _packet.getLength()
+        _context.aes().decrypt(_data, _packet.getOffset() + MAC_SIZE + IV_SIZE, _data, _packet.getOffset() + MAC_SIZE + IV_SIZE, cipherKey, iv.getData(), len - MAC_SIZE - IV_SIZE);
         _ivCache.release(iv);
     }
     
     public String toString() {
+        verifyNotReleased(); 
         StringBuffer buf = new StringBuffer(64);
-        buf.append(_packet.getLength());
+        buf.append(_packetDataLength);
         buf.append(" byte packet with ");
         buf.append(_packet.getAddress().getHostAddress()).append(":");
         buf.append(_packet.getPort());
+        buf.append(" id=").append(System.identityHashCode(this));
+            buf.append(" data=").append(Base64.encode(_packet.getData(), _packet.getOffset(), _packet.getLength()));
         return buf.toString();
     }
     
-    
     public static UDPPacket acquire(I2PAppContext ctx) {
+        UDPPacket rv = null;
         if (CACHE) {
             synchronized (_packetCache) {
                 if (_packetCache.size() > 0) {
-                    UDPPacket rv = (UDPPacket)_packetCache.remove(0);
+                    rv = (UDPPacket)_packetCache.remove(0);
+                }
+            }
+            /*
+            if (rv != null) {
                 rv._context = ctx;
-                    rv._log = ctx.logManager().getLog(UDPPacket.class);
+                //rv._log = ctx.logManager().getLog(UDPPacket.class);
                 rv.resetBegin();
                 Arrays.fill(rv._data, (byte)0x00);
                 rv._markedType = -1;
+                rv._dataBuf.setValid(0);
+                rv._released = false;
+                rv._releasedBy = null;
+                rv._acquiredBy = null;
+                rv.setPacketDataLength(0);
+                synchronized (rv._packet) {
+                    //rv._packet.setLength(0);
+                    //rv._packet.setPort(1);
+                }
+            }
+             */
+        }
+        if (rv == null)
+            rv = new UDPPacket(ctx);
+        //if (rv._acquiredBy != null) {
+        //    _log.log(Log.CRIT, "Already acquired!  current stack trace is:", new Exception());
+        //    _log.log(Log.CRIT, "Earlier acquired:", rv._acquiredBy);
+        //}
+        //rv._acquiredBy = new Exception("acquired on");
         return rv;
     }
-            }
-        }
-        return new UDPPacket(ctx);
-    }
 
     public void release() {
-        _dataCache.release(_dataBuf);
+        verifyNotReleased();
+        _released = true;
+        //_releasedBy = new Exception("released by");
+        //_acquiredBy = null;
+        //_dataCache.release(_dataBuf);
         if (!CACHE) return;
         synchronized (_packetCache) {
-            _packet.setLength(0);
+            if (_packetCache.size() <= 64) {
-            _packet.setPort(1);
-            if (_packetCache.size() <= 64)
                 _packetCache.add(this);
             }
         }
+    }
+    
+    private void verifyNotReleased() {
+        if (CACHE) return;
+        if (_released) {
+            _log.log(Log.CRIT, "Already released.  current stack trace is:", new Exception());
+            _log.log(Log.CRIT, "Released by: ", _releasedBy);
+            _log.log(Log.CRIT, "Acquired by: ", _acquiredBy);
+        }
+    }
 }

router/java/src/net/i2p/router/transport/udp/UDPPacketReader.java

@@ -37,7 +37,7 @@ public class UDPPacketReader {
     
     public void initialize(UDPPacket packet) {
         int off = packet.getPacket().getOffset();
-        int len = packet.getPacket().getLength();
+        int len = packet.getPacketDataLength(); //packet.getPacket().getLength();
         off += UDPPacket.MAC_SIZE + UDPPacket.IV_SIZE;
         len -= UDPPacket.MAC_SIZE + UDPPacket.IV_SIZE;
         initialize(packet.getPacket().getData(), off, len);
@@ -234,11 +234,8 @@ public class UDPPacketReader {
         public boolean readACKsIncluded() {
             return flagSet(UDPPacket.DATA_FLAG_EXPLICIT_ACK);
         }
-        public boolean readNACKsIncluded() {
+        public boolean readACKBitfieldsIncluded() {
-            return flagSet(UDPPacket.DATA_FLAG_EXPLICIT_NACK);
+            return flagSet(UDPPacket.DATA_FLAG_ACK_BITFIELDS);
-        }
-        public boolean readNumACKsIncluded() {
-            return flagSet(UDPPacket.DATA_FLAG_NUMACKS);
         }
         public boolean readECN() {
             return flagSet(UDPPacket.DATA_FLAG_ECN);
@@ -264,8 +261,8 @@ public class UDPPacketReader {
             }
             return rv;
         }
-        public long[] readNACKs() {
+        public ACKBitfield[] readACKBitfields() {
-            if (!readNACKsIncluded()) return null;
+            if (!readACKBitfieldsIncluded()) return null;
             int off = readBodyOffset() + 1;
             if (readACKsIncluded()) {
                 int numACKs = (int)DataHelper.fromLong(_message, off, 1);
@@ -273,31 +270,16 @@ public class UDPPacketReader {
                 off += 4 * numACKs;
             }
             
-            int numNACKs = (int)DataHelper.fromLong(_message, off, 1);
+            int numBitfields = (int)DataHelper.fromLong(_message, off, 1);
             off++;
-            long rv[] = new long[numNACKs];
+            
-            for (int i = 0; i < numNACKs; i++) {
+            PacketACKBitfield rv[] = new PacketACKBitfield[numBitfields];
-                rv[i] = DataHelper.fromLong(_message, off, 4);
+            for (int i = 0; i < numBitfields; i++) {
-                off += 4;
+                rv[i] = new PacketACKBitfield(off);
+                off += rv[i].getByteLength();
             }
             return rv;
         }
-        public int readNumACKs() {
-            if (!readNumACKsIncluded()) return -1;
-            int off = readBodyOffset() + 1;
-            
-            if (readACKsIncluded()) {
-                int numACKs = (int)DataHelper.fromLong(_message, off, 1);
-                off++;
-                off += 4 * numACKs;
-            }
-            if (readNACKsIncluded()) {
-                int numNACKs = (int)DataHelper.fromLong(_message, off, 1);
-                off++;
-                off += 4 * numNACKs;
-            }
-            return (int)DataHelper.fromLong(_message, off, 2);
-        }
         
         public int readFragmentCount() {
             int off = readBodyOffset() + 1;
@@ -306,13 +288,15 @@ public class UDPPacketReader {
                 off++;
                 off += 4 * numACKs;
             }
-            if (readNACKsIncluded()) {
+            if (readACKBitfieldsIncluded()) {
-                int numNACKs = (int)DataHelper.fromLong(_message, off, 1);
+                int numBitfields = (int)DataHelper.fromLong(_message, off, 1);
                 off++;
-                off += 4 * numNACKs;
+
+                for (int i = 0; i < numBitfields; i++) {
+                    PacketACKBitfield bf = new PacketACKBitfield(off);
+                    off += bf.getByteLength();
+                }
             }
-            if (readNumACKsIncluded())
-                off += 2;
             if (readExtendedDataIncluded()) {
                 int size = (int)DataHelper.fromLong(_message, off, 1);
                 off++;
@@ -328,24 +312,24 @@ public class UDPPacketReader {
         public int readMessageFragmentNum(int fragmentNum) {
             int off = getFragmentBegin(fragmentNum);
             off += 4; // messageId
-            return (_message[off] & 0xFF) >>> 3;
+            return (_message[off] & 0xFF) >>> 1;
         }
         public boolean readMessageIsLast(int fragmentNum) {
             int off = getFragmentBegin(fragmentNum);
             off += 4; // messageId
-            return ((_message[off] & (1 << 2)) != 0);
+            return ((_message[off] & 1) != 0);
         }
         public int readMessageFragmentSize(int fragmentNum) {
             int off = getFragmentBegin(fragmentNum);
             off += 4; // messageId
             off++; // fragment info
-            return (int)DataHelper.fromLong(_message, off, 2);
+            return ((int)DataHelper.fromLong(_message, off, 2)) & 0x3FFF;
         }
         public void readMessageFragment(int fragmentNum, byte target[], int targetOffset) {
             int off = getFragmentBegin(fragmentNum);
             off += 4; // messageId
             off++; // fragment info
-            int size = (int)DataHelper.fromLong(_message, off, 2);
+            int size = ((int)DataHelper.fromLong(_message, off, 2)) & 0x3FFF;
             off += 2;
             System.arraycopy(_message, off, target, targetOffset, size);
         }
@@ -357,13 +341,16 @@ public class UDPPacketReader {
                 off++;
                 off += 4 * numACKs;
             }
-            if (readNACKsIncluded()) {
+            if (readACKBitfieldsIncluded()) {
-                int numNACKs = (int)DataHelper.fromLong(_message, off, 1);
+                int numBitfields = (int)DataHelper.fromLong(_message, off, 1);
                 off++;
-                off += 5 * numNACKs;
+
+                PacketACKBitfield bf[] = new PacketACKBitfield[numBitfields];
+                for (int i = 0; i < numBitfields; i++) {
+                    bf[i] = new PacketACKBitfield(off);
+                    off += bf[i].getByteLength();
+                }
             }
-            if (readNumACKsIncluded())
-                off += 2;
             if (readExtendedDataIncluded()) {
                 int size = (int)DataHelper.fromLong(_message, off, 1);
                 off++;
@@ -376,7 +363,7 @@ public class UDPPacketReader {
             } else {
                 for (int i = 0; i < fragmentNum; i++) {
                     off += 5; // messageId+info
-                    off += (int)DataHelper.fromLong(_message, off, 2);
+                    off += ((int)DataHelper.fromLong(_message, off, 2)) & 0x3FFF;
                     off += 2;
                 }
                 return off;
@@ -405,21 +392,16 @@ public class UDPPacketReader {
                     off += 4;
                 }
             }
-            if (readNACKsIncluded()) {
+            if (readACKBitfieldsIncluded()) {
-                int numNACKs = (int)DataHelper.fromLong(_message, off, 1);
+                int numBitfields = (int)DataHelper.fromLong(_message, off, 1);
                 off++;
-                buf.append("with NACKs for ");
+                buf.append("with partial ACKs for ");
-                for (int i = 0; i < numNACKs; i++) {
+
-                    buf.append(DataHelper.fromLong(_message, off, 4)).append(' ');
+                for (int i = 0; i < numBitfields; i++) {
-                    off += 5;
+                    PacketACKBitfield bf = new PacketACKBitfield(off);
+                    buf.append(bf.getMessageId()).append(' ');
+                    off += bf.getByteLength();
                 }
-                off += 5 * numNACKs;
-            }
-            if (readNumACKsIncluded()) {
-                buf.append("with numACKs of ");
-                buf.append(DataHelper.fromLong(_message, off, 2));
-                buf.append(' ');
-                off += 2;
             }
             if (readExtendedDataIncluded()) {
                 int size = (int)DataHelper.fromLong(_message, off, 1);
@@ -440,13 +422,13 @@ public class UDPPacketReader {
                 buf.append("containing messageId ");
                 buf.append(DataHelper.fromLong(_message, off, 4));
                 off += 4;
-                int fragNum = (_message[off] & 0XFF) >>> 3;
+                int fragNum = (_message[off] & 0xFF) >>> 1;
-                boolean isLast = (_message[off] & (1 << 2)) != 0;
+                boolean isLast = (_message[off] & 1) != 0;
                 off++;
                 buf.append(" frag# ").append(fragNum);
                 buf.append(" isLast? ").append(isLast);
                 buf.append(" info ").append((int)_message[off-1]);
-                int size = (int)DataHelper.fromLong(_message, off, 2);
+                int size = ((int)DataHelper.fromLong(_message, off, 2)) & 0x3FFF;
                 buf.append(" with ").append(size).append(" bytes");
                 buf.append(' ');
                 off += size;
@@ -463,9 +445,50 @@ public class UDPPacketReader {
             int off = getFragmentBegin(0); // first fragment
             off += 4; // messageId
             off++; // fragment info
-            int size = (int)DataHelper.fromLong(_message, off, 2);
+            int size = ((int)DataHelper.fromLong(_message, off, 2)) & 0x3FFF;
             off += 2;
             buf.append(Base64.encode(_message, off, size));
         }
     }
+    
+    /**
+     * Helper class to fetch the particular bitfields from the raw packet
+     */   
+    private class PacketACKBitfield implements ACKBitfield {
+        private int _start;
+        private int _bitfieldStart;
+        private int _bitfieldSize;
+        public PacketACKBitfield(int start) {
+            _start = start;
+            _bitfieldStart = start + 4;
+            _bitfieldSize = 1;
+            // bitfield is an array of bytes where the high bit is 1 if 
+            // further bytes in the bitfield follow
+            while ((_message[_bitfieldStart + _bitfieldSize - 1] & UDPPacket.BITFIELD_CONTINUATION) != 0x0)
+                _bitfieldSize++;
+        }
+        public long getMessageId() { return DataHelper.fromLong(_message, _start, 4); }
+        public int getByteLength() { return 4 + _bitfieldSize; }
+        public int fragmentCount() { return _bitfieldSize * 7; }
+        public boolean receivedComplete() { return false; }
+        public boolean received(int fragmentNum) {
+            if ( (fragmentNum < 0) || (fragmentNum >= _bitfieldSize*7) )
+                return false;
+            // the fragment has been received if the bit is set
+            int byteNum = _bitfieldStart + (fragmentNum/7);
+            int flagNum = fragmentNum % 7;
+            return (_message[byteNum] & (1 << flagNum)) != 0x0;
+        }
+        public String toString() { 
+            StringBuffer buf = new StringBuffer(64);
+            buf.append("Read partial ACK of ");
+            buf.append(getMessageId());
+            buf.append(" with ACKs for: ");
+            int numFrags = fragmentCount();
+            for (int i = 0; i < numFrags; i++)
+                if (received(i))
+                    buf.append(i).append(" ");
+            return buf.toString();
+        }
+    }
 }

router/java/src/net/i2p/router/transport/udp/UDPReceiver.java

@@ -8,6 +8,7 @@ import java.util.ArrayList;
 import java.util.List;
 import net.i2p.router.RouterContext;
 import net.i2p.router.transport.FIFOBandwidthLimiter;
+import net.i2p.util.SimpleTimer;
 import net.i2p.util.I2PThread;
 import net.i2p.util.Log;
 
@@ -66,33 +67,50 @@ public class UDPReceiver {
     /** if a packet been sitting in the queue for 2 seconds, drop subsequent packets */
     private static final long MAX_QUEUE_PERIOD = 2*1000;
     
-    private static final float ARTIFICIAL_DROP_PROBABILITY = 0f; //0.02f;
+    private static final float ARTIFICIAL_DROP_PROBABILITY = 0.0f; // 0.02f; // 0.0f;
     
-    private void receive(UDPPacket packet) {
+    private static final int ARTIFICIAL_DELAY = 0; // 100;
+    private static final int ARTIFICIAL_DELAY_BASE = 0; //100;
+    
+    private int receive(UDPPacket packet) {
         if (ARTIFICIAL_DROP_PROBABILITY > 0) { 
             // the first check is to let the compiler optimize away this 
             // random block on the live system when the probability is == 0
             if (_context.random().nextFloat() <= ARTIFICIAL_DROP_PROBABILITY)
-                return;
+                return -1;
         }
         
+        if ( (ARTIFICIAL_DELAY > 0) || (ARTIFICIAL_DELAY_BASE > 0) ) {
+            SimpleTimer.getInstance().addEvent(new ArtificiallyDelayedReceive(packet), ARTIFICIAL_DELAY_BASE + _context.random().nextInt(ARTIFICIAL_DELAY));
+        }
+        
+        return doReceive(packet);
+    }
+    private final int doReceive(UDPPacket packet) {
         synchronized (_inboundQueue) {
             int queueSize = _inboundQueue.size();
             if (queueSize > 0) {
                 long headPeriod = ((UDPPacket)_inboundQueue.get(0)).getLifetime();
                 if (headPeriod > MAX_QUEUE_PERIOD) {
                     _context.statManager().addRateData("udp.droppedInbound", queueSize, headPeriod);
-                    if (_log.shouldLog(Log.WARN))
+                    if (_log.shouldLog(Log.ERROR))
-                        _log.warn("Dropping inbound packet with " + queueSize + " queued for " + headPeriod);
+                        _log.error("Dropping inbound packet with " + queueSize + " queued for " + headPeriod);
                     _inboundQueue.notifyAll();
-                    return;
+                    return queueSize;
                 }
             }
             _inboundQueue.add(packet);
             _inboundQueue.notifyAll();
+            return queueSize + 1;
         }
     }
     
+    private class ArtificiallyDelayedReceive implements SimpleTimer.TimedEvent {
+        private UDPPacket _packet;
+        public ArtificiallyDelayedReceive(UDPPacket packet) { _packet = packet; }
+        public void timeReached() { doReceive(_packet); }
+    }
+    
     /**
      * Blocking call to retrieve the next inbound packet, or null if we have
      * shut down.
@@ -100,16 +118,18 @@ public class UDPReceiver {
      */
     public UDPPacket receiveNext() {
         while (_keepRunning) {
-            synchronized (_inboundQueue) {
-                if (_inboundQueue.size() <= 0) {
             try {
+                synchronized (_inboundQueue) {
+                    if (_inboundQueue.size() > 0) {
+                        UDPPacket rv = (UDPPacket)_inboundQueue.remove(0);
+                        _inboundQueue.notifyAll();
+                        return rv;
+                    } else {
                         _inboundQueue.wait();
+                    }
+                }
             } catch (InterruptedException ie) {}
         }
-                if (_inboundQueue.size() > 0)
-                    return (UDPPacket)_inboundQueue.remove(0);
-            }
-        }
         return null;
     }
     
@@ -125,16 +145,22 @@ public class UDPReceiver {
                 UDPPacket packet = UDPPacket.acquire(_context);
                 
                 // block before we read...
+                if (_log.shouldLog(Log.DEBUG))
+                    _log.debug("Before throttling receive");
                 while (!_context.throttle().acceptNetworkMessage())
                     try { Thread.sleep(10); } catch (InterruptedException ie) {}
                 
                 try {
+                    if (_log.shouldLog(Log.DEBUG))
+                        _log.debug("Before blocking socket.receive");
                     synchronized (Runner.this) {
                         _socket.receive(packet.getPacket());
                     }
                     int size = packet.getPacket().getLength();
+                    if (_log.shouldLog(Log.DEBUG))
+                        _log.debug("After blocking socket.receive: packet is " + size + " bytes!");
+                    packet.setPacketDataLength(size);
                     packet.resetBegin();
-                    _context.statManager().addRateData("udp.receivePacketSize", size, 0);
             
                     // and block after we know how much we read but before
                     // we release the packet to the inbound queue
@@ -144,7 +170,8 @@ public class UDPReceiver {
                             req.waitForNextAllocation();
                     }
                     
-                    receive(packet);
+                    int queued = receive(packet);
+                    _context.statManager().addRateData("udp.receivePacketSize", size, queued);
                 } catch (IOException ioe) {
                     if (_socketChanged) {
                         if (_log.shouldLog(Log.INFO))
@@ -156,6 +183,8 @@ public class UDPReceiver {
                     packet.release();
                 }
             }
+            if (_log.shouldLog(Log.DEBUG))
+                _log.debug("Stop receiving...");
         }
         
         public DatagramSocket updateListeningPort(DatagramSocket socket, int newPort) {

router/java/src/net/i2p/router/transport/udp/UDPSender.java

@@ -43,6 +43,8 @@ public class UDPSender {
     }
     
     public void startup() {
+        if (_log.shouldLog(Log.DEBUG))
+            _log.debug("Starting the runner: " + _name);
         _keepRunning = true;
         I2PThread t = new I2PThread(_runner, _name);
         t.setDaemon(true);
@@ -72,10 +74,12 @@ public class UDPSender {
     public int add(UDPPacket packet, int blockTime) {
         long expiration = _context.clock().now() + blockTime;
         int remaining = -1;
+        long lifetime = -1;
         while ( (_keepRunning) && (remaining < 0) ) {
             try {
                 synchronized (_outboundQueue) {
                     if (_outboundQueue.size() < MAX_QUEUED) {
+                        lifetime = packet.getLifetime();
                         _outboundQueue.add(packet);
                         remaining = _outboundQueue.size();
                         _outboundQueue.notifyAll();
@@ -91,7 +95,9 @@ public class UDPSender {
                 }
             } catch (InterruptedException ie) {}
         }
-        _context.statManager().addRateData("udp.sendQueueSize", remaining, packet.getLifetime());
+        _context.statManager().addRateData("udp.sendQueueSize", remaining, lifetime);
+        if (_log.shouldLog(Log.DEBUG))
+            _log.debug("Added the packet onto the queue with " + remaining + " remaining and a lifetime of " + lifetime);
         return remaining;
     }
     
@@ -101,18 +107,24 @@ public class UDPSender {
      */
     public int add(UDPPacket packet) {
         int size = 0;
+        long lifetime = -1;
         synchronized (_outboundQueue) {
+            lifetime = packet.getLifetime();
             _outboundQueue.add(packet);
             size = _outboundQueue.size();
             _outboundQueue.notifyAll();
         }
-        _context.statManager().addRateData("udp.sendQueueSize", size, packet.getLifetime());
+        _context.statManager().addRateData("udp.sendQueueSize", size, lifetime);
+        if (_log.shouldLog(Log.DEBUG))
+            _log.debug("Added the packet onto the queue with " + size + " remaining and a lifetime of " + lifetime);
         return size;
     }
     
     private class Runner implements Runnable {
         private boolean _socketChanged;
         public void run() {
+            if (_log.shouldLog(Log.DEBUG))
+                _log.debug("Running the UDP sender");
             _socketChanged = false;
             while (_keepRunning) {
                 if (_socketChanged) {
@@ -122,8 +134,11 @@ public class UDPSender {
                 
                 UDPPacket packet = getNextPacket();
                 if (packet != null) {
+                    if (_log.shouldLog(Log.DEBUG))
+                        _log.debug("Packet to send known: " + packet);
                     long acquireTime = _context.clock().now();
-                    int size = packet.getPacket().getLength();
+                    int size = packet.getPacketDataLength(); // packet.getPacket().getLength();
+                    int size2 = packet.getPacket().getLength();
                     if (size > 0) {
                         FIFOBandwidthLimiter.Request req = _context.bandwidthLimiter().requestOutbound(size, "UDP sender");
                         while (req.getPendingOutboundRequested() > 0)
@@ -133,17 +148,23 @@ public class UDPSender {
                     long afterBW = _context.clock().now();
                     
                     if (_log.shouldLog(Log.DEBUG)) {
-                        int len = packet.getPacket().getLength();
                         //if (len > 128)
                         //    len = 128;
-                        _log.debug("Sending packet: \nraw: " + Base64.encode(packet.getPacket().getData(), 0, len));
+                        //_log.debug("Sending packet: (size="+size + "/"+size2 +")\nraw: " + Base64.encode(packet.getPacket().getData(), 0, size));
                     }
                     
+                    //packet.getPacket().setLength(size);
                     try {
                         long before = _context.clock().now();
                         synchronized (Runner.this) {
                             // synchronization lets us update safely
-                            _socket.send(packet.getPacket());
+                            //_log.debug("Break out datagram for " + packet);
+                            DatagramPacket dp = packet.getPacket();
+                            if (_log.shouldLog(Log.DEBUG))
+                                _log.debug("Just before socket.send of " + packet);
+                            _socket.send(dp);
+                            if (_log.shouldLog(Log.DEBUG))
+                                _log.debug("Just after socket.send of " + packet);
                         }
                         long sendTime = _context.clock().now() - before;
                         _context.statManager().addRateData("udp.socketSendTime", sendTime, packet.getLifetime());
@@ -151,7 +172,7 @@ public class UDPSender {
                         if (packet.getMarkedType() == 1)
                             _context.statManager().addRateData("udp.sendACKTime", afterBW - acquireTime, packet.getLifetime());
                         _context.statManager().addRateData("udp.pushTime", packet.getLifetime(), packet.getLifetime());
-                        _context.statManager().addRateData("udp.sendPacketSize", packet.getPacket().getLength(), packet.getLifetime());
+                        _context.statManager().addRateData("udp.sendPacketSize", size, packet.getLifetime());
                     } catch (IOException ioe) {
                         if (_log.shouldLog(Log.ERROR))
                             _log.error("Error sending", ioe);
@@ -161,6 +182,8 @@ public class UDPSender {
                     packet.release();
                 }
             }
+            if (_log.shouldLog(Log.DEBUG))
+                _log.debug("Stop sending...");
         }
         
         private UDPPacket getNextPacket() {

router/java/src/net/i2p/router/transport/udp/UDPTransport.java

@@ -40,7 +40,7 @@ public class UDPTransport extends TransportImpl implements TimedWeightedPriority
     private UDPEndpoint _endpoint;
     /** Peer (Hash) to PeerState */
     private Map _peersByIdent;
-    /** Remote host (ip+port as a string) to PeerState */
+    /** RemoteHostId to PeerState */
     private Map _peersByRemoteHost;
     /** Relay tag (base64 String) to PeerState */
     private Map _peersByRelayTag;
@@ -98,7 +98,7 @@ public class UDPTransport extends TransportImpl implements TimedWeightedPriority
     private static final int PRIORITY_WEIGHT[] = new int[] { 1, 1, 1, 1, 1, 2 };
 
     /** should we flood all UDP peers with the configured rate? */
-    private static final boolean SHOULD_FLOOD_PEERS = false;
+    private static final boolean SHOULD_FLOOD_PEERS = true;
     
     private static final int MAX_CONSECUTIVE_FAILED = 5;
     
@@ -270,7 +270,7 @@ public class UDPTransport extends TransportImpl implements TimedWeightedPriority
      * if no state exists
      */
     public PeerState getPeerState(InetAddress remoteHost, int remotePort) {
-        String hostInfo = PeerState.calculateRemoteHostString(remoteHost.getAddress(), remotePort);
+        RemoteHostId hostInfo = new RemoteHostId(remoteHost.getAddress(), remotePort);
         synchronized (_peersByRemoteHost) {
             return (PeerState)_peersByRemoteHost.get(hostInfo);
         }
@@ -316,11 +316,11 @@ public class UDPTransport extends TransportImpl implements TimedWeightedPriority
             }
         }
         
-        String remoteString = peer.getRemoteHostString();
+        RemoteHostId remoteId = peer.getRemoteHostId();
-        if (remoteString == null) return false;
+        if (remoteId == null) return false;
         
         synchronized (_peersByRemoteHost) {
-            PeerState oldPeer = (PeerState)_peersByRemoteHost.put(remoteString, peer);
+            PeerState oldPeer = (PeerState)_peersByRemoteHost.put(remoteId, peer);
             if ( (oldPeer != null) && (oldPeer != peer) ) {
                 //_peersByRemoteHost.put(remoteString, oldPeer);
                 //return false;
@@ -348,10 +348,10 @@ public class UDPTransport extends TransportImpl implements TimedWeightedPriority
             }
         }
         
-        String remoteString = peer.getRemoteHostString();
+        RemoteHostId remoteId = peer.getRemoteHostId();
-        if (remoteString != null) {
+        if (remoteId != null) {
             synchronized (_peersByRemoteHost) {
-                _peersByRemoteHost.remove(remoteString);
+                _peersByRemoteHost.remove(remoteId);
             }
         }
         
@@ -568,6 +568,7 @@ public class UDPTransport extends TransportImpl implements TimedWeightedPriority
         buf.append("     <td><b>cwnd</b></td><td><b>ssthresh</b></td>\n");
         buf.append("     <td><b>rtt</b></td><td><b>dev</b></td><td><b>rto</b></td>\n");
         buf.append("     <td><b>send</b></td><td><b>recv</b></td>\n");
+        buf.append("     <td><b>resent</b></td><td><b>dupRecv</b></td>\n");
         buf.append(" </tr>\n");
         out.write(buf.toString());
         buf.setLength(0);
@@ -640,11 +641,22 @@ public class UDPTransport extends TransportImpl implements TimedWeightedPriority
             buf.append("</td>");
             
             buf.append("<td>");
-            buf.append(peer.getMessagesSent());
+            buf.append(peer.getPacketsTransmitted());
             buf.append("</td>");
             
             buf.append("<td>");
-            buf.append(peer.getMessagesReceived());
+            buf.append(peer.getPacketsReceived());
+            buf.append("</td>");
+            
+            double sendLostPct = (double)peer.getPacketsRetransmitted()/(double)PeerState.RETRANSMISSION_PERIOD_WIDTH;
+            buf.append("<td>");
+            //buf.append(formatPct(sendLostPct));
+            buf.append(peer.getPacketRetransmissionRate());
+            buf.append("</td>");
+            
+            double recvDupPct = (double)peer.getPacketsReceivedDuplicate()/(double)peer.getPacketsReceived();
+            buf.append("<td>");
+            buf.append(formatPct(recvDupPct));
             buf.append("</td>");
 
             buf.append("</tr>");
@@ -655,12 +667,29 @@ public class UDPTransport extends TransportImpl implements TimedWeightedPriority
         out.write("</table>\n");
     }
 
+    public PartialACKSource getPartialACKSource() { return _inboundFragments; }
+    
+    /** help us grab partial ACKs */
+    public interface PartialACKSource {
+        /** 
+         * build partial ACKs of messages received from the peer and store
+         * them in the ackBitfields
+         */
+        public void fetchPartialACKs(Hash fromPeer, List ackBitfields);
+    }
+    
     private static final DecimalFormat _fmt = new DecimalFormat("#,##0.00");
     private static final String formatKBps(int bps) {
         synchronized (_fmt) {
             return _fmt.format((float)bps/1024);
         }
     }
+    private static final DecimalFormat _pctFmt = new DecimalFormat("#0.0%");
+    private static final String formatPct(double pct) {
+        synchronized (_pctFmt) {
+            return _pctFmt.format(pct);
+        }
+    }
     
     /**
      * Cache the bid to reduce object churn

router/java/src/net/i2p/router/tunnel/BloomFilterIVValidator.java

@@ -1,6 +1,9 @@
 package net.i2p.router.tunnel;
 
 import net.i2p.I2PAppContext;
+import net.i2p.data.ByteArray;
+import net.i2p.data.DataHelper;
+import net.i2p.util.ByteCache;
 import net.i2p.util.DecayingBloomFilter;
 
 /**
@@ -11,6 +14,7 @@ import net.i2p.util.DecayingBloomFilter;
 public class BloomFilterIVValidator implements IVValidator {
     private I2PAppContext _context;
     private DecayingBloomFilter _filter;
+    private ByteCache _ivXorCache = ByteCache.getInstance(32, HopProcessor.IV_LENGTH);
     
     /**
      * After 2*halflife, an entry is completely forgotten from the bloom filter.
@@ -24,11 +28,13 @@ public class BloomFilterIVValidator implements IVValidator {
         _filter = new DecayingBloomFilter(ctx, HALFLIFE_MS, 16);
         ctx.statManager().createRateStat("tunnel.duplicateIV", "Note that a duplicate IV was received", "Tunnels", 
                                          new long[] { 10*60*1000l, 60*60*1000l, 3*60*60*1000l, 24*60*60*1000l });
-
     }
     
-    public boolean receiveIV(byte[] iv) { 
+    public boolean receiveIV(byte ivData[], int ivOffset, byte payload[], int payloadOffset) {
-        boolean dup = _filter.add(iv); 
+        ByteArray buf = _ivXorCache.acquire();
+        DataHelper.xor(ivData, ivOffset, payload, payloadOffset, buf.getData(), 0, HopProcessor.IV_LENGTH);
+        boolean dup = _filter.add(buf.getData()); 
+        _ivXorCache.release(buf);
         if (dup) _context.statManager().addRateData("tunnel.duplicateIV", 1, 1);
         return !dup; // return true if it is OK, false if it isn't
     }

router/java/src/net/i2p/router/tunnel/DummyValidator.java

@@ -8,5 +8,6 @@ class DummyValidator implements IVValidator {
     public static DummyValidator getInstance() { return _instance; }
     private DummyValidator() {}
     
-    public boolean receiveIV(byte[] iv) { return true; }
+    public boolean receiveIV(byte ivData[], int ivOffset, byte payload[], int payloadOffset) { return true; }
+
 }

router/java/src/net/i2p/router/tunnel/FragmentHandler.java

@@ -143,7 +143,8 @@ public class FragmentHandler {
         boolean eq = DataHelper.eq(v.getData(), 0, preprocessed, offset + HopProcessor.IV_LENGTH, 4);
         if (!eq) {
             if (_log.shouldLog(Log.ERROR))
-                _log.error("Corrupt tunnel message - verification fails");
+                _log.error("Corrupt tunnel message - verification fails: \n" + Base64.encode(preprocessed, offset+HopProcessor.IV_LENGTH, 4)
+                           + "\n" + Base64.encode(v.getData(), 0, 4));
             if (_log.shouldLog(Log.WARN))
                 _log.warn("nomatching endpoint: # pad bytes: " + (paddingEnd-(HopProcessor.IV_LENGTH+4)-1) + "\n" 
                            + " offset=" + offset + " length=" + length + " paddingEnd=" + paddingEnd

router/java/src/net/i2p/router/tunnel/HashSetIVValidator.java

@@ -2,6 +2,7 @@ package net.i2p.router.tunnel;
 
 import java.util.HashSet;
 import net.i2p.data.ByteArray;
+import net.i2p.data.DataHelper;
 
 /** 
  * waste lots of RAM 
@@ -12,9 +13,12 @@ class HashSetIVValidator implements IVValidator {
     public HashSetIVValidator() {
         _received = new HashSet();
     }
-    public boolean receiveIV(byte[] iv) {
+    
+    public boolean receiveIV(byte ivData[], int ivOffset, byte payload[], int payloadOffset) {
         //if (true) // foo!
         //    return true; 
+        byte iv[] = new byte[HopProcessor.IV_LENGTH];
+        DataHelper.xor(ivData, ivOffset, payload, payloadOffset, iv, 0, HopProcessor.IV_LENGTH);
         ByteArray ba = new ByteArray(iv);
         boolean isNew = false;
         synchronized (_received) {

router/java/src/net/i2p/router/tunnel/HopProcessor.java

@@ -66,10 +66,7 @@ public class HopProcessor {
             }
         }
         
-        ByteArray ba = _cache.acquire();
+        boolean okIV = _validator.receiveIV(orig, offset, orig, offset + IV_LENGTH);
-        byte iv[] = ba.getData(); // new byte[IV_LENGTH];
-        System.arraycopy(orig, offset, iv, 0, IV_LENGTH);
-        boolean okIV = _validator.receiveIV(iv);
         if (!okIV) {
             if (_log.shouldLog(Log.WARN)) 
                 _log.warn("Invalid IV received on tunnel " + _config.getReceiveTunnelId());
@@ -88,7 +85,6 @@ public class HopProcessor {
             //_log.debug("Data after processing: " + Base64.encode(orig, IV_LENGTH, orig.length - IV_LENGTH));
             //_log.debug("IV sent: " + Base64.encode(orig, 0, IV_LENGTH));
         }
-        _cache.release(ba);
         return true;
     }
     

router/java/src/net/i2p/router/tunnel/IVValidator.java

@@ -8,9 +8,14 @@ package net.i2p.router.tunnel;
  */
 public interface IVValidator {
     /** 
-     * receive the IV for the tunnel, returning true if it is valid,
+     * receive the IV for the tunnel message, returning true if it is valid,
-     * or false if it has already been used (or is otherwise invalid).
+     * or false if it has already been used (or is otherwise invalid).  To
+     * prevent colluding attackers from successfully tagging the tunnel by
+     * switching the IV and the first block of the message, the validator should
+     * treat the XOR of the IV and the first block as the unique identifier, 
+     * not the IV alone (since the tunnel is encrypted via AES/CBC).  Thanks to
+     * dvorak for pointing out that tagging!
      *
      */
-    public boolean receiveIV(byte iv[]);
+    public boolean receiveIV(byte iv[], int ivOffset, byte payload[], int payloadOffset);
 }

router/java/src/net/i2p/router/tunnel/InboundEndpointProcessor.java

@@ -58,7 +58,7 @@ public class InboundEndpointProcessor {
         //if (_config.getLength() > 1)
         //    _log.debug("IV at inbound endpoint before decrypt: " + Base64.encode(iv));
 
-        boolean ok = _validator.receiveIV(iv);
+        boolean ok = _validator.receiveIV(iv, 0, orig, offset + HopProcessor.IV_LENGTH);
         if (!ok) {
             if (_log.shouldLog(Log.WARN)) 
                 _log.warn("Invalid IV received");

router/java/src/net/i2p/router/tunnel/TunnelDispatcher.java

@@ -356,8 +356,7 @@ public class TunnelDispatcher implements Service {
             if (_log.shouldLog(Log.DEBUG))
                 _log.debug("dispatch to participant " + participant + ": " + msg.getUniqueId() + " from " 
                            + recvFrom.toBase64().substring(0,4));
-            _context.messageHistory().tunnelDispatched("message " + msg.getUniqueId() + " on tunnel " 
+            _context.messageHistory().tunnelDispatched(msg.getUniqueId(), msg.getTunnelId().getTunnelId(), "participant");
-                                                       + msg.getTunnelId().getTunnelId() + " as participant");
             participant.dispatch(msg, recvFrom);
             _context.statManager().addRateData("tunnel.dispatchParticipant", 1, 0);
         } else {
@@ -370,8 +369,7 @@ public class TunnelDispatcher implements Service {
                 if (_log.shouldLog(Log.DEBUG))
                     _log.debug("dispatch where we are the outbound endpoint: " + endpoint + ": " 
                                + msg + " from " + recvFrom.toBase64().substring(0,4));
-                _context.messageHistory().tunnelDispatched("message " + msg.getUniqueId() + " on tunnel " 
+                _context.messageHistory().tunnelDispatched(msg.getUniqueId(), msg.getTunnelId().getTunnelId(), "outbound endpoint");
-                                                           + msg.getTunnelId().getTunnelId() + " as outbound endpoint");
                 endpoint.dispatch(msg, recvFrom);
                 
                 _context.statManager().addRateData("tunnel.dispatchEndpoint", 1, 0);
@@ -481,9 +479,9 @@ public class TunnelDispatcher implements Service {
                                + (before-msg.getMessageExpiration()) + "ms ago? " 
                                + msg, new Exception("cause"));
             }
-            _context.messageHistory().tunnelDispatched("message " + msg.getUniqueId() + " on tunnel " 
+            long tid1 = (outboundTunnel != null ? outboundTunnel.getTunnelId() : -1);
-                                                           + outboundTunnel + "/" + targetTunnel + " to "
+            long tid2 = (targetTunnel != null ? targetTunnel.getTunnelId() : -1);
-                                                           + targetPeer + " as outbound gateway");
+            _context.messageHistory().tunnelDispatched(msg.getUniqueId(), tid1, tid2, targetPeer, "outbound gateway");
             gw.add(msg, targetPeer, targetTunnel);
             if (targetTunnel == null)
                 _context.statManager().addRateData("tunnel.dispatchOutboundPeer", 1, 0);