* Tunnel Building:

- Add getRecordCount() to TunnelBuildMessage and TunnelBuildReplyMessage
        so they can be extended.
      - New I2NP Messages VariableTunnelBuildMessage and VariableTunnelBuildReplyMessage,
        which contain the number of request slots in them.
      - Convert all static assumptions of 8 slots to getRecordCount()
      - Use the new VTBM if all hops in the tunnel and the OBEP or IBGW of the reply tunnel
        support it, and the tunnel is 4 hops or shorter.
      - Reply to a VTBM with a VTBRM of the same size
      - Make BuildReplyHandler static
      - Convert the currentlyBuilding List to a ConcurrentHashMap to speed reply lookups
        and eliminate a global lock; don't put fallback tunnels in there
      - Add new tunnel.corruptBuildReply stat
      - Various cleanups and javadoc

    Tested as compatible with current network, new messages untested.

router/java/src/net/i2p/data/i2np/I2NPMessageImpl.java

@@ -11,8 +11,8 @@ package net.i2p.data.i2np;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
-import java.util.HashMap;
 import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
 
 import net.i2p.I2PAppContext;
 import net.i2p.data.DataFormatException;
@@ -39,10 +39,11 @@ public abstract class I2NPMessageImpl extends DataStructureImpl implements I2NPM
     
     private static final boolean RAW_FULL_SIZE = false;
     
-    /** unsynchronized as its pretty much read only (except at startup) */
+    /** unused */
-    private static final Map _builders = new HashMap(8);
+    private static final Map<Integer, Builder> _builders = new ConcurrentHashMap(1);
+    /** @deprecated unused */
     public static final void registerBuilder(Builder builder, int type) { _builders.put(Integer.valueOf(type), builder); }
-    /** interface for extending the types of messages handled */
+    /** interface for extending the types of messages handled - unused */
     public interface Builder {
         /** instantiate a new I2NPMessage to be populated shortly */
         public I2NPMessage build(I2PAppContext ctx);
@@ -383,16 +384,19 @@ public abstract class I2NPMessageImpl extends DataStructureImpl implements I2NPM
                 return new TunnelGatewayMessage(context);
             case DataMessage.MESSAGE_TYPE:
                 return new DataMessage(context);
-            //case TunnelCreateMessage.MESSAGE_TYPE:
-            //    return new TunnelCreateMessage(context);
-            //case TunnelCreateStatusMessage.MESSAGE_TYPE:
-            //    return new TunnelCreateStatusMessage(context);
             case TunnelBuildMessage.MESSAGE_TYPE:
                 return new TunnelBuildMessage(context);
             case TunnelBuildReplyMessage.MESSAGE_TYPE:
                 return new TunnelBuildReplyMessage(context);
+            // since 0.7.10
+            case VariableTunnelBuildMessage.MESSAGE_TYPE:
+                return new VariableTunnelBuildMessage(context);
+            // since 0.7.10
+            case VariableTunnelBuildReplyMessage.MESSAGE_TYPE:
+                return new VariableTunnelBuildReplyMessage(context);
             default:
-                Builder builder = (Builder)_builders.get(Integer.valueOf(type));
+                // unused
+                Builder builder = _builders.get(Integer.valueOf(type));
                 if (builder == null)
                     return null;
                 else

router/java/src/net/i2p/data/i2np/TunnelBuildMessage.java

@@ -9,18 +9,30 @@ import net.i2p.data.ByteArray;
  *
  */
 public class TunnelBuildMessage extends I2NPMessageImpl {
-    private ByteArray _records[];
+    protected ByteArray _records[];
+    protected int RECORD_COUNT;
+    public static final int MAX_RECORD_COUNT = 8;
     
     public static final int MESSAGE_TYPE = 21;
-    public static final int RECORD_COUNT = 8;
 
     public TunnelBuildMessage(I2PAppContext context) {
+        this(context, MAX_RECORD_COUNT);
+    }
+
+    /** @since 0.7.10 */
+    protected TunnelBuildMessage(I2PAppContext context, int records) {
         super(context);
-        _records = new ByteArray[RECORD_COUNT];
+        if (records > 0) {
+            RECORD_COUNT = records;
+            _records = new ByteArray[records];
+        }
+        // else will be initialized by readMessage() in VTBM
     }
 
     public void setRecord(int index, ByteArray record) { _records[index] = record; }
     public ByteArray getRecord(int index) { return _records[index]; }
+    /** @since 0.7.10 */
+    public int getRecordCount() { return RECORD_COUNT; }
     
     public static final int RECORD_SIZE = 512+16;
     
@@ -50,4 +62,9 @@ public class TunnelBuildMessage extends I2NPMessageImpl {
         }
         return curIndex;
     }
+
+    @Override
+    public String toString() {
+        return "[TunnelBuildMessage]";
+    }
 }

router/java/src/net/i2p/data/i2np/TunnelBuildReplyMessage.java

@@ -10,18 +10,30 @@ import net.i2p.data.ByteArray;
  * reply tunnel
  */
 public class TunnelBuildReplyMessage extends I2NPMessageImpl {
-    private ByteArray _records[];
+    protected ByteArray _records[];
+    protected int RECORD_COUNT;
+    public static final int MAX_RECORD_COUNT = TunnelBuildMessage.MAX_RECORD_COUNT;
     
     public static final int MESSAGE_TYPE = 22;
-    public static final int RECORD_COUNT = TunnelBuildMessage.RECORD_COUNT;
 
     public TunnelBuildReplyMessage(I2PAppContext context) {
+        this(context, MAX_RECORD_COUNT);
+    }
+
+    /** @since 0.7.10 */
+    protected TunnelBuildReplyMessage(I2PAppContext context, int records) {
         super(context);
-        _records = new ByteArray[RECORD_COUNT];
+        if (records > 0) {
+            RECORD_COUNT = records;
+            _records = new ByteArray[records];
+        }
+        // else will be initialized by readMessage() in VTBRM
     }
 
     public void setRecord(int index, ByteArray record) { _records[index] = record; }
     public ByteArray getRecord(int index) { return _records[index]; }
+    /** @since 0.7.10 */
+    public int getRecordCount() { return RECORD_COUNT; }
     
     public static final int RECORD_SIZE = TunnelBuildMessage.RECORD_SIZE;
     
@@ -53,4 +65,9 @@ public class TunnelBuildReplyMessage extends I2NPMessageImpl {
         }
         return curIndex;
     }
+
+    @Override
+    public String toString() {
+        return "[TunnelBuildReplyMessage]";
+    }
 }

router/java/src/net/i2p/data/i2np/VariableTunnelBuildMessage.java

@@ -0,0 +1,69 @@
+package net.i2p.data.i2np;
+
+import java.io.IOException;
+
+import net.i2p.I2PAppContext;
+import net.i2p.data.ByteArray;
+import net.i2p.data.DataHelper;
+
+/**
+ * @since 0.7.11
+ */
+public class VariableTunnelBuildMessage extends TunnelBuildMessage {
+    public static final int MESSAGE_TYPE = 23;
+
+    /** zero record count, will be set with readMessage() */
+    public VariableTunnelBuildMessage(I2PAppContext context) {
+        super(context, 0);
+    }
+
+    public VariableTunnelBuildMessage(I2PAppContext context, int records) {
+        super(context, records);
+    }
+
+    @Override
+    protected int calculateWrittenLength() { return 1 + super.calculateWrittenLength(); }
+
+    @Override
+    public int getType() { return MESSAGE_TYPE; }
+
+    @Override
+    public void readMessage(byte[] data, int offset, int dataSize, int type) throws I2NPMessageException, IOException {
+        if (type != MESSAGE_TYPE) 
+            throw new I2NPMessageException("Message type is incorrect for this message");
+        if (dataSize != calculateWrittenLength()) 
+            throw new I2NPMessageException("Wrong length (expects " + calculateWrittenLength() + ", recv " + dataSize + ")");
+        int r = (int)DataHelper.fromLong(data, offset, 1);
+        if (r <= 0 || r > MAX_RECORD_COUNT)
+            throw new I2NPMessageException("Bad record count " + r);
+        RECORD_COUNT = r;
+        _records = new ByteArray[RECORD_COUNT];
+        super.readMessage(data, offset + 1, dataSize, TunnelBuildMessage.MESSAGE_TYPE);
+    }
+    
+    @Override
+    protected int writeMessageBody(byte[] out, int curIndex) throws I2NPMessageException {
+        int remaining = out.length - (curIndex + calculateWrittenLength());
+        if (remaining < 0)
+            throw new I2NPMessageException("Not large enough (too short by " + remaining + ")");
+        if (RECORD_COUNT <= 0 || RECORD_COUNT > MAX_RECORD_COUNT)
+            throw new I2NPMessageException("Bad record count " + RECORD_COUNT);
+        DataHelper.toLong(out, curIndex, 1, RECORD_COUNT);
+        // can't call super, written length check will fail
+        //return super.writeMessageBody(out, curIndex + 1);
+        for (int i = 0; i < RECORD_COUNT; i++) {
+            System.arraycopy(_records[i].getData(), _records[i].getOffset(), out, curIndex, RECORD_SIZE);
+            curIndex += RECORD_SIZE;
+        }
+        return curIndex;
+    }
+    
+    @Override
+    public String toString() {
+        StringBuilder buf = new StringBuilder(64);
+        buf.append("[VariableTunnelBuildMessage: " +
+                   "\n\tRecords: ").append(getRecordCount())
+           .append(']');
+        return buf.toString();
+    }
+}

router/java/src/net/i2p/data/i2np/VariableTunnelBuildReplyMessage.java

@@ -0,0 +1,71 @@
+package net.i2p.data.i2np;
+
+import java.io.IOException;
+
+import net.i2p.I2PAppContext;
+import net.i2p.data.ByteArray;
+import net.i2p.data.DataHelper;
+
+/**
+ * Transmitted from the new outbound endpoint to the creator through a
+ * reply tunnel
+ *
+ * @since 0.7.11
+ */
+public class VariableTunnelBuildReplyMessage extends TunnelBuildReplyMessage {
+    public static final int MESSAGE_TYPE = 24;
+
+    /** zero record count, will be set with readMessage() */
+    public VariableTunnelBuildReplyMessage(I2PAppContext context) {
+        super(context, 0);
+    }
+
+    public VariableTunnelBuildReplyMessage(I2PAppContext context, int records) {
+        super(context, records);
+    }
+
+    @Override
+    protected int calculateWrittenLength() { return 1 + super.calculateWrittenLength(); }
+
+    @Override
+    public int getType() { return MESSAGE_TYPE; }
+
+    public void readMessage(byte[] data, int offset, int dataSize, int type) throws I2NPMessageException, IOException {
+        if (type != MESSAGE_TYPE) 
+            throw new I2NPMessageException("Message type is incorrect for this message");
+        if (dataSize != calculateWrittenLength()) 
+            throw new I2NPMessageException("Wrong length (expects " + calculateWrittenLength() + ", recv " + dataSize + ")");
+        
+        int r = (int)DataHelper.fromLong(data, offset, 1);
+        if (r <= 0 || r > MAX_RECORD_COUNT)
+            throw new I2NPMessageException("Bad record count " + r);
+        RECORD_COUNT = r;
+        _records = new ByteArray[RECORD_COUNT];
+        super.readMessage(data, offset + 1, dataSize, TunnelBuildMessage.MESSAGE_TYPE);
+    }
+    
+    protected int writeMessageBody(byte[] out, int curIndex) throws I2NPMessageException {
+        int remaining = out.length - (curIndex + calculateWrittenLength());
+        if (remaining < 0)
+            throw new I2NPMessageException("Not large enough (too short by " + remaining + ")");
+        if (RECORD_COUNT <= 0 || RECORD_COUNT > MAX_RECORD_COUNT)
+            throw new I2NPMessageException("Bad record count " + RECORD_COUNT);
+        DataHelper.toLong(out, curIndex, 1, RECORD_COUNT);
+        // can't call super, written length check will fail
+        //return super.writeMessageBody(out, curIndex + 1);
+        for (int i = 0; i < RECORD_COUNT; i++) {
+            System.arraycopy(_records[i].getData(), _records[i].getOffset(), out, curIndex, RECORD_SIZE);
+            curIndex += RECORD_SIZE;
+        }
+        return curIndex;
+    }
+    
+    @Override
+    public String toString() {
+        StringBuilder buf = new StringBuilder(64);
+        buf.append("[VariableTunnelBuildReplyMessage: " +
+                   "\n\tRecords: ").append(getRecordCount())
+           .append(']');
+        return buf.toString();
+    }
+}

router/java/src/net/i2p/router/tunnel/BuildMessageGenerator.java

@@ -22,18 +22,24 @@ import net.i2p.util.Log;
  */
 public class BuildMessageGenerator {
     // cached, rather than creating lots of temporary Integer objects whenever we build a tunnel
-    public static final Integer ORDER[] = new Integer[TunnelBuildMessage.RECORD_COUNT];
+    public static final Integer ORDER[] = new Integer[TunnelBuildMessage.MAX_RECORD_COUNT];
     static { for (int i = 0; i < ORDER.length; i++) ORDER[i] = Integer.valueOf(i); }
     
     /** return null if it is unable to find a router's public key (etc) */
+/****
     public TunnelBuildMessage createInbound(RouterContext ctx, TunnelCreatorConfig cfg) {
         return create(ctx, cfg, null, -1);
     }
+****/
+
     /** return null if it is unable to find a router's public key (etc) */
+/****
     public TunnelBuildMessage createOutbound(RouterContext ctx, TunnelCreatorConfig cfg, Hash replyRouter, long replyTunnel) {
         return create(ctx, cfg, replyRouter, replyTunnel);
     }
+****/
     
+/****
     private TunnelBuildMessage create(RouterContext ctx, TunnelCreatorConfig cfg, Hash replyRouter, long replyTunnel) {
         TunnelBuildMessage msg = new TunnelBuildMessage(ctx);
         List order = new ArrayList(ORDER.length);
@@ -50,14 +56,15 @@ public class BuildMessageGenerator {
         layeredEncrypt(ctx, msg, cfg, order);
         return msg;
     }
+****/
     
     /**
      * Place the asymmetrically encrypted record in the specified record slot, 
      * containing the hop's configuration (as well as the reply info, if it is an outbound endpoint)
      */
-    public void createRecord(int recordNum, int hop, TunnelBuildMessage msg, TunnelCreatorConfig cfg, Hash replyRouter, long replyTunnel, I2PAppContext ctx, PublicKey peerKey) {
+    public static void createRecord(int recordNum, int hop, TunnelBuildMessage msg, TunnelCreatorConfig cfg, Hash replyRouter, long replyTunnel, I2PAppContext ctx, PublicKey peerKey) {
         byte encrypted[] = new byte[TunnelBuildMessage.RECORD_SIZE];
-        Log log = ctx.logManager().getLog(getClass());
+        Log log = ctx.logManager().getLog(BuildMessageGenerator.class);
         if (peerKey != null) {
             BuildRequestRecord req = null;
             if ( (!cfg.isInbound()) && (hop + 1 == cfg.getLength()) ) //outbound endpoint
@@ -79,7 +86,7 @@ public class BuildMessageGenerator {
         msg.setRecord(recordNum, new ByteArray(encrypted));
     }
     
-    private BuildRequestRecord createUnencryptedRecord(I2PAppContext ctx, TunnelCreatorConfig cfg, int hop, Hash replyRouter, long replyTunnel) {
+    private static BuildRequestRecord createUnencryptedRecord(I2PAppContext ctx, TunnelCreatorConfig cfg, int hop, Hash replyRouter, long replyTunnel) {
         Log log = ctx.logManager().getLog(BuildMessageGenerator.class);
         if (hop < cfg.getLength()) {
             // ok, now lets fill in some data
@@ -143,10 +150,10 @@ public class BuildMessageGenerator {
      * Encrypt the records so their hop ident is visible at the appropriate times
      * @param order list of hop #s as Integers.  For instance, if (order.get(1) is 4), it is peer cfg.getPeer(4)
      */
-    public void layeredEncrypt(I2PAppContext ctx, TunnelBuildMessage msg, TunnelCreatorConfig cfg, List order) {
+    public static void layeredEncrypt(I2PAppContext ctx, TunnelBuildMessage msg, TunnelCreatorConfig cfg, List order) {
         Log log = ctx.logManager().getLog(BuildMessageGenerator.class);
         // encrypt the records so that the right elements will be visible at the right time
-        for (int i = 0; i < TunnelBuildMessage.RECORD_COUNT; i++) {
+        for (int i = 0; i < msg.getRecordCount(); i++) {
             ByteArray rec = msg.getRecord(i);
             Integer hopNum = (Integer)order.get(i);
             int hop = hopNum.intValue();

router/java/src/net/i2p/router/tunnel/BuildMessageProcessor.java

@@ -43,7 +43,7 @@ public class BuildMessageProcessor {
         long totalEq = 0;
         long totalDup = 0;
         long beforeLoop = System.currentTimeMillis();
-        for (int i = 0; i < TunnelBuildMessage.RECORD_COUNT; i++) {
+        for (int i = 0; i < msg.getRecordCount(); i++) {
             ByteArray rec = msg.getRecord(i);
             int off = rec.getOffset();
             int len = BuildRequestRecord.PEER_SIZE;
@@ -87,7 +87,7 @@ public class BuildMessageProcessor {
         SessionKey replyKey = rv.readReplyKey();
         byte iv[] = rv.readReplyIV();
         int ivOff = 0;
-        for (int i = 0; i < TunnelBuildMessage.RECORD_COUNT; i++) {
+        for (int i = 0; i < msg.getRecordCount(); i++) {
             if (i != ourHop) {
                 ByteArray data = msg.getRecord(i);
                 if (log.shouldLog(Log.DEBUG))

router/java/src/net/i2p/router/tunnel/BuildReplyHandler.java

@@ -17,7 +17,6 @@ import net.i2p.util.Log;
  *
  */
 public class BuildReplyHandler {
-    public BuildReplyHandler() {}
 
     /**
      * Decrypt the tunnel build reply records.  This overwrites the contents of the reply
@@ -25,11 +24,16 @@ public class BuildReplyHandler {
      * @return status for the records (in record order), or null if the replies were not valid.  Fake records
      *         always have 0 as their value
      */
-    public int[] decrypt(I2PAppContext ctx, TunnelBuildReplyMessage reply, TunnelCreatorConfig cfg, List recordOrder) {
+    public static int[] decrypt(I2PAppContext ctx, TunnelBuildReplyMessage reply, TunnelCreatorConfig cfg, List<Integer> recordOrder) {
-        Log log = ctx.logManager().getLog(getClass());
+        Log log = ctx.logManager().getLog(BuildReplyHandler.class);
-        int rv[] = new int[TunnelBuildReplyMessage.RECORD_COUNT];
+        if (reply.getRecordCount() != recordOrder.size()) {
+            // somebody messed with us
+            log.error("Corrupted build reply, expected " + recordOrder.size() + " records, got " + reply.getRecordCount());
+            return null;
+        }
+        int rv[] = new int[reply.getRecordCount()];
         for (int i = 0; i < rv.length; i++) {
-            int hop = ((Integer)recordOrder.get(i)).intValue();
+            int hop = recordOrder.get(i).intValue();
             if (BuildMessageGenerator.isBlank(cfg, hop)) {
                 // self...
                 if (log.shouldLog(Log.DEBUG))
@@ -56,8 +60,8 @@ public class BuildReplyHandler {
      *
      * @return -1 on decrypt failure
      */
-    private int decryptRecord(I2PAppContext ctx, TunnelBuildReplyMessage reply, TunnelCreatorConfig cfg, int recordNum, int hop) {
+    private static int decryptRecord(I2PAppContext ctx, TunnelBuildReplyMessage reply, TunnelCreatorConfig cfg, int recordNum, int hop) {
-        Log log = ctx.logManager().getLog(getClass());
+        Log log = ctx.logManager().getLog(BuildReplyHandler.class);
         if (BuildMessageGenerator.isBlank(cfg, hop)) {
             if (log.shouldLog(Log.DEBUG))
                 log.debug(reply.getUniqueId() + ": Record " + recordNum + "/" + hop + " is fake, so consider it valid...");

router/java/src/net/i2p/router/tunnel/TunnelCreatorConfig.java

@@ -26,7 +26,7 @@ public class TunnelCreatorConfig implements TunnelInfo {
     /** gateway first */
     private Hash _peers[];
     private long _expiration;
-    private List _order;
+    private List<Integer> _order;
     private long _replyMessageId;
     private boolean _isInbound;
     private long _messagesProcessed;
@@ -54,7 +54,11 @@ public class TunnelCreatorConfig implements TunnelInfo {
         _failures = 0;
     }
     
-    /** how many hops are there in the tunnel? */
+    /**
+     *  How many hops are there in the tunnel?
+     *  INCLUDING US.
+     *  i.e. one more than the TunnelCreatorConfig length.
+     */
     public int getLength() { return _config.length; }
     
     public Properties getOptions() { return null; }
@@ -91,8 +95,8 @@ public class TunnelCreatorConfig implements TunnelInfo {
     public void setExpiration(long when) { _expiration = when; }
     
     /** component ordering in the new style request */
-    public List getReplyOrder() { return _order; }
+    public List<Integer> getReplyOrder() { return _order; }
-    public void setReplyOrder(List order) { _order = order; }
+    public void setReplyOrder(List<Integer> order) { _order = order; }
     /** new style reply message id */
     public long getReplyMessageId() { return _replyMessageId; }
     public void setReplyMessageId(long id) { _replyMessageId = id; }

router/java/src/net/i2p/router/tunnel/pool/BuildExecutor.java

@@ -3,9 +3,12 @@ package net.i2p.router.tunnel.pool;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Comparator;
+import java.util.Iterator;
 import java.util.List;
+import java.util.concurrent.ConcurrentHashMap;
 
 import net.i2p.data.Hash;
+import net.i2p.data.i2np.I2NPMessage;
 import net.i2p.data.RouterInfo;
 import net.i2p.router.RouterContext;
 import net.i2p.router.TunnelManagerFacade;
@@ -28,7 +31,9 @@ class BuildExecutor implements Runnable {
     private Log _log;
     private TunnelPoolManager _manager;
     /** list of TunnelCreatorConfig elements of tunnels currently being built */
-    private final List<PooledTunnelCreatorConfig> _currentlyBuilding;
+    private final Object _currentlyBuilding;
+    /** indexed by ptcc.getReplyMessageId() */
+    private final ConcurrentHashMap<Long, PooledTunnelCreatorConfig> _currentlyBuildingMap;
     private boolean _isRunning;
     private BuildHandler _handler;
     private boolean _repoll;
@@ -38,7 +43,8 @@ class BuildExecutor implements Runnable {
         _context = ctx;
         _log = ctx.logManager().getLog(getClass());
         _manager = mgr;
-        _currentlyBuilding = new ArrayList(MAX_CONCURRENT_BUILDS);
+        _currentlyBuilding = new Object();
+        _currentlyBuildingMap = new ConcurrentHashMap(MAX_CONCURRENT_BUILDS);
         _context.statManager().createRateStat("tunnel.concurrentBuilds", "How many builds are going at once", "Tunnels", new long[] { 60*1000, 5*60*1000, 60*60*1000 });
         _context.statManager().createRateStat("tunnel.concurrentBuildsLagged", "How many builds are going at once when we reject further builds, due to job lag (period is lag)", "Tunnels", new long[] { 60*1000, 5*60*1000, 60*60*1000 });
         _context.statManager().createRateStat("tunnel.buildExploratoryExpire", "How often an exploratory tunnel times out during creation", "Tunnels", new long[] { 10*60*1000, 60*60*1000 });
@@ -82,21 +88,18 @@ class BuildExecutor implements Runnable {
         int concurrent = 0;
         // Todo: Make expiration variable
         long expireBefore = _context.clock().now() + 10*60*1000 - BuildRequestor.REQUEST_TIMEOUT;
-        synchronized (_currentlyBuilding) {
             // expire any old requests
-            for (int i = 0; i < _currentlyBuilding.size(); i++) {
+            for (Iterator<PooledTunnelCreatorConfig> iter = _currentlyBuildingMap.values().iterator(); iter.hasNext(); ) {
-                PooledTunnelCreatorConfig cfg = _currentlyBuilding.get(i);
+                PooledTunnelCreatorConfig cfg = iter.next();
                 if (cfg.getExpiration() <= expireBefore) {
-                    _currentlyBuilding.remove(i);
+                    iter.remove();
-                    i--;
                     if (expired == null)
                         expired = new ArrayList();
                     expired.add(cfg);
                 }
             }
-            concurrent = _currentlyBuilding.size();
+            concurrent = _currentlyBuildingMap.size();
             allowed -= concurrent;
-        }
         
         if (expired != null) {
             for (int i = 0; i < expired.size(); i++) {
@@ -303,9 +306,6 @@ class BuildExecutor implements Runnable {
                                 }
                                 if (_log.shouldLog(Log.DEBUG))
                                     _log.debug("Configuring new tunnel " + i + " for " + pool + ": " + cfg);
-                                synchronized (_currentlyBuilding) {
-                                    _currentlyBuilding.add(cfg);
-                                }
                                 buildTunnel(pool, cfg);
                                 realBuilt++;
                                 
@@ -400,9 +400,6 @@ class BuildExecutor implements Runnable {
                 if (cfg != null) {
                     if (_log.shouldLog(Log.DEBUG))
                         _log.debug("Configuring short tunnel " + i + " for " + pool + ": " + cfg);
-                    synchronized (_currentlyBuilding) {
-                        _currentlyBuilding.add(cfg);
-                    }
                     buildTunnel(pool, cfg);
                     if (cfg.getLength() > 1) {
                         allowed--; // oops... shouldn't have done that, but hey, its not that bad...
@@ -422,6 +419,15 @@ class BuildExecutor implements Runnable {
     
     void buildTunnel(TunnelPool pool, PooledTunnelCreatorConfig cfg) {
         long beforeBuild = System.currentTimeMillis();
+        if (cfg.getLength() > 1) {
+            // should we allow an ID of 0?
+            cfg.setReplyMessageId(_context.random().nextLong(I2NPMessage.MAX_ID_VALUE));
+            if (addToBuilding(cfg)) {
+                _log.error("Dup reply ID: " + cfg.getReplyMessageId());
+                // fail
+                return;
+            }
+        }
         BuildRequestor.request(_context, pool, cfg, this);
         long buildTime = System.currentTimeMillis() - beforeBuild;
         if (cfg.getLength() <= 1)
@@ -445,8 +451,9 @@ class BuildExecutor implements Runnable {
         if (_log.shouldLog(Log.DEBUG))
             _log.debug("Build complete for " + cfg);
         pool.buildComplete(cfg);
+        if (cfg.getLength() > 1)
+            removeFromBuilding(cfg.getReplyMessageId());
         synchronized (_currentlyBuilding) { 
-            _currentlyBuilding.remove(cfg);
             _currentlyBuilding.notifyAll();
         }
         
@@ -479,6 +486,22 @@ class BuildExecutor implements Runnable {
             _log.info(tunnel + ": Peer " + peer.toBase64() + " did not reply to the tunnel join request");
     }
     
-    List locked_getCurrentlyBuilding() { return _currentlyBuilding; }
+    /**
+     *  Only do this for non-fallback tunnels.
+     *  @return true if refused because of a duplicate key
+     */
+    private boolean addToBuilding(PooledTunnelCreatorConfig cfg) {
+        //_log.error("Adding ID: " + cfg.getReplyMessageId() + "; size was: " + _currentlyBuildingMap.size());
+        return _currentlyBuildingMap.putIfAbsent(Long.valueOf(cfg.getReplyMessageId()), cfg) != null;
+    }
+
+    /**
+     *  @return ptcc or null
+     */
+    PooledTunnelCreatorConfig removeFromBuilding(long id) {
+        //_log.error("Removing ID: " + id + "; size was: " + _currentlyBuildingMap.size());
+        return _currentlyBuildingMap.remove(Long.valueOf(id));
+    }
+
     public int getInboundBuildQueueSize() { return _handler.getInboundBuildQueueSize(); }
 }

router/java/src/net/i2p/router/tunnel/pool/BuildHandler.java

@@ -16,6 +16,8 @@ import net.i2p.data.i2np.I2NPMessage;
 import net.i2p.data.i2np.TunnelBuildMessage;
 import net.i2p.data.i2np.TunnelBuildReplyMessage;
 import net.i2p.data.i2np.TunnelGatewayMessage;
+import net.i2p.data.i2np.VariableTunnelBuildMessage;
+import net.i2p.data.i2np.VariableTunnelBuildReplyMessage;
 import net.i2p.router.HandlerJobBuilder;
 import net.i2p.router.Job;
 import net.i2p.router.JobImpl;
@@ -87,12 +89,18 @@ class BuildHandler {
         _context.statManager().createRateStat("tunnel.receiveRejectionTransient", "How often we are rejected due to transient overload?", "Tunnels", new long[] { 10*60*1000l, 60*60*1000l, 24*60*60*1000l });
         _context.statManager().createRateStat("tunnel.receiveRejectionBandwidth", "How often we are rejected due to bandwidth overload?", "Tunnels", new long[] { 10*60*1000l, 60*60*1000l, 24*60*60*1000l });
         _context.statManager().createRateStat("tunnel.receiveRejectionCritical", "How often we are rejected due to critical failure?", "Tunnels", new long[] { 10*60*1000l, 60*60*1000l, 24*60*60*1000l });
+
+        _context.statManager().createRateStat("tunnel.corruptBuildReply", "", "Tunnels", new long[] { 24*60*60*1000l });
         
         _processor = new BuildMessageProcessor(ctx);
         _buildMessageHandlerJob = new TunnelBuildMessageHandlerJob(ctx);
         _buildReplyMessageHandlerJob = new TunnelBuildReplyMessageHandlerJob(ctx);
-        ctx.inNetMessagePool().registerHandlerJobBuilder(TunnelBuildMessage.MESSAGE_TYPE, new TunnelBuildMessageHandlerJobBuilder());
+        TunnelBuildMessageHandlerJobBuilder tbmhjb = new TunnelBuildMessageHandlerJobBuilder();
-        ctx.inNetMessagePool().registerHandlerJobBuilder(TunnelBuildReplyMessage.MESSAGE_TYPE, new TunnelBuildReplyMessageHandlerJobBuilder());
+        TunnelBuildReplyMessageHandlerJobBuilder tbrmhjb = new TunnelBuildReplyMessageHandlerJobBuilder();
+        ctx.inNetMessagePool().registerHandlerJobBuilder(TunnelBuildMessage.MESSAGE_TYPE, tbmhjb);
+        ctx.inNetMessagePool().registerHandlerJobBuilder(TunnelBuildReplyMessage.MESSAGE_TYPE, tbrmhjb);
+        ctx.inNetMessagePool().registerHandlerJobBuilder(VariableTunnelBuildMessage.MESSAGE_TYPE, tbmhjb);
+        ctx.inNetMessagePool().registerHandlerJobBuilder(VariableTunnelBuildReplyMessage.MESSAGE_TYPE, tbrmhjb);
     }
     
     private static final int MAX_HANDLE_AT_ONCE = 2;
@@ -219,28 +227,13 @@ class BuildHandler {
     private void handleReply(BuildReplyMessageState state) {
         // search through the tunnels for a reply
         long replyMessageId = state.msg.getUniqueId();
-        PooledTunnelCreatorConfig cfg = null;
+        PooledTunnelCreatorConfig cfg = _exec.removeFromBuilding(replyMessageId);
-        List building = _exec.locked_getCurrentlyBuilding();
         StringBuilder buf = null;
-        synchronized (building) {
-            for (int i = 0; i < building.size(); i++) {
-                PooledTunnelCreatorConfig cur = (PooledTunnelCreatorConfig)building.get(i);
-                if (cur.getReplyMessageId() == replyMessageId) {
-                    building.remove(i);
-                    cfg = cur;
-                    break;
-                }
-            }
-            if ( (cfg == null) && (_log.shouldLog(Log.DEBUG)) )
-                buf = new StringBuilder(building.toString());
-        }
         
         if (cfg == null) {
             // cannot handle - not pending... took too long?
             if (_log.shouldLog(Log.WARN))
                 _log.warn("The reply " + replyMessageId + " did not match any pending tunnels");
-            if (_log.shouldLog(Log.DEBUG))
-                _log.debug("Pending tunnels: " + buf.toString());
             _context.statManager().addRateData("tunnel.buildReplyTooSlow", 1, 0);
         } else {
             handleReply(state.msg, cfg, System.currentTimeMillis()-state.recvTime);
@@ -253,9 +246,8 @@ class BuildHandler {
         if (_log.shouldLog(Log.INFO))
             _log.info(msg.getUniqueId() + ": Handling the reply after " + rtt + ", delayed " + delay + " waiting for " + cfg);
         
-        BuildReplyHandler handler = new BuildReplyHandler();
+        List<Integer> order = cfg.getReplyOrder();
-        List order = cfg.getReplyOrder();
+        int statuses[] = BuildReplyHandler.decrypt(_context, msg, cfg, order);
-        int statuses[] = handler.decrypt(_context, msg, cfg, order);
         if (statuses != null) {
             boolean allAgree = true;
             // For each peer in the tunnel
@@ -338,6 +330,7 @@ class BuildHandler {
         } else {
             if (_log.shouldLog(Log.WARN))
                 _log.warn(msg.getUniqueId() + ": Tunnel reply could not be decrypted for tunnel " + cfg);
+            _context.statManager().addRateData("tunnel.corruptBuildReply", 1, 0);
             // don't leak
             _exec.buildComplete(cfg, cfg.getTunnelPool());
         }
@@ -403,8 +396,13 @@ class BuildHandler {
      * This request is actually a reply, process it as such
      */
     private void handleRequestAsInboundEndpoint(BuildEndMessageState state) {
-        TunnelBuildReplyMessage msg = new TunnelBuildReplyMessage(_context);
+        int records = state.msg.getRecordCount();
-        for (int i = 0; i < TunnelBuildMessage.RECORD_COUNT; i++)
+        TunnelBuildReplyMessage msg;
+        if (records == TunnelBuildMessage.MAX_RECORD_COUNT)
+            msg = new TunnelBuildReplyMessage(_context);
+        else
+            msg = new VariableTunnelBuildReplyMessage(_context, records);
+        for (int i = 0; i < records; i++)
             msg.setRecord(i, state.msg.getRecord(i));
         msg.setUniqueId(state.msg.getUniqueId());
         handleReply(msg, state.cfg, System.currentTimeMillis() - state.recvTime);
@@ -490,7 +488,6 @@ class BuildHandler {
      *  If we did credit the reply to the tunnel, it would
      *  prevent the classification of the tunnel as 'inactive' on tunnels.jsp.
      */
-    @SuppressWarnings("static-access")
     private void handleReq(RouterInfo nextPeerInfo, BuildMessageState state, BuildRequestRecord req, Hash nextPeer) {
         long ourId = req.readReceiveTunnelId();
         long nextId = req.readNextTunnelId();
@@ -613,7 +610,8 @@ class BuildHandler {
         }
 
         byte reply[] = BuildResponseRecord.create(_context, response, req.readReplyKey(), req.readReplyIV(), state.msg.getUniqueId());
-        for (int j = 0; j < TunnelBuildMessage.RECORD_COUNT; j++) {
+        int records = state.msg.getRecordCount();
+        for (int j = 0; j < records; j++) {
             if (state.msg.getRecord(j) == null) {
                 ourSlot = j;
                 state.msg.setRecord(j, new ByteArray(reply));
@@ -648,9 +646,12 @@ class BuildHandler {
         } else {
             // send it to the reply tunnel on the reply peer within a new TunnelBuildReplyMessage
             // (enough layers jrandom?)
-            TunnelBuildReplyMessage replyMsg = new TunnelBuildReplyMessage(_context);
+            TunnelBuildReplyMessage replyMsg;
-            /* FIXME Accessing static field "RECORD_COUNT" FIXME */
+            if (records == TunnelBuildMessage.MAX_RECORD_COUNT)
-            for (int i = 0; i < state.msg.RECORD_COUNT; i++)
+                replyMsg = new TunnelBuildReplyMessage(_context);
+            else
+                replyMsg = new VariableTunnelBuildReplyMessage(_context, records);
+            for (int i = 0; i < records; i++)
                 replyMsg.setRecord(i, state.msg.getRecord(i));
             replyMsg.setUniqueId(req.readReplyMessageId());
             replyMsg.setMessageExpiration(_context.clock().now() + 10*1000);
@@ -693,28 +694,16 @@ class BuildHandler {
             // need to figure out if this is a reply to an inbound tunnel request (where we are the
             // endpoint, receiving the request at the last hop)
             long reqId = receivedMessage.getUniqueId();
-            PooledTunnelCreatorConfig cfg = null;
+            PooledTunnelCreatorConfig cfg = _exec.removeFromBuilding(reqId);
-            List building = _exec.locked_getCurrentlyBuilding();
-            List ids = new ArrayList();
-            synchronized (building) {
-                for (int i = 0; i < building.size(); i++) {
-                    PooledTunnelCreatorConfig cur = (PooledTunnelCreatorConfig)building.get(i);
-                    ids.add(new Long(cur.getReplyMessageId()));
-                    if ( (cur.isInbound()) && (cur.getReplyMessageId() == reqId) ) {
-                        building.remove(i);
-                        cfg = cur;
-                        break;
-                    } else if (cur.getReplyMessageId() == reqId) {
-                        _log.error("received it, but its not inbound? " + cur);
-                    }
-                }
-            }
             if (_log.shouldLog(Log.DEBUG))
                 _log.debug("Receive tunnel build message " + reqId + " from " 
                            + (from != null ? from.calculateHash().toBase64() : fromHash != null ? fromHash.toBase64() : "tunnels") 
-                           + ", waiting ids: " + ids + ", found matching tunnel? " + (cfg != null), 
+                           + ", found matching tunnel? " + (cfg != null));
-                           null);//new Exception("source"));
             if (cfg != null) {
+                if (!cfg.isInbound()) {
+                    // shouldnt happen - should we put it back?
+                    _log.error("received it, but its not inbound? " + cfg);
+                }
                 BuildEndMessageState state = new BuildEndMessageState(cfg, receivedMessage);
                 if (HANDLE_REPLIES_INLINE) {
                     handleRequestAsInboundEndpoint(state);

router/java/src/net/i2p/router/tunnel/pool/BuildRequestor.java

@@ -12,18 +12,20 @@ import net.i2p.data.RouterInfo;
 import net.i2p.data.TunnelId;
 import net.i2p.data.i2np.I2NPMessage;
 import net.i2p.data.i2np.TunnelBuildMessage;
+import net.i2p.data.i2np.VariableTunnelBuildMessage;
 import net.i2p.router.JobImpl;
 import net.i2p.router.OutNetMessage;
 import net.i2p.router.RouterContext;
 import net.i2p.router.TunnelInfo;
 import net.i2p.router.tunnel.BuildMessageGenerator;
 import net.i2p.util.Log;
+import net.i2p.util.VersionComparator;
 
 /**
  *
  */
 class BuildRequestor {
-    private static final List ORDER = new ArrayList(BuildMessageGenerator.ORDER.length);
+    private static final List<Integer> ORDER = new ArrayList(BuildMessageGenerator.ORDER.length);
     static {
         for (int i = 0; i < BuildMessageGenerator.ORDER.length; i++)
             ORDER.add(Integer.valueOf(i));
@@ -50,7 +52,7 @@ class BuildRequestor {
     }
     
     /** new style requests need to fill in the tunnel IDs before hand */
-    public static void prepare(RouterContext ctx, PooledTunnelCreatorConfig cfg) {
+    private static void prepare(RouterContext ctx, PooledTunnelCreatorConfig cfg) {
         for (int i = 0; i < cfg.getLength(); i++) {
             if ( (!cfg.isInbound()) && (i == 0) ) {
                 // outbound gateway (us) doesn't receive on a tunnel id
@@ -67,8 +69,14 @@ class BuildRequestor {
             cfg.getConfig(i).setReplyIV(new ByteArray(iv));
             cfg.getConfig(i).setReplyKey(ctx.keyGenerator().generateSessionKey());
         }
-        cfg.setReplyMessageId(ctx.random().nextLong(I2NPMessage.MAX_ID_VALUE));
+        // This is in BuildExecutor.buildTunnel() now
+        // And it was overwritten by the one in createTunnelBuildMessage() anyway!
+        //cfg.setReplyMessageId(ctx.random().nextLong(I2NPMessage.MAX_ID_VALUE));
     }
+
+    /**
+     *  @param cfg ReplyMessageId must be set
+     */
     public static void request(RouterContext ctx, TunnelPool pool, PooledTunnelCreatorConfig cfg, BuildExecutor exec) {
         // new style crypto fills in all the blanks, while the old style waits for replies to fill in the next hop, etc
         prepare(ctx, cfg);
@@ -156,33 +164,94 @@ class BuildRequestor {
                       + "ms and dispatched in " + (System.currentTimeMillis()-beforeDispatch));
     }
     
+    private static final String MIN_VARIABLE_VERSION = "0.7.11";
+    /** 5 (~2600 bytes) fits nicely in 3 tunnel messages */
+    private static final int SHORT_RECORDS = 5;
+    private static final int LONG_RECORDS = TunnelBuildMessage.MAX_RECORD_COUNT;
+    private static final VersionComparator _versionComparator = new VersionComparator();
+    private static final List<Integer> SHORT_ORDER = new ArrayList(SHORT_RECORDS);
+    static {
+        for (int i = 0; i < SHORT_RECORDS; i++)
+            SHORT_ORDER.add(Integer.valueOf(i));
+    }
+
+    private static boolean supportsVariable(RouterContext ctx, Hash h) {
+        RouterInfo ri = ctx.netDb().lookupRouterInfoLocally(h);
+        if (ri == null)
+            return false;
+        String v = ri.getOption("router.version");
+        if (v == null)
+            return false;
+        return _versionComparator.compare(v, MIN_VARIABLE_VERSION) >= 0;
+    }
+
+    /**
+     *  If the tunnel is short enough, and everybody in the tunnel, and the
+     *  OBEP or IBGW for the paired tunnel, all support the new variable-sized tunnel build message,
+     *  then use that, otherwise the old 8-entry version.
+     */
     private static TunnelBuildMessage createTunnelBuildMessage(RouterContext ctx, TunnelPool pool, PooledTunnelCreatorConfig cfg, TunnelInfo pairedTunnel, BuildExecutor exec) {
         Log log = ctx.logManager().getLog(BuildRequestor.class);
         long replyTunnel = 0;
         Hash replyRouter = null;
+        boolean useVariable = cfg.getLength() <= SHORT_RECORDS;
         if (cfg.isInbound()) {
-            replyTunnel = 0;
+            //replyTunnel = 0; // as above
             replyRouter = ctx.routerHash();
+            if (useVariable) {
+                // check the reply OBEP and all the tunnel peers except ourselves
+                if (!supportsVariable(ctx, pairedTunnel.getPeer(pairedTunnel.getLength() - 1))) {
+                    useVariable = false;
+                } else {
+                    for (int i = 0; i < cfg.getLength() - 1; i++) {
+                        if (!supportsVariable(ctx, cfg.getPeer(i))) {
+                            useVariable = false;
+                            break;
+                        }
+                    }
+                }
+            }
         } else {
             replyTunnel = pairedTunnel.getReceiveTunnelId(0).getTunnelId();
             replyRouter = pairedTunnel.getPeer(0);
+            if (useVariable) {
+                // check the reply IBGW and all the tunnel peers except ourselves
+                if (!supportsVariable(ctx, replyRouter)) {
+                    useVariable = false;
+                } else {
+                    for (int i = 1; i < cfg.getLength() - 1; i++) {
+                        if (!supportsVariable(ctx, cfg.getPeer(i))) {
+                            useVariable = false;
+                            break;
+                        }
+                    }
+                }
+            }
         }
 
         // populate and encrypt the message
-        BuildMessageGenerator gen = new BuildMessageGenerator();
+        TunnelBuildMessage msg;
-        TunnelBuildMessage msg = new TunnelBuildMessage(ctx);
+        List<Integer> order;
+        if (useVariable) {
+            msg = new VariableTunnelBuildMessage(ctx, SHORT_RECORDS);
+            order = new ArrayList(SHORT_ORDER);
+            log.log(Log.CRIT, "Using new VTBM");
+        } else {
+            msg = new TunnelBuildMessage(ctx);
+            order = new ArrayList(ORDER);
+        }
 
-        long replyMessageId = ctx.random().nextLong(I2NPMessage.MAX_ID_VALUE);
+        // This is in BuildExecutor.buildTunnel() now
-        cfg.setReplyMessageId(replyMessageId);
+        //long replyMessageId = ctx.random().nextLong(I2NPMessage.MAX_ID_VALUE);
+        //cfg.setReplyMessageId(replyMessageId);
         
-        List order = new ArrayList(ORDER);
         Collections.shuffle(order, ctx.random()); // randomized placement within the message
         cfg.setReplyOrder(order);
         
         if (log.shouldLog(Log.DEBUG))
             log.debug("Build order: " + order + " for " + cfg);
         
-        for (int i = 0; i < BuildMessageGenerator.ORDER.length; i++) {
+        for (int i = 0; i < msg.getRecordCount(); i++) {
             int hop = ((Integer)order.get(i)).intValue();
             PublicKey key = null;
     
@@ -202,9 +271,9 @@ class BuildRequestor {
             }
             if (log.shouldLog(Log.DEBUG))
                 log.debug(cfg.getReplyMessageId() + ": record " + i + "/" + hop + " has key " + key);
-            gen.createRecord(i, hop, msg, cfg, replyRouter, replyTunnel, ctx, key);
+            BuildMessageGenerator.createRecord(i, hop, msg, cfg, replyRouter, replyTunnel, ctx, key);
         }
-        gen.layeredEncrypt(ctx, msg, cfg, order);
+        BuildMessageGenerator.layeredEncrypt(ctx, msg, cfg, order);
         
         return msg;
     }