merge of '1d753ff05fc1c942cf6ce8feeadfaa577bff27ab'

and 'e20c0075f57440edcebaac0009a0ef0327d563cc'

checklist.txt

@@ -3,6 +3,7 @@ Release checklist
 
 Sync with mtn.i2p2.i2p
 Start with a clean checkout mtn -d i2p.mtn co --branch=i2p.i2p
+Double-check trust list
 
 Change revision in:
 	history.txt
@@ -14,7 +15,7 @@ Change revision in:
 Build and tag:
 	ant dist
 	mtn ci
-	mtn tag i2p-0.6.1.xx
+	mtn tag i2p-0.6.1.xx h:
 	Sync with mtn.i2p2.i2p
 
 Create a signed update file with:
@@ -27,11 +28,12 @@ Verify signed update file with:
 
 Make the source tarball:
 	Start with a clean checkout mtn -d i2p.mtn co --branch=i2p.i2p i2p-0.6.1.xx
-	tar tjf i2p-0.6.1.xx.tar.bz2 --exclude i2p-0.6.1.xx/_MTN i2p-0.6.1.xx	
+        Double-check trust list
+	tar cjf i2psource-0.6.1.xx.tar.bz2 --exclude i2p-0.6.1.xx/_MTN i2p-0.6.1.xx	
 	mv i2p-0.6.1.xx.tar.bz2 i2p.i2p
 
 More signatures:
-	sha1sum i2pinstall.exe i2p.tar.bz2 i2p-0.6.1.xx.tar.bz2 i2pupdate.zip
+	sha1sum i2pinstall.exe i2p.tar.bz2 i2psource-0.6.1.xx.tar.bz2 i2pupdate.zip
 	gpg -b i2pinstall.exe
 	gpg -b i2p.tar.bz2
 	gpg -b i2p-0.6.1.xx.tar.bz2

history.txt

@@ -1,3 +1,13 @@
+2008-02-10 zzz
+    * Add new tunnel build algorithm (preliminary)
+    * Change NTCP backlogged message from error to warning
+    * Checklist updates
+
+* 2008-02-10  0.6.1.31 released
+
+2008-02-10 Complication
+    * Update news and version numbers
+
 2008-02-06 zzz
     * build.xml: Add some apps to javadoc
     * checklist.txt: Add some things

initialNews.xml

@@ -1,4 +1,4 @@
-<i2p.news date="$Date: 2007-08-23 19:33:29 $">
+<i2p.news date="$Date: 2008-02-10 15:00:00 $">
  <i2p.release version="0.6.1.31" date="2008/02/10" minVersion="0.6"
               anonurl="http://i2p/NF2RLVUxVulR3IqK0sGJR0dHQcGXAzwa6rEO4WAWYXOHw-DoZhKnlbf1nzHXwMEJoex5nFTyiNMqxJMWlY54cvU~UenZdkyQQeUSBZXyuSweflUXFqKN-y8xIoK2w9Ylq1k8IcrAFDsITyOzjUKoOPfVq34rKNDo7fYyis4kT5bAHy~2N1EVMs34pi2RFabATIOBk38Qhab57Umpa6yEoE~rbyR~suDRvD7gjBvBiIKFqhFueXsR2uSrPB-yzwAGofTXuklofK3DdKspciclTVzqbDjsk5UXfu2nTrC1agkhLyqlOfjhyqC~t1IXm-Vs2o7911k7KKLGjB4lmH508YJ7G9fLAUyjuB-wwwhejoWqvg7oWvqo4oIok8LG6ECR71C3dzCvIjY2QcrhoaazA9G4zcGMm6NKND-H4XY6tUWhpB~5GefB3YczOqMbHq4wi0O9MzBFrOJEOs3X4hwboKWANf7DT5PZKJZ5KorQPsYRSq0E3wSOsFCSsdVCKUGsAAAA/i2p/i2pupdate.sud"
               publicurl="http://dev.i2p.net/i2p/i2pupdate.sud"

news.xml

@@ -1,5 +1,5 @@
-<i2p.news date="$Date: 2008-02-05 00:00:00 $">
+<i2p.news date="$Date: 2008-02-10 15:00:00 $">
- <i2p.release version="0.6.1.30" date="2007/02/15" minVersion="0.6"
+ <i2p.release version="0.6.1.31" date="2008/02/10" minVersion="0.6"
               anonurl="http://i2p/NF2RLVUxVulR3IqK0sGJR0dHQcGXAzwa6rEO4WAWYXOHw-DoZhKnlbf1nzHXwMEJoex5nFTyiNMqxJMWlY54cvU~UenZdkyQQeUSBZXyuSweflUXFqKN-y8xIoK2w9Ylq1k8IcrAFDsITyOzjUKoOPfVq34rKNDo7fYyis4kT5bAHy~2N1EVMs34pi2RFabATIOBk38Qhab57Umpa6yEoE~rbyR~suDRvD7gjBvBiIKFqhFueXsR2uSrPB-yzwAGofTXuklofK3DdKspciclTVzqbDjsk5UXfu2nTrC1agkhLyqlOfjhyqC~t1IXm-Vs2o7911k7KKLGjB4lmH508YJ7G9fLAUyjuB-wwwhejoWqvg7oWvqo4oIok8LG6ECR71C3dzCvIjY2QcrhoaazA9G4zcGMm6NKND-H4XY6tUWhpB~5GefB3YczOqMbHq4wi0O9MzBFrOJEOs3X4hwboKWANf7DT5PZKJZ5KorQPsYRSq0E3wSOsFCSsdVCKUGsAAAA/i2p/i2pupdate.sud"
               publicurl="http://dev.i2p.net/i2p/i2pupdate.sud"
               anonannouncement="http://i2p/NF2RLVUxVulR3IqK0sGJR0dHQcGXAzwa6rEO4WAWYXOHw-DoZhKnlbf1nzHXwMEJoex5nFTyiNMqxJMWlY54cvU~UenZdkyQQeUSBZXyuSweflUXFqKN-y8xIoK2w9Ylq1k8IcrAFDsITyOzjUKoOPfVq34rKNDo7fYyis4kT5bAHy~2N1EVMs34pi2RFabATIOBk38Qhab57Umpa6yEoE~rbyR~suDRvD7gjBvBiIKFqhFueXsR2uSrPB-yzwAGofTXuklofK3DdKspciclTVzqbDjsk5UXfu2nTrC1agkhLyqlOfjhyqC~t1IXm-Vs2o7911k7KKLGjB4lmH508YJ7G9fLAUyjuB-wwwhejoWqvg7oWvqo4oIok8LG6ECR71C3dzCvIjY2QcrhoaazA9G4zcGMm6NKND-H4XY6tUWhpB~5GefB3YczOqMbHq4wi0O9MzBFrOJEOs3X4hwboKWANf7DT5PZKJZ5KorQPsYRSq0E3wSOsFCSsdVCKUGsAAAA/pipermail/i2p/2005-September/000878.html" 
@@ -9,25 +9,22 @@
             publicurl="http://dev.i2p.net/pipermail/i2p/2005-July/000826.html"
             anonlogs="http://i2p/Nf3ab-ZFkmI-LyMt7GjgT-jfvZ3zKDl0L96pmGQXF1B82W2Bfjf0n7~288vafocjFLnQnVcmZd~-p0-Oolfo9aW2Rm-AhyqxnxyLlPBqGxsJBXjPhm1JBT4Ia8FB-VXt0BuY0fMKdAfWwN61-tj4zIcQWRxv3DFquwEf035K~Ra4SWOqiuJgTRJu7~o~DzHVljVgWIzwf8Z84cz0X33pv-mdG~~y0Bsc2qJVnYwjjR178YMcRSmNE0FVMcs6f17c6zqhMw-11qjKpY~EJfHYCx4lBWF37CD0obbWqTNUIbL~78vxqZRT3dgAgnLixog9nqTO-0Rh~NpVUZnoUi7fNR~awW5U3Cf7rU7nNEKKobLue78hjvRcWn7upHUF45QqTDuaM3yZa7OsjbcH-I909DOub2Q0Dno6vIwuA7yrysccN1sbnkwZbKlf4T6~iDdhaSLJd97QCyPOlbyUfYy9QLNExlRqKgNVJcMJRrIual~Lb1CLbnzt0uvobM57UpqSAAAA/meeting141"
             publiclogs="http://www.i2p.net/meeting141" />
-&#149;
-2008-02-05: <b>*.i2p[.net] Replacements</b>
 <p>
-As you probably know, <a href="http://www.i2p.net/">www.i2p.net</a>
+&#149;
-and its related sites and services went down January 13 after
+2008-02-10: <b>0.6.1.31 released</b>
-a power outage at the hosting facility, and did not return fully to service.
-While some pages are still available at
-<a href="http://66.111.51.110/">66.111.51.110</a>
-we have a new official site at
-<a href="http://www.i2p2.de/">www.i2p2.de</a>
-and
-<a href="http://www.i2p2.i2p/">www.i2p2.i2p</a>.
-Extensive information about replacements for the various services
-is available at
-<a href="http://forum.i2p/viewtopic.php?t=2500">the forums</a>,
-<a href="http://complication.i2p/">complication.i2p</a>, and
-<a href="http://zzz.i2p/">zzz.i2p</a>.
-
 </p><p>
+The transitional 0.6.1.31 release will make the I2P
+release process independent of jrandom and *.i2p.net servers,
+introducing two new update verification keys, adding support
+for distributed update hosting inside I2P, and changing many URLs.
+</p><p>
+It introduces version 2 of the SAM protocol,
+along with substantial improvements to the address book,
+I2PSnark and ircproxy. Unless you have already done that,
+read about the update procedure below.
+</p>
+
+<p>
 &#149;
 2008-02-05: <b>Upcoming 0.6.1.31 Release</b>
 </p><p>
@@ -90,6 +87,26 @@ Thank you for your support during this transition. For help please contact us on
 Amiga, Complication, welterde, zzz
 </p>
 
+<p>
+&#149;
+2008-02-05: <b>*.i2p[.net] Replacements</b>
+<p>
+As you probably know, <a href="http://www.i2p.net/">www.i2p.net</a>
+and its related sites and services went down January 13 after
+a power outage at the hosting facility, and did not return fully to service.
+While some pages are still available at
+<a href="http://66.111.51.110/">66.111.51.110</a>
+we have a new official site at
+<a href="http://www.i2p2.de/">www.i2p2.de</a>
+and
+<a href="http://www.i2p2.i2p/">www.i2p2.i2p</a>.
+Extensive information about replacements for the various services
+is available at
+<a href="http://forum.i2p/viewtopic.php?t=2500">the forums</a>,
+<a href="http://complication.i2p/">complication.i2p</a>, and
+<a href="http://zzz.i2p/">zzz.i2p</a>.
+</p>
+
 <!--
 &#149;
 2007-04-10:

router/java/src/net/i2p/router/RouterVersion.java

@@ -15,9 +15,9 @@ import net.i2p.CoreVersion;
  *
  */
 public class RouterVersion {
-    public final static String ID = "$Revision: 1.548 $ $Date: 2008-01-08 21:15:43 $";
+    public final static String ID = "$Revision: 1.548 $ $Date: 2008-02-10 15:00:00 $";
-    public final static String VERSION = "0.6.1.30";
+    public final static String VERSION = "0.6.1.31";
-    public final static long BUILD = 27;
+    public final static long BUILD = 1;
     public static void main(String args[]) {
         System.out.println("I2P Router version: " + VERSION + "-" + BUILD);
         System.out.println("Router ID: " + RouterVersion.ID);

router/java/src/net/i2p/router/transport/ntcp/NTCPConnection.java

@@ -268,8 +268,8 @@ public class NTCPConnection implements FIFOBandwidthLimiter.CompleteListener {
                 try { wantsWrite = ( (_conKey.interestOps() & SelectionKey.OP_WRITE) != 0); } catch (Exception e) {}
                 int blocks = 0;
 		synchronized (_writeBufs) { blocks = _writeBufs.size(); }
-                if (_log.shouldLog(Log.ERROR))
+                if (_log.shouldLog(Log.WARN))
-                    _log.error("Too backlogged for too long (" + _consecutiveBacklog + " messages for " + DataHelper.formatDuration(queueTime()) + ", sched? " + wantsWrite + ", blocks: " + blocks + ") sending to " + _remotePeer.calculateHash().toBase64());
+                    _log.warn("Too backlogged for too long (" + _consecutiveBacklog + " messages for " + DataHelper.formatDuration(queueTime()) + ", sched? " + wantsWrite + ", blocks: " + blocks + ") sending to " + _remotePeer.calculateHash().toBase64());
                 _context.statManager().addRateData("ntcp.closeOnBacklog", getUptime(), getUptime());
                 close();
             }

router/java/src/net/i2p/router/tunnel/pool/ExpireJob.java

@@ -19,9 +19,12 @@ class ExpireJob extends JobImpl {
         // so we rebuild out of sync.  otoh, we will honor tunnel messages on it
         // up through the full lifetime of the tunnel, plus a clock skew, since
         // others may be sending to the published lease expirations
+        // Also skew the inbound away from the outbound
         long expire = cfg.getExpiration();
         _dropAfter = expire + Router.CLOCK_FUDGE_FACTOR;
-        expire -= ctx.random().nextLong(5*60*1000);
+        expire -= ctx.random().nextLong(60*1000);
+        if (_pool.getSettings().isInbound())
+            expire -= ctx.random().nextLong(15*1000);
         cfg.setExpiration(expire);
         getTiming().setStartAfter(expire);
     }

router/java/src/net/i2p/router/tunnel/pool/TunnelPool.java

@@ -1,5 +1,6 @@
 package net.i2p.router.tunnel.pool;
 
+import java.util.Arrays;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Date;
@@ -16,6 +17,8 @@ import net.i2p.router.RouterContext;
 import net.i2p.router.TunnelPoolSettings;
 import net.i2p.router.TunnelInfo;
 import net.i2p.router.tunnel.HopConfig;
+import net.i2p.stat.Rate;
+import net.i2p.stat.RateStat;
 import net.i2p.util.Log;
 
 /**
@@ -36,7 +39,9 @@ public class TunnelPool {
     private long _started;
     private long _lastRateUpdate;
     private long _lastLifetimeProcessed;
-    private String _rateName;
+    private final String _rateName;
+    private final String _buildStatName;
+    private static final int TUNNEL_LIFETIME = 10*60*1000;
     
     public TunnelPool(RouterContext ctx, TunnelPoolManager mgr, TunnelPoolSettings settings, TunnelPeerSelector sel) {
         _context = ctx;
@@ -56,6 +61,9 @@ public class TunnelPool {
         _rateName = "tunnel.Bps." +
                     (_settings.isExploratory() ? "exploratory" : _settings.getDestinationNickname()) +
                     (_settings.isInbound() ? ".in" : ".out");
+        _buildStatName = "tunnel.build." +
+                    (_settings.isExploratory() ? "exploratory" : _settings.getDestinationNickname()) +
+                    (_settings.isInbound() ? ".in" : ".out");
         refreshSettings();
     }
     
@@ -79,6 +87,9 @@ public class TunnelPool {
         _context.statManager().createRateStat(_rateName,
                                "Tunnel Bandwidth", "Tunnels", 
                                new long[] { 5*60*1000l });
+        _context.statManager().createRateStat(_buildStatName,
+                               "Tunnel Build Frequency", "Tunnels", 
+                               new long[] { TUNNEL_LIFETIME });
     }
     
     public void shutdown() {
@@ -148,25 +159,43 @@ public class TunnelPool {
                 
                 // if there are nonzero hop tunnels and the zero hop tunnels are fallbacks, 
                 // avoid the zero hop tunnels
+                TunnelInfo backloggedTunnel = null;
                 if (avoidZeroHop) {
                     for (int i = 0; i < _tunnels.size(); i++) {
                         TunnelInfo info = (TunnelInfo)_tunnels.get(i);
                         if ( (info.getLength() > 1) && (info.getExpiration() > _context.clock().now()) ) {
-                            _lastSelected = info;
+                            // avoid outbound tunnels where the 1st hop is backlogged
-                            return info;
+                            if (_settings.isInbound() || !_context.commSystem().isBacklogged(info.getPeer(1))) {
+                                _lastSelected = info;
+                                return info;
+                            } else {
+                                backloggedTunnel = info;
+                            }
                         }
                     }
+                    // return a random backlogged tunnel
+                    if (backloggedTunnel != null)
+                        return backloggedTunnel;
                 }
                 // ok, either we are ok using zero hop tunnels, or only fallback tunnels remain.  pick 'em
                 // randomly
                 for (int i = 0; i < _tunnels.size(); i++) {
                     TunnelInfo info = (TunnelInfo)_tunnels.get(i);
                     if (info.getExpiration() > _context.clock().now()) {
-                        //_log.debug("Selecting tunnel: " + info + " - " + _tunnels);
+                        // avoid outbound tunnels where the 1st hop is backlogged
-                        _lastSelected = info;
+                        if (_settings.isInbound() || info.getLength() <= 1 ||
-                        return info;
+                            !_context.commSystem().isBacklogged(info.getPeer(1))) {
+                            //_log.debug("Selecting tunnel: " + info + " - " + _tunnels);
+                            _lastSelected = info;
+                            return info;
+                        } else {
+                            backloggedTunnel = info;
+                        }
                     }
                 }
+                // return a random backlogged tunnel
+                if (backloggedTunnel != null)
+                    return backloggedTunnel;
                 if (_log.shouldLog(Log.WARN))
                     _log.warn(toString() + ": after " + _tunnels.size() + " tries, no unexpired ones were found: " + _tunnels);
             }
@@ -445,6 +474,106 @@ public class TunnelPool {
         
         boolean allowZeroHop = ((getSettings().getLength() + getSettings().getLengthVariance()) <= 0);
           
+        /**
+         * This algorithm builds based on the previous average length of time it takes
+         * to build a tunnel. This average is kept in the _buildRateName stat.
+         * It is a separate stat for each pool, since in and out building use different methods,
+         * and each pool can have separate length and length variance settings.
+         * We add one minute to the stat for safety.
+         *
+         * We linearly increase the number of builds per expiring tunnel from
+         * 1 to PANIC_FACTOR as the time-to-expire gets shorter.
+         *
+         * The stat will be 0 for first 10m of uptime so we will use the conservative algorithm
+         * further below instead. It will take about 30m of uptime to settle down.
+         * Or, if we are building more than 33% of the time something is seriously wrong,
+         * we also use the conservative algorithm instead
+         *
+         **/
+
+        int avg = 0;
+        RateStat rs = _context.statManager().getRate(_buildStatName);
+        if (rs != null) {
+            Rate r = rs.getRate(TUNNEL_LIFETIME);
+            if (r != null)
+                avg = (int) ( TUNNEL_LIFETIME * r.getAverageValue() / wanted);
+        }
+
+        if (avg > 0 && avg < TUNNEL_LIFETIME / 3) {
+            final int PANIC_FACTOR = 4;  // how many builds to kick off when time gets short
+            avg += 60*1000;   // one minute safety factor
+            if (_settings.isExploratory())
+                avg += 60*1000;   // two minute safety factor
+            long now = _context.clock().now();
+
+            int expireSoon = 0;
+            int expireLater = 0;
+            int expireTime[];
+            int fallback = 0;
+            synchronized (_tunnels) {
+                expireTime = new int[_tunnels.size()];
+                for (int i = 0; i < _tunnels.size(); i++) {
+                    TunnelInfo info = (TunnelInfo)_tunnels.get(i);
+                    if (allowZeroHop || (info.getLength() > 1)) {
+                        int timeToExpire = (int) (info.getExpiration() - now);
+                        if (timeToExpire > 0 && timeToExpire < avg) {
+                            expireTime[expireSoon++] = timeToExpire;
+                        } else {
+                            expireLater++;
+                        }
+                    } else if (info.getExpiration() - now > avg) {
+                        fallback++;
+                    }
+                }
+            }
+
+            int inProgress;
+            synchronized (_inProgress) {
+                inProgress = _inProgress.size();
+            }
+            int remainingWanted = (wanted - expireLater) - inProgress;
+            if (allowZeroHop)
+                remainingWanted -= fallback;
+
+            int rv = 0;
+            int latesttime = 0;
+            if (remainingWanted > 0) {
+                if (remainingWanted > expireSoon) {
+                    rv = PANIC_FACTOR * (remainingWanted - expireSoon);  // for tunnels completely missing
+                    remainingWanted = expireSoon;
+                }
+                // add from 1 to PANIC_FACTOR builds, depending on how late it is
+                // only use the expire times of the latest-expiring tunnels,
+                // the other ones are extras
+                for (int i = 0; i < remainingWanted; i++) {
+                    int latestidx = 0;
+                    // given the small size of the array this is efficient enough
+                    for (int j = 0; j < expireSoon; j++) {
+                        if (expireTime[j] > latesttime) {
+                            latesttime = expireTime[j];
+                            latestidx = j;
+                        }
+                    }
+                    expireTime[latestidx] = 0;
+                    if (latesttime > avg / 2)
+                        rv += 1;
+                    else
+                        rv += 2 + ((PANIC_FACTOR - 2) * (((avg / 2) - latesttime) / (avg / 2)));
+                }
+            }
+
+            if (rv > 0 && _log.shouldLog(Log.DEBUG))
+                _log.debug("New Count: rv: " + rv + " allow? " + allowZeroHop
+                       + " avg " + avg + " latesttime " + latesttime
+                       + " soon " + expireSoon + " later " + expireLater
+                       + " std " + wanted + " inProgress " + inProgress + " fallback " + fallback 
+                       + " for " + toString());
+            _context.statManager().addRateData(_buildStatName, rv + inProgress, 0);
+            return rv;
+        }
+
+        // fixed, conservative algorithm - starts building 3 1/2 - 6m before expiration
+        // (210 or 270s) + (0..90s random)
         long expireAfter = _context.clock().now() + _expireSkew; // + _settings.getRebuildPeriod() + _expireSkew;
         int expire30s = 0;
         int expire90s = 0;
@@ -491,8 +620,11 @@ public class TunnelPool {
             }
         }
         
-        return countHowManyToBuild(allowZeroHop, expire30s, expire90s, expire150s, expire210s, expire270s, 
+        int rv = countHowManyToBuild(allowZeroHop, expire30s, expire90s, expire150s, expire210s, expire270s, 
                                    expireLater, wanted, inProgress, fallback);
+        _context.statManager().addRateData(_buildStatName, (rv > 0 || inProgress > 0) ? 1 : 0, 0);
+        return rv;
+
     }
     
     /**
@@ -590,7 +722,7 @@ public class TunnelPool {
         if ( (lifetime < 60*1000) && (rv + inProgress + fallback >= standardAmount) )
                 rv = standardAmount - inProgress - fallback;
         
-        if (_log.shouldLog(Log.DEBUG))
+        if (rv > 0 && _log.shouldLog(Log.DEBUG))
             _log.debug("Count: rv: " + rv + " allow? " + allowZeroHop
                        + " 30s " + expire30s + " 90s " + expire90s + " 150s " + expire150s + " 210s " + expire210s
                        + " 270s " + expire270s + " later " + expireLater