2006-04-05 jrandom

* Fix during the ssu handshake to avoid an unnecessary failure on
      packet retransmission (thanks ripple!)
    * Fix during the SSU handshake to use the negotiated session key asap,
      rather than using the intro key for more than we should (thanks ripple!)
    * Fixes to the message reply registry (thanks Complication!)
    * More comprehensive syndie banning (for repeated pushes)
    * Publish the router's ballpark bandwidth limit (w/in a power of 2), for
      testing purposes
    * Put a floor back on the capacity threshold, so too many failing peers
      won't cause us to pick very bad peers (unless we have very few good
      ones)
    * Bugfix to cut down on peers using introducers unneessarily (thanks
      Complication!)
    * Reduced the default streaming lib message size to fit into a single
      tunnel message, rather than require 5 tunnel messages to be transferred
      without loss before recomposition.  This reduces throughput, but should
      increase reliability, at least for the time being.
    * Misc small bugfixes in the router (thanks all!)
    * More tweaking for Syndie's CSS (thanks Doubtful Salmon!)

apps/routerconsole/java/src/net/i2p/router/web/ConfigNetHandler.java

@@ -250,7 +250,7 @@ public class ConfigNetHandler extends FormHandler {
             // If hidden mode value changes, restart is required
             if (_hiddenMode && "false".equalsIgnoreCase(_context.getProperty(Router.PROP_HIDDEN, "false"))) {
                 _context.router().setConfigSetting(Router.PROP_HIDDEN, "true");
-                _context.router().getRouterInfo().addCapability(RouterInfo.CAPABILITY_HIDDEN);
+                _context.router().addCapabilities(_context.router().getRouterInfo());
                 addFormNotice("Gracefully restarting into Hidden Router Mode. Make sure you have no 0-1 length "
                               + "<a href=\"configtunnels.jsp\">tunnels!</a>");
                 hiddenSwitch();

apps/streaming/java/src/net/i2p/client/streaming/ConnectionOptions.java

@@ -101,7 +101,7 @@ public class ConnectionOptions extends I2PSocketOptionsImpl {
         setMaxWindowSize(getInt(opts, PROP_MAX_WINDOW_SIZE, Connection.MAX_WINDOW_SIZE));
         setConnectDelay(getInt(opts, PROP_CONNECT_DELAY, -1));
         setProfile(getInt(opts, PROP_PROFILE, PROFILE_BULK));
-        setMaxMessageSize(getInt(opts, PROP_MAX_MESSAGE_SIZE, 4*1024));
+        setMaxMessageSize(getInt(opts, PROP_MAX_MESSAGE_SIZE, 960)); // 960 fits inside a single tunnel message
         setRTT(getInt(opts, PROP_INITIAL_RTT, 10*1000));
         setReceiveWindow(getInt(opts, PROP_INITIAL_RECEIVE_WINDOW, 1));
         setResendDelay(getInt(opts, PROP_INITIAL_RESEND_DELAY, 1000));

apps/syndie/java/src/net/i2p/syndie/BlogManager.java

@@ -890,6 +890,8 @@ public class BlogManager {
         try {
             BlogInfo info = new BlogInfo();
             info.load(metadataStream);
+            if (isBanned(info.getKey().calculateHash()))
+                return false;
             return _archive.storeBlogInfo(info);
         } catch (IOException ioe) {
             _log.error("Error importing meta", ioe);
@@ -906,6 +908,8 @@ public class BlogManager {
         try {
             EntryContainer c = new EntryContainer();
             c.load(entryStream);
+            if (isBanned(c.getURI().getKeyHash()))
+                return false;
             return _archive.storeEntry(c);
         } catch (IOException ioe) {
             _log.error("Error importing entry", ioe);

apps/syndie/jsp/syndie.css

@@ -134,9 +134,9 @@ select {
    display: inline;
 }
 .controlBar {
-	margin: 0em;
+        border-bottom: thick double #CCF;
-	padding: 0em;
+        border-left: medium solid #CCF;
-//	border: medium solid #DDF;
+        border-right: medium solid #CCF;
 	background-color: #EEF;
 	color: inherit;
 	font-size: small;

history.txt

@@ -1,4 +1,25 @@
-$Id: history.txt,v 1.442 2006/04/01 14:05:37 jrandom Exp $
+$Id: history.txt,v 1.443 2006/04/03 05:07:24 jrandom Exp $
+
+2006-04-05  jrandom
+    * Fix during the ssu handshake to avoid an unnecessary failure on
+      packet retransmission (thanks ripple!)
+    * Fix during the SSU handshake to use the negotiated session key asap,
+      rather than using the intro key for more than we should (thanks ripple!)
+    * Fixes to the message reply registry (thanks Complication!)
+    * More comprehensive syndie banning (for repeated pushes)
+    * Publish the router's ballpark bandwidth limit (w/in a power of 2), for 
+      testing purposes
+    * Put a floor back on the capacity threshold, so too many failing peers
+      won't cause us to pick very bad peers (unless we have very few good
+      ones)
+    * Bugfix to cut down on peers using introducers unneessarily (thanks
+      Complication!)
+    * Reduced the default streaming lib message size to fit into a single
+      tunnel message, rather than require 5 tunnel messages to be transferred
+      without loss before recomposition.  This reduces throughput, but should
+      increase reliability, at least for the time being.
+    * Misc small bugfixes in the router (thanks all!)
+    * More tweaking for Syndie's CSS (thanks Doubtful Salmon!)
 
 2006-04-01  jrandom
     * Take out the router watchdog's teeth (don't restart on leaseset failure)

router/java/src/net/i2p/router/Router.java

@@ -323,13 +323,8 @@ public class Router {
             stats.setProperty(RouterInfo.PROP_NETWORK_ID, NETWORK_ID+"");
             ri.setOptions(stats);
             ri.setAddresses(_context.commSystem().createAddresses());
-            if (FloodfillNetworkDatabaseFacade.floodfillEnabled(_context))
-                ri.addCapability(FloodfillNetworkDatabaseFacade.CAPACITY_FLOODFILL);
-            if("true".equalsIgnoreCase(_context.getProperty(Router.PROP_HIDDEN, "false"))) {
-                ri.addCapability(RouterInfo.CAPABILITY_HIDDEN);
-            }
 
-            addReachabilityCapability(ri);
+            addCapabilities(ri);
             SigningPrivateKey key = _context.keyManager().getSigningPrivateKey();
             if (key == null) {
                 _log.log(Log.CRIT, "Internal error - signing private key not known?  wtf");
@@ -358,15 +353,43 @@ public class Router {
         }
     }
     
+    // publicize our ballpark capacity - this does not affect anything at
+    // the moment
+    public static final char CAPABILITY_BW16 = 'K';
+    public static final char CAPABILITY_BW32 = 'L';
+    public static final char CAPABILITY_BW64 = 'M';
+    public static final char CAPABILITY_BW128 = 'N';
+    public static final char CAPABILITY_BW256 = 'O';
+    
     public static final char CAPABILITY_REACHABLE = 'R';
     public static final char CAPABILITY_UNREACHABLE = 'U';
     public static final String PROP_FORCE_UNREACHABLE = "router.forceUnreachable";
 
     public static final char CAPABILITY_NEW_TUNNEL = 'T';
     
-    public void addReachabilityCapability(RouterInfo ri) {
+    public void addCapabilities(RouterInfo ri) {
-        // routers who can understand TunnelBuildMessages 
+        int bwLim = Math.min(_context.bandwidthLimiter().getInboundKBytesPerSecond(),
-        ////ri.addCapability(CAPABILITY_NEW_TUNNEL);
+                             _context.bandwidthLimiter().getInboundKBytesPerSecond());
+        if (_log.shouldLog(Log.WARN))
+            _log.warn("Adding capabilities w/ bw limit @ " + bwLim, new Exception("caps"));
+        
+        if (bwLim <= 16) {
+            ri.addCapability(CAPABILITY_BW16);
+        } else if (bwLim <= 32) {
+            ri.addCapability(CAPABILITY_BW32);
+        } else if (bwLim <= 64) {
+            ri.addCapability(CAPABILITY_BW64);
+        } else if (bwLim <= 128) {
+            ri.addCapability(CAPABILITY_BW128);
+        } else { // ok, more than 128KBps... aka "lots"
+            ri.addCapability(CAPABILITY_BW256);
+        }
+        
+        if (FloodfillNetworkDatabaseFacade.floodfillEnabled(_context))
+            ri.addCapability(FloodfillNetworkDatabaseFacade.CAPACITY_FLOODFILL);
+        
+        if("true".equalsIgnoreCase(_context.getProperty(Router.PROP_HIDDEN, "false")))
+            ri.addCapability(RouterInfo.CAPABILITY_HIDDEN);
         
         String forceUnreachable = _context.getProperty(PROP_FORCE_UNREACHABLE);
         if ( (forceUnreachable != null) && ("true".equalsIgnoreCase(forceUnreachable)) ) {

router/java/src/net/i2p/router/networkdb/PublishLocalRouterInfoJob.java

@@ -45,13 +45,8 @@ public class PublishLocalRouterInfoJob extends JobImpl {
             ri.setPublished(getContext().clock().now());
             ri.setOptions(stats);
             ri.setAddresses(getContext().commSystem().createAddresses());
-            if (FloodfillNetworkDatabaseFacade.floodfillEnabled(getContext()))
-                ri.addCapability(FloodfillNetworkDatabaseFacade.CAPACITY_FLOODFILL);
 
-            if ("true".equalsIgnoreCase(getContext().getProperty(Router.PROP_HIDDEN, "false")))
+            getContext().router().addCapabilities(ri);
-                ri.addCapability(RouterInfo.CAPABILITY_HIDDEN);
-
-            getContext().router().addReachabilityCapability(ri);
             SigningPrivateKey key = getContext().keyManager().getSigningPrivateKey();
             if (key == null) {
                 _log.log(Log.CRIT, "Internal error - signing private key not known?  rescheduling publish for 30s");

router/java/src/net/i2p/router/peermanager/ProfileOrganizer.java

@@ -682,6 +682,12 @@ public class ProfileOrganizer {
                           + "], but there aren't enough of them " + numExceedingMean);
             _thresholdCapacityValue = Math.max(thresholdAtMinHighCap, thresholdAtLowest);
         }
+        
+        // the base growth factor is the value we give to new routers that we don't
+        // know anything about.  dont go under that limit unless you want to expose
+        // the selection to simple ident flooding attacks
+        if (_thresholdCapacityValue <= CapacityCalculator.GROWTH_FACTOR)
+            _thresholdCapacityValue = CapacityCalculator.GROWTH_FACTOR + 0.0001;
     }
     
     /**

router/java/src/net/i2p/router/startup/CreateRouterInfoJob.java

@@ -54,12 +54,8 @@ public class CreateRouterInfoJob extends JobImpl {
             info.setAddresses(getContext().commSystem().createAddresses());
             Properties stats = getContext().statPublisher().publishStatistics();
             stats.setProperty(RouterInfo.PROP_NETWORK_ID, Router.NETWORK_ID+"");
-            getContext().router().addReachabilityCapability(info);
+            getContext().router().addCapabilities(info);
             info.setOptions(stats);
-            if (FloodfillNetworkDatabaseFacade.floodfillEnabled(getContext()))
-                info.addCapability(FloodfillNetworkDatabaseFacade.CAPACITY_FLOODFILL);
-            if ("true".equalsIgnoreCase(getContext().getProperty(Router.PROP_HIDDEN, "false")))
-                info.addCapability(RouterInfo.CAPABILITY_HIDDEN);
             info.setPeers(new HashSet());
             info.setPublished(getCurrentPublishDate(getContext()));
             RouterIdentity ident = new RouterIdentity();

router/java/src/net/i2p/router/startup/RebuildRouterInfoJob.java

@@ -128,14 +128,7 @@ public class RebuildRouterInfoJob extends JobImpl {
                 Properties stats = getContext().statPublisher().publishStatistics();
                 stats.setProperty(RouterInfo.PROP_NETWORK_ID, ""+Router.NETWORK_ID);
                 info.setOptions(stats);
-                if (FloodfillNetworkDatabaseFacade.floodfillEnabled(getContext()))
+                getContext().router().addCapabilities(info);
-                    info.addCapability(FloodfillNetworkDatabaseFacade.CAPACITY_FLOODFILL);
-
-                // Set caps=H for hidden mode routers
-                if ("true".equalsIgnoreCase(getContext().getProperty(Router.PROP_HIDDEN, "false"))) 
-                    info.addCapability(RouterInfo.CAPABILITY_HIDDEN);
-
-                getContext().router().addReachabilityCapability(info);
                 // info.setPeers(new HashSet()); // this would have the trusted peers
                 info.setPublished(CreateRouterInfoJob.getCurrentPublishDate(getContext()));
                 

router/java/src/net/i2p/router/transport/OutboundMessageRegistry.java

@@ -25,7 +25,7 @@ public class OutboundMessageRegistry {
     private Log _log;
     /** list of currently active MessageSelector instances */
     private List _selectors;
-    /** map of active MessageSelector to the OutNetMessage causing it (for quick removal) */
+    /** map of active MessageSelector to either an OutNetMessage or a List of OutNetMessages causing it (for quick removal) */
     private Map _selectorToMessage;
     /** set of active OutNetMessage (for quick removal and selector fetching) */
     private Set _activeMessages;
@@ -61,6 +61,8 @@ public class OutboundMessageRegistry {
         synchronized (_selectors) {
             for (int i = 0; i < _selectors.size(); i++) {
                 MessageSelector sel = (MessageSelector)_selectors.get(i);
+                if (sel == null)
+                    continue;
                 boolean isMatch = sel.isMatch(message);
                 if (isMatch) {
                     if (matchedSelectors == null) matchedSelectors = new ArrayList(1);
@@ -82,19 +84,36 @@ public class OutboundMessageRegistry {
                 MessageSelector sel = (MessageSelector)matchedSelectors.get(i);
                 boolean removed = false;
                 OutNetMessage msg = null;
+                List msgs = null;
                 synchronized (_selectorToMessage) {
+                    Object o = null;
                     if ( (removedSelectors != null) && (removedSelectors.contains(sel)) ) {
-                        msg = (OutNetMessage)_selectorToMessage.remove(sel);
+                        o = _selectorToMessage.remove(sel);
                         removed = true;
                     } else {
-                        msg = (OutNetMessage)_selectorToMessage.get(sel);
+                        o = _selectorToMessage.get(sel);
+                    }
+                    
+                    if (o instanceof OutNetMessage) {
+                        msg = (OutNetMessage)o;
+                        if (msg != null)
+                            rv.add(msg);
+                    } else if (o instanceof List) {
+                        msgs = (List)o;
+                        if (msgs != null)
+                            for (int j = 0; j < msgs.size(); j++)
+                                rv.add(msgs.get(j));
                     }
-                    if (msg != null)
-                        rv.add(msg);
                 }
-                if (removed && msg != null) {
+                if (removed) {
-                    synchronized (_activeMessages) {
+                    if (msg != null) {
-                        _activeMessages.remove(msg);
+                        synchronized (_activeMessages) {
+                            _activeMessages.remove(msg);
+                        }
+                    } else if (msgs != null) {
+                        synchronized (_activeMessages) {
+                            _activeMessages.removeAll(msgs);
+                        }
                     }
                 }
             }
@@ -128,7 +147,24 @@ public class OutboundMessageRegistry {
             if (!_activeMessages.add(msg))
                 return; // dont add dups
         }
-        synchronized (_selectorToMessage) { _selectorToMessage.put(sel, msg); }
+        synchronized (_selectorToMessage) { 
+            Object oldMsg = _selectorToMessage.put(sel, msg);
+            if (oldMsg != null) {
+                List multi = null;
+                if (oldMsg instanceof OutNetMessage) {
+                    multi = new ArrayList(4);
+                    multi.add(oldMsg);
+                    multi.add(msg);
+                    _selectorToMessage.put(sel, multi);
+                } else if (oldMsg instanceof List) {
+                    multi = (List)oldMsg;
+                    multi.add(msg);
+                    _selectorToMessage.put(sel, multi);
+                }
+                if (_log.shouldLog(Log.WARN))
+                    _log.warn("a single message selector [" + sel + "] with multiple messages ("+ multi + ")");
+            }
+        }
         synchronized (_selectors) { _selectors.add(sel); }
 
         _cleanupTask.scheduleExpiration(sel);
@@ -136,9 +172,22 @@ public class OutboundMessageRegistry {
     
     public void unregisterPending(OutNetMessage msg) {
         MessageSelector sel = msg.getReplySelector();
-        // remember, order matters
+        boolean stillActive = false;
-        synchronized (_selectors) { _selectors.add(sel); }
+        synchronized (_selectorToMessage) { 
-        synchronized (_selectorToMessage) { _selectorToMessage.put(sel, msg); }
+            Object old = _selectorToMessage.remove(sel);
+            if (old != null) {
+                if (old instanceof List) {
+                    List l = (List)old;
+                    l.remove(msg);
+                    if (l.size() > 0) {
+                        _selectorToMessage.put(sel, l);
+                        stillActive = true;
+                    }
+                }
+            }
+        }
+        if (!stillActive)
+            synchronized (_selectors) { _selectors.remove(sel); }
         synchronized (_activeMessages) { _activeMessages.remove(msg); }
     }
 
@@ -156,6 +205,7 @@ public class OutboundMessageRegistry {
             synchronized (_selectors) {
                 for (int i = 0; i < _selectors.size(); i++) {
                     MessageSelector sel = (MessageSelector)_selectors.get(i);
+                    if (sel == null) continue;
                     long expiration = sel.getExpiration();
                     if (expiration <= now) {
                         _removing.add(sel);
@@ -170,8 +220,13 @@ public class OutboundMessageRegistry {
                 for (int i = 0; i < _removing.size(); i++) {
                     MessageSelector sel = (MessageSelector)_removing.get(i);
                     OutNetMessage msg = null;
+                    List msgs = null;
                     synchronized (_selectorToMessage) {
-                        msg = (OutNetMessage)_selectorToMessage.remove(sel);
+                        Object o = _selectorToMessage.remove(sel);
+                        if (o instanceof OutNetMessage)
+                            msg = (OutNetMessage)o;
+                        else if (o instanceof List)
+                            msgs = (List)o;
                     }
                     if (msg != null) {
                         synchronized (_activeMessages) {
@@ -180,6 +235,16 @@ public class OutboundMessageRegistry {
                         Job fail = msg.getOnFailedReplyJob();
                         if (fail != null)
                             _context.jobQueue().addJob(fail);
+                    } else if (msgs != null) {
+                        synchronized (_activeMessages) {
+                            _activeMessages.removeAll(msgs);
+                        }
+                        for (int j = 0; j < msgs.size(); j++) {
+                            msg = (OutNetMessage)msgs.get(i);
+                            Job fail = msg.getOnFailedReplyJob();
+                            if (fail != null)
+                                _context.jobQueue().addJob(fail);
+                        }
                     }
                 }
                 _removing.clear();

router/java/src/net/i2p/router/transport/udp/EstablishmentManager.java

@@ -552,7 +552,11 @@ public class EstablishmentManager {
             // offer to relay
             // (perhaps we should check our bw usage and/or how many peers we are 
             //  already offering introducing?)
-            state.setSentRelayTag(_context.random().nextLong(MAX_TAG_VALUE));
+            if (state.getSentRelayTag() < 0) {
+                state.setSentRelayTag(_context.random().nextLong(MAX_TAG_VALUE));
+            } else {
+                // don't change it, since we've already prepared our sig
+            }
         } else {
             // don't offer to relay
             state.setSentRelayTag(0);

router/java/src/net/i2p/router/transport/udp/PacketBuilder.java

@@ -567,7 +567,7 @@ public class PacketBuilder {
             if ( (off % 16) != 0)
                 off += 16 - (off % 16);
             packet.getPacket().setLength(off);
-            authenticate(packet, state.getIntroKey(), state.getIntroKey());
+            authenticate(packet, state.getCipherKey(), state.getMACKey());
         } 
         
         setTo(packet, to, state.getSentPort());

router/java/src/net/i2p/router/transport/udp/UDPTransport.java

@@ -337,6 +337,7 @@ public class UDPTransport extends TransportImpl implements TimedWeightedPriority
                 if ( (_externalListenHost == null) ||
                      (!eq(_externalListenHost.getAddress(), _externalListenPort, ourIP, ourPort)) ) {
                     if ( (_reachabilityStatus == CommSystemFacade.STATUS_UNKNOWN) ||
+                         (_externalListenHost == null) || (_externalListenPort <= 0) ||
                          (_context.clock().now() - _reachabilityStatusLastUpdated > 2*TEST_FREQUENCY) ) {
                         // they told us something different and our tests are either old or failing
                         if (_log.shouldLog(Log.INFO))
@@ -358,7 +359,7 @@ public class UDPTransport extends TransportImpl implements TimedWeightedPriority
                         // so lets test again
                         fireTest = true;
                         if (_log.shouldLog(Log.INFO))
-                            _log.info("Different address, but we're fine..");
+                            _log.info("Different address, but we're fine.. (" + _reachabilityStatus + ")");
                     }
                 } else {
                     // matched what we expect

router/java/src/net/i2p/router/tunnel/pool/ExploratoryPeerSelector.java

@@ -3,6 +3,8 @@ package net.i2p.router.tunnel.pool;
 import java.util.*;
 import net.i2p.router.RouterContext;
 import net.i2p.router.TunnelPoolSettings;
+import net.i2p.stat.Rate;
+import net.i2p.stat.RateStat;
 import net.i2p.util.Log;
 
 /**
@@ -30,7 +32,7 @@ class ExploratoryPeerSelector extends TunnelPeerSelector {
         Set exclude = getExclude(ctx, settings.isInbound(), settings.isExploratory());
         exclude.add(ctx.routerHash());
         HashSet matches = new HashSet(length);
-        boolean exploreHighCap = Boolean.valueOf(ctx.getProperty("router.exploreHighCapacity", "false")).booleanValue();
+        boolean exploreHighCap = shouldPickHighCap(ctx);
         if (exploreHighCap) 
             ctx.profileOrganizer().selectHighCapacityPeers(length, exclude, matches);
         else
@@ -48,4 +50,38 @@ class ExploratoryPeerSelector extends TunnelPeerSelector {
             rv.add(ctx.routerHash());
         return rv;
     }
+    
+    private boolean shouldPickHighCap(RouterContext ctx) {
+        if (Boolean.valueOf(ctx.getProperty("router.exploreHighCapacity", "false")).booleanValue())
+            return true;
+        // no need to explore too wildly at first
+        if (ctx.router().getUptime() <= 10*1000)
+            return true;
+        // ok, if we aren't explicitly asking for it, we should try to pick peers
+        // randomly from the 'not failing' pool.  However, if we are having a
+        // hard time building exploratory tunnels, lets fall back again on the
+        // high capacity peers, at least for a little bit.
+        int failPct = getExploratoryFailPercentage(ctx);
+        return (failPct >= ctx.random().nextInt(100));
+    }
+    
+    private int getExploratoryFailPercentage(RouterContext ctx) {
+        int timeout = getEvents(ctx, "tunnel.buildExploratoryExpire", 10*60*1000);
+        int reject = getEvents(ctx, "tunnel.buildExploratoryReject", 10*60*1000);
+        int accept = getEvents(ctx, "tunnel.buildExploratorySuccess", 10*60*1000);
+        if (accept + reject + timeout <= 0)
+            return 0;
+        double pct = (double)(reject + timeout) / (accept + reject + timeout);
+        return (int)(100 * pct);
+    }
+    
+    private int getEvents(RouterContext ctx, String stat, long period) {
+        RateStat rs = ctx.statManager().getRate(stat);
+        if (rs == null) 
+            return 0;
+        Rate r = rs.getRate(period);
+        if (r == null)
+            return 0;
+        return (int)r.getLastEventCount();
+    }
 }

router/java/src/net/i2p/router/tunnel/pool/TunnelPool.java

@@ -601,6 +601,7 @@ public class TunnelPool {
             peers.add(_context.routerHash());
         }
         PooledTunnelCreatorConfig cfg = new PooledTunnelCreatorConfig(_context, peers.size(), settings.isInbound(), settings.getDestination());
+        cfg.setTunnelPool(this);
         // peers[] is ordered endpoint first, but cfg.getPeer() is ordered gateway first
         for (int i = 0; i < peers.size(); i++) {
             int j = peers.size() - 1 - i;