Router: More peer selection fixes for hidden mode

Fix excludes after calling SANFP
Pick connected OBEP when hidden and paired tunnel is zero-hop

history.txt

@@ -1,5 +1,17 @@
+2018-03-11 zzz
+ * Router: More peer selection fixes for hidden mode
+
+2018-03-10 zzz
+ * Console: Fix compression logic
+ * i2ptunnel: Strip server Date header
+ * NetDB: Wake up FloodfillMonitor when setting changes
+ * Router: Fix rekey after hidden config change
+
 2018-03-09 zzz
- * Console: Enable compression (ticket #2157)
+ * Console:
+   - Enable compression (ticket #2157)
+   - Close output stream on redirect
+   - Add Accept-Ranges header
 
 2018-03-08 zzz
  * Crypto: Generate non-CA cert for family

router/java/src/net/i2p/router/RouterVersion.java

@@ -18,7 +18,7 @@ public class RouterVersion {
     /** deprecated */
     public final static String ID = "Monotone";
     public final static String VERSION = CoreVersion.VERSION;
-    public final static long BUILD = 14;
+    public final static long BUILD = 15;
 
     /** for example "-test" */
     public final static String EXTRA = "";

router/java/src/net/i2p/router/TunnelManagerFacade.java

@@ -182,6 +182,18 @@ public interface TunnelManagerFacade extends Service {
     /** for TunnelRenderer in router console */
     public TunnelPool getOutboundExploratoryPool();
 
+    /**
+     *  @return pool or null
+     *  @since 0.9.34
+     */
+    public TunnelPool getInboundPool(Hash client);
+
+    /**
+     *  @return pool or null
+     *  @since 0.9.34
+     */
+    public TunnelPool getOutboundPool(Hash client);
+
     /** @since 0.8.13 */
     public void fail(Hash peer);
 }

router/java/src/net/i2p/router/dummy/DummyTunnelManagerFacade.java

@@ -75,4 +75,12 @@ public class DummyTunnelManagerFacade implements TunnelManagerFacade {
     public TunnelPool getInboundExploratoryPool() { return null; }
     public TunnelPool getOutboundExploratoryPool() { return null; }
     public void fail(Hash peer) {}
+
+    public TunnelPool getInboundPool(Hash client) {
+        return null;
+    }
+
+    public TunnelPool getOutboundPool(Hash client) {
+        return null;
+    }
 }

router/java/src/net/i2p/router/peermanager/ProfileOrganizer.java

@@ -571,7 +571,7 @@ public class ProfileOrganizer {
      * Caution, this does NOT cascade further to non-connected peers, so it should only
      * be used when there is a good number of connected peers.
      *
-     * @param exclude non-null
+     * @param exclude non-null, WARNING - side effect, all not-connected peers are added
      * No mask parameter, to be fixed
      */
     public void selectActiveNotFailingPeers(int howMany, Set<Hash> exclude, Set<Hash> matches) {

router/java/src/net/i2p/router/tunnel/pool/ClientPeerSelector.java

@@ -8,6 +8,8 @@ import java.util.Set;
 import net.i2p.data.DataHelper;
 import net.i2p.data.Hash;
 import net.i2p.router.RouterContext;
+import net.i2p.router.TunnelInfo;
+import net.i2p.router.TunnelManagerFacade;
 import net.i2p.router.TunnelPoolSettings;
 import static net.i2p.router.peermanager.ProfileOrganizer.Slice.*;
 
@@ -52,6 +54,7 @@ class ClientPeerSelector extends TunnelPeerSelector {
             boolean hidden = ctx.router().isHidden() ||
                              ctx.router().getRouterInfo().getAddressCount() <= 0;
             boolean hiddenInbound = hidden && isInbound;
+            boolean hiddenOutbound = hidden && !isInbound;
 
             if (shouldSelectExplicit(settings))
                 return selectExplicit(settings, length);
@@ -65,8 +68,11 @@ class ClientPeerSelector extends TunnelPeerSelector {
                     if (moreExclude != null)
                         exclude.addAll(moreExclude);
                 }
-                if (hiddenInbound)
+                if (hiddenInbound) {
-                    ctx.profileOrganizer().selectActiveNotFailingPeers(1, exclude, matches);
+                    // SANFP adds all not-connected to exclude, so make a copy
+                    Set<Hash> SANFPExclude = new HashSet<Hash>(exclude);
+                    ctx.profileOrganizer().selectActiveNotFailingPeers(1, SANFPExclude, matches);
+                }
                 if (matches.isEmpty()) {
                     // ANFP does not fall back to non-connected
                     ctx.profileOrganizer().selectFastPeers(length, exclude, matches, 0);
@@ -78,9 +84,10 @@ class ClientPeerSelector extends TunnelPeerSelector {
                 // For a 2-hop tunnel, the first hop comes from subtiers 0-1 and the last from subtiers 2-3.
                 // For a longer tunnels, the first hop comes from subtier 0, the middle from subtiers 2-3, and the last from subtier 1.
                 rv = new ArrayList<Hash>(length + 1);
+                Hash randomKey = settings.getRandomKey();
                 // OBEP or IB last hop
                 // group 0 or 1 if two hops, otherwise group 0
-                Set<Hash> firstHopExclude;
+                Set<Hash> lastHopExclude;
                 if (isInbound) {
                     // exclude existing OBEPs to get some diversity ?
                     // closest-hop restrictions
@@ -88,26 +95,101 @@ class ClientPeerSelector extends TunnelPeerSelector {
                         Set<Hash> moreExclude = getClosestHopExclude(false);
                         if (moreExclude != null) {
                             moreExclude.addAll(exclude);
-                            firstHopExclude = moreExclude;
+                            lastHopExclude = moreExclude;
                         } else {
-                            firstHopExclude = exclude;
+                            lastHopExclude = exclude;
                         }
                     } else {
-                         firstHopExclude = exclude;
+                         lastHopExclude = exclude;
                     }
                 } else {
-                    firstHopExclude = exclude;
+                    lastHopExclude = exclude;
                 }
                 if (hiddenInbound) {
-                    ctx.profileOrganizer().selectActiveNotFailingPeers(1, exclude, matches);
+                    // IB closest hop 
+                    if (log.shouldInfo())
+                        log.info("CPS SANFP closest IB exclude " + lastHopExclude.size());
+                    // SANFP adds all not-connected to exclude, so make a copy
+                    Set<Hash> SANFPExclude = new HashSet<Hash>(lastHopExclude);
+                    ctx.profileOrganizer().selectActiveNotFailingPeers(1, SANFPExclude, matches);
                     if (matches.isEmpty()) {
+                        if (log.shouldInfo())
+                            log.info("CPS SFP closest IB exclude " + lastHopExclude.size());
                         // ANFP does not fall back to non-connected
-                        ctx.profileOrganizer().selectFastPeers(1, firstHopExclude, matches, settings.getRandomKey(), length == 2 ? SLICE_0_1 : SLICE_0);
+                        ctx.profileOrganizer().selectFastPeers(1, lastHopExclude, matches, randomKey, length == 2 ? SLICE_0_1 : SLICE_0);
+                    }
+                } else if (hiddenOutbound) {
+                    // OBEP
+                    // check for hidden and outbound, and the paired (inbound) tunnel is zero-hop
+                    // if so, we need the OBEP to be connected to us, so we get the build reply back
+                    // This should be rare except at startup
+                    TunnelManagerFacade tmf = ctx.tunnelManager();
+                    TunnelPool tp = tmf.getInboundPool(settings.getDestination());
+                    boolean pickFurthest;
+                    if (tp != null) {
+                        pickFurthest = true;
+                        TunnelPoolSettings tps = tp.getSettings();
+                        int len = tps.getLength();
+                        if (len <= 0 ||
+                            tps.getLengthOverride() == 0 ||
+                            len + tps.getLengthVariance() <= 0) {
+                            // leave it true
+                        } else {
+                            List<TunnelInfo> tunnels = tp.listTunnels();
+                            if (!tunnels.isEmpty()) {
+                                for (TunnelInfo ti : tp.listTunnels()) {
+                                    if (ti.getLength() > 1) {
+                                        pickFurthest = false;
+                                        break;
+                                    }
+                                }
+                            } else {
+                                // no tunnels in the paired tunnel pool
+                                // BuildRequester will be using exploratory
+                                tp = tmf.getInboundExploratoryPool();
+                                tps = tp.getSettings();
+                                len = tps.getLength();
+                                if (len <= 0 ||
+                                    tps.getLengthOverride() == 0 ||
+                                    len + tps.getLengthVariance() <= 0) {
+                                    // leave it true
+                                } else {
+                                    tunnels = tp.listTunnels();
+                                    if (!tunnels.isEmpty()) {
+                                        for (TunnelInfo ti : tp.listTunnels()) {
+                                            if (ti.getLength() > 1) {
+                                                pickFurthest = false;
+                                                break;
+                                            }
+                                        }
+                                    }
+                                }
+                            }
+                        }
+                    } else {
+                        // shouldn't happen
+                        pickFurthest = false;
+                    }
+                    if (pickFurthest) {
+                        if (log.shouldInfo())
+                            log.info("CPS SANFP OBEP exclude " + lastHopExclude.size());
+                        // SANFP adds all not-connected to exclude, so make a copy
+                        Set<Hash> SANFPExclude = new HashSet<Hash>(lastHopExclude);
+                        ctx.profileOrganizer().selectActiveNotFailingPeers(1, SANFPExclude, matches);
+                        if (matches.isEmpty()) {
+                            // ANFP does not fall back to non-connected
+                            if (log.shouldInfo())
+                                log.info("CPS SFP OBEP exclude " + lastHopExclude.size());
+                            ctx.profileOrganizer().selectFastPeers(1, lastHopExclude, matches, randomKey, length == 2 ? SLICE_0_1 : SLICE_0);
+                        }
+                    } else {
+                        ctx.profileOrganizer().selectFastPeers(1, lastHopExclude, matches, randomKey, length == 2 ? SLICE_0_1 : SLICE_0);
                     }
                 } else {
                     // TODO exclude IPv6-only at OBEP? Caught in checkTunnel() below
-                    ctx.profileOrganizer().selectFastPeers(1, firstHopExclude, matches, settings.getRandomKey(), length == 2 ? SLICE_0_1 : SLICE_0);
+                    ctx.profileOrganizer().selectFastPeers(1, lastHopExclude, matches, randomKey, length == 2 ? SLICE_0_1 : SLICE_0);
                 }
+
                 matches.remove(ctx.routerHash());
                 exclude.addAll(matches);
                 rv.addAll(matches);
@@ -115,12 +197,12 @@ class ClientPeerSelector extends TunnelPeerSelector {
                 if (length > 2) {
                     // middle hop(s)
                     // group 2 or 3
-                    ctx.profileOrganizer().selectFastPeers(length - 2, exclude, matches, settings.getRandomKey(), SLICE_2_3);
+                    ctx.profileOrganizer().selectFastPeers(length - 2, exclude, matches, randomKey, SLICE_2_3);
                     matches.remove(ctx.routerHash());
                     if (matches.size() > 1) {
                         // order the middle peers for tunnels >= 4 hops
                         List<Hash> ordered = new ArrayList<Hash>(matches);
-                        orderPeers(ordered, settings.getRandomKey());
+                        orderPeers(ordered, randomKey);
                         rv.addAll(ordered);
                     } else {
                         rv.addAll(matches);
@@ -128,6 +210,7 @@ class ClientPeerSelector extends TunnelPeerSelector {
                     exclude.addAll(matches);
                     matches.clear();
                 }
+
                 // IBGW or OB first hop
                 // group 2 or 3 if two hops, otherwise group 1
                 if (!isInbound) {
@@ -140,7 +223,7 @@ class ClientPeerSelector extends TunnelPeerSelector {
                     }
                 }
                 // TODO exclude IPv6-only at IBGW? Caught in checkTunnel() below
-                ctx.profileOrganizer().selectFastPeers(1, exclude, matches, settings.getRandomKey(), length == 2 ? SLICE_2_3 : SLICE_1);
+                ctx.profileOrganizer().selectFastPeers(1, exclude, matches, randomKey, length == 2 ? SLICE_2_3 : SLICE_1);
                 matches.remove(ctx.routerHash());
                 rv.addAll(matches);
             }

router/java/src/net/i2p/router/tunnel/pool/ExploratoryPeerSelector.java

@@ -8,6 +8,8 @@ import java.util.Set;
 import net.i2p.data.DataHelper;
 import net.i2p.data.Hash;
 import net.i2p.router.RouterContext;
+import net.i2p.router.TunnelInfo;
+import net.i2p.router.TunnelManagerFacade;
 import net.i2p.router.TunnelPoolSettings;
 import net.i2p.stat.Rate;
 import net.i2p.stat.RateStat;
@@ -65,6 +67,7 @@ class ExploratoryPeerSelector extends TunnelPeerSelector {
         boolean hidden = nonzero && (ctx.router().isHidden() ||
                                      ctx.router().getRouterInfo().getAddressCount() <= 0);
         boolean hiddenInbound = hidden && isInbound;
+        boolean hiddenOutbound = hidden && !isInbound;
         boolean lowOutbound = nonzero && !isInbound && !ctx.commSystem().haveHighOutboundCapacity();
 
 
@@ -93,7 +96,9 @@ class ExploratoryPeerSelector extends TunnelPeerSelector {
                 // use only connected peers so we don't make more connections
                 if (log.shouldLog(Log.INFO))
                     log.info("EPS SANFP closest " + (isInbound ? "IB" : "OB") + " exclude " + closestExclude.size());
-                ctx.profileOrganizer().selectActiveNotFailingPeers(1, closestExclude, closest);
+                // SANFP adds all not-connected to exclude, so make a copy
+                Set<Hash> SANFPExclude = new HashSet<Hash>(closestExclude);
+                ctx.profileOrganizer().selectActiveNotFailingPeers(1, SANFPExclude, closest);
                 if (closest.isEmpty()) {
                     // ANFP does not fall back to non-connected
                     if (log.shouldLog(Log.INFO))
@@ -116,6 +121,55 @@ class ExploratoryPeerSelector extends TunnelPeerSelector {
             }
         }
 
+        // furthest-hop restrictions
+        // Since we're applying orderPeers() later, we don't know
+        // which will be the furthest hop, so select the furthest one here if necessary.
+
+        Hash furthestHop = null;
+        if (hiddenOutbound && length > 0) {
+            // OBEP
+            // check for hidden and outbound, and the paired (inbound) tunnel is zero-hop
+            // if so, we need the OBEP to be connected to us, so we get the build reply back
+            // This should be rare except at startup
+            TunnelManagerFacade tmf = ctx.tunnelManager();
+            TunnelPool tp = tmf.getInboundExploratoryPool();
+            TunnelPoolSettings tps = tp.getSettings();
+            int len = tps.getLength();
+            boolean pickFurthest = true;
+            if (len <= 0 ||
+                tps.getLengthOverride() == 0 ||
+                len + tps.getLengthVariance() <= 0) {
+                // leave it true
+            } else {
+                for (TunnelInfo ti : tp.listTunnels()) {
+                    if (ti.getLength() > 1) {
+                        pickFurthest = false;
+                        break;
+                    }
+                }
+            }
+            if (pickFurthest) {
+                Set<Hash> furthest = new HashSet<Hash>(1);
+                if (log.shouldLog(Log.INFO))
+                    log.info("EPS SANFP furthest OB exclude " + exclude.size());
+                // ANFP adds all not-connected to exclude, so make a copy
+                Set<Hash> SANFPExclude = new HashSet<Hash>(exclude);
+                ctx.profileOrganizer().selectActiveNotFailingPeers(1, SANFPExclude, furthest);
+                if (furthest.isEmpty()) {
+                    // ANFP does not fall back to non-connected
+                    if (log.shouldLog(Log.INFO))
+                        log.info("EPS SFP furthest OB exclude " + exclude.size());
+                    ctx.profileOrganizer().selectFastPeers(1, exclude, furthest);
+                }
+                if (!furthest.isEmpty()) {
+                    furthestHop = furthest.iterator().next();
+                    exclude.add(furthestHop);
+                    length--;
+                }
+            }
+        }
+
+
         // Don't use ff peers for exploratory tunnels to lessen exposure to netDb searches and stores
         // Hmm if they don't get explored they don't get a speed/capacity rating
         // so they don't get used for client tunnels either.
@@ -154,6 +208,14 @@ class ExploratoryPeerSelector extends TunnelPeerSelector {
                 rv.add(closestHop);
             length++;
         }
+        if (furthestHop != null) {
+            // always OBEP for now, nothing special for IBGW
+            if (isInbound)
+                rv.add(furthestHop);
+            else
+                rv.add(0, furthestHop);
+            length++;
+        }
         //if (length != rv.size() && log.shouldWarn())
         //    log.warn("EPS requested " + length + " got " + rv.size() + ": " + DataHelper.toString(rv));
         //else if (log.shouldDebug())

router/java/src/net/i2p/router/tunnel/pool/TunnelPoolManager.java

@@ -729,6 +729,22 @@ public class TunnelPoolManager implements TunnelManagerFacade {
         return _outboundExploratory;
     }
 
+    /**
+     *  @return pool or null
+     *  @since 0.9.34
+     */
+    public TunnelPool getInboundPool(Hash client) {
+        return _clientInboundPools.get(client);
+    }
+
+    /**
+     *  @return pool or null
+     *  @since 0.9.34
+     */
+    public TunnelPool getOutboundPool(Hash client) {
+        return _clientOutboundPools.get(client);
+    }
+
     /**
      *  Fail all outbound tunnels with this peer as first hop,
      *  and all inbound tunnels with this peer as the last hop,