NetDb: Search for new leaseset before expiration

Reduce expiration for router infos with introducers
More negative cache checks
Log tweaks
SSU: Switch introducers less often

history.txt

@@ -1,3 +1,12 @@
+2016-03-01 zzz
+ * i2psnark: Fix handling of HAVE messages received before metainfo
+ * i2ptunnel: Don't default to a private key file that exists (ticket #1628)
+ * NetDb:
+   - Search for new leaseset before expiration;
+   - Reduce expiration for router infos with introducers
+   - Add missing reseed cert
+ * SSU: Switch introducers less often
+
 2016-02-26 zzz
  * Console:
    - Add X-Content-Type-Options header everywhere (ticket #1763)
@@ -8,6 +17,8 @@
    - Add QR code generation
  * Router: Log full path to wrapper.log when dumping threads
  * Transports: Increase connection limits for class N and higher
+ * Utils: Add main classes to i2p.jar and router.jar
+   for simple command line access to utilities
 
 2016-02-22 zzz
  * Console: Improve news CSS (ticket #1710)

router/java/src/net/i2p/router/NetworkDatabaseFacade.java

@@ -52,6 +52,16 @@ public abstract class NetworkDatabaseFacade implements Service {
     public abstract LeaseSet lookupLeaseSetLocally(Hash key);
     public abstract void lookupRouterInfo(Hash key, Job onFindJob, Job onFailedLookupJob, long timeoutMs);
     public abstract RouterInfo lookupRouterInfoLocally(Hash key);
+    
+    /**
+     *  Unconditionally lookup using the client's tunnels.
+     *  No success or failed jobs, no local lookup, no checks.
+     *  Use this to refresh a leaseset before expiration.
+     *
+     *  @param fromLocalDest use these tunnels for the lookup, or null for exploratory
+     *  @since 0.9.25
+     */
+    public abstract void lookupLeaseSetRemotely(Hash key, Hash fromLocalDest);
 
     /**
      *  Lookup using the client's tunnels

router/java/src/net/i2p/router/RouterVersion.java

@@ -18,10 +18,10 @@ public class RouterVersion {
     /** deprecated */
     public final static String ID = "Monotone";
     public final static String VERSION = CoreVersion.VERSION;
-    public final static long BUILD = 9;
+    public final static long BUILD = 10;
 
     /** for example "-test" */
-    public final static String EXTRA = "";
+    public final static String EXTRA = "-rc";
     public final static String FULL_VERSION = VERSION + "-" + BUILD + EXTRA;
     public static void main(String args[]) {
         System.out.println("I2P Router version: " + FULL_VERSION);

router/java/src/net/i2p/router/dummy/DummyNetworkDatabaseFacade.java

@@ -43,6 +43,7 @@ public class DummyNetworkDatabaseFacade extends NetworkDatabaseFacade {
     public void lookupLeaseSet(Hash key, Job onFindJob, Job onFailedLookupJob, long timeoutMs) {}
     public void lookupLeaseSet(Hash key, Job onFindJob, Job onFailedLookupJob, long timeoutMs, Hash fromLocalDest) {}
     public LeaseSet lookupLeaseSetLocally(Hash key) { return null; }
+    public void lookupLeaseSetRemotely(Hash key, Hash fromLocalDest) {}
 
     public void lookupDestination(Hash key, Job onFinishedJob, long timeoutMs, Hash fromLocalDest) {}
 

router/java/src/net/i2p/router/message/OutboundClientMessageOneShotJob.java

@@ -281,6 +281,19 @@ public class OutboundClientMessageOneShotJob extends JobImpl {
             //getContext().statManager().addRateData("client.leaseSetFoundLocally", 1);
             if (_log.shouldLog(Log.DEBUG))
                 _log.debug(getJobId() + ": Send outbound client message - leaseSet found locally for " + _toString);
+
+            if (!_leaseSet.isCurrent(Router.CLOCK_FUDGE_FACTOR / 4)) {
+                // If it's about to expire, refetch in the background, we'll
+                // probably need it again. This will prevent stalls later.
+                // We don't know if the other end is actually publishing his LS, so this could be a waste of time.
+                // When we move to LS2, we will have a bit that tells us if it is published.
+                if (_log.shouldWarn()) {
+                    long exp = now - _leaseSet.getLatestLeaseDate();
+                    _log.warn(getJobId() + ": leaseSet expired " + DataHelper.formatDuration(exp) + " ago, firing search: " + _leaseSet.getHash().toBase32());
+                }
+                getContext().netDb().lookupLeaseSetRemotely(_leaseSet.getHash(), _from.calculateHash());
+            }
+
             success.runJob();
         } else {
             _leaseSetLookupBegin = getContext().clock().now();

router/java/src/net/i2p/router/networkdb/kademlia/FloodfillNetworkDatabaseFacade.java

@@ -343,7 +343,7 @@ public class FloodfillNetworkDatabaseFacade extends KademliaNetworkDatabaseFacad
         //if (true) return super.search(key, onFindJob, onFailedLookupJob, timeoutMs, isLease);
         if (key == null) throw new IllegalArgumentException("searchin for nothin, eh?");
         boolean isNew = false;
-        FloodSearchJob searchJob = null;
+        FloodSearchJob searchJob;
         synchronized (_activeFloodQueries) {
             searchJob = _activeFloodQueries.get(key);
             if (searchJob == null) {

router/java/src/net/i2p/router/networkdb/kademlia/KademliaNetworkDatabaseFacade.java

@@ -133,6 +133,7 @@ public class KademliaNetworkDatabaseFacade extends NetworkDatabaseFacade {
     private final static long ROUTER_INFO_EXPIRATION_MIN = 90*60*1000l;
     private final static long ROUTER_INFO_EXPIRATION_SHORT = 75*60*1000l;
     private final static long ROUTER_INFO_EXPIRATION_FLOODFILL = 60*60*1000l;
+    private final static long ROUTER_INFO_EXPIRATION_INTRODUCED = 45*60*1000l;
     
     private final static long EXPLORE_JOB_DELAY = 10*60*1000l;
 
@@ -493,7 +494,7 @@ public class KademliaNetworkDatabaseFacade extends NetworkDatabaseFacade {
 
     /**
      *  Lookup using exploratory tunnels.
-     *  Use lookupDestination() if you don't need the LS or need it validated.
+     *  Use lookupDestination() if you don't need the LS or don't need it validated.
      */
     public void lookupLeaseSet(Hash key, Job onFindJob, Job onFailedLookupJob, long timeoutMs) {
         lookupLeaseSet(key, onFindJob, onFailedLookupJob, timeoutMs, null);
@@ -501,7 +502,7 @@ public class KademliaNetworkDatabaseFacade extends NetworkDatabaseFacade {
 
     /**
      *  Lookup using the client's tunnels
-     *  Use lookupDestination() if you don't need the LS or need it validated.
+     *  Use lookupDestination() if you don't need the LS or don't need it validated.
      *
      *  @param fromLocalDest use these tunnels for the lookup, or null for exploratory
      *  @since 0.9.10
@@ -511,26 +512,39 @@ public class KademliaNetworkDatabaseFacade extends NetworkDatabaseFacade {
         if (!_initialized) return;
         LeaseSet ls = lookupLeaseSetLocally(key);
         if (ls != null) {
-            if (_log.shouldLog(Log.DEBUG))
+            //if (_log.shouldLog(Log.DEBUG))
-                _log.debug("leaseSet found locally, firing " + onFindJob);
+            //    _log.debug("leaseSet found locally, firing " + onFindJob);
             if (onFindJob != null)
                 _context.jobQueue().addJob(onFindJob);
         } else if (isNegativeCached(key)) {
             if (_log.shouldLog(Log.WARN))
-                _log.warn("Negative cached, not searching: " + key);
+                _log.warn("Negative cached, not searching LS: " + key);
             if (onFailedLookupJob != null)
                 _context.jobQueue().addJob(onFailedLookupJob);
         } else {
-            if (_log.shouldLog(Log.DEBUG))
+            //if (_log.shouldLog(Log.DEBUG))
-                _log.debug("leaseSet not found locally, running search");
+            //    _log.debug("leaseSet not found locally, running search");
             search(key, onFindJob, onFailedLookupJob, timeoutMs, true, fromLocalDest);
         }
-        if (_log.shouldLog(Log.DEBUG))
+        //if (_log.shouldLog(Log.DEBUG))
-            _log.debug("after lookupLeaseSet");
+        //    _log.debug("after lookupLeaseSet");
     }
     
     /**
-     *  Use lookupDestination() if you don't need the LS or need it validated.
+     *  Unconditionally lookup using the client's tunnels.
+     *  No success or failed jobs, no local lookup, no checks.
+     *  Use this to refresh a leaseset before expiration.
+     *
+     *  @param fromLocalDest use these tunnels for the lookup, or null for exploratory
+     *  @since 0.9.25
+     */
+    public void lookupLeaseSetRemotely(Hash key, Hash fromLocalDest) {
+        if (!_initialized) return;
+        search(key, null, null, 20*1000, true, fromLocalDest);
+    }
+
+    /**
+     *  Use lookupDestination() if you don't need the LS or don't need it validated.
      */
     public LeaseSet lookupLeaseSetLocally(Hash key) {
         if (!_initialized) return null;
@@ -571,6 +585,9 @@ public class KademliaNetworkDatabaseFacade extends NetworkDatabaseFacade {
         Destination d = lookupDestinationLocally(key);
         if (d != null) {
             _context.jobQueue().addJob(onFinishedJob);
+        } else if (isNegativeCached(key)) {
+            if (_log.shouldLog(Log.WARN))
+                _log.warn("Negative cached, not searching dest: " + key);
         } else {
             search(key, onFinishedJob, onFinishedJob, timeoutMs, true, fromLocalDest);
         }
@@ -605,6 +622,9 @@ public class KademliaNetworkDatabaseFacade extends NetworkDatabaseFacade {
         } else if (_context.banlist().isBanlistedForever(key)) {
             if (onFailedLookupJob != null)
                 _context.jobQueue().addJob(onFailedLookupJob);
+        } else if (isNegativeCached(key)) {
+            if (_log.shouldLog(Log.WARN))
+                _log.warn("Negative cached, not searching RI: " + key);
         } else {
             search(key, onFindJob, onFailedLookupJob, timeoutMs, false);
         }
@@ -957,25 +977,28 @@ public class KademliaNetworkDatabaseFacade extends NetworkDatabaseFacade {
                           + new Date(routerInfo.getPublished()) + "]", new Exception());
             return "Peer published " + DataHelper.formatDuration(age) + " in the future?!";
         }
+        if (!routerInfo.isCurrent(ROUTER_INFO_EXPIRATION_INTRODUCED)) {
+            if (routerInfo.getAddresses().isEmpty())
+                return "Old peer with no addresses";
+            // This should cover the introducers case below too
+            // And even better, catches the case where the router is unreachable but knows no introducers
+            if (routerInfo.getCapabilities().indexOf(Router.CAPABILITY_UNREACHABLE) >= 0)
+                return "Old peer and thinks it is unreachable";
+            // FIXME check all SSU addresses, not just first
+            RouterAddress ra = routerInfo.getTargetAddress("SSU");
+            if (ra != null) {
+                // Introducers change often, introducee will ping introducer for 2 hours
+                if (ra.getOption("ihost0") != null)
+                    return "Old peer with SSU Introducers";
+            }
+        }
         if (upLongEnough && (routerInfo.getPublished() < now - 2*24*60*60*1000l) ) {
             long age = _context.clock().now() - routerInfo.getPublished();
             return "Peer published " + DataHelper.formatDuration(age) + " ago";
         }
         if (upLongEnough && !routerInfo.isCurrent(ROUTER_INFO_EXPIRATION_SHORT)) {
-            if (routerInfo.getAddresses().isEmpty())
+            if (routerInfo.getTargetAddress("NTCP") == null)
-                return "Peer published > 75m ago with no addresses";
+                return "Peer published > 75m ago, SSU only without introducers";
-            // This should cover the introducers case below too
-            // And even better, catches the case where the router is unreachable but knows no introducers
-            if (routerInfo.getCapabilities().indexOf(Router.CAPABILITY_UNREACHABLE) >= 0)
-                return "Peer published > 75m ago and thinks it is unreachable";
-            RouterAddress ra = routerInfo.getTargetAddress("SSU");
-            if (ra != null) {
-                // Introducers change often, introducee will ping introducer for 2 hours
-                if (ra.getOption("ihost0") != null)
-                    return "Peer published > 75m ago with SSU Introducers";
-                if (routerInfo.getTargetAddress("NTCP") == null)
-                    return "Peer published > 75m ago, SSU only without introducers";
-            }
         }
         return null;
     }

router/java/src/net/i2p/router/transport/udp/UDPTransport.java

@@ -1500,7 +1500,7 @@ public class UDPTransport extends TransportImpl implements TimedWeightedPriority
             long sinceSelected = _context.clock().now() - _introducersSelectedOn;
             if (valid >= PUBLIC_RELAY_COUNT) {
                 // try to shift 'em around every 10 minutes or so
-                if (sinceSelected > 10*60*1000) {
+                if (sinceSelected > 17*60*1000) {
                     if (_log.shouldLog(Log.WARN))
                         _log.warn("Our introducers are valid, but haven't changed in " + DataHelper.formatDuration(sinceSelected) + ", so lets rechoose");
                     return true;