DNS (ticket #1998):

- Data: Cache hostname lookups in RouterAddress
- Job Queue: Make search jobs droppable
- Router: Increase JVM DNS cache time
- Util: Add negative DNS lookup cache, increase cache size

core/java/src/net/i2p/util/Addresses.java

@@ -293,6 +293,8 @@ public abstract class Addresses {
      *  @since 0.9.3
      */
     private static final Map<String, byte[]> _IPAddress;
+    private static final Map<String, Long> _negativeCache;
+    private static final long NEG_CACHE_TIME = 60*60*1000L;
 
     static {
         int size;
@@ -301,12 +303,13 @@ public abstract class Addresses {
             long maxMemory = SystemVersion.getMaxMemory();
             long min = 256;
             long max = 4096;
-            // 1024 nominal for 128 MB
+            // 2048 nominal for 128 MB
-            size = (int) Math.max(min, Math.min(max, 1 + (maxMemory / (128*1024))));
+            size = (int) Math.max(min, Math.min(max, 1 + (maxMemory / (64*1024))));
         } else {
             size = 32;
         }
         _IPAddress = new LHMCache<String, byte[]>(size);
+        _negativeCache = new LHMCache<String, Long>(128);
     }
 
     /**
@@ -329,6 +332,14 @@ public abstract class Addresses {
             rv = _IPAddress.get(host);
         }
         if (rv == null) {
+            synchronized(_negativeCache) {
+                Long when = _negativeCache.get(host);
+                if (when != null) {
+                    if (when.longValue() > System.currentTimeMillis() - NEG_CACHE_TIME)
+                        return null;
+                    _negativeCache.remove(host);
+                }
+            }
             try {
                 rv = InetAddress.getByName(host).getAddress();
                 if (InetAddressUtils.isIPv4Address(host) ||
@@ -337,7 +348,12 @@ public abstract class Addresses {
                         _IPAddress.put(host, rv);
                     }
                 }
-            } catch (UnknownHostException uhe) {}
+                // else we do not cache hostnames here, we rely on the JVM
+            } catch (UnknownHostException uhe) {
+                synchronized(_negativeCache) {
+                    _negativeCache.put(host, Long.valueOf(System.currentTimeMillis()));
+                }
+            }
         }
         return rv;
     }
@@ -357,6 +373,14 @@ public abstract class Addresses {
             return null;
         if (InetAddressUtils.isIPv4Address(host) || InetAddressUtils.isIPv6Address(host))
             return getIP(host);
+        synchronized(_negativeCache) {
+            Long when = _negativeCache.get(host);
+            if (when != null) {
+                if (when.longValue() > System.currentTimeMillis() - NEG_CACHE_TIME)
+                    return null;
+                _negativeCache.remove(host);
+            }
+        }
         byte[] rv = null;
         try {
             InetAddress[] addrs = InetAddress.getAllByName(host);
@@ -372,7 +396,11 @@ public abstract class Addresses {
                         break;
                 }
             }
-        } catch (UnknownHostException uhe) {}
+        } catch (UnknownHostException uhe) {
+            synchronized(_negativeCache) {
+                _negativeCache.put(host, Long.valueOf(System.currentTimeMillis()));
+            }
+        }
         return rv;
     }
 
@@ -402,6 +430,14 @@ public abstract class Addresses {
                 return null;
             return Collections.singletonList(brv);
         }
+        synchronized(_negativeCache) {
+            Long when = _negativeCache.get(host);
+            if (when != null) {
+                if (when.longValue() > System.currentTimeMillis() - NEG_CACHE_TIME)
+                    return null;
+                _negativeCache.remove(host);
+            }
+        }
         try {
             InetAddress[] addrs = InetAddress.getAllByName(host);
             if (addrs == null || addrs.length == 0)
@@ -411,7 +447,11 @@ public abstract class Addresses {
                 rv.add(addrs[i].getAddress());
             }
             return rv;
-        } catch (UnknownHostException uhe) {}
+        } catch (UnknownHostException uhe) {
+            synchronized(_negativeCache) {
+                _negativeCache.put(host, Long.valueOf(System.currentTimeMillis()));
+            }
+        }
         return null;
     }
 
@@ -552,6 +592,9 @@ public abstract class Addresses {
         synchronized(_IPAddress) {
             _IPAddress.clear();
         }
+        synchronized(_negativeCache) {
+            _negativeCache.clear();
+        }
         if (_ifCache != null) {
             synchronized(_ifCache) {
                 _ifCache.clear();

history.txt

@@ -1,3 +1,9 @@
+2017-06-15 zzz
+ * Data: Cache hostname lookups in RouterAddress (ticket #1998)
+ * Job Queue: Make search jobs droppable (ticket #1998)
+ * Router: Increase JVM DNS cache time (ticket #1998)
+ * Util: Add negative DNS lookup cache, increase cache size (ticket #1998)
+
 2017-06-12 str4d
  * Console:
    - /logs: fixed environment/running info table resize on focus (ticket #1996)

router/java/src/net/i2p/data/router/RouterAddress.java

@@ -50,11 +50,12 @@ public class RouterAddress extends DataStructureImpl {
     private String _transportStyle;
     private final Properties _options;
     // cached values
-    private byte[] _ip;
+    private byte[] _ip = NOT_LOOKED_UP;
     private int _port;
 
     public static final String PROP_HOST = "host";
     public static final String PROP_PORT = "port";
+    private static final byte[] NOT_LOOKED_UP = new byte[0];
 
     public RouterAddress() {
         _options = new OrderedProperties();
@@ -213,26 +214,23 @@ public class RouterAddress extends DataStructureImpl {
     
     /**
      *  Caching version of InetAddress.getByName(getOption("host")).getAddress(), which is slow.
-     *  Caches numeric host names only.
+     *  Caches numeric host names AND DNS host names, and negative caches also.
-     *  Will resolve but not cache resolution of DNS host names.
      *
      *  @return IP or null
      *  @since 0.9.3
      */
-    public byte[] getIP() {
+    public synchronized byte[] getIP() {
-        if (_ip != null)
+        if (_ip == NOT_LOOKED_UP) {
-            return _ip;
+            // Only look up once, even if it fails, so we don't generate excessive DNS lookups.
-        byte[] rv = null;
+            // The lifetime of a RouterAddress object is a few hours at most,
-        String host = getHost();
+            // it will get republished or expired, so it's OK even for host names.
-        if (host != null) {
+            String host = getHost();
-            rv = Addresses.getIP(host);
+            if (host != null)
-            if (rv != null &&
+                _ip = Addresses.getIP(host);
-                (InetAddressUtils.isIPv4Address(host) ||
+            else
-                 InetAddressUtils.isIPv6Address(host))) {
+                _ip = null;
-                _ip = rv;
-            }
         }
-        return rv;
+        return _ip;
     }
     
     /**

router/java/src/net/i2p/router/JobQueue.java

@@ -27,6 +27,7 @@ import java.util.concurrent.atomic.AtomicInteger;
 import net.i2p.data.DataHelper;
 import net.i2p.router.message.HandleGarlicMessageJob;
 import net.i2p.router.networkdb.kademlia.HandleFloodfillDatabaseLookupMessageJob;
+import net.i2p.router.networkdb.kademlia.IterativeSearchJob;
 import net.i2p.router.RouterClock;
 import net.i2p.util.Clock;
 import net.i2p.util.I2PThread;
@@ -316,10 +317,13 @@ public class JobQueue {
             // Garlic added in 0.9.19, floodfills were getting overloaded
             // with encrypted lookups
             //
+            // ISJ added in 0.9.31, can get backed up due to DNS
+            //
             // Obviously we can only drop one-shot jobs, not those that requeue
             //
             if (cls == HandleFloodfillDatabaseLookupMessageJob.class ||
-                cls == HandleGarlicMessageJob.class) {
+                cls == HandleGarlicMessageJob.class ||
+                cls == IterativeSearchJob.class) {
                 // this tail drops based on the lag at the tail, which
                 // makes no sense...
                 //JobTiming jt = job.getTiming();

router/java/src/net/i2p/router/Router.java

@@ -117,7 +117,6 @@ public class Router implements RouterClock.ClockShiftListener {
     private final static String PROP_SHUTDOWN_IN_PROGRESS = "__shutdownInProgress";
     public static final String PROP_IB_RANDOM_KEY = TunnelPoolSettings.PREFIX_INBOUND_EXPLORATORY + TunnelPoolSettings.PROP_RANDOM_KEY;
     public static final String PROP_OB_RANDOM_KEY = TunnelPoolSettings.PREFIX_OUTBOUND_EXPLORATORY + TunnelPoolSettings.PROP_RANDOM_KEY;
-    private final static String DNS_CACHE_TIME = "" + (5*60);
     private static final String EVENTLOG = "eventlog.txt";
     private static final String PROP_JBIGI = "jbigi.loadedResource";
     public static final String UPDATE_FILE = "i2pupdate.zip";
@@ -133,10 +132,12 @@ public class Router implements RouterClock.ClockShiftListener {
         if (System.getProperty("I2P_DISABLE_DNS_CACHE_OVERRIDE") == null) {
             // grumble about sun's java caching DNS entries *forever* by default
             // so lets just keep 'em for a short time
+            String DNS_CACHE_TIME = Integer.toString(2*60*60);
+            String DNS_NEG_CACHE_TIME = Integer.toString(30*60);
             System.setProperty("sun.net.inetaddr.ttl", DNS_CACHE_TIME);
-            System.setProperty("sun.net.inetaddr.negative.ttl", DNS_CACHE_TIME);
+            System.setProperty("sun.net.inetaddr.negative.ttl", DNS_NEG_CACHE_TIME);
             System.setProperty("networkaddress.cache.ttl", DNS_CACHE_TIME);
-            System.setProperty("networkaddress.cache.negative.ttl", DNS_CACHE_TIME);
+            System.setProperty("networkaddress.cache.negative.ttl", DNS_NEG_CACHE_TIME);
         }
         if (System.getProperty("I2P_DISABLE_HTTP_AGENT_OVERRIDE") == null) {
             System.setProperty("http.agent", "I2P");

router/java/src/net/i2p/router/RouterVersion.java

@@ -18,7 +18,7 @@ public class RouterVersion {
     /** deprecated */
     public final static String ID = "Monotone";
     public final static String VERSION = CoreVersion.VERSION;
-    public final static long BUILD = 11;
+    public final static long BUILD = 12;
 
     /** for example "-test" */
     public final static String EXTRA = "";

router/java/src/net/i2p/router/networkdb/kademlia/IterativeSearchJob.java

@@ -54,9 +54,11 @@ import net.i2p.util.VersionComparator;
  * Halves search traffic for successful searches, as this doesn't do
  * two sesarches in parallel like FOSJ does.
  *
+ * Public only for JobQueue, not a public API, not for external use.
+ *
  * @since 0.8.9
  */
-class IterativeSearchJob extends FloodSearchJob {
+public class IterativeSearchJob extends FloodSearchJob {
     /** peers not sent to yet, sorted closest-to-the-routing-key */
     private final SortedSet<Hash> _toTry;
     /** query sent, no reply yet */
@@ -559,6 +561,15 @@ class IterativeSearchJob extends FloodSearchJob {
         return rv == null ? -1 : rv.longValue();
     }
 
+    /**
+     *  Dropped by the job queue
+     *  @since 0.9.31
+     */
+    @Override
+    public void dropped() {
+        failed();
+    }
+
     /**
      *  Total failure
      */