2007-07-14 Complication

* Take the post-download routerInfo size check back out of ReseedHandler,
      since it wasn't helpful, and a lower limit caused false warnings.
    * Give EepGet ability to enforce a min/max HTTP response size.
    * Enforce a maximum response size of 8 MB when ReseedHandler
      downloads into a ByteArrayOutputStream.
    * Refactor ReseedHandler/ReseedRunner from static to ordinary classes,
      change invocation from RouterConsoleRunner accordingly.
    * Add an EepGet status listener to ReseedHandler to log causes of reseed failure,
      provide status reports to indicate the progress of reseeding.
    * Enable icon for default eepsite, and the index page
      of the router console (more later).

apps/routerconsole/java/src/net/i2p/router/web/ReseedHandler.java

@@ -12,6 +12,7 @@ import java.util.Iterator;
 import java.util.Set;
 
 import net.i2p.I2PAppContext;
+import net.i2p.router.RouterContext;
 import net.i2p.util.Log;
 import net.i2p.util.I2PThread;
 import net.i2p.util.EepGet;
@@ -23,9 +24,39 @@ import net.i2p.util.EepGet;
  *
  */
 public class ReseedHandler {
+    private static ReseedRunner _reseedRunner;
+    private RouterContext _context;
+    private Log _log;
+
+    // Reject unreasonably big files, because we download into a ByteArrayOutputStream.
+    private static final long MAX_RESEED_RESPONSE_SIZE = 8 * 1024 * 1024;
+
+    private static final String DEFAULT_SEED_URL = "http://dev.i2p.net/i2pdb2/";
+
+    public ReseedHandler() {
+        this(ContextHelper.getContext(null));
+    }
+    public ReseedHandler(RouterContext ctx) {
+        _context = ctx;
+        _log = ctx.logManager().getLog(ReseedHandler.class);
+    }
+
+    /**
+     * Configure this bean to query a particular router context
+     *
+     * @param contextId begging few characters of the routerHash, or null to pick
+     *                  the first one we come across.
+     */
+    public void setContextId(String contextId) {
+        try {
+            _context = ContextHelper.getContext(contextId);
+            _log = _context.logManager().getLog(ReseedHandler.class);
+        } catch (Throwable t) {
+            t.printStackTrace();
+        }
+    }
+
 
-    private static ReseedRunner _reseedRunner = new ReseedRunner();
-    
     public void setReseedNonce(String nonce) { 
         if (nonce == null) return;
         if (nonce.equals(System.getProperty("net.i2p.router.web.ReseedHandler.nonce")) ||
@@ -34,22 +65,28 @@ public class ReseedHandler {
         }
     }
     
-    public static void requestReseed() {
+    public void requestReseed() {
-        synchronized (_reseedRunner) {
+        synchronized (ReseedHandler.class) {
+            if (_reseedRunner == null)
+                _reseedRunner = new ReseedRunner();
             if (_reseedRunner.isRunning()) {
                 return;
             } else {
                 System.setProperty("net.i2p.router.web.ReseedHandler.reseedInProgress", "true");
-                System.out.println("Reseeding");
                 I2PThread reseed = new I2PThread(_reseedRunner, "Reseed");
                 reseed.start();
             }
         }
+
     }
 
-    public static class ReseedRunner implements Runnable {
+    public class ReseedRunner implements Runnable, EepGet.StatusListener {
         private boolean _isRunning;
-        public ReseedRunner() { _isRunning = false; }
+
+        public ReseedRunner() {
+            _isRunning = false; 
+            System.setProperty("net.i2p.router.web.ReseedHandler.statusMessage","Reseeding.");
+        }
         public boolean isRunning() { return _isRunning; }
         public void run() {
             _isRunning = true;
@@ -58,132 +95,148 @@ public class ReseedHandler {
             System.setProperty("net.i2p.router.web.ReseedHandler.reseedInProgress", "false");
             _isRunning = false;
         }
-    }
-    
-    static final String DEFAULT_SEED_URL = "http://dev.i2p.net/i2pdb2/";
-    /**
-     * Reseed has been requested, so lets go ahead and do it.  Fetch all of
-     * the routerInfo-*.dat files from the specified URL (or the default) and
-     * save them into this router's netDb dir.
-     *
-     */
-    private static void reseed(boolean echoStatus) {
-        I2PAppContext context = I2PAppContext.getGlobalContext();
-        Log log = context.logManager().getLog(ReseedHandler.class);
-        
-        String seedURL = context.getProperty("i2p.reseedURL", DEFAULT_SEED_URL);
-        if ( (seedURL == null) || (seedURL.trim().length() <= 0) ) 
-            seedURL = DEFAULT_SEED_URL;
-        System.setProperty("net.i2p.router.web.ReseedHandler.errorMessage","");
-        try {
-            URL dir = new URL(seedURL);
-            byte contentRaw[] = readURL(dir);
-            if (contentRaw == null) {
-                System.setProperty("net.i2p.router.web.ReseedHandler.errorMessage",
-                    "Last reseed failed fully (failed reading seed URL). " +
-                    "Ensure that nothing blocks outbound HTTP, check <a href=logs.jsp>logs</a> " +
-                    "and if nothing helps, read FAQ about reseeding manually.");
-                log.error("Failed reading seed URL: " + seedURL);
-                return;
-            }
-            String content = new String(contentRaw);
-            Set urls = new HashSet();
-            int cur = 0;
-            while (true) {
-                int start = content.indexOf("href=\"routerInfo-", cur);
-                if (start < 0)
-                    break;
 
-                int end = content.indexOf(".dat\">", start);
+        // EepGet status listeners
-                String name = content.substring(start+"href=\"routerInfo-".length(), end);
+        public void attemptFailed(String url, long bytesTransferred, long bytesRemaining, int currentAttempt, int numRetries, Exception cause) {
-                urls.add(name);
+            // Since readURL() runs an EepGet with 0 retries,
-                cur = end + 1;
+            // we can report errors with attemptFailed() instead of transferFailed().
-            }
+            // It has the benefit of providing cause of failure, which helps resolve issues.
-            if (urls.size() <= 0) {
+            if (_log.shouldLog(Log.ERROR)) _log.error("EepGet failed on " + url, cause);
-                log.error("Read " + contentRaw.length + " bytes from seed " + seedURL + ", but found no routerInfo URLs.");
+        }
-                System.setProperty("net.i2p.router.web.ReseedHandler.errorMessage",
+        public void bytesTransferred(long alreadyTransferred, int currentWrite, long bytesTransferred, long bytesRemaining, String url) {}
-                    "Last reseed failed fully (no routerInfo URLs at seed URL). " +
+        public void transferComplete(long alreadyTransferred, long bytesTransferred, long bytesRemaining, String url, String outputFile, boolean notModified) {}
-                    "Ensure that nothing blocks outbound HTTP, check <a href=logs.jsp>logs</a> " +
+        public void transferFailed(String url, long bytesTransferred, long bytesRemaining, int currentAttempt) {}
-                    "and if nothing helps, read FAQ about reseeding manually.");
+        public void headerReceived(String url, int attemptNum, String key, String val) {}
-                return;
+        // End of EepGet status listeners
-            }
 
-            int fetched = 0;
+        /**
-            int errors = 0;
+        * Reseed has been requested, so lets go ahead and do it.  Fetch all of
-            for (Iterator iter = urls.iterator(); iter.hasNext(); ) {
+        * the routerInfo-*.dat files from the specified URL (or the default) and
-                try {
+        * save them into this router's netDb dir.
-                    fetchSeed(seedURL, (String)iter.next());
+        *
-                    fetched++;
+        */
-                    if (echoStatus) {
+        private void reseed(boolean echoStatus) {
-                        System.out.print(".");
+
-                        if (fetched % 60 == 0)
+            String seedURL = _context.getProperty("i2p.reseedURL", DEFAULT_SEED_URL);
-                            System.out.println();
+            if ( (seedURL == null) || (seedURL.trim().length() <= 0) ) 
-                    }
+                seedURL = DEFAULT_SEED_URL;
-                } catch (Exception e) {
+            try {
-                    errors++;
+                System.setProperty("net.i2p.router.web.ReseedHandler.errorMessage","");
+                System.setProperty("net.i2p.router.web.ReseedHandler.statusMessage","Reseeding: fetching seed URL.");
+                URL dir = new URL(seedURL);
+                byte contentRaw[] = readURL(dir);
+                if (contentRaw == null) {
+                    System.setProperty("net.i2p.router.web.ReseedHandler.errorMessage",
+                        "Last reseed failed fully (failed reading seed URL). " +
+                        "Ensure that nothing blocks outbound HTTP, check <a href=logs.jsp>logs</a> " +
+                        "and if nothing helps, read FAQ about reseeding manually.");
+                    // Logging deprecated here since attemptFailed() provides better info
+                    _log.debug("Failed reading seed URL: " + seedURL);
+                    return;
                 }
-            }
+                String content = new String(contentRaw);
-            if (echoStatus) System.out.println();
+                Set urls = new HashSet();
-            if (errors > 0) {
+                int cur = 0;
+                while (true) {
+                    int start = content.indexOf("href=\"routerInfo-", cur);
+                    if (start < 0)
+                        break;
+
+                    int end = content.indexOf(".dat\">", start);
+                    String name = content.substring(start+"href=\"routerInfo-".length(), end);
+                    urls.add(name);
+                    cur = end + 1;
+                }
+                if (urls.size() <= 0) {
+                    _log.error("Read " + contentRaw.length + " bytes from seed " + seedURL + ", but found no routerInfo URLs.");
+                    System.setProperty("net.i2p.router.web.ReseedHandler.errorMessage",
+                        "Last reseed failed fully (no routerInfo URLs at seed URL). " +
+                        "Ensure that nothing blocks outbound HTTP, check <a href=logs.jsp>logs</a> " +
+                        "and if nothing helps, read FAQ about reseeding manually.");
+                    return;
+                }
+
+                int fetched = 0;
+                int errors = 0;
+                for (Iterator iter = urls.iterator(); iter.hasNext(); ) {
+                    try {
+                        System.setProperty("net.i2p.router.web.ReseedHandler.statusMessage",
+                            "Reseeding: fetching router info from seed URL (" +
+                            fetched + " successful, " + errors + " errors, " + urls.size() + " total).");
+
+                        fetchSeed(seedURL, (String)iter.next());
+                        fetched++;
+                        if (echoStatus) {
+                            System.out.print(".");
+                            if (fetched % 60 == 0)
+                                System.out.println();
+                        }
+                    } catch (Exception e) {
+                        errors++;
+                    }
+                }
+                if (echoStatus) System.out.println();
+                if (errors > 0) {
+                    System.setProperty("net.i2p.router.web.ReseedHandler.errorMessage",
+                        "Last reseed failed partly (" + errors + " of " + urls.size() + "). " +
+                        "Ensure that nothing blocks outbound HTTP, check <a href=logs.jsp>logs</a> " +
+                        "and if nothing helps, read FAQ about reseeding manually.");
+                }
+            } catch (Throwable t) {
                 System.setProperty("net.i2p.router.web.ReseedHandler.errorMessage",
-                    "Last reseed failed partly (" + errors + " of " + urls.size() + "). " +
+                    "Last reseed failed fully (exception caught). " +
                     "Ensure that nothing blocks outbound HTTP, check <a href=logs.jsp>logs</a> " +
                     "and if nothing helps, read FAQ about reseeding manually.");
+                _log.error("Error reseeding", t);
             }
-        } catch (Throwable t) {
-            System.setProperty("net.i2p.router.web.ReseedHandler.errorMessage",
-                "Last reseed failed fully (exception caught). " +
-                "Ensure that nothing blocks outbound HTTP, check <a href=logs.jsp>logs</a> " +
-                "and if nothing helps, read FAQ about reseeding manually.");
-            log.error("Error reseeding", t);
         }
-    }
     
-    /* Since we don't return a value, we should always throw an exception if something fails. */
+        /* Since we don't return a value, we should always throw an exception if something fails. */
-    private static void fetchSeed(String seedURL, String peer) throws Exception {
+        private void fetchSeed(String seedURL, String peer) throws Exception {
-        Log log = I2PAppContext.getGlobalContext().logManager().getLog(ReseedHandler.class);
+            Log log = I2PAppContext.getGlobalContext().logManager().getLog(ReseedHandler.class);
-        URL url = new URL(seedURL + (seedURL.endsWith("/") ? "" : "/") + "routerInfo-" + peer + ".dat");
+            URL url = new URL(seedURL + (seedURL.endsWith("/") ? "" : "/") + "routerInfo-" + peer + ".dat");
 
-        byte data[] = readURL(url);
+            byte data[] = readURL(url);
-        if (data == null) {
+            if (data == null) {
-            log.error("Failed fetching seed: " + url.toString());
+                // Logging deprecated here since attemptFailed() provides better info
-            throw new Exception ("Failed fetching seed.");
+                _log.debug("Failed fetching seed: " + url.toString());
+                throw new Exception ("Failed fetching seed.");
+            }
+            //System.out.println("read: " + (data != null ? data.length : -1));
+            writeSeed(peer, data);
         }
-        if (data.length < 1024) {
-            log.error("Fetched data too small to contain a routerInfo: " + url.toString());
-            throw new Exception ("Fetched data too small.");
-        }
-        //System.out.println("read: " + (data != null ? data.length : -1));
-        writeSeed(peer, data);
-    }
-    
-    private static byte[] readURL(URL url) throws Exception {
-        ByteArrayOutputStream baos = new ByteArrayOutputStream(4*1024);
-        
-        // Do a non-proxied eepget into our ByteArrayOutputStream with 0 retries
-        EepGet get = new EepGet( I2PAppContext.getGlobalContext(), false, null, -1, 0,
-            null, baos, url.toString(), false, null, null);
 
-        if (get.fetch()) return baos.toByteArray(); else return null;
+        private byte[] readURL(URL url) throws Exception {
-    }
+            ByteArrayOutputStream baos = new ByteArrayOutputStream(4*1024);
-    
+
-    private static void writeSeed(String name, byte data[]) throws Exception {
+            // Do a non-proxied eepget into our ByteArrayOutputStream with 0 retries
-        String dirName = "netDb"; // _context.getProperty("router.networkDatabase.dbDir", "netDb");
+            EepGet get = new EepGet( I2PAppContext.getGlobalContext(), false, null, -1, 0, 0, MAX_RESEED_RESPONSE_SIZE,
-        File netDbDir = new File(dirName);
+                null, baos, url.toString(), false, null, null);
-        if (!netDbDir.exists()) {
+            get.addStatusListener(ReseedRunner.this);
-            boolean ok = netDbDir.mkdirs();
+            if (get.fetch()) return baos.toByteArray(); else return null;
         }
-        FileOutputStream fos = new FileOutputStream(new File(netDbDir, "routerInfo-" + name + ".dat"));
-        fos.write(data);
-        fos.close();
-    }
     
+        private void writeSeed(String name, byte data[]) throws Exception {
+            String dirName = "netDb"; // _context.getProperty("router.networkDatabase.dbDir", "netDb");
+            File netDbDir = new File(dirName);
+            if (!netDbDir.exists()) {
+                boolean ok = netDbDir.mkdirs();
+            }
+            FileOutputStream fos = new FileOutputStream(new File(netDbDir, "routerInfo-" + name + ".dat"));
+            fos.write(data);
+            fos.close();
+        }
+
+    }
+
     public static void main(String args[]) {
         if ( (args != null) && (args.length == 1) && (!Boolean.valueOf(args[0]).booleanValue()) ) {
             System.out.println("Not reseeding, as requested");
             return; // not reseeding on request
         }
         System.out.println("Reseeding");
-        reseed(true);
+        ReseedHandler reseedHandler = new ReseedHandler();
+        reseedHandler.requestReseed();
     }
+
 }

apps/routerconsole/java/src/net/i2p/router/web/RouterConsoleRunner.java

@@ -88,7 +88,8 @@ public class RouterConsoleRunner {
             // get i2p started - they can reseed later in the web console)
             String names[] = (netDb.exists() ? netDb.list() : null);
             if ( (names == null) || (names.length < 15) ) {
-                ReseedHandler.requestReseed();
+                ReseedHandler reseedHandler = new ReseedHandler();
+                reseedHandler.requestReseed();
             }
         }
         

apps/routerconsole/jsp/index.jsp

@@ -5,6 +5,7 @@
 <html><head>
 <title>I2P Router Console - home</title>
 <link rel="stylesheet" href="default.css" type="text/css" />
+<link rel="shortcut icon" href="favicon.ico" />
 </head><body>
 <%
 if (System.getProperty("router.consoleNonce") == null) {

apps/routerconsole/jsp/summary.jsp

@@ -46,10 +46,13 @@
     if (helper.getActivePeers() <= 0) {
         %><b><a href="config.jsp">check your NAT/firewall</a></b><br /><%
     }
+    // If showing the reseed link is allowed
     if (helper.allowReseed()) {
         if ("true".equals(System.getProperty("net.i2p.router.web.ReseedHandler.reseedInProgress", "false"))) {
-            out.print(" <i>reseeding</i><br />");
+            // While reseed occurring, show status message instead
+            out.print("<i>" + System.getProperty("net.i2p.router.web.ReseedHandler.statusMessage","") + "</i><br />");
         } else {
+            // While no reseed occurring, show reseed link
             long nonce = new java.util.Random().nextLong();
             String prev = System.getProperty("net.i2p.router.web.ReseedHandler.nonce");
             if (prev != null) System.setProperty("net.i2p.router.web.ReseedHandler.noncePrev", prev);
@@ -62,7 +65,7 @@
             out.print(" <a href=\"" + uri + "\">reseed</a><br />");
         }
     }
-    // If a new reseed ain't running, show how the last reseed finished
+    // If a new reseed ain't running, and the last reseed had errors, show error message
     if ("false".equals(System.getProperty("net.i2p.router.web.ReseedHandler.reseedInProgress", "false"))) {
         String reseedErrorMessage = System.getProperty("net.i2p.router.web.ReseedHandler.errorMessage","");
         if (reseedErrorMessage.length() > 0) {

core/java/src/net/i2p/util/EepGet.java

@@ -33,6 +33,8 @@ public class EepGet {
     private String _proxyHost;
     private int _proxyPort;
     private int _numRetries;
+    private long _minSize; // minimum and maximum acceptable response size, -1 signifies unlimited,
+    private long _maxSize; // applied both against whole responses and chunks
     private String _outputFile;
     private OutputStream _outputStream;
     private String _url;
@@ -76,14 +78,14 @@ public class EepGet {
     }
     // Constructor 2, calls 0 with: no output buffer, do caching, no etag
     public EepGet(I2PAppContext ctx, boolean shouldProxy, String proxyHost, int proxyPort, int numRetries, String outputFile, String url, String postData) {
-        this(ctx, shouldProxy, proxyHost, proxyPort, numRetries, outputFile, null, url, true, null, postData);
+        this(ctx, shouldProxy, proxyHost, proxyPort, numRetries, -1, -1, outputFile, null, url, true, null, postData);
     }
     // Constructor 1, calls 0 with: no output buffer, no postdata
     public EepGet(I2PAppContext ctx, boolean shouldProxy, String proxyHost, int proxyPort, int numRetries, String outputFile, String url, boolean allowCaching, String etag) {
-        this(ctx, shouldProxy, proxyHost, proxyPort, numRetries, outputFile, null, url, allowCaching, etag, null);
+        this(ctx, shouldProxy, proxyHost, proxyPort, numRetries, -1, -1, outputFile, null, url, allowCaching, etag, null);
     }
     // Constructor 0, real constructor
-    public EepGet(I2PAppContext ctx, boolean shouldProxy, String proxyHost, int proxyPort, int numRetries,
+    public EepGet(I2PAppContext ctx, boolean shouldProxy, String proxyHost, int proxyPort, int numRetries, long minSize, long maxSize,
                   String outputFile, OutputStream outputStream, String url, boolean allowCaching,
                   String etag, String postData) {
         _context = ctx;
@@ -92,6 +94,8 @@ public class EepGet {
         _proxyHost = proxyHost;
         _proxyPort = proxyPort;
         _numRetries = numRetries;
+        _minSize = minSize;
+        _maxSize = maxSize;
         _outputFile = outputFile;     // if outputFile is set, outputStream must be null
         _outputStream = outputStream; // if both are set, outputStream overrides outputFile
         _url = url;
@@ -371,7 +375,13 @@ public class EepGet {
             _log.debug("Headers read completely, reading " + _bytesRemaining);
         
         boolean strictSize = (_bytesRemaining >= 0);
-            
+
+        // If minimum or maximum size defined, ensure they aren't exceeded
+        if ((_minSize > -1) && (_bytesRemaining < _minSize))
+            throw new IOException("HTTP response size " + _bytesRemaining + " violates minimum of " + _minSize + " bytes");
+        if ((_maxSize > -1) && (_bytesRemaining > _maxSize))
+            throw new IOException("HTTP response size " + _bytesRemaining + " violates maximum of " + _maxSize + " bytes");
+
         int remaining = (int)_bytesRemaining;
         byte buf[] = new byte[1024];
         while (_keepFetching && ( (remaining > 0) || !strictSize )) {

history.txt

@@ -1,4 +1,17 @@
-$Id: history.txt,v 1.574 2007-07-09 20:20:38 zzz Exp $
+$Id: history.txt,v 1.575 2007-07-14 13:44:11 zzz Exp $
+
+2007-07-14  Complication
+    * Take the post-download routerInfo size check back out of ReseedHandler,
+      since it wasn't helpful, and a lower limit caused false warnings.
+    * Give EepGet ability to enforce a min/max HTTP response size.
+    * Enforce a maximum response size of 8 MB when ReseedHandler
+      downloads into a ByteArrayOutputStream.
+    * Refactor ReseedHandler/ReseedRunner from static to ordinary classes,
+      change invocation from RouterConsoleRunner accordingly.
+    * Add an EepGet status listener to ReseedHandler to log causes of reseed failure,
+      provide status reports to indicate the progress of reseeding.
+    * Enable icon for default eepsite, and the index page
+      of the router console (more later).
 
 2007-07-14  zzz
     * Clean up graphs.jsp - set K=1024 where appropriate,

installer/resources/eepsite_index.html

@@ -1,6 +1,7 @@
 <html>
  <head>
   <title>Welcome to your eepsite</title>
+  <link rel="shortcut icon" href="favicon.ico" />
  </head>
  <body>
  <h1>Welcome to your eepsite</h1>

router/java/src/net/i2p/router/RouterVersion.java

@@ -15,9 +15,9 @@ import net.i2p.CoreVersion;
  *
  */
 public class RouterVersion {
-    public final static String ID = "$Revision: 1.509 $ $Date: 2007-07-09 20:20:37 $";
+    public final static String ID = "$Revision: 1.510 $ $Date: 2007-07-14 13:44:12 $";
     public final static String VERSION = "0.6.1.28";
-    public final static long BUILD = 12;
+    public final static long BUILD = 13;
     public static void main(String args[]) {
         System.out.println("I2P Router version: " + VERSION + "-" + BUILD);
         System.out.println("Router ID: " + RouterVersion.ID);