2007-06-16 Complication

* First pass on EepGet and ReseedHandler improvements,
      please avoid use on routers which matter!
    * Give EepGet ability of downloading into an OutputStream,
      such as the ByteArrayOutputStream of ReseedHandler.
    * Detect failure to reseed better, report it persistently
      and more verbosely, provide a link to logs
      and suggest manual reseed.

apps/routerconsole/java/src/net/i2p/router/web/ReseedHandler.java

@@ -3,7 +3,6 @@ package net.i2p.router.web;
 import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.FileOutputStream;
-import java.io.InputStream;
 import java.io.OutputStream;
 
 import java.net.Socket;
@@ -13,11 +12,13 @@ import java.util.Iterator;
 import java.util.Set;
 
 import net.i2p.I2PAppContext;
+import net.i2p.util.Log;
 import net.i2p.util.I2PThread;
+import net.i2p.util.EepGet;
 
 /**
  * Handler to deal with reseed requests.  This reseed from the URL
- * http://dev.i2p.net/i2pdb2/ unless the java env property "i2p.reseedURL" is
+ * http://dev.i2p.net/i2pdb2/ unless the I2P configuration property "i2p.reseedURL" is
  * set.  It always writes to ./netDb/, so don't mess with that.
  *
  */
@@ -67,13 +68,24 @@ public class ReseedHandler {
      *
      */
     private static void reseed(boolean echoStatus) {
-        String seedURL = I2PAppContext.getGlobalContext().getProperty("i2p.reseedURL", DEFAULT_SEED_URL);
+        I2PAppContext context = I2PAppContext.getGlobalContext();
+        Log log = context.logManager().getLog(ReseedHandler.class);
+        
+        String seedURL = context.getProperty("i2p.reseedURL", DEFAULT_SEED_URL);
         if ( (seedURL == null) || (seedURL.trim().length() <= 0) ) 
             seedURL = DEFAULT_SEED_URL;
+        System.setProperty("net.i2p.router.web.ReseedHandler.errorMessage","");
         try {
             URL dir = new URL(seedURL);
             byte contentRaw[] = readURL(dir);
-            if (contentRaw == null) return;
+            if (contentRaw == null) {
+                System.setProperty("net.i2p.router.web.ReseedHandler.errorMessage",
+                    "Last reseed failed fully (failed reading seed URL). " +
+                    "Ensure that nothing blocks outbound HTTP, check <a href=logs.jsp>logs</a> " +
+                    "and if nothing helps, read FAQ about reseeding manually.");
+                log.error("Failed reading seed URL: " + seedURL);
+                return;
+            }
             String content = new String(contentRaw);
             Set urls = new HashSet();
             int cur = 0;
@@ -87,6 +99,14 @@ public class ReseedHandler {
                 urls.add(name);
                 cur = end + 1;
             }
+            if (urls.size() <= 0) {
+                log.error("Read " + contentRaw.length + " bytes from seed " + seedURL + ", but found no routerInfo URLs.");
+                System.setProperty("net.i2p.router.web.ReseedHandler.errorMessage",
+                    "Last reseed failed fully (no routerInfo URLs at seed URL). " +
+                    "Ensure that nothing blocks outbound HTTP, check <a href=logs.jsp>logs</a> " +
+                    "and if nothing helps, read FAQ about reseeding manually.");
+                return;
+            }
 
             int fetched = 0;
             int errors = 0;
@@ -104,75 +124,47 @@ public class ReseedHandler {
                 }
             }
             if (echoStatus) System.out.println();
+            if (errors > 0) {
+                System.setProperty("net.i2p.router.web.ReseedHandler.errorMessage",
+                    "Last reseed failed partly (" + errors + " of " + urls.size() + "). " +
+                    "Ensure that nothing blocks outbound HTTP, check <a href=logs.jsp>logs</a> " +
+                    "and if nothing helps, read FAQ about reseeding manually.");
+            }
         } catch (Throwable t) {
-            I2PAppContext.getGlobalContext().logManager().getLog(ReseedHandler.class).error("Error reseeding", t);
+            System.setProperty("net.i2p.router.web.ReseedHandler.errorMessage",
+                "Last reseed failed fully (exception caught). " +
+                "Ensure that nothing blocks outbound HTTP, check <a href=logs.jsp>logs</a> " +
+                "and if nothing helps, read FAQ about reseeding manually.");
+            log.error("Error reseeding", t);
         }
     }
     
+    /* Since we don't return a value, we should always throw an exception if something fails. */
     private static void fetchSeed(String seedURL, String peer) throws Exception {
+        Log log = I2PAppContext.getGlobalContext().logManager().getLog(ReseedHandler.class);
         URL url = new URL(seedURL + (seedURL.endsWith("/") ? "" : "/") + "routerInfo-" + peer + ".dat");
 
         byte data[] = readURL(url);
+        if (data == null) {
+            log.error("Failed fetching seed: " + url.toString());
+            throw new Exception ("Failed fetching seed.");
+        }
+        if (data.length < 1024) {
+            log.error("Fetched data too small to contain a routerInfo: " + url.toString());
+            throw new Exception ("Fetched data too small.");
+        }
         //System.out.println("read: " + (data != null ? data.length : -1));
         writeSeed(peer, data);
     }
     
     private static byte[] readURL(URL url) throws Exception {
         ByteArrayOutputStream baos = new ByteArrayOutputStream(4*1024);
-        String hostname = url.getHost();
+        
-        int port = url.getPort();
+        // Do a non-proxied eepget into our ByteArrayOutputStream with 0 retries
-        if (port < 0)
+        EepGet get = new EepGet( I2PAppContext.getGlobalContext(), false, null, -1, 0,
-            port = 80;
+            null, baos, url.toString(), false, null, null);
-        Socket s = new Socket(hostname, port);
+
-        OutputStream out = s.getOutputStream();
+        if (get.fetch()) return baos.toByteArray(); else return null;
-        InputStream in = s.getInputStream();
-        String request = getRequest(url);
-        //System.out.println("Sending to " + hostname +":"+ port + ": " + request);
-        out.write(request.getBytes());
-        out.flush();
-        // skip the HTTP response headers
-        // (if we were smart, we'd check for HTTP 200, content-length, etc)
-        int consecutiveNL = 0;
-        while (true) {
-            int cur = in.read();
-            switch (cur) {
-                case -1: 
-                    return null;
-                case '\n':
-                case '\r':
-                    consecutiveNL++;
-                    break;
-                default:
-                    consecutiveNL = 0;
-            }
-            if (consecutiveNL == 4)
-                break;
-        }
-        // ok, past the headers, grab the goods
-        byte buf[] = new byte[1024];
-        while (true) {
-            int read = in.read(buf);
-            if (read < 0)
-                break;
-            baos.write(buf, 0, read);
-        }
-        in.close();
-        s.close();
-        return baos.toByteArray();
-    }
-    
-    private static String getRequest(URL url) {
-        StringBuffer buf = new StringBuffer(512);
-        String path = url.getPath();
-        if ("".equals(path))
-            path = "/";
-        buf.append("GET ").append(path).append(" HTTP/1.0\n");
-        buf.append("Host: ").append(url.getHost());
-        int port = url.getPort();
-        if ( (port > 0) && (port != 80) )
-            buf.append(":").append(port);
-        buf.append("\nConnection: close\n\n");
-        return buf.toString();
     }
     
     private static void writeSeed(String name, byte data[]) throws Exception {

apps/routerconsole/jsp/summary.jsp

@@ -43,12 +43,12 @@
  <b>Failing:</b> <jsp:getProperty name="helper" property="failingPeers" /><br />
  <!-- <b>Shitlisted:</b> <jsp:getProperty name="helper" property="shitlistedPeers" /><br /> -->
  <b>Known:</b> <jsp:getProperty name="helper" property="allPeers" /><br /><%
-     if (helper.getActivePeers() <= 0) {
+    if (helper.getActivePeers() <= 0) {
         %><b><a href="config.jsp">check your NAT/firewall</a></b><br /><%
-     }
+    }
     if (helper.allowReseed()) {
         if ("true".equals(System.getProperty("net.i2p.router.web.ReseedHandler.reseedInProgress", "false"))) {
-            out.print(" <i>reseeding</i>");
+            out.print(" <i>reseeding</i><br />");
         } else {
             long nonce = new java.util.Random().nextLong();
             String prev = System.getProperty("net.i2p.router.web.ReseedHandler.nonce");
@@ -59,7 +59,14 @@
                 uri = uri + "&reseedNonce=" + nonce;
             else
                 uri = uri + "?reseedNonce=" + nonce;
-            out.print(" <a href=\"" + uri + "\">reseed</a>");
+            out.print(" <a href=\"" + uri + "\">reseed</a><br />");
+        }
+    }
+    // If a new reseed ain't running, show how the last reseed finished
+    if ("false".equals(System.getProperty("net.i2p.router.web.ReseedHandler.reseedInProgress", "false"))) {
+        String reseedErrorMessage = System.getProperty("net.i2p.router.web.ReseedHandler.errorMessage","");
+        if (reseedErrorMessage.length() > 0) {
+            out.print("<i>" + reseedErrorMessage + "</i><br />");
         }
     }
  %><hr />

core/java/src/net/i2p/util/EepGet.java

@@ -34,6 +34,7 @@ public class EepGet {
     private int _proxyPort;
     private int _numRetries;
     private String _outputFile;
+    private OutputStream _outputStream;
     private String _url;
     private String _postData;
     private boolean _allowCaching;
@@ -53,35 +54,46 @@ public class EepGet {
     private boolean _notModified;
     private String _contentType;
     
+    // Constructor 7, calls 3 with: do proxy
     public EepGet(I2PAppContext ctx, String proxyHost, int proxyPort, int numRetries, String outputFile, String url) {
         this(ctx, true, proxyHost, proxyPort, numRetries, outputFile, url);
     }
+    // Constructor 6, calls 1 with: do proxy, no etag
     public EepGet(I2PAppContext ctx, String proxyHost, int proxyPort, int numRetries, String outputFile, String url, boolean allowCaching) {
         this(ctx, true, proxyHost, proxyPort, numRetries, outputFile, url, allowCaching, null);
     }
+    // Constructor 5, calls 3 with: no proxy
     public EepGet(I2PAppContext ctx, int numRetries, String outputFile, String url) {
         this(ctx, false, null, -1, numRetries, outputFile, url);
     }
+    // Constructor 4, calls 1 with: no proxy, no etag
     public EepGet(I2PAppContext ctx, int numRetries, String outputFile, String url, boolean allowCaching) {
         this(ctx, false, null, -1, numRetries, outputFile, url, allowCaching, null);
     }
+    // Constructor 3, calls 1 with: do caching, no etag
     public EepGet(I2PAppContext ctx, boolean shouldProxy, String proxyHost, int proxyPort, int numRetries, String outputFile, String url) {
         this(ctx, shouldProxy, proxyHost, proxyPort, numRetries, outputFile, url, true, null);
     }
+    // Constructor 2, calls 0 with: no output buffer, do caching, no etag
     public EepGet(I2PAppContext ctx, boolean shouldProxy, String proxyHost, int proxyPort, int numRetries, String outputFile, String url, String postData) {
-        this(ctx, shouldProxy, proxyHost, proxyPort, numRetries, outputFile, url, true, null, postData);
+        this(ctx, shouldProxy, proxyHost, proxyPort, numRetries, outputFile, null, url, true, null, postData);
     }
+    // Constructor 1, calls 0 with: no output buffer, no postdata
     public EepGet(I2PAppContext ctx, boolean shouldProxy, String proxyHost, int proxyPort, int numRetries, String outputFile, String url, boolean allowCaching, String etag) {
-        this(ctx, shouldProxy, proxyHost, proxyPort, numRetries, outputFile, url, allowCaching, etag, null);
+        this(ctx, shouldProxy, proxyHost, proxyPort, numRetries, outputFile, null, url, allowCaching, etag, null);
     }
-    public EepGet(I2PAppContext ctx, boolean shouldProxy, String proxyHost, int proxyPort, int numRetries, String outputFile, String url, boolean allowCaching, String etag, String postData) {
+    // Constructor 0, real constructor
+    public EepGet(I2PAppContext ctx, boolean shouldProxy, String proxyHost, int proxyPort, int numRetries,
+                  String outputFile, OutputStream outputStream, String url, boolean allowCaching,
+                  String etag, String postData) {
         _context = ctx;
         _log = ctx.logManager().getLog(EepGet.class);
         _shouldProxy = shouldProxy;
         _proxyHost = proxyHost;
         _proxyPort = proxyPort;
         _numRetries = numRetries;
-        _outputFile = outputFile;
+        _outputFile = outputFile;     // if outputFile is set, outputStream must be null
+        _outputStream = outputStream; // if both are set, outputStream overrides outputFile
         _url = url;
         _postData = postData;
         _alreadyTransferred = 0;
@@ -352,7 +364,7 @@ public class EepGet {
         return false;
     }
 
-    /** return true if the URL was completely retrieved */
+    /** a single fetch attempt */
     private void doFetch() throws IOException {
         readHeaders();
         if (_log.shouldLog(Log.DEBUG))
@@ -371,6 +383,11 @@ public class EepGet {
                 break;
             _out.write(buf, 0, read);
             _bytesTransferred += read;
+            // This seems necessary to properly resume a partial download into a stream,
+            // as nothing else increments _alreadyTransferred, and there's no file length to check.
+            // Hopefully this won't break compatibility with existing status listeners
+            // (cause them to behave weird, or show weird numbers).
+            _alreadyTransferred += read;
             remaining -= read;
             if (remaining==0 && _encodingChunked) {
                 if(_proxyIn.read()=='\r' && _proxyIn.read()=='\n') {
@@ -419,12 +436,14 @@ public class EepGet {
         boolean rcOk = false;
         switch (responseCode) {
             case 200: // full
-                _out = new FileOutputStream(_outputFile, false);
+                if (_outputStream != null) _out = _outputStream;
+                    else _out = new FileOutputStream(_outputFile, false);
                 _alreadyTransferred = 0;
                 rcOk = true;
                 break;
             case 206: // partial
-                _out = new FileOutputStream(_outputFile, true);
+                if (_outputStream != null) _out = _outputStream;
+                    else _out = new FileOutputStream(_outputFile, true);
                 rcOk = true;
                 break;
             case 304: // not modified
@@ -588,9 +607,16 @@ public class EepGet {
     private boolean isNL(byte b) { return (b == NL); }
 
     private void sendRequest() throws IOException {
-        File outFile = new File(_outputFile);
+        if (_outputStream != null) {
-        if (outFile.exists())
+            // We are reading into a stream supplied by a caller,
-            _alreadyTransferred = outFile.length();
+            // for which we cannot easily determine how much we've written.
+            // Assume that _alreadyTransferred holds the right value
+            // (we should never be restarted to work on an old stream).
+        } else {
+            File outFile = new File(_outputFile);
+            if (outFile.exists())
+                _alreadyTransferred = outFile.length();
+        }
 
         String req = getRequest();
 

history.txt

@@ -1,4 +1,13 @@
-$Id: history.txt,v 1.567 2007-05-06 14:52:39 complication Exp $
+$Id: history.txt,v 1.568 2007-05-06 15:02:04 complication Exp $
+
+2007-06-16  Complication
+    * First pass on EepGet and ReseedHandler improvements,
+      please avoid use on routers which matter!
+    * Give EepGet ability of downloading into an OutputStream,
+      such as the ByteArrayOutputStream of ReseedHandler.
+    * Detect failure to reseed better, report it persistently
+      and more verbosely, provide a link to logs
+      and suggest manual reseed.
 
 2007-05-06  Complication
     * Fix the build.xml file, so the preppkg build target won't try copying files

router/java/src/net/i2p/router/RouterVersion.java

@@ -15,9 +15,9 @@ import net.i2p.CoreVersion;
  *
  */
 public class RouterVersion {
-    public final static String ID = "$Revision: 1.502 $ $Date: 2007-03-31 16:50:51 $";
+    public final static String ID = "$Revision: 1.503 $ $Date: 2007-05-06 14:52:43 $";
     public final static String VERSION = "0.6.1.28";
-    public final static long BUILD = 5;
+    public final static long BUILD = 6;
     public static void main(String args[]) {
         System.out.println("I2P Router version: " + VERSION + "-" + BUILD);
         System.out.println("Router ID: " + RouterVersion.ID);