* EepGet, I2PSnark:

- New I2PSocketEepGet fetches through existing tunnels
        rather than through the proxy
      - Use new eepget for i2psnark
      - Add a fake user agent for non-proxied fetches
      - Cleanups

apps/i2psnark/java/src/org/klomp/snark/I2PSnarkUtil.java

@@ -17,6 +17,7 @@ import net.i2p.I2PException;
 import net.i2p.client.I2PSession;
 import net.i2p.client.streaming.I2PServerSocket;
 import net.i2p.client.streaming.I2PSocket;
+import net.i2p.client.streaming.I2PSocketEepGet;
 import net.i2p.client.streaming.I2PSocketManager;
 import net.i2p.client.streaming.I2PSocketManagerFactory;
 import net.i2p.data.DataFormatException;
@@ -231,7 +232,13 @@ public class I2PSnarkUtil {
         if (rewrite)
             fetchURL = rewriteAnnounce(url);
         //_log.debug("Rewritten url [" + fetchURL + "]");
-        EepGet get = new EepGet(_context, _shouldProxy, _proxyHost, _proxyPort, retries, out.getAbsolutePath(), fetchURL);
+        //EepGet get = new EepGet(_context, _shouldProxy, _proxyHost, _proxyPort, retries, out.getAbsolutePath(), fetchURL);
+        // Use our tunnel for announces and .torrent fetches too! Make sure we're connected first...
+        if (!connected()) {
+            if (!connect())
+                return null;
+        }
+        EepGet get = new I2PSocketEepGet(_context, _manager, retries, out.getAbsolutePath(), fetchURL);
         if (get.fetch()) {
             _log.debug("Fetch successful [" + url + "]: size=" + out.length());
             return out;

apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java

@@ -751,10 +751,10 @@ public class I2PSnarkServlet extends HttpServlet {
                   + openTrackers + "\" size=\"50\" /><br>\n");
 
         //out.write("\n");
-        out.write("EepProxy host: <input type=\"text\" name=\"eepHost\" value=\""
+        //out.write("EepProxy host: <input type=\"text\" name=\"eepHost\" value=\""
-                  + _manager.util().getEepProxyHost() + "\" size=\"15\" /> ");
+        //          + _manager.util().getEepProxyHost() + "\" size=\"15\" /> ");
-        out.write("port: <input type=\"text\" name=\"eepPort\" value=\""
+        //out.write("port: <input type=\"text\" name=\"eepPort\" value=\""
-                  + _manager.util().getEepProxyPort() + "\" size=\"5\" maxlength=\"5\" /><br>\n");
+        //          + _manager.util().getEepProxyPort() + "\" size=\"5\" maxlength=\"5\" /><br>\n");
         out.write("I2CP host: <input type=\"text\" name=\"i2cpHost\" value=\"" 
                   + _manager.util().getI2CPHost() + "\" size=\"15\" /> ");
         out.write("port: <input type=\"text\" name=\"i2cpPort\" value=\"" +

apps/i2ptunnel/java/src/net/i2p/i2ptunnel/I2PTunnelHTTPClient.java

@@ -37,7 +37,7 @@ import net.i2p.util.Log;
  * or
  *   $method $path $protocolVersion\nHost: $site
  * or
- *   $method http://i2p/$site/$path $protocolVersion
+ *   $method http://i2p/$b64key/$path $protocolVersion
  * or 
  *   $method /$site/$path $protocolVersion
  * </pre>

apps/ministreaming/java/src/net/i2p/client/streaming/I2PSocketEepGet.java

@@ -0,0 +1,242 @@
+package net.i2p.client.streaming;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.MalformedURLException;
+import java.net.UnknownHostException;
+import java.net.URL;
+import java.util.Properties;
+
+import net.i2p.I2PAppContext;
+import net.i2p.I2PException;
+import net.i2p.data.DataHelper;
+import net.i2p.data.Destination;
+import net.i2p.util.EepGet;
+import net.i2p.util.SocketTimeout;
+
+/**
+ *  Fetch a URL using a socket from the supplied I2PSocketManager.
+ *  Hostname must resolve to an i2p destination - no routing to an outproxy.
+ *  Does not support response gzip decompression (unlike I2PTunnelHTTPProxy) (yet),
+ *  but of course there is still gzip at the I2CP layer.
+ *
+ *  This is designed for Java apps such as bittorrent clients that wish to
+ *  do HTTP fetches and use other protocols on a single set of tunnels.
+ *  This may provide anonymity benefits over using the shared clients HTTP proxy,
+ *  preventing inadvertent outproxy usage, reduce resource usage by eliminating
+ *  a second set of tunnels, and eliminate the requirement to
+ *  to separately configure the proxy host and port.
+ *
+ *  For additional documentation see the superclass.
+ *
+ *  Supports http://example.i2p/blah
+ *  Supports http://B32KEY.b32.i2p/blah
+ *  Supports http://i2p/B64KEY/blah for compatibility with the eepproxy
+ *  Supports http://B64KEY/blah for compatibility with the eepproxy
+ *  Warning - does not support /eepproxy/blah, address helpers, http://B64KEY.i2p/blah,
+ *  or other odd things that may be found in the HTTP proxy.
+ *
+ *  @author zzz
+ */
+public class I2PSocketEepGet extends EepGet {
+    private I2PSocketManager _socketManager;
+    /** this replaces _proxy in the superclass. Sadly, I2PSocket does not extend Socket. */
+    private I2PSocket _socket;
+    
+    public I2PSocketEepGet(I2PAppContext ctx, I2PSocketManager mgr, int numRetries, String outputFile, String url) {
+        this(ctx, mgr, numRetries, -1, -1, outputFile, null, url);
+    }
+
+    public I2PSocketEepGet(I2PAppContext ctx, I2PSocketManager mgr, int numRetries, long minSize, long maxSize,
+                           String outputFile, OutputStream outputStream, String url) {
+        // we're using this constructor:
+        // public EepGet(I2PAppContext ctx, boolean shouldProxy, String proxyHost, int proxyPort, int numRetries, long minSize, long maxSize, String outputFile, OutputStream outputStream, String url, boolean allowCaching, String etag, String postData) {
+        super(ctx, false, null, -1, numRetries, minSize, maxSize, outputFile, outputStream, url, true, null, null);
+        _socketManager = mgr;
+        _log = ctx.logManager().getLog(I2PSocketEepGet.class);
+    }
+   
+    /**
+     *  We have to override this to close _socket, since we can't use _proxy in super as the I2PSocket.
+     */
+    @Override
+    public boolean fetch(long fetchHeaderTimeout, long totalTimeout, long inactivityTimeout) {
+        boolean rv = super.fetch(fetchHeaderTimeout, totalTimeout, inactivityTimeout);
+        if (_socket != null) {
+            try { 
+                _socket.close(); 
+                _socket = null;
+            } catch (IOException ioe) {}
+        }
+        return rv;
+    }
+
+    /**
+     *  Look up the address, get a socket from the I2PSocketManager supplied in the constructor,
+     *  and send the request.
+     *
+     *  @param timeout ignored 
+     */
+    @Override
+    protected void sendRequest(SocketTimeout timeout) throws IOException {
+        if (_outputStream == null) {
+            File outFile = new File(_outputFile);
+            if (outFile.exists())
+                _alreadyTransferred = outFile.length();
+        }
+
+        if (_proxyIn != null) try { _proxyIn.close(); } catch (IOException ioe) {}
+        if (_proxyOut != null) try { _proxyOut.close(); } catch (IOException ioe) {}
+        if (_socket != null) try { _socket.close(); } catch (IOException ioe) {}
+
+        try {
+            URL url = new URL(_actualURL);
+            if ("http".equals(url.getProtocol())) {
+                String host = url.getHost();
+                int port = url.getPort();
+                if (port != -1)
+                    throw new IOException("Ports not supported in i2p: " + _actualURL);
+
+                // HTTP Proxy compatibility http://i2p/B64KEY/blah
+                // Rewrite the url to strip out the /i2p/,
+                // as the naming service accepts B64KEY (but not B64KEY.i2p atm)
+                if ("i2p".equals(host)) {
+                    String file = url.getFile();
+                    try {
+                        int slash = 1 + file.substring(1).indexOf("/");
+                        host = file.substring(1, slash);
+                        _actualURL = "http:/" + file.substring(slash);  // get the extra slash from the substring
+                    } catch (IndexOutOfBoundsException ioobe) {
+                        throw new IOException("Bad /i2p/ format: " + _actualURL);
+                    }
+                }
+
+                Destination dest = _context.namingService().lookup(host);
+                if (dest == null)
+                    throw new UnknownHostException("Unknown or non-i2p host");
+
+                // Set the timeouts, using the other existing options in the socket manager
+                // This currently duplicates what SocketTimeout is doing in EepGet,
+                // but when that's ripped out of EepGet to use setsotimeout, we'll need this.
+                Properties props = new Properties();
+                props.setProperty(I2PSocketOptions.PROP_CONNECT_TIMEOUT, "" + CONNECT_TIMEOUT);
+                props.setProperty(I2PSocketOptions.PROP_READ_TIMEOUT, "" + INACTIVITY_TIMEOUT);
+                I2PSocketOptions opts = _socketManager.buildOptions(props);
+                _socket = _socketManager.connect(dest, opts);
+            } else {
+                throw new IOException("Unsupported protocol: " + _actualURL);
+            }
+        } catch (MalformedURLException mue) {
+            throw new IOException("Request URL is invalid: " + _actualURL);
+        } catch (I2PException ie) {
+            throw new IOException(ie.toString());
+        }
+
+        _proxyIn = _socket.getInputStream();
+        _proxyOut = _socket.getOutputStream();
+        
+        // SocketTimeout doesn't take an I2PSocket, but no matter, because we
+        // always close our socket in fetch() above.
+        //timeout.setSocket(_socket);
+        
+        String req = getRequest();
+        _proxyOut.write(DataHelper.getUTF8(req));
+        _proxyOut.flush();
+    }
+
+    /**
+     *  Guess we have to override this since
+     *  super doesn't strip the http://host from the GET line
+     *  which hoses some servers (opentracker)
+     *  HTTP proxy was kind enough to do this for us
+     */
+    @Override
+    protected String getRequest() throws IOException {
+        StringBuilder buf = new StringBuilder(2048);
+        URL url = new URL(_actualURL);
+        String host = url.getHost();
+        String path = url.getPath();
+        String query = url.getQuery();
+        if (query != null)
+            path = path + '?' + query;
+        if (!path.startsWith("/"))
+	    path = '/' + path;
+        buf.append("GET ").append(path).append(" HTTP/1.1\r\n" +
+                   "Host: ").append(url.getHost()).append("\r\n");
+        if (_alreadyTransferred > 0) {
+            buf.append("Range: bytes=");
+            buf.append(_alreadyTransferred);
+            buf.append("-\r\n");
+        }
+        buf.append("Accept-Encoding: \r\n" +
+                   "Cache-control: no-cache\r\n" +
+                   "Pragma: no-cache\r\n" +
+                   "User-Agent: " + USER_AGENT + "\r\n" +
+                   "Connection: close\r\n\r\n");
+        return buf.toString();
+    }
+
+    /**
+     * I2PSocketEepGet [-n #retries] [-t timeout] url
+     * Uses I2CP at localhost:7654 with a single 1-hop tunnel each direction.
+     * Tunnel build time not included in the timeout.
+     *
+     * This is just for testing, it will be commented out someday.
+     * Real command line apps should use EepGet.main(),
+     * which has more options, and you don't have to wait for tunnels to be built.
+     */ 
+    public static void main(String args[]) {
+        int numRetries = 0;
+        long inactivityTimeout = INACTIVITY_TIMEOUT;
+        String url = null;
+        try {
+            for (int i = 0; i < args.length; i++) {
+                if (args[i].equals("-n")) {
+                    numRetries = Integer.parseInt(args[i+1]);
+                    i++;
+                } else if (args[i].equals("-t")) {
+                    inactivityTimeout = 1000 * Integer.parseInt(args[i+1]);
+                    i++;
+                } else if (args[i].startsWith("-")) {
+                    usage();
+                    return;
+                } else {
+                    url = args[i];
+                }
+            }
+        } catch (Exception e) {
+            e.printStackTrace();
+            usage();
+            return;
+        }
+        
+        if (url == null) {
+            usage();
+            return;
+        }
+
+        Properties opts = new Properties();
+        opts.setProperty("i2cp.dontPublishLeaseSet", "true");
+        opts.setProperty("inbound.quantity", "1");
+        opts.setProperty("outbound.quantity", "1");
+        opts.setProperty("inbound.length", "1");
+        opts.setProperty("outbound.length", "1");
+        opts.setProperty("inbound.nickname", "I2PSocketEepGet");
+        I2PSocketManager mgr = I2PSocketManagerFactory.createManager(opts);
+        if (mgr == null) {
+            System.err.println("Error creating the socket manager");
+            return;
+        }
+        I2PSocketEepGet get = new I2PSocketEepGet(I2PAppContext.getGlobalContext(),
+                                                  mgr, numRetries, suggestName(url), url);
+        get.addStatusListener(get.new CLIStatusListener(1024, 40));
+        get.fetch(inactivityTimeout, -1, inactivityTimeout);
+        mgr.destroySocketManager();
+    }
+    
+    private static void usage() {
+        System.err.println("I2PSocketEepGet [-n #retries] [-t timeout] url");
+    }
+}

apps/streaming/java/src/net/i2p/client/streaming/ConnectionOptions.java

@@ -266,6 +266,7 @@ public class ConnectionOptions extends I2PSocketOptionsImpl {
         if (opts.contains(PROP_SLOW_START_GROWTH_RATE_FACTOR))
             setSlowStartGrowthRateFactor(getInt(opts, PROP_SLOW_START_GROWTH_RATE_FACTOR, 2));
         if (opts.containsKey(PROP_CONNECT_TIMEOUT))
+            // wow 5 minutes!!! FIXME!!
             setConnectTimeout(getInt(opts, PROP_CONNECT_TIMEOUT, Connection.DISCONNECT_TIMEOUT));
         if (opts.containsKey(PROP_ANSWER_PINGS))
             setAnswerPings(getBool(opts, PROP_ANSWER_PINGS, DEFAULT_ANSWER_PINGS));

core/java/src/net/i2p/util/EepGet.java

@@ -27,7 +27,7 @@ import net.i2p.data.DataHelper;
  * Bug: a malformed url http://example.i2p (no trailing '/') fails cryptically
  */
 public class EepGet {
-    private I2PAppContext _context;
+    protected I2PAppContext _context;
     protected Log _log;
     protected boolean _shouldProxy;
     private String _proxyHost;
@@ -35,8 +35,8 @@ public class EepGet {
     protected int _numRetries;
     private long _minSize; // minimum and maximum acceptable response size, -1 signifies unlimited,
     private long _maxSize; // applied both against whole responses and chunks
-    private String _outputFile;
+    protected String _outputFile;
-    private OutputStream _outputStream;
+    protected OutputStream _outputStream;
     /** url we were asked to fetch */
     protected String _url;
     /** the URL we actually fetch from (may differ from the _url in case of redirect) */
@@ -47,10 +47,10 @@ public class EepGet {
     
     private boolean _keepFetching;
     private Socket _proxy;
-    private OutputStream _proxyOut;
+    protected OutputStream _proxyOut;
-    private InputStream _proxyIn;
+    protected InputStream _proxyIn;
     protected OutputStream _out;
-    private long _alreadyTransferred;
+    protected long _alreadyTransferred;
     private long _bytesTransferred;
     protected long _bytesRemaining;
     protected int _currentAttempt;
@@ -67,6 +67,10 @@ public class EepGet {
     protected long _fetchInactivityTimeout;
     protected int _redirects;
     protected String _redirectLocation;
+    /** this will be replaced by the HTTP Proxy if we are using it */
+    protected static final String USER_AGENT = "Wget/1.11.4";
+    protected static final long CONNECT_TIMEOUT = 45*1000;
+    protected static final long INACTIVITY_TIMEOUT = 60*1000;
     
     public EepGet(I2PAppContext ctx, String proxyHost, int proxyPort, int numRetries, String outputFile, String url) {
         this(ctx, true, proxyHost, proxyPort, numRetries, outputFile, url);
@@ -118,7 +122,7 @@ public class EepGet {
         _transferFailed = false;
         _headersRead = false;
         _aborted = false;
-        _fetchHeaderTimeout = 45*1000;
+        _fetchHeaderTimeout = CONNECT_TIMEOUT;
         _listeners = new ArrayList(1);
         _etag = etag;
         _lastModified = lastModified;
@@ -134,7 +138,7 @@ public class EepGet {
         int numRetries = 5;
         int markSize = 1024;
         int lineLen = 40;
-        int inactivityTimeout = 60*1000;
+        long inactivityTimeout = INACTIVITY_TIMEOUT;
         String etag = null;
         String saveAs = null;
         String url = null;
@@ -183,7 +187,7 @@ public class EepGet {
 
         EepGet get = new EepGet(I2PAppContext.getGlobalContext(), true, proxyHost, proxyPort, numRetries, saveAs, url, true, etag);
         get.addStatusListener(get.new CLIStatusListener(markSize, lineLen));
-        get.fetch(45*1000, -1, inactivityTimeout);
+        get.fetch(CONNECT_TIMEOUT, -1, inactivityTimeout);
     }
     
     public static String suggestName(String url) {
@@ -216,7 +220,7 @@ public class EepGet {
         return buf.toString();
     }
     
-    protected static void usage() {
+    private static void usage() {
         System.err.println("EepGet [-p 127.0.0.1:4444] [-n #retries] [-o outputFile] [-m markSize lineLen] [-t timeout] url");
     }
     
@@ -247,7 +251,7 @@ public class EepGet {
         public void headerReceived(String url, int currentAttempt, String key, String val);
         public void attempting(String url);
     }
-    private class CLIStatusListener implements StatusListener {
+    protected class CLIStatusListener implements StatusListener {
         private int _markSize;
         private int _lineSize;
         private long _startedOn;
@@ -497,7 +501,7 @@ public class EepGet {
         if (_fetchInactivityTimeout > 0)
             timeout.setInactivityTimeout(_fetchInactivityTimeout);
         else
-            timeout.setInactivityTimeout(60*1000);
+            timeout.setInactivityTimeout(INACTIVITY_TIMEOUT);
         
         if (_redirectLocation != null) {
             try {
@@ -829,12 +833,12 @@ public class EepGet {
         }
     }
 
-    private void increment(byte[] lookahead, int cur) {
+    private static void increment(byte[] lookahead, int cur) {
         lookahead[0] = lookahead[1];
         lookahead[1] = lookahead[2];
         lookahead[2] = (byte)cur;
     }
-    private boolean isEndOfHeaders(byte lookahead[]) {
+    private static boolean isEndOfHeaders(byte lookahead[]) {
         byte first = lookahead[0];
         byte second = lookahead[1];
         byte third = lookahead[2];
@@ -844,7 +848,7 @@ public class EepGet {
 
     /** we ignore any potential \r, since we trim it on write anyway */
     private static final byte NL = '\n';
-    private boolean isNL(byte b) { return (b == NL); }
+    private static boolean isNL(byte b) { return (b == NL); }
 
     protected void sendRequest(SocketTimeout timeout) throws IOException {
         if (_outputStream != null) {
@@ -895,7 +899,7 @@ public class EepGet {
     }
     
     protected String getRequest() throws IOException {
-        StringBuilder buf = new StringBuilder(512);
+        StringBuilder buf = new StringBuilder(2048);
         boolean post = false;
         if ( (_postData != null) && (_postData.length() > 0) )
             post = true;
@@ -906,7 +910,7 @@ public class EepGet {
         String path = url.getPath();
         String query = url.getQuery();
         if (query != null)
-            path = path + "?" + query;
+            path = path + '?' + query;
         if (!path.startsWith("/"))
 	    path = "/" + path;
         if ( (port == 80) || (port == 443) || (port <= 0) ) path = proto + "://" + host + path;
@@ -923,12 +927,11 @@ public class EepGet {
             buf.append(_alreadyTransferred);
             buf.append("-\r\n");
         }
-        buf.append("Accept-Encoding: \r\n");
         if (_shouldProxy)
             buf.append("X-Accept-Encoding: x-i2p-gzip;q=1.0, identity;q=0.5, deflate;q=0, gzip;q=0, *;q=0\r\n");
         if (!_allowCaching) {
-            buf.append("Cache-control: no-cache\r\n");
+            buf.append("Cache-control: no-cache\r\n" +
-            buf.append("Pragma: no-cache\r\n");
+                       "Pragma: no-cache\r\n");
         }
         if ((_etag != null) && (_alreadyTransferred <= 0)) {
             buf.append("If-None-Match: ");
@@ -942,7 +945,10 @@ public class EepGet {
         }
         if (post)
             buf.append("Content-length: ").append(_postData.length()).append("\r\n");
-        buf.append("Connection: close\r\n\r\n");
+        // This will be replaced if we are going through I2PTunnelHTTPClient
+        buf.append("User-Agent: " + USER_AGENT + "\r\n" +
+                   "Accept-Encoding: \r\n" +
+                   "Connection: close\r\n\r\n");
         if (post)
             buf.append(_postData);
         if (_log.shouldLog(Log.DEBUG))

core/java/src/net/i2p/util/EepGetScheduler.java

@@ -7,7 +7,7 @@ import java.util.List;
 import net.i2p.I2PAppContext;
 
 /**
- *
+ *  @deprecated unused a webapp version would be nice though
  */
 public class EepGetScheduler implements EepGet.StatusListener {
     private I2PAppContext _context;

core/java/src/net/i2p/util/EepHead.java

@@ -93,7 +93,7 @@ public class EepHead extends EepGet {
         }
     }
     
-    protected static void usage() {
+    private static void usage() {
         System.err.println("EepHead [-p 127.0.0.1:4444] [-n #retries] [-t timeout] url");
     }
     
@@ -191,6 +191,8 @@ public class EepHead extends EepGet {
         buf.append("Accept-Encoding: \r\n");
         if (_shouldProxy)
             buf.append("X-Accept-Encoding: x-i2p-gzip;q=1.0, identity;q=0.5, deflate;q=0, gzip;q=0, *;q=0\r\n");
+        // This will be replaced if we are going through I2PTunnelHTTPClient
+        buf.append("User-Agent: " + USER_AGENT + "\r\n");
         buf.append("Connection: close\r\n\r\n");
         if (_log.shouldLog(Log.DEBUG))
             _log.debug("Request: [" + buf.toString() + "]");

core/java/src/net/i2p/util/SocketTimeout.java

@@ -5,6 +5,15 @@ import java.net.Socket;
 import java.text.SimpleDateFormat;
 import java.util.Date;
 
+/**
+ *  This should be deprecated.
+ *  It is only used by EepGet, and it uses the inefficient SimpleTimer.
+ *  The only advantage seems to be a total timeout period, which is the second
+ *  argument to EepGet.fetch(headerTimeout, totalTimeout, inactivityTimeout),
+ *  which is most likely always set to -1.
+ *
+ *  Use socket.setsotimeout instead?
+ */
 public class SocketTimeout implements SimpleTimer.TimedEvent {
     private Socket _targetSocket;
     private long _startTime;