I2P_Developers/i2p.itoopie
3
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

2005-09-16 jrandom

Browse Source 
* Added the i2p.exe and i2pinstall.exe for windows users, using launch4j.
    * Added runplain.sh for *nix/osx users having problems using the java
      service wrapper (called from the install dir as: sh runplain.sh)
    * Bundle susidns and syndie, with links on the top nav
    * Have I2PTunnelHTTPClient and I2PTunnelHTTPServer use the x-i2p-gzip
      content-encoding (if offered), reducing the payload size before it
      reaches the streaming lib.  The existing compression is at the i2cp
      level, so we've been packetizing 4KB of uncompressed data and then
      compressing those messages, rather than compressing and then packetizing
      4KB of compressed data.  This should reduce the number of round trips
      to fetch web pages substantially.
    * Adjust the startup and timing of the addressbook so that susidns always
      has config to work off, and expose a method for susidns to tell it to
      reload its config and rerun.
This commit is contained in:
jrandom
2005-09-16 04:12:24 +00:00
committed by zzz
parent 8c1895e04f
commit d89f589f2b
17 changed files with 396 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files

107
apps/i2ptunnel/java/src/net/i2p/i2ptunnel/HTTPResponseOutputStream.java
View File

@ -8,36 +8,49 @@ package net.i2p.i2ptunnel;
*
*/
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.FilterOutputStream;
import java.io.OutputStream;
import java.io.*;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.zip.GZIPInputStream;
import net.i2p.I2PAppContext;
import net.i2p.data.ByteArray;
import net.i2p.util.ByteCache;
import net.i2p.util.I2PThread;
import net.i2p.util.Log;
/**
* Simple stream for delivering an HTTP response to
* the client, trivially filtered to make sure "Connection: close"
* is always in the response.
* is always in the response. Perhaps add transparent handling of the
* Content-encoding: x-i2p-gzip, adjusting the headers to say Content-encoding: identity?
* Content-encoding: gzip is trivial as well, but Transfer-encoding: chunked makes it
* more work than is worthwhile at the moment.
*
*/
class HTTPResponseOutputStream extends FilterOutputStream {
private static final Log _log = new Log(HTTPResponseOutputStream.class);
private I2PAppContext _context;
private Log _log;
private ByteCache _cache;
protected ByteArray _headerBuffer;
private boolean _headerWritten;
private byte _buf1[];
protected boolean _gzip;
private long _dataWritten;
private static final int CACHE_SIZE = 8*1024;
public HTTPResponseOutputStream(OutputStream raw) {
super(raw);
_context = I2PAppContext.getGlobalContext();
_context.statManager().createRateStat("i2ptunnel.http.compressionRatio", "ratio of compressed size to decompressed size after transfer", "i2ptunnel", new long[] { 60*1000, 30*60*1000 });
_context.statManager().createRateStat("i2ptunnel.http.compressed", "compressed size transferred", "i2ptunnel", new long[] { 60*1000, 30*60*1000 });
_context.statManager().createRateStat("i2ptunnel.http.expanded", "size transferred after expansion", "i2ptunnel", new long[] { 60*1000, 30*60*1000 });
_log = _context.logManager().getLog(getClass());
_cache = ByteCache.getInstance(8, CACHE_SIZE);
_headerBuffer = _cache.acquire();
_headerWritten = false;
_gzip = false;
_dataWritten = 0;
_buf1 = new byte[1];
}
@ -51,6 +64,7 @@ class HTTPResponseOutputStream extends FilterOutputStream {
public void write(byte buf[], int off, int len) throws IOException {
if (_headerWritten) {
out.write(buf, off, len);
_dataWritten += len;
return;
}
@ -62,8 +76,11 @@ class HTTPResponseOutputStream extends FilterOutputStream {
if (headerReceived()) {
writeHeader();
_headerWritten = true;
if (i + 1 < len) // write out the remaining
if (i + 1 < len) {
// write out the remaining
out.write(buf, off+i+1, len-i-1);
_dataWritten += len-i-1;
}
return;
}
}
@ -128,7 +145,10 @@ class HTTPResponseOutputStream extends FilterOutputStream {
if ( (keyLen <= 0) || (valLen <= 0) )
throw new IOException("Invalid header @ " + j);
String key = new String(_headerBuffer.getData(), lastEnd+1, keyLen);
String val = new String(_headerBuffer.getData(), j+2, valLen);
String val = new String(_headerBuffer.getData(), j+2, valLen).trim();
if (_log.shouldLog(Log.INFO))
_log.info("Response header [" + key + "] = [" + val + "]");
if ("Connection".equalsIgnoreCase(key)) {
out.write("Connection: close\n".getBytes());
@ -136,6 +156,8 @@ class HTTPResponseOutputStream extends FilterOutputStream {
} else if ("Proxy-Connection".equalsIgnoreCase(key)) {
out.write("Proxy-Connection: close\n".getBytes());
proxyConnectionSent = true;
} else if ( ("Content-encoding".equalsIgnoreCase(key)) && ("x-i2p-gzip".equalsIgnoreCase(val)) ) {
_gzip = true;
} else {
out.write((key.trim() + ": " + val.trim() + "\n").getBytes());
}
@ -152,13 +174,80 @@ class HTTPResponseOutputStream extends FilterOutputStream {
if (!proxyConnectionSent)
out.write("Proxy-Connection: close\n".getBytes());
out.write("\n".getBytes()); // end of the headers
finishHeaders();
// done, shove off
if (_headerBuffer.getData().length == CACHE_SIZE)
_cache.release(_headerBuffer);
else
_headerBuffer = null;
if (_log.shouldLog(Log.INFO))
_log.info("After headers: gzip? " + _gzip);
if (shouldCompress()) {
beginProcessing();
}
}
protected boolean shouldCompress() { return _gzip; }
protected void finishHeaders() throws IOException {
out.write("\n".getBytes()); // end of the headers
}
protected void beginProcessing() throws IOException {
out.flush();
PipedInputStream pi = new PipedInputStream();
PipedOutputStream po = new PipedOutputStream(pi);
new I2PThread(new Pusher(pi, out), "HTTP decompresser").start();
out = po;
}
private class Pusher implements Runnable {
private InputStream _in;
private OutputStream _out;
public Pusher(InputStream in, OutputStream out) {
_in = in;
_out = out;
}
public void run() {
OutputStream to = null;
InternalGZIPInputStream in = null;
try {
long start = System.currentTimeMillis();
in = new InternalGZIPInputStream(_in);
byte buf[] = new byte[8192];
int read = -1;
while ( (read = in.read(buf)) != -1) {
if (_log.shouldLog(Log.INFO))
_log.info("Read " + read + " and writing it to the browser/streams");
_out.write(buf, 0, read);
}
if (_log.shouldLog(Log.INFO))
_log.info("Decompressed: " + _dataWritten + ", " + in.getTotalRead() + "/" + in.getTotalExpanded());
long end = System.currentTimeMillis();
long compressed = in.getTotalRead();
long expanded = in.getTotalExpanded();
double ratio = 0;
if (expanded > 0)
ratio = compressed/expanded;
_context.statManager().addRateData("i2ptunnel.http.compressionRatio", (int)(100d*ratio), end-start);
_context.statManager().addRateData("i2ptunnel.http.compressed", compressed, end-start);
_context.statManager().addRateData("i2ptunnel.http.expanded", expanded, end-start);
} catch (IOException ioe) {
if (_log.shouldLog(Log.WARN))
_log.warn("Error decompressing: " + _dataWritten + ", " + in.getTotalRead() + "/" + in.getTotalExpanded(), ioe);
} finally {
if (_out != null) try { _out.close(); } catch (IOException ioe) {}
}
}
}
private class InternalGZIPInputStream extends GZIPInputStream {
public InternalGZIPInputStream(InputStream in) throws IOException {
super(in);
}
public long getTotalRead() { return super.inf.getTotalIn(); }
public long getTotalExpanded() { return super.inf.getTotalOut(); }
}
public static void main(String args[]) {

7
apps/i2ptunnel/java/src/net/i2p/i2ptunnel/I2PTunnelHTTPClient.java
View File

@ -369,6 +369,13 @@ public class I2PTunnelHTTPClient extends I2PTunnelClientBase implements Runnable
}
if (line.length() == 0) {
String ok = getTunnel().getContext().getProperty("i2ptunnel.gzip");
boolean gzip = true;
if (ok != null)
gzip = Boolean.valueOf(ok).booleanValue();
if (gzip)
newRequest.append("Accept-Encoding: x-i2p-gzip\r\n");
newRequest.append("User-Agent: MYOB/6.66 (AN/ON)\r\n");
newRequest.append("Connection: close\r\n\r\n");
break;

146
apps/i2ptunnel/java/src/net/i2p/i2ptunnel/I2PTunnelHTTPServer.java
View File

@ -3,14 +3,13 @@
*/
package net.i2p.i2ptunnel;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.*;
import java.net.InetAddress;
import java.net.Socket;
import java.net.SocketException;
import java.util.Iterator;
import java.util.Properties;
import java.util.zip.GZIPOutputStream;
import net.i2p.I2PAppContext;
import net.i2p.I2PException;
@ -24,7 +23,9 @@ import net.i2p.util.Log;
/**
* Simple extension to the I2PTunnelServer that filters the HTTP
* headers sent from the client to the server, replacing the Host
* header with whatever this instance has been configured with.
* header with whatever this instance has been configured with, and
* if the browser set Accept-encoding: x-i2p-gzip, gzip the http
* message body and set Content-encoding: x-i2p-gzip.
*
*/
public class I2PTunnelHTTPServer extends I2PTunnelServer {
@ -62,14 +63,35 @@ public class I2PTunnelHTTPServer extends I2PTunnelServer {
try {
// give them 5 seconds to send in the HTTP request
socket.setReadTimeout(5*1000);
String modifiedHeader = getModifiedHeader(socket);
InputStream in = socket.getInputStream();
StringBuffer command = new StringBuffer(128);
Properties headers = readHeaders(in, command);
headers.setProperty("Host", _spoofHost);
headers.setProperty("Connection", "close");
String modifiedHeader = formatHeaders(headers, command);
//String modifiedHeader = getModifiedHeader(socket);
if (_log.shouldLog(Log.DEBUG))
_log.debug("Modified header: [" + modifiedHeader + "]");
socket.setReadTimeout(readTimeout);
Socket s = new Socket(remoteHost, remotePort);
afterSocket = getTunnel().getContext().clock().now();
new I2PTunnelRunner(s, socket, slock, null, modifiedHeader.getBytes(), null);
// instead of i2ptunnelrunner, use something that reads the HTTP
// request from the socket, modifies the headers, sends the request to the
// server, reads the response headers, rewriting to include Content-encoding: x-i2p-gzip
// if it was one of the Accept-encoding: values, and gzip the payload
String enc = headers.getProperty("Accept-encoding");
if (_log.shouldLog(Log.INFO))
_log.info("HTTP server encoding header: " + enc);
if ( (enc != null) && (enc.indexOf("x-i2p-gzip") >= 0) ) {
I2PThread req = new I2PThread(new CompressedRequestor(s, socket, modifiedHeader), "http compressor");
req.start();
} else {
new I2PTunnelRunner(s, socket, slock, null, modifiedHeader.getBytes(), null);
}
} catch (SocketException ex) {
try {
socket.close();
@ -88,17 +110,107 @@ public class I2PTunnelHTTPServer extends I2PTunnelServer {
if ( (timeToHandle > 1000) && (_log.shouldLog(Log.WARN)) )
_log.warn("Took a while to handle the request [" + timeToHandle + ", socket create: " + (afterSocket-afterAccept) + "]");
}
private String getModifiedHeader(I2PSocket handleSocket) throws IOException {
InputStream in = handleSocket.getInputStream();
StringBuffer command = new StringBuffer(128);
Properties headers = readHeaders(in, command);
headers.setProperty("Host", _spoofHost);
headers.setProperty("Connection", "close");
return formatHeaders(headers, command);
}
private class CompressedRequestor implements Runnable {
private Socket _webserver;
private I2PSocket _browser;
private String _headers;
public CompressedRequestor(Socket webserver, I2PSocket browser, String headers) {
_webserver = webserver;
_browser = browser;
_headers = headers;
}
public void run() {
if (_log.shouldLog(Log.INFO))
_log.info("Compressed requestor running");
OutputStream serverout = null;
OutputStream browserout = null;
InputStream browserin = null;
InputStream serverin = null;
try {
serverout = _webserver.getOutputStream();
if (_log.shouldLog(Log.INFO))
_log.info("request headers: " + _headers);
serverout.write(_headers.getBytes());
browserin = _browser.getInputStream();
I2PThread sender = new I2PThread(new Sender(serverout, browserin, "server: browser to server"), "http compressed sender");
sender.start();
browserout = _browser.getOutputStream();
serverin = _webserver.getInputStream();
Sender s = new Sender(new CompressedResponseOutputStream(browserout), serverin, "server: server to browser");
if (_log.shouldLog(Log.INFO))
_log.info("Before pumping the compressed response");
s.run(); // same thread
if (_log.shouldLog(Log.INFO))
_log.info("After pumping the compressed response");
} catch (IOException ioe) {
if (_log.shouldLog(Log.WARN))
_log.warn("error compressing", ioe);
} finally {
if (browserout != null) try { browserout.close(); } catch (IOException ioe) {}
if (serverout != null) try { serverout.close(); } catch (IOException ioe) {}
if (browserin != null) try { browserin.close(); } catch (IOException ioe) {}
if (serverin != null) try { serverin.close(); } catch (IOException ioe) {}
}
}
}
private class Sender implements Runnable {
private OutputStream _out;
private InputStream _in;
private String _name;
public Sender(OutputStream out, InputStream in, String name) {
_out = out;
_in = in;
_name = name;
}
public void run() {
if (_log.shouldLog(Log.INFO))
_log.info(_name + ": Begin sending");
try {
byte buf[] = new byte[4096];
int read = 0;
int total = 0;
while ( (read = _in.read(buf)) != -1) {
if (_log.shouldLog(Log.INFO))
_log.info(_name + ": read " + read + " and sending through the stream");
_out.write(buf, 0, read);
total += read;
}
if (_log.shouldLog(Log.INFO))
_log.info(_name + ": Done sending: " + total);
_out.flush();
} catch (IOException ioe) {
if (_log.shouldLog(Log.WARN))
_log.warn("Error sending", ioe);
} finally {
if (_in != null) try { _in.close(); } catch (IOException ioe) {}
if (_out != null) try { _out.close(); } catch (IOException ioe) {}
}
}
}
private class CompressedResponseOutputStream extends HTTPResponseOutputStream {
public CompressedResponseOutputStream(OutputStream o) {
super(o);
}
protected boolean shouldCompress() { return true; }
protected void finishHeaders() throws IOException {
if (_log.shouldLog(Log.INFO))
_log.info("Including x-i2p-gzip as the content encoding in the response");
out.write("Content-encoding: x-i2p-gzip\n".getBytes());
super.finishHeaders();
}
protected void beginProcessing() throws IOException {
if (_log.shouldLog(Log.INFO))
_log.info("Beginning compression processing");
out.flush();
out = new GZIPOutputStream(out);
}
}
private String formatHeaders(Properties headers, StringBuffer command) {
StringBuffer buf = new StringBuffer(command.length() + headers.size() * 64);
buf.append(command.toString()).append('\n');
@ -133,6 +245,8 @@ public class I2PTunnelHTTPServer extends I2PTunnelServer {
if (split <= 0) throw new IOException("Invalid HTTP header, missing colon [" + buf.toString() + "]");
String name = buf.substring(0, split);
String value = buf.substring(split+2); // ": "
if ("Accept-encoding".equalsIgnoreCase(name))
name = "Accept-encoding";
headers.setProperty(name, value);
if (_log.shouldLog(Log.DEBUG))
_log.debug("Read the header [" + name + "] = [" + value + "]");