I2P_Developers/i2p.itoopie
3
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

EepHead

Browse Source
This commit is contained in:
zzz
2009-08-08 14:29:18 +00:00
parent 2436ea1131
commit bdcad06ece
2 changed files with 226 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
core/java/src/net/i2p/util/EepGet.java
View File

@ -23,48 +23,50 @@ import net.i2p.data.DataHelper;
* [-o outputFile]
* [-m markSize lineLen]
* url
*
* Bug: a malformed url http://example.i2p (no trailing '/') fails cryptically
*/
public class EepGet {
private I2PAppContext _context;
private Log _log;
private boolean _shouldProxy;
protected Log _log;
protected boolean _shouldProxy;
private String _proxyHost;
private int _proxyPort;
private int _numRetries;
protected int _numRetries;
private long _minSize; // minimum and maximum acceptable response size, -1 signifies unlimited,
private long _maxSize; // applied both against whole responses and chunks
private String _outputFile;
private OutputStream _outputStream;
/** url we were asked to fetch */
private String _url;
protected String _url;
/** the URL we actually fetch from (may differ from the _url in case of redirect) */
private String _actualURL;
protected String _actualURL;
private String _postData;
private boolean _allowCaching;
private List _listeners;
protected List _listeners;
private boolean _keepFetching;
private Socket _proxy;
private OutputStream _proxyOut;
private InputStream _proxyIn;
private OutputStream _out;
protected OutputStream _out;
private long _alreadyTransferred;
private long _bytesTransferred;
private long _bytesRemaining;
private int _currentAttempt;
protected long _bytesRemaining;
protected int _currentAttempt;
private String _etag;
private String _lastModified;
private boolean _encodingChunked;
private boolean _notModified;
private String _contentType;
private boolean _transferFailed;
private boolean _headersRead;
private boolean _aborted;
protected boolean _transferFailed;
protected boolean _headersRead;
protected boolean _aborted;
private long _fetchHeaderTimeout;
private long _fetchEndTime;
private long _fetchInactivityTimeout;
private int _redirects;
private String _redirectLocation;
protected long _fetchInactivityTimeout;
protected int _redirects;
protected String _redirectLocation;
public EepGet(I2PAppContext ctx, String proxyHost, int proxyPort, int numRetries, String outputFile, String url) {
this(ctx, true, proxyHost, proxyPort, numRetries, outputFile, url);
@ -214,7 +216,7 @@ public class EepGet {
return buf.toString();
}
private static void usage() {
protected static void usage() {
System.err.println("EepGet [-p 127.0.0.1:4444] [-n #retries] [-o outputFile] [-m markSize lineLen] [-t timeout] url");
}
@ -480,7 +482,7 @@ public class EepGet {
}
/** return true if the URL was completely retrieved */
private void doFetch(SocketTimeout timeout) throws IOException {
protected void doFetch(SocketTimeout timeout) throws IOException {
_headersRead = false;
_aborted = false;
try {
@ -625,7 +627,7 @@ public class EepGet {
}
}
private void readHeaders() throws IOException {
protected void readHeaders() throws IOException {
String key = null;
StringBuilder buf = new StringBuilder(32);
@ -844,7 +846,7 @@ public class EepGet {
private static final byte NL = '\n';
private boolean isNL(byte b) { return (b == NL); }
private void sendRequest(SocketTimeout timeout) throws IOException {
protected void sendRequest(SocketTimeout timeout) throws IOException {
if (_outputStream != null) {
// We are reading into a stream supplied by a caller,
// for which we cannot easily determine how much we've written.
@ -892,7 +894,7 @@ public class EepGet {
_log.debug("Request flushed");
}
private String getRequest() throws IOException {
protected String getRequest() throws IOException {
StringBuilder buf = new StringBuilder(512);
boolean post = false;
if ( (_postData != null) && (_postData.length() > 0) )
@ -963,5 +965,4 @@ public class EepGet {
public String getContentType() {
return _contentType;
}
}

204
core/java/src/net/i2p/util/EepHead.java Normal file
View File

@ -0,0 +1,204 @@
package net.i2p.util;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.net.MalformedURLException;
import java.net.URL;
import net.i2p.I2PAppContext;
/**
* This is a quick hack to get a working EepHead, primarily for the following usage:
*
* EepHead foo = new EepHead(...);
* if (foo.fetch()) {
* String lastmod = foo.getLastModified();
* if (lastmod != null) {
* parse the string...
* ...
* }
* }
*
* Other use cases (command line, listeners, etc...) lightly- or un-tested.
*
* Writing from scratch rather than extending EepGet would maybe have been less bloated memory-wise.
* This way gets us redirect handling, among other benefits.
*
* @author zzz
*/
public class EepHead extends EepGet {
/** EepGet needs either a non-null file or a stream... shouldn't actually be written to... */
static final OutputStream _dummyStream = new ByteArrayOutputStream(0);
public EepHead(I2PAppContext ctx, String proxyHost, int proxyPort, int numRetries, String url) {
// we're using this constructor:
// public EepGet(I2PAppContext ctx, boolean shouldProxy, String proxyHost, int proxyPort, int numRetries, long minSize, long maxSize, String outputFile, OutputStream outputStream, String url, boolean allowCaching, String etag, String postData) {
super(ctx, true, proxyHost, proxyPort, numRetries, -1, -1, null, _dummyStream, url, true, null, null);
}
/**
* EepHead [-p 127.0.0.1:4444] [-n #retries] url
*
* This doesn't really do much since it doesn't register a listener.
* EepGet doesn't have a method to store and return all the headers, so just print
* out the ones we have methods for.
* Turn on logging to use it for a decent test.
*/
public static void main(String args[]) {
String proxyHost = "127.0.0.1";
int proxyPort = 4444;
int numRetries = 0;
int inactivityTimeout = 60*1000;
String url = null;
try {
for (int i = 0; i < args.length; i++) {
if (args[i].equals("-p")) {
proxyHost = args[i+1].substring(0, args[i+1].indexOf(':'));
String port = args[i+1].substring(args[i+1].indexOf(':')+1);
proxyPort = Integer.parseInt(port);
i++;
} else if (args[i].equals("-n")) {
numRetries = Integer.parseInt(args[i+1]);
i++;
} else if (args[i].equals("-t")) {
inactivityTimeout = 1000 * Integer.parseInt(args[i+1]);
i++;
} else if (args[i].startsWith("-")) {
usage();
return;
} else {
url = args[i];
}
}
} catch (Exception e) {
e.printStackTrace();
usage();
return;
}
if (url == null) {
usage();
return;
}
EepHead get = new EepHead(I2PAppContext.getGlobalContext(), proxyHost, proxyPort, numRetries, url);
if (get.fetch(45*1000, -1, inactivityTimeout)) {
System.err.println("Content-Type: " + get.getContentType());
System.err.println("Content-Length: " + get.getContentLength());
System.err.println("Last-Modified: " + get.getLastModified());
System.err.println("Etag: " + get.getETag());
} else {
System.err.println("Failed " + url);
}
}
protected static void usage() {
System.err.println("EepHead [-p 127.0.0.1:4444] [-n #retries] [-t timeout] url");
}
/** return true if the URL was completely retrieved */
@Override
protected void doFetch(SocketTimeout timeout) throws IOException {
_headersRead = false;
_aborted = false;
try {
readHeaders();
} finally {
_headersRead = true;
}
if (_aborted)
throw new IOException("Timed out reading the HTTP headers");
timeout.resetTimer();
if (_fetchInactivityTimeout > 0)
timeout.setInactivityTimeout(_fetchInactivityTimeout);
else
timeout.setInactivityTimeout(60*1000);
if (_redirectLocation != null) {
try {
URL oldURL = new URL(_actualURL);
String query = oldURL.getQuery();
if (query == null) query = "";
if (_redirectLocation.startsWith("http://")) {
if ( (_redirectLocation.indexOf('?') < 0) && (query.length() > 0) )
_actualURL = _redirectLocation + "?" + query;
else
_actualURL = _redirectLocation;
} else {
URL url = new URL(_actualURL);
if (_redirectLocation.startsWith("/"))
_actualURL = "http://" + url.getHost() + ":" + url.getPort() + _redirectLocation;
else
_actualURL = "http://" + url.getHost() + ":" + url.getPort() + "/" + _redirectLocation;
if ( (_actualURL.indexOf('?') < 0) && (query.length() > 0) )
_actualURL = _actualURL + "?" + query;
}
} catch (MalformedURLException mue) {
throw new IOException("Redirected from an invalid URL");
}
_redirects++;
if (_redirects > 5)
throw new IOException("Too many redirects: to " + _redirectLocation);
if (_log.shouldLog(Log.INFO)) _log.info("Redirecting to " + _redirectLocation);
sendRequest(timeout);
doFetch(timeout);
return;
}
if (_log.shouldLog(Log.DEBUG))
_log.debug("Headers read completely");
if (_out != null)
_out.close();
_out = null;
if (_aborted)
throw new IOException("Timed out reading the HTTP data");
timeout.cancel();
if (_transferFailed) {
// 404, etc - transferFailed is called after all attempts fail, by fetch() above
for (int i = 0; i < _listeners.size(); i++)
((StatusListener)_listeners.get(i)).attemptFailed(_url, 0, 0, _currentAttempt, _numRetries, new Exception("Attempt failed"));
} else {
for (int i = 0; i < _listeners.size(); i++)
((StatusListener)_listeners.get(i)).transferComplete(
0, 0, 0, _url, "dummy", false);
}
}
@Override
protected String getRequest() throws IOException {
StringBuilder buf = new StringBuilder(512);
URL url = new URL(_actualURL);
String proto = url.getProtocol();
String host = url.getHost();
int port = url.getPort();
String path = url.getPath();
String query = url.getQuery();
if (query != null)
path = path + "?" + query;
if (!path.startsWith("/"))
path = "/" + path;
if ( (port == 80) || (port == 443) || (port <= 0) ) path = proto + "://" + host + path;
else path = proto + "://" + host + ":" + port + path;
if (_log.shouldLog(Log.DEBUG)) _log.debug("Requesting " + path);
buf.append("HEAD ").append(_actualURL).append(" HTTP/1.1\r\n");
buf.append("Host: ").append(url.getHost()).append("\r\n");
buf.append("Accept-Encoding: \r\n");
if (_shouldProxy)
buf.append("X-Accept-Encoding: x-i2p-gzip;q=1.0, identity;q=0.5, deflate;q=0, gzip;q=0, *;q=0\r\n");
buf.append("Connection: close\r\n\r\n");
if (_log.shouldLog(Log.DEBUG))
_log.debug("Request: [" + buf.toString() + "]");
return buf.toString();
}
/** We don't decrement the variable (unlike in EepGet), so this is valid */
public long getContentLength() {
return _bytesRemaining;
}
}