echelon/i2p.i2p
1
0
Fork 0
forked from I2P_Developers/i2p.i2p
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

* Reseeding / NetDb:

Browse Source 
- Move reseeding from the routerconsole app to
        the router, so that we can bootstrap an embedded router lacking a routerconsole
        (iMule or android for example), without additional modifications.
        This allows better integration between the reseeding function
        and the netDb.
      - Call reseed from PersistentDataStore, not from the
        routerconsole init, and start seeding as soon as the netdb has read
        the netDb/ directory, not when the console starts.
      - Wake up the netdb reader as soon as reseeding is done,
        rather than waiting up to 60s.
      - Don't display the reseed button on the console until the
        netdb initialization is done.
    * NetDb:
      - Fix an NPE on early shutdown
    * RouterConsoleRunner:
      - Catch a class not found error better
This commit is contained in:
zzz
2009-06-15 21:58:28 +00:00
parent 71f3cd648f
commit 7aa9949332
10 changed files with 365 additions and 287 deletions
Download Patch File Download Diff File Expand all files Collapse all files

266
apps/routerconsole/java/src/net/i2p/router/web/ReseedHandler.java
View File

@ -1,63 +1,21 @@
package net.i2p.router.web;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.net.URL;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import java.util.StringTokenizer;
import net.i2p.I2PAppContext;
import net.i2p.router.RouterContext;
import net.i2p.util.EepGet;
import net.i2p.util.I2PThread;
import net.i2p.util.Log;
import net.i2p.router.networkdb.reseed.Reseeder;
/**
* Handler to deal with reseed requests. This will reseed from the URL
* http://i2pdb.tin0.de/netDb/ unless the I2P configuration property "i2p.reseedURL" is
* set. It always writes to ./netDb/, so don't mess with that.
*
* Handler to deal with reseed requests.
*/
public class ReseedHandler {
private static ReseedRunner _reseedRunner;
private RouterContext _context;
private Log _log;
// Reject unreasonably big files, because we download into a ByteArrayOutputStream.
private static final long MAX_RESEED_RESPONSE_SIZE = 8 * 1024 * 1024;
private static final String DEFAULT_SEED_URL = "http://i2pdb.tin0.de/netDb/,http://netdb.i2p2.de/";
public class ReseedHandler extends HelperBase {
private static Reseeder _reseedRunner;
public ReseedHandler() {
this(ContextHelper.getContext(null));
}
public ReseedHandler(RouterContext ctx) {
_context = ctx;
_log = ctx.logManager().getLog(ReseedHandler.class);
}
/**
* Configure this bean to query a particular router context
*
* @param contextId begging few characters of the routerHash, or null to pick
* the first one we come across.
*/
public void setContextId(String contextId) {
try {
_context = ContextHelper.getContext(contextId);
_log = _context.logManager().getLog(ReseedHandler.class);
} catch (Throwable t) {
t.printStackTrace();
}
}
public void setReseedNonce(String nonce) {
if (nonce == null) return;
if (nonce.equals(System.getProperty("net.i2p.router.web.ReseedHandler.nonce")) ||
@ -69,220 +27,8 @@ public class ReseedHandler {
public void requestReseed() {
synchronized (ReseedHandler.class) {
if (_reseedRunner == null)
_reseedRunner = new ReseedRunner();
if (_reseedRunner.isRunning()) {
return;
} else {
System.setProperty("net.i2p.router.web.ReseedHandler.reseedInProgress", "true");
I2PThread reseed = new I2PThread(_reseedRunner, "Reseed");
reseed.start();
}
_reseedRunner = new Reseeder(_context);
_reseedRunner.requestReseed();
}
}
public class ReseedRunner implements Runnable, EepGet.StatusListener {
private boolean _isRunning;
public ReseedRunner() {
_isRunning = false;
System.setProperty("net.i2p.router.web.ReseedHandler.statusMessage","Reseeding.");
}
public boolean isRunning() { return _isRunning; }
public void run() {
_isRunning = true;
System.out.println("Reseed start");
reseed(false);
System.out.println("Reseed complete");
System.setProperty("net.i2p.router.web.ReseedHandler.reseedInProgress", "false");
_isRunning = false;
}
// EepGet status listeners
public void attemptFailed(String url, long bytesTransferred, long bytesRemaining, int currentAttempt, int numRetries, Exception cause) {
// Since readURL() runs an EepGet with 0 retries,
// we can report errors with attemptFailed() instead of transferFailed().
// It has the benefit of providing cause of failure, which helps resolve issues.
if (_log.shouldLog(Log.ERROR)) _log.error("EepGet failed on " + url, cause);
}
public void bytesTransferred(long alreadyTransferred, int currentWrite, long bytesTransferred, long bytesRemaining, String url) {}
public void transferComplete(long alreadyTransferred, long bytesTransferred, long bytesRemaining, String url, String outputFile, boolean notModified) {}
public void transferFailed(String url, long bytesTransferred, long bytesRemaining, int currentAttempt) {}
public void headerReceived(String url, int attemptNum, String key, String val) {}
public void attempting(String url) {}
// End of EepGet status listeners
/**
* Reseed has been requested, so lets go ahead and do it. Fetch all of
* the routerInfo-*.dat files from the specified URL (or the default) and
* save them into this router's netDb dir.
*
*/
private static final String RESEED_TIPS =
"Ensure that nothing blocks outbound HTTP, check <a href=logs.jsp>logs</a> " +
"and if nothing helps, read FAQ about reseeding manually.";
private void reseed(boolean echoStatus) {
List URLList = new ArrayList();
String URLs = _context.getProperty("i2p.reseedURL", DEFAULT_SEED_URL);
StringTokenizer tok = new StringTokenizer(URLs, " ,");
while (tok.hasMoreTokens())
URLList.add(tok.nextToken().trim());
Collections.shuffle(URLList);
for (int i = 0; i < URLList.size() && _isRunning; i++)
reseedOne((String) URLList.get(i), echoStatus);
}
/**
* Fetch a directory listing and then up to 200 routerInfo files in the listing.
* The listing must contain (exactly) strings that match:
* href="routerInfo-{hash}.dat">
* OR
* HREF="routerInfo-{hash}.dat">
* and then it fetches the files
* {seedURL}routerInfo-{hash}.dat
* after appending a '/' to seedURL if it doesn't have one.
* Essentially this means that the seedURL must be a directory, it
* can't end with 'index.html', for example.
*
* Jetty directory listings are not compatible, as they look like
* HREF="/full/path/to/routerInfo-...
**/
private void reseedOne(String seedURL, boolean echoStatus) {
try {
System.setProperty("net.i2p.router.web.ReseedHandler.errorMessage","");
System.setProperty("net.i2p.router.web.ReseedHandler.statusMessage","Reseeding: fetching seed URL.");
System.err.println("Reseed from " + seedURL);
URL dir = new URL(seedURL);
byte contentRaw[] = readURL(dir);
if (contentRaw == null) {
System.setProperty("net.i2p.router.web.ReseedHandler.errorMessage",
"Last reseed failed fully (failed reading seed URL). " +
RESEED_TIPS);
// Logging deprecated here since attemptFailed() provides better info
_log.debug("Failed reading seed URL: " + seedURL);
return;
}
String content = new String(contentRaw);
Set urls = new HashSet();
int cur = 0;
int total = 0;
while (total++ < 1000) {
int start = content.indexOf("href=\"routerInfo-", cur);
if (start < 0) {
start = content.indexOf("HREF=\"routerInfo-", cur);
if (start < 0)
break;
}
int end = content.indexOf(".dat\">", start);
String name = content.substring(start+"href=\"routerInfo-".length(), end);
urls.add(name);
cur = end + 1;
}
if (total <= 0) {
_log.error("Read " + contentRaw.length + " bytes from seed " + seedURL + ", but found no routerInfo URLs.");
System.setProperty("net.i2p.router.web.ReseedHandler.errorMessage",
"Last reseed failed fully (no routerInfo URLs at seed URL). " +
RESEED_TIPS);
return;
}
List urlList = new ArrayList(urls);
Collections.shuffle(urlList);
int fetched = 0;
int errors = 0;
// 200 max from one URL
for (Iterator iter = urlList.iterator(); iter.hasNext() && fetched < 200; ) {
try {
System.setProperty("net.i2p.router.web.ReseedHandler.statusMessage",
"Reseeding: fetching router info from seed URL (" +
fetched + " successful, " + errors + " errors, " + total + " total).");
fetchSeed(seedURL, (String)iter.next());
fetched++;
if (echoStatus) {
System.out.print(".");
if (fetched % 60 == 0)
System.out.println();
}
} catch (Exception e) {
errors++;
}
}
System.err.println("Reseed got " + fetched + " router infos from " + seedURL);
int failPercent = 100 * errors / total;
// Less than 10% of failures is considered success,
// because some routerInfos will always fail.
if ((failPercent >= 10) && (failPercent < 90)) {
System.setProperty("net.i2p.router.web.ReseedHandler.errorMessage",
"Last reseed failed partly (" + failPercent + "% of " + total + "). " +
RESEED_TIPS);
}
if (failPercent >= 90) {
System.setProperty("net.i2p.router.web.ReseedHandler.errorMessage",
"Last reseed failed (" + failPercent + "% of " + total + "). " +
RESEED_TIPS);
}
// Don't go on to the next URL if we have enough
if (fetched >= 100)
_isRunning = false;
} catch (Throwable t) {
System.setProperty("net.i2p.router.web.ReseedHandler.errorMessage",
"Last reseed failed fully (exception caught). " +
RESEED_TIPS);
_log.error("Error reseeding", t);
}
}
/* Since we don't return a value, we should always throw an exception if something fails. */
private void fetchSeed(String seedURL, String peer) throws Exception {
URL url = new URL(seedURL + (seedURL.endsWith("/") ? "" : "/") + "routerInfo-" + peer + ".dat");
byte data[] = readURL(url);
if (data == null) {
// Logging deprecated here since attemptFailed() provides better info
_log.debug("Failed fetching seed: " + url.toString());
throw new Exception ("Failed fetching seed.");
}
//System.out.println("read: " + (data != null ? data.length : -1));
writeSeed(peer, data);
}
private byte[] readURL(URL url) throws Exception {
ByteArrayOutputStream baos = new ByteArrayOutputStream(4*1024);
// Do a non-proxied eepget into our ByteArrayOutputStream with 0 retries
EepGet get = new EepGet( I2PAppContext.getGlobalContext(), false, null, -1, 0, 0, MAX_RESEED_RESPONSE_SIZE,
null, baos, url.toString(), false, null, null);
get.addStatusListener(ReseedRunner.this);
if (get.fetch()) return baos.toByteArray(); else return null;
}
private void writeSeed(String name, byte data[]) throws Exception {
String dirName = "netDb"; // _context.getProperty("router.networkDatabase.dbDir", "netDb");
File netDbDir = new File(_context.getRouterDir(), dirName);
if (!netDbDir.exists()) {
boolean ok = netDbDir.mkdirs();
}
FileOutputStream fos = new FileOutputStream(new File(netDbDir, "routerInfo-" + name + ".dat"));
fos.write(data);
fos.close();
}
}
public static void main(String args[]) {
if ( (args != null) && (args.length == 1) && (!Boolean.valueOf(args[0]).booleanValue()) ) {
System.out.println("Not reseeding, as requested");
return; // not reseeding on request
}
System.out.println("Reseeding");
ReseedHandler reseedHandler = new ReseedHandler();
reseedHandler.requestReseed();
}
}

22
apps/routerconsole/java/src/net/i2p/router/web/RouterConsoleRunner.java
View File

@ -144,7 +144,8 @@ public class RouterConsoleRunner {
storeWebAppProperties(props);
try {
_server.start();
} catch (Exception me) {
} catch (Throwable me) {
// NoClassFoundDefError from a webapp is a throwable, not an exception
System.err.println("WARNING: Error starting one or more listeners of the Router Console server.\n" +
"If your console is still accessible at http://127.0.0.1:7657/,\n" +
"this may be a problem only with binding to the IPV6 address ::1.\n" +
@ -158,25 +159,6 @@ public class RouterConsoleRunner {
t.printStackTrace();
}
// we check the i2p installation directory (.) for a flag telling us not to reseed,
// but also check the home directory for that flag too, since new users installing i2p
// don't have an installation directory that they can put the flag in yet.
File noReseedFile = new File(new File(System.getProperty("user.home")), ".i2pnoreseed");
File noReseedFileAlt1 = new File(new File(System.getProperty("user.home")), "noreseed.i2p");
File noReseedFileAlt2 = new File(I2PAppContext.getGlobalContext().getConfigDir(), ".i2pnoreseed");
File noReseedFileAlt3 = new File(I2PAppContext.getGlobalContext().getConfigDir(), "noreseed.i2p");
if (!noReseedFile.exists() && !noReseedFileAlt1.exists() && !noReseedFileAlt2.exists() && !noReseedFileAlt3.exists()) {
File netDb = new File(I2PAppContext.getGlobalContext().getRouterDir(), "netDb");
// sure, some of them could be "my.info" or various leaseSet- files, but chances are,
// if someone has those files, they've already been seeded (at least enough to let them
// get i2p started - they can reseed later in the web console)
String names[] = (netDb.exists() ? netDb.list() : null);
if ( (names == null) || (names.length < 15) ) {
ReseedHandler reseedHandler = new ReseedHandler();
reseedHandler.requestReseed();
}
}
NewsFetcher fetcher = NewsFetcher.getInstance(I2PAppContext.getGlobalContext());
I2PThread t = new I2PThread(fetcher, "NewsFetcher");
t.setDaemon(true);

5
apps/routerconsole/java/src/net/i2p/router/web/SummaryHelper.java
View File

@ -85,8 +85,9 @@ public class SummaryHelper extends HelperBase {
}
public boolean allowReseed() {
return (_context.netDb().getKnownRouters() < 30) ||
Boolean.valueOf(_context.getProperty("i2p.alwaysAllowReseed", "false")).booleanValue();
return _context.netDb().isInitialized() &&
((_context.netDb().getKnownRouters() < 30) ||
Boolean.valueOf(_context.getProperty("i2p.alwaysAllowReseed")).booleanValue());
}
public int getAllPeers() { return _context.netDb().getKnownRouters(); }