From 7aa9949332ca0d390bef8f2bba2a630ad32d99ea Mon Sep 17 00:00:00 2001 From: zzz Date: Mon, 15 Jun 2009 21:58:28 +0000 Subject: [PATCH] * Reseeding / NetDb: - Move reseeding from the routerconsole app to the router, so that we can bootstrap an embedded router lacking a routerconsole (iMule or android for example), without additional modifications. This allows better integration between the reseeding function and the netDb. - Call reseed from PersistentDataStore, not from the routerconsole init, and start seeding as soon as the netdb has read the netDb/ directory, not when the console starts. - Wake up the netdb reader as soon as reseeding is done, rather than waiting up to 60s. - Don't display the reseed button on the console until the netdb initialization is done. * NetDb: - Fix an NPE on early shutdown * RouterConsoleRunner: - Catch a class not found error better --- .../src/net/i2p/router/web/ReseedHandler.java | 266 +----------------- .../i2p/router/web/RouterConsoleRunner.java | 22 +- .../src/net/i2p/router/web/SummaryHelper.java | 5 +- .../net/i2p/router/NetworkDatabaseFacade.java | 2 + .../router/networkdb/kademlia/DataStore.java | 2 + .../KademliaNetworkDatabaseFacade.java | 14 +- .../kademlia/PersistentDataStore.java | 25 +- .../kademlia/TransientDataStore.java | 4 + .../networkdb/reseed/ReseedChecker.java | 46 +++ .../i2p/router/networkdb/reseed/Reseeder.java | 266 ++++++++++++++++++ 10 files changed, 365 insertions(+), 287 deletions(-) create mode 100644 router/java/src/net/i2p/router/networkdb/reseed/ReseedChecker.java create mode 100644 router/java/src/net/i2p/router/networkdb/reseed/Reseeder.java diff --git a/apps/routerconsole/java/src/net/i2p/router/web/ReseedHandler.java b/apps/routerconsole/java/src/net/i2p/router/web/ReseedHandler.java index 944c4e910..31c5efc41 100644 --- a/apps/routerconsole/java/src/net/i2p/router/web/ReseedHandler.java +++ b/apps/routerconsole/java/src/net/i2p/router/web/ReseedHandler.java @@ -1,63 +1,21 @@ package net.i2p.router.web; -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.FileOutputStream; -import java.net.URL; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Set; -import java.util.StringTokenizer; - -import net.i2p.I2PAppContext; import net.i2p.router.RouterContext; -import net.i2p.util.EepGet; -import net.i2p.util.I2PThread; -import net.i2p.util.Log; +import net.i2p.router.networkdb.reseed.Reseeder; /** - * Handler to deal with reseed requests. This will reseed from the URL - * http://i2pdb.tin0.de/netDb/ unless the I2P configuration property "i2p.reseedURL" is - * set. It always writes to ./netDb/, so don't mess with that. - * + * Handler to deal with reseed requests. */ -public class ReseedHandler { - private static ReseedRunner _reseedRunner; - private RouterContext _context; - private Log _log; - - // Reject unreasonably big files, because we download into a ByteArrayOutputStream. - private static final long MAX_RESEED_RESPONSE_SIZE = 8 * 1024 * 1024; - - private static final String DEFAULT_SEED_URL = "http://i2pdb.tin0.de/netDb/,http://netdb.i2p2.de/"; +public class ReseedHandler extends HelperBase { + private static Reseeder _reseedRunner; public ReseedHandler() { this(ContextHelper.getContext(null)); } public ReseedHandler(RouterContext ctx) { _context = ctx; - _log = ctx.logManager().getLog(ReseedHandler.class); } - /** - * Configure this bean to query a particular router context - * - * @param contextId begging few characters of the routerHash, or null to pick - * the first one we come across. - */ - public void setContextId(String contextId) { - try { - _context = ContextHelper.getContext(contextId); - _log = _context.logManager().getLog(ReseedHandler.class); - } catch (Throwable t) { - t.printStackTrace(); - } - } - - public void setReseedNonce(String nonce) { if (nonce == null) return; if (nonce.equals(System.getProperty("net.i2p.router.web.ReseedHandler.nonce")) || @@ -69,220 +27,8 @@ public class ReseedHandler { public void requestReseed() { synchronized (ReseedHandler.class) { if (_reseedRunner == null) - _reseedRunner = new ReseedRunner(); - if (_reseedRunner.isRunning()) { - return; - } else { - System.setProperty("net.i2p.router.web.ReseedHandler.reseedInProgress", "true"); - I2PThread reseed = new I2PThread(_reseedRunner, "Reseed"); - reseed.start(); - } + _reseedRunner = new Reseeder(_context); + _reseedRunner.requestReseed(); } - } - - public class ReseedRunner implements Runnable, EepGet.StatusListener { - private boolean _isRunning; - - public ReseedRunner() { - _isRunning = false; - System.setProperty("net.i2p.router.web.ReseedHandler.statusMessage","Reseeding."); - } - public boolean isRunning() { return _isRunning; } - public void run() { - _isRunning = true; - System.out.println("Reseed start"); - reseed(false); - System.out.println("Reseed complete"); - System.setProperty("net.i2p.router.web.ReseedHandler.reseedInProgress", "false"); - _isRunning = false; - } - - // EepGet status listeners - public void attemptFailed(String url, long bytesTransferred, long bytesRemaining, int currentAttempt, int numRetries, Exception cause) { - // Since readURL() runs an EepGet with 0 retries, - // we can report errors with attemptFailed() instead of transferFailed(). - // It has the benefit of providing cause of failure, which helps resolve issues. - if (_log.shouldLog(Log.ERROR)) _log.error("EepGet failed on " + url, cause); - } - public void bytesTransferred(long alreadyTransferred, int currentWrite, long bytesTransferred, long bytesRemaining, String url) {} - public void transferComplete(long alreadyTransferred, long bytesTransferred, long bytesRemaining, String url, String outputFile, boolean notModified) {} - public void transferFailed(String url, long bytesTransferred, long bytesRemaining, int currentAttempt) {} - public void headerReceived(String url, int attemptNum, String key, String val) {} - public void attempting(String url) {} - // End of EepGet status listeners - - /** - * Reseed has been requested, so lets go ahead and do it. Fetch all of - * the routerInfo-*.dat files from the specified URL (or the default) and - * save them into this router's netDb dir. - * - */ - private static final String RESEED_TIPS = - "Ensure that nothing blocks outbound HTTP, check logs " + - "and if nothing helps, read FAQ about reseeding manually."; - - private void reseed(boolean echoStatus) { - List URLList = new ArrayList(); - String URLs = _context.getProperty("i2p.reseedURL", DEFAULT_SEED_URL); - StringTokenizer tok = new StringTokenizer(URLs, " ,"); - while (tok.hasMoreTokens()) - URLList.add(tok.nextToken().trim()); - Collections.shuffle(URLList); - for (int i = 0; i < URLList.size() && _isRunning; i++) - reseedOne((String) URLList.get(i), echoStatus); - } - - /** - * Fetch a directory listing and then up to 200 routerInfo files in the listing. - * The listing must contain (exactly) strings that match: - * href="routerInfo-{hash}.dat"> - * OR - * HREF="routerInfo-{hash}.dat"> - * and then it fetches the files - * {seedURL}routerInfo-{hash}.dat - * after appending a '/' to seedURL if it doesn't have one. - * Essentially this means that the seedURL must be a directory, it - * can't end with 'index.html', for example. - * - * Jetty directory listings are not compatible, as they look like - * HREF="/full/path/to/routerInfo-... - **/ - private void reseedOne(String seedURL, boolean echoStatus) { - - try { - System.setProperty("net.i2p.router.web.ReseedHandler.errorMessage",""); - System.setProperty("net.i2p.router.web.ReseedHandler.statusMessage","Reseeding: fetching seed URL."); - System.err.println("Reseed from " + seedURL); - URL dir = new URL(seedURL); - byte contentRaw[] = readURL(dir); - if (contentRaw == null) { - System.setProperty("net.i2p.router.web.ReseedHandler.errorMessage", - "Last reseed failed fully (failed reading seed URL). " + - RESEED_TIPS); - // Logging deprecated here since attemptFailed() provides better info - _log.debug("Failed reading seed URL: " + seedURL); - return; - } - String content = new String(contentRaw); - Set urls = new HashSet(); - int cur = 0; - int total = 0; - while (total++ < 1000) { - int start = content.indexOf("href=\"routerInfo-", cur); - if (start < 0) { - start = content.indexOf("HREF=\"routerInfo-", cur); - if (start < 0) - break; - } - - int end = content.indexOf(".dat\">", start); - String name = content.substring(start+"href=\"routerInfo-".length(), end); - urls.add(name); - cur = end + 1; - } - if (total <= 0) { - _log.error("Read " + contentRaw.length + " bytes from seed " + seedURL + ", but found no routerInfo URLs."); - System.setProperty("net.i2p.router.web.ReseedHandler.errorMessage", - "Last reseed failed fully (no routerInfo URLs at seed URL). " + - RESEED_TIPS); - return; - } - - List urlList = new ArrayList(urls); - Collections.shuffle(urlList); - int fetched = 0; - int errors = 0; - // 200 max from one URL - for (Iterator iter = urlList.iterator(); iter.hasNext() && fetched < 200; ) { - try { - System.setProperty("net.i2p.router.web.ReseedHandler.statusMessage", - "Reseeding: fetching router info from seed URL (" + - fetched + " successful, " + errors + " errors, " + total + " total)."); - - fetchSeed(seedURL, (String)iter.next()); - fetched++; - if (echoStatus) { - System.out.print("."); - if (fetched % 60 == 0) - System.out.println(); - } - } catch (Exception e) { - errors++; - } - } - System.err.println("Reseed got " + fetched + " router infos from " + seedURL); - - int failPercent = 100 * errors / total; - - // Less than 10% of failures is considered success, - // because some routerInfos will always fail. - if ((failPercent >= 10) && (failPercent < 90)) { - System.setProperty("net.i2p.router.web.ReseedHandler.errorMessage", - "Last reseed failed partly (" + failPercent + "% of " + total + "). " + - RESEED_TIPS); - } - if (failPercent >= 90) { - System.setProperty("net.i2p.router.web.ReseedHandler.errorMessage", - "Last reseed failed (" + failPercent + "% of " + total + "). " + - RESEED_TIPS); - } - // Don't go on to the next URL if we have enough - if (fetched >= 100) - _isRunning = false; - } catch (Throwable t) { - System.setProperty("net.i2p.router.web.ReseedHandler.errorMessage", - "Last reseed failed fully (exception caught). " + - RESEED_TIPS); - _log.error("Error reseeding", t); - } - } - - /* Since we don't return a value, we should always throw an exception if something fails. */ - private void fetchSeed(String seedURL, String peer) throws Exception { - URL url = new URL(seedURL + (seedURL.endsWith("/") ? "" : "/") + "routerInfo-" + peer + ".dat"); - - byte data[] = readURL(url); - if (data == null) { - // Logging deprecated here since attemptFailed() provides better info - _log.debug("Failed fetching seed: " + url.toString()); - throw new Exception ("Failed fetching seed."); - } - //System.out.println("read: " + (data != null ? data.length : -1)); - writeSeed(peer, data); - } - - private byte[] readURL(URL url) throws Exception { - ByteArrayOutputStream baos = new ByteArrayOutputStream(4*1024); - - // Do a non-proxied eepget into our ByteArrayOutputStream with 0 retries - EepGet get = new EepGet( I2PAppContext.getGlobalContext(), false, null, -1, 0, 0, MAX_RESEED_RESPONSE_SIZE, - null, baos, url.toString(), false, null, null); - get.addStatusListener(ReseedRunner.this); - if (get.fetch()) return baos.toByteArray(); else return null; - } - - private void writeSeed(String name, byte data[]) throws Exception { - String dirName = "netDb"; // _context.getProperty("router.networkDatabase.dbDir", "netDb"); - File netDbDir = new File(_context.getRouterDir(), dirName); - if (!netDbDir.exists()) { - boolean ok = netDbDir.mkdirs(); - } - FileOutputStream fos = new FileOutputStream(new File(netDbDir, "routerInfo-" + name + ".dat")); - fos.write(data); - fos.close(); - } - - } - - public static void main(String args[]) { - if ( (args != null) && (args.length == 1) && (!Boolean.valueOf(args[0]).booleanValue()) ) { - System.out.println("Not reseeding, as requested"); - return; // not reseeding on request - } - System.out.println("Reseeding"); - ReseedHandler reseedHandler = new ReseedHandler(); - reseedHandler.requestReseed(); - } - } diff --git a/apps/routerconsole/java/src/net/i2p/router/web/RouterConsoleRunner.java b/apps/routerconsole/java/src/net/i2p/router/web/RouterConsoleRunner.java index 25b990f04..bbf9852c3 100644 --- a/apps/routerconsole/java/src/net/i2p/router/web/RouterConsoleRunner.java +++ b/apps/routerconsole/java/src/net/i2p/router/web/RouterConsoleRunner.java @@ -144,7 +144,8 @@ public class RouterConsoleRunner { storeWebAppProperties(props); try { _server.start(); - } catch (Exception me) { + } catch (Throwable me) { + // NoClassFoundDefError from a webapp is a throwable, not an exception System.err.println("WARNING: Error starting one or more listeners of the Router Console server.\n" + "If your console is still accessible at http://127.0.0.1:7657/,\n" + "this may be a problem only with binding to the IPV6 address ::1.\n" + @@ -158,25 +159,6 @@ public class RouterConsoleRunner { t.printStackTrace(); } - // we check the i2p installation directory (.) for a flag telling us not to reseed, - // but also check the home directory for that flag too, since new users installing i2p - // don't have an installation directory that they can put the flag in yet. - File noReseedFile = new File(new File(System.getProperty("user.home")), ".i2pnoreseed"); - File noReseedFileAlt1 = new File(new File(System.getProperty("user.home")), "noreseed.i2p"); - File noReseedFileAlt2 = new File(I2PAppContext.getGlobalContext().getConfigDir(), ".i2pnoreseed"); - File noReseedFileAlt3 = new File(I2PAppContext.getGlobalContext().getConfigDir(), "noreseed.i2p"); - if (!noReseedFile.exists() && !noReseedFileAlt1.exists() && !noReseedFileAlt2.exists() && !noReseedFileAlt3.exists()) { - File netDb = new File(I2PAppContext.getGlobalContext().getRouterDir(), "netDb"); - // sure, some of them could be "my.info" or various leaseSet- files, but chances are, - // if someone has those files, they've already been seeded (at least enough to let them - // get i2p started - they can reseed later in the web console) - String names[] = (netDb.exists() ? netDb.list() : null); - if ( (names == null) || (names.length < 15) ) { - ReseedHandler reseedHandler = new ReseedHandler(); - reseedHandler.requestReseed(); - } - } - NewsFetcher fetcher = NewsFetcher.getInstance(I2PAppContext.getGlobalContext()); I2PThread t = new I2PThread(fetcher, "NewsFetcher"); t.setDaemon(true); diff --git a/apps/routerconsole/java/src/net/i2p/router/web/SummaryHelper.java b/apps/routerconsole/java/src/net/i2p/router/web/SummaryHelper.java index 0810dd950..66e9d9f25 100644 --- a/apps/routerconsole/java/src/net/i2p/router/web/SummaryHelper.java +++ b/apps/routerconsole/java/src/net/i2p/router/web/SummaryHelper.java @@ -85,8 +85,9 @@ public class SummaryHelper extends HelperBase { } public boolean allowReseed() { - return (_context.netDb().getKnownRouters() < 30) || - Boolean.valueOf(_context.getProperty("i2p.alwaysAllowReseed", "false")).booleanValue(); + return _context.netDb().isInitialized() && + ((_context.netDb().getKnownRouters() < 30) || + Boolean.valueOf(_context.getProperty("i2p.alwaysAllowReseed")).booleanValue()); } public int getAllPeers() { return _context.netDb().getKnownRouters(); } diff --git a/router/java/src/net/i2p/router/NetworkDatabaseFacade.java b/router/java/src/net/i2p/router/NetworkDatabaseFacade.java index 1ff7af132..865fbf8ec 100644 --- a/router/java/src/net/i2p/router/NetworkDatabaseFacade.java +++ b/router/java/src/net/i2p/router/NetworkDatabaseFacade.java @@ -58,6 +58,8 @@ public abstract class NetworkDatabaseFacade implements Service { public abstract Set getAllRouters(); public int getKnownRouters() { return 0; } public int getKnownLeaseSets() { return 0; } + public boolean isInitialized() { return true; } + public void rescan() {} public void renderRouterInfoHTML(Writer out, String s) throws IOException {} public void renderLeaseSetHTML(Writer out) throws IOException {} public void renderStatusHTML(Writer out, boolean b) throws IOException {} diff --git a/router/java/src/net/i2p/router/networkdb/kademlia/DataStore.java b/router/java/src/net/i2p/router/networkdb/kademlia/DataStore.java index 75329f896..ae4132678 100644 --- a/router/java/src/net/i2p/router/networkdb/kademlia/DataStore.java +++ b/router/java/src/net/i2p/router/networkdb/kademlia/DataStore.java @@ -14,6 +14,7 @@ import net.i2p.data.DataStructure; import net.i2p.data.Hash; public interface DataStore { + public boolean isInitialized(); public boolean isKnown(Hash key); public DataStructure get(Hash key); public DataStructure get(Hash key, boolean persist); @@ -24,6 +25,7 @@ public interface DataStore { public Set getKeys(); public void stop(); public void restart(); + public void rescan(); public int countLeaseSets(); } diff --git a/router/java/src/net/i2p/router/networkdb/kademlia/KademliaNetworkDatabaseFacade.java b/router/java/src/net/i2p/router/networkdb/kademlia/KademliaNetworkDatabaseFacade.java index 31d6dc04b..ae5dd9d80 100644 --- a/router/java/src/net/i2p/router/networkdb/kademlia/KademliaNetworkDatabaseFacade.java +++ b/router/java/src/net/i2p/router/networkdb/kademlia/KademliaNetworkDatabaseFacade.java @@ -139,6 +139,11 @@ public class KademliaNetworkDatabaseFacade extends NetworkDatabaseFacade { context.statManager().createRateStat("netDb.exploreKeySet", "how many keys are queued for exploration?", "NetworkDatabase", new long[] { 10*60*1000 }); } + @Override + public boolean isInitialized() { + return _initialized && _ds != null && _ds.isInitialized(); + } + protected PeerSelector createPeerSelector() { return new PeerSelector(_context); } public PeerSelector getPeerSelector() { return _peerSelector; } @@ -177,7 +182,8 @@ public class KademliaNetworkDatabaseFacade extends NetworkDatabaseFacade { public void shutdown() { _initialized = false; _kb = null; - _ds.stop(); + if (_ds != null) + _ds.stop(); _ds = null; _exploreKeys.clear(); // hope this doesn't cause an explosion, it shouldn't. // _exploreKeys = null; @@ -203,6 +209,12 @@ public class KademliaNetworkDatabaseFacade extends NetworkDatabaseFacade { publish(ri); } + @Override + public void rescan() { + if (isInitialized()) + _ds.rescan(); + } + String getDbDir() { return _dbDir; } public void startup() { diff --git a/router/java/src/net/i2p/router/networkdb/kademlia/PersistentDataStore.java b/router/java/src/net/i2p/router/networkdb/kademlia/PersistentDataStore.java index 92124d690..d80a3248b 100644 --- a/router/java/src/net/i2p/router/networkdb/kademlia/PersistentDataStore.java +++ b/router/java/src/net/i2p/router/networkdb/kademlia/PersistentDataStore.java @@ -26,6 +26,7 @@ import net.i2p.data.RouterInfo; import net.i2p.router.JobImpl; import net.i2p.router.Router; import net.i2p.router.RouterContext; +import net.i2p.router.networkdb.reseed.ReseedChecker; import net.i2p.util.I2PThread; import net.i2p.util.Log; @@ -39,6 +40,8 @@ class PersistentDataStore extends TransientDataStore { private String _dbDir; private KademliaNetworkDatabaseFacade _facade; private Writer _writer; + private ReadJob _readJob; + private boolean _initialized; private final static int READ_DELAY = 60*1000; @@ -47,7 +50,8 @@ class PersistentDataStore extends TransientDataStore { _log = ctx.logManager().getLog(PersistentDataStore.class); _dbDir = dbDir; _facade = facade; - _context.jobQueue().addJob(new ReadJob()); + _readJob = new ReadJob(); + _context.jobQueue().addJob(_readJob); ctx.statManager().createRateStat("netDb.writeClobber", "How often we clobber a pending netDb write", "NetworkDatabase", new long[] { 20*60*1000 }); ctx.statManager().createRateStat("netDb.writePending", "How many pending writes are there", "NetworkDatabase", new long[] { 60*1000 }); ctx.statManager().createRateStat("netDb.writeOut", "How many we wrote", "NetworkDatabase", new long[] { 20*60*1000 }); @@ -58,7 +62,10 @@ class PersistentDataStore extends TransientDataStore { //writer.setDaemon(true); writer.start(); } - + + public boolean isInitialized() { return _initialized; } + + // this doesn't stop the read job or the writer, maybe it should? @Override public void stop() { super.stop(); @@ -71,6 +78,11 @@ class PersistentDataStore extends TransientDataStore { _dbDir = _facade.getDbDir(); } + public void rescan() { + if (_initialized) + _readJob.wakeup(); + } + @Override public DataStructure get(Hash key) { return get(key, true); @@ -317,6 +329,10 @@ class PersistentDataStore extends TransientDataStore { requeue(READ_DELAY); } + public void wakeup() { + requeue(0); + } + private void readFiles() { int routerCount = 0; try { @@ -336,9 +352,10 @@ class PersistentDataStore extends TransientDataStore { _log.error("Error reading files in the db dir", ioe); } - if ( (routerCount <= 5) && (!_alreadyWarned) ) { - _log.error("Very few routerInfo files remaining - please reseed"); + if (!_alreadyWarned) { + ReseedChecker.checkReseed(_context, routerCount); _alreadyWarned = true; + _initialized = true; } } } diff --git a/router/java/src/net/i2p/router/networkdb/kademlia/TransientDataStore.java b/router/java/src/net/i2p/router/networkdb/kademlia/TransientDataStore.java index 5028b5ea1..ace0a9666 100644 --- a/router/java/src/net/i2p/router/networkdb/kademlia/TransientDataStore.java +++ b/router/java/src/net/i2p/router/networkdb/kademlia/TransientDataStore.java @@ -35,6 +35,8 @@ class TransientDataStore implements DataStore { _log.info("Data Store initialized"); } + public boolean isInitialized() { return true; } + public void stop() { _data.clear(); } @@ -43,6 +45,8 @@ class TransientDataStore implements DataStore { stop(); } + public void rescan() {} + public Set getKeys() { return new HashSet(_data.keySet()); } diff --git a/router/java/src/net/i2p/router/networkdb/reseed/ReseedChecker.java b/router/java/src/net/i2p/router/networkdb/reseed/ReseedChecker.java new file mode 100644 index 000000000..aa38be5e7 --- /dev/null +++ b/router/java/src/net/i2p/router/networkdb/reseed/ReseedChecker.java @@ -0,0 +1,46 @@ +package net.i2p.router.networkdb.reseed; + +import java.io.File; + +import net.i2p.router.RouterContext; +import net.i2p.util.Log; + +/** + * Moved from RouterConsoleRunner.java + * + * Reseeding is not strictly a router function, it used to be + * in the routerconsole app, but this made it impossible to + * bootstrap an embedded router lacking a routerconsole, + * in iMule or android for example, without additional modifications. + * + * Also, as this is now called from PersistentDataStore, not from the + * routerconsole, we can get started as soon as the netdb has read + * the netDb/ directory, not when the console starts, +router/java/src/net/i2p/router/networkdb/eseed/ReseedChecker.java + */ +public class ReseedChecker { + + private static final int MINIMUM = 15; + + public static void checkReseed(RouterContext context, int count) { + if (count >= MINIMUM) + return; + + // we check the i2p installation directory for a flag telling us not to reseed, + // but also check the home directory for that flag too, since new users installing i2p + // don't have an installation directory that they can put the flag in yet. + File noReseedFile = new File(new File(System.getProperty("user.home")), ".i2pnoreseed"); + File noReseedFileAlt1 = new File(new File(System.getProperty("user.home")), "noreseed.i2p"); + File noReseedFileAlt2 = new File(context.getConfigDir(), ".i2pnoreseed"); + File noReseedFileAlt3 = new File(context.getConfigDir(), "noreseed.i2p"); + if (!noReseedFile.exists() && !noReseedFileAlt1.exists() && !noReseedFileAlt2.exists() && !noReseedFileAlt3.exists()) { + Log _log = context.logManager().getLog(ReseedChecker.class); + if (count <= 1) + _log.error("Downloading peer router information for a new I2P installation"); + else + _log.error("Very few routerInfo files remaining - reseeding now"); + Reseeder reseeder = new Reseeder(context); + reseeder.requestReseed(); + } + } +} diff --git a/router/java/src/net/i2p/router/networkdb/reseed/Reseeder.java b/router/java/src/net/i2p/router/networkdb/reseed/Reseeder.java new file mode 100644 index 000000000..bcecf0753 --- /dev/null +++ b/router/java/src/net/i2p/router/networkdb/reseed/Reseeder.java @@ -0,0 +1,266 @@ +package net.i2p.router.networkdb.reseed; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.net.URL; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Set; +import java.util.StringTokenizer; + +import net.i2p.I2PAppContext; +import net.i2p.router.RouterContext; +import net.i2p.util.EepGet; +import net.i2p.util.I2PThread; +import net.i2p.util.Log; + +/** + * Moved from ReseedHandler in routerconsole. See ReseedChecker for additional comments. + * + * Handler to deal with reseed requests. This will reseed from the URLs + * specified below unless the I2P configuration property "i2p.reseedURL" is + * set. It always writes to ./netDb/, so don't mess with that. + * + */ +public class Reseeder { + private static ReseedRunner _reseedRunner; + private RouterContext _context; + private Log _log; + + // Reject unreasonably big files, because we download into a ByteArrayOutputStream. + private static final long MAX_RESEED_RESPONSE_SIZE = 8 * 1024 * 1024; + + private static final String DEFAULT_SEED_URL = "http://i2pdb.tin0.de/netDb/,http://netdb.i2p2.de/"; + + public Reseeder(RouterContext ctx) { + _context = ctx; + _log = ctx.logManager().getLog(Reseeder.class); + } + + public void requestReseed() { + synchronized (Reseeder.class) { + if (_reseedRunner == null) + _reseedRunner = new ReseedRunner(); + if (_reseedRunner.isRunning()) { + return; + } else { + System.setProperty("net.i2p.router.web.Reseeder.reseedInProgress", "true"); + I2PThread reseed = new I2PThread(_reseedRunner, "Reseed"); + reseed.start(); + } + } + + } + + public class ReseedRunner implements Runnable, EepGet.StatusListener { + private boolean _isRunning; + + public ReseedRunner() { + _isRunning = false; + System.setProperty("net.i2p.router.web.Reseeder.statusMessage","Reseeding."); + } + public boolean isRunning() { return _isRunning; } + public void run() { + _isRunning = true; + System.out.println("Reseed start"); + reseed(false); + System.out.println("Reseed complete"); + System.setProperty("net.i2p.router.web.Reseeder.reseedInProgress", "false"); + _isRunning = false; + } + + // EepGet status listeners + public void attemptFailed(String url, long bytesTransferred, long bytesRemaining, int currentAttempt, int numRetries, Exception cause) { + // Since readURL() runs an EepGet with 0 retries, + // we can report errors with attemptFailed() instead of transferFailed(). + // It has the benefit of providing cause of failure, which helps resolve issues. + if (_log.shouldLog(Log.ERROR)) _log.error("EepGet failed on " + url, cause); + } + public void bytesTransferred(long alreadyTransferred, int currentWrite, long bytesTransferred, long bytesRemaining, String url) {} + public void transferComplete(long alreadyTransferred, long bytesTransferred, long bytesRemaining, String url, String outputFile, boolean notModified) {} + public void transferFailed(String url, long bytesTransferred, long bytesRemaining, int currentAttempt) {} + public void headerReceived(String url, int attemptNum, String key, String val) {} + public void attempting(String url) {} + // End of EepGet status listeners + + /** + * Reseed has been requested, so lets go ahead and do it. Fetch all of + * the routerInfo-*.dat files from the specified URL (or the default) and + * save them into this router's netDb dir. + * + */ + private static final String RESEED_TIPS = + "Ensure that nothing blocks outbound HTTP, check logs " + + "and if nothing helps, read FAQ about reseeding manually."; + + private void reseed(boolean echoStatus) { + List URLList = new ArrayList(); + String URLs = _context.getProperty("i2p.reseedURL", DEFAULT_SEED_URL); + StringTokenizer tok = new StringTokenizer(URLs, " ,"); + while (tok.hasMoreTokens()) + URLList.add(tok.nextToken().trim()); + Collections.shuffle(URLList); + for (int i = 0; i < URLList.size() && _isRunning; i++) + reseedOne((String) URLList.get(i), echoStatus); + } + + /** + * Fetch a directory listing and then up to 200 routerInfo files in the listing. + * The listing must contain (exactly) strings that match: + * href="routerInfo-{hash}.dat"> + * OR + * HREF="routerInfo-{hash}.dat"> + * and then it fetches the files + * {seedURL}routerInfo-{hash}.dat + * after appending a '/' to seedURL if it doesn't have one. + * Essentially this means that the seedURL must be a directory, it + * can't end with 'index.html', for example. + * + * Jetty directory listings are not compatible, as they look like + * HREF="/full/path/to/routerInfo-... + **/ + private void reseedOne(String seedURL, boolean echoStatus) { + + try { + System.setProperty("net.i2p.router.web.Reseeder.errorMessage",""); + System.setProperty("net.i2p.router.web.Reseeder.statusMessage","Reseeding: fetching seed URL."); + System.err.println("Reseed from " + seedURL); + URL dir = new URL(seedURL); + byte contentRaw[] = readURL(dir); + if (contentRaw == null) { + System.setProperty("net.i2p.router.web.Reseeder.errorMessage", + "Last reseed failed fully (failed reading seed URL). " + + RESEED_TIPS); + // Logging deprecated here since attemptFailed() provides better info + _log.debug("Failed reading seed URL: " + seedURL); + return; + } + String content = new String(contentRaw); + Set urls = new HashSet(); + int cur = 0; + int total = 0; + while (total++ < 1000) { + int start = content.indexOf("href=\"routerInfo-", cur); + if (start < 0) { + start = content.indexOf("HREF=\"routerInfo-", cur); + if (start < 0) + break; + } + + int end = content.indexOf(".dat\">", start); + String name = content.substring(start+"href=\"routerInfo-".length(), end); + urls.add(name); + cur = end + 1; + } + if (total <= 0) { + _log.error("Read " + contentRaw.length + " bytes from seed " + seedURL + ", but found no routerInfo URLs."); + System.setProperty("net.i2p.router.web.Reseeder.errorMessage", + "Last reseed failed fully (no routerInfo URLs at seed URL). " + + RESEED_TIPS); + return; + } + + List urlList = new ArrayList(urls); + Collections.shuffle(urlList); + int fetched = 0; + int errors = 0; + // 200 max from one URL + for (Iterator iter = urlList.iterator(); iter.hasNext() && fetched < 200; ) { + try { + System.setProperty("net.i2p.router.web.Reseeder.statusMessage", + "Reseeding: fetching router info from seed URL (" + + fetched + " successful, " + errors + " errors, " + total + " total)."); + + fetchSeed(seedURL, (String)iter.next()); + fetched++; + if (echoStatus) { + System.out.print("."); + if (fetched % 60 == 0) + System.out.println(); + } + } catch (Exception e) { + errors++; + } + } + System.err.println("Reseed got " + fetched + " router infos from " + seedURL); + + int failPercent = 100 * errors / total; + + // Less than 10% of failures is considered success, + // because some routerInfos will always fail. + if ((failPercent >= 10) && (failPercent < 90)) { + System.setProperty("net.i2p.router.web.Reseeder.errorMessage", + "Last reseed failed partly (" + failPercent + "% of " + total + "). " + + RESEED_TIPS); + } + if (failPercent >= 90) { + System.setProperty("net.i2p.router.web.Reseeder.errorMessage", + "Last reseed failed (" + failPercent + "% of " + total + "). " + + RESEED_TIPS); + } + if (fetched > 0) + _context.netDb().rescan(); + // Don't go on to the next URL if we have enough + if (fetched >= 100) + _isRunning = false; + } catch (Throwable t) { + System.setProperty("net.i2p.router.web.Reseeder.errorMessage", + "Last reseed failed fully (exception caught). " + + RESEED_TIPS); + _log.error("Error reseeding", t); + } + } + + /* Since we don't return a value, we should always throw an exception if something fails. */ + private void fetchSeed(String seedURL, String peer) throws Exception { + URL url = new URL(seedURL + (seedURL.endsWith("/") ? "" : "/") + "routerInfo-" + peer + ".dat"); + + byte data[] = readURL(url); + if (data == null) { + // Logging deprecated here since attemptFailed() provides better info + _log.debug("Failed fetching seed: " + url.toString()); + throw new Exception ("Failed fetching seed."); + } + //System.out.println("read: " + (data != null ? data.length : -1)); + writeSeed(peer, data); + } + + private byte[] readURL(URL url) throws Exception { + ByteArrayOutputStream baos = new ByteArrayOutputStream(4*1024); + + // Do a non-proxied eepget into our ByteArrayOutputStream with 0 retries + EepGet get = new EepGet( I2PAppContext.getGlobalContext(), false, null, -1, 0, 0, MAX_RESEED_RESPONSE_SIZE, + null, baos, url.toString(), false, null, null); + get.addStatusListener(ReseedRunner.this); + if (get.fetch()) return baos.toByteArray(); else return null; + } + + private void writeSeed(String name, byte data[]) throws Exception { + String dirName = "netDb"; // _context.getProperty("router.networkDatabase.dbDir", "netDb"); + File netDbDir = new File(_context.getRouterDir(), dirName); + if (!netDbDir.exists()) { + boolean ok = netDbDir.mkdirs(); + } + FileOutputStream fos = new FileOutputStream(new File(netDbDir, "routerInfo-" + name + ".dat")); + fos.write(data); + fos.close(); + } + + } + +/****** + public static void main(String args[]) { + if ( (args != null) && (args.length == 1) && (!Boolean.valueOf(args[0]).booleanValue()) ) { + System.out.println("Not reseeding, as requested"); + return; // not reseeding on request + } + System.out.println("Reseeding"); + Reseeder reseedHandler = new Reseeder(); + reseedHandler.requestReseed(); + } +******/ +}