* Reseeding / NetDb:
- Move reseeding from the routerconsole app to
the router, so that we can bootstrap an embedded router lacking a routerconsole
(iMule or android for example), without additional modifications.
This allows better integration between the reseeding function
and the netDb.
- Call reseed from PersistentDataStore, not from the
routerconsole init, and start seeding as soon as the netdb has read
the netDb/ directory, not when the console starts.
- Wake up the netdb reader as soon as reseeding is done,
rather than waiting up to 60s.
- Don't display the reseed button on the console until the
netdb initialization is done.
* NetDb:
- Fix an NPE on early shutdown
* RouterConsoleRunner:
- Catch a class not found error better
This commit is contained in:
@ -1,63 +1,21 @@
|
||||
package net.i2p.router.web;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.File;
|
||||
import java.io.FileOutputStream;
|
||||
import java.net.URL;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.HashSet;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.StringTokenizer;
|
||||
|
||||
import net.i2p.I2PAppContext;
|
||||
import net.i2p.router.RouterContext;
|
||||
import net.i2p.util.EepGet;
|
||||
import net.i2p.util.I2PThread;
|
||||
import net.i2p.util.Log;
|
||||
import net.i2p.router.networkdb.reseed.Reseeder;
|
||||
|
||||
/**
|
||||
* Handler to deal with reseed requests. This will reseed from the URL
|
||||
* http://i2pdb.tin0.de/netDb/ unless the I2P configuration property "i2p.reseedURL" is
|
||||
* set. It always writes to ./netDb/, so don't mess with that.
|
||||
*
|
||||
* Handler to deal with reseed requests.
|
||||
*/
|
||||
public class ReseedHandler {
|
||||
private static ReseedRunner _reseedRunner;
|
||||
private RouterContext _context;
|
||||
private Log _log;
|
||||
|
||||
// Reject unreasonably big files, because we download into a ByteArrayOutputStream.
|
||||
private static final long MAX_RESEED_RESPONSE_SIZE = 8 * 1024 * 1024;
|
||||
|
||||
private static final String DEFAULT_SEED_URL = "http://i2pdb.tin0.de/netDb/,http://netdb.i2p2.de/";
|
||||
public class ReseedHandler extends HelperBase {
|
||||
private static Reseeder _reseedRunner;
|
||||
|
||||
public ReseedHandler() {
|
||||
this(ContextHelper.getContext(null));
|
||||
}
|
||||
public ReseedHandler(RouterContext ctx) {
|
||||
_context = ctx;
|
||||
_log = ctx.logManager().getLog(ReseedHandler.class);
|
||||
}
|
||||
|
||||
/**
|
||||
* Configure this bean to query a particular router context
|
||||
*
|
||||
* @param contextId begging few characters of the routerHash, or null to pick
|
||||
* the first one we come across.
|
||||
*/
|
||||
public void setContextId(String contextId) {
|
||||
try {
|
||||
_context = ContextHelper.getContext(contextId);
|
||||
_log = _context.logManager().getLog(ReseedHandler.class);
|
||||
} catch (Throwable t) {
|
||||
t.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public void setReseedNonce(String nonce) {
|
||||
if (nonce == null) return;
|
||||
if (nonce.equals(System.getProperty("net.i2p.router.web.ReseedHandler.nonce")) ||
|
||||
@ -69,220 +27,8 @@ public class ReseedHandler {
|
||||
public void requestReseed() {
|
||||
synchronized (ReseedHandler.class) {
|
||||
if (_reseedRunner == null)
|
||||
_reseedRunner = new ReseedRunner();
|
||||
if (_reseedRunner.isRunning()) {
|
||||
return;
|
||||
} else {
|
||||
System.setProperty("net.i2p.router.web.ReseedHandler.reseedInProgress", "true");
|
||||
I2PThread reseed = new I2PThread(_reseedRunner, "Reseed");
|
||||
reseed.start();
|
||||
_reseedRunner = new Reseeder(_context);
|
||||
_reseedRunner.requestReseed();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public class ReseedRunner implements Runnable, EepGet.StatusListener {
|
||||
private boolean _isRunning;
|
||||
|
||||
public ReseedRunner() {
|
||||
_isRunning = false;
|
||||
System.setProperty("net.i2p.router.web.ReseedHandler.statusMessage","Reseeding.");
|
||||
}
|
||||
public boolean isRunning() { return _isRunning; }
|
||||
public void run() {
|
||||
_isRunning = true;
|
||||
System.out.println("Reseed start");
|
||||
reseed(false);
|
||||
System.out.println("Reseed complete");
|
||||
System.setProperty("net.i2p.router.web.ReseedHandler.reseedInProgress", "false");
|
||||
_isRunning = false;
|
||||
}
|
||||
|
||||
// EepGet status listeners
|
||||
public void attemptFailed(String url, long bytesTransferred, long bytesRemaining, int currentAttempt, int numRetries, Exception cause) {
|
||||
// Since readURL() runs an EepGet with 0 retries,
|
||||
// we can report errors with attemptFailed() instead of transferFailed().
|
||||
// It has the benefit of providing cause of failure, which helps resolve issues.
|
||||
if (_log.shouldLog(Log.ERROR)) _log.error("EepGet failed on " + url, cause);
|
||||
}
|
||||
public void bytesTransferred(long alreadyTransferred, int currentWrite, long bytesTransferred, long bytesRemaining, String url) {}
|
||||
public void transferComplete(long alreadyTransferred, long bytesTransferred, long bytesRemaining, String url, String outputFile, boolean notModified) {}
|
||||
public void transferFailed(String url, long bytesTransferred, long bytesRemaining, int currentAttempt) {}
|
||||
public void headerReceived(String url, int attemptNum, String key, String val) {}
|
||||
public void attempting(String url) {}
|
||||
// End of EepGet status listeners
|
||||
|
||||
/**
|
||||
* Reseed has been requested, so lets go ahead and do it. Fetch all of
|
||||
* the routerInfo-*.dat files from the specified URL (or the default) and
|
||||
* save them into this router's netDb dir.
|
||||
*
|
||||
*/
|
||||
private static final String RESEED_TIPS =
|
||||
"Ensure that nothing blocks outbound HTTP, check <a href=logs.jsp>logs</a> " +
|
||||
"and if nothing helps, read FAQ about reseeding manually.";
|
||||
|
||||
private void reseed(boolean echoStatus) {
|
||||
List URLList = new ArrayList();
|
||||
String URLs = _context.getProperty("i2p.reseedURL", DEFAULT_SEED_URL);
|
||||
StringTokenizer tok = new StringTokenizer(URLs, " ,");
|
||||
while (tok.hasMoreTokens())
|
||||
URLList.add(tok.nextToken().trim());
|
||||
Collections.shuffle(URLList);
|
||||
for (int i = 0; i < URLList.size() && _isRunning; i++)
|
||||
reseedOne((String) URLList.get(i), echoStatus);
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch a directory listing and then up to 200 routerInfo files in the listing.
|
||||
* The listing must contain (exactly) strings that match:
|
||||
* href="routerInfo-{hash}.dat">
|
||||
* OR
|
||||
* HREF="routerInfo-{hash}.dat">
|
||||
* and then it fetches the files
|
||||
* {seedURL}routerInfo-{hash}.dat
|
||||
* after appending a '/' to seedURL if it doesn't have one.
|
||||
* Essentially this means that the seedURL must be a directory, it
|
||||
* can't end with 'index.html', for example.
|
||||
*
|
||||
* Jetty directory listings are not compatible, as they look like
|
||||
* HREF="/full/path/to/routerInfo-...
|
||||
**/
|
||||
private void reseedOne(String seedURL, boolean echoStatus) {
|
||||
|
||||
try {
|
||||
System.setProperty("net.i2p.router.web.ReseedHandler.errorMessage","");
|
||||
System.setProperty("net.i2p.router.web.ReseedHandler.statusMessage","Reseeding: fetching seed URL.");
|
||||
System.err.println("Reseed from " + seedURL);
|
||||
URL dir = new URL(seedURL);
|
||||
byte contentRaw[] = readURL(dir);
|
||||
if (contentRaw == null) {
|
||||
System.setProperty("net.i2p.router.web.ReseedHandler.errorMessage",
|
||||
"Last reseed failed fully (failed reading seed URL). " +
|
||||
RESEED_TIPS);
|
||||
// Logging deprecated here since attemptFailed() provides better info
|
||||
_log.debug("Failed reading seed URL: " + seedURL);
|
||||
return;
|
||||
}
|
||||
String content = new String(contentRaw);
|
||||
Set urls = new HashSet();
|
||||
int cur = 0;
|
||||
int total = 0;
|
||||
while (total++ < 1000) {
|
||||
int start = content.indexOf("href=\"routerInfo-", cur);
|
||||
if (start < 0) {
|
||||
start = content.indexOf("HREF=\"routerInfo-", cur);
|
||||
if (start < 0)
|
||||
break;
|
||||
}
|
||||
|
||||
int end = content.indexOf(".dat\">", start);
|
||||
String name = content.substring(start+"href=\"routerInfo-".length(), end);
|
||||
urls.add(name);
|
||||
cur = end + 1;
|
||||
}
|
||||
if (total <= 0) {
|
||||
_log.error("Read " + contentRaw.length + " bytes from seed " + seedURL + ", but found no routerInfo URLs.");
|
||||
System.setProperty("net.i2p.router.web.ReseedHandler.errorMessage",
|
||||
"Last reseed failed fully (no routerInfo URLs at seed URL). " +
|
||||
RESEED_TIPS);
|
||||
return;
|
||||
}
|
||||
|
||||
List urlList = new ArrayList(urls);
|
||||
Collections.shuffle(urlList);
|
||||
int fetched = 0;
|
||||
int errors = 0;
|
||||
// 200 max from one URL
|
||||
for (Iterator iter = urlList.iterator(); iter.hasNext() && fetched < 200; ) {
|
||||
try {
|
||||
System.setProperty("net.i2p.router.web.ReseedHandler.statusMessage",
|
||||
"Reseeding: fetching router info from seed URL (" +
|
||||
fetched + " successful, " + errors + " errors, " + total + " total).");
|
||||
|
||||
fetchSeed(seedURL, (String)iter.next());
|
||||
fetched++;
|
||||
if (echoStatus) {
|
||||
System.out.print(".");
|
||||
if (fetched % 60 == 0)
|
||||
System.out.println();
|
||||
}
|
||||
} catch (Exception e) {
|
||||
errors++;
|
||||
}
|
||||
}
|
||||
System.err.println("Reseed got " + fetched + " router infos from " + seedURL);
|
||||
|
||||
int failPercent = 100 * errors / total;
|
||||
|
||||
// Less than 10% of failures is considered success,
|
||||
// because some routerInfos will always fail.
|
||||
if ((failPercent >= 10) && (failPercent < 90)) {
|
||||
System.setProperty("net.i2p.router.web.ReseedHandler.errorMessage",
|
||||
"Last reseed failed partly (" + failPercent + "% of " + total + "). " +
|
||||
RESEED_TIPS);
|
||||
}
|
||||
if (failPercent >= 90) {
|
||||
System.setProperty("net.i2p.router.web.ReseedHandler.errorMessage",
|
||||
"Last reseed failed (" + failPercent + "% of " + total + "). " +
|
||||
RESEED_TIPS);
|
||||
}
|
||||
// Don't go on to the next URL if we have enough
|
||||
if (fetched >= 100)
|
||||
_isRunning = false;
|
||||
} catch (Throwable t) {
|
||||
System.setProperty("net.i2p.router.web.ReseedHandler.errorMessage",
|
||||
"Last reseed failed fully (exception caught). " +
|
||||
RESEED_TIPS);
|
||||
_log.error("Error reseeding", t);
|
||||
}
|
||||
}
|
||||
|
||||
/* Since we don't return a value, we should always throw an exception if something fails. */
|
||||
private void fetchSeed(String seedURL, String peer) throws Exception {
|
||||
URL url = new URL(seedURL + (seedURL.endsWith("/") ? "" : "/") + "routerInfo-" + peer + ".dat");
|
||||
|
||||
byte data[] = readURL(url);
|
||||
if (data == null) {
|
||||
// Logging deprecated here since attemptFailed() provides better info
|
||||
_log.debug("Failed fetching seed: " + url.toString());
|
||||
throw new Exception ("Failed fetching seed.");
|
||||
}
|
||||
//System.out.println("read: " + (data != null ? data.length : -1));
|
||||
writeSeed(peer, data);
|
||||
}
|
||||
|
||||
private byte[] readURL(URL url) throws Exception {
|
||||
ByteArrayOutputStream baos = new ByteArrayOutputStream(4*1024);
|
||||
|
||||
// Do a non-proxied eepget into our ByteArrayOutputStream with 0 retries
|
||||
EepGet get = new EepGet( I2PAppContext.getGlobalContext(), false, null, -1, 0, 0, MAX_RESEED_RESPONSE_SIZE,
|
||||
null, baos, url.toString(), false, null, null);
|
||||
get.addStatusListener(ReseedRunner.this);
|
||||
if (get.fetch()) return baos.toByteArray(); else return null;
|
||||
}
|
||||
|
||||
private void writeSeed(String name, byte data[]) throws Exception {
|
||||
String dirName = "netDb"; // _context.getProperty("router.networkDatabase.dbDir", "netDb");
|
||||
File netDbDir = new File(_context.getRouterDir(), dirName);
|
||||
if (!netDbDir.exists()) {
|
||||
boolean ok = netDbDir.mkdirs();
|
||||
}
|
||||
FileOutputStream fos = new FileOutputStream(new File(netDbDir, "routerInfo-" + name + ".dat"));
|
||||
fos.write(data);
|
||||
fos.close();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public static void main(String args[]) {
|
||||
if ( (args != null) && (args.length == 1) && (!Boolean.valueOf(args[0]).booleanValue()) ) {
|
||||
System.out.println("Not reseeding, as requested");
|
||||
return; // not reseeding on request
|
||||
}
|
||||
System.out.println("Reseeding");
|
||||
ReseedHandler reseedHandler = new ReseedHandler();
|
||||
reseedHandler.requestReseed();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -144,7 +144,8 @@ public class RouterConsoleRunner {
|
||||
storeWebAppProperties(props);
|
||||
try {
|
||||
_server.start();
|
||||
} catch (Exception me) {
|
||||
} catch (Throwable me) {
|
||||
// NoClassFoundDefError from a webapp is a throwable, not an exception
|
||||
System.err.println("WARNING: Error starting one or more listeners of the Router Console server.\n" +
|
||||
"If your console is still accessible at http://127.0.0.1:7657/,\n" +
|
||||
"this may be a problem only with binding to the IPV6 address ::1.\n" +
|
||||
@ -158,25 +159,6 @@ public class RouterConsoleRunner {
|
||||
t.printStackTrace();
|
||||
}
|
||||
|
||||
// we check the i2p installation directory (.) for a flag telling us not to reseed,
|
||||
// but also check the home directory for that flag too, since new users installing i2p
|
||||
// don't have an installation directory that they can put the flag in yet.
|
||||
File noReseedFile = new File(new File(System.getProperty("user.home")), ".i2pnoreseed");
|
||||
File noReseedFileAlt1 = new File(new File(System.getProperty("user.home")), "noreseed.i2p");
|
||||
File noReseedFileAlt2 = new File(I2PAppContext.getGlobalContext().getConfigDir(), ".i2pnoreseed");
|
||||
File noReseedFileAlt3 = new File(I2PAppContext.getGlobalContext().getConfigDir(), "noreseed.i2p");
|
||||
if (!noReseedFile.exists() && !noReseedFileAlt1.exists() && !noReseedFileAlt2.exists() && !noReseedFileAlt3.exists()) {
|
||||
File netDb = new File(I2PAppContext.getGlobalContext().getRouterDir(), "netDb");
|
||||
// sure, some of them could be "my.info" or various leaseSet- files, but chances are,
|
||||
// if someone has those files, they've already been seeded (at least enough to let them
|
||||
// get i2p started - they can reseed later in the web console)
|
||||
String names[] = (netDb.exists() ? netDb.list() : null);
|
||||
if ( (names == null) || (names.length < 15) ) {
|
||||
ReseedHandler reseedHandler = new ReseedHandler();
|
||||
reseedHandler.requestReseed();
|
||||
}
|
||||
}
|
||||
|
||||
NewsFetcher fetcher = NewsFetcher.getInstance(I2PAppContext.getGlobalContext());
|
||||
I2PThread t = new I2PThread(fetcher, "NewsFetcher");
|
||||
t.setDaemon(true);
|
||||
|
||||
@ -85,8 +85,9 @@ public class SummaryHelper extends HelperBase {
|
||||
}
|
||||
|
||||
public boolean allowReseed() {
|
||||
return (_context.netDb().getKnownRouters() < 30) ||
|
||||
Boolean.valueOf(_context.getProperty("i2p.alwaysAllowReseed", "false")).booleanValue();
|
||||
return _context.netDb().isInitialized() &&
|
||||
((_context.netDb().getKnownRouters() < 30) ||
|
||||
Boolean.valueOf(_context.getProperty("i2p.alwaysAllowReseed")).booleanValue());
|
||||
}
|
||||
|
||||
public int getAllPeers() { return _context.netDb().getKnownRouters(); }
|
||||
|
||||
@ -58,6 +58,8 @@ public abstract class NetworkDatabaseFacade implements Service {
|
||||
public abstract Set<Hash> getAllRouters();
|
||||
public int getKnownRouters() { return 0; }
|
||||
public int getKnownLeaseSets() { return 0; }
|
||||
public boolean isInitialized() { return true; }
|
||||
public void rescan() {}
|
||||
public void renderRouterInfoHTML(Writer out, String s) throws IOException {}
|
||||
public void renderLeaseSetHTML(Writer out) throws IOException {}
|
||||
public void renderStatusHTML(Writer out, boolean b) throws IOException {}
|
||||
|
||||
@ -14,6 +14,7 @@ import net.i2p.data.DataStructure;
|
||||
import net.i2p.data.Hash;
|
||||
|
||||
public interface DataStore {
|
||||
public boolean isInitialized();
|
||||
public boolean isKnown(Hash key);
|
||||
public DataStructure get(Hash key);
|
||||
public DataStructure get(Hash key, boolean persist);
|
||||
@ -24,6 +25,7 @@ public interface DataStore {
|
||||
public Set getKeys();
|
||||
public void stop();
|
||||
public void restart();
|
||||
public void rescan();
|
||||
public int countLeaseSets();
|
||||
|
||||
}
|
||||
|
||||
@ -139,6 +139,11 @@ public class KademliaNetworkDatabaseFacade extends NetworkDatabaseFacade {
|
||||
context.statManager().createRateStat("netDb.exploreKeySet", "how many keys are queued for exploration?", "NetworkDatabase", new long[] { 10*60*1000 });
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isInitialized() {
|
||||
return _initialized && _ds != null && _ds.isInitialized();
|
||||
}
|
||||
|
||||
protected PeerSelector createPeerSelector() { return new PeerSelector(_context); }
|
||||
public PeerSelector getPeerSelector() { return _peerSelector; }
|
||||
|
||||
@ -177,6 +182,7 @@ public class KademliaNetworkDatabaseFacade extends NetworkDatabaseFacade {
|
||||
public void shutdown() {
|
||||
_initialized = false;
|
||||
_kb = null;
|
||||
if (_ds != null)
|
||||
_ds.stop();
|
||||
_ds = null;
|
||||
_exploreKeys.clear(); // hope this doesn't cause an explosion, it shouldn't.
|
||||
@ -203,6 +209,12 @@ public class KademliaNetworkDatabaseFacade extends NetworkDatabaseFacade {
|
||||
publish(ri);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void rescan() {
|
||||
if (isInitialized())
|
||||
_ds.rescan();
|
||||
}
|
||||
|
||||
String getDbDir() { return _dbDir; }
|
||||
|
||||
public void startup() {
|
||||
|
||||
@ -26,6 +26,7 @@ import net.i2p.data.RouterInfo;
|
||||
import net.i2p.router.JobImpl;
|
||||
import net.i2p.router.Router;
|
||||
import net.i2p.router.RouterContext;
|
||||
import net.i2p.router.networkdb.reseed.ReseedChecker;
|
||||
import net.i2p.util.I2PThread;
|
||||
import net.i2p.util.Log;
|
||||
|
||||
@ -39,6 +40,8 @@ class PersistentDataStore extends TransientDataStore {
|
||||
private String _dbDir;
|
||||
private KademliaNetworkDatabaseFacade _facade;
|
||||
private Writer _writer;
|
||||
private ReadJob _readJob;
|
||||
private boolean _initialized;
|
||||
|
||||
private final static int READ_DELAY = 60*1000;
|
||||
|
||||
@ -47,7 +50,8 @@ class PersistentDataStore extends TransientDataStore {
|
||||
_log = ctx.logManager().getLog(PersistentDataStore.class);
|
||||
_dbDir = dbDir;
|
||||
_facade = facade;
|
||||
_context.jobQueue().addJob(new ReadJob());
|
||||
_readJob = new ReadJob();
|
||||
_context.jobQueue().addJob(_readJob);
|
||||
ctx.statManager().createRateStat("netDb.writeClobber", "How often we clobber a pending netDb write", "NetworkDatabase", new long[] { 20*60*1000 });
|
||||
ctx.statManager().createRateStat("netDb.writePending", "How many pending writes are there", "NetworkDatabase", new long[] { 60*1000 });
|
||||
ctx.statManager().createRateStat("netDb.writeOut", "How many we wrote", "NetworkDatabase", new long[] { 20*60*1000 });
|
||||
@ -59,6 +63,9 @@ class PersistentDataStore extends TransientDataStore {
|
||||
writer.start();
|
||||
}
|
||||
|
||||
public boolean isInitialized() { return _initialized; }
|
||||
|
||||
// this doesn't stop the read job or the writer, maybe it should?
|
||||
@Override
|
||||
public void stop() {
|
||||
super.stop();
|
||||
@ -71,6 +78,11 @@ class PersistentDataStore extends TransientDataStore {
|
||||
_dbDir = _facade.getDbDir();
|
||||
}
|
||||
|
||||
public void rescan() {
|
||||
if (_initialized)
|
||||
_readJob.wakeup();
|
||||
}
|
||||
|
||||
@Override
|
||||
public DataStructure get(Hash key) {
|
||||
return get(key, true);
|
||||
@ -317,6 +329,10 @@ class PersistentDataStore extends TransientDataStore {
|
||||
requeue(READ_DELAY);
|
||||
}
|
||||
|
||||
public void wakeup() {
|
||||
requeue(0);
|
||||
}
|
||||
|
||||
private void readFiles() {
|
||||
int routerCount = 0;
|
||||
try {
|
||||
@ -336,9 +352,10 @@ class PersistentDataStore extends TransientDataStore {
|
||||
_log.error("Error reading files in the db dir", ioe);
|
||||
}
|
||||
|
||||
if ( (routerCount <= 5) && (!_alreadyWarned) ) {
|
||||
_log.error("Very few routerInfo files remaining - please reseed");
|
||||
if (!_alreadyWarned) {
|
||||
ReseedChecker.checkReseed(_context, routerCount);
|
||||
_alreadyWarned = true;
|
||||
_initialized = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -35,6 +35,8 @@ class TransientDataStore implements DataStore {
|
||||
_log.info("Data Store initialized");
|
||||
}
|
||||
|
||||
public boolean isInitialized() { return true; }
|
||||
|
||||
public void stop() {
|
||||
_data.clear();
|
||||
}
|
||||
@ -43,6 +45,8 @@ class TransientDataStore implements DataStore {
|
||||
stop();
|
||||
}
|
||||
|
||||
public void rescan() {}
|
||||
|
||||
public Set getKeys() {
|
||||
return new HashSet(_data.keySet());
|
||||
}
|
||||
|
||||
@ -0,0 +1,46 @@
|
||||
package net.i2p.router.networkdb.reseed;
|
||||
|
||||
import java.io.File;
|
||||
|
||||
import net.i2p.router.RouterContext;
|
||||
import net.i2p.util.Log;
|
||||
|
||||
/**
|
||||
* Moved from RouterConsoleRunner.java
|
||||
*
|
||||
* Reseeding is not strictly a router function, it used to be
|
||||
* in the routerconsole app, but this made it impossible to
|
||||
* bootstrap an embedded router lacking a routerconsole,
|
||||
* in iMule or android for example, without additional modifications.
|
||||
*
|
||||
* Also, as this is now called from PersistentDataStore, not from the
|
||||
* routerconsole, we can get started as soon as the netdb has read
|
||||
* the netDb/ directory, not when the console starts,
|
||||
router/java/src/net/i2p/router/networkdb/eseed/ReseedChecker.java
|
||||
*/
|
||||
public class ReseedChecker {
|
||||
|
||||
private static final int MINIMUM = 15;
|
||||
|
||||
public static void checkReseed(RouterContext context, int count) {
|
||||
if (count >= MINIMUM)
|
||||
return;
|
||||
|
||||
// we check the i2p installation directory for a flag telling us not to reseed,
|
||||
// but also check the home directory for that flag too, since new users installing i2p
|
||||
// don't have an installation directory that they can put the flag in yet.
|
||||
File noReseedFile = new File(new File(System.getProperty("user.home")), ".i2pnoreseed");
|
||||
File noReseedFileAlt1 = new File(new File(System.getProperty("user.home")), "noreseed.i2p");
|
||||
File noReseedFileAlt2 = new File(context.getConfigDir(), ".i2pnoreseed");
|
||||
File noReseedFileAlt3 = new File(context.getConfigDir(), "noreseed.i2p");
|
||||
if (!noReseedFile.exists() && !noReseedFileAlt1.exists() && !noReseedFileAlt2.exists() && !noReseedFileAlt3.exists()) {
|
||||
Log _log = context.logManager().getLog(ReseedChecker.class);
|
||||
if (count <= 1)
|
||||
_log.error("Downloading peer router information for a new I2P installation");
|
||||
else
|
||||
_log.error("Very few routerInfo files remaining - reseeding now");
|
||||
Reseeder reseeder = new Reseeder(context);
|
||||
reseeder.requestReseed();
|
||||
}
|
||||
}
|
||||
}
|
||||
266
router/java/src/net/i2p/router/networkdb/reseed/Reseeder.java
Normal file
266
router/java/src/net/i2p/router/networkdb/reseed/Reseeder.java
Normal file
@ -0,0 +1,266 @@
|
||||
package net.i2p.router.networkdb.reseed;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.File;
|
||||
import java.io.FileOutputStream;
|
||||
import java.net.URL;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.HashSet;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.StringTokenizer;
|
||||
|
||||
import net.i2p.I2PAppContext;
|
||||
import net.i2p.router.RouterContext;
|
||||
import net.i2p.util.EepGet;
|
||||
import net.i2p.util.I2PThread;
|
||||
import net.i2p.util.Log;
|
||||
|
||||
/**
|
||||
* Moved from ReseedHandler in routerconsole. See ReseedChecker for additional comments.
|
||||
*
|
||||
* Handler to deal with reseed requests. This will reseed from the URLs
|
||||
* specified below unless the I2P configuration property "i2p.reseedURL" is
|
||||
* set. It always writes to ./netDb/, so don't mess with that.
|
||||
*
|
||||
*/
|
||||
public class Reseeder {
|
||||
private static ReseedRunner _reseedRunner;
|
||||
private RouterContext _context;
|
||||
private Log _log;
|
||||
|
||||
// Reject unreasonably big files, because we download into a ByteArrayOutputStream.
|
||||
private static final long MAX_RESEED_RESPONSE_SIZE = 8 * 1024 * 1024;
|
||||
|
||||
private static final String DEFAULT_SEED_URL = "http://i2pdb.tin0.de/netDb/,http://netdb.i2p2.de/";
|
||||
|
||||
public Reseeder(RouterContext ctx) {
|
||||
_context = ctx;
|
||||
_log = ctx.logManager().getLog(Reseeder.class);
|
||||
}
|
||||
|
||||
public void requestReseed() {
|
||||
synchronized (Reseeder.class) {
|
||||
if (_reseedRunner == null)
|
||||
_reseedRunner = new ReseedRunner();
|
||||
if (_reseedRunner.isRunning()) {
|
||||
return;
|
||||
} else {
|
||||
System.setProperty("net.i2p.router.web.Reseeder.reseedInProgress", "true");
|
||||
I2PThread reseed = new I2PThread(_reseedRunner, "Reseed");
|
||||
reseed.start();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public class ReseedRunner implements Runnable, EepGet.StatusListener {
|
||||
private boolean _isRunning;
|
||||
|
||||
public ReseedRunner() {
|
||||
_isRunning = false;
|
||||
System.setProperty("net.i2p.router.web.Reseeder.statusMessage","Reseeding.");
|
||||
}
|
||||
public boolean isRunning() { return _isRunning; }
|
||||
public void run() {
|
||||
_isRunning = true;
|
||||
System.out.println("Reseed start");
|
||||
reseed(false);
|
||||
System.out.println("Reseed complete");
|
||||
System.setProperty("net.i2p.router.web.Reseeder.reseedInProgress", "false");
|
||||
_isRunning = false;
|
||||
}
|
||||
|
||||
// EepGet status listeners
|
||||
public void attemptFailed(String url, long bytesTransferred, long bytesRemaining, int currentAttempt, int numRetries, Exception cause) {
|
||||
// Since readURL() runs an EepGet with 0 retries,
|
||||
// we can report errors with attemptFailed() instead of transferFailed().
|
||||
// It has the benefit of providing cause of failure, which helps resolve issues.
|
||||
if (_log.shouldLog(Log.ERROR)) _log.error("EepGet failed on " + url, cause);
|
||||
}
|
||||
public void bytesTransferred(long alreadyTransferred, int currentWrite, long bytesTransferred, long bytesRemaining, String url) {}
|
||||
public void transferComplete(long alreadyTransferred, long bytesTransferred, long bytesRemaining, String url, String outputFile, boolean notModified) {}
|
||||
public void transferFailed(String url, long bytesTransferred, long bytesRemaining, int currentAttempt) {}
|
||||
public void headerReceived(String url, int attemptNum, String key, String val) {}
|
||||
public void attempting(String url) {}
|
||||
// End of EepGet status listeners
|
||||
|
||||
/**
|
||||
* Reseed has been requested, so lets go ahead and do it. Fetch all of
|
||||
* the routerInfo-*.dat files from the specified URL (or the default) and
|
||||
* save them into this router's netDb dir.
|
||||
*
|
||||
*/
|
||||
private static final String RESEED_TIPS =
|
||||
"Ensure that nothing blocks outbound HTTP, check <a href=logs.jsp>logs</a> " +
|
||||
"and if nothing helps, read FAQ about reseeding manually.";
|
||||
|
||||
private void reseed(boolean echoStatus) {
|
||||
List URLList = new ArrayList();
|
||||
String URLs = _context.getProperty("i2p.reseedURL", DEFAULT_SEED_URL);
|
||||
StringTokenizer tok = new StringTokenizer(URLs, " ,");
|
||||
while (tok.hasMoreTokens())
|
||||
URLList.add(tok.nextToken().trim());
|
||||
Collections.shuffle(URLList);
|
||||
for (int i = 0; i < URLList.size() && _isRunning; i++)
|
||||
reseedOne((String) URLList.get(i), echoStatus);
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch a directory listing and then up to 200 routerInfo files in the listing.
|
||||
* The listing must contain (exactly) strings that match:
|
||||
* href="routerInfo-{hash}.dat">
|
||||
* OR
|
||||
* HREF="routerInfo-{hash}.dat">
|
||||
* and then it fetches the files
|
||||
* {seedURL}routerInfo-{hash}.dat
|
||||
* after appending a '/' to seedURL if it doesn't have one.
|
||||
* Essentially this means that the seedURL must be a directory, it
|
||||
* can't end with 'index.html', for example.
|
||||
*
|
||||
* Jetty directory listings are not compatible, as they look like
|
||||
* HREF="/full/path/to/routerInfo-...
|
||||
**/
|
||||
private void reseedOne(String seedURL, boolean echoStatus) {
|
||||
|
||||
try {
|
||||
System.setProperty("net.i2p.router.web.Reseeder.errorMessage","");
|
||||
System.setProperty("net.i2p.router.web.Reseeder.statusMessage","Reseeding: fetching seed URL.");
|
||||
System.err.println("Reseed from " + seedURL);
|
||||
URL dir = new URL(seedURL);
|
||||
byte contentRaw[] = readURL(dir);
|
||||
if (contentRaw == null) {
|
||||
System.setProperty("net.i2p.router.web.Reseeder.errorMessage",
|
||||
"Last reseed failed fully (failed reading seed URL). " +
|
||||
RESEED_TIPS);
|
||||
// Logging deprecated here since attemptFailed() provides better info
|
||||
_log.debug("Failed reading seed URL: " + seedURL);
|
||||
return;
|
||||
}
|
||||
String content = new String(contentRaw);
|
||||
Set urls = new HashSet();
|
||||
int cur = 0;
|
||||
int total = 0;
|
||||
while (total++ < 1000) {
|
||||
int start = content.indexOf("href=\"routerInfo-", cur);
|
||||
if (start < 0) {
|
||||
start = content.indexOf("HREF=\"routerInfo-", cur);
|
||||
if (start < 0)
|
||||
break;
|
||||
}
|
||||
|
||||
int end = content.indexOf(".dat\">", start);
|
||||
String name = content.substring(start+"href=\"routerInfo-".length(), end);
|
||||
urls.add(name);
|
||||
cur = end + 1;
|
||||
}
|
||||
if (total <= 0) {
|
||||
_log.error("Read " + contentRaw.length + " bytes from seed " + seedURL + ", but found no routerInfo URLs.");
|
||||
System.setProperty("net.i2p.router.web.Reseeder.errorMessage",
|
||||
"Last reseed failed fully (no routerInfo URLs at seed URL). " +
|
||||
RESEED_TIPS);
|
||||
return;
|
||||
}
|
||||
|
||||
List urlList = new ArrayList(urls);
|
||||
Collections.shuffle(urlList);
|
||||
int fetched = 0;
|
||||
int errors = 0;
|
||||
// 200 max from one URL
|
||||
for (Iterator iter = urlList.iterator(); iter.hasNext() && fetched < 200; ) {
|
||||
try {
|
||||
System.setProperty("net.i2p.router.web.Reseeder.statusMessage",
|
||||
"Reseeding: fetching router info from seed URL (" +
|
||||
fetched + " successful, " + errors + " errors, " + total + " total).");
|
||||
|
||||
fetchSeed(seedURL, (String)iter.next());
|
||||
fetched++;
|
||||
if (echoStatus) {
|
||||
System.out.print(".");
|
||||
if (fetched % 60 == 0)
|
||||
System.out.println();
|
||||
}
|
||||
} catch (Exception e) {
|
||||
errors++;
|
||||
}
|
||||
}
|
||||
System.err.println("Reseed got " + fetched + " router infos from " + seedURL);
|
||||
|
||||
int failPercent = 100 * errors / total;
|
||||
|
||||
// Less than 10% of failures is considered success,
|
||||
// because some routerInfos will always fail.
|
||||
if ((failPercent >= 10) && (failPercent < 90)) {
|
||||
System.setProperty("net.i2p.router.web.Reseeder.errorMessage",
|
||||
"Last reseed failed partly (" + failPercent + "% of " + total + "). " +
|
||||
RESEED_TIPS);
|
||||
}
|
||||
if (failPercent >= 90) {
|
||||
System.setProperty("net.i2p.router.web.Reseeder.errorMessage",
|
||||
"Last reseed failed (" + failPercent + "% of " + total + "). " +
|
||||
RESEED_TIPS);
|
||||
}
|
||||
if (fetched > 0)
|
||||
_context.netDb().rescan();
|
||||
// Don't go on to the next URL if we have enough
|
||||
if (fetched >= 100)
|
||||
_isRunning = false;
|
||||
} catch (Throwable t) {
|
||||
System.setProperty("net.i2p.router.web.Reseeder.errorMessage",
|
||||
"Last reseed failed fully (exception caught). " +
|
||||
RESEED_TIPS);
|
||||
_log.error("Error reseeding", t);
|
||||
}
|
||||
}
|
||||
|
||||
/* Since we don't return a value, we should always throw an exception if something fails. */
|
||||
private void fetchSeed(String seedURL, String peer) throws Exception {
|
||||
URL url = new URL(seedURL + (seedURL.endsWith("/") ? "" : "/") + "routerInfo-" + peer + ".dat");
|
||||
|
||||
byte data[] = readURL(url);
|
||||
if (data == null) {
|
||||
// Logging deprecated here since attemptFailed() provides better info
|
||||
_log.debug("Failed fetching seed: " + url.toString());
|
||||
throw new Exception ("Failed fetching seed.");
|
||||
}
|
||||
//System.out.println("read: " + (data != null ? data.length : -1));
|
||||
writeSeed(peer, data);
|
||||
}
|
||||
|
||||
private byte[] readURL(URL url) throws Exception {
|
||||
ByteArrayOutputStream baos = new ByteArrayOutputStream(4*1024);
|
||||
|
||||
// Do a non-proxied eepget into our ByteArrayOutputStream with 0 retries
|
||||
EepGet get = new EepGet( I2PAppContext.getGlobalContext(), false, null, -1, 0, 0, MAX_RESEED_RESPONSE_SIZE,
|
||||
null, baos, url.toString(), false, null, null);
|
||||
get.addStatusListener(ReseedRunner.this);
|
||||
if (get.fetch()) return baos.toByteArray(); else return null;
|
||||
}
|
||||
|
||||
private void writeSeed(String name, byte data[]) throws Exception {
|
||||
String dirName = "netDb"; // _context.getProperty("router.networkDatabase.dbDir", "netDb");
|
||||
File netDbDir = new File(_context.getRouterDir(), dirName);
|
||||
if (!netDbDir.exists()) {
|
||||
boolean ok = netDbDir.mkdirs();
|
||||
}
|
||||
FileOutputStream fos = new FileOutputStream(new File(netDbDir, "routerInfo-" + name + ".dat"));
|
||||
fos.write(data);
|
||||
fos.close();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/******
|
||||
public static void main(String args[]) {
|
||||
if ( (args != null) && (args.length == 1) && (!Boolean.valueOf(args[0]).booleanValue()) ) {
|
||||
System.out.println("Not reseeding, as requested");
|
||||
return; // not reseeding on request
|
||||
}
|
||||
System.out.println("Reseeding");
|
||||
Reseeder reseedHandler = new Reseeder();
|
||||
reseedHandler.requestReseed();
|
||||
}
|
||||
******/
|
||||
}
|
||||
Reference in New Issue
Block a user