echelon/i2p.i2p
1
0
Fork 0
forked from I2P_Developers/i2p.i2p
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

* Reseeding / NetDb:

Browse Source 
- Move reseeding from the routerconsole app to
        the router, so that we can bootstrap an embedded router lacking a routerconsole
        (iMule or android for example), without additional modifications.
        This allows better integration between the reseeding function
        and the netDb.
      - Call reseed from PersistentDataStore, not from the
        routerconsole init, and start seeding as soon as the netdb has read
        the netDb/ directory, not when the console starts.
      - Wake up the netdb reader as soon as reseeding is done,
        rather than waiting up to 60s.
      - Don't display the reseed button on the console until the
        netdb initialization is done.
    * NetDb:
      - Fix an NPE on early shutdown
    * RouterConsoleRunner:
      - Catch a class not found error better
This commit is contained in:
zzz
2009-06-15 21:58:28 +00:00
parent 71f3cd648f
commit 7aa9949332
10 changed files with 365 additions and 287 deletions
Download Patch File Download Diff File Expand all files Collapse all files

266
apps/routerconsole/java/src/net/i2p/router/web/ReseedHandler.java
View File

@ -1,63 +1,21 @@
package net.i2p.router.web; package net.i2p.router.web;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.net.URL;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import java.util.StringTokenizer;
import net.i2p.I2PAppContext;
import net.i2p.router.RouterContext; import net.i2p.router.RouterContext;
import net.i2p.util.EepGet; import net.i2p.router.networkdb.reseed.Reseeder;
import net.i2p.util.I2PThread;
import net.i2p.util.Log;
/** /**
* Handler to deal with reseed requests. This will reseed from the URL * Handler to deal with reseed requests.
* http://i2pdb.tin0.de/netDb/ unless the I2P configuration property "i2p.reseedURL" is
* set. It always writes to ./netDb/, so don't mess with that.
*
*/ */
public class ReseedHandler { public class ReseedHandler extends HelperBase {
private static ReseedRunner _reseedRunner; private static Reseeder _reseedRunner;
private RouterContext _context;
private Log _log;
// Reject unreasonably big files, because we download into a ByteArrayOutputStream.
private static final long MAX_RESEED_RESPONSE_SIZE = 8 * 1024 * 1024;
private static final String DEFAULT_SEED_URL = "http://i2pdb.tin0.de/netDb/,http://netdb.i2p2.de/";
public ReseedHandler() { public ReseedHandler() {
this(ContextHelper.getContext(null)); this(ContextHelper.getContext(null));
} }
public ReseedHandler(RouterContext ctx) { public ReseedHandler(RouterContext ctx) {
_context = ctx; _context = ctx;
_log = ctx.logManager().getLog(ReseedHandler.class);
} }
/**
* Configure this bean to query a particular router context
*
* @param contextId begging few characters of the routerHash, or null to pick
* the first one we come across.
*/
public void setContextId(String contextId) {
try {
_context = ContextHelper.getContext(contextId);
_log = _context.logManager().getLog(ReseedHandler.class);
} catch (Throwable t) {
t.printStackTrace();
}
}
public void setReseedNonce(String nonce) { public void setReseedNonce(String nonce) {
if (nonce == null) return; if (nonce == null) return;
if (nonce.equals(System.getProperty("net.i2p.router.web.ReseedHandler.nonce")) || if (nonce.equals(System.getProperty("net.i2p.router.web.ReseedHandler.nonce")) ||
@ -69,220 +27,8 @@ public class ReseedHandler {
public void requestReseed() { public void requestReseed() {
synchronized (ReseedHandler.class) { synchronized (ReseedHandler.class) {
if (_reseedRunner == null) if (_reseedRunner == null)
_reseedRunner = new ReseedRunner(); _reseedRunner = new Reseeder(_context);
if (_reseedRunner.isRunning()) { _reseedRunner.requestReseed();
return;
} else {
System.setProperty("net.i2p.router.web.ReseedHandler.reseedInProgress", "true");
I2PThread reseed = new I2PThread(_reseedRunner, "Reseed");
reseed.start();
}
} }
} }
public class ReseedRunner implements Runnable, EepGet.StatusListener {
private boolean _isRunning;
public ReseedRunner() {
_isRunning = false;
System.setProperty("net.i2p.router.web.ReseedHandler.statusMessage","Reseeding.");
}
public boolean isRunning() { return _isRunning; }
public void run() {
_isRunning = true;
System.out.println("Reseed start");
reseed(false);
System.out.println("Reseed complete");
System.setProperty("net.i2p.router.web.ReseedHandler.reseedInProgress", "false");
_isRunning = false;
}
// EepGet status listeners
public void attemptFailed(String url, long bytesTransferred, long bytesRemaining, int currentAttempt, int numRetries, Exception cause) {
// Since readURL() runs an EepGet with 0 retries,
// we can report errors with attemptFailed() instead of transferFailed().
// It has the benefit of providing cause of failure, which helps resolve issues.
if (_log.shouldLog(Log.ERROR)) _log.error("EepGet failed on " + url, cause);
}
public void bytesTransferred(long alreadyTransferred, int currentWrite, long bytesTransferred, long bytesRemaining, String url) {}
public void transferComplete(long alreadyTransferred, long bytesTransferred, long bytesRemaining, String url, String outputFile, boolean notModified) {}
public void transferFailed(String url, long bytesTransferred, long bytesRemaining, int currentAttempt) {}
public void headerReceived(String url, int attemptNum, String key, String val) {}
public void attempting(String url) {}
// End of EepGet status listeners
/**
* Reseed has been requested, so lets go ahead and do it. Fetch all of
* the routerInfo-*.dat files from the specified URL (or the default) and
* save them into this router's netDb dir.
*
*/
private static final String RESEED_TIPS =
"Ensure that nothing blocks outbound HTTP, check <a href=logs.jsp>logs</a> " +
"and if nothing helps, read FAQ about reseeding manually.";
private void reseed(boolean echoStatus) {
List URLList = new ArrayList();
String URLs = _context.getProperty("i2p.reseedURL", DEFAULT_SEED_URL);
StringTokenizer tok = new StringTokenizer(URLs, " ,");
while (tok.hasMoreTokens())
URLList.add(tok.nextToken().trim());
Collections.shuffle(URLList);
for (int i = 0; i < URLList.size() && _isRunning; i++)
reseedOne((String) URLList.get(i), echoStatus);
}
/**
* Fetch a directory listing and then up to 200 routerInfo files in the listing.
* The listing must contain (exactly) strings that match:
* href="routerInfo-{hash}.dat">
* OR
* HREF="routerInfo-{hash}.dat">
* and then it fetches the files
* {seedURL}routerInfo-{hash}.dat
* after appending a '/' to seedURL if it doesn't have one.
* Essentially this means that the seedURL must be a directory, it
* can't end with 'index.html', for example.
*
* Jetty directory listings are not compatible, as they look like
* HREF="/full/path/to/routerInfo-...
**/
private void reseedOne(String seedURL, boolean echoStatus) {
try {
System.setProperty("net.i2p.router.web.ReseedHandler.errorMessage","");
System.setProperty("net.i2p.router.web.ReseedHandler.statusMessage","Reseeding: fetching seed URL.");
System.err.println("Reseed from " + seedURL);
URL dir = new URL(seedURL);
byte contentRaw[] = readURL(dir);
if (contentRaw == null) {
System.setProperty("net.i2p.router.web.ReseedHandler.errorMessage",
"Last reseed failed fully (failed reading seed URL). " +
RESEED_TIPS);
// Logging deprecated here since attemptFailed() provides better info
_log.debug("Failed reading seed URL: " + seedURL);
return;
}
String content = new String(contentRaw);
Set urls = new HashSet();
int cur = 0;
int total = 0;
while (total++ < 1000) {
int start = content.indexOf("href=\"routerInfo-", cur);
if (start < 0) {
start = content.indexOf("HREF=\"routerInfo-", cur);
if (start < 0)
break;
}
int end = content.indexOf(".dat\">", start);
String name = content.substring(start+"href=\"routerInfo-".length(), end);
urls.add(name);
cur = end + 1;
}
if (total <= 0) {
_log.error("Read " + contentRaw.length + " bytes from seed " + seedURL + ", but found no routerInfo URLs.");
System.setProperty("net.i2p.router.web.ReseedHandler.errorMessage",
"Last reseed failed fully (no routerInfo URLs at seed URL). " +
RESEED_TIPS);
return;
}
List urlList = new ArrayList(urls);
Collections.shuffle(urlList);
int fetched = 0;
int errors = 0;
// 200 max from one URL
for (Iterator iter = urlList.iterator(); iter.hasNext() && fetched < 200; ) {
try {
System.setProperty("net.i2p.router.web.ReseedHandler.statusMessage",
"Reseeding: fetching router info from seed URL (" +
fetched + " successful, " + errors + " errors, " + total + " total).");
fetchSeed(seedURL, (String)iter.next());
fetched++;
if (echoStatus) {
System.out.print(".");
if (fetched % 60 == 0)
System.out.println();
}
} catch (Exception e) {
errors++;
}
}
System.err.println("Reseed got " + fetched + " router infos from " + seedURL);
int failPercent = 100 * errors / total;
// Less than 10% of failures is considered success,
// because some routerInfos will always fail.
if ((failPercent >= 10) && (failPercent < 90)) {
System.setProperty("net.i2p.router.web.ReseedHandler.errorMessage",
"Last reseed failed partly (" + failPercent + "% of " + total + "). " +
RESEED_TIPS);
}
if (failPercent >= 90) {
System.setProperty("net.i2p.router.web.ReseedHandler.errorMessage",
"Last reseed failed (" + failPercent + "% of " + total + "). " +
RESEED_TIPS);
}
// Don't go on to the next URL if we have enough
if (fetched >= 100)
_isRunning = false;
} catch (Throwable t) {
System.setProperty("net.i2p.router.web.ReseedHandler.errorMessage",
"Last reseed failed fully (exception caught). " +
RESEED_TIPS);
_log.error("Error reseeding", t);
}
}
/* Since we don't return a value, we should always throw an exception if something fails. */
private void fetchSeed(String seedURL, String peer) throws Exception {
URL url = new URL(seedURL + (seedURL.endsWith("/") ? "" : "/") + "routerInfo-" + peer + ".dat");
byte data[] = readURL(url);
if (data == null) {
// Logging deprecated here since attemptFailed() provides better info
_log.debug("Failed fetching seed: " + url.toString());
throw new Exception ("Failed fetching seed.");
}
//System.out.println("read: " + (data != null ? data.length : -1));
writeSeed(peer, data);
}
private byte[] readURL(URL url) throws Exception {
ByteArrayOutputStream baos = new ByteArrayOutputStream(4*1024);
// Do a non-proxied eepget into our ByteArrayOutputStream with 0 retries
EepGet get = new EepGet( I2PAppContext.getGlobalContext(), false, null, -1, 0, 0, MAX_RESEED_RESPONSE_SIZE,
null, baos, url.toString(), false, null, null);
get.addStatusListener(ReseedRunner.this);
if (get.fetch()) return baos.toByteArray(); else return null;
}
private void writeSeed(String name, byte data[]) throws Exception {
String dirName = "netDb"; // _context.getProperty("router.networkDatabase.dbDir", "netDb");
File netDbDir = new File(_context.getRouterDir(), dirName);
if (!netDbDir.exists()) {
boolean ok = netDbDir.mkdirs();
}
FileOutputStream fos = new FileOutputStream(new File(netDbDir, "routerInfo-" + name + ".dat"));
fos.write(data);
fos.close();
}
}
public static void main(String args[]) {
if ( (args != null) && (args.length == 1) && (!Boolean.valueOf(args[0]).booleanValue()) ) {
System.out.println("Not reseeding, as requested");
return; // not reseeding on request
}
System.out.println("Reseeding");
ReseedHandler reseedHandler = new ReseedHandler();
reseedHandler.requestReseed();
}
} }

22
apps/routerconsole/java/src/net/i2p/router/web/RouterConsoleRunner.java
View File

@ -144,7 +144,8 @@ public class RouterConsoleRunner {
storeWebAppProperties(props); storeWebAppProperties(props);
try { try {
_server.start(); _server.start();
} catch (Exception me) { } catch (Throwable me) {
// NoClassFoundDefError from a webapp is a throwable, not an exception
System.err.println("WARNING: Error starting one or more listeners of the Router Console server.\n" + System.err.println("WARNING: Error starting one or more listeners of the Router Console server.\n" +
"If your console is still accessible at http://127.0.0.1:7657/,\n" + "If your console is still accessible at http://127.0.0.1:7657/,\n" +
"this may be a problem only with binding to the IPV6 address ::1.\n" + "this may be a problem only with binding to the IPV6 address ::1.\n" +
@ -158,25 +159,6 @@ public class RouterConsoleRunner {
t.printStackTrace(); t.printStackTrace();
} }
// we check the i2p installation directory (.) for a flag telling us not to reseed,
// but also check the home directory for that flag too, since new users installing i2p
// don't have an installation directory that they can put the flag in yet.
File noReseedFile = new File(new File(System.getProperty("user.home")), ".i2pnoreseed");
File noReseedFileAlt1 = new File(new File(System.getProperty("user.home")), "noreseed.i2p");
File noReseedFileAlt2 = new File(I2PAppContext.getGlobalContext().getConfigDir(), ".i2pnoreseed");
File noReseedFileAlt3 = new File(I2PAppContext.getGlobalContext().getConfigDir(), "noreseed.i2p");
if (!noReseedFile.exists() && !noReseedFileAlt1.exists() && !noReseedFileAlt2.exists() && !noReseedFileAlt3.exists()) {
File netDb = new File(I2PAppContext.getGlobalContext().getRouterDir(), "netDb");
// sure, some of them could be "my.info" or various leaseSet- files, but chances are,
// if someone has those files, they've already been seeded (at least enough to let them
// get i2p started - they can reseed later in the web console)
String names[] = (netDb.exists() ? netDb.list() : null);
if ( (names == null) || (names.length < 15) ) {
ReseedHandler reseedHandler = new ReseedHandler();
reseedHandler.requestReseed();
}
}
NewsFetcher fetcher = NewsFetcher.getInstance(I2PAppContext.getGlobalContext()); NewsFetcher fetcher = NewsFetcher.getInstance(I2PAppContext.getGlobalContext());
I2PThread t = new I2PThread(fetcher, "NewsFetcher"); I2PThread t = new I2PThread(fetcher, "NewsFetcher");
t.setDaemon(true); t.setDaemon(true);

5
apps/routerconsole/java/src/net/i2p/router/web/SummaryHelper.java
View File

@ -85,8 +85,9 @@ public class SummaryHelper extends HelperBase {
} }
public boolean allowReseed() { public boolean allowReseed() {
return (_context.netDb().getKnownRouters() < 30) || return _context.netDb().isInitialized() &&
Boolean.valueOf(_context.getProperty("i2p.alwaysAllowReseed", "false")).booleanValue(); ((_context.netDb().getKnownRouters() < 30) ||
Boolean.valueOf(_context.getProperty("i2p.alwaysAllowReseed")).booleanValue());
} }
public int getAllPeers() { return _context.netDb().getKnownRouters(); } public int getAllPeers() { return _context.netDb().getKnownRouters(); }

2
router/java/src/net/i2p/router/NetworkDatabaseFacade.java
View File

@ -58,6 +58,8 @@ public abstract class NetworkDatabaseFacade implements Service {
public abstract Set<Hash> getAllRouters(); public abstract Set<Hash> getAllRouters();
public int getKnownRouters() { return 0; } public int getKnownRouters() { return 0; }
public int getKnownLeaseSets() { return 0; } public int getKnownLeaseSets() { return 0; }
public boolean isInitialized() { return true; }
public void rescan() {}
public void renderRouterInfoHTML(Writer out, String s) throws IOException {} public void renderRouterInfoHTML(Writer out, String s) throws IOException {}
public void renderLeaseSetHTML(Writer out) throws IOException {} public void renderLeaseSetHTML(Writer out) throws IOException {}
public void renderStatusHTML(Writer out, boolean b) throws IOException {} public void renderStatusHTML(Writer out, boolean b) throws IOException {}

2
router/java/src/net/i2p/router/networkdb/kademlia/DataStore.java
View File

@ -14,6 +14,7 @@ import net.i2p.data.DataStructure;
import net.i2p.data.Hash; import net.i2p.data.Hash;
public interface DataStore { public interface DataStore {
public boolean isInitialized();
public boolean isKnown(Hash key); public boolean isKnown(Hash key);
public DataStructure get(Hash key); public DataStructure get(Hash key);
public DataStructure get(Hash key, boolean persist); public DataStructure get(Hash key, boolean persist);
@ -24,6 +25,7 @@ public interface DataStore {
public Set getKeys(); public Set getKeys();
public void stop(); public void stop();
public void restart(); public void restart();
public void rescan();
public int countLeaseSets(); public int countLeaseSets();
} }

14
router/java/src/net/i2p/router/networkdb/kademlia/KademliaNetworkDatabaseFacade.java
View File

@ -139,6 +139,11 @@ public class KademliaNetworkDatabaseFacade extends NetworkDatabaseFacade {
context.statManager().createRateStat("netDb.exploreKeySet", "how many keys are queued for exploration?", "NetworkDatabase", new long[] { 10*60*1000 }); context.statManager().createRateStat("netDb.exploreKeySet", "how many keys are queued for exploration?", "NetworkDatabase", new long[] { 10*60*1000 });
} }
@Override
public boolean isInitialized() {
return _initialized && _ds != null && _ds.isInitialized();
}
protected PeerSelector createPeerSelector() { return new PeerSelector(_context); } protected PeerSelector createPeerSelector() { return new PeerSelector(_context); }
public PeerSelector getPeerSelector() { return _peerSelector; } public PeerSelector getPeerSelector() { return _peerSelector; }
@ -177,7 +182,8 @@ public class KademliaNetworkDatabaseFacade extends NetworkDatabaseFacade {
public void shutdown() { public void shutdown() {
_initialized = false; _initialized = false;
_kb = null; _kb = null;
_ds.stop(); if (_ds != null)
_ds.stop();
_ds = null; _ds = null;
_exploreKeys.clear(); // hope this doesn't cause an explosion, it shouldn't. _exploreKeys.clear(); // hope this doesn't cause an explosion, it shouldn't.
// _exploreKeys = null; // _exploreKeys = null;
@ -203,6 +209,12 @@ public class KademliaNetworkDatabaseFacade extends NetworkDatabaseFacade {
publish(ri); publish(ri);
} }
@Override
public void rescan() {
if (isInitialized())
_ds.rescan();
}
String getDbDir() { return _dbDir; } String getDbDir() { return _dbDir; }
public void startup() { public void startup() {

25
router/java/src/net/i2p/router/networkdb/kademlia/PersistentDataStore.java
View File

@ -26,6 +26,7 @@ import net.i2p.data.RouterInfo;
import net.i2p.router.JobImpl; import net.i2p.router.JobImpl;
import net.i2p.router.Router; import net.i2p.router.Router;
import net.i2p.router.RouterContext; import net.i2p.router.RouterContext;
import net.i2p.router.networkdb.reseed.ReseedChecker;
import net.i2p.util.I2PThread; import net.i2p.util.I2PThread;
import net.i2p.util.Log; import net.i2p.util.Log;
@ -39,6 +40,8 @@ class PersistentDataStore extends TransientDataStore {
private String _dbDir; private String _dbDir;
private KademliaNetworkDatabaseFacade _facade; private KademliaNetworkDatabaseFacade _facade;
private Writer _writer; private Writer _writer;
private ReadJob _readJob;
private boolean _initialized;
private final static int READ_DELAY = 60*1000; private final static int READ_DELAY = 60*1000;
@ -47,7 +50,8 @@ class PersistentDataStore extends TransientDataStore {
_log = ctx.logManager().getLog(PersistentDataStore.class); _log = ctx.logManager().getLog(PersistentDataStore.class);
_dbDir = dbDir; _dbDir = dbDir;
_facade = facade; _facade = facade;
_context.jobQueue().addJob(new ReadJob()); _readJob = new ReadJob();
_context.jobQueue().addJob(_readJob);
ctx.statManager().createRateStat("netDb.writeClobber", "How often we clobber a pending netDb write", "NetworkDatabase", new long[] { 20*60*1000 }); ctx.statManager().createRateStat("netDb.writeClobber", "How often we clobber a pending netDb write", "NetworkDatabase", new long[] { 20*60*1000 });
ctx.statManager().createRateStat("netDb.writePending", "How many pending writes are there", "NetworkDatabase", new long[] { 60*1000 }); ctx.statManager().createRateStat("netDb.writePending", "How many pending writes are there", "NetworkDatabase", new long[] { 60*1000 });
ctx.statManager().createRateStat("netDb.writeOut", "How many we wrote", "NetworkDatabase", new long[] { 20*60*1000 }); ctx.statManager().createRateStat("netDb.writeOut", "How many we wrote", "NetworkDatabase", new long[] { 20*60*1000 });
@ -58,7 +62,10 @@ class PersistentDataStore extends TransientDataStore {
//writer.setDaemon(true); //writer.setDaemon(true);
writer.start(); writer.start();
} }
public boolean isInitialized() { return _initialized; }
// this doesn't stop the read job or the writer, maybe it should?
@Override @Override
public void stop() { public void stop() {
super.stop(); super.stop();
@ -71,6 +78,11 @@ class PersistentDataStore extends TransientDataStore {
_dbDir = _facade.getDbDir(); _dbDir = _facade.getDbDir();
} }
public void rescan() {
if (_initialized)
_readJob.wakeup();
}
@Override @Override
public DataStructure get(Hash key) { public DataStructure get(Hash key) {
return get(key, true); return get(key, true);
@ -317,6 +329,10 @@ class PersistentDataStore extends TransientDataStore {
requeue(READ_DELAY); requeue(READ_DELAY);
} }
public void wakeup() {
requeue(0);
}
private void readFiles() { private void readFiles() {
int routerCount = 0; int routerCount = 0;
try { try {
@ -336,9 +352,10 @@ class PersistentDataStore extends TransientDataStore {
_log.error("Error reading files in the db dir", ioe); _log.error("Error reading files in the db dir", ioe);
} }
if ( (routerCount <= 5) && (!_alreadyWarned) ) { if (!_alreadyWarned) {
_log.error("Very few routerInfo files remaining - please reseed"); ReseedChecker.checkReseed(_context, routerCount);
_alreadyWarned = true; _alreadyWarned = true;
_initialized = true;
} }
} }
} }

4
router/java/src/net/i2p/router/networkdb/kademlia/TransientDataStore.java
View File

@ -35,6 +35,8 @@ class TransientDataStore implements DataStore {
_log.info("Data Store initialized"); _log.info("Data Store initialized");
} }
public boolean isInitialized() { return true; }
public void stop() { public void stop() {
_data.clear(); _data.clear();
} }
@ -43,6 +45,8 @@ class TransientDataStore implements DataStore {
stop(); stop();
} }
public void rescan() {}
public Set getKeys() { public Set getKeys() {
return new HashSet(_data.keySet()); return new HashSet(_data.keySet());
} }

46
router/java/src/net/i2p/router/networkdb/reseed/ReseedChecker.java Normal file
View File

@ -0,0 +1,46 @@
package net.i2p.router.networkdb.reseed;
import java.io.File;
import net.i2p.router.RouterContext;
import net.i2p.util.Log;
/**
* Moved from RouterConsoleRunner.java
*
* Reseeding is not strictly a router function, it used to be
* in the routerconsole app, but this made it impossible to
* bootstrap an embedded router lacking a routerconsole,
* in iMule or android for example, without additional modifications.
*
* Also, as this is now called from PersistentDataStore, not from the
* routerconsole, we can get started as soon as the netdb has read
* the netDb/ directory, not when the console starts,
router/java/src/net/i2p/router/networkdb/eseed/ReseedChecker.java
*/
public class ReseedChecker {
private static final int MINIMUM = 15;
public static void checkReseed(RouterContext context, int count) {
if (count >= MINIMUM)
return;
// we check the i2p installation directory for a flag telling us not to reseed,
// but also check the home directory for that flag too, since new users installing i2p
// don't have an installation directory that they can put the flag in yet.
File noReseedFile = new File(new File(System.getProperty("user.home")), ".i2pnoreseed");
File noReseedFileAlt1 = new File(new File(System.getProperty("user.home")), "noreseed.i2p");
File noReseedFileAlt2 = new File(context.getConfigDir(), ".i2pnoreseed");
File noReseedFileAlt3 = new File(context.getConfigDir(), "noreseed.i2p");
if (!noReseedFile.exists() && !noReseedFileAlt1.exists() && !noReseedFileAlt2.exists() && !noReseedFileAlt3.exists()) {
Log _log = context.logManager().getLog(ReseedChecker.class);
if (count <= 1)
_log.error("Downloading peer router information for a new I2P installation");
else
_log.error("Very few routerInfo files remaining - reseeding now");
Reseeder reseeder = new Reseeder(context);
reseeder.requestReseed();
}
}
}

266
router/java/src/net/i2p/router/networkdb/reseed/Reseeder.java Normal file
View File

@ -0,0 +1,266 @@
package net.i2p.router.networkdb.reseed;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.net.URL;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import java.util.StringTokenizer;
import net.i2p.I2PAppContext;
import net.i2p.router.RouterContext;
import net.i2p.util.EepGet;
import net.i2p.util.I2PThread;
import net.i2p.util.Log;
/**
* Moved from ReseedHandler in routerconsole. See ReseedChecker for additional comments.
*
* Handler to deal with reseed requests. This will reseed from the URLs
* specified below unless the I2P configuration property "i2p.reseedURL" is
* set. It always writes to ./netDb/, so don't mess with that.
*
*/
public class Reseeder {
private static ReseedRunner _reseedRunner;
private RouterContext _context;
private Log _log;
// Reject unreasonably big files, because we download into a ByteArrayOutputStream.
private static final long MAX_RESEED_RESPONSE_SIZE = 8 * 1024 * 1024;
private static final String DEFAULT_SEED_URL = "http://i2pdb.tin0.de/netDb/,http://netdb.i2p2.de/";
public Reseeder(RouterContext ctx) {
_context = ctx;
_log = ctx.logManager().getLog(Reseeder.class);
}
public void requestReseed() {
synchronized (Reseeder.class) {
if (_reseedRunner == null)
_reseedRunner = new ReseedRunner();
if (_reseedRunner.isRunning()) {
return;
} else {
System.setProperty("net.i2p.router.web.Reseeder.reseedInProgress", "true");
I2PThread reseed = new I2PThread(_reseedRunner, "Reseed");
reseed.start();
}
}
}
public class ReseedRunner implements Runnable, EepGet.StatusListener {
private boolean _isRunning;
public ReseedRunner() {
_isRunning = false;
System.setProperty("net.i2p.router.web.Reseeder.statusMessage","Reseeding.");
}
public boolean isRunning() { return _isRunning; }
public void run() {
_isRunning = true;
System.out.println("Reseed start");
reseed(false);
System.out.println("Reseed complete");
System.setProperty("net.i2p.router.web.Reseeder.reseedInProgress", "false");
_isRunning = false;
}
// EepGet status listeners
public void attemptFailed(String url, long bytesTransferred, long bytesRemaining, int currentAttempt, int numRetries, Exception cause) {
// Since readURL() runs an EepGet with 0 retries,
// we can report errors with attemptFailed() instead of transferFailed().
// It has the benefit of providing cause of failure, which helps resolve issues.
if (_log.shouldLog(Log.ERROR)) _log.error("EepGet failed on " + url, cause);
}
public void bytesTransferred(long alreadyTransferred, int currentWrite, long bytesTransferred, long bytesRemaining, String url) {}
public void transferComplete(long alreadyTransferred, long bytesTransferred, long bytesRemaining, String url, String outputFile, boolean notModified) {}
public void transferFailed(String url, long bytesTransferred, long bytesRemaining, int currentAttempt) {}
public void headerReceived(String url, int attemptNum, String key, String val) {}
public void attempting(String url) {}
// End of EepGet status listeners
/**
* Reseed has been requested, so lets go ahead and do it. Fetch all of
* the routerInfo-*.dat files from the specified URL (or the default) and
* save them into this router's netDb dir.
*
*/
private static final String RESEED_TIPS =
"Ensure that nothing blocks outbound HTTP, check <a href=logs.jsp>logs</a> " +
"and if nothing helps, read FAQ about reseeding manually.";
private void reseed(boolean echoStatus) {
List URLList = new ArrayList();
String URLs = _context.getProperty("i2p.reseedURL", DEFAULT_SEED_URL);
StringTokenizer tok = new StringTokenizer(URLs, " ,");
while (tok.hasMoreTokens())
URLList.add(tok.nextToken().trim());
Collections.shuffle(URLList);
for (int i = 0; i < URLList.size() && _isRunning; i++)
reseedOne((String) URLList.get(i), echoStatus);
}
/**
* Fetch a directory listing and then up to 200 routerInfo files in the listing.
* The listing must contain (exactly) strings that match:
* href="routerInfo-{hash}.dat">
* OR
* HREF="routerInfo-{hash}.dat">
* and then it fetches the files
* {seedURL}routerInfo-{hash}.dat
* after appending a '/' to seedURL if it doesn't have one.
* Essentially this means that the seedURL must be a directory, it
* can't end with 'index.html', for example.
*
* Jetty directory listings are not compatible, as they look like
* HREF="/full/path/to/routerInfo-...
**/
private void reseedOne(String seedURL, boolean echoStatus) {
try {
System.setProperty("net.i2p.router.web.Reseeder.errorMessage","");
System.setProperty("net.i2p.router.web.Reseeder.statusMessage","Reseeding: fetching seed URL.");
System.err.println("Reseed from " + seedURL);
URL dir = new URL(seedURL);
byte contentRaw[] = readURL(dir);
if (contentRaw == null) {
System.setProperty("net.i2p.router.web.Reseeder.errorMessage",
"Last reseed failed fully (failed reading seed URL). " +
RESEED_TIPS);
// Logging deprecated here since attemptFailed() provides better info
_log.debug("Failed reading seed URL: " + seedURL);
return;
}
String content = new String(contentRaw);
Set urls = new HashSet();
int cur = 0;
int total = 0;
while (total++ < 1000) {
int start = content.indexOf("href=\"routerInfo-", cur);
if (start < 0) {
start = content.indexOf("HREF=\"routerInfo-", cur);
if (start < 0)
break;
}
int end = content.indexOf(".dat\">", start);
String name = content.substring(start+"href=\"routerInfo-".length(), end);
urls.add(name);
cur = end + 1;
}
if (total <= 0) {
_log.error("Read " + contentRaw.length + " bytes from seed " + seedURL + ", but found no routerInfo URLs.");
System.setProperty("net.i2p.router.web.Reseeder.errorMessage",
"Last reseed failed fully (no routerInfo URLs at seed URL). " +
RESEED_TIPS);
return;
}
List urlList = new ArrayList(urls);
Collections.shuffle(urlList);
int fetched = 0;
int errors = 0;
// 200 max from one URL
for (Iterator iter = urlList.iterator(); iter.hasNext() && fetched < 200; ) {
try {
System.setProperty("net.i2p.router.web.Reseeder.statusMessage",
"Reseeding: fetching router info from seed URL (" +
fetched + " successful, " + errors + " errors, " + total + " total).");
fetchSeed(seedURL, (String)iter.next());
fetched++;
if (echoStatus) {
System.out.print(".");
if (fetched % 60 == 0)
System.out.println();
}
} catch (Exception e) {
errors++;
}
}
System.err.println("Reseed got " + fetched + " router infos from " + seedURL);
int failPercent = 100 * errors / total;
// Less than 10% of failures is considered success,
// because some routerInfos will always fail.
if ((failPercent >= 10) && (failPercent < 90)) {
System.setProperty("net.i2p.router.web.Reseeder.errorMessage",
"Last reseed failed partly (" + failPercent + "% of " + total + "). " +
RESEED_TIPS);
}
if (failPercent >= 90) {
System.setProperty("net.i2p.router.web.Reseeder.errorMessage",
"Last reseed failed (" + failPercent + "% of " + total + "). " +
RESEED_TIPS);
}
if (fetched > 0)
_context.netDb().rescan();
// Don't go on to the next URL if we have enough
if (fetched >= 100)
_isRunning = false;
} catch (Throwable t) {
System.setProperty("net.i2p.router.web.Reseeder.errorMessage",
"Last reseed failed fully (exception caught). " +
RESEED_TIPS);
_log.error("Error reseeding", t);
}
}
/* Since we don't return a value, we should always throw an exception if something fails. */
private void fetchSeed(String seedURL, String peer) throws Exception {
URL url = new URL(seedURL + (seedURL.endsWith("/") ? "" : "/") + "routerInfo-" + peer + ".dat");
byte data[] = readURL(url);
if (data == null) {
// Logging deprecated here since attemptFailed() provides better info
_log.debug("Failed fetching seed: " + url.toString());
throw new Exception ("Failed fetching seed.");
}
//System.out.println("read: " + (data != null ? data.length : -1));
writeSeed(peer, data);
}
private byte[] readURL(URL url) throws Exception {
ByteArrayOutputStream baos = new ByteArrayOutputStream(4*1024);
// Do a non-proxied eepget into our ByteArrayOutputStream with 0 retries
EepGet get = new EepGet( I2PAppContext.getGlobalContext(), false, null, -1, 0, 0, MAX_RESEED_RESPONSE_SIZE,
null, baos, url.toString(), false, null, null);
get.addStatusListener(ReseedRunner.this);
if (get.fetch()) return baos.toByteArray(); else return null;
}
private void writeSeed(String name, byte data[]) throws Exception {
String dirName = "netDb"; // _context.getProperty("router.networkDatabase.dbDir", "netDb");
File netDbDir = new File(_context.getRouterDir(), dirName);
if (!netDbDir.exists()) {
boolean ok = netDbDir.mkdirs();
}
FileOutputStream fos = new FileOutputStream(new File(netDbDir, "routerInfo-" + name + ".dat"));
fos.write(data);
fos.close();
}
}
/******
public static void main(String args[]) {
if ( (args != null) && (args.length == 1) && (!Boolean.valueOf(args[0]).booleanValue()) ) {
System.out.println("Not reseeding, as requested");
return; // not reseeding on request
}
System.out.println("Reseeding");
Reseeder reseedHandler = new Reseeder();
reseedHandler.requestReseed();
}
******/
}