From 6e981874a5736961ee2cb6acb266f2cf776bd4e5 Mon Sep 17 00:00:00 2001 From: zzz Date: Wed, 19 Mar 2008 00:20:15 +0000 Subject: [PATCH] * ExploratoryPeerSelector: - Exclude floodfill peers - Tweak the HighCap vs. NonFailing decision * i2psnark: Increase retries for .torrent fetch * IRC Proxy: Prevent mIRC from sending an alternate DCC request containing an IP * readme.html: Reorder some items * Stats: Add some more required stats * Streaming lib: Fix slow start to be exponential growth, fix congestion avoidance to be linear growth. Should speed up local connections a lot, and remote connections a little. --- .../src/org/klomp/snark/I2PSnarkUtil.java | 8 +++-- .../java/src/org/klomp/snark/Snark.java | 2 +- .../net/i2p/i2ptunnel/I2PTunnelIRCClient.java | 10 ++++++ .../streaming/ConnectionPacketHandler.java | 35 ++++++++++++------- core/java/src/net/i2p/stat/StatManager.java | 3 ++ history.txt | 16 ++++++++- readme.html | 10 +++--- .../src/net/i2p/router/RouterVersion.java | 2 +- .../OutboundClientMessageOneShotJob.java | 3 ++ .../tunnel/pool/ClientPeerSelector.java | 4 +-- .../tunnel/pool/ExploratoryPeerSelector.java | 31 +++++++++++++--- 11 files changed, 95 insertions(+), 29 deletions(-) diff --git a/apps/i2psnark/java/src/org/klomp/snark/I2PSnarkUtil.java b/apps/i2psnark/java/src/org/klomp/snark/I2PSnarkUtil.java index bd187aaf4e..4fc75342f1 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/I2PSnarkUtil.java +++ b/apps/i2psnark/java/src/org/klomp/snark/I2PSnarkUtil.java @@ -160,8 +160,10 @@ public class I2PSnarkUtil { /** * fetch the given URL, returning the file it is stored in, or null on error */ - public File get(String url) { return get(url, true); } - public File get(String url, boolean rewrite) { + public File get(String url) { return get(url, true, 1); } + public File get(String url, boolean rewrite) { return get(url, rewrite, 1); } + public File get(String url, int retries) { return get(url, true, retries); } + public File get(String url, boolean rewrite, int retries) { _log.debug("Fetching [" + url + "] proxy=" + _proxyHost + ":" + _proxyPort + ": " + _shouldProxy); File out = null; try { @@ -175,7 +177,7 @@ public class I2PSnarkUtil { if (rewrite) fetchURL = rewriteAnnounce(url); //_log.debug("Rewritten url [" + fetchURL + "]"); - EepGet get = new EepGet(_context, _shouldProxy, _proxyHost, _proxyPort, 1, out.getAbsolutePath(), fetchURL); + EepGet get = new EepGet(_context, _shouldProxy, _proxyHost, _proxyPort, retries, out.getAbsolutePath(), fetchURL); if (get.fetch()) { _log.debug("Fetch successful [" + url + "]: size=" + out.length()); return out; diff --git a/apps/i2psnark/java/src/org/klomp/snark/Snark.java b/apps/i2psnark/java/src/org/klomp/snark/Snark.java index ad2f4aea0a..71bb49f53f 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/Snark.java +++ b/apps/i2psnark/java/src/org/klomp/snark/Snark.java @@ -312,7 +312,7 @@ public class Snark else { activity = "Getting torrent"; - File torrentFile = I2PSnarkUtil.instance().get(torrent); + File torrentFile = I2PSnarkUtil.instance().get(torrent, 3); if (torrentFile == null) { fatal("Unable to fetch " + torrent); if (false) return; // never reached - fatal(..) throws diff --git a/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/I2PTunnelIRCClient.java b/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/I2PTunnelIRCClient.java index 7a24e39957..ad2f55b2b1 100644 --- a/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/I2PTunnelIRCClient.java +++ b/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/I2PTunnelIRCClient.java @@ -435,6 +435,16 @@ public class I2PTunnelIRCClient extends I2PTunnelClientBase implements Runnable return s; } + // mIRC sends "NOTICE user :DCC Send file (IP)" + // in addition to the CTCP version + if("NOTICE".equalsIgnoreCase(command)) + { + String msg = field[2]; + if(msg.startsWith(":DCC ")) + return null; + // fall through + } + // Allow PRIVMSG, but block CTCP (except ACTION). if("PRIVMSG".equalsIgnoreCase(command) || "NOTICE".equalsIgnoreCase(command)) { diff --git a/apps/streaming/java/src/net/i2p/client/streaming/ConnectionPacketHandler.java b/apps/streaming/java/src/net/i2p/client/streaming/ConnectionPacketHandler.java index 3079273ce5..980ecd86ce 100644 --- a/apps/streaming/java/src/net/i2p/client/streaming/ConnectionPacketHandler.java +++ b/apps/streaming/java/src/net/i2p/client/streaming/ConnectionPacketHandler.java @@ -302,24 +302,35 @@ public class ConnectionPacketHandler { _context.statManager().addRateData("stream.trend", trend, newWindowSize); if ( (!congested) && (acked > 0) && (numResends <= 0) ) { - if (trend < 0) { + if (newWindowSize < con.getLastCongestionSeenAt() / 2) { + // Don't make this <= LastCongestion/2 or we'll jump right back to where we were + // slow start - exponential growth + // grow acked/N times (where N = the slow start factor) + // always grow at least 1 + int factor = con.getOptions().getSlowStartGrowthRateFactor(); + if (factor <= 1) + newWindowSize += acked; + else if (acked < factor) + newWindowSize++; + else + newWindowSize += acked / factor; + if (_log.shouldLog(Log.DEBUG)) + _log.debug("slow start acks = " + acked + " for " + con); + } else if (trend < 0) { // rtt is shrinking, so lets increment the cwin newWindowSize++; - } else if (newWindowSize > con.getLastCongestionSeenAt() / 2) { + if (_log.shouldLog(Log.DEBUG)) + _log.debug("trend < 0 for " + con); + } else { // congestion avoidance - - // we can't use newWindowSize += 1/newWindowSize, since we're + // linear growth - increase window 1/N per RTT + // we can't use newWindowSize += acked/(oldWindow*N) (where N = the cong. avoid. factor), since we're // integers, so lets use a random distribution instead int shouldIncrement = _context.random().nextInt(con.getOptions().getCongestionAvoidanceGrowthRateFactor()*newWindowSize); - if (shouldIncrement <= 0) - newWindowSize += 1; - } else { - // slow start, but modified to take into account the fact - // that windows in the streaming lib are messages, not bytes, - // so we only grow 1 every N times (where N = the slow start factor) - int shouldIncrement = _context.random().nextInt(con.getOptions().getSlowStartGrowthRateFactor()); - if (shouldIncrement <= 0) + if (shouldIncrement < acked) newWindowSize += 1; + if (_log.shouldLog(Log.DEBUG)) + _log.debug("cong. avoid acks = " + acked + " for " + con); } } diff --git a/core/java/src/net/i2p/stat/StatManager.java b/core/java/src/net/i2p/stat/StatManager.java index 9d7635415e..f2573f501c 100644 --- a/core/java/src/net/i2p/stat/StatManager.java +++ b/core/java/src/net/i2p/stat/StatManager.java @@ -45,6 +45,9 @@ public class StatManager { "bw.recvRate,bw.sendBps,bw.sendRate,client.sendAckTime,clock.skew,crypto.elGamal.encrypt," + "jobQueue.jobLag,netDb.successTime,router.fastPeers," + "transport.receiveMessageSize,transport.sendMessageSize,transport.sendProcessingTime," + + "tunnel.acceptLoad,tunnel.buildRequestTime,tunnel.rejectOverloaded,tunnel.rejectTimeout" + + "tunnel.buildClientExpire,tunnel.buildClientReject,tunnel.buildClientSuccess," + + "tunnel.buildExploratoryExpire,tunnel.buildExploratoryReject,tunnel.buildExploratorySuccess," + "tunnel.buildRatio.*,tunnel.buildFailure,tunnel.buildSuccess,tunnel.corruptMessage," + "tunnel.decryptRequestTime,tunnel.fragmentedDropped,tunnel.participatingMessageCount,"+ "tunnel.participatingTunnels,tunnel.testFailedTime,tunnel.testSuccessTime" ; diff --git a/history.txt b/history.txt index ab21011729..642dd46080 100644 --- a/history.txt +++ b/history.txt @@ -1,3 +1,17 @@ +2008-03-19 zzz + * ExploratoryPeerSelector: + - Exclude floodfill peers + - Tweak the HighCap vs. NonFailing decision + * i2psnark: Increase retries for .torrent fetch + * IRC Proxy: Prevent mIRC from sending an alternate DCC request + containing an IP + * readme.html: Reorder some items + * Stats: Add some more required stats + * Streaming lib: Fix slow start to be exponential growth, + fix congestion avoidance to be linear growth. + Should speed up local connections a lot, and remote + connections a little. + 2008-03-14 zzz * Floodfill Search: - Prefer heard-from, unfailing, unshitlisted floodfill peers @@ -45,7 +59,7 @@ * NetDb: Comment out published netDb stats disabled for .32 2008-03-08 zzz - * ClientPeerSelector: Implement strict ordering of peers, + * TunnelPeerSelectors: Implement strict ordering of peers, based on XOR distance from a random hash separately generated for each tunnel pool diff --git a/readme.html b/readme.html index 4441c53b1a..228f340420 100644 --- a/readme.html +++ b/readme.html @@ -3,10 +3,6 @@ grow over the next few minutes and you'll see a "shared clients" local destinati on the left (if not, see below). Once those show up, you can:

diff --git a/router/java/src/net/i2p/router/RouterVersion.java b/router/java/src/net/i2p/router/RouterVersion.java index 6dee5391b0..6cade9a886 100644 --- a/router/java/src/net/i2p/router/RouterVersion.java +++ b/router/java/src/net/i2p/router/RouterVersion.java @@ -17,7 +17,7 @@ import net.i2p.CoreVersion; public class RouterVersion { public final static String ID = "$Revision: 1.548 $ $Date: 2008-02-10 15:00:00 $"; public final static String VERSION = "0.6.1.32"; - public final static long BUILD = 7; + public final static long BUILD = 8; public static void main(String args[]) { System.out.println("I2P Router version: " + VERSION + "-" + BUILD); System.out.println("Router ID: " + RouterVersion.ID); diff --git a/router/java/src/net/i2p/router/message/OutboundClientMessageOneShotJob.java b/router/java/src/net/i2p/router/message/OutboundClientMessageOneShotJob.java index df95eb9b89..c66c83ede1 100644 --- a/router/java/src/net/i2p/router/message/OutboundClientMessageOneShotJob.java +++ b/router/java/src/net/i2p/router/message/OutboundClientMessageOneShotJob.java @@ -277,6 +277,9 @@ public class OutboundClientMessageOneShotJob extends JobImpl { _lease = (Lease)orderedLeases.get(orderedLeases.firstKey()); } else { + // strangely, _lease isn't used anywhere except for log messages - ?!?!??!?!?! + // Apparently the outbound endpoint gets to pick the inbound gateway + // and this whole function is pointless. _lease = (Lease)leases.get(0); } return true; diff --git a/router/java/src/net/i2p/router/tunnel/pool/ClientPeerSelector.java b/router/java/src/net/i2p/router/tunnel/pool/ClientPeerSelector.java index fc5871b143..a5c905f47e 100644 --- a/router/java/src/net/i2p/router/tunnel/pool/ClientPeerSelector.java +++ b/router/java/src/net/i2p/router/tunnel/pool/ClientPeerSelector.java @@ -5,8 +5,8 @@ import net.i2p.router.RouterContext; import net.i2p.router.TunnelPoolSettings; /** - * Pick peers randomly out of the fast pool, and put them into tunnels in a - * random order + * Pick peers randomly out of the fast pool, and put them into tunnels + * ordered by XOR distance from a random key. * */ class ClientPeerSelector extends TunnelPeerSelector { diff --git a/router/java/src/net/i2p/router/tunnel/pool/ExploratoryPeerSelector.java b/router/java/src/net/i2p/router/tunnel/pool/ExploratoryPeerSelector.java index 25e0378b62..cbd6cd7d66 100644 --- a/router/java/src/net/i2p/router/tunnel/pool/ExploratoryPeerSelector.java +++ b/router/java/src/net/i2p/router/tunnel/pool/ExploratoryPeerSelector.java @@ -3,13 +3,14 @@ package net.i2p.router.tunnel.pool; import java.util.*; import net.i2p.router.RouterContext; import net.i2p.router.TunnelPoolSettings; +import net.i2p.router.networkdb.kademlia.FloodfillNetworkDatabaseFacade; import net.i2p.stat.Rate; import net.i2p.stat.RateStat; import net.i2p.util.Log; /** - * Pick peers randomly out of the not-failing pool, and put them into randomly - * ordered tunnels. + * Pick peers randomly out of the not-failing pool, and put them into a tunnel + * ordered by XOR distance from a random key. * */ class ExploratoryPeerSelector extends TunnelPeerSelector { @@ -31,6 +32,9 @@ class ExploratoryPeerSelector extends TunnelPeerSelector { Set exclude = getExclude(ctx, settings.isInbound(), settings.isExploratory()); exclude.add(ctx.routerHash()); + // Don't use ff peers for exploratory tunnels to lessen exposure to netDb searches and stores + FloodfillNetworkDatabaseFacade fac = (FloodfillNetworkDatabaseFacade)ctx.netDb(); + exclude.addAll(fac.getFloodfillPeers()); HashSet matches = new HashSet(length); boolean exploreHighCap = shouldPickHighCap(ctx); if (exploreHighCap) @@ -52,6 +56,7 @@ class ExploratoryPeerSelector extends TunnelPeerSelector { return rv; } + private static final int MIN_NONFAILING_PCT = 25; private boolean shouldPickHighCap(RouterContext ctx) { if (Boolean.valueOf(ctx.getProperty("router.exploreHighCapacity", "false")).booleanValue()) return true; @@ -62,10 +67,27 @@ class ExploratoryPeerSelector extends TunnelPeerSelector { // randomly from the 'not failing' pool. However, if we are having a // hard time building exploratory tunnels, lets fall back again on the // high capacity peers, at least for a little bit. - int failPct = getExploratoryFailPercentage(ctx); + int failPct; + // getEvents() will be 0 for first 10 minutes + if (ctx.router().getUptime() <= 11*60*1000) { + failPct = 100 - MIN_NONFAILING_PCT; + } else { + failPct = getExploratoryFailPercentage(ctx); + // always try a little, this helps keep the failPct stat accurate too + if (failPct > 100 - MIN_NONFAILING_PCT) + failPct = 100 - MIN_NONFAILING_PCT; + } + Log l = ctx.logManager().getLog(getClass()); + if (l.shouldLog(Log.DEBUG)) + l.error("Fail pct: " + failPct); return (failPct >= ctx.random().nextInt(100)); } + // We should really use the difference between the exploratory fail rate + // and the client fail rate. + // (return 100 * ((Efail - Cfail) / (1 - Cfail))) + // Even this isn't the "true" rate for the NonFailingPeers pool, since we + // are often building exploratory tunnels using the HighCapacity pool. private int getExploratoryFailPercentage(RouterContext ctx) { int timeout = getEvents(ctx, "tunnel.buildExploratoryExpire", 10*60*1000); int reject = getEvents(ctx, "tunnel.buildExploratoryReject", 10*60*1000); @@ -76,6 +98,7 @@ class ExploratoryPeerSelector extends TunnelPeerSelector { return (int)(100 * pct); } + // Use current + last to get more recent and smoother data private int getEvents(RouterContext ctx, String stat, long period) { RateStat rs = ctx.statManager().getRate(stat); if (rs == null) @@ -83,6 +106,6 @@ class ExploratoryPeerSelector extends TunnelPeerSelector { Rate r = rs.getRate(period); if (r == null) return 0; - return (int)r.getLastEventCount(); + return (int) (r.getLastEventCount() + r.getCurrentEventCount()); } }