diff --git a/history.txt b/history.txt index afce95851..32687e266 100644 --- a/history.txt +++ b/history.txt @@ -1,3 +1,12 @@ +2008-04-17 zzz + * Reachability: + - Track unreachable peers persistently + (i.e. separately from shitlist, and not cleared when they contact us) + - Exclude detected unreachable peers from inbound tunnels + - Exclude detected unreachable peers from selected leases + - Exclude detected unreachable floodfill peers from lookups + - Show unreachable status on profiles.jsp + 2008-04-16 zzz * SSU/Reachability: - Extend shitlist time from 4-8m to 40-60m diff --git a/router/java/src/net/i2p/router/CommSystemFacade.java b/router/java/src/net/i2p/router/CommSystemFacade.java index 4f0a532e1..2978ce928 100644 --- a/router/java/src/net/i2p/router/CommSystemFacade.java +++ b/router/java/src/net/i2p/router/CommSystemFacade.java @@ -54,6 +54,7 @@ public abstract class CommSystemFacade implements Service { public short getReachabilityStatus() { return STATUS_OK; } public void recheckReachability() {} public boolean isBacklogged(Hash dest) { return false; } + public boolean wasUnreachable(Hash dest) { return false; } /** * Tell other transports our address changed diff --git a/router/java/src/net/i2p/router/RouterVersion.java b/router/java/src/net/i2p/router/RouterVersion.java index 1bdcecca6..d7909dfd6 100644 --- a/router/java/src/net/i2p/router/RouterVersion.java +++ b/router/java/src/net/i2p/router/RouterVersion.java @@ -17,7 +17,7 @@ import net.i2p.CoreVersion; public class RouterVersion { public final static String ID = "$Revision: 1.548 $ $Date: 2008-02-10 15:00:00 $"; public final static String VERSION = "0.6.1.32"; - public final static long BUILD = 18; + public final static long BUILD = 19; public static void main(String args[]) { System.out.println("I2P Router version: " + VERSION + "-" + BUILD); System.out.println("Router ID: " + RouterVersion.ID); diff --git a/router/java/src/net/i2p/router/message/OutboundClientMessageOneShotJob.java b/router/java/src/net/i2p/router/message/OutboundClientMessageOneShotJob.java index 04f6a67f1..d9357b563 100644 --- a/router/java/src/net/i2p/router/message/OutboundClientMessageOneShotJob.java +++ b/router/java/src/net/i2p/router/message/OutboundClientMessageOneShotJob.java @@ -319,15 +319,29 @@ public class OutboundClientMessageOneShotJob extends JobImpl { _lease = (Lease)orderedLeases.get(orderedLeases.firstKey()); } else { ****/ + + // Avoid a lease on a gateway we think is unreachable, if possible + for (int i = 0; i < _leaseSet.getLeaseCount(); i++) { + Lease l = _leaseSet.getLease(i); + if (!getContext().commSystem().wasUnreachable(l.getGateway())) { + _lease = l; + break; + } + if (_log.shouldLog(Log.WARN)) + _log.warn(getJobId() + ": Skipping unreachable (by us) gateway " + l.getGateway()); + } + if (_lease == null) { _lease = (Lease)leases.get(0); -// } + if (_log.shouldLog(Log.WARN)) + _log.warn(getJobId() + ": All leases are unreachable (by us) for " + _toString); + } /*** removed until we fix SSU reachability synchronized (_leaseCache) { _leaseCache.put(_to, _lease); } -***/ if (_log.shouldLog(Log.WARN)) _log.warn("Added to cache - lease for " + _toString); +***/ return true; } diff --git a/router/java/src/net/i2p/router/networkdb/kademlia/FloodOnlySearchJob.java b/router/java/src/net/i2p/router/networkdb/kademlia/FloodOnlySearchJob.java index d7152eead..f293b59ad 100644 --- a/router/java/src/net/i2p/router/networkdb/kademlia/FloodOnlySearchJob.java +++ b/router/java/src/net/i2p/router/networkdb/kademlia/FloodOnlySearchJob.java @@ -76,7 +76,7 @@ class FloodOnlySearchJob extends FloodSearchJob { // We need to randomize our ff selection, else we stay with the same ones since // getFloodfillPeers() is sorted by closest distance. Always using the same // ones didn't help reliability. - // Also, query the unheard-from, unprofiled, failing and shitlisted ones last. + // Also, query the unheard-from, unprofiled, failing, unreachable and shitlisted ones last. // We should hear from floodfills pretty frequently so set a 30m time limit. // If unprofiled we haven't talked to them in a long time. // We aren't contacting the peer directly, so shitlist doesn't strictly matter, @@ -90,14 +90,15 @@ class FloodOnlySearchJob extends FloodSearchJob { Hash peer = (Hash)floodfillPeers.get(i); PeerProfile profile = getContext().profileOrganizer().getProfile(peer); if (profile == null || profile.getLastHeardFrom() < before || - profile.getIsFailing() || getContext().shitlist().isShitlisted(peer)) { + profile.getIsFailing() || getContext().shitlist().isShitlisted(peer) || + getContext().commSystem().wasUnreachable(peer)) { failcount++; ffp.add(peer); } else ffp.add(0, peer); } if (_log.shouldLog(Log.INFO) && failcount > 0) - _log.info(getJobId() + ": " + failcount + " of " + floodfillPeers.size() + " floodfills are not heard from, unprofiled, failing or shitlisted"); + _log.info(getJobId() + ": " + failcount + " of " + floodfillPeers.size() + " floodfills are not heard from, unprofiled, failing, unreachable or shitlisted"); floodfillPeers = ffp; } diff --git a/router/java/src/net/i2p/router/peermanager/ProfileOrganizer.java b/router/java/src/net/i2p/router/peermanager/ProfileOrganizer.java index 6747bd6f1..7c8d1eba3 100644 --- a/router/java/src/net/i2p/router/peermanager/ProfileOrganizer.java +++ b/router/java/src/net/i2p/router/peermanager/ProfileOrganizer.java @@ -424,6 +424,24 @@ public class ProfileOrganizer { return; } + public List selectPeersLocallyUnreachable() { + List n; + int count; + synchronized (_reorganizeLock) { + count = _notFailingPeers.size(); + n = new ArrayList(_notFailingPeers.keySet()); + } + List l = new ArrayList(count / 4); + for (Iterator iter = n.iterator(); iter.hasNext(); ) { + Hash peer = (Hash)iter.next(); + if (_context.commSystem().wasUnreachable(peer)) + l.add(peer); + } + if (_log.shouldLog(Log.INFO)) + _log.info("Unreachable: " + l); + return l; + } + /** * Find the hashes for all peers we are actively profiling * diff --git a/router/java/src/net/i2p/router/peermanager/ProfileOrganizerRenderer.java b/router/java/src/net/i2p/router/peermanager/ProfileOrganizerRenderer.java index ebc87f060..31ed10b23 100644 --- a/router/java/src/net/i2p/router/peermanager/ProfileOrganizerRenderer.java +++ b/router/java/src/net/i2p/router/peermanager/ProfileOrganizerRenderer.java @@ -137,6 +137,7 @@ class ProfileOrganizerRenderer { buf.append(""); if (_context.shitlist().isShitlisted(peer)) buf.append("Shitlist"); if (prof.getIsFailing()) buf.append(" Failing"); + if (_context.commSystem().wasUnreachable(peer)) buf.append(" Unreachable"); buf.append(" "); //buf.append("profile.txt "); //buf.append(" netDb"); diff --git a/router/java/src/net/i2p/router/transport/CommSystemFacadeImpl.java b/router/java/src/net/i2p/router/transport/CommSystemFacadeImpl.java index c41d1dc91..c2b024c05 100644 --- a/router/java/src/net/i2p/router/transport/CommSystemFacadeImpl.java +++ b/router/java/src/net/i2p/router/transport/CommSystemFacadeImpl.java @@ -125,6 +125,10 @@ public class CommSystemFacadeImpl extends CommSystemFacade { return _manager.isBacklogged(dest); } + public boolean wasUnreachable(Hash dest) { + return _manager.wasUnreachable(dest); + } + public List getMostRecentErrorMessages() { return _manager.getMostRecentErrorMessages(); } diff --git a/router/java/src/net/i2p/router/transport/Transport.java b/router/java/src/net/i2p/router/transport/Transport.java index d67c45294..499e47d69 100644 --- a/router/java/src/net/i2p/router/transport/Transport.java +++ b/router/java/src/net/i2p/router/transport/Transport.java @@ -48,6 +48,7 @@ public interface Transport { public short getReachabilityStatus(); public void recheckReachability(); public boolean isBacklogged(Hash dest); + public boolean wasUnreachable(Hash dest); public boolean isUnreachable(Hash peer); } diff --git a/router/java/src/net/i2p/router/transport/TransportImpl.java b/router/java/src/net/i2p/router/transport/TransportImpl.java index 02451e8fd..c00a5485e 100644 --- a/router/java/src/net/i2p/router/transport/TransportImpl.java +++ b/router/java/src/net/i2p/router/transport/TransportImpl.java @@ -15,6 +15,7 @@ import java.util.*; import net.i2p.data.Hash; import net.i2p.data.RouterAddress; import net.i2p.data.RouterIdentity; +import net.i2p.data.RouterInfo; import net.i2p.data.i2np.I2NPMessage; import net.i2p.router.CommSystemFacade; import net.i2p.router.Job; @@ -36,6 +37,7 @@ public abstract class TransportImpl implements Transport { protected RouterContext _context; /** map from routerIdentHash to timestamp (Long) that the peer was last unreachable */ private Map _unreachableEntries; + private Set _wasUnreachableEntries; /** * Initialize the new transport @@ -54,6 +56,7 @@ public abstract class TransportImpl implements Transport { _context.statManager().createRateStat("transport.expiredOnQueueLifetime", "How long a message that expires on our outbound queue is processed", "Transport", new long[] { 60*1000l, 10*60*1000l, 60*60*1000l, 24*60*60*1000l } ); _sendPool = new ArrayList(16); _unreachableEntries = new HashMap(16); + _wasUnreachableEntries = new HashSet(16); _currentAddress = null; } @@ -399,19 +402,23 @@ public abstract class TransportImpl implements Transport { } } /** called when we can't reach a peer */ + /** This isn't very useful since it is cleared when they contact us */ public void markUnreachable(Hash peer) { long now = _context.clock().now(); synchronized (_unreachableEntries) { _unreachableEntries.put(peer, new Long(now)); } + markWasUnreachable(peer, true); } /** called when we establish a peer connection (outbound or inbound) */ - public void markReachable(Hash peer) { + public void markReachable(Hash peer, boolean isInbound) { // if *some* transport can reach them, then we shouldn't shitlist 'em _context.shitlist().unshitlistRouter(peer); synchronized (_unreachableEntries) { _unreachableEntries.remove(peer); } + if (!isInbound) + markWasUnreachable(peer, false); } private class CleanupUnreachable extends JobImpl { public CleanupUnreachable(RouterContext ctx) { @@ -432,6 +439,34 @@ public abstract class TransportImpl implements Transport { } } + /** + * Was the peer UNreachable (outbound only) the last time we tried it? + * This is NOT reset if the peer contacts us and it is never expired. + */ + public boolean wasUnreachable(Hash peer) { + synchronized (_wasUnreachableEntries) { + if (_wasUnreachableEntries.contains(peer)) + return true; + } + RouterInfo ri = _context.netDb().lookupRouterInfoLocally(peer); + if (ri == null) + return false; + return null == ri.getTargetAddress(this.getStyle()); + } + /** + * Maintain the WasUnreachable list + */ + public void markWasUnreachable(Hash peer, boolean yes) { + synchronized (_wasUnreachableEntries) { + if (yes) + _wasUnreachableEntries.add(peer); + else + _wasUnreachableEntries.remove(peer); + } + if (_log.shouldLog(Log.WARN)) + _log.warn(this.getStyle() + " setting wasUnreachable to " + yes + " for " + peer); + } + public static boolean isPubliclyRoutable(byte addr[]) { if (addr.length == 4) { if ((addr[0]&0xFF) == 127) return false; diff --git a/router/java/src/net/i2p/router/transport/TransportManager.java b/router/java/src/net/i2p/router/transport/TransportManager.java index bc622db4c..fefbb9df8 100644 --- a/router/java/src/net/i2p/router/transport/TransportManager.java +++ b/router/java/src/net/i2p/router/transport/TransportManager.java @@ -202,6 +202,20 @@ public class TransportManager implements TransportEventListener { return false; } + /** + * Was the peer UNreachable (outbound only) on any transport, + * based on the last time we tried it for each transport? + * This is NOT reset if the peer contacts us. + */ + public boolean wasUnreachable(Hash dest) { + for (int i = 0; i < _transports.size(); i++) { + Transport t = (Transport)_transports.get(i); + if (!t.wasUnreachable(dest)) + return false; + } + return true; + } + Map getAddresses() { Map rv = new HashMap(_transports.size()); for (int i = 0; i < _transports.size(); i++) { diff --git a/router/java/src/net/i2p/router/transport/ntcp/EstablishState.java b/router/java/src/net/i2p/router/transport/ntcp/EstablishState.java index 2e6de34bb..a3e06e09a 100644 --- a/router/java/src/net/i2p/router/transport/ntcp/EstablishState.java +++ b/router/java/src/net/i2p/router/transport/ntcp/EstablishState.java @@ -351,7 +351,7 @@ public class EstablishState { long diff = 1000*Math.abs(_tsA-_tsB); if (diff >= Router.CLOCK_FUDGE_FACTOR) { _context.statManager().addRateData("ntcp.invalidOutboundSkew", diff, 0); - _transport.markReachable(_con.getRemotePeer().calculateHash()); + _transport.markReachable(_con.getRemotePeer().calculateHash(), false); _context.shitlist().shitlistRouter(_con.getRemotePeer().calculateHash(), "Outbound clock skew of " + diff + " ms"); fail("Clocks too skewed (" + diff + " ms)", null, true); return; @@ -536,7 +536,7 @@ public class EstablishState { long diff = 1000*Math.abs(tsA-_tsB); if (diff >= Router.CLOCK_FUDGE_FACTOR) { _context.statManager().addRateData("ntcp.invalidInboundSkew", diff, 0); - _transport.markReachable(alice.calculateHash()); + _transport.markReachable(alice.calculateHash(), true); _context.shitlist().shitlistRouter(alice.calculateHash(), "Clock skew of " + diff + " ms"); fail("Clocks too skewed (" + diff + " ms)", null, true); return; diff --git a/router/java/src/net/i2p/router/transport/ntcp/NTCPConnection.java b/router/java/src/net/i2p/router/transport/ntcp/NTCPConnection.java index 9bf31fa06..648e02e46 100644 --- a/router/java/src/net/i2p/router/transport/ntcp/NTCPConnection.java +++ b/router/java/src/net/i2p/router/transport/ntcp/NTCPConnection.java @@ -419,7 +419,7 @@ public class NTCPConnection implements FIFOBandwidthLimiter.CompleteListener { _established = true; _establishedOn = System.currentTimeMillis(); _establishState = null; - _transport.markReachable(getRemotePeer().calculateHash()); + _transport.markReachable(getRemotePeer().calculateHash(), false); //_context.shitlist().unshitlistRouter(getRemotePeer().calculateHash(), NTCPTransport.STYLE); boolean msgs = false; synchronized (_outbound) { diff --git a/router/java/src/net/i2p/router/transport/ntcp/NTCPTransport.java b/router/java/src/net/i2p/router/transport/ntcp/NTCPTransport.java index 06a0d04a7..38128abc2 100644 --- a/router/java/src/net/i2p/router/transport/ntcp/NTCPTransport.java +++ b/router/java/src/net/i2p/router/transport/ntcp/NTCPTransport.java @@ -132,7 +132,7 @@ public class NTCPTransport extends TransportImpl { void inboundEstablished(NTCPConnection con) { _context.statManager().addRateData("ntcp.inboundEstablished", 1, 0); - markReachable(con.getRemotePeer().calculateHash()); + markReachable(con.getRemotePeer().calculateHash(), true); //_context.shitlist().unshitlistRouter(con.getRemotePeer().calculateHash()); NTCPConnection old = null; synchronized (_conLock) { diff --git a/router/java/src/net/i2p/router/transport/udp/UDPTransport.java b/router/java/src/net/i2p/router/transport/udp/UDPTransport.java index 77c1e3b7c..1e7fed417 100644 --- a/router/java/src/net/i2p/router/transport/udp/UDPTransport.java +++ b/router/java/src/net/i2p/router/transport/udp/UDPTransport.java @@ -555,7 +555,7 @@ public class UDPTransport extends TransportImpl implements TimedWeightedPriority _log.warn("Peer already connected: old=" + oldPeer + " new=" + peer, new Exception("dup")); _activeThrottle.unchoke(peer.getRemotePeer()); - markReachable(peer.getRemotePeer()); + markReachable(peer.getRemotePeer(), peer.isInbound()); //_context.shitlist().unshitlistRouter(peer.getRemotePeer(), STYLE); if (SHOULD_FLOOD_PEERS) diff --git a/router/java/src/net/i2p/router/tunnel/pool/TunnelPeerSelector.java b/router/java/src/net/i2p/router/tunnel/pool/TunnelPeerSelector.java index 97a129978..1a553fec7 100644 --- a/router/java/src/net/i2p/router/tunnel/pool/TunnelPeerSelector.java +++ b/router/java/src/net/i2p/router/tunnel/pool/TunnelPeerSelector.java @@ -155,8 +155,12 @@ public abstract class TunnelPeerSelector { // // Unreachable peers at the inbound gateway is a major cause of problems. // Due to a bug in SSU peer testing in 0.6.1.32 and earlier, peers don't know - // if they are unreachable, so this won't help much. As of 0.6.1.33 we should have - // lots of unreachables, so enable this for now. + // if they are unreachable, so the netdb indication won't help much. + // As of 0.6.1.33 we should have lots of unreachables, so enable this for now. + // Also (and more effectively) exclude peers we detect are unreachable, + // this should be much more effective, especially on a router that has been + // up a few hours. + // // We could just try and exclude them as the inbound gateway but that's harder // (and even worse for anonymity?). // @@ -168,6 +172,9 @@ public abstract class TunnelPeerSelector { List caps = ctx.peerManager().getPeersByCapability(Router.CAPABILITY_UNREACHABLE); if (caps != null) peers.addAll(caps); + caps = ctx.profileOrganizer().selectPeersLocallyUnreachable(); + if (caps != null) + peers.addAll(caps); } if (filterSlow(ctx, isInbound, isExploratory)) { Log log = ctx.logManager().getLog(TunnelPeerSelector.class);