From d8080278b349fa9222767986454d25062d7dc3c5 Mon Sep 17 00:00:00 2001 From: zzz Date: Sun, 6 Feb 2011 00:12:54 +0000 Subject: [PATCH 1/9] initial DHT code, needs work --- .../src/org/klomp/snark/dht/DHTNodes.java | 110 ++ .../src/org/klomp/snark/dht/DHTTracker.java | 132 ++ .../src/org/klomp/snark/dht/InfoHash.java | 19 + .../java/src/org/klomp/snark/dht/KRPC.java | 1278 +++++++++++++++++ .../java/src/org/klomp/snark/dht/MsgID.java | 32 + .../java/src/org/klomp/snark/dht/NID.java | 19 + .../src/org/klomp/snark/dht/NodeInfo.java | 181 +++ .../klomp/snark/dht/NodeInfoComparator.java | 31 + .../java/src/org/klomp/snark/dht/Peer.java | 30 + .../java/src/org/klomp/snark/dht/Peers.java | 21 + .../org/klomp/snark/dht/SHA1Comparator.java | 31 + .../java/src/org/klomp/snark/dht/Token.java | 39 + .../src/org/klomp/snark/dht/TokenKey.java | 20 + .../src/org/klomp/snark/dht/Torrents.java | 19 + 14 files changed, 1962 insertions(+) create mode 100644 apps/i2psnark/java/src/org/klomp/snark/dht/DHTNodes.java create mode 100644 apps/i2psnark/java/src/org/klomp/snark/dht/DHTTracker.java create mode 100644 apps/i2psnark/java/src/org/klomp/snark/dht/InfoHash.java create mode 100644 apps/i2psnark/java/src/org/klomp/snark/dht/KRPC.java create mode 100644 apps/i2psnark/java/src/org/klomp/snark/dht/MsgID.java create mode 100644 apps/i2psnark/java/src/org/klomp/snark/dht/NID.java create mode 100644 apps/i2psnark/java/src/org/klomp/snark/dht/NodeInfo.java create mode 100644 apps/i2psnark/java/src/org/klomp/snark/dht/NodeInfoComparator.java create mode 100644 apps/i2psnark/java/src/org/klomp/snark/dht/Peer.java create mode 100644 apps/i2psnark/java/src/org/klomp/snark/dht/Peers.java create mode 100644 apps/i2psnark/java/src/org/klomp/snark/dht/SHA1Comparator.java create mode 100644 apps/i2psnark/java/src/org/klomp/snark/dht/Token.java create mode 100644 apps/i2psnark/java/src/org/klomp/snark/dht/TokenKey.java create mode 100644 apps/i2psnark/java/src/org/klomp/snark/dht/Torrents.java diff --git a/apps/i2psnark/java/src/org/klomp/snark/dht/DHTNodes.java b/apps/i2psnark/java/src/org/klomp/snark/dht/DHTNodes.java new file mode 100644 index 0000000000..cd6d0f37ca --- /dev/null +++ b/apps/i2psnark/java/src/org/klomp/snark/dht/DHTNodes.java @@ -0,0 +1,110 @@ +package org.klomp.snark.dht; +/* + * From zzzot, modded and relicensed to GPLv2 + */ + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.Iterator; +import java.util.List; +import java.util.Set; +import java.util.TreeSet; +import java.util.concurrent.ConcurrentHashMap; + +import net.i2p.I2PAppContext; +import net.i2p.crypto.SHA1Hash; +import net.i2p.data.DataHelper; +import net.i2p.util.Log; +import net.i2p.util.SimpleScheduler; +import net.i2p.util.SimpleTimer; + +/** + * All the nodes we know about, stored as a mapping from + * node ID to a Destination and Port. + * Also uses the keySet as a subsitute for kbuckets. + * + * Swap this out for a real DHT later. + * + * @since 0.8.4 + * @author zzz + */ +public class DHTNodes extends ConcurrentHashMap { + + private final I2PAppContext _context; + private long _expireTime; + private final Log _log; + + /** stagger with other cleaners */ + private static final long CLEAN_TIME = 237*1000; + private static final long MAX_EXPIRE_TIME = 60*60*1000; + private static final long MIN_EXPIRE_TIME = 5*60*1000; + private static final long DELTA_EXPIRE_TIME = 7*60*1000; + private static final int MAX_PEERS = 9999; + + public DHTNodes(I2PAppContext ctx) { + super(); + _context = ctx; + _expireTime = MAX_EXPIRE_TIME; + _log = _context.logManager().getLog(DHTNodes.class); + SimpleScheduler.getInstance().addPeriodicEvent(new Cleaner(), CLEAN_TIME); + } + + /** + * Fake DHT + * @param sha1 either a InfoHash or a NID + */ + List findClosest(SHA1Hash h, int numWant) { + // sort the whole thing + Set all = new TreeSet(new SHA1Comparator(h)); + all.addAll(keySet()); + int sz = all.size(); + int max = Math.min(numWant, sz); + + // return the first ones + List rv = new ArrayList(max); + int count = 0; + for (NID nid : all) { + if (count++ >= max) + break; + NodeInfo nInfo = get(nid); + if (nInfo == null) + continue; + rv.add(nInfo); + } + return rv; + } + + /**** used CHM methods to be replaced: + public Collection values() {} + public NodeInfo get(NID nID) {} + public NodeInfo putIfAbssent(NID nID, NodeInfo nInfo) {} + public int size() {} + ****/ + + /** */ + private class Cleaner implements SimpleTimer.TimedEvent { + + public void timeReached() { + long now = _context.clock().now(); + int peerCount = 0; + for (Iterator iter = DHTNodes.this.values().iterator(); iter.hasNext(); ) { + NodeInfo peer = iter.next(); + if (peer.lastSeen() < now - _expireTime) + iter.remove(); + else + peerCount++; + } + + if (peerCount > MAX_PEERS) + _expireTime = Math.max(_expireTime - DELTA_EXPIRE_TIME, MIN_EXPIRE_TIME); + else + _expireTime = Math.min(_expireTime + DELTA_EXPIRE_TIME, MAX_EXPIRE_TIME); + + if (_log.shouldLog(Log.INFO)) + _log.info("DHT storage cleaner done, now with " + + peerCount + " peers, " + + DataHelper.formatDuration(_expireTime) + " expiration"); + + } + } +} diff --git a/apps/i2psnark/java/src/org/klomp/snark/dht/DHTTracker.java b/apps/i2psnark/java/src/org/klomp/snark/dht/DHTTracker.java new file mode 100644 index 0000000000..712b89e4f0 --- /dev/null +++ b/apps/i2psnark/java/src/org/klomp/snark/dht/DHTTracker.java @@ -0,0 +1,132 @@ +package org.klomp.snark.dht; +/* + * From zzzot, relicensed to GPLv2 + */ + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; + +import net.i2p.I2PAppContext; +import net.i2p.data.DataHelper; +import net.i2p.data.Hash; +import net.i2p.util.Log; +import net.i2p.util.SimpleScheduler; +import net.i2p.util.SimpleTimer; + +/** + * The tracker stores peers, i.e. Dest hashes (not nodes). + * + * @since 0.8.4 + * @author zzz + */ +class DHTTracker { + + private final I2PAppContext _context; + private final Torrents _torrents; + private long _expireTime; + private final Log _log; + + /** stagger with other cleaners */ + private static final long CLEAN_TIME = 199*1000; + /** make this longer than postman's tracker */ + private static final long MAX_EXPIRE_TIME = 95*60*1000; + private static final long MIN_EXPIRE_TIME = 5*60*1000; + private static final long DELTA_EXPIRE_TIME = 7*60*1000; + private static final int MAX_PEERS = 9999; + + DHTTracker(I2PAppContext ctx) { + _context = ctx; + _torrents = new Torrents(); + _expireTime = MAX_EXPIRE_TIME; + _log = _context.logManager().getLog(DHTTracker.class); + SimpleScheduler.getInstance().addPeriodicEvent(new Cleaner(), CLEAN_TIME); + } + + void stop() { + _torrents.clear(); + // no way to stop the cleaner + } + + void announce(InfoHash ih, Hash hash) { + if (_log.shouldLog(Log.DEBUG)) + _log.debug("Announce " + hash + " for " + ih); + Peers peers = _torrents.get(ih); + if (peers == null) { + peers = new Peers(); + Peers peers2 = _torrents.putIfAbsent(ih, peers); + if (peers2 != null) + peers = peers2; + } + + Peer peer = new Peer(hash.getData()); + Peer peer2 = peers.putIfAbsent(peer, peer); + if (peer2 != null) + peer = peer2; + peer.setLastSeen(_context.clock().now()); + } + + void unannounce(InfoHash ih, Hash hash) { + Peers peers = _torrents.get(ih); + if (peers == null) + return; + Peer peer = new Peer(hash.getData()); + peers.remove(peer); + } + + /** + * Caller's responsibility to remove himself from the list + * @return list or empty list (never null) + */ + List getPeers(InfoHash ih, int max) { + Peers peers = _torrents.get(ih); + if (peers == null) + return Collections.EMPTY_LIST; + + int size = peers.size(); + List rv = new ArrayList(peers.values()); + if (max < size) { + Collections.shuffle(rv, _context.random()); + rv = rv.subList(0, max); + } + return rv; + } + + private class Cleaner implements SimpleTimer.TimedEvent { + + public void timeReached() { + long now = _context.clock().now(); + int torrentCount = 0; + int peerCount = 0; + for (Iterator iter = _torrents.values().iterator(); iter.hasNext(); ) { + Peers p = iter.next(); + int recent = 0; + for (Iterator iterp = p.values().iterator(); iterp.hasNext(); ) { + Peer peer = iterp.next(); + if (peer.lastSeen() < now - _expireTime) + iterp.remove(); + else { + recent++; + peerCount++; + } + } + if (recent <= 0) + iter.remove(); + else + torrentCount++; + } + + if (peerCount > MAX_PEERS) + _expireTime = Math.max(_expireTime - DELTA_EXPIRE_TIME, MIN_EXPIRE_TIME); + else + _expireTime = Math.min(_expireTime + DELTA_EXPIRE_TIME, MAX_EXPIRE_TIME); + + if (_log.shouldLog(Log.INFO)) + _log.info("DHT tracker cleaner done, now with " + + torrentCount + " torrents, " + + peerCount + " peers, " + + DataHelper.formatDuration(_expireTime) + " expiration"); + } + } +} diff --git a/apps/i2psnark/java/src/org/klomp/snark/dht/InfoHash.java b/apps/i2psnark/java/src/org/klomp/snark/dht/InfoHash.java new file mode 100644 index 0000000000..2b439c6c4a --- /dev/null +++ b/apps/i2psnark/java/src/org/klomp/snark/dht/InfoHash.java @@ -0,0 +1,19 @@ +package org.klomp.snark.dht; +/* + * From zzzot, modded and relicensed to GPLv2 + */ + +import net.i2p.crypto.SHA1Hash; + +/** + * A 20-byte SHA1 info hash + * + * @since 0.8.4 + * @author zzz + */ +public class InfoHash extends SHA1Hash { + + public InfoHash(byte[] data) { + super(data); + } +} diff --git a/apps/i2psnark/java/src/org/klomp/snark/dht/KRPC.java b/apps/i2psnark/java/src/org/klomp/snark/dht/KRPC.java new file mode 100644 index 0000000000..ef6757a831 --- /dev/null +++ b/apps/i2psnark/java/src/org/klomp/snark/dht/KRPC.java @@ -0,0 +1,1278 @@ +package org.klomp.snark.dht; + +/* + * GPLv2 + */ + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.Set; +import java.util.SortedSet; +import java.util.TreeSet; +import java.util.concurrent.ConcurrentHashMap; + +import net.i2p.I2PAppContext; +import net.i2p.client.I2PClient; +import net.i2p.client.I2PSession; +import net.i2p.client.I2PSessionException; +import net.i2p.client.I2PSessionMuxedListener; +import net.i2p.client.datagram.I2PDatagramDissector; +import net.i2p.client.datagram.I2PDatagramMaker; +import net.i2p.client.datagram.I2PInvalidDatagramException; +import net.i2p.crypto.SHA1Hash; +import net.i2p.data.DataFormatException; +import net.i2p.data.DataHelper; +import net.i2p.data.Destination; +import net.i2p.data.Hash; +import net.i2p.data.SimpleDataStructure; +import net.i2p.util.Log; +import net.i2p.util.SimpleScheduler; +import net.i2p.util.SimpleTimer; +import net.i2p.util.SimpleTimer2; + +import org.klomp.snark.bencode.BDecoder; +import org.klomp.snark.bencode.BEncoder; +import org.klomp.snark.bencode.BEValue; +import org.klomp.snark.bencode.InvalidBEncodingException; + + +/** + * Standard BEP 5 + * Mods for I2P: + * 
+ * - The UDP port need not be pinged after receiving a PORT message.
+ *
+ * - The UDP (datagram) port listed in the compact node info is used
+ *   to receive repliable (signed) datagrams.
+ *   This is used for queries, except for announces.
+ *   We call this the "query port".
+ *   In addition to that UDP port, we use a second datagram
+ *   port equal to the signed port + 1. This is used to receive
+ *   unsigned (raw) datagrams for replies, errors, and announce queries..
+ *   We call this the "response port".
+ *
+ * - Compact peer info is 32 bytes (32 byte SHA256 Hash)
+ *   instead of 4 byte IP + 2 byte port. There is no peer port.
+ *
+ * - Compact node info is 54 bytes (20 byte SHA1 Hash + 32 byte SHA256 Hash + 2 byte port)
+ *   instead of 20 byte SHA1 Hash + 4 byte IP + 2 byte port.
+ *   Port is the query port, the response port is always the query port + 1.
+ *
+ * - The trackerless torrent dictionary "nodes" key is a list of
+ *   32 byte binary strings (SHA256 Hashes) instead of a list of lists
+ *   containing a host string and a port integer.
+ *
+ * + * Questions: + * - nodes (in the find_node and get_peers response) is one concatenated string, not a list of strings, right? + * - Node ID enforcement, keyspace rotation? + * + * @since 0.8.4 + * @author zzz + */ +public class KRPC implements I2PSessionMuxedListener, DHT { + + private final I2PAppContext _context; + private final Log _log; + + /** our tracker */ + private final DHTTracker _tracker; + /** who we know */ + private final DHTNodes _knownNodes; + /** index to sent queries awaiting reply */ + private final ConcurrentHashMap _sentQueries; + /** index to outgoing tokens, sent in reply to a get_peers query */ + private final ConcurrentHashMap _outgoingTokens; + /** index to incoming tokens, received in a peers or nodes reply */ + private final ConcurrentHashMap _incomingTokens; + + /** hook to inject and receive datagrams */ + private final I2PSession _session; + /** 20 byte random id + 32 byte Hash + 2 byte port */ + private final NodeInfo _myNodeInfo; + /** unsigned dgrams */ + private final int _rPort; + /** signed dgrams */ + private final int _qPort; + + /** all-zero NID used for pings */ + private static final NID _fakeNID = new NID(new byte[NID.HASH_LENGTH]); + + /** Max number of nodes to return. BEP 5 says 8 */ + private static final int K = 8; + /** Max number of peers to return. BEP 5 doesn't say. We'll use the same as I2PSnarkUtil.MAX_CONNECTIONS */ + private static final int MAX_WANT = 16; + + /** overloads error codes which start with 201 */ + private static final int REPLY_NONE = 0; + private static final int REPLY_PONG = 1; + private static final int REPLY_PEERS = 2; + private static final int REPLY_NODES = 3; + + /** how long since last heard from do we delete - BEP 5 says 15 minutes */ + private static final long MAX_NODEINFO_AGE = 60*60*1000; + /** how long since generated do we delete - BEP 5 says 10 minutes */ + private static final long MAX_TOKEN_AGE = 60*60*1000; + /** how long since sent do we wait for a reply */ + private static final long MAX_MSGID_AGE = 2*60*1000; + /** how long since sent do we wait for a reply */ + private static final long DEFAULT_QUERY_TIMEOUT = 75*1000; + /** stagger with other cleaners */ + private static final long CLEAN_TIME = 63*1000; + + public KRPC (I2PAppContext ctx, I2PSession session) { + _context = ctx; + _session = session; + _log = ctx.logManager().getLog(KRPC.class); + _tracker = new DHTTracker(ctx); + + // in place of a DHT, store everybody we hear from for now + _knownNodes = new DHTNodes(ctx); + _sentQueries = new ConcurrentHashMap(); + _outgoingTokens = new ConcurrentHashMap(); + _incomingTokens = new ConcurrentHashMap(); + + // Construct my NodeInfo + // ports can really be fixed, just do this for testing + _qPort = 30000 + ctx.random().nextInt(99); + _rPort = _qPort + 1; + byte[] myID = new byte[NID.HASH_LENGTH]; + ctx.random().nextBytes(myID); + NID myNID = new NID(myID); + _myNodeInfo = new NodeInfo(myNID, session.getMyDestination(), _qPort); + + session.addMuxedSessionListener(this, I2PSession.PROTO_DATAGRAM, _rPort); + session.addMuxedSessionListener(this, I2PSession.PROTO_DATAGRAM, _qPort); + // can't be stopped + SimpleScheduler.getInstance().addPeriodicEvent(new Cleaner(), CLEAN_TIME); + } + + ///////////////// Public methods + + /** + * For bootstrapping if loaded from config file. + * @param when when did we hear from them + */ + public void addNode(NodeInfo nInfo, long when) { + heardFrom(nInfo, when); + } + + /** + * NodeInfo heard from + */ + public void addNode(NodeInfo nInfo) { + heardFrom(nInfo); + } + + /** + * For saving in a config file. + * @return the values, not a copy, could change, use an iterator + */ + public Collection getNodes() { + return _knownNodes.values(); + } + + /** + * @return The UDP port that should be included in a PORT message. + */ + public int getPort() { + return _qPort; + } + + /** + * Ping. We don't have a NID yet so the node is presumed + * to be absent from our DHT. + * Non-blocking, does not wait for pong. + * If and when the pong is received the node will be inserted in our DHT. + */ + public void ping(Destination dest, int port) { + NodeInfo nInfo = new NodeInfo(_fakeNID, dest, port); + sendPing(nInfo); + } + + /** + * Bootstrapping or background thread. + * Blocking! + * This is almost the same as getPeers() + * + * @param maxNodes how many to contact + * @param maxWait how long to wait for each to reply (not total) must be > 0 + * @param parallel how many outstanding at once (unimplemented, always 1) + */ + public void explore(int maxNodes, long maxWait, int parallel) { + // Initial set to try, will get added to as we go + NID myNID = _myNodeInfo.getNID(); + List nodes = _knownNodes.findClosest(myNID, maxNodes); + if (nodes.isEmpty()) { + if (_log.shouldLog(Log.WARN)) + _log.info("DHT is empty, cannot explore"); + return; + } + SortedSet toTry = new TreeSet(new NodeInfoComparator(myNID)); + toTry.addAll(nodes); + Set tried = new HashSet(); + + if (_log.shouldLog(Log.INFO)) + _log.info("Starting explore"); + for (int i = 0; i < maxNodes; i++) { + NodeInfo nInfo; + try { + nInfo = toTry.first(); + } catch (NoSuchElementException nsee) { + break; + } + toTry.remove(nInfo); + tried.add(nInfo); + + // this isn't going to work, he will just return our own? + ReplyWaiter waiter = sendFindNode(nInfo, _myNodeInfo); + if (waiter == null) + continue; + synchronized(waiter) { + try { + waiter.wait(maxWait); + } catch (InterruptedException ie) {} + } + + int replyType = waiter.getReplyCode(); + if (replyType == REPLY_NONE) { + if (_log.shouldLog(Log.INFO)) + _log.info("Got no reply"); + } else if (replyType == REPLY_NODES) { + List reply = (List) waiter.getReplyObject(); + // It seems like we are just going to get back ourselves all the time + if (_log.shouldLog(Log.INFO)) + _log.info("Got " + reply.size() + " nodes"); + for (NodeInfo ni : reply) { + if (! (ni.equals(_myNodeInfo) || (toTry.contains(ni) && tried.contains(ni)))) + toTry.add(ni); + } + } else { + if (_log.shouldLog(Log.INFO)) + _log.info("Got unexpected reply " + replyType + ": " + waiter.getReplyObject()); + } + } + if (_log.shouldLog(Log.INFO)) + _log.info("Finished explore"); + } + + /** + * Local lookup only + * @param ih a 20-byte info hash + * @param max max to return + * @return list or empty list (never null) + */ + public List findClosest(byte[] ih, int max) { + List nodes = _knownNodes.findClosest(new InfoHash(ih), max); + return nodes; + } + + /** + * Get peers for a torrent. + * Blocking! + * Caller should run in a thread. + * + * @param ih the Info Hash (torrent) + * @param max maximum number of peers to return + * @param maxWait the maximum time to wait (ms) must be > 0 + * @return list or empty list (never null) + */ + public List getPeers(byte[] ih, int max, long maxWait) { + // check local tracker first + InfoHash iHash = new InfoHash(ih); + List rv = _tracker.getPeers(iHash, max); + rv.remove(_myNodeInfo.getHash()); + if (!rv.isEmpty()) + return rv; // TODO get DHT too? + + // Initial set to try, will get added to as we go + List nodes = _knownNodes.findClosest(iHash, max); + SortedSet toTry = new TreeSet(new NodeInfoComparator(iHash)); + toTry.addAll(nodes); + Set tried = new HashSet(); + + if (_log.shouldLog(Log.INFO)) + _log.info("Starting getPeers"); + for (int i = 0; i < max; i++) { + NodeInfo nInfo; + try { + nInfo = toTry.first(); + } catch (NoSuchElementException nsee) { + break; + } + toTry.remove(nInfo); + tried.add(nInfo); + + ReplyWaiter waiter = sendGetPeers(nInfo, iHash); + if (waiter == null) + continue; + synchronized(waiter) { + try { + waiter.wait(maxWait); + } catch (InterruptedException ie) {} + } + + int replyType = waiter.getReplyCode(); + if (replyType == REPLY_NONE) { + if (_log.shouldLog(Log.INFO)) + _log.info("Got no reply"); + } else if (replyType == REPLY_PONG) { + if (_log.shouldLog(Log.INFO)) + _log.info("Got pong"); + } else if (replyType == REPLY_PEERS) { + if (_log.shouldLog(Log.INFO)) + _log.info("Got peers"); + List reply = (List) waiter.getReplyObject(); + if (!reply.isEmpty()) { + if (_log.shouldLog(Log.INFO)) + _log.info("Finished get Peers, returning " + reply.size()); + return reply; + } + } else if (replyType == REPLY_NODES) { + List reply = (List) waiter.getReplyObject(); + if (_log.shouldLog(Log.INFO)) + _log.info("Got " + reply.size() + " nodes"); + for (NodeInfo ni : reply) { + if (! (ni.equals(_myNodeInfo) || tried.contains(ni) || toTry.contains(ni))) + toTry.add(ni); + } + } else { + if (_log.shouldLog(Log.INFO)) + _log.info("Got unexpected reply " + replyType + ": " + waiter.getReplyObject()); + } + } + if (_log.shouldLog(Log.INFO)) + _log.info("Finished get Peers, fail"); + return Collections.EMPTY_LIST; + } + + /** + * Announce to ourselves. + * Non-blocking. + * + * @param ih the Info Hash (torrent) + */ + public void announce(byte[] ih) { + InfoHash iHash = new InfoHash(ih); + _tracker.announce(iHash, _myNodeInfo.getHash()); + } + + /** + * Announce somebody else we know about. + * Non-blocking. + * + * @param ih the Info Hash (torrent) + * @param peer the peer's Hash + */ + public void announce(byte[] ih, byte[] peerHash) { + InfoHash iHash = new InfoHash(ih); + _tracker.announce(iHash, new Hash(peerHash)); + } + + /** + * Remove reference to ourselves in the local tracker. + * Use when shutting down the torrent locally. + * Non-blocking. + * + * @param ih the Info Hash (torrent) + */ + public void unannounce(byte[] ih) { + InfoHash iHash = new InfoHash(ih); + _tracker.unannounce(iHash, _myNodeInfo.getHash()); + } + + /** + * Announce to the closest DHT peers. + * Blocking unless maxWait <= 0 + * Caller should run in a thread. + * This also automatically announces ourself to our local tracker. + * For best results do a getPeers() first so we have tokens. + * + * @param ih the Info Hash (torrent) + * @param maxWait the maximum total time to wait (ms) or 0 to do all in parallel and return immediately. + * @return the number of successful announces, not counting ourselves. + */ + public int announce(byte[] ih, int max, long maxWait) { + announce(ih); + int rv = 0; + long start = _context.clock().now(); + List nodes = _knownNodes.findClosest(new InfoHash(ih), max); + for (NodeInfo nInfo : nodes) { + if (announce(ih, nInfo, Math.min(maxWait, 60*1000))) + rv++; + maxWait -= _context.clock().now() - start; + if (maxWait < 1000) + break; + } + return rv; + } + + /** + * Announce to a single DHT peer. + * Blocking unless maxWait <= 0 + * Caller should run in a thread. + * For best results do a getPeers() first so we have a token. + * + * @param ih the Info Hash (torrent) + * @param nInfo the peer to announce to + * @param maxWait the maximum time to wait (ms) or 0 to return immediately. + * @return success + */ + public boolean announce(byte[] ih, NodeInfo nInfo, long maxWait) { + InfoHash iHash = new InfoHash(ih); + TokenKey tokenKey = new TokenKey(nInfo.getNID(), iHash); + Token token = _incomingTokens.get(tokenKey); + if (token == null) { + // we have no token, have to do a getPeers first to get a token + if (maxWait <= 0) + return false; + ReplyWaiter waiter = sendGetPeers(nInfo, iHash); + if (waiter == null) + return false; + long start = _context.clock().now(); + synchronized(waiter) { + try { + waiter.wait(maxWait); + } catch (InterruptedException ie) {} + } + int replyType = waiter.getReplyCode(); + if (!(replyType == REPLY_PEERS || replyType == REPLY_NODES)) + return false; + // we should have a token now + token = _incomingTokens.get(tokenKey); + if (token == null) + return false; + maxWait -= _context.clock().now() - start; + if (maxWait < 1000) + return false; + } + + // send and wait on rcv msg lock unless maxWait <= 0 + ReplyWaiter waiter = sendAnnouncePeer(nInfo, iHash, token); + if (waiter == null) + return false; + if (maxWait <= 0) + return true; + synchronized(waiter) { + try { + waiter.wait(maxWait); + } catch (InterruptedException ie) {} + } + int replyType = waiter.getReplyCode(); + return replyType == REPLY_PONG; + } + + /** + * Does nothing yet, everything is prestarted. + * Can't be restarted after stopping? + */ + public void start() { + // start the explore thread + } + + /** + * Does nothing yet. + */ + public void stop() { + // stop the explore thread + // unregister port listeners + // does not clear the DHT or tracker yet. + } + + /** + * Clears the tracker and DHT data. + * Call after saving DHT data to disk. + */ + public void clear() { + _tracker.stop(); + _knownNodes.clear(); + } + + ////////// All private below here ///////////////////////////////////// + + ///// Sending..... + + // Queries..... + // The first 3 queries use the query port. + // Announces use the response port. + + /** + * @param nInfo who to send it to + * @return null on error + */ + private ReplyWaiter sendPing(NodeInfo nInfo) { + Map map = new HashMap(); + map.put("q", "ping"); + Map args = new HashMap(); + map.put("a", args); + return sendQuery(nInfo, map, true); + } + + /** + * @param nInfo who to send it to + * @return null on error + */ + private ReplyWaiter sendFindNode(NodeInfo nInfo, NodeInfo tID) { + Map map = new HashMap(); + map.put("q", "find_node"); + Map args = new HashMap(); + args.put("target", tID.getData()); + map.put("a", args); + return sendQuery(nInfo, map, true); + } + + /** + * @param nInfo who to send it to + * @return null on error + */ + private ReplyWaiter sendGetPeers(NodeInfo nInfo, InfoHash ih) { + Map map = new HashMap(); + map.put("q", "get_peers"); + Map args = new HashMap(); + args.put("info_hash", ih.getData()); + map.put("a", args); + return sendQuery(nInfo, map, true); + } + + /** + * @param nInfo who to send it to + * @return null on error + */ + private ReplyWaiter sendAnnouncePeer(NodeInfo nInfo, InfoHash ih, Token token) { + Map map = new HashMap(); + map.put("q", "announce_peer"); + Map args = new HashMap(); + args.put("info_hash", ih.getData()); + // port ignored + args.put("port", Integer.valueOf(6881)); + args.put("token", token.getData()); + map.put("a", args); + // an announce need not be signed, we have a token + ReplyWaiter rv = sendQuery(nInfo, map, false); + // save the InfoHash so we can get it later + if (rv != null) + rv.setSentObject(ih); + return rv; + } + + // Responses..... + // All responses use the response port. + + /** + * @param nInfo who to send it to + * @return success + */ + private boolean sendPong(NodeInfo nInfo, MsgID msgID) { + Map map = new HashMap(); + Map resps = new HashMap(); + map.put("r", resps); + return sendResponse(nInfo, msgID, map); + } + + /** response to find_node (no token) */ + private boolean sendNodes(NodeInfo nInfo, MsgID msgID, byte[] ids) { + return sendNodes(nInfo, msgID, null, ids); + } + + /** + * response to find_node (token is null) or get_peers (has a token) + * @param nInfo who to send it to + * @return success + */ + private boolean sendNodes(NodeInfo nInfo, MsgID msgID, Token token, byte[] ids) { + Map map = new HashMap(); + Map resps = new HashMap(); + map.put("r", resps); + if (token != null) + resps.put("token", token.getData()); + resps.put("nodes", ids); + return sendResponse(nInfo, msgID, map); + } + + /** @param token non-null */ + private boolean sendPeers(NodeInfo nInfo, MsgID msgID, Token token, List peers) { + Map map = new HashMap(); + Map resps = new HashMap(); + map.put("r", resps); + resps.put("token", token.getData()); + resps.put("values", peers); + return sendResponse(nInfo, msgID, map); + } + + // All errors use the response port. + + /** + * @param nInfo who to send it to + * @return success + */ + private boolean sendError(NodeInfo nInfo, MsgID msgID, int err, String msg) { + Map map = new HashMap(); + Map resps = new HashMap(); + map.put("r", resps); + return sendResponse(nInfo, msgID, map); + } + + // Low-level send methods + + // TODO sendQuery with onReply / onTimeout args + + /** + * @param repliable true for all but announce + * @return null on error + */ + private ReplyWaiter sendQuery(NodeInfo nInfo, Map map, boolean repliable) { + if (nInfo.equals(_myNodeInfo)) + throw new IllegalArgumentException("wtf don't send to ourselves"); + if (_log.shouldLog(Log.INFO)) + _log.info("Sending query to: " + nInfo); + if (nInfo.getDestination() == null) { + NodeInfo newInfo = _knownNodes.get(nInfo.getNID()); + if (newInfo != null && newInfo.getDestination() != null) { + nInfo = newInfo; + } else { + // lookup b32? + if (_log.shouldLog(Log.WARN)) + _log.warn("No destination for: " + nInfo); + return null; + } + } + map.put("y", "q"); + MsgID mID = new MsgID(_context); + map.put("t", mID.getData()); + Map args = (Map) map.get("a"); + if (args == null) + throw new IllegalArgumentException("no args"); + args.put("id", _myNodeInfo.getData()); + int port = nInfo.getPort(); + if (!repliable) + port++; + boolean success = sendMessage(nInfo.getDestination(), port, map, true); + if (success) { + // save for the caller to get + ReplyWaiter rv = new ReplyWaiter(mID, nInfo, null, null); + _sentQueries.put(mID, rv); + return rv; + } + return null; + } + + /** + * @param toPort the query port, we will increment here + * @return success + */ + private boolean sendResponse(NodeInfo nInfo, MsgID msgID, Map map) { + if (nInfo.equals(_myNodeInfo)) + throw new IllegalArgumentException("wtf don't send to ourselves"); + if (_log.shouldLog(Log.INFO)) + _log.info("Sending response to: " + nInfo); + if (nInfo.getDestination() == null) { + NodeInfo newInfo = _knownNodes.get(nInfo.getNID()); + if (newInfo != null && newInfo.getDestination() != null) { + nInfo = newInfo; + } else { + // lookup b32? + if (_log.shouldLog(Log.WARN)) + _log.warn("No destination for: " + nInfo); + return false; + } + } + map.put("y", "r"); + map.put("t", msgID.getData()); + Map resps = (Map) map.get("r"); + if (resps == null) + throw new IllegalArgumentException("no resps"); + resps.put("id", _myNodeInfo.getData()); + return sendMessage(nInfo.getDestination(), nInfo.getPort() + 1, map, false); + } + + /** + * @param toPort the query port, we will increment here + * @return success + */ + private boolean sendError(NodeInfo nInfo, MsgID msgID, Map map) { + if (nInfo.equals(_myNodeInfo)) + throw new IllegalArgumentException("wtf don't send to ourselves"); + if (_log.shouldLog(Log.INFO)) + _log.info("Sending error to: " + nInfo); + if (nInfo.getDestination() == null) { + NodeInfo newInfo = _knownNodes.get(nInfo.getNID()); + if (newInfo != null && newInfo.getDestination() != null) { + nInfo = newInfo; + } else { + // lookup b32? + if (_log.shouldLog(Log.WARN)) + _log.warn("No destination for: " + nInfo); + return false; + } + } + map.put("y", "e"); + map.put("t", msgID.getData()); + return sendMessage(nInfo.getDestination(), nInfo.getPort() + 1, map, false); + } + + /** + * Lowest-level send message call. + * @param repliable true for all but announce + * @return success + */ + private boolean sendMessage(Destination dest, int toPort, Map map, boolean repliable) { + if (_session.isClosed()) { + // Don't allow DHT to open a closed session + if (_log.shouldLog(Log.WARN)) + _log.warn("Not sending message, session is closed"); + return false; + } + if (dest.calculateHash().equals(_myNodeInfo.getHash())) + throw new IllegalArgumentException("wtf don't send to ourselves"); + byte[] payload = BEncoder.bencode(map); + if (_log.shouldLog(Log.DEBUG)) { + ByteArrayInputStream bais = new ByteArrayInputStream(payload); + try { + _log.debug("Sending to: " + dest.calculateHash() + ' ' + BDecoder.bdecode(bais).toString()); + } catch (IOException ioe) {} + } + + // Always send query port, peer will increment for unsigned replies + int fromPort = _qPort; + if (repliable) { + I2PDatagramMaker dgMaker = new I2PDatagramMaker(_session); + payload = dgMaker.makeI2PDatagram(payload); + if (payload == null) { + if (_log.shouldLog(Log.WARN)) + _log.warn("WTF DGM fail"); + } + } + + try { + boolean success = _session.sendMessage(dest, payload, 0, payload.length, null, null, 60*1000, + I2PSession.PROTO_DATAGRAM, fromPort, toPort); + if (!success) { + if (_log.shouldLog(Log.WARN)) + _log.warn("WTF sendMessage fail"); + } + return success; + } catch (I2PSessionException ise) { + if (_log.shouldLog(Log.WARN)) + _log.warn("sendMessage fail", ise); + return false; + } + } + + ///// Reception..... + + /** + * @param from dest or null if it didn't come in on signed port + */ + private void receiveMessage(Destination from, int fromPort, byte[] payload) { + + try { + InputStream is = new ByteArrayInputStream(payload); + BDecoder dec = new BDecoder(is); + BEValue bev = dec.bdecodeMap(); + Map map = bev.getMap(); + if (_log.shouldLog(Log.DEBUG)) + _log.debug("Got KRPC message " + bev.toString()); + + // Lazy here, just let missing Map entries throw NPEs, caught below + + byte[] msgIDBytes = map.get("t").getBytes(); + MsgID mID = new MsgID(msgIDBytes); + String type = map.get("y").getString(); + if (type.equals("q") && from != null) { + // queries must be repliable + String method = map.get("q").getString(); + Map args = map.get("a").getMap(); + receiveQuery(mID, from, fromPort, method, args); + } else if (type.equals("r") || type.equals("e")) { + // get dest from id->dest map + ReplyWaiter waiter = _sentQueries.remove(mID); + if (waiter != null) { + // TODO verify waiter NID and port? + if (type.equals("r")) { + Map response = map.get("r").getMap(); + receiveResponse(waiter, response); + } else { + List error = map.get("e").getList(); + receiveError(waiter, error); + } + } else { + if (_log.shouldLog(Log.WARN)) + _log.warn("Rcvd msg with no one waiting: " + bev.toString()); + } + } else { + if (_log.shouldLog(Log.WARN)) + _log.warn("Unknown msg type rcvd: " + bev.toString()); + throw new InvalidBEncodingException("Unknown type: " + type); + } + // success + /*** + } catch (InvalidBEncodingException e) { + } catch (IOException e) { + } catch (ArrayIndexOutOfBoundsException e) { + } catch (IllegalArgumentException e) { + } catch (ClassCastException e) { + } catch (NullPointerException e) { + ***/ + } catch (Exception e) { + if (_log.shouldLog(Log.WARN)) + _log.warn("Receive error for message", e); + } + } + + + // Queries..... + + /** + * Adds sender to our DHT. + * @param dest non-null + * @throws NPE too + */ + private void receiveQuery(MsgID msgID, Destination dest, int fromPort, String method, Map args) throws InvalidBEncodingException { + byte[] nid = args.get("id").getBytes(); + NodeInfo nInfo = new NodeInfo(nid); + nInfo = heardFrom(nInfo); + nInfo.setDestination(dest); +// ninfo.checkport ? + + if (method.equals("ping")) { + receivePing(msgID, nInfo); + } else if (method.equals("find_node")) { + byte[] tid = args.get("target").getBytes(); + NodeInfo tID = new NodeInfo(tid); + receiveFindNode(msgID, nInfo, tID); + } else if (method.equals("get_peers")) { + byte[] hash = args.get("info_hash").getBytes(); + InfoHash ih = new InfoHash(hash); + receiveGetPeers(msgID, nInfo, ih); + } else if (method.equals("announce_peer")) { + byte[] hash = args.get("info_hash").getBytes(); + InfoHash ih = new InfoHash(hash); + // this is the "TCP" port, we don't care + //int port = args.get("port").getInt(); + byte[] token = args.get("token").getBytes(); + receiveAnnouncePeer(msgID, nInfo, ih, token); + } else { + if (_log.shouldLog(Log.WARN)) + _log.warn("Unknown query method rcvd: " + method); + } + } + + /** + * Called for a request or response + * @return old NodeInfo or nInfo if none, use this to reduce object churn + */ + private NodeInfo heardFrom(NodeInfo nInfo) { + return heardFrom(nInfo, _context.clock().now()); + } + + /** + * Used for initialization + * @return old NodeInfo or nInfo if none, use this to reduce object churn + */ + private NodeInfo heardFrom(NodeInfo nInfo, long when) { + // try to keep ourselves out of the DHT + if (nInfo.equals(_myNodeInfo)) + return _myNodeInfo; + NID nID = nInfo.getNID(); + NodeInfo oldInfo = _knownNodes.get(nID); + if (oldInfo == null) { + if (_log.shouldLog(Log.INFO)) + _log.info("Adding node: " + nInfo); + oldInfo = nInfo; + NodeInfo nInfo2 = _knownNodes.putIfAbsent(nID, nInfo); + if (nInfo2 != null) + oldInfo = nInfo2; + } + if (when > oldInfo.lastSeen()) + oldInfo.setLastSeen(when); + return oldInfo; + } + + /** + * Handle and respond to the query + */ + private void receivePing(MsgID msgID, NodeInfo nInfo) throws InvalidBEncodingException { + if (_log.shouldLog(Log.INFO)) + _log.info("Rcvd ping from: " + nInfo); + sendPong(nInfo, msgID); + } + + /** + * Handle and respond to the query + */ + private void receiveFindNode(MsgID msgID, NodeInfo nInfo, NodeInfo tID) throws InvalidBEncodingException { + if (_log.shouldLog(Log.INFO)) + _log.info("Rcvd find_node from: " + nInfo + " for: " + tID); + NodeInfo peer = _knownNodes.get(tID); + if (peer != null) { + // success, one answer + sendNodes(nInfo, msgID, peer.getData()); + } else { + // get closest from DHT + List nodes = _knownNodes.findClosest(tID.getNID(), K); + nodes.remove(nInfo); // him + nodes.remove(_myNodeInfo); // me + byte[] nodeArray = new byte[nodes.size() * NodeInfo.LENGTH]; + for (int i = 0; i < nodes.size(); i ++) { + System.arraycopy(nodes.get(i).getData(), 0, nodeArray, i * NodeInfo.LENGTH, NodeInfo.LENGTH); + } + sendNodes(nInfo, msgID, nodeArray); + } + } + + /** + * Handle and respond to the query + */ + private void receiveGetPeers(MsgID msgID, NodeInfo nInfo, InfoHash ih) throws InvalidBEncodingException { + if (_log.shouldLog(Log.INFO)) + _log.info("Rcvd get_peers from: " + nInfo + " for: " + ih); + // generate and save random token + Token token = new Token(_context); + _outgoingTokens.put(ih, token); + + List peers = _tracker.getPeers(ih, MAX_WANT); + if (peers.isEmpty()) { + // similar to find node, but with token + // get closest from DHT + List nodes = _knownNodes.findClosest(ih, K); + nodes.remove(nInfo); // him + nodes.remove(_myNodeInfo); // me + byte[] nodeArray = new byte[nodes.size() * NodeInfo.LENGTH]; + for (int i = 0; i < nodes.size(); i ++) { + System.arraycopy(nodes.get(i).getData(), 0, nodeArray, i * NodeInfo.LENGTH, NodeInfo.LENGTH); + } + sendNodes(nInfo, msgID, token, nodeArray); + } else { + List hashes = new ArrayList(peers.size()); + Hash him = nInfo.getHash(); + for (Hash peer : peers) { + if (!peer.equals(him)) + hashes.add(peer.getData()); + } + sendPeers(nInfo, msgID, token, hashes); + } + } + + /** + * Handle and respond to the query + */ + private void receiveAnnouncePeer(MsgID msgID, NodeInfo nInfo, InfoHash ih, byte[] token) throws InvalidBEncodingException { + if (_log.shouldLog(Log.INFO)) + _log.info("Rcvd announce from: " + nInfo + " for: " + ih); + // check token + // get desthash from token->dest map + Token oldToken = _outgoingTokens.get(ih); + if (oldToken == null || !DataHelper.eq(oldToken.getData(), token)) { + if (_log.shouldLog(Log.WARN)) + _log.warn("Bad token"); + return; + } + + //msg ID -> NodeInfo -> Dest -> Hash + //verify with token -> nid or dest or hash ???? + + _tracker.announce(ih, nInfo.getHash()); + // the reply for an announce is the same as the reply for a ping + sendPong(nInfo, msgID); + } + + // Responses..... + + /** + * Handle the response and alert whoever sent the query it is responding to. + * Adds sender nodeinfo to our DHT. + * @throws NPE, IllegalArgumentException, and others too + */ + private void receiveResponse(ReplyWaiter waiter, Map response) throws InvalidBEncodingException { + NodeInfo nInfo = waiter; + + BEValue nodes = response.get("nodes"); + BEValue values = response.get("values"); + + // token handling - save it for later announces + if (nodes != null || values != null) { + BEValue btok = response.get("token"); + InfoHash ih = (InfoHash) waiter.getSentObject(); + if (btok != null && ih != null) { + byte[] tok = btok.getBytes(); + _incomingTokens.put(new TokenKey(nInfo.getNID(), ih), new Token(_context, tok)); + if (_log.shouldLog(Log.INFO)) + _log.info("Got token, must be a response to get_peers"); + } else { + if (_log.shouldLog(Log.INFO)) + _log.info("No token and saved infohash, must be a response to find_node"); + } + } + + // now do the right thing + if (nodes != null) { + // find node or get peers response - concatenated NodeInfos + byte[] ids = nodes.getBytes(); + List rlist = receiveNodes(nInfo, ids); + waiter.gotReply(REPLY_NODES, rlist); + } else if (values != null) { + // get peers response - list of Hashes + List peers = values.getList(); + List rlist = receivePeers(nInfo, peers); + waiter.gotReply(REPLY_PEERS, rlist); + } else { + // a ping response or an announce peer response + receivePong(nInfo); + waiter.gotReply(REPLY_PONG, null); + } + } + + /** + * rcv concatenated 54 byte NodeInfos, return as a List + * Adds all received nodeinfos to our DHT. + * @throws NPE, IllegalArgumentException, and others too + */ + private List receiveNodes(NodeInfo nInfo, byte[] ids) throws InvalidBEncodingException { + List rv = new ArrayList(ids.length / NodeInfo.LENGTH); + long fakeTime = _context.clock().now() - (MAX_NODEINFO_AGE * 3 / 4); + for (int off = 0; off < ids.length; off += NodeInfo.LENGTH) { + NodeInfo nInf = new NodeInfo(ids, off); + // anti-churn + // TODO do we need heardAbout too? + nInf = heardFrom(nInf, fakeTime); + rv.add(nInf); + } + if (_log.shouldLog(Log.INFO)) + _log.info("Rcvd nodes from: " + nInfo + ": " + DataHelper.toString(rv)); + return rv; + } + + /** + * rcv 32 byte Hashes, return as a List + * @throws NPE, IllegalArgumentException, and others too + */ + private List receivePeers(NodeInfo nInfo, List peers) throws InvalidBEncodingException { + if (_log.shouldLog(Log.INFO)) + _log.info("Rcvd peers from: " + nInfo); + List rv = new ArrayList(peers.size()); + for (BEValue bev : peers) { + byte[] b = bev.getBytes(); + Hash h = new Hash(b); + rv.add(h); + } + if (_log.shouldLog(Log.INFO)) + _log.info("Rcvd peers from: " + nInfo + ": " + DataHelper.toString(rv)); + return rv; + } + + /** does nothing, but node was already added to our DHT */ + private void receivePong(NodeInfo nInfo) { + if (_log.shouldLog(Log.INFO)) + _log.info("Rcvd pong from: " + nInfo); + } + + // Errors..... + + /** + * @throws NPE, and others too + */ + private void receiveError(ReplyWaiter waiter, List error) throws InvalidBEncodingException { + int errorCode = error.get(0).getInt(); + String errorString = error.get(1).getString(); + if (_log.shouldLog(Log.WARN)) + _log.warn("Rcvd error from: " + waiter + + " num: " + errorCode + + " msg: " + errorString); + // this calls heardFrom() + waiter.gotReply(errorCode, errorString); + } + + /** + * Callback for replies + */ + private class ReplyWaiter extends NodeInfo { + private final MsgID mid; + private final Runnable onReply; + private final Runnable onTimeout; + private final SimpleTimer2.TimedEvent event; + private int replyCode; + private Object sentObject; + private Object replyObject; + + /** + * Either wait on this object with a timeout, or use non-null Runnables. + * Any sent data to be rememberd may be stored by setSentObject(). + * Reply object may be in getReplyObject(). + * @param onReply must be fast, otherwise set to null and wait on this + * @param onTimeout must be fast, otherwise set to null and wait on this + */ + public ReplyWaiter(MsgID mID, NodeInfo nInfo, Runnable onReply, Runnable onTimeout) { + super(nInfo.getData()); + this.mid = mID; + this.onReply = onReply; + this.onTimeout = onTimeout; + if (onTimeout != null) + this.event = new Event(); + else + this.event = null; + } + + /** only used for announce, to save the Info Hash */ + public void setSentObject(Object o) { + sentObject = o; + } + + /** @return that stored with setSentObject() */ + public Object getSentObject() { + return sentObject; + } + + + /** + * Should contain null if getReplyCode is REPLY_PONG. + * Should contain List if getReplyCode is REPLY_PEERS. + * Should contain List if getReplyCode is REPLY_NODES. + * Should contain String if getReplyCode is > 200. + * @return may be null depending on what happened. Cast to expected type. + */ + public Object getReplyObject() { + return replyObject; + } + + /** + * If nonzero, we got a reply, and getReplyObject() may contain something. + * @return code or 0 if no error + */ + public int getReplyCode() { + return replyCode; + } + + /** + * Will notify this and run onReply. + * Also removes from _sentQueries and calls heardFrom(). + */ + public void gotReply(int code, Object o) { + replyCode = code; + replyObject = o; + if (event != null) + event.cancel(); + _sentQueries.remove(mid); + heardFrom(this); + if (onReply != null) + onReply.run(); + synchronized(this) { + this.notifyAll(); + } + } + + private class Event extends SimpleTimer2.TimedEvent { + public Event() { + super(SimpleTimer2.getInstance(), DEFAULT_QUERY_TIMEOUT); + } + + public void timeReached() { + _sentQueries.remove(mid); + if (onTimeout != null) + onTimeout.run(); + if (_log.shouldLog(Log.INFO)) + _log.warn("timeout waiting for reply from " + this.toString()); + } + } + } + + // I2PSessionMuxedListener interface ---------------- + + /** + * Instruct the client that the given session has received a message + * + * Will be called only if you register via addMuxedSessionListener(). + * Will be called only for the proto(s) and toPort(s) you register for. + * + * @param session session to notify + * @param msgId message number available + * @param size size of the message - why it's a long and not an int is a mystery + * @param proto 1-254 or 0 for unspecified + * @param fromPort 1-65535 or 0 for unspecified + * @param toPort 1-65535 or 0 for unspecified + */ + public void messageAvailable(I2PSession session, int msgId, long size, int proto, int fromPort, int toPort) { + try { + byte[] payload = session.receiveMessage(msgId); + if (toPort == _qPort) { + // repliable + I2PDatagramDissector dgDiss = new I2PDatagramDissector(); + dgDiss.loadI2PDatagram(payload); + payload = dgDiss.getPayload(); + Destination from = dgDiss.getSender(); + receiveMessage(from, fromPort, payload); + } else if (toPort == _rPort) { + // raw + receiveMessage(null, fromPort, payload); + } else { + if (_log.shouldLog(Log.WARN)) + _log.warn("msg on bad port"); + } + } catch (DataFormatException e) { + if (_log.shouldLog(Log.WARN)) + _log.warn("bad msg"); + } catch (I2PInvalidDatagramException e) { + if (_log.shouldLog(Log.WARN)) + _log.warn("bad msg"); + } catch (I2PSessionException e) { + if (_log.shouldLog(Log.WARN)) + _log.warn("bad msg"); + } + } + + /** for non-muxed */ + public void messageAvailable(I2PSession session, int msgId, long size) {} + + public void reportAbuse(I2PSession session, int severity) {} + + public void disconnected(I2PSession session) { + if (_log.shouldLog(Log.WARN)) + _log.warn("KRPC disconnected"); + } + + public void errorOccurred(I2PSession session, String message, Throwable error) { + if (_log.shouldLog(Log.WARN)) + _log.warn("KRPC got error msg: ", error); + } + + /** + * Cleaner-upper + */ + private class Cleaner implements SimpleTimer.TimedEvent { + + public void timeReached() { + long now = _context.clock().now(); + for (Iterator iter = _outgoingTokens.values().iterator(); iter.hasNext(); ) { + Token tok = iter.next(); + if (tok.lastSeen() < now - MAX_TOKEN_AGE) + iter.remove(); + } + for (Iterator iter = _incomingTokens.values().iterator(); iter.hasNext(); ) { + Token tok = iter.next(); + if (tok.lastSeen() < now - MAX_TOKEN_AGE) + iter.remove(); + } + // TODO sent queries? + for (Iterator iter = _knownNodes.values().iterator(); iter.hasNext(); ) { + NodeInfo ni = iter.next(); + if (ni.lastSeen() < now - MAX_NODEINFO_AGE) + iter.remove(); + } + if (_log.shouldLog(Log.INFO)) + _log.info("KRPC cleaner done, now with " + + _outgoingTokens.size() + " sent Tokens, " + + _incomingTokens.size() + " rcvd Tokens, " + + _knownNodes.size() + " known peers, " + + _sentQueries.size() + " queries awaiting response"); + } + } +} diff --git a/apps/i2psnark/java/src/org/klomp/snark/dht/MsgID.java b/apps/i2psnark/java/src/org/klomp/snark/dht/MsgID.java new file mode 100644 index 0000000000..94b37a690b --- /dev/null +++ b/apps/i2psnark/java/src/org/klomp/snark/dht/MsgID.java @@ -0,0 +1,32 @@ +package org.klomp.snark.dht; +/* + * GPLv2 + */ + +import net.i2p.I2PAppContext; +import net.i2p.data.ByteArray; + +/** + * Used for both incoming and outgoing message IDs + * + * @since 0.8.4 + * @author zzz + */ +public class MsgID extends ByteArray { + + private static final int MY_TOK_LEN = 8; + + /** outgoing - generate a random ID */ + public MsgID(I2PAppContext ctx) { + super(null); + byte[] data = new byte[MY_TOK_LEN]; + ctx.random().nextBytes(data); + setData(data); + setValid(MY_TOK_LEN); + } + + /** incoming - save the ID (arbitrary length) */ + public MsgID(byte[] data) { + super(data); + } +} diff --git a/apps/i2psnark/java/src/org/klomp/snark/dht/NID.java b/apps/i2psnark/java/src/org/klomp/snark/dht/NID.java new file mode 100644 index 0000000000..3d0d0a496e --- /dev/null +++ b/apps/i2psnark/java/src/org/klomp/snark/dht/NID.java @@ -0,0 +1,19 @@ +package org.klomp.snark.dht; +/* + * From zzzot, modded and relicensed to GPLv2 + */ + +import net.i2p.crypto.SHA1Hash; + +/** + * A 20-byte peer ID, used as a Map key in lots of places. + * + * @since 0.8.4 + * @author zzz + */ +public class NID extends SHA1Hash { + + public NID(byte[] data) { + super(data); + } +} diff --git a/apps/i2psnark/java/src/org/klomp/snark/dht/NodeInfo.java b/apps/i2psnark/java/src/org/klomp/snark/dht/NodeInfo.java new file mode 100644 index 0000000000..9e73ca03be --- /dev/null +++ b/apps/i2psnark/java/src/org/klomp/snark/dht/NodeInfo.java @@ -0,0 +1,181 @@ +package org.klomp.snark.dht; +/* + * From zzzot, modded and relicensed to GPLv2 + */ + +import net.i2p.data.DataHelper; +import net.i2p.data.Destination; +import net.i2p.data.Hash; +import net.i2p.data.SimpleDataStructure; + +/* + * A Node ID, Hash, and port, and an optional Destination. + * This is what DHTNodes remembers. The DHT tracker just stores Hashes. + * getData() returns the 54 byte compact info (NID, Hash, port). + * + * Things are a little tricky in KRPC since we exchange Hashes and don't + * always have the Destination. + * + * @since 0.8.4 + * @author zzz + */ + +public class NodeInfo extends SimpleDataStructure { + + private long lastSeen; + private NID nID; + private Hash hash; + private Destination dest; + private int port; + + public static final int LENGTH = NID.HASH_LENGTH + Hash.HASH_LENGTH + 2; + + /** + * Use this if we have the full destination + * @throws IllegalArgumentException + */ + public NodeInfo(NID nID, Destination dest, int port) { + super(); + this.nID = nID; + this.dest = dest; + this.hash = dest.calculateHash(); + this.port = port; + initialize(nID, this.hash, port); + } + + /** + * No Destination yet available + * @deprecated unused + * @throws IllegalArgumentException + */ + public NodeInfo(NID nID, Hash hash, int port) { + super(); + this.nID = nID; + this.hash = hash; + this.port = port; + initialize(nID, hash, port); + } + + /** + * No Destination yet available + * @param compactInfo 20 byte node ID, 32 byte destHash, 2 byte port + * @throws IllegalArgumentException + */ + public NodeInfo(byte[] compactInfo) { + super(compactInfo); + initialize(compactInfo); + } + + /** + * No Destination yet available + * @param compactInfo 20 byte node ID, 32 byte destHash, 2 byte port + * @param offset starting at this offset in compactInfo + * @throws IllegalArgumentException + * @throws AIOOBE + */ + public NodeInfo(byte[] compactInfo, int offset) { + super(); + byte[] d = new byte[LENGTH]; + System.arraycopy(compactInfo, offset, d, 0, LENGTH); + setData(d); + initialize(d); + } + + /** + * Creates data structures from the compact info + * @throws IllegalArgumentException + */ + private void initialize(byte[] compactInfo) { + if (compactInfo.length != LENGTH) + throw new IllegalArgumentException("Bad compact info length"); + byte[] ndata = new byte[NID.HASH_LENGTH]; + System.arraycopy(compactInfo, 0, ndata, 0, NID.HASH_LENGTH); + this.nID = new NID(ndata); + byte[] hdata = new byte[Hash.HASH_LENGTH]; + System.arraycopy(compactInfo, NID.HASH_LENGTH, hdata, 0, Hash.HASH_LENGTH); + this.hash = new Hash(hdata); + this.port = (int) DataHelper.fromLong(compactInfo, NID.HASH_LENGTH + Hash.HASH_LENGTH, 2); + } + + /** + * Creates 54-byte compact info + * @throws IllegalArgumentException + */ + private void initialize(NID nID, Hash hash, int port) { + if (port < 0 || port > 65535) + throw new IllegalArgumentException("Bad port"); + byte[] compactInfo = new byte[LENGTH]; + System.arraycopy(nID.getData(), 0, compactInfo, 0, NID.HASH_LENGTH); + System.arraycopy(hash.getData(), 0, compactInfo, NID.HASH_LENGTH, Hash.HASH_LENGTH); + DataHelper.toLong(compactInfo, NID.HASH_LENGTH + Hash.HASH_LENGTH, 2, port); + setData(compactInfo); + } + + public int length() { + return LENGTH; + } + + public NID getNID() { + return this.nID; + } + + /** @return may be null if we don't have it */ + public Destination getDestination() { + return this.dest; + } + + public Hash getHash() { + return this.hash; + } + + @Override + public Hash calculateHash() { + return this.hash; + } + + /** + * This can come in later but the hash must match. + * @throws IllegalArgumentException if hash of dest doesn't match previous hash + */ + public void setDestination(Destination dest) throws IllegalArgumentException { + if (!dest.calculateHash().equals(this.hash)) + throw new IllegalArgumentException("Hash mismatch, was: " + this.hash + " new: " + dest.calculateHash()); + if (this.dest == null) + this.dest = dest; + else if (!this.dest.equals(dest)) + throw new IllegalArgumentException("Dest mismatch, was: " + this.dest+ " new: " + dest); + // else keep the old to reduce object churn + } + + public int getPort() { + return this.port; + } + + public long lastSeen() { + return lastSeen; + } + + public void setLastSeen(long now) { + lastSeen = now; + } + + @Override + public int hashCode() { + return super.hashCode() ^ nID.hashCode() ^ port; + } + + @Override + public boolean equals(Object o) { + try { + NodeInfo ni = (NodeInfo) o; + // assume dest matches, ignore it + return this.hash.equals(ni.hash) && nID.equals(ni.nID) && port == ni.port; + } catch (Exception e) { + return false; + } + } + + public String toString() { + return "NodeInfo: " + nID + ' ' + hash + " port: " + port; + } +} diff --git a/apps/i2psnark/java/src/org/klomp/snark/dht/NodeInfoComparator.java b/apps/i2psnark/java/src/org/klomp/snark/dht/NodeInfoComparator.java new file mode 100644 index 0000000000..66eb57bbab --- /dev/null +++ b/apps/i2psnark/java/src/org/klomp/snark/dht/NodeInfoComparator.java @@ -0,0 +1,31 @@ +package org.klomp.snark.dht; +/* + * From zzzot, modded and relicensed to GPLv2 + */ + +import java.util.Comparator; + +import net.i2p.crypto.SHA1Hash; +import net.i2p.data.DataHelper; + +/** + * Closest to a InfoHash or NID key. + * Use for NodeInfos. + * + * @since 0.8.4 + * @author zzz + */ +class NodeInfoComparator implements Comparator { + private final SHA1Hash _base; + + public NodeInfoComparator(SHA1Hash h) { + _base = h; + } + + public int compare(NodeInfo lhs, NodeInfo rhs) { + byte lhsDelta[] = DataHelper.xor(lhs.getNID().getData(), _base.getData()); + byte rhsDelta[] = DataHelper.xor(rhs.getNID().getData(), _base.getData()); + return DataHelper.compareTo(lhsDelta, rhsDelta); + } + +} diff --git a/apps/i2psnark/java/src/org/klomp/snark/dht/Peer.java b/apps/i2psnark/java/src/org/klomp/snark/dht/Peer.java new file mode 100644 index 0000000000..8943c06b5f --- /dev/null +++ b/apps/i2psnark/java/src/org/klomp/snark/dht/Peer.java @@ -0,0 +1,30 @@ +package org.klomp.snark.dht; +/* + * From zzzot, modded and relicensed to GPLv2 + */ + +import net.i2p.data.Hash; + +/** + * A single peer for a single torrent. + * This is what the DHT tracker remembers. + * + * @since 0.8.4 + * @author zzz + */ +public class Peer extends Hash { + + private long lastSeen; + + public Peer(byte[] data) { + super(data); + } + + public long lastSeen() { + return lastSeen; + } + + public void setLastSeen(long now) { + lastSeen = now; + } +} diff --git a/apps/i2psnark/java/src/org/klomp/snark/dht/Peers.java b/apps/i2psnark/java/src/org/klomp/snark/dht/Peers.java new file mode 100644 index 0000000000..2a2452e956 --- /dev/null +++ b/apps/i2psnark/java/src/org/klomp/snark/dht/Peers.java @@ -0,0 +1,21 @@ +package org.klomp.snark.dht; +/* + * From zzzot, modded and relicensed to GPLv2 + */ + +import java.util.concurrent.ConcurrentHashMap; + +import net.i2p.data.Hash; + +/** + * All the peers for a single torrent + * + * @since 0.8.4 + * @author zzz + */ +public class Peers extends ConcurrentHashMap { + + public Peers() { + super(); + } +} diff --git a/apps/i2psnark/java/src/org/klomp/snark/dht/SHA1Comparator.java b/apps/i2psnark/java/src/org/klomp/snark/dht/SHA1Comparator.java new file mode 100644 index 0000000000..36c355e491 --- /dev/null +++ b/apps/i2psnark/java/src/org/klomp/snark/dht/SHA1Comparator.java @@ -0,0 +1,31 @@ +package org.klomp.snark.dht; +/* + * From zzzot, modded and relicensed to GPLv2 + */ + +import java.util.Comparator; + +import net.i2p.crypto.SHA1Hash; +import net.i2p.data.DataHelper; + +/** + * Closest to a InfoHash or NID key. + * Use for InfoHashes and NIDs. + * + * @since 0.8.4 + * @author zzz + */ +class SHA1Comparator implements Comparator { + private final byte[] _base; + + public SHA1Comparator(SHA1Hash h) { + _base = h.getData(); + } + + public int compare(SHA1Hash lhs, SHA1Hash rhs) { + byte lhsDelta[] = DataHelper.xor(lhs.getData(), _base); + byte rhsDelta[] = DataHelper.xor(rhs.getData(), _base); + return DataHelper.compareTo(lhsDelta, rhsDelta); + } + +} diff --git a/apps/i2psnark/java/src/org/klomp/snark/dht/Token.java b/apps/i2psnark/java/src/org/klomp/snark/dht/Token.java new file mode 100644 index 0000000000..c8859155df --- /dev/null +++ b/apps/i2psnark/java/src/org/klomp/snark/dht/Token.java @@ -0,0 +1,39 @@ +package org.klomp.snark.dht; +/* + * GPLv2 + */ + +import net.i2p.I2PAppContext; +import net.i2p.data.ByteArray; +import net.i2p.data.DataHelper; + +/** + * Used for Both outgoing and incoming tokens + * + * @since 0.8.4 + * @author zzz + */ +public class Token extends ByteArray { + + private static final int MY_TOK_LEN = 8; + private final long lastSeen; + + /** outgoing - generate a random token */ + public Token(I2PAppContext ctx) { + super(null); + byte[] data = new byte[MY_TOK_LEN]; + ctx.random().nextBytes(data); + setData(data); + lastSeen = ctx.clock().now(); + } + + /** incoming - save the token (arbitrary length) */ + public Token(I2PAppContext ctx, byte[] data) { + super(data); + lastSeen = ctx.clock().now(); + } + + public long lastSeen() { + return lastSeen; + } +} diff --git a/apps/i2psnark/java/src/org/klomp/snark/dht/TokenKey.java b/apps/i2psnark/java/src/org/klomp/snark/dht/TokenKey.java new file mode 100644 index 0000000000..d2217a015e --- /dev/null +++ b/apps/i2psnark/java/src/org/klomp/snark/dht/TokenKey.java @@ -0,0 +1,20 @@ +package org.klomp.snark.dht; +/* + * GPLv2 + */ + +import net.i2p.crypto.SHA1Hash; +import net.i2p.data.DataHelper; + +/** + * Used to index incoming Tokens + * + * @since 0.8.4 + * @author zzz + */ +public class TokenKey extends SHA1Hash { + + public TokenKey(NID nID, InfoHash ih) { + super(DataHelper.xor(nID.getData(), ih.getData())); + } +} diff --git a/apps/i2psnark/java/src/org/klomp/snark/dht/Torrents.java b/apps/i2psnark/java/src/org/klomp/snark/dht/Torrents.java new file mode 100644 index 0000000000..c791e70776 --- /dev/null +++ b/apps/i2psnark/java/src/org/klomp/snark/dht/Torrents.java @@ -0,0 +1,19 @@ +package org.klomp.snark.dht; +/* + * From zzzot, relicensed to GPLv2 + */ + +import java.util.concurrent.ConcurrentHashMap; + +/** + * All the torrents + * + * @since 0.8.4 + * @author zzz + */ +public class Torrents extends ConcurrentHashMap { + + public Torrents() { + super(); + } +} From 7b07eb89a36d9ae8aaa9bc84865b14662607d617 Mon Sep 17 00:00:00 2001 From: zzz Date: Sat, 2 Jun 2012 18:52:46 +0000 Subject: [PATCH 2/9] - Uncomment DHT - Change DHT from option bit to extension message - Add DHT start/stop code - Add UI for DHT enabling - Add raw datagram protocol type and use for response port --- .../src/org/klomp/snark/ExtensionHandler.java | 57 ++++++++++++++++++- .../src/org/klomp/snark/I2PSnarkUtil.java | 32 +++++++++-- .../java/src/org/klomp/snark/Peer.java | 18 +++--- .../src/org/klomp/snark/PeerCoordinator.java | 27 ++++++++- .../src/org/klomp/snark/PeerListener.java | 7 ++- .../java/src/org/klomp/snark/PeerState.java | 8 ++- .../src/org/klomp/snark/SnarkManager.java | 15 ++++- .../java/src/org/klomp/snark/dht/DHT.java | 12 +++- .../java/src/org/klomp/snark/dht/KRPC.java | 24 ++++++-- .../org/klomp/snark/web/I2PSnarkServlet.java | 12 +++- core/java/src/net/i2p/client/I2PSession.java | 11 ++++ 11 files changed, 195 insertions(+), 28 deletions(-) diff --git a/apps/i2psnark/java/src/org/klomp/snark/ExtensionHandler.java b/apps/i2psnark/java/src/org/klomp/snark/ExtensionHandler.java index e67466f4a5..abde95e801 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/ExtensionHandler.java +++ b/apps/i2psnark/java/src/org/klomp/snark/ExtensionHandler.java @@ -28,6 +28,9 @@ abstract class ExtensionHandler { public static final int ID_PEX = 2; /** not ut_pex since the compact format is different */ public static final String TYPE_PEX = "i2p_pex"; + public static final int ID_DHT = 3; + /** not using the option bit since the compact format is different */ + public static final String TYPE_DHT = "i2p_dht"; /** Pieces * SHA1 Hash length, + 25% extra for file names, benconding overhead, etc */ private static final int MAX_METADATA_SIZE = Storage.MAX_PIECES * 20 * 5 / 4; private static final int PARALLEL_REQUESTS = 3; @@ -36,9 +39,10 @@ abstract class ExtensionHandler { /** * @param metasize -1 if unknown * @param pexAndMetadata advertise these capabilities + * @param dht advertise DHT capability * @return bencoded outgoing handshake message */ - public static byte[] getHandshake(int metasize, boolean pexAndMetadata) { + public static byte[] getHandshake(int metasize, boolean pexAndMetadata, boolean dht) { Map handshake = new HashMap(); Map m = new HashMap(); if (pexAndMetadata) { @@ -47,6 +51,9 @@ abstract class ExtensionHandler { if (metasize >= 0) handshake.put("metadata_size", Integer.valueOf(metasize)); } + if (dht) { + m.put(TYPE_DHT, Integer.valueOf(ID_DHT)); + } // include the map even if empty so the far-end doesn't NPE handshake.put("m", m); handshake.put("p", Integer.valueOf(6881)); @@ -65,6 +72,8 @@ abstract class ExtensionHandler { handleMetadata(peer, listener, bs, log); else if (id == ID_PEX) handlePEX(peer, listener, bs, log); + else if (id == ID_DHT) + handleDHT(peer, listener, bs, log); else if (log.shouldLog(Log.INFO)) log.info("Unknown extension msg " + id + " from " + peer); } @@ -87,6 +96,12 @@ abstract class ExtensionHandler { // peer state calls peer listener calls sendPEX() } + if (msgmap.get(TYPE_DHT) != null) { + if (log.shouldLog(Log.DEBUG)) + log.debug("Peer supports DHT extension: " + peer); + // peer state calls peer listener calls sendDHT() + } + MagnetState state = peer.getMagnetState(); if (msgmap.get(TYPE_METADATA) == null) { @@ -332,6 +347,28 @@ abstract class ExtensionHandler { } } + /** + * Receive the DHT port numbers + * @since DHT + */ + private static void handleDHT(Peer peer, PeerListener listener, byte[] bs, Log log) { + if (log.shouldLog(Log.DEBUG)) + log.debug("Got DHT msg from " + peer); + try { + InputStream is = new ByteArrayInputStream(bs); + BDecoder dec = new BDecoder(is); + BEValue bev = dec.bdecodeMap(); + Map map = bev.getMap(); + int qport = map.get("port").getInt(); + int rport = map.get("rport").getInt(); + listener.gotPort(peer, qport, rport); + } catch (Exception e) { + if (log.shouldLog(Log.INFO)) + log.info("DHT msg exception from " + peer, e); + //peer.disconnect(false); + } + } + /** * added.f and dropped unsupported * @param pList non-null @@ -359,4 +396,22 @@ abstract class ExtensionHandler { } } + /** + * Send the DHT port numbers + * @since DHT + */ + public static void sendDHT(Peer peer, int qport, int rport) { + Map map = new HashMap(); + map.put("port", Integer.valueOf(qport)); + map.put("rport", Integer.valueOf(rport)); + byte[] payload = BEncoder.bencode(map); + try { + int hisMsgCode = peer.getHandshakeMap().get("m").getMap().get(TYPE_DHT).getInt(); + peer.sendExtension(hisMsgCode, payload); + } catch (Exception e) { + // NPE, no DHT caps + //if (log.shouldLog(Log.INFO)) + // log.info("DHT msg exception to " + peer, e); + } + } } diff --git a/apps/i2psnark/java/src/org/klomp/snark/I2PSnarkUtil.java b/apps/i2psnark/java/src/org/klomp/snark/I2PSnarkUtil.java index 320b5005a2..dca9fbaf2b 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/I2PSnarkUtil.java +++ b/apps/i2psnark/java/src/org/klomp/snark/I2PSnarkUtil.java @@ -36,7 +36,7 @@ import net.i2p.util.SimpleTimer; import net.i2p.util.Translate; import org.klomp.snark.dht.DHT; -//import org.klomp.snark.dht.KRPC; +import org.klomp.snark.dht.KRPC; /** * I2P specific helpers for I2PSnark @@ -63,6 +63,7 @@ public class I2PSnarkUtil { private final File _tmpDir; private int _startupDelay; private boolean _shouldUseOT; + private boolean _shouldUseDHT; private boolean _areFilesPublic; private String _openTrackerString; private DHT _dht; @@ -73,7 +74,7 @@ public class I2PSnarkUtil { public static final int DEFAULT_MAX_UP_BW = 8; //KBps public static final int MAX_CONNECTIONS = 16; // per torrent public static final String PROP_MAX_BW = "i2cp.outboundBytesPerSecond"; - //private static final boolean ENABLE_DHT = true; + public static final boolean DEFAULT_USE_DHT = true; public I2PSnarkUtil(I2PAppContext ctx) { _context = ctx; @@ -88,6 +89,7 @@ public class I2PSnarkUtil { _maxConnections = MAX_CONNECTIONS; _startupDelay = DEFAULT_STARTUP_DELAY; _shouldUseOT = DEFAULT_USE_OPENTRACKERS; + _shouldUseDHT = DEFAULT_USE_DHT; // This is used for both announce replies and .torrent file downloads, // so it must be available even if not connected to I2CP. // so much for multiple instances @@ -234,8 +236,8 @@ public class I2PSnarkUtil { _manager = I2PSocketManagerFactory.createManager(_i2cpHost, _i2cpPort, opts); } // FIXME this only instantiates krpc once, left stuck with old manager - //if (ENABLE_DHT && _manager != null && _dht == null) - // _dht = new KRPC(_context, _manager.getSession()); + if (_shouldUseDHT && _manager != null && _dht == null) + _dht = new KRPC(_context, _manager.getSession()); return (_manager != null); } @@ -250,7 +252,11 @@ public class I2PSnarkUtil { /** * Destroy the destination itself */ - public void disconnect() { + public synchronized void disconnect() { + if (_dht != null) { + _dht.stop(); + _dht = null; + } I2PSocketManager mgr = _manager; // FIXME this can cause race NPEs elsewhere _manager = null; @@ -490,6 +496,22 @@ public class I2PSnarkUtil { public boolean shouldUseOpenTrackers() { return _shouldUseOT; } + + /** @since DHT */ + public synchronized void setUseDHT(boolean yes) { + _shouldUseDHT = yes; + if (yes && _manager != null && _dht == null) { + _dht = new KRPC(_context, _manager.getSession()); + } else if (!yes && _dht != null) { + _dht.stop(); + _dht = null; + } + } + + /** @since DHT */ + public boolean shouldUseDHT() { + return _shouldUseDHT; + } /** * Like DataHelper.toHexString but ensures no loss of leading zero bytes diff --git a/apps/i2psnark/java/src/org/klomp/snark/Peer.java b/apps/i2psnark/java/src/org/klomp/snark/Peer.java index 02fb635605..8f33c01a72 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/Peer.java +++ b/apps/i2psnark/java/src/org/klomp/snark/Peer.java @@ -80,7 +80,9 @@ public class Peer implements Comparable static final long OPTION_FAST = 0x0000000000000004l; static final long OPTION_DHT = 0x0000000000000001l; /** we use a different bit since the compact format is different */ +/* no, let's use an extension message static final long OPTION_I2P_DHT = 0x0000000040000000l; +*/ static final long OPTION_AZMP = 0x1000000000000000l; private long options; @@ -269,15 +271,17 @@ public class Peer implements Comparable _log.debug("Peer supports extensions, sending reply message"); int metasize = metainfo != null ? metainfo.getInfoBytes().length : -1; boolean pexAndMetadata = metainfo == null || !metainfo.isPrivate(); - out.sendExtension(0, ExtensionHandler.getHandshake(metasize, pexAndMetadata)); + boolean dht = util.getDHT() != null; + out.sendExtension(0, ExtensionHandler.getHandshake(metasize, pexAndMetadata, dht)); } - if ((options & OPTION_I2P_DHT) != 0 && util.getDHT() != null) { - if (_log.shouldLog(Log.DEBUG)) - _log.debug("Peer supports DHT, sending PORT message"); - int port = util.getDHT().getPort(); - out.sendPort(port); - } + // Old DHT PORT message + //if ((options & OPTION_I2P_DHT) != 0 && util.getDHT() != null) { + // if (_log.shouldLog(Log.DEBUG)) + // _log.debug("Peer supports DHT, sending PORT message"); + // int port = util.getDHT().getPort(); + // out.sendPort(port); + //} // Send our bitmap if (bitfield != null) diff --git a/apps/i2psnark/java/src/org/klomp/snark/PeerCoordinator.java b/apps/i2psnark/java/src/org/klomp/snark/PeerCoordinator.java index 45b6ef82ac..845b989046 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/PeerCoordinator.java +++ b/apps/i2psnark/java/src/org/klomp/snark/PeerCoordinator.java @@ -1273,6 +1273,7 @@ class PeerCoordinator implements PeerListener } } else if (id == ExtensionHandler.ID_HANDSHAKE) { sendPeers(peer); + sendDHT(peer); } } @@ -1301,6 +1302,26 @@ class PeerCoordinator implements PeerListener } catch (InvalidBEncodingException ibee) {} } + /** + * Send a DHT message to the peer, if we both support DHT. + * @since DHT + */ + void sendDHT(Peer peer) { + DHT dht = _util.getDHT(); + if (dht == null) + return; + Map handshake = peer.getHandshakeMap(); + if (handshake == null) + return; + BEValue bev = handshake.get("m"); + if (bev == null) + return; + try { + if (bev.getMap().get(ExtensionHandler.TYPE_DHT) != null) + ExtensionHandler.sendDHT(peer, dht.getPort(), dht.getRPort()); + } catch (InvalidBEncodingException ibee) {} + } + /** * Sets the storage after transition out of magnet mode * Snark calls this after we call gotMetaInfo() @@ -1318,11 +1339,13 @@ class PeerCoordinator implements PeerListener /** * PeerListener callback * Tell the DHT to ping it, this will get back the node info + * @param rport must be port + 1 * @since 0.8.4 */ - public void gotPort(Peer peer, int port) { + public void gotPort(Peer peer, int port, int rport) { DHT dht = _util.getDHT(); - if (dht != null) + if (dht != null && + port > 0 && port < 65535 && rport == port + 1) dht.ping(peer.getDestination(), port); } diff --git a/apps/i2psnark/java/src/org/klomp/snark/PeerListener.java b/apps/i2psnark/java/src/org/klomp/snark/PeerListener.java index f573d44552..ee2de562d2 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/PeerListener.java +++ b/apps/i2psnark/java/src/org/klomp/snark/PeerListener.java @@ -190,13 +190,14 @@ interface PeerListener void gotExtension(Peer peer, int id, byte[] bs); /** - * Called when a port message is received. + * Called when a DHT port message is received. * * @param peer the Peer that got the message. - * @param port the port + * @param port the query port + * @param port the response port * @since 0.8.4 */ - void gotPort(Peer peer, int port); + void gotPort(Peer peer, int port, int qport); /** * Called when peers are received via PEX diff --git a/apps/i2psnark/java/src/org/klomp/snark/PeerState.java b/apps/i2psnark/java/src/org/klomp/snark/PeerState.java index f41bf1b575..2ed9c373df 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/PeerState.java +++ b/apps/i2psnark/java/src/org/klomp/snark/PeerState.java @@ -526,10 +526,14 @@ class PeerState implements DataLoader setInteresting(true); } - /** @since 0.8.4 */ + /** + * Unused + * @since 0.8.4 + */ void portMessage(int port) { - listener.gotPort(peer, port); + // for compatibility with old DHT PORT message + listener.gotPort(peer, port, port + 1); } void unknownMessage(int type, byte[] bs) diff --git a/apps/i2psnark/java/src/org/klomp/snark/SnarkManager.java b/apps/i2psnark/java/src/org/klomp/snark/SnarkManager.java index 7c1c421a8c..56c6ce431a 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/SnarkManager.java +++ b/apps/i2psnark/java/src/org/klomp/snark/SnarkManager.java @@ -85,6 +85,7 @@ public class SnarkManager implements Snark.CompleteListener { public static final String DEFAULT_THEME = "ubergine"; private static final String PROP_USE_OPENTRACKERS = "i2psnark.useOpentrackers"; public static final String PROP_OPENTRACKERS = "i2psnark.opentrackers"; + private static final String PROP_USE_DHT = "i2psnark.enableDHT"; public static final int MIN_UP_BW = 2; public static final int DEFAULT_MAX_UP_BW = 10; @@ -273,6 +274,8 @@ public class SnarkManager implements Snark.CompleteListener { _config.setProperty(PROP_STARTUP_DELAY, Integer.toString(DEFAULT_STARTUP_DELAY)); if (!_config.containsKey(PROP_THEME)) _config.setProperty(PROP_THEME, DEFAULT_THEME); + if (!_config.containsKey(PROP_USE_DHT)) + _config.setProperty(PROP_USE_DHT, Boolean.toString(I2PSnarkUtil.DEFAULT_USE_DHT)); updateConfig(); } /** @@ -347,6 +350,7 @@ public class SnarkManager implements Snark.CompleteListener { String useOT = _config.getProperty(PROP_USE_OPENTRACKERS); boolean bOT = useOT == null || Boolean.valueOf(useOT).booleanValue(); _util.setUseOpenTrackers(bOT); + _util.setUseDHT(Boolean.valueOf(PROP_USE_DHT).booleanValue()); getDataDir().mkdirs(); initTrackerMap(); } @@ -365,7 +369,7 @@ public class SnarkManager implements Snark.CompleteListener { public void updateConfig(String dataDir, boolean filesPublic, boolean autoStart, String refreshDelay, String startDelay, String seedPct, String eepHost, String eepPort, String i2cpHost, String i2cpPort, String i2cpOpts, - String upLimit, String upBW, boolean useOpenTrackers, String theme) { + String upLimit, String upBW, boolean useOpenTrackers, boolean useDHT, String theme) { boolean changed = false; //if (eepHost != null) { // // unused, we use socket eepget @@ -549,6 +553,15 @@ public class SnarkManager implements Snark.CompleteListener { _util.setUseOpenTrackers(useOpenTrackers); changed = true; } + if (_util.shouldUseDHT() != useDHT) { + _config.setProperty(PROP_USE_DHT, Boolean.toString(useDHT)); + if (useDHT) + addMessage(_("Enabled DHT.")); + else + addMessage(_("Disabled DHT.")); + _util.setUseDHT(useDHT); + changed = true; + } if (theme != null) { if(!theme.equals(_config.getProperty(PROP_THEME))) { _config.setProperty(PROP_THEME, theme); diff --git a/apps/i2psnark/java/src/org/klomp/snark/dht/DHT.java b/apps/i2psnark/java/src/org/klomp/snark/dht/DHT.java index 6a16e4e605..2e401f060d 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/dht/DHT.java +++ b/apps/i2psnark/java/src/org/klomp/snark/dht/DHT.java @@ -17,10 +17,15 @@ public interface DHT { /** - * @return The UDP port that should be included in a PORT message. + * @return The UDP query port */ public int getPort(); + /** + * @return The UDP response port + */ + public int getRPort(); + /** * Ping. We don't have a NID yet so the node is presumed * to be absent from our DHT. @@ -79,4 +84,9 @@ public interface DHT { * @return the number of successful announces, not counting ourselves. */ public int announce(byte[] ih, int max, long maxWait); + + /** + * Stop everything. + */ + public void stop(); } diff --git a/apps/i2psnark/java/src/org/klomp/snark/dht/KRPC.java b/apps/i2psnark/java/src/org/klomp/snark/dht/KRPC.java index ef6757a831..fc81f5479b 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/dht/KRPC.java +++ b/apps/i2psnark/java/src/org/klomp/snark/dht/KRPC.java @@ -151,7 +151,7 @@ public class KRPC implements I2PSessionMuxedListener, DHT { NID myNID = new NID(myID); _myNodeInfo = new NodeInfo(myNID, session.getMyDestination(), _qPort); - session.addMuxedSessionListener(this, I2PSession.PROTO_DATAGRAM, _rPort); + session.addMuxedSessionListener(this, I2PSession.PROTO_DATAGRAM_RAW, _rPort); session.addMuxedSessionListener(this, I2PSession.PROTO_DATAGRAM, _qPort); // can't be stopped SimpleScheduler.getInstance().addPeriodicEvent(new Cleaner(), CLEAN_TIME); @@ -183,12 +183,19 @@ public class KRPC implements I2PSessionMuxedListener, DHT { } /** - * @return The UDP port that should be included in a PORT message. + * @return The UDP query port */ public int getPort() { return _qPort; } + /** + * @return The UDP response port + */ + public int getRPort() { + return _rPort; + } + /** * Ping. We don't have a NID yet so the node is presumed * to be absent from our DHT. @@ -481,12 +488,19 @@ public class KRPC implements I2PSessionMuxedListener, DHT { } /** - * Does nothing yet. + * Stop everything. */ public void stop() { - // stop the explore thread + // FIXME stop the explore thread // unregister port listeners - // does not clear the DHT or tracker yet. + _session.removeListener(I2PSession.PROTO_DATAGRAM, _qPort); + _session.removeListener(I2PSession.PROTO_DATAGRAM_RAW, _rPort); + // clear the DHT and tracker + _tracker.stop(); + _knownNodes.clear(); + _sentQueries.clear(); + _outgoingTokens.clear(); + _incomingTokens.clear(); } /** diff --git a/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java b/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java index dd9b14ba3d..515ee18bf2 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java +++ b/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java @@ -693,11 +693,12 @@ public class I2PSnarkServlet extends DefaultServlet { String refreshDel = req.getParameter("refreshDelay"); String startupDel = req.getParameter("startupDelay"); boolean useOpenTrackers = req.getParameter("useOpenTrackers") != null; + boolean useDHT = req.getParameter("useDHT") != null; //String openTrackers = req.getParameter("openTrackers"); String theme = req.getParameter("theme"); _manager.updateConfig(dataDir, filesPublic, autoStart, refreshDel, startupDel, seedPct, eepHost, eepPort, i2cpHost, i2cpPort, i2cpOpts, - upLimit, upBW, useOpenTrackers, theme); + upLimit, upBW, useOpenTrackers, useDHT, theme); } else if ("Save2".equals(action)) { String taction = req.getParameter("taction"); if (taction != null) @@ -1438,6 +1439,7 @@ public class I2PSnarkServlet extends DefaultServlet { boolean autoStart = _manager.shouldAutoStart(); boolean useOpenTrackers = _manager.util().shouldUseOpenTrackers(); //String openTrackers = _manager.util().getOpenTrackerString(); + boolean useDHT = _manager.util().shouldUseDHT(); //int seedPct = 0; out.write("
\n" + @@ -1551,6 +1553,14 @@ public class I2PSnarkServlet extends DefaultServlet { + (useOpenTrackers ? "checked " : "") + "title=\""); out.write(_("If checked, announce torrents to open trackers as well as the tracker listed in the torrent file")); + out.write("\" >\n" + + + ""); + out.write(_("Enable DHT")); + out.write(": \n"); // ""); diff --git a/core/java/src/net/i2p/client/I2PSession.java b/core/java/src/net/i2p/client/I2PSession.java index 06b094488c..d91c3f20d3 100644 --- a/core/java/src/net/i2p/client/I2PSession.java +++ b/core/java/src/net/i2p/client/I2PSession.java @@ -252,5 +252,16 @@ public interface I2PSession { public static final int PROTO_ANY = 0; public static final int PROTO_UNSPECIFIED = 0; public static final int PROTO_STREAMING = 6; + + /** + * Generally a signed datagram, but could + * also be a raw datagram, depending on the application + */ public static final int PROTO_DATAGRAM = 17; + + /** + * A raw (unsigned) datagram + * @since 0.9.1 + */ + public static final int PROTO_DATAGRAM_RAW = 18; } From 558bb2f4f36ec9a9ec5d95f603dc61ead3a8664f Mon Sep 17 00:00:00 2001 From: zzz Date: Sat, 2 Jun 2012 18:56:10 +0000 Subject: [PATCH 3/9] select proto on UDP send --- apps/i2psnark/java/src/org/klomp/snark/dht/KRPC.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/i2psnark/java/src/org/klomp/snark/dht/KRPC.java b/apps/i2psnark/java/src/org/klomp/snark/dht/KRPC.java index fc81f5479b..49874f4595 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/dht/KRPC.java +++ b/apps/i2psnark/java/src/org/klomp/snark/dht/KRPC.java @@ -769,7 +769,8 @@ public class KRPC implements I2PSessionMuxedListener, DHT { try { boolean success = _session.sendMessage(dest, payload, 0, payload.length, null, null, 60*1000, - I2PSession.PROTO_DATAGRAM, fromPort, toPort); + repliable ? I2PSession.PROTO_DATAGRAM : I2PSession.PROTO_DATAGRAM_RAW, + fromPort, toPort); if (!success) { if (_log.shouldLog(Log.WARN)) _log.warn("WTF sendMessage fail"); From f8c185d09fe52e14d35a65b3b990d4874c001a71 Mon Sep 17 00:00:00 2001 From: zzz Date: Sat, 2 Jun 2012 21:44:23 +0000 Subject: [PATCH 4/9] prep for merging --- apps/i2psnark/java/src/org/klomp/snark/I2PSnarkUtil.java | 2 +- apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/i2psnark/java/src/org/klomp/snark/I2PSnarkUtil.java b/apps/i2psnark/java/src/org/klomp/snark/I2PSnarkUtil.java index dca9fbaf2b..425d51aeb6 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/I2PSnarkUtil.java +++ b/apps/i2psnark/java/src/org/klomp/snark/I2PSnarkUtil.java @@ -74,7 +74,7 @@ public class I2PSnarkUtil { public static final int DEFAULT_MAX_UP_BW = 8; //KBps public static final int MAX_CONNECTIONS = 16; // per torrent public static final String PROP_MAX_BW = "i2cp.outboundBytesPerSecond"; - public static final boolean DEFAULT_USE_DHT = true; + public static final boolean DEFAULT_USE_DHT = false; public I2PSnarkUtil(I2PAppContext ctx) { _context = ctx; diff --git a/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java b/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java index 515ee18bf2..7f47fe3e22 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java +++ b/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java @@ -1556,7 +1556,7 @@ public class I2PSnarkServlet extends DefaultServlet { out.write("\" >\n" + ""); - out.write(_("Enable DHT")); + out.write(_("Enable DHT") + " (**BETA**)"); out.write(": Date: Sun, 3 Jun 2012 15:25:51 +0000 Subject: [PATCH 5/9] - Fix node ID / node info confusion - Fix updating node ID when receiving pong - Fix getting DHT enable setting from config file - Fix handling of get_peers replies - Fix sending and receiving announces without signing - Fix incoming/outgoing token handling - Set cleanup timer for all queries - More debug logging --- .../src/org/klomp/snark/SnarkManager.java | 2 +- .../java/src/org/klomp/snark/dht/KRPC.java | 210 ++++++++++++------ .../src/org/klomp/snark/dht/NodeInfo.java | 10 +- .../java/src/org/klomp/snark/dht/Token.java | 32 +++ 4 files changed, 178 insertions(+), 76 deletions(-) diff --git a/apps/i2psnark/java/src/org/klomp/snark/SnarkManager.java b/apps/i2psnark/java/src/org/klomp/snark/SnarkManager.java index 56c6ce431a..9bfb885cf3 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/SnarkManager.java +++ b/apps/i2psnark/java/src/org/klomp/snark/SnarkManager.java @@ -350,7 +350,7 @@ public class SnarkManager implements Snark.CompleteListener { String useOT = _config.getProperty(PROP_USE_OPENTRACKERS); boolean bOT = useOT == null || Boolean.valueOf(useOT).booleanValue(); _util.setUseOpenTrackers(bOT); - _util.setUseDHT(Boolean.valueOf(PROP_USE_DHT).booleanValue()); + _util.setUseDHT(Boolean.valueOf(_config.getProperty(PROP_USE_DHT)).booleanValue()); getDataDir().mkdirs(); initTrackerMap(); } diff --git a/apps/i2psnark/java/src/org/klomp/snark/dht/KRPC.java b/apps/i2psnark/java/src/org/klomp/snark/dht/KRPC.java index 49874f4595..e0608df4e4 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/dht/KRPC.java +++ b/apps/i2psnark/java/src/org/klomp/snark/dht/KRPC.java @@ -91,13 +91,17 @@ public class KRPC implements I2PSessionMuxedListener, DHT { private final DHTNodes _knownNodes; /** index to sent queries awaiting reply */ private final ConcurrentHashMap _sentQueries; - /** index to outgoing tokens, sent in reply to a get_peers query */ - private final ConcurrentHashMap _outgoingTokens; - /** index to incoming tokens, received in a peers or nodes reply */ - private final ConcurrentHashMap _incomingTokens; + /** index to outgoing tokens we generated, sent in reply to a get_peers query */ + private final ConcurrentHashMap _outgoingTokens; + /** index to incoming opaque tokens, received in a peers or nodes reply */ + private final ConcurrentHashMap _incomingTokens; /** hook to inject and receive datagrams */ private final I2PSession _session; + /** 20 byte random id */ + private final byte[] _myID; + /** 20 byte random id */ + private final NID _myNID; /** 20 byte random id + 32 byte Hash + 2 byte port */ private final NodeInfo _myNodeInfo; /** unsigned dgrams */ @@ -146,10 +150,10 @@ public class KRPC implements I2PSessionMuxedListener, DHT { // ports can really be fixed, just do this for testing _qPort = 30000 + ctx.random().nextInt(99); _rPort = _qPort + 1; - byte[] myID = new byte[NID.HASH_LENGTH]; - ctx.random().nextBytes(myID); - NID myNID = new NID(myID); - _myNodeInfo = new NodeInfo(myNID, session.getMyDestination(), _qPort); + _myID = new byte[NID.HASH_LENGTH]; + ctx.random().nextBytes(_myID); + _myNID = new NID(_myID); + _myNodeInfo = new NodeInfo(_myNID, session.getMyDestination(), _qPort); session.addMuxedSessionListener(this, I2PSession.PROTO_DATAGRAM_RAW, _rPort); session.addMuxedSessionListener(this, I2PSession.PROTO_DATAGRAM, _qPort); @@ -242,7 +246,7 @@ public class KRPC implements I2PSessionMuxedListener, DHT { tried.add(nInfo); // this isn't going to work, he will just return our own? - ReplyWaiter waiter = sendFindNode(nInfo, _myNodeInfo); + ReplyWaiter waiter = sendFindNode(nInfo, _myNID); if (waiter == null) continue; synchronized(waiter) { @@ -309,7 +313,7 @@ public class KRPC implements I2PSessionMuxedListener, DHT { Set tried = new HashSet(); if (_log.shouldLog(Log.INFO)) - _log.info("Starting getPeers"); + _log.info("Starting getPeers with " + nodes.size() + " to try"); for (int i = 0; i < max; i++) { NodeInfo nInfo; try { @@ -413,7 +417,10 @@ public class KRPC implements I2PSessionMuxedListener, DHT { announce(ih); int rv = 0; long start = _context.clock().now(); - List nodes = _knownNodes.findClosest(new InfoHash(ih), max); + InfoHash iHash = new InfoHash(ih); + List nodes = _knownNodes.findClosest(iHash, max); + if (_log.shouldLog(Log.INFO)) + _log.info("Found " + nodes.size() + " to announce to for " + iHash); for (NodeInfo nInfo : nodes) { if (announce(ih, nInfo, Math.min(maxWait, 60*1000))) rv++; @@ -435,14 +442,18 @@ public class KRPC implements I2PSessionMuxedListener, DHT { * @param maxWait the maximum time to wait (ms) or 0 to return immediately. * @return success */ - public boolean announce(byte[] ih, NodeInfo nInfo, long maxWait) { + private boolean announce(byte[] ih, NodeInfo nInfo, long maxWait) { InfoHash iHash = new InfoHash(ih); - TokenKey tokenKey = new TokenKey(nInfo.getNID(), iHash); - Token token = _incomingTokens.get(tokenKey); + // it isn't clear from BEP 5 if a token is bound to a single infohash? + // for now, just bind to the NID + //TokenKey tokenKey = new TokenKey(nInfo.getNID(), iHash); + Token token = _incomingTokens.get(nInfo.getNID()); if (token == null) { // we have no token, have to do a getPeers first to get a token if (maxWait <= 0) return false; + if (_log.shouldLog(Log.INFO)) + _log.info("No token for announce to " + nInfo + " sending get_peers first"); ReplyWaiter waiter = sendGetPeers(nInfo, iHash); if (waiter == null) return false; @@ -453,15 +464,24 @@ public class KRPC implements I2PSessionMuxedListener, DHT { } catch (InterruptedException ie) {} } int replyType = waiter.getReplyCode(); - if (!(replyType == REPLY_PEERS || replyType == REPLY_NODES)) + if (!(replyType == REPLY_PEERS || replyType == REPLY_NODES)) { + if (_log.shouldLog(Log.INFO)) + _log.info("Get_peers failed to " + nInfo); return false; + } // we should have a token now - token = _incomingTokens.get(tokenKey); - if (token == null) + token = _incomingTokens.get(nInfo.getNID()); + if (token == null) { + if (_log.shouldLog(Log.INFO)) + _log.info("Huh? no token after get_peers succeeded to " + nInfo); return false; + } maxWait -= _context.clock().now() - start; - if (maxWait < 1000) + if (maxWait < 1000) { + if (_log.shouldLog(Log.INFO)) + _log.info("Ran out of time after get_peers succeeded to " + nInfo); return false; + } } // send and wait on rcv msg lock unless maxWait <= 0 @@ -525,6 +545,8 @@ public class KRPC implements I2PSessionMuxedListener, DHT { * @return null on error */ private ReplyWaiter sendPing(NodeInfo nInfo) { + if (_log.shouldLog(Log.INFO)) + _log.info("Sending ping to: " + nInfo); Map map = new HashMap(); map.put("q", "ping"); Map args = new HashMap(); @@ -534,9 +556,12 @@ public class KRPC implements I2PSessionMuxedListener, DHT { /** * @param nInfo who to send it to + * @param tID target ID we are looking for * @return null on error */ - private ReplyWaiter sendFindNode(NodeInfo nInfo, NodeInfo tID) { + private ReplyWaiter sendFindNode(NodeInfo nInfo, NID tID) { + if (_log.shouldLog(Log.INFO)) + _log.info("Sending find node of " + tID + " to: " + nInfo); Map map = new HashMap(); map.put("q", "find_node"); Map args = new HashMap(); @@ -550,12 +575,18 @@ public class KRPC implements I2PSessionMuxedListener, DHT { * @return null on error */ private ReplyWaiter sendGetPeers(NodeInfo nInfo, InfoHash ih) { + if (_log.shouldLog(Log.INFO)) + _log.info("Sending get peers of " + ih + " to: " + nInfo); Map map = new HashMap(); map.put("q", "get_peers"); Map args = new HashMap(); args.put("info_hash", ih.getData()); map.put("a", args); - return sendQuery(nInfo, map, true); + ReplyWaiter rv = sendQuery(nInfo, map, true); + // save the InfoHash so we can get it later + if (rv != null) + rv.setSentObject(ih); + return rv; } /** @@ -563,6 +594,8 @@ public class KRPC implements I2PSessionMuxedListener, DHT { * @return null on error */ private ReplyWaiter sendAnnouncePeer(NodeInfo nInfo, InfoHash ih, Token token) { + if (_log.shouldLog(Log.INFO)) + _log.info("Sending announce of " + ih + " to: " + nInfo); Map map = new HashMap(); map.put("q", "announce_peer"); Map args = new HashMap(); @@ -573,9 +606,6 @@ public class KRPC implements I2PSessionMuxedListener, DHT { map.put("a", args); // an announce need not be signed, we have a token ReplyWaiter rv = sendQuery(nInfo, map, false); - // save the InfoHash so we can get it later - if (rv != null) - rv.setSentObject(ih); return rv; } @@ -587,6 +617,8 @@ public class KRPC implements I2PSessionMuxedListener, DHT { * @return success */ private boolean sendPong(NodeInfo nInfo, MsgID msgID) { + if (_log.shouldLog(Log.INFO)) + _log.info("Sending pong to: " + nInfo); Map map = new HashMap(); Map resps = new HashMap(); map.put("r", resps); @@ -604,6 +636,8 @@ public class KRPC implements I2PSessionMuxedListener, DHT { * @return success */ private boolean sendNodes(NodeInfo nInfo, MsgID msgID, Token token, byte[] ids) { + if (_log.shouldLog(Log.INFO)) + _log.info("Sending nodes to: " + nInfo); Map map = new HashMap(); Map resps = new HashMap(); map.put("r", resps); @@ -615,6 +649,8 @@ public class KRPC implements I2PSessionMuxedListener, DHT { /** @param token non-null */ private boolean sendPeers(NodeInfo nInfo, MsgID msgID, Token token, List peers) { + if (_log.shouldLog(Log.INFO)) + _log.info("Sending peers to: " + nInfo); Map map = new HashMap(); Map resps = new HashMap(); map.put("r", resps); @@ -630,6 +666,8 @@ public class KRPC implements I2PSessionMuxedListener, DHT { * @return success */ private boolean sendError(NodeInfo nInfo, MsgID msgID, int err, String msg) { + if (_log.shouldLog(Log.INFO)) + _log.info("Sending error " + msg + " to: " + nInfo); Map map = new HashMap(); Map resps = new HashMap(); map.put("r", resps); @@ -647,8 +685,8 @@ public class KRPC implements I2PSessionMuxedListener, DHT { private ReplyWaiter sendQuery(NodeInfo nInfo, Map map, boolean repliable) { if (nInfo.equals(_myNodeInfo)) throw new IllegalArgumentException("wtf don't send to ourselves"); - if (_log.shouldLog(Log.INFO)) - _log.info("Sending query to: " + nInfo); + if (_log.shouldLog(Log.DEBUG)) + _log.debug("Sending query to: " + nInfo); if (nInfo.getDestination() == null) { NodeInfo newInfo = _knownNodes.get(nInfo.getNID()); if (newInfo != null && newInfo.getDestination() != null) { @@ -666,11 +704,11 @@ public class KRPC implements I2PSessionMuxedListener, DHT { Map args = (Map) map.get("a"); if (args == null) throw new IllegalArgumentException("no args"); - args.put("id", _myNodeInfo.getData()); + args.put("id", _myID); int port = nInfo.getPort(); if (!repliable) port++; - boolean success = sendMessage(nInfo.getDestination(), port, map, true); + boolean success = sendMessage(nInfo.getDestination(), port, map, repliable); if (success) { // save for the caller to get ReplyWaiter rv = new ReplyWaiter(mID, nInfo, null, null); @@ -687,8 +725,8 @@ public class KRPC implements I2PSessionMuxedListener, DHT { private boolean sendResponse(NodeInfo nInfo, MsgID msgID, Map map) { if (nInfo.equals(_myNodeInfo)) throw new IllegalArgumentException("wtf don't send to ourselves"); - if (_log.shouldLog(Log.INFO)) - _log.info("Sending response to: " + nInfo); + if (_log.shouldLog(Log.DEBUG)) + _log.debug("Sending response to: " + nInfo); if (nInfo.getDestination() == null) { NodeInfo newInfo = _knownNodes.get(nInfo.getNID()); if (newInfo != null && newInfo.getDestination() != null) { @@ -705,7 +743,7 @@ public class KRPC implements I2PSessionMuxedListener, DHT { Map resps = (Map) map.get("r"); if (resps == null) throw new IllegalArgumentException("no resps"); - resps.put("id", _myNodeInfo.getData()); + resps.put("id", _myID); return sendMessage(nInfo.getDestination(), nInfo.getPort() + 1, map, false); } @@ -803,7 +841,7 @@ public class KRPC implements I2PSessionMuxedListener, DHT { byte[] msgIDBytes = map.get("t").getBytes(); MsgID mID = new MsgID(msgIDBytes); String type = map.get("y").getString(); - if (type.equals("q") && from != null) { + if (type.equals("q")) { // queries must be repliable String method = map.get("q").getString(); Map args = map.get("a").getMap(); @@ -849,21 +887,31 @@ public class KRPC implements I2PSessionMuxedListener, DHT { /** * Adds sender to our DHT. - * @param dest non-null + * @param dest may be null for announce_peer method only * @throws NPE too */ private void receiveQuery(MsgID msgID, Destination dest, int fromPort, String method, Map args) throws InvalidBEncodingException { + if (dest == null && !method.equals("announce_peer")) { + if (_log.shouldLog(Log.WARN)) + _log.warn("Received non-announce_peer query method on reply port: " + method); + return; + } byte[] nid = args.get("id").getBytes(); - NodeInfo nInfo = new NodeInfo(nid); - nInfo = heardFrom(nInfo); - nInfo.setDestination(dest); -// ninfo.checkport ? + NodeInfo nInfo; + if (dest != null) { + nInfo = new NodeInfo(new NID(nid), dest, fromPort); + nInfo = heardFrom(nInfo); + nInfo.setDestination(dest); + // ninfo.checkport ? + } else { + nInfo = null; + } if (method.equals("ping")) { receivePing(msgID, nInfo); } else if (method.equals("find_node")) { byte[] tid = args.get("target").getBytes(); - NodeInfo tID = new NodeInfo(tid); + NID tID = new NID(tid); receiveFindNode(msgID, nInfo, tID); } else if (method.equals("get_peers")) { byte[] hash = args.get("info_hash").getBytes(); @@ -875,7 +923,7 @@ public class KRPC implements I2PSessionMuxedListener, DHT { // this is the "TCP" port, we don't care //int port = args.get("port").getInt(); byte[] token = args.get("token").getBytes(); - receiveAnnouncePeer(msgID, nInfo, ih, token); + receiveAnnouncePeer(msgID, ih, token); } else { if (_log.shouldLog(Log.WARN)) _log.warn("Unknown query method rcvd: " + method); @@ -907,6 +955,9 @@ public class KRPC implements I2PSessionMuxedListener, DHT { NodeInfo nInfo2 = _knownNodes.putIfAbsent(nID, nInfo); if (nInfo2 != null) oldInfo = nInfo2; + } else { + if (oldInfo.getDestination() == null && nInfo.getDestination() != null) + oldInfo.setDestination(nInfo.getDestination()); } if (when > oldInfo.lastSeen()) oldInfo.setLastSeen(when); @@ -924,8 +975,9 @@ public class KRPC implements I2PSessionMuxedListener, DHT { /** * Handle and respond to the query + * @param tID target ID they are looking for */ - private void receiveFindNode(MsgID msgID, NodeInfo nInfo, NodeInfo tID) throws InvalidBEncodingException { + private void receiveFindNode(MsgID msgID, NodeInfo nInfo, NID tID) throws InvalidBEncodingException { if (_log.shouldLog(Log.INFO)) _log.info("Rcvd find_node from: " + nInfo + " for: " + tID); NodeInfo peer = _knownNodes.get(tID); @@ -934,7 +986,7 @@ public class KRPC implements I2PSessionMuxedListener, DHT { sendNodes(nInfo, msgID, peer.getData()); } else { // get closest from DHT - List nodes = _knownNodes.findClosest(tID.getNID(), K); + List nodes = _knownNodes.findClosest(tID, K); nodes.remove(nInfo); // him nodes.remove(_myNodeInfo); // me byte[] nodeArray = new byte[nodes.size() * NodeInfo.LENGTH]; @@ -953,7 +1005,9 @@ public class KRPC implements I2PSessionMuxedListener, DHT { _log.info("Rcvd get_peers from: " + nInfo + " for: " + ih); // generate and save random token Token token = new Token(_context); - _outgoingTokens.put(ih, token); + _outgoingTokens.put(token, nInfo.getNID()); + if (_log.shouldLog(Log.INFO)) + _log.info("Stored new OB token: " + token + " for: " + nInfo); List peers = _tracker.getPeers(ih, MAX_WANT); if (peers.isEmpty()) { @@ -979,22 +1033,27 @@ public class KRPC implements I2PSessionMuxedListener, DHT { } /** - * Handle and respond to the query + * Handle and respond to the query. + * We have no node info here, it came on response port, we have to get it from the token */ - private void receiveAnnouncePeer(MsgID msgID, NodeInfo nInfo, InfoHash ih, byte[] token) throws InvalidBEncodingException { - if (_log.shouldLog(Log.INFO)) - _log.info("Rcvd announce from: " + nInfo + " for: " + ih); - // check token - // get desthash from token->dest map - Token oldToken = _outgoingTokens.get(ih); - if (oldToken == null || !DataHelper.eq(oldToken.getData(), token)) { + private void receiveAnnouncePeer(MsgID msgID, InfoHash ih, byte[] tok) throws InvalidBEncodingException { + Token token = new Token(tok); + NID nid = _outgoingTokens.get(token); + if (nid == null) { if (_log.shouldLog(Log.WARN)) - _log.warn("Bad token"); + _log.warn("Unknown token in announce_peer: " + token); + if (_log.shouldLog(Log.INFO)) + _log.info("Current known tokens: " + _outgoingTokens.keySet()); return; } - - //msg ID -> NodeInfo -> Dest -> Hash - //verify with token -> nid or dest or hash ???? + NodeInfo nInfo = _knownNodes.get(nid); + if (nInfo == null) { + if (_log.shouldLog(Log.WARN)) + _log.warn("Unknown node in announce_peer for: " + nid); + return; + } + if (_log.shouldLog(Log.INFO)) + _log.info("Rcvd announce from: " + nInfo + " for: " + ih); _tracker.announce(ih, nInfo.getHash()); // the reply for an announce is the same as the reply for a ping @@ -1020,9 +1079,10 @@ public class KRPC implements I2PSessionMuxedListener, DHT { InfoHash ih = (InfoHash) waiter.getSentObject(); if (btok != null && ih != null) { byte[] tok = btok.getBytes(); - _incomingTokens.put(new TokenKey(nInfo.getNID(), ih), new Token(_context, tok)); + Token token = new Token(_context, tok); + _incomingTokens.put(nInfo.getNID(), token); if (_log.shouldLog(Log.INFO)) - _log.info("Got token, must be a response to get_peers"); + _log.info("Got token: " + token + ", must be a response to get_peers"); } else { if (_log.shouldLog(Log.INFO)) _log.info("No token and saved infohash, must be a response to find_node"); @@ -1042,7 +1102,8 @@ public class KRPC implements I2PSessionMuxedListener, DHT { waiter.gotReply(REPLY_PEERS, rlist); } else { // a ping response or an announce peer response - receivePong(nInfo); + byte[] nid = response.get("id").getBytes(); + receivePong(nInfo, nid); waiter.gotReply(REPLY_PONG, null); } } @@ -1085,8 +1146,18 @@ public class KRPC implements I2PSessionMuxedListener, DHT { return rv; } - /** does nothing, but node was already added to our DHT */ - private void receivePong(NodeInfo nInfo) { + /** + * If node info was previously created with the dummy NID, + * replace it with the received NID. + */ + private void receivePong(NodeInfo nInfo, byte[] nid) { + if (nInfo.getNID().equals(_fakeNID)) { + NodeInfo newInfo = new NodeInfo(new NID(nid), nInfo.getHash(), nInfo.getPort()); + Destination dest = nInfo.getDestination(); + if (dest != null) + newInfo.setDestination(dest); + heardFrom(newInfo); + } if (_log.shouldLog(Log.INFO)) _log.info("Rcvd pong from: " + nInfo); } @@ -1121,23 +1192,23 @@ public class KRPC implements I2PSessionMuxedListener, DHT { /** * Either wait on this object with a timeout, or use non-null Runnables. - * Any sent data to be rememberd may be stored by setSentObject(). + * Any sent data to be remembered may be stored by setSentObject(). * Reply object may be in getReplyObject(). * @param onReply must be fast, otherwise set to null and wait on this * @param onTimeout must be fast, otherwise set to null and wait on this */ public ReplyWaiter(MsgID mID, NodeInfo nInfo, Runnable onReply, Runnable onTimeout) { - super(nInfo.getData()); + super(nInfo.getNID(), nInfo.getHash(), nInfo.getPort()); + Destination dest = nInfo.getDestination(); + if (dest != null) + setDestination(dest); this.mid = mID; this.onReply = onReply; this.onTimeout = onTimeout; - if (onTimeout != null) - this.event = new Event(); - else - this.event = null; + this.event = new Event(); } - /** only used for announce, to save the Info Hash */ + /** only used for get_peers, to save the Info Hash */ public void setSentObject(Object o) { sentObject = o; } @@ -1174,10 +1245,11 @@ public class KRPC implements I2PSessionMuxedListener, DHT { public void gotReply(int code, Object o) { replyCode = code; replyObject = o; - if (event != null) - event.cancel(); + event.cancel(); _sentQueries.remove(mid); - heardFrom(this); + // if it is fake, heardFrom is called by receivePong() + if (!getNID().equals(_fakeNID)) + heardFrom(this); if (onReply != null) onReply.run(); synchronized(this) { @@ -1266,7 +1338,7 @@ public class KRPC implements I2PSessionMuxedListener, DHT { public void timeReached() { long now = _context.clock().now(); - for (Iterator iter = _outgoingTokens.values().iterator(); iter.hasNext(); ) { + for (Iterator iter = _outgoingTokens.keySet().iterator(); iter.hasNext(); ) { Token tok = iter.next(); if (tok.lastSeen() < now - MAX_TOKEN_AGE) iter.remove(); diff --git a/apps/i2psnark/java/src/org/klomp/snark/dht/NodeInfo.java b/apps/i2psnark/java/src/org/klomp/snark/dht/NodeInfo.java index 9e73ca03be..7c74a506e4 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/dht/NodeInfo.java +++ b/apps/i2psnark/java/src/org/klomp/snark/dht/NodeInfo.java @@ -15,6 +15,7 @@ import net.i2p.data.SimpleDataStructure; * * Things are a little tricky in KRPC since we exchange Hashes and don't * always have the Destination. + * The conpact info is immutable. The Destination may be added later. * * @since 0.8.4 * @author zzz @@ -45,7 +46,6 @@ public class NodeInfo extends SimpleDataStructure { /** * No Destination yet available - * @deprecated unused * @throws IllegalArgumentException */ public NodeInfo(NID nID, Hash hash, int port) { @@ -138,13 +138,11 @@ public class NodeInfo extends SimpleDataStructure { * @throws IllegalArgumentException if hash of dest doesn't match previous hash */ public void setDestination(Destination dest) throws IllegalArgumentException { + if (this.dest != null) + return; if (!dest.calculateHash().equals(this.hash)) throw new IllegalArgumentException("Hash mismatch, was: " + this.hash + " new: " + dest.calculateHash()); - if (this.dest == null) - this.dest = dest; - else if (!this.dest.equals(dest)) - throw new IllegalArgumentException("Dest mismatch, was: " + this.dest+ " new: " + dest); - // else keep the old to reduce object churn + this.dest = dest; } public int getPort() { diff --git a/apps/i2psnark/java/src/org/klomp/snark/dht/Token.java b/apps/i2psnark/java/src/org/klomp/snark/dht/Token.java index c8859155df..46b3e10979 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/dht/Token.java +++ b/apps/i2psnark/java/src/org/klomp/snark/dht/Token.java @@ -3,6 +3,8 @@ package org.klomp.snark.dht; * GPLv2 */ +import java.util.Date; + import net.i2p.I2PAppContext; import net.i2p.data.ByteArray; import net.i2p.data.DataHelper; @@ -24,6 +26,7 @@ public class Token extends ByteArray { byte[] data = new byte[MY_TOK_LEN]; ctx.random().nextBytes(data); setData(data); + setValid(MY_TOK_LEN); lastSeen = ctx.clock().now(); } @@ -33,7 +36,36 @@ public class Token extends ByteArray { lastSeen = ctx.clock().now(); } + /** incoming - for lookup only, not storage, lastSeen is 0 */ + public Token(byte[] data) { + super(data); + lastSeen = 0; + } + public long lastSeen() { return lastSeen; } + + @Override + public String toString() { + StringBuilder buf = new StringBuilder(64); + buf.append("[Token: "); + byte[] bs = getData(); + if (bs.length == 0) { + buf.append("0 bytes"); + } else { + buf.append(bs.length).append(" bytes: 0x"); + // backwards, but the same way BEValue does it + for (int i = 0; i < bs.length; i++) { + int b = bs[i] & 0xff; + if (b < 16) + buf.append('0'); + buf.append(Integer.toHexString(b)); + } + } + if (lastSeen > 0) + buf.append(" created ").append((new Date(lastSeen)).toString()); + buf.append(']'); + return buf.toString(); + } } From 121491a3be3d2cb6e1f42868e657c9d7f271da5a Mon Sep 17 00:00:00 2001 From: zzz Date: Sun, 3 Jun 2012 16:05:38 +0000 Subject: [PATCH 6/9] - B32 lookup if required for non-announce queries only - Token timeout tweaks - Most classes package private --- .../src/org/klomp/snark/dht/DHTNodes.java | 2 +- .../src/org/klomp/snark/dht/InfoHash.java | 2 +- .../java/src/org/klomp/snark/dht/KRPC.java | 48 ++++++++++++++++--- .../java/src/org/klomp/snark/dht/MsgID.java | 2 +- .../java/src/org/klomp/snark/dht/NID.java | 2 +- .../src/org/klomp/snark/dht/NodeInfo.java | 2 +- .../java/src/org/klomp/snark/dht/Peer.java | 2 +- .../java/src/org/klomp/snark/dht/Peers.java | 2 +- .../java/src/org/klomp/snark/dht/Token.java | 2 +- .../src/org/klomp/snark/dht/TokenKey.java | 2 +- .../src/org/klomp/snark/dht/Torrents.java | 2 +- 11 files changed, 52 insertions(+), 16 deletions(-) diff --git a/apps/i2psnark/java/src/org/klomp/snark/dht/DHTNodes.java b/apps/i2psnark/java/src/org/klomp/snark/dht/DHTNodes.java index cd6d0f37ca..14c9f9a3fd 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/dht/DHTNodes.java +++ b/apps/i2psnark/java/src/org/klomp/snark/dht/DHTNodes.java @@ -28,7 +28,7 @@ import net.i2p.util.SimpleTimer; * @since 0.8.4 * @author zzz */ -public class DHTNodes extends ConcurrentHashMap { +class DHTNodes extends ConcurrentHashMap { private final I2PAppContext _context; private long _expireTime; diff --git a/apps/i2psnark/java/src/org/klomp/snark/dht/InfoHash.java b/apps/i2psnark/java/src/org/klomp/snark/dht/InfoHash.java index 2b439c6c4a..221d79af42 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/dht/InfoHash.java +++ b/apps/i2psnark/java/src/org/klomp/snark/dht/InfoHash.java @@ -11,7 +11,7 @@ import net.i2p.crypto.SHA1Hash; * @since 0.8.4 * @author zzz */ -public class InfoHash extends SHA1Hash { +class InfoHash extends SHA1Hash { public InfoHash(byte[] data) { super(data); diff --git a/apps/i2psnark/java/src/org/klomp/snark/dht/KRPC.java b/apps/i2psnark/java/src/org/klomp/snark/dht/KRPC.java index e0608df4e4..c4190bd3b4 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/dht/KRPC.java +++ b/apps/i2psnark/java/src/org/klomp/snark/dht/KRPC.java @@ -127,6 +127,7 @@ public class KRPC implements I2PSessionMuxedListener, DHT { private static final long MAX_NODEINFO_AGE = 60*60*1000; /** how long since generated do we delete - BEP 5 says 10 minutes */ private static final long MAX_TOKEN_AGE = 60*60*1000; + private static final long MAX_INBOUND_TOKEN_AGE = MAX_TOKEN_AGE - 5*60*1000; /** how long since sent do we wait for a reply */ private static final long MAX_MSGID_AGE = 2*60*1000; /** how long since sent do we wait for a reply */ @@ -679,6 +680,7 @@ public class KRPC implements I2PSessionMuxedListener, DHT { // TODO sendQuery with onReply / onTimeout args /** + * Blocking if repliable and we must lookup b32 * @param repliable true for all but announce * @return null on error */ @@ -691,11 +693,19 @@ public class KRPC implements I2PSessionMuxedListener, DHT { NodeInfo newInfo = _knownNodes.get(nInfo.getNID()); if (newInfo != null && newInfo.getDestination() != null) { nInfo = newInfo; - } else { - // lookup b32? + } else if (!repliable) { + // Don't lookup for announce query, we should already have it if (_log.shouldLog(Log.WARN)) - _log.warn("No destination for: " + nInfo); + _log.warn("Dropping non-repliable query, no dest for " + nInfo); return null; + } else { + // Lookup the dest for the hash + // TODO spin off into thread or queue? We really don't want to block here + if (!lookupDest(nInfo)) { + if (_log.shouldLog(Log.WARN)) + _log.warn("Dropping repliable query, no dest for " + nInfo); + return null; + } } } map.put("y", "q"); @@ -734,7 +744,7 @@ public class KRPC implements I2PSessionMuxedListener, DHT { } else { // lookup b32? if (_log.shouldLog(Log.WARN)) - _log.warn("No destination for: " + nInfo); + _log.warn("Dropping response, no dest for " + nInfo); return false; } } @@ -763,7 +773,7 @@ public class KRPC implements I2PSessionMuxedListener, DHT { } else { // lookup b32? if (_log.shouldLog(Log.WARN)) - _log.warn("No destination for: " + nInfo); + _log.warn("Dropping sendError, no dest for " + nInfo); return false; } } @@ -772,6 +782,32 @@ public class KRPC implements I2PSessionMuxedListener, DHT { return sendMessage(nInfo.getDestination(), nInfo.getPort() + 1, map, false); } + /** + * Get the dest for a NodeInfo lacking it, and store it there. + * Blocking. + * @return success + */ + private boolean lookupDest(NodeInfo nInfo) { + if (_log.shouldLog(Log.INFO)) + _log.info("looking up dest for " + nInfo); + try { + // use a short timeout for now + Destination dest = _session.lookupDest(nInfo.getHash(), 5*1000); + if (dest != null) { + nInfo.setDestination(dest); + if (_log.shouldLog(Log.INFO)) + _log.info("lookup success for " + nInfo); + return true; + } + } catch (I2PSessionException ise) { + if (_log.shouldLog(Log.WARN)) + _log.warn("lookup fail", ise); + } + if (_log.shouldLog(Log.INFO)) + _log.info("lookup fail for " + nInfo); + return false; + } + /** * Lowest-level send message call. * @param repliable true for all but announce @@ -1345,7 +1381,7 @@ public class KRPC implements I2PSessionMuxedListener, DHT { } for (Iterator iter = _incomingTokens.values().iterator(); iter.hasNext(); ) { Token tok = iter.next(); - if (tok.lastSeen() < now - MAX_TOKEN_AGE) + if (tok.lastSeen() < now - MAX_INBOUND_TOKEN_AGE) iter.remove(); } // TODO sent queries? diff --git a/apps/i2psnark/java/src/org/klomp/snark/dht/MsgID.java b/apps/i2psnark/java/src/org/klomp/snark/dht/MsgID.java index 94b37a690b..6d53f7c8fb 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/dht/MsgID.java +++ b/apps/i2psnark/java/src/org/klomp/snark/dht/MsgID.java @@ -12,7 +12,7 @@ import net.i2p.data.ByteArray; * @since 0.8.4 * @author zzz */ -public class MsgID extends ByteArray { +class MsgID extends ByteArray { private static final int MY_TOK_LEN = 8; diff --git a/apps/i2psnark/java/src/org/klomp/snark/dht/NID.java b/apps/i2psnark/java/src/org/klomp/snark/dht/NID.java index 3d0d0a496e..87407720bd 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/dht/NID.java +++ b/apps/i2psnark/java/src/org/klomp/snark/dht/NID.java @@ -11,7 +11,7 @@ import net.i2p.crypto.SHA1Hash; * @since 0.8.4 * @author zzz */ -public class NID extends SHA1Hash { +class NID extends SHA1Hash { public NID(byte[] data) { super(data); diff --git a/apps/i2psnark/java/src/org/klomp/snark/dht/NodeInfo.java b/apps/i2psnark/java/src/org/klomp/snark/dht/NodeInfo.java index 7c74a506e4..1d82b268a4 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/dht/NodeInfo.java +++ b/apps/i2psnark/java/src/org/klomp/snark/dht/NodeInfo.java @@ -21,7 +21,7 @@ import net.i2p.data.SimpleDataStructure; * @author zzz */ -public class NodeInfo extends SimpleDataStructure { +class NodeInfo extends SimpleDataStructure { private long lastSeen; private NID nID; diff --git a/apps/i2psnark/java/src/org/klomp/snark/dht/Peer.java b/apps/i2psnark/java/src/org/klomp/snark/dht/Peer.java index 8943c06b5f..84fc263a7d 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/dht/Peer.java +++ b/apps/i2psnark/java/src/org/klomp/snark/dht/Peer.java @@ -12,7 +12,7 @@ import net.i2p.data.Hash; * @since 0.8.4 * @author zzz */ -public class Peer extends Hash { +class Peer extends Hash { private long lastSeen; diff --git a/apps/i2psnark/java/src/org/klomp/snark/dht/Peers.java b/apps/i2psnark/java/src/org/klomp/snark/dht/Peers.java index 2a2452e956..f16d903ec6 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/dht/Peers.java +++ b/apps/i2psnark/java/src/org/klomp/snark/dht/Peers.java @@ -13,7 +13,7 @@ import net.i2p.data.Hash; * @since 0.8.4 * @author zzz */ -public class Peers extends ConcurrentHashMap { +class Peers extends ConcurrentHashMap { public Peers() { super(); diff --git a/apps/i2psnark/java/src/org/klomp/snark/dht/Token.java b/apps/i2psnark/java/src/org/klomp/snark/dht/Token.java index 46b3e10979..37a43575db 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/dht/Token.java +++ b/apps/i2psnark/java/src/org/klomp/snark/dht/Token.java @@ -15,7 +15,7 @@ import net.i2p.data.DataHelper; * @since 0.8.4 * @author zzz */ -public class Token extends ByteArray { +class Token extends ByteArray { private static final int MY_TOK_LEN = 8; private final long lastSeen; diff --git a/apps/i2psnark/java/src/org/klomp/snark/dht/TokenKey.java b/apps/i2psnark/java/src/org/klomp/snark/dht/TokenKey.java index d2217a015e..996d43351e 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/dht/TokenKey.java +++ b/apps/i2psnark/java/src/org/klomp/snark/dht/TokenKey.java @@ -12,7 +12,7 @@ import net.i2p.data.DataHelper; * @since 0.8.4 * @author zzz */ -public class TokenKey extends SHA1Hash { +class TokenKey extends SHA1Hash { public TokenKey(NID nID, InfoHash ih) { super(DataHelper.xor(nID.getData(), ih.getData())); diff --git a/apps/i2psnark/java/src/org/klomp/snark/dht/Torrents.java b/apps/i2psnark/java/src/org/klomp/snark/dht/Torrents.java index c791e70776..304b7c9491 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/dht/Torrents.java +++ b/apps/i2psnark/java/src/org/klomp/snark/dht/Torrents.java @@ -11,7 +11,7 @@ import java.util.concurrent.ConcurrentHashMap; * @since 0.8.4 * @author zzz */ -public class Torrents extends ConcurrentHashMap { +class Torrents extends ConcurrentHashMap { public Torrents() { super(); From d5cb443925e4f91f9058a0d54acc2738a8e9dd2f Mon Sep 17 00:00:00 2001 From: zzz Date: Mon, 4 Jun 2012 14:15:38 +0000 Subject: [PATCH 7/9] - Switch back from storing NID to full NodeInfo for outgoing tokens so they don't get expired early - Announce only to the single closest DHT peer - Increase random port range - Decrease max local tracker and DHT size --- .../src/org/klomp/snark/TrackerClient.java | 3 ++- .../src/org/klomp/snark/dht/DHTNodes.java | 2 +- .../src/org/klomp/snark/dht/DHTTracker.java | 2 +- .../java/src/org/klomp/snark/dht/KRPC.java | 26 +++++++++---------- 4 files changed, 17 insertions(+), 16 deletions(-) diff --git a/apps/i2psnark/java/src/org/klomp/snark/TrackerClient.java b/apps/i2psnark/java/src/org/klomp/snark/TrackerClient.java index 01b6a829b1..ac5c532855 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/TrackerClient.java +++ b/apps/i2psnark/java/src/org/klomp/snark/TrackerClient.java @@ -374,7 +374,8 @@ public class TrackerClient extends I2PAppThread // announce ourselves while the token is still good // FIXME this needs to be in its own thread if (!stop) { - int good = _util.getDHT().announce(snark.getInfoHash(), 8, 5*60*1000); + // announce only to the 1 closest + int good = _util.getDHT().announce(snark.getInfoHash(), 1, 5*60*1000); _util.debug("Sent " + good + " good announces to DHT", Snark.INFO); } diff --git a/apps/i2psnark/java/src/org/klomp/snark/dht/DHTNodes.java b/apps/i2psnark/java/src/org/klomp/snark/dht/DHTNodes.java index 14c9f9a3fd..0e57d1f26a 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/dht/DHTNodes.java +++ b/apps/i2psnark/java/src/org/klomp/snark/dht/DHTNodes.java @@ -39,7 +39,7 @@ class DHTNodes extends ConcurrentHashMap { private static final long MAX_EXPIRE_TIME = 60*60*1000; private static final long MIN_EXPIRE_TIME = 5*60*1000; private static final long DELTA_EXPIRE_TIME = 7*60*1000; - private static final int MAX_PEERS = 9999; + private static final int MAX_PEERS = 999; public DHTNodes(I2PAppContext ctx) { super(); diff --git a/apps/i2psnark/java/src/org/klomp/snark/dht/DHTTracker.java b/apps/i2psnark/java/src/org/klomp/snark/dht/DHTTracker.java index 712b89e4f0..d7e9df93ed 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/dht/DHTTracker.java +++ b/apps/i2psnark/java/src/org/klomp/snark/dht/DHTTracker.java @@ -34,7 +34,7 @@ class DHTTracker { private static final long MAX_EXPIRE_TIME = 95*60*1000; private static final long MIN_EXPIRE_TIME = 5*60*1000; private static final long DELTA_EXPIRE_TIME = 7*60*1000; - private static final int MAX_PEERS = 9999; + private static final int MAX_PEERS = 2000; DHTTracker(I2PAppContext ctx) { _context = ctx; diff --git a/apps/i2psnark/java/src/org/klomp/snark/dht/KRPC.java b/apps/i2psnark/java/src/org/klomp/snark/dht/KRPC.java index c4190bd3b4..f34a6a1dda 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/dht/KRPC.java +++ b/apps/i2psnark/java/src/org/klomp/snark/dht/KRPC.java @@ -92,7 +92,7 @@ public class KRPC implements I2PSessionMuxedListener, DHT { /** index to sent queries awaiting reply */ private final ConcurrentHashMap _sentQueries; /** index to outgoing tokens we generated, sent in reply to a get_peers query */ - private final ConcurrentHashMap _outgoingTokens; + private final ConcurrentHashMap _outgoingTokens; /** index to incoming opaque tokens, received in a peers or nodes reply */ private final ConcurrentHashMap _incomingTokens; @@ -148,8 +148,9 @@ public class KRPC implements I2PSessionMuxedListener, DHT { _incomingTokens = new ConcurrentHashMap(); // Construct my NodeInfo - // ports can really be fixed, just do this for testing - _qPort = 30000 + ctx.random().nextInt(99); + // Pick ports over a big range to marginally increase security + // If we add a search DHT, adjust to stay out of each other's way + _qPort = 2555 + ctx.random().nextInt(61111); _rPort = _qPort + 1; _myID = new byte[NID.HASH_LENGTH]; ctx.random().nextBytes(_myID); @@ -291,6 +292,7 @@ public class KRPC implements I2PSessionMuxedListener, DHT { /** * Get peers for a torrent. + * This is an iterative lookup in the DHT. * Blocking! * Caller should run in a thread. * @@ -404,13 +406,16 @@ public class KRPC implements I2PSessionMuxedListener, DHT { } /** - * Announce to the closest DHT peers. + * Announce to the closest peers in the local DHT. + * This is NOT iterative - call getPeers() first to get the closest + * peers into the local DHT. * Blocking unless maxWait <= 0 * Caller should run in a thread. * This also automatically announces ourself to our local tracker. * For best results do a getPeers() first so we have tokens. * * @param ih the Info Hash (torrent) + * @param max maximum number of peers to announce to * @param maxWait the maximum total time to wait (ms) or 0 to do all in parallel and return immediately. * @return the number of successful announces, not counting ourselves. */ @@ -842,6 +847,7 @@ public class KRPC implements I2PSessionMuxedListener, DHT { } try { + // TODO I2CP per-packet options boolean success = _session.sendMessage(dest, payload, 0, payload.length, null, null, 60*1000, repliable ? I2PSession.PROTO_DATAGRAM : I2PSession.PROTO_DATAGRAM_RAW, fromPort, toPort); @@ -1041,7 +1047,7 @@ public class KRPC implements I2PSessionMuxedListener, DHT { _log.info("Rcvd get_peers from: " + nInfo + " for: " + ih); // generate and save random token Token token = new Token(_context); - _outgoingTokens.put(token, nInfo.getNID()); + _outgoingTokens.put(token, nInfo); if (_log.shouldLog(Log.INFO)) _log.info("Stored new OB token: " + token + " for: " + nInfo); @@ -1074,20 +1080,14 @@ public class KRPC implements I2PSessionMuxedListener, DHT { */ private void receiveAnnouncePeer(MsgID msgID, InfoHash ih, byte[] tok) throws InvalidBEncodingException { Token token = new Token(tok); - NID nid = _outgoingTokens.get(token); - if (nid == null) { + NodeInfo nInfo = _outgoingTokens.get(token); + if (nInfo == null) { if (_log.shouldLog(Log.WARN)) _log.warn("Unknown token in announce_peer: " + token); if (_log.shouldLog(Log.INFO)) _log.info("Current known tokens: " + _outgoingTokens.keySet()); return; } - NodeInfo nInfo = _knownNodes.get(nid); - if (nInfo == null) { - if (_log.shouldLog(Log.WARN)) - _log.warn("Unknown node in announce_peer for: " + nid); - return; - } if (_log.shouldLog(Log.INFO)) _log.info("Rcvd announce from: " + nInfo + " for: " + ih); From 3f40487c996477d47b51be5f99439596eee0f50c Mon Sep 17 00:00:00 2001 From: zzz Date: Mon, 4 Jun 2012 22:34:56 +0000 Subject: [PATCH 8/9] - Add persistent local DHT storage - Shutdown now closes tunnel - Delay after sending stop announces at shutdown - Stub out using Hash cache - Implement stop for all cleaners - Log tweaks --- .../src/org/klomp/snark/I2PSnarkUtil.java | 4 +- .../src/org/klomp/snark/SnarkManager.java | 12 ++- .../src/org/klomp/snark/dht/DHTNodes.java | 28 +++++-- .../src/org/klomp/snark/dht/DHTTracker.java | 25 ++++-- .../java/src/org/klomp/snark/dht/KRPC.java | 39 +++++++--- .../src/org/klomp/snark/dht/NodeInfo.java | 61 ++++++++++++++- .../src/org/klomp/snark/dht/PersistDHT.java | 77 +++++++++++++++++++ 7 files changed, 216 insertions(+), 30 deletions(-) create mode 100644 apps/i2psnark/java/src/org/klomp/snark/dht/PersistDHT.java diff --git a/apps/i2psnark/java/src/org/klomp/snark/I2PSnarkUtil.java b/apps/i2psnark/java/src/org/klomp/snark/I2PSnarkUtil.java index 425d51aeb6..0d71dcd346 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/I2PSnarkUtil.java +++ b/apps/i2psnark/java/src/org/klomp/snark/I2PSnarkUtil.java @@ -261,7 +261,8 @@ public class I2PSnarkUtil { // FIXME this can cause race NPEs elsewhere _manager = null; _shitlist.clear(); - mgr.destroySocketManager(); + if (mgr != null) + mgr.destroySocketManager(); // this will delete a .torrent file d/l in progress so don't do that... FileUtil.rmdir(_tmpDir, false); // in case the user will d/l a .torrent file next... @@ -405,6 +406,7 @@ public class I2PSnarkUtil { byte[] b = Base32.decode(ip.substring(0, BASE32_HASH_LENGTH)); if (b != null) { Hash h = new Hash(b); + //Hash h = Hash.create(b); if (_log.shouldLog(Log.INFO)) _log.info("Using existing session for lookup of " + ip); try { diff --git a/apps/i2psnark/java/src/org/klomp/snark/SnarkManager.java b/apps/i2psnark/java/src/org/klomp/snark/SnarkManager.java index 9bfb885cf3..42927df00f 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/SnarkManager.java +++ b/apps/i2psnark/java/src/org/klomp/snark/SnarkManager.java @@ -1514,15 +1514,23 @@ public class SnarkManager implements Snark.CompleteListener { } } - public class SnarkManagerShutdown extends I2PAppThread { + private class SnarkManagerShutdown extends I2PAppThread { @Override public void run() { Set names = listTorrentFiles(); + int running = 0; for (Iterator iter = names.iterator(); iter.hasNext(); ) { Snark snark = getTorrent((String)iter.next()); - if ( (snark != null) && (!snark.isStopped()) ) + if (snark != null && !snark.isStopped()) { snark.stopTorrent(); + running++; + } } + _snarks.clear(); + if (running > 0) { + try { sleep(1500); } catch (InterruptedException ie) {}; + } + _util.disconnect(); } } diff --git a/apps/i2psnark/java/src/org/klomp/snark/dht/DHTNodes.java b/apps/i2psnark/java/src/org/klomp/snark/dht/DHTNodes.java index 0e57d1f26a..e18fca6fef 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/dht/DHTNodes.java +++ b/apps/i2psnark/java/src/org/klomp/snark/dht/DHTNodes.java @@ -15,8 +15,7 @@ import net.i2p.I2PAppContext; import net.i2p.crypto.SHA1Hash; import net.i2p.data.DataHelper; import net.i2p.util.Log; -import net.i2p.util.SimpleScheduler; -import net.i2p.util.SimpleTimer; +import net.i2p.util.SimpleTimer2; /** * All the nodes we know about, stored as a mapping from @@ -33,6 +32,7 @@ class DHTNodes extends ConcurrentHashMap { private final I2PAppContext _context; private long _expireTime; private final Log _log; + private volatile boolean _isRunning; /** stagger with other cleaners */ private static final long CLEAN_TIME = 237*1000; @@ -46,7 +46,16 @@ class DHTNodes extends ConcurrentHashMap { _context = ctx; _expireTime = MAX_EXPIRE_TIME; _log = _context.logManager().getLog(DHTNodes.class); - SimpleScheduler.getInstance().addPeriodicEvent(new Cleaner(), CLEAN_TIME); + } + + public void start() { + _isRunning = true; + new Cleaner(); + } + + public void stop() { + clear(); + _isRunning = false; } /** @@ -82,9 +91,15 @@ class DHTNodes extends ConcurrentHashMap { ****/ /** */ - private class Cleaner implements SimpleTimer.TimedEvent { + private class Cleaner extends SimpleTimer2.TimedEvent { + + public Cleaner() { + super(SimpleTimer2.getInstance(), CLEAN_TIME); + } public void timeReached() { + if (!_isRunning) + return; long now = _context.clock().now(); int peerCount = 0; for (Iterator iter = DHTNodes.this.values().iterator(); iter.hasNext(); ) { @@ -100,11 +115,12 @@ class DHTNodes extends ConcurrentHashMap { else _expireTime = Math.min(_expireTime + DELTA_EXPIRE_TIME, MAX_EXPIRE_TIME); - if (_log.shouldLog(Log.INFO)) - _log.info("DHT storage cleaner done, now with " + + if (_log.shouldLog(Log.DEBUG)) + _log.debug("DHT storage cleaner done, now with " + peerCount + " peers, " + DataHelper.formatDuration(_expireTime) + " expiration"); + schedule(CLEAN_TIME); } } } diff --git a/apps/i2psnark/java/src/org/klomp/snark/dht/DHTTracker.java b/apps/i2psnark/java/src/org/klomp/snark/dht/DHTTracker.java index d7e9df93ed..97fbcef73d 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/dht/DHTTracker.java +++ b/apps/i2psnark/java/src/org/klomp/snark/dht/DHTTracker.java @@ -12,8 +12,7 @@ import net.i2p.I2PAppContext; import net.i2p.data.DataHelper; import net.i2p.data.Hash; import net.i2p.util.Log; -import net.i2p.util.SimpleScheduler; -import net.i2p.util.SimpleTimer; +import net.i2p.util.SimpleTimer2; /** * The tracker stores peers, i.e. Dest hashes (not nodes). @@ -27,6 +26,7 @@ class DHTTracker { private final Torrents _torrents; private long _expireTime; private final Log _log; + private volatile boolean _isRunning; /** stagger with other cleaners */ private static final long CLEAN_TIME = 199*1000; @@ -41,12 +41,16 @@ class DHTTracker { _torrents = new Torrents(); _expireTime = MAX_EXPIRE_TIME; _log = _context.logManager().getLog(DHTTracker.class); - SimpleScheduler.getInstance().addPeriodicEvent(new Cleaner(), CLEAN_TIME); + } + + public void start() { + _isRunning = true; + new Cleaner(); } void stop() { _torrents.clear(); - // no way to stop the cleaner + _isRunning = false; } void announce(InfoHash ih, Hash hash) { @@ -93,9 +97,15 @@ class DHTTracker { return rv; } - private class Cleaner implements SimpleTimer.TimedEvent { + private class Cleaner extends SimpleTimer2.TimedEvent { + + public Cleaner() { + super(SimpleTimer2.getInstance(), CLEAN_TIME); + } public void timeReached() { + if (!_isRunning) + return; long now = _context.clock().now(); int torrentCount = 0; int peerCount = 0; @@ -122,11 +132,12 @@ class DHTTracker { else _expireTime = Math.min(_expireTime + DELTA_EXPIRE_TIME, MAX_EXPIRE_TIME); - if (_log.shouldLog(Log.INFO)) - _log.info("DHT tracker cleaner done, now with " + + if (_log.shouldLog(Log.DEBUG)) + _log.debug("DHT tracker cleaner done, now with " + torrentCount + " torrents, " + peerCount + " peers, " + DataHelper.formatDuration(_expireTime) + " expiration"); + schedule(CLEAN_TIME); } } } diff --git a/apps/i2psnark/java/src/org/klomp/snark/dht/KRPC.java b/apps/i2psnark/java/src/org/klomp/snark/dht/KRPC.java index f34a6a1dda..2c44deb68f 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/dht/KRPC.java +++ b/apps/i2psnark/java/src/org/klomp/snark/dht/KRPC.java @@ -5,6 +5,7 @@ package org.klomp.snark.dht; */ import java.io.ByteArrayInputStream; +import java.io.File; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; @@ -36,8 +37,6 @@ import net.i2p.data.Destination; import net.i2p.data.Hash; import net.i2p.data.SimpleDataStructure; import net.i2p.util.Log; -import net.i2p.util.SimpleScheduler; -import net.i2p.util.SimpleTimer; import net.i2p.util.SimpleTimer2; import org.klomp.snark.bencode.BDecoder; @@ -108,6 +107,8 @@ public class KRPC implements I2PSessionMuxedListener, DHT { private final int _rPort; /** signed dgrams */ private final int _qPort; + private final File _dhtFile; + private volatile boolean _isRunning; /** all-zero NID used for pings */ private static final NID _fakeNID = new NID(new byte[NID.HASH_LENGTH]); @@ -134,6 +135,7 @@ public class KRPC implements I2PSessionMuxedListener, DHT { private static final long DEFAULT_QUERY_TIMEOUT = 75*1000; /** stagger with other cleaners */ private static final long CLEAN_TIME = 63*1000; + private static final String DHT_FILE = "i2psnark.dht.dat"; public KRPC (I2PAppContext ctx, I2PSession session) { _context = ctx; @@ -156,11 +158,11 @@ public class KRPC implements I2PSessionMuxedListener, DHT { ctx.random().nextBytes(_myID); _myNID = new NID(_myID); _myNodeInfo = new NodeInfo(_myNID, session.getMyDestination(), _qPort); + _dhtFile = new File(ctx.getConfigDir(), DHT_FILE); session.addMuxedSessionListener(this, I2PSession.PROTO_DATAGRAM_RAW, _rPort); session.addMuxedSessionListener(this, I2PSession.PROTO_DATAGRAM, _qPort); - // can't be stopped - SimpleScheduler.getInstance().addPeriodicEvent(new Cleaner(), CLEAN_TIME); + start(); } ///////////////// Public methods @@ -391,6 +393,7 @@ public class KRPC implements I2PSessionMuxedListener, DHT { public void announce(byte[] ih, byte[] peerHash) { InfoHash iHash = new InfoHash(ih); _tracker.announce(iHash, new Hash(peerHash)); +// _tracker.announce(iHash, Hash.create(peerHash)); } /** @@ -506,24 +509,32 @@ public class KRPC implements I2PSessionMuxedListener, DHT { } /** - * Does nothing yet, everything is prestarted. + * Loads the DHT from file. * Can't be restarted after stopping? */ public void start() { + _knownNodes.start(); + _tracker.start(); + PersistDHT.loadDHT(this, _dhtFile); // start the explore thread + _isRunning = true; + // no need to keep ref, it will eventually stop + new Cleaner(); } /** * Stop everything. */ public void stop() { + _isRunning = false; // FIXME stop the explore thread // unregister port listeners _session.removeListener(I2PSession.PROTO_DATAGRAM, _qPort); _session.removeListener(I2PSession.PROTO_DATAGRAM_RAW, _rPort); // clear the DHT and tracker _tracker.stop(); - _knownNodes.clear(); + PersistDHT.saveDHT(_knownNodes, _dhtFile); + _knownNodes.stop(); _sentQueries.clear(); _outgoingTokens.clear(); _incomingTokens.clear(); @@ -1175,6 +1186,7 @@ public class KRPC implements I2PSessionMuxedListener, DHT { for (BEValue bev : peers) { byte[] b = bev.getBytes(); Hash h = new Hash(b); + //Hash h = Hash.create(b); rv.add(h); } if (_log.shouldLog(Log.INFO)) @@ -1303,7 +1315,7 @@ public class KRPC implements I2PSessionMuxedListener, DHT { if (onTimeout != null) onTimeout.run(); if (_log.shouldLog(Log.INFO)) - _log.warn("timeout waiting for reply from " + this.toString()); + _log.warn("timeout waiting for reply from " + ReplyWaiter.this.toString()); } } } @@ -1370,9 +1382,15 @@ public class KRPC implements I2PSessionMuxedListener, DHT { /** * Cleaner-upper */ - private class Cleaner implements SimpleTimer.TimedEvent { + private class Cleaner extends SimpleTimer2.TimedEvent { + + public Cleaner() { + super(SimpleTimer2.getInstance(), CLEAN_TIME); + } public void timeReached() { + if (!_isRunning) + return; long now = _context.clock().now(); for (Iterator iter = _outgoingTokens.keySet().iterator(); iter.hasNext(); ) { Token tok = iter.next(); @@ -1390,12 +1408,13 @@ public class KRPC implements I2PSessionMuxedListener, DHT { if (ni.lastSeen() < now - MAX_NODEINFO_AGE) iter.remove(); } - if (_log.shouldLog(Log.INFO)) - _log.info("KRPC cleaner done, now with " + + if (_log.shouldLog(Log.DEBUG)) + _log.debug("KRPC cleaner done, now with " + _outgoingTokens.size() + " sent Tokens, " + _incomingTokens.size() + " rcvd Tokens, " + _knownNodes.size() + " known peers, " + _sentQueries.size() + " queries awaiting response"); + schedule(CLEAN_TIME); } } } diff --git a/apps/i2psnark/java/src/org/klomp/snark/dht/NodeInfo.java b/apps/i2psnark/java/src/org/klomp/snark/dht/NodeInfo.java index 1d82b268a4..e4f52091bf 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/dht/NodeInfo.java +++ b/apps/i2psnark/java/src/org/klomp/snark/dht/NodeInfo.java @@ -3,6 +3,8 @@ package org.klomp.snark.dht; * From zzzot, modded and relicensed to GPLv2 */ +import net.i2p.data.Base64; +import net.i2p.data.DataFormatException; import net.i2p.data.DataHelper; import net.i2p.data.Destination; import net.i2p.data.Hash; @@ -41,7 +43,7 @@ class NodeInfo extends SimpleDataStructure { this.dest = dest; this.hash = dest.calculateHash(); this.port = port; - initialize(nID, this.hash, port); + initialize(); } /** @@ -53,7 +55,7 @@ class NodeInfo extends SimpleDataStructure { this.nID = nID; this.hash = hash; this.port = port; - initialize(nID, hash, port); + initialize(); } /** @@ -81,6 +83,36 @@ class NodeInfo extends SimpleDataStructure { initialize(d); } + /** + * Form persistent storage string. + * Format: NID:Hash:Destination:port + * First 3 in base 64; Destination may be empty string + * @throws IllegalArgumentException + */ + public NodeInfo(String s) throws DataFormatException { + super(); + String[] parts = s.split(":", 4); + if (parts.length != 4) + throw new DataFormatException("Bad format"); + byte[] nid = Base64.decode(parts[0]); + if (nid == null) + throw new DataFormatException("Bad NID"); + nID = new NID(nid); + byte[] h = Base64.decode(parts[1]); + if (h == null) + throw new DataFormatException("Bad hash"); + hash = new Hash(h); + //hash = Hash.create(h); + if (parts[2].length() > 0) + dest = new Destination(parts[2]); + try { + port = Integer.parseInt(parts[3]); + } catch (NumberFormatException nfe) { + throw new DataFormatException("Bad port", nfe); + } + initialize(); + } + /** * Creates data structures from the compact info * @throws IllegalArgumentException @@ -91,18 +123,22 @@ class NodeInfo extends SimpleDataStructure { byte[] ndata = new byte[NID.HASH_LENGTH]; System.arraycopy(compactInfo, 0, ndata, 0, NID.HASH_LENGTH); this.nID = new NID(ndata); + //3 lines or... byte[] hdata = new byte[Hash.HASH_LENGTH]; System.arraycopy(compactInfo, NID.HASH_LENGTH, hdata, 0, Hash.HASH_LENGTH); this.hash = new Hash(hdata); + //this.hash = Hash.create(compactInfo, NID.HASH_LENGTH); this.port = (int) DataHelper.fromLong(compactInfo, NID.HASH_LENGTH + Hash.HASH_LENGTH, 2); + if (port <= 0 || port >= 65535) + throw new IllegalArgumentException("Bad port"); } /** * Creates 54-byte compact info * @throws IllegalArgumentException */ - private void initialize(NID nID, Hash hash, int port) { - if (port < 0 || port > 65535) + private void initialize() { + if (port <= 0 || port >= 65535) throw new IllegalArgumentException("Bad port"); byte[] compactInfo = new byte[LENGTH]; System.arraycopy(nID.getData(), 0, compactInfo, 0, NID.HASH_LENGTH); @@ -173,7 +209,24 @@ class NodeInfo extends SimpleDataStructure { } } + @Override public String toString() { return "NodeInfo: " + nID + ' ' + hash + " port: " + port; } + + /** + * To persistent storage string. + * Format: NID:Hash:Destination:port + * First 3 in base 64; Destination may be empty string + */ + public String toPersistentString() { + StringBuilder buf = new StringBuilder(650); + buf.append(nID.toBase64()).append(':'); + buf.append(hash.toBase64()).append(':'); + if (dest != null) + buf.append(dest.toBase64()); + buf.append(':').append(port); + return buf.toString(); + } + } diff --git a/apps/i2psnark/java/src/org/klomp/snark/dht/PersistDHT.java b/apps/i2psnark/java/src/org/klomp/snark/dht/PersistDHT.java new file mode 100644 index 0000000000..dde495a04d --- /dev/null +++ b/apps/i2psnark/java/src/org/klomp/snark/dht/PersistDHT.java @@ -0,0 +1,77 @@ +package org.klomp.snark.dht; + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileInputStream; +import java.io.InputStreamReader; +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.io.PrintWriter; + +import net.i2p.I2PAppContext; +import net.i2p.data.DataFormatException; +import net.i2p.util.Log; +import net.i2p.util.SecureFileOutputStream; + +/** + * Retrieve / Store the local DHT in a file + * + */ +abstract class PersistDHT { + + public static synchronized void loadDHT(KRPC krpc, File file) { + Log log = I2PAppContext.getGlobalContext().logManager().getLog(PersistDHT.class); + int count = 0; + FileInputStream in = null; + try { + in = new FileInputStream(file); + BufferedReader br = new BufferedReader(new InputStreamReader(in, "ISO-8859-1")); + String line = null; + while ( (line = br.readLine()) != null) { + if (line.startsWith("#")) + continue; + try { + krpc.addNode(new NodeInfo(line)); + count++; + // TODO limit number? this will flush the router's SDS caches + } catch (IllegalArgumentException iae) { + if (log.shouldLog(Log.WARN)) + log.warn("Error reading DHT entry", iae); + } catch (DataFormatException dfe) { + if (log.shouldLog(Log.WARN)) + log.warn("Error reading DHT entry", dfe); + } + } + } catch (IOException ioe) { + if (log.shouldLog(Log.WARN) && file.exists()) + log.warn("Error reading the DHT File", ioe); + } finally { + if (in != null) try { in.close(); } catch (IOException ioe) {} + } + if (log.shouldLog(Log.INFO)) + log.info("Loaded " + count + " nodes from " + file); + } + + public static synchronized void saveDHT(DHTNodes nodes, File file) { + Log log = I2PAppContext.getGlobalContext().logManager().getLog(PersistDHT.class); + int count = 0; + PrintWriter out = null; + try { + out = new PrintWriter(new BufferedWriter(new OutputStreamWriter(new SecureFileOutputStream(file), "ISO-8859-1"))); + out.println("# DHT nodes, format is NID:Hash:Destination:port"); + for (NodeInfo ni : nodes.values()) { + // DHTNodes shouldn't contain us, if that changes check here + out.println(ni.toPersistentString()); + count++; + } + } catch (IOException ioe) { + if (log.shouldLog(Log.WARN)) + log.warn("Error writing the DHT File", ioe); + } finally { + if (out != null) out.close(); + } + if (log.shouldLog(Log.INFO)) + log.info("Stored " + count + " nodes to " + file); + } +} From 6a1b90f8f83c22d2c6841fbdddbc8ac4bbf50ee5 Mon Sep 17 00:00:00 2001 From: zzz Date: Tue, 5 Jun 2012 01:03:39 +0000 Subject: [PATCH 9/9] hash caching --- .../java/src/org/klomp/snark/I2PSnarkUtil.java | 4 ++-- .../i2psnark/java/src/org/klomp/snark/dht/KRPC.java | 7 ++++--- .../java/src/org/klomp/snark/dht/NodeInfo.java | 13 ++++++------- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/apps/i2psnark/java/src/org/klomp/snark/I2PSnarkUtil.java b/apps/i2psnark/java/src/org/klomp/snark/I2PSnarkUtil.java index 0d71dcd346..f44a30bf16 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/I2PSnarkUtil.java +++ b/apps/i2psnark/java/src/org/klomp/snark/I2PSnarkUtil.java @@ -405,8 +405,8 @@ public class I2PSnarkUtil { if (sess != null) { byte[] b = Base32.decode(ip.substring(0, BASE32_HASH_LENGTH)); if (b != null) { - Hash h = new Hash(b); - //Hash h = Hash.create(b); + //Hash h = new Hash(b); + Hash h = Hash.create(b); if (_log.shouldLog(Log.INFO)) _log.info("Using existing session for lookup of " + ip); try { diff --git a/apps/i2psnark/java/src/org/klomp/snark/dht/KRPC.java b/apps/i2psnark/java/src/org/klomp/snark/dht/KRPC.java index 2c44deb68f..0ee7994111 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/dht/KRPC.java +++ b/apps/i2psnark/java/src/org/klomp/snark/dht/KRPC.java @@ -393,7 +393,8 @@ public class KRPC implements I2PSessionMuxedListener, DHT { public void announce(byte[] ih, byte[] peerHash) { InfoHash iHash = new InfoHash(ih); _tracker.announce(iHash, new Hash(peerHash)); -// _tracker.announce(iHash, Hash.create(peerHash)); + // Do NOT do this, corrupts the Hash cache and the Peer ID + //_tracker.announce(iHash, Hash.create(peerHash)); } /** @@ -1185,8 +1186,8 @@ public class KRPC implements I2PSessionMuxedListener, DHT { List rv = new ArrayList(peers.size()); for (BEValue bev : peers) { byte[] b = bev.getBytes(); - Hash h = new Hash(b); - //Hash h = Hash.create(b); + //Hash h = new Hash(b); + Hash h = Hash.create(b); rv.add(h); } if (_log.shouldLog(Log.INFO)) diff --git a/apps/i2psnark/java/src/org/klomp/snark/dht/NodeInfo.java b/apps/i2psnark/java/src/org/klomp/snark/dht/NodeInfo.java index e4f52091bf..8ff21efc41 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/dht/NodeInfo.java +++ b/apps/i2psnark/java/src/org/klomp/snark/dht/NodeInfo.java @@ -101,8 +101,8 @@ class NodeInfo extends SimpleDataStructure { byte[] h = Base64.decode(parts[1]); if (h == null) throw new DataFormatException("Bad hash"); - hash = new Hash(h); - //hash = Hash.create(h); + //hash = new Hash(h); + hash = Hash.create(h); if (parts[2].length() > 0) dest = new Destination(parts[2]); try { @@ -123,11 +123,10 @@ class NodeInfo extends SimpleDataStructure { byte[] ndata = new byte[NID.HASH_LENGTH]; System.arraycopy(compactInfo, 0, ndata, 0, NID.HASH_LENGTH); this.nID = new NID(ndata); - //3 lines or... - byte[] hdata = new byte[Hash.HASH_LENGTH]; - System.arraycopy(compactInfo, NID.HASH_LENGTH, hdata, 0, Hash.HASH_LENGTH); - this.hash = new Hash(hdata); - //this.hash = Hash.create(compactInfo, NID.HASH_LENGTH); + //byte[] hdata = new byte[Hash.HASH_LENGTH]; + //System.arraycopy(compactInfo, NID.HASH_LENGTH, hdata, 0, Hash.HASH_LENGTH); + //this.hash = new Hash(hdata); + this.hash = Hash.create(compactInfo, NID.HASH_LENGTH); this.port = (int) DataHelper.fromLong(compactInfo, NID.HASH_LENGTH + Hash.HASH_LENGTH, 2); if (port <= 0 || port >= 65535) throw new IllegalArgumentException("Bad port");