From afe57512abf92c486f36740fc88d3f52d5b2c7f6 Mon Sep 17 00:00:00 2001 From: zzz Date: Sun, 19 Dec 2010 15:37:11 +0000 Subject: [PATCH 01/27] PORT msg support; port and extension listener stubs --- .../java/src/org/klomp/snark/Message.java | 7 +++++ .../java/src/org/klomp/snark/Peer.java | 18 ++++++++++--- .../src/org/klomp/snark/PeerConnectionIn.java | 5 ++++ .../org/klomp/snark/PeerConnectionOut.java | 8 ++++++ .../src/org/klomp/snark/PeerCoordinator.java | 6 +++++ .../src/org/klomp/snark/PeerListener.java | 19 ++++++++++++++ .../java/src/org/klomp/snark/PeerState.java | 26 +++++-------------- 7 files changed, 67 insertions(+), 22 deletions(-) diff --git a/apps/i2psnark/java/src/org/klomp/snark/Message.java b/apps/i2psnark/java/src/org/klomp/snark/Message.java index a9d1e23f25..57fecf43d9 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/Message.java +++ b/apps/i2psnark/java/src/org/klomp/snark/Message.java @@ -53,6 +53,7 @@ class Message // Used for HAVE, REQUEST, PIECE and CANCEL messages. // low byte used for EXTENSION message + // low two bytes used for PORT message int piece; // Used for REQUEST, PIECE and CANCEL messages. @@ -107,6 +108,9 @@ class Message if (type == EXTENSION) datalen += 1; + if (type == PORT) + datalen += 2; + // add length of data for piece or bitfield array. if (type == BITFIELD || type == PIECE || type == EXTENSION) datalen += len; @@ -130,6 +134,9 @@ class Message if (type == EXTENSION) dos.writeByte((byte) piece & 0xff); + if (type == PORT) + dos.writeShort(piece & 0xffff); + // Send actual data if (type == BITFIELD || type == PIECE || type == EXTENSION) dos.write(data, off, len); diff --git a/apps/i2psnark/java/src/org/klomp/snark/Peer.java b/apps/i2psnark/java/src/org/klomp/snark/Peer.java index ae053b8c8a..f42adddeff 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/Peer.java +++ b/apps/i2psnark/java/src/org/klomp/snark/Peer.java @@ -72,7 +72,6 @@ public class Peer implements Comparable * relevant MetaInfo. */ public Peer(PeerID peerID, byte[] my_id, MetaInfo metainfo) - throws IOException { this.peerID = peerID; this.my_id = my_id; @@ -253,6 +252,13 @@ public class Peer implements Comparable out.sendExtension(0, ExtensionHandshake.getPayload()); } + if ((options & OPTION_DHT) != 0 && util.getDHT() != null) { + if (_log.shouldLog(Log.DEBUG)) + _log.debug("Peer supports DHT, sending PORT message"); + int port = util.getDHT().getPort(); + out.sendPort(port); + } + // Send our bitmap if (bitfield != null) s.out.sendBitfield(bitfield); @@ -303,7 +309,8 @@ public class Peer implements Comparable dout.write(19); dout.write("BitTorrent protocol".getBytes("UTF-8")); // Handshake write - options - dout.writeLong(OPTION_EXTENSION); + // FIXME not if DHT disabled + dout.writeLong(OPTION_EXTENSION | OPTION_DHT); // Handshake write - metainfo hash byte[] shared_hash = metainfo.getInfoHash(); dout.write(shared_hash); @@ -343,7 +350,7 @@ public class Peer implements Comparable _log.debug("Read the remote side's hash and peerID fully from " + toString()); if (options != 0) { - // send them something + // send them something in runConnection() above if (_log.shouldLog(Log.DEBUG)) _log.debug("Peer supports options 0x" + Long.toString(options, 16) + ": " + toString()); } @@ -351,6 +358,11 @@ public class Peer implements Comparable return bs; } + /** @since 0.8.4 */ + public long getOptions() { + return options; + } + public boolean isConnected() { return state != null; diff --git a/apps/i2psnark/java/src/org/klomp/snark/PeerConnectionIn.java b/apps/i2psnark/java/src/org/klomp/snark/PeerConnectionIn.java index 43bb9a7122..90755a9371 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/PeerConnectionIn.java +++ b/apps/i2psnark/java/src/org/klomp/snark/PeerConnectionIn.java @@ -171,6 +171,11 @@ class PeerConnectionIn implements Runnable if (_log.shouldLog(Log.DEBUG)) _log.debug("Received cancel(" + piece + "," + begin + ") from " + peer + " on " + peer.metainfo.getName()); break; + case 9: // PORT message + int port = din.readUnsignedShort(); + ps.portMessage(port); + if (_log.shouldLog(Log.DEBUG)) + _log.debug("Received port message from " + peer + " on " + peer.metainfo.getName()); case 20: // Extension message int id = din.readUnsignedByte(); byte[] payload = new byte[i-2]; diff --git a/apps/i2psnark/java/src/org/klomp/snark/PeerConnectionOut.java b/apps/i2psnark/java/src/org/klomp/snark/PeerConnectionOut.java index 38cb29e3b4..19614bc52e 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/PeerConnectionOut.java +++ b/apps/i2psnark/java/src/org/klomp/snark/PeerConnectionOut.java @@ -547,4 +547,12 @@ class PeerConnectionOut implements Runnable m.len = bytes.length; addMessage(m); } + + /** @since 0.8.4 */ + void sendPort(int port) { + Message m = new Message(); + m.type = Message.PORT; + m.piece = port; + addMessage(m); + } } diff --git a/apps/i2psnark/java/src/org/klomp/snark/PeerCoordinator.java b/apps/i2psnark/java/src/org/klomp/snark/PeerCoordinator.java index 034f74ae71..c9cdf61419 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/PeerCoordinator.java +++ b/apps/i2psnark/java/src/org/klomp/snark/PeerCoordinator.java @@ -1086,6 +1086,12 @@ public class PeerCoordinator implements PeerListener } } + /** @since 0.8.4 */ + public void gotExtension(Peer peer, int id, byte[] bs) {} + + /** @since 0.8.4 */ + public void gotPort(Peer peer, int port) {} + /** Return number of allowed uploaders for this torrent. ** Check with Snark to see if we are over the total upload limit. */ diff --git a/apps/i2psnark/java/src/org/klomp/snark/PeerListener.java b/apps/i2psnark/java/src/org/klomp/snark/PeerListener.java index 975c12c106..6d9e681cb6 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/PeerListener.java +++ b/apps/i2psnark/java/src/org/klomp/snark/PeerListener.java @@ -179,4 +179,23 @@ interface PeerListener * @since 0.8.2 */ PartialPiece getPartialPiece(Peer peer, BitField havePieces); + + /** + * Called when an extension message is received. + * + * @param peer the Peer that got the message. + * @param id the message ID + * @param bs the message payload + * @since 0.8.4 + */ + void gotExtension(Peer peer, int id, byte[] bs); + + /** + * Called when an extension message is received. + * + * @param peer the Peer that got the message. + * @param port the port + * @since 0.8.4 + */ + void gotPort(Peer peer, int port); } diff --git a/apps/i2psnark/java/src/org/klomp/snark/PeerState.java b/apps/i2psnark/java/src/org/klomp/snark/PeerState.java index 0c3c9eee88..ad70df9dc6 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/PeerState.java +++ b/apps/i2psnark/java/src/org/klomp/snark/PeerState.java @@ -32,9 +32,6 @@ import java.util.Set; import net.i2p.I2PAppContext; import net.i2p.util.Log; -import org.klomp.snark.bencode.BDecoder; -import org.klomp.snark.bencode.BEValue; - class PeerState implements DataLoader { private final Log _log = I2PAppContext.getGlobalContext().logManager().getLog(PeerState.class); @@ -485,22 +482,13 @@ class PeerState implements DataLoader /** @since 0.8.2 */ void extensionMessage(int id, byte[] bs) { - if (id == 0) { - InputStream is = new ByteArrayInputStream(bs); - try { - BDecoder dec = new BDecoder(is); - BEValue bev = dec.bdecodeMap(); - Map map = bev.getMap(); - if (_log.shouldLog(Log.DEBUG)) - _log.debug("Got extension handshake message " + bev.toString()); - } catch (Exception e) { - if (_log.shouldLog(Log.DEBUG)) - _log.debug("Failed extension decode", e); - } - } else { - if (_log.shouldLog(Log.DEBUG)) - _log.debug("Got extended message type: " + id + " length: " + bs.length); - } + listener.gotExtension(peer, id, bs); + } + + /** @since 0.8.4 */ + void portMessage(int port) + { + listener.gotPort(peer, port); } void unknownMessage(int type, byte[] bs) From 4899a6d306d644fd880d3a062b5fd720c8eaefe3 Mon Sep 17 00:00:00 2001 From: zzz Date: Sun, 19 Dec 2010 22:14:02 +0000 Subject: [PATCH 02/27] Refactor fields to private and replace with getters, and lots of prep for lack of metainfo and storage. --- .../src/org/klomp/snark/PeerAcceptor.java | 2 +- .../src/org/klomp/snark/PeerCheckerTask.java | 3 - .../src/org/klomp/snark/PeerCoordinator.java | 58 +++- .../java/src/org/klomp/snark/Snark.java | 263 ++++++++++++++++-- .../src/org/klomp/snark/SnarkManager.java | 73 +++-- .../java/src/org/klomp/snark/StaticSnark.java | 3 +- .../src/org/klomp/snark/TrackerClient.java | 30 +- .../org/klomp/snark/web/I2PSnarkServlet.java | 140 +++++----- 8 files changed, 409 insertions(+), 163 deletions(-) diff --git a/apps/i2psnark/java/src/org/klomp/snark/PeerAcceptor.java b/apps/i2psnark/java/src/org/klomp/snark/PeerAcceptor.java index 58ef3ae2db..050884e5c0 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/PeerAcceptor.java +++ b/apps/i2psnark/java/src/org/klomp/snark/PeerAcceptor.java @@ -120,7 +120,7 @@ public class PeerAcceptor else { if (_log.shouldLog(Log.DEBUG)) - _log.debug("Rejecting new peer for " + cur.snark.torrent); + _log.debug("Rejecting new peer for " + cur.getName()); socket.close(); return; } diff --git a/apps/i2psnark/java/src/org/klomp/snark/PeerCheckerTask.java b/apps/i2psnark/java/src/org/klomp/snark/PeerCheckerTask.java index aa9cf2187d..d6e2c60895 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/PeerCheckerTask.java +++ b/apps/i2psnark/java/src/org/klomp/snark/PeerCheckerTask.java @@ -51,10 +51,7 @@ class PeerCheckerTask extends TimerTask { List peerList = coordinator.peerList(); if (peerList.isEmpty() || coordinator.halted()) { - coordinator.peerCount = 0; - coordinator.interestedAndChoking = 0; coordinator.setRateHistory(0, 0); - coordinator.uploaders = 0; if (coordinator.halted()) cancel(); return; diff --git a/apps/i2psnark/java/src/org/klomp/snark/PeerCoordinator.java b/apps/i2psnark/java/src/org/klomp/snark/PeerCoordinator.java index c9cdf61419..ba9727d8a6 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/PeerCoordinator.java +++ b/apps/i2psnark/java/src/org/klomp/snark/PeerCoordinator.java @@ -42,18 +42,33 @@ import net.i2p.util.SimpleTimer2; public class PeerCoordinator implements PeerListener { private final Log _log = I2PAppContext.getGlobalContext().logManager().getLog(PeerCoordinator.class); + + /** + * External use by PeerMonitorTask only. + */ final MetaInfo metainfo; + + /** + * External use by PeerMonitorTask only. + */ final Storage storage; - final Snark snark; + private final Snark snark; // package local for access by CheckDownLoadersTask final static long CHECK_PERIOD = 40*1000; // 40 seconds final static int MAX_UPLOADERS = 6; - // Approximation of the number of current uploaders. - // Resynced by PeerChecker once in a while. - int uploaders = 0; - int interestedAndChoking = 0; + /** + * Approximation of the number of current uploaders. + * Resynced by PeerChecker once in a while. + * External use by PeerCheckerTask only. + */ + int uploaders; + + /** + * External use by PeerCheckerTask only. + */ + int interestedAndChoking; // final static int MAX_DOWNLOADERS = MAX_CONNECTIONS; // int downloaders = 0; @@ -61,14 +76,18 @@ public class PeerCoordinator implements PeerListener private long uploaded; private long downloaded; final static int RATE_DEPTH = 3; // make following arrays RATE_DEPTH long - private long uploaded_old[] = {-1,-1,-1}; - private long downloaded_old[] = {-1,-1,-1}; + private final long uploaded_old[] = {-1,-1,-1}; + private final long downloaded_old[] = {-1,-1,-1}; - // synchronize on this when changing peers or downloaders - // This is a Queue, not a Set, because PeerCheckerTask keeps things in order for choking/unchoking + /** + * synchronize on this when changing peers or downloaders. + * This is a Queue, not a Set, because PeerCheckerTask keeps things in order for choking/unchoking. + * External use by PeerMonitorTask only. + */ final Queue peers; + /** estimate of the peers, without requiring any synchronization */ - volatile int peerCount; + private volatile int peerCount; /** Timer to handle all periodical tasks. */ private final CheckEvent timer; @@ -86,12 +105,9 @@ public class PeerCoordinator implements PeerListener private boolean halted = false; private final CoordinatorListener listener; - public I2PSnarkUtil _util; + private final I2PSnarkUtil _util; private static final Random _random = I2PAppContext.getGlobalContext().random(); - public String trackerProblems = null; - public int trackerSeenPeers = 0; - public PeerCoordinator(I2PSnarkUtil util, byte[] id, MetaInfo metainfo, Storage storage, CoordinatorListener listener, Snark torrent) { @@ -153,7 +169,6 @@ public class PeerCoordinator implements PeerListener } public Storage getStorage() { return storage; } - public CoordinatorListener getListener() { return listener; } // for web page detailed stats public List peerList() @@ -166,6 +181,11 @@ public class PeerCoordinator implements PeerListener return id; } + public String getName() + { + return snark.getName(); + } + public boolean completed() { return storage.complete(); @@ -1107,6 +1127,14 @@ public class PeerCoordinator implements PeerListener return MAX_UPLOADERS; } + /** + * @return current + * @since 0.8.4 + */ + public int getUploaders() { + return uploaders; + } + public boolean overUpBWLimit() { if (listener != null) diff --git a/apps/i2psnark/java/src/org/klomp/snark/Snark.java b/apps/i2psnark/java/src/org/klomp/snark/Snark.java index 31105a2693..79807d68f2 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/Snark.java +++ b/apps/i2psnark/java/src/org/klomp/snark/Snark.java @@ -26,6 +26,7 @@ import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; +import java.util.Collections; import java.util.Iterator; import java.util.List; import java.util.Properties; @@ -112,6 +113,8 @@ public class Snark } +/******** No, not maintaining a command-line client + public static void main(String[] args) { System.out.println(copyright); @@ -235,19 +238,27 @@ public class Snark } } +***********/ + public static final String PROP_MAX_CONNECTIONS = "i2psnark.maxConnections"; - public String torrent; - public MetaInfo meta; - public Storage storage; - public PeerCoordinator coordinator; - public ConnectionAcceptor acceptor; - public TrackerClient trackerclient; - public String rootDataDir = "."; - public CompleteListener completeListener; - public boolean stopped; - byte[] id; - public I2PSnarkUtil _util; - private PeerCoordinatorSet _peerCoordinatorSet; + + /** most of these used to be public, use accessors below instead */ + private final String torrent; + private MetaInfo meta; + private Storage storage; + private PeerCoordinator coordinator; + private ConnectionAcceptor acceptor; + private TrackerClient trackerclient; + private String rootDataDir = "."; + private final CompleteListener completeListener; + private boolean stopped; + private byte[] id; + private byte[] infoHash; + private final I2PSnarkUtil _util; + private final PeerCoordinatorSet _peerCoordinatorSet; + private String trackerProblems; + private int trackerSeenPeers; + /** from main() via parseArguments() single torrent */ Snark(I2PSnarkUtil util, String torrent, String ip, int user_port, @@ -486,7 +497,7 @@ public class Snark // single torrent acceptor = new ConnectionAcceptor(_util, serversocket, new PeerAcceptor(coordinator)); } - trackerclient = new TrackerClient(_util, meta, coordinator); + trackerclient = new TrackerClient(_util, meta, coordinator, this); } stopped = false; @@ -496,8 +507,7 @@ public class Snark // restart safely, so lets build a new one to replace the old if (_peerCoordinatorSet != null) _peerCoordinatorSet.remove(coordinator); - PeerCoordinator newCoord = new PeerCoordinator(_util, coordinator.getID(), coordinator.getMetaInfo(), - coordinator.getStorage(), coordinator.getListener(), this); + PeerCoordinator newCoord = new PeerCoordinator(_util, id, meta, storage, this, this); if (_peerCoordinatorSet != null) _peerCoordinatorSet.add(newCoord); coordinator = newCoord; @@ -517,7 +527,7 @@ public class Snark } fatal("Could not reopen storage", ioe); } - TrackerClient newClient = new TrackerClient(_util, coordinator.getMetaInfo(), coordinator); + TrackerClient newClient = new TrackerClient(_util, coordinator.getMetaInfo(), coordinator, this); if (!trackerclient.halted()) trackerclient.halt(); trackerclient = newClient; @@ -553,18 +563,223 @@ public class Snark _util.disconnect(); } - static Snark parseArguments(String[] args) + private static Snark parseArguments(String[] args) { return parseArguments(args, null, null); } + // Accessors + + /** + * @return file name of .torrent file (should be full absolute path), or a fake name if in magnet mode. + * @since 0.8.4 + */ + public String getName() { + return torrent; + } + + /** + * @return base name of torrent [filtered version of getMetaInfo.getName()], or a fake name if in magnet mode + * @since 0.8.4 + */ + public String getBaseName() { + if (storage != null) + return storage.getBaseName(); + return torrent; + } + + /** + * @return always will be valid even in magnet mode + * @since 0.8.4 + */ + public byte[] getID() { + return id; + } + + /** + * @return always will be valid even in magnet mode + * @since 0.8.4 + */ + public byte[] getInfoHash() { + if (meta != null) + return meta.getInfoHash(); + return infoHash; + } + + /** + * @return may be null if in magnet mode + * @since 0.8.4 + */ + public MetaInfo getMetaInfo() { + return meta; + } + + /** + * @return may be null if in magnet mode + * @since 0.8.4 + */ + public Storage getStorage() { + return storage; + } + + /** + * @since 0.8.4 + */ + public boolean isStopped() { + return stopped; + } + + /** + * @since 0.8.4 + */ + public long getDownloadRate() { + PeerCoordinator coord = coordinator; + if (coord != null) + return coord.getDownloadRate(); + return 0; + } + + /** + * @since 0.8.4 + */ + public long getUploadRate() { + PeerCoordinator coord = coordinator; + if (coord != null) + return coord.getUploadRate(); + return 0; + } + + /** + * @since 0.8.4 + */ + public long getDownloaded() { + PeerCoordinator coord = coordinator; + if (coord != null) + return coord.getDownloaded(); + return 0; + } + + /** + * @since 0.8.4 + */ + public long getUploaded() { + PeerCoordinator coord = coordinator; + if (coord != null) + return coord.getUploaded(); + return 0; + } + + /** + * @since 0.8.4 + */ + public int getPeerCount() { + PeerCoordinator coord = coordinator; + if (coord != null) + return coord.getPeerCount(); + return 0; + } + + /** + * @since 0.8.4 + */ + public List getPeerList() { + PeerCoordinator coord = coordinator; + if (coord != null) + return coord.peerList(); + return Collections.EMPTY_LIST; + } + + /** + * @return String returned from tracker, or null if no error + * @since 0.8.4 + */ + public String getTrackerProblems() { + return trackerProblems; + } + + /** + * @param p tracker error string or null + * @since 0.8.4 + */ + public void setTrackerProblems(String p) { + trackerProblems = p; + } + + /** + * @return count returned from tracker + * @since 0.8.4 + */ + public int getTrackerSeenPeers() { + return trackerSeenPeers; + } + + /** + * @since 0.8.4 + */ + public void setTrackerSeenPeers(int p) { + trackerSeenPeers = p; + } + + /** + * @since 0.8.4 + */ + public void updatePiecePriorities() { + PeerCoordinator coord = coordinator; + if (coord != null) + coord.updatePiecePriorities(); + } + + /** + * @return total of all torrent files, or total of metainfo file if fetching magnet, or -1 + * @since 0.8.4 + */ + public long getTotalLength() { + if (meta != null) + return meta.getTotalLength(); + // FIXME else return metainfo length if available + return -1; + } + + /** + * @return needed of all torrent files, or total of metainfo file if fetching magnet, or -1 + * @since 0.8.4 + */ + public long getNeeded() { + if (storage != null) + return storage.needed(); + // FIXME else return metainfo length if available + return -1; + } + + /** + * @param p the piece number + * @return metainfo piece length or 16K if fetching magnet + * @since 0.8.4 + */ + public int getPieceLength(int p) { + if (meta != null) + return meta.getPieceLength(p); + return 16*1024; + } + + /** + * @return true if restarted + * @since 0.8.4 + */ + public boolean restartAcceptor() { + if (acceptor == null) + return false; + acceptor.restart(); + return true; + } + /** * Sets debug, ip and torrent variables then creates a Snark * instance. Calls usage(), which terminates the program, if * non-valid argument list. The given listeners will be * passed to all components that take one. */ - static Snark parseArguments(String[] args, + private static Snark parseArguments(String[] args, StorageListener slistener, CoordinatorListener clistener) { @@ -719,7 +934,7 @@ public class Snark /** * Aborts program abnormally. */ - public void fatal(String s) + private void fatal(String s) { fatal(s, null); } @@ -727,7 +942,7 @@ public class Snark /** * Aborts program abnormally. */ - public void fatal(String s, Throwable t) + private void fatal(String s, Throwable t) { _util.debug(s, ERROR, t); //System.err.println("snark: " + s + ((t == null) ? "" : (": " + t))); @@ -751,7 +966,7 @@ public class Snark // System.out.println(peer.toString()); } - boolean allocating = false; + private boolean allocating = false; public void storageCreateFile(Storage storage, String name, long length) { //if (allocating) @@ -774,9 +989,9 @@ public class Snark // System.out.println(); // We have all the disk space we need. } - boolean allChecked = false; - boolean checking = false; - boolean prechecking = true; + private boolean allChecked = false; + private boolean checking = false; + private boolean prechecking = true; public void storageChecked(Storage storage, int num, boolean checked) { allocating = false; diff --git a/apps/i2psnark/java/src/org/klomp/snark/SnarkManager.java b/apps/i2psnark/java/src/org/klomp/snark/SnarkManager.java index 82d1d80f4f..6a4eb4c7c5 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/SnarkManager.java +++ b/apps/i2psnark/java/src/org/klomp/snark/SnarkManager.java @@ -359,7 +359,7 @@ public class SnarkManager implements Snark.CompleteListener { Set names = listTorrentFiles(); for (Iterator iter = names.iterator(); iter.hasNext(); ) { Snark snark = getTorrent((String)iter.next()); - if ( (snark != null) && (!snark.stopped) ) { + if ( (snark != null) && (!snark.isStopped()) ) { snarksActive = true; break; } @@ -398,9 +398,8 @@ public class SnarkManager implements Snark.CompleteListener { for (Iterator iter = names.iterator(); iter.hasNext(); ) { String name = (String)iter.next(); Snark snark = getTorrent(name); - if ( (snark != null) && (snark.acceptor != null) ) { - snark.acceptor.restart(); - addMessage(_("I2CP listener restarted for \"{0}\"", snark.meta.getName())); + if (snark != null && snark.restartAcceptor()) { + addMessage(_("I2CP listener restarted for \"{0}\"", snark.getBaseName())); } } } @@ -478,7 +477,7 @@ public class SnarkManager implements Snark.CompleteListener { public Snark getTorrentByBaseName(String filename) { synchronized (_snarks) { for (Snark s : _snarks.values()) { - if (s.storage.getBaseName().equals(filename)) + if (s.getBaseName().equals(filename)) return s; } } @@ -554,7 +553,6 @@ public class SnarkManager implements Snark.CompleteListener { _peerCoordinatorSet, _connectionAcceptor, false, dataDir.getPath()); loadSavedFilePriorities(torrent); - torrent.completeListener = this; synchronized (_snarks) { _snarks.put(filename, torrent); } @@ -572,12 +570,11 @@ public class SnarkManager implements Snark.CompleteListener { return; } // ok, snark created, now lets start it up or configure it further - File f = new File(filename); if (!dontAutoStart && shouldAutoStart()) { torrent.startTorrent(); - addMessage(_("Torrent added and started: \"{0}\"", torrent.storage.getBaseName())); + addMessage(_("Torrent added and started: \"{0}\"", torrent.getBaseName())); } else { - addMessage(_("Torrent added: \"{0}\"", torrent.storage.getBaseName())); + addMessage(_("Torrent added: \"{0}\"", torrent.getBaseName())); } } @@ -585,8 +582,7 @@ public class SnarkManager implements Snark.CompleteListener { * Get the timestamp for a torrent from the config file */ public long getSavedTorrentTime(Snark snark) { - MetaInfo metainfo = snark.meta; - byte[] ih = metainfo.getInfoHash(); + byte[] ih = snark.getInfoHash(); String infohash = Base64.encode(ih); infohash = infohash.replace('=', '$'); String time = _config.getProperty(PROP_META_PREFIX + infohash + PROP_META_BITFIELD_SUFFIX); @@ -605,8 +601,10 @@ public class SnarkManager implements Snark.CompleteListener { * Convert "." to a full bitfield. */ public BitField getSavedTorrentBitField(Snark snark) { - MetaInfo metainfo = snark.meta; - byte[] ih = metainfo.getInfoHash(); + MetaInfo metainfo = snark.getMetaInfo(); + if (metainfo == null) + return null; + byte[] ih = snark.getInfoHash(); String infohash = Base64.encode(ih); infohash = infohash.replace('=', '$'); String bf = _config.getProperty(PROP_META_PREFIX + infohash + PROP_META_BITFIELD_SUFFIX); @@ -636,10 +634,13 @@ public class SnarkManager implements Snark.CompleteListener { * @since 0.8.1 */ public void loadSavedFilePriorities(Snark snark) { - MetaInfo metainfo = snark.meta; + MetaInfo metainfo = snark.getMetaInfo(); + Storage storage = snark.getStorage(); + if (metainfo == null || storage == null) + return; if (metainfo.getFiles() == null) return; - byte[] ih = metainfo.getInfoHash(); + byte[] ih = snark.getInfoHash(); String infohash = Base64.encode(ih); infohash = infohash.replace('=', '$'); String pri = _config.getProperty(PROP_META_PREFIX + infohash + PROP_META_PRIORITY_SUFFIX); @@ -655,7 +656,7 @@ public class SnarkManager implements Snark.CompleteListener { } catch (Throwable t) {} } } - snark.storage.setFilePriorities(rv); + storage.setFilePriorities(rv); } /** @@ -777,21 +778,15 @@ public class SnarkManager implements Snark.CompleteListener { remaining = _snarks.size(); } if (torrent != null) { - boolean wasStopped = torrent.stopped; + boolean wasStopped = torrent.isStopped(); torrent.stopTorrent(); if (remaining == 0) { // should we disconnect/reconnect here (taking care to deal with the other thread's // I2PServerSocket.accept() call properly?) ////_util. } - String name; - if (torrent.storage != null) { - name = torrent.storage.getBaseName(); - } else { - name = sfile.getName(); - } if (!wasStopped) - addMessage(_("Torrent stopped: \"{0}\"", name)); + addMessage(_("Torrent stopped: \"{0}\"", torrent.getBaseName())); } return torrent; } @@ -804,14 +799,10 @@ public class SnarkManager implements Snark.CompleteListener { if (torrent != null) { File torrentFile = new File(filename); torrentFile.delete(); - String name; - if (torrent.storage != null) { - removeTorrentStatus(torrent.storage.getMetaInfo()); - name = torrent.storage.getBaseName(); - } else { - name = torrentFile.getName(); - } - addMessage(_("Torrent removed: \"{0}\"", name)); + Storage storage = torrent.getStorage(); + if (storage != null) + removeTorrentStatus(storage.getMetaInfo()); + addMessage(_("Torrent removed: \"{0}\"", torrent.getBaseName())); } } @@ -843,18 +834,24 @@ public class SnarkManager implements Snark.CompleteListener { /** two listeners */ public void torrentComplete(Snark snark) { + MetaInfo meta = snark.getMetaInfo(); + Storage storage = snark.getStorage(); + if (meta == null || storage == null) + return; StringBuilder buf = new StringBuilder(256); - buf.append("").append(snark.storage.getBaseName()).append(""); - long len = snark.meta.getTotalLength(); + buf.append("\">").append(storage.getBaseName()).append(""); addMessage(_("Download finished: {0}", buf.toString())); // + " (" + _("size: {0}B", DataHelper.formatSize2(len)) + ')'); updateStatus(snark); } public void updateStatus(Snark snark) { - saveTorrentStatus(snark.meta, snark.storage.getBitField(), snark.storage.getFilePriorities()); + MetaInfo meta = snark.getMetaInfo(); + Storage storage = snark.getStorage(); + if (meta != null && storage != null) + saveTorrentStatus(meta, storage.getBitField(), storage.getFilePriorities()); } private void monitorTorrents(File dir) { @@ -984,7 +981,7 @@ public class SnarkManager implements Snark.CompleteListener { Set names = listTorrentFiles(); for (Iterator iter = names.iterator(); iter.hasNext(); ) { Snark snark = getTorrent((String)iter.next()); - if ( (snark != null) && (!snark.stopped) ) + if ( (snark != null) && (!snark.isStopped()) ) snark.stopTorrent(); } } diff --git a/apps/i2psnark/java/src/org/klomp/snark/StaticSnark.java b/apps/i2psnark/java/src/org/klomp/snark/StaticSnark.java index 38b470a7c9..52bef12b47 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/StaticSnark.java +++ b/apps/i2psnark/java/src/org/klomp/snark/StaticSnark.java @@ -38,6 +38,7 @@ public class StaticSnark //Security.addProvider(gnu); // And finally call the normal starting point. - Snark.main(args); + //Snark.main(args); + System.err.println("unsupported"); } } diff --git a/apps/i2psnark/java/src/org/klomp/snark/TrackerClient.java b/apps/i2psnark/java/src/org/klomp/snark/TrackerClient.java index a1a3f4cc21..6bded21008 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/TrackerClient.java +++ b/apps/i2psnark/java/src/org/klomp/snark/TrackerClient.java @@ -34,9 +34,11 @@ import java.util.Random; import java.util.Set; import net.i2p.I2PAppContext; +import net.i2p.data.Hash; import net.i2p.util.I2PAppThread; import net.i2p.util.Log; + /** * Informs metainfo tracker of events and gets new peers for peer * coordinator. @@ -63,6 +65,7 @@ public class TrackerClient extends I2PAppThread private I2PSnarkUtil _util; private final MetaInfo meta; private final PeerCoordinator coordinator; + private final Snark snark; private final int port; private boolean stop; @@ -70,15 +73,16 @@ public class TrackerClient extends I2PAppThread private List trackers; - public TrackerClient(I2PSnarkUtil util, MetaInfo meta, PeerCoordinator coordinator) + public TrackerClient(I2PSnarkUtil util, MetaInfo meta, PeerCoordinator coordinator, Snark snark) { super(); // Set unique name. - String id = urlencode(coordinator.getID()); + String id = urlencode(snark.getID()); setName("TrackerClient " + id.substring(id.length() - 12)); _util = util; this.meta = meta; this.coordinator = coordinator; + this.snark = snark; this.port = 6881; //(port == -1) ? 9 : port; @@ -119,10 +123,10 @@ public class TrackerClient extends I2PAppThread public void run() { String infoHash = urlencode(meta.getInfoHash()); - String peerID = urlencode(coordinator.getID()); + String peerID = urlencode(snark.getID()); _log.debug("Announce: [" + meta.getAnnounce() + "] infoHash: " + infoHash); - + // Construct the list of trackers for this torrent, // starting with the primary one listed in the metainfo, // followed by the secondary open trackers @@ -200,7 +204,7 @@ public class TrackerClient extends I2PAppThread firstTime = false; } else if (completed && runStarted) delay = 3*SLEEP*60*1000 + random; - else if (coordinator.trackerProblems != null && ++consecutiveFails < MAX_CONSEC_FAILS) + else if (snark.getTrackerProblems() != null && ++consecutiveFails < MAX_CONSEC_FAILS) delay = INITIAL_SLEEP; else // sleep a while, when we wake up we will contact only the trackers whose intervals have passed @@ -251,7 +255,7 @@ public class TrackerClient extends I2PAppThread uploaded, downloaded, left, event); - coordinator.trackerProblems = null; + snark.setTrackerProblems(null); tr.trackerProblems = null; tr.registerFails = 0; tr.consecutiveFails = 0; @@ -262,8 +266,8 @@ public class TrackerClient extends I2PAppThread Set peers = info.getPeers(); tr.seenPeers = info.getPeerCount(); - if (coordinator.trackerSeenPeers < tr.seenPeers) // update rising number quickly - coordinator.trackerSeenPeers = tr.seenPeers; + if (snark.getTrackerSeenPeers() < tr.seenPeers) // update rising number quickly + snark.setTrackerSeenPeers(tr.seenPeers); if ( (left > 0) && (!completed) ) { // we only want to talk to new people if we need things // from them (duh) @@ -293,12 +297,12 @@ public class TrackerClient extends I2PAppThread tr.trackerProblems = ioe.getMessage(); // don't show secondary tracker problems to the user if (tr.isPrimary) - coordinator.trackerProblems = tr.trackerProblems; + snark.setTrackerProblems(tr.trackerProblems); if (tr.trackerProblems.toLowerCase().startsWith(NOT_REGISTERED)) { // Give a guy some time to register it if using opentrackers too if (trackers.size() == 1) { stop = true; - coordinator.snark.stopTorrent(); + snark.stopTorrent(); } else { // hopefully each on the opentrackers list is really open if (tr.registerFails++ > MAX_REGISTER_FAILS) tr.stop = true; @@ -316,7 +320,7 @@ public class TrackerClient extends I2PAppThread } // *** end of trackers loop here // we could try and total the unique peers but that's too hard for now - coordinator.trackerSeenPeers = maxSeenPeers; + snark.setTrackerSeenPeers(maxSeenPeers); if (!runStarted) _util.debug(" Retrying in one minute...", Snark.DEBUG); } // *** end of while loop @@ -377,8 +381,8 @@ public class TrackerClient extends I2PAppThread try { in = new FileInputStream(fetched); - TrackerInfo info = new TrackerInfo(in, coordinator.getID(), - coordinator.getMetaInfo()); + TrackerInfo info = new TrackerInfo(in, snark.getID(), + snark.getMetaInfo()); _util.debug("TrackerClient response: " + info, Snark.INFO); String failure = info.getFailureReason(); diff --git a/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java b/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java index d74426ee12..57e8bd4461 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java +++ b/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java @@ -26,6 +26,7 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import net.i2p.I2PAppContext; +import net.i2p.data.Base32; import net.i2p.data.Base64; import net.i2p.data.DataHelper; import net.i2p.util.FileUtil; @@ -361,7 +362,7 @@ public class I2PSnarkServlet extends Default { for (int i = 0; i < snarks.size(); i++) { Snark snark = (Snark)snarks.get(i); boolean showDebug = "2".equals(peerParam); - boolean showPeers = showDebug || "1".equals(peerParam) || Base64.encode(snark.meta.getInfoHash()).equals(peerParam); + boolean showPeers = showDebug || "1".equals(peerParam) || Base64.encode(snark.getInfoHash()).equals(peerParam); displaySnark(out, snark, uri, i, stats, showPeers, isDegraded, showDebug); } @@ -452,7 +453,7 @@ public class I2PSnarkServlet extends Default { if (newURL != null) { if (newURL.startsWith("http://")) { _manager.addMessage(_("Fetching {0}", urlify(newURL))); - I2PAppThread fetch = new I2PAppThread(new FetchAndAdd(_manager, newURL), "Fetch and add"); + I2PAppThread fetch = new I2PAppThread(new FetchAndAdd(_manager, newURL), "Fetch and add", true); fetch.start(); } else { _manager.addMessage(_("Invalid URL - must start with http://")); @@ -468,7 +469,7 @@ public class I2PSnarkServlet extends Default { for (Iterator iter = _manager.listTorrentFiles().iterator(); iter.hasNext(); ) { String name = (String)iter.next(); Snark snark = _manager.getTorrent(name); - if ( (snark != null) && (DataHelper.eq(infoHash, snark.meta.getInfoHash())) ) { + if ( (snark != null) && (DataHelper.eq(infoHash, snark.getInfoHash())) ) { _manager.stopTorrent(name, false); break; } @@ -482,11 +483,9 @@ public class I2PSnarkServlet extends Default { if ( (infoHash != null) && (infoHash.length == 20) ) { // valid sha1 for (String name : _manager.listTorrentFiles()) { Snark snark = _manager.getTorrent(name); - if ( (snark != null) && (DataHelper.eq(infoHash, snark.meta.getInfoHash())) ) { + if ( (snark != null) && (DataHelper.eq(infoHash, snark.getInfoHash())) ) { snark.startTorrent(); - if (snark.storage != null) - name = snark.storage.getBaseName(); - _manager.addMessage(_("Starting up torrent {0}", name)); + _manager.addMessage(_("Starting up torrent {0}", snark.getBaseName())); break; } } @@ -500,8 +499,12 @@ public class I2PSnarkServlet extends Default { for (Iterator iter = _manager.listTorrentFiles().iterator(); iter.hasNext(); ) { String name = (String)iter.next(); Snark snark = _manager.getTorrent(name); - if ( (snark != null) && (DataHelper.eq(infoHash, snark.meta.getInfoHash())) ) { + if ( (snark != null) && (DataHelper.eq(infoHash, snark.getInfoHash())) ) { _manager.stopTorrent(name, true); + MetaInfo meta = snark.getMetaInfo(); + if (meta == null) { + return; + } // should we delete the torrent file? // yeah, need to, otherwise it'll get autoadded again (at the moment File f = new File(name); @@ -520,13 +523,17 @@ public class I2PSnarkServlet extends Default { for (Iterator iter = _manager.listTorrentFiles().iterator(); iter.hasNext(); ) { String name = (String)iter.next(); Snark snark = _manager.getTorrent(name); - if ( (snark != null) && (DataHelper.eq(infoHash, snark.meta.getInfoHash())) ) { + if ( (snark != null) && (DataHelper.eq(infoHash, snark.getInfoHash())) ) { _manager.stopTorrent(name, true); + MetaInfo meta = snark.getMetaInfo(); + if (meta == null) { + return; + } File f = new File(name); f.delete(); _manager.addMessage(_("Torrent file deleted: {0}", f.getAbsolutePath())); - List files = snark.meta.getFiles(); - String dataFile = snark.meta.getName(); + List files = meta.getFiles(); + String dataFile = snark.getBaseName(); f = new File(_manager.getDataDir(), dataFile); if (files == null) { // single file torrent if (f.delete()) @@ -617,8 +624,8 @@ public class I2PSnarkServlet extends Default { List snarks = getSortedSnarks(req); for (int i = 0; i < snarks.size(); i++) { Snark snark = (Snark)snarks.get(i); - if (!snark.stopped) - _manager.stopTorrent(snark.torrent, false); + if (!snark.isStopped()) + _manager.stopTorrent(snark.getName(), false); } if (_manager.util().connected()) { // Give the stopped announces time to get out @@ -631,7 +638,7 @@ public class I2PSnarkServlet extends Default { List snarks = getSortedSnarks(req); for (int i = 0; i < snarks.size(); i++) { Snark snark = (Snark)snarks.get(i); - if (snark.stopped) + if (snark.isStopped()) snark.startTorrent(); } } else { @@ -699,7 +706,7 @@ public class I2PSnarkServlet extends Default { private static final int MAX_DISPLAYED_ERROR_LENGTH = 43; private void displaySnark(PrintWriter out, Snark snark, String uri, int row, long stats[], boolean showPeers, boolean isDegraded, boolean showDebug) throws IOException { - String filename = snark.torrent; + String filename = snark.getName(); File f = new File(filename); filename = f.getName(); // the torrent may be the canonical name, so lets just grab the local name int i = filename.lastIndexOf(".torrent"); @@ -710,28 +717,21 @@ public class I2PSnarkServlet extends Default { fullFilename = new String(filename); filename = filename.substring(0, MAX_DISPLAYED_FILENAME_LENGTH) + "…"; } - long total = snark.meta.getTotalLength(); + long total = snark.getTotalLength(); // Early typecast, avoid possibly overflowing a temp integer - long remaining = (long) snark.storage.needed() * (long) snark.meta.getPieceLength(0); + long remaining = (long) snark.getNeeded() * (long) snark.getPieceLength(0); if (remaining > total) remaining = total; - long downBps = 0; - long upBps = 0; - if (snark.coordinator != null) { - downBps = snark.coordinator.getDownloadRate(); - upBps = snark.coordinator.getUploadRate(); - } + long downBps = snark.getDownloadRate(); + long upBps = snark.getUploadRate(); long remainingSeconds; if (downBps > 0) remainingSeconds = remaining / downBps; else remainingSeconds = -1; - boolean isRunning = !snark.stopped; - long uploaded = 0; - if (snark.coordinator != null) { - uploaded = snark.coordinator.getUploaded(); - stats[0] += snark.coordinator.getDownloaded(); - } + boolean isRunning = !snark.isStopped(); + long uploaded = snark.getUploaded(); + stats[0] += snark.getDownloaded(); stats[1] += uploaded; if (isRunning) { stats[2] += downBps; @@ -739,25 +739,21 @@ public class I2PSnarkServlet extends Default { } stats[5] += total; - boolean isValid = snark.meta != null; - boolean singleFile = snark.meta.getFiles() == null; + MetaInfo meta = snark.getMetaInfo(); + boolean isValid = meta != null; + boolean singleFile = (!isValid) || meta.getFiles() == null; - String err = null; - int curPeers = 0; - int knownPeers = 0; - if (snark.coordinator != null) { - err = snark.coordinator.trackerProblems; - curPeers = snark.coordinator.getPeerCount(); - stats[4] += curPeers; - knownPeers = Math.max(curPeers, snark.coordinator.trackerSeenPeers); - } + String err = snark.getTrackerProblems(); + int curPeers = snark.getPeerCount(); + stats[4] += curPeers; + int knownPeers = Math.max(curPeers, snark.getTrackerSeenPeers()); String rowClass = (row % 2 == 0 ? "snarkTorrentEven" : "snarkTorrentOdd"); String statusString; if (err != null) { if (isRunning && curPeers > 0 && !showPeers) statusString = "\"\"" + _("Tracker Error") + - ": " + + ": " + curPeers + thinsp(isDegraded) + ngettext("1 peer", "{0} peers", knownPeers) + ""; else if (isRunning) @@ -773,7 +769,7 @@ public class I2PSnarkServlet extends Default { } else if (remaining <= 0) { if (isRunning && curPeers > 0 && !showPeers) statusString = "\"\"" + _("Seeding") + - ": " + + ": " + curPeers + thinsp(isDegraded) + ngettext("1 peer", "{0} peers", knownPeers) + ""; else if (isRunning) @@ -785,7 +781,7 @@ public class I2PSnarkServlet extends Default { } else { if (isRunning && curPeers > 0 && downBps > 0 && !showPeers) statusString = "\"\"" + _("OK") + - ": " + + ": " + curPeers + thinsp(isDegraded) + ngettext("1 peer", "{0} peers", knownPeers) + ""; else if (isRunning && curPeers > 0 && downBps > 0) @@ -794,7 +790,7 @@ public class I2PSnarkServlet extends Default { ngettext("1 peer", "{0} peers", knownPeers); else if (isRunning && curPeers > 0 && !showPeers) statusString = "\"\"" + _("Stalled") + - ": " + + ": " + curPeers + thinsp(isDegraded) + ngettext("1 peer", "{0} peers", knownPeers) + ""; else if (isRunning && curPeers > 0) @@ -816,7 +812,7 @@ public class I2PSnarkServlet extends Default { out.write(""); // temporarily hardcoded for postman* and anonymity, requires bytemonsoon patch for lookup by info_hash - String announce = snark.meta.getAnnounce(); + String announce = meta.getAnnounce(); if (announce.startsWith("http://YRgrgTLG") || announce.startsWith("http://8EoJZIKr") || announce.startsWith("http://lnQ6yoBT") || announce.startsWith("http://tracker2.postman.i2p/") || announce.startsWith("http://ahsplxkbhemefwvvml7qovzl5a2b5xo5i7lyai7ntdunvcyfdtna.b32.i2p/")) { Map trackers = _manager.getTrackers(); @@ -833,7 +829,7 @@ public class I2PSnarkServlet extends Default { continue; baseURL = baseURL.substring(e + 1); out.write(""); out.write("\"""); out.write(""); @@ -843,13 +839,13 @@ public class I2PSnarkServlet extends Default { out.write("\n"); StringBuilder buf = null; - if (remaining == 0 || snark.meta.getFiles() != null) { + if (remaining == 0 || meta.getFiles() != null) { buf = new StringBuilder(128); - buf.append(""); } else { out.write(toImg(icon)); } out.write(""); - if (remaining == 0 || snark.meta.getFiles() != null) + if (remaining == 0 || meta.getFiles() != null) out.write(buf.toString()); out.write(filename); - if (remaining == 0 || snark.meta.getFiles() != null) + if (remaining == 0 || meta.getFiles() != null) out.write(""); out.write(""); @@ -897,8 +893,8 @@ public class I2PSnarkServlet extends Default { out.write(formatSize(upBps) + "ps"); out.write("\n\t"); out.write(""); - String parameters = "&nonce=" + _nonce + "&torrent=" + Base64.encode(snark.meta.getInfoHash()); - String b64 = Base64.encode(snark.meta.getInfoHash()); + String parameters = "&nonce=" + _nonce + "&torrent=" + Base64.encode(snark.getInfoHash()); + String b64 = Base64.encode(snark.getInfoHash()); if (showPeers) parameters = parameters + "&p=1"; if (isRunning) { @@ -963,7 +959,7 @@ public class I2PSnarkServlet extends Default { out.write("\n\n"); if(showPeers && isRunning && curPeers > 0) { - List peers = snark.coordinator.peerList(); + List peers = snark.getPeerList(); if (!showDebug) Collections.sort(peers, new PeerComparator()); for (Peer peer : peers) { @@ -996,7 +992,7 @@ public class I2PSnarkServlet extends Default { out.write(""); out.write("\n\t"); out.write(""); - float pct = (float) (100.0 * (float) peer.completed() / snark.meta.getPieces()); + float pct = (float) (100.0 * (float) peer.completed() / meta.getPieces()); if (pct == 100.0) out.write(_("Seed")); else { @@ -1358,6 +1354,11 @@ public class I2PSnarkServlet extends Default { return _manager.util().getString(s, o); } + /** translate */ + private String _(String s, Object o, Object o2) { + return _manager.util().getString(s, o, o2); + } + /** translate (ngettext) @since 0.7.14 */ private String ngettext(String s, String p, int n) { return _manager.util().getString(n, s, p); @@ -1469,7 +1470,7 @@ public class I2PSnarkServlet extends Default { if (parent) // always true buf.append("
"); - boolean showPriority = snark != null && !snark.storage.complete(); + boolean showPriority = snark != null && snark.getStorage() != null && !snark.getStorage().complete(); if (showPriority) buf.append("
\n"); buf.append("" + @@ -1516,15 +1517,16 @@ public class I2PSnarkServlet extends Default { complete = true; status = toImg("tick") + ' ' + _("Directory"); } else { - if (snark == null) { + if (snark == null || snark.getStorage() == null) { // Assume complete, perhaps he removed a completed torrent but kept a bookmark complete = true; status = toImg("cancel") + ' ' + _("Torrent not found?"); } else { + Storage storage = snark.getStorage(); try { File f = item.getFile(); if (f != null) { - long remaining = snark.storage.remaining(f.getCanonicalPath()); + long remaining = storage.remaining(f.getCanonicalPath()); if (remaining < 0) { complete = true; status = toImg("cancel") + ' ' + _("File not found in torrent?"); @@ -1532,7 +1534,7 @@ public class I2PSnarkServlet extends Default { complete = true; status = toImg("tick") + ' ' + _("Complete"); } else { - int priority = snark.storage.getPriority(f.getCanonicalPath()); + int priority = storage.getPriority(f.getCanonicalPath()); if (priority < 0) status = toImg("cancel"); else if (priority == 0) @@ -1588,7 +1590,7 @@ public class I2PSnarkServlet extends Default { buf.append("\n\n\t"); out.write("\n\t"); out.write("\n\t"); out.write("\n\t"); out.write("\n\t"); out.write("\n\t"); out.write("\n\t"); out.write("
"); File f = item.getFile(); if ((!complete) && (!item.isDirectory()) && f != null) { - int pri = snark.storage.getPriority(f.getCanonicalPath()); + int pri = snark.getStorage().getPriority(f.getCanonicalPath()); buf.append(" 0) buf.append("checked=\"true\""); @@ -1690,6 +1692,9 @@ public class I2PSnarkServlet extends Default { /** @since 0.8.1 */ private void savePriorities(Snark snark, Map postParams) { + Storage storage = snark.getStorage(); + if (storage == null) + return; Set entries = postParams.entrySet(); for (Map.Entry entry : entries) { String key = (String)entry.getKey(); @@ -1698,14 +1703,13 @@ public class I2PSnarkServlet extends Default { String file = key.substring(4); String val = ((String[])entry.getValue())[0]; // jetty arrays int pri = Integer.parseInt(val); - snark.storage.setPriority(file, pri); + storage.setPriority(file, pri); //System.err.println("Priority now " + pri + " for " + file); } catch (Throwable t) { t.printStackTrace(); } } } - if (snark.coordinator != null) - snark.coordinator.updatePiecePriorities(); - _manager.saveTorrentStatus(snark.storage.getMetaInfo(), snark.storage.getBitField(), snark.storage.getFilePriorities()); + snark.updatePiecePriorities(); + _manager.saveTorrentStatus(snark.getMetaInfo(), storage.getBitField(), storage.getFilePriorities()); } From 76029992742698022c6699ff30ec30d5f731b7ff Mon Sep 17 00:00:00 2001 From: zzz Date: Mon, 20 Dec 2010 00:05:03 +0000 Subject: [PATCH 03/27] more prep and stubs for no metainfo --- .../src/org/klomp/snark/I2PSnarkUtil.java | 15 +++ .../java/src/org/klomp/snark/Peer.java | 14 ++- .../src/org/klomp/snark/PeerAcceptor.java | 14 +-- .../src/org/klomp/snark/PeerCoordinator.java | 66 +++++++++-- .../java/src/org/klomp/snark/PeerState.java | 19 ++- .../java/src/org/klomp/snark/Snark.java | 112 ++++++++++++------ .../src/org/klomp/snark/SnarkManager.java | 38 ++++++ .../src/org/klomp/snark/TrackerClient.java | 50 ++++++++ .../java/src/org/klomp/snark/TrackerInfo.java | 4 +- .../org/klomp/snark/web/I2PSnarkServlet.java | 54 ++++++++- 10 files changed, 318 insertions(+), 68 deletions(-) diff --git a/apps/i2psnark/java/src/org/klomp/snark/I2PSnarkUtil.java b/apps/i2psnark/java/src/org/klomp/snark/I2PSnarkUtil.java index 001928d479..e8ff0881fa 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/I2PSnarkUtil.java +++ b/apps/i2psnark/java/src/org/klomp/snark/I2PSnarkUtil.java @@ -32,6 +32,8 @@ import net.i2p.util.SimpleScheduler; import net.i2p.util.SimpleTimer; import net.i2p.util.Translate; +import org.klomp.snark.dht.KRPC; + /** * I2P specific helpers for I2PSnark * We use this class as a sort of context for i2psnark @@ -56,6 +58,7 @@ public class I2PSnarkUtil { private int _maxConnections; private File _tmpDir; private int _startupDelay; + private KRPC _krpc; public static final int DEFAULT_STARTUP_DELAY = 3; public static final String PROP_USE_OPENTRACKERS = "i2psnark.useOpentrackers"; @@ -64,6 +67,8 @@ public class I2PSnarkUtil { public static final String DEFAULT_OPENTRACKERS = "http://tracker.welterde.i2p/a"; public static final int DEFAULT_MAX_UP_BW = 8; //KBps public static final int MAX_CONNECTIONS = 16; // per torrent + private static final boolean ENABLE_DHT = true; + public I2PSnarkUtil(I2PAppContext ctx) { _context = ctx; _log = _context.logManager().getLog(Snark.class); @@ -185,10 +190,20 @@ public class I2PSnarkUtil { // opts.setProperty("i2p.streaming.readTimeout", "120000"); _manager = I2PSocketManagerFactory.createManager(_i2cpHost, _i2cpPort, opts); } + // FIXME this only instantiates krpc once, left stuck with old manager + if (ENABLE_DHT && _manager != null && _krpc == null) + _krpc = new KRPC(_context, _manager.getSession()); return (_manager != null); } + /** + * @return null if disabled or not started + * @since 0.8.4 + */ + public KRPC getDHT() { return _krpc; } + public boolean connected() { return _manager != null; } + /** * Destroy the destination itself */ diff --git a/apps/i2psnark/java/src/org/klomp/snark/Peer.java b/apps/i2psnark/java/src/org/klomp/snark/Peer.java index f42adddeff..a65e458876 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/Peer.java +++ b/apps/i2psnark/java/src/org/klomp/snark/Peer.java @@ -39,6 +39,7 @@ public class Peer implements Comparable private final PeerID peerID; private final byte[] my_id; + private final byte[] infohash; final MetaInfo metainfo; // The data in/output streams set during the handshake and used by @@ -70,11 +71,13 @@ public class Peer implements Comparable * Outgoing connection. * Creates a disconnected peer given a PeerID, your own id and the * relevant MetaInfo. + * @param metainfo null if in magnet mode */ - public Peer(PeerID peerID, byte[] my_id, MetaInfo metainfo) + public Peer(PeerID peerID, byte[] my_id, byte[] infohash, MetaInfo metainfo) { this.peerID = peerID; this.my_id = my_id; + this.infohash = infohash; this.metainfo = metainfo; _id = ++__id; //_log.debug("Creating a new peer with " + peerID.getAddress().calculateHash().toBase64(), new Exception("creating")); @@ -88,12 +91,14 @@ public class Peer implements Comparable * get the remote peer id. To completely start the connection call * the connect() method. * + * @param metainfo null if in magnet mode * @exception IOException when an error occurred during the handshake. */ - public Peer(final I2PSocket sock, InputStream in, OutputStream out, byte[] my_id, MetaInfo metainfo) + public Peer(final I2PSocket sock, InputStream in, OutputStream out, byte[] my_id, byte[] infohash, MetaInfo metainfo) throws IOException { this.my_id = my_id; + this.infohash = infohash; this.metainfo = metainfo; this.sock = sock; @@ -312,8 +317,7 @@ public class Peer implements Comparable // FIXME not if DHT disabled dout.writeLong(OPTION_EXTENSION | OPTION_DHT); // Handshake write - metainfo hash - byte[] shared_hash = metainfo.getInfoHash(); - dout.write(shared_hash); + dout.write(infohash); // Handshake write - peer id dout.write(my_id); dout.flush(); @@ -341,7 +345,7 @@ public class Peer implements Comparable // Handshake read - metainfo hash bs = new byte[20]; din.readFully(bs); - if (!Arrays.equals(shared_hash, bs)) + if (!Arrays.equals(infohash, bs)) throw new IOException("Unexpected MetaInfo hash"); // Handshake read - peer id diff --git a/apps/i2psnark/java/src/org/klomp/snark/PeerAcceptor.java b/apps/i2psnark/java/src/org/klomp/snark/PeerAcceptor.java index 050884e5c0..0adb13d9b7 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/PeerAcceptor.java +++ b/apps/i2psnark/java/src/org/klomp/snark/PeerAcceptor.java @@ -88,12 +88,11 @@ public class PeerAcceptor } if (coordinator != null) { // single torrent capability - MetaInfo meta = coordinator.getMetaInfo(); - if (DataHelper.eq(meta.getInfoHash(), peerInfoHash)) { + if (DataHelper.eq(coordinator.getInfoHash(), peerInfoHash)) { if (coordinator.needPeers()) { Peer peer = new Peer(socket, in, out, coordinator.getID(), - coordinator.getMetaInfo()); + coordinator.getInfoHash(), coordinator.getMetaInfo()); coordinator.addPeer(peer); } else @@ -101,19 +100,18 @@ public class PeerAcceptor } else { // its for another infohash, but we are only single torrent capable. b0rk. throw new IOException("Peer wants another torrent (" + Base64.encode(peerInfoHash) - + ") while we only support (" + Base64.encode(meta.getInfoHash()) + ")"); + + ") while we only support (" + Base64.encode(coordinator.getInfoHash()) + ")"); } } else { // multitorrent capable, so lets see what we can handle for (Iterator iter = coordinators.iterator(); iter.hasNext(); ) { PeerCoordinator cur = (PeerCoordinator)iter.next(); - MetaInfo meta = cur.getMetaInfo(); - - if (DataHelper.eq(meta.getInfoHash(), peerInfoHash)) { + + if (DataHelper.eq(cur.getInfoHash(), peerInfoHash)) { if (cur.needPeers()) { Peer peer = new Peer(socket, in, out, cur.getID(), - cur.getMetaInfo()); + cur.getInfoHash(), cur.getMetaInfo()); cur.addPeer(peer); return; } diff --git a/apps/i2psnark/java/src/org/klomp/snark/PeerCoordinator.java b/apps/i2psnark/java/src/org/klomp/snark/PeerCoordinator.java index ba9727d8a6..f7a3a42fc8 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/PeerCoordinator.java +++ b/apps/i2psnark/java/src/org/klomp/snark/PeerCoordinator.java @@ -93,6 +93,7 @@ public class PeerCoordinator implements PeerListener private final CheckEvent timer; private final byte[] id; + private final byte[] infohash; /** The wanted pieces. We could use a TreeSet but we'd have to clear and re-add everything * when priorities change. @@ -108,11 +109,16 @@ public class PeerCoordinator implements PeerListener private final I2PSnarkUtil _util; private static final Random _random = I2PAppContext.getGlobalContext().random(); - public PeerCoordinator(I2PSnarkUtil util, byte[] id, MetaInfo metainfo, Storage storage, + /** + * @param metainfo null if in magnet mode + * @param storage null if in magnet mode + */ + public PeerCoordinator(I2PSnarkUtil util, byte[] id, byte[] infohash, MetaInfo metainfo, Storage storage, CoordinatorListener listener, Snark torrent) { _util = util; this.id = id; + this.infohash = infohash; this.metainfo = metainfo; this.storage = storage; this.listener = listener; @@ -149,6 +155,8 @@ public class PeerCoordinator implements PeerListener // only called externally from Storage after the double-check fails public void setWantedPieces() { + if (metainfo == null || storage == null) + return; // Make a list of pieces synchronized(wantedPieces) { wantedPieces.clear(); @@ -188,6 +196,9 @@ public class PeerCoordinator implements PeerListener public boolean completed() { + // FIXME return metainfo complete status + if (storage == null) + return false; return storage.complete(); } @@ -204,9 +215,12 @@ public class PeerCoordinator implements PeerListener /** * Returns how many bytes are still needed to get the complete file. + * @return -1 if in magnet mode */ public long getLeft() { + if (metainfo == null | storage == null) + return -1; // XXX - Only an approximation. return ((long) storage.needed()) * metainfo.getPieceLength(0); } @@ -291,6 +305,12 @@ public class PeerCoordinator implements PeerListener return metainfo; } + /** @since 0.8.4 */ + public byte[] getInfoHash() + { + return infohash; + } + public boolean needPeers() { return !halted && peers.size() < getMaxConnections(); @@ -301,6 +321,8 @@ public class PeerCoordinator implements PeerListener * @return 512K: 16; 1M: 11; 2M: 6 */ private int getMaxConnections() { + if (metainfo == null) + return 6; int size = metainfo.getPieceLength(0); int max = _util.getMaxConnections(); if (size <= 512*1024 || completed()) @@ -375,8 +397,15 @@ public class PeerCoordinator implements PeerListener } else { - if (_log.shouldLog(Log.INFO)) - _log.info("New connection to peer: " + peer + " for " + metainfo.getName()); + if (_log.shouldLog(Log.INFO)) { + // just for logging + String name; + if (metainfo == null) + name = "Magnet"; + else + name = metainfo.getName(); + _log.info("New connection to peer: " + peer + " for " + metainfo.getName()); + } // Add it to the beginning of the list. // And try to optimistically make it a uploader. @@ -435,12 +464,22 @@ public class PeerCoordinator implements PeerListener if (need_more) { - if (_log.shouldLog(Log.DEBUG)) - _log.debug("Adding a peer " + peer.getPeerID().toString() + " for " + metainfo.getName(), new Exception("add/run")); - + if (_log.shouldLog(Log.DEBUG)) { + // just for logging + String name; + if (metainfo == null) + name = "Magnet"; + else + name = metainfo.getName(); + _log.debug("Adding a peer " + peer.getPeerID().toString() + " for " + name, new Exception("add/run")); + } // Run the peer with us as listener and the current bitfield. final PeerListener listener = this; - final BitField bitfield = storage.getBitField(); + final BitField bitfield; + if (storage != null) + bitfield = storage.getBitField(); + else + bitfield = null; Runnable r = new Runnable() { public void run() @@ -506,11 +545,6 @@ public class PeerCoordinator implements PeerListener interestedAndChoking = count; } - public byte[] getBitMap() - { - return storage.getBitField().getFieldBytes(); - } - /** * @return true if we still want the given piece */ @@ -679,6 +713,8 @@ public class PeerCoordinator implements PeerListener * @since 0.8.1 */ public void updatePiecePriorities() { + if (storage == null) + return; int[] pri = storage.getPiecePriorities(); if (pri == null) { _log.debug("Updated piece priorities called but no priorities to set?"); @@ -745,6 +781,8 @@ public class PeerCoordinator implements PeerListener { if (halted) return null; + if (metainfo == null || storage == null) + return null; try { @@ -787,6 +825,8 @@ public class PeerCoordinator implements PeerListener */ public boolean gotPiece(Peer peer, int piece, byte[] bs) { + if (metainfo == null || storage == null) + return true; if (halted) { _log.info("Got while-halted piece " + piece + "/" + metainfo.getPieces() +" from " + peer + " for " + metainfo.getName()); return true; // We don't actually care anymore. @@ -983,6 +1023,8 @@ public class PeerCoordinator implements PeerListener * @since 0.8.2 */ public PartialPiece getPartialPiece(Peer peer, BitField havePieces) { + if (metainfo == null) + return null; synchronized(wantedPieces) { // sorts by remaining bytes, least first Collections.sort(partialPieces); diff --git a/apps/i2psnark/java/src/org/klomp/snark/PeerState.java b/apps/i2psnark/java/src/org/klomp/snark/PeerState.java index ad70df9dc6..3ec3d219f6 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/PeerState.java +++ b/apps/i2psnark/java/src/org/klomp/snark/PeerState.java @@ -71,6 +71,9 @@ class PeerState implements DataLoader public final static int PARTSIZE = 16*1024; // outbound request private final static int MAX_PARTSIZE = 64*1024; // Don't let anybody request more than this + /** + * @param metainfo null if in magnet mode + */ PeerState(Peer peer, PeerListener listener, MetaInfo metainfo, PeerConnectionIn in, PeerConnectionOut out) { @@ -132,6 +135,9 @@ class PeerState implements DataLoader { if (_log.shouldLog(Log.DEBUG)) _log.debug(peer + " rcv have(" + piece + ")"); + // FIXME we will lose these until we get the metainfo + if (metainfo == null) + return; // Sanity check if (piece < 0 || piece >= metainfo.getPieces()) { @@ -169,8 +175,15 @@ class PeerState implements DataLoader } // XXX - Check for weird bitfield and disconnect? - bitfield = new BitField(bitmap, metainfo.getPieces()); + // FIXME will have to regenerate the bitfield after we know exactly + // how many pieces there are, as we don't know how many spare bits there are. + if (metainfo == null) + bitfield = new BitField(bitmap, bitmap.length * 8); + else + bitfield = new BitField(bitmap, metainfo.getPieces()); } + if (metainfo == null) + return; boolean interest = listener.gotBitField(peer, bitfield); setInteresting(interest); if (bitfield.complete() && !interest) { @@ -188,6 +201,8 @@ class PeerState implements DataLoader if (_log.shouldLog(Log.DEBUG)) _log.debug(peer + " rcv request(" + piece + ", " + begin + ", " + length + ") "); + if (metainfo == null) + return; if (choking) { if (_log.shouldLog(Log.INFO)) @@ -606,6 +621,8 @@ class PeerState implements DataLoader // no bitfield yet? nothing to request then. if (bitfield == null) return; + if (metainfo == null) + return; boolean more_pieces = true; while (more_pieces) { diff --git a/apps/i2psnark/java/src/org/klomp/snark/Snark.java b/apps/i2psnark/java/src/org/klomp/snark/Snark.java index 79807d68f2..2460e9f530 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/Snark.java +++ b/apps/i2psnark/java/src/org/klomp/snark/Snark.java @@ -317,31 +317,7 @@ public class Snark stopped = true; activity = "Network setup"; - // "Taking Three as the subject to reason about-- - // A convenient number to state-- - // We add Seven, and Ten, and then multiply out - // By One Thousand diminished by Eight. - // - // "The result we proceed to divide, as you see, - // By Nine Hundred and Ninety Two: - // Then subtract Seventeen, and the answer must be - // Exactly and perfectly true. - - // Create a new ID and fill it with something random. First nine - // zeros bytes, then three bytes filled with snark and then - // sixteen random bytes. - byte snark = (((3 + 7 + 10) * (1000 - 8)) / 992) - 17; - id = new byte[20]; - Random random = I2PAppContext.getGlobalContext().random(); - int i; - for (i = 0; i < 9; i++) - id[i] = 0; - id[i++] = snark; - id[i++] = snark; - id[i++] = snark; - while (i < 20) - id[i++] = (byte)random.nextInt(256); - + id = generateID(); debug("My peer id: " + PeerID.idencode(id), Snark.INFO); int port; @@ -468,6 +444,64 @@ public class Snark if (start) startTorrent(); } + + /** + * multitorrent, magnet + * + * @param torrent a fake name for now (not a file name) + * @param ih 20-byte info hash + * @since 0.8.4 + */ + public Snark(I2PSnarkUtil util, String torrent, byte[] ih, + CompleteListener complistener, PeerCoordinatorSet peerCoordinatorSet, + ConnectionAcceptor connectionAcceptor, boolean start, String rootDir) + { + completeListener = complistener; + _util = util; + _peerCoordinatorSet = peerCoordinatorSet; + acceptor = connectionAcceptor; + this.torrent = torrent; + this.infoHash = ih; + this.rootDataDir = rootDir; + stopped = true; + id = generateID(); + + // All we have is an infoHash + // meta remains null + // storage remains null + + if (start) + startTorrent(); + } + + private static byte[] generateID() { + // "Taking Three as the subject to reason about-- + // A convenient number to state-- + // We add Seven, and Ten, and then multiply out + // By One Thousand diminished by Eight. + // + // "The result we proceed to divide, as you see, + // By Nine Hundred and Ninety Two: + // Then subtract Seventeen, and the answer must be + // Exactly and perfectly true. + + // Create a new ID and fill it with something random. First nine + // zeros bytes, then three bytes filled with snark and then + // sixteen random bytes. + byte snark = (((3 + 7 + 10) * (1000 - 8)) / 992) - 17; + byte[] rv = new byte[20]; + Random random = I2PAppContext.getGlobalContext().random(); + int i; + for (i = 0; i < 9; i++) + rv[i] = 0; + rv[i++] = snark; + rv[i++] = snark; + rv[i++] = snark; + while (i < 20) + rv[i++] = (byte)random.nextInt(256); + return rv; + } + /** * Start up contacting peers and querying the tracker */ @@ -484,7 +518,7 @@ public class Snark } debug("Starting PeerCoordinator, ConnectionAcceptor, and TrackerClient", NOTICE); activity = "Collecting pieces"; - coordinator = new PeerCoordinator(_util, id, meta, storage, this, this); + coordinator = new PeerCoordinator(_util, id, infoHash, meta, storage, this, this); if (_peerCoordinatorSet != null) { // multitorrent _peerCoordinatorSet.add(coordinator); @@ -507,7 +541,7 @@ public class Snark // restart safely, so lets build a new one to replace the old if (_peerCoordinatorSet != null) _peerCoordinatorSet.remove(coordinator); - PeerCoordinator newCoord = new PeerCoordinator(_util, id, meta, storage, this, this); + PeerCoordinator newCoord = new PeerCoordinator(_util, id, infoHash, meta, storage, this, this); if (_peerCoordinatorSet != null) _peerCoordinatorSet.add(newCoord); coordinator = newCoord; @@ -516,18 +550,17 @@ public class Snark if (!trackerclient.started() && !coordinatorChanged) { trackerclient.start(); } else if (trackerclient.halted() || coordinatorChanged) { - try - { - storage.reopen(rootDataDir); - } - catch (IOException ioe) - { - try { storage.close(); } catch (IOException ioee) { - ioee.printStackTrace(); - } - fatal("Could not reopen storage", ioe); - } - TrackerClient newClient = new TrackerClient(_util, coordinator.getMetaInfo(), coordinator, this); + if (storage != null) { + try { + storage.reopen(rootDataDir); + } catch (IOException ioe) { + try { storage.close(); } catch (IOException ioee) { + ioee.printStackTrace(); + } + fatal("Could not reopen storage", ioe); + } + } + TrackerClient newClient = new TrackerClient(_util, meta, coordinator, this); if (!trackerclient.halted()) trackerclient.halt(); trackerclient = newClient; @@ -601,6 +634,7 @@ public class Snark * @since 0.8.4 */ public byte[] getInfoHash() { + // should always be the same if (meta != null) return meta.getInfoHash(); return infoHash; diff --git a/apps/i2psnark/java/src/org/klomp/snark/SnarkManager.java b/apps/i2psnark/java/src/org/klomp/snark/SnarkManager.java index 6a4eb4c7c5..270937688e 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/SnarkManager.java +++ b/apps/i2psnark/java/src/org/klomp/snark/SnarkManager.java @@ -578,6 +578,44 @@ public class SnarkManager implements Snark.CompleteListener { } } + /** + * Add a torrent with the info hash alone (magnet / maggot) + * + * @param name hex or b32 name from the magnet link + * @param ih 20 byte info hash + * @since 0.8.4 + */ + public void addMagnet(String name, byte[] ih) { + Snark torrent = new Snark(_util, name, ih, this, + _peerCoordinatorSet, _connectionAcceptor, + false, getDataDir().getPath()); + + // TODO tell the dir monitor not to delete us + synchronized (_snarks) { + _snarks.put(name, torrent); + } + if (shouldAutoStart()) { + torrent.startTorrent(); + addMessage(_("Fetching {0}", name)); + boolean haveSavedPeers = false; + if ((!util().connected()) && !haveSavedPeers) { + addMessage(_("We have no saved peers and no other torrents are running. " + + "Fetch of {0} will not succeed until you start another torrent.", name)); + } + } else { + addMessage(_("Adding {0}", name)); + } + } + + /** + * Delete a torrent with the info hash alone (magnet / maggot) + * + * @param ih 20 byte info hash + * @since 0.8.4 + */ + public void deleteMagnet(byte[] ih) { + } + /** * Get the timestamp for a torrent from the config file */ diff --git a/apps/i2psnark/java/src/org/klomp/snark/TrackerClient.java b/apps/i2psnark/java/src/org/klomp/snark/TrackerClient.java index 6bded21008..d14abaa0c9 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/TrackerClient.java +++ b/apps/i2psnark/java/src/org/klomp/snark/TrackerClient.java @@ -38,6 +38,7 @@ import net.i2p.data.Hash; import net.i2p.util.I2PAppThread; import net.i2p.util.Log; +import org.klomp.snark.dht.KRPC; /** * Informs metainfo tracker of events and gets new peers for peer @@ -73,6 +74,9 @@ public class TrackerClient extends I2PAppThread private List trackers; + /** + * @param meta null if in magnet mode + */ public TrackerClient(I2PSnarkUtil util, MetaInfo meta, PeerCoordinator coordinator, Snark snark) { super(); @@ -173,6 +177,7 @@ public class TrackerClient extends I2PAppThread // FIXME really need to get this message to the gui stop = true; _log.error("No valid trackers for infoHash: " + infoHash); + // FIXME keep going if DHT enabled return; } @@ -192,6 +197,9 @@ public class TrackerClient extends I2PAppThread Random r = I2PAppContext.getGlobalContext().random(); while(!stop) { + // Local DHT tracker announce + if (_util.getDHT() != null) + _util.getDHT().announce(snark.getInfoHash()); try { // Sleep some minutes... @@ -319,6 +327,45 @@ public class TrackerClient extends I2PAppThread maxSeenPeers = tr.seenPeers; } // *** end of trackers loop here + // Get peers from DHT + // FIXME this needs to be in its own thread + if (_util.getDHT() != null && !stop) { + int numwant; + if (left <= 0 || event.equals(STOPPED_EVENT) || !coordinator.needPeers()) + numwant = 1; + else + numwant = _util.getMaxConnections(); + List hashes = _util.getDHT().getPeers(snark.getInfoHash(), numwant, 2*60*1000); + _util.debug("Got " + hashes + " from DHT", Snark.INFO); + // announce ourselves while the token is still good + // FIXME this needs to be in its own thread + if (!stop) { + int good = _util.getDHT().announce(snark.getInfoHash(), 8, 5*60*1000); + _util.debug("Sent " + good + " good announces to DHT", Snark.INFO); + } + + // now try these peers + if ((!stop) && !hashes.isEmpty()) { + List peers = new ArrayList(hashes.size()); + for (Hash h : hashes) { + PeerID pID = new PeerID(h.getData()); + peers.add(new Peer(pID, snark.getID(), snark.getInfoHash(), meta)); + } + Collections.shuffle(peers, r); + Iterator it = peers.iterator(); + while ((!stop) && it.hasNext()) { + Peer cur = it.next(); + if (coordinator.addPeer(cur)) { + int delay = DELAY_MUL; + delay *= ((int)cur.getPeerID().getAddress().calculateHash().toBase64().charAt(0)) % 10; + delay += DELAY_MIN; + try { Thread.sleep(delay); } catch (InterruptedException ie) {} + } + } + } + } + + // we could try and total the unique peers but that's too hard for now snark.setTrackerSeenPeers(maxSeenPeers); if (!runStarted) @@ -333,6 +380,9 @@ public class TrackerClient extends I2PAppThread } finally { + // Local DHT tracker unannounce + if (_util.getDHT() != null) + _util.getDHT().unannounce(snark.getInfoHash()); try { // try to contact everybody we can diff --git a/apps/i2psnark/java/src/org/klomp/snark/TrackerInfo.java b/apps/i2psnark/java/src/org/klomp/snark/TrackerInfo.java index 360a4f47e4..ef67fc58b1 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/TrackerInfo.java +++ b/apps/i2psnark/java/src/org/klomp/snark/TrackerInfo.java @@ -144,7 +144,7 @@ public class TrackerInfo continue; } } - peers.add(new Peer(peerID, my_id, metainfo)); + peers.add(new Peer(peerID, my_id, metainfo.getInfoHash(), metainfo)); } return peers; @@ -172,7 +172,7 @@ public class TrackerInfo // won't happen continue; } - peers.add(new Peer(peerID, my_id, metainfo)); + peers.add(new Peer(peerID, my_id, metainfo.getInfoHash(), metainfo)); } return peers; diff --git a/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java b/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java index 57e8bd4461..917652bc04 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java +++ b/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java @@ -61,6 +61,10 @@ public class I2PSnarkServlet extends Default { private String _imgPath; public static final String PROP_CONFIG_FILE = "i2psnark.configFile"; + /** BEP 9 */ + private static final String MAGNET = "magnet:?xt=urn:btih:"; + /** http://sponge.i2p/files/maggotspec.txt */ + private static final String MAGGOT = "maggot://"; @Override public void init(ServletConfig cfg) throws ServletException { @@ -455,8 +459,10 @@ public class I2PSnarkServlet extends Default { _manager.addMessage(_("Fetching {0}", urlify(newURL))); I2PAppThread fetch = new I2PAppThread(new FetchAndAdd(_manager, newURL), "Fetch and add", true); fetch.start(); + } else if (newURL.startsWith(MAGNET) || newURL.startsWith(MAGGOT)) { + addMagnet(newURL); } else { - _manager.addMessage(_("Invalid URL - must start with http://")); + _manager.addMessage(_("Invalid URL - must start with http://, {0} or {1}", MAGNET, MAGGOT)); } } else { // no file or URL specified @@ -503,6 +509,8 @@ public class I2PSnarkServlet extends Default { _manager.stopTorrent(name, true); MetaInfo meta = snark.getMetaInfo(); if (meta == null) { + // magnet + _manager.deleteMagnet(snark.getInfoHash()); return; } // should we delete the torrent file? @@ -527,6 +535,8 @@ public class I2PSnarkServlet extends Default { _manager.stopTorrent(name, true); MetaInfo meta = snark.getMetaInfo(); if (meta == null) { + // magnet + _manager.deleteMagnet(snark.getInfoHash()); return; } File f = new File(name); @@ -1314,6 +1324,48 @@ public class I2PSnarkServlet extends Default { out.write("\n"); } + /** + * @param url in base32 or hex, xt must be first magnet param + * @since 0.8.4 + */ + private void addMagnet(String url) { + String ihash; + String name; + if (url.startsWith(MAGNET)) { + ihash = url.substring(MAGNET.length()).trim(); + int amp = ihash.indexOf('&'); + if (amp >= 0) + ihash = url.substring(0, amp); + name = "Magnet " + ihash; + } else if (url.startsWith(MAGGOT)) { + ihash = url.substring(MAGGOT.length()).trim(); + int col = ihash.indexOf(':'); + if (col >= 0) + ihash = url.substring(0, col); + name = "Maggot " + ihash; + } else { + return; + } + byte[] ih = null; + if (ihash.length() == 32) { + ih = Base32.decode(ihash); + } else if (ihash.length() == 40) { + ih = new byte[20]; + try { + for (int i = 0; i < 20; i++) { + ih[i] = (byte) (Integer.parseInt(ihash.substring(i*2, (i*2) + 2), 16) & 0xff); + } + } catch (NumberFormatException nfe) { + ih = null; + } + } + if (ih == null || ih.length != 20) { + _manager.addMessage(_("Invalid info hash in magnet URL {0}", url)); + return; + } + _manager.addMagnet(ihash, ih); + } + /** copied from ConfigTunnelsHelper */ private static final String HOP = "hop"; private static final String TUNNEL = "tunnel"; From ebe7f3b127f81049c7e71351d06fdc770c91e8e6 Mon Sep 17 00:00:00 2001 From: zzz Date: Mon, 20 Dec 2010 18:55:10 +0000 Subject: [PATCH 04/27] UI adjustments when no metainfo yet --- .../java/src/org/klomp/snark/Snark.java | 8 +- .../src/org/klomp/snark/SnarkManager.java | 66 ++++++--- .../src/org/klomp/snark/TrackerClient.java | 17 ++- .../org/klomp/snark/web/I2PSnarkServlet.java | 126 ++++++++++-------- 4 files changed, 140 insertions(+), 77 deletions(-) diff --git a/apps/i2psnark/java/src/org/klomp/snark/Snark.java b/apps/i2psnark/java/src/org/klomp/snark/Snark.java index 2460e9f530..27e38b5ca0 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/Snark.java +++ b/apps/i2psnark/java/src/org/klomp/snark/Snark.java @@ -531,6 +531,7 @@ public class Snark // single torrent acceptor = new ConnectionAcceptor(_util, serversocket, new PeerAcceptor(coordinator)); } + // TODO pass saved closest DHT nodes to the tracker? or direct to the coordinator? trackerclient = new TrackerClient(_util, meta, coordinator, this); } @@ -781,8 +782,11 @@ public class Snark public long getNeeded() { if (storage != null) return storage.needed(); - // FIXME else return metainfo length if available - return -1; + if (meta != null) + // FIXME subtract chunks we have + return meta.getTotalLength(); + // FIXME fake + return 16 * 16 * 1024; } /** diff --git a/apps/i2psnark/java/src/org/klomp/snark/SnarkManager.java b/apps/i2psnark/java/src/org/klomp/snark/SnarkManager.java index 270937688e..24e8ebc7f6 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/SnarkManager.java +++ b/apps/i2psnark/java/src/org/klomp/snark/SnarkManager.java @@ -20,6 +20,7 @@ import java.util.Collection; import net.i2p.I2PAppContext; import net.i2p.data.Base64; import net.i2p.data.DataHelper; +import net.i2p.util.ConcurrentHashSet; import net.i2p.util.I2PAppThread; import net.i2p.util.Log; import net.i2p.util.OrderedProperties; @@ -34,6 +35,8 @@ public class SnarkManager implements Snark.CompleteListener { /** map of (canonical) filename of the .torrent file to Snark instance (unsynchronized) */ private final Map _snarks; + /** used to prevent DirMonitor from deleting torrents that don't have a torrent file yet */ + private final Set _magnets; private final Object _addSnarkLock; private /* FIXME final FIXME */ File _configFile; private Properties _config; @@ -72,6 +75,7 @@ public class SnarkManager implements Snark.CompleteListener { public static final int DEFAULT_STARTUP_DELAY = 3; private SnarkManager() { _snarks = new HashMap(); + _magnets = new ConcurrentHashSet(); _addSnarkLock = new Object(); _context = I2PAppContext.getGlobalContext(); _log = _context.logManager().getLog(SnarkManager.class); @@ -90,8 +94,6 @@ public class SnarkManager implements Snark.CompleteListener { _running = true; _peerCoordinatorSet = new PeerCoordinatorSet(); _connectionAcceptor = new ConnectionAcceptor(_util); - int minutes = getStartupDelayMinutes(); - _messages.add(_("Adding torrents in {0} minutes", minutes)); _monitor = new I2PAppThread(new DirMonitor(), "Snark DirMonitor", true); _monitor.start(); _context.addShutdownTask(new SnarkManagerShutdown()); @@ -321,7 +323,7 @@ public class SnarkManager implements Snark.CompleteListener { _util.setStartupDelay(minutes); changed = true; _config.setProperty(PROP_STARTUP_DELAY, "" + minutes); - addMessage(_("Startup delay limit changed to {0} minutes", minutes)); + addMessage(_("Startup delay changed to {0}", DataHelper.formatDuration2(minutes * 60 * 1000))); } } @@ -549,6 +551,7 @@ public class SnarkManager implements Snark.CompleteListener { addMessage(rejectMessage); return; } else { + // TODO load saved closest DHT nodes and pass to the Snark ? torrent = new Snark(_util, filename, null, -1, null, null, this, _peerCoordinatorSet, _connectionAcceptor, false, dataDir.getPath()); @@ -583,6 +586,7 @@ public class SnarkManager implements Snark.CompleteListener { * * @param name hex or b32 name from the magnet link * @param ih 20 byte info hash + * @throws RuntimeException via Snark.fatal() * @since 0.8.4 */ public void addMagnet(String name, byte[] ih) { @@ -590,7 +594,8 @@ public class SnarkManager implements Snark.CompleteListener { _peerCoordinatorSet, _connectionAcceptor, false, getDataDir().getPath()); - // TODO tell the dir monitor not to delete us + // Tell the dir monitor not to delete us + _magnets.add(name); synchronized (_snarks) { _snarks.put(name, torrent); } @@ -608,12 +613,17 @@ public class SnarkManager implements Snark.CompleteListener { } /** - * Delete a torrent with the info hash alone (magnet / maggot) + * Stop and delete a torrent running in magnet mode * - * @param ih 20 byte info hash + * @param snark a torrent with a fake file name ("Magnet xxxx") * @since 0.8.4 */ - public void deleteMagnet(byte[] ih) { + public void deleteMagnet(Snark snark) { + synchronized (_snarks) { + _snarks.remove(snark.getName()); + } + snark.stopTorrent(); + _magnets.remove(snark.getName()); } /** @@ -748,6 +758,8 @@ public class SnarkManager implements Snark.CompleteListener { _config.remove(prop); } + // TODO save closest DHT nodes too + saveConfig(); } @@ -828,6 +840,23 @@ public class SnarkManager implements Snark.CompleteListener { } return torrent; } + + /** + * Stop the torrent, leaving it on the list of torrents unless told to remove it + * @since 0.8.4 + */ + public void stopTorrent(Snark torrent, boolean shouldRemove) { + if (shouldRemove) { + synchronized (_snarks) { + _snarks.remove(torrent.getName()); + } + } + boolean wasStopped = torrent.isStopped(); + torrent.stopTorrent(); + if (!wasStopped) + addMessage(_("Torrent stopped: \"{0}\"", torrent.getBaseName())); + } + /** * Stop the torrent and delete the torrent file itself, but leaving the data * behind. @@ -846,11 +875,16 @@ public class SnarkManager implements Snark.CompleteListener { private class DirMonitor implements Runnable { public void run() { - try { Thread.sleep(60*1000*getStartupDelayMinutes()); } catch (InterruptedException ie) {} - // the first message was a "We are starting up in 1m" - synchronized (_messages) { - if (_messages.size() == 1) - _messages.remove(0); + // don't bother delaying if auto start is false + long delay = 60 * 1000 * getStartupDelayMinutes(); + if (delay > 0 && shouldAutoStart()) { + _messages.add(_("Adding torrents in {0}", DataHelper.formatDuration2(delay))); + try { Thread.sleep(delay); } catch (InterruptedException ie) {} + // the first message was a "We are starting up in 1m" + synchronized (_messages) { + if (_messages.size() == 1) + _messages.remove(0); + } } // here because we need to delay until I2CP is up @@ -922,6 +956,8 @@ public class SnarkManager implements Snark.CompleteListener { } } } + // Don't remove magnet torrents that don't have a torrent file yet + existingNames.removeAll(_magnets); // now lets see which ones have been removed... for (Iterator iter = existingNames.iterator(); iter.hasNext(); ) { String name = (String)iter.next(); @@ -975,12 +1011,12 @@ public class SnarkManager implements Snark.CompleteListener { /** comma delimited list of name=announceURL=baseURL for the trackers to be displayed */ public static final String PROP_TRACKERS = "i2psnark.trackers"; - private static Map trackerMap = null; + private static Map trackerMap = null; /** sorted map of name to announceURL=baseURL */ - public Map getTrackers() { + public Map getTrackers() { if (trackerMap != null) // only do this once, can't be updated while running return trackerMap; - Map rv = new TreeMap(); + Map rv = new TreeMap(); String trackers = _config.getProperty(PROP_TRACKERS); if ( (trackers == null) || (trackers.trim().length() <= 0) ) trackers = _context.getProperty(PROP_TRACKERS); diff --git a/apps/i2psnark/java/src/org/klomp/snark/TrackerClient.java b/apps/i2psnark/java/src/org/klomp/snark/TrackerClient.java index d14abaa0c9..f437beeeb6 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/TrackerClient.java +++ b/apps/i2psnark/java/src/org/klomp/snark/TrackerClient.java @@ -126,10 +126,9 @@ public class TrackerClient extends I2PAppThread @Override public void run() { - String infoHash = urlencode(meta.getInfoHash()); + String infoHash = urlencode(snark.getInfoHash()); String peerID = urlencode(snark.getID()); - _log.debug("Announce: [" + meta.getAnnounce() + "] infoHash: " + infoHash); // Construct the list of trackers for this torrent, // starting with the primary one listed in the metainfo, @@ -138,12 +137,18 @@ public class TrackerClient extends I2PAppThread // the primary tracker, that we don't add it twice. // todo: check for b32 matches as well trackers = new ArrayList(2); - String primary = meta.getAnnounce(); - if (isValidAnnounce(primary)) { - trackers.add(new Tracker(meta.getAnnounce(), true)); + String primary; + if (meta != null) { + primary = meta.getAnnounce(); + if (isValidAnnounce(primary)) { + trackers.add(new Tracker(meta.getAnnounce(), true)); + } else { + _log.warn("Skipping invalid or non-i2p announce: " + primary); + } } else { - _log.warn("Skipping invalid or non-i2p announce: " + primary); + primary = ""; } + _log.debug("Announce: [" + primary + "] infoHash: " + infoHash); List tlist = _util.getOpenTrackers(); if (tlist != null) { for (int i = 0; i < tlist.size(); i++) { diff --git a/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java b/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java index 917652bc04..c6ba8b9fd8 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java +++ b/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java @@ -462,7 +462,7 @@ public class I2PSnarkServlet extends Default { } else if (newURL.startsWith(MAGNET) || newURL.startsWith(MAGGOT)) { addMagnet(newURL); } else { - _manager.addMessage(_("Invalid URL - must start with http://, {0} or {1}", MAGNET, MAGGOT)); + _manager.addMessage(_("Invalid URL: Must start with \"http://\", \"{0}\", or \"{1}\"", MAGNET, MAGGOT)); } } else { // no file or URL specified @@ -476,7 +476,7 @@ public class I2PSnarkServlet extends Default { String name = (String)iter.next(); Snark snark = _manager.getTorrent(name); if ( (snark != null) && (DataHelper.eq(infoHash, snark.getInfoHash())) ) { - _manager.stopTorrent(name, false); + _manager.stopTorrent(snark, false); break; } } @@ -506,13 +506,14 @@ public class I2PSnarkServlet extends Default { String name = (String)iter.next(); Snark snark = _manager.getTorrent(name); if ( (snark != null) && (DataHelper.eq(infoHash, snark.getInfoHash())) ) { - _manager.stopTorrent(name, true); MetaInfo meta = snark.getMetaInfo(); if (meta == null) { - // magnet - _manager.deleteMagnet(snark.getInfoHash()); + // magnet - remove and delete are the same thing + _manager.deleteMagnet(snark); + _manager.addMessage(_("Magnet deleted: {0}", name)); return; } + _manager.stopTorrent(snark, true); // should we delete the torrent file? // yeah, need to, otherwise it'll get autoadded again (at the moment File f = new File(name); @@ -532,13 +533,14 @@ public class I2PSnarkServlet extends Default { String name = (String)iter.next(); Snark snark = _manager.getTorrent(name); if ( (snark != null) && (DataHelper.eq(infoHash, snark.getInfoHash())) ) { - _manager.stopTorrent(name, true); MetaInfo meta = snark.getMetaInfo(); if (meta == null) { - // magnet - _manager.deleteMagnet(snark.getInfoHash()); + // magnet - remove and delete are the same thing + _manager.deleteMagnet(snark); + _manager.addMessage(_("Magnet deleted: {0}", name)); return; } + _manager.stopTorrent(snark, true); File f = new File(name); f.delete(); _manager.addMessage(_("Torrent file deleted: {0}", f.getAbsolutePath())); @@ -635,7 +637,7 @@ public class I2PSnarkServlet extends Default { for (int i = 0; i < snarks.size(); i++) { Snark snark = (Snark)snarks.get(i); if (!snark.isStopped()) - _manager.stopTorrent(snark.getName(), false); + _manager.stopTorrent(snark, false); } if (_manager.util().connected()) { // Give the stopped announces time to get out @@ -750,8 +752,9 @@ public class I2PSnarkServlet extends Default { stats[5] += total; MetaInfo meta = snark.getMetaInfo(); + // isValid means isNotMagnet boolean isValid = meta != null; - boolean singleFile = (!isValid) || meta.getFiles() == null; + boolean isMultiFile = isValid && meta.getFiles() != null; String err = snark.getTrackerProblems(); int curPeers = snark.getPeerCount(); @@ -776,7 +779,7 @@ public class I2PSnarkServlet extends Default { statusString = "\"\"" + _("Tracker Error") + "
" + err; } - } else if (remaining <= 0) { + } else if (remaining == 0) { // < 0 means no meta size yet if (isRunning && curPeers > 0 && !showPeers) statusString = "\"\"		" + _("Seeding") + ": " + @@ -822,9 +825,12 @@ public class I2PSnarkServlet extends Default { out.write(""); // temporarily hardcoded for postman* and anonymity, requires bytemonsoon patch for lookup by info_hash - String announce = meta.getAnnounce(); - if (announce.startsWith("http://YRgrgTLG") || announce.startsWith("http://8EoJZIKr") || - announce.startsWith("http://lnQ6yoBT") || announce.startsWith("http://tracker2.postman.i2p/") || announce.startsWith("http://ahsplxkbhemefwvvml7qovzl5a2b5xo5i7lyai7ntdunvcyfdtna.b32.i2p/")) { + String announce = null; + if (isValid) + announce = meta.getAnnounce(); + if (announce != null && (announce.startsWith("http://YRgrgTLG") || announce.startsWith("http://8EoJZIKr") || + announce.startsWith("http://lnQ6yoBT") || announce.startsWith("http://tracker2.postman.i2p/") || + announce.startsWith("http://ahsplxkbhemefwvvml7qovzl5a2b5xo5i7lyai7ntdunvcyfdtna.b32.i2p/"))) { Map trackers = _manager.getTrackers(); for (Iterator iter = trackers.entrySet().iterator(); iter.hasNext(); ) { Map.Entry entry = (Map.Entry)iter.next(); @@ -849,13 +855,13 @@ public class I2PSnarkServlet extends Default { out.write(""); StringBuilder buf = null; - if (remaining == 0 || meta.getFiles() != null) { + if (remaining == 0 || isMultiFile) { buf = new StringBuilder(128); buf.append(""); } else { out.write(toImg(icon)); } out.write(""); - if (remaining == 0 || meta.getFiles() != null) + if (remaining == 0 || isMultiFile) out.write(buf.toString()); out.write(filename); - if (remaining == 0 || meta.getFiles() != null) + if (remaining == 0 || isMultiFile) out.write(""); out.write(""); @@ -887,19 +896,21 @@ public class I2PSnarkServlet extends Default { out.write(""); if (remaining > 0) out.write(formatSize(total-remaining) + thinsp(isDegraded) + formatSize(total)); - else + else if (remaining == 0) out.write(formatSize(total)); // 3GB + else + out.write("??"); // no meta size yet out.write(""); - if(isRunning) + if(isRunning && isValid) out.write(formatSize(uploaded)); out.write(""); - if(isRunning && remaining > 0) + if(isRunning && remaining != 0) out.write(formatSize(downBps) + "ps"); out.write(""); - if(isRunning) + if(isRunning && isValid) out.write(formatSize(upBps) + "ps"); out.write(""); @@ -919,7 +930,6 @@ public class I2PSnarkServlet extends Default { if (isDegraded) out.write(""); } else { - if (isValid) { if (isDegraded) out.write(""); if (isDegraded) out.write(""); - } - if (isDegraded) - out.write("\"");"); - if (isDegraded) - out.write(""); + if (isValid) { + if (isDegraded) + out.write("\"");"); + if (isDegraded) + out.write(""); + } if (isDegraded) out.write(""); out.write(""); - float pct = (float) (100.0 * (float) peer.completed() / meta.getPieces()); - if (pct == 100.0) - out.write(_("Seed")); - else { - String ps = String.valueOf(pct); - if (ps.length() > 5) - ps = ps.substring(0, 5); - out.write(ps + "%"); + float pct; + if (isValid) { + pct = (float) (100.0 * (float) peer.completed() / meta.getPieces()); + if (pct == 100.0) + out.write(_("Seed")); + else { + String ps = String.valueOf(pct); + if (ps.length() > 5) + ps = ps.substring(0, 5); + out.write(ps + "%"); + } + } else { + pct = (float) 101.0; + // until we get the metainfo we don't know how many pieces there are + out.write("??"); } out.write(""); @@ -1031,7 +1049,7 @@ public class I2PSnarkServlet extends Default { } out.write(""); - if (pct != 100.0) { + if (isValid && pct < 100.0) { if (peer.isInterested() && !peer.isChoking()) { out.write(""); out.write(formatSize(peer.getUploadRate()) + "ps"); @@ -1363,7 +1381,7 @@ public class I2PSnarkServlet extends Default { _manager.addMessage(_("Invalid info hash in magnet URL {0}", url)); return; } - _manager.addMagnet(ihash, ih); + _manager.addMagnet(name, ih); } /** copied from ConfigTunnelsHelper */ From ad00c16f857ef80929cb466601c60600c3763d53 Mon Sep 17 00:00:00 2001 From: zzz Date: Mon, 20 Dec 2010 19:36:49 +0000 Subject: [PATCH 05/27] fix NPE --- .../i2psnark/java/src/org/klomp/snark/PeerCheckerTask.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/apps/i2psnark/java/src/org/klomp/snark/PeerCheckerTask.java b/apps/i2psnark/java/src/org/klomp/snark/PeerCheckerTask.java index d6e2c60895..65a8698453 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/PeerCheckerTask.java +++ b/apps/i2psnark/java/src/org/klomp/snark/PeerCheckerTask.java @@ -244,8 +244,9 @@ class PeerCheckerTask extends TimerTask coordinator.setRateHistory(uploaded, downloaded); // close out unused files, but we don't need to do it every time - if (random.nextInt(4) == 0) - coordinator.getStorage().cleanRAFs(); - + Storage storage = coordinator.getStorage(); + if (storage != null && random.nextInt(4) == 0) { + storage.cleanRAFs(); + } } } From 8451610737e5577aff530d2c1567a17eaea566a5 Mon Sep 17 00:00:00 2001 From: zzz Date: Mon, 20 Dec 2010 19:37:38 +0000 Subject: [PATCH 06/27] Several connect-to-self checks --- .../org/klomp/snark/ConnectionAcceptor.java | 5 ++++ .../src/org/klomp/snark/I2PSnarkUtil.java | 26 +++++++++++++------ .../java/src/org/klomp/snark/Peer.java | 4 +++ 3 files changed, 27 insertions(+), 8 deletions(-) diff --git a/apps/i2psnark/java/src/org/klomp/snark/ConnectionAcceptor.java b/apps/i2psnark/java/src/org/klomp/snark/ConnectionAcceptor.java index 52099d1d2e..34fa5054ea 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/ConnectionAcceptor.java +++ b/apps/i2psnark/java/src/org/klomp/snark/ConnectionAcceptor.java @@ -137,6 +137,11 @@ public class ConnectionAcceptor implements Runnable } } } else { + if (socket.getPeerDestination().equals(_util.getMyDestination())) { + _util.debug("Incoming connection from myself", Snark.ERROR); + try { socket.close(); } catch (IOException ioe) {} + continue; + } Thread t = new I2PAppThread(new Handler(socket), "I2PSnark incoming connection"); t.start(); } diff --git a/apps/i2psnark/java/src/org/klomp/snark/I2PSnarkUtil.java b/apps/i2psnark/java/src/org/klomp/snark/I2PSnarkUtil.java index e8ff0881fa..c0c272ae79 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/I2PSnarkUtil.java +++ b/apps/i2psnark/java/src/org/klomp/snark/I2PSnarkUtil.java @@ -227,6 +227,8 @@ public class I2PSnarkUtil { Destination addr = peer.getAddress(); if (addr == null) throw new IOException("Null address"); + if (addr.equals(getMyDestination())) + throw new IOException("Attempt to connect to myself"); Hash dest = addr.calculateHash(); if (_shitlist.contains(dest)) throw new IOException("Not trying to contact " + dest.toBase64() + ", as they are shitlisted"); @@ -300,17 +302,25 @@ public class I2PSnarkUtil { } String getOurIPString() { - if (_manager == null) - return "unknown"; - I2PSession sess = _manager.getSession(); - if (sess != null) { - Destination dest = sess.getMyDestination(); - if (dest != null) - return dest.toBase64(); - } + Destination dest = getMyDestination(); + if (dest != null) + return dest.toBase64(); return "unknown"; } + /** + * @return dest or null + * @since 0.8.4 + */ + Destination getMyDestination() { + if (_manager == null) + return null; + I2PSession sess = _manager.getSession(); + if (sess != null) + return sess.getMyDestination(); + return null; + } + /** Base64 only - static (no naming service) */ static Destination getDestinationFromBase64(String ip) { if (ip == null) return null; diff --git a/apps/i2psnark/java/src/org/klomp/snark/Peer.java b/apps/i2psnark/java/src/org/klomp/snark/Peer.java index a65e458876..f5d200fdad 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/Peer.java +++ b/apps/i2psnark/java/src/org/klomp/snark/Peer.java @@ -30,6 +30,7 @@ import java.util.Arrays; import java.util.List; import net.i2p.client.streaming.I2PSocket; +import net.i2p.data.DataHelper; import net.i2p.util.Log; public class Peer implements Comparable @@ -353,6 +354,9 @@ public class Peer implements Comparable if (_log.shouldLog(Log.DEBUG)) _log.debug("Read the remote side's hash and peerID fully from " + toString()); + if (DataHelper.eq(my_id, bs)) + throw new IOException("Connected to myself"); + if (options != 0) { // send them something in runConnection() above if (_log.shouldLog(Log.DEBUG)) From 8e40b35210254719ee14dd085a65817d1bba8558 Mon Sep 17 00:00:00 2001 From: zzz Date: Tue, 21 Dec 2010 03:04:10 +0000 Subject: [PATCH 07/27] metadata handling - untested, still some stubs --- .../src/org/klomp/snark/ExtensionHandler.java | 224 ++++++++++++++++++ .../org/klomp/snark/ExtensionHandshake.java | 36 --- .../java/src/org/klomp/snark/MagnetState.java | 204 ++++++++++++++++ .../java/src/org/klomp/snark/MetaInfo.java | 56 +++-- .../java/src/org/klomp/snark/Peer.java | 51 +++- .../src/org/klomp/snark/PeerConnectionIn.java | 28 +-- .../org/klomp/snark/PeerConnectionOut.java | 2 +- .../src/org/klomp/snark/PeerCoordinator.java | 28 ++- .../java/src/org/klomp/snark/PeerState.java | 12 + .../src/org/klomp/snark/SnarkManager.java | 10 +- 10 files changed, 572 insertions(+), 79 deletions(-) create mode 100644 apps/i2psnark/java/src/org/klomp/snark/ExtensionHandler.java delete mode 100644 apps/i2psnark/java/src/org/klomp/snark/ExtensionHandshake.java create mode 100644 apps/i2psnark/java/src/org/klomp/snark/MagnetState.java diff --git a/apps/i2psnark/java/src/org/klomp/snark/ExtensionHandler.java b/apps/i2psnark/java/src/org/klomp/snark/ExtensionHandler.java new file mode 100644 index 0000000000..41cfcdf8d2 --- /dev/null +++ b/apps/i2psnark/java/src/org/klomp/snark/ExtensionHandler.java @@ -0,0 +1,224 @@ +package org.klomp.snark; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.HashMap; +import java.util.Map; + +import net.i2p.I2PAppContext; +import net.i2p.util.Log; + +import org.klomp.snark.bencode.BDecoder; +import org.klomp.snark.bencode.BEncoder; +import org.klomp.snark.bencode.BEValue; +import org.klomp.snark.bencode.InvalidBEncodingException; + +/** + * REF: BEP 10 Extension Protocol + * @since 0.8.2 + * @author zzz + */ +abstract class ExtensionHandler { + + private static final byte[] _handshake = buildHandshake(); + private static final Log _log = I2PAppContext.getGlobalContext().logManager().getLog(ExtensionHandler.class); + + public static final int ID_METADATA = 3; + private static final String TYPE_METADATA = "ut_metadata"; + private static final int MAX_METADATA_SIZE = Storage.MAX_PIECES * 32 * 5 / 4; + private static final int PARALLEL_REQUESTS = 3; + + + /** + * @return bencoded outgoing handshake message + */ + public static byte[] getHandshake() { + return _handshake; + } + + /** outgoing handshake message */ + private static byte[] buildHandshake() { + Map handshake = new HashMap(); + Map m = new HashMap(); + m.put(TYPE_METADATA, Integer.valueOf(ID_METADATA)); + handshake.put("m", m); + handshake.put("p", Integer.valueOf(6881)); + handshake.put("v", "I2PSnark"); + handshake.put("reqq", Integer.valueOf(5)); + return BEncoder.bencode(handshake); + } + + public static void handleMessage(Peer peer, int id, byte[] bs) { + if (id == 0) + handleHandshake(peer, bs); + else if (id == ID_METADATA) + handleMetadata(peer, bs); + else if (_log.shouldLog(Log.INFO)) + _log.info("Unknown extension msg " + id + " from " + peer); + } + + private static void handleHandshake(Peer peer, byte[] bs) { + if (_log.shouldLog(Log.DEBUG)) + _log.debug("Got handshake msg from " + peer); + try { + // this throws NPE on missing keys + InputStream is = new ByteArrayInputStream(bs); + BDecoder dec = new BDecoder(is); + BEValue bev = dec.bdecodeMap(); + Map map = bev.getMap(); + Map msgmap = map.get("m").getMap(); + peer.setHandshakeMap(map); + + // not used, just to throw out of here + int hisMsgCode = peer.getHandshakeMap().get("m").getMap().get(TYPE_METADATA).getInt(); + + int metaSize = map.get("metadata_size").getInt(); + MagnetState state = peer.getMagnetState(); + int remaining; + synchronized(state) { + if (state.isComplete()) + return; + + if (state.isInitialized()) { + if (state.getSize() != metaSize) { + peer.disconnect(); + return; + } + } else { + // initialize it + if (metaSize > MAX_METADATA_SIZE) { + peer.disconnect(false); + return; + } + if (_log.shouldLog(Log.INFO)) + _log.info("Initialized state, metadata size = " + metaSize + " from " + peer); + state.initialize(metaSize); + } + remaining = state.chunksRemaining(); + } + + // send requests for chunks + int count = Math.min(remaining, PARALLEL_REQUESTS); + for (int i = 0; i < count; i++) { + int chk; + synchronized(state) { + chk = state.getNextRequest(); + } + if (_log.shouldLog(Log.INFO)) + _log.info("Request chunk " + chk + " from " + peer); + sendRequest(peer, chk); + } + } catch (Exception e) { + if (_log.shouldLog(Log.WARN)) + _log.info("Handshake exception from " + peer, e); + //peer.disconnect(false); + } + } + + private static final int TYPE_REQUEST = 0; + private static final int TYPE_DATA = 1; + private static final int TYPE_REJECT = 2; + + private static final int CHUNK_SIZE = 16*1024; + /** 25% extra for file names, benconding overhead, etc */ + + /** + * REF: BEP 9 + * @since 0.8.4 + */ + private static void handleMetadata(Peer peer, byte[] bs) { + if (_log.shouldLog(Log.DEBUG)) + _log.debug("Got metadata msg from " + peer); + try { + InputStream is = new ByteArrayInputStream(bs); + BDecoder dec = new BDecoder(is); + BEValue bev = dec.bdecodeMap(); + Map map = bev.getMap(); + int type = map.get("msg_type").getInt(); + int piece = map.get("piece").getInt(); + + MagnetState state = peer.getMagnetState(); + if (type == TYPE_REQUEST) { + byte[] pc; + synchronized(state) { + pc = state.getChunk(piece); + } + sendPiece(peer, piece, pc); + } else if (type == TYPE_DATA) { + int size = map.get("total_size").getInt(); + boolean done; + int chk = -1; + synchronized(state) { + if (state.isComplete()) + return; + int len = is.available(); + done = state.saveChunk(piece, bs, bs.length - len, len); + if (_log.shouldLog(Log.INFO)) + _log.info("Got chunk " + piece + " from " + peer); + if (!done) + chk = state.getNextRequest(); + } + // out of the lock + if (done) { + // Done! + // PeerState will call the listener (peer coord), who will + // check to see if the MagnetState has it + if (_log.shouldLog(Log.WARN)) + _log.warn("Got last chunk from " + peer); + } else { + // get the next chunk + if (_log.shouldLog(Log.INFO)) + _log.info("Request chunk " + chk + " from " + peer); + sendRequest(peer, chk); + } + } else if (type == TYPE_REJECT) { + if (_log.shouldLog(Log.WARN)) + _log.warn("Got reject msg from " + peer); + peer.disconnect(false); + } else { + if (_log.shouldLog(Log.WARN)) + _log.warn("Got unknown metadata msg from " + peer); + peer.disconnect(false); + } + } catch (Exception e) { + if (_log.shouldLog(Log.WARN)) + _log.info("Metadata ext. msg. exception from " + peer, e); + peer.disconnect(false); + } + } + + private static void sendRequest(Peer peer, int piece) { + Map map = new HashMap(); + map.put("msg_type", TYPE_REQUEST); + map.put("piece", Integer.valueOf(piece)); + byte[] payload = BEncoder.bencode(map); + try { + int hisMsgCode = peer.getHandshakeMap().get("m").getMap().get(TYPE_METADATA).getInt(); + peer.sendExtension(hisMsgCode, payload); + } catch (Exception e) { + // NPE, no metadata capability + if (_log.shouldLog(Log.WARN)) + _log.info("Metadata send req msg exception to " + peer, e); + } + } + + private static void sendPiece(Peer peer, int piece, byte[] data) { + Map map = new HashMap(); + map.put("msg_type", TYPE_REQUEST); + map.put("piece", Integer.valueOf(piece)); + map.put("total_size", Integer.valueOf(data.length)); + byte[] dict = BEncoder.bencode(map); + byte[] payload = new byte[dict.length + data.length]; + System.arraycopy(dict, 0, payload, 0, dict.length); + System.arraycopy(data, 0, payload, dict.length, payload.length); + try { + int hisMsgCode = peer.getHandshakeMap().get("m").getMap().get("METADATA").getInt(); + peer.sendExtension(hisMsgCode, payload); + } catch (Exception e) { + // NPE, no metadata caps + if (_log.shouldLog(Log.WARN)) + _log.info("Metadata send piece msg exception to " + peer, e); + } + } +} diff --git a/apps/i2psnark/java/src/org/klomp/snark/ExtensionHandshake.java b/apps/i2psnark/java/src/org/klomp/snark/ExtensionHandshake.java deleted file mode 100644 index fb69b044d2..0000000000 --- a/apps/i2psnark/java/src/org/klomp/snark/ExtensionHandshake.java +++ /dev/null @@ -1,36 +0,0 @@ -package org.klomp.snark; - -import java.util.HashMap; -import java.util.Map; - -import org.klomp.snark.bencode.BEncoder; -import org.klomp.snark.bencode.BEValue; - -/** - * REF: BEP 10 Extension Protocol - * @since 0.8.2 - */ -class ExtensionHandshake { - - private static final byte[] _payload = buildPayload(); - - /** - * @return bencoded data - */ - static byte[] getPayload() { - return _payload; - } - - /** just a test for now */ - private static byte[] buildPayload() { - Map handshake = new HashMap(); - Map m = new HashMap(); - m.put("foo", Integer.valueOf(99)); - m.put("bar", Integer.valueOf(101)); - handshake.put("m", m); - handshake.put("p", Integer.valueOf(6881)); - handshake.put("v", "I2PSnark"); - handshake.put("reqq", Integer.valueOf(5)); - return BEncoder.bencode(handshake); - } -} diff --git a/apps/i2psnark/java/src/org/klomp/snark/MagnetState.java b/apps/i2psnark/java/src/org/klomp/snark/MagnetState.java new file mode 100644 index 0000000000..fb7e5a542d --- /dev/null +++ b/apps/i2psnark/java/src/org/klomp/snark/MagnetState.java @@ -0,0 +1,204 @@ +package org.klomp.snark; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.HashMap; +import java.util.Map; +import java.util.Random; + +import net.i2p.I2PAppContext; +import net.i2p.data.DataHelper; + +import org.klomp.snark.bencode.BDecoder; +import org.klomp.snark.bencode.BEValue; + +/** + * Simple state for the download of the metainfo, shared between + * Peer and ExtensionHandler. + * + * Nothing is synchronized here! + * Caller must synchronize on this for everything! + * + * Reference: BEP 9 + * + * @since 0.8.4 + * author zzz + */ +class MagnetState { + public static final int CHUNK_SIZE = 16*1024; + private static final Random random = I2PAppContext.getGlobalContext().random(); + + private final byte[] infohash; + private boolean complete; + /** if false, nothing below is valid */ + private boolean isInitialized; + + private int metaSize; + private int totalChunks; + /** bitfield for the metainfo chunks - will remain null if we start out complete */ + private BitField requested; + private BitField have; + /** bitfield for the metainfo */ + private byte[] metainfoBytes; + /** only valid when finished */ + private MetaInfo metainfo; + + /** + * @param meta null for new magnet + */ + public MagnetState(byte[] iHash, MetaInfo meta) { + infohash = iHash; + if (meta != null) { + metainfo = meta; + initialize(meta.getInfoBytes().length); + complete = true; + } else { + metainfoBytes = new byte[metaSize]; + } + } + + /** + * @param call this for a new magnet when you have the size + * @throws IllegalArgumentException + */ + public void initialize(int size) { + if (isInitialized) + throw new IllegalArgumentException("already set"); + isInitialized = true; + metaSize = size; + totalChunks = (size + (CHUNK_SIZE - 1)) / CHUNK_SIZE; + if (metainfo == null) { + // we don't need these if complete + have = new BitField(totalChunks); + requested = new BitField(totalChunks); + } + } + + /** + * @param Call this for a new magnet when the download is complete. + * @throws IllegalArgumentException + */ + public void setMetaInfo(MetaInfo meta) { + metainfo = meta; + } + + /** + * @throws IllegalArgumentException + */ + public MetaInfo getMetaInfo() { + if (!complete) + throw new IllegalArgumentException("not complete"); + return metainfo; + } + + /** + * @throws IllegalArgumentException + */ + public int getSize() { + if (!isInitialized) + throw new IllegalArgumentException("not initialized"); + return metaSize; + } + + public boolean isInitialized() { + return isInitialized; + } + + public boolean isComplete() { + return complete; + } + + public int chunkSize(int chunk) { + return Math.min(CHUNK_SIZE, metaSize - (chunk * CHUNK_SIZE)); + } + + /** @return chunk count */ + public int chunksRemaining() { + if (!isInitialized) + throw new IllegalArgumentException("not initialized"); + if (complete) + return 0; + return totalChunks - have.count(); + } + + /** @return chunk number */ + public int getNextRequest() { + if (!isInitialized) + throw new IllegalArgumentException("not initialized"); + if (complete) + throw new IllegalArgumentException("complete"); + int rand = random.nextInt(totalChunks); + for (int i = 0; i < totalChunks; i++) { + int chk = (i + rand) % totalChunks; + if (!(have.get(chk) || requested.get(chk))) { + requested.set(chk); + return chk; + } + } + // all requested - end game + for (int i = 0; i < totalChunks; i++) { + int chk = (i + rand) % totalChunks; + if (!have.get(chk)) + return chk; + } + throw new IllegalArgumentException("complete"); + } + + /** + * @throws IllegalArgumentException + */ + public byte[] getChunk(int chunk) { + if (!complete) + throw new IllegalArgumentException("not complete"); + if (chunk < 0 || chunk >= totalChunks) + throw new IllegalArgumentException("bad chunk number"); + int size = chunkSize(chunk); + byte[] rv = new byte[size]; + System.arraycopy(metainfoBytes, chunk * CHUNK_SIZE, rv, 0, size); + // use meta.getInfoBytes() so we don't save it in memory + return rv; + } + + /** + * @return true if this was the last piece + * @throws NPE, IllegalArgumentException, IOException, ... + */ + public boolean saveChunk(int chunk, byte[] data, int off, int length) throws Exception { + if (!isInitialized) + throw new IllegalArgumentException("not initialized"); + if (chunk < 0 || chunk >= totalChunks) + throw new IllegalArgumentException("bad chunk number"); + if (have.get(chunk)) + return false; // shouldn't happen if synced + int size = chunkSize(chunk); + if (size != length) + throw new IllegalArgumentException("bad chunk length"); + System.arraycopy(data, off, metainfoBytes, chunk * CHUNK_SIZE, size); + have.set(chunk); + boolean done = have.complete(); + if (done) { + metainfo = buildMetaInfo(); + complete = true; + } + return done; + } + + /** + * @return true if this was the last piece + * @throws NPE, IllegalArgumentException, IOException, ... + */ + public MetaInfo buildMetaInfo() throws Exception { + // top map has nothing in it but the info map (no announce) + Map map = new HashMap(); + InputStream is = new ByteArrayInputStream(metainfoBytes); + BDecoder dec = new BDecoder(is); + BEValue bev = dec.bdecodeMap(); + Map info = bev.getMap(); + map.put("info", info); + MetaInfo newmeta = new MetaInfo(map); + if (!DataHelper.eq(newmeta.getInfoHash(), infohash)) + throw new IOException("info hash mismatch"); + return newmeta; + } +} diff --git a/apps/i2psnark/java/src/org/klomp/snark/MetaInfo.java b/apps/i2psnark/java/src/org/klomp/snark/MetaInfo.java index 140d3cb468..a2522cfc4c 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/MetaInfo.java +++ b/apps/i2psnark/java/src/org/klomp/snark/MetaInfo.java @@ -25,6 +25,7 @@ import java.io.InputStream; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.Iterator; import java.util.List; @@ -59,10 +60,11 @@ public class MetaInfo private final int piece_length; private final byte[] piece_hashes; private final long length; - private final Map infoMap; - - private byte[] torrentdata; + private Map infoMap; + /** + * Called by Storage when creating a new torrent from local data + */ MetaInfo(String announce, String name, String name_utf8, List files, List lengths, int piece_length, byte[] piece_hashes, long length) { @@ -77,7 +79,7 @@ public class MetaInfo this.length = length; this.info_hash = calculateInfoHash(); - infoMap = null; + //infoMap = null; } /** @@ -104,7 +106,7 @@ public class MetaInfo * Creates a new MetaInfo from a Map of BEValues and the SHA1 over * the original bencoded info dictonary (this is a hack, we could * reconstruct the bencoded stream and recalculate the hash). Will - * throw a InvalidBEncodingException if the given map does not + * NOT throw a InvalidBEncodingException if the given map does not * contain a valid announce string or info dictonary. */ public MetaInfo(Map m) throws InvalidBEncodingException @@ -112,9 +114,13 @@ public class MetaInfo if (_log.shouldLog(Log.DEBUG)) _log.debug("Creating a metaInfo: " + m, new Exception("source")); BEValue val = (BEValue)m.get("announce"); - if (val == null) - throw new InvalidBEncodingException("Missing announce string"); - this.announce = val.getString(); + // Disabled check, we can get info from a magnet now + if (val == null) { + //throw new InvalidBEncodingException("Missing announce string"); + this.announce = null; + } else { + this.announce = val.getString(); + } val = (BEValue)m.get("info"); if (val == null) @@ -215,6 +221,7 @@ public class MetaInfo /** * Returns the string representing the URL of the tracker for this torrent. + * @return may be null! */ public String getAnnounce() { @@ -388,26 +395,34 @@ public class MetaInfo piece_hashes, length); } - public byte[] getTorrentData() + /** + * Called by servlet to save a new torrent file generated from local data + */ + public synchronized byte[] getTorrentData() { - if (torrentdata == null) - { Map m = new HashMap(); m.put("announce", announce); Map info = createInfoMap(); m.put("info", info); - torrentdata = BEncoder.bencode(m); - } - return torrentdata; + // don't save this locally, we should only do this once + return BEncoder.bencode(m); } - private Map createInfoMap() + /** @since 0.8.4 */ + public synchronized byte[] getInfoBytes() { + if (infoMap == null) + createInfoMap(); + return BEncoder.bencode(infoMap); + } + + /** @return an unmodifiable view of the Map */ + private Map createInfoMap() { + // if we loaded this metainfo from a file, we have the map + if (infoMap != null) + return Collections.unmodifiableMap(infoMap); + // otherwise we must create it Map info = new HashMap(); - if (infoMap != null) { - info.putAll(infoMap); - return info; - } info.put("name", name); if (name_utf8 != null) info.put("name.utf-8", name_utf8); @@ -429,7 +444,8 @@ public class MetaInfo } info.put("files", l); } - return info; + infoMap = info; + return Collections.unmodifiableMap(infoMap); } private byte[] calculateInfoHash() diff --git a/apps/i2psnark/java/src/org/klomp/snark/Peer.java b/apps/i2psnark/java/src/org/klomp/snark/Peer.java index f5d200fdad..681a1a5806 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/Peer.java +++ b/apps/i2psnark/java/src/org/klomp/snark/Peer.java @@ -28,11 +28,14 @@ import java.io.InputStream; import java.io.OutputStream; import java.util.Arrays; import java.util.List; +import java.util.Map; import net.i2p.client.streaming.I2PSocket; import net.i2p.data.DataHelper; import net.i2p.util.Log; +import org.klomp.snark.bencode.BEValue; + public class Peer implements Comparable { private Log _log = new Log(Peer.class); @@ -41,7 +44,9 @@ public class Peer implements Comparable private final byte[] my_id; private final byte[] infohash; - final MetaInfo metainfo; + /** will start out null in magnet mode */ + private MetaInfo metainfo; + private Map handshakeMap; // The data in/output streams set during the handshake and used by // the actual connections. @@ -52,6 +57,9 @@ public class Peer implements Comparable // was successful, the connection setup and runs PeerState state; + /** shared across all peers on this torrent */ + MagnetState magnetState; + private I2PSocket sock; private boolean deregister = true; @@ -197,7 +205,7 @@ public class Peer implements Comparable * If the given BitField is non-null it is send to the peer as first * message. */ - public void runConnection(I2PSnarkUtil util, PeerListener listener, BitField bitfield) + public void runConnection(I2PSnarkUtil util, PeerListener listener, BitField bitfield, MagnetState mState) { if (state != null) throw new IllegalStateException("Peer already started"); @@ -255,7 +263,7 @@ public class Peer implements Comparable if ((options & OPTION_EXTENSION) != 0) { if (_log.shouldLog(Log.DEBUG)) _log.debug("Peer supports extensions, sending test message"); - out.sendExtension(0, ExtensionHandshake.getPayload()); + out.sendExtension(0, ExtensionHandler.getHandshake()); } if ((options & OPTION_DHT) != 0 && util.getDHT() != null) { @@ -271,6 +279,7 @@ public class Peer implements Comparable // We are up and running! state = s; + magnetState = mState; listener.connected(this); if (_log.shouldLog(Log.DEBUG)) @@ -371,6 +380,42 @@ public class Peer implements Comparable return options; } + /** + * Shared state across all peers, callers must sync on returned object + * @return non-null + * @since 0.8.4 + */ + public MagnetState getMagnetState() { + return magnetState; + } + + /** @return could be null @since 0.8.4 */ + public Map getHandshakeMap() { + return handshakeMap; + } + + /** @since 0.8.4 */ + public void setHandshakeMap(Map map) { + handshakeMap = map; + } + + /** @since 0.8.4 */ + public void sendExtension(int type, byte[] payload) { + PeerState s = state; + if (s != null) + s.out.sendExtension(type, payload); + } + + /** + * Switch from magnet mode to normal mode + * @since 0.8.4 + */ + public void gotMetaInfo(MetaInfo meta) { + PeerState s = state; + if (s != null) + s.gotMetaInfo(meta); + } + public boolean isConnected() { return state != null; diff --git a/apps/i2psnark/java/src/org/klomp/snark/PeerConnectionIn.java b/apps/i2psnark/java/src/org/klomp/snark/PeerConnectionIn.java index 90755a9371..a27fc1c9dd 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/PeerConnectionIn.java +++ b/apps/i2psnark/java/src/org/klomp/snark/PeerConnectionIn.java @@ -90,7 +90,7 @@ class PeerConnectionIn implements Runnable { ps.keepAliveMessage(); if (_log.shouldLog(Log.DEBUG)) - _log.debug("Received keepalive from " + peer + " on " + peer.metainfo.getName()); + _log.debug("Received keepalive from " + peer); continue; } @@ -102,35 +102,35 @@ class PeerConnectionIn implements Runnable case 0: ps.chokeMessage(true); if (_log.shouldLog(Log.DEBUG)) - _log.debug("Received choke from " + peer + " on " + peer.metainfo.getName()); + _log.debug("Received choke from " + peer); break; case 1: ps.chokeMessage(false); if (_log.shouldLog(Log.DEBUG)) - _log.debug("Received unchoke from " + peer + " on " + peer.metainfo.getName()); + _log.debug("Received unchoke from " + peer); break; case 2: ps.interestedMessage(true); if (_log.shouldLog(Log.DEBUG)) - _log.debug("Received interested from " + peer + " on " + peer.metainfo.getName()); + _log.debug("Received interested from " + peer); break; case 3: ps.interestedMessage(false); if (_log.shouldLog(Log.DEBUG)) - _log.debug("Received not interested from " + peer + " on " + peer.metainfo.getName()); + _log.debug("Received not interested from " + peer); break; case 4: piece = din.readInt(); ps.haveMessage(piece); if (_log.shouldLog(Log.DEBUG)) - _log.debug("Received havePiece(" + piece + ") from " + peer + " on " + peer.metainfo.getName()); + _log.debug("Received havePiece(" + piece + ") from " + peer); break; case 5: byte[] bitmap = new byte[i-1]; din.readFully(bitmap); ps.bitfieldMessage(bitmap); if (_log.shouldLog(Log.DEBUG)) - _log.debug("Received bitmap from " + peer + " on " + peer.metainfo.getName() + ": size=" + (i-1) /* + ": " + ps.bitfield */ ); + _log.debug("Received bitmap from " + peer + ": size=" + (i-1) /* + ": " + ps.bitfield */ ); break; case 6: piece = din.readInt(); @@ -138,7 +138,7 @@ class PeerConnectionIn implements Runnable len = din.readInt(); ps.requestMessage(piece, begin, len); if (_log.shouldLog(Log.DEBUG)) - _log.debug("Received request(" + piece + "," + begin + ") from " + peer + " on " + peer.metainfo.getName()); + _log.debug("Received request(" + piece + "," + begin + ") from " + peer); break; case 7: piece = din.readInt(); @@ -152,7 +152,7 @@ class PeerConnectionIn implements Runnable din.readFully(piece_bytes, begin, len); ps.pieceMessage(req); if (_log.shouldLog(Log.DEBUG)) - _log.debug("Received data(" + piece + "," + begin + ") from " + peer + " on " + peer.metainfo.getName()); + _log.debug("Received data(" + piece + "," + begin + ") from " + peer); } else { @@ -160,7 +160,7 @@ class PeerConnectionIn implements Runnable piece_bytes = new byte[len]; din.readFully(piece_bytes); if (_log.shouldLog(Log.DEBUG)) - _log.debug("Received UNWANTED data(" + piece + "," + begin + ") from " + peer + " on " + peer.metainfo.getName()); + _log.debug("Received UNWANTED data(" + piece + "," + begin + ") from " + peer); } break; case 8: @@ -169,27 +169,27 @@ class PeerConnectionIn implements Runnable len = din.readInt(); ps.cancelMessage(piece, begin, len); if (_log.shouldLog(Log.DEBUG)) - _log.debug("Received cancel(" + piece + "," + begin + ") from " + peer + " on " + peer.metainfo.getName()); + _log.debug("Received cancel(" + piece + "," + begin + ") from " + peer); break; case 9: // PORT message int port = din.readUnsignedShort(); ps.portMessage(port); if (_log.shouldLog(Log.DEBUG)) - _log.debug("Received port message from " + peer + " on " + peer.metainfo.getName()); + _log.debug("Received port message from " + peer); case 20: // Extension message int id = din.readUnsignedByte(); byte[] payload = new byte[i-2]; din.readFully(payload); ps.extensionMessage(id, payload); if (_log.shouldLog(Log.DEBUG)) - _log.debug("Received extension message from " + peer + " on " + peer.metainfo.getName()); + _log.debug("Received extension message from " + peer); break; default: byte[] bs = new byte[i-1]; din.readFully(bs); ps.unknownMessage(b, bs); if (_log.shouldLog(Log.DEBUG)) - _log.debug("Received unknown message from " + peer + " on " + peer.metainfo.getName()); + _log.debug("Received unknown message from " + peer); } } } diff --git a/apps/i2psnark/java/src/org/klomp/snark/PeerConnectionOut.java b/apps/i2psnark/java/src/org/klomp/snark/PeerConnectionOut.java index 19614bc52e..181ca49695 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/PeerConnectionOut.java +++ b/apps/i2psnark/java/src/org/klomp/snark/PeerConnectionOut.java @@ -151,7 +151,7 @@ class PeerConnectionOut implements Runnable if (m != null) { if (_log.shouldLog(Log.DEBUG)) - _log.debug("Send " + peer + ": " + m + " on " + peer.metainfo.getName()); + _log.debug("Send " + peer + ": " + m); // This can block for quite a while. // To help get slow peers going, and track the bandwidth better, diff --git a/apps/i2psnark/java/src/org/klomp/snark/PeerCoordinator.java b/apps/i2psnark/java/src/org/klomp/snark/PeerCoordinator.java index f7a3a42fc8..6eb730fc9d 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/PeerCoordinator.java +++ b/apps/i2psnark/java/src/org/klomp/snark/PeerCoordinator.java @@ -105,6 +105,7 @@ public class PeerCoordinator implements PeerListener private boolean halted = false; + private final MagnetState magnetState; private final CoordinatorListener listener; private final I2PSnarkUtil _util; private static final Random _random = I2PAppContext.getGlobalContext().random(); @@ -128,6 +129,7 @@ public class PeerCoordinator implements PeerListener setWantedPieces(); partialPieces = new ArrayList(getMaxConnections() + 1); peers = new LinkedBlockingQueue(); + magnetState = new MagnetState(infohash, metainfo); // Install a timer to check the uploaders. // Randomize the first start time so multiple tasks are spread out, @@ -484,7 +486,7 @@ public class PeerCoordinator implements PeerListener { public void run() { - peer.runConnection(_util, listener, bitfield); + peer.runConnection(_util, listener, bitfield, magnetState); } }; String threadName = "Snark peer " + peer.toString(); @@ -1149,10 +1151,30 @@ public class PeerCoordinator implements PeerListener } /** @since 0.8.4 */ - public void gotExtension(Peer peer, int id, byte[] bs) {} + public void gotExtension(Peer peer, int id, byte[] bs) { + if (_log.shouldLog(Log.DEBUG)) + _log.debug("Got extension message " + id + " from " + peer); + // basic handling done in PeerState... here we just check if we are done + if (metainfo == null && id == ExtensionHandler.ID_METADATA) { + synchronized (magnetState) { + if (magnetState.isComplete()) { + if (_log.shouldLog(Log.WARN)) + _log.warn("Got completed metainfo via extension"); + MetaInfo newinfo = magnetState.getMetaInfo(); + // more validation + // set global + // instantiate storage + // tell Snark listener + // tell all peers + } + } + } + } /** @since 0.8.4 */ - public void gotPort(Peer peer, int port) {} + public void gotPort(Peer peer, int port) { + // send to DHT + } /** Return number of allowed uploaders for this torrent. ** Check with Snark to see if we are over the total upload limit. diff --git a/apps/i2psnark/java/src/org/klomp/snark/PeerState.java b/apps/i2psnark/java/src/org/klomp/snark/PeerState.java index 3ec3d219f6..a85207476b 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/PeerState.java +++ b/apps/i2psnark/java/src/org/klomp/snark/PeerState.java @@ -497,9 +497,21 @@ class PeerState implements DataLoader /** @since 0.8.2 */ void extensionMessage(int id, byte[] bs) { + ExtensionHandler.handleMessage(peer, id, bs); + // Peer coord will get metadata from MagnetState, + // verify, and then call gotMetaInfo() listener.gotExtension(peer, id, bs); } + /** + * Switch from magnet mode to normal mode + * @since 0.8.4 + */ + public void gotMetaInfo(MetaInfo meta) { + // set metainfo + // fix bitfield + } + /** @since 0.8.4 */ void portMessage(int port) { diff --git a/apps/i2psnark/java/src/org/klomp/snark/SnarkManager.java b/apps/i2psnark/java/src/org/klomp/snark/SnarkManager.java index 24e8ebc7f6..dca9c83242 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/SnarkManager.java +++ b/apps/i2psnark/java/src/org/klomp/snark/SnarkManager.java @@ -594,9 +594,15 @@ public class SnarkManager implements Snark.CompleteListener { _peerCoordinatorSet, _connectionAcceptor, false, getDataDir().getPath()); - // Tell the dir monitor not to delete us - _magnets.add(name); synchronized (_snarks) { + for (Snark snark : _snarks.values()) { + if (DataHelper.eq(ih, snark.getInfoHash())) { + addMessage(_("Torrent already running: {0}", snark.getBaseName())); + return; + } + } + // Tell the dir monitor not to delete us + _magnets.add(name); _snarks.put(name, torrent); } if (shouldAutoStart()) { From f15b32987406785f555ef556b909d38a1e826663 Mon Sep 17 00:00:00 2001 From: zzz Date: Tue, 21 Dec 2010 16:36:08 +0000 Subject: [PATCH 08/27] - Move some torrent file creation code from the servlet to the manager, to allow locking and prevent interference by the DirMonitor - More checks for whether torrent is already running - Consistent filename filtering in all cases - Allow null announce string - Move snarks map to a CHM - Remember last tracker selection - Add callback for reception of metainfo --- .../java/src/org/klomp/snark/MetaInfo.java | 10 +- .../java/src/org/klomp/snark/Snark.java | 10 + .../src/org/klomp/snark/SnarkManager.java | 223 ++++++++++++++++-- .../java/src/org/klomp/snark/Storage.java | 5 +- .../org/klomp/snark/web/I2PSnarkServlet.java | 59 ++--- 5 files changed, 255 insertions(+), 52 deletions(-) diff --git a/apps/i2psnark/java/src/org/klomp/snark/MetaInfo.java b/apps/i2psnark/java/src/org/klomp/snark/MetaInfo.java index a2522cfc4c..2f01fe62c2 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/MetaInfo.java +++ b/apps/i2psnark/java/src/org/klomp/snark/MetaInfo.java @@ -64,6 +64,8 @@ public class MetaInfo /** * Called by Storage when creating a new torrent from local data + * + * @param announce may be null */ MetaInfo(String announce, String name, String name_utf8, List files, List lengths, int piece_length, byte[] piece_hashes, long length) @@ -86,6 +88,7 @@ public class MetaInfo * Creates a new MetaInfo from the given InputStream. The * InputStream must start with a correctly bencoded dictonary * describing the torrent. + * Caller must close the stream. */ public MetaInfo(InputStream in) throws IOException { @@ -107,7 +110,9 @@ public class MetaInfo * the original bencoded info dictonary (this is a hack, we could * reconstruct the bencoded stream and recalculate the hash). Will * NOT throw a InvalidBEncodingException if the given map does not - * contain a valid announce string or info dictonary. + * contain a valid announce string. + * WILL throw a InvalidBEncodingException if the given map does not + * contain a valid info dictionary. */ public MetaInfo(Map m) throws InvalidBEncodingException { @@ -401,7 +406,8 @@ public class MetaInfo public synchronized byte[] getTorrentData() { Map m = new HashMap(); - m.put("announce", announce); + if (announce != null) + m.put("announce", announce); Map info = createInfoMap(); m.put("info", info); // don't save this locally, we should only do this once diff --git a/apps/i2psnark/java/src/org/klomp/snark/Snark.java b/apps/i2psnark/java/src/org/klomp/snark/Snark.java index 27e38b5ca0..fd7a88b57a 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/Snark.java +++ b/apps/i2psnark/java/src/org/klomp/snark/Snark.java @@ -1084,6 +1084,16 @@ public class Snark public interface CompleteListener { public void torrentComplete(Snark snark); public void updateStatus(Snark snark); + + /** + * We transitioned from magnet mode, we have now initialized our + * metainfo and storage. The listener should now call getMetaInfo() + * and save the data to disk. + * + * @since 0.8.4 + */ + public void gotMetaInfo(Snark snark); + // not really listeners but the easiest way to get back to an optional SnarkManager public long getSavedTorrentTime(Snark snark); public BitField getSavedTorrentBitField(Snark snark); diff --git a/apps/i2psnark/java/src/org/klomp/snark/SnarkManager.java b/apps/i2psnark/java/src/org/klomp/snark/SnarkManager.java index dca9c83242..d5a51c584c 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/SnarkManager.java +++ b/apps/i2psnark/java/src/org/klomp/snark/SnarkManager.java @@ -5,6 +5,7 @@ import java.io.FileFilter; import java.io.FileInputStream; import java.io.FilenameFilter; import java.io.IOException; +import java.io.OutputStream; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; @@ -16,15 +17,18 @@ import java.util.Set; import java.util.StringTokenizer; import java.util.TreeMap; import java.util.Collection; +import java.util.concurrent.ConcurrentHashMap; import net.i2p.I2PAppContext; import net.i2p.data.Base64; import net.i2p.data.DataHelper; import net.i2p.util.ConcurrentHashSet; +import net.i2p.util.FileUtil; import net.i2p.util.I2PAppThread; import net.i2p.util.Log; import net.i2p.util.OrderedProperties; import net.i2p.util.SecureDirectory; +import net.i2p.util.SecureFileOutputStream; /** * Manage multiple snarks @@ -33,7 +37,11 @@ public class SnarkManager implements Snark.CompleteListener { private static SnarkManager _instance = new SnarkManager(); public static SnarkManager instance() { return _instance; } - /** map of (canonical) filename of the .torrent file to Snark instance (unsynchronized) */ + /** + * Map of (canonical) filename of the .torrent file to Snark instance. + * This is a CHM so listTorrentFiles() need not be synced, but + * all adds, deletes, and the DirMonitor should sync on it. + */ private final Map _snarks; /** used to prevent DirMonitor from deleting torrents that don't have a torrent file yet */ private final Set _magnets; @@ -74,7 +82,7 @@ public class SnarkManager implements Snark.CompleteListener { public static final int DEFAULT_MAX_UP_BW = 10; public static final int DEFAULT_STARTUP_DELAY = 3; private SnarkManager() { - _snarks = new HashMap(); + _snarks = new ConcurrentHashMap(); _magnets = new ConcurrentHashSet(); _addSnarkLock = new Object(); _context = I2PAppContext.getGlobalContext(); @@ -462,8 +470,13 @@ public class SnarkManager implements Snark.CompleteListener { /** hardcoded for sanity. perhaps this should be customizable, for people who increase their ulimit, etc. */ private static final int MAX_FILES_PER_TORRENT = 512; - /** set of canonical .torrent filenames that we are dealing with */ - public Set listTorrentFiles() { synchronized (_snarks) { return new HashSet(_snarks.keySet()); } } + /** + * Set of canonical .torrent filenames that we are dealing with. + * An unsynchronized copy. + */ + public Set listTorrentFiles() { + return new HashSet(_snarks.keySet()); + } /** * Grab the torrent given the (canonical) filename of the .torrent file @@ -486,10 +499,31 @@ public class SnarkManager implements Snark.CompleteListener { return null; } - /** @throws RuntimeException via Snark.fatal() */ + /** + * Grab the torrent given the info hash + * @return Snark or null + * @since 0.8.4 + */ + public Snark getTorrentByInfoHash(byte[] infohash) { + synchronized (_snarks) { + for (Snark s : _snarks.values()) { + if (DataHelper.eq(infohash, s.getInfoHash())) + return s; + } + } + return null; + } + + /** + * Caller must verify this torrent is not already added. + * @throws RuntimeException via Snark.fatal() + */ public void addTorrent(String filename) { addTorrent(filename, false); } - /** @throws RuntimeException via Snark.fatal() */ + /** + * Caller must verify this torrent is not already added. + * @throws RuntimeException via Snark.fatal() + */ public void addTorrent(String filename, boolean dontAutoStart) { if ((!dontAutoStart) && !_util.connected()) { addMessage(_("Connecting to I2P")); @@ -545,13 +579,14 @@ public class SnarkManager implements Snark.CompleteListener { dontAutoStart = true; } } - String rejectMessage = locked_validateTorrent(info); + String rejectMessage = validateTorrent(info); if (rejectMessage != null) { sfile.delete(); addMessage(rejectMessage); return; } else { // TODO load saved closest DHT nodes and pass to the Snark ? + // This may take a LONG time torrent = new Snark(_util, filename, null, -1, null, null, this, _peerCoordinatorSet, _connectionAcceptor, false, dataDir.getPath()); @@ -595,11 +630,10 @@ public class SnarkManager implements Snark.CompleteListener { false, getDataDir().getPath()); synchronized (_snarks) { - for (Snark snark : _snarks.values()) { - if (DataHelper.eq(ih, snark.getInfoHash())) { - addMessage(_("Torrent already running: {0}", snark.getBaseName())); - return; - } + Snark snark = getTorrentByInfoHash(ih); + if (snark != null) { + addMessage(_("Torrent with this info hash is already running: {0}", snark.getBaseName())); + return; } // Tell the dir monitor not to delete us _magnets.add(name); @@ -633,7 +667,99 @@ public class SnarkManager implements Snark.CompleteListener { } /** - * Get the timestamp for a torrent from the config file + * Add a torrent from a MetaInfo. Save the MetaInfo data to filename. + * Holds the snarks lock to prevent interference from the DirMonitor. + * This verifies that a torrent with this infohash is not already added. + * This may take a LONG time to create or check the storage. + * + * @param metainfo the metainfo for the torrent + * @param bitfield the current completion status of the torrent + * @param filename the absolute path to save the metainfo to, generally ending in ".torrent", which is also the name of the torrent + * Must be a filesystem-safe name. + * @throws RuntimeException via Snark.fatal() + * @since 0.8.4 + */ + public void addTorrent(MetaInfo metainfo, BitField bitfield, String filename, boolean dontAutoStart) throws IOException { + // prevent interference by DirMonitor + synchronized (_snarks) { + Snark snark = getTorrentByInfoHash(metainfo.getInfoHash()); + if (snark != null) { + addMessage(_("Torrent with this info hash is already running: {0}", snark.getBaseName())); + return; + } + // so addTorrent won't recheck + saveTorrentStatus(metainfo, bitfield, null); // no file priorities + try { + locked_writeMetaInfo(metainfo, filename); + // hold the lock for a long time + addTorrent(filename, dontAutoStart); + } catch (IOException ioe) { + addMessage(_("Failed to copy torrent file to {0}", filename)); + _log.error("Failed to write torrent file", ioe); + } + } + } + + /** + * Add a torrent from a file not in the torrent directory. Copy the file to filename. + * Holds the snarks lock to prevent interference from the DirMonitor. + * Caller must verify this torrent is not already added. + * This may take a LONG time to create or check the storage. + * + * @param fromfile where the file is now, presumably in a temp directory somewhere + * @param filename the absolute path to save the metainfo to, generally ending in ".torrent", which is also the name of the torrent + * Must be a filesystem-safe name. + * @throws RuntimeException via Snark.fatal() + * @since 0.8.4 + */ + public void copyAndAddTorrent(File fromfile, String filename) throws IOException { + // prevent interference by DirMonitor + synchronized (_snarks) { + boolean success = FileUtil.copy(fromfile.getAbsolutePath(), filename, false); + if (!success) { + addMessage(_("Failed to copy torrent file to {0}", filename)); + _log.error("Failed to write torrent file to " + filename); + return; + } + SecureFileOutputStream.setPerms(new File(filename)); + // hold the lock for a long time + addTorrent(filename); + } + } + + /** + * Write the metainfo to the file, caller must hold the snarks lock + * to prevent interference from the DirMonitor. + * + * @param metainfo The metainfo for the torrent + * @param filename The absolute path to save the metainfo to, generally ending in ".torrent". + * Must be a filesystem-safe name. + * @since 0.8.4 + */ + private void locked_writeMetaInfo(MetaInfo metainfo, String filename) throws IOException { + // prevent interference by DirMonitor + File file = new File(filename); + if (file.exists()) + throw new IOException("Cannot overwrite an existing .torrent file: " + file.getPath()); + OutputStream out = null; + try { + out = new SecureFileOutputStream(filename); + out.write(metainfo.getTorrentData()); + } catch (IOException ioe) { + // remove any partial + file.delete(); + throw ioe; + } finally { + try { + if (out == null) + out.close(); + } catch (IOException ioe) {} + } + } + + /** + * Get the timestamp for a torrent from the config file. + * A Snark.CompleteListener method. */ public long getSavedTorrentTime(Snark snark) { byte[] ih = snark.getInfoHash(); @@ -653,6 +779,7 @@ public class SnarkManager implements Snark.CompleteListener { /** * Get the saved bitfield for a torrent from the config file. * Convert "." to a full bitfield. + * A Snark.CompleteListener method. */ public BitField getSavedTorrentBitField(Snark snark) { MetaInfo metainfo = snark.getMetaInfo(); @@ -721,6 +848,8 @@ public class SnarkManager implements Snark.CompleteListener { * The time is a standard long converted to string. * The status is either a bitfield converted to Base64 or "." for a completed * torrent to save space in the config file and in memory. + * + * @param bitfield non-null * @param priorities may be null */ public void saveTorrentStatus(MetaInfo metainfo, BitField bitfield, int[] priorities) { @@ -783,9 +912,11 @@ public class SnarkManager implements Snark.CompleteListener { } /** + * Does not really delete on failure, that's the caller's responsibility. * Warning - does not validate announce URL - use TrackerClient.isValidAnnounce() + * @return failure message or null on success */ - private String locked_validateTorrent(MetaInfo info) throws IOException { + private String validateTorrent(MetaInfo info) { List files = info.getFiles(); if ( (files != null) && (files.size() > MAX_FILES_PER_TORRENT) ) { return _("Too many files in \"{0}\" ({1}), deleting it!", info.getName(), files.size()); @@ -866,17 +997,22 @@ public class SnarkManager implements Snark.CompleteListener { /** * Stop the torrent and delete the torrent file itself, but leaving the data * behind. + * Holds the snarks lock to prevent interference from the DirMonitor. */ public void removeTorrent(String filename) { - Snark torrent = stopTorrent(filename, true); - if (torrent != null) { + Snark torrent; + // prevent interference by DirMonitor + synchronized (_snarks) { + torrent = stopTorrent(filename, true); + if (torrent == null) + return; File torrentFile = new File(filename); torrentFile.delete(); - Storage storage = torrent.getStorage(); - if (storage != null) - removeTorrentStatus(storage.getMetaInfo()); - addMessage(_("Torrent removed: \"{0}\"", torrent.getBaseName())); } + Storage storage = torrent.getStorage(); + if (storage != null) + removeTorrentStatus(storage.getMetaInfo()); + addMessage(_("Torrent removed: \"{0}\"", torrent.getBaseName())); } private class DirMonitor implements Runnable { @@ -901,7 +1037,10 @@ public class SnarkManager implements Snark.CompleteListener { if (_log.shouldLog(Log.DEBUG)) _log.debug("Directory Monitor loop over " + dir.getAbsolutePath()); try { - monitorTorrents(dir); + // Don't let this interfere with .torrent files being added or deleted + synchronized (_snarks) { + monitorTorrents(dir); + } } catch (Exception e) { _log.error("Error in the DirectoryMonitor", e); } @@ -910,7 +1049,11 @@ public class SnarkManager implements Snark.CompleteListener { } } - /** two listeners */ + // Begin Snark.CompleteListeners + + /** + * A Snark.CompleteListener method. + */ public void torrentComplete(Snark snark) { MetaInfo meta = snark.getMetaInfo(); Storage storage = snark.getStorage(); @@ -925,6 +1068,9 @@ public class SnarkManager implements Snark.CompleteListener { updateStatus(snark); } + /** + * A Snark.CompleteListener method. + */ public void updateStatus(Snark snark) { MetaInfo meta = snark.getMetaInfo(); Storage storage = snark.getStorage(); @@ -932,6 +1078,39 @@ public class SnarkManager implements Snark.CompleteListener { saveTorrentStatus(meta, storage.getBitField(), storage.getFilePriorities()); } + /** + * We transitioned from magnet mode, we have now initialized our + * metainfo and storage. The listener should now call getMetaInfo() + * and save the data to disk. + * A Snark.CompleteListener method. + * + * @since 0.8.4 + */ + public void gotMetaInfo(Snark snark) { + MetaInfo meta = snark.getMetaInfo(); + Storage storage = snark.getStorage(); + if (meta != null && storage != null) { + String rejectMessage = validateTorrent(meta); + if (rejectMessage != null) { + addMessage(rejectMessage); + snark.stopTorrent(); + return; + } + saveTorrentStatus(meta, storage.getBitField(), null); // no file priorities + String name = (new File(getDataDir(), storage.getBaseName() + ".torrent")).getAbsolutePath(); + try { + synchronized (_snarks) { + locked_writeMetaInfo(meta, name); + } + } catch (IOException ioe) { + addMessage(_("Failed to copy torrent file to {0}", name)); + _log.error("Failed to write torrent file", ioe); + } + } + } + + // End Snark.CompleteListeners + private void monitorTorrents(File dir) { String fileNames[] = dir.list(TorrentFilenameFilter.instance()); List foundNames = new ArrayList(0); diff --git a/apps/i2psnark/java/src/org/klomp/snark/Storage.java b/apps/i2psnark/java/src/org/klomp/snark/Storage.java index 17fae52352..dcd78620d9 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/Storage.java +++ b/apps/i2psnark/java/src/org/klomp/snark/Storage.java @@ -87,6 +87,9 @@ public class Storage * Creates a storage from the existing file or directory together * with an appropriate MetaInfo file as can be announced on the * given announce String location. + * + * @param announce may be null + * @param listener may be null */ public Storage(I2PSnarkUtil util, File baseFile, String announce, StorageListener listener) throws IOException @@ -590,7 +593,7 @@ public class Storage * Removes 'suspicious' characters from the given file name. * http://msdn.microsoft.com/en-us/library/aa365247%28VS.85%29.aspx */ - private static String filterName(String name) + public static String filterName(String name) { if (name.equals(".") || name.equals(" ")) return "_"; diff --git a/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java b/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java index c6ba8b9fd8..fefc808485 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java +++ b/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java @@ -59,6 +59,7 @@ public class I2PSnarkServlet extends Default { private Resource _resourceBase; private String _themePath; private String _imgPath; + private String _lastAnnounceURL = ""; public static final String PROP_CONFIG_FILE = "i2psnark.configFile"; /** BEP 9 */ @@ -605,23 +606,23 @@ public class I2PSnarkServlet extends Default { if (announceURL == null || announceURL.length() <= 0) _manager.addMessage(_("Error creating torrent - you must select a tracker")); else if (baseFile.exists()) { + _lastAnnounceURL = announceURL; + if (announceURL.equals("none")) + announceURL = null; try { + // This may take a long time to check the storage, but since it already exists, + // it shouldn't be THAT bad, so keep it in this thread. Storage s = new Storage(_manager.util(), baseFile, announceURL, null); s.create(); s.close(); // close the files... maybe need a way to pass this Storage to addTorrent rather than starting over MetaInfo info = s.getMetaInfo(); - File torrentFile = new File(baseFile.getParent(), baseFile.getName() + ".torrent"); - if (torrentFile.exists()) - throw new IOException("Cannot overwrite an existing .torrent file: " + torrentFile.getPath()); - _manager.saveTorrentStatus(info, s.getBitField(), null); // so addTorrent won't recheck - // DirMonitor could grab this first, maybe hold _snarks lock? - FileOutputStream out = new FileOutputStream(torrentFile); - out.write(info.getTorrentData()); - out.close(); + File torrentFile = new File(_manager.getDataDir(), s.getBaseName() + ".torrent"); + // FIXME is the storage going to stay around thanks to the info reference? + // now add it, but don't automatically start it + _manager.addTorrent(info, s.getBitField(), torrentFile.getAbsolutePath(), true); _manager.addMessage(_("Torrent created for \"{0}\"", baseFile.getName()) + ": " + torrentFile.getAbsolutePath()); - // now fire it up, but don't automatically seed it - _manager.addTorrent(torrentFile.getCanonicalPath(), true); - _manager.addMessage(_("Many I2P trackers require you to register new torrents before seeding - please do so before starting \"{0}\"", baseFile.getName())); + if (announceURL != null) + _manager.addMessage(_("Many I2P trackers require you to register new torrents before seeding - please do so before starting \"{0}\"", baseFile.getName())); } catch (IOException ioe) { _manager.addMessage(_("Error creating a torrent for \"{0}\"", baseFile.getAbsolutePath()) + ": " + ioe.getMessage()); } @@ -1165,6 +1166,10 @@ public class I2PSnarkServlet extends Default { out.write(":\n"); @@ -1801,15 +1808,18 @@ private static class FetchAndAdd implements Runnable { FileInputStream in = null; try { in = new FileInputStream(file); + // we do not retain this MetaInfo object, hopefully it will go away quickly MetaInfo info = new MetaInfo(in); - String name = info.getName(); - name = DataHelper.stripHTML(name); // XSS - name = name.replace('/', '_'); - name = name.replace('\\', '_'); - name = name.replace('&', '+'); - name = name.replace('\'', '_'); - name = name.replace('"', '_'); - name = name.replace('`', '_'); + try { in.close(); } catch (IOException ioe) {} + Snark snark = _manager.getTorrentByInfoHash(info.getInfoHash()); + if (snark != null) { + _manager.addMessage(_("Torrent with this info hash is already running: {0}", snark.getBaseName())); + return; + } + + // don't hold object from this MetaInfo + String name = new String(info.getName()); + name = Storage.filterName(name); name = name + ".torrent"; File torrentFile = new File(_manager.getDataDir(), name); @@ -1821,18 +1831,13 @@ private static class FetchAndAdd implements Runnable { else _manager.addMessage(_("Torrent already in the queue: {0}", name)); } else { - boolean success = FileUtil.copy(file.getAbsolutePath(), canonical, false); - if (success) { - SecureFileOutputStream.setPerms(torrentFile); - _manager.addTorrent(canonical); - } else { - _manager.addMessage(_("Failed to copy torrent file to {0}", canonical)); - } + // This may take a LONG time to create the storage. + _manager.copyAndAddTorrent(file, canonical); } } catch (IOException ioe) { _manager.addMessage(_("Torrent at {0} was not valid", urlify(_url)) + ": " + ioe.getMessage()); } finally { - try { in.close(); } catch (IOException ioe) {} + try { if (in != null) in.close(); } catch (IOException ioe) {} } } else { _manager.addMessage(_("Torrent was not retrieved from {0}", urlify(_url))); From 690aea255b03ef4cd29f8be3a20c3777a38c375b Mon Sep 17 00:00:00 2001 From: zzz Date: Tue, 21 Dec 2010 23:43:13 +0000 Subject: [PATCH 09/27] - Rework traffic counters - Record the metadata bandwidth - More null announce handling - Callbacks for got MetaInfo event - Cleanups --- .../org/klomp/snark/CoordinatorListener.java | 6 +++ .../src/org/klomp/snark/ExtensionHandler.java | 38 +++++++++++++------ .../java/src/org/klomp/snark/Peer.java | 38 +++++++++++++------ .../src/org/klomp/snark/PeerCoordinator.java | 36 +++++++++++++----- .../java/src/org/klomp/snark/PeerState.java | 33 ++++++++-------- .../java/src/org/klomp/snark/Snark.java | 24 ++++++++++++ .../src/org/klomp/snark/SnarkManager.java | 6 ++- 7 files changed, 129 insertions(+), 52 deletions(-) diff --git a/apps/i2psnark/java/src/org/klomp/snark/CoordinatorListener.java b/apps/i2psnark/java/src/org/klomp/snark/CoordinatorListener.java index a86afa781b..478c17bb50 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/CoordinatorListener.java +++ b/apps/i2psnark/java/src/org/klomp/snark/CoordinatorListener.java @@ -31,6 +31,12 @@ public interface CoordinatorListener */ void peerChange(PeerCoordinator coordinator, Peer peer); + /** + * Called when the PeerCoordinator got the MetaInfo via magnet. + * @since 0.8.4 + */ + void gotMetaInfo(PeerCoordinator coordinator, MetaInfo metainfo); + public boolean overUploadLimit(int uploaders); public boolean overUpBWLimit(); public boolean overUpBWLimit(long total); diff --git a/apps/i2psnark/java/src/org/klomp/snark/ExtensionHandler.java b/apps/i2psnark/java/src/org/klomp/snark/ExtensionHandler.java index 41cfcdf8d2..e212f6f613 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/ExtensionHandler.java +++ b/apps/i2psnark/java/src/org/klomp/snark/ExtensionHandler.java @@ -26,7 +26,8 @@ abstract class ExtensionHandler { public static final int ID_METADATA = 3; private static final String TYPE_METADATA = "ut_metadata"; - private static final int MAX_METADATA_SIZE = Storage.MAX_PIECES * 32 * 5 / 4; + /** Pieces * SHA1 Hash length, + 25% extra for file names, benconding overhead, etc */ + private static final int MAX_METADATA_SIZE = Storage.MAX_PIECES * 20 * 5 / 4; private static final int PARALLEL_REQUESTS = 3; @@ -49,16 +50,16 @@ abstract class ExtensionHandler { return BEncoder.bencode(handshake); } - public static void handleMessage(Peer peer, int id, byte[] bs) { + public static void handleMessage(Peer peer, PeerListener listener, int id, byte[] bs) { if (id == 0) - handleHandshake(peer, bs); + handleHandshake(peer, listener, bs); else if (id == ID_METADATA) - handleMetadata(peer, bs); + handleMetadata(peer, listener, bs); else if (_log.shouldLog(Log.INFO)) _log.info("Unknown extension msg " + id + " from " + peer); } - private static void handleHandshake(Peer peer, byte[] bs) { + private static void handleHandshake(Peer peer, PeerListener listener, byte[] bs) { if (_log.shouldLog(Log.DEBUG)) _log.debug("Got handshake msg from " + peer); try { @@ -67,11 +68,11 @@ abstract class ExtensionHandler { BDecoder dec = new BDecoder(is); BEValue bev = dec.bdecodeMap(); Map map = bev.getMap(); - Map msgmap = map.get("m").getMap(); peer.setHandshakeMap(map); + Map msgmap = map.get("m").getMap(); - // not used, just to throw out of here - int hisMsgCode = peer.getHandshakeMap().get("m").getMap().get(TYPE_METADATA).getInt(); + // rv not used, just to throw an NPE to get out of here + msgmap.get(TYPE_METADATA).getInt(); int metaSize = map.get("metadata_size").getInt(); MagnetState state = peer.getMagnetState(); @@ -121,13 +122,12 @@ abstract class ExtensionHandler { private static final int TYPE_REJECT = 2; private static final int CHUNK_SIZE = 16*1024; - /** 25% extra for file names, benconding overhead, etc */ /** * REF: BEP 9 * @since 0.8.4 */ - private static void handleMetadata(Peer peer, byte[] bs) { + private static void handleMetadata(Peer peer, PeerListener listener, byte[] bs) { if (_log.shouldLog(Log.DEBUG)) _log.debug("Got metadata msg from " + peer); try { @@ -145,6 +145,9 @@ abstract class ExtensionHandler { pc = state.getChunk(piece); } sendPiece(peer, piece, pc); + // Do this here because PeerConnectionOut only reports for PIECE messages + peer.uploaded(pc.length); + listener.uploaded(peer, pc.length); } else if (type == TYPE_DATA) { int size = map.get("total_size").getInt(); boolean done; @@ -153,6 +156,8 @@ abstract class ExtensionHandler { if (state.isComplete()) return; int len = is.available(); + peer.downloaded(len); + listener.downloaded(peer, len); done = state.saveChunk(piece, bs, bs.length - len, len); if (_log.shouldLog(Log.INFO)) _log.info("Got chunk " + piece + " from " + peer); @@ -189,8 +194,17 @@ abstract class ExtensionHandler { } private static void sendRequest(Peer peer, int piece) { + sendMessage(peer, TYPE_REQUEST, piece); + } + + private static void sendReject(Peer peer, int piece) { + sendMessage(peer, TYPE_REJECT, piece); + } + + /** REQUEST and REJECT are the same except for message type */ + private static void sendMessage(Peer peer, int type, int piece) { Map map = new HashMap(); - map.put("msg_type", TYPE_REQUEST); + map.put("msg_type", Integer.valueOf(type)); map.put("piece", Integer.valueOf(piece)); byte[] payload = BEncoder.bencode(map); try { @@ -205,7 +219,7 @@ abstract class ExtensionHandler { private static void sendPiece(Peer peer, int piece, byte[] data) { Map map = new HashMap(); - map.put("msg_type", TYPE_REQUEST); + map.put("msg_type", Integer.valueOf(TYPE_REQUEST)); map.put("piece", Integer.valueOf(piece)); map.put("total_size", Integer.valueOf(data.length)); byte[] dict = BEncoder.bencode(map); diff --git a/apps/i2psnark/java/src/org/klomp/snark/Peer.java b/apps/i2psnark/java/src/org/klomp/snark/Peer.java index 681a1a5806..0f13d1470d 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/Peer.java +++ b/apps/i2psnark/java/src/org/klomp/snark/Peer.java @@ -53,6 +53,10 @@ public class Peer implements Comparable private DataInputStream din; private DataOutputStream dout; + /** running counters */ + private long downloaded; + private long uploaded; + // Keeps state for in/out connections. Non-null when the handshake // was successful, the connection setup and runs PeerState state; @@ -410,10 +414,10 @@ public class Peer implements Comparable * Switch from magnet mode to normal mode * @since 0.8.4 */ - public void gotMetaInfo(MetaInfo meta) { + public void setMetaInfo(MetaInfo meta) { PeerState s = state; if (s != null) - s.gotMetaInfo(meta); + s.setMetaInfo(meta); } public boolean isConnected() @@ -577,14 +581,29 @@ public class Peer implements Comparable return (s == null) || s.choked; } + /** + * Increment the counter. + * @since 0.8.4 + */ + public void downloaded(int size) { + downloaded += size; + } + + /** + * Increment the counter. + * @since 0.8.4 + */ + public void uploaded(int size) { + uploaded += size; + } + /** * Returns the number of bytes that have been downloaded. * Can be reset to zero with resetCounters()/ */ public long getDownloaded() { - PeerState s = state; - return (s != null) ? s.downloaded : 0; + return downloaded; } /** @@ -593,8 +612,7 @@ public class Peer implements Comparable */ public long getUploaded() { - PeerState s = state; - return (s != null) ? s.uploaded : 0; + return uploaded; } /** @@ -602,12 +620,8 @@ public class Peer implements Comparable */ public void resetCounters() { - PeerState s = state; - if (s != null) - { - s.downloaded = 0; - s.uploaded = 0; - } + downloaded = 0; + uploaded = 0; } public long getInactiveTime() { diff --git a/apps/i2psnark/java/src/org/klomp/snark/PeerCoordinator.java b/apps/i2psnark/java/src/org/klomp/snark/PeerCoordinator.java index 6eb730fc9d..0ad2c1e372 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/PeerCoordinator.java +++ b/apps/i2psnark/java/src/org/klomp/snark/PeerCoordinator.java @@ -45,13 +45,15 @@ public class PeerCoordinator implements PeerListener /** * External use by PeerMonitorTask only. + * Will be null when in magnet mode. */ - final MetaInfo metainfo; + MetaInfo metainfo; /** * External use by PeerMonitorTask only. + * Will be null when in magnet mode. */ - final Storage storage; + Storage storage; private final Snark snark; // package local for access by CheckDownLoadersTask @@ -1150,7 +1152,10 @@ public class PeerCoordinator implements PeerListener } } - /** @since 0.8.4 */ + /** + * PeerListener callback + * @since 0.8.4 + */ public void gotExtension(Peer peer, int id, byte[] bs) { if (_log.shouldLog(Log.DEBUG)) _log.debug("Got extension message " + id + " from " + peer); @@ -1160,18 +1165,29 @@ public class PeerCoordinator implements PeerListener if (magnetState.isComplete()) { if (_log.shouldLog(Log.WARN)) _log.warn("Got completed metainfo via extension"); - MetaInfo newinfo = magnetState.getMetaInfo(); - // more validation - // set global - // instantiate storage - // tell Snark listener - // tell all peers + metainfo = magnetState.getMetaInfo(); + listener.gotMetaInfo(this, metainfo); + for (Peer p : peers) { + p.setMetaInfo(metainfo); + } } } } } - /** @since 0.8.4 */ + /** + * Sets the storage after transition out of magnet mode + * Snark calls this after we call gotMetaInfo() + * @since 0.8.4 + */ + public void setStorage(Storage stg) { + storage = stg; + } + + /** + * PeerListener callback + * @since 0.8.4 + */ public void gotPort(Peer peer, int port) { // send to DHT } diff --git a/apps/i2psnark/java/src/org/klomp/snark/PeerState.java b/apps/i2psnark/java/src/org/klomp/snark/PeerState.java index a85207476b..4164e310a2 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/PeerState.java +++ b/apps/i2psnark/java/src/org/klomp/snark/PeerState.java @@ -36,8 +36,9 @@ class PeerState implements DataLoader { private final Log _log = I2PAppContext.getGlobalContext().logManager().getLog(PeerState.class); private final Peer peer; + /** Fixme, used by Peer.disconnect() to get to the coordinator */ final PeerListener listener; - private final MetaInfo metainfo; + private MetaInfo metainfo; // Interesting and choking describes whether we are interested in or // are choking the other side. @@ -49,10 +50,6 @@ class PeerState implements DataLoader boolean interested = false; boolean choked = true; - // Package local for use by Peer. - long downloaded; - long uploaded; - /** the pieces the peer has */ BitField bitfield; @@ -285,7 +282,7 @@ class PeerState implements DataLoader */ void uploaded(int size) { - uploaded += size; + peer.uploaded(size); listener.uploaded(peer, size); } @@ -305,7 +302,7 @@ class PeerState implements DataLoader void pieceMessage(Request req) { int size = req.len; - downloaded += size; + peer.downloaded(size); listener.downloaded(peer, size); if (_log.shouldLog(Log.DEBUG)) @@ -326,9 +323,6 @@ class PeerState implements DataLoader { if (_log.shouldLog(Log.WARN)) _log.warn("Got BAD " + req.piece + " from " + peer); - // XXX ARGH What now !?! - // FIXME Why would we set downloaded to 0? - downloaded = 0; } } @@ -372,7 +366,6 @@ class PeerState implements DataLoader _log.info("Unrequested 'piece: " + piece + ", " + begin + ", " + length + "' received from " + peer); - downloaded = 0; // XXX - punishment? return null; } @@ -397,7 +390,6 @@ class PeerState implements DataLoader + begin + ", " + length + "' received from " + peer); - downloaded = 0; // XXX - punishment? return null; } @@ -497,7 +489,7 @@ class PeerState implements DataLoader /** @since 0.8.2 */ void extensionMessage(int id, byte[] bs) { - ExtensionHandler.handleMessage(peer, id, bs); + ExtensionHandler.handleMessage(peer, listener, id, bs); // Peer coord will get metadata from MagnetState, // verify, and then call gotMetaInfo() listener.gotExtension(peer, id, bs); @@ -507,9 +499,18 @@ class PeerState implements DataLoader * Switch from magnet mode to normal mode * @since 0.8.4 */ - public void gotMetaInfo(MetaInfo meta) { - // set metainfo - // fix bitfield + public void setMetaInfo(MetaInfo meta) { + BitField oldBF = bitfield; + if (oldBF != null) { + if (oldBF.size() != meta.getPieces()) + // fix bitfield, it was too big by 1-7 bits + bitfield = new BitField(oldBF.getFieldBytes(), meta.getPieces()); + // else no extra + } else { + // it will be initialized later + //bitfield = new BitField(meta.getPieces()); + } + metainfo = meta; } /** @since 0.8.4 */ diff --git a/apps/i2psnark/java/src/org/klomp/snark/Snark.java b/apps/i2psnark/java/src/org/klomp/snark/Snark.java index fd7a88b57a..bd3d0d20dd 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/Snark.java +++ b/apps/i2psnark/java/src/org/klomp/snark/Snark.java @@ -1004,6 +1004,30 @@ public class Snark // System.out.println(peer.toString()); } + /** + * Called when the PeerCoordinator got the MetaInfo via magnet. + * CoordinatorListener. + * Create the storage, tell SnarkManager, and give the storage + * back to the coordinator. + * + * @throws RuntimeException via fatal() + * @since 0.8.4 + */ + public void gotMetaInfo(PeerCoordinator coordinator, MetaInfo metainfo) { + meta = metainfo; + try { + storage = new Storage(_util, meta, this); + if (completeListener != null) + completeListener.gotMetaInfo(this); + coordinator.setStorage(storage); + } catch (IOException ioe) { + if (storage != null) { + try { storage.close(); } catch (IOException ioee) {} + } + fatal("Could not check or create storage", ioe); + } + } + private boolean allocating = false; public void storageCreateFile(Storage storage, String name, long length) { diff --git a/apps/i2psnark/java/src/org/klomp/snark/SnarkManager.java b/apps/i2psnark/java/src/org/klomp/snark/SnarkManager.java index d5a51c584c..c1410e7fa4 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/SnarkManager.java +++ b/apps/i2psnark/java/src/org/klomp/snark/SnarkManager.java @@ -573,9 +573,11 @@ public class SnarkManager implements Snark.CompleteListener { if (!TrackerClient.isValidAnnounce(info.getAnnounce())) { if (_util.shouldUseOpenTrackers() && _util.getOpenTrackers() != null) { - addMessage(_("Warning - Ignoring non-i2p tracker in \"{0}\", will announce to i2p open trackers only", info.getName())); + addMessage(_("Warning - No I2P trackers in \"{0}\", will announce to I2P open trackers and DHT only.", info.getName())); + } else if (_util.getDHT() != null) { + addMessage(_("Warning - No I2P trackers in \"{0}\", and open trackers are disabled, will announce to DHT only.", info.getName())); } else { - addMessage(_("Warning - Ignoring non-i2p tracker in \"{0}\", and open trackers are disabled, you must enable open trackers before starting the torrent!", info.getName())); + addMessage(_("Warning - No I2P trackers in \"{0}\", and DHT and open trackers are disabled, you should enable open trackers or DHT before starting the torrent.", info.getName())); dontAutoStart = true; } } From 973407498b74489d1aeb836cacc6f49b1f01bc16 Mon Sep 17 00:00:00 2001 From: zzz Date: Wed, 22 Dec 2010 21:45:09 +0000 Subject: [PATCH 10/27] Magnet icon from http://findicons.com/icon/178214/magnet Courtesy led24.de Freeware --- apps/i2psnark/_icons/magnet.png | Bin 0 -> 591 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 apps/i2psnark/_icons/magnet.png diff --git a/apps/i2psnark/_icons/magnet.png b/apps/i2psnark/_icons/magnet.png new file mode 100644 index 0000000000000000000000000000000000000000..3430f2e5a9c08050cb90248f42315876fd04cb6a GIT binary patch literal 591 zcmV-V0DY!e-pP^Qvt5!>}3Z-d&8th+eay!Hi^4a8_-r+sZeb3=4 zYc0)s5{nH38Q?aM1oFTFFn7OLT;V^dX28>U?6E8AL2uX+4S{Glti$#`_jZ$AlmZd-7>mfpS-hG ze*G`=bG|29{l2(#Z6G0ZJMarT;Q=&&D3ZajL0n;PV-ry1W_b? z00$7)AVGtw)tJhctIdF^e7UCUbtGsoA%oKx@vVRf2-;d4{Qts}m+`z0APU3?S}R1s z%*@O&z_0>w#aqb_aIi=~qaDG)R$bR^*+AMgD2(3|MS>Q^pT4#t?}_@O(S9$~2BBLL!mC^X=3s;{oQf*=)x1 dZRT5MjsQ}{qk5&)8Mgoc002ovPDHLkV1mRJ2SES; literal 0 HcmV?d00001 From 97f0c13c159147769752e9091a826a232a498cde Mon Sep 17 00:00:00 2001 From: zzz Date: Wed, 22 Dec 2010 22:22:38 +0000 Subject: [PATCH 11/27] - Protection against modifying metainfos - Announce peers to local tracker - Ping node on port reception - More info on directory pages - Cleanups --- .../java/src/org/klomp/snark/MetaInfo.java | 55 +++++++++--------- .../java/src/org/klomp/snark/Peer.java | 8 +++ .../src/org/klomp/snark/PeerCheckerTask.java | 15 ++++- .../src/org/klomp/snark/PeerCoordinator.java | 7 ++- .../java/src/org/klomp/snark/Snark.java | 15 ++++- .../src/org/klomp/snark/SnarkManager.java | 5 ++ .../java/src/org/klomp/snark/Storage.java | 6 +- .../src/org/klomp/snark/TrackerClient.java | 16 ++++-- .../org/klomp/snark/web/I2PSnarkServlet.java | 57 +++++++++++++++++-- 9 files changed, 141 insertions(+), 43 deletions(-) diff --git a/apps/i2psnark/java/src/org/klomp/snark/MetaInfo.java b/apps/i2psnark/java/src/org/klomp/snark/MetaInfo.java index 2f01fe62c2..3fc4977f6e 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/MetaInfo.java +++ b/apps/i2psnark/java/src/org/klomp/snark/MetaInfo.java @@ -54,28 +54,30 @@ public class MetaInfo private final byte[] info_hash; private final String name; private final String name_utf8; - private final List files; - private final List files_utf8; - private final List lengths; + private final List> files; + private final List> files_utf8; + private final List lengths; private final int piece_length; private final byte[] piece_hashes; private final long length; - private Map infoMap; + private Map infoMap; /** * Called by Storage when creating a new torrent from local data * * @param announce may be null + * @param files null for single-file torrent + * @param lengths null for single-file torrent */ - MetaInfo(String announce, String name, String name_utf8, List files, List lengths, + MetaInfo(String announce, String name, String name_utf8, List> files, List lengths, int piece_length, byte[] piece_hashes, long length) { this.announce = announce; this.name = name; this.name_utf8 = name_utf8; - this.files = files; + this.files = files == null ? null : Collections.unmodifiableList(files); this.files_utf8 = null; - this.lengths = lengths; + this.lengths = lengths == null ? null : Collections.unmodifiableList(lengths); this.piece_length = piece_length; this.piece_hashes = piece_hashes; this.length = length; @@ -131,7 +133,7 @@ public class MetaInfo if (val == null) throw new InvalidBEncodingException("Missing info map"); Map info = val.getMap(); - infoMap = info; + infoMap = Collections.unmodifiableMap(info); val = (BEValue)info.get("name"); if (val == null) @@ -171,39 +173,39 @@ public class MetaInfo throw new InvalidBEncodingException ("Missing length number and/or files list"); - List list = val.getList(); + List list = val.getList(); int size = list.size(); if (size == 0) throw new InvalidBEncodingException("zero size files list"); - files = new ArrayList(size); - files_utf8 = new ArrayList(size); - lengths = new ArrayList(size); + List> m_files = new ArrayList(size); + List> m_files_utf8 = new ArrayList(size); + List m_lengths = new ArrayList(size); long l = 0; for (int i = 0; i < list.size(); i++) { - Map desc = ((BEValue)list.get(i)).getMap(); - val = (BEValue)desc.get("length"); + Map desc = list.get(i).getMap(); + val = desc.get("length"); if (val == null) throw new InvalidBEncodingException("Missing length number"); long len = val.getLong(); - lengths.add(new Long(len)); + m_lengths.add(new Long(len)); l += len; val = (BEValue)desc.get("path"); if (val == null) throw new InvalidBEncodingException("Missing path list"); - List path_list = val.getList(); + List path_list = val.getList(); int path_length = path_list.size(); if (path_length == 0) throw new InvalidBEncodingException("zero size file path list"); - List file = new ArrayList(path_length); - Iterator it = path_list.iterator(); + List file = new ArrayList(path_length); + Iterator it = path_list.iterator(); while (it.hasNext()) - file.add(((BEValue)it.next()).getString()); + file.add(it.next().getString()); - files.add(file); + m_files.add(Collections.unmodifiableList(file)); val = (BEValue)desc.get("path.utf-8"); if (val != null) { @@ -213,11 +215,14 @@ public class MetaInfo file = new ArrayList(path_length); it = path_list.iterator(); while (it.hasNext()) - file.add(((BEValue)it.next()).getString()); - files_utf8.add(file); + file.add(it.next().getString()); + m_files_utf8.add(Collections.unmodifiableList(file)); } } } + files = Collections.unmodifiableList(m_files); + files_utf8 = Collections.unmodifiableList(m_files_utf8); + lengths = Collections.unmodifiableList(m_lengths); length = l; } @@ -265,9 +270,8 @@ public class MetaInfo * a single name. It has the same size as the list returned by * getLengths(). */ - public List getFiles() + public List> getFiles() { - // XXX - Immutable? return files; } @@ -276,9 +280,8 @@ public class MetaInfo * files, or null if it is a single file. It has the same size as * the list returned by getFiles(). */ - public List getLengths() + public List getLengths() { - // XXX - Immutable? return lengths; } diff --git a/apps/i2psnark/java/src/org/klomp/snark/Peer.java b/apps/i2psnark/java/src/org/klomp/snark/Peer.java index 0f13d1470d..b31ba65e2f 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/Peer.java +++ b/apps/i2psnark/java/src/org/klomp/snark/Peer.java @@ -32,6 +32,7 @@ import java.util.Map; import net.i2p.client.streaming.I2PSocket; import net.i2p.data.DataHelper; +import net.i2p.data.Destination; import net.i2p.util.Log; import org.klomp.snark.bencode.BEValue; @@ -384,6 +385,13 @@ public class Peer implements Comparable return options; } + /** @since 0.8.4 */ + public Destination getDestination() { + if (sock == null) + return null; + return sock.getPeerDestination(); + } + /** * Shared state across all peers, callers must sync on returned object * @return non-null diff --git a/apps/i2psnark/java/src/org/klomp/snark/PeerCheckerTask.java b/apps/i2psnark/java/src/org/klomp/snark/PeerCheckerTask.java index 65a8698453..6370eaca64 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/PeerCheckerTask.java +++ b/apps/i2psnark/java/src/org/klomp/snark/PeerCheckerTask.java @@ -37,7 +37,8 @@ class PeerCheckerTask extends TimerTask private static final long KILOPERSECOND = 1024*(PeerCoordinator.CHECK_PERIOD/1000); private final PeerCoordinator coordinator; - public I2PSnarkUtil _util; + private final I2PSnarkUtil _util; + private int _runCount; PeerCheckerTask(I2PSnarkUtil util, PeerCoordinator coordinator) { @@ -49,6 +50,7 @@ class PeerCheckerTask extends TimerTask public void run() { + _runCount++; List peerList = coordinator.peerList(); if (peerList.isEmpty() || coordinator.halted()) { coordinator.setRateHistory(0, 0); @@ -204,6 +206,10 @@ class PeerCheckerTask extends TimerTask } peer.retransmitRequests(); peer.keepAlive(); + // announce them to local tracker (TrackerClient does this too) + if (_util.getDHT() != null && (_runCount % 5) == 0) { + _util.getDHT().announce(coordinator.getInfoHash(), peer.getPeerID().getDestHash()); + } } // Resync actual uploaders value @@ -245,8 +251,13 @@ class PeerCheckerTask extends TimerTask // close out unused files, but we don't need to do it every time Storage storage = coordinator.getStorage(); - if (storage != null && random.nextInt(4) == 0) { + if (storage != null && (_runCount % 4) == 0) { storage.cleanRAFs(); } + + // announce ourselves to local tracker (TrackerClient does this too) + if (_util.getDHT() != null && (_runCount % 16) == 0) { + _util.getDHT().announce(coordinator.getInfoHash()); + } } } diff --git a/apps/i2psnark/java/src/org/klomp/snark/PeerCoordinator.java b/apps/i2psnark/java/src/org/klomp/snark/PeerCoordinator.java index 0ad2c1e372..504749b625 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/PeerCoordinator.java +++ b/apps/i2psnark/java/src/org/klomp/snark/PeerCoordinator.java @@ -36,6 +36,8 @@ import net.i2p.util.I2PAppThread; import net.i2p.util.Log; import net.i2p.util.SimpleTimer2; +import org.klomp.snark.dht.KRPC; + /** * Coordinates what peer does what. */ @@ -1186,10 +1188,13 @@ public class PeerCoordinator implements PeerListener /** * PeerListener callback + * Tell the DHT to ping it, this will get back the node info * @since 0.8.4 */ public void gotPort(Peer peer, int port) { - // send to DHT + KRPC krpc = _util.getDHT(); + if (krpc != null) + krpc.ping(peer.getDestination(), port); } /** Return number of allowed uploaders for this torrent. diff --git a/apps/i2psnark/java/src/org/klomp/snark/Snark.java b/apps/i2psnark/java/src/org/klomp/snark/Snark.java index bd3d0d20dd..45231965fc 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/Snark.java +++ b/apps/i2psnark/java/src/org/klomp/snark/Snark.java @@ -776,7 +776,7 @@ public class Snark } /** - * @return needed of all torrent files, or total of metainfo file if fetching magnet, or -1 + * @return number of pieces still needed (magnet mode or not), or -1 if unknown * @since 0.8.4 */ public long getNeeded() { @@ -786,7 +786,7 @@ public class Snark // FIXME subtract chunks we have return meta.getTotalLength(); // FIXME fake - return 16 * 16 * 1024; + return -1; } /** @@ -800,6 +800,17 @@ public class Snark return 16*1024; } + /** + * @return number of pieces + * @since 0.8.4 + */ + public int getPieces() { + if (meta != null) + return meta.getPieces(); + // FIXME else return metainfo pieces if available + return -1; + } + /** * @return true if restarted * @since 0.8.4 diff --git a/apps/i2psnark/java/src/org/klomp/snark/SnarkManager.java b/apps/i2psnark/java/src/org/klomp/snark/SnarkManager.java index c1410e7fa4..27640d53ba 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/SnarkManager.java +++ b/apps/i2psnark/java/src/org/klomp/snark/SnarkManager.java @@ -68,6 +68,7 @@ public class SnarkManager implements Snark.CompleteListener { public static final String PROP_META_PREFIX = "i2psnark.zmeta."; public static final String PROP_META_BITFIELD_SUFFIX = ".bitfield"; public static final String PROP_META_PRIORITY_SUFFIX = ".priority"; + public static final String PROP_META_MAGNET_SUFFIX = ".magnet"; private static final String CONFIG_FILE = "i2psnark.config"; public static final String PROP_AUTO_START = "i2snark.autoStart"; // oops @@ -1031,6 +1032,9 @@ public class SnarkManager implements Snark.CompleteListener { } } +//start magnets + + // here because we need to delay until I2CP is up // although the user will see the default until then getBWLimit(); @@ -1245,6 +1249,7 @@ public class SnarkManager implements Snark.CompleteListener { if ( (snark != null) && (!snark.isStopped()) ) snark.stopTorrent(); } +//save magnets } } } diff --git a/apps/i2psnark/java/src/org/klomp/snark/Storage.java b/apps/i2psnark/java/src/org/klomp/snark/Storage.java index dcd78620d9..77012de77d 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/Storage.java +++ b/apps/i2psnark/java/src/org/klomp/snark/Storage.java @@ -100,7 +100,7 @@ public class Storage getFiles(baseFile); long total = 0; - ArrayList lengthsList = new ArrayList(); + ArrayList lengthsList = new ArrayList(); for (int i = 0; i < lengths.length; i++) { long length = lengths[i]; @@ -122,10 +122,10 @@ public class Storage bitfield = new BitField(pieces); needed = 0; - List files = new ArrayList(); + List> files = new ArrayList(); for (int i = 0; i < names.length; i++) { - List file = new ArrayList(); + List file = new ArrayList(); StringTokenizer st = new StringTokenizer(names[i], File.separator); while (st.hasMoreTokens()) { diff --git a/apps/i2psnark/java/src/org/klomp/snark/TrackerClient.java b/apps/i2psnark/java/src/org/klomp/snark/TrackerClient.java index f437beeeb6..720a662b3a 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/TrackerClient.java +++ b/apps/i2psnark/java/src/org/klomp/snark/TrackerClient.java @@ -277,18 +277,26 @@ public class TrackerClient extends I2PAppThread runStarted = true; tr.started = true; - Set peers = info.getPeers(); + Set peers = info.getPeers(); tr.seenPeers = info.getPeerCount(); if (snark.getTrackerSeenPeers() < tr.seenPeers) // update rising number quickly snark.setTrackerSeenPeers(tr.seenPeers); + + // pass everybody over to our tracker + if (_util.getDHT() != null) { + for (Peer peer : peers) { + _util.getDHT().announce(snark.getInfoHash(), peer.getPeerID().getDestHash()); + } + } + if ( (left > 0) && (!completed) ) { // we only want to talk to new people if we need things // from them (duh) - List ordered = new ArrayList(peers); + List ordered = new ArrayList(peers); Collections.shuffle(ordered, r); - Iterator it = ordered.iterator(); + Iterator it = ordered.iterator(); while ((!stop) && it.hasNext()) { - Peer cur = (Peer)it.next(); + Peer cur = it.next(); // FIXME if id == us || dest == us continue; // only delay if we actually make an attempt to add peer if(coordinator.addPeer(cur)) { diff --git a/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java b/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java index fefc808485..61c9c8e572 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java +++ b/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java @@ -6,6 +6,7 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.PrintWriter; import java.text.Collator; +import java.text.DecimalFormat; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -875,8 +876,7 @@ public class I2PSnarkServlet extends Default { else if (isValid) icon = toIcon(meta.getName()); else - // todo get a nice magnet icon? - icon = "page_white"; + icon = "magnet"; if (remaining == 0 || isMultiFile) { out.write(toImg(icon, _("Open"))); out.write(""); @@ -1375,6 +1375,7 @@ public class I2PSnarkServlet extends Default { if (ihash.length() == 32) { ih = Base32.decode(ihash); } else if (ihash.length() == 40) { + // Like DataHelper.fromHexString() but ensures no loss of leading zero bytes ih = new byte[20]; try { for (int i = 0; i < 20; i++) { @@ -1539,6 +1540,7 @@ public class I2PSnarkServlet extends Default { if (title.endsWith("/")) title = title.substring(0, title.length() - 1); + String directory = title; title = _("Torrent") + ": " + title; buf.append(title); buf.append("").append(HEADER_A).append(_themePath).append(HEADER_B).append("" + @@ -1550,10 +1552,40 @@ public class I2PSnarkServlet extends Default { boolean showPriority = snark != null && snark.getStorage() != null && !snark.getStorage().complete(); if (showPriority) buf.append("\n"); - buf.append("" + - "
") + buf.append(""); + if (snark != null) { + // first row - torrent info + // FIXME center + buf.append(""); + } + // second row - dir info + buf.append("\n\t"); out.write("\n\t"); out.write("\n\t"); out.write("\n
") + .append(_("Torrent")).append(": ").append(snark.getBaseName()); + int pieces = snark.getPieces(); + double completion = (pieces - snark.getNeeded()) / (double) pieces; + if (completion < 1.0) + buf.append("
").append(_("Completion")).append(": ").append((new DecimalFormat("0.00%")).format(completion)); + else + buf.append("
").append(_("Complete")); + // else unknown + buf.append("
").append(_("Size")).append(": ").append(formatSize(snark.getTotalLength())); + MetaInfo meta = snark.getMetaInfo(); + if (meta != null) { + List files = meta.getFiles(); + int fileCount = files != null ? files.size() : 1; + buf.append("
").append(_("Files")).append(": ").append(fileCount); + } + buf.append("
").append(_("Pieces")).append(": ").append(pieces); + buf.append("
").append(_("Piece size")).append(": ").append(formatSize(snark.getPieceLength(0))); + String hex = toHex(snark.getInfoHash()); + buf.append("
").append(_("Magnet link")).append(": ") + .append(MAGNET).append(hex).append(""); + // We don't have the hash of the torrent file + //buf.append("
").append(_("Maggot link")).append(": ") + // .append(MAGGOT).append(hex).append(':').append(hex).append(""); + buf.append("
") .append("\"\" ") - .append(title).append("") + .append(_("Directory")).append(": ").append(directory).append("") .append("\"\" ") .append(_("Size")); buf.append("") @@ -1767,6 +1799,21 @@ public class I2PSnarkServlet extends Default { return "\"""; } + /** + * Like DataHelper.toHexString but ensures no loss of leading zero bytes + * @since 0.8.4 + */ + private static String toHex(byte[] b) { + StringBuilder buf = new StringBuilder(40); + for (int i = 0; i < b.length; i++) { + int bi = b[i] & 0xff; + if (bi < 16) + buf.append('0'); + buf.append(Integer.toHexString(bi)); + } + return buf.toString(); + } + /** @since 0.8.1 */ private void savePriorities(Snark snark, Map postParams) { Storage storage = snark.getStorage(); From b97ad6c5f02822756955916c724dd0411295d08a Mon Sep 17 00:00:00 2001 From: zzz Date: Wed, 22 Dec 2010 22:23:34 +0000 Subject: [PATCH 12/27] generics --- .../java/src/org/klomp/snark/bencode/BEncoder.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/apps/i2psnark/java/src/org/klomp/snark/bencode/BEncoder.java b/apps/i2psnark/java/src/org/klomp/snark/bencode/BEncoder.java index b8129f4772..1a92362481 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/bencode/BEncoder.java +++ b/apps/i2psnark/java/src/org/klomp/snark/bencode/BEncoder.java @@ -59,7 +59,7 @@ public class BEncoder else if (o instanceof List) bencode((List)o, out); else if (o instanceof Map) - bencode((Map)o, out); + bencode((Map)o, out); else if (o instanceof BEValue) bencode(((BEValue)o).getValue(), out); else @@ -153,7 +153,7 @@ public class BEncoder out.write(bs); } - public static byte[] bencode(Map m) + public static byte[] bencode(Map m) { try { @@ -167,20 +167,20 @@ public class BEncoder } } - public static void bencode(Map m, OutputStream out) throws IOException + public static void bencode(Map m, OutputStream out) throws IOException { out.write('d'); // Keys must be sorted. XXX - But is this the correct order? - Set s = m.keySet(); - List l = new ArrayList(s); + Set s = m.keySet(); + List l = new ArrayList(s); Collections.sort(l); - Iterator it = l.iterator(); + Iterator it = l.iterator(); while(it.hasNext()) { // Keys must be Strings. - String key = (String)it.next(); + String key = it.next(); Object value = m.get(key); bencode(key, out); bencode(value, out); From 12fae66948dffc247956756c1c64b8d8ce711fcb Mon Sep 17 00:00:00 2001 From: zzz Date: Sun, 26 Dec 2010 12:27:40 +0000 Subject: [PATCH 13/27] - Fix about 9 NPEs - Fix numwant in magnet mode - Send metadata size in extension handshake - Open trackers are primary if we don't have primary trackers - Add missing break in port message handling - Increase max msg size to account for metadata msg - Remember magnets across restarts - Drop peers w/o extensions if we need metainfo - Fix DATA messages - Fix tracker transition to non-magnet - Fix infohash for non-magnet - Fix up peer transition to non-magnet - More logging --- .../src/org/klomp/snark/bencode/BEncoder.java | 2 ++ .../org/klomp/snark/web/I2PSnarkServlet.java | 20 +++---------------- 2 files changed, 5 insertions(+), 17 deletions(-) diff --git a/apps/i2psnark/java/src/org/klomp/snark/bencode/BEncoder.java b/apps/i2psnark/java/src/org/klomp/snark/bencode/BEncoder.java index 1a92362481..9584b0d9db 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/bencode/BEncoder.java +++ b/apps/i2psnark/java/src/org/klomp/snark/bencode/BEncoder.java @@ -50,6 +50,8 @@ public class BEncoder public static void bencode(Object o, OutputStream out) throws IOException, IllegalArgumentException { + if (o == null) + throw new NullPointerException("Cannot bencode null"); if (o instanceof String) bencode((String)o, out); else if (o instanceof byte[]) diff --git a/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java b/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java index 61c9c8e572..bd08dc46d3 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java +++ b/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java @@ -35,6 +35,7 @@ import net.i2p.util.I2PAppThread; import net.i2p.util.Log; import net.i2p.util.SecureFileOutputStream; +import org.klomp.snark.I2PSnarkUtil; import org.klomp.snark.MetaInfo; import org.klomp.snark.Peer; import org.klomp.snark.Snark; @@ -1389,7 +1390,7 @@ public class I2PSnarkServlet extends Default { _manager.addMessage(_("Invalid info hash in magnet URL {0}", url)); return; } - _manager.addMagnet(name, ih); + _manager.addMagnet(name, ih, true); } /** copied from ConfigTunnelsHelper */ @@ -1574,7 +1575,7 @@ public class I2PSnarkServlet extends Default { } buf.append("
").append(_("Pieces")).append(": ").append(pieces); buf.append("
").append(_("Piece size")).append(": ").append(formatSize(snark.getPieceLength(0))); - String hex = toHex(snark.getInfoHash()); + String hex = I2PSnarkUtil.toHex(snark.getInfoHash()); buf.append("
").append(_("Magnet link")).append(": ") .append(MAGNET).append(hex).append(""); // We don't have the hash of the torrent file @@ -1799,21 +1800,6 @@ public class I2PSnarkServlet extends Default { return "\"""; } - /** - * Like DataHelper.toHexString but ensures no loss of leading zero bytes - * @since 0.8.4 - */ - private static String toHex(byte[] b) { - StringBuilder buf = new StringBuilder(40); - for (int i = 0; i < b.length; i++) { - int bi = b[i] & 0xff; - if (bi < 16) - buf.append('0'); - buf.append(Integer.toHexString(bi)); - } - return buf.toString(); - } - /** @since 0.8.1 */ private void savePriorities(Snark snark, Map postParams) { Storage storage = snark.getStorage(); From 71043c41f1c490a02ec1eaef4448426ba06e4718 Mon Sep 17 00:00:00 2001 From: zzz Date: Mon, 27 Dec 2010 17:13:24 +0000 Subject: [PATCH 14/27] - Hide I2CP settings when in router context - Better BEValue.toString() (most of the following got missed in the last checkin) - Fix about 9 NPEs - Fix numwant in magnet mode - Send metadata size in extension handshake - Open trackers are primary if we don't have primary trackers - Add missing break in port message handling - Increase max msg size to account for metadata msg - Remember magnets across restarts - Drop peers w/o extensions if we need metainfo - Fix DATA messages - Fix tracker transition to non-magnet - Fix infohash for non-magnet - Fix up peer transition to non-magnet - More logging --- .../src/org/klomp/snark/ExtensionHandler.java | 55 ++++++++++---- .../src/org/klomp/snark/I2PSnarkUtil.java | 15 ++++ .../java/src/org/klomp/snark/MagnetState.java | 10 +-- .../java/src/org/klomp/snark/Message.java | 4 +- .../java/src/org/klomp/snark/Peer.java | 15 +++- .../src/org/klomp/snark/PeerConnectionIn.java | 16 ++-- .../src/org/klomp/snark/PeerCoordinator.java | 10 ++- .../java/src/org/klomp/snark/PeerState.java | 2 + .../java/src/org/klomp/snark/Snark.java | 15 +++- .../src/org/klomp/snark/SnarkManager.java | 74 +++++++++++++++++-- .../src/org/klomp/snark/TrackerClient.java | 21 +++--- .../java/src/org/klomp/snark/TrackerInfo.java | 23 +++--- .../src/org/klomp/snark/bencode/BEValue.java | 12 ++- .../org/klomp/snark/web/I2PSnarkServlet.java | 18 +++-- 14 files changed, 213 insertions(+), 77 deletions(-) diff --git a/apps/i2psnark/java/src/org/klomp/snark/ExtensionHandler.java b/apps/i2psnark/java/src/org/klomp/snark/ExtensionHandler.java index e212f6f613..5b7b0b0382 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/ExtensionHandler.java +++ b/apps/i2psnark/java/src/org/klomp/snark/ExtensionHandler.java @@ -21,7 +21,6 @@ import org.klomp.snark.bencode.InvalidBEncodingException; */ abstract class ExtensionHandler { - private static final byte[] _handshake = buildHandshake(); private static final Log _log = I2PAppContext.getGlobalContext().logManager().getLog(ExtensionHandler.class); public static final int ID_METADATA = 3; @@ -32,17 +31,15 @@ abstract class ExtensionHandler { /** + * @param metasize -1 if unknown * @return bencoded outgoing handshake message */ - public static byte[] getHandshake() { - return _handshake; - } - - /** outgoing handshake message */ - private static byte[] buildHandshake() { + public static byte[] getHandshake(int metasize) { Map handshake = new HashMap(); Map m = new HashMap(); m.put(TYPE_METADATA, Integer.valueOf(ID_METADATA)); + if (metasize >= 0) + handshake.put("metadata_size", Integer.valueOf(metasize)); handshake.put("m", m); handshake.put("p", Integer.valueOf(6881)); handshake.put("v", "I2PSnark"); @@ -51,6 +48,8 @@ abstract class ExtensionHandler { } public static void handleMessage(Peer peer, PeerListener listener, int id, byte[] bs) { + if (_log.shouldLog(Log.INFO)) + _log.info("Got extension msg " + id + " length " + bs.length + " from " + peer); if (id == 0) handleHandshake(peer, listener, bs); else if (id == ID_METADATA) @@ -71,10 +70,24 @@ abstract class ExtensionHandler { peer.setHandshakeMap(map); Map msgmap = map.get("m").getMap(); - // rv not used, just to throw an NPE to get out of here - msgmap.get(TYPE_METADATA).getInt(); + if (msgmap.get(TYPE_METADATA) == null) { + if (_log.shouldLog(Log.WARN)) + _log.debug("Peer does not support metadata extension: " + peer); + // drop if we need metainfo ? + return; + } + + BEValue msize = map.get("metadata_size"); + if (msize == null) { + if (_log.shouldLog(Log.WARN)) + _log.debug("Peer does not have the metainfo size yet: " + peer); + // drop if we need metainfo ? + return; + } + int metaSize = msize.getInt(); + if (_log.shouldLog(Log.WARN)) + _log.debug("Got the metainfo size: " + metaSize); - int metaSize = map.get("metadata_size").getInt(); MagnetState state = peer.getMagnetState(); int remaining; synchronized(state) { @@ -83,12 +96,16 @@ abstract class ExtensionHandler { if (state.isInitialized()) { if (state.getSize() != metaSize) { + if (_log.shouldLog(Log.WARN)) + _log.debug("Wrong metainfo size " + metaSize + " from: " + peer); peer.disconnect(); return; } } else { // initialize it if (metaSize > MAX_METADATA_SIZE) { + if (_log.shouldLog(Log.WARN)) + _log.debug("Huge metainfo size " + metaSize + " from: " + peer); peer.disconnect(false); return; } @@ -112,8 +129,7 @@ abstract class ExtensionHandler { } } catch (Exception e) { if (_log.shouldLog(Log.WARN)) - _log.info("Handshake exception from " + peer, e); - //peer.disconnect(false); + _log.warn("Handshake exception from " + peer, e); } } @@ -140,6 +156,8 @@ abstract class ExtensionHandler { MagnetState state = peer.getMagnetState(); if (type == TYPE_REQUEST) { + if (_log.shouldLog(Log.DEBUG)) + _log.debug("Got request for " + piece + " from: " + peer); byte[] pc; synchronized(state) { pc = state.getChunk(piece); @@ -150,12 +168,19 @@ abstract class ExtensionHandler { listener.uploaded(peer, pc.length); } else if (type == TYPE_DATA) { int size = map.get("total_size").getInt(); + if (_log.shouldLog(Log.DEBUG)) + _log.debug("Got data for " + piece + " length " + size + " from: " + peer); boolean done; int chk = -1; synchronized(state) { if (state.isComplete()) return; int len = is.available(); + if (len != size) { + // probably fatal + if (_log.shouldLog(Log.WARN)) + _log.warn("total_size " + size + " but avail data " + len); + } peer.downloaded(len); listener.downloaded(peer, len); done = state.saveChunk(piece, bs, bs.length - len, len); @@ -219,15 +244,15 @@ abstract class ExtensionHandler { private static void sendPiece(Peer peer, int piece, byte[] data) { Map map = new HashMap(); - map.put("msg_type", Integer.valueOf(TYPE_REQUEST)); + map.put("msg_type", Integer.valueOf(TYPE_DATA)); map.put("piece", Integer.valueOf(piece)); map.put("total_size", Integer.valueOf(data.length)); byte[] dict = BEncoder.bencode(map); byte[] payload = new byte[dict.length + data.length]; System.arraycopy(dict, 0, payload, 0, dict.length); - System.arraycopy(data, 0, payload, dict.length, payload.length); + System.arraycopy(data, 0, payload, dict.length, data.length); try { - int hisMsgCode = peer.getHandshakeMap().get("m").getMap().get("METADATA").getInt(); + int hisMsgCode = peer.getHandshakeMap().get("m").getMap().get(TYPE_METADATA).getInt(); peer.sendExtension(hisMsgCode, payload); } catch (Exception e) { // NPE, no metadata caps diff --git a/apps/i2psnark/java/src/org/klomp/snark/I2PSnarkUtil.java b/apps/i2psnark/java/src/org/klomp/snark/I2PSnarkUtil.java index ef194ac3f4..775d4cc4ab 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/I2PSnarkUtil.java +++ b/apps/i2psnark/java/src/org/klomp/snark/I2PSnarkUtil.java @@ -446,6 +446,21 @@ public class I2PSnarkUtil { return Boolean.valueOf(rv).booleanValue(); } + /** + * Like DataHelper.toHexString but ensures no loss of leading zero bytes + * @since 0.8.4 + */ + public static String toHex(byte[] b) { + StringBuilder buf = new StringBuilder(40); + for (int i = 0; i < b.length; i++) { + int bi = b[i] & 0xff; + if (bi < 16) + buf.append('0'); + buf.append(Integer.toHexString(bi)); + } + return buf.toString(); + } + /** hook between snark's logger and an i2p log */ void debug(String msg, int snarkDebugLevel) { debug(msg, snarkDebugLevel, null); diff --git a/apps/i2psnark/java/src/org/klomp/snark/MagnetState.java b/apps/i2psnark/java/src/org/klomp/snark/MagnetState.java index fb7e5a542d..e9eb563896 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/MagnetState.java +++ b/apps/i2psnark/java/src/org/klomp/snark/MagnetState.java @@ -53,8 +53,6 @@ class MagnetState { metainfo = meta; initialize(meta.getInfoBytes().length); complete = true; - } else { - metainfoBytes = new byte[metaSize]; } } @@ -68,10 +66,13 @@ class MagnetState { isInitialized = true; metaSize = size; totalChunks = (size + (CHUNK_SIZE - 1)) / CHUNK_SIZE; - if (metainfo == null) { + if (metainfo != null) { + metainfoBytes = metainfo.getInfoBytes(); + } else { // we don't need these if complete have = new BitField(totalChunks); requested = new BitField(totalChunks); + metainfoBytes = new byte[metaSize]; } } @@ -194,8 +195,7 @@ class MagnetState { InputStream is = new ByteArrayInputStream(metainfoBytes); BDecoder dec = new BDecoder(is); BEValue bev = dec.bdecodeMap(); - Map info = bev.getMap(); - map.put("info", info); + map.put("info", bev); MetaInfo newmeta = new MetaInfo(map); if (!DataHelper.eq(newmeta.getInfoHash(), infohash)) throw new IOException("info hash mismatch"); diff --git a/apps/i2psnark/java/src/org/klomp/snark/Message.java b/apps/i2psnark/java/src/org/klomp/snark/Message.java index 57fecf43d9..01c474216e 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/Message.java +++ b/apps/i2psnark/java/src/org/klomp/snark/Message.java @@ -104,7 +104,7 @@ class Message if (type == REQUEST || type == CANCEL) datalen += 4; - // length is 1 byte + // msg type is 1 byte if (type == EXTENSION) datalen += 1; @@ -167,6 +167,8 @@ class Message return "PIECE(" + piece + "," + begin + "," + length + ")"; case CANCEL: return "CANCEL(" + piece + "," + begin + "," + length + ")"; + case PORT: + return "PORT(" + piece + ")"; case EXTENSION: return "EXTENSION(" + piece + ',' + data.length + ')'; default: diff --git a/apps/i2psnark/java/src/org/klomp/snark/Peer.java b/apps/i2psnark/java/src/org/klomp/snark/Peer.java index de8d2b1d09..18c9ad229f 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/Peer.java +++ b/apps/i2psnark/java/src/org/klomp/snark/Peer.java @@ -120,7 +120,7 @@ public class Peer implements Comparable this.peerID = new PeerID(id, sock.getPeerDestination()); _id = ++__id; if (_log.shouldLog(Log.DEBUG)) - _log.debug("Creating a new peer with " + peerID.toString(), new Exception("creating " + _id)); + _log.debug("Creating a new peer " + peerID.toString(), new Exception("creating " + _id)); } /** @@ -261,14 +261,22 @@ public class Peer implements Comparable _log.debug("Already have din [" + sock + "] with " + toString()); } + // bad idea? + if (metainfo == null && (options & OPTION_EXTENSION) == 0) { + if (_log.shouldLog(Log.INFO)) + _log.info("Peer does not support extensions and we need metainfo, dropping"); + throw new IOException("Peer does not support extensions and we need metainfo, dropping"); + } + PeerConnectionIn in = new PeerConnectionIn(this, din); PeerConnectionOut out = new PeerConnectionOut(this, dout); PeerState s = new PeerState(this, listener, metainfo, in, out); if ((options & OPTION_EXTENSION) != 0) { if (_log.shouldLog(Log.DEBUG)) - _log.debug("Peer supports extensions, sending test message"); - out.sendExtension(0, ExtensionHandler.getHandshake()); + _log.debug("Peer supports extensions, sending reply message"); + int metasize = metainfo != null ? metainfo.getInfoBytes().length : -1; + out.sendExtension(0, ExtensionHandler.getHandshake(metasize)); } if ((options & OPTION_DHT) != 0 && util.getDHT() != null) { @@ -423,6 +431,7 @@ public class Peer implements Comparable * @since 0.8.4 */ public void setMetaInfo(MetaInfo meta) { + metainfo = meta; PeerState s = state; if (s != null) s.setMetaInfo(meta); diff --git a/apps/i2psnark/java/src/org/klomp/snark/PeerConnectionIn.java b/apps/i2psnark/java/src/org/klomp/snark/PeerConnectionIn.java index a27fc1c9dd..33da75263e 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/PeerConnectionIn.java +++ b/apps/i2psnark/java/src/org/klomp/snark/PeerConnectionIn.java @@ -32,6 +32,13 @@ class PeerConnectionIn implements Runnable private final Peer peer; private final DataInputStream din; + // The max length of a complete message in bytes. + // The biggest is the piece message, for which the length is the + // request size (32K) plus 9. (we could also check if Storage.MAX_PIECES / 8 + // in the bitfield message is bigger but it's currently 5000/8 = 625 so don't bother) + private static final int MAX_MSG_SIZE = Math.max(PeerState.PARTSIZE + 9, + MagnetState.CHUNK_SIZE + 100); // 100 for the ext msg dictionary + private Thread thread; private volatile boolean quit; @@ -77,13 +84,9 @@ class PeerConnectionIn implements Runnable int len; // Wait till we hear something... - // The length of a complete message in bytes. - // The biggest is the piece message, for which the length is the - // request size (32K) plus 9. (we could also check if Storage.MAX_PIECES / 8 - // in the bitfield message is bigger but it's currently 5000/8 = 625 so don't bother) int i = din.readInt(); lastRcvd = System.currentTimeMillis(); - if (i < 0 || i > PeerState.PARTSIZE + 9) + if (i < 0 || i > MAX_MSG_SIZE) throw new IOException("Unexpected length prefix: " + i); if (i == 0) @@ -176,13 +179,14 @@ class PeerConnectionIn implements Runnable ps.portMessage(port); if (_log.shouldLog(Log.DEBUG)) _log.debug("Received port message from " + peer); + break; case 20: // Extension message int id = din.readUnsignedByte(); byte[] payload = new byte[i-2]; din.readFully(payload); - ps.extensionMessage(id, payload); if (_log.shouldLog(Log.DEBUG)) _log.debug("Received extension message from " + peer); + ps.extensionMessage(id, payload); break; default: byte[] bs = new byte[i-1]; diff --git a/apps/i2psnark/java/src/org/klomp/snark/PeerCoordinator.java b/apps/i2psnark/java/src/org/klomp/snark/PeerCoordinator.java index 504749b625..74a840fc7b 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/PeerCoordinator.java +++ b/apps/i2psnark/java/src/org/klomp/snark/PeerCoordinator.java @@ -410,7 +410,7 @@ public class PeerCoordinator implements PeerListener name = "Magnet"; else name = metainfo.getName(); - _log.info("New connection to peer: " + peer + " for " + metainfo.getName()); + _log.info("New connection to peer: " + peer + " for " + name); } // Add it to the beginning of the list. @@ -1169,9 +1169,6 @@ public class PeerCoordinator implements PeerListener _log.warn("Got completed metainfo via extension"); metainfo = magnetState.getMetaInfo(); listener.gotMetaInfo(this, metainfo); - for (Peer p : peers) { - p.setMetaInfo(metainfo); - } } } } @@ -1184,6 +1181,11 @@ public class PeerCoordinator implements PeerListener */ public void setStorage(Storage stg) { storage = stg; + setWantedPieces(); + // ok we should be in business + for (Peer p : peers) { + p.setMetaInfo(metainfo); + } } /** diff --git a/apps/i2psnark/java/src/org/klomp/snark/PeerState.java b/apps/i2psnark/java/src/org/klomp/snark/PeerState.java index 4164e310a2..6989f1ea65 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/PeerState.java +++ b/apps/i2psnark/java/src/org/klomp/snark/PeerState.java @@ -511,6 +511,8 @@ class PeerState implements DataLoader //bitfield = new BitField(meta.getPieces()); } metainfo = meta; + if (bitfield.count() > 0) + setInteresting(true); } /** @since 0.8.4 */ diff --git a/apps/i2psnark/java/src/org/klomp/snark/Snark.java b/apps/i2psnark/java/src/org/klomp/snark/Snark.java index 45231965fc..d114fe0ee2 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/Snark.java +++ b/apps/i2psnark/java/src/org/klomp/snark/Snark.java @@ -243,7 +243,7 @@ public class Snark public static final String PROP_MAX_CONNECTIONS = "i2psnark.maxConnections"; /** most of these used to be public, use accessors below instead */ - private final String torrent; + private String torrent; private MetaInfo meta; private Storage storage; private PeerCoordinator coordinator; @@ -360,6 +360,7 @@ public class Snark } } meta = new MetaInfo(new BDecoder(in)); + infoHash = meta.getInfoHash(); } catch(IOException ioe) { @@ -1028,8 +1029,13 @@ public class Snark meta = metainfo; try { storage = new Storage(_util, meta, this); - if (completeListener != null) - completeListener.gotMetaInfo(this); + storage.check(rootDataDir); + if (completeListener != null) { + String newName = completeListener.gotMetaInfo(this); + if (newName != null) + torrent = newName; + // else some horrible problem + } coordinator.setStorage(storage); } catch (IOException ioe) { if (storage != null) { @@ -1125,9 +1131,10 @@ public class Snark * metainfo and storage. The listener should now call getMetaInfo() * and save the data to disk. * + * @return the new name for the torrent or null on error * @since 0.8.4 */ - public void gotMetaInfo(Snark snark); + public String gotMetaInfo(Snark snark); // not really listeners but the easiest way to get back to an optional SnarkManager public long getSavedTorrentTime(Snark snark); diff --git a/apps/i2psnark/java/src/org/klomp/snark/SnarkManager.java b/apps/i2psnark/java/src/org/klomp/snark/SnarkManager.java index 27640d53ba..1e5ba10bf2 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/SnarkManager.java +++ b/apps/i2psnark/java/src/org/klomp/snark/SnarkManager.java @@ -68,7 +68,7 @@ public class SnarkManager implements Snark.CompleteListener { public static final String PROP_META_PREFIX = "i2psnark.zmeta."; public static final String PROP_META_BITFIELD_SUFFIX = ".bitfield"; public static final String PROP_META_PRIORITY_SUFFIX = ".priority"; - public static final String PROP_META_MAGNET_SUFFIX = ".magnet"; + public static final String PROP_META_MAGNET_PREFIX = "i2psnark.magnet."; private static final String CONFIG_FILE = "i2psnark.config"; public static final String PROP_AUTO_START = "i2snark.autoStart"; // oops @@ -340,7 +340,9 @@ public class SnarkManager implements Snark.CompleteListener { int oldI2CPPort = _util.getI2CPPort(); String oldI2CPHost = _util.getI2CPHost(); int port = oldI2CPPort; - try { port = Integer.parseInt(i2cpPort); } catch (NumberFormatException nfe) {} + if (i2cpPort != null) { + try { port = Integer.parseInt(i2cpPort); } catch (NumberFormatException nfe) {} + } String host = oldI2CPHost; Map opts = new HashMap(); if (i2cpOpts == null) i2cpOpts = ""; @@ -627,7 +629,7 @@ public class SnarkManager implements Snark.CompleteListener { * @throws RuntimeException via Snark.fatal() * @since 0.8.4 */ - public void addMagnet(String name, byte[] ih) { + public void addMagnet(String name, byte[] ih, boolean updateStatus) { Snark torrent = new Snark(_util, name, ih, this, _peerCoordinatorSet, _connectionAcceptor, false, getDataDir().getPath()); @@ -640,6 +642,8 @@ public class SnarkManager implements Snark.CompleteListener { } // Tell the dir monitor not to delete us _magnets.add(name); + if (updateStatus) + saveMagnetStatus(ih); _snarks.put(name, torrent); } if (shouldAutoStart()) { @@ -667,6 +671,7 @@ public class SnarkManager implements Snark.CompleteListener { } snark.stopTorrent(); _magnets.remove(snark.getName()); + removeMagnetStatus(snark.getInfoHash()); } /** @@ -914,6 +919,28 @@ public class SnarkManager implements Snark.CompleteListener { saveConfig(); } + /** + * Just remember we have it + * @since 0.8.4 + */ + public void saveMagnetStatus(byte[] ih) { + String infohash = Base64.encode(ih); + infohash = infohash.replace('=', '$'); + _config.setProperty(PROP_META_MAGNET_PREFIX + infohash, "."); + saveConfig(); + } + + /** + * Remove the magnet marker from the config file. + * @since 0.8.4 + */ + public void removeMagnetStatus(byte[] ih) { + String infohash = Base64.encode(ih); + infohash = infohash.replace('=', '$'); + _config.remove(PROP_META_MAGNET_PREFIX + infohash); + saveConfig(); + } + /** * Does not really delete on failure, that's the caller's responsibility. * Warning - does not validate announce URL - use TrackerClient.isValidAnnounce() @@ -1032,12 +1059,10 @@ public class SnarkManager implements Snark.CompleteListener { } } -//start magnets - - // here because we need to delay until I2CP is up // although the user will see the default until then getBWLimit(); + boolean doMagnets = true; while (true) { File dir = getDataDir(); if (_log.shouldLog(Log.DEBUG)) @@ -1050,6 +1075,10 @@ public class SnarkManager implements Snark.CompleteListener { } catch (Exception e) { _log.error("Error in the DirectoryMonitor", e); } + if (doMagnets) { + addMagnets(); + doMagnets = false; + } try { Thread.sleep(60*1000); } catch (InterruptedException ie) {} } } @@ -1090,9 +1119,10 @@ public class SnarkManager implements Snark.CompleteListener { * and save the data to disk. * A Snark.CompleteListener method. * + * @return the new name for the torrent or null on error * @since 0.8.4 */ - public void gotMetaInfo(Snark snark) { + public String gotMetaInfo(Snark snark) { MetaInfo meta = snark.getMetaInfo(); Storage storage = snark.getStorage(); if (meta != null && storage != null) { @@ -1100,23 +1130,51 @@ public class SnarkManager implements Snark.CompleteListener { if (rejectMessage != null) { addMessage(rejectMessage); snark.stopTorrent(); - return; + return null; } saveTorrentStatus(meta, storage.getBitField(), null); // no file priorities String name = (new File(getDataDir(), storage.getBaseName() + ".torrent")).getAbsolutePath(); try { synchronized (_snarks) { locked_writeMetaInfo(meta, name); + // put it in the list under the new name + _snarks.remove(snark.getName()); + _snarks.put(name, snark); } + _magnets.remove(snark.getName()); + removeMagnetStatus(snark.getInfoHash()); + addMessage(_("Metainfo received for {0}", snark.getName())); + addMessage(_("Starting up torrent {0}", storage.getBaseName())); + return name; } catch (IOException ioe) { addMessage(_("Failed to copy torrent file to {0}", name)); _log.error("Failed to write torrent file", ioe); } } + return null; } // End Snark.CompleteListeners + /** + * Add all magnets from the config file + * @since 0.8.4 + */ + private void addMagnets() { + for (Object o : _config.keySet()) { + String k = (String) o; + if (k.startsWith(PROP_META_MAGNET_PREFIX)) { + String b64 = k.substring(PROP_META_MAGNET_PREFIX.length()); + b64 = b64.replace('$', '='); + byte[] ih = Base64.decode(b64); + // ignore value + if (ih != null && ih.length == 20) + addMagnet("Magnet: " + I2PSnarkUtil.toHex(ih), ih, false); + // else remove from config? + } + } + } + private void monitorTorrents(File dir) { String fileNames[] = dir.list(TorrentFilenameFilter.instance()); List foundNames = new ArrayList(0); diff --git a/apps/i2psnark/java/src/org/klomp/snark/TrackerClient.java b/apps/i2psnark/java/src/org/klomp/snark/TrackerClient.java index 719edee69b..d03753d6bd 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/TrackerClient.java +++ b/apps/i2psnark/java/src/org/klomp/snark/TrackerClient.java @@ -173,7 +173,8 @@ public class TrackerClient extends I2PAppThread continue; if (primary.startsWith("http://i2p/" + dest)) continue; - trackers.add(new Tracker(url, false)); + // opentrackers are primary if we don't have primary + trackers.add(new Tracker(url, primary.equals(""))); _log.debug("Additional announce: [" + url + "] for infoHash: " + infoHash); } } @@ -238,7 +239,7 @@ public class TrackerClient extends I2PAppThread uploaded = coordinator.getUploaded(); downloaded = coordinator.getDownloaded(); - left = coordinator.getLeft(); + left = coordinator.getLeft(); // -1 in magnet mode // First time we got a complete download? String event; @@ -289,7 +290,7 @@ public class TrackerClient extends I2PAppThread } } - if ( (left > 0) && (!completed) ) { + if ( (left != 0) && (!completed) ) { // we only want to talk to new people if we need things // from them (duh) List ordered = new ArrayList(peers); @@ -344,7 +345,7 @@ public class TrackerClient extends I2PAppThread // FIXME this needs to be in its own thread if (_util.getDHT() != null && !stop) { int numwant; - if (left <= 0 || event.equals(STOPPED_EVENT) || !coordinator.needPeers()) + if (left == 0 || event.equals(STOPPED_EVENT) || !coordinator.needPeers()) numwant = 1; else numwant = _util.getMaxConnections(); @@ -362,7 +363,7 @@ public class TrackerClient extends I2PAppThread List peers = new ArrayList(hashes.size()); for (Hash h : hashes) { PeerID pID = new PeerID(h.getData()); - peers.add(new Peer(pID, snark.getID(), snark.getInfoHash(), meta)); + peers.add(new Peer(pID, snark.getID(), snark.getInfoHash(), snark.getMetaInfo())); } Collections.shuffle(peers, r); Iterator it = peers.iterator(); @@ -370,7 +371,7 @@ public class TrackerClient extends I2PAppThread Peer cur = it.next(); if (coordinator.addPeer(cur)) { int delay = DELAY_MUL; - delay *= ((int)cur.getPeerID().getAddress().calculateHash().toBase64().charAt(0)) % 10; + delay *= r.nextInt(10); delay += DELAY_MIN; try { Thread.sleep(delay); } catch (InterruptedException ie) {} } @@ -418,6 +419,8 @@ public class TrackerClient extends I2PAppThread long downloaded, long left, String event) throws IOException { + // What do we send for left in magnet mode? Can we omit it? + long tleft = left >= 0 ? left : 1; String s = tr.announce + "?info_hash=" + infoHash + "&peer_id=" + peerID @@ -425,10 +428,10 @@ public class TrackerClient extends I2PAppThread + "&ip=" + _util.getOurIPString() + ".i2p" + "&uploaded=" + uploaded + "&downloaded=" + downloaded - + "&left=" + left + + "&left=" + tleft + "&compact=1" // NOTE: opentracker will return 400 for &compact alone + ((! event.equals(NO_EVENT)) ? ("&event=" + event) : ""); - if (left <= 0 || event.equals(STOPPED_EVENT) || !coordinator.needPeers()) + if (left == 0 || event.equals(STOPPED_EVENT) || !coordinator.needPeers()) s += "&numwant=0"; else s += "&numwant=" + _util.getMaxConnections(); @@ -445,7 +448,7 @@ public class TrackerClient extends I2PAppThread in = new FileInputStream(fetched); TrackerInfo info = new TrackerInfo(in, snark.getID(), - snark.getMetaInfo()); + snark.getInfoHash(), snark.getMetaInfo()); _util.debug("TrackerClient response: " + info, Snark.INFO); String failure = info.getFailureReason(); diff --git a/apps/i2psnark/java/src/org/klomp/snark/TrackerInfo.java b/apps/i2psnark/java/src/org/klomp/snark/TrackerInfo.java index ef67fc58b1..1b829d0eef 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/TrackerInfo.java +++ b/apps/i2psnark/java/src/org/klomp/snark/TrackerInfo.java @@ -46,19 +46,20 @@ public class TrackerInfo private int complete; private int incomplete; - public TrackerInfo(InputStream in, byte[] my_id, MetaInfo metainfo) + /** @param metainfo may be null */ + public TrackerInfo(InputStream in, byte[] my_id, byte[] infohash, MetaInfo metainfo) throws IOException { - this(new BDecoder(in), my_id, metainfo); + this(new BDecoder(in), my_id, infohash, metainfo); } - public TrackerInfo(BDecoder be, byte[] my_id, MetaInfo metainfo) + private TrackerInfo(BDecoder be, byte[] my_id, byte[] infohash, MetaInfo metainfo) throws IOException { - this(be.bdecodeMap().getMap(), my_id, metainfo); + this(be.bdecodeMap().getMap(), my_id, infohash, metainfo); } - public TrackerInfo(Map m, byte[] my_id, MetaInfo metainfo) + private TrackerInfo(Map m, byte[] my_id, byte[] infohash, MetaInfo metainfo) throws IOException { BEValue reason = (BEValue)m.get("failure reason"); @@ -84,10 +85,10 @@ public class TrackerInfo Set p; try { // One big string (the official compact format) - p = getPeers(bePeers.getBytes(), my_id, metainfo); + p = getPeers(bePeers.getBytes(), my_id, infohash, metainfo); } catch (InvalidBEncodingException ibe) { // List of Dictionaries or List of Strings - p = getPeers(bePeers.getList(), my_id, metainfo); + p = getPeers(bePeers.getList(), my_id, infohash, metainfo); } peers = p; } @@ -123,7 +124,7 @@ public class TrackerInfo ******/ /** List of Dictionaries or List of Strings */ - private static Set getPeers(List l, byte[] my_id, MetaInfo metainfo) + private static Set getPeers(List l, byte[] my_id, byte[] infohash, MetaInfo metainfo) throws IOException { Set peers = new HashSet(l.size()); @@ -144,7 +145,7 @@ public class TrackerInfo continue; } } - peers.add(new Peer(peerID, my_id, metainfo.getInfoHash(), metainfo)); + peers.add(new Peer(peerID, my_id, infohash, metainfo)); } return peers; @@ -156,7 +157,7 @@ public class TrackerInfo * One big string of concatenated 32-byte hashes * @since 0.8.1 */ - private static Set getPeers(byte[] l, byte[] my_id, MetaInfo metainfo) + private static Set getPeers(byte[] l, byte[] my_id, byte[] infohash, MetaInfo metainfo) throws IOException { int count = l.length / HASH_LENGTH; @@ -172,7 +173,7 @@ public class TrackerInfo // won't happen continue; } - peers.add(new Peer(peerID, my_id, metainfo.getInfoHash(), metainfo)); + peers.add(new Peer(peerID, my_id, infohash, metainfo)); } return peers; diff --git a/apps/i2psnark/java/src/org/klomp/snark/bencode/BEValue.java b/apps/i2psnark/java/src/org/klomp/snark/bencode/BEValue.java index 986e456437..75a63bab19 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/bencode/BEValue.java +++ b/apps/i2psnark/java/src/org/klomp/snark/bencode/BEValue.java @@ -24,6 +24,8 @@ import java.io.UnsupportedEncodingException; import java.util.List; import java.util.Map; +import net.i2p.data.Base64; + /** * Holds different types that a bencoded byte array can represent. * You need to call the correct get method to get the correct java @@ -180,10 +182,14 @@ public class BEValue { byte[] bs = (byte[])value; // XXX - Stupid heuristic... and not UTF-8 - if (bs.length <= 12) - valueString = new String(bs); + //if (bs.length <= 12) + // valueString = new String(bs); + //else + // valueString = "bytes:" + bs.length; + if (bs.length <= 32) + valueString = bs.length + " bytes: " + Base64.encode(bs); else - valueString = "bytes:" + bs.length; + valueString = bs.length + " bytes"; } else valueString = value.toString(); diff --git a/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java b/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java index bd08dc46d3..077b24351f 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java +++ b/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java @@ -1314,15 +1314,17 @@ public class I2PSnarkServlet extends Default { out.write("     "); out.write(renderOptions(0, 4, options.remove("outbound.length"), "outbound.length", HOP)); - out.write("
"); - out.write(_("I2CP host")); - out.write(": "); + if (!_context.isRouterContext()) { + out.write("
"); + out.write(_("I2CP host")); + out.write(": "); - out.write("
"); - out.write(_("I2CP port")); - out.write(":
\n"); + out.write("
"); + out.write(_("I2CP port")); + out.write(":
\n"); + } StringBuilder opts = new StringBuilder(64); for (Iterator iter = options.entrySet().iterator(); iter.hasNext(); ) { From 2ae91a9801ab7cd266e130248c21aed8632b6111 Mon Sep 17 00:00:00 2001 From: zzz Date: Mon, 27 Dec 2010 18:08:43 +0000 Subject: [PATCH 15/27] listing info tweaks --- .../org/klomp/snark/web/I2PSnarkServlet.java | 81 +++++++++++++------ 1 file changed, 56 insertions(+), 25 deletions(-) diff --git a/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java b/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java index 077b24351f..6b82734555 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java +++ b/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java @@ -827,32 +827,13 @@ public class I2PSnarkServlet extends Default { out.write(statusString + "		"); - // temporarily hardcoded for postman* and anonymity, requires bytemonsoon patch for lookup by info_hash String announce = null; - if (isValid) + if (isValid) { announce = meta.getAnnounce(); - if (announce != null && (announce.startsWith("http://YRgrgTLG") || announce.startsWith("http://8EoJZIKr") || - announce.startsWith("http://lnQ6yoBT") || announce.startsWith("http://tracker2.postman.i2p/") || - announce.startsWith("http://ahsplxkbhemefwvvml7qovzl5a2b5xo5i7lyai7ntdunvcyfdtna.b32.i2p/"))) { - Map trackers = _manager.getTrackers(); - for (Iterator iter = trackers.entrySet().iterator(); iter.hasNext(); ) { - Map.Entry entry = (Map.Entry)iter.next(); - String name = (String)entry.getKey(); - String baseURL = (String)entry.getValue(); - if (!(baseURL.startsWith(announce) || // vvv hack for non-b64 announce in list vvv - (announce.startsWith("http://lnQ6yoBT") && baseURL.startsWith("http://tracker2.postman.i2p/")) || - (announce.startsWith("http://ahsplxkbhemefwvvml7qovzl5a2b5xo5i7lyai7ntdunvcyfdtna.b32.i2p/") && baseURL.startsWith("http://tracker2.postman.i2p/")))) - continue; - int e = baseURL.indexOf('='); - if (e < 0) - continue; - baseURL = baseURL.substring(e + 1); - out.write(""); - out.write("\"""); - out.write(""); - break; + if (announce != null) { + String trackerLink = getTrackerLink(announce, snark.getInfoHash()); + if (trackerLink != null) + out.write(trackerLink); } } @@ -1094,6 +1075,39 @@ public class I2PSnarkServlet extends Default { } } + /** + * @return string or null + * @since 0.8.4 + */ + private String getTrackerLink(String announce, byte[] infohash) { + // temporarily hardcoded for postman* and anonymity, requires bytemonsoon patch for lookup by info_hash + if (announce != null && (announce.startsWith("http://YRgrgTLG") || announce.startsWith("http://8EoJZIKr") || + announce.startsWith("http://lnQ6yoBT") || announce.startsWith("http://tracker2.postman.i2p/") || + announce.startsWith("http://ahsplxkbhemefwvvml7qovzl5a2b5xo5i7lyai7ntdunvcyfdtna.b32.i2p/"))) { + Map trackers = _manager.getTrackers(); + for (Map.Entry entry : trackers.entrySet()) { + String baseURL = entry.getValue(); + if (!(baseURL.startsWith(announce) || // vvv hack for non-b64 announce in list vvv + (announce.startsWith("http://lnQ6yoBT") && baseURL.startsWith("http://tracker2.postman.i2p/")) || + (announce.startsWith("http://ahsplxkbhemefwvvml7qovzl5a2b5xo5i7lyai7ntdunvcyfdtna.b32.i2p/") && baseURL.startsWith("http://tracker2.postman.i2p/")))) + continue; + int e = baseURL.indexOf('='); + if (e < 0) + continue; + baseURL = baseURL.substring(e + 1); + String name = entry.getKey(); + StringBuilder buf = new StringBuilder(128); + buf.append("" + + "\"").append(_("Info")).append("\""); + return buf.toString(); + } + } + return null; + } + private void writeAddForm(PrintWriter out, HttpServletRequest req) throws IOException { String uri = req.getRequestURI(); String newURL = req.getParameter("newURL"); @@ -1577,8 +1591,25 @@ public class I2PSnarkServlet extends Default { } buf.append("
").append(_("Pieces")).append(": ").append(pieces); buf.append("
").append(_("Piece size")).append(": ").append(formatSize(snark.getPieceLength(0))); + + if (meta != null) { + String announce = meta.getAnnounce(); + if (announce != null) { + String trackerLink = getTrackerLink(announce, snark.getInfoHash()); + if (trackerLink != null) { + if (announce.startsWith("http://")) + announce = announce.substring(7); + int slsh = announce.indexOf('/'); + if (slsh > 0) + announce = announce.substring(0, slsh); + buf.append("
").append(trackerLink).append(' ').append(announce); + } + } + } + String hex = I2PSnarkUtil.toHex(snark.getInfoHash()); - buf.append("
").append(_("Magnet link")).append(": ") + buf.append("
").append(toImg("magnet", _("Magnet link"))).append(" ") .append(MAGNET).append(hex).append(""); // We don't have the hash of the torrent file //buf.append("
").append(_("Maggot link")).append(": ") From 01b67acfa017ef92e5215431396b1f8c4b56a2a8 Mon Sep 17 00:00:00 2001 From: zzz Date: Tue, 28 Dec 2010 15:46:45 +0000 Subject: [PATCH 16/27] - Catch OOM when creating torrent (tickets #364 and #366) - Fix changing 'use open trackers' config setting (old bug) - More TrackerClient fixes for no primary announce - More BEValue.toString() improvements for debugging --- .../src/org/klomp/snark/I2PSnarkUtil.java | 15 +++++--- .../java/src/org/klomp/snark/Snark.java | 2 + .../src/org/klomp/snark/SnarkManager.java | 7 +++- .../src/org/klomp/snark/TrackerClient.java | 10 ++--- .../src/org/klomp/snark/bencode/BEValue.java | 37 +++++++++++++++---- .../org/klomp/snark/web/I2PSnarkServlet.java | 2 + 6 files changed, 53 insertions(+), 20 deletions(-) diff --git a/apps/i2psnark/java/src/org/klomp/snark/I2PSnarkUtil.java b/apps/i2psnark/java/src/org/klomp/snark/I2PSnarkUtil.java index 775d4cc4ab..16edc12f65 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/I2PSnarkUtil.java +++ b/apps/i2psnark/java/src/org/klomp/snark/I2PSnarkUtil.java @@ -60,6 +60,7 @@ public class I2PSnarkUtil { private int _maxConnections; private File _tmpDir; private int _startupDelay; + private boolean _shouldUseOT; private KRPC _krpc; public static final int DEFAULT_STARTUP_DELAY = 3; @@ -83,6 +84,7 @@ public class I2PSnarkUtil { _maxUpBW = DEFAULT_MAX_UP_BW; _maxConnections = MAX_CONNECTIONS; _startupDelay = DEFAULT_STARTUP_DELAY; + _shouldUseOT = DEFAULT_USE_OPENTRACKERS; // This is used for both announce replies and .torrent file downloads, // so it must be available even if not connected to I2CP. // so much for multiple instances @@ -425,10 +427,10 @@ public class I2PSnarkUtil { /** comma delimited list open trackers to use as backups */ /** sorted map of name to announceURL=baseURL */ - public List getOpenTrackers() { + public List getOpenTrackers() { if (!shouldUseOpenTrackers()) return null; - List rv = new ArrayList(1); + List rv = new ArrayList(1); String trackers = getOpenTrackerString(); StringTokenizer tok = new StringTokenizer(trackers, ", "); while (tok.hasMoreTokens()) @@ -439,11 +441,12 @@ public class I2PSnarkUtil { return rv; } + public void setUseOpenTrackers(boolean yes) { + _shouldUseOT = yes; + } + public boolean shouldUseOpenTrackers() { - String rv = (String) _opts.get(PROP_USE_OPENTRACKERS); - if (rv == null) - return DEFAULT_USE_OPENTRACKERS; - return Boolean.valueOf(rv).booleanValue(); + return _shouldUseOT; } /** diff --git a/apps/i2psnark/java/src/org/klomp/snark/Snark.java b/apps/i2psnark/java/src/org/klomp/snark/Snark.java index d114fe0ee2..e3a99bdc9b 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/Snark.java +++ b/apps/i2psnark/java/src/org/klomp/snark/Snark.java @@ -394,6 +394,8 @@ public class Snark */ else fatal("Cannot open '" + torrent + "'", ioe); + } catch (OutOfMemoryError oom) { + fatal("ERROR - Out of memory, cannot create torrent " + torrent + ": " + oom.getMessage()); } finally { if (in != null) try { in.close(); } catch (IOException ioe) {} diff --git a/apps/i2psnark/java/src/org/klomp/snark/SnarkManager.java b/apps/i2psnark/java/src/org/klomp/snark/SnarkManager.java index 1e5ba10bf2..6ac7261e3e 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/SnarkManager.java +++ b/apps/i2psnark/java/src/org/klomp/snark/SnarkManager.java @@ -263,7 +263,9 @@ public class SnarkManager implements Snark.CompleteListener { String ot = _config.getProperty(I2PSnarkUtil.PROP_OPENTRACKERS); if (ot != null) _util.setOpenTrackerString(ot); - // FIXME set util use open trackers property somehow + String useOT = _config.getProperty(I2PSnarkUtil.PROP_USE_OPENTRACKERS); + boolean bOT = useOT == null || Boolean.valueOf(useOT).booleanValue(); + _util.setUseOpenTrackers(bOT); getDataDir().mkdirs(); } @@ -434,6 +436,7 @@ public class SnarkManager implements Snark.CompleteListener { addMessage(_("Enabled open trackers - torrent restart required to take effect.")); else addMessage(_("Disabled open trackers - torrent restart required to take effect.")); + _util.setUseOpenTrackers(useOpenTrackers); changed = true; } if (openTrackers != null) { @@ -605,6 +608,8 @@ public class SnarkManager implements Snark.CompleteListener { if (sfile.exists()) sfile.delete(); return; + } catch (OutOfMemoryError oom) { + addMessage(_("ERROR - Out of memory, cannot create torrent from {0}", sfile.getName()) + ": " + oom.getMessage()); } finally { if (fis != null) try { fis.close(); } catch (IOException ioe) {} } diff --git a/apps/i2psnark/java/src/org/klomp/snark/TrackerClient.java b/apps/i2psnark/java/src/org/klomp/snark/TrackerClient.java index d03753d6bd..9cf9f7f097 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/TrackerClient.java +++ b/apps/i2psnark/java/src/org/klomp/snark/TrackerClient.java @@ -137,18 +137,18 @@ public class TrackerClient extends I2PAppThread // the primary tracker, that we don't add it twice. // todo: check for b32 matches as well trackers = new ArrayList(2); - String primary; + String primary = null; if (meta != null) { primary = meta.getAnnounce(); if (isValidAnnounce(primary)) { trackers.add(new Tracker(meta.getAnnounce(), true)); + _log.debug("Announce: [" + primary + "] infoHash: " + infoHash); } else { _log.warn("Skipping invalid or non-i2p announce: " + primary); } - } else { - primary = ""; } - _log.debug("Announce: [" + primary + "] infoHash: " + infoHash); + if (primary == null) + primary = ""; List tlist = _util.getOpenTrackers(); if (tlist != null) { for (int i = 0; i < tlist.size(); i++) { @@ -179,7 +179,7 @@ public class TrackerClient extends I2PAppThread } } - if (tlist.isEmpty()) { + if (trackers.isEmpty()) { // FIXME really need to get this message to the gui stop = true; _log.error("No valid trackers for infoHash: " + infoHash); diff --git a/apps/i2psnark/java/src/org/klomp/snark/bencode/BEValue.java b/apps/i2psnark/java/src/org/klomp/snark/bencode/BEValue.java index 75a63bab19..4cae2881ae 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/bencode/BEValue.java +++ b/apps/i2psnark/java/src/org/klomp/snark/bencode/BEValue.java @@ -180,15 +180,36 @@ public class BEValue String valueString; if (value instanceof byte[]) { + // try to do a nice job for debugging byte[] bs = (byte[])value; - // XXX - Stupid heuristic... and not UTF-8 - //if (bs.length <= 12) - // valueString = new String(bs); - //else - // valueString = "bytes:" + bs.length; - if (bs.length <= 32) - valueString = bs.length + " bytes: " + Base64.encode(bs); - else + if (bs.length == 0) + valueString = "0 bytes"; + else if (bs.length <= 32) { + StringBuilder buf = new StringBuilder(32); + boolean bin = false; + for (int i = 0; i < bs.length; i++) { + int b = bs[i] & 0xff; + // no UTF-8 + if (b < ' ' || b > 0x7e) { + bin = true; + break; + } + } + if (bin && bs.length <= 8) { + buf.append(bs.length).append(" bytes: 0x"); + for (int i = 0; i < bs.length; i++) { + int b = bs[i] & 0xff; + if (b < 16) + buf.append('0'); + buf.append(Integer.toHexString(b)); + } + } else if (bin) { + buf.append(bs.length).append(" bytes: ").append(Base64.encode(bs)); + } else { + buf.append('"').append(new String(bs)).append('"'); + } + valueString = buf.toString(); + } else valueString = bs.length + " bytes"; } else diff --git a/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java b/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java index 6b82734555..5890c0df65 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java +++ b/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java @@ -1902,6 +1902,8 @@ private static class FetchAndAdd implements Runnable { } } catch (IOException ioe) { _manager.addMessage(_("Torrent at {0} was not valid", urlify(_url)) + ": " + ioe.getMessage()); + } catch (OutOfMemoryError oom) { + _manager.addMessage(_("ERROR - Out of memory, cannot create torrent from {0}", urlify(_url)) + ": " + oom.getMessage()); } finally { try { if (in != null) in.close(); } catch (IOException ioe) {} } From 5a4af0eeb8a56967963e052ffeb40e3499326f02 Mon Sep 17 00:00:00 2001 From: zzz Date: Thu, 30 Dec 2010 15:33:36 +0000 Subject: [PATCH 17/27] fixup after prop --- .../src/org/klomp/snark/web/I2PSnarkServlet.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java b/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java index 058461fc4b..c71c06cdfb 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java +++ b/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java @@ -374,7 +374,7 @@ public class I2PSnarkServlet extends Default { Snark snark = (Snark)snarks.get(i); boolean showDebug = "2".equals(peerParam); boolean showPeers = showDebug || "1".equals(peerParam) || Base64.encode(snark.getInfoHash()).equals(peerParam); - displaySnark(out, snark, uri, i, stats, showPeers, isDegraded, showDebug); + displaySnark(out, snark, uri, i, stats, showPeers, isDegraded, noThinsp, showDebug); } if (snarks.isEmpty()) { @@ -735,7 +735,7 @@ public class I2PSnarkServlet extends Default { private static final int MAX_DISPLAYED_FILENAME_LENGTH = 50; private static final int MAX_DISPLAYED_ERROR_LENGTH = 43; private void displaySnark(PrintWriter out, Snark snark, String uri, int row, long stats[], boolean showPeers, - boolean isDegraded, boolean showDebug) throws IOException { + boolean isDegraded, boolean noThinsp, boolean showDebug) throws IOException { String filename = snark.getName(); File f = new File(filename); filename = f.getName(); // the torrent may be the canonical name, so lets just grab the local name @@ -785,7 +785,7 @@ public class I2PSnarkServlet extends Default { if (isRunning && curPeers > 0 && !showPeers) statusString = "\"\"		" + _("Tracker Error") + ": " + - curPeers + thinsp(isDegraded) + + curPeers + thinsp(noThinsp) + ngettext("1 peer", "{0} peers", knownPeers) + ""; else if (isRunning) statusString = "\"\"" + _("Tracker Error") + @@ -801,7 +801,7 @@ public class I2PSnarkServlet extends Default { if (isRunning && curPeers > 0 && !showPeers) statusString = "\"\"" + _("Seeding") + ": " + - curPeers + thinsp(isDegraded) + + curPeers + thinsp(noThinsp) + ngettext("1 peer", "{0} peers", knownPeers) + ""; else if (isRunning) statusString = "\"\"" + _("Seeding") + @@ -813,7 +813,7 @@ public class I2PSnarkServlet extends Default { if (isRunning && curPeers > 0 && downBps > 0 && !showPeers) statusString = "\"\"" + _("OK") + ": " + - curPeers + thinsp(isDegraded) + + curPeers + thinsp(noThinsp) + ngettext("1 peer", "{0} peers", knownPeers) + ""; else if (isRunning && curPeers > 0 && downBps > 0) statusString = "\"\"" + _("OK") + @@ -822,7 +822,7 @@ public class I2PSnarkServlet extends Default { else if (isRunning && curPeers > 0 && !showPeers) statusString = "\"\"" + _("Stalled") + ": " + - curPeers + thinsp(isDegraded) + + curPeers + thinsp(noThinsp) + ngettext("1 peer", "{0} peers", knownPeers) + ""; else if (isRunning && curPeers > 0) statusString = "\"\"" + _("Stalled") + @@ -893,7 +893,7 @@ public class I2PSnarkServlet extends Default { out.write(""); if (remaining > 0) - out.write(formatSize(total-remaining) + thinsp(isDegraded) + formatSize(total)); + out.write(formatSize(total-remaining) + thinsp(noThinsp) + formatSize(total)); else if (remaining == 0) out.write(formatSize(total)); // 3GB else From 4d4bfa00b3082f3d865c1261c36e3f5352fa8afe Mon Sep 17 00:00:00 2001 From: zzz Date: Sat, 1 Jan 2011 14:59:41 +0000 Subject: [PATCH 18/27] add dht stub --- .../src/org/klomp/snark/I2PSnarkUtil.java | 9 +- .../src/org/klomp/snark/PeerCoordinator.java | 8 +- .../src/org/klomp/snark/TrackerClient.java | 2 +- .../java/src/org/klomp/snark/dht/DHT.java | 83 +++++++++++++++++++ 4 files changed, 93 insertions(+), 9 deletions(-) create mode 100644 apps/i2psnark/java/src/org/klomp/snark/dht/DHT.java diff --git a/apps/i2psnark/java/src/org/klomp/snark/I2PSnarkUtil.java b/apps/i2psnark/java/src/org/klomp/snark/I2PSnarkUtil.java index 16edc12f65..89f5087b15 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/I2PSnarkUtil.java +++ b/apps/i2psnark/java/src/org/klomp/snark/I2PSnarkUtil.java @@ -34,6 +34,7 @@ import net.i2p.util.SimpleScheduler; import net.i2p.util.SimpleTimer; import net.i2p.util.Translate; +import org.klomp.snark.dht.DHT; import org.klomp.snark.dht.KRPC; /** @@ -61,7 +62,7 @@ public class I2PSnarkUtil { private File _tmpDir; private int _startupDelay; private boolean _shouldUseOT; - private KRPC _krpc; + private DHT _dht; public static final int DEFAULT_STARTUP_DELAY = 3; public static final String PROP_USE_OPENTRACKERS = "i2psnark.useOpentrackers"; @@ -195,8 +196,8 @@ public class I2PSnarkUtil { _manager = I2PSocketManagerFactory.createManager(_i2cpHost, _i2cpPort, opts); } // FIXME this only instantiates krpc once, left stuck with old manager - if (ENABLE_DHT && _manager != null && _krpc == null) - _krpc = new KRPC(_context, _manager.getSession()); + if (ENABLE_DHT && _manager != null && _dht == null) + _dht = new KRPC(_context, _manager.getSession()); return (_manager != null); } @@ -204,7 +205,7 @@ public class I2PSnarkUtil { * @return null if disabled or not started * @since 0.8.4 */ - public KRPC getDHT() { return _krpc; } + public DHT getDHT() { return _dht; } public boolean connected() { return _manager != null; } diff --git a/apps/i2psnark/java/src/org/klomp/snark/PeerCoordinator.java b/apps/i2psnark/java/src/org/klomp/snark/PeerCoordinator.java index 74a840fc7b..dbecc757e6 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/PeerCoordinator.java +++ b/apps/i2psnark/java/src/org/klomp/snark/PeerCoordinator.java @@ -36,7 +36,7 @@ import net.i2p.util.I2PAppThread; import net.i2p.util.Log; import net.i2p.util.SimpleTimer2; -import org.klomp.snark.dht.KRPC; +import org.klomp.snark.dht.DHT; /** * Coordinates what peer does what. @@ -1194,9 +1194,9 @@ public class PeerCoordinator implements PeerListener * @since 0.8.4 */ public void gotPort(Peer peer, int port) { - KRPC krpc = _util.getDHT(); - if (krpc != null) - krpc.ping(peer.getDestination(), port); + DHT dht = _util.getDHT(); + if (dht != null) + dht.ping(peer.getDestination(), port); } /** Return number of allowed uploaders for this torrent. diff --git a/apps/i2psnark/java/src/org/klomp/snark/TrackerClient.java b/apps/i2psnark/java/src/org/klomp/snark/TrackerClient.java index 9cf9f7f097..d03ca55db3 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/TrackerClient.java +++ b/apps/i2psnark/java/src/org/klomp/snark/TrackerClient.java @@ -38,7 +38,7 @@ import net.i2p.data.Hash; import net.i2p.util.I2PAppThread; import net.i2p.util.Log; -import org.klomp.snark.dht.KRPC; +import org.klomp.snark.dht.DHT; /** * Informs metainfo tracker of events and gets new peers for peer diff --git a/apps/i2psnark/java/src/org/klomp/snark/dht/DHT.java b/apps/i2psnark/java/src/org/klomp/snark/dht/DHT.java new file mode 100644 index 0000000000..29e0fa20b4 --- /dev/null +++ b/apps/i2psnark/java/src/org/klomp/snark/dht/DHT.java @@ -0,0 +1,83 @@ +package org.klomp.snark.dht; + +/* + * Copyright 2010 zzz (zzz@mail.i2p) + * GPLv2 + */ + +import java.util.List; + +import net.i2p.data.Destination; +import net.i2p.data.Hash; + + +/** + * Stub for KRPC + */ +public interface DHT { + + + /** + * @return The UDP port that should be included in a PORT message. + */ + public int getPort(); + + /** + * Ping. We don't have a NID yet so the node is presumed + * to be absent from our DHT. + * Non-blocking, does not wait for pong. + * If and when the pong is received the node will be inserted in our DHT. + */ + public void ping(Destination dest, int port); + + /** + * Get peers for a torrent. + * Blocking! + * Caller should run in a thread. + * + * @param ih the Info Hash (torrent) + * @param max maximum number of peers to return + * @param maxWait the maximum time to wait (ms) must be > 0 + * @return list or empty list (never null) + */ + public List getPeers(byte[] ih, int max, long maxWait); + + /** + * Announce to ourselves. + * Non-blocking. + * + * @param ih the Info Hash (torrent) + */ + public void announce(byte[] ih); + + /** + * Announce somebody else we know about. + * Non-blocking. + * + * @param ih the Info Hash (torrent) + * @param peer the peer's Hash + */ + public void announce(byte[] ih, byte[] peerHash); + + /** + * Remove reference to ourselves in the local tracker. + * Use when shutting down the torrent locally. + * Non-blocking. + * + * @param ih the Info Hash (torrent) + */ + public void unannounce(byte[] ih); + + /** + * Announce to the closest DHT peers. + * Blocking unless maxWait <= 0 + * Caller should run in a thread. + * This also automatically announces ourself to our local tracker. + * For best results do a getPeers() first so we have tokens. + * + * @param ih the Info Hash (torrent) + * @param maxWait the maximum total time to wait (ms) or 0 to do all in parallel and return immediately. + * @return the number of successful announces, not counting ourselves. + */ + public int announce(byte[] ih, int max, long maxWait); +} From 8d59fd1cbe6105d888a4280dd6b6241656f53ec1 Mon Sep 17 00:00:00 2001 From: zzz Date: Mon, 3 Jan 2011 16:21:51 +0000 Subject: [PATCH 19/27] some more defines --- apps/i2psnark/java/src/org/klomp/snark/ExtensionHandler.java | 4 +++- apps/i2psnark/java/src/org/klomp/snark/Peer.java | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/apps/i2psnark/java/src/org/klomp/snark/ExtensionHandler.java b/apps/i2psnark/java/src/org/klomp/snark/ExtensionHandler.java index 5b7b0b0382..3748b6239f 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/ExtensionHandler.java +++ b/apps/i2psnark/java/src/org/klomp/snark/ExtensionHandler.java @@ -23,8 +23,10 @@ abstract class ExtensionHandler { private static final Log _log = I2PAppContext.getGlobalContext().logManager().getLog(ExtensionHandler.class); - public static final int ID_METADATA = 3; + public static final int ID_METADATA = 1; private static final String TYPE_METADATA = "ut_metadata"; + public static final int ID_PEX = 2; + private static final String TYPE_PEX = "ut_pex"; /** Pieces * SHA1 Hash length, + 25% extra for file names, benconding overhead, etc */ private static final int MAX_METADATA_SIZE = Storage.MAX_PIECES * 20 * 5 / 4; private static final int PARALLEL_REQUESTS = 3; diff --git a/apps/i2psnark/java/src/org/klomp/snark/Peer.java b/apps/i2psnark/java/src/org/klomp/snark/Peer.java index 18c9ad229f..dd98e5dd92 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/Peer.java +++ b/apps/i2psnark/java/src/org/klomp/snark/Peer.java @@ -79,6 +79,7 @@ public class Peer implements Comparable static final long OPTION_EXTENSION = 0x0000000000100000l; static final long OPTION_FAST = 0x0000000000000004l; static final long OPTION_DHT = 0x0000000000000001l; + static final long OPTION_AZMP = 0x1000000000000000l; private long options; /** From 3305f1790c88f953504e433aa603d7d642818286 Mon Sep 17 00:00:00 2001 From: zzz Date: Mon, 3 Jan 2011 18:02:26 +0000 Subject: [PATCH 20/27] details page for single-file torrents --- .../org/klomp/snark/web/I2PSnarkServlet.java | 53 ++++++++++--------- 1 file changed, 29 insertions(+), 24 deletions(-) diff --git a/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java b/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java index 51bb548bbc..616df5b83a 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java +++ b/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java @@ -161,7 +161,7 @@ public class I2PSnarkServlet extends Default { resp.setCharacterEncoding("UTF-8"); resp.setContentType("text/html; charset=UTF-8"); Resource resource = getResource(pathInContext); - if (resource == null || (!resource.exists()) || !resource.isDirectory()) { + if (resource == null || (!resource.exists())) { resp.sendError(HttpResponse.__404_Not_Found); } else { String base = URI.addPaths(req.getRequestURI(), "/"); @@ -853,14 +853,13 @@ public class I2PSnarkServlet extends Default { out.write(statusString + ""); - String announce = null; if (isValid) { - announce = meta.getAnnounce(); - if (announce != null) { - String trackerLink = getTrackerLink(announce, snark.getInfoHash()); - if (trackerLink != null) - out.write(trackerLink); - } + StringBuilder buf = new StringBuilder(128); + buf.append("\"").append(_("Info")).append("\""); + out.write(buf.toString()); } out.write(""); @@ -1555,13 +1554,11 @@ public class I2PSnarkServlet extends Default { private String getListHTML(Resource r, String base, boolean parent, Map postParams) throws IOException { - if (!r.isDirectory()) - return null; - - String[] ls = r.list(); - if (ls==null) - return null; - Arrays.sort(ls, Collator.getInstance()); + String[] ls = null; + if (r.isDirectory()) { + ls = r.list(); + Arrays.sort(ls, Collator.getInstance()); + } // if r is not a directory, we are only showing torrent info section StringBuilder buf=new StringBuilder(4096); buf.append(DOCTYPE + ""); @@ -1592,7 +1589,7 @@ public class I2PSnarkServlet extends Default { if (parent) // always true buf.append("<div class=\"page\"><div class=\"mainsection\">"); - boolean showPriority = snark != null && snark.getStorage() != null && !snark.getStorage().complete(); + boolean showPriority = ls != null && snark != null && snark.getStorage() != null && !snark.getStorage().complete(); if (showPriority) buf.append("<form action=\"").append(base).append("\" method=\"POST\">\n"); buf.append("<TABLE BORDER=0 class=\"snarkTorrents\" ><thead>"); @@ -1621,15 +1618,17 @@ public class I2PSnarkServlet extends Default { if (meta != null) { String announce = meta.getAnnounce(); if (announce != null) { + buf.append("<br>"); String trackerLink = getTrackerLink(announce, snark.getInfoHash()); - if (trackerLink != null) { - if (announce.startsWith("http://")) - announce = announce.substring(7); - int slsh = announce.indexOf('/'); - if (slsh > 0) - announce = announce.substring(0, slsh); - buf.append("<br>").append(trackerLink).append(' ').append(announce); - } + if (trackerLink != null) + buf.append(trackerLink).append(' '); + buf.append(_("Tracker")).append(": "); + if (announce.startsWith("http://")) + announce = announce.substring(7); + int slsh = announce.indexOf('/'); + if (slsh > 0) + announce = announce.substring(0, slsh); + buf.append(announce); } } @@ -1642,6 +1641,12 @@ public class I2PSnarkServlet extends Default { // .append(MAGGOT).append(hex).append(':').append(hex).append("</a>"); buf.append("</div></th></tr>"); } + if (ls == null) { + // We are only showing the torrent info section + buf.append("</thead></table></div></div></BODY></HTML>"); + return buf.toString(); + } + // second row - dir info buf.append("<tr><th>") .append("<img alt=\"\" border=\"0\" src=\"" + _imgPath + "file.png\" > ") From 976544bd1dc50dc3dc9f8e24d4397efb35e7cb01 Mon Sep 17 00:00:00 2001 From: zzz <zzz@mail.i2p> Date: Tue, 4 Jan 2011 00:36:24 +0000 Subject: [PATCH 21/27] stub out PEX --- .../src/org/klomp/snark/ExtensionHandler.java | 88 ++++++++++++++++++- .../src/org/klomp/snark/PeerCoordinator.java | 18 ++++ .../src/org/klomp/snark/PeerListener.java | 11 ++- 3 files changed, 113 insertions(+), 4 deletions(-) diff --git a/apps/i2psnark/java/src/org/klomp/snark/ExtensionHandler.java b/apps/i2psnark/java/src/org/klomp/snark/ExtensionHandler.java index 3748b6239f..7b2979b8ab 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/ExtensionHandler.java +++ b/apps/i2psnark/java/src/org/klomp/snark/ExtensionHandler.java @@ -3,10 +3,13 @@ package org.klomp.snark; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; import net.i2p.I2PAppContext; +import net.i2p.data.DataHelper; import net.i2p.util.Log; import org.klomp.snark.bencode.BDecoder; @@ -23,10 +26,11 @@ abstract class ExtensionHandler { private static final Log _log = I2PAppContext.getGlobalContext().logManager().getLog(ExtensionHandler.class); + public static final int ID_HANDSHAKE = 0; public static final int ID_METADATA = 1; - private static final String TYPE_METADATA = "ut_metadata"; + public static final String TYPE_METADATA = "ut_metadata"; public static final int ID_PEX = 2; - private static final String TYPE_PEX = "ut_pex"; + public static final String TYPE_PEX = "ut_pex"; /** Pieces * SHA1 Hash length, + 25% extra for file names, benconding overhead, etc */ private static final int MAX_METADATA_SIZE = Storage.MAX_PIECES * 20 * 5 / 4; private static final int PARALLEL_REQUESTS = 3; @@ -40,6 +44,7 @@ abstract class ExtensionHandler { Map<String, Object> handshake = new HashMap(); Map<String, Integer> m = new HashMap(); m.put(TYPE_METADATA, Integer.valueOf(ID_METADATA)); + m.put(TYPE_PEX, Integer.valueOf(ID_PEX)); if (metasize >= 0) handshake.put("metadata_size", Integer.valueOf(metasize)); handshake.put("m", m); @@ -52,10 +57,12 @@ abstract class ExtensionHandler { public static void handleMessage(Peer peer, PeerListener listener, int id, byte[] bs) { if (_log.shouldLog(Log.INFO)) _log.info("Got extension msg " + id + " length " + bs.length + " from " + peer); - if (id == 0) + if (id == ID_HANDSHAKE) handleHandshake(peer, listener, bs); else if (id == ID_METADATA) handleMetadata(peer, listener, bs); + else if (id == ID_PEX) + handlePEX(peer, listener, bs); else if (_log.shouldLog(Log.INFO)) _log.info("Unknown extension msg " + id + " from " + peer); } @@ -72,6 +79,12 @@ abstract class ExtensionHandler { peer.setHandshakeMap(map); Map<String, BEValue> msgmap = map.get("m").getMap(); + if (msgmap.get(TYPE_PEX) != null) { + if (_log.shouldLog(Log.WARN)) + _log.debug("Peer supports PEX extension: " + peer); + // peer state calls peer listener calls sendPEX() + } + if (msgmap.get(TYPE_METADATA) == null) { if (_log.shouldLog(Log.WARN)) _log.debug("Peer does not support metadata extension: " + peer); @@ -216,6 +229,7 @@ abstract class ExtensionHandler { } catch (Exception e) { if (_log.shouldLog(Log.WARN)) _log.info("Metadata ext. msg. exception from " + peer, e); + // fatal ? peer.disconnect(false); } } @@ -262,4 +276,72 @@ abstract class ExtensionHandler { _log.info("Metadata send piece msg exception to " + peer, e); } } + + private static final int HASH_LENGTH = 32; + + /** + * Can't find a published standard for this anywhere. + * See the libtorrent code. + * Here we use the "added" key as a single string of concatenated + * 32-byte peer hashes. + * added.f and dropped unsupported + * @since 0.8.4 + */ + private static void handlePEX(Peer peer, PeerListener listener, byte[] bs) { + if (_log.shouldLog(Log.DEBUG)) + _log.debug("Got PEX msg from " + peer); + try { + InputStream is = new ByteArrayInputStream(bs); + BDecoder dec = new BDecoder(is); + BEValue bev = dec.bdecodeMap(); + Map<String, BEValue> map = bev.getMap(); + byte[] ids = map.get("added").getBytes(); + if (ids.length < HASH_LENGTH) + return; + int len = Math.min(ids.length, (I2PSnarkUtil.MAX_CONNECTIONS - 1) * HASH_LENGTH); + List<PeerID> peers = new ArrayList(len / HASH_LENGTH); + for (int off = 0; off < len; off += HASH_LENGTH) { + byte[] hash = new byte[HASH_LENGTH]; + System.arraycopy(ids, off, hash, 0, HASH_LENGTH); + if (DataHelper.eq(hash, peer.getPeerID().getDestHash())) + continue; + PeerID pID = new PeerID(hash); + peers.add(pID); + } + // could include ourselves, listener must remove + listener.gotPeers(peer, peers); + } catch (Exception e) { + if (_log.shouldLog(Log.WARN)) + _log.info("PEX msg exception from " + peer, e); + //peer.disconnect(false); + } + } + + /** + * added.f and dropped unsupported + * @param pList non-null + * @since 0.8.4 + */ + public static void sendPEX(Peer peer, List<Peer> pList) { + if (pList.isEmpty()) + return; + Map<String, Object> map = new HashMap(); + byte[] peers = new byte[HASH_LENGTH * pList.size()]; + int off = 0; + for (Peer p : pList) { + System.arraycopy(p.getPeerID().getDestHash(), 0, peers, off, HASH_LENGTH); + off += HASH_LENGTH; + } + map.put("added", peers); + byte[] payload = BEncoder.bencode(map); + try { + int hisMsgCode = peer.getHandshakeMap().get("m").getMap().get(TYPE_PEX).getInt(); + peer.sendExtension(hisMsgCode, payload); + } catch (Exception e) { + // NPE, no PEX caps + if (_log.shouldLog(Log.WARN)) + _log.info("PEX msg exception to " + peer, e); + } + } + } diff --git a/apps/i2psnark/java/src/org/klomp/snark/PeerCoordinator.java b/apps/i2psnark/java/src/org/klomp/snark/PeerCoordinator.java index dbecc757e6..7852840ccb 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/PeerCoordinator.java +++ b/apps/i2psnark/java/src/org/klomp/snark/PeerCoordinator.java @@ -1171,6 +1171,16 @@ public class PeerCoordinator implements PeerListener listener.gotMetaInfo(this, metainfo); } } + } else if (id == ExtensionHandler.ID_HANDSHAKE) { + try { + if (peer.getHandshakeMap().get("m").getMap().get(ExtensionHandler.TYPE_PEX) != null) { + List<Peer> pList = peerList(); + pList.remove(peer); + ExtensionHandler.sendPEX(peer, pList); + } + } catch (Exception e) { + // NPE, no map + } } } @@ -1199,6 +1209,14 @@ public class PeerCoordinator implements PeerListener dht.ping(peer.getDestination(), port); } + /** + * PeerListener callback + * @since 0.8.4 + */ + public void gotPeers(Peer peer, List<PeerID> peers) { + + } + /** Return number of allowed uploaders for this torrent. ** Check with Snark to see if we are over the total upload limit. */ diff --git a/apps/i2psnark/java/src/org/klomp/snark/PeerListener.java b/apps/i2psnark/java/src/org/klomp/snark/PeerListener.java index 6d9e681cb6..c7650a5524 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/PeerListener.java +++ b/apps/i2psnark/java/src/org/klomp/snark/PeerListener.java @@ -191,11 +191,20 @@ interface PeerListener void gotExtension(Peer peer, int id, byte[] bs); /** - * Called when an extension message is received. + * Called when a port message is received. * * @param peer the Peer that got the message. * @param port the port * @since 0.8.4 */ void gotPort(Peer peer, int port); + + /** + * Called when peers are received via PEX + * + * @param peer the Peer that got the message. + * @param pIDList the peer IDs (dest hashes) + * @since 0.8.4 + */ + void gotPeers(Peer peer, List<PeerID> pIDList); } From 6670209cb3061c1f81337283e72876a94274f3a4 Mon Sep 17 00:00:00 2001 From: zzz <zzz@mail.i2p> Date: Mon, 10 Jan 2011 17:14:34 +0000 Subject: [PATCH 22/27] i2psnark: - Update session options when max bandwidth changes - BufferedStream cleanup and comments - torrent name display tweak - findbugs --- .../org/klomp/snark/ConnectionAcceptor.java | 7 ++----- .../src/org/klomp/snark/I2PSnarkUtil.java | 19 ++++++++++++++++++- .../java/src/org/klomp/snark/MetaInfo.java | 4 ++-- .../java/src/org/klomp/snark/Peer.java | 18 +++--------------- .../src/org/klomp/snark/PeerCheckerTask.java | 1 + .../src/org/klomp/snark/PeerCoordinator.java | 2 +- .../java/src/org/klomp/snark/Piece.java | 2 +- .../java/src/org/klomp/snark/Snark.java | 6 +++--- .../src/org/klomp/snark/SnarkManager.java | 11 ++++++----- .../java/src/org/klomp/snark/Storage.java | 5 +++-- .../org/klomp/snark/web/I2PSnarkServlet.java | 16 ++++++++-------- 11 files changed, 48 insertions(+), 43 deletions(-) diff --git a/apps/i2psnark/java/src/org/klomp/snark/ConnectionAcceptor.java b/apps/i2psnark/java/src/org/klomp/snark/ConnectionAcceptor.java index 34fa5054ea..b4ba0f2996 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/ConnectionAcceptor.java +++ b/apps/i2psnark/java/src/org/klomp/snark/ConnectionAcceptor.java @@ -179,11 +179,8 @@ public class ConnectionAcceptor implements Runnable try { InputStream in = _socket.getInputStream(); OutputStream out = _socket.getOutputStream(); - - if (true) { - in = new BufferedInputStream(in); - //out = new BufferedOutputStream(out); - } + // this is for the readahead in PeerAcceptor.connection() + in = new BufferedInputStream(in); if (_log.shouldLog(Log.DEBUG)) _log.debug("Handling socket from " + _socket.getPeerDestination().calculateHash().toBase64()); peeracceptor.connection(_socket, in, out); diff --git a/apps/i2psnark/java/src/org/klomp/snark/I2PSnarkUtil.java b/apps/i2psnark/java/src/org/klomp/snark/I2PSnarkUtil.java index 89f5087b15..8fd7745678 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/I2PSnarkUtil.java +++ b/apps/i2psnark/java/src/org/klomp/snark/I2PSnarkUtil.java @@ -71,6 +71,7 @@ public class I2PSnarkUtil { public static final String DEFAULT_OPENTRACKERS = "http://tracker.welterde.i2p/a"; public static final int DEFAULT_MAX_UP_BW = 8; //KBps public static final int MAX_CONNECTIONS = 16; // per torrent + private static final String PROP_MAX_BW = "i2cp.outboundBytesPerSecond"; private static final boolean ENABLE_DHT = true; public I2PSnarkUtil(I2PAppContext ctx) { @@ -132,9 +133,21 @@ public class I2PSnarkUtil { _configured = true; } + /** + * @param KBps + */ public void setMaxUpBW(int limit) { _maxUpBW = limit; + _opts.put(PROP_MAX_BW, Integer.toString(limit * (1024 * 6 / 5))); // add a little for overhead _configured = true; + if (_manager != null) { + I2PSession sess = _manager.getSession(); + if (sess != null) { + Properties newProps = new Properties(); + newProps.putAll(_opts); + sess.updateOptions(newProps); + } + } } public void setMaxConnections(int limit) { @@ -154,6 +167,10 @@ public class I2PSnarkUtil { public int getEepProxyPort() { return _proxyPort; } public boolean getEepProxySet() { return _shouldProxy; } public int getMaxUploaders() { return _maxUploaders; } + + /** + * @return KBps + */ public int getMaxUpBW() { return _maxUpBW; } public int getMaxConnections() { return _maxConnections; } public int getStartupDelay() { return _startupDelay; } @@ -166,7 +183,7 @@ public class I2PSnarkUtil { // try to find why reconnecting after stop if (_log.shouldLog(Log.DEBUG)) _log.debug("Connecting to I2P", new Exception("I did it")); - Properties opts = new Properties(); + Properties opts = _context.getProperties(); if (_opts != null) { for (Iterator iter = _opts.keySet().iterator(); iter.hasNext(); ) { String key = (String)iter.next(); diff --git a/apps/i2psnark/java/src/org/klomp/snark/MetaInfo.java b/apps/i2psnark/java/src/org/klomp/snark/MetaInfo.java index d6a113ce61..3e13c7a421 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/MetaInfo.java +++ b/apps/i2psnark/java/src/org/klomp/snark/MetaInfo.java @@ -189,7 +189,7 @@ public class MetaInfo if (val == null) throw new InvalidBEncodingException("Missing length number"); long len = val.getLong(); - m_lengths.add(new Long(len)); + m_lengths.add(Long.valueOf(len)); l += len; val = (BEValue)desc.get("path"); @@ -438,7 +438,7 @@ public class MetaInfo info.put("piece length", Integer.valueOf(piece_length)); info.put("pieces", piece_hashes); if (files == null) - info.put("length", new Long(length)); + info.put("length", Long.valueOf(length)); else { List l = new ArrayList(); diff --git a/apps/i2psnark/java/src/org/klomp/snark/Peer.java b/apps/i2psnark/java/src/org/klomp/snark/Peer.java index bda2c28fd6..5a003cac23 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/Peer.java +++ b/apps/i2psnark/java/src/org/klomp/snark/Peer.java @@ -20,7 +20,6 @@ package org.klomp.snark; -import java.io.BufferedInputStream; import java.io.DataInputStream; import java.io.DataOutputStream; import java.io.IOException; @@ -231,19 +230,8 @@ public class Peer implements Comparable throw new IOException("Unable to reach " + peerID); } InputStream in = sock.getInputStream(); - OutputStream out = sock.getOutputStream(); //new BufferedOutputStream(sock.getOutputStream()); - if (true) { - // buffered output streams are internally synchronized, so we can't get through to the underlying - // I2PSocket's MessageOutputStream to close() it if we are blocking on a write(...). Oh, and the - // buffer is unnecessary anyway, as unbuffered access lets the streaming lib do the 'right thing'. - //out = new BufferedOutputStream(out); - in = new BufferedInputStream(sock.getInputStream()); - } - //BufferedInputStream bis - // = new BufferedInputStream(sock.getInputStream()); - //BufferedOutputStream bos - // = new BufferedOutputStream(sock.getOutputStream()); - byte [] id = handshake(in, out); //handshake(bis, bos); + OutputStream out = sock.getOutputStream(); + byte [] id = handshake(in, out); byte [] expected_id = peerID.getID(); if (expected_id == null) { peerID.setID(id); @@ -328,7 +316,7 @@ public class Peer implements Comparable * Sets DataIn/OutputStreams, does the handshake and returns the id * reported by the other side. */ - private byte[] handshake(InputStream in, OutputStream out) //BufferedInputStream bis, BufferedOutputStream bos) + private byte[] handshake(InputStream in, OutputStream out) throws IOException { din = new DataInputStream(in); diff --git a/apps/i2psnark/java/src/org/klomp/snark/PeerCheckerTask.java b/apps/i2psnark/java/src/org/klomp/snark/PeerCheckerTask.java index 6370eaca64..b767f875a7 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/PeerCheckerTask.java +++ b/apps/i2psnark/java/src/org/klomp/snark/PeerCheckerTask.java @@ -210,6 +210,7 @@ class PeerCheckerTask extends TimerTask if (_util.getDHT() != null && (_runCount % 5) == 0) { _util.getDHT().announce(coordinator.getInfoHash(), peer.getPeerID().getDestHash()); } + // send PEX } // Resync actual uploaders value diff --git a/apps/i2psnark/java/src/org/klomp/snark/PeerCoordinator.java b/apps/i2psnark/java/src/org/klomp/snark/PeerCoordinator.java index d01f1ef2e8..5e75f17643 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/PeerCoordinator.java +++ b/apps/i2psnark/java/src/org/klomp/snark/PeerCoordinator.java @@ -1185,7 +1185,7 @@ public class PeerCoordinator implements PeerListener * @since 0.8.4 */ public void gotPeers(Peer peer, List<PeerID> peers) { - + // spin off thread or timer task to do a new Peer() and an addPeer() for each one } /** Return number of allowed uploaders for this torrent. diff --git a/apps/i2psnark/java/src/org/klomp/snark/Piece.java b/apps/i2psnark/java/src/org/klomp/snark/Piece.java index 6855d36a0e..dd48508a9a 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/Piece.java +++ b/apps/i2psnark/java/src/org/klomp/snark/Piece.java @@ -35,8 +35,8 @@ class Piece implements Comparable { @Override public boolean equals(Object o) { + if (o == null) return false; if (o instanceof Piece) { - if (o == null) return false; return this.id == ((Piece)o).id; } return false; diff --git a/apps/i2psnark/java/src/org/klomp/snark/Snark.java b/apps/i2psnark/java/src/org/klomp/snark/Snark.java index e3a99bdc9b..98bcd4eed4 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/Snark.java +++ b/apps/i2psnark/java/src/org/klomp/snark/Snark.java @@ -108,7 +108,7 @@ public class Snark } catch (Throwable t) { System.out.println("OOM in the OOM"); } - System.exit(0); + //System.exit(0); } } @@ -980,7 +980,6 @@ public class Snark (" <file> \tEither a local .torrent metainfo file to download"); System.out.println (" \tor (with --share) a file to share."); - System.exit(-1); } /** @@ -1117,11 +1116,12 @@ public class Snark coordinator.setWantedPieces(); } + /** SnarkSnutdown callback unused */ public void shutdown() { // Should not be necessary since all non-deamon threads should // have died. But in reality this does not always happen. - System.exit(0); + //System.exit(0); } public interface CompleteListener { diff --git a/apps/i2psnark/java/src/org/klomp/snark/SnarkManager.java b/apps/i2psnark/java/src/org/klomp/snark/SnarkManager.java index 6ac7261e3e..026363f6f2 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/SnarkManager.java +++ b/apps/i2psnark/java/src/org/klomp/snark/SnarkManager.java @@ -247,11 +247,9 @@ public class SnarkManager implements Snark.CompleteListener { i2cpOpts.put(pair.substring(0, split), pair.substring(split+1)); } } - if (i2cpHost != null) { - _util.setI2CPConfig(i2cpHost, i2cpPort, i2cpOpts); - if (_log.shouldLog(Log.DEBUG)) - _log.debug("Configuring with I2CP options " + i2cpOpts); - } + _util.setI2CPConfig(i2cpHost, i2cpPort, i2cpOpts); + if (_log.shouldLog(Log.DEBUG)) + _log.debug("Configuring with I2CP options " + i2cpOpts); //I2PSnarkUtil.instance().setI2CPConfig("66.111.51.110", 7654, new Properties()); //String eepHost = _config.getProperty(PROP_EEP_HOST); //int eepPort = getInt(PROP_EEP_PORT, 4444); @@ -338,6 +336,7 @@ public class SnarkManager implements Snark.CompleteListener { } } + // FIXME do this even if == null if (i2cpHost != null) { int oldI2CPPort = _util.getI2CPPort(); String oldI2CPHost = _util.getI2CPHost(); @@ -383,6 +382,7 @@ public class SnarkManager implements Snark.CompleteListener { Properties p = new Properties(); p.putAll(opts); _util.setI2CPConfig(i2cpHost, port, p); + _util.setMaxUpBW(getInt(PROP_UPBW_MAX, DEFAULT_MAX_UP_BW)); addMessage(_("I2CP and tunnel changes will take effect after stopping all torrents")); if (_log.shouldLog(Log.DEBUG)) _log.debug("i2cp host [" + i2cpHost + "] i2cp port " + port + " opts [" + opts @@ -396,6 +396,7 @@ public class SnarkManager implements Snark.CompleteListener { p.putAll(opts); addMessage(_("I2CP settings changed to {0}", i2cpHost + ":" + port + " (" + i2cpOpts.trim() + ")")); _util.setI2CPConfig(i2cpHost, port, p); + _util.setMaxUpBW(getInt(PROP_UPBW_MAX, DEFAULT_MAX_UP_BW)); boolean ok = _util.connect(); if (!ok) { addMessage(_("Unable to connect with the new settings, reverting to the old I2CP settings")); diff --git a/apps/i2psnark/java/src/org/klomp/snark/Storage.java b/apps/i2psnark/java/src/org/klomp/snark/Storage.java index 77012de77d..23fba7f357 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/Storage.java +++ b/apps/i2psnark/java/src/org/klomp/snark/Storage.java @@ -105,7 +105,7 @@ public class Storage { long length = lengths[i]; total += length; - lengthsList.add(new Long(length)); + lengthsList.add(Long.valueOf(length)); } piece_size = MIN_PIECE_SIZE; @@ -753,7 +753,8 @@ public class Storage openRAF(nr, false); // RW // XXX - Is this the best way to make sure we have enough space for // the whole file? - listener.storageCreateFile(this, names[nr], lengths[nr]); + if (listener != null) + listener.storageCreateFile(this, names[nr], lengths[nr]); final int ZEROBLOCKSIZE = metainfo.getPieceLength(0); byte[] zeros; try { diff --git a/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java b/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java index 616df5b83a..c859c436cf 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java +++ b/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java @@ -755,8 +755,12 @@ public class I2PSnarkServlet extends Default { filename = filename.substring(0, i); String fullFilename = filename; if (filename.length() > MAX_DISPLAYED_FILENAME_LENGTH) { - fullFilename = new String(filename); - filename = filename.substring(0, MAX_DISPLAYED_FILENAME_LENGTH) + "…"; + String start = filename.substring(0, MAX_DISPLAYED_FILENAME_LENGTH); + if (start.indexOf(" ") < 0 && start.indexOf("-") < 0) { + // browser has nowhere to break it + fullFilename = filename; + filename = start + "…"; + } } long total = snark.getTotalLength(); // Early typecast, avoid possibly overflowing a temp integer @@ -1035,7 +1039,7 @@ public class I2PSnarkServlet extends Default { } else { pct = (float) 101.0; // until we get the metainfo we don't know how many pieces there are - out.write("??"); + //out.write("??"); } out.write("</td>\n\t"); out.write("<td class=\"snarkTorrentStatus " + rowClass + "\">"); @@ -1134,7 +1138,6 @@ public class I2PSnarkServlet extends Default { } private void writeAddForm(PrintWriter out, HttpServletRequest req) throws IOException { - String uri = req.getRequestURI(); String newURL = req.getParameter("newURL"); if ( (newURL == null) || (newURL.trim().length() <= 0) ) newURL = ""; @@ -1175,7 +1178,6 @@ public class I2PSnarkServlet extends Default { } private void writeSeedForm(PrintWriter out, HttpServletRequest req) throws IOException { - String uri = req.getRequestURI(); String baseFile = req.getParameter("baseFile"); if (baseFile == null || baseFile.trim().length() <= 0) baseFile = ""; @@ -1235,7 +1237,6 @@ public class I2PSnarkServlet extends Default { } private void writeConfigForm(PrintWriter out, HttpServletRequest req) throws IOException { - String uri = req.getRequestURI(); String dataDir = _manager.getDataDir().getAbsolutePath(); boolean autoStart = _manager.shouldAutoStart(); boolean useOpenTrackers = _manager.util().shouldUseOpenTrackers(); @@ -1914,8 +1915,7 @@ private static class FetchAndAdd implements Runnable { return; } - // don't hold object from this MetaInfo - String name = new String(info.getName()); + String name = info.getName(); name = Storage.filterName(name); name = name + ".torrent"; File torrentFile = new File(_manager.getDataDir(), name); From 1ae6c28592eed4735b30c1248aac0460f41cefb3 Mon Sep 17 00:00:00 2001 From: zzz <zzz@mail.i2p> Date: Mon, 10 Jan 2011 17:49:00 +0000 Subject: [PATCH 23/27] Don't expire outgoing piece messages, since we no longer prefetch the data --- .../java/src/org/klomp/snark/Message.java | 3 ++- .../src/org/klomp/snark/PeerConnectionOut.java | 18 +++++++++++------- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/apps/i2psnark/java/src/org/klomp/snark/Message.java b/apps/i2psnark/java/src/org/klomp/snark/Message.java index 01c474216e..b4344f3ba5 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/Message.java +++ b/apps/i2psnark/java/src/org/klomp/snark/Message.java @@ -68,7 +68,8 @@ class Message // Used to do deferred fetch of data DataLoader dataLoader; - SimpleTimer.TimedEvent expireEvent; + // now unused + //SimpleTimer.TimedEvent expireEvent; /** Utility method for sending a message through a DataStream. */ void sendMessage(DataOutputStream dos) throws IOException diff --git a/apps/i2psnark/java/src/org/klomp/snark/PeerConnectionOut.java b/apps/i2psnark/java/src/org/klomp/snark/PeerConnectionOut.java index 181ca49695..225edd4923 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/PeerConnectionOut.java +++ b/apps/i2psnark/java/src/org/klomp/snark/PeerConnectionOut.java @@ -29,8 +29,8 @@ import java.util.List; import net.i2p.I2PAppContext; import net.i2p.util.I2PAppThread; import net.i2p.util.Log; -import net.i2p.util.SimpleScheduler; -import net.i2p.util.SimpleTimer; +//import net.i2p.util.SimpleScheduler; +//import net.i2p.util.SimpleTimer; class PeerConnectionOut implements Runnable { @@ -124,27 +124,27 @@ class PeerConnectionOut implements Runnable { if (state.choking) { it.remove(); - SimpleTimer.getInstance().removeEvent(nm.expireEvent); + //SimpleTimer.getInstance().removeEvent(nm.expireEvent); } nm = null; } else if (nm.type == Message.REQUEST && state.choked) { it.remove(); - SimpleTimer.getInstance().removeEvent(nm.expireEvent); + //SimpleTimer.getInstance().removeEvent(nm.expireEvent); nm = null; } if (m == null && nm != null) { m = nm; - SimpleTimer.getInstance().removeEvent(nm.expireEvent); + //SimpleTimer.getInstance().removeEvent(nm.expireEvent); it.remove(); } } if (m == null && !sendQueue.isEmpty()) { m = (Message)sendQueue.remove(0); - SimpleTimer.getInstance().removeEvent(m.expireEvent); + //SimpleTimer.getInstance().removeEvent(m.expireEvent); } } } @@ -241,6 +241,8 @@ class PeerConnectionOut implements Runnable /** remove messages not sent in 3m */ private static final int SEND_TIMEOUT = 3*60*1000; + +/***** private class RemoveTooSlow implements SimpleTimer.TimedEvent { private Message _m; public RemoveTooSlow(Message m) { @@ -258,6 +260,7 @@ class PeerConnectionOut implements Runnable _log.info("Took too long to send " + _m + " to " + peer); } } +*****/ /** * Removes a particular message type from the queue. @@ -474,7 +477,8 @@ class PeerConnectionOut implements Runnable m.off = 0; m.len = length; // since we have the data already loaded, queue a timeout to remove it - SimpleScheduler.getInstance().addEvent(new RemoveTooSlow(m), SEND_TIMEOUT); + // no longer prefetched + //SimpleScheduler.getInstance().addEvent(new RemoveTooSlow(m), SEND_TIMEOUT); addMessage(m); } From b4e0fe121c7b241e355364eada5d1ace4e72c259 Mon Sep 17 00:00:00 2001 From: zzz <zzz@mail.i2p> Date: Wed, 12 Jan 2011 13:54:41 +0000 Subject: [PATCH 24/27] dont update globals until end of storage check --- .../java/src/org/klomp/snark/Storage.java | 36 +++++++++++++------ 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/apps/i2psnark/java/src/org/klomp/snark/Storage.java b/apps/i2psnark/java/src/org/klomp/snark/Storage.java index 23fba7f357..dfc229dce2 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/Storage.java +++ b/apps/i2psnark/java/src/org/klomp/snark/Storage.java @@ -538,7 +538,7 @@ public class Storage } else { // the following sets the needed variable changed = true; - checkCreateFiles(); + checkCreateFiles(false); } if (complete()) { _util.debug("Torrent is complete", Snark.NOTICE); @@ -649,15 +649,26 @@ public class Storage /** * This is called at the beginning, and at presumed completion, * so we have to be careful about locking. + * + * @param recheck if true, this is a check after we downloaded the + * last piece, and we don't modify the global bitfield unless + * the check fails. */ - private void checkCreateFiles() throws IOException + private void checkCreateFiles(boolean recheck) throws IOException { // Whether we are resuming or not, // if any of the files already exists we assume we are resuming. boolean resume = false; _probablyComplete = true; - needed = metainfo.getPieces(); + // use local variables during the check + int need = metainfo.getPieces(); + BitField bfield; + if (recheck) { + bfield = new BitField(need); + } else { + bfield = bitfield; + } // Make sure all files are available and of correct length for (int i = 0; i < rafs.length; i++) @@ -718,8 +729,8 @@ public class Storage } if (correctHash) { - bitfield.set(i); - needed--; + bfield.set(i); + need--; } if (listener != null) @@ -739,6 +750,15 @@ public class Storage // } //} + // do this here so we don't confuse the user during checking + needed = need; + if (recheck && need > 0) { + // whoops, recheck failed + synchronized(bitfield) { + bitfield = bfield; + } + } + if (listener != null) { listener.storageAllChecked(this); if (needed <= 0) @@ -903,11 +923,7 @@ public class Storage // checkCreateFiles() which will set 'needed' and 'bitfield' // and also call listener.storageCompleted() if the double-check // was successful. - // Todo: set a listener variable so the web shows "checking" and don't - // have the user panic when completed amount goes to zero temporarily? - needed = metainfo.getPieces(); - bitfield = new BitField(needed); - checkCreateFiles(); + checkCreateFiles(true); if (needed > 0) { if (listener != null) listener.setWantedPieces(this); From c1c4e50b5bd3583ff002ed932a16509fe4819f3c Mon Sep 17 00:00:00 2001 From: zzz <zzz@mail.i2p> Date: Thu, 13 Jan 2011 19:54:56 +0000 Subject: [PATCH 25/27] - Finish PEX send/rcv - Disable KRPC - Shorten tracker string --- .../src/org/klomp/snark/ExtensionHandler.java | 19 ++++- .../src/org/klomp/snark/I2PSnarkUtil.java | 6 +- .../java/src/org/klomp/snark/Peer.java | 3 +- .../src/org/klomp/snark/PeerCheckerTask.java | 4 +- .../src/org/klomp/snark/PeerCoordinator.java | 71 ++++++++++++++++--- .../src/org/klomp/snark/TrackerClient.java | 33 ++++++--- .../org/klomp/snark/web/I2PSnarkServlet.java | 12 +++- 7 files changed, 120 insertions(+), 28 deletions(-) diff --git a/apps/i2psnark/java/src/org/klomp/snark/ExtensionHandler.java b/apps/i2psnark/java/src/org/klomp/snark/ExtensionHandler.java index 7b2979b8ab..44490c37be 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/ExtensionHandler.java +++ b/apps/i2psnark/java/src/org/klomp/snark/ExtensionHandler.java @@ -85,10 +85,18 @@ abstract class ExtensionHandler { // peer state calls peer listener calls sendPEX() } + MagnetState state = peer.getMagnetState(); + if (msgmap.get(TYPE_METADATA) == null) { if (_log.shouldLog(Log.WARN)) _log.debug("Peer does not support metadata extension: " + peer); - // drop if we need metainfo ? + // drop if we need metainfo and we haven't found anybody yet + synchronized(state) { + if (!state.isInitialized()) { + _log.debug("Dropping peer, we need metadata! " + peer); + peer.disconnect(); + } + } return; } @@ -96,14 +104,19 @@ abstract class ExtensionHandler { if (msize == null) { if (_log.shouldLog(Log.WARN)) _log.debug("Peer does not have the metainfo size yet: " + peer); - // drop if we need metainfo ? + // drop if we need metainfo and we haven't found anybody yet + synchronized(state) { + if (!state.isInitialized()) { + _log.debug("Dropping peer, we need metadata! " + peer); + peer.disconnect(); + } + } return; } int metaSize = msize.getInt(); if (_log.shouldLog(Log.WARN)) _log.debug("Got the metainfo size: " + metaSize); - MagnetState state = peer.getMagnetState(); int remaining; synchronized(state) { if (state.isComplete()) diff --git a/apps/i2psnark/java/src/org/klomp/snark/I2PSnarkUtil.java b/apps/i2psnark/java/src/org/klomp/snark/I2PSnarkUtil.java index 8fd7745678..a3ecb1777a 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/I2PSnarkUtil.java +++ b/apps/i2psnark/java/src/org/klomp/snark/I2PSnarkUtil.java @@ -35,7 +35,7 @@ import net.i2p.util.SimpleTimer; import net.i2p.util.Translate; import org.klomp.snark.dht.DHT; -import org.klomp.snark.dht.KRPC; +//import org.klomp.snark.dht.KRPC; /** * I2P specific helpers for I2PSnark @@ -213,8 +213,8 @@ public class I2PSnarkUtil { _manager = I2PSocketManagerFactory.createManager(_i2cpHost, _i2cpPort, opts); } // FIXME this only instantiates krpc once, left stuck with old manager - if (ENABLE_DHT && _manager != null && _dht == null) - _dht = new KRPC(_context, _manager.getSession()); + //if (ENABLE_DHT && _manager != null && _dht == null) + // _dht = new KRPC(_context, _manager.getSession()); return (_manager != null); } diff --git a/apps/i2psnark/java/src/org/klomp/snark/Peer.java b/apps/i2psnark/java/src/org/klomp/snark/Peer.java index 5a003cac23..5110ae9a5f 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/Peer.java +++ b/apps/i2psnark/java/src/org/klomp/snark/Peer.java @@ -29,6 +29,7 @@ import java.util.Arrays; import java.util.List; import java.util.Map; +import net.i2p.I2PAppContext; import net.i2p.client.streaming.I2PSocket; import net.i2p.data.DataHelper; import net.i2p.data.Destination; @@ -38,7 +39,7 @@ import org.klomp.snark.bencode.BEValue; public class Peer implements Comparable { - private Log _log = new Log(Peer.class); + private final Log _log = I2PAppContext.getGlobalContext().logManager().getLog(Peer.class); // Identifying property, the peer id of the other side. private final PeerID peerID; diff --git a/apps/i2psnark/java/src/org/klomp/snark/PeerCheckerTask.java b/apps/i2psnark/java/src/org/klomp/snark/PeerCheckerTask.java index b767f875a7..5d7b6d66db 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/PeerCheckerTask.java +++ b/apps/i2psnark/java/src/org/klomp/snark/PeerCheckerTask.java @@ -205,12 +205,14 @@ class PeerCheckerTask extends TimerTask } } peer.retransmitRequests(); + // send PEX + if ((_runCount % 17) == 0 && !peer.isCompleted()) + coordinator.sendPeers(peer); peer.keepAlive(); // announce them to local tracker (TrackerClient does this too) if (_util.getDHT() != null && (_runCount % 5) == 0) { _util.getDHT().announce(coordinator.getInfoHash(), peer.getPeerID().getDestHash()); } - // send PEX } // Resync actual uploaders value diff --git a/apps/i2psnark/java/src/org/klomp/snark/PeerCoordinator.java b/apps/i2psnark/java/src/org/klomp/snark/PeerCoordinator.java index 5e75f17643..cdf1e58b5e 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/PeerCoordinator.java +++ b/apps/i2psnark/java/src/org/klomp/snark/PeerCoordinator.java @@ -27,15 +27,22 @@ import java.util.Collections; import java.util.Iterator; import java.util.LinkedList; import java.util.List; +import java.util.Map; import java.util.Queue; import java.util.Random; +import java.util.Set; import java.util.concurrent.LinkedBlockingQueue; import net.i2p.I2PAppContext; +import net.i2p.data.DataHelper; +import net.i2p.data.Destination; +import net.i2p.util.ConcurrentHashSet; import net.i2p.util.I2PAppThread; import net.i2p.util.Log; import net.i2p.util.SimpleTimer2; +import org.klomp.snark.bencode.BEValue; +import org.klomp.snark.bencode.InvalidBEncodingException; import org.klomp.snark.dht.DHT; /** @@ -90,6 +97,11 @@ public class PeerCoordinator implements PeerListener */ final Queue<Peer> peers; + /** + * Peers we heard about via PEX + */ + private final Set<PeerID> pexPeers; + /** estimate of the peers, without requiring any synchronization */ private volatile int peerCount; @@ -134,6 +146,7 @@ public class PeerCoordinator implements PeerListener partialPieces = new ArrayList(getMaxConnections() + 1); peers = new LinkedBlockingQueue(); magnetState = new MagnetState(infohash, metainfo); + pexPeers = new ConcurrentHashSet(); // Install a timer to check the uploaders. // Randomize the first start time so multiple tasks are spread out, @@ -1143,18 +1156,33 @@ public class PeerCoordinator implements PeerListener } } } else if (id == ExtensionHandler.ID_HANDSHAKE) { - try { - if (peer.getHandshakeMap().get("m").getMap().get(ExtensionHandler.TYPE_PEX) != null) { - List<Peer> pList = peerList(); - pList.remove(peer); - ExtensionHandler.sendPEX(peer, pList); - } - } catch (Exception e) { - // NPE, no map - } + sendPeers(peer); } } + /** + * Send a PEX message to the peer, if he supports PEX. + * This just sends everybody we are connected to, we don't + * track new vs. old peers yet. + * @since 0.8.4 + */ + void sendPeers(Peer peer) { + Map<String, BEValue> handshake = peer.getHandshakeMap(); + if (handshake == null) + return; + BEValue bev = handshake.get("m"); + if (bev == null) + return; + try { + if (bev.getMap().get(ExtensionHandler.TYPE_PEX) != null) { + List<Peer> pList = peerList(); + pList.remove(peer); + if (!pList.isEmpty()) + ExtensionHandler.sendPEX(peer, pList); + } + } catch (InvalidBEncodingException ibee) {} + } + /** * Sets the storage after transition out of magnet mode * Snark calls this after we call gotMetaInfo() @@ -1185,7 +1213,30 @@ public class PeerCoordinator implements PeerListener * @since 0.8.4 */ public void gotPeers(Peer peer, List<PeerID> peers) { - // spin off thread or timer task to do a new Peer() and an addPeer() for each one + if (completed() || !needPeers()) + return; + Destination myDest = _util.getMyDestination(); + if (myDest == null) + return; + byte[] myHash = myDest.calculateHash().getData(); + List<Peer> pList = peerList(); + for (PeerID id : peers) { + if (peerIDInList(id, pList) != null) + continue; + if (DataHelper.eq(myHash, id.getDestHash())) + continue; + pexPeers.add(id); + } + // TrackerClient will poll for pexPeers and do the add in its thread, + // rather than running another thread here. + } + + /** + * Called by TrackerClient + * @since 0.8.4 + */ + Set<PeerID> getPEXPeers() { + return pexPeers; } /** Return number of allowed uploaders for this torrent. diff --git a/apps/i2psnark/java/src/org/klomp/snark/TrackerClient.java b/apps/i2psnark/java/src/org/klomp/snark/TrackerClient.java index d03ca55db3..c8f1cd6c96 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/TrackerClient.java +++ b/apps/i2psnark/java/src/org/klomp/snark/TrackerClient.java @@ -300,10 +300,8 @@ public class TrackerClient extends I2PAppThread Peer cur = it.next(); // FIXME if id == us || dest == us continue; // only delay if we actually make an attempt to add peer - if(coordinator.addPeer(cur)) { - int delay = DELAY_MUL; - delay *= r.nextInt(10); - delay += DELAY_MIN; + if(coordinator.addPeer(cur) && it.hasNext()) { + int delay = (DELAY_MUL * r.nextInt(10)) + DELAY_MIN; sleptTime += delay; try { Thread.sleep(delay); } catch (InterruptedException ie) {} } @@ -341,6 +339,27 @@ public class TrackerClient extends I2PAppThread maxSeenPeers = tr.seenPeers; } // *** end of trackers loop here + // Get peers from PEX + if (left > 0 && coordinator.needPeers() && !stop) { + Set<PeerID> pids = coordinator.getPEXPeers(); + if (!pids.isEmpty()) { + _util.debug("Got " + pids.size() + " from PEX", Snark.INFO); + List<Peer> peers = new ArrayList(pids.size()); + for (PeerID pID : pids) { + peers.add(new Peer(pID, snark.getID(), snark.getInfoHash(), snark.getMetaInfo())); + } + Collections.shuffle(peers, r); + Iterator<Peer> it = peers.iterator(); + while ((!stop) && it.hasNext()) { + Peer cur = it.next(); + if (coordinator.addPeer(cur) && it.hasNext()) { + int delay = (DELAY_MUL * r.nextInt(10)) + DELAY_MIN; + try { Thread.sleep(delay); } catch (InterruptedException ie) {} + } + } + } + } + // Get peers from DHT // FIXME this needs to be in its own thread if (_util.getDHT() != null && !stop) { @@ -369,10 +388,8 @@ public class TrackerClient extends I2PAppThread Iterator<Peer> it = peers.iterator(); while ((!stop) && it.hasNext()) { Peer cur = it.next(); - if (coordinator.addPeer(cur)) { - int delay = DELAY_MUL; - delay *= r.nextInt(10); - delay += DELAY_MIN; + if (coordinator.addPeer(cur) && it.hasNext()) { + int delay = (DELAY_MUL * r.nextInt(10)) + DELAY_MIN; try { Thread.sleep(delay); } catch (InterruptedException ie) {} } } diff --git a/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java b/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java index c859c436cf..9bbfec8fee 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java +++ b/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java @@ -910,8 +910,8 @@ public class I2PSnarkServlet extends Default { out.write(formatSize(total-remaining) + thinsp(noThinsp) + formatSize(total)); else if (remaining == 0) out.write(formatSize(total)); // 3GB - else - out.write("??"); // no meta size yet + //else + // out.write("??"); // no meta size yet out.write("</td>\n\t"); out.write("<td align=\"right\" class=\"snarkTorrentUploaded " + rowClass + "\">"); if(isRunning && isValid) @@ -1058,6 +1058,12 @@ public class I2PSnarkServlet extends Default { out.write("\">"); out.write(formatSize(peer.getDownloadRate()) + "ps</a></span>"); } + } else if (!isValid) { + //if (peer supports metadata extension) { + out.write("<span class=\"unchoked\">"); + out.write(formatSize(peer.getDownloadRate()) + "ps</span>"); + //} else { + //} } out.write("</td>\n\t"); out.write("<td align=\"right\" class=\"snarkTorrentStatus " + rowClass + "\">"); @@ -1629,6 +1635,8 @@ public class I2PSnarkServlet extends Default { int slsh = announce.indexOf('/'); if (slsh > 0) announce = announce.substring(0, slsh); + if (announce.length() > 67) + announce = announce.substring(0, 40) + "…" + announce.substring(announce.length() - 8); buf.append(announce); } } From f046eb79bd9315f7cf04916225a920f717395df3 Mon Sep 17 00:00:00 2001 From: zzz <zzz@mail.i2p> Date: Fri, 21 Jan 2011 16:04:45 +0000 Subject: [PATCH 26/27] javadoc --- core/java/src/net/i2p/data/ByteArray.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/java/src/net/i2p/data/ByteArray.java b/core/java/src/net/i2p/data/ByteArray.java index 7369a27d19..9f460fe475 100644 --- a/core/java/src/net/i2p/data/ByteArray.java +++ b/core/java/src/net/i2p/data/ByteArray.java @@ -24,6 +24,7 @@ public class ByteArray implements Serializable, Comparable { public ByteArray() { } + /** Sets valid */ public ByteArray(byte[] data) { _data = data; _valid = (data != null ? data.length : 0); @@ -38,6 +39,7 @@ public class ByteArray implements Serializable, Comparable { return _data; } + /** Warning, does not set valid */ public void setData(byte[] data) { _data = data; } From 6a54aa7e4e1d2dc287147bb904523c997d2ccb29 Mon Sep 17 00:00:00 2001 From: zzz <zzz@mail.i2p> Date: Sat, 5 Feb 2011 22:42:07 +0000 Subject: [PATCH 27/27] cleanups --- .../java/src/org/klomp/snark/ExtensionHandler.java | 3 ++- .../java/src/org/klomp/snark/I2PSnarkUtil.java | 2 +- apps/i2psnark/java/src/org/klomp/snark/Peer.java | 11 ++++++++--- .../java/src/org/klomp/snark/SnarkManager.java | 4 +--- apps/i2psnark/java/src/org/klomp/snark/dht/DHT.java | 1 - 5 files changed, 12 insertions(+), 9 deletions(-) diff --git a/apps/i2psnark/java/src/org/klomp/snark/ExtensionHandler.java b/apps/i2psnark/java/src/org/klomp/snark/ExtensionHandler.java index 44490c37be..b3770070f4 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/ExtensionHandler.java +++ b/apps/i2psnark/java/src/org/klomp/snark/ExtensionHandler.java @@ -30,7 +30,8 @@ abstract class ExtensionHandler { public static final int ID_METADATA = 1; public static final String TYPE_METADATA = "ut_metadata"; public static final int ID_PEX = 2; - public static final String TYPE_PEX = "ut_pex"; + /** not ut_pex since the compact format is different */ + public static final String TYPE_PEX = "i2p_pex"; /** Pieces * SHA1 Hash length, + 25% extra for file names, benconding overhead, etc */ private static final int MAX_METADATA_SIZE = Storage.MAX_PIECES * 20 * 5 / 4; private static final int PARALLEL_REQUESTS = 3; diff --git a/apps/i2psnark/java/src/org/klomp/snark/I2PSnarkUtil.java b/apps/i2psnark/java/src/org/klomp/snark/I2PSnarkUtil.java index a3ecb1777a..13bda2e557 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/I2PSnarkUtil.java +++ b/apps/i2psnark/java/src/org/klomp/snark/I2PSnarkUtil.java @@ -72,7 +72,7 @@ public class I2PSnarkUtil { public static final int DEFAULT_MAX_UP_BW = 8; //KBps public static final int MAX_CONNECTIONS = 16; // per torrent private static final String PROP_MAX_BW = "i2cp.outboundBytesPerSecond"; - private static final boolean ENABLE_DHT = true; + //private static final boolean ENABLE_DHT = true; public I2PSnarkUtil(I2PAppContext ctx) { _context = ctx; diff --git a/apps/i2psnark/java/src/org/klomp/snark/Peer.java b/apps/i2psnark/java/src/org/klomp/snark/Peer.java index 5110ae9a5f..5489148121 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/Peer.java +++ b/apps/i2psnark/java/src/org/klomp/snark/Peer.java @@ -79,6 +79,8 @@ public class Peer implements Comparable static final long OPTION_EXTENSION = 0x0000000000100000l; static final long OPTION_FAST = 0x0000000000000004l; static final long OPTION_DHT = 0x0000000000000001l; + /** we use a different bit since the compact format is different */ + static final long OPTION_I2P_DHT = 0x0000000040000000l; static final long OPTION_AZMP = 0x1000000000000000l; private long options; @@ -269,7 +271,7 @@ public class Peer implements Comparable out.sendExtension(0, ExtensionHandler.getHandshake(metasize)); } - if ((options & OPTION_DHT) != 0 && util.getDHT() != null) { + if ((options & OPTION_I2P_DHT) != 0 && util.getDHT() != null) { if (_log.shouldLog(Log.DEBUG)) _log.debug("Peer supports DHT, sending PORT message"); int port = util.getDHT().getPort(); @@ -327,8 +329,11 @@ public class Peer implements Comparable dout.write(19); dout.write("BitTorrent protocol".getBytes("UTF-8")); // Handshake write - options - // FIXME not if DHT disabled - dout.writeLong(OPTION_EXTENSION | OPTION_DHT); + long myOptions = OPTION_EXTENSION; + // FIXME get util here somehow + //if (util.getDHT() != null) + // myOptions |= OPTION_I2P_DHT; + dout.writeLong(myOptions); // Handshake write - metainfo hash dout.write(infohash); // Handshake write - peer id diff --git a/apps/i2psnark/java/src/org/klomp/snark/SnarkManager.java b/apps/i2psnark/java/src/org/klomp/snark/SnarkManager.java index 026363f6f2..3f9f90edfb 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/SnarkManager.java +++ b/apps/i2psnark/java/src/org/klomp/snark/SnarkManager.java @@ -750,8 +750,7 @@ public class SnarkManager implements Snark.CompleteListener { * Must be a filesystem-safe name. * @since 0.8.4 */ - private void locked_writeMetaInfo(MetaInfo metainfo, String filename) throws IOException { - // prevent interference by DirMonitor + private static void locked_writeMetaInfo(MetaInfo metainfo, String filename) throws IOException { File file = new File(filename); if (file.exists()) throw new IOException("Cannot overwrite an existing .torrent file: " + file.getPath()); @@ -1313,7 +1312,6 @@ public class SnarkManager implements Snark.CompleteListener { if ( (snark != null) && (!snark.isStopped()) ) snark.stopTorrent(); } -//save magnets } } } diff --git a/apps/i2psnark/java/src/org/klomp/snark/dht/DHT.java b/apps/i2psnark/java/src/org/klomp/snark/dht/DHT.java index 29e0fa20b4..a074890b93 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/dht/DHT.java +++ b/apps/i2psnark/java/src/org/klomp/snark/dht/DHT.java @@ -1,7 +1,6 @@ package org.klomp.snark.dht; /* - * Copyright 2010 zzz (zzz@mail.i2p) * GPLv2 */