From f9b2c0bc63b986a8c1fff8f2f078a34d92d9d021 Mon Sep 17 00:00:00 2001 From: zzz Date: Tue, 8 Mar 2011 03:01:02 +0000 Subject: [PATCH] * i2psnark: - More efficient metainfo handling, reduce instantiations - Improved handling of storage errors - Improved handling of duplicate file names - More metainfo sanity checks - Metadata transfer error handling improvements - Code cleanup, remove dead and duplicated code --- .../java/src/org/klomp/snark/MagnetState.java | 11 +- .../java/src/org/klomp/snark/MetaInfo.java | 161 +++++++++---- .../src/org/klomp/snark/PeerCoordinator.java | 18 +- .../java/src/org/klomp/snark/Snark.java | 10 +- .../src/org/klomp/snark/SnarkManager.java | 4 + .../java/src/org/klomp/snark/Storage.java | 225 +++++++++--------- .../src/org/klomp/snark/bencode/BDecoder.java | 49 ++-- .../org/klomp/snark/web/I2PSnarkServlet.java | 8 +- 8 files changed, 286 insertions(+), 200 deletions(-) diff --git a/apps/i2psnark/java/src/org/klomp/snark/MagnetState.java b/apps/i2psnark/java/src/org/klomp/snark/MagnetState.java index f19f1091a4..da89f5577a 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/MagnetState.java +++ b/apps/i2psnark/java/src/org/klomp/snark/MagnetState.java @@ -191,14 +191,21 @@ class MagnetState { */ public MetaInfo buildMetaInfo() throws Exception { // top map has nothing in it but the info map (no announce) - Map map = new HashMap(); + Map map = new HashMap(); InputStream is = new ByteArrayInputStream(metainfoBytes); BDecoder dec = new BDecoder(is); BEValue bev = dec.bdecodeMap(); map.put("info", bev); MetaInfo newmeta = new MetaInfo(map); - if (!DataHelper.eq(newmeta.getInfoHash(), infohash)) + if (!DataHelper.eq(newmeta.getInfoHash(), infohash)) { + // Disaster. Start over. ExtensionHandler will catch + // the IOE and disconnect the peer, hopefully we will + // find a new peer. + // TODO: Count fails and give up eventually + have = new BitField(totalChunks); + requested = new BitField(totalChunks); throw new IOException("info hash mismatch"); + } return newmeta; } } diff --git a/apps/i2psnark/java/src/org/klomp/snark/MetaInfo.java b/apps/i2psnark/java/src/org/klomp/snark/MetaInfo.java index 3e13c7a421..5889db89ec 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/MetaInfo.java +++ b/apps/i2psnark/java/src/org/klomp/snark/MetaInfo.java @@ -20,6 +20,7 @@ package org.klomp.snark; +import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.security.MessageDigest; @@ -34,6 +35,7 @@ import java.util.Map; import net.i2p.I2PAppContext; import net.i2p.crypto.SHA1; import net.i2p.data.Base64; +import net.i2p.data.DataHelper; import net.i2p.util.Log; import org.klomp.snark.bencode.BDecoder; @@ -82,6 +84,12 @@ public class MetaInfo this.piece_hashes = piece_hashes; this.length = length; + // TODO if we add a parameter for other keys + //if (other != null) { + // otherInfo = new HashMap(2); + // otherInfo.putAll(other); + //} + this.info_hash = calculateInfoHash(); //infoMap = null; } @@ -101,10 +109,14 @@ public class MetaInfo * Creates a new MetaInfo from the given BDecoder. The BDecoder * must have a complete dictionary describing the torrent. */ - public MetaInfo(BDecoder be) throws IOException + private MetaInfo(BDecoder be) throws IOException { // Note that evaluation order matters here... this(be.bdecodeMap().getMap()); + byte[] origInfohash = be.get_special_map_digest(); + // shouldn't ever happen + if (!DataHelper.eq(origInfohash, info_hash)) + throw new InvalidBEncodingException("Infohash mismatch, please report"); } /** @@ -116,11 +128,11 @@ public class MetaInfo * WILL throw a InvalidBEncodingException if the given map does not * contain a valid info dictionary. */ - public MetaInfo(Map m) throws InvalidBEncodingException + public MetaInfo(Map m) throws InvalidBEncodingException { if (_log.shouldLog(Log.DEBUG)) _log.debug("Creating a metaInfo: " + m, new Exception("source")); - BEValue val = (BEValue)m.get("announce"); + BEValue val = m.get("announce"); // Disabled check, we can get info from a magnet now if (val == null) { //throw new InvalidBEncodingException("Missing announce string"); @@ -129,34 +141,37 @@ public class MetaInfo this.announce = val.getString(); } - val = (BEValue)m.get("info"); + val = m.get("info"); if (val == null) throw new InvalidBEncodingException("Missing info map"); - Map info = val.getMap(); + Map info = val.getMap(); infoMap = Collections.unmodifiableMap(info); - val = (BEValue)info.get("name"); + val = info.get("name"); if (val == null) throw new InvalidBEncodingException("Missing name string"); name = val.getString(); + // We could silently replace the '/', but that messes up the info hash, so just throw instead. + if (name.indexOf('/') >= 0) + throw new InvalidBEncodingException("Invalid name containing '/' " + name); - val = (BEValue)info.get("name.utf-8"); + val = info.get("name.utf-8"); if (val != null) name_utf8 = val.getString(); else name_utf8 = null; - val = (BEValue)info.get("piece length"); + val = info.get("piece length"); if (val == null) throw new InvalidBEncodingException("Missing piece length number"); piece_length = val.getInt(); - val = (BEValue)info.get("pieces"); + val = info.get("pieces"); if (val == null) throw new InvalidBEncodingException("Missing piece bytes"); piece_hashes = val.getBytes(); - val = (BEValue)info.get("length"); + val = info.get("length"); if (val != null) { // Single file case. @@ -168,7 +183,7 @@ public class MetaInfo else { // Multi file case. - val = (BEValue)info.get("files"); + val = info.get("files"); if (val == null) throw new InvalidBEncodingException ("Missing length number and/or files list"); @@ -189,8 +204,14 @@ public class MetaInfo if (val == null) throw new InvalidBEncodingException("Missing length number"); long len = val.getLong(); + if (len < 0) + throw new InvalidBEncodingException("Negative file length"); m_lengths.add(Long.valueOf(len)); + // check for overflowing the long + long oldTotal = l; l += len; + if (l < oldTotal) + throw new InvalidBEncodingException("Huge total length"); val = (BEValue)desc.get("path"); if (val == null) @@ -202,8 +223,19 @@ public class MetaInfo List file = new ArrayList(path_length); Iterator it = path_list.iterator(); - while (it.hasNext()) - file.add(it.next().getString()); + while (it.hasNext()) { + String s = it.next().getString(); + // We could throw an IBEE, but just silently replace instead. + if (s.indexOf('/') >= 0) + s = s.replace("/", "_"); + file.add(s); + } + + // quick dup check - case sensitive, etc. - Storage does a better job + for (int j = 0; j < i; j++) { + if (file.equals(m_files.get(j))) + throw new InvalidBEncodingException("Duplicate file path " + DataHelper.toString(file)); + } m_files.add(Collections.unmodifiableList(file)); @@ -229,6 +261,28 @@ public class MetaInfo info_hash = calculateInfoHash(); } + /** + * Efficiently returns the name and the 20 byte SHA1 hash of the info dictionary in a torrent file + * Caller must close stream. + * + * @param infoHashOut 20-byte out parameter + * @since 0.8.5 + */ + public static String getNameAndInfoHash(InputStream in, byte[] infoHashOut) throws IOException { + BDecoder bd = new BDecoder(in); + Map m = bd.bdecodeMap().getMap(); + BEValue ibev = m.get("info"); + if (ibev == null) + throw new InvalidBEncodingException("Missing info map"); + Map i = ibev.getMap(); + BEValue rvbev = i.get("name"); + if (rvbev == null) + throw new InvalidBEncodingException("Missing name"); + byte[] h = bd.get_special_map_digest(); + System.arraycopy(h, 0, infoHashOut, 0, 20); + return rvbev.getString(); + } + /** * Returns the string representing the URL of the tracker for this torrent. * @return may be null! @@ -318,11 +372,13 @@ public class MetaInfo */ public boolean checkPiece(int piece, byte[] bs, int off, int length) { - if (true) + //if (true) return fast_checkPiece(piece, bs, off, length); - else - return orig_checkPiece(piece, bs, off, length); + //else + // return orig_checkPiece(piece, bs, off, length); } + +/**** private boolean orig_checkPiece(int piece, byte[] bs, int off, int length) { // Check digest MessageDigest sha1; @@ -342,6 +398,7 @@ public class MetaInfo return false; return true; } +****/ private boolean fast_checkPiece(int piece, byte[] bs, int off, int length) { SHA1 sha1 = new SHA1(); @@ -365,7 +422,7 @@ public class MetaInfo @Override public String toString() { - return "MetaInfo[info_hash='" + hexencode(info_hash) + return "MetaInfo[info_hash='" + I2PSnarkUtil.toHex(info_hash) + "', announce='" + announce + "', name='" + name + "', files=" + files @@ -375,23 +432,6 @@ public class MetaInfo + "']"; } - /** - * Encode a byte array as a hex encoded string. - */ - private static String hexencode(byte[] bs) - { - StringBuilder sb = new StringBuilder(bs.length*2); - for (int i = 0; i < bs.length; i++) - { - int c = bs[i] & 0xFF; - if (c < 16) - sb.append('0'); - sb.append(Integer.toHexString(c)); - } - - return sb.toString(); - } - /** * Creates a copy of this MetaInfo that shares everything except the * announce URL. @@ -427,7 +467,8 @@ public class MetaInfo /** @return an unmodifiable view of the Map */ private Map createInfoMap() { - // if we loaded this metainfo from a file, we have the map + // If we loaded this metainfo from a file, we have the map, and we must use it + // or else we will lose any non-standard keys and corrupt the infohash. if (infoMap != null) return Collections.unmodifiableMap(infoMap); // otherwise we must create it @@ -453,27 +494,29 @@ public class MetaInfo } info.put("files", l); } + + // TODO if we add the ability for other keys in the first constructor + //if (otherInfo != null) + // info.putAll(otherInfo); + infoMap = info; return Collections.unmodifiableMap(infoMap); } private byte[] calculateInfoHash() { - Map info = createInfoMap(); - StringBuilder buf = new StringBuilder(128); - buf.append("info: "); - for (Iterator iter = info.entrySet().iterator(); iter.hasNext(); ) { - Map.Entry entry = (Map.Entry)iter.next(); - String key = (String)entry.getKey(); - Object val = entry.getValue(); - buf.append(key).append('='); - if (val instanceof byte[]) - buf.append(Base64.encode((byte[])val, true)); - else + Map info = createInfoMap(); + if (_log.shouldLog(Log.DEBUG)) { + StringBuilder buf = new StringBuilder(128); + buf.append("info: "); + for (Map.Entry entry : info.entrySet()) { + String key = entry.getKey(); + Object val = entry.getValue(); + buf.append(key).append('='); buf.append(val.toString()); - } - if (_log.shouldLog(Log.DEBUG)) + } _log.debug(buf.toString()); + } byte[] infoBytes = BEncoder.bencode(info); //_log.debug("info bencoded: [" + Base64.encode(infoBytes, true) + "]"); try @@ -481,7 +524,7 @@ public class MetaInfo MessageDigest digest = MessageDigest.getInstance("SHA"); byte hash[] = digest.digest(infoBytes); if (_log.shouldLog(Log.DEBUG)) - _log.debug("info hash: [" + net.i2p.data.Base64.encode(hash) + "]"); + _log.debug("info hash: " + I2PSnarkUtil.toHex(hash)); return hash; } catch(NoSuchAlgorithmException nsa) @@ -490,5 +533,23 @@ public class MetaInfo } } - + /** @since 0.8.5 */ + public static void main(String[] args) { + if (args.length <= 0) { + System.err.println("Usage: MetaInfo files..."); + return; + } + for (int i = 0; i < args.length; i++) { + InputStream in = null; + try { + in = new FileInputStream(args[i]); + MetaInfo meta = new MetaInfo(in); + System.out.println(args[i] + " InfoHash: " + I2PSnarkUtil.toHex(meta.getInfoHash())); + } catch (IOException ioe) { + System.err.println("Error in file " + args[i] + ": " + ioe); + } finally { + try { if (in != null) in.close(); } catch (IOException ioe) {} + } + } + } } diff --git a/apps/i2psnark/java/src/org/klomp/snark/PeerCoordinator.java b/apps/i2psnark/java/src/org/klomp/snark/PeerCoordinator.java index cdf1e58b5e..3aec9394c1 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/PeerCoordinator.java +++ b/apps/i2psnark/java/src/org/klomp/snark/PeerCoordinator.java @@ -783,6 +783,8 @@ public class PeerCoordinator implements PeerListener /** * Returns a byte array containing the requested piece or null of * the piece is unknown. + * + * @throws RuntimeException on IOE getting the data */ public byte[] gotRequest(Peer peer, int piece, int off, int len) { @@ -798,8 +800,11 @@ public class PeerCoordinator implements PeerListener catch (IOException ioe) { snark.stopTorrent(); - _log.error("Error reading the storage for " + metainfo.getName(), ioe); - throw new RuntimeException("B0rked"); + String msg = "Error reading the storage (piece " + piece + ") for " + metainfo.getName() + ": " + ioe; + _log.error(msg, ioe); + SnarkManager.instance().addMessage(msg); + SnarkManager.instance().addMessage("Fatal storage error: Stopping torrent " + metainfo.getName()); + throw new RuntimeException(msg, ioe); } } @@ -829,6 +834,8 @@ public class PeerCoordinator implements PeerListener * Returns false if the piece is no good (according to the hash). * In that case the peer that supplied the piece should probably be * blacklisted. + * + * @throws RuntimeException on IOE saving the piece */ public boolean gotPiece(Peer peer, int piece, byte[] bs) { @@ -872,8 +879,11 @@ public class PeerCoordinator implements PeerListener catch (IOException ioe) { snark.stopTorrent(); - _log.error("Error writing storage for " + metainfo.getName(), ioe); - throw new RuntimeException("B0rked"); + String msg = "Error writing storage (piece " + piece + ") for " + metainfo.getName() + ": " + ioe; + _log.error(msg, ioe); + SnarkManager.instance().addMessage(msg); + SnarkManager.instance().addMessage("Fatal storage error: Stopping torrent " + metainfo.getName()); + throw new RuntimeException(msg, ioe); } wantedPieces.remove(p); } diff --git a/apps/i2psnark/java/src/org/klomp/snark/Snark.java b/apps/i2psnark/java/src/org/klomp/snark/Snark.java index 1908a11937..c43679ed39 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/Snark.java +++ b/apps/i2psnark/java/src/org/klomp/snark/Snark.java @@ -40,8 +40,6 @@ import net.i2p.client.streaming.I2PServerSocket; import net.i2p.data.Destination; import net.i2p.util.I2PThread; -import org.klomp.snark.bencode.BDecoder; - /** * Main Snark program startup class. * @@ -360,7 +358,7 @@ public class Snark in = new FileInputStream(torrentFile); } } - meta = new MetaInfo(new BDecoder(in)); + meta = new MetaInfo(in); infoHash = meta.getInfoHash(); } catch(IOException ioe) @@ -589,7 +587,7 @@ public class Snark pc.halt(); Storage st = storage; if (st != null) { - boolean changed = storage.changed; + boolean changed = storage.isChanged(); try { storage.close(); } catch (IOException ioe) { @@ -1013,7 +1011,7 @@ public class Snark //if (debug >= INFO && t != null) // t.printStackTrace(); stopTorrent(); - throw new RuntimeException(s + (t == null ? "" : ": " + t)); + throw new RuntimeException(s, t); } /** @@ -1111,7 +1109,7 @@ public class Snark allChecked = true; checking = false; - if (storage.changed && completeListener != null) + if (storage.isChanged() && completeListener != null) completeListener.updateStatus(this); } diff --git a/apps/i2psnark/java/src/org/klomp/snark/SnarkManager.java b/apps/i2psnark/java/src/org/klomp/snark/SnarkManager.java index e47ffcc0c8..365eb14b37 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/SnarkManager.java +++ b/apps/i2psnark/java/src/org/klomp/snark/SnarkManager.java @@ -572,6 +572,9 @@ public class SnarkManager implements Snark.CompleteListener { } try { + // This is somewhat wasteful as this metainfo is thrown away, + // the real one is created in the Snark constructor. + // TODO: Make a Snark constructor where we pass the MetaInfo in as a parameter. MetaInfo info = new MetaInfo(fis); try { fis.close(); @@ -622,6 +625,7 @@ public class SnarkManager implements Snark.CompleteListener { return; } catch (OutOfMemoryError oom) { addMessage(_("ERROR - Out of memory, cannot create torrent from {0}", sfile.getName()) + ": " + oom.getMessage()); + return; } finally { if (fis != null) try { fis.close(); } catch (IOException ioe) {} } diff --git a/apps/i2psnark/java/src/org/klomp/snark/Storage.java b/apps/i2psnark/java/src/org/klomp/snark/Storage.java index dfc229dce2..f9d93ca073 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/Storage.java +++ b/apps/i2psnark/java/src/org/klomp/snark/Storage.java @@ -53,10 +53,10 @@ public class Storage private int needed; // Number of pieces needed private boolean _probablyComplete; // use this to decide whether to open files RO - // XXX - Not always set correctly - int piece_size; - int pieces; - boolean changed; + private final int piece_size; + private final int pieces; + private final long total_length; + private boolean changed; /** The default piece size. */ private static final int MIN_PIECE_SIZE = 256*1024; @@ -81,6 +81,9 @@ public class Storage needed = metainfo.getPieces(); _probablyComplete = false; bitfield = new BitField(needed); + piece_size = metainfo.getPieceLength(0); + pieces = needed; + total_length = metainfo.getTotalLength(); } /** @@ -108,17 +111,17 @@ public class Storage lengthsList.add(Long.valueOf(length)); } - piece_size = MIN_PIECE_SIZE; - pieces = (int) ((total - 1)/piece_size) + 1; - while (pieces > MAX_PIECES && piece_size < MAX_PIECE_SIZE) + int pc_size = MIN_PIECE_SIZE; + int pcs = (int) ((total - 1)/pc_size) + 1; + while (pcs > MAX_PIECES && pc_size < MAX_PIECE_SIZE) { - piece_size = piece_size*2; - pieces = (int) ((total - 1)/piece_size) +1; + pc_size *= 2; + pcs = (int) ((total - 1)/pc_size) +1; } + piece_size = pc_size; + pieces = pcs; + total_length = total; - // Note that piece_hashes and the bitfield will be filled after - // the MetaInfo is created. - byte[] piece_hashes = new byte[20*pieces]; bitfield = new BitField(pieces); needed = 0; @@ -142,69 +145,26 @@ public class Storage lengthsList = null; } - // Note that the piece_hashes are not correctly setup yet. + byte[] piece_hashes = fast_digestCreate(); metainfo = new MetaInfo(announce, baseFile.getName(), null, files, lengthsList, piece_size, piece_hashes, total); } - // Creates piece hashes for a new storage. - // This does NOT create the files, just the hashes - public void create() throws IOException - { -// if (true) { - fast_digestCreate(); -// } else { -// orig_digestCreate(); -// } - } - -/* - private void orig_digestCreate() throws IOException { - // Calculate piece_hashes - MessageDigest digest = null; - try - { - digest = MessageDigest.getInstance("SHA"); - } - catch(NoSuchAlgorithmException nsa) - { - throw new InternalError(nsa.toString()); - } - - byte[] piece_hashes = metainfo.getPieceHashes(); - - byte[] piece = new byte[piece_size]; - for (int i = 0; i < pieces; i++) - { - int length = getUncheckedPiece(i, piece); - digest.update(piece, 0, length); - byte[] hash = digest.digest(); - for (int j = 0; j < 20; j++) - piece_hashes[20 * i + j] = hash[j]; - - bitfield.set(i); - - if (listener != null) - listener.storageChecked(this, i, true); - } - - if (listener != null) - listener.storageAllChecked(this); - - // Reannounce to force recalculating the info_hash. - metainfo = metainfo.reannounce(metainfo.getAnnounce()); - } -*/ - - /** FIXME we can run out of fd's doing this, + /** + * Creates piece hashes for a new storage. + * This does NOT create the files, just the hashes. + * Also sets all the bitfield bits. + * + * FIXME we can run out of fd's doing this, * maybe some sort of global close-RAF-right-away flag - * would do the trick */ - private void fast_digestCreate() throws IOException { + * would do the trick + */ + private byte[] fast_digestCreate() throws IOException { // Calculate piece_hashes SHA1 digest = new SHA1(); - byte[] piece_hashes = metainfo.getPieceHashes(); + byte[] piece_hashes = new byte[20 * pieces]; byte[] piece = new byte[piece_size]; for (int i = 0; i < pieces; i++) @@ -212,14 +172,10 @@ public class Storage int length = getUncheckedPiece(i, piece); digest.update(piece, 0, length); byte[] hash = digest.digest(); - for (int j = 0; j < 20; j++) - piece_hashes[20 * i + j] = hash[j]; - + System.arraycopy(hash, 0, piece_hashes, 20 * i, 20); bitfield.set(i); } - - // Reannounce to force recalculating the info_hash. - metainfo = metainfo.reannounce(metainfo.getAnnounce()); + return piece_hashes; } private void getFiles(File base) throws IOException @@ -294,6 +250,14 @@ public class Storage return needed == 0; } + /** + * Has the storage changed since instantiation? + * @since 0.8.5 + */ + public boolean isChanged() { + return changed; + } + /** * @param file canonical path (non-directory) * @return number of bytes remaining; -1 if unknown file @@ -315,14 +279,13 @@ public class Storage if (f != null && canonical.equals(file)) { if (complete()) return 0; - int psz = metainfo.getPieceLength(0); + int psz = piece_size; long start = bytes; long end = start + lengths[i]; int pc = (int) (bytes / psz); long rv = 0; if (!bitfield.get(pc)) rv = Math.min(psz - (start % psz), lengths[i]); - int pieces = metainfo.getPieces(); for (int j = pc + 1; (((long)j) * psz) < end && j < pieces; j++) { if (!bitfield.get(j)) { if (((long)(j+1))*psz < end) @@ -418,7 +381,7 @@ public class Storage int file = 0; long pcEnd = -1; long fileEnd = lengths[0] - 1; - int psz = metainfo.getPieceLength(0); + int psz = piece_size; for (int i = 0; i < rv.length; i++) { pcEnd += psz; int pri = priorities[file]; @@ -469,7 +432,7 @@ public class Storage File base = new SecureFile(rootDir, filterName(metainfo.getName())); boolean useSavedBitField = savedTime > 0 && savedBitField != null; - List files = metainfo.getFiles(); + List> files = metainfo.getFiles(); if (files == null) { // Create base as file. @@ -500,7 +463,7 @@ public class Storage if (!base.mkdir() && !base.isDirectory()) throw new IOException("Could not create directory " + base); - List ls = metainfo.getLengths(); + List ls = metainfo.getLengths(); int size = files.size(); long total = 0; lengths = new long[size]; @@ -511,8 +474,28 @@ public class Storage RAFfile = new File[size]; for (int i = 0; i < size; i++) { - File f = createFileFromNames(base, (List)files.get(i)); - lengths[i] = ((Long)ls.get(i)).longValue(); + List path = files.get(i); + File f = createFileFromNames(base, path); + // dup file name check after filtering + for (int j = 0; j < i; j++) { + if (f.equals(RAFfile[j])) { + // Rename and start the check over again + // Copy path since metainfo list is unmodifiable + path = new ArrayList(path); + int last = path.size() - 1; + String lastPath = path.get(last); + int dot = lastPath.lastIndexOf('.'); + // foo.mp3 -> foo_.mp3; foo -> _foo + if (dot >= 0) + lastPath = lastPath.substring(0, dot) + '_' + lastPath.substring(dot); + else + lastPath = '_' + lastPath; + path.set(last, lastPath); + f = createFileFromNames(base, path); + j = 0; + } + } + lengths[i] = ls.get(i).longValue(); RAFlock[i] = new Object(); RAFfile[i] = f; total += lengths[i]; @@ -551,36 +534,19 @@ public class Storage } /** - * Reopen the file descriptors for a restart - * Do existence check but no length check or data reverification + * Doesn't really reopen the file descriptors for a restart. + * Just does an existence check but no length check or data reverification + * + * @param rootDir ignored + * @throws IOE on fail */ public void reopen(String rootDir) throws IOException { - File base = new File(rootDir, filterName(metainfo.getName())); - - List files = metainfo.getFiles(); - if (files == null) - { - // Reopen base as file. - _util.debug("Reopening file: " + base, Snark.NOTICE); - if (!base.exists()) - throw new IOException("Could not reopen file " + base); - } - else - { - // Reopen base as dir. - _util.debug("Reopening directory: " + base, Snark.NOTICE); - if (!base.isDirectory()) - throw new IOException("Could not reopen directory " + base); - - int size = files.size(); - for (int i = 0; i < size; i++) - { - File f = getFileFromNames(base, (List)files.get(i)); - if (!f.exists()) - throw new IOException("Could not reopen file " + f); - } - + if (RAFfile == null) + throw new IOException("Storage not checked yet"); + for (int i = 0; i < RAFfile.length; i++) { + if (!RAFfile[i].exists()) + throw new IOException("File does not exist: " + RAFfile[i]); } } @@ -609,13 +575,18 @@ public class Storage return rv; } - private File createFileFromNames(File base, List names) throws IOException + /** + * Note that filtering each path element individually may lead to + * things going in the wrong place if there are duplicates + * in intermediate path elements after filtering. + */ + private static File createFileFromNames(File base, List names) throws IOException { File f = null; - Iterator it = names.iterator(); + Iterator it = names.iterator(); while (it.hasNext()) { - String name = filterName((String)it.next()); + String name = filterName(it.next()); if (it.hasNext()) { // Another dir in the hierarchy. @@ -635,12 +606,12 @@ public class Storage return f; } - public static File getFileFromNames(File base, List names) + public static File getFileFromNames(File base, List names) { - Iterator it = names.iterator(); + Iterator it = names.iterator(); while (it.hasNext()) { - String name = filterName((String)it.next()); + String name = filterName(it.next()); base = new File(base, name); } return base; @@ -690,7 +661,10 @@ public class Storage } catch (IOException ioe) {} } } else { - _util.debug("File '" + names[i] + "' exists, but has wrong length - repairing corruption", Snark.ERROR); + String msg = "File '" + names[i] + "' exists, but has wrong length (expected " + + lengths[i] + " but found " + length + ") - repairing corruption"; + SnarkManager.instance().addMessage(msg); + _util.debug(msg, Snark.ERROR); changed = true; _probablyComplete = false; // to force RW synchronized(RAFlock[i]) { @@ -706,8 +680,7 @@ public class Storage // Check which pieces match and which don't if (resume) { - pieces = metainfo.getPieces(); - byte[] piece = new byte[metainfo.getPieceLength(0)]; + byte[] piece = new byte[piece_size]; int file = 0; long fileEnd = lengths[0]; long pieceEnd = 0; @@ -775,7 +748,7 @@ public class Storage // the whole file? if (listener != null) listener.storageCreateFile(this, names[nr], lengths[nr]); - final int ZEROBLOCKSIZE = metainfo.getPieceLength(0); + final int ZEROBLOCKSIZE = piece_size; byte[] zeros; try { zeros = new byte[ZEROBLOCKSIZE]; @@ -868,7 +841,7 @@ public class Storage } // Early typecast, avoid possibly overflowing a temp integer - long start = (long) piece * (long) metainfo.getPieceLength(0); + long start = (long) piece * (long) piece_size; int i = 0; long raflen = lengths[i]; while (start > raflen) @@ -935,10 +908,24 @@ public class Storage return true; } + /** + * This is a dup of MetaInfo.getPieceLength() but we need it + * before the MetaInfo is created in our second constructor. + * @since 0.8.5 + */ + private int getPieceLength(int piece) { + if (piece >= 0 && piece < pieces -1) + return piece_size; + else if (piece == pieces -1) + return (int)(total_length - ((long)piece * piece_size)); + else + throw new IndexOutOfBoundsException("no piece: " + piece); + } + private int getUncheckedPiece(int piece, byte[] bs) throws IOException { - return getUncheckedPiece(piece, bs, 0, metainfo.getPieceLength(piece)); + return getUncheckedPiece(piece, bs, 0, getPieceLength(piece)); } private int getUncheckedPiece(int piece, byte[] bs, int off, int length) @@ -947,7 +934,7 @@ public class Storage // XXX - copy/paste code from putPiece(). // Early typecast, avoid possibly overflowing a temp integer - long start = ((long) piece * (long) metainfo.getPieceLength(0)) + off; + long start = ((long) piece * (long) piece_size) + off; int i = 0; long raflen = lengths[i]; diff --git a/apps/i2psnark/java/src/org/klomp/snark/bencode/BDecoder.java b/apps/i2psnark/java/src/org/klomp/snark/bencode/BDecoder.java index 4c59bcb93b..4f394d01e5 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/bencode/BDecoder.java +++ b/apps/i2psnark/java/src/org/klomp/snark/bencode/BDecoder.java @@ -60,22 +60,30 @@ public class BDecoder private int indicator = 0; // Used for ugly hack to get SHA hash over the metainfo info map - private String special_map = "info"; + private final String special_map = "info"; private boolean in_special_map = false; - private final MessageDigest sha_digest; + /** creation deferred until we encounter the special map, to make processing of announce replies more efficient */ + private MessageDigest sha_digest; - // Ugly hack. Return the SHA has over bytes that make up the special map. + /** + * Ugly hack. Return the SHA has over bytes that make up the special map. + * @return null if there was no special map + */ public byte[] get_special_map_digest() { + if (sha_digest == null) + return null; byte[] result = sha_digest.digest(); return result; } +/**** // Ugly hack. Name defaults to "info". public void set_special_map_name(String name) { special_map = name; } +****/ /** * Initalizes a new BDecoder. Nothing is read from the given @@ -84,15 +92,6 @@ public class BDecoder public BDecoder(InputStream in) { this.in = in; - // XXX - Used for ugly hack. - try - { - sha_digest = MessageDigest.getInstance("SHA"); - } - catch(NoSuchAlgorithmException nsa) - { - throw new InternalError(nsa.toString()); - } } /** @@ -112,6 +111,24 @@ public class BDecoder return new BDecoder(in).bdecode(); } + /** + * Used for SHA1 hack + * @since 0.8.5 + */ + private void createDigest() { + if (sha_digest == null) { + try { + sha_digest = MessageDigest.getInstance("SHA"); + } catch(NoSuchAlgorithmException nsa) { + throw new InternalError(nsa.toString()); + } + } else { + // there are two info maps, but not one inside the other, + // the resulting hash will be incorrect + // throw something? - no, the check in the MetaInfo constructor will catch it. + } + } + /** * Returns what the next bencoded object will be on the stream or -1 * when the end of stream has been reached. Can return something @@ -294,9 +311,13 @@ public class BDecoder String key = bdecode().getString(); // XXX ugly hack - boolean special = special_map.equals(key); - if (special) + // This will not screw up if an info map contains an info map, + // but it will if there are two info maps (not one inside the other) + boolean special = (!in_special_map) && special_map.equals(key); + if (special) { + createDigest(); in_special_map = true; + } BEValue value = bdecode(); result.put(key, value); diff --git a/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java b/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java index 788741bd8f..f3ef456dce 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java +++ b/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java @@ -643,7 +643,6 @@ public class I2PSnarkServlet extends Default { // This may take a long time to check the storage, but since it already exists, // it shouldn't be THAT bad, so keep it in this thread. Storage s = new Storage(_manager.util(), baseFile, announceURL, null); - s.create(); s.close(); // close the files... maybe need a way to pass this Storage to addTorrent rather than starting over MetaInfo info = s.getMetaInfo(); File torrentFile = new File(_manager.getDataDir(), s.getBaseName() + ".torrent"); @@ -1968,16 +1967,15 @@ private static class FetchAndAdd implements Runnable { FileInputStream in = null; try { in = new FileInputStream(file); - // we do not retain this MetaInfo object, hopefully it will go away quickly - MetaInfo info = new MetaInfo(in); + byte[] fileInfoHash = new byte[20]; + String name = MetaInfo.getNameAndInfoHash(in, fileInfoHash); try { in.close(); } catch (IOException ioe) {} - Snark snark = _manager.getTorrentByInfoHash(info.getInfoHash()); + Snark snark = _manager.getTorrentByInfoHash(fileInfoHash); if (snark != null) { _manager.addMessage(_("Torrent with this info hash is already running: {0}", snark.getBaseName())); return; } - String name = info.getName(); name = Storage.filterName(name); name = name + ".torrent"; File torrentFile = new File(_manager.getDataDir(), name);