echelon/i2p.i2p
1
0
Fork 0
forked from I2P_Developers/i2p.i2p
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

* i2psnark:

Browse Source 
- More efficient metainfo handling, reduce instantiations
      - Improved handling of storage errors
      - Improved handling of duplicate file names
      - More metainfo sanity checks
      - Metadata transfer error handling improvements
      - Code cleanup, remove dead and duplicated code
This commit is contained in:
zzz
2011-03-08 03:01:02 +00:00
parent 0e854623c1
commit f9b2c0bc63
8 changed files with 286 additions and 200 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
apps/i2psnark/java/src/org/klomp/snark/MagnetState.java
View File

@ -191,14 +191,21 @@ class MagnetState {
*/ */
public MetaInfo buildMetaInfo() throws Exception { public MetaInfo buildMetaInfo() throws Exception {
// top map has nothing in it but the info map (no announce) // top map has nothing in it but the info map (no announce)
Map<String, Object> map = new HashMap(); Map<String, BEValue> map = new HashMap();
InputStream is = new ByteArrayInputStream(metainfoBytes); InputStream is = new ByteArrayInputStream(metainfoBytes);
BDecoder dec = new BDecoder(is); BDecoder dec = new BDecoder(is);
BEValue bev = dec.bdecodeMap(); BEValue bev = dec.bdecodeMap();
map.put("info", bev); map.put("info", bev);
MetaInfo newmeta = new MetaInfo(map); MetaInfo newmeta = new MetaInfo(map);
if (!DataHelper.eq(newmeta.getInfoHash(), infohash)) if (!DataHelper.eq(newmeta.getInfoHash(), infohash)) {
// Disaster. Start over. ExtensionHandler will catch
// the IOE and disconnect the peer, hopefully we will
// find a new peer.
// TODO: Count fails and give up eventually
have = new BitField(totalChunks);
requested = new BitField(totalChunks);
throw new IOException("info hash mismatch"); throw new IOException("info hash mismatch");
}
return newmeta; return newmeta;
} }
} }

161
apps/i2psnark/java/src/org/klomp/snark/MetaInfo.java
View File

@ -20,6 +20,7 @@
package org.klomp.snark; package org.klomp.snark;
import java.io.FileInputStream;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.security.MessageDigest; import java.security.MessageDigest;
@ -34,6 +35,7 @@ import java.util.Map;
import net.i2p.I2PAppContext; import net.i2p.I2PAppContext;
import net.i2p.crypto.SHA1; import net.i2p.crypto.SHA1;
import net.i2p.data.Base64; import net.i2p.data.Base64;
import net.i2p.data.DataHelper;
import net.i2p.util.Log; import net.i2p.util.Log;
import org.klomp.snark.bencode.BDecoder; import org.klomp.snark.bencode.BDecoder;
@ -82,6 +84,12 @@ public class MetaInfo
this.piece_hashes = piece_hashes; this.piece_hashes = piece_hashes;
this.length = length; this.length = length;
// TODO if we add a parameter for other keys
//if (other != null) {
// otherInfo = new HashMap(2);
// otherInfo.putAll(other);
//}
this.info_hash = calculateInfoHash(); this.info_hash = calculateInfoHash();
//infoMap = null; //infoMap = null;
} }
@ -101,10 +109,14 @@ public class MetaInfo
* Creates a new MetaInfo from the given BDecoder. The BDecoder * Creates a new MetaInfo from the given BDecoder. The BDecoder
* must have a complete dictionary describing the torrent. * must have a complete dictionary describing the torrent.
*/ */
public MetaInfo(BDecoder be) throws IOException private MetaInfo(BDecoder be) throws IOException
{ {
// Note that evaluation order matters here... // Note that evaluation order matters here...
this(be.bdecodeMap().getMap()); this(be.bdecodeMap().getMap());
byte[] origInfohash = be.get_special_map_digest();
// shouldn't ever happen
if (!DataHelper.eq(origInfohash, info_hash))
throw new InvalidBEncodingException("Infohash mismatch, please report");
} }
/** /**
@ -116,11 +128,11 @@ public class MetaInfo
* WILL throw a InvalidBEncodingException if the given map does not * WILL throw a InvalidBEncodingException if the given map does not
* contain a valid info dictionary. * contain a valid info dictionary.
*/ */
public MetaInfo(Map m) throws InvalidBEncodingException public MetaInfo(Map<String, BEValue> m) throws InvalidBEncodingException
{ {
if (_log.shouldLog(Log.DEBUG)) if (_log.shouldLog(Log.DEBUG))
_log.debug("Creating a metaInfo: " + m, new Exception("source")); _log.debug("Creating a metaInfo: " + m, new Exception("source"));
BEValue val = (BEValue)m.get("announce"); BEValue val = m.get("announce");
// Disabled check, we can get info from a magnet now // Disabled check, we can get info from a magnet now
if (val == null) { if (val == null) {
//throw new InvalidBEncodingException("Missing announce string"); //throw new InvalidBEncodingException("Missing announce string");
@ -129,34 +141,37 @@ public class MetaInfo
this.announce = val.getString(); this.announce = val.getString();
} }
val = (BEValue)m.get("info"); val = m.get("info");
if (val == null) if (val == null)
throw new InvalidBEncodingException("Missing info map"); throw new InvalidBEncodingException("Missing info map");
Map info = val.getMap(); Map<String, BEValue> info = val.getMap();
infoMap = Collections.unmodifiableMap(info); infoMap = Collections.unmodifiableMap(info);
val = (BEValue)info.get("name"); val = info.get("name");
if (val == null) if (val == null)
throw new InvalidBEncodingException("Missing name string"); throw new InvalidBEncodingException("Missing name string");
name = val.getString(); name = val.getString();
// We could silently replace the '/', but that messes up the info hash, so just throw instead.
if (name.indexOf('/') >= 0)
throw new InvalidBEncodingException("Invalid name containing '/' " + name);
val = (BEValue)info.get("name.utf-8"); val = info.get("name.utf-8");
if (val != null) if (val != null)
name_utf8 = val.getString(); name_utf8 = val.getString();
else else
name_utf8 = null; name_utf8 = null;
val = (BEValue)info.get("piece length"); val = info.get("piece length");
if (val == null) if (val == null)
throw new InvalidBEncodingException("Missing piece length number"); throw new InvalidBEncodingException("Missing piece length number");
piece_length = val.getInt(); piece_length = val.getInt();
val = (BEValue)info.get("pieces"); val = info.get("pieces");
if (val == null) if (val == null)
throw new InvalidBEncodingException("Missing piece bytes"); throw new InvalidBEncodingException("Missing piece bytes");
piece_hashes = val.getBytes(); piece_hashes = val.getBytes();
val = (BEValue)info.get("length"); val = info.get("length");
if (val != null) if (val != null)
{ {
// Single file case. // Single file case.
@ -168,7 +183,7 @@ public class MetaInfo
else else
{ {
// Multi file case. // Multi file case.
val = (BEValue)info.get("files"); val = info.get("files");
if (val == null) if (val == null)
throw new InvalidBEncodingException throw new InvalidBEncodingException
("Missing length number and/or files list"); ("Missing length number and/or files list");
@ -189,8 +204,14 @@ public class MetaInfo
if (val == null) if (val == null)
throw new InvalidBEncodingException("Missing length number"); throw new InvalidBEncodingException("Missing length number");
long len = val.getLong(); long len = val.getLong();
if (len < 0)
throw new InvalidBEncodingException("Negative file length");
m_lengths.add(Long.valueOf(len)); m_lengths.add(Long.valueOf(len));
// check for overflowing the long
long oldTotal = l;
l += len; l += len;
if (l < oldTotal)
throw new InvalidBEncodingException("Huge total length");
val = (BEValue)desc.get("path"); val = (BEValue)desc.get("path");
if (val == null) if (val == null)
@ -202,8 +223,19 @@ public class MetaInfo
List<String> file = new ArrayList(path_length); List<String> file = new ArrayList(path_length);
Iterator<BEValue> it = path_list.iterator(); Iterator<BEValue> it = path_list.iterator();
while (it.hasNext()) while (it.hasNext()) {
file.add(it.next().getString()); String s = it.next().getString();
// We could throw an IBEE, but just silently replace instead.
if (s.indexOf('/') >= 0)
s = s.replace("/", "_");
file.add(s);
}
// quick dup check - case sensitive, etc. - Storage does a better job
for (int j = 0; j < i; j++) {
if (file.equals(m_files.get(j)))
throw new InvalidBEncodingException("Duplicate file path " + DataHelper.toString(file));
}
m_files.add(Collections.unmodifiableList(file)); m_files.add(Collections.unmodifiableList(file));
@ -229,6 +261,28 @@ public class MetaInfo
info_hash = calculateInfoHash(); info_hash = calculateInfoHash();
} }
/**
* Efficiently returns the name and the 20 byte SHA1 hash of the info dictionary in a torrent file
* Caller must close stream.
*
* @param infoHashOut 20-byte out parameter
* @since 0.8.5
*/
public static String getNameAndInfoHash(InputStream in, byte[] infoHashOut) throws IOException {
BDecoder bd = new BDecoder(in);
Map<String, BEValue> m = bd.bdecodeMap().getMap();
BEValue ibev = m.get("info");
if (ibev == null)
throw new InvalidBEncodingException("Missing info map");
Map<String, BEValue> i = ibev.getMap();
BEValue rvbev = i.get("name");
if (rvbev == null)
throw new InvalidBEncodingException("Missing name");
byte[] h = bd.get_special_map_digest();
System.arraycopy(h, 0, infoHashOut, 0, 20);
return rvbev.getString();
}
/** /**
* Returns the string representing the URL of the tracker for this torrent. * Returns the string representing the URL of the tracker for this torrent.
* @return may be null! * @return may be null!
@ -318,11 +372,13 @@ public class MetaInfo
*/ */
public boolean checkPiece(int piece, byte[] bs, int off, int length) public boolean checkPiece(int piece, byte[] bs, int off, int length)
{ {
if (true) //if (true)
return fast_checkPiece(piece, bs, off, length); return fast_checkPiece(piece, bs, off, length);
else //else
return orig_checkPiece(piece, bs, off, length); // return orig_checkPiece(piece, bs, off, length);
} }
/****
private boolean orig_checkPiece(int piece, byte[] bs, int off, int length) { private boolean orig_checkPiece(int piece, byte[] bs, int off, int length) {
// Check digest // Check digest
MessageDigest sha1; MessageDigest sha1;
@ -342,6 +398,7 @@ public class MetaInfo
return false; return false;
return true; return true;
} }
****/
private boolean fast_checkPiece(int piece, byte[] bs, int off, int length) { private boolean fast_checkPiece(int piece, byte[] bs, int off, int length) {
SHA1 sha1 = new SHA1(); SHA1 sha1 = new SHA1();
@ -365,7 +422,7 @@ public class MetaInfo
@Override @Override
public String toString() public String toString()
{ {
return "MetaInfo[info_hash='" + hexencode(info_hash) return "MetaInfo[info_hash='" + I2PSnarkUtil.toHex(info_hash)
+ "', announce='" + announce + "', announce='" + announce
+ "', name='" + name + "', name='" + name
+ "', files=" + files + "', files=" + files
@ -375,23 +432,6 @@ public class MetaInfo
+ "']"; + "']";
} }
/**
* Encode a byte array as a hex encoded string.
*/
private static String hexencode(byte[] bs)
{
StringBuilder sb = new StringBuilder(bs.length*2);
for (int i = 0; i < bs.length; i++)
{
int c = bs[i] & 0xFF;
if (c < 16)
sb.append('0');
sb.append(Integer.toHexString(c));
}
return sb.toString();
}
/** /**
* Creates a copy of this MetaInfo that shares everything except the * Creates a copy of this MetaInfo that shares everything except the
* announce URL. * announce URL.
@ -427,7 +467,8 @@ public class MetaInfo
/** @return an unmodifiable view of the Map */ /** @return an unmodifiable view of the Map */
private Map<String, BEValue> createInfoMap() private Map<String, BEValue> createInfoMap()
{ {
// if we loaded this metainfo from a file, we have the map // If we loaded this metainfo from a file, we have the map, and we must use it
// or else we will lose any non-standard keys and corrupt the infohash.
if (infoMap != null) if (infoMap != null)
return Collections.unmodifiableMap(infoMap); return Collections.unmodifiableMap(infoMap);
// otherwise we must create it // otherwise we must create it
@ -453,27 +494,29 @@ public class MetaInfo
} }
info.put("files", l); info.put("files", l);
} }
// TODO if we add the ability for other keys in the first constructor
//if (otherInfo != null)
// info.putAll(otherInfo);
infoMap = info; infoMap = info;
return Collections.unmodifiableMap(infoMap); return Collections.unmodifiableMap(infoMap);
} }
private byte[] calculateInfoHash() private byte[] calculateInfoHash()
{ {
Map info = createInfoMap(); Map<String, BEValue> info = createInfoMap();
StringBuilder buf = new StringBuilder(128); if (_log.shouldLog(Log.DEBUG)) {
buf.append("info: "); StringBuilder buf = new StringBuilder(128);
for (Iterator iter = info.entrySet().iterator(); iter.hasNext(); ) { buf.append("info: ");
Map.Entry entry = (Map.Entry)iter.next(); for (Map.Entry<String, BEValue> entry : info.entrySet()) {
String key = (String)entry.getKey(); String key = entry.getKey();
Object val = entry.getValue(); Object val = entry.getValue();
buf.append(key).append('='); buf.append(key).append('=');
if (val instanceof byte[])
buf.append(Base64.encode((byte[])val, true));
else
buf.append(val.toString()); buf.append(val.toString());
} }
if (_log.shouldLog(Log.DEBUG))
_log.debug(buf.toString()); _log.debug(buf.toString());
}
byte[] infoBytes = BEncoder.bencode(info); byte[] infoBytes = BEncoder.bencode(info);
//_log.debug("info bencoded: [" + Base64.encode(infoBytes, true) + "]"); //_log.debug("info bencoded: [" + Base64.encode(infoBytes, true) + "]");
try try
@ -481,7 +524,7 @@ public class MetaInfo
MessageDigest digest = MessageDigest.getInstance("SHA"); MessageDigest digest = MessageDigest.getInstance("SHA");
byte hash[] = digest.digest(infoBytes); byte hash[] = digest.digest(infoBytes);
if (_log.shouldLog(Log.DEBUG)) if (_log.shouldLog(Log.DEBUG))
_log.debug("info hash: [" + net.i2p.data.Base64.encode(hash) + "]"); _log.debug("info hash: " + I2PSnarkUtil.toHex(hash));
return hash; return hash;
} }
catch(NoSuchAlgorithmException nsa) catch(NoSuchAlgorithmException nsa)
@ -490,5 +533,23 @@ public class MetaInfo
} }
} }
/** @since 0.8.5 */
public static void main(String[] args) {
if (args.length <= 0) {
System.err.println("Usage: MetaInfo files...");
return;
}
for (int i = 0; i < args.length; i++) {
InputStream in = null;
try {
in = new FileInputStream(args[i]);
MetaInfo meta = new MetaInfo(in);
System.out.println(args[i] + " InfoHash: " + I2PSnarkUtil.toHex(meta.getInfoHash()));
} catch (IOException ioe) {
System.err.println("Error in file " + args[i] + ": " + ioe);
} finally {
try { if (in != null) in.close(); } catch (IOException ioe) {}
}
}
}
} }

18
apps/i2psnark/java/src/org/klomp/snark/PeerCoordinator.java
View File

@ -783,6 +783,8 @@ public class PeerCoordinator implements PeerListener
/** /**
* Returns a byte array containing the requested piece or null of * Returns a byte array containing the requested piece or null of
* the piece is unknown. * the piece is unknown.
*
* @throws RuntimeException on IOE getting the data
*/ */
public byte[] gotRequest(Peer peer, int piece, int off, int len) public byte[] gotRequest(Peer peer, int piece, int off, int len)
{ {
@ -798,8 +800,11 @@ public class PeerCoordinator implements PeerListener
catch (IOException ioe) catch (IOException ioe)
{ {
snark.stopTorrent(); snark.stopTorrent();
_log.error("Error reading the storage for " + metainfo.getName(), ioe); String msg = "Error reading the storage (piece " + piece + ") for " + metainfo.getName() + ": " + ioe;
throw new RuntimeException("B0rked"); _log.error(msg, ioe);
SnarkManager.instance().addMessage(msg);
SnarkManager.instance().addMessage("Fatal storage error: Stopping torrent " + metainfo.getName());
throw new RuntimeException(msg, ioe);
} }
} }
@ -829,6 +834,8 @@ public class PeerCoordinator implements PeerListener
* Returns false if the piece is no good (according to the hash). * Returns false if the piece is no good (according to the hash).
* In that case the peer that supplied the piece should probably be * In that case the peer that supplied the piece should probably be
* blacklisted. * blacklisted.
*
* @throws RuntimeException on IOE saving the piece
*/ */
public boolean gotPiece(Peer peer, int piece, byte[] bs) public boolean gotPiece(Peer peer, int piece, byte[] bs)
{ {
@ -872,8 +879,11 @@ public class PeerCoordinator implements PeerListener
catch (IOException ioe) catch (IOException ioe)
{ {
snark.stopTorrent(); snark.stopTorrent();
_log.error("Error writing storage for " + metainfo.getName(), ioe); String msg = "Error writing storage (piece " + piece + ") for " + metainfo.getName() + ": " + ioe;
throw new RuntimeException("B0rked"); _log.error(msg, ioe);
SnarkManager.instance().addMessage(msg);
SnarkManager.instance().addMessage("Fatal storage error: Stopping torrent " + metainfo.getName());
throw new RuntimeException(msg, ioe);
} }
wantedPieces.remove(p); wantedPieces.remove(p);
} }

10
apps/i2psnark/java/src/org/klomp/snark/Snark.java
View File

@ -40,8 +40,6 @@ import net.i2p.client.streaming.I2PServerSocket;
import net.i2p.data.Destination; import net.i2p.data.Destination;
import net.i2p.util.I2PThread; import net.i2p.util.I2PThread;
import org.klomp.snark.bencode.BDecoder;
/** /**
* Main Snark program startup class. * Main Snark program startup class.
* *
@ -360,7 +358,7 @@ public class Snark
in = new FileInputStream(torrentFile); in = new FileInputStream(torrentFile);
} }
} }
meta = new MetaInfo(new BDecoder(in)); meta = new MetaInfo(in);
infoHash = meta.getInfoHash(); infoHash = meta.getInfoHash();
} }
catch(IOException ioe) catch(IOException ioe)
@ -589,7 +587,7 @@ public class Snark
pc.halt(); pc.halt();
Storage st = storage; Storage st = storage;
if (st != null) { if (st != null) {
boolean changed = storage.changed; boolean changed = storage.isChanged();
try { try {
storage.close(); storage.close();
} catch (IOException ioe) { } catch (IOException ioe) {
@ -1013,7 +1011,7 @@ public class Snark
//if (debug >= INFO && t != null) //if (debug >= INFO && t != null)
// t.printStackTrace(); // t.printStackTrace();
stopTorrent(); stopTorrent();
throw new RuntimeException(s + (t == null ? "" : ": " + t)); throw new RuntimeException(s, t);
} }
/** /**
@ -1111,7 +1109,7 @@ public class Snark
allChecked = true; allChecked = true;
checking = false; checking = false;
if (storage.changed && completeListener != null) if (storage.isChanged() && completeListener != null)
completeListener.updateStatus(this); completeListener.updateStatus(this);
} }

4
apps/i2psnark/java/src/org/klomp/snark/SnarkManager.java
View File

@ -572,6 +572,9 @@ public class SnarkManager implements Snark.CompleteListener {
} }
try { try {
// This is somewhat wasteful as this metainfo is thrown away,
// the real one is created in the Snark constructor.
// TODO: Make a Snark constructor where we pass the MetaInfo in as a parameter.
MetaInfo info = new MetaInfo(fis); MetaInfo info = new MetaInfo(fis);
try { try {
fis.close(); fis.close();
@ -622,6 +625,7 @@ public class SnarkManager implements Snark.CompleteListener {
return; return;
} catch (OutOfMemoryError oom) { } catch (OutOfMemoryError oom) {
addMessage(_("ERROR - Out of memory, cannot create torrent from {0}", sfile.getName()) + ": " + oom.getMessage()); addMessage(_("ERROR - Out of memory, cannot create torrent from {0}", sfile.getName()) + ": " + oom.getMessage());
return;
} finally { } finally {
if (fis != null) try { fis.close(); } catch (IOException ioe) {} if (fis != null) try { fis.close(); } catch (IOException ioe) {}
} }

225
apps/i2psnark/java/src/org/klomp/snark/Storage.java
View File

@ -53,10 +53,10 @@ public class Storage
private int needed; // Number of pieces needed private int needed; // Number of pieces needed
private boolean _probablyComplete; // use this to decide whether to open files RO private boolean _probablyComplete; // use this to decide whether to open files RO
// XXX - Not always set correctly private final int piece_size;
int piece_size; private final int pieces;
int pieces; private final long total_length;
boolean changed; private boolean changed;
/** The default piece size. */ /** The default piece size. */
private static final int MIN_PIECE_SIZE = 256*1024; private static final int MIN_PIECE_SIZE = 256*1024;
@ -81,6 +81,9 @@ public class Storage
needed = metainfo.getPieces(); needed = metainfo.getPieces();
_probablyComplete = false; _probablyComplete = false;
bitfield = new BitField(needed); bitfield = new BitField(needed);
piece_size = metainfo.getPieceLength(0);
pieces = needed;
total_length = metainfo.getTotalLength();
} }
/** /**
@ -108,17 +111,17 @@ public class Storage
lengthsList.add(Long.valueOf(length)); lengthsList.add(Long.valueOf(length));
} }
piece_size = MIN_PIECE_SIZE; int pc_size = MIN_PIECE_SIZE;
pieces = (int) ((total - 1)/piece_size) + 1; int pcs = (int) ((total - 1)/pc_size) + 1;
while (pieces > MAX_PIECES && piece_size < MAX_PIECE_SIZE) while (pcs > MAX_PIECES && pc_size < MAX_PIECE_SIZE)
{ {
piece_size = piece_size*2; pc_size *= 2;
pieces = (int) ((total - 1)/piece_size) +1; pcs = (int) ((total - 1)/pc_size) +1;
} }
piece_size = pc_size;
pieces = pcs;
total_length = total;
// Note that piece_hashes and the bitfield will be filled after
// the MetaInfo is created.
byte[] piece_hashes = new byte[20*pieces];
bitfield = new BitField(pieces); bitfield = new BitField(pieces);
needed = 0; needed = 0;
@ -142,69 +145,26 @@ public class Storage
lengthsList = null; lengthsList = null;
} }
// Note that the piece_hashes are not correctly setup yet. byte[] piece_hashes = fast_digestCreate();
metainfo = new MetaInfo(announce, baseFile.getName(), null, files, metainfo = new MetaInfo(announce, baseFile.getName(), null, files,
lengthsList, piece_size, piece_hashes, total); lengthsList, piece_size, piece_hashes, total);
} }
// Creates piece hashes for a new storage. /**
// This does NOT create the files, just the hashes * Creates piece hashes for a new storage.
public void create() throws IOException * This does NOT create the files, just the hashes.
{ * Also sets all the bitfield bits.
// if (true) { *
fast_digestCreate(); * FIXME we can run out of fd's doing this,
// } else {
// orig_digestCreate();
// }
}
/*
private void orig_digestCreate() throws IOException {
// Calculate piece_hashes
MessageDigest digest = null;
try
{
digest = MessageDigest.getInstance("SHA");
}
catch(NoSuchAlgorithmException nsa)
{
throw new InternalError(nsa.toString());
}
byte[] piece_hashes = metainfo.getPieceHashes();
byte[] piece = new byte[piece_size];
for (int i = 0; i < pieces; i++)
{
int length = getUncheckedPiece(i, piece);
digest.update(piece, 0, length);
byte[] hash = digest.digest();
for (int j = 0; j < 20; j++)
piece_hashes[20 * i + j] = hash[j];
bitfield.set(i);
if (listener != null)
listener.storageChecked(this, i, true);
}
if (listener != null)
listener.storageAllChecked(this);
// Reannounce to force recalculating the info_hash.
metainfo = metainfo.reannounce(metainfo.getAnnounce());
}
*/
/** FIXME we can run out of fd's doing this,
* maybe some sort of global close-RAF-right-away flag * maybe some sort of global close-RAF-right-away flag
* would do the trick */ * would do the trick
private void fast_digestCreate() throws IOException { */
private byte[] fast_digestCreate() throws IOException {
// Calculate piece_hashes // Calculate piece_hashes
SHA1 digest = new SHA1(); SHA1 digest = new SHA1();
byte[] piece_hashes = metainfo.getPieceHashes(); byte[] piece_hashes = new byte[20 * pieces];
byte[] piece = new byte[piece_size]; byte[] piece = new byte[piece_size];
for (int i = 0; i < pieces; i++) for (int i = 0; i < pieces; i++)
@ -212,14 +172,10 @@ public class Storage
int length = getUncheckedPiece(i, piece); int length = getUncheckedPiece(i, piece);
digest.update(piece, 0, length); digest.update(piece, 0, length);
byte[] hash = digest.digest(); byte[] hash = digest.digest();
for (int j = 0; j < 20; j++) System.arraycopy(hash, 0, piece_hashes, 20 * i, 20);
piece_hashes[20 * i + j] = hash[j];
bitfield.set(i); bitfield.set(i);
} }
return piece_hashes;
// Reannounce to force recalculating the info_hash.
metainfo = metainfo.reannounce(metainfo.getAnnounce());
} }
private void getFiles(File base) throws IOException private void getFiles(File base) throws IOException
@ -294,6 +250,14 @@ public class Storage
return needed == 0; return needed == 0;
} }
/**
* Has the storage changed since instantiation?
* @since 0.8.5
*/
public boolean isChanged() {
return changed;
}
/** /**
* @param file canonical path (non-directory) * @param file canonical path (non-directory)
* @return number of bytes remaining; -1 if unknown file * @return number of bytes remaining; -1 if unknown file
@ -315,14 +279,13 @@ public class Storage
if (f != null && canonical.equals(file)) { if (f != null && canonical.equals(file)) {
if (complete()) if (complete())
return 0; return 0;
int psz = metainfo.getPieceLength(0); int psz = piece_size;
long start = bytes; long start = bytes;
long end = start + lengths[i]; long end = start + lengths[i];
int pc = (int) (bytes / psz); int pc = (int) (bytes / psz);
long rv = 0; long rv = 0;
if (!bitfield.get(pc)) if (!bitfield.get(pc))
rv = Math.min(psz - (start % psz), lengths[i]); rv = Math.min(psz - (start % psz), lengths[i]);
int pieces = metainfo.getPieces();
for (int j = pc + 1; (((long)j) * psz) < end && j < pieces; j++) { for (int j = pc + 1; (((long)j) * psz) < end && j < pieces; j++) {
if (!bitfield.get(j)) { if (!bitfield.get(j)) {
if (((long)(j+1))*psz < end) if (((long)(j+1))*psz < end)
@ -418,7 +381,7 @@ public class Storage
int file = 0; int file = 0;
long pcEnd = -1; long pcEnd = -1;
long fileEnd = lengths[0] - 1; long fileEnd = lengths[0] - 1;
int psz = metainfo.getPieceLength(0); int psz = piece_size;
for (int i = 0; i < rv.length; i++) { for (int i = 0; i < rv.length; i++) {
pcEnd += psz; pcEnd += psz;
int pri = priorities[file]; int pri = priorities[file];
@ -469,7 +432,7 @@ public class Storage
File base = new SecureFile(rootDir, filterName(metainfo.getName())); File base = new SecureFile(rootDir, filterName(metainfo.getName()));
boolean useSavedBitField = savedTime > 0 && savedBitField != null; boolean useSavedBitField = savedTime > 0 && savedBitField != null;
List files = metainfo.getFiles(); List<List<String>> files = metainfo.getFiles();
if (files == null) if (files == null)
{ {
// Create base as file. // Create base as file.
@ -500,7 +463,7 @@ public class Storage
if (!base.mkdir() && !base.isDirectory()) if (!base.mkdir() && !base.isDirectory())
throw new IOException("Could not create directory " + base); throw new IOException("Could not create directory " + base);
List ls = metainfo.getLengths(); List<Long> ls = metainfo.getLengths();
int size = files.size(); int size = files.size();
long total = 0; long total = 0;
lengths = new long[size]; lengths = new long[size];
@ -511,8 +474,28 @@ public class Storage
RAFfile = new File[size]; RAFfile = new File[size];
for (int i = 0; i < size; i++) for (int i = 0; i < size; i++)
{ {
File f = createFileFromNames(base, (List)files.get(i)); List<String> path = files.get(i);
lengths[i] = ((Long)ls.get(i)).longValue(); File f = createFileFromNames(base, path);
// dup file name check after filtering
for (int j = 0; j < i; j++) {
if (f.equals(RAFfile[j])) {
// Rename and start the check over again
// Copy path since metainfo list is unmodifiable
path = new ArrayList(path);
int last = path.size() - 1;
String lastPath = path.get(last);
int dot = lastPath.lastIndexOf('.');
// foo.mp3 -> foo_.mp3; foo -> _foo
if (dot >= 0)
lastPath = lastPath.substring(0, dot) + '_' + lastPath.substring(dot);
else
lastPath = '_' + lastPath;
path.set(last, lastPath);
f = createFileFromNames(base, path);
j = 0;
}
}
lengths[i] = ls.get(i).longValue();
RAFlock[i] = new Object(); RAFlock[i] = new Object();
RAFfile[i] = f; RAFfile[i] = f;
total += lengths[i]; total += lengths[i];
@ -551,36 +534,19 @@ public class Storage
} }
/** /**
* Reopen the file descriptors for a restart * Doesn't really reopen the file descriptors for a restart.
* Do existence check but no length check or data reverification * Just does an existence check but no length check or data reverification
*
* @param rootDir ignored
* @throws IOE on fail
*/ */
public void reopen(String rootDir) throws IOException public void reopen(String rootDir) throws IOException
{ {
File base = new File(rootDir, filterName(metainfo.getName())); if (RAFfile == null)
throw new IOException("Storage not checked yet");
List files = metainfo.getFiles(); for (int i = 0; i < RAFfile.length; i++) {
if (files == null) if (!RAFfile[i].exists())
{ throw new IOException("File does not exist: " + RAFfile[i]);
// Reopen base as file.
_util.debug("Reopening file: " + base, Snark.NOTICE);
if (!base.exists())
throw new IOException("Could not reopen file " + base);
}
else
{
// Reopen base as dir.
_util.debug("Reopening directory: " + base, Snark.NOTICE);
if (!base.isDirectory())
throw new IOException("Could not reopen directory " + base);
int size = files.size();
for (int i = 0; i < size; i++)
{
File f = getFileFromNames(base, (List)files.get(i));
if (!f.exists())
throw new IOException("Could not reopen file " + f);
}
} }
} }
@ -609,13 +575,18 @@ public class Storage
return rv; return rv;
} }
private File createFileFromNames(File base, List names) throws IOException /**
* Note that filtering each path element individually may lead to
* things going in the wrong place if there are duplicates
* in intermediate path elements after filtering.
*/
private static File createFileFromNames(File base, List<String> names) throws IOException
{ {
File f = null; File f = null;
Iterator it = names.iterator(); Iterator<String> it = names.iterator();
while (it.hasNext()) while (it.hasNext())
{ {
String name = filterName((String)it.next()); String name = filterName(it.next());
if (it.hasNext()) if (it.hasNext())
{ {
// Another dir in the hierarchy. // Another dir in the hierarchy.
@ -635,12 +606,12 @@ public class Storage
return f; return f;
} }
public static File getFileFromNames(File base, List names) public static File getFileFromNames(File base, List<String> names)
{ {
Iterator it = names.iterator(); Iterator<String> it = names.iterator();
while (it.hasNext()) while (it.hasNext())
{ {
String name = filterName((String)it.next()); String name = filterName(it.next());
base = new File(base, name); base = new File(base, name);
} }
return base; return base;
@ -690,7 +661,10 @@ public class Storage
} catch (IOException ioe) {} } catch (IOException ioe) {}
} }
} else { } else {
_util.debug("File '" + names[i] + "' exists, but has wrong length - repairing corruption", Snark.ERROR); String msg = "File '" + names[i] + "' exists, but has wrong length (expected " +
lengths[i] + " but found " + length + ") - repairing corruption";
SnarkManager.instance().addMessage(msg);
_util.debug(msg, Snark.ERROR);
changed = true; changed = true;
_probablyComplete = false; // to force RW _probablyComplete = false; // to force RW
synchronized(RAFlock[i]) { synchronized(RAFlock[i]) {
@ -706,8 +680,7 @@ public class Storage
// Check which pieces match and which don't // Check which pieces match and which don't
if (resume) if (resume)
{ {
pieces = metainfo.getPieces(); byte[] piece = new byte[piece_size];
byte[] piece = new byte[metainfo.getPieceLength(0)];
int file = 0; int file = 0;
long fileEnd = lengths[0]; long fileEnd = lengths[0];
long pieceEnd = 0; long pieceEnd = 0;
@ -775,7 +748,7 @@ public class Storage
// the whole file? // the whole file?
if (listener != null) if (listener != null)
listener.storageCreateFile(this, names[nr], lengths[nr]); listener.storageCreateFile(this, names[nr], lengths[nr]);
final int ZEROBLOCKSIZE = metainfo.getPieceLength(0); final int ZEROBLOCKSIZE = piece_size;
byte[] zeros; byte[] zeros;
try { try {
zeros = new byte[ZEROBLOCKSIZE]; zeros = new byte[ZEROBLOCKSIZE];
@ -868,7 +841,7 @@ public class Storage
} }
// Early typecast, avoid possibly overflowing a temp integer // Early typecast, avoid possibly overflowing a temp integer
long start = (long) piece * (long) metainfo.getPieceLength(0); long start = (long) piece * (long) piece_size;
int i = 0; int i = 0;
long raflen = lengths[i]; long raflen = lengths[i];
while (start > raflen) while (start > raflen)
@ -935,10 +908,24 @@ public class Storage
return true; return true;
} }
/**
* This is a dup of MetaInfo.getPieceLength() but we need it
* before the MetaInfo is created in our second constructor.
* @since 0.8.5
*/
private int getPieceLength(int piece) {
if (piece >= 0 && piece < pieces -1)
return piece_size;
else if (piece == pieces -1)
return (int)(total_length - ((long)piece * piece_size));
else
throw new IndexOutOfBoundsException("no piece: " + piece);
}
private int getUncheckedPiece(int piece, byte[] bs) private int getUncheckedPiece(int piece, byte[] bs)
throws IOException throws IOException
{ {
return getUncheckedPiece(piece, bs, 0, metainfo.getPieceLength(piece)); return getUncheckedPiece(piece, bs, 0, getPieceLength(piece));
} }
private int getUncheckedPiece(int piece, byte[] bs, int off, int length) private int getUncheckedPiece(int piece, byte[] bs, int off, int length)
@ -947,7 +934,7 @@ public class Storage
// XXX - copy/paste code from putPiece(). // XXX - copy/paste code from putPiece().
// Early typecast, avoid possibly overflowing a temp integer // Early typecast, avoid possibly overflowing a temp integer
long start = ((long) piece * (long) metainfo.getPieceLength(0)) + off; long start = ((long) piece * (long) piece_size) + off;
int i = 0; int i = 0;
long raflen = lengths[i]; long raflen = lengths[i];

49
apps/i2psnark/java/src/org/klomp/snark/bencode/BDecoder.java
View File

@ -60,22 +60,30 @@ public class BDecoder
private int indicator = 0; private int indicator = 0;
// Used for ugly hack to get SHA hash over the metainfo info map // Used for ugly hack to get SHA hash over the metainfo info map
private String special_map = "info"; private final String special_map = "info";
private boolean in_special_map = false; private boolean in_special_map = false;
private final MessageDigest sha_digest; /** creation deferred until we encounter the special map, to make processing of announce replies more efficient */
private MessageDigest sha_digest;
// Ugly hack. Return the SHA has over bytes that make up the special map. /**
* Ugly hack. Return the SHA has over bytes that make up the special map.
* @return null if there was no special map
*/
public byte[] get_special_map_digest() public byte[] get_special_map_digest()
{ {
if (sha_digest == null)
return null;
byte[] result = sha_digest.digest(); byte[] result = sha_digest.digest();
return result; return result;
} }
/****
// Ugly hack. Name defaults to "info". // Ugly hack. Name defaults to "info".
public void set_special_map_name(String name) public void set_special_map_name(String name)
{ {
special_map = name; special_map = name;
} }
****/
/** /**
* Initalizes a new BDecoder. Nothing is read from the given * Initalizes a new BDecoder. Nothing is read from the given
@ -84,15 +92,6 @@ public class BDecoder
public BDecoder(InputStream in) public BDecoder(InputStream in)
{ {
this.in = in; this.in = in;
// XXX - Used for ugly hack.
try
{
sha_digest = MessageDigest.getInstance("SHA");
}
catch(NoSuchAlgorithmException nsa)
{
throw new InternalError(nsa.toString());
}
} }
/** /**
@ -112,6 +111,24 @@ public class BDecoder
return new BDecoder(in).bdecode(); return new BDecoder(in).bdecode();
} }
/**
* Used for SHA1 hack
* @since 0.8.5
*/
private void createDigest() {
if (sha_digest == null) {
try {
sha_digest = MessageDigest.getInstance("SHA");
} catch(NoSuchAlgorithmException nsa) {
throw new InternalError(nsa.toString());
}
} else {
// there are two info maps, but not one inside the other,
// the resulting hash will be incorrect
// throw something? - no, the check in the MetaInfo constructor will catch it.
}
}
/** /**
* Returns what the next bencoded object will be on the stream or -1 * Returns what the next bencoded object will be on the stream or -1
* when the end of stream has been reached. Can return something * when the end of stream has been reached. Can return something
@ -294,9 +311,13 @@ public class BDecoder
String key = bdecode().getString(); String key = bdecode().getString();
// XXX ugly hack // XXX ugly hack
boolean special = special_map.equals(key); // This will not screw up if an info map contains an info map,
if (special) // but it will if there are two info maps (not one inside the other)
boolean special = (!in_special_map) && special_map.equals(key);
if (special) {
createDigest();
in_special_map = true; in_special_map = true;
}
BEValue value = bdecode(); BEValue value = bdecode();
result.put(key, value); result.put(key, value);

8
apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java
View File

@ -643,7 +643,6 @@ public class I2PSnarkServlet extends Default {
// This may take a long time to check the storage, but since it already exists, // This may take a long time to check the storage, but since it already exists,
// it shouldn't be THAT bad, so keep it in this thread. // it shouldn't be THAT bad, so keep it in this thread.
Storage s = new Storage(_manager.util(), baseFile, announceURL, null); Storage s = new Storage(_manager.util(), baseFile, announceURL, null);
s.create();
s.close(); // close the files... maybe need a way to pass this Storage to addTorrent rather than starting over s.close(); // close the files... maybe need a way to pass this Storage to addTorrent rather than starting over
MetaInfo info = s.getMetaInfo(); MetaInfo info = s.getMetaInfo();
File torrentFile = new File(_manager.getDataDir(), s.getBaseName() + ".torrent"); File torrentFile = new File(_manager.getDataDir(), s.getBaseName() + ".torrent");
@ -1968,16 +1967,15 @@ private static class FetchAndAdd implements Runnable {
FileInputStream in = null; FileInputStream in = null;
try { try {
in = new FileInputStream(file); in = new FileInputStream(file);
// we do not retain this MetaInfo object, hopefully it will go away quickly byte[] fileInfoHash = new byte[20];
MetaInfo info = new MetaInfo(in); String name = MetaInfo.getNameAndInfoHash(in, fileInfoHash);
try { in.close(); } catch (IOException ioe) {} try { in.close(); } catch (IOException ioe) {}
Snark snark = _manager.getTorrentByInfoHash(info.getInfoHash()); Snark snark = _manager.getTorrentByInfoHash(fileInfoHash);
if (snark != null) { if (snark != null) {
_manager.addMessage(_("Torrent with this info hash is already running: {0}", snark.getBaseName())); _manager.addMessage(_("Torrent with this info hash is already running: {0}", snark.getBaseName()));
return; return;
} }
String name = info.getName();
name = Storage.filterName(name); name = Storage.filterName(name);
name = name + ".torrent"; name = name + ".torrent";
File torrentFile = new File(_manager.getDataDir(), name); File torrentFile = new File(_manager.getDataDir(), name);