* i2psnark:

- Refactor Storage file data structures
   - Sort files when creating torrents
This commit is contained in:
zzz
2013-09-18 15:41:10 +00:00
parent ab8900f910
commit a80c34c1df

View File

@ -28,6 +28,7 @@ import java.nio.charset.Charset;
import java.nio.charset.CharsetEncoder; import java.nio.charset.CharsetEncoder;
import java.security.MessageDigest; import java.security.MessageDigest;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collections;
import java.util.Iterator; import java.util.Iterator;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
@ -50,17 +51,7 @@ import net.i2p.util.SystemVersion;
public class Storage public class Storage
{ {
private final MetaInfo metainfo; private final MetaInfo metainfo;
private long[] lengths; private final List<TorrentFile> _torrentFiles;
private RandomAccessFile[] rafs;
private String[] names;
private Object[] RAFlock; // lock on RAF access
private long[] RAFtime; // when was RAF last accessed, or 0 if closed
private File[] RAFfile; // File to make it easier to reopen
/** priorities by file; default 0; may be null. @since 0.8.1 */
private int[] priorities;
/** is the file empty and sparse? */
private boolean[] isSparse;
private final StorageListener listener; private final StorageListener listener;
private final I2PSnarkUtil _util; private final I2PSnarkUtil _util;
private final Log _log; private final Log _log;
@ -108,6 +99,9 @@ public class Storage
piece_size = metainfo.getPieceLength(0); piece_size = metainfo.getPieceLength(0);
pieces = needed; pieces = needed;
total_length = metainfo.getTotalLength(); total_length = metainfo.getTotalLength();
List<List<String>> files = metainfo.getFiles();
int sz = files != null ? files.size() : 1;
_torrentFiles = new ArrayList(sz);
} }
/** /**
@ -130,13 +124,13 @@ public class Storage
_log = util.getContext().logManager().getLog(Storage.class); _log = util.getContext().logManager().getLog(Storage.class);
this.listener = listener; this.listener = listener;
// Create names, rafs and lengths arrays. // Create names, rafs and lengths arrays.
getFiles(baseFile); _torrentFiles = getFiles(baseFile);
long total = 0; long total = 0;
ArrayList<Long> lengthsList = new ArrayList(); ArrayList<Long> lengthsList = new ArrayList();
for (int i = 0; i < lengths.length; i++) for (TorrentFile tf : _torrentFiles)
{ {
long length = lengths[i]; long length = tf.length;
total += length; total += length;
lengthsList.add(Long.valueOf(length)); lengthsList.add(Long.valueOf(length));
} }
@ -167,10 +161,10 @@ public class Storage
needed = 0; needed = 0;
List<List<String>> files = new ArrayList(); List<List<String>> files = new ArrayList();
for (int i = 0; i < names.length; i++) for (TorrentFile tf : _torrentFiles)
{ {
List<String> file = new ArrayList(); List<String> file = new ArrayList();
StringTokenizer st = new StringTokenizer(names[i], File.separator); StringTokenizer st = new StringTokenizer(tf.name, File.separator);
while (st.hasMoreTokens()) while (st.hasMoreTokens())
{ {
String part = st.nextToken(); String part = st.nextToken();
@ -220,42 +214,29 @@ public class Storage
return piece_hashes; return piece_hashes;
} }
private void getFiles(File base) throws IOException private List<TorrentFile> getFiles(File base) throws IOException
{ {
if (base.getAbsolutePath().equals("/")) if (base.getAbsolutePath().equals("/"))
throw new IOException("Don't seed root"); throw new IOException("Don't seed root");
ArrayList files = new ArrayList(); List<File> files = new ArrayList();
addFiles(files, base); addFiles(files, base);
int size = files.size(); int size = files.size();
names = new String[size]; List<TorrentFile> rv = new ArrayList(size);
lengths = new long[size];
rafs = new RandomAccessFile[size];
RAFlock = new Object[size];
RAFtime = new long[size];
RAFfile = new File[size];
priorities = new int[size];
isSparse = new boolean[size];
int i = 0; for (File f : files) {
Iterator it = files.iterator(); rv.add(new TorrentFile(base, f));
while (it.hasNext()) }
{ // Sort to prevent exposing OS type, and to make it more likely
File f = (File)it.next(); // the same torrent created twice will have the same infohash.
names[i] = f.getPath(); Collections.sort(rv);
if (base.isDirectory() && names[i].startsWith(base.getPath())) return rv;
names[i] = names[i].substring(base.getPath().length() + 1);
lengths[i] = f.length();
RAFlock[i] = new Object();
RAFfile[i] = f;
i++;
}
} }
/** /**
* @throws IOException if too many total files * @throws IOException if too many total files
*/ */
private void addFiles(List l, File f) throws IOException { private void addFiles(List<File> l, File f) throws IOException {
if (!f.isDirectory()) { if (!f.isDirectory()) {
if (l.size() >= SnarkManager.MAX_FILES_PER_TORRENT) if (l.size() >= SnarkManager.MAX_FILES_PER_TORRENT)
throw new IOException("Too many files, limit is " + SnarkManager.MAX_FILES_PER_TORRENT + ", zip them?"); throw new IOException("Too many files, limit is " + SnarkManager.MAX_FILES_PER_TORRENT + ", zip them?");
@ -330,8 +311,8 @@ public class Storage
*/ */
public long remaining(String file) { public long remaining(String file) {
long bytes = 0; long bytes = 0;
for (int i = 0; i < rafs.length; i++) { for (TorrentFile tf : _torrentFiles) {
File f = RAFfile[i]; File f = tf.RAFfile;
// use canonical in case snark dir or sub dirs are symlinked // use canonical in case snark dir or sub dirs are symlinked
String canonical = null; String canonical = null;
if (f != null) { if (f != null) {
@ -346,11 +327,11 @@ public class Storage
return 0; return 0;
int psz = piece_size; int psz = piece_size;
long start = bytes; long start = bytes;
long end = start + lengths[i]; long end = start + tf.length;
int pc = (int) (bytes / psz); int pc = (int) (bytes / psz);
long rv = 0; long rv = 0;
if (!bitfield.get(pc)) if (!bitfield.get(pc))
rv = Math.min(psz - (start % psz), lengths[i]); rv = Math.min(psz - (start % psz), tf.length);
for (int j = pc + 1; (((long)j) * psz) < end && j < pieces; j++) { for (int j = pc + 1; (((long)j) * psz) < end && j < pieces; j++) {
if (!bitfield.get(j)) { if (!bitfield.get(j)) {
if (((long)(j+1))*psz < end) if (((long)(j+1))*psz < end)
@ -361,7 +342,7 @@ public class Storage
} }
return rv; return rv;
} }
bytes += lengths[i]; bytes += tf.length;
} }
return -1; return -1;
} }
@ -371,16 +352,16 @@ public class Storage
* @since 0.8.1 * @since 0.8.1
*/ */
public int getPriority(String file) { public int getPriority(String file) {
if (complete() || metainfo.getFiles() == null || priorities == null) if (complete() || metainfo.getFiles() == null)
return 0; return 0;
for (int i = 0; i < rafs.length; i++) { for (TorrentFile tf : _torrentFiles) {
File f = RAFfile[i]; File f = tf.RAFfile;
// use canonical in case snark dir or sub dirs are symlinked // use canonical in case snark dir or sub dirs are symlinked
if (f != null) { if (f != null) {
try { try {
String canonical = f.getCanonicalPath(); String canonical = f.getCanonicalPath();
if (canonical.equals(file)) if (canonical.equals(file))
return priorities[i]; return tf.priority;
} catch (IOException ioe) {} } catch (IOException ioe) {}
} }
} }
@ -395,16 +376,16 @@ public class Storage
* @since 0.8.1 * @since 0.8.1
*/ */
public void setPriority(String file, int pri) { public void setPriority(String file, int pri) {
if (complete() || metainfo.getFiles() == null || priorities == null) if (complete() || metainfo.getFiles() == null)
return; return;
for (int i = 0; i < rafs.length; i++) { for (TorrentFile tf : _torrentFiles) {
File f = RAFfile[i]; File f = tf.RAFfile;
// use canonical in case snark dir or sub dirs are symlinked // use canonical in case snark dir or sub dirs are symlinked
if (f != null) { if (f != null) {
try { try {
String canonical = f.getCanonicalPath(); String canonical = f.getCanonicalPath();
if (canonical.equals(file)) { if (canonical.equals(file)) {
priorities[i] = pri; tf.priority = pri;
return; return;
} }
} catch (IOException ioe) {} } catch (IOException ioe) {}
@ -418,6 +399,15 @@ public class Storage
* @since 0.8.1 * @since 0.8.1
*/ */
public int[] getFilePriorities() { public int[] getFilePriorities() {
if (complete())
return null;
int sz = _torrentFiles.size();
if (sz <= 1)
return null;
int[] priorities = new int[sz];
for (int i = 0; i < sz; i++) {
priorities[i] = _torrentFiles.get(i).priority;
}
return priorities; return priorities;
} }
@ -428,7 +418,18 @@ public class Storage
* @since 0.8.1 * @since 0.8.1
*/ */
void setFilePriorities(int[] p) { void setFilePriorities(int[] p) {
priorities = p; if (p == null) {
for (TorrentFile tf : _torrentFiles) {
tf.priority = 0;
}
} else {
int sz = _torrentFiles.size();
if (p.length != sz)
throw new IllegalArgumentException();
for (int i = 0; i < sz; i++) {
_torrentFiles.get(i).priority = p[i];
}
}
} }
/** /**
@ -441,22 +442,23 @@ public class Storage
* @since 0.8.1 * @since 0.8.1
*/ */
public int[] getPiecePriorities() { public int[] getPiecePriorities() {
if (complete() || metainfo.getFiles() == null || priorities == null) if (complete() || metainfo.getFiles() == null)
return null; return null;
int[] rv = new int[metainfo.getPieces()]; int[] rv = new int[metainfo.getPieces()];
int file = 0; int file = 0;
long pcEnd = -1; long pcEnd = -1;
long fileEnd = lengths[0] - 1; long fileEnd = _torrentFiles.get(0).length - 1;
int psz = piece_size; int psz = piece_size;
for (int i = 0; i < rv.length; i++) { for (int i = 0; i < rv.length; i++) {
pcEnd += psz; pcEnd += psz;
int pri = priorities[file]; int pri = _torrentFiles.get(file).priority;
while (fileEnd <= pcEnd && file < lengths.length - 1) { while (fileEnd <= pcEnd && file < _torrentFiles.size() - 1) {
file++; file++;
TorrentFile tf = _torrentFiles.get(file);
long oldFileEnd = fileEnd; long oldFileEnd = fileEnd;
fileEnd += lengths[file]; fileEnd += tf.length;
if (priorities[file] > pri && oldFileEnd < pcEnd) if (tf.priority > pri && oldFileEnd < pcEnd)
pri = priorities[file]; pri = tf.priority;
} }
rv[i] = pri; rv[i] = pri;
} }
@ -486,13 +488,18 @@ public class Storage
/** /**
* Creates (and/or checks) all files from the metainfo file list. * Creates (and/or checks) all files from the metainfo file list.
* Only call this once, and only after the constructor with the metainfo.
*/ */
public void check(String rootDir) throws IOException public void check(String rootDir) throws IOException
{ {
check(rootDir, 0, null); check(rootDir, 0, null);
} }
/** use a saved bitfield and timestamp from a config file */ /**
* Creates (and/or checks) all files from the metainfo file list.
* Use a saved bitfield and timestamp from a config file.
* Only call this once, and only after the constructor with the metainfo.
*/
public void check(String rootDir, long savedTime, BitField savedBitField) throws IOException public void check(String rootDir, long savedTime, BitField savedBitField) throws IOException
{ {
File base; File base;
@ -503,6 +510,8 @@ public class Storage
base = new SecureFile(rootDir, filterName(metainfo.getName())); base = new SecureFile(rootDir, filterName(metainfo.getName()));
boolean useSavedBitField = savedTime > 0 && savedBitField != null; boolean useSavedBitField = savedTime > 0 && savedBitField != null;
if (!_torrentFiles.isEmpty())
throw new IllegalStateException();
List<List<String>> files = metainfo.getFiles(); List<List<String>> files = metainfo.getFiles();
if (files == null) if (files == null)
{ {
@ -512,22 +521,12 @@ public class Storage
if (!base.createNewFile() && !base.exists()) if (!base.createNewFile() && !base.exists())
throw new IOException("Could not create file " + base); throw new IOException("Could not create file " + base);
lengths = new long[1]; _torrentFiles.add(new TorrentFile(base, base, metainfo.getTotalLength()));
rafs = new RandomAccessFile[1];
names = new String[1];
RAFlock = new Object[1];
RAFtime = new long[1];
RAFfile = new File[1];
isSparse = new boolean[1];
lengths[0] = metainfo.getTotalLength();
RAFlock[0] = new Object();
RAFfile[0] = base;
if (useSavedBitField) { if (useSavedBitField) {
long lm = base.lastModified(); long lm = base.lastModified();
if (lm <= 0 || lm > savedTime) if (lm <= 0 || lm > savedTime)
useSavedBitField = false; useSavedBitField = false;
} }
names[0] = base.getName();
} }
else else
{ {
@ -540,20 +539,13 @@ public class Storage
List<Long> ls = metainfo.getLengths(); List<Long> ls = metainfo.getLengths();
int size = files.size(); int size = files.size();
long total = 0; long total = 0;
lengths = new long[size];
rafs = new RandomAccessFile[size];
names = new String[size];
RAFlock = new Object[size];
RAFtime = new long[size];
RAFfile = new File[size];
isSparse = new boolean[size];
for (int i = 0; i < size; i++) for (int i = 0; i < size; i++)
{ {
List<String> path = files.get(i); List<String> path = files.get(i);
File f = createFileFromNames(base, path, areFilesPublic); File f = createFileFromNames(base, path, areFilesPublic);
// dup file name check after filtering // dup file name check after filtering
for (int j = 0; j < i; j++) { for (int j = 0; j < i; j++) {
if (f.equals(RAFfile[j])) { if (f.equals(_torrentFiles.get(j).RAFfile)) {
// Rename and start the check over again // Rename and start the check over again
// Copy path since metainfo list is unmodifiable // Copy path since metainfo list is unmodifiable
path = new ArrayList(path); path = new ArrayList(path);
@ -570,16 +562,14 @@ public class Storage
j = 0; j = 0;
} }
} }
lengths[i] = ls.get(i).longValue(); long len = ls.get(i).longValue();
RAFlock[i] = new Object(); _torrentFiles.add(new TorrentFile(base, f, len));
RAFfile[i] = f; total += len;
total += lengths[i];
if (useSavedBitField) { if (useSavedBitField) {
long lm = f.lastModified(); long lm = f.lastModified();
if (lm <= 0 || lm > savedTime) if (lm <= 0 || lm > savedTime)
useSavedBitField = false; useSavedBitField = false;
} }
names[i] = f.getName();
} }
// Sanity check for metainfo file. // Sanity check for metainfo file.
@ -604,8 +594,6 @@ public class Storage
_log.info("Torrent is complete"); _log.info("Torrent is complete");
} else { } else {
// fixme saved priorities // fixme saved priorities
if (files != null)
priorities = new int[files.size()];
if (_log.shouldLog(Log.INFO)) if (_log.shouldLog(Log.INFO))
_log.info("Still need " + needed + " out of " + metainfo.getPieces() + " pieces"); _log.info("Still need " + needed + " out of " + metainfo.getPieces() + " pieces");
} }
@ -620,11 +608,11 @@ public class Storage
*/ */
public void reopen(String rootDir) throws IOException public void reopen(String rootDir) throws IOException
{ {
if (RAFfile == null) if (_torrentFiles.isEmpty())
throw new IOException("Storage not checked yet"); throw new IOException("Storage not checked yet");
for (int i = 0; i < RAFfile.length; i++) { for (TorrentFile tf : _torrentFiles) {
if (!RAFfile[i].exists()) if (!tf.RAFfile.exists())
throw new IOException("File does not exist: " + RAFfile[i]); throw new IOException("File does not exist: " + tf);
} }
} }
@ -778,10 +766,10 @@ public class Storage
// Make sure all files are available and of correct length // Make sure all files are available and of correct length
// The files should all exist as they have been created with zero length by createFilesFromNames() // The files should all exist as they have been created with zero length by createFilesFromNames()
for (int i = 0; i < rafs.length; i++) for (TorrentFile tf : _torrentFiles)
{ {
long length = RAFfile[i].length(); long length = tf.RAFfile.length();
if(RAFfile[i].exists() && length == lengths[i]) if(tf.RAFfile.exists() && length == tf.length)
{ {
if (listener != null) if (listener != null)
listener.storageAllocated(this, length); listener.storageAllocated(this, length);
@ -789,27 +777,27 @@ public class Storage
} }
else if (length == 0) { else if (length == 0) {
changed = true; changed = true;
synchronized(RAFlock[i]) { synchronized(tf) {
allocateFile(i); allocateFile(tf);
// close as we go so we don't run out of file descriptors // close as we go so we don't run out of file descriptors
try { try {
closeRAF(i); tf.closeRAF();
} catch (IOException ioe) {} } catch (IOException ioe) {}
} }
} else { } else {
String msg = "File '" + names[i] + "' exists, but has wrong length (expected " + String msg = "File '" + tf.name + "' exists, but has wrong length (expected " +
lengths[i] + " but found " + length + ") - repairing corruption"; tf.length + " but found " + length + ") - repairing corruption";
if (listener != null) if (listener != null)
listener.addMessage(msg); listener.addMessage(msg);
_log.error(msg); _log.error(msg);
changed = true; changed = true;
resume = true; resume = true;
_probablyComplete = false; // to force RW _probablyComplete = false; // to force RW
synchronized(RAFlock[i]) { synchronized(tf) {
checkRAF(i); RandomAccessFile raf = tf.checkRAF();
rafs[i].setLength(lengths[i]); raf.setLength(tf.length);
try { try {
closeRAF(i); tf.closeRAF();
} catch (IOException ioe) {} } catch (IOException ioe) {}
} }
} }
@ -820,7 +808,7 @@ public class Storage
{ {
byte[] piece = new byte[piece_size]; byte[] piece = new byte[piece_size];
int file = 0; int file = 0;
long fileEnd = lengths[0]; long fileEnd = _torrentFiles.get(0).length;
long pieceEnd = 0; long pieceEnd = 0;
for (int i = 0; i < pieces; i++) for (int i = 0; i < pieces; i++)
{ {
@ -829,14 +817,15 @@ public class Storage
// close as we go so we don't run out of file descriptors // close as we go so we don't run out of file descriptors
pieceEnd += length; pieceEnd += length;
while (fileEnd <= pieceEnd) { while (fileEnd <= pieceEnd) {
synchronized(RAFlock[file]) { TorrentFile tf = _torrentFiles.get(file);
synchronized(tf) {
try { try {
closeRAF(file); tf.closeRAF();
} catch (IOException ioe) {} } catch (IOException ioe) {}
} }
if (++file >= rafs.length) if (++file >= _torrentFiles.size())
break; break;
fileEnd += lengths[file]; fileEnd += tf.length;
} }
if (correctHash) if (correctHash)
{ {
@ -882,59 +871,17 @@ public class Storage
* Sets isSparse[nr] = true. balloonFile(nr) should be called later to * Sets isSparse[nr] = true. balloonFile(nr) should be called later to
* defrag the file. * defrag the file.
* *
* This calls openRAF(); caller must synchronize and call closeRAF(). * This calls OpenRAF(); caller must synchronize and call closeRAF().
*/ */
private void allocateFile(int nr) throws IOException private void allocateFile(TorrentFile tf) throws IOException
{ {
// caller synchronized // caller synchronized
openRAF(nr, false); // RW tf.allocateFile();
long remaining = lengths[nr]; if (listener != null) {
if (listener != null) listener.storageCreateFile(this, tf.name, tf.length);
listener.storageCreateFile(this, names[nr], remaining); listener.storageAllocated(this, tf.length);
rafs[nr].setLength(remaining);
// don't bother ballooning later on Windows since there is no sparse file support
// until JDK7 using the JSR-203 interface.
// RAF seeks/writes do not create sparse files.
// Windows will zero-fill up to the point of the write, which
// will make the file fairly unfragmented, on average, at least until
// near the end where it will get exponentially more fragmented.
if (!_isWindows)
isSparse[nr] = true;
// caller will close rafs[nr]
if (listener != null)
listener.storageAllocated(this, lengths[nr]);
}
/**
* This "balloons" the file with zeros to eliminate disk fragmentation.,
* Overwrites the entire file with zeros. Sets isSparse[nr] = false.
*
* Caller must synchronize and call checkRAF() or openRAF().
* @since 0.9.1
*/
private void balloonFile(int nr) throws IOException
{
if (_log.shouldLog(Log.INFO))
_log.info("Ballooning " + nr + ": " + RAFfile[nr]);
long remaining = lengths[nr];
final int ZEROBLOCKSIZE = (int) Math.min(remaining, 32*1024);
byte[] zeros = new byte[ZEROBLOCKSIZE];
rafs[nr].seek(0);
// don't bother setting flag for small files
if (remaining > 20*1024*1024)
_allocateCount.incrementAndGet();
try {
while (remaining > 0) {
int size = (int) Math.min(remaining, ZEROBLOCKSIZE);
rafs[nr].write(zeros, 0, size);
remaining -= size;
}
} finally {
remaining = lengths[nr];
if (remaining > 20*1024*1024)
_allocateCount.decrementAndGet();
} }
isSparse[nr] = false; // caller will close rafs[nr]
} }
@ -944,18 +891,14 @@ public class Storage
*/ */
public void close() throws IOException public void close() throws IOException
{ {
if (rafs == null) return; for (TorrentFile tf : _torrentFiles)
for (int i = 0; i < rafs.length; i++)
{ {
// if we had an IOE in check(), the RAFlock may be null
if (RAFlock[i] == null)
continue;
try { try {
synchronized(RAFlock[i]) { synchronized(tf) {
closeRAF(i); tf.closeRAF();
} }
} catch (IOException ioe) { } catch (IOException ioe) {
_log.error("Error closing " + RAFfile[i], ioe); _log.error("Error closing " + tf, ioe);
// gobble gobble // gobble gobble
} }
} }
@ -1020,11 +963,11 @@ public class Storage
// Early typecast, avoid possibly overflowing a temp integer // Early typecast, avoid possibly overflowing a temp integer
long start = (long) piece * (long) piece_size; long start = (long) piece * (long) piece_size;
int i = 0; int i = 0;
long raflen = lengths[i]; long raflen = _torrentFiles.get(i).length;
while (start > raflen) { while (start > raflen) {
i++; i++;
start -= raflen; start -= raflen;
raflen = lengths[i]; raflen = _torrentFiles.get(i).length;
} }
int written = 0; int written = 0;
@ -1033,27 +976,31 @@ public class Storage
while (written < length) { while (written < length) {
int need = length - written; int need = length - written;
int len = (start + need < raflen) ? need : (int)(raflen - start); int len = (start + need < raflen) ? need : (int)(raflen - start);
synchronized(RAFlock[i]) { TorrentFile tf = _torrentFiles.get(i);
checkRAF(i); synchronized(tf) {
if (isSparse[i]) { RandomAccessFile raf = tf.checkRAF();
if (tf.isSparse) {
// If the file is a newly created sparse file, // If the file is a newly created sparse file,
// AND we aren't skipping it, balloon it with all // AND we aren't skipping it, balloon it with all
// zeros to un-sparse it by allocating the space. // zeros to un-sparse it by allocating the space.
// Obviously this could take a while. // Obviously this could take a while.
// Once we have written to it, it isn't empty/sparse any more. // Once we have written to it, it isn't empty/sparse any more.
if (priorities == null || priorities[i] >= 0) if (tf.priority >= 0) {
balloonFile(i); if (_log.shouldLog(Log.INFO))
else _log.info("Ballooning " + tf);
isSparse[i] = false; tf.balloonFile();
} else {
tf.isSparse = false;
}
} }
rafs[i].seek(start); raf.seek(start);
//rafs[i].write(bs, off + written, len); //rafs[i].write(bs, off + written, len);
pp.write(rafs[i], off + written, len); pp.write(raf, off + written, len);
} }
written += len; written += len;
if (need - len > 0) { if (need - len > 0) {
i++; i++;
raflen = lengths[i]; raflen = tf.length;
start = 0; start = 0;
} }
} }
@ -1130,12 +1077,12 @@ public class Storage
long start = ((long) piece * (long) piece_size) + off; long start = ((long) piece * (long) piece_size) + off;
int i = 0; int i = 0;
long raflen = lengths[i]; long raflen = _torrentFiles.get(i).length;
while (start > raflen) while (start > raflen)
{ {
i++; i++;
start -= raflen; start -= raflen;
raflen = lengths[i]; raflen = _torrentFiles.get(i).length;
} }
int read = 0; int read = 0;
@ -1143,17 +1090,18 @@ public class Storage
{ {
int need = length - read; int need = length - read;
int len = (start + need < raflen) ? need : (int)(raflen - start); int len = (start + need < raflen) ? need : (int)(raflen - start);
synchronized(RAFlock[i]) TorrentFile tf = _torrentFiles.get(i);
synchronized(tf)
{ {
checkRAF(i); RandomAccessFile raf = tf.checkRAF();
rafs[i].seek(start); raf.seek(start);
rafs[i].readFully(bs, read, len); raf.readFully(bs, read, len);
} }
read += len; read += len;
if (need - len > 0) if (need - len > 0)
{ {
i++; i++;
raflen = lengths[i]; raflen = _torrentFiles.get(i).length;
start = 0; start = 0;
} }
} }
@ -1161,58 +1109,182 @@ public class Storage
return length; return length;
} }
private static final long RAFCloseDelay = 4*60*1000;
/** /**
* Close unused RAFs - call periodically * Close unused RAFs - call periodically
*/ */
private static final long RAFCloseDelay = 4*60*1000;
public void cleanRAFs() { public void cleanRAFs() {
long cutoff = System.currentTimeMillis() - RAFCloseDelay; long cutoff = System.currentTimeMillis() - RAFCloseDelay;
for (int i = 0; i < RAFlock.length; i++) { for (TorrentFile tf : _torrentFiles) {
synchronized(RAFlock[i]) { synchronized(tf) {
if (RAFtime[i] > 0 && RAFtime[i] < cutoff) { tf.closeRAF(cutoff);
try {
closeRAF(i);
} catch (IOException ioe) {}
}
} }
} }
} }
/*
* For each of the following,
* caller must synchronize on RAFlock[i]
* ... except at the beginning if you're careful
*/
/** /**
* This must be called before using the RAF to ensure it is open * A single file in a torrent.
* @since 0.9.9
*/ */
private void checkRAF(int i) throws IOException { private class TorrentFile implements Comparable<TorrentFile> {
if (RAFtime[i] > 0) { public final long length;
RAFtime[i] = System.currentTimeMillis(); public final String name;
return; public final File RAFfile;
} /**
openRAF(i); * when was RAF last accessed, or 0 if closed
} * locking: this
*/
private long RAFtime;
/**
* null when closed
* locking: this
*/
private RandomAccessFile raf;
/**
* is the file empty and sparse?
* locking: this
*/
public boolean isSparse;
/** priority by file; default 0 */
public volatile int priority;
private void openRAF(int i) throws IOException { /**
openRAF(i, _probablyComplete); * For new metainfo from files;
} * use base == f for single-file torrent
*/
public TorrentFile(File base, File f) {
this(base, f, f.length());
}
private void openRAF(int i, boolean readonly) throws IOException { /**
rafs[i] = new RandomAccessFile(RAFfile[i], (readonly || !RAFfile[i].canWrite()) ? "r" : "rw"); * For existing metainfo with specified file length;
RAFtime[i] = System.currentTimeMillis(); * use base == f for single-file torrent
} */
public TorrentFile(File base, File f, long len) {
String n = f.getPath();
if (base.isDirectory() && n.startsWith(base.getPath()))
n = n.substring(base.getPath().length() + 1);
name = n;
length = len;
RAFfile = f;
}
/** /*
* Can be called even if not open * For each of the following,
*/ * caller must synchronize on RAFlock[i]
private void closeRAF(int i) throws IOException { * ... except at the beginning if you're careful
RAFtime[i] = 0; */
if (rafs[i] == null)
return; /**
rafs[i].close(); * This must be called before using the RAF to ensure it is open
rafs[i] = null; * locking: this
*/
public RandomAccessFile checkRAF() throws IOException {
if (RAFtime > 0)
RAFtime = System.currentTimeMillis();
else
openRAF();
return raf;
}
/**
* locking: this
*/
private void openRAF() throws IOException {
openRAF(_probablyComplete);
}
/**
* locking: this
*/
private void openRAF(boolean readonly) throws IOException {
raf = new RandomAccessFile(RAFfile, (readonly || !RAFfile.canWrite()) ? "r" : "rw");
RAFtime = System.currentTimeMillis();
}
/**
* Close if last used time older than cutoff.
* locking: this
*/
public void closeRAF(long cutoff) {
if (RAFtime > 0 && RAFtime < cutoff) {
try {
closeRAF();
} catch (IOException ioe) {}
}
}
/**
* Can be called even if not open
* locking: this
*/
public void closeRAF() throws IOException {
RAFtime = 0;
if (raf == null)
return;
raf.close();
raf = null;
}
/**
* This creates a (presumably) sparse file so that reads won't fail with IOE.
* Sets isSparse[nr] = true. balloonFile(nr) should be called later to
* defrag the file.
*
* This calls openRAF(); caller must synchronize and call closeRAF().
*/
public void allocateFile() throws IOException {
// caller synchronized
openRAF(false); // RW
raf.setLength(length);
// don't bother ballooning later on Windows since there is no sparse file support
// until JDK7 using the JSR-203 interface.
// RAF seeks/writes do not create sparse files.
// Windows will zero-fill up to the point of the write, which
// will make the file fairly unfragmented, on average, at least until
// near the end where it will get exponentially more fragmented.
if (!_isWindows)
isSparse = true;
}
/**
* This "balloons" the file with zeros to eliminate disk fragmentation.,
* Overwrites the entire file with zeros. Sets isSparse[nr] = false.
*
* Caller must synchronize and call checkRAF() or openRAF().
* @since 0.9.1
*/
public void balloonFile() throws IOException
{
long remaining = length;
final int ZEROBLOCKSIZE = (int) Math.min(remaining, 32*1024);
byte[] zeros = new byte[ZEROBLOCKSIZE];
raf.seek(0);
// don't bother setting flag for small files
if (remaining > 20*1024*1024)
_allocateCount.incrementAndGet();
try {
while (remaining > 0) {
int size = (int) Math.min(remaining, ZEROBLOCKSIZE);
raf.write(zeros, 0, size);
remaining -= size;
}
} finally {
remaining = length;
if (remaining > 20*1024*1024)
_allocateCount.decrementAndGet();
}
isSparse = false;
}
public int compareTo(TorrentFile tf) {
return name.compareTo(tf.name);
}
@Override
public String toString() { return name; }
} }
/** /**