echelon/i2p.i2p
1
0
Fork 0
forked from I2P_Developers/i2p.i2p
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

propagate from branch 'i2p.i2p.zzz.test4' (head 50c95d70238ff224e57acfca3d8797990ff3d01d)

Browse Source 
to branch 'i2p.i2p' (head 1fa62c230eba5af7849024338fecf51bb2719c4e)
This commit is contained in:
zzz
2011-06-30 12:22:12 +00:00
parent 14e7a8eb12 01b4b227ae
commit 3a2e9ad5d3
11 changed files with 379 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files

65
core/java/src/net/i2p/client/naming/BlockfileNamingService.java
View File

@ -33,6 +33,7 @@ import net.i2p.data.Hash;
import net.i2p.util.Log;
import net.i2p.util.SecureFileOutputStream;
import net.metanotion.io.RAIFile;
import net.metanotion.io.Serializer;
import net.metanotion.io.block.BlockFile;
import net.metanotion.io.data.UTF8StringBytes;
@ -76,10 +77,11 @@ import net.metanotion.util.skiplist.SkipList;
public class BlockfileNamingService extends DummyNamingService {
private final BlockFile _bf;
private final RandomAccessFile _raf;
private final RAIFile _raf;
private final List<String> _lists;
private final List<InvalidEntry> _invalid;
private volatile boolean _isClosed;
private final boolean _readOnly;
private static final Serializer _infoSerializer = new PropertiesSerializer();
private static final Serializer _stringSerializer = new UTF8StringBytes();
@ -87,6 +89,7 @@ public class BlockfileNamingService extends DummyNamingService {
private static final String HOSTS_DB = "hostsdb.blockfile";
private static final String FALLBACK_LIST = "hosts.txt";
private static final String PROP_FORCE = "i2p.naming.blockfile.writeInAppContext";
private static final String INFO_SKIPLIST = "%%__INFO__%%";
private static final String PROP_INFO = "info";
@ -100,6 +103,13 @@ public class BlockfileNamingService extends DummyNamingService {
private static final String PROP_SOURCE = "s";
/**
* Opens the database at hostsdb.blockfile or creates a new
* one and imports entries from hosts.txt, userhosts.txt, and privatehosts.txt.
*
* If not in router context, the database will be opened read-only
* unless the property i2p.naming.blockfile.writeInAppContext is true.
* Not designed for simultaneous access by multiple processes.
*
* @throws RuntimeException on fatal error
*/
public BlockfileNamingService(I2PAppContext context) {
@ -107,14 +117,31 @@ public class BlockfileNamingService extends DummyNamingService {
_lists = new ArrayList();
_invalid = new ArrayList();
BlockFile bf = null;
RandomAccessFile raf = null;
RAIFile raf = null;
boolean readOnly = false;
File f = new File(_context.getRouterDir(), HOSTS_DB);
if (f.exists()) {
try {
// closing a BlockFile does not close the underlying file,
// so we must create and retain a RAF so we may close it later
raf = new RandomAccessFile(f, "rw");
// *** Open readonly if not in router context (unless forced)
readOnly = (!f.canWrite()) ||
((!context.isRouterContext()) && (!context.getBooleanProperty(PROP_FORCE)));
raf = new RAIFile(f, true, !readOnly);
bf = initExisting(raf);
if (readOnly && context.isRouterContext())
_log.logAlways(Log.WARN, "Read-only hosts database in router context");
if (bf.wasMounted()) {
if (context.isRouterContext())
_log.logAlways(Log.WARN, "The hosts database was not closed cleanly or is still open by another process");
else
_log.logAlways(Log.WARN, "The hosts database is possibly in use by another process, perhaps the router? " +
"The database is not designed for simultaneous access by multiple processes.\n" +
"If you are using clients outside the router JVM, consider using the hosts.txt " +
"naming service with " +
"i2p.naming.impl=net.i2p.client.naming.HostsTxtNamingService");
}
} catch (IOException ioe) {
if (raf != null) {
try { raf.close(); } catch (IOException e) {}
@ -131,7 +158,7 @@ public class BlockfileNamingService extends DummyNamingService {
try {
// closing a BlockFile does not close the underlying file,
// so we must create and retain a RAF so we may close it later
raf = new RandomAccessFile(f, "rw");
raf = new RAIFile(f, true, true);
SecureFileOutputStream.setPerms(f);
bf = init(raf);
} catch (IOException ioe) {
@ -141,9 +168,11 @@ public class BlockfileNamingService extends DummyNamingService {
_log.log(Log.CRIT, "Failed to initialize database", ioe);
throw new RuntimeException(ioe);
}
readOnly = false;
}
_bf = bf;
_raf = raf;
_readOnly = readOnly;
_context.addShutdownTask(new Shutdown());
}
@ -152,7 +181,7 @@ public class BlockfileNamingService extends DummyNamingService {
* privatehosts.txt, userhosts.txt, and hosts.txt,
* creating a skiplist in the database for each.
*/
private BlockFile init(RandomAccessFile f) throws IOException {
private BlockFile init(RAIFile f) throws IOException {
long start = _context.clock().now();
try {
BlockFile rv = new BlockFile(f, true);
@ -221,7 +250,7 @@ public class BlockfileNamingService extends DummyNamingService {
/**
* Read the info block of an existing database.
*/
private BlockFile initExisting(RandomAccessFile raf) throws IOException {
private BlockFile initExisting(RAIFile raf) throws IOException {
long start = _context.clock().now();
try {
BlockFile bf = new BlockFile(raf, false);
@ -427,6 +456,10 @@ public class BlockfileNamingService extends DummyNamingService {
}
private boolean put(String hostname, Destination d, Properties options, boolean checkExisting) {
if (_readOnly) {
_log.error("Add entry failed, read-only hosts database");
return false;
}
String key = hostname.toLowerCase();
String listname = FALLBACK_LIST;
Properties props = new Properties();
@ -475,6 +508,10 @@ public class BlockfileNamingService extends DummyNamingService {
*/
@Override
public boolean remove(String hostname, Properties options) {
if (_readOnly) {
_log.error("Remove entry failed, read-only hosts database");
return false;
}
String key = hostname.toLowerCase();
String listname = FALLBACK_LIST;
if (options != null) {
@ -657,7 +694,7 @@ public class BlockfileNamingService extends DummyNamingService {
de != null &&
de.dest != null &&
de.dest.getPublicKey() != null;
if (!rv)
if ((!rv) && (!_readOnly))
_invalid.add(new InvalidEntry(key, listname));
return rv;
}
@ -731,7 +768,11 @@ public class BlockfileNamingService extends DummyNamingService {
try {
_bf.close();
} catch (IOException ioe) {
if (_log.shouldLog(Log.WARN))
_log.warn("Error closing", ioe);
} catch (RuntimeException e) {
if (_log.shouldLog(Log.WARN))
_log.warn("Error closing", e);
}
try {
_raf.close();
@ -870,8 +911,16 @@ public class BlockfileNamingService extends DummyNamingService {
}
}
/**
* BlockfileNamingService [force]
* force = force writable
*/
public static void main(String[] args) {
BlockfileNamingService bns = new BlockfileNamingService(I2PAppContext.getGlobalContext());
Properties ctxProps = new Properties();
if (args.length > 0 && args[0].equals("force"))
ctxProps.setProperty(PROP_FORCE, "true");
I2PAppContext ctx = new I2PAppContext(ctxProps);
BlockfileNamingService bns = new BlockfileNamingService(ctx);
List<String> names = null;
Properties props = new Properties();
try {

3
core/java/src/net/i2p/client/naming/DummyNamingService.java
View File

@ -58,7 +58,8 @@ class DummyNamingService extends NamingService {
if (hostname.length() >= 516) {
d = lookupBase64(hostname);
// What the heck, cache these too
putCache(hostname, d);
if (d != null)
putCache(hostname, d);
return d;
}

62
core/java/src/net/i2p/crypto/TrustedUpdate.java
View File

@ -22,6 +22,7 @@ import net.i2p.data.SigningPrivateKey;
import net.i2p.data.SigningPublicKey;
import net.i2p.util.Log;
import net.i2p.util.VersionComparator;
import net.i2p.util.ZipFileComment;
/**
* <p>Handles DSA signing and verification of update files.
@ -35,6 +36,7 @@ import net.i2p.util.VersionComparator;
* java net.i2p.crypto.TrustedUpdate sign <i>inputFile signedFile privateKeyFile version</i>
* java net.i2p.crypto.TrustedUpdate verifysig <i>signedFile</i>
* java net.i2p.crypto.TrustedUpdate verifyupdate <i>signedFile</i>
* java net.i2p.crypto.TrustedUpdate verifyversion <i>signedFile</i>
* </pre>
*
* @author jrandom and smeghead
@ -242,6 +244,8 @@ JXQAnA28vDmMMMH/WPbC5ixmJeGGNUiR
ok = verifySigCLI(args[1]);
} else if ("verifyupdate".equals(args[0])) {
ok = verifyUpdateCLI(args[1]);
} else if ("verifyversion".equals(args[0])) {
ok = verifyVersionCLI(args[1]);
} else {
showUsageCLI();
}
@ -301,11 +305,12 @@ JXQAnA28vDmMMMH/WPbC5ixmJeGGNUiR
}
private static final void showUsageCLI() {
System.err.println("Usage: TrustedUpdate keygen publicKeyFile privateKeyFile");
System.err.println(" TrustedUpdate showversion signedFile");
System.err.println(" TrustedUpdate sign inputFile signedFile privateKeyFile version");
System.err.println(" TrustedUpdate verifysig signedFile");
System.err.println(" TrustedUpdate verifyupdate signedFile");
System.err.println("Usage: TrustedUpdate keygen publicKeyFile privateKeyFile");
System.err.println(" TrustedUpdate showversion signedFile");
System.err.println(" TrustedUpdate sign inputFile signedFile privateKeyFile version");
System.err.println(" TrustedUpdate verifysig signedFile");
System.err.println(" TrustedUpdate verifyupdate signedFile");
System.err.println(" TrustedUpdate verifyversion signedFile");
}
/** @return success */
@ -349,9 +354,29 @@ JXQAnA28vDmMMMH/WPbC5ixmJeGGNUiR
System.out.println("File version is newer than current version.");
else
System.out.println("File version is older than or equal to current version.");
return isUpdate;
}
/**
* @return true if there's no version mismatch
* @since 0.8.8
*/
private static final boolean verifyVersionCLI(String signedFile) {
TrustedUpdate tu = new TrustedUpdate();
File file = new File(signedFile);
// ignore result, just used to read in version
tu.isUpdatedVersion("0", file);
boolean isMatch = tu.verifyVersionMatch(file);
if (isMatch)
System.out.println("Version verified");
else
System.out.println("Version mismatch, header version does not match zip comment version");
return isMatch;
}
/**
* Fetches the trusted keys for the current instance.
*
@ -490,6 +515,13 @@ JXQAnA28vDmMMMH/WPbC5ixmJeGGNUiR
* file's version is newer than the given current version, migrates the data
* out of <code>signedFile</code> and into <code>outputFile</code>.
*
* As of 0.8.8, the embedded file must be a zip file with
* a standard zip header and a UTF-8 zip file comment
* matching the version in the sud header. This prevents spoofing the version,
* since the sud signature does NOT cover the version in the header.
* (We do this for sud/su2 files but not plugin xpi2p files -
* don't use this method for plugin files)
*
* @param currentVersion The current version to check against.
* @param signedFile A signed update file.
* @param outputFile The file to write the verified data to.
@ -507,12 +539,32 @@ JXQAnA28vDmMMMH/WPbC5ixmJeGGNUiR
return "Downloaded version is not greater than current version";
}
if (!verifyVersionMatch(signedFile))
return "Update file invalid - signed version mismatch";
if (!verify(signedFile))
return "Unknown signing key or corrupt file";
return migrateFile(signedFile, outputFile);
}
/**
* Verify the version in the sud header matches the version in the zip comment
* (and that the embedded file is a zip file at all)
* isUpdatedVersion() must be called first to set _newVersion.
*
* @return true if matches
*
* @since 0.8.8
*/
private boolean verifyVersionMatch(File signedFile) {
try {
String zipComment = ZipFileComment.getComment(signedFile, VERSION_BYTES, HEADER_BYTES);
return zipComment.equals(_newVersion);
} catch (IOException ioe) {}
return false;
}
/**
* Extract the file. Skips and ignores the signature and version. No verification.
*

33
core/java/src/net/i2p/data/DataHelper.java
View File

@ -1298,16 +1298,30 @@ public class DataHelper {
private static final int MAX_UNCOMPRESSED = 40*1024;
public static final int MAX_COMPRESSION = Deflater.BEST_COMPRESSION;
public static final int NO_COMPRESSION = Deflater.NO_COMPRESSION;
/** compress the data and return a new GZIP compressed array */
/**
* Compress the data and return a new GZIP compressed byte array.
* @throws IllegalArgumentException if size is over 40KB
*/
public static byte[] compress(byte orig[]) {
return compress(orig, 0, orig.length);
}
/**
* Compress the data and return a new GZIP compressed byte array.
* @throws IllegalArgumentException if size is over 40KB
*/
public static byte[] compress(byte orig[], int offset, int size) {
return compress(orig, offset, size, MAX_COMPRESSION);
}
/**
* Compress the data and return a new GZIP compressed byte array.
* @throws IllegalArgumentException if size is over 40KB
*/
public static byte[] compress(byte orig[], int offset, int size, int level) {
if ((orig == null) || (orig.length <= 0)) return orig;
if (size >= MAX_UNCOMPRESSED)
if (size > MAX_UNCOMPRESSED)
throw new IllegalArgumentException("tell jrandom size=" + size);
ReusableGZIPOutputStream out = ReusableGZIPOutputStream.acquire();
out.setLevel(level);
@ -1335,10 +1349,18 @@ public class DataHelper {
}
/** decompress the GZIP compressed data (returning null on error) */
/**
* Decompress the GZIP compressed data (returning null on error).
* @throws IOE if uncompressed is over 40 KB
*/
public static byte[] decompress(byte orig[]) throws IOException {
return (orig != null ? decompress(orig, 0, orig.length) : null);
}
/**
* Decompress the GZIP compressed data (returning null on error).
* @throws IOE if uncompressed is over 40 KB
*/
public static byte[] decompress(byte orig[], int offset, int length) throws IOException {
if ((orig == null) || (orig.length <= 0)) return orig;
@ -1354,6 +1376,11 @@ public class DataHelper {
if (read == -1)
break;
written += read;
if (written >= MAX_UNCOMPRESSED) {
if (in.available() > 0)
throw new IOException("Uncompressed data larger than " + MAX_UNCOMPRESSED);
break;
}
}
byte rv[] = new byte[written];
System.arraycopy(outBuf.getData(), 0, rv, 0, written);

111
core/java/src/net/i2p/util/ZipFileComment.java Normal file
View File

@ -0,0 +1,111 @@
package net.i2p.util;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.InputStream;
import java.io.IOException;
import java.util.zip.ZipException;
import net.i2p.data.DataHelper;
/**
* Not available in ZipFile until Java 7. Refs:
* https://secure.wikimedia.org/wikipedia/en/wiki/ZIP_%28file_format%29
* http://download.oracle.com/javase/1.5.0/docs/api/java/util/zip/ZipFile.html
* http://bugs.sun.com/view_bug.do?bug_id=6646605
*
* Code modified from:
* http://www.flattermann.net/2009/01/read-a-zip-file-comment-with-java/
* Beerware.
*
* since 0.8.8
*/
public abstract class ZipFileComment {
private static final int BLOCK_LEN = 22;
private static final byte[] magicStart = {0x50, 0x4b, 0x03, 0x04};
private static final int HEADER_LEN = magicStart.length;
private static final byte[] magicDirEnd = {0x50, 0x4b, 0x05, 0x06};
private static final int MAGIC_LEN = magicDirEnd.length;
/**
* @param max The max length of the comment in bytes.
* If the actual comment is longer, it will not be found and
* this method will throw an IOE
*
* @return empty string if no comment, or the comment.
* The string is decoded with UTF-8
*
* @throws IOE if no valid end-of-central-directory record found
*/
public static String getComment(File file, int max) throws IOException {
return getComment(file, max, 0);
}
/**
* @param max The max length of the comment in bytes.
* If the actual comment is longer, it will not be found and
* this method will throw an IOE
* @param skip Number of bytes to skip in the file before looking for the
* zip header. Use 56 for sud/su2 files.
*
* @return empty string if no comment, or the comment.
* The string is decoded with UTF-8
*
* @throws IOE if no valid end-of-central-directory record found
*/
public static String getComment(File file, int max, int skip) throws IOException {
if (!file.exists())
throw new FileNotFoundException("File not found: " + file);
long len = file.length();
if (len < BLOCK_LEN + HEADER_LEN + skip)
throw new ZipException("File too short: " + file);
if (len > Integer.MAX_VALUE)
throw new ZipException("File too long: " + file);
int fileLen = (int) len;
byte[] buffer = new byte[Math.min(fileLen - skip, max + BLOCK_LEN)];
InputStream in = null;
try {
in = new FileInputStream(file);
if (skip > 0)
in.skip(skip);
byte[] hdr = new byte[HEADER_LEN];
DataHelper.read(in, hdr);
if (!DataHelper.eq(hdr, magicStart))
throw new ZipException("Not a zip file: " + file);
in.skip(fileLen - (skip + HEADER_LEN + buffer.length));
DataHelper.read(in, buffer);
return getComment(buffer);
} finally {
if (in != null) try { in.close(); } catch (IOException ioe) {}
}
}
/**
* go backwards from the end
*/
private static String getComment(byte[] buffer) throws IOException {
for (int i = buffer.length - (1 + BLOCK_LEN - MAGIC_LEN); i >= 0; i--) {
if (DataHelper.eq(buffer, i, magicDirEnd, 0, MAGIC_LEN)) {
int commentLen = (buffer[i + BLOCK_LEN - 2] & 0xff) +
((buffer[i + BLOCK_LEN - 1] & 0xff) * 256);
return new String(buffer, i + BLOCK_LEN, commentLen, "UTF-8");
}
}
throw new ZipException("No comment block found");
}
public static void main(String args[]) throws IOException {
if (args.length != 1) {
System.err.println("Usage: ZipFileComment file");
return;
}
int skip = 0;
String file = args[0];
if (file.endsWith(".sud") || file.endsWith(".su2"))
skip = 56;
String c = getComment(new File(file), 256, skip);
System.out.println("comment is: \"" + c + '\"');
}
}

15
core/java/src/net/metanotion/io/RAIFile.java
View File

@ -38,13 +38,17 @@ import java.io.RandomAccessFile;
public class RAIFile implements RandomAccessInterface, DataInput, DataOutput {
private File f;
private RandomAccessFile delegate;
private boolean r=false, w=false;
private final boolean r, w;
public RAIFile(RandomAccessFile file) throws FileNotFoundException {
this.f = null;
this.delegate = file;
this.r = true;
// fake, we don't really know
this.w = true;
}
/** @param read must be true */
public RAIFile(File file, boolean read, boolean write) throws FileNotFoundException {
this.f = file;
this.r = read;
@ -55,6 +59,15 @@ public class RAIFile implements RandomAccessInterface, DataInput, DataOutput {
this.delegate = new RandomAccessFile(file, mode);
}
/**
* I2P is the file writable?
* Only valid if the File constructor was used, not the RAF constructor
* @since 0.8.8
*/
public boolean canWrite() {
return this.w;
}
public long getFilePointer() throws IOException { return delegate.getFilePointer(); }
public long length() throws IOException { return delegate.length(); }
public int read() throws IOException { return delegate.read(); }

7
core/java/src/net/metanotion/io/RandomAccessInterface.java
View File

@ -39,6 +39,13 @@ public interface RandomAccessInterface {
public void seek(long pos) throws IOException;
public void setLength(long newLength) throws IOException;
/**
* I2P is the file writable?
* Only valid if the File constructor was used, not the RAF constructor
* @since 0.8.8
*/
public boolean canWrite();
// Closeable Methods
public void close() throws IOException;

31
core/java/src/net/metanotion/io/block/BlockFile.java
View File

@ -90,6 +90,9 @@ public class BlockFile {
private int mounted = 0;
public int spanSize = 16;
/** I2P was the file locked when we opened it? */
private final boolean _wasMounted;
private BSkipList metaIndex;
/** cached list of free pages, only valid if freListStart > 0 */
private FreeListBlock flb;
@ -258,11 +261,19 @@ public class BlockFile {
return curPage;
}
/** Use this constructor with a readonly RAI for a readonly blockfile */
public BlockFile(RandomAccessInterface rai) throws IOException { this(rai, false); }
/** RAF must be writable */
public BlockFile(RandomAccessFile raf) throws IOException { this(new RAIFile(raf), false); }
/** RAF must be writable */
public BlockFile(RandomAccessFile raf, boolean init) throws IOException { this(new RAIFile(raf), init); }
/** File must be writable */
public BlockFile(File f, boolean init) throws IOException { this(new RAIFile(f, true, true), init); }
/** Use this constructor with a readonly RAI and init = false for a readonly blockfile */
public BlockFile(RandomAccessInterface rai, boolean init) throws IOException {
if(rai==null) { throw new NullPointerException(); }
@ -283,16 +294,26 @@ public class BlockFile {
throw new IOException("Bad magic number");
}
}
if (mounted != 0)
_wasMounted = mounted != 0;
if (_wasMounted)
log.warn("Warning - file was not previously closed");
if(fileLen != file.length())
throw new IOException("Expected file length " + fileLen +
" but actually " + file.length());
mount();
if (rai.canWrite())
mount();
metaIndex = new BSkipList(spanSize, this, METAINDEX_PAGE, new StringBytes(), new IntBytes());
}
/**
* I2P was the file locked when we opened it?
* @since 0.8.8
*/
public boolean wasMounted() {
return _wasMounted;
}
/**
* Go to any page but the superblock.
* Page 1 is the superblock, must use file.seek(0) to get there.
@ -454,8 +475,10 @@ public class BlockFile {
}
// Unmount.
file.seek(BlockFile.OFFSET_MOUNTED);
file.writeShort(0);
if (file.canWrite()) {
file.seek(BlockFile.OFFSET_MOUNTED);
file.writeShort(0);
}
}
public void bfck(boolean fix) {

5
core/java/src/net/metanotion/io/block/index/BSkipList.java
View File

@ -98,7 +98,8 @@ public class BSkipList extends SkipList {
}
if (BlockFile.log.shouldLog(Log.DEBUG))
BlockFile.log.debug("Loaded " + this + " cached " + levelHash.size() + " levels and " + spanHash.size() + " spans with " + total + " entries");
if (levelCount != levelHash.size() || spans != spanHash.size() || size != total) {
if (bf.file.canWrite() &&
(levelCount != levelHash.size() || spans != spanHash.size() || size != total)) {
if (BlockFile.log.shouldLog(Log.WARN))
BlockFile.log.warn("On-disk counts were " + levelCount + " / " + spans + " / " + size + ", correcting");
size = total;
@ -117,6 +118,8 @@ public class BSkipList extends SkipList {
@Override
public void flush() {
if (!bf.file.canWrite())
return;
if (isClosed) {
BlockFile.log.error("Already closed!! " + this, new Exception());
return;