* BlockfileNamingService:

- Support readonly blockfiles
    - Open blockfile readonly if not in router context
    - Log warning if blockfile is locked
This commit is contained in:
zzz
2011-06-23 14:35:48 +00:00
parent facbb8c950
commit bda6d7819c
5 changed files with 109 additions and 14 deletions

View File

@ -33,6 +33,7 @@ import net.i2p.data.Hash;
import net.i2p.util.Log; import net.i2p.util.Log;
import net.i2p.util.SecureFileOutputStream; import net.i2p.util.SecureFileOutputStream;
import net.metanotion.io.RAIFile;
import net.metanotion.io.Serializer; import net.metanotion.io.Serializer;
import net.metanotion.io.block.BlockFile; import net.metanotion.io.block.BlockFile;
import net.metanotion.io.data.UTF8StringBytes; import net.metanotion.io.data.UTF8StringBytes;
@ -76,10 +77,11 @@ import net.metanotion.util.skiplist.SkipList;
public class BlockfileNamingService extends DummyNamingService { public class BlockfileNamingService extends DummyNamingService {
private final BlockFile _bf; private final BlockFile _bf;
private final RandomAccessFile _raf; private final RAIFile _raf;
private final List<String> _lists; private final List<String> _lists;
private final List<InvalidEntry> _invalid; private final List<InvalidEntry> _invalid;
private volatile boolean _isClosed; private volatile boolean _isClosed;
private final boolean _readOnly;
private static final Serializer _infoSerializer = new PropertiesSerializer(); private static final Serializer _infoSerializer = new PropertiesSerializer();
private static final Serializer _stringSerializer = new UTF8StringBytes(); private static final Serializer _stringSerializer = new UTF8StringBytes();
@ -87,6 +89,7 @@ public class BlockfileNamingService extends DummyNamingService {
private static final String HOSTS_DB = "hostsdb.blockfile"; private static final String HOSTS_DB = "hostsdb.blockfile";
private static final String FALLBACK_LIST = "hosts.txt"; private static final String FALLBACK_LIST = "hosts.txt";
private static final String PROP_FORCE = "i2p.naming.blockfile.writeInAppContext";
private static final String INFO_SKIPLIST = "%%__INFO__%%"; private static final String INFO_SKIPLIST = "%%__INFO__%%";
private static final String PROP_INFO = "info"; private static final String PROP_INFO = "info";
@ -100,6 +103,13 @@ public class BlockfileNamingService extends DummyNamingService {
private static final String PROP_SOURCE = "s"; private static final String PROP_SOURCE = "s";
/** /**
* Opens the database at hostsdb.blockfile or creates a new
* one and imports entries from hosts.txt, userhosts.txt, and privatehosts.txt.
*
* If not in router context, the database will be opened read-only
* unless the property i2p.naming.blockfile.writeInAppContext is true.
* Not designed for simultaneous access by multiple processes.
*
* @throws RuntimeException on fatal error * @throws RuntimeException on fatal error
*/ */
public BlockfileNamingService(I2PAppContext context) { public BlockfileNamingService(I2PAppContext context) {
@ -107,14 +117,31 @@ public class BlockfileNamingService extends DummyNamingService {
_lists = new ArrayList(); _lists = new ArrayList();
_invalid = new ArrayList(); _invalid = new ArrayList();
BlockFile bf = null; BlockFile bf = null;
RandomAccessFile raf = null; RAIFile raf = null;
boolean readOnly = false;
File f = new File(_context.getRouterDir(), HOSTS_DB); File f = new File(_context.getRouterDir(), HOSTS_DB);
if (f.exists()) { if (f.exists()) {
try { try {
// closing a BlockFile does not close the underlying file, // closing a BlockFile does not close the underlying file,
// so we must create and retain a RAF so we may close it later // so we must create and retain a RAF so we may close it later
raf = new RandomAccessFile(f, "rw");
// *** Open readonly if not in router context (unless forced)
readOnly = (!f.canWrite()) ||
((!context.isRouterContext()) && (!context.getBooleanProperty(PROP_FORCE)));
raf = new RAIFile(f, true, !readOnly);
bf = initExisting(raf); bf = initExisting(raf);
if (readOnly && context.isRouterContext())
_log.logAlways(Log.WARN, "Read-only hosts database in router context");
if (bf.wasMounted()) {
if (context.isRouterContext())
_log.logAlways(Log.WARN, "The hosts database was not closed cleanly or is still open by another process");
else
_log.logAlways(Log.WARN, "The hosts database is possibly in use by another process, perhaps the router? " +
"The database is not designed for simultaneous access by multiple processes.\n" +
"If you are using clients outside the router JVM, consider using the hosts.txt " +
"naming service with " +
"i2p.naming.impl=net.i2p.client.naming.HostsTxtNamingService");
}
} catch (IOException ioe) { } catch (IOException ioe) {
if (raf != null) { if (raf != null) {
try { raf.close(); } catch (IOException e) {} try { raf.close(); } catch (IOException e) {}
@ -131,7 +158,7 @@ public class BlockfileNamingService extends DummyNamingService {
try { try {
// closing a BlockFile does not close the underlying file, // closing a BlockFile does not close the underlying file,
// so we must create and retain a RAF so we may close it later // so we must create and retain a RAF so we may close it later
raf = new RandomAccessFile(f, "rw"); raf = new RAIFile(f, true, true);
SecureFileOutputStream.setPerms(f); SecureFileOutputStream.setPerms(f);
bf = init(raf); bf = init(raf);
} catch (IOException ioe) { } catch (IOException ioe) {
@ -141,9 +168,11 @@ public class BlockfileNamingService extends DummyNamingService {
_log.log(Log.CRIT, "Failed to initialize database", ioe); _log.log(Log.CRIT, "Failed to initialize database", ioe);
throw new RuntimeException(ioe); throw new RuntimeException(ioe);
} }
readOnly = false;
} }
_bf = bf; _bf = bf;
_raf = raf; _raf = raf;
_readOnly = readOnly;
_context.addShutdownTask(new Shutdown()); _context.addShutdownTask(new Shutdown());
} }
@ -152,7 +181,7 @@ public class BlockfileNamingService extends DummyNamingService {
* privatehosts.txt, userhosts.txt, and hosts.txt, * privatehosts.txt, userhosts.txt, and hosts.txt,
* creating a skiplist in the database for each. * creating a skiplist in the database for each.
*/ */
private BlockFile init(RandomAccessFile f) throws IOException { private BlockFile init(RAIFile f) throws IOException {
long start = _context.clock().now(); long start = _context.clock().now();
try { try {
BlockFile rv = new BlockFile(f, true); BlockFile rv = new BlockFile(f, true);
@ -221,7 +250,7 @@ public class BlockfileNamingService extends DummyNamingService {
/** /**
* Read the info block of an existing database. * Read the info block of an existing database.
*/ */
private BlockFile initExisting(RandomAccessFile raf) throws IOException { private BlockFile initExisting(RAIFile raf) throws IOException {
long start = _context.clock().now(); long start = _context.clock().now();
try { try {
BlockFile bf = new BlockFile(raf, false); BlockFile bf = new BlockFile(raf, false);
@ -427,6 +456,10 @@ public class BlockfileNamingService extends DummyNamingService {
} }
private boolean put(String hostname, Destination d, Properties options, boolean checkExisting) { private boolean put(String hostname, Destination d, Properties options, boolean checkExisting) {
if (_readOnly) {
_log.error("Add entry failed, read-only hosts database");
return false;
}
String key = hostname.toLowerCase(); String key = hostname.toLowerCase();
String listname = FALLBACK_LIST; String listname = FALLBACK_LIST;
Properties props = new Properties(); Properties props = new Properties();
@ -475,6 +508,10 @@ public class BlockfileNamingService extends DummyNamingService {
*/ */
@Override @Override
public boolean remove(String hostname, Properties options) { public boolean remove(String hostname, Properties options) {
if (_readOnly) {
_log.error("Remove entry failed, read-only hosts database");
return false;
}
String key = hostname.toLowerCase(); String key = hostname.toLowerCase();
String listname = FALLBACK_LIST; String listname = FALLBACK_LIST;
if (options != null) { if (options != null) {
@ -657,7 +694,7 @@ public class BlockfileNamingService extends DummyNamingService {
de != null && de != null &&
de.dest != null && de.dest != null &&
de.dest.getPublicKey() != null; de.dest.getPublicKey() != null;
if (!rv) if ((!rv) && (!_readOnly))
_invalid.add(new InvalidEntry(key, listname)); _invalid.add(new InvalidEntry(key, listname));
return rv; return rv;
} }
@ -731,7 +768,11 @@ public class BlockfileNamingService extends DummyNamingService {
try { try {
_bf.close(); _bf.close();
} catch (IOException ioe) { } catch (IOException ioe) {
if (_log.shouldLog(Log.WARN))
_log.warn("Error closing", ioe);
} catch (RuntimeException e) { } catch (RuntimeException e) {
if (_log.shouldLog(Log.WARN))
_log.warn("Error closing", e);
} }
try { try {
_raf.close(); _raf.close();
@ -870,8 +911,16 @@ public class BlockfileNamingService extends DummyNamingService {
} }
} }
/**
* BlockfileNamingService [force]
* force = force writable
*/
public static void main(String[] args) { public static void main(String[] args) {
BlockfileNamingService bns = new BlockfileNamingService(I2PAppContext.getGlobalContext()); Properties ctxProps = new Properties();
if (args.length > 0 && args[0].equals("force"))
ctxProps.setProperty(PROP_FORCE, "true");
I2PAppContext ctx = new I2PAppContext(ctxProps);
BlockfileNamingService bns = new BlockfileNamingService(ctx);
List<String> names = null; List<String> names = null;
Properties props = new Properties(); Properties props = new Properties();
try { try {

View File

@ -38,13 +38,17 @@ import java.io.RandomAccessFile;
public class RAIFile implements RandomAccessInterface, DataInput, DataOutput { public class RAIFile implements RandomAccessInterface, DataInput, DataOutput {
private File f; private File f;
private RandomAccessFile delegate; private RandomAccessFile delegate;
private boolean r=false, w=false; private final boolean r, w;
public RAIFile(RandomAccessFile file) throws FileNotFoundException { public RAIFile(RandomAccessFile file) throws FileNotFoundException {
this.f = null; this.f = null;
this.delegate = file; this.delegate = file;
this.r = true;
// fake, we don't really know
this.w = true;
} }
/** @param read must be true */
public RAIFile(File file, boolean read, boolean write) throws FileNotFoundException { public RAIFile(File file, boolean read, boolean write) throws FileNotFoundException {
this.f = file; this.f = file;
this.r = read; this.r = read;
@ -55,6 +59,15 @@ public class RAIFile implements RandomAccessInterface, DataInput, DataOutput {
this.delegate = new RandomAccessFile(file, mode); this.delegate = new RandomAccessFile(file, mode);
} }
/**
* I2P is the file writable?
* Only valid if the File constructor was used, not the RAF constructor
* @since 0.8.8
*/
public boolean canWrite() {
return this.w;
}
public long getFilePointer() throws IOException { return delegate.getFilePointer(); } public long getFilePointer() throws IOException { return delegate.getFilePointer(); }
public long length() throws IOException { return delegate.length(); } public long length() throws IOException { return delegate.length(); }
public int read() throws IOException { return delegate.read(); } public int read() throws IOException { return delegate.read(); }

View File

@ -39,6 +39,13 @@ public interface RandomAccessInterface {
public void seek(long pos) throws IOException; public void seek(long pos) throws IOException;
public void setLength(long newLength) throws IOException; public void setLength(long newLength) throws IOException;
/**
* I2P is the file writable?
* Only valid if the File constructor was used, not the RAF constructor
* @since 0.8.8
*/
public boolean canWrite();
// Closeable Methods // Closeable Methods
public void close() throws IOException; public void close() throws IOException;

View File

@ -90,6 +90,9 @@ public class BlockFile {
private int mounted = 0; private int mounted = 0;
public int spanSize = 16; public int spanSize = 16;
/** I2P was the file locked when we opened it? */
private final boolean _wasMounted;
private BSkipList metaIndex; private BSkipList metaIndex;
/** cached list of free pages, only valid if freListStart > 0 */ /** cached list of free pages, only valid if freListStart > 0 */
private FreeListBlock flb; private FreeListBlock flb;
@ -258,11 +261,19 @@ public class BlockFile {
return curPage; return curPage;
} }
/** Use this constructor with a readonly RAI for a readonly blockfile */
public BlockFile(RandomAccessInterface rai) throws IOException { this(rai, false); } public BlockFile(RandomAccessInterface rai) throws IOException { this(rai, false); }
/** RAF must be writable */
public BlockFile(RandomAccessFile raf) throws IOException { this(new RAIFile(raf), false); } public BlockFile(RandomAccessFile raf) throws IOException { this(new RAIFile(raf), false); }
/** RAF must be writable */
public BlockFile(RandomAccessFile raf, boolean init) throws IOException { this(new RAIFile(raf), init); } public BlockFile(RandomAccessFile raf, boolean init) throws IOException { this(new RAIFile(raf), init); }
/** File must be writable */
public BlockFile(File f, boolean init) throws IOException { this(new RAIFile(f, true, true), init); } public BlockFile(File f, boolean init) throws IOException { this(new RAIFile(f, true, true), init); }
/** Use this constructor with a readonly RAI and init = false for a readonly blockfile */
public BlockFile(RandomAccessInterface rai, boolean init) throws IOException { public BlockFile(RandomAccessInterface rai, boolean init) throws IOException {
if(rai==null) { throw new NullPointerException(); } if(rai==null) { throw new NullPointerException(); }
@ -283,16 +294,26 @@ public class BlockFile {
throw new IOException("Bad magic number"); throw new IOException("Bad magic number");
} }
} }
if (mounted != 0) _wasMounted = mounted != 0;
if (_wasMounted)
log.warn("Warning - file was not previously closed"); log.warn("Warning - file was not previously closed");
if(fileLen != file.length()) if(fileLen != file.length())
throw new IOException("Expected file length " + fileLen + throw new IOException("Expected file length " + fileLen +
" but actually " + file.length()); " but actually " + file.length());
mount(); if (rai.canWrite())
mount();
metaIndex = new BSkipList(spanSize, this, METAINDEX_PAGE, new StringBytes(), new IntBytes()); metaIndex = new BSkipList(spanSize, this, METAINDEX_PAGE, new StringBytes(), new IntBytes());
} }
/**
* I2P was the file locked when we opened it?
* @since 0.8.8
*/
public boolean wasMounted() {
return _wasMounted;
}
/** /**
* Go to any page but the superblock. * Go to any page but the superblock.
* Page 1 is the superblock, must use file.seek(0) to get there. * Page 1 is the superblock, must use file.seek(0) to get there.
@ -454,8 +475,10 @@ public class BlockFile {
} }
// Unmount. // Unmount.
file.seek(BlockFile.OFFSET_MOUNTED); if (file.canWrite()) {
file.writeShort(0); file.seek(BlockFile.OFFSET_MOUNTED);
file.writeShort(0);
}
} }
public void bfck(boolean fix) { public void bfck(boolean fix) {

View File

@ -98,7 +98,8 @@ public class BSkipList extends SkipList {
} }
if (BlockFile.log.shouldLog(Log.DEBUG)) if (BlockFile.log.shouldLog(Log.DEBUG))
BlockFile.log.debug("Loaded " + this + " cached " + levelHash.size() + " levels and " + spanHash.size() + " spans with " + total + " entries"); BlockFile.log.debug("Loaded " + this + " cached " + levelHash.size() + " levels and " + spanHash.size() + " spans with " + total + " entries");
if (levelCount != levelHash.size() || spans != spanHash.size() || size != total) { if (bf.file.canWrite() &&
(levelCount != levelHash.size() || spans != spanHash.size() || size != total)) {
if (BlockFile.log.shouldLog(Log.WARN)) if (BlockFile.log.shouldLog(Log.WARN))
BlockFile.log.warn("On-disk counts were " + levelCount + " / " + spans + " / " + size + ", correcting"); BlockFile.log.warn("On-disk counts were " + levelCount + " / " + spans + " / " + size + ", correcting");
size = total; size = total;
@ -117,6 +118,8 @@ public class BSkipList extends SkipList {
@Override @Override
public void flush() { public void flush() {
if (!bf.file.canWrite())
return;
if (isClosed) { if (isClosed) {
BlockFile.log.error("Already closed!! " + this, new Exception()); BlockFile.log.error("Already closed!! " + this, new Exception());
return; return;