echelon/i2p.i2p
1
0
Fork 0
forked from I2P_Developers/i2p.i2p
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

- Fix several bugs with BSkipLevels persistence

Browse Source 
- Logging and debug code
- New serializers
- Cleanups and javadocs
This commit is contained in:
zzz
2011-03-26 02:40:17 +00:00
parent 078056f163
commit 8f9f102baf
10 changed files with 298 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files

52
core/java/src/net/metanotion/io/block/BlockFile.java
View File

@ -38,11 +38,14 @@ import java.util.Set;
import net.metanotion.io.RAIFile;
import net.metanotion.io.RandomAccessInterface;
import net.metanotion.io.Serializer;
import net.metanotion.io.data.IdentityBytes;
import net.metanotion.io.data.IntBytes;
import net.metanotion.io.data.LongBytes;
import net.metanotion.io.data.NullBytes;
import net.metanotion.io.data.StringBytes;
import net.metanotion.io.data.UTF8StringBytes;
import net.metanotion.io.block.index.BSkipList;
import net.metanotion.util.skiplist.SkipIterator;
import net.metanotion.util.skiplist.SkipList;
import net.i2p.I2PAppContext;
@ -93,17 +96,15 @@ public class BlockFile {
}
public static void main(String args[]) {
if (args.length != 1) {
System.err.println("Usage: BlockFile file");
return;
}
boolean init = !(new File(args[0])).exists();
try {
RAIFile raif = new RAIFile(new File(args[0]), true, true);
BlockFile bf = new BlockFile(raif, true);
//bf.metaIndex.delete();
bf.makeIndex("foo", new NullBytes(), new NullBytes());
BSkipList b = bf.getIndex("foo", new NullBytes(), new NullBytes());
System.out.println(bf.allocPage());
BlockFile bf = new BlockFile(raif, init);
bf.bfck(true);
bf.close();
raif.close();
} catch (Exception e) {
@ -190,7 +191,8 @@ public class BlockFile {
throw new BadFileFormatException();
}
}
// if(mounted != 0) { throw new CorruptFileException(); }
if (mounted != 0)
log.warn("Warning - file was not previously closed");
if(fileLen != file.length()) { throw new CorruptFileException(); }
mount();
@ -275,7 +277,7 @@ public class BlockFile {
public void delIndex(String name) throws IOException {
Integer page = (Integer) metaIndex.remove(name);
if (page == null) { return; }
NullBytes nb = new NullBytes();
Serializer nb = new IdentityBytes();
BSkipList bsl = new BSkipList(spanSize, this, page.intValue(), nb, nb, true);
bsl.delete();
}
@ -314,4 +316,32 @@ public class BlockFile {
file.seek(BlockFile.OFFSET_MOUNTED);
file.writeShort(0);
}
public void bfck(boolean fix) {
log.warn("magic bytes " + magicBytes);
log.warn("fileLen " + fileLen);
log.warn("freeListStart " + freeListStart);
log.warn("mounted " + mounted);
log.warn("spanSize " + spanSize);
log.warn("Metaindex");
metaIndex.bslck(true, fix);
int items = 0;
for (SkipIterator iter = metaIndex.iterator(); iter.hasNext(); ) {
String slname = (String) iter.nextKey();
Integer page = (Integer) iter.next();
log.warn("List " + slname + " page " + page);
try {
BSkipList bsl = getIndex(slname, new UTF8StringBytes(), new IdentityBytes());
if (bsl == null) {
log.error("Can't find list? " + slname);
continue;
}
bsl.bslck(false, fix);
items++;
} catch (IOException ioe) {
log.error("Error with list " + slname, ioe);
}
}
log.warn("actual size " + items);
}
}

60
core/java/src/net/metanotion/io/block/index/BSkipLevels.java
View File

@ -36,12 +36,18 @@ import net.metanotion.util.skiplist.SkipList;
import net.metanotion.util.skiplist.SkipLevels;
import net.metanotion.util.skiplist.SkipSpan;
/**
* On-disk format:
* max height (unsigned short)
* non-null height (unsigned short)
* span page (unsigned int)
* height number of level pages (unsigned ints)
*/
public class BSkipLevels extends SkipLevels {
public int levelPage;
public int spanPage;
public BlockFile bf;
public final int levelPage;
public final int spanPage;
public final BlockFile bf;
protected BSkipLevels() { }
public BSkipLevels(BlockFile bf, int levelPage, BSkipList bsl) throws IOException {
this.levelPage = levelPage;
this.bf = bf;
@ -56,20 +62,27 @@ public class BSkipLevels extends SkipLevels {
bottom = (BSkipSpan) bsl.spanHash.get(new Integer(spanPage));
this.levels = new BSkipLevels[maxLen];
int lp;
for(int i=0;i<nonNull;i++) {
lp = bf.file.readUnsignedInt();
BlockFile.log.debug("Reading New BSkipLevels with " + nonNull + " valid levels out of " + maxLen + " page " + levelPage);
// We have to read now because new BSkipLevels() will move the file pointer
int[] lps = new int[nonNull];
for(int i = 0; i < nonNull; i++) {
lps[i] = bf.file.readUnsignedInt();
}
for(int i = 0; i < nonNull; i++) {
int lp = lps[i];
if(lp != 0) {
levels[i] = (BSkipLevels) bsl.levelHash.get(new Integer(lp));
if(levels[i] == null) {
levels[i] = new BSkipLevels(bf, lp, bsl);
bsl.levelHash.put(new Integer(lp), levels[i]);
} else {
}
} else {
BlockFile.log.warn("WTF page " + levelPage + " i = " + i + " of " + nonNull + " valid levels out of " + maxLen + " but level page is zero");
levels[i] = null;
}
}
}
public static void init(BlockFile bf, int page, int spanPage, int maxHeight) throws IOException {
@ -83,13 +96,15 @@ public class BSkipLevels extends SkipLevels {
try {
BlockFile.pageSeek(bf.file, levelPage);
bf.file.writeShort((short) levels.length);
int i=0;
for(i=0;i<levels.length;i++) { if(levels[i] == null) { break; } }
int i = 0;
for( ; i < levels.length; i++) {
if(levels[i] == null)
break;
}
bf.file.writeShort(i);
bf.file.writeInt(((BSkipSpan) bottom).page);
for(i=0;i<levels.length;i++) {
if(levels[i]==null) { break; }
bf.file.writeInt(((BSkipLevels) levels[i]).levelPage);
for(int j = 0; j < i; j++) {
bf.file.writeInt(((BSkipLevels) levels[j]).levelPage);
}
} catch (IOException ioe) { throw new RuntimeException("Error writing to database", ioe); }
}
@ -106,7 +121,26 @@ public class BSkipLevels extends SkipLevels {
BSkipList bsl = (BSkipList) sl;
int page = bf.allocPage();
BSkipLevels.init(bf, page, bss.page, levels);
BlockFile.log.info("New BSkipLevels height " + levels + " page " + page);
return new BSkipLevels(bf, page, bsl);
} catch (IOException ioe) { throw new RuntimeException("Error creating database page", ioe); }
}
@Override
public void blvlck(boolean fix, int width) {
BlockFile.log.warn(" Skip level at width " + width);
BlockFile.log.warn(" levels " + this.levels.length);
BlockFile.log.warn(" first key " + this.key());
BlockFile.log.warn(" spanPage " + this.spanPage);
BlockFile.log.warn(" levelPage " + this.levelPage);
for (int i = levels.length - 1; i >= 0; i--) {
if (levels[i] != null) {
BlockFile.log.warn(" level " + i + " -> " + levels[i].key() + " ");
} else {
BlockFile.log.warn(" level " + i + " empty");
}
}
if (levels[0] != null)
levels[0].blvlck(fix, width + 1);
}
}

29
core/java/src/net/metanotion/io/block/index/BSkipList.java
View File

@ -158,4 +158,33 @@ public class BSkipList extends SkipList {
return new IBSkipIterator(ss, search[0]);
}
public void bslck(boolean isMeta, boolean fix) {
BlockFile.log.warn(" size " + this.size);
BlockFile.log.warn(" spans " + this.spans);
BlockFile.log.warn(" skipPage " + this.skipPage);
BlockFile.log.warn(" firstSpanPage " + this.firstSpanPage);
BlockFile.log.warn(" firstLevelPage " + this.firstLevelPage);
BlockFile.log.warn(" maxLevels " + this.maxLevels());
BlockFile.log.warn("*** printSL() ***");
printSL();
BlockFile.log.warn("*** print() ***");
print();
BlockFile.log.warn("*** Lvlck() ***");
stack.blvlck(fix, 0);
int items = 0;
for (SkipIterator iter = this.iterator(); iter.hasNext(); ) {
String key = (String) iter.nextKey();
if (isMeta) {
int sz = ((Integer) iter.next()).intValue();
BlockFile.log.warn(" Item " + key + " page " + sz);
} else {
String cls= iter.next().getClass().getSimpleName();
BlockFile.log.warn(" Item " + key + " size " + cls);
}
items++;
}
BlockFile.log.warn(" actual size " + items);
if (items != this.size)
BlockFile.log.warn("****** size mismatch, header = " + this.size + " actual = " + items);
}
}

1
core/java/src/net/metanotion/io/block/index/BSkipSpan.java
View File

@ -33,7 +33,6 @@ import java.io.IOException;
import net.metanotion.io.RandomAccessInterface;
import net.metanotion.io.Serializer;
import net.metanotion.io.block.BlockFile;
import net.metanotion.io.data.NullBytes;
import net.metanotion.util.skiplist.SkipList;
import net.metanotion.util.skiplist.SkipSpan;

23
core/java/src/net/metanotion/io/block/index/IBSkipSpan.java
View File

@ -35,6 +35,8 @@ import net.metanotion.io.block.BlockFile;
import net.metanotion.util.skiplist.SkipList;
import net.metanotion.util.skiplist.SkipSpan;
import net.i2p.util.Log;
/**
* I2P version of BSkipSpan
*
@ -55,12 +57,11 @@ import net.metanotion.util.skiplist.SkipSpan;
public class IBSkipSpan extends BSkipSpan {
private Comparable firstKey;
private static final boolean DEBUG = false;
@Override
public SkipSpan newInstance(SkipList sl) {
if (DEBUG)
System.err.println("Splitting page " + this.page + " containing " + this.nKeys + '/' + this.spanSize);
if (BlockFile.log.shouldLog(Log.DEBUG))
BlockFile.log.debug("Splitting page " + this.page + " containing " + this.nKeys + '/' + this.spanSize);
try {
int newPage = bf.allocPage();
init(bf, newPage, bf.spanSize);
@ -84,8 +85,8 @@ public class IBSkipSpan extends BSkipSpan {
this.firstKey = null;
this.keys = null;
this.vals = null;
if (DEBUG)
System.err.println("Flushed data for page " + this.page + " containing " + this.nKeys + '/' + this.spanSize);
if (BlockFile.log.shouldLog(Log.DEBUG))
BlockFile.log.debug("Flushed data for page " + this.page + " containing " + this.nKeys + '/' + this.spanSize);
}
/**
@ -97,8 +98,8 @@ public class IBSkipSpan extends BSkipSpan {
super.loadData();
if (this.nKeys > 0)
this.firstKey = this.keys[0];
if (DEBUG)
System.err.println("Loaded data for page " + this.page + " containing " + this.nKeys + '/' + this.spanSize + " first key: " + this.firstKey);
if (BlockFile.log.shouldLog(Log.DEBUG))
BlockFile.log.debug("Loaded data for page " + this.page + " containing " + this.nKeys + '/' + this.spanSize + " first key: " + this.firstKey);
}
/**
@ -124,8 +125,8 @@ public class IBSkipSpan extends BSkipSpan {
BlockFile.log.error("Null deserialized first key in page " + curPage);
repair(1);
}
if (DEBUG)
System.err.println("Loaded header for page " + this.page + " containing " + this.nKeys + '/' + this.spanSize + " first key: " + this.firstKey);
if (BlockFile.log.shouldLog(Log.DEBUG))
BlockFile.log.debug("Loaded header for page " + this.page + " containing " + this.nKeys + '/' + this.spanSize + " first key: " + this.firstKey);
}
/**
@ -221,8 +222,8 @@ public class IBSkipSpan extends BSkipSpan {
protected IBSkipSpan() { }
public IBSkipSpan(BlockFile bf, BSkipList bsl, int spanPage, Serializer key, Serializer val) throws IOException {
if (DEBUG)
System.err.println("New ibss page " + spanPage);
if (BlockFile.log.shouldLog(Log.DEBUG))
BlockFile.log.debug("New ibss page " + spanPage);
BSkipSpan.loadInit(this, bf, bsl, spanPage, key, val);
loadFirstKey();
this.next = null;

45
core/java/src/net/metanotion/io/data/IdentityBytes.java Normal file
View File

@ -0,0 +1,45 @@
/*
Copyright (c) 2006, Matthew Estes
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
* Neither the name of Metanotion Software nor the names of its
contributors may be used to endorse or promote products derived from this
software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package net.metanotion.io.data;
import net.metanotion.io.Serializer;
/**
* May be used to scan and repair the database nondestructively.
* Will never return null.
* Added by I2P.
*/
public class IdentityBytes implements Serializer {
/** @return byte[] */
public byte[] getBytes(Object o) { return (byte[])o; }
/** @return b */
public Object construct(byte[] b) { return b; }
}

50
core/java/src/net/metanotion/io/data/UTF8StringBytes.java Normal file
View File

@ -0,0 +1,50 @@
/*
Copyright (c) 2006, Matthew Estes
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
* Neither the name of Metanotion Software nor the names of its
contributors may be used to endorse or promote products derived from this
software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package net.metanotion.io.data;
import java.io.UnsupportedEncodingException;
import net.metanotion.io.Serializer;
/**
* Added by I2P
*/
public class UTF8StringBytes implements Serializer {
public byte[] getBytes(Object o) {
try {
return ((String) o).getBytes("UTF-8");
} catch (UnsupportedEncodingException uee) { throw new Error("Unsupported Encoding"); }
}
public Object construct(byte[] b) {
try {
return new String(b, "UTF-8");
} catch (UnsupportedEncodingException uee) { throw new Error("Unsupported Encoding"); }
}
}

66
core/java/src/net/metanotion/util/skiplist/SkipLevels.java
View File

@ -28,12 +28,17 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package net.metanotion.util.skiplist;
import net.metanotion.io.block.BlockFile;
public class SkipLevels {
/* "Next" pointers
The highest indexed level is the "highest" level in the list.
The "bottom" level is the direct pointer to a SkipSpan.
*/
/*
* "Next" pointers
* The highest indexed level is the "highest" level in the list.
* The "bottom" level is the direct pointer to a SkipSpan.
*/
// levels is almost final
public SkipLevels[] levels;
// bottom is final
public SkipSpan bottom;
public SkipLevels newInstance(int levels, SkipSpan ss, SkipList sl) { return new SkipLevels(levels, ss); }
@ -42,18 +47,15 @@ public class SkipLevels {
protected SkipLevels() { }
public SkipLevels(int size, SkipSpan span) {
if(size < 1) { throw new Error("Invalid Level Skip size"); }
if(size < 1) { throw new RuntimeException("Invalid Level Skip size"); }
levels = new SkipLevels[size];
bottom = span;
}
public void print() {
SkipLevels prev = null;
SkipLevels max = null;
System.out.print("SL:" + key() + "::");
for(int i=0;i<levels.length;i++) {
if(levels[i] != null) {
max = levels[i];
System.out.print(i + "->" + levels[i].key() + " ");
} else {
System.out.print(i + "->() ");
@ -93,7 +95,6 @@ public class SkipLevels {
}
public Object[] remove(int start, Comparable key, SkipList sl) {
SkipSpan ss = null;
Object[] res = null;
SkipLevels slvls = null;
for(int i=Math.min(start, levels.length - 1);i>=0;i--) {
@ -127,42 +128,61 @@ public class SkipLevels {
return res;
}
/**
* @return the new level if it caused a split and we made a new level,
* and the new level is taller than our level;
* else null if it went in an existing level or the new level is our height or less.
*/
public SkipLevels put(int start, Comparable key, Object val, SkipList sl) {
SkipSpan ss = null;
SkipLevels slvls = null;
for(int i=Math.min(start, levels.length - 1);i>=0;i--) {
boolean modified = false;
for(int i = Math.min(start, levels.length - 1); i >= 0; i--) {
// is key equal to or after the start of the level?
if((levels[i] != null) && (levels[i].key().compareTo(key) <= 0)) {
slvls = levels[i].put(i, key, val, sl);
SkipLevels slvls = levels[i].put(i, key, val, sl);
if(slvls != null) {
for(int j=i+1;j<(Math.min(slvls.levels.length,levels.length));j++) {
for (int j = i + 1; j < Math.min(slvls.levels.length, levels.length); j++) {
// he points to where we used to point
// and we now point to him
slvls.levels[j] = levels[j];
levels[j] = slvls;
modified = true;
}
if(levels.length < slvls.levels.length) {
this.flush();
if (modified) {
this.flush();
slvls.flush();
}
return slvls;
}
}
this.flush();
if (modified)
this.flush();
return null;
}
}
ss = bottom.put(key,val,sl);
SkipSpan ss = bottom.put(key,val,sl);
if(ss!=null) {
int height = sl.generateColHeight();
if(height != 0) {
slvls = this.newInstance(height, ss, sl);
SkipLevels slvls = this.newInstance(height, ss, sl);
for(int i=0;i<(Math.min(height,levels.length));i++) {
// he points to where we used to point
// and we now point to him
slvls.levels[i] = levels[i];
levels[i] = slvls;
modified = true;
}
if (modified) {
this.flush();
slvls.flush();
}
if(levels.length < height)
return slvls;
}
this.flush();
if(levels.length >= height) { return null; }
return slvls;
} else {
return null;
}
return null;
}
public void blvlck(boolean fix, int depth) {}
}

30
core/java/src/net/metanotion/util/skiplist/SkipList.java
View File

@ -32,6 +32,8 @@ import java.util.Random;
import net.i2p.util.RandomSource;
import net.metanotion.io.block.BlockFile;
public class SkipList {
protected SkipSpan first;
protected SkipLevels stack;
@ -45,7 +47,7 @@ public class SkipList {
protected SkipList() { }
public SkipList(int span) {
if(span < 1) { throw new Error("Span size too small"); }
if(span < 1) { throw new RuntimeException("Span size too small"); }
first = new SkipSpan(span);
stack = new SkipLevels(1, first);
spans = 1;
@ -54,24 +56,30 @@ public class SkipList {
public int size() { return size; }
/**
* @return log2(spans), minimum 4
*/
public int maxLevels() {
int hob = 0, s = spans;
while(spans > 0) {
while(s > 0) {
hob++;
spans = spans / 2;
s /= 2;
}
return (hob > 4) ? hob : 4;
return Math.max(hob, 4);
}
/**
* @return 0..maxLevels()
*/
public int generateColHeight() {
int bits = rng.nextInt();
boolean cont = true;
int res=0;
for(res=0; cont; res++) {
cont = ((bits % 2) == 0) ? true : false;
bits = bits / 2;
int max = maxLevels();
for(int res = 0; res < max; res++) {
if (bits % 2 == 0)
return res;
bits /= 2;
}
return Math.max(0, Math.min(res, maxLevels()));
return max;
}
public void put(Comparable key, Object val) {
@ -79,6 +87,8 @@ public class SkipList {
if(val == null) { throw new NullPointerException(); }
SkipLevels slvls = stack.put(stack.levels.length - 1, key, val, this);
if(slvls != null) {
// grow our stack
BlockFile.log.info("Top level old hgt " + stack.levels.length + " new hgt " + slvls.levels.length);
SkipLevels[] levels = new SkipLevels[slvls.levels.length];
for(int i=0;i < slvls.levels.length; i++) {
if(i < stack.levels.length) {

14
core/java/src/net/metanotion/util/skiplist/SkipSpan.java
View File

@ -45,9 +45,11 @@ public class SkipSpan {
}
public void print() {
System.out.println("Span");
for(int i=0;i<nKeys;i++) {
System.out.println("\t" + keys[i] + " => " + vals[i]);
System.out.println("Span containing " + nKeys + " keys");
if (nKeys > 0 && keys != null && vals != null) {
for(int i=0;i<nKeys;i++) {
System.out.println("\t" + keys[i] + " => " + vals[i]);
}
}
if(next != null) { next.print(); }
}
@ -155,6 +157,9 @@ public class SkipSpan {
this.next.flush();
}
/**
* @return the new span if it caused a split, else null if it went in this span
*/
private SkipSpan insert(int loc, Comparable key, Object val, SkipList sl) {
sl.size++;
if(nKeys == keys.length) {
@ -170,6 +175,9 @@ public class SkipSpan {
}
}
/**
* @return the new span if it caused a split, else null if it went in an existing span
*/
public SkipSpan put(Comparable key, Object val, SkipList sl) {
if(nKeys == 0) {
sl.size++;