echelon/i2p.i2p
1
0
Fork 0
forked from I2P_Developers/i2p.i2p
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

- Isolate span read failures to that span

Browse Source 
- Don't keep separate count of spans and levels, use BSL HashMap sizes
- Fix SkipList counts when read in
This commit is contained in:
zzz
2011-03-28 20:33:57 +00:00
parent a818e84dcf
commit 8ae398d786
5 changed files with 107 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
core/java/src/net/metanotion/io/block/index/BSkipList.java
View File

@ -82,8 +82,8 @@ public class BSkipList extends SkipList {
firstSpanPage = bf.file.readUnsignedInt();
firstLevelPage = bf.file.readUnsignedInt();
size = bf.file.readUnsignedInt();
spans = bf.file.readUnsignedInt();
levelCount = bf.file.readUnsignedInt();
int spans = bf.file.readInt();
int levelCount = bf.file.readInt();
//System.out.println(size + " " + spans);
this.fileOnly = fileOnly;
@ -92,8 +92,18 @@ public class BSkipList extends SkipList {
else
first = new BSkipSpan(bf, this, firstSpanPage, key, val);
stack = new BSkipLevels(bf, firstLevelPage, this);
int total = 0;
for (BSkipSpan ss : spanHash.values()) {
total += ss.nKeys;
}
if (BlockFile.log.shouldLog(Log.DEBUG))
BlockFile.log.debug("Loaded " + this + " cached " + levelHash.size() + " levels and " + spanHash.size() + " spans");
BlockFile.log.debug("Loaded " + this + " cached " + levelHash.size() + " levels and " + spanHash.size() + " spans with " + total + " entries");
if (levelCount != levelHash.size() || spans != spanHash.size() || size != total) {
if (BlockFile.log.shouldLog(Log.WARN))
BlockFile.log.warn("On-disk counts were " + levelCount + " / " + spans + " / " + size + ", correcting");
size = total;
flush();
}
//rng = new Random(System.currentTimeMillis());
}
@ -117,8 +127,8 @@ public class BSkipList extends SkipList {
bf.file.writeInt(firstSpanPage);
bf.file.writeInt(firstLevelPage);
bf.file.writeInt(Math.max(0, size));
bf.file.writeInt(Math.max(0, spans));
bf.file.writeInt(Math.max(0, levelCount));
bf.file.writeInt(spanHash.size());
bf.file.writeInt(levelHash.size());
} catch (IOException ioe) { throw new RuntimeException("Error writing to database", ioe); }
}
@ -164,11 +174,21 @@ public class BSkipList extends SkipList {
BSkipLevels.init(bf, firstLevel, firstSpan, 4);
}
/**
* @return log2(span count), minimum 4
*/
@Override
public int maxLevels() {
int max = super.maxLevels();
int cells = (BlockFile.PAGESIZE - BSkipLevels.HEADER_LEN) / 4;
return Math.min(cells, max);
int hob = 0;
int s = spanHash.size();
while(s > 0) {
hob++;
s /= P;
}
int max = Math.max(hob, super.maxLevels());
// 252
//int cells = (BlockFile.PAGESIZE - BSkipLevels.HEADER_LEN) / 4;
return Math.min(BSkipLevels.MAX_SIZE, max);
}
@Override
@ -203,8 +223,8 @@ public class BSkipList extends SkipList {
public void bslck(boolean isMeta, boolean fix) {
BlockFile.log.warn(" size " + this.size);
BlockFile.log.warn(" spans " + this.spans);
BlockFile.log.warn(" levels " + this.levelCount);
BlockFile.log.warn(" spans " + this.spanHash.size());
BlockFile.log.warn(" levels " + this.levelHash.size());
BlockFile.log.warn(" skipPage " + this.skipPage);
BlockFile.log.warn(" firstSpanPage " + this.firstSpanPage);
BlockFile.log.warn(" firstLevelPage " + this.firstLevelPage);

56
core/java/src/net/metanotion/io/block/index/BSkipSpan.java
View File

@ -161,6 +161,8 @@ public class BSkipSpan extends SkipSpan {
return;
bf.file.writeShort((short) keys.length);
bf.file.writeShort((short) nKeys);
if (nKeys <= 0 && prev != null)
BlockFile.log.error("Flushing with no entries?" + this, new Exception());
int ksz, vsz;
int curPage = this.page;
@ -264,8 +266,11 @@ public class BSkipSpan extends SkipSpan {
bss.nextPage = bf.file.readUnsignedInt();
bss.spanSize = bf.file.readUnsignedShort();
bss.nKeys = bf.file.readUnsignedShort();
if(bss.spanSize < 1 || bss.spanSize > SkipSpan.MAX_SIZE || bss.nKeys > bss.spanSize)
throw new IOException("Invalid span size " + bss.nKeys + " / "+ bss.spanSize);
if(bss.spanSize < 1 || bss.spanSize > SkipSpan.MAX_SIZE || bss.nKeys > bss.spanSize) {
BlockFile.log.error("Invalid span size " + bss.nKeys + " / "+ bss.spanSize);
bss.nKeys = 0;
bss.spanSize = bf.spanSize;
}
}
/**
@ -298,10 +303,13 @@ public class BSkipSpan extends SkipSpan {
for(int i=0;i<this.nKeys;i++) {
if((pageCounter[0] + 4) > BlockFile.PAGESIZE) {
BlockFile.pageSeek(this.bf.file, curNextPage[0]);
curPage = curNextPage[0];
int magic = bf.file.readInt();
if (magic != BlockFile.MAGIC_CONT)
throw new IOException("Bad SkipSpan magic number 0x" + Integer.toHexString(magic) + " on page " + curPage);
if (magic != BlockFile.MAGIC_CONT) {
BlockFile.log.error("Lost " + (this.nKeys - i) + " entries - Bad SkipSpan magic number 0x" + Integer.toHexString(magic) + " on page " + curNextPage[0]);
lostEntries(i, curPage);
break;
}
curPage = curNextPage[0];
curNextPage[0] = this.bf.file.readUnsignedInt();
pageCounter[0] = CONT_HEADER_LEN;
}
@ -310,8 +318,15 @@ public class BSkipSpan extends SkipSpan {
pageCounter[0] +=4;
byte[] k = new byte[ksz];
byte[] v = new byte[vsz];
curPage = this.bf.readMultiPageData(k, curPage, pageCounter, curNextPage);
curPage = this.bf.readMultiPageData(v, curPage, pageCounter, curNextPage);
int lastGood = curPage;
try {
curPage = this.bf.readMultiPageData(k, curPage, pageCounter, curNextPage);
curPage = this.bf.readMultiPageData(v, curPage, pageCounter, curNextPage);
} catch (IOException ioe) {
BlockFile.log.error("Lost " + (this.nKeys - i) + " entries - Error loading " + this + " on page " + curPage, ioe);
lostEntries(i, lastGood);
break;
}
// System.out.println("i=" + i + ", Page " + curPage + ", offset " + pageCounter[0] + " ksz " + ksz + " vsz " + vsz);
this.keys[i] = (Comparable) this.keySer.construct(k);
this.vals[i] = this.valSer.construct(v);
@ -336,6 +351,33 @@ public class BSkipSpan extends SkipSpan {
}
}
/**
* Attempt to recover from corrupt data in this span.
* All entries starting with firstBadEntry are lost.
* Zero out the overflow page on lastGoodPage,
* and corect the number of entries in the first page.
* We don't attempt to free the lost continuation pages.
*/
protected void lostEntries(int firstBadEntry, int lastGoodPage) {
try {
this.nKeys = firstBadEntry;
// zero overflow page pointer
BlockFile.pageSeek(this.bf.file, lastGoodPage);
bf.file.skipBytes(4); // skip magic
bf.file.writeInt(0);
// write new number of keys
if (lastGoodPage != this.page) {
BlockFile.pageSeek(this.bf.file, this.page);
bf.file.skipBytes(18);
} else {
bf.file.skipBytes(10);
}
bf.file.writeShort(this.nKeys);
} catch (IOException ioe) {
BlockFile.log.error("Error while recovering from corruption of " + this, ioe);
}
}
protected BSkipSpan(BlockFile bf, BSkipList bsl) {
this.bf = bf;
this.bsl = bsl;

27
core/java/src/net/metanotion/io/block/index/IBSkipSpan.java
View File

@ -172,10 +172,13 @@ public class IBSkipSpan extends BSkipSpan {
for(int i=0;i<this.nKeys;i++) {
if((pageCounter[0] + 4) > BlockFile.PAGESIZE) {
BlockFile.pageSeek(this.bf.file, curNextPage[0]);
curPage = curNextPage[0];
int magic = bf.file.readInt();
if (magic != BlockFile.MAGIC_CONT)
throw new IOException("Bad SkipSpan continuation magic number 0x" + Integer.toHexString(magic) + " on page " + curPage);
if (magic != BlockFile.MAGIC_CONT) {
BlockFile.log.error("Lost " + (this.nKeys - i) + " entries - Bad SkipSpan magic number 0x" + Integer.toHexString(magic) + " on page " + curNextPage[0]);
lostEntries(i, curPage);
break;
}
curPage = curNextPage[0];
curNextPage[0] = this.bf.file.readUnsignedInt();
pageCounter[0] = CONT_HEADER_LEN;
}
@ -183,7 +186,13 @@ public class IBSkipSpan extends BSkipSpan {
int vsz = this.bf.file.readUnsignedShort();
pageCounter[0] +=4;
byte[] k = new byte[ksz];
curPage = this.bf.readMultiPageData(k, curPage, pageCounter, curNextPage);
try {
curPage = this.bf.readMultiPageData(k, curPage, pageCounter, curNextPage);
} catch (IOException ioe) {
BlockFile.log.error("Lost " + (this.nKeys - i) + " entries - Error loading " + this + " on page " + curPage, ioe);
lostEntries(i, curPage);
break;
}
//System.out.println("i=" + i + ", Page " + curPage + ", offset " + pageCounter[0] + " ksz " + ksz + " vsz " + vsz);
Comparable ckey = (Comparable) this.keySer.construct(k);
if (ckey == null) {
@ -197,7 +206,13 @@ public class IBSkipSpan extends BSkipSpan {
if (diff == 0) {
//System.err.println("Found " + key + " at " + i + " (first: " + this.firstKey + ')');
byte[] v = new byte[vsz];
curPage = this.bf.readMultiPageData(v, curPage, pageCounter, curNextPage);
try {
curPage = this.bf.readMultiPageData(v, curPage, pageCounter, curNextPage);
} catch (IOException ioe) {
BlockFile.log.error("Lost " + (this.nKeys - i) + " entries - Error loading " + this + " on page " + curPage, ioe);
lostEntries(i, curPage);
break;
}
Object rv = this.valSer.construct(v);
if (rv == null) {
BlockFile.log.error("Null deserialized value in entry " + i + " page " + curPage +
@ -224,6 +239,7 @@ public class IBSkipSpan extends BSkipSpan {
}
private void repair(int fail) {
/***** needs work
try {
loadData(false);
if (this.nKeys > 0)
@ -233,6 +249,7 @@ public class IBSkipSpan extends BSkipSpan {
} catch (IOException ioe) {
BlockFile.log.error("Failed to repair corruption of " + fail + " entries", ioe);
}
*****/
}
private IBSkipSpan(BlockFile bf, BSkipList bsl) {

2
core/java/src/net/metanotion/util/skiplist/SkipLevels.java
View File

@ -187,7 +187,6 @@ public class SkipLevels {
}
res[1] = null;
}
sl.delSpan(res[1] != null);
}
if((bottom.nKeys == 0) && (sl.first != bottom)) { this.killInstance(); }
return res;
@ -244,7 +243,6 @@ public class SkipLevels {
if(levels.length < height)
return slvls;
}
sl.addSpan(height != 0);
}
return null;
}

33
core/java/src/net/metanotion/util/skiplist/SkipList.java
View File

@ -36,7 +36,7 @@ import net.i2p.util.RandomSource;
public class SkipList {
/** the probability of each next higher level */
private static final int P = 2;
protected static final int P = 2;
private static final int MIN_SLOTS = 4;
// these two are really final
protected SkipSpan first;
@ -45,8 +45,6 @@ public class SkipList {
public static final Random rng = RandomSource.getInstance();
protected int size;
protected int spans;
protected int levelCount;
public void flush() { }
protected SkipList() { }
@ -60,8 +58,6 @@ public class SkipList {
throw new IllegalArgumentException("Invalid span size");
first = new SkipSpan(span);
stack = new SkipLevels(1, first);
spans = 1;
levelCount = 1;
//rng = new Random(System.currentTimeMillis());
}
@ -76,29 +72,12 @@ public class SkipList {
size--;
}
public void addSpan(boolean addLevel) {
spans++;
if (addLevel)
levelCount++;
}
public void delSpan(boolean delLevel) {
if (spans > 0)
spans--;
if (delLevel && levelCount > 0)
levelCount--;
}
/**
* @return log2(spans), minimum 4
* @return 4 since we don't track span count here any more - see override
* Fix if for some reason you want a huge in-memory skiplist.
*/
public int maxLevels() {
int hob = 0, s = spans;
while(s > 0) {
hob++;
s /= P;
}
return Math.max(hob, MIN_SLOTS);
return MIN_SLOTS;
}
/**
@ -157,13 +136,13 @@ public class SkipList {
/** dumps all the skip levels */
public void printSL() {
System.out.println("List size " + size + " spans " + spans);
System.out.println("List size " + size);
System.out.println(stack.printAll());
}
/** dumps all the data */
public void print() {
System.out.println("List size " + size + " spans " + spans);
System.out.println("List size " + size);
System.out.println(first.print());
}