Blockfile: Fixes for finishing interrupted database migration

by tracking version of each list, and catching some exceptions

core/java/src/net/i2p/client/naming/BlockfileNamingService.java

@@ -121,6 +121,7 @@ public class BlockfileNamingService extends DummyNamingService {
     private static final String REVERSE_SKIPLIST = "%%__REVERSE__%%";
     private static final String PROP_INFO = "info";
     private static final String PROP_VERSION = "version";
+    private static final String PROP_LISTVERSION = "listversion";
     private static final String PROP_LISTS = "lists";
     private static final String PROP_CREATED = "created";
     private static final String PROP_UPGRADED = "upgraded";
@@ -406,12 +407,29 @@ public class BlockfileNamingService extends DummyNamingService {
             // version 3 -> version 4
             // support multiple destinations per hostname
             if (VersionComparator.comp(_version, "4") < 0) {
+                SkipList<String, Properties> hdr = _bf.getIndex(INFO_SKIPLIST, _stringSerializer, _infoSerializer);
+                if (hdr == null)
+                    throw new IOException("No db header");
+                Properties info = hdr.get(PROP_INFO);
+                if (info == null)
+                    throw new IOException("No header info");
                 for (String list : _lists) { 
                     try {
-                        if (_log.shouldWarn())
+                        // so that we can handle an aborted upgrade,
-                            _log.warn("Upgrading " + list + " from database version 3 to 4");
+                        // we keep track of the version of each list
-                        _bf.reformatIndex(list, _stringSerializer, _destSerializerV1,
+                        String vprop = PROP_LISTVERSION + '_' + list;
-                                          _stringSerializer, _destSerializerV4);
+                        String listVersion = info.getProperty(vprop);
+                        if (listVersion == null || VersionComparator.comp(listVersion, "4") < 0) {
+                            if (_log.shouldWarn())
+                                _log.warn("Upgrading " + list + " from database version 3 to 4");
+                            _bf.reformatIndex(list, _stringSerializer, _destSerializerV1,
+                                              _stringSerializer, _destSerializerV4);
+                            info.setProperty(vprop, "4");
+                            hdr.put(PROP_INFO, info);
+                        } else {
+                            if (_log.shouldWarn())
+                                _log.warn("Partial upgrade, " + list + " already at version " + listVersion);
+                        }
                     } catch (IOException ioe) {
                         _log.error("Failed upgrade of list " + list + " to version 4", ioe);
                     }
@@ -1537,7 +1555,6 @@ public class BlockfileNamingService extends DummyNamingService {
             ctxProps.setProperty(PROP_FORCE, "true");
         I2PAppContext ctx = new I2PAppContext(ctxProps);
         BlockfileNamingService bns = new BlockfileNamingService(ctx);
-/****
         Properties sprops = new Properties();
         String lname = "privatehosts.txt";
         sprops.setProperty("list", lname);
@@ -1549,6 +1566,7 @@ public class BlockfileNamingService extends DummyNamingService {
         sprops.setProperty("list", lname);
         System.out.println("List " + lname + " contains " + bns.size(sprops));
 
+/****
         List<String> names = null;
         Properties props = new Properties();
         try {

core/java/src/net/metanotion/io/RAIFile.java

@@ -36,8 +36,8 @@ import java.io.IOException;
 import java.io.RandomAccessFile;
 
 public class RAIFile implements RandomAccessInterface, DataInput, DataOutput {
-	private File f;
+	private final File f;
-	private RandomAccessFile delegate;
+	private final RandomAccessFile delegate;
 	private final boolean r, w;
 
 	public RAIFile(RandomAccessFile file) throws FileNotFoundException {

core/java/src/net/metanotion/io/block/BlockFile.java

@@ -36,6 +36,7 @@ import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
+import java.util.NoSuchElementException;
 import java.util.Set;
 
 import net.metanotion.io.RAIFile;
@@ -526,7 +527,12 @@ public class BlockFile implements Closeable {
 			return;
 		long start = System.currentTimeMillis();
 		String tmpName = "---tmp---" + name + "---tmp---";
-		BSkipList<K, V> tmp = makeIndex(tmpName, newKey, newVal);
+		BSkipList<K, V> tmp = getIndex(tmpName, newKey, newVal);
+		if (tmp != null) {
+			log.logAlways(Log.WARN, "Continuing on aborted reformat of list " + name);
+		} else {
+			tmp = makeIndex(tmpName, newKey, newVal);
+		}
 
 		// It could be much more efficient to do this at the
 		// SkipSpan layer but that's way too hard.
@@ -536,8 +542,12 @@ public class BlockFile implements Closeable {
 		while (true) {
 			SkipIterator<K, V> iter = old.iterator();
 			for (int i = 0; iter.hasNext() && i < loop; i++) {
-				keys.add(iter.nextKey());
+				try {
-				vals.add(iter.next());
+					keys.add(iter.nextKey());
+					vals.add(iter.next());
+				} catch (NoSuchElementException nsee) {
+					throw new IOException("Unable to reformat corrupt list " + name, nsee);
+				}
 			}
 			// save state, as deleting corrupts the iterator
 			boolean done = !iter.hasNext();

core/java/src/net/metanotion/io/block/index/BSkipList.java

@@ -104,7 +104,8 @@ public class BSkipList<K extends Comparable<? super K>, V> extends SkipList<K, V
 		if (bf.file.canWrite() &&
 		    (levelCount != levelHash.size() || spans != spanHash.size() || size != total)) {
 			if (bf.log.shouldLog(Log.WARN))
-				bf.log.warn("On-disk counts were " + levelCount + " / " + spans + " / " +  size + ", correcting");
+				bf.log.warn("On-disk counts were " + levelCount + " levels / " + spans +
+				            " spans / " +  size + " entries, correcting to " + total + " entries");
 			size = total;
 			flush();
 		}

history.txt

@@ -1,3 +1,16 @@
+2016-04-20 zzz
+ * Addressbook:
+   - Add sign/verify methods for inner signature
+   - Add preliminary handling of incoming actions
+ * Blockfile:
+   - Add generics
+   - Add method to change serialization schema for a skiplist
+   - Fix delIndex() method
+ * BlockfileNamingService:
+   - New database version 4, for multiple destinations per hostname
+   - Disallow database version higher than supported
+ * NamingService: Add new API methods for multiple Destinations per hostname
+
 2016-04-17 zzz
  * Addressbook:
    - Several cleanups and refactoring

router/java/src/net/i2p/router/RouterVersion.java

@@ -18,7 +18,7 @@ public class RouterVersion {
     /** deprecated */
     public final static String ID = "Monotone";
     public final static String VERSION = CoreVersion.VERSION;
-    public final static long BUILD = 3;
+    public final static long BUILD = 4;
 
     /** for example "-test" */
     public final static String EXTRA = "";