Sybil: Refactoring to separate analysis and display, part 4

Load/store offline analysis
Split up sections into separate tabs

apps/routerconsole/java/src/net/i2p/router/sybil/Analysis.java

@@ -249,8 +249,10 @@ public class Analysis {
         return avg;
     }
 
-    private double closestDistance(Hash h, List<RouterInfo> ris) {
+    private static final BigInteger BI_MAX = (new BigInteger("2")).pow(256);
-        BigInteger min = (new BigInteger("2")).pow(256);
+
+    private static double closestDistance(Hash h, List<RouterInfo> ris) {
+        BigInteger min = BI_MAX;
         for (RouterInfo info : ris) {
             BigInteger dist = HashDistance.getDistance(h, info.getHash());
             if (dist.compareTo(min) < 0)

apps/routerconsole/java/src/net/i2p/router/sybil/PersistSybil.java

@@ -95,7 +95,7 @@ public class PersistSybil {
                 rv.add(Long.valueOf(d));
             } catch (NumberFormatException nfe) {}
         }
-        Collections.sort(rv);
+        Collections.sort(rv, Collections.reverseOrder());
         return rv;
     }
 

apps/routerconsole/java/src/net/i2p/router/web/helpers/NetDbHelper.java

@@ -13,7 +13,8 @@ public class NetDbHelper extends HelperBase {
     private String _version;
     private String _country;
     private String _family, _caps, _ip, _sybil, _mtu, _ssucaps, _ipv6, _transport;
-    private int _full, _port, _cost, _page;
+    private int _full, _port, _cost, _page, _mode;
+    private long _date;
     private int _limit = DEFAULT_LIMIT;
     private boolean _lease;
     private boolean _debug;
@@ -141,6 +142,20 @@ public class NetDbHelper extends HelperBase {
         } catch (NumberFormatException nfe) {}
     }
 
+    /** @since 0.9.38 */
+    public void setMode(String f) {
+        try {
+            _mode = Integer.parseInt(f);
+        } catch (NumberFormatException nfe) {}
+    }
+
+    /** @since 0.9.38 */
+    public void setDate(String f) {
+        try {
+            _date = Long.parseLong(f);
+        } catch (NumberFormatException nfe) {}
+    }
+
     public void setFull(String f) {
         try {
             _full = Integer.parseInt(f);
@@ -198,7 +213,7 @@ public class NetDbHelper extends HelperBase {
             else if (_lease)
                 renderer.renderLeaseSetHTML(_out, _debug);
             else if (_full == 3)
-                (new SybilRenderer(_context)).getNetDbSummary(_out);
+                (new SybilRenderer(_context)).getNetDbSummary(_out, _mode, _date);
             else if (_full == 4)
                 renderLookupForm();
             else

apps/routerconsole/java/src/net/i2p/router/web/helpers/SybilRenderer.java

@@ -12,6 +12,7 @@ import java.util.Collections;
 import java.util.Comparator;
 import java.util.Date;
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
@@ -32,6 +33,7 @@ import net.i2p.router.peermanager.DBHistory;
 import net.i2p.router.peermanager.PeerProfile;
 import net.i2p.router.sybil.Analysis;
 import net.i2p.router.sybil.Pair;
+import net.i2p.router.sybil.PersistSybil;
 import net.i2p.router.sybil.Points;
 import static net.i2p.router.sybil.Util.biLog2;
 import net.i2p.router.tunnel.pool.TunnelPool;
@@ -70,9 +72,12 @@ public class SybilRenderer {
 
     /**
      *   Entry point
+     *
+     *  @param mode what tab to show
+     *  @param date only for mode = 12
      */
-    public String getNetDbSummary(Writer out) throws IOException {
+    public String getNetDbSummary(Writer out, int mode, long date) throws IOException {
-        renderRouterInfoHTML(out, (String)null);
+        renderRouterInfoHTML(out, mode, date);
         return "";
     }
 
@@ -115,11 +120,11 @@ public class SybilRenderer {
     /**
      *  The whole thing
      *
-     *  @param routerPrefix ignored
+     *  @param mode what tab to show
+     *  @param date only for mode = 12
      */
-    private void renderRouterInfoHTML(Writer out, String routerPrefix) throws IOException {
+    private void renderRouterInfoHTML(Writer out, int mode, long date) throws IOException {
         Hash us = _context.routerHash();
-        Hash ourRKey = _context.router().getRouterInfo().getRoutingKey();
         Analysis analysis = new Analysis(_context);
         List<RouterInfo> ris = analysis.getFloodfills(us);
         if (ris.isEmpty()) {
@@ -131,24 +136,117 @@ public class SybilRenderer {
         buf.append("<p id=\"sybilinfo\"><b>This is an experimental network database tool for debugging and analysis. Do not panic even if you see warnings below. " +
                    "Possible \"threats\" are summarized at the bottom, however these are unlikely to be real threats. " +
                    "If you see anything you would like to discuss with the devs, contact us on IRC #i2p-dev.</b></p>" +
-                   "<div id=\"sybilnav\"><ul><li><a href=\"#known\">FF Summary</a>" +
+                   "<div id=\"sybilnav\"><ul><li><a href=\"netdb?f=3\">Review stored analysis</a>" +
-                   "</li><li><a href=\"#family\">Same Family</a>" +
+                   "</li><li><a href=\"netdb?f=3&amp;m=13\">Run new analysis</a>" +
-                   "</li><li><a href=\"#ourIP\">IP close to us</a>" +
+                   "</li><li><a href=\"netdb?f=3&amp;m=1\">Floodfill Summary</a>" +
-                   "</li><li><a href=\"#sameIP\">Same IP</a>" +
+                   "</li><li><a href=\"netdb?f=3&amp;m=2\">Same Family</a>" +
-                   "</li><li><a href=\"#same24\">Same /24</a>" +
+                   "</li><li><a href=\"netdb?f=3&amp;m=3\">IP close to us</a>" +
-                   "</li><li><a href=\"#same16\">Same /16</a>" +
+                   "</li><li><a href=\"netdb?f=3&amp;m=4\">Same IP</a>" +
-                   "</li><li><a href=\"#pairs\">Pair distance</a>" +
+                   "</li><li><a href=\"netdb?f=3&amp;m=5\">Same /24</a>" +
-                   "</li><li><a href=\"#ritoday\">Close to us</a>" +
+                   "</li><li><a href=\"netdb?f=3&amp;m=6\">Same /16</a>" +
-                   "</li><li><a href=\"#ritmrw\">Close to us tomorrow</a>" +
+                   "</li><li><a href=\"netdb?f=3&amp;m=7\">Pair distance</a>" +
-                   "</li><li><a href=\"#dht\">DHT neighbors</a>" +
+                   "</li><li><a href=\"netdb?f=3&amp;m=8\">Close to us</a>" +
-                   "</li><li><a href=\"#dest\">Close to our destinations</a>" +
+                   "</li><li><a href=\"netdb?f=3&amp;m=9\">Close to us tomorrow</a>" +
-                   "</li><li><a href=\"#threats\">Highest threats</a>" +
+                   "</li><li><a href=\"netdb?f=3&amp;m=10\">DHT neighbors</a>" +
+                   "</li><li><a href=\"netdb?f=3&amp;m=11\">Close to our destinations</a>" +
                    "</li></ul></div>");
-
+        writeBuf(out, buf);
-        renderRouterInfo(buf, _context.router().getRouterInfo(), null, true, false);
-        buf.append("<h3 id=\"known\" class=\"sybils\">Known Floodfills: ").append(ris.size()).append("</h3>");
 
         double avgMinDist = analysis.getAvgMinDist(ris);
+        Map<Hash, Points> points = new HashMap<Hash, Points>(64);
+
+        if (mode == 0) {
+            renderOverview(out, buf, analysis);
+        } else if (mode == 1) {
+            renderFFSummary(out, buf, ris, avgMinDist);
+        } else if (mode == 2) {
+            renderFamilySummary(out, buf, analysis, ris, points);
+        } else if (mode == 3) {
+            renderIPUsSummary(out, buf, analysis, ris, points);
+        } else if (mode == 4) {
+            renderIP32Summary(out, buf, analysis, ris, points);
+        } else if (mode == 5) {
+            renderIP24Summary(out, buf, analysis, ris, points);
+        } else if (mode == 6) {
+            renderIP16Summary(out, buf, analysis, ris, points);
+        } else if (mode == 7) {
+            renderPairSummary(out, buf, analysis, ris, points);
+        } else if (mode == 8) {
+            renderCloseSummary(out, buf, analysis, avgMinDist, ris, points);
+        } else if (mode == 9) {
+            renderCloseTmrwSummary(out, buf, analysis, us, avgMinDist, ris, points);
+        } else if (mode == 10) {
+            renderDHTSummary(out, buf, analysis, us, avgMinDist, ris, points);
+        } else if (mode == 11) {
+            renderDestSummary(out, buf, analysis, avgMinDist, ris, points);
+        } else if (mode == 12) {
+            PersistSybil ps = new PersistSybil(_context);
+            try {
+                points = ps.load(date);
+            } catch (IOException ioe) {
+                out.write("<b>No analysis found for " + new Date(date) + "</b>");
+                return;
+            }
+            if (points.isEmpty()) {
+                out.write("<b>No analysis found for " + new Date(date) + "</b>");
+            } else {
+                renderThreatsHTML(out, buf, date, points);
+            }
+        } else if (mode == 13) {
+            long now = _context.clock().now();
+            points = analysis.backgroundAnalysis();
+            if (!points.isEmpty()) {
+                PersistSybil ps = new PersistSybil(_context);
+                try {
+                    ps.store(now, points);
+                } catch (IOException ioe) {
+                    out.write("<b>Failed to store analysis " + ioe + "</b>");
+                }
+            }
+            renderThreatsHTML(out, buf, now, points);
+        } else {
+            out.write("Unknown mode " + mode);
+        }
+        writeBuf(out, buf);
+    }
+
+    /**
+     *  @since 0.9.38
+     */
+    private void renderOverview(Writer out, StringBuilder buf, Analysis analysis) throws IOException {
+        PersistSybil ps = new PersistSybil(_context);
+        List<Long> dates = ps.load();
+        if (dates.isEmpty()) {
+            out.write("No stored analysis");
+        } else {
+            buf.append("<form action=\"netdb\" method=\"GET\">\n" +
+                       "<input type=\"hidden\" name=\"f\" value=\"3\">\n" +
+                       "<input type=\"hidden\" name=\"m\" value=\"12\">\n" +
+                       "Select stored analysis: " +
+                       "<select name=\"date\">\n");
+            boolean first = true;
+            for (Long date : dates) {
+                buf.append("<option value=\"").append(date).append('\"');
+                if (first) {
+                    buf.append(" selected=\"selected\"");
+                    first = false;
+                }
+                buf.append('>').append(new Date(date.longValue())).append("</option>\n");
+            }        
+            buf.append("</select>\n" +
+                       "<input type=\"submit\" name=\"action\" class=\"go\" value=\"Review analysis\" />" +
+                       "</form>\n");
+        }        
+        buf.append("<br><a href=\"netdb?f=3&amp;m=13\">Run new analysis</a>");
+        writeBuf(out, buf);
+    }
+
+    /**
+     *  @since 0.9.38 split out from renderRouterInfoHTML()
+     */
+    private void renderFFSummary(Writer out, StringBuilder buf, List<RouterInfo> ris, double avgMinDist) throws IOException {
+        renderRouterInfo(buf, _context.router().getRouterInfo(), null, true, false);
+        buf.append("<h3 id=\"known\" class=\"sybils\">Known Floodfills: ").append(ris.size()).append("</h3>");
         buf.append("<div id=\"sybils_summary\">\n" +
                    "<b>Average closest floodfill distance:</b> ").append(fmt.format(avgMinDist)).append("<br>\n" +
                    "<b>Routing Data:</b> \"").append(DataHelper.getUTF8(_context.routerKeyGenerator().getModData()))
@@ -156,56 +254,119 @@ public class SybilRenderer {
                    "<b>Next Routing Data:</b> \"").append(DataHelper.getUTF8(_context.routerKeyGenerator().getNextModData()))
            .append("\" <b>Rotates in:</b> ").append(DataHelper.formatDuration(_context.routerKeyGenerator().getTimeTillMidnight())).append("\n" +
                    "</div>\n");
+        writeBuf(out, buf);
+    }
 
-        Map<Hash, Points> points = new HashMap<Hash, Points>(64);
+    /**
-
+     *  @since 0.9.38 split out from renderRouterInfoHTML()
-        // IP analysis
+     */
+    private void renderFamilySummary(Writer out, StringBuilder buf, Analysis analysis, List<RouterInfo> ris, Map<Hash, Points> points) throws IOException {
         Map<String, List<RouterInfo>> fmap = analysis.calculateIPGroupsFamily(ris, points);
         renderIPGroupsFamily(out, buf, fmap);
+    }
+
+    /**
+     *  @since 0.9.38 split out from renderRouterInfoHTML()
+     */
+    private void renderIPUsSummary(Writer out, StringBuilder buf, Analysis analysis, List<RouterInfo> ris, Map<Hash, Points> points) throws IOException {
         List<RouterInfo> ri32 = new ArrayList<RouterInfo>(4);
         List<RouterInfo> ri24 = new ArrayList<RouterInfo>(4);
         List<RouterInfo> ri16 = new ArrayList<RouterInfo>(4);
         analysis.calculateIPGroupsUs(ris, points, ri32, ri24, ri16);
         renderIPGroupsUs(out, buf, ri32, ri24, ri16);
+    }
+
+    /**
+     *  @since 0.9.38 split out from renderRouterInfoHTML()
+     */
+    private void renderIP32Summary(Writer out, StringBuilder buf, Analysis analysis, List<RouterInfo> ris, Map<Hash, Points> points) throws IOException {
         Map<Integer, List<RouterInfo>> map = analysis.calculateIPGroups32(ris, points);
         renderIPGroups32(out, buf, map);
-        map = analysis.calculateIPGroups24(ris, points);
+    }
-        renderIPGroups24(out, buf, map);
-        map = analysis.calculateIPGroups16(ris, points);
-        //renderIPGroups16(out, buf, map);
 
+    /**
+     *  @since 0.9.38 split out from renderRouterInfoHTML()
+     */
+    private void renderIP24Summary(Writer out, StringBuilder buf, Analysis analysis, List<RouterInfo> ris, Map<Hash, Points> points) throws IOException {
+        Map<Integer, List<RouterInfo>> map = analysis.calculateIPGroups24(ris, points);
+        renderIPGroups24(out, buf, map);
+    }
+
+    /**
+     *  @since 0.9.38 split out from renderRouterInfoHTML()
+     */
+    private void renderIP16Summary(Writer out, StringBuilder buf, Analysis analysis, List<RouterInfo> ris, Map<Hash, Points> points) throws IOException {
+        Map<Integer, List<RouterInfo>> map = analysis.calculateIPGroups16(ris, points);
+        renderIPGroups16(out, buf, map);
+    }
+
+    /**
+     *  @since 0.9.38 split out from renderRouterInfoHTML()
+     */
+    private void renderPairSummary(Writer out, StringBuilder buf, Analysis analysis, List<RouterInfo> ris, Map<Hash, Points> points) throws IOException {
         // Pairwise distance analysis
         List<Pair> pairs = new ArrayList<Pair>(PAIRMAX);
         double avg = analysis.calculatePairDistance(ris, points, pairs);
         renderPairDistance(out, buf, pairs, avg);
+    }
 
+    /**
+     *  @since 0.9.38 split out from renderRouterInfoHTML()
+     */
+    private void renderCloseSummary(Writer out, StringBuilder buf, Analysis analysis, double avgMinDist, List<RouterInfo> ris, Map<Hash, Points> points) throws IOException {
         // Distance to our router analysis
         buf.append("<h3 id=\"ritoday\" class=\"sybils\">Closest Floodfills to Our Routing Key (Where we Store our RI)</h3>");
         buf.append("<p class=\"sybil_info\"><a href=\"/netdb?caps=f&amp;sybil\">See all</a></p>");
+        Hash ourRKey = _context.router().getRouterInfo().getRoutingKey();
         analysis.calculateRouterInfo(ourRKey, "our rkey", ris, points);
         renderRouterInfoHTML(out, buf, ourRKey, avgMinDist, ris);
+    }
+
+    /**
+     *  @since 0.9.38 split out from renderRouterInfoHTML()
+     */
+    private void renderCloseTmrwSummary(Writer out, StringBuilder buf, Analysis analysis, Hash us, double avgMinDist, List<RouterInfo> ris, Map<Hash, Points> points) throws IOException {
+        // Distance to our router analysis
         RouterKeyGenerator rkgen = _context.routerKeyGenerator();
         Hash nkey = rkgen.getNextRoutingKey(us);
         buf.append("<h3 id=\"ritmrw\" class=\"sybils\">Closest Floodfills to Tomorrow's Routing Key (Where we will Store our RI)</h3>");
         buf.append("<p class=\"sybil_info\"><a href=\"/netdb?caps=f&amp;sybil\">See all</a></p>");
         analysis.calculateRouterInfo(nkey, "our rkey (tomorrow)", ris, points);
         renderRouterInfoHTML(out, buf, nkey, avgMinDist, ris);
+    }
 
+    /**
+     *  @since 0.9.38 split out from renderRouterInfoHTML()
+     */
+    private void renderDHTSummary(Writer out, StringBuilder buf, Analysis analysis, Hash us, double avgMinDist, List<RouterInfo> ris, Map<Hash, Points> points) throws IOException {
         buf.append("<h3 id=\"dht\" class=\"sybils\">Closest Floodfills to Our Router Hash (DHT Neighbors if we are Floodfill)</h3>");
         analysis.calculateRouterInfo(us, "our router", ris, points);
         renderRouterInfoHTML(out, buf, us, avgMinDist, ris);
+    }
 
+    /**
+     *  @since 0.9.38 split out from renderRouterInfoHTML()
+     */
+    private void renderDestSummary(Writer out, StringBuilder buf, Analysis analysis, double avgMinDist, List<RouterInfo> ris, Map<Hash, Points> points) throws IOException {
         // Distance to our published destinations analysis
+        RouterKeyGenerator rkgen = _context.routerKeyGenerator();
         buf.append("<h3 id=\"dest\" class=\"sybils\">Floodfills Close to Our Destinations</h3>");
         Map<Hash, TunnelPool> clientInboundPools = _context.tunnelManager().getInboundClientPools();
         List<Hash> destinations = new ArrayList<Hash>(clientInboundPools.keySet());
-        //boolean debug = _context.getBooleanProperty(HelperBase.PROP_ADVANCED);
+        for (Iterator<Hash> iter = destinations.iterator(); iter.hasNext(); ) {
+            Hash client = iter.next();
+            if (!_context.clientManager().isLocal(client) ||
+                !_context.clientManager().shouldPublishLeaseSet(client) ||
+                _context.netDb().lookupLeaseSetLocally(client) == null) {
+                iter.remove();
+            }
+        }
+        if (destinations.isEmpty()) {
+            buf.append("<p class=\"notfound\">None</p>");
+            writeBuf(out, buf);
+            return;
+        }
         for (Hash client : destinations) {
-            boolean isLocal = _context.clientManager().isLocal(client);
-            if (!isLocal)
-                continue;
-            if (! _context.clientManager().shouldPublishLeaseSet(client))
-                continue;
             LeaseSet ls = _context.netDb().lookupLeaseSetLocally(client);
             if (ls == null)
                 continue;
@@ -216,21 +377,22 @@ public class SybilRenderer {
             buf.append("<p class=\"sybil_info\"><a href=\"/netdb?caps=f&amp;sybil=" + ls.getHash().toBase64() + "\">See all</a></p>");
             analysis.calculateRouterInfo(rkey, name, ris, points);
             renderRouterInfoHTML(out, buf, rkey, avgMinDist, ris);
-            nkey = rkgen.getNextRoutingKey(ls.getHash());
+            Hash nkey = rkgen.getNextRoutingKey(ls.getHash());
             buf.append("<h3 class=\"sybils\">Closest floodfills to Tomorrow's Routing Key for " + name + " (where we will store our LS)</h3>");
             buf.append("<p class=\"sybil_info\"><a href=\"/netdb?caps=f&amp;sybil=" + ls.getHash().toBase64() + "\">See all</a></p>");
             analysis.calculateRouterInfo(nkey, name + " (tomorrow)", ris, points);
             renderRouterInfoHTML(out, buf, nkey, avgMinDist, ris);
         }
+    }
 
-        // Profile analysis
+    /**
-        analysis.addProfilePoints(ris, points);
+     *  @since 0.9.38 split out from renderRouterInfoHTML()
-        analysis.addVersionPoints(ris, points);
+     */
-
+    private void renderThreatsHTML(Writer out, StringBuilder buf, long date, Map<Hash, Points> points) throws IOException {
         if (!points.isEmpty()) {
             List<Hash> warns = new ArrayList<Hash>(points.keySet());
             Collections.sort(warns, new PointsComparator(points));
-            buf.append("<h3 id=\"threats\" class=\"sybils\">Routers with Most Threat Points</h3>");
+            buf.append("<h3 id=\"threats\" class=\"sybils\">Routers with Most Threat Points as of " + new Date(date) + "</h3>");
             for (Hash h : warns) {
                 RouterInfo ri = _context.netDb().lookupRouterInfoLocally(h);
                 if (ri == null)
@@ -253,10 +415,7 @@ public class SybilRenderer {
                 renderRouterInfo(buf, ri, null, false, false);
             }
         }
-
+        writeBuf(out, buf);
-        out.write(buf.toString());
-        out.flush();
-        buf.setLength(0);
     }
 
     /**
@@ -277,10 +436,7 @@ public class SybilRenderer {
             renderRouterInfo(buf, p.r1, null, false, false);
             renderRouterInfo(buf, p.r2, null, false, false);
         }
-
+        writeBuf(out, buf);
-        out.write(buf.toString());
-        out.flush();
-        buf.setLength(0);
     }
 
     private static class FooComparator implements Comparator<Integer>, Serializable {
@@ -340,9 +496,7 @@ public class SybilRenderer {
         }
         if (!found)
             buf.append("<p class=\"notfound\">None</p>");
-        out.write(buf.toString());
+        writeBuf(out, buf);
-        out.flush();
-        buf.setLength(0);
     }
 
     /**
@@ -372,9 +526,7 @@ public class SybilRenderer {
         }
         if (!found)
             buf.append("<p class=\"notfound\">None</p>");
-        out.write(buf.toString());
+        writeBuf(out, buf);
-        out.flush();
-        buf.setLength(0);
     }
 
     /**
@@ -403,9 +555,7 @@ public class SybilRenderer {
         }
         if (!found)
             buf.append("<p class=\"notfound\">None</p>");
-        out.write(buf.toString());
+        writeBuf(out, buf);
-        out.flush();
-        buf.setLength(0);
     }
 
     /**
@@ -435,9 +585,7 @@ public class SybilRenderer {
         }
         if (!found)
             buf.append("<p class=\"notfound\">None</p>");
-        out.write(buf.toString());
+        writeBuf(out, buf);
-        out.flush();
-        buf.setLength(0);
     }
 
     /**
@@ -467,9 +615,7 @@ public class SybilRenderer {
         if (!found)
             buf.append("<p class=\"notfound\">None</p>");
         buf.append("</div>");
-        out.write(buf.toString());
+        writeBuf(out, buf);
-        out.flush();
-        buf.setLength(0);
     }
 
     /**
@@ -520,6 +666,10 @@ public class SybilRenderer {
                    "&nbsp; <span class=\"netdb_name\">MEDIAN:</span> " + fmt.format(median) +
                    "&nbsp; <span class=\"netdb_name\">MAX:</span> " + fmt.format(max) +
                    "</p>\n");
+        writeBuf(out, buf);
+    }
+
+    private static void writeBuf(Writer out, StringBuilder buf) throws IOException {
         out.write(buf.toString());
         out.flush();
         buf.setLength(0);

apps/routerconsole/jsp/netdb.jsp

@@ -38,5 +38,7 @@
  <jsp:setProperty name="netdbHelper" property="transport" value="<%=request.getParameter(\"tr\")%>" />
  <jsp:setProperty name="netdbHelper" property="limit" value="<%=request.getParameter(\"ps\")%>" />
  <jsp:setProperty name="netdbHelper" property="page" value="<%=request.getParameter(\"pg\")%>" />
+ <jsp:setProperty name="netdbHelper" property="mode" value="<%=request.getParameter(\"m\")%>" />
+ <jsp:setProperty name="netdbHelper" property="date" value="<%=request.getParameter(\"date\")%>" />
  <jsp:getProperty name="netdbHelper" property="netDbSummary" />
 </div></body></html>