2004-09-27 jrandom

* Limit the number of connection tags saved to 10,000.  This is a huge
      limit, but consumes no more than 1MB of RAM.  For now, we drop them
      randomly after reaching that size, forcing those dropped peers to use
      a full DH negotiation.
    * HTML cleanup in the console.

apps/routerconsole/java/src/net/i2p/router/web/LogsHelper.java

@@ -28,7 +28,7 @@ public class LogsHelper {
     public String getLogs() {
         List msgs = _context.logManager().getBuffer().getMostRecentMessages();
         StringBuffer buf = new StringBuffer(16*1024); 
-        buf.append("<h2>Most recent console messages:</h2><ul>");
+        buf.append("<ul>");
         buf.append("<code>\n");
         for (int i = msgs.size(); i > 0; i--) { 
             String msg = (String)msgs.get(i - 1);
@@ -48,4 +48,20 @@ public class LogsHelper {
         else
             return "<pre>" + str + "</pre>";
     }
+    
+    public String getConnectionLogs() {
+        List msgs = _context.commSystem().getMostRecentErrorMessages();
+        StringBuffer buf = new StringBuffer(16*1024); 
+        buf.append("<ul>");
+        buf.append("<code>\n");
+        for (int i = msgs.size(); i > 0; i--) { 
+            String msg = (String)msgs.get(i - 1);
+            buf.append("<li>");
+            buf.append(msg);
+            buf.append("</li>\n");
+        }
+        buf.append("</code></ul>\n");
+        
+        return buf.toString();
+    }
 }

apps/routerconsole/jsp/config.jsp

@@ -35,8 +35,11 @@
      <input name="port" type="text" size="4" value="<jsp:getProperty name="nethelper" property="port" />" /> <br />
  <i>The hostname/IP address and TCP port must be reachable from the outside world.  If
  you are behind a firewall or NAT, this means you must poke a hole for this port.  If
- you are using DHCP and do not have a static IP address, you must use a service like
- <a href="http://dyndns.org/">dyndns</a>.  The "guess" functionality makes an HTTP request
+ you are using DHCP and do not have a static IP address, you should either use a service like
+ <a href="http://dyndns.org/">dyndns</a> or leave the hostname blank.  If you leave it blank,
+ your router will autodetect the 'correct' IP address by asking a peer (and unconditionally 
+ believing them if the address is routable and you don't have any established connections yet).
+ The "guess" functionality makes an HTTP request 
  to <a href="http://www.whatismyip.com/">www.whatismyip.com</a>.</i>
  <hr />
  <b>Enable internal time synchronization?</b> <input type="checkbox" <jsp:getProperty name="nethelper" property="enableTimeSyncChecked" /> name="enabletimesync" /><br />

apps/routerconsole/jsp/logs.jsp

@@ -16,6 +16,9 @@
  <h4>Router logs:</h4>
  <jsp:getProperty name="logsHelper" property="logs" />
  <hr />
+ <h4>Connection logs:</h4><a name="connectionlogs"> </a>
+ <jsp:getProperty name="logsHelper" property="connectionLogs" />
+ <hr />
  <h4>Service logs:</h4><a name="servicelogs"> </a>
  <jsp:getProperty name="logsHelper" property="serviceLogs" />
 </div>

core/java/src/net/i2p/data/SessionKey.java

@@ -28,7 +28,10 @@ public class SessionKey extends DataStructureImpl {
     public final static int KEYSIZE_BYTES = 32;
 
     public SessionKey() {
-        setData(null);
+        this(null);
+    } 
+    public SessionKey(byte data[]) {
+        setData(data);
     }
 
     public byte[] getData() {

history.txt

@@ -1,4 +1,11 @@
-$Id: history.txt,v 1.19 2004/09/21 19:10:26 jrandom Exp $
+$Id: history.txt,v 1.20 2004/09/26 10:16:44 jrandom Exp $
+
+2004-09-27  jrandom
+    * Limit the number of connection tags saved to 10,000.  This is a huge 
+      limit, but consumes no more than 1MB of RAM.  For now, we drop them 
+      randomly after reaching that size, forcing those dropped peers to use
+      a full DH negotiation.
+    * HTML cleanup in the console.
 
 2004-09-26  jrandom
     * Complete rewrite of the TCP transport with IP autodetection and 

router/java/src/net/i2p/router/CommSystemFacade.java

@@ -10,7 +10,9 @@ package net.i2p.router;
 
 import java.io.IOException;
 import java.io.OutputStream;
+import java.util.Collections;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Set;
 
 /**
@@ -27,6 +29,7 @@ public abstract class CommSystemFacade implements Service {
     public Set createAddresses() { return new HashSet(); }
     
     public int countActivePeers() { return 0; }
+    public List getMostRecentErrorMessages() { return Collections.EMPTY_LIST; }
 }
 
 class DummyCommSystemFacade extends CommSystemFacade {

router/java/src/net/i2p/router/transport/CommSystemFacadeImpl.java

@@ -62,6 +62,10 @@ public class CommSystemFacadeImpl extends CommSystemFacade {
         j.runJob();
     }
     
+    public List getMostRecentErrorMessages() { 
+        return _manager.getMostRecentErrorMessages(); 
+    }
+    
     public void renderStatusHTML(OutputStream out) throws IOException { 
         _manager.renderStatusHTML(out); 
     }

router/java/src/net/i2p/router/transport/Transport.java

@@ -8,6 +8,7 @@ package net.i2p.router.transport;
  *
  */
 
+import java.util.List;
 import java.util.Properties;
 import java.util.Set;
 
@@ -35,7 +36,8 @@ public interface Transport {
     public void setListener(TransportEventListener listener);
     public String getStyle();
     
-    public int countActivePeers();
+    public int countActivePeers();    
+    public List getMostRecentErrorMessages();
     
     public String renderStatusHTML();
 }

router/java/src/net/i2p/router/transport/TransportImpl.java

@@ -9,6 +9,7 @@ package net.i2p.router.transport;
  */
 
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.Date;
 import java.util.HashSet;
 import java.util.Iterator;
@@ -62,6 +63,7 @@ public abstract class TransportImpl implements Transport {
      */
     public int countActivePeers() { return 0; }
     
+    public List getMostRecentErrorMessages() { return Collections.EMPTY_LIST; }
     /**
      * Nonblocking call to pull the next outbound message
      * off the queue.  

router/java/src/net/i2p/router/transport/TransportManager.java

@@ -251,6 +251,15 @@ public class TransportManager implements TransportEventListener {
             _log.debug("Added to in pool: "+ num);
     }
     
+    public List getMostRecentErrorMessages() { 
+        List rv = new ArrayList(16);
+        for (int i = 0; i < _transports.size(); i++) {
+            Transport t = (Transport)_transports.get(i);
+            rv.addAll(t.getMostRecentErrorMessages());
+        }
+        return rv;
+    }
+    
     public void renderStatusHTML(OutputStream out) throws IOException {
         StringBuffer buf = new StringBuffer(8*1024);
         buf.append("<h2>Transport Manager</h2>\n");

router/java/src/net/i2p/router/transport/tcp/ConnectionBuilder.java

@@ -326,7 +326,9 @@ public class ConnectionBuilder {
         System.arraycopy(h.getData(), 0, _iv, 0, 16);
      
         updateNextTagExisting();
-        
+
+        _rawOut = new BufferedOutputStream(_rawOut, ConnectionBuilder.WRITE_BUFFER_SIZE);
+
         _rawOut = new AESOutputStream(_context, _rawOut, _key, _iv);
         _rawIn = new AESInputStream(_context, _rawIn, _key, _iv);
         
@@ -474,7 +476,9 @@ public class ConnectionBuilder {
             _log.debug("\nNew session[X]: key=" + _key.toBase64() + " iv=" 
                        + Base64.encode(_iv) + " nonce=" + Base64.encode(_nonce.getData())
                        + " socket: " + _socket);
-        
+
+        _rawOut = new BufferedOutputStream(_rawOut, ConnectionBuilder.WRITE_BUFFER_SIZE);
+
         _rawOut = new AESOutputStream(_context, _rawOut, _key, _iv);
         _rawIn = new AESInputStream(_context, _rawIn, _key, _iv);
         
@@ -498,7 +502,7 @@ public class ConnectionBuilder {
             byte val[] = new byte[32];
             int read = DataHelper.read(_rawIn, val);
             if (read != 32) {
-                fail("Not enough data to read the verification from " 
+                fail("Not enough data (" + read + ") to read the verification from " 
                      + _target.getIdentity().calculateHash().toBase64().substring(0,6));
                 return false;
             }
@@ -632,7 +636,7 @@ public class ConnectionBuilder {
     private void establishComplete() {
         _connectionIn = new BandwidthLimitedInputStream(_context, _rawIn, _actualPeer.getIdentity());
         OutputStream blos = new BandwidthLimitedOutputStream(_context, _rawOut, _actualPeer.getIdentity());
-        _connectionOut = new BufferedOutputStream(blos, WRITE_BUFFER_SIZE);
+        _connectionOut = blos;
         
         Hash peer = _actualPeer.getIdentity().getHash();
         _context.netDb().store(peer, _actualPeer);

router/java/src/net/i2p/router/transport/tcp/ConnectionHandler.java

@@ -322,7 +322,9 @@ public class ConnectionHandler {
         System.arraycopy(h.getData(), 0, _iv, 0, 16);
         
         updateNextTagExisting();
-        
+
+        _rawOut = new BufferedOutputStream(_rawOut, ConnectionBuilder.WRITE_BUFFER_SIZE);
+
         _rawOut = new AESOutputStream(_context, _rawOut, _key, _iv);
         _rawIn = new AESInputStream(_context, _rawIn, _key, _iv);
         
@@ -473,6 +475,8 @@ public class ConnectionHandler {
                        + Base64.encode(_iv) + " nonce=" + Base64.encode(_nonce.getData())
                        + " socket: " + _socket);
         
+        _rawOut = new BufferedOutputStream(_rawOut, ConnectionBuilder.WRITE_BUFFER_SIZE);
+        
         _rawOut = new AESOutputStream(_context, _rawOut, _key, _iv);
         _rawIn = new AESInputStream(_context, _rawIn, _key, _iv);
         
@@ -774,7 +778,7 @@ public class ConnectionHandler {
     private void establishComplete() {
         _connectionIn = new BandwidthLimitedInputStream(_context, _rawIn, _actualPeer.getIdentity());
         OutputStream blos = new BandwidthLimitedOutputStream(_context, _rawOut, _actualPeer.getIdentity());
-        _connectionOut = new BufferedOutputStream(blos, ConnectionBuilder.WRITE_BUFFER_SIZE);
+        _connectionOut = blos;
         
         Hash peer = _actualPeer.getIdentity().getHash();
         _context.netDb().store(peer, _actualPeer);

router/java/src/net/i2p/router/transport/tcp/ConnectionTagManager.java

@@ -8,57 +8,101 @@ import net.i2p.data.ByteArray;
 import net.i2p.data.Hash;
 import net.i2p.data.SessionKey;
 import net.i2p.router.RouterContext;
+import net.i2p.util.Log;
 
 /**
  * Organize the tags used to connect with peers.
  *
  */
 public class ConnectionTagManager {
+    protected Log _log;
     private RouterContext _context;
     /** H(routerIdentity) to ByteArray */
-    private Map _tags;
+    private Map _tagByPeer;
+    /** ByteArray to H(routerIdentity) */
+    private Map _peerByTag;
     /** H(routerIdentity) to SessionKey */
-    private Map _keys;
+    private Map _keyByPeer;
     
     /** synchronize against this when dealing with the data */
     private Object _lock;
     
+    /** 
+     * Only keep the keys and tags for up to *cough* 10,000 peers (everyone 
+     * else will need to use a full DH rekey).  Ok, yeah, 10,000 is absurd for 
+     * the TCP transport anyway, but we need a limit, and this eats up at most 
+     * 1MB (96 bytes per peer).  Later we may add another mapping to drop the
+     * oldest ones first, but who cares for now.
+     * 
+     */
+    public static final int MAX_CONNECTION_TAGS = 10*1000;
+    
     public ConnectionTagManager(RouterContext context) {
         _context = context;
-        _tags = new HashMap(128);
-        _keys = new HashMap(128);
+        _log = context.logManager().getLog(getClass());
+        initialize();
         _lock = new Object();
     }
     
+    protected void initialize() {
+        initializeData(new HashMap(128), new HashMap(128), new HashMap(128));
+    }
+    
+    protected void initializeData(Map keyByPeer, Map tagByPeer, Map peerByTag) {
+        _keyByPeer = keyByPeer;
+        _tagByPeer = tagByPeer;
+        _peerByTag = peerByTag;
+    }
+    
     /** Retrieve the associated tag (but do not consume it) */
     public ByteArray getTag(Hash peer) {
         synchronized (_lock) {
-            return (ByteArray)_tags.get(peer);
+            return (ByteArray)_tagByPeer.get(peer);
         }
     }
     
     public SessionKey getKey(Hash peer) {
         synchronized (_lock) { // 
-            return (SessionKey)_keys.get(peer);
+            return (SessionKey)_keyByPeer.get(peer);
         }
     }
     public SessionKey getKey(ByteArray tag) {
         synchronized (_lock) { // 
-            for (Iterator iter = _tags.keySet().iterator(); iter.hasNext(); ) {
-                Hash peer = (Hash)iter.next();
-                ByteArray cur = (ByteArray)_tags.get(peer);
-                if (cur.equals(tag))
-                    return (SessionKey)_keys.get(peer);
-            }
-            return null;
+            Hash peer = (Hash)_peerByTag.get(tag);
+            if (peer == null) return null;
+            return (SessionKey)_keyByPeer.get(peer);
         }
     }
     
     /** Update the tag associated with a peer, dropping the old one */
     public void replaceTag(Hash peer, ByteArray newTag, SessionKey key) {
         synchronized (_lock) {
-            _tags.put(peer, newTag);
-            _keys.put(peer, key);
+            while (_keyByPeer.size() > MAX_CONNECTION_TAGS) {
+                Hash rmPeer = (Hash)_keyByPeer.keySet().iterator().next();
+                ByteArray tag = (ByteArray)_tagByPeer.remove(peer);
+                SessionKey oldKey = (SessionKey)_keyByPeer.remove(peer);
+                rmPeer = (Hash)_peerByTag.remove(tag);
+                
+                if (_log.shouldLog(Log.INFO))
+                    _log.info("Too many tags, dropping the one for " + rmPeer.toBase64().substring(0,6));
+            }
+            _keyByPeer.put(peer, key);
+            _peerByTag.put(newTag, peer);
+            _tagByPeer.put(peer, newTag);
+            
+            saveTags(_keyByPeer, _tagByPeer);
         }
     }
+    
+    /**
+     * Save the tags/keys associated with the peer.
+     *
+     * @param keyByPeer H(routerIdentity) to SessionKey
+     * @param tagByPeer H(routerIdentity) to ByteArray
+     */
+    protected void saveTags(Map keyByPeer, Map tagByPeer) {
+        // noop, in memory only
+    }
+    
+    protected RouterContext getContext() { return _context; }
 }

router/java/src/net/i2p/router/transport/tcp/PersistentConnectionTagManager.java

@@ -0,0 +1,199 @@
+package net.i2p.router.transport.tcp;
+
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+
+import net.i2p.data.ByteArray;
+import net.i2p.data.DataHelper;
+import net.i2p.data.DataFormatException;
+import net.i2p.data.Hash;
+import net.i2p.data.SessionKey;
+import net.i2p.router.RouterContext;
+import net.i2p.util.Log;
+
+/**
+ *
+ */
+public class PersistentConnectionTagManager extends ConnectionTagManager {
+    private Object _ioLock;
+    
+    public PersistentConnectionTagManager(RouterContext context) {
+        super(context);
+        _ioLock = new Object();
+    }
+
+    public static final String PROP_TAG_FILE = "i2np.tcp.tagFile";
+    public static final String DEFAULT_TAG_FILE = "connectionTag.keys";
+
+    protected void initialize() {
+        loadTags();
+    }
+    
+    /**
+     * Save the tags/keys associated with the peer.
+     *
+     * @param keyByPeer H(routerIdentity) to SessionKey
+     * @param tagByPeer H(routerIdentity) to ByteArray
+     */
+    protected void saveTags(Map keyByPeer, Map tagByPeer) {
+        byte data[] = prepareData(keyByPeer, tagByPeer);
+        if (data == null) return;
+        
+        synchronized (_ioLock) {
+            File tagFile = getFile();
+            if (tagFile == null) return;
+            
+            FileOutputStream fos = null;
+            try {
+                fos = new FileOutputStream(tagFile);
+                fos.write(data);
+                fos.flush();
+                
+                if (_log.shouldLog(Log.INFO))
+                    _log.info("Wrote connection tags for " + keyByPeer.size() + " peers");
+            } catch (IOException ioe) {
+                if (_log.shouldLog(Log.WARN))
+                    _log.warn("Error writing out the tags", ioe);
+            } finally {
+                if (fos != null) try { fos.close(); } catch (IOException ioe) {}
+            }
+        }
+    }
+    
+    /**
+     * Get the raw data to be written to disk.
+     *
+     * @param keyByPeer H(routerIdentity) to SessionKey
+     * @param tagByPeer H(routerIdentity) to ByteArray
+     */
+    private byte[] prepareData(Map keyByPeer, Map tagByPeer) {
+        ByteArrayOutputStream baos = new ByteArrayOutputStream(keyByPeer.size() * 32 * 3 + 32);
+        try {
+            for (Iterator iter = keyByPeer.keySet().iterator(); iter.hasNext(); ) {
+                Hash peer = (Hash)iter.next();
+                SessionKey key = (SessionKey)keyByPeer.get(peer);
+                ByteArray tag = (ByteArray)tagByPeer.get(peer);
+                
+                if ( (key == null) || (tag == null) ) continue;
+                
+                baos.write(peer.getData());
+                baos.write(key.getData());
+                baos.write(tag.getData());
+                
+                if (_log.shouldLog(Log.DEBUG))
+                    _log.debug("Wrote connection tag for " + peer.toBase64().substring(0,6));
+            }
+            byte pre[] = baos.toByteArray();
+            Hash check = getContext().sha().calculateHash(pre);
+            baos.write(check.getData());
+            return baos.toByteArray();
+        } catch (IOException ioe) {
+            if (_log.shouldLog(Log.WARN))
+                _log.warn("Error preparing the tags", ioe);
+            return null;
+        }
+    }
+    
+    private void loadTags() {
+        File tagFile = getFile();
+        if ( (tagFile == null) || (tagFile.length() <= 31) ) {
+            initializeData(new HashMap(), new HashMap(), new HashMap());
+            return;
+        }
+        
+        FileInputStream fin = null;
+        try {
+            fin = new FileInputStream(tagFile);
+            byte data[] = getData(tagFile, fin);
+            if (data == null) {
+                initializeData(new HashMap(), new HashMap(), new HashMap());
+                return;
+            }
+        
+            int entries = data.length / (32 * 3);
+            Map keyByPeer = new HashMap(entries);
+            Map tagByPeer = new HashMap(entries);
+            Map peerByTag = new HashMap(entries);
+        
+            for (int i = 0; i < data.length; i += 32*3) {
+                byte peer[] = new byte[32];
+                byte key[] = new byte[32];
+                byte tag[] = new byte[32];
+                System.arraycopy(data, i, peer, 0, 32);
+                System.arraycopy(data, i + 32, key, 0, 32);
+                System.arraycopy(data, i + 64, tag, 0, 32);
+                
+                Hash peerData = new Hash(peer);
+                SessionKey keyData = new SessionKey(key);
+                ByteArray tagData = new ByteArray(tag);
+                
+                keyByPeer.put(peerData, keyData);
+                tagByPeer.put(peerData, tagData);
+                peerByTag.put(tagData, peerData);
+                
+                if (_log.shouldLog(Log.DEBUG))
+                    _log.debug("Loaded connection tag for " + peerData.toBase64().substring(0,6));
+                
+                if (keyByPeer.size() > ConnectionTagManager.MAX_CONNECTION_TAGS)
+                    break;
+            }
+            
+            if (_log.shouldLog(Log.INFO))
+                _log.info("Loaded connection tags for " + keyByPeer.size() + " peers");
+            initializeData(keyByPeer, tagByPeer, peerByTag);
+        } catch (IOException ioe) {
+            if (_log.shouldLog(Log.ERROR))        
+                _log.error("Connection tag file is corrupt, removing it");
+            try { fin.close(); } catch (IOException ioe2) {}
+            tagFile.delete(); // ignore rv
+            fin = null;
+            initializeData(new HashMap(), new HashMap(), new HashMap());
+            return;
+        } finally {
+            if (fin != null) try { fin.close(); } catch (IOException ioe) {}
+        }
+    }
+    
+    private byte[] getData(File tagFile, FileInputStream fin) throws IOException {
+        byte data[] = new byte[(int)tagFile.length() - 32];
+        int read = DataHelper.read(fin, data);
+        if (read != data.length) {
+            if (_log.shouldLog(Log.ERROR))
+                _log.error("Connection tag file is corrupt (too short), removing it");
+            try { fin.close(); } catch (IOException ioe) {}
+            tagFile.delete(); // ignore rv
+            fin = null;
+            return null;
+        }
+
+        Hash readHash = new Hash();
+        try {
+            readHash.readBytes(fin);
+        } catch (DataFormatException dfe) {
+            readHash = null;
+        }
+
+        Hash calcHash = getContext().sha().calculateHash(data);
+        if ( (readHash == null) || (!calcHash.equals(readHash)) ) {
+            if (_log.shouldLog(Log.ERROR))
+                _log.error("Connection tag file is corrupt, removing it");
+            try { fin.close(); } catch (IOException ioe) {}
+            tagFile.delete(); // ignore rv
+            fin = null;
+            return null;
+        }
+        
+        return data;
+    }
+    
+    private File getFile() {
+        return new File(getContext().getProperty(PROP_TAG_FILE, DEFAULT_TAG_FILE));
+    }
+}

router/java/src/net/i2p/router/transport/tcp/TCPTransport.java

@@ -91,7 +91,7 @@ public class TCPTransport extends TransportImpl {
         _log = context.logManager().getLog(TCPTransport.class);
         _listener = new TCPListener(context, this);
         _myAddress = null;
-        _tagManager = new ConnectionTagManager(context);
+        _tagManager = new PersistentConnectionTagManager(context);
         _connectionsByIdent = new HashMap(16);
         _connectionsByAddress = new HashMap(16);
         _pendingConnectionsByIdent = new HashSet(16);
@@ -453,6 +453,9 @@ public class TCPTransport extends TransportImpl {
         return -1;
     }
 
+    public List getMostRecentErrorMessages() { 
+        return _lastConnectionErrors; 
+    }
     
     /**
      * How many peers can we talk to right now?