2004-11-02 jrandom

* Fixed up the configuration overrides for the streaming socket lib
      integration so that it properly honors env settings.
    * More memory usage streamlining (last major revamp for now, i promise)

apps/ministreaming/java/src/net/i2p/client/streaming/I2PSocketManagerFactory.java

@@ -4,6 +4,7 @@ import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.util.Iterator;
 import java.util.Properties;
 
 import net.i2p.I2PAppContext;
@@ -81,6 +82,13 @@ public class I2PSocketManagerFactory {
     public static I2PSocketManager createManager(InputStream myPrivateKeyStream, String i2cpHost, int i2cpPort,
                                                  Properties opts) {
         I2PClient client = I2PClientFactory.createClient();
+        if (opts == null)
+            opts = new Properties();
+        for (Iterator iter = System.getProperties().keySet().iterator(); iter.hasNext(); ) {
+            String name = (String)iter.next();
+            if (!opts.containsKey(name))
+                opts.setProperty(name, System.getProperty(name));
+        }
         if (true) {
             // for the old streaming lib
             opts.setProperty(I2PClient.PROP_RELIABILITY, I2PClient.PROP_RELIABILITY_GUARANTEED);
@@ -88,11 +96,12 @@ public class I2PSocketManagerFactory {
         } else {
             // for new streaming lib:
             opts.setProperty(I2PClient.PROP_RELIABILITY, I2PClient.PROP_RELIABILITY_BEST_EFFORT);
-            //opts.setProperty("tunnels.depthInbound", "0");
+            //p.setProperty("tunnels.depthInbound", "0");
         }
 
         opts.setProperty(I2PClient.PROP_TCP_HOST, i2cpHost);
         opts.setProperty(I2PClient.PROP_TCP_PORT, "" + i2cpPort);
+        
         try {
             I2PSession session = client.createSession(myPrivateKeyStream, opts);
             session.connect();

apps/streaming/java/src/net/i2p/client/streaming/Packet.java

@@ -3,6 +3,7 @@ package net.i2p.client.streaming;
 import java.util.Arrays;
 import net.i2p.I2PAppContext;
 import net.i2p.data.Base64;
+import net.i2p.data.DataFormatException;
 import net.i2p.data.DataHelper;
 import net.i2p.data.Destination;
 import net.i2p.data.Signature;
@@ -439,7 +440,11 @@ public class Packet {
         }
         if (isFlagSet(FLAG_FROM_INCLUDED)) {
             _optionFrom = new Destination();
-            cur += _optionFrom.readBytes(buffer, cur);
+            try {
+                cur += _optionFrom.readBytes(buffer, cur);
+            } catch (DataFormatException dfe) {
+                throw new IllegalArgumentException("Bad from field: " + dfe.getMessage());
+            }
         }
         if (isFlagSet(FLAG_MAX_PACKET_SIZE_INCLUDED)) {
             _optionMaxSize = (int)DataHelper.fromLong(buffer, cur, 2);

core/java/src/net/i2p/data/Certificate.java

@@ -104,13 +104,21 @@ public class Certificate extends DataStructureImpl {
         return cur - offset;
     }
     
-    public int readBytes(byte source[], int offset) {
+    public int readBytes(byte source[], int offset) throws DataFormatException {
+        if (source == null) throw new DataFormatException("Cert is null");
+        if (source.length <= offset + 3)
+            throw new DataFormatException("Cert is too small [" + source.length + " off=" + offset + "]");
+
         int cur = offset;
         _type = (int)DataHelper.fromLong(source, cur, 1);
         cur++;
         int length = (int)DataHelper.fromLong(source, cur, 2);
         cur += 2;
         if (length > 0) {
+            if (length + cur > source.length)
+                throw new DataFormatException("Payload on the certificate is insufficient (len=" 
+                                              + source.length + " off=" + offset + " cur=" + cur
+                                              + " payloadLen=" + length);
             _payload = new byte[length];
             System.arraycopy(source, cur, _payload, 0, length);
             cur += length;

core/java/src/net/i2p/data/DataHelper.java

@@ -10,6 +10,7 @@ package net.i2p.data;
  */
 
 import java.io.BufferedReader;
+import java.io.BufferedInputStream;
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.File;
@@ -255,8 +256,9 @@ public class DataHelper {
     public static void writeLong(OutputStream rawStream, int numBytes, long value) 
         throws DataFormatException, IOException {
         try {
-            UnsignedInteger i = new UnsignedInteger(value);
+            UnsignedInteger.writeBytes(rawStream, numBytes, value);
-            rawStream.write(i.getBytes(numBytes));
+            //UnsignedInteger i = new UnsignedInteger(value);
+            //rawStream.write(i.getBytes(numBytes));
         } catch (IllegalArgumentException iae) {
             throw new DataFormatException("Invalid value (must be positive)", iae);
         }
@@ -360,12 +362,18 @@ public class DataHelper {
         else
             return toLong(DATE_LENGTH, date.getTime());
     }
-    public static Date fromDate(byte src[], int offset) throws IllegalArgumentException {
+    public static Date fromDate(byte src[], int offset) throws DataFormatException {
-        long when = fromLong(src, offset, DATE_LENGTH);
+        if ( (src == null) || (offset + DATE_LENGTH > src.length) )
-        if (when <= 0) 
+            throw new DataFormatException("Not enough data to read a date");
-            return null;
+        try {
-        else
+            long when = fromLong(src, offset, DATE_LENGTH);
-            return new Date(when);
+            if (when <= 0) 
+                return null;
+            else
+                return new Date(when);
+        } catch (IllegalArgumentException iae) {
+            throw new DataFormatException(iae.getMessage());
+        }
     }
     
     public static final int DATE_LENGTH = 8;

core/java/src/net/i2p/data/Destination.java

@@ -90,7 +90,10 @@ public class Destination extends DataStructureImpl {
         return cur - offset;
     }
     
-    public int readBytes(byte source[], int offset) {
+    public int readBytes(byte source[], int offset) throws DataFormatException {
+        if (source == null) throw new DataFormatException("Null source");
+        if (source.length <= offset + PublicKey.KEYSIZE_BYTES + SigningPublicKey.KEYSIZE_BYTES) 
+            throw new DataFormatException("Not enough data (len=" + source.length + " off=" + offset + ")");
         int cur = offset;
         
         _publicKey = new PublicKey();

core/java/src/net/i2p/data/RouterInfo.java

@@ -378,7 +378,11 @@ public class RouterInfo extends DataStructureImpl {
     public synchronized void readBytes(InputStream in) throws DataFormatException, IOException {
         _identity = new RouterIdentity();
         _identity.readBytes(in);
-        _published = DataHelper.readDate(in).getTime();
+        Date when = DataHelper.readDate(in);
+        if (when == null)
+            _published = 0;
+        else
+            _published = when.getTime();
         int numAddresses = (int) DataHelper.readLong(in, 1);
         for (int i = 0; i < numAddresses; i++) {
             RouterAddress address = new RouterAddress();
@@ -402,7 +406,7 @@ public class RouterInfo extends DataStructureImpl {
 
     public synchronized void writeBytes(OutputStream out) throws DataFormatException, IOException {
         if (_identity == null) throw new DataFormatException("Missing identity");
-        if (_published <= 0) throw new DataFormatException("Invalid published date: " + _published);
+        if (_published < 0) throw new DataFormatException("Invalid published date: " + _published);
         if (_signature == null) throw new DataFormatException("Signature is null");
         //if (!isValid())
         //    throw new DataFormatException("Data is not valid");

core/java/src/net/i2p/data/UnsignedInteger.java

@@ -9,6 +9,8 @@ package net.i2p.data;
  *
  */
 
+import java.io.IOException;
+import java.io.OutputStream;
 import java.math.BigInteger;
 
 import net.i2p.util.Log;
@@ -190,6 +192,16 @@ public class UnsignedInteger {
         return data;
     }
     
+    
+    public static void writeBytes(OutputStream rawStream, int numBytes, long value) 
+        throws DataFormatException, IOException {
+        if (value < 0) throw new DataFormatException("Invalid value (" + value + ")");
+        for (int i = numBytes - 1; i >= 0; i--) {
+            byte cur = (byte)( (value >>> (i*8) ) & 0xFF);
+            rawStream.write(cur);
+        }
+    }
+
     public BigInteger getBigInteger() {
         return new BigInteger(1, _data);
     }
@@ -238,6 +250,7 @@ public class UnsignedInteger {
         testNum(1024 * 1024 * 1024 * 4L + 1L);
         _log.debug("Testing MaxLong");
         testNum(Long.MAX_VALUE);
+        testWrite();
         } catch (Throwable t) { t.printStackTrace(); }
         try {
             Thread.sleep(1000);
@@ -260,4 +273,18 @@ public class UnsignedInteger {
         BigInteger tbi = new BigInteger(1, calculateBytes(num));
         _log.debug(num + " As a shifted   : 0x" + tbi.toString(16));
     }
+    
+    private static void testWrite() throws Exception {
+        java.io.ByteArrayOutputStream baos = new java.io.ByteArrayOutputStream(8);
+        UnsignedInteger i = new UnsignedInteger(12345);
+        baos.write(i.getBytes(8));
+        byte v1[] = baos.toByteArray();
+        baos.reset();
+        UnsignedInteger.writeBytes(baos, 8, 12345);
+        byte v2[] = baos.toByteArray();
+        System.out.println("v1 len: " + v1.length + " v2 len: " + v2.length);
+        System.out.println("v1: " + DataHelper.toHexString(v1));
+        System.out.println("v2: " + DataHelper.toHexString(v2));
+        
+    }
 }

history.txt

@@ -1,4 +1,9 @@
-$Id: history.txt,v 1.59 2004/10/30 18:44:01 jrandom Exp $
+$Id: history.txt,v 1.60 2004/11/01 08:31:31 jrandom Exp $
+
+2004-11-02  jrandom
+    * Fixed up the configuration overrides for the streaming socket lib 
+      integration so that it properly honors env settings.
+    * More memory usage streamlining (last major revamp for now, i promise)
 
 2004-11-01  jrandom
     * Increase the tunnel test timeout rapidly if our tunnels are failing.

router/java/src/net/i2p/data/i2np/DeliveryStatusMessage.java

@@ -48,7 +48,11 @@ public class DeliveryStatusMessage extends I2NPMessageImpl {
         
         _id = DataHelper.fromLong(data, curIndex, 4);
         curIndex += 4;
-        _arrival = DataHelper.fromDate(data, curIndex);
+        try {
+            _arrival = DataHelper.fromDate(data, curIndex);
+        } catch (DataFormatException dfe) {
+            throw new I2NPMessageException("Unable to read the arrival");
+        }
     }
     
     /** calculate the message body's length (not including the header and footer */

router/java/src/net/i2p/data/i2np/GarlicClove.java

@@ -78,6 +78,34 @@ public class GarlicClove extends DataStructureImpl {
             _log.debug("Read cert: " + _certificate);
     }
 
+    public int readBytes(byte source[], int offset) throws DataFormatException {
+        int cur = offset;
+        _instructions = new DeliveryInstructions();
+        cur += _instructions.readBytes(source, cur);
+        if (_log.shouldLog(Log.DEBUG))
+            _log.debug("Read instructions: " + _instructions);
+        try {
+            cur += _handler.readMessage(source, cur);
+            _msg = _handler.lastRead();
+        } catch (I2NPMessageException ime) {
+            throw new DataFormatException("Unable to read the message from a garlic clove", ime);
+        } catch (IOException ioe) {
+            throw new DataFormatException("Not enough data to read the clove", ioe);
+        }
+        _cloveId = DataHelper.fromLong(source, cur, 4);
+        cur += 4;
+        _expiration = DataHelper.fromDate(source, cur);
+        cur += DataHelper.DATE_LENGTH;
+        if (_log.shouldLog(Log.DEBUG))
+            _log.debug("CloveID read: " + _cloveId + " expiration read: " + _expiration);
+        _certificate = new Certificate();
+        cur += _certificate.readBytes(source, cur);
+        if (_log.shouldLog(Log.DEBUG))
+            _log.debug("Read cert: " + _certificate);
+        return cur - offset;
+    }
+
+    
     public void writeBytes(OutputStream out) throws DataFormatException, IOException {
         StringBuffer error = new StringBuffer();
         if (_instructions == null)

router/java/src/net/i2p/data/i2np/I2NPMessageImpl.java

@@ -95,30 +95,36 @@ public abstract class I2NPMessageImpl extends DataStructureImpl implements I2NPM
         }
     }
     public int readBytes(byte data[], int type, int offset) throws I2NPMessageException, IOException {
+        int cur = offset;
         if (type < 0) {
-            type = (int)DataHelper.fromLong(data, offset, 1);
+            type = (int)DataHelper.fromLong(data, cur, 1);
-            offset++;
+            cur++;
         }
-        _uniqueId = DataHelper.fromLong(data, offset, 4);
+        _uniqueId = DataHelper.fromLong(data, cur, 4);
-        offset += 4;
+        cur += 4;
-        _expiration = DataHelper.fromDate(data, offset);
+        try {
-        offset += DataHelper.DATE_LENGTH;
+            _expiration = DataHelper.fromDate(data, cur);
-        int size = (int)DataHelper.fromLong(data, offset, 2);
+            cur += DataHelper.DATE_LENGTH;
-        offset += 2;
+        } catch (DataFormatException dfe) {
+            throw new I2NPMessageException("Unable to read the expiration", dfe);
+        }
+        int size = (int)DataHelper.fromLong(data, cur, 2);
+        cur += 2;
         Hash h = new Hash();
         byte hdata[] = new byte[Hash.HASH_LENGTH];
-        System.arraycopy(data, offset, hdata, 0, Hash.HASH_LENGTH);
+        System.arraycopy(data, cur, hdata, 0, Hash.HASH_LENGTH);
-        offset += Hash.HASH_LENGTH;
+        cur += Hash.HASH_LENGTH;
         h.setData(hdata);
 
-        if (offset + size > data.length)
+        if (cur + size > data.length)
             throw new I2NPMessageException("Payload is too short [" 
                                            + "data.len=" + data.length
                                            + " offset=" + offset
+                                           + " cur=" + cur 
                                            + " wanted=" + size + "]");
 
         SHA256EntryCache.CacheEntry cache = _context.sha().cache().acquire(size);
-        Hash calc = _context.sha().calculateHash(data, offset, size, cache);
+        Hash calc = _context.sha().calculateHash(data, cur, size, cache);
         boolean eq = calc.equals(h);
         _context.sha().cache().release(cache);
         if (!eq)
@@ -127,11 +133,12 @@ public abstract class I2NPMessageImpl extends DataStructureImpl implements I2NPM
         long start = _context.clock().now();
         if (_log.shouldLog(Log.DEBUG))
             _log.debug("Reading bytes: type = " + type + " / uniqueId : " + _uniqueId + " / expiration : " + _expiration);
-        readMessage(data, offset, size, type);
+        readMessage(data, cur, size, type);
+        cur += size;
         long time = _context.clock().now() - start;
         if (time > 50)
             _context.statManager().addRateData("i2np.readTime", time, time);
-        return size + Hash.HASH_LENGTH + 1 + 4 + DataHelper.DATE_LENGTH;
+        return cur - offset;
     }
     
     public void writeBytes(OutputStream out) throws DataFormatException, IOException {

router/java/src/net/i2p/router/RouterVersion.java

@@ -15,9 +15,9 @@ import net.i2p.CoreVersion;
  *
  */
 public class RouterVersion {
-    public final static String ID = "$Revision: 1.65 $ $Date: 2004/10/30 18:44:01 $";
+    public final static String ID = "$Revision: 1.66 $ $Date: 2004/11/01 08:31:30 $";
     public final static String VERSION = "0.4.1.3";
-    public final static long BUILD = 6;
+    public final static long BUILD = 7;
     public static void main(String args[]) {
         System.out.println("I2P Router version: " + VERSION);
         System.out.println("Router ID: " + RouterVersion.ID);

router/java/src/net/i2p/router/message/GarlicMessageParser.java

@@ -40,57 +40,57 @@ public class GarlicMessageParser {
         byte encData[] = message.getData();
         byte decrData[] = null;
         try {
-            _log.debug("Decrypting with private key " + encryptionKey);
+            if (_log.shouldLog(Log.DEBUG))
+                _log.debug("Decrypting with private key " + encryptionKey);
             decrData = _context.elGamalAESEngine().decrypt(encData, encryptionKey);
         } catch (DataFormatException dfe) {
-            _log.warn("Error decrypting", dfe);
+            if (_log.shouldLog(Log.WARN))
+                _log.warn("Error decrypting", dfe);
         }
         if (decrData == null) {
             if (_log.shouldLog(Log.WARN))
                 _log.warn("Decryption of garlic message failed (data = " + encData + ")", new Exception("Decrypt fail"));
             return null;
         } else {
-            return readCloveSet(decrData);
+            try {
+                return readCloveSet(decrData); 
+            } catch (DataFormatException dfe) {
+                if (_log.shouldLog(Log.WARN))
+                    _log.warn("Unable to read cloveSet", dfe);
+                return null;
+            }
         }
     }
     
-    private CloveSet readCloveSet(byte data[]) {
+    private CloveSet readCloveSet(byte data[]) throws DataFormatException {
         Set cloves = new HashSet();
-        ByteArrayInputStream bais = new ByteArrayInputStream(data);
+        int offset = 0;
-        try {
-            CloveSet set = new CloveSet();
         
-            int numCloves = (int)DataHelper.readLong(bais, 1);
+        CloveSet set = new CloveSet();
+
+        int numCloves = (int)DataHelper.fromLong(data, offset, 1);
+        offset++;
+        if (_log.shouldLog(Log.DEBUG))
             _log.debug("# cloves to read: " + numCloves);
-            for (int i = 0; i < numCloves; i++) {
+        for (int i = 0; i < numCloves; i++) {
+            if (_log.shouldLog(Log.DEBUG))
                 _log.debug("Reading clove " + i);
-                try {
+                GarlicClove clove = new GarlicClove(_context);
-                    GarlicClove clove = new GarlicClove(_context);
+                offset += clove.readBytes(data, offset);
-                    clove.readBytes(bais);
+                set.addClove(clove);
-                    set.addClove(clove);
+            if (_log.shouldLog(Log.WARN))
-                } catch (DataFormatException dfe) {
-                    _log.warn("Unable to read clove " + i, dfe);
-                } catch (IOException ioe) {
-                    _log.warn("Unable to read clove " + i, ioe);
-                }
                 _log.debug("After reading clove " + i);
-            }
-            Certificate cert = new Certificate();
-            cert.readBytes(bais);
-            long msgId = DataHelper.readLong(bais, 4);
-            Date expiration = DataHelper.readDate(bais);
-            
-            set.setCertificate(cert);
-            set.setMessageId(msgId);
-            set.setExpiration(expiration.getTime());
-            
-            return set;
-        } catch (IOException ioe) {
-            _log.error("Error reading clove set", ioe);
-            return null;
-        } catch (DataFormatException dfe) {
-            _log.error("Error reading clove set", dfe);
-            return null;
         }
+        Certificate cert = new Certificate();
+        offset += cert.readBytes(data, offset);
+        long msgId = DataHelper.fromLong(data, offset, 4);
+        offset += 4;
+        Date expiration = DataHelper.fromDate(data, offset);
+        offset += DataHelper.DATE_LENGTH;
+
+        set.setCertificate(cert);
+        set.setMessageId(msgId);
+        set.setExpiration(expiration.getTime());
+        return set;
     }
 }

router/java/src/net/i2p/router/message/HandleGarlicMessageJob.java

@@ -132,21 +132,24 @@ public class HandleGarlicMessageJob extends JobImpl {
     
     private boolean isValid(GarlicClove clove) {
         if (isKnown(clove.getCloveId())) {
-            _log.error("Duplicate garlic clove received - replay attack in progress? [cloveId = "
+            if (_log.shouldLog(Log.ERROR))
-                       + clove.getCloveId() + " expiration = " + clove.getExpiration());
+                _log.error("Duplicate garlic clove received - replay attack in progress? [cloveId = "
+                           + clove.getCloveId() + " expiration = " + clove.getExpiration());
             return false;
         } else {
-            _log.debug("Clove " + clove.getCloveId() + " expiring on " + clove.getExpiration() 
+            if (_log.shouldLog(Log.DEBUG))
-                       + " is not known");
+                _log.debug("Clove " + clove.getCloveId() + " expiring on " + clove.getExpiration() 
+                           + " is not known");
         }
         long now = getContext().clock().now();
         if (clove.getExpiration().getTime() < now) {
             if (clove.getExpiration().getTime() < now + Router.CLOCK_FUDGE_FACTOR) {
-                _log.warn("Expired garlic received, but within our fudge factor [" 
+                if (_log.shouldLog(Log.WARN))
-                          + clove.getExpiration() + "]");
+                    _log.warn("Expired garlic received, but within our fudge factor [" 
+                              + clove.getExpiration() + "]");
             } else {
                 if (_log.shouldLog(Log.DEBUG))
-                    _log.error("Expired garlic clove received - replay attack in progress? [cloveId = " 
+                    _log.debug("Expired garlic clove received - replay attack in progress? [cloveId = " 
                                + clove.getCloveId() + " expiration = " + clove.getExpiration() 
                                + " now = " + (new Date(getContext().clock().now())));
                 return false;

router/java/src/net/i2p/router/transport/OutboundMessageRegistry.java

@@ -76,7 +76,7 @@ public class OutboundMessageRegistry {
         long continueTime = 0;
         int numMessages = messages.size();
 
-        StringBuffer slow = new StringBuffer(256);
+        StringBuffer slow = null; // new StringBuffer(256);
         long afterSync1 = _context.clock().now();
 
         ArrayList matchedRemove = null; // new ArrayList(32);
@@ -93,6 +93,7 @@ public class OutboundMessageRegistry {
                     if (_log.shouldLog(Log.WARN))
                         _log.warn("Matching with selector took too long (" + diff + "ms) : " 
                                   + selector.getClass().getName());
+                    if (slow == null) slow = new StringBuffer(256);
                     slow.append(selector.getClass().getName()).append(": ");
                     slow.append(diff).append(" ");
                 }
@@ -152,7 +153,9 @@ public class OutboundMessageRegistry {
                 buf.append(0);
             else
                 buf.append(matchedRemove.size());
-            buf.append(" removed, ").append(matches.size()).append(" matches: slow = ").append(slow.toString());
+            buf.append(" removed, ").append(matches.size()).append(" matches: slow = ");
+            if (slow != null)
+                buf.append(slow.toString());
             _log.log(level, buf.toString());
         }