I2PSSLSocketFactory:

- Add hostname verification using code from
Apache HttpClient 4.4.1 (Apache 2.0 license)
and one small class from HttpCore 4.4.1,
slightly modified to remove additional Apache dependencies
and unneeded code.
- Includes support for public suffix list;
use basic list with standard TLDs,
and also support loading the big Mozilla list,
but don't bundle the 150KB Mozilla list for now.
- For Android, use its default verifier, which
should actually work (unlike Oracle)
- Java 7 not required, although servers requiring SNI will now
fail on Java 6, which does not support SNI
SSLEepGet:
- Rework recent setSoTimeout code changes, as they broke SNI
- Add option to save certs even if no errors
- Add option to disable hostname verification

core/java/src/net/i2p/util/I2PSSLSocketFactory.java

@@ -21,8 +21,44 @@ package net.i2p.util;
  *  ========================================================================
  */
 
+/*
+ * Contains code adapted from:
+ * Apache httpcomponents PublicSuffixMatcherLoader.java
+ *
+ * ====================================================================
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ * ====================================================================
+ *
+ * This software consists of voluntary contributions made by many
+ * individuals on behalf of the Apache Software Foundation.  For more
+ * information on the Apache Software Foundation, please see
+ * <http://www.apache.org/>.
+ *
+ */
+import java.io.BufferedReader;
 import java.io.File;
+import java.io.FileInputStream;
 import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Locale;
 import java.net.InetAddress;
 import java.net.Socket;
 import java.security.KeyStore;
@@ -34,8 +70,13 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 
+import javax.net.ssl.HostnameVerifier;
+import javax.net.ssl.HttpsURLConnection;
 import javax.net.ssl.SSLContext;
+import javax.net.ssl.SSLException;
+import javax.net.ssl.SSLHandshakeException;
 import javax.net.ssl.SSLServerSocket;
+import javax.net.ssl.SSLSession;
 import javax.net.ssl.SSLSocket;
 import javax.net.ssl.SSLSocketFactory;
 import javax.net.ssl.TrustManagerFactory;
@@ -43,14 +84,41 @@ import javax.net.ssl.TrustManagerFactory;
 import net.i2p.I2PAppContext;
 import net.i2p.crypto.KeyStoreUtil;
 
+import org.apache.http.conn.ssl.DefaultHostnameVerifier;
+import org.apache.http.conn.util.PublicSuffixList;
+import org.apache.http.conn.util.PublicSuffixListParser;
+import org.apache.http.conn.util.PublicSuffixMatcher;
+
 /**
  * Loads trusted ASCII certs from ~/.i2p/certificates/ and $I2P/certificates/.
  *
+ * TODO extend SSLSocketFactory
+ *
  * @author zzz
  * @since 0.9.9 moved from ../client, original since 0.8.3
  */
 public class I2PSSLSocketFactory {
 
+    private static final String PROP_DISABLE = "i2p.disableSSLHostnameVerification";
+    private static final String PROP_GEOIP_DIR = "geoip.dir";
+    private static final String GEOIP_DIR_DEFAULT = "geoip";
+    private static final String GEOIP_FILE_DEFAULT = "geoip.txt";
+    private static final String COUNTRY_FILE_DEFAULT = "countries.txt";
+    private static final String PUBLIC_SUFFIX_LIST = "public-suffix-list.txt";
+    private static PublicSuffixMatcher DEFAULT_MATCHER;
+    private static boolean _matcherLoaded;
+    // not in countries.txt, but only the public ones, not the private ones
+    private static final String[] DEFAULT_TLDS = {
+        "arpa", "asia", "biz", "cat", "com", "coop",
+        "edu", "gov", "info", "int", "jobs", "mil",
+        "mobi", "museum", "name", "net", "org", "post",
+        "pro", "tel", "travel", "xxx"
+    };
+    // not in countries.txt or public-suffix-list.txt
+    private static final String[] ADDITIONAL_TLDS = {
+        "i2p", "mooo.com", "onion"
+    };
+
     /**
      *  Unmodifiable.
      *  Public for RouterConsoleRunner.
@@ -148,7 +216,9 @@ public class I2PSSLSocketFactory {
      */
     public static final List<String> INCLUDE_CIPHERS = Collections.emptyList();
 
+    /** the "real" factory */
     private final SSLSocketFactory _factory;
+    private final I2PAppContext _context;
 
     /**
      * @param relativeCertPath e.g. "certificates/i2cp"
@@ -157,27 +227,230 @@ public class I2PSSLSocketFactory {
     public I2PSSLSocketFactory(I2PAppContext context, boolean loadSystemCerts, String relativeCertPath)
                                throws GeneralSecurityException {
         _factory = initSSLContext(context, loadSystemCerts, relativeCertPath);
+        _context = context;
     }
 
     /**
      * Returns a socket to the host.
+     *
+     * A host argument that's an IP address (instead of a host name)
+     * is not recommended, as this will probably fail
+     * SSL certificate validation.
+     *
+     * Hostname validation is skipped for localhost addresses, but you still
+     * must trust the certificate.
+     *
      */
     public Socket createSocket(String host, int port) throws IOException {
         SSLSocket rv = (SSLSocket) _factory.createSocket(host, port);
         setProtocolsAndCiphers(rv);
+        verifyHostname(_context, rv, host);
         return rv;
     }
 
     /**
      * Returns a socket to the host.
+     *
+     * An InetAddress argument created with an IP address (instead of a host name)
+     * is not recommended, as this will perform a reverse DNS lookup to
+     * get the host name for certificate validation, which will probably then fail.
+     *
+     * Hostname validation is skipped for localhost addresses, but you still
+     * must trust the certificate.
+     *
      * @since 0.9.9
      */
     public Socket createSocket(InetAddress host, int port) throws IOException {
         SSLSocket rv = (SSLSocket) _factory.createSocket(host, port);
         setProtocolsAndCiphers(rv);
+        String name = host.getHostName();
+        verifyHostname(_context, rv, name);
         return rv;
     }
 
+    /**
+     *  Validate the hostname
+     *
+     *  ref: https://developer.android.com/training/articles/security-ssl.html
+     *  ref: http://op-co.de/blog/posts/java_sslsocket_mitm/
+     *  ref: http://kevinlocke.name/bits/2012/10/03/ssl-certificate-verification-in-dispatch-and-asynchttpclient/
+     *
+     *  @throws SSLException on hostname verification failure
+     *  @since 0.9.20
+     */
+    public static void verifyHostname(I2PAppContext ctx, SSLSocket socket, String host) throws SSLException {
+        Log log = ctx.logManager().getLog(I2PSSLSocketFactory.class);
+        if (ctx.getBooleanProperty(PROP_DISABLE) ||
+            host.equals("localhost") ||
+            host.equals("127.0.0.1") ||
+            host.equals("::1") ||
+            host.equals("0:0:0:0:0:0:0::1")) {
+            if (log.shouldWarn())
+                log.warn("Skipping hostname validation for " + host);
+            return;
+        }
+        HostnameVerifier hv;
+        if (SystemVersion.isAndroid()) {
+            // https://developer.android.com/training/articles/security-ssl.html
+            hv = HttpsURLConnection.getDefaultHostnameVerifier();
+        } else {
+            // haha the above may work for Android but it doesn't in Oracle
+            //
+            // quote http://kevinlocke.name/bits/2012/10/03/ssl-certificate-verification-in-dispatch-and-asynchttpclient/ :
+            // Unlike SSLContext, using the Java default (HttpsURLConnection.getDefaultHostnameVerifier)
+            // is not a viable option because the default HostnameVerifier expects to only be called
+            // in the case that there is a mismatch (and therefore always returns false) while some
+            // of the AsyncHttpClient providers (e.g. Netty, the default) call it on all connections.
+            // in the case that there is a mismatch (and therefore always returns false) while some
+            // To make matters worse, the check is not trivial (consider SAN and wildcard matching)
+            // and is implemented in sun.security.util.HostnameChecker (a Sun internal proprietary API).
+            // This leaves the developer in the position of either depending on an internal API or
+            // finding/copying/creating another implementation of this functionality.
+            //
+            hv = new DefaultHostnameVerifier(getDefaultMatcher(ctx));
+        }
+        SSLSession sess = socket.getSession();
+        // Verify that the certicate hostname is for mail.google.com
+        // This is due to lack of SNI support in the current SSLSocket.
+        if (!hv.verify(host, sess)) {
+            throw new SSLHandshakeException("SSL hostname verify failed, Expected " + host +
+                                            // throws SSLPeerUnverifiedException
+                                            //", found " + sess.getPeerPrincipal() +
+                                            // returns null
+                                            //", found " + sess.getPeerHost() +
+                                            // enable logging for DefaultHostnameVerifier to find out the CN and SANs
+                                            " - set " + PROP_DISABLE +
+                                            "=true to disable verification (dangerous!)");
+        }
+        // At this point SSLSocket performed certificate verificaiton and
+        // we have performed hostname verification, so it is safe to proceed.
+    }
+
+    /**
+     *  From Apache PublicSuffixMatcherLoader.getDefault()
+     *
+     *  https://publicsuffix.org/list/effective_tld_names.dat
+     *  What does this get us?
+     *  Deciding whether to issue or accept an SSL wildcard certificate for *.public.suffix.
+     *
+     *  @return null on failure
+     *  @since 0.9.20
+     */
+    private static PublicSuffixMatcher getDefaultMatcher(I2PAppContext ctx) {
+        synchronized (I2PSSLSocketFactory.class) {
+            if (!_matcherLoaded) {
+                String geoDir = ctx.getProperty(PROP_GEOIP_DIR, GEOIP_DIR_DEFAULT);
+                File geoFile = new File(geoDir);
+                if (!geoFile.isAbsolute())
+                    geoFile = new File(ctx.getBaseDir(), geoDir);
+                geoFile = new File(geoFile, PUBLIC_SUFFIX_LIST);
+                Log log = ctx.logManager().getLog(I2PSSLSocketFactory.class);
+                if (geoFile.exists()) {
+                    try {
+                        // we can't use PublicSuffixMatcherLoader.load() here because we
+                        // want to add some of our own and a PublicSuffixMatcher's
+                        // underlying PublicSuffixList is immutable and inaccessible
+                        long begin = System.currentTimeMillis();
+                        InputStream in = null;
+                        PublicSuffixList list = new PublicSuffixList(Arrays.asList(ADDITIONAL_TLDS),
+                                                                     Collections.<String>emptyList());
+                        try {
+                            in = new FileInputStream(geoFile);
+                            PublicSuffixList list2 = new PublicSuffixListParser().parse(
+                                 new InputStreamReader(in, "UTF-8"));
+                            list = merge(list, list2);
+                        } finally {
+                            try { if (in != null) in.close(); } catch (IOException ioe) {}
+                        }
+                        DEFAULT_MATCHER = new PublicSuffixMatcher(list.getRules(), list.getExceptions());
+                        if (log.shouldWarn())
+                            log.warn("Loaded " + geoFile + " in " + (System.currentTimeMillis() - begin) +
+                                     " ms and created list with " + list.getRules().size() + " entries and " +
+                                     list.getExceptions().size() + " exceptions");
+                    } catch (IOException ex) {
+                         log.error("Failure loading public suffix list from " + geoFile, ex);
+                         // DEFAULT_MATCHER remains null
+                    }
+                } else {
+                    List<String> list = new ArrayList<String>(320);
+                    addCountries(ctx, list);
+                    list.addAll(Arrays.asList(DEFAULT_TLDS));
+                    list.addAll(Arrays.asList(ADDITIONAL_TLDS));
+                    DEFAULT_MATCHER = new PublicSuffixMatcher(list, null);
+                    if (log.shouldWarn())
+                        log.warn("No public suffix list found at " + geoFile +
+                                 " - created default with " + list.size() + " entries");
+                }
+            }
+            _matcherLoaded = true;
+        }
+        return DEFAULT_MATCHER;
+    }
+
+   /**
+    *  Merge two PublicSuffixLists
+    *  Have to do this because they are unmodifiable
+    *
+    *  @since 0.9.20
+    */
+    private static PublicSuffixList merge(PublicSuffixList a, PublicSuffixList b) {
+        List<String> ar = a.getRules();
+        List<String> ae = a.getExceptions();
+        List<String> br = b.getRules();
+        List<String> be = b.getExceptions();
+        List<String> cr = new ArrayList<String>(ar.size() + br.size());
+        List<String> ce = new ArrayList<String>(ae.size() + be.size());
+        cr.addAll(ar);
+        cr.addAll(br);
+        ce.addAll(ae);
+        ce.addAll(be);
+        return new PublicSuffixList(cr, ce);
+    }
+
+   /**
+    *  Read in the country file and add all TLDs to the list.
+    *  It would almost be easier just to add all possible 26*26 two-letter codes.
+    *
+    *  @param tlds out parameter
+    *  @since 0.9.20 adapted from GeoIP.loadCountryFile()
+    */
+    private static void addCountries(I2PAppContext ctx, List<String> tlds) {
+        Log log = ctx.logManager().getLog(I2PSSLSocketFactory.class);
+        String geoDir = ctx.getProperty(PROP_GEOIP_DIR, GEOIP_DIR_DEFAULT);
+        File geoFile = new File(geoDir);
+        if (!geoFile.isAbsolute())
+            geoFile = new File(ctx.getBaseDir(), geoDir);
+        geoFile = new File(geoFile, COUNTRY_FILE_DEFAULT);
+        if (!geoFile.exists()) {
+            if (log.shouldWarn())
+                log.warn("Country file not found: " + geoFile.getAbsolutePath());
+            return;
+        }
+        BufferedReader br = null;
+        try {
+            br = new BufferedReader(new InputStreamReader(
+                    new FileInputStream(geoFile), "UTF-8"));
+            String line = null;
+            int i = 0;
+            while ( (line = br.readLine()) != null) {
+                try {
+                    if (line.charAt(0) == '#')
+                        continue;
+                    String[] s = line.split(",");
+                    String lc = s[0].toLowerCase(Locale.US);
+                    tlds.add(lc);
+                    i++;
+                } catch (IndexOutOfBoundsException ioobe) {}
+            }
+            if (log.shouldInfo())
+                log.info("Loaded " + i + " TLDs from " + geoFile.getAbsolutePath());
+        } catch (IOException ioe) {
+            log.error("Error reading the Country File", ioe);
+        } finally {
+            if (br != null) try { br.close(); } catch (IOException ioe) {}
+        }
+    }
+
     /**
      *  Loads certs from
      *  the ~/.i2p/certificates/ and $I2P/certificates/ directories.

core/java/src/net/i2p/util/SSLEepGet.java

@@ -55,7 +55,7 @@ import java.util.Arrays;
 import java.util.Locale;
 import javax.net.ssl.SSLContext;
 import javax.net.ssl.SSLEngine;
-import javax.net.ssl.SSLHandshakeException;
+import javax.net.ssl.SSLException;
 import javax.net.ssl.SSLParameters;
 import javax.net.ssl.SSLSocket;
 import javax.net.ssl.SSLSocketFactory;
@@ -83,7 +83,9 @@ import net.i2p.data.DataHelper;
  */
 public class SSLEepGet extends EepGet {
     /** if true, save cert chain on cert error */
-    private boolean _saveCerts;
+    private int _saveCerts;
+    /** if true, don't do hostname verification */
+    private boolean _bypassVerification;
     /** true if called from main(), used for logging */
     private boolean _commandLine;
     /** may be null if init failed */
@@ -154,15 +156,20 @@ public class SSLEepGet extends EepGet {
      * SSLEepGet -s https://foo/bar
      */ 
     public static void main(String args[]) {
-        boolean saveCerts = false;
+        int saveCerts = 0;
+        boolean noVerify = false;
         boolean error = false;
-        Getopt g = new Getopt("ssleepget", args, "s");
+        Getopt g = new Getopt("ssleepget", args, "sz");
         try {
             int c;
             while ((c = g.getopt()) != -1) {
               switch (c) {
                 case 's':
-                    saveCerts = true;
+                    saveCerts++;
+                    break;
+
+                case 'z':
+                    noVerify = true;
                     break;
 
                 case '?':
@@ -194,8 +201,10 @@ public class SSLEepGet extends EepGet {
         }
 
         SSLEepGet get = new SSLEepGet(I2PAppContext.getGlobalContext(), out, url);
-        if (saveCerts)
-            get._saveCerts = true;
+        if (saveCerts > 0)
+            get._saveCerts = saveCerts;
+        if (noVerify)
+            get._bypassVerification = true;
         get._commandLine = true;
         get.addStatusListener(get.new CLIStatusListener(1024, 40));
         if(!get.fetch(45*1000, -1, 60*1000))
@@ -203,8 +212,10 @@ public class SSLEepGet extends EepGet {
     }
     
     private static void usage() {
-        System.err.println("Usage: SSLEepGet https://url\n" +
-                           "To save unknown certs, use: SSLEepGet -s https://url");
+        System.err.println("Usage: SSLEepGet [-sz] https://url\n" +
+                           "  -s save unknown certs\n" +
+                           "  -s -s save all certs\n" +
+                           "  -z bypass hostname verification");
     }
 
     /**
@@ -352,7 +363,7 @@ public class SSLEepGet extends EepGet {
         for (int k = 0; k < chain.length; k++) {
             X509Certificate cert = chain[k];
             String name = host + '-' + (k + 1) + ".crt";
-            System.out.println("NOTE: Saving untrusted X509 certificate as " + name);
+            System.out.println("NOTE: Saving X509 certificate as " + name);
             System.out.println("      Issuer:     " + cert.getIssuerX500Principal());
             System.out.println("      Valid From: " + cert.getNotBefore());
             System.out.println("      Valid To:   " + cert.getNotAfter());
@@ -364,7 +375,6 @@ public class SSLEepGet extends EepGet {
             CertUtil.saveCert(cert, new File(name));
         }
         System.out.println("NOTE: To trust them, copy the certificate file(s) to the certificates directory and rerun without the -s option");
-        System.out.println("NOTE: EepGet failed, certificate error follows:");
     }
 
     /**
@@ -554,18 +564,28 @@ public class SSLEepGet extends EepGet {
                 port = url.getPort();
                 if (port == -1)
                     port = 443;
+                // Warning, createSocket() followed by connect(InetSocketAddress)
+                // disables SNI, at least on Java 7.
+                // So we must do createSocket(host, port) and then setSoTimeout;
+                // we can't crate a disconnected socket and then call setSoTimeout, sadly.
                 if (_sslContext != null)
-                    _proxy = _sslContext.getSocketFactory().createSocket();
+                    _proxy = _sslContext.getSocketFactory().createSocket(host, port);
                 else
-                    _proxy = SSLSocketFactory.getDefault().createSocket();
+                    _proxy = SSLSocketFactory.getDefault().createSocket(host, port);
                 if (_fetchHeaderTimeout > 0) {
                     _proxy.setSoTimeout(_fetchHeaderTimeout);
-                    _proxy.connect(new InetSocketAddress(host, port), _fetchHeaderTimeout);
-                } else {
-                    _proxy.connect(new InetSocketAddress(host, port));
                 }
                 SSLSocket socket = (SSLSocket) _proxy;
                 I2PSSLSocketFactory.setProtocolsAndCiphers(socket);
+                if (!_bypassVerification) {
+                    try {
+                        I2PSSLSocketFactory.verifyHostname(_context, socket, host);
+                    } catch (SSLException ssle) {
+                        if (_saveCerts > 0 && _stm != null)
+                            saveCerts(host, _stm);
+                        throw ssle;
+                    }
+                }
             } else {
                 throw new MalformedURLException("Only https supported: " + _actualURL);
             }
@@ -581,12 +601,14 @@ public class SSLEepGet extends EepGet {
         try {
             _proxyOut.write(DataHelper.getUTF8(req));
             _proxyOut.flush();
-        } catch (SSLHandshakeException sslhe) {
+            if (_saveCerts > 1 && _stm != null)
+                saveCerts(host, _stm);
+        } catch (SSLException sslhe) {
             // this maybe would be better done in the catch in super.fetch(), but
             // then we'd have to copy it all over here.
             _log.error("SSL negotiation error with " + host + ':' + port +
                        " - self-signed certificate or untrusted certificate authority?", sslhe);
-            if (_saveCerts && _stm != null)
+            if (_saveCerts > 0 && _stm != null)
                 saveCerts(host, _stm);
             else if (_commandLine) {
                 System.out.println("FAILED (probably due to untrusted certificates) - Run with -s option to save certificates");