GeoIP: Add methods to get all IPs for a country

from a geoip 1 or 2 database,
to prep for a country blocklist (ticket #2759)
WIP - not hooked in yet

router/java/src/com/maxmind/db/Reader.java

@@ -4,9 +4,12 @@ import java.io.Closeable;
 import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.Writer;
 import java.net.InetAddress;
 import java.nio.ByteBuffer;
+import java.util.HashSet;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.atomic.AtomicReference;
 
 /**
@@ -149,6 +152,127 @@ public final class Reader implements Closeable {
         return this.resolveDataPointer(buffer, pointer);
     }
 
+    /**
+     *  I2P -
+     *  Write all IPv4 address ranges for the given country to out.
+     *
+     *  @param country two-letter uppper-case
+     *  @param out caller must close
+     *  @since 0.9.48
+     */
+    public void countryToIP(String country, Writer out) throws IOException {
+        Walker walker = new Walker(country, out);
+        walker.walk();
+    }
+
+    /**
+     *  I2P
+     *  @since 0.9.48
+     */
+    private class Walker {
+        private final String _country;
+        private final Writer _out;
+        private final int _nodeCount;
+        private final ByteBuffer _buffer;
+        private final Set<Integer> _negativeCache;
+        //private boolean _ipv6;
+        private int _countryRecord = -1;
+
+        /**
+         *  Write all IPv4 address ranges for the given country to out.
+         *
+         *  @param country two-letter uppper-case
+         */
+        public Walker(String country, Writer out) throws IOException {
+            _country = country;
+            _out = out;
+            _nodeCount = metadata.getNodeCount();
+            _buffer = getBufferHolder().get();
+            _negativeCache = new HashSet<Integer>(256);
+        }
+
+        /** only call once */
+        public void walk() throws IOException {
+            _out.write("# IPs for country " + _country + " from GeoIP2 database\n");
+            int record = startNode(32);
+            walk(record, 0, 0);
+            // TODO if supported in blocklist
+            //record = startNode(128);
+            //_ipv6 = true;
+            //walk(record, 0, 0);
+        }
+
+        /**
+         * Recursive, depth first
+         * @param ip big endian
+         */
+        private void walk(int record, int ip, int depth) throws IOException {
+            if (record == _nodeCount) {
+                return;
+            }
+            if (record > _nodeCount) {
+                boolean found;
+                if (_countryRecord < 0) {
+                    // Without a negative cache, perversely, the rarest
+                    // countries take the longest, because it takes longer to
+                    // find the right record and set _countryRecord.
+                    // The negative cache speeds up the search for an unknown country by 25x.
+                    // If we could find the country record in advance that would
+                    // be even better, but there's no way to do that other than
+                    // "priming" it with a known IP.
+                    if (_negativeCache.contains(Integer.valueOf(record)))
+                        return;
+                    // This is the slow part
+                    // we only need to read and parse the country record once
+                    // wouldn't work on a city database?
+                    Object o = resolveDataPointer(_buffer, record);
+                    if (!(o instanceof Map))
+                        return;
+                    Map m = (Map) o;
+                    o = m.get("country");
+                    if (!(o instanceof Map))
+                        return;
+                    m = (Map) o;
+                    o = m.get("iso_code");
+                    found = _country.equals(o);
+                    if (found) {
+                        _countryRecord = record;
+                        _negativeCache.clear();
+                    } else {
+                        if (_negativeCache.size() < 10000)  // don't blow up on city database?
+                            _negativeCache.add(Integer.valueOf(record));
+                    }
+                } else {
+                    found = record == _countryRecord;
+                }
+                if (found) {
+                    String sip;
+                    //if (_ipv6) {
+                    //    sip = Integer.toHexString((ip >> 16) & 0xffff) + ":" +
+                    //          Integer.toHexString(ip & 0xffff) + "::/" + depth;
+                    //} else {
+                        sip = ((ip >> 24) & 0xff) + "." +
+                                 ((ip >> 16) & 0xff) + '.' +
+                                 ((ip >> 8) & 0xff) + '.' +
+                                 (ip & 0xff);
+                    //}
+                    _out.write(sip);
+                    if (depth < 32) {
+                        _out.write('/');
+                        _out.write(Integer.toString(depth));
+                    }
+                    _out.write('\n');
+                }
+                return;
+            }
+            if (depth >= 32)
+                return;
+            walk(readNode(_buffer, record, 0), ip, depth + 1);
+            ip |= 1 << (31 - depth);
+            walk(readNode(_buffer, record, 1), ip, depth + 1);
+        }
+    }
+
     private BufferHolder getBufferHolder() throws ClosedDatabaseException {
         BufferHolder bufferHolder = this.bufferHolderReference.get();
         if (bufferHolder == null) {

router/java/src/com/maxmind/geoip/LookupService.java

@@ -23,6 +23,7 @@ package com.maxmind.geoip;
 import java.io.File;
 import java.io.IOException;
 import java.io.RandomAccessFile;
+import java.io.Writer;
 import java.net.InetAddress;
 import java.net.UnknownHostException;
 import java.nio.ByteBuffer;
@@ -31,6 +32,7 @@ import java.nio.charset.Charset;
 import java.nio.charset.CharsetDecoder;
 import java.util.Arrays;
 import java.util.List;
+import java.util.Locale;
 
 /**
  * Provides a lookup service for information based on an IP address. The
@@ -501,6 +503,90 @@ public class LookupService {
         }
     }
 
+    /**
+     *  I2P -
+     *  Write all IPv4 address ranges for the given country to out.
+     *
+     *  @param country two-letter case-insensitive
+     *  @param out caller must close
+     *  @since 0.9.48
+     */
+    public synchronized void countryToIP(String country, Writer out) throws IOException {
+        if (file == null && (dboptions & GEOIP_MEMORY_CACHE) == 0) {
+            throw new IllegalStateException("Database has been closed.");
+        }
+        country = country.toUpperCase(Locale.US);
+        // get the country ID we're looking for
+        int id = 0;
+        for (int i = 0; i < countryCode.length; i++) {
+            if (countryCode[i].equals(country)) {
+                id = i;
+                break;
+            }
+        }
+        if (id <= 0)
+            return;
+        // segment
+        id += COUNTRY_BEGIN;
+        Walker walker = new Walker(id, out);
+        out.write("# IPs for country " + country + " from GeoIP database\n");
+        walker.walk();
+    }
+
+    /**
+     *  I2P
+     *  @since 0.9.48
+     */
+    private class Walker {
+        private final int _country;
+        private final Writer _out;
+        private final byte[] _buf = new byte[2 * MAX_RECORD_LENGTH];
+        private final int[] _x = new int[2];
+        private final int _dbs0 = databaseSegments[0];
+
+        /**
+         * @param country the segment
+         */
+        public Walker(int country, Writer out) throws IOException {
+            _country = country;
+            _out = out;
+        }
+
+        /** only call once */
+        public void walk() throws IOException {
+            walk(0, 0, 31);
+        }
+
+        /**
+         * Recursive, depth first
+         * @param ip big endian
+         */
+        private void walk(int offset, int ip, int depth) throws IOException {
+            if (offset >= _dbs0) {
+                if (offset == _country) {
+                    String sip = ((ip >> 24) & 0xff) + "." +
+                                 ((ip >> 16) & 0xff) + '.' +
+                                 ((ip >> 8) & 0xff) + '.' +
+                                 (ip & 0xff);
+                    _out.write(sip);
+                    if (depth >= 0) {
+                        _out.write('/');
+                        _out.write(Integer.toString(31 - depth));
+                    }
+                    _out.write('\n');
+                }
+                return;
+            }
+            if (depth < 0)
+                return;
+            readNode(_buf, _x, offset);
+            int x1 = _x[1];
+            walk(_x[0], ip, depth - 1);
+            ip |= 1 << depth;
+            walk(x1, ip, depth - 1);
+        }
+    }
+
     public int getID(String ipAddress) {
         InetAddress addr;
         try {

router/java/src/com/maxmind/geoip2/DatabaseReader.java

@@ -7,9 +7,11 @@ import java.io.Closeable;
 import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.Writer;
 import java.net.InetAddress;
 import java.util.Collections;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 
 /**
@@ -228,6 +230,18 @@ public class DatabaseReader implements Closeable {
 
     }
 
+    /**
+     *  I2P -
+     *  Write all IPv4 address ranges for the given country to out.
+     *
+     *  @param country two-letter case-insensitive
+     *  @param out caller must close
+     *  @since 0.9.48
+     */
+    public void countryToIP(String country, Writer out) throws IOException {
+        reader.countryToIP(country.toUpperCase(Locale.US), out);
+    }
+
     /**
      * @return the metadata for the open MaxMind DB file.
      */