GeoIP: Add MaxMind GeoLite2-Country database 2018113, gzipped

Database and Contents Copyright (c) 2018 MaxMind, Inc.
This work is licensed under the Creative Commons Attribution-ShareAlike 4.0 International License.
To view a copy of this license, visit http://creativecommons.org/licenses/by-sa/4.0/.
This database incorporates GeoNames [http://www.geonames.org] geographical data,
which is made available under the Creative Commons Attribution 3.0 License.
To view a copy of this license, visit http://www.creativecommons.org/licenses/by/3.0/us/.
- Drop old ipv4 and ipv6 data files and build scripts
- Build changes for new file

LICENSE.txt

@@ -87,6 +87,7 @@ Public domain except as listed below:
    See licenses/LICENSE-Apache2.0.txt
 
    json-simple 1.1.1
+   (not included in most distribution packages)
    See licenses/LICENSE-Apache2.0.txt
 
 
@@ -159,9 +160,7 @@ Installer:
 
 GeoIP Data:
    (not included in most distribution packages)
-   GeoLite databases are licensed under a Creative Commons Attribution-ShareAlike 3.0 Unported License.
+   This product includes GeoLite2 data created by MaxMind, available from http://www.maxmind.com/
-   http://creativecommons.org/licenses/by-sa/3.0/
-   This product includes GeoLite data created by MaxMind, available from http://www.maxmind.com/
    See licenses/LICENSE-GeoIP.txt
 
 Launchers:

build.xml

@@ -76,7 +76,7 @@
             <echo message="The following command will install the build dependencies for you:" />
             <echo message=" " />
             <echo message="sudo apt-get install debhelper ant debconf default-jdk gettext libgmp-dev po-debconf fakeroot \" />
-            <echo message="  build-essential quilt dh-apparmor dh-systemd libservice-wrapper-java \"  />
+            <echo message="  build-essential quilt dh-apparmor dh-systemd libservice-wrapper-java libjson-simple-java \"  />
             <echo message="  devscripts libjetty9-java libtomcat8-java libtaglibs-standard-jstlel-java libgetopt-java"  />
             <echo message=" " />
             <echo message="The following command will install the additional runtime dependencies:" />
@@ -1686,8 +1686,6 @@
         <copy file="build/addressbook.war" todir="pkg-temp/webapps/" />
         <!-- decapitalized the file in 0.7.8 -->
         <copy file="installer/resources/countries.txt" todir="pkg-temp/geoip/" />
-        <!-- small enough to include for now -->
-        <copy file="installer/resources/geoipv6.dat.gz" todir="pkg-temp/geoip/" />
         <copy file="installer/resources/continents.txt" todir="pkg-temp/geoip/" />
       <!--
         <copy file="installer/resources/public-suffix-list.txt" todir="pkg-temp/geoip/" />
@@ -1716,8 +1714,12 @@
     <!-- GeoIP files, set withGeoIPDatabase=true in override.properties to prevent -->
     <!-- As of 0.9.26, the files are not included in Debian/Ubuntu builds. -->
     <target name="prepgeoupdate-unlesspkg" unless="${with-geoip-database}" >
+        <mkdir dir="pkg-temp/geoip" />
+      <!--
         <copy file="installer/resources/geoip.txt" todir="pkg-temp/geoip/" />
         <copy file="installer/resources/geoipv6.dat.gz" todir="pkg-temp/geoip/" />
+       -->
+        <gunzip src="installer/resources/GeoLite2-Country.mmdb.gz" dest="pkg-temp/geoip/GeoLite2-Country.mmdb" />
     </target>
 
     <!-- All jetty jars required for update.
@@ -2113,9 +2115,9 @@
 <!--
     <target name="release" depends="distclean, updaterWithJettyFixesAndJbigi , updater200WithJettyFixes, preppkg, installer, getReleaseNumber" >
     <target name="release" depends="verifyReleaseBuildNumbers, -pre-release, distclean, testscripts, updaterWithJettyAndJbigiRepack, updater200WithJettyAndJbigi, delete-j6-update, installer-nowindows, delete-nonwindows, installer-windows">
-    <target name="release" depends="verifyReleaseBuildNumbers, -pre-release, distclean, testscripts, updaterWithJettyAndGeoIPRepack, updater200WithJettyAndGeoIP, delete-j6-update, installer-nowindows, delete-nonwindows, installer-windows">
--->
     <target name="release" depends="verifyReleaseBuildNumbers, -pre-release, distclean, testscripts, updaterWithJettyRepack, updater200WithJetty, delete-j6-update, installer-nowindows, delete-nonwindows, installer-windows">
+-->
+    <target name="release" depends="verifyReleaseBuildNumbers, -pre-release, distclean, testscripts, updaterWithJettyAndGeoIPRepack, updater200WithJettyAndGeoIP, delete-j6-update, installer-nowindows, delete-nonwindows, installer-windows">
         <echo message="New version number is ${release.number}" />
         <copy file="i2pupdate.zip" tofile="i2pupdate_${release.number}.zip" />
         <copy file="i2pinstall_${full.version}.jar" tofile="i2pinstall_${release.number}.jar" />
@@ -2467,6 +2469,7 @@
             <!-- geoip-database -->
             <file name="../i2p-${Extended.Version}/installer/resources/geoip.txt" />
             <file name="../i2p-${Extended.Version}/installer/resources/geoipv6.dat.gz" />
+            <file name="../i2p-${Extended.Version}/installer/resources/GeoLite2-Country.mmdb.gz" />
             <!-- libjetty9-java -->
             <fileset dir="../i2p-${Extended.Version}/apps/jetty/jetty-distribution-${jetty.ver}" />
             <!-- libtomcat8-java -->
@@ -2548,6 +2551,7 @@
             <!-- geoip-database -->
             <file name="../i2p-${Extended.Version}/installer/resources/geoip.txt" />
             <file name="../i2p-${Extended.Version}/installer/resources/geoipv6.dat.gz" />
+            <file name="../i2p-${Extended.Version}/installer/resources/GeoLite2-Country.mmdb.gz" />
             <!-- libjetty9-java -->
           <!--
             <fileset dir="../i2p-${Extended.Version}/apps/jetty/jetty-distribution-${jetty.ver}" />
@@ -2631,6 +2635,7 @@
            -->
             <file name="../i2p-${Extended.Version}/installer/resources/geoip.txt" />
             <file name="../i2p-${Extended.Version}/installer/resources/geoipv6.dat.gz" />
+            <file name="../i2p-${Extended.Version}/installer/resources/GeoLite2-Country.mmdb.gz" />
             <fileset dir="../i2p-${Extended.Version}/apps/jetty/jetty-distribution-${jetty.ver}" />
             <fileset dir="../i2p-${Extended.Version}/apps/jetty/apache-tomcat-${tomcat.ver}" />
           <!--
@@ -2705,6 +2710,7 @@
            -->
             <file name="../i2p-${Extended.Version}/installer/resources/geoip.txt" />
             <file name="../i2p-${Extended.Version}/installer/resources/geoipv6.dat.gz" />
+            <file name="../i2p-${Extended.Version}/installer/resources/GeoLite2-Country.mmdb.gz" />
             <fileset dir="../i2p-${Extended.Version}/apps/jetty/jetty-distribution-${jetty.ver}" />
             <fileset dir="../i2p-${Extended.Version}/apps/jetty/apache-tomcat-${tomcat.ver}" />
             <file name="../i2p-${Extended.Version}/apps/susidns/src/lib/jstl.jar" />

history.txt

@@ -1,3 +1,10 @@
+2018-11-20 zzz
+ * GeoIP: Add support for Maxmind GeoLite2 format (ticket #2268)
+
+2018-11-19 zzz
+ * Debian: Add libjson-simple-java dependency
+ * Util: Change json lib from net.minidev.json to com.json.simple
+
 2018-11-17 zzz
  * Console: Wizard fixes and improvements
 
@@ -6,8 +13,8 @@
 
 2018-11-13 zzz
  * Console:
-   - New M-Lab NDT subsystem
+   - New M-Lab NDT subsystem (ticket #2321)
-   - Setup wizard (WIP)
+   - Setup wizard (WIP) (ticket #1473)
  * Router: LS2 handling for proposal 123
 
 2018-10-13 zzz

installer/resources/checklist.md

@@ -10,14 +10,9 @@
 - Push to Transifex: `tx push -s`
 - Make announcement on Transifex with checkin deadline
 
-- GeoIP: Maxmind update is usually first week of the month, time accordingly
+- GeoIP: Maxmind update is usually first Tuesday of the month, time accordingly
-- wget http://geolite.maxmind.com/download/geoip/database/GeoIPCountryCSV.zip
+- installer/resources/makegeoip.sh
-- unzip GeoIPCountryCSV.zip
+- mtn ci installer/resources/GeoLite2-Country.mmdb
-- cut -d, -f3-5 < GeoIPCountryWhois.csv|sed 's/"//g' > geoip.txt
-- replace header
-- fix date in header (get date from HEAD)
-- ./makegeoipv6.sh
-- mtn ci geoip.txt geoipv6.dat.gz
 
 - BuildTime: Don't have to do this every release, but update the
   EARLIEST and EARLIEST_LONG values in core/java/src/net/i2p/time/BuildTime.java

installer/resources/makegeoip.sh

@@ -0,0 +1,14 @@
+#!/bin/sh
+#
+# Fetch the latest file from Maxmind,
+# and pull out what we need
+#
+rm -rf GeoLite2-Country_20*
+mv GeoLite2-Country.tar.gz GeoIPCountry2-Country.tar.gz.bak
+mv GeoLite2-Country.mmdb.gz GeoIPCountry2-Country.mmdb.gz.bak
+wget http://geolite.maxmind.com/download/geoip/database/GeoLite2-Country.tar.gz || exit 1
+tar xzf GeoLite2-Country.tar.gz || exit 1
+mv GeoLite2-Country_20*/GeoLite2-Country.mmdb . || exit 1
+gzip GeoLite2-Country.mmdb || exit 1
+rm -rf GeoLite2-Country_20*
+ls -l GeoLite2-Country.mmdb.gz

installer/resources/makegeoipv4.sh

@@ -1,26 +0,0 @@
-#!/bin/sh
-#
-# Fetch the latest file from Maxmind,
-# and pull out what we need
-#
-mv GeoIPCountryCSV.zip GeoIPCountryCSV.zip.bak
-mv GeoIPCountryWhois.csv GeoIPCountryWhois.csv.bak
-wget http://geolite.maxmind.com/download/geoip/database/GeoIPCountryCSV.zip
-unzip GeoIPCountryCSV.zip
-echo 'old entry count'
-wc -l geoip.txt
-GDATE=`ls -l --time-style=long-iso GeoIPCountryCSV.zip | cut -d ' ' -f 6`
-echo '# Last updated based on Maxmind GeoLite Country' > geoip.txt
-echo "# dated $GDATE" >> geoip.txt
-cat << EOF >> geoip.txt
-# Script borrowed from Tor
-#
-# wget http://geolite.maxmind.com/download/geoip/database/GeoIPCountryCSV.zip
-# unzip GeoIPCountryCSV.zip
-# cut -d, -f3-5 < GeoIPCountryWhois.csv|sed 's/"//g' > geoip.txt
-# cut -d, -f5,6 < GeoIPCountryWhois.csv |sed 's/"//g' | sort | uniq > countries.txt
-EOF
-cut -d, -f3-5 < GeoIPCountryWhois.csv|sed 's/"//g' >> geoip.txt
-echo 'new entry count'
-wc -l geoip.txt
-# cut -d, -f5,6 < GeoIPCountryWhois.csv |sed 's/"//g' | sort | uniq > countries.txt

installer/resources/makegeoipv6.sh

@@ -1,19 +0,0 @@
-#!/bin/sh
-#
-# Fetch the latest file from Maxmind, merge with
-# our additions, and compress.
-#
-
-FILE1=GeoIPv6.csv.gz
-FILE2=geoipv6-extras.csv
-FILEOUT=geoipv6.dat.gz
-
-rm -f $FILE1 $FILEOUT
-wget http://geolite.maxmind.com/download/geoip/database/$FILE1
-if [ "$?" -ne "0" ]
-then
-	echo 'Cannot fetch'
-	exit 1
-fi
-java -cp ../../build/i2p.jar:../../build/router.jar net.i2p.router.transport.GeoIPv6 $FILE1 $FILE2 $FILEOUT
-exit $?

licenses/LICENSE-GeoIP.txt

@@ -1,8 +1,14 @@
-The GeoLite databases are distributed under the
+Database and Contents Copyright (c) 2018 MaxMind, Inc.
-Creative Commons Attribution-ShareAlike 3.0 Unported License
+This work is licensed under the Creative Commons Attribution-ShareAlike 4.0 International License.
-http://creativecommons.org/licenses/by-sa/3.0/ .
+To view a copy of this license, visit http://creativecommons.org/licenses/by-sa/4.0/.
+No changes were made.
+
+This database incorporates GeoNames [http://www.geonames.org] geographical data,
+which is made available under the Creative Commons Attribution 3.0 License.
+To view a copy of this license, visit http://www.creativecommons.org/licenses/by/3.0/us/.
+
 The attribution requirement may be met by including the following in
 all advertising and documentation mentioning features of or use of this database:
 
-"This product includes GeoLite data created by MaxMind, available from
+"This product includes GeoLite2 data created by MaxMind, available from
 http://www.maxmind.com/"

router/java/src/net/i2p/router/RouterVersion.java

@@ -18,7 +18,7 @@ public class RouterVersion {
     /** deprecated */
     public final static String ID = "Monotone";
     public final static String VERSION = CoreVersion.VERSION;
-    public final static long BUILD = 5;
+    public final static long BUILD = 6;
 
     /** for example "-test" */
     public final static String EXTRA = "";