echelon/i2p.i2p
1
0
Fork 0
forked from I2P_Developers/i2p.i2p
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

GeoIP: Hook maxmind geoip2 code into transport lookup.

Browse Source
This commit is contained in:
zzz
2018-11-20 14:14:34 +00:00
parent f1297e7c62
commit ef44c36f0c
1 changed files with 108 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

120
router/java/src/net/i2p/router/transport/GeoIP.java
View File

@ -16,8 +16,10 @@ import java.util.Set;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicBoolean;
import com.maxmind.db.CHMCache;
import com.maxmind.geoip.InvalidDatabaseException; import com.maxmind.geoip.InvalidDatabaseException;
import com.maxmind.geoip.LookupService; import com.maxmind.geoip.LookupService;
import com.maxmind.geoip2.DatabaseReader;
import net.i2p.I2PAppContext; import net.i2p.I2PAppContext;
import net.i2p.data.DataHelper; import net.i2p.data.DataHelper;
@ -64,13 +66,15 @@ public class GeoIP {
public static final String PROP_GEOIP_DIR = "geoip.dir"; public static final String PROP_GEOIP_DIR = "geoip.dir";
public static final String GEOIP_DIR_DEFAULT = "geoip"; public static final String GEOIP_DIR_DEFAULT = "geoip";
static final String GEOIP_FILE_DEFAULT = "geoip.txt"; static final String GEOIP_FILE_DEFAULT = "geoip.txt";
static final String GEOIP2_FILE_DEFAULT = "GeoLite2-Country.mmdb";
static final String COUNTRY_FILE_DEFAULT = "countries.txt"; static final String COUNTRY_FILE_DEFAULT = "countries.txt";
public static final String PROP_IP_COUNTRY = "i2np.lastCountry"; public static final String PROP_IP_COUNTRY = "i2np.lastCountry";
public static final String PROP_DEBIAN_GEOIP = "geoip.dat"; public static final String PROP_DEBIAN_GEOIP = "geoip.dat";
public static final String PROP_DEBIAN_GEOIPV6 = "geoip.v6.dat"; public static final String PROP_DEBIAN_GEOIPV6 = "geoip.v6.dat";
private static final String DEBIAN_GEOIP_FILE = "/usr/share/GeoIP/GeoIP.dat"; private static final String DEBIAN_GEOIP_FILE = "/usr/share/GeoIP/GeoIP.dat";
private static final String DEBIAN_GEOIPV6_FILE = "/usr/share/GeoIP/GeoIPv6.dat"; private static final String DEBIAN_GEOIPV6_FILE = "/usr/share/GeoIP/GeoIPv6.dat";
private static final boolean ENABLE_DEBIAN = !(SystemVersion.isWindows() || SystemVersion.isAndroid()); private static final boolean DISABLE_DEBIAN = false;
private static final boolean ENABLE_DEBIAN = !DISABLE_DEBIAN && !(SystemVersion.isWindows() || SystemVersion.isAndroid());
/** maxmind API */ /** maxmind API */
private static final String UNKNOWN_COUNTRY_CODE = "--"; private static final String UNKNOWN_COUNTRY_CODE = "--";
@ -147,6 +151,8 @@ public class GeoIP {
public void run() { public void run() {
if (_lock.getAndSet(true)) if (_lock.getAndSet(true))
return; return;
File geoip2 = getGeoIP2();
DatabaseReader dbr = null;
try { try {
// clear the negative cache every few runs, to prevent it from getting too big // clear the negative cache every few runs, to prevent it from getting too big
if (((++_lookupRunCount) % CLEAR) == 0) if (((++_lookupRunCount) % CLEAR) == 0)
@ -158,19 +164,20 @@ public class GeoIP {
Arrays.sort(search); Arrays.sort(search);
File f = new File(_context.getProperty(PROP_DEBIAN_GEOIP, DEBIAN_GEOIP_FILE)); File f = new File(_context.getProperty(PROP_DEBIAN_GEOIP, DEBIAN_GEOIP_FILE));
if (ENABLE_DEBIAN && f.exists()) { if (ENABLE_DEBIAN && f.exists()) {
// Maxmind database // Maxmind v1 database
LookupService ls = null; LookupService ls = null;
try { try {
ls = new LookupService(f, LookupService.GEOIP_STANDARD); ls = new LookupService(f, LookupService.GEOIP_STANDARD);
for (int i = 0; i < search.length; i++) { for (int i = 0; i < search.length; i++) {
long ip = search[i].longValue(); Long ipl = search[i];
long ip = ipl.longValue();
// returns upper case or "--" // returns upper case or "--"
String uc = ls.getCountry(ip).getCode(); String uc = ls.getCountry(ip).getCode();
if (!uc.equals(UNKNOWN_COUNTRY_CODE)) { if (!uc.equals(UNKNOWN_COUNTRY_CODE)) {
String cached = _codeCache.get(uc.toLowerCase(Locale.US)); String cached = _codeCache.get(uc.toLowerCase(Locale.US));
_IPToCountry.put(search[i], cached); _IPToCountry.put(ipl, cached);
} else { } else {
_notFound.add(search[i]); _notFound.add(ipl);
} }
} }
} catch (IOException ioe) { } catch (IOException ioe) {
@ -180,6 +187,25 @@ public class GeoIP {
} finally { } finally {
if (ls != null) ls.close(); if (ls != null) ls.close();
} }
} else if (geoip2 != null) {
// Maxmind v2 database
try {
dbr = openGeoIP2(geoip2);
for (int i = 0; i < search.length; i++) {
Long ipl = search[i];
String ipv4 = toV4(ipl);
// returns upper case or null
String uc = dbr.country(ipv4);
if (uc != null) {
String cached = _codeCache.get(uc.toLowerCase(Locale.US));
_IPToCountry.put(ipl, cached);
} else {
_notFound.add(ipl);
}
}
} catch (IOException ioe) {
_log.error("GeoIP2 failure", ioe);
}
} else { } else {
// Tor-style database // Tor-style database
String[] countries = readGeoIPFile(search); String[] countries = readGeoIPFile(search);
@ -198,12 +224,13 @@ public class GeoIP {
Arrays.sort(search); Arrays.sort(search);
File f = new File(_context.getProperty(PROP_DEBIAN_GEOIPV6, DEBIAN_GEOIPV6_FILE)); File f = new File(_context.getProperty(PROP_DEBIAN_GEOIPV6, DEBIAN_GEOIPV6_FILE));
if (ENABLE_DEBIAN && f.exists()) { if (ENABLE_DEBIAN && f.exists()) {
// Maxmind database // Maxmind v1 database
LookupService ls = null; LookupService ls = null;
try { try {
ls = new LookupService(f, LookupService.GEOIP_STANDARD); ls = new LookupService(f, LookupService.GEOIP_STANDARD);
for (int i = 0; i < search.length; i++) { for (int i = 0; i < search.length; i++) {
long ip = search[i].longValue(); Long ipl = search[i];
long ip = ipl.longValue();
String ipv6 = toV6(ip); String ipv6 = toV6(ip);
// returns upper case or "--" // returns upper case or "--"
String uc = ls.getCountryV6(ipv6).getCode(); String uc = ls.getCountryV6(ipv6).getCode();
@ -212,9 +239,9 @@ public class GeoIP {
String cached = _codeCache.get(lc); String cached = _codeCache.get(lc);
if (cached == null) if (cached == null)
cached = lc; cached = lc;
_IPToCountry.put(search[i], cached); _IPToCountry.put(ipl, cached);
} else { } else {
_notFound.add(search[i]); _notFound.add(ipl);
} }
} }
} catch (IOException ioe) { } catch (IOException ioe) {
@ -224,8 +251,28 @@ public class GeoIP {
} finally { } finally {
if (ls != null) ls.close(); if (ls != null) ls.close();
} }
} else { } else if (geoip2 != null) {
// Tor-style database // Maxmind v2 database
try {
if (dbr == null)
dbr = openGeoIP2(geoip2);
for (int i = 0; i < search.length; i++) {
Long ipl = search[i];
String ipv6 = toV6(ipl);
// returns upper case or null
String uc = dbr.country(ipv6);
if (uc != null) {
String cached = _codeCache.get(uc.toLowerCase(Locale.US));
_IPToCountry.put(ipl, cached);
} else {
_notFound.add(ipl);
}
}
} catch (IOException ioe) {
_log.error("GeoIP2 failure", ioe);
}
} else {
// I2P format IPv6 database
String[] countries = GeoIPv6.readGeoIPFile(_context, search, _codeCache); String[] countries = GeoIPv6.readGeoIPFile(_context, search, _codeCache);
for (int i = 0; i < countries.length; i++) { for (int i = 0; i < countries.length; i++) {
if (countries[i] != null) if (countries[i] != null)
@ -236,11 +283,45 @@ public class GeoIP {
} }
} }
} finally { } finally {
if (dbr != null) try { dbr.close(); } catch (IOException ioe) {}
_lock.set(false); _lock.set(false);
} }
} }
} }
/**
* Get the GeoIP2 database file
*
* @return null if not found
* @since 0.9.38
*/
private File getGeoIP2() {
String geoDir = _context.getProperty(PROP_GEOIP_DIR, GEOIP_DIR_DEFAULT);
File geoFile = new File(geoDir);
if (!geoFile.isAbsolute())
geoFile = new File(_context.getBaseDir(), geoDir);
geoFile = new File(geoFile, GEOIP2_FILE_DEFAULT);
if (!geoFile.exists()) {
if (_log.shouldLog(Log.WARN))
_log.warn("GeoIP2 file not found: " + geoFile.getAbsolutePath());
return null;
}
return geoFile;
}
/**
* Open a GeoIP2 database
* @since 0.9.38
*/
private DatabaseReader openGeoIP2(File geoFile) throws IOException {
DatabaseReader.Builder b = new DatabaseReader.Builder(geoFile);
b.withCache(new CHMCache(256));
DatabaseReader rv = b.build();
if (_log.shouldDebug())
_log.debug("Opened GeoIP2 Database, Metadata: " + rv.getMetadata());
return rv;
}
/** /**
* Read in and parse the country file. * Read in and parse the country file.
* The file need not be sorted. * The file need not be sorted.
@ -473,6 +554,21 @@ public class GeoIP {
} }
} }
/**
* @return e.g. 1.2.3.4
* @since 0.9.38 for maxmind
*/
private static String toV4(long ip) {
StringBuilder buf = new StringBuilder(15);
for (int i = 0; i < 4; i++) {
buf.append(Long.toString((ip >> ((3-i)*8)) & 0xff));
if (i == 3)
break;
buf.append('.');
}
return buf.toString();
}
/** /**
* @return e.g. aabb:ccdd:eeff:1122:: * @return e.g. aabb:ccdd:eeff:1122::
* @since 0.9.26 for maxmind * @since 0.9.26 for maxmind
@ -498,7 +594,7 @@ public class GeoIP {
/*** /***
public static void main(String args[]) { public static void main(String args[]) {
GeoIP g = new GeoIP(new Router().getContext()); GeoIP g = new GeoIP(I2PAppContext.getGlobalContext());
String tests[] = {"0.0.0.0", "0.0.0.1", "0.0.0.2", "0.0.0.255", "1.0.0.0", String tests[] = {"0.0.0.0", "0.0.0.1", "0.0.0.2", "0.0.0.255", "1.0.0.0",
"94.3.3.3", "77.1.2.3", "127.0.0.0", "127.127.127.127", "128.0.0.0", "94.3.3.3", "77.1.2.3", "127.0.0.0", "127.127.127.127", "128.0.0.0",
"89.8.9.3", "72.5.6.8", "217.4.9.7", "175.107.027.107", "135.6.5.2", "89.8.9.3", "72.5.6.8", "217.4.9.7", "175.107.027.107", "135.6.5.2",