echelon/i2p.i2p
1
0
Fork 0
forked from I2P_Developers/i2p.i2p
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

News: Support blocklist in the news feed (proposal 129)

Browse Source
This commit is contained in:
zzz
2016-11-23 13:54:05 +00:00
parent 86c0fe327b
commit 62064da081
6 changed files with 501 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

279
apps/routerconsole/java/src/net/i2p/router/news/BlocklistEntries.java Normal file
View File

@ -0,0 +1,279 @@
package net.i2p.router.news;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStreamReader;
import java.io.IOException;
import java.security.GeneralSecurityException;
import java.security.PrivateKey;
import java.security.PublicKey;
import java.util.List;
import java.util.ArrayList;
import net.i2p.I2PAppContext;
import net.i2p.crypto.DirKeyRing;
import net.i2p.crypto.KeyRing;
import net.i2p.crypto.KeyStoreUtil;
import net.i2p.crypto.SigType;
import net.i2p.crypto.SigUtil;
import net.i2p.data.Base64;
import net.i2p.data.DataFormatException;
import net.i2p.data.DataHelper;
import net.i2p.data.Signature;
import net.i2p.data.SigningPrivateKey;
import net.i2p.data.SigningPublicKey;
import net.i2p.util.Log;
/**
* One Blocklist.
* Any String fields may be null.
*
* @since 0.9.28
*/
public class BlocklistEntries {
public final List<String> entries, removes;
public String signer;
public String sig;
public String supdated;
public long updated;
private boolean verified;
public static final int MAX_ENTRIES = 2000;
private static final String CONTENT_ROUTER = "router";
public BlocklistEntries(int capacity) {
entries = new ArrayList<String>(capacity);
removes = new ArrayList<String>(4);
}
public synchronized boolean isVerified() {
return verified;
}
public synchronized boolean verify(I2PAppContext ctx) {
if (verified)
return true;
if (signer == null || sig == null || supdated == null)
return false;
Log log = ctx.logManager().getLog(BlocklistEntries.class);
String[] ss = DataHelper.split(sig, ":", 2);
if (ss.length != 2) {
log.error("blocklist feed bad sig: " + sig);
return false;
}
SigType type = SigType.parseSigType(ss[0]);
if (type == null) {
log.error("blocklist feed bad sig: " + sig);
return false;
}
if (!type.isAvailable()) {
log.error("blocklist feed sigtype unavailable: " + sig);
return false;
}
byte[] bsig = Base64.decode(ss[1]);
if (bsig == null) {
log.error("blocklist feed bad sig: " + sig);
return false;
}
Signature ssig;
try {
ssig = new Signature(type, bsig);
} catch (IllegalArgumentException iae) {
log.error("blocklist feed bad sig: " + sig);
return false;
}
// look in both install dir and config dir for the signer cert
KeyRing ring = new DirKeyRing(new File(ctx.getBaseDir(), "certificates"));
PublicKey pubkey;
try {
pubkey = ring.getKey(signer, CONTENT_ROUTER, type);
} catch (IOException ioe) {
log.error("blocklist feed error", ioe);
return false;
} catch (GeneralSecurityException gse) {
log.error("blocklist feed error", gse);
return false;
}
if (pubkey == null) {
boolean diff = true;
try {
diff = !ctx.getBaseDir().getCanonicalPath().equals(ctx.getConfigDir().getCanonicalPath());
} catch (IOException ioe) {}
if (diff) {
ring = new DirKeyRing(new File(ctx.getConfigDir(), "certificates"));
try {
pubkey = ring.getKey(signer, CONTENT_ROUTER, type);
} catch (IOException ioe) {
log.error("blocklist feed error", ioe);
return false;
} catch (GeneralSecurityException gse) {
log.error("blocklist feed error", gse);
return false;
}
}
if (pubkey == null) {
log.error("unknown signer for blocklist feed: " + signer);
return false;
}
}
SigningPublicKey spubkey;
try {
spubkey = SigUtil.fromJavaKey(pubkey, type);
} catch (GeneralSecurityException gse) {
log.error("blocklist feed bad sig: " + sig, gse);
return false;
}
StringBuilder buf = new StringBuilder(256);
buf.append(supdated).append('\n');
for (String s : entries) {
buf.append(s).append('\n');
}
for (String s : removes) {
buf.append('!').append(s).append('\n');
}
byte[] data = DataHelper.getUTF8(buf.toString());
boolean rv = ctx.dsa().verifySignature(ssig, data, spubkey);
if (rv)
log.info("blocklist feed sig ok");
else
log.error("blocklist feed sig verify fail: " + signer);
verified = rv;
return rv;
}
/**
* BlocklistEntries [-p keystorepw] input.txt keystore.ks you@mail.i2p
* File format: One entry per line, # starts a comment, ! starts an unblock entry.
* Single IPv4 or IPv6 address only (no mask allowed), or 44-char base 64 router hash.
* See MAX_ENTRIES above.
*/
public static void main(String[] args) {
if (args.length < 3) {
System.err.println("Usage: BlocklistEntries [-p keystorepw] input.txt keystore.ks you@mail.i2p");
System.exit(1);
}
int st;
String kspass;
if (args[0].equals("-p")) {
kspass = args[1];
st = 2;
} else {
kspass = KeyStoreUtil.DEFAULT_KEYSTORE_PASSWORD;
st = 0;
}
String inputFile = args[st++];
String privateKeyFile = args[st++];
String signerName = args[st];
I2PAppContext ctx = new I2PAppContext();
List<String> elist = new ArrayList<String>(16);
List<String> rlist = new ArrayList<String>(4);
StringBuilder buf = new StringBuilder();
long now = System.currentTimeMillis();
String date = RFC3339Date.to3339Date(now);
buf.append(date).append('\n');;
BufferedReader br = null;
try {
br = new BufferedReader(new InputStreamReader(new FileInputStream(inputFile), "UTF-8"));
String s = null;
while ((s = br.readLine()) != null) {
int index = s.indexOf('#');
if (index == 0)
continue; // comment
if (index > 0)
s = s.substring(0, index);
s = s.trim();
if (s.length() < 7) {
if (s.length() > 0)
System.err.println("Bad line: " + s);
continue;
}
if (s.startsWith("!")) {
rlist.add(s.substring(1));
} else {
elist.add(s);
buf.append(s).append('\n');;
}
}
} catch (IOException ioe) {
System.err.println("load error from " + args[0]);
ioe.printStackTrace();
System.exit(1);
} finally {
if (br != null) try { br.close(); } catch (IOException ioe) {}
}
if (elist.isEmpty() && rlist.isEmpty()) {
System.err.println("nothing to sign");
System.exit(1);
}
if (elist.size() > MAX_ENTRIES) {
System.err.println("too many blocks, max is " + MAX_ENTRIES);
System.exit(1);
}
for (String s : rlist) {
buf.append('!').append(s).append('\n');
}
SigningPrivateKey spk = null;
try {
String keypw = "";
while (keypw.length() < 6) {
System.err.print("Enter password for key \"" + signerName + "\": ");
keypw = DataHelper.readLine(System.in);
if (keypw == null) {
System.out.println("\nEOF reading password");
System.exit(1);
}
keypw = keypw.trim();
if (keypw.length() > 0 && keypw.length() < 6)
System.out.println("Key password must be at least 6 characters");
}
File pkfile = new File(privateKeyFile);
PrivateKey pk = KeyStoreUtil.getPrivateKey(pkfile, kspass, signerName, keypw);
if (pk == null) {
System.out.println("Private key for " + signerName + " not found in keystore " + privateKeyFile);
System.exit(1);
}
spk = SigUtil.fromJavaKey(pk);
} catch (GeneralSecurityException gse) {
System.out.println("Error signing input file '" + inputFile + "'");
gse.printStackTrace();
System.exit(1);
} catch (IOException ioe) {
System.out.println("Error signing input file '" + inputFile + "'");
ioe.printStackTrace();
System.exit(1);
}
SigType type = spk.getType();
byte[] data = DataHelper.getUTF8(buf.toString());
Signature ssig = ctx.dsa().sign(data, spk);
if (ssig == null) {
System.err.println("sign failed");
System.exit(1);
}
String bsig = Base64.encode(ssig.getData());
// verify
BlocklistEntries ble = new BlocklistEntries(elist.size());
ble.entries.addAll(elist);
ble.removes.addAll(rlist);
ble.supdated = date;
ble.signer = signerName;
ble.sig = type.getCode() + ":" + bsig;
boolean ok = ble.verify(ctx);
if (!ok) {
System.err.println("verify failed");
System.exit(1);
}
System.out.println(" <i2p:blocklist updated=\"" + date + "\" signed-by=\"" + signerName + "\" sig=\"" + type.getCode() + ':' + bsig + "\">");
for (String e : elist) {
System.out.println(" <i2p:block>" + e + "</i2p:block>");
}
for (String e : rlist) {
System.out.println(" <i2p:unblock>" + e + "</i2p:unblock>");
}
System.out.println(" </i2p:blocklist>");
}
}

62
apps/routerconsole/java/src/net/i2p/router/news/NewsXMLParser.java
View File

@ -33,6 +33,7 @@ public class NewsXMLParser {
private final Log _log;
private List<NewsEntry> _entries;
private List<CRLEntry> _crlEntries;
private BlocklistEntries _blocklistEntries;
private NewsMetadata _metadata;
private XHTMLMode _mode;
@ -157,12 +158,24 @@ public class NewsXMLParser {
return _crlEntries;
}
/**
* The blocklist entries.
* Must call parse() first.
*
* @return null if none
* @since 0.9.28
*/
public BlocklistEntries getBlocklistEntries() {
return _blocklistEntries;
}
private void extract(Node root) throws I2PParserException {
if (!root.getName().equals("feed"))
throw new I2PParserException("no feed in XML");
_metadata = extractNewsMetadata(root);
_entries = extractNewsEntries(root);
_crlEntries = extractCRLEntries(root);
_blocklistEntries = extractBlocklistEntries(root);
}
private static NewsMetadata extractNewsMetadata(Node feed) throws I2PParserException {
@ -370,7 +383,7 @@ public class NewsXMLParser {
* @return null if none
* @since 0.9.26
*/
private List<CRLEntry> extractCRLEntries(Node feed) throws I2PParserException {
private static List<CRLEntry> extractCRLEntries(Node feed) throws I2PParserException {
Node rev = feed.getNode("i2p:revocations");
if (rev == null)
return null;
@ -397,6 +410,53 @@ public class NewsXMLParser {
return rv;
}
/**
* This does not check for any missing values.
* Any field in a BlocklistEntry may be null.
* Signature is verified here.
*
* @return null if none
* @since 0.9.28
*/
private BlocklistEntries extractBlocklistEntries(Node feed) throws I2PParserException {
Node bl = feed.getNode("i2p:blocklist");
if (bl == null)
return null;
List<Node> entries = getNodes(bl, "i2p:block");
BlocklistEntries rv = new BlocklistEntries(entries.size());
String a = bl.getAttributeValue("signed-by");
if (a.length() > 0)
rv.signer = a;
a = bl.getAttributeValue("sig");
if (a.length() > 0) {
rv.sig = a;
}
a = bl.getAttributeValue("updated");
if (a.length() > 0) {
rv.supdated = a;
long time = RFC3339Date.parse3339Date(a.trim());
if (time > 0)
rv.updated = time;
}
for (Node entry : entries) {
a = entry.getValue();
if (a != null) {
rv.entries.add(a.trim());
}
}
List<Node> rentries = getNodes(bl, "i2p:unblock");
if (entries.isEmpty() && rentries.isEmpty())
return null;
for (Node entry : rentries) {
a = entry.getValue();
if (a != null) {
rv.removes.add(a.trim());
}
}
rv.verify(_context);
return rv;
}
/**
* Helper to get all Nodes matching the name
*

107
apps/routerconsole/java/src/net/i2p/router/update/NewsFetcher.java
View File

@ -17,6 +17,7 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Locale;
import java.util.Map;
@ -28,8 +29,12 @@ import net.i2p.crypto.SU3File;
import net.i2p.crypto.TrustedUpdate;
import net.i2p.data.Base64;
import net.i2p.data.DataHelper;
import net.i2p.data.Hash;
import net.i2p.router.Banlist;
import net.i2p.router.Blocklist;
import net.i2p.router.RouterContext;
import net.i2p.router.RouterVersion;
import net.i2p.router.news.BlocklistEntries;
import net.i2p.router.news.CRLEntry;
import net.i2p.router.news.NewsEntry;
import net.i2p.router.news.NewsManager;
@ -41,6 +46,7 @@ import net.i2p.router.web.NewsHelper;
import net.i2p.update.*;
import static net.i2p.update.UpdateType.*;
import static net.i2p.update.UpdateMethod.*;
import net.i2p.util.Addresses;
import net.i2p.util.EepGet;
import net.i2p.util.FileUtil;
import net.i2p.util.Log;
@ -71,6 +77,9 @@ class NewsFetcher extends UpdateRunner {
private boolean _success;
private static final String TEMP_NEWS_FILE = "news.xml.temp";
private static final String PROP_BLOCKLIST_TIME = "router.blocklistVersion";
private static final String BLOCKLIST_DIR = "docs/feed/blocklist";
private static final String BLOCKLIST_FILE = "blocklist.txt";
public NewsFetcher(RouterContext ctx, ConsoleUpdateManager mgr, List<URI> uris) {
super(ctx, mgr, NEWS, uris);
@ -521,6 +530,14 @@ class NewsFetcher extends UpdateRunner {
persistCRLEntries(crlEntries);
else
_log.info("No CRL entries found in news feed");
// Block any new blocklist entries
BlocklistEntries ble = parser.getBlocklistEntries();
if (ble != null && ble.isVerified())
processBlocklistEntries(ble);
else
_log.info("No blocklist entries found in news feed");
// store entries and metadata in old news.xml format
String sudVersion = su3.getVersionString();
String signingKeyName = su3.getSignerString();
@ -607,6 +624,94 @@ class NewsFetcher extends UpdateRunner {
_log.logAlways(Log.WARN, "Stored " + i + " new CRL " + (i > 1 ? "entries" : "entry"));
}
/**
* Process blocklist entries
*
* @since 0.9.28
*/
private void processBlocklistEntries(BlocklistEntries ble) {
long oldTime = _context.getProperty(PROP_BLOCKLIST_TIME, 0L);
if (ble.updated <= oldTime) {
if (_log.shouldWarn())
_log.warn("Not processing blocklist " + new Date(ble.updated) +
", already have " + new Date(oldTime));
return;
}
Blocklist bl = _context.blocklist();
Banlist ban = _context.banlist();
int banned = 0;
for (Iterator<String> iter = ble.entries.iterator(); iter.hasNext(); ) {
String s = iter.next();
if (s.length() == 44) {
byte[] b = Base64.decode(s);
if (b == null || b.length != Hash.HASH_LENGTH) {
iter.remove();
continue;
}
Hash h = Hash.create(b);
if (!ban.isBanlistedForever(h))
ban.banlistRouterForever(h, "News feed");
} else {
byte[] ip = Addresses.getIP(s);
if (ip == null) {
iter.remove();
continue;
}
if (!bl.isBlocklisted(ip))
bl.add(ip);
}
if (++banned >= BlocklistEntries.MAX_ENTRIES) {
// prevent somebody from destroying the whole network
break;
}
}
for (String s : ble.removes) {
if (s.length() == 44) {
byte[] b = Base64.decode(s);
if (b == null || b.length != Hash.HASH_LENGTH)
continue;
Hash h = Hash.create(b);
if (ban.isBanlistedForever(h))
ban.unbanlistRouter(h);
} else {
byte[] ip = Addresses.getIP(s);
if (ip == null)
continue;
if (bl.isBlocklisted(ip))
bl.remove(ip);
}
}
// Save the blocks. We do not save the unblocks.
File f = new SecureFile(_context.getConfigDir(), BLOCKLIST_DIR);
f.mkdirs();
f = new File(f, BLOCKLIST_FILE);
boolean fail = false;
BufferedWriter out = null;
try {
out = new BufferedWriter(new OutputStreamWriter(new SecureFileOutputStream(f), "UTF-8"));
out.write("# ");
out.write(ble.supdated);
out.newLine();
for (String s : ble.entries) {
s = s.replace(':', ';'); // IPv6
out.write("Blocklist Feed:");
out.write(s);
out.newLine();
}
} catch (IOException ioe) {
_log.error("Error writing blocklist", ioe);
fail = true;
} finally {
if (out != null) try {
out.close();
} catch (IOException ioe) {}
}
if (!fail)
_context.router().saveConfig(PROP_BLOCKLIST_TIME, Long.toString(ble.updated));
if (_log.shouldWarn())
_log.warn("Processed " + ble.entries.size() + " blocks and " + ble.removes.size() + " unblocks from news feed");
}
/**
* Output in the old format.
*
@ -617,7 +722,7 @@ class NewsFetcher extends UpdateRunner {
NewsMetadata.Release latestRelease = data.releases.get(0);
Writer out = null;
try {
out = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(to), "UTF-8"));
out = new BufferedWriter(new OutputStreamWriter(new SecureFileOutputStream(to), "UTF-8"));
out.write("<!--\n");
// update metadata in old format
out.write("<i2p.release ");

4
core/java/src/net/i2p/crypto/DirKeyRing.java
View File

@ -21,9 +21,9 @@ import net.i2p.util.SystemVersion;
* Simple storage of each cert in a separate file in a directory.
* Limited sanitization of filenames.
*
* @since 0.9.9
* @since 0.9.9, public since 0.9.28
*/
class DirKeyRing implements KeyRing {
public class DirKeyRing implements KeyRing {
private final File _base;

11
router/java/src/net/i2p/router/Banlist.java
View File

@ -162,16 +162,17 @@ public class Banlist {
*/
public boolean banlistRouter(Hash peer, String reason, String reasonCode, String transport, long expireOn) {
if (peer == null) {
_log.error("why did we try to banlist null?", new Exception("banfaced"));
_log.error("ban null?", new Exception());
return false;
}
if (peer.equals(_context.routerHash())) {
_log.error("why did we try to banlist ourselves?", new Exception("banfaced"));
if (_log.shouldWarn())
_log.warn("not banning us", new Exception());
return false;
}
boolean wasAlready = false;
if (_log.shouldLog(Log.INFO))
_log.info("Banlisting router " + peer.toBase64() +
_log.info("Banlist " + peer.toBase64() +
((transport != null) ? " on transport " + transport : ""), new Exception("Banlist cause: " + reason));
Entry e = new Entry();
@ -228,7 +229,7 @@ public class Banlist {
private void unbanlistRouter(Hash peer, boolean realUnbanlist, String transport) {
if (peer == null) return;
if (_log.shouldLog(Log.DEBUG))
_log.debug("Calling unbanlistRouter " + peer.toBase64()
_log.debug("unbanlist " + peer.toBase64()
+ (transport != null ? "/" + transport : ""));
boolean fully = false;
@ -282,7 +283,7 @@ public class Banlist {
// prof.unbanlist();
_context.messageHistory().unbanlist(peer);
if (_log.shouldLog(Log.INFO))
_log.info("Unbanlisting router (expired) " + peer.toBase64());
_log.info("Unbanlisting (expired) " + peer.toBase64());
}
return rv;

51
router/java/src/net/i2p/router/Blocklist.java
View File

@ -76,6 +76,7 @@ public class Blocklist {
private final Object _lock = new Object();
private Entry _wrapSave;
private final Set<Hash> _inProcess = new HashSet<Hash>(4);
private final File _blocklistFeedFile;
// temp
private Map<Hash, String> _peerBlocklist = new HashMap<Hash, String>(4);
@ -95,18 +96,21 @@ public class Blocklist {
public Blocklist(RouterContext context) {
_context = context;
_log = context.logManager().getLog(Blocklist.class);
_blocklistFeedFile = new File(context.getConfigDir(), BLOCKLIST_FEED_FILE);
}
/** only for testing with main() */
private Blocklist() {
_context = null;
_log = new Log(Blocklist.class);
_blocklistFeedFile = new File(BLOCKLIST_FEED_FILE);
}
private static final String PROP_BLOCKLIST_ENABLED = "router.blocklist.enable";
private static final String PROP_BLOCKLIST_DETAIL = "router.blocklist.detail";
private static final String PROP_BLOCKLIST_FILE = "router.blocklist.file";
private static final String BLOCKLIST_FILE_DEFAULT = "blocklist.txt";
private static final String BLOCKLIST_FEED_FILE = "docs/feed/blocklist/blocklist.txt";
/**
* Loads the following files in-order:
@ -117,7 +121,7 @@ public class Blocklist {
public void startup() {
if (! _context.getBooleanPropertyDefaultTrue(PROP_BLOCKLIST_ENABLED))
return;
List<File> files = new ArrayList<File>(3);
List<File> files = new ArrayList<File>(4);
// install dir
File blFile = new File(_context.getBaseDir(), BLOCKLIST_FILE_DEFAULT);
@ -135,6 +139,7 @@ public class Blocklist {
blFile = new File(_context.getConfigDir(), file);
files.add(blFile);
}
files.add(_blocklistFeedFile);
Job job = new ReadinJob(files);
job.getTiming().setStartAfter(_context.clock().now() + 30*1000);
_context.jobQueue().addJob(job);
@ -277,6 +282,7 @@ public class Blocklist {
int badcount = 0;
int peercount = 0;
long ipcount = 0;
final boolean isFeedFile = blFile.equals(_blocklistFeedFile);
BufferedReader br = null;
try {
br = new BufferedReader(new InputStreamReader(
@ -295,9 +301,14 @@ public class Blocklist {
}
byte[] ip1 = e.ip1;
if (ip1.length == 4) {
byte[] ip2 = e.ip2;
store(ip1, ip2, count++);
ipcount += 1 + toInt(ip2) - toInt(ip1); // includes dups, oh well
if (isFeedFile) {
// temporary
add(ip1);
} else {
byte[] ip2 = e.ip2;
store(ip1, ip2, count++);
ipcount += 1 + toInt(ip2) - toInt(ip1); // includes dups, oh well
}
} else {
// IPv6
add(ip1);
@ -575,12 +586,34 @@ public class Blocklist {
_log.warn("Adding IP to blocklist: " + Addresses.toString(ip));
}
/**
* Remove from the in-memory single-IP blocklist.
* This is only works to undo add()s, NOT for the main list
* of IP ranges read in from the file.
*
* @param ip IPv4 or IPv6
* @since 0.9.28
*/
public void remove(byte ip[]) {
if (ip.length == 4)
remove(toInt(ip));
else if (ip.length == 16)
remove(new BigInteger(1, ip));
}
private boolean add(int ip) {
if (_singleIPBlocklist.size() >= MAX_IPV4_SINGLES)
return false;
return _singleIPBlocklist.add(Integer.valueOf(ip));
}
/**
* @since 0.9.28
*/
private void remove(int ip) {
_singleIPBlocklist.remove(Integer.valueOf(ip));
}
private boolean isOnSingleList(int ip) {
return _singleIPBlocklist.contains(Integer.valueOf(ip));
}
@ -595,6 +628,16 @@ public class Blocklist {
}
}
/**
* @param ip IPv6 non-negative
* @since 0.9.28
*/
private void remove(BigInteger ip) {
synchronized(_singleIPv6Blocklist) {
_singleIPv6Blocklist.remove(ip);
}
}
/**
* @param ip IPv6 non-negative
* @since IPv6