echelon/i2p.i2p
1
0
Fork 0
forked from I2P_Developers/i2p.i2p
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

News: Store/load individual news entries as XML in separate files by UUID.

Browse Source 
WIP, not yet hooked in.
This commit is contained in:
zzz
2015-09-14 11:06:35 +00:00
parent ed3e444d1e
commit 6f5739b9d8
2 changed files with 286 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
apps/routerconsole/java/src/net/i2p/router/news/NewsXMLParser.java
View File

@ -247,6 +247,10 @@ public class NewsXMLParser {
return rv;
}
/**
* This does not check for any missing values.
* Any fields in any NewsEntry may be null.
*/
private List<NewsEntry> extractNewsEntries(Node feed) throws I2PParserException {
List<NewsEntry> rv = new ArrayList<NewsEntry>();
List<Node> entries = getNodes(feed, "entry");
@ -345,8 +349,10 @@ public class NewsXMLParser {
/**
* Helper to get all Nodes matching the name
*
* @return non-null
*/
private static List<Node> getNodes(Node node, String name) {
static List<Node> getNodes(Node node, String name) {
List<Node> rv = new ArrayList<Node>();
int count = node.getNNodes();
for (int i = 0; i < count; i++) {
@ -432,8 +438,8 @@ public class NewsXMLParser {
}
public static void main(String[] args) {
if (args.length != 1) {
System.err.println("Usage: NewsXMLParser file.xml");
if (args.length <= 0 || args.length > 2) {
System.err.println("Usage: NewsXMLParser file.xml [parserMode]");
System.exit(1);
}
try {
@ -454,9 +460,22 @@ public class NewsXMLParser {
System.out.println("Release timestamp: " + latestRelease.date);
System.out.println("Feed timestamp: " + ud.feedUpdated);
System.out.println("Found " + entries.size() + " news entries");
Set<String> uuids = new HashSet<String>(entries.size());
for (int i = 0; i < entries.size(); i++) {
NewsEntry e = entries.get(i);
System.out.println("News #" + (i+1) + ": " + e.title + '\n' + e.content);
System.out.println("\n****** News #" + (i+1) + ": " + e.title + '\n' + e.content);
if (e.id == null)
throw new IOException("missing ID");
if (e.title == null)
throw new IOException("missing title");
if (e.content == null)
throw new IOException("missing content");
if (e.authorName == null)
throw new IOException("missing author");
if (e.updated == 0)
throw new IOException("missing updated");
if (!uuids.add(e.id))
throw new IOException("duplicate ID");
}
} catch (IOException ioe) {
ioe.printStackTrace();

263
apps/routerconsole/java/src/net/i2p/router/news/PersistNews.java Normal file
View File

@ -0,0 +1,263 @@
package net.i2p.router.news;
import java.io.BufferedInputStream;
import java.util.Collections;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.Writer;
import java.util.ArrayList;
import java.util.List;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;
import net.i2p.I2PAppContext;
import net.i2p.data.Base64;
import net.i2p.data.DataHelper;
import net.i2p.data.Hash;
import net.i2p.util.Log;
import net.i2p.util.SecureDirectory;
import net.i2p.util.SecureFileOutputStream;
import org.cybergarage.util.Debug;
import org.cybergarage.xml.Node;
import org.cybergarage.xml.ParserException;
/**
* Store and retrieve news entries from disk.
* Each entry is stored in a separate file, with the name
* derived from the UUID.
*
* @since 0.9.23
*/
public class PersistNews {
private static final String DIR = "docs/news";
private static final String PFX = "news-";
private static final String SFX = ".xml.gz";
private static final String XML_START = "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n";
/**
* Store each entry.
* Old entries are always overwritten, as they may change even without the updated date changing.
*
* @param entries each one should be "entry" at the root
* @return success
*/
public static boolean store(I2PAppContext ctx, List<Node> entries) {
Log log = ctx.logManager().getLog(PersistNews.class);
File dir = new SecureDirectory(ctx.getConfigDir(), DIR);
if (!dir.exists())
dir.mkdirs();
StringBuilder buf = new StringBuilder();
boolean rv = true;
for (Node entry : entries) {
Node nid = entry.getNode("id");
if (nid == null) {
if (log.shouldWarn())
log.warn("entry without UUID");
continue;
}
String id = nid.getValue();
if (id == null) {
if (log.shouldWarn())
log.warn("entry without UUID");
continue;
}
String name = idToName(ctx, id);
File file = new File(dir, name);
Writer out = null;
try {
out = new OutputStreamWriter(new GZIPOutputStream(new SecureFileOutputStream(file)));
out.write(XML_START);
XMLParser.toString(buf, entry);
out.write(buf.toString());
buf.setLength(0);
} catch (IOException ioe) {
if (log.shouldWarn())
log.warn("failed store to " + file, ioe);
rv = false;
} finally {
if (out != null) try { out.close(); } catch (IOException ioe) {}
}
}
return rv;
}
/**
* This does not check for any missing values.
* Any fields in any NewsEntry may be null.
* Content is not sanitized by NewsXMLParser here, do that before storing.
*
* @return non-null, sorted by updated date, newest first
*/
public static List<NewsEntry> load(I2PAppContext ctx) {
Log log = ctx.logManager().getLog(PersistNews.class);
File dir = new File(ctx.getConfigDir(), DIR);
List<NewsEntry> rv = new ArrayList<NewsEntry>();
File[] files = dir.listFiles();
if (files == null)
return rv;
for (File file : files) {
String name = file.getName();
if (!name.startsWith(PFX) || !name.endsWith(SFX))
continue;
XMLParser parser = new XMLParser(ctx);
InputStream in = null;
Node node;
boolean error = false;
try {
in = new GZIPInputStream(new FileInputStream(file));
node = parser.parse(in);
NewsEntry entry = extract(node);
if (entry != null) {
rv.add(entry);
} else {
if (log.shouldWarn())
log.warn("load error from " + file);
error = true;
}
} catch (ParserException pe) {
if (log.shouldWarn())
log.warn("load error from " + file, pe);
error = true;
} catch (IOException ioe) {
if (log.shouldWarn())
log.warn("load error from " + file, ioe);
error = true;
} finally {
if (in != null) try { in.close(); } catch (IOException ioe) {}
}
if (error)
file.delete();
}
Collections.sort(rv);
return rv;
}
/**
* This does not check for any missing values.
* Any fields in any NewsEntry may be null.
* Content is not sanitized by NewsXMLParser here, do that before storing.
*
* @return non-null, throws on errors
*/
private static NewsEntry extract(Node entry) {
NewsEntry e = new NewsEntry();
Node n = entry.getNode("title");
if (n != null) {
e.title = n.getValue();
if (e.title != null)
e.title = e.title.trim();
}
n = entry.getNode("link");
if (n != null) {
e.link = n.getValue();
if (e.link != null)
e.link = e.link.trim();
}
n = entry.getNode("id");
if (n != null) {
e.id = n.getValue();
if (e.id != null)
e.id = e.id.trim();
}
n = entry.getNode("updated");
if (n != null) {
String v = n.getValue();
if (v != null) {
long time = RFC3339Date.parse3339Date(v.trim());
if (time > 0)
e.updated = time;
}
}
n = entry.getNode("summary");
if (n != null) {
e.summary = n.getValue();
if (e.summary != null)
e.summary = e.summary.trim();
}
n = entry.getNode("author");
if (n != null) {
n = n.getNode("name");
if (n != null) {
e.authorName = n.getValue();
if (e.authorName != null)
e.authorName = e.authorName.trim();
}
}
n = entry.getNode("content");
if (n != null) {
String a = n.getAttributeValue("type");
if (a.length() > 0)
e.contentType = a;
// now recursively sanitize
// and convert everything in the content to string
StringBuilder buf = new StringBuilder(256);
for (int i = 0; i < n.getNNodes(); i++) {
Node sn = n.getNode(i);
XMLParser.toString(buf, sn);
}
e.content = buf.toString();
}
return e;
}
/**
* @return success
*/
public static boolean delete(I2PAppContext ctx, NewsEntry entry) {
String id = entry.id;
if (id == null)
return false;
String name = idToName(ctx, id);
File dir = new File(ctx.getConfigDir(), DIR);
File file = new File(dir, name);
return file.delete();
}
/**
* @param id non-null
*/
private static String idToName(I2PAppContext ctx, String id) {
byte[] bid = DataHelper.getUTF8(id);
byte[] hash = new byte[Hash.HASH_LENGTH];
ctx.sha().calculateHash(bid, 0, bid.length, hash, 0);
return PFX + Base64.encode(hash) + SFX;
}
public static void main(String[] args) {
if (args.length != 1) {
System.err.println("Usage: PersistNews file.xml");
System.exit(1);
}
I2PAppContext ctx = new I2PAppContext();
Debug.initialize(ctx);
XMLParser parser = new XMLParser(ctx);
InputStream in = null;
try {
in = new FileInputStream(args[0]);
Node root = parser.parse(in);
List<Node> entries = NewsXMLParser.getNodes(root, "entry");
store(ctx, entries);
System.out.println("Stored " + entries.size() + " entries");
} catch (ParserException pe) {
System.out.println("load error from " + args[0]);
pe.printStackTrace();
} catch (IOException ioe) {
System.out.println("load error from " + args[0]);
ioe.printStackTrace();
} finally {
if (in != null) try { in.close(); } catch (IOException ioe) {}
}
List<NewsEntry> entries = load(ctx);
System.out.println("Loaded " + entries.size() + " news entries");
for (int i = 0; i < entries.size(); i++) {
NewsEntry e = entries.get(i);
System.out.println("\n****** News #" + (i+1) + ": " + e.title + '\n' + e.content);
}
}
}