News: connect it all together (ticket #1425):

- Enable new NewsManager to load/store feed items on disk by UUID
 - News items are stored forever, not lost when they are removed from feed
 - News read in once at startup, not at every summary bar refresh
 - Convert old initialNews.xml and news.xml to NewsEntry format
 - Limit display to 2 news items in summary bar, /home and /console
 - New /news page to show all news

apps/routerconsole/java/src/net/i2p/router/news/NewsManager.java

@@ -4,10 +4,13 @@ import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.Reader;
+import java.text.DateFormat;
+import java.text.ParseException;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Date;
 import java.util.List;
+import java.util.TimeZone;
 
 import net.i2p.I2PAppContext;
 import net.i2p.app.ClientAppManager;
@@ -34,7 +37,13 @@ public class NewsManager implements RouterApp {
     private final ClientAppManager _cmgr;
     private volatile ClientAppState _state = UNINITIALIZED;
     private List<NewsEntry> _currentNews;
-    private NewsMetadata _currentMetadata;
+    // TODO
+    // Metadata is persisted in the old news.xml format by
+    // NewsFetcher.outputOldNewsXML() and read in at startup by
+    // ConsoleUpdateManager.startup() and NewsFetcher.checkForUpdates().
+    // While running, the UpdateManager keeps the metadata.
+    // NewsHelper looks at the news.xml timestamp.
+    //private NewsMetadata _currentMetadata;
 
     public static final String APP_NAME = "news";
     private static final String BUNDLE_NAME = "net.i2p.router.news.messages";
@@ -93,10 +102,13 @@ public class NewsManager implements RouterApp {
             String id = e.id;
             if (id == null)
                 continue;
+            String title = e.title;
             boolean found = false;
             for (int i = 0; i < _currentNews.size(); i++) {
                 NewsEntry old = _currentNews.get(i);
-                if (id.equals(old.id)) {
+                // try to prevent dups with those created from old news.xml,
+                // where the UUID is the title
+                if (id.equals(old.id) || (title != null && title.equals(old.id))) {
                     _currentNews.set(i, e);
                     found = true;
                     break;
@@ -156,7 +168,7 @@ public class NewsManager implements RouterApp {
         String newsContent = FileUtil.readTextFile(file.toString(), -1, true);
         if (newsContent == null || newsContent.equals(""))
             return Collections.emptyList();
-        return parseNews(newsContent);
+        return parseNews(newsContent, false);
     }
 
     private List<NewsEntry> parseInitialNews() {
@@ -171,7 +183,7 @@ public class NewsManager implements RouterApp {
             while((len = reader.read(buf)) > 0) {
                 out.append(buf, 0, len);
             }
-            List<NewsEntry> rv = parseNews(out.toString());
+            List<NewsEntry> rv = parseNews(out.toString(), true);
             if (!rv.isEmpty()) {
                 rv.get(0).updated = RFC3339Date.parse3339Date("2015-01-01");
             } else {
@@ -191,7 +203,12 @@ public class NewsManager implements RouterApp {
         }
     }
 
-    private List<NewsEntry> parseNews(String newsContent) {
+    /**
+     *  Used for initialNews.xml and news.xml
+     *
+     *  @param addMissingDiv true for initialNews, false for news.xml
+     */
+    private List<NewsEntry> parseNews(String newsContent, boolean addMissingDiv) {
         List<NewsEntry> rv = new ArrayList<NewsEntry>();
         // Parse news content for headings.
         boolean foundEntry = false;
@@ -205,10 +222,28 @@ public class NewsManager implements RouterApp {
             if (newsContent.length() > start + 16 &&
                 newsContent.substring(start + 4, start + 6).equals("20") &&
                 newsContent.substring(start + 14, start + 16).equals(": ")) {
+                // initialNews.xml, or old news.xml from server
                 entry.updated = RFC3339Date.parse3339Date(newsContent.substring(start + 4, start + 14));
                 newsContent = newsContent.substring(start+16);
             } else {
                 newsContent = newsContent.substring(start+4);
+                int colon = newsContent.indexOf(": ");
+                if (colon > 0 && colon <= 10) {
+                    //  Parse the format we wrote it out in, in NewsFetcher.outputOldNewsXML()
+                    //  Doesn't work if the date has a : in it, but SHORT hopefully does not
+                    DateFormat fmt = DateFormat.getDateInstance(DateFormat.SHORT);
+                    // the router sets the JVM time zone to UTC but saves the original here so we can get it
+                    String systemTimeZone = _context.getProperty("i2p.systemTimeZone");
+                    if (systemTimeZone != null)
+                        fmt.setTimeZone(TimeZone.getTimeZone(systemTimeZone));
+                    try {
+                        Date date = fmt.parse(newsContent.substring(0, colon));
+                        entry.updated = date.getTime();
+                        newsContent = newsContent.substring(colon + 2);
+                    } catch (ParseException pe) {
+                        // can't find date, will be zero
+                    }
+                }
             }
             int end = newsContent.indexOf("</h3>");
             if (end >= 0) {
@@ -222,6 +257,10 @@ public class NewsManager implements RouterApp {
                     entry.content = newsContent.substring(0, end);
                 else
                     entry.content = newsContent;
+                // initialNews.xml has the <div> before the <h3>, not after, so we lose it...
+                // add it back.
+                if (addMissingDiv)
+                    entry.content = "<div>\n" + entry.content;
                 rv.add(entry);
                 start = end;
             }

apps/routerconsole/java/src/net/i2p/router/news/NewsXMLParser.java

@@ -97,10 +97,11 @@ public class NewsXMLParser {
      *
      *  @param file XML content only. Any su3 or gunzip handling must have
      *              already happened.
+     *  @return the root node
      *  @throws IOException on any parse error
      */
-    public void parse(File file) throws IOException {
+    public Node parse(File file) throws IOException {
-        parse(new BufferedInputStream(new FileInputStream(file)));
+        return parse(new BufferedInputStream(new FileInputStream(file)));
     }
 
     /**
@@ -108,15 +109,17 @@ public class NewsXMLParser {
      *
      *  @param in XML content only. Any su3 or gunzip handling must have
      *            already happened.
+     *  @return the root node
      *  @throws IOException on any parse error
      */
-    public void parse(InputStream in) throws IOException {
+    public Node parse(InputStream in) throws IOException {
         _entries = null;
         _metadata = null;
         XMLParser parser = new XMLParser(_context);
         try {
             Node root = parser.parse(in);
             extract(root);
+            return root;
         } catch (ParserException pe) {
             throw new I2PParserException(pe);
         }
@@ -352,7 +355,7 @@ public class NewsXMLParser {
      *
      *  @return non-null
      */
-    static List<Node> getNodes(Node node, String name) {
+    public static List<Node> getNodes(Node node, String name) {
         List<Node> rv = new ArrayList<Node>();
         int count = node.getNNodes();
         for (int i = 0; i < count; i++) {

apps/routerconsole/java/src/net/i2p/router/news/PersistNews.java

@@ -35,7 +35,7 @@ import org.cybergarage.xml.ParserException;
  */
 class PersistNews {
 
-    private static final String DIR = "docs/news";
+    private static final String DIR = "docs/feed/news";
     private static final String PFX = "news-";
     private static final String SFX = ".xml.gz";
     private static final String XML_START = "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n";
@@ -207,6 +207,8 @@ class PersistNews {
     }
 
     /**
+     *  Unused for now, as we don't have any way to remember it's deleted.
+     *
      *  @return success
      */
     public static boolean delete(I2PAppContext ctx, NewsEntry entry) {

apps/routerconsole/java/src/net/i2p/router/update/NewsFetcher.java

@@ -21,12 +21,14 @@ import java.util.Map;
 import java.util.StringTokenizer;
 import java.util.TimeZone;
 
+import net.i2p.app.ClientAppManager;
 import net.i2p.crypto.SU3File;
 import net.i2p.crypto.TrustedUpdate;
 import net.i2p.data.DataHelper;
 import net.i2p.router.RouterContext;
 import net.i2p.router.RouterVersion;
 import net.i2p.router.news.NewsEntry;
+import net.i2p.router.news.NewsManager;
 import net.i2p.router.news.NewsMetadata;
 import net.i2p.router.news.NewsXMLParser;
 import net.i2p.router.util.RFC822Date;
@@ -45,6 +47,8 @@ import net.i2p.util.SSLEepGet;
 import net.i2p.util.Translate;
 import net.i2p.util.VersionComparator;
 
+import org.cybergarage.xml.Node;
+
 /**
  * Task to fetch updates to the news.xml, and to keep
  * track of whether that has an announcement for a new version.
@@ -475,10 +479,21 @@ class NewsFetcher extends UpdateRunner {
                 xml = to1;
             }
             NewsXMLParser parser = new NewsXMLParser(_context);
-            parser.parse(xml);
+            Node root = parser.parse(xml);
             xml.delete();
             NewsMetadata data = parser.getMetadata();
             List<NewsEntry> entries = parser.getEntries();
+            // add entries to the news manager
+            ClientAppManager cmgr = _context.clientAppManager();
+            if (cmgr != null) {
+                NewsManager nmgr = (NewsManager) cmgr.getRegisteredApp(NewsManager.APP_NAME);
+                if (nmgr != null) {
+                    nmgr.addEntries(entries);
+                    List<Node> nodes = NewsXMLParser.getNodes(root, "entry");
+                    nmgr.storeEntries(nodes);
+                }
+            }
+            // store entries and metadata in old news.xml format
             String sudVersion = su3.getVersionString();
             String signingKeyName = su3.getSignerString();
             File to3 = new File(_context.getTempDir(), "tmp3-" + _context.random().nextInt() + ".xml");

apps/routerconsole/java/src/net/i2p/router/web/NewsFeedHelper.java

@@ -20,7 +20,7 @@ import net.i2p.router.news.NewsManager;
 public class NewsFeedHelper extends HelperBase {
     
     private int _start = 0;
-    private int _limit = 3;
+    private int _limit = 2;
 
     /**
      *  @param limit less than or equal to zero means all
@@ -62,16 +62,16 @@ public class NewsFeedHelper extends HelperBase {
             for (NewsEntry entry : entries) {
                 if (i++ < start)
                     continue;
-                buf.append("<h3>");
+                buf.append("<div class=\"newsentry\"><h3>");
                 if (entry.updated > 0) {
                     Date date = new Date(entry.updated);
                     buf.append(fmt.format(date))
                        .append(": ");
                 }
                 buf.append(entry.title)
-                   .append("</h3>\n")
+                   .append("</h3>\n<div class=\"newscontent\">\n")
                    .append(entry.content)
-                   .append("\n");
+                   .append("\n</div></div>\n");
                 if (i >= start + max)
                     break;
             }

apps/routerconsole/java/src/net/i2p/router/web/NewsHelper.java

@@ -204,37 +204,12 @@ public class NewsHelper extends ContentHelper {
         return mgr.getStatus();
     }
 
-    private static final String BUNDLE_NAME = "net.i2p.router.news.messages";
-
     /**
      *  If we haven't downloaded news yet, use the translated initial news file
      */
     @Override
     public String getContent() {
-        File news = new File(_page);
+        return NewsFeedHelper.getEntries(_context, 0, 2);
-        if (!news.exists()) {
-            _page = (new File(_context.getBaseDir(), "docs/initialNews/initialNews.xml")).getAbsolutePath();
-            // don't use super, translate on-the-fly
-            Reader reader = null;
-            try {
-                char[] buf = new char[512];
-                StringBuilder out = new StringBuilder(2048);
-                reader = new TranslateReader(_context, BUNDLE_NAME, new FileInputStream(_page));
-                int len;
-                while((len = reader.read(buf)) > 0) {
-                    out.append(buf, 0, len);
-                }
-                return out.toString();
-            } catch (IOException ioe) {
-                return "";
-            } finally {
-                try {
-                    if (reader != null)
-                        reader.close();
-                } catch (IOException foo) {}
-            }
-        }
-        return super.getContent();
     }
 
     /**
@@ -312,7 +287,10 @@ public class NewsHelper extends ContentHelper {
                  buf.append(" <a href=\"/?news=1&amp;consoleNonce=").append(consoleNonce).append("\">")
                     .append(Messages.getString("Show news", ctx));
              }
-             buf.append("</a>");
+             buf.append("</a>" +
+                        " - <a href=\"/news\">")
+                .append(Messages.getString("Show all news", ctx))
+                .append("</a>");
          }
          return buf.toString();
     }

apps/routerconsole/java/src/net/i2p/router/web/RouterConsoleRunner.java

@@ -26,8 +26,9 @@ import net.i2p.crypto.KeyStoreUtil;
 import net.i2p.data.DataHelper;
 import net.i2p.jetty.I2PLogger;
 import net.i2p.router.RouterContext;
-import net.i2p.router.update.ConsoleUpdateManager;
 import net.i2p.router.app.RouterApp;
+import net.i2p.router.news.NewsManager;
+import net.i2p.router.update.ConsoleUpdateManager;
 import net.i2p.util.Addresses;
 import net.i2p.util.FileUtil;
 import net.i2p.util.I2PAppThread;
@@ -706,6 +707,8 @@ public class RouterConsoleRunner implements RouterApp {
         
             ConsoleUpdateManager um = new ConsoleUpdateManager(_context, _mgr, null);
             um.start();
+            NewsManager nm = new NewsManager(_context, _mgr, null);
+            nm.startup();
         
             if (PluginStarter.pluginsEnabled(_context)) {
                 t = new I2PAppThread(new PluginStarter(_context), "PluginStarter", true);

apps/routerconsole/java/src/net/i2p/router/web/SummaryBarRenderer.java

@@ -3,14 +3,20 @@ package net.i2p.router.web;
 import java.io.File;
 import java.io.IOException;
 import java.io.Writer;
+import java.text.DateFormat;
 import java.util.Collections;
+import java.util.Date;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.TimeZone;
 
+import net.i2p.app.ClientAppManager;
 import net.i2p.crypto.SigType;
 import net.i2p.data.DataHelper;
 import net.i2p.router.RouterContext;
+import net.i2p.router.news.NewsEntry;
+import net.i2p.router.news.NewsManager;
 import net.i2p.util.PortMapper;
 
 /**
@@ -604,49 +610,45 @@ public class SummaryBarRenderer {
         String consoleNonce = CSSHelper.getNonce();
         if (consoleNonce != null) {
             // Set up title and pre-headings stuff.
-            buf.append("<h3><a href=\"/configupdate\">")
+            //buf.append("<h3><a href=\"/configupdate\">")
+            buf.append("<h3><a href=\"/news\">")
                .append(_("News &amp; Updates"))
                .append("</a></h3><hr class=\"b\"><div class=\"newsheadings\">\n");
             // Get news content.
-            String newsContent = newshelper.getContent();
+            List<NewsEntry> entries = Collections.emptyList();
-            if (newsContent != "") {
+            ClientAppManager cmgr = _context.clientAppManager();
+            if (cmgr != null) {
+                NewsManager nmgr = (NewsManager) cmgr.getRegisteredApp(NewsManager.APP_NAME);
+                if (nmgr != null)
+                    entries = nmgr.getEntries();
+            }
+            if (!entries.isEmpty()) {
                 buf.append("<ul>\n");
-                // Parse news content for headings.
+                DateFormat fmt = DateFormat.getDateInstance(DateFormat.SHORT);
-                boolean foundEntry = false;
+                // the router sets the JVM time zone to UTC but saves the original here so we can get it
-                int start = newsContent.indexOf("<h3>");
+                String systemTimeZone = _context.getProperty("i2p.systemTimeZone");
-                while (start >= 0) {
+                if (systemTimeZone != null)
-                    // Add offset to start:
+                    fmt.setTimeZone(TimeZone.getTimeZone(systemTimeZone));
-                    // 4 - gets rid of <h3>
+                int i = 0;
-                    // 16 - gets rid of the date as well (assuming form "<h3>yyyy-mm-dd: Foobarbaz...")
+                final int max = 2;
-                    // Don't truncate the "congratulations" in initial news
+                for (NewsEntry entry : entries) {
-                    if (newsContent.length() > start + 16 &&
+                    buf.append("<li><a href=\"/?news=1&amp;consoleNonce=")
-                        newsContent.substring(start + 4, start + 6).equals("20") &&
+                       .append(consoleNonce)
-                        newsContent.substring(start + 14, start + 16).equals(": "))
+                       .append("\">");
-                        newsContent = newsContent.substring(start+16, newsContent.length());
+                    if (entry.updated > 0) {
-                    else
+                        Date date = new Date(entry.updated);
-                        newsContent = newsContent.substring(start+4, newsContent.length());
+                        buf.append(fmt.format(date))
-                    int end = newsContent.indexOf("</h3>");
+                           .append(": ");
-                    if (end >= 0) {
-                        String heading = newsContent.substring(0, end);
-                        buf.append("<li><a href=\"/?news=1&amp;consoleNonce=")
-                           .append(consoleNonce)
-                           .append("\">")
-                           .append(heading)
-                           .append("</a></li>\n");
-                        foundEntry = true;
                     }
-                    start = newsContent.indexOf("<h3>");
+                    buf.append(entry.title)
+                       .append("</a></li>\n");
+                    if (++i >= max)
+                        break;
                 }
                 buf.append("</ul>\n");
-                // Set up string containing <a> to show news.
+                //buf.append("<a href=\"/news\">")
-                String requestURI = _helper.getRequestURI();
+                //   .append(_("Show all news"))
-                if (requestURI.contains("/home") && !foundEntry) {
+                //   .append("</a>\n");
-                    buf.append("<a href=\"/?news=1&amp;consoleNonce=")
-                       .append(consoleNonce)
-                       .append("\">")
-                       .append(_("Show news"))
-                       .append("</a>\n");
-                }
             } else {
                 buf.append("<center><i>")
                    .append(_("none"))

apps/routerconsole/jsp/news.jsp

@@ -14,5 +14,6 @@
 <jsp:useBean class="net.i2p.router.web.NewsFeedHelper" id="feedHelper" scope="request" />
 <jsp:setProperty name="feedHelper" property="contextId" value="<%=(String)session.getAttribute(\"i2p.contextId\")%>" />
 <% feedHelper.setLimit(0); %>
+<div class="fixme" id="fixme">
 <jsp:getProperty name="feedHelper" property="entries" />
-</div></body></html>
+</div></div></body></html>

history.txt

@@ -1,3 +1,9 @@
+2015-09-15 zzz
+ * Console:
+   - Store news feed items separately on disk in XML, like a real feed reader
+   - Limit display to 2 news items in summary bar, /home and /console
+   - New /news page to show all news (ticket #1425)
+
 * 2015-09-12 0.9.22 released
 
 2015-09-11 kytv

router/java/src/net/i2p/router/RouterVersion.java

@@ -18,7 +18,7 @@ public class RouterVersion {
     /** deprecated */
     public final static String ID = "Monotone";
     public final static String VERSION = CoreVersion.VERSION;
-    public final static long BUILD = 1;
+    public final static long BUILD = 2;
 
     /** for example "-test" */
     public final static String EXTRA = "";