2005-10-30 dust

* Merge sucker into syndie with a rssimport.jsp page.
    * Add getContentType() to EepGet.
    * Make chunked transfer work (better) with EepGet.
    * Do replaceAll("<","&lt;") for logs.

apps/routerconsole/java/src/net/i2p/router/web/LogsHelper.java

@@ -30,6 +30,7 @@ public class LogsHelper {
         buf.append("<code>\n");
         for (int i = msgs.size(); i > 0; i--) { 
             String msg = (String)msgs.get(i - 1);
+            msg = msg.replaceAll("<","&lt;");
             buf.append("<li>");
             buf.append(msg);
             buf.append("</li>\n");
@@ -46,6 +47,7 @@ public class LogsHelper {
         buf.append("<code>\n");
         for (int i = msgs.size(); i > 0; i--) { 
             String msg = (String)msgs.get(i - 1);
+            msg = msg.replaceAll("<","&lt;");
             buf.append("<li>");
             buf.append(msg);
             buf.append("</li>\n");
@@ -59,9 +61,11 @@ public class LogsHelper {
         String str = FileUtil.readTextFile("wrapper.log", 500, false);
         if (str == null) 
             return "";
-        else
+        else {
+            str = str.replaceAll("<","&lt;");
             return "<pre>" + str + "</pre>";
         }
+    }
     
     public String getConnectionLogs() {
         List msgs = _context.commSystem().getMostRecentErrorMessages();

apps/syndie/java/src/net/i2p/syndie/BlogManager.java

@@ -324,8 +324,10 @@ public class BlogManager {
     }
     
     public boolean isConfigured() {
-        File cfg = getConfigFile();
+        String p = _context.getProperty("syndie.configurationCheck");
-        return (cfg.exists());
+        if(p==null)
+            return false;
+        return true;
     }
     
     /**
@@ -348,6 +350,73 @@ public class BlogManager {
         return delay;
     }
     
+    public List getRssFeeds() {
+        List feedList = new ArrayList();
+        int i=0;
+        while(true) {
+            String url = _context.getProperty("syndie.rssFeed."+i+".url");
+            String blog = _context.getProperty("syndie.rssFeed."+i+".blog");
+            String tagPrefix = _context.getProperty("syndie.rssFeed."+i+".tagPrefix");
+            if(url==null || blog==null || tagPrefix==null)
+                break;
+            String feed[] = new String[3];
+            feed[0]=url.trim();
+            feed[1]=blog.trim();
+            feed[2]=tagPrefix.trim();
+            feedList.add(feed);
+            i++;
+        }
+        return feedList;
+    }
+    public boolean addRssFeed(String url, String blog, String tagPrefix) {
+        
+        List feedList = getRssFeeds();
+        int nextIdx=feedList.size();
+        
+        String baseFeedProp="syndie.rssFeed."+nextIdx;
+        System.setProperty(baseFeedProp+".url",url);
+        System.setProperty(baseFeedProp+".blog",blog);
+        System.setProperty(baseFeedProp+".tagPrefix",tagPrefix);
+        _log.info("addRssFeed("+nextIdx+"): "+url);
+        writeConfig();
+        Updater.wakeup();
+        return true;
+    }
+    public boolean deleteRssFeed(String url, String blog, String tagPrefix) {
+        List feedList = getRssFeeds();
+        Iterator iter = feedList.iterator();
+        int idx=0;
+        while(iter.hasNext()) {
+            String fields[] = (String[])iter.next();
+            if(fields[0].equals(url) &&
+               fields[1].equals(blog) &&
+               fields[2].equals(tagPrefix)) {
+                break;
+            }
+            idx++;
+        }
+        
+        // copy any remaining to idx-1
+        while(iter.hasNext()) {
+            String fields[] = (String[])iter.next();
+            String baseFeedProp="syndie.rssFeed."+idx;
+            System.setProperty(baseFeedProp+".url",fields[0]);
+            System.setProperty(baseFeedProp+".blog",fields[1]);
+            System.setProperty(baseFeedProp+".tagPrefix",fields[2]);
+            idx++;
+        }
+        
+        // Delete last idx from properties
+        String baseFeedProp="syndie.rssFeed."+idx;
+        System.getProperties().remove(baseFeedProp+".url");
+        System.getProperties().remove(baseFeedProp+".blog");
+        System.getProperties().remove(baseFeedProp+".tagPrefix");
+        _log.info("deleteRssFeed("+idx+"): "+url);
+        writeConfig();
+        return true;
+    }
+     
+    
     public boolean authorizeAdmin(String pass) {
         if (isSingleUser()) return true;
         String admin = getAdminPasswordHash();
@@ -388,6 +457,7 @@ public class BlogManager {
             if (defaultProxyPort > 0)
                 out.write("syndie.defaultProxyPort="+defaultProxyPort + "\n");
             out.write("syndie.singleUser=" + isSingleUser + "\n");
+            out.write("syndie.configurationCheck=foo\n");
             if (opts != null) {
                 for (Iterator iter = opts.keySet().iterator(); iter.hasNext(); ) {
                     String key = (String)iter.next();
@@ -525,10 +595,13 @@ public class BlogManager {
     }
     
     public BlogURI createBlogEntry(User user, String subject, String tags, String entryHeaders, String sml) {
-        return createBlogEntry(user, subject, tags, entryHeaders, sml, null, null, null);
+        return createBlogEntry(user, true, subject, tags, entryHeaders, sml, null, null, null);
     }
     public BlogURI createBlogEntry(User user, String subject, String tags, String entryHeaders, String sml, List fileNames, List fileStreams, List fileTypes) {
-        if (!user.getAuthenticated()) return null;
+        return createBlogEntry(user, true, subject, tags, entryHeaders, sml, fileNames, fileStreams, fileTypes);        
+    }
+    public BlogURI createBlogEntry(User user, boolean shouldAuthenticate, String subject, String tags, String entryHeaders, String sml, List fileNames, List fileStreams, List fileTypes) {
+        if (shouldAuthenticate && !user.getAuthenticated()) return null;
         BlogInfo info = getArchive().getBlogInfo(user.getBlog());
         if (info == null) return null;
         SigningPrivateKey privkey = getMyPrivateKey(info);
@@ -601,7 +674,10 @@ public class BlogManager {
             if (ok) {
                 getArchive().regenerateIndex();
                 user.setMostRecentEntry(entryId);
+                if(shouldAuthenticate)
                     saveUser(user);
+                else
+                    storeUser(user);
                 return uri;
             } else {
                 return null;

apps/syndie/java/src/net/i2p/syndie/Sucker.java

@@ -6,16 +6,14 @@ import java.io.FileInputStream;
 import java.io.FileNotFoundException;
 import java.io.FileOutputStream;
 import java.io.IOException;
-import java.io.InputStream;
 import java.net.MalformedURLException;
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
+import java.util.ArrayList;
 import java.util.Date;
 import java.util.Iterator;
 import java.util.List;
-import java.util.ListIterator;
+import java.util.Properties;
-
-//import sun.security.provider.SHA;
 
 import com.sun.syndication.feed.synd.SyndCategory;
 import com.sun.syndication.feed.synd.SyndContent;
@@ -27,9 +25,15 @@ import com.sun.syndication.io.XmlReader;
 
 import net.i2p.I2PAppContext;
 import net.i2p.data.Base64;
+import net.i2p.data.DataFormatException;
+import net.i2p.data.DataHelper;
+import net.i2p.data.Hash;
+import net.i2p.syndie.data.BlogURI;
 import net.i2p.util.EepGet;
+import net.i2p.util.Log;
 
 public class Sucker {
+    private static final Log _log = I2PAppContext.getGlobalContext().logManager().getLog(Sucker.class);
     private String urlToLoad;
     private String outputDir="./sucker_out";
     private String historyPath="./sucker.history";
@@ -46,8 +50,49 @@ public class Sucker {
     private boolean pendingEndLink;
     private boolean shouldProxy;
     private int proxyPortNum;
+    private String blog;
+    private boolean pushToSyndie;
+    private long messageNumber=0;
+    private BlogManager bm;
+    private User user;
     
-    public Sucker() {}
+    //
+    private List fileNames;
+    private List fileStreams;
+    private List fileTypes;
+    
+    public Sucker() {
+    }
+    
+    /**
+     * Constructor for BlogManager. 
+     */
+    public Sucker(String[] strings) throws IllegalArgumentException {
+        pushToSyndie=true;
+        urlToLoad = strings[0];
+        blog = strings[1];
+        feedTag = strings[2];
+        outputDir = "blog-"+blog;
+        try {
+            historyPath=BlogManager.instance().getRootDir().getCanonicalPath()+"/rss.history";
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
+        proxyPort="4444";
+        proxyHost="localhost";
+
+        bm = BlogManager.instance();
+        Hash blogHash = new Hash();
+        try {
+            blogHash.fromBase64(blog);
+        } catch (DataFormatException e1) {
+            throw new IllegalArgumentException("ooh, bad $blog");
+        }
+     
+        user = bm.getUser(blogHash);
+        if(user==null)
+            throw new IllegalArgumentException("wtf, user==null? hash:"+blogHash);
+    }
 
     public boolean parseArgs(String args[]) {
         for (int i = 0; i < args.length; i++) {
@@ -78,44 +123,33 @@ public class Sucker {
         if (urlToLoad == null)
             return false;
 
-        // Find base url, gah HELP
-        int idx=urlToLoad.length();
-        int x=urlToLoad.indexOf('?');
-        if(x>0)
-            idx=x;
-        while(idx>0)
-        {
-            idx--;
-            if(urlToLoad.charAt(idx)=='/')
-                break;
-        }
-        if(idx==0)
-            idx=x;
-        baseUrl=urlToLoad.substring(0,idx);
-
-	System.out.println("Processing: "+urlToLoad);
-
         return true;
     }
     
+    /**
+     * Fetch urlToLoad and call convertToHtml() on any new entries.
+     */
     public void suck() {
         SyndFeed feed;
         
+        // Find base url
+        int idx=urlToLoad.lastIndexOf('/');
+        if(idx>0)
+            baseUrl=urlToLoad.substring(0,idx);
+        else
+            baseUrl=urlToLoad;
+
+        debugLog("Processing: "+urlToLoad);
+        debugLog("Base url: "+baseUrl);
+
         //
         try {
+            File lastIdFile=null;
          
-            // Create outputDir if missing
+            // Get next message number to use (for messageId in history only)
-            File f = new File(outputDir);
+            if(!pushToSyndie) {
-            f.mkdirs();
                 
-            // Create historyFile if missing
+                lastIdFile = new File(historyPath + ".lastId");
-            historyFile = new File(historyPath);
-            if (!historyFile.exists())
-                historyFile.createNewFile();
-
-            int messageNumber;
-
-            File lastIdFile = new File(historyPath + ".lastId");
                 if (!lastIdFile.exists())
                     lastIdFile.createNewFile();
                 
@@ -127,7 +161,17 @@ public class Sucker {
                     messageNumber = 0;
                 }
 
-            SyndFeedInput input = new SyndFeedInput();
+                // Create outputDir if missing
+                File f = new File(outputDir);
+                f.mkdirs();
+            } else {
+                messageNumber=bm.getNextBlogEntry(user);
+            }
+
+            // Create historyFile if missing
+            historyFile = new File(historyPath);
+            if (!historyFile.exists())
+                historyFile.createNewFile();
 
             shouldProxy = false;
             proxyPortNum = -1;
@@ -140,11 +184,11 @@ public class Sucker {
                     nfe.printStackTrace();
                 }
             }
+            
+            // fetch
             int numRetries = 2;
-            // perhaps specify a temp dir?
             File fetched = File.createTempFile("sucker", ".fetch");
             fetched.deleteOnExit();
-            // we use eepGet, since it retries and doesn't leak DNS requests like URL does
             EepGet get = new EepGet(I2PAppContext.getGlobalContext(), shouldProxy, proxyHost, proxyPortNum, 
                                     numRetries, fetched.getAbsolutePath(), urlToLoad);
             SuckerFetchListener lsnr = new SuckerFetchListener();
@@ -155,89 +199,71 @@ public class Sucker {
                 System.err.println("Unable to retrieve the url after " + numRetries + " tries.");
                 return;
             }
+            if(get.getNotModified()) {
+                debugLog("not modified, saving network bytes from useless fetch");
+                return;
+            }
 
+            // Build entry list from fetched rss file
+            SyndFeedInput input = new SyndFeedInput();
             feed = input.build(new XmlReader(fetched));
 
             List entries = feed.getEntries();
 
             FileOutputStream hos = new FileOutputStream(historyFile, true);
 
-            ListIterator iter = entries.listIterator();
+            // Process list backwards to get syndie to display the 
-            while (iter.hasNext()) {
+            // entries in the right order. (most recent at top)
+            for (int i = entries.size()-1; i >= 0; i--) { 
+                SyndEntry e = (SyndEntry) entries.get(i);
                 
                 attachmentCounter=0;
                 
-                SyndEntry e = (SyndEntry) iter.next();
+                String messageId = convertToSml(e);
-                // Calculate messageId
+                if (messageId!=null) {
-                String feedHash = sha1(urlToLoad);
-                String itemHash = sha1(e.getTitle() + e.getDescription());
-                Date d = e.getPublishedDate();
-                String time;
-                if(d!=null)
-                    time = "" + d.getTime();
-                else
-                    time = "" + new Date().getTime();
-                    
-                String outputFileName = outputDir + "/" + messageNumber;
-
-                /*
-                 * $feedHash:$itemHash:$time:$outputfile. $feedHash would be the
-                 * hash (md5? sha1? sha2?) of the $urlToFeed, $itemHash is some
-                 * hash of the SyndEntry, $time would be the time that the entry
-                 * was posted $outputfile would be the $outputdir/$uniqueid
-                 */
-                String messageId = feedHash + ":" + itemHash + ":" + time + ":"
-                                   + outputFileName;
-
-                // Check if we already have this
-                if (!existsInHistory(messageId)) {
-                    System.out.println("new: " + messageId);
-
-                    if (convertToSml(e, ""+messageNumber)) {
-
-                        if (pushScript != null) {
-                            if (!execPushScript(""+messageNumber, time)) {
-                                System.out.println("################## push failed");
-                            }else {
-                                System.out.println("push success");
                     hos.write(messageId.getBytes());
                     hos.write("\n".getBytes());
                 }
             }
-                    }
-                    messageNumber++;
-                }
-            }
             
+            if(!pushToSyndie) {
                 FileOutputStream fos = new FileOutputStream(lastIdFile);
                 fos.write(("" + messageNumber).getBytes());
+            }
         } catch (MalformedURLException e) {
-            // TODO Auto-generated catch block
             e.printStackTrace();
         } catch (IllegalArgumentException e) {
-            // TODO Auto-generated catch block
             e.printStackTrace();
         } catch (FeedException e) {
-            // TODO Auto-generated catch block
             e.printStackTrace();
         } catch (IOException e) {
-            // TODO Auto-generated catch block
             e.printStackTrace();
         }
-        System.out.println("Done.");
+        debugLog("Done.");
     }
 
     public static void main(String[] args) {
         Sucker sucker = new Sucker();
         boolean ok = sucker.parseArgs(args);
         if (!ok) {
-            usage();
+            System.out.println("sucker --load $urlToFeed \n"
-            return;
+                    + "--proxyhost <host> \n" 
+                    + "--proxyport <port> \n"
+                    + "--importenclosures true \n" 
+                    + "--importrefs true \n"
+                    + "--tag feed \n" 
+                    + "--outputdir ./sucker_out \n"
+                    + "--exec pushscript.sh OUTPUTDIR UNIQUEID ENTRYTIMESTAMP \n"
+                    + "--history ./sucker.history");
+            System.exit(1);
         }
         
         sucker.suck();
     }
 
+    /**
+     * Call the specified script with "$outputDir $id and $time". 
+     */
     private boolean execPushScript(String id, String time) {
         try {
             String ls_str;
@@ -251,7 +277,7 @@ public class Sucker {
 
             try {
                 while ((ls_str = ls_in.readLine()) != null) {
-                    System.out.println(pushScript + ": " + ls_str);
+                    debugLog(pushScript + ": " + ls_str);
                 }
             } catch (IOException e) {
                 return false;
@@ -261,7 +287,6 @@ public class Sucker {
                 if(pushScript_proc.exitValue()==0)
                     return true;
             } catch (InterruptedException e) {
-                // TODO Auto-generated catch block
                 e.printStackTrace();
             }
             return false;
@@ -271,40 +296,57 @@ public class Sucker {
         }
     }
 
-    private boolean convertToSml(SyndEntry e, String messageName) {
+    /** 
+     * Converts the SyndEntry e to sml and fetches any images as attachments 
+     */ 
+    private String convertToSml(SyndEntry e) {
+        String subject;
+        
+        // Calculate messageId, and check if we have got the message already
+        String feedHash = sha1(urlToLoad);
+        String itemHash = sha1(e.getTitle() + e.getDescription());
+        Date d = e.getPublishedDate();
+        String time;
+        if(d!=null)
+            time = "" + d.getTime();
+        else
+            time = "" + new Date().getTime();
+        String outputFileName = outputDir + "/" + messageNumber;
+        String messageId = feedHash + ":" + itemHash + ":" + time + ":" + outputFileName;
+        // Check if we already have this
+        if (existsInHistory(messageId))
+            return null;
+        
+        debugLog("new: " + messageId);
             
-        // Create message
-        FileOutputStream fos;
-        messagePath=outputDir+"/"+messageName;
         try {
-            fos = new FileOutputStream(messagePath);
 
-            String sml;
+            String sml="";
-            sml = "Subject: " + e.getTitle() + "\n";
+            subject=e.getTitle();
             List cats = e.getCategories();
             Iterator iter = cats.iterator();
             String tags = feedTag;
             while (iter.hasNext()) {
                 SyndCategory c = (SyndCategory) iter.next();
+                debugLog("Name: "+c.getName());
+                debugLog("uri:"+c.getTaxonomyUri());
                 String tag=c.getName();
                 tag=tag.replaceAll("[^a-zA-z.-_:]","_");
                 tags += "\t" + feedTag + "." + tag;
             }
-            sml += "Tags: " + tags + "\n";
-            sml += "\n";
 
             SyndContent content;
 
             List l = e.getContents();
             if(l!=null)
             {
-            System.out.println("There is content");
+            debugLog("There is content");
                 iter = l.iterator();
                 while(iter.hasNext())
                 {
                     content = (SyndContent)iter.next();
                     String c = content.getValue();
-                    System.out.println("Content: "+c);
+                    debugLog("Content: "+c);
                                   sml += htmlToSml(c);
                                   sml += "\n";
                 }
@@ -314,17 +356,53 @@ public class Sucker {
             source=baseUrl+source;
             sml += "[link schema=\"web\" location=\""+source+"\"]source[/link]\n";
 
-            fos.write(sml.getBytes());
+            if(pushToSyndie) {
+                debugLog("user.blog: "+user.getBlogStr());
+                debugLog("user.id: "+bm.getNextBlogEntry(user));
+                debugLog("subject: "+subject);
+                debugLog("tags: "+tags);
+                debugLog("sml: "+sml);
+                debugLog("");
+                BlogURI uri = bm.createBlogEntry(
+                        user, 
+                        false,
+                        subject, 
+                        tags, 
+                        null,
+                        sml, 
+                        fileNames, 
+                        fileStreams, 
+                        fileTypes);
 
-            return true;
+                if(uri==null) {
+                    debugLog("pushToSyndie failure.");
+                    return null;
+                }
+                else
+                    debugLog("pushToSyndie success, uri: "+uri.toString());
+            }
+            else
+            {
+                FileOutputStream fos;
+                fos = new FileOutputStream(messagePath);
+                sml=subject + "\nTags: " + tags + "\n\n" + sml;
+                fos.write(sml.getBytes());
+                if (pushScript != null) {
+                    if (!execPushScript(""+messageNumber, time)) {
+                        debugLog("################## push failed");
+                    } else {
+                        debugLog("push success");
+                    }
+                }
+            }
+            messageNumber++;
+            return messageId;
         } catch (FileNotFoundException e1) {
-            // TODO Auto-generated catch block
             e1.printStackTrace();
         } catch (IOException e2) {
-            // TODO Auto-generated catch block
             e2.printStackTrace();
         }
-        return false;
+        return null;
     }
 
     private String htmlToSml(String html) {
@@ -338,29 +416,32 @@ public class Sucker {
         {
             if(html.charAt(i)=='<')
             {
-                //System.out.println("html: "+html.substring(i));
+                //log("html: "+html.substring(i));
                 
                 int tagLen = findTagLen(html.substring(i));
                 if(tagLen<=0)
                 {
-                        System.out.println("Bad html? ("+html+")");
+                        debugLog("Bad html? ("+html+")");
                         break;
                 }
                 //
                 String htmlTag = html.substring(i,i+tagLen);
                 
-                //System.out.println("htmlTag: "+htmlTag);
+                //log("htmlTag: "+htmlTag);
                 
                 String smlTag = htmlTagToSmlTag(htmlTag);
                 if(smlTag!=null)
                     sml+=smlTag;
                 i+=tagLen;
-                //System.out.println("tagLen: "+tagLen);
+                //log("tagLen: "+tagLen);
                 sml+=" "; 
             }
             else
             {
-                sml+=html.charAt(i++);
+                char c=html.charAt(i++);
+                sml+=c;
+                if(c=='[' || c==']')
+                    sml+=c;
             }
         }
         
@@ -373,7 +454,7 @@ public class Sucker {
 
         if(importEnclosures && htmlTagLowerCase.startsWith("<img"))
         {
-            System.out.println("Found image tag: "+htmlTag);
+            debugLog("Found image tag: "+htmlTag);
             int a,b;
             a=htmlTagLowerCase.indexOf("src=\"")+5;
             b=a+1;
@@ -391,12 +472,14 @@ public class Sucker {
             a=htmlTagLowerCase.indexOf("alt=\"")+5;
             if(a>=5)
             {
-                b=a+1;
+                b=a;
+                if(htmlTagLowerCase.charAt(b)!='\"') {
                     while(htmlTagLowerCase.charAt(b)!='\"')
                         b++;
                     String altText=htmlTag.substring(a,b);
                     ret+=altText;
                 }
+            }
             
             ret+="[/img]";
             
@@ -405,14 +488,14 @@ public class Sucker {
             
             fetchAttachment(imageLink);
 
-            System.out.println("Converted to: "+ret);
+            debugLog("Converted to: "+ret);
             
             return ret;
             
         }
         if(importRefs && htmlTagLowerCase.startsWith("<a "))
         {
-            System.out.println("Found link tag: "+htmlTag);
+            debugLog("Found link tag: "+htmlTag);
             int a,b;
             
             a=htmlTagLowerCase.indexOf("href=\"")+6;
@@ -426,9 +509,12 @@ public class Sucker {
             String schema="web";
             
             ret += "[link schema=\""+schema+"\" location=\""+link+"\"]";
+            if(htmlTagLowerCase.endsWith("/>"))
+                ret += "[/link]";
+            else
                 pendingEndLink=true;
         
-            System.out.println("Converted to: "+ret);
+            debugLog("Converted to: "+ret);
 
             return ret;
         }
@@ -446,19 +532,36 @@ public class Sucker {
             return "[i]";
         if("</i>".equals(htmlTagLowerCase))
             return "[/i]";
+        if("<em>".equals(htmlTagLowerCase))
+            return "[i]";
+        if("</em>".equals(htmlTagLowerCase))
+            return "[/i]";
         
         return null;
     }
 
     private void fetchAttachment(String link) {
         
-        System.out.println("Fetch attachment from: "+link);
-        
-        String attachmentPath = messagePath+"."+attachmentCounter;
         link=link.replaceAll("&amp;","&");
         
+        debugLog("Fetch attachment from: "+link);
+        
+        File fetched;
+        if(pushToSyndie) {
+            try {
+                // perhaps specify a temp dir?
+                fetched = File.createTempFile("sucker",".attachment");
+                fetched.deleteOnExit();
+            } catch (IOException e) {
+                e.printStackTrace();
+                return;
+            }
+            fetched.deleteOnExit();
+        } else {
+            String attachmentPath = messagePath+"."+attachmentCounter;
+            fetched = new File(attachmentPath);
+        }
         int numRetries = 2;
-        File fetched = new File(attachmentPath);
         // we use eepGet, since it retries and doesn't leak DNS requests like URL does
         EepGet get = new EepGet(I2PAppContext.getGlobalContext(), shouldProxy, proxyHost, proxyPortNum, 
                                 numRetries, fetched.getAbsolutePath(), link);
@@ -471,9 +574,31 @@ public class Sucker {
             fetched.delete();
             return;
         }
+        String filename=EepGet.suggestName(link);
+        String contentType = get.getContentType();
+        if(contentType==null)
+            contentType="text/plain";
+        debugLog("successful fetch of filename "+filename);
+        if(fileNames==null) fileNames = new ArrayList();
+        if(fileTypes==null) fileTypes = new ArrayList();
+        if(fileStreams==null) fileStreams = new ArrayList();
+        fileNames.add(filename);
+        fileTypes.add(contentType);
+        try {
+            fileStreams.add(new FileInputStream(fetched));
+        } catch (FileNotFoundException e) {
+            e.printStackTrace();
+        }
         attachmentCounter++;
     }
 
+    private void debugLog(String string) {
+        if (_log.shouldLog(Log.DEBUG))
+            _log.debug(string);
+        if(!pushToSyndie)
+            System.out.println(string);
+    }
+
     private static int findTagLen(String s) {
         int i;
         for(i=0;i<s.length();i++)
@@ -487,7 +612,7 @@ public class Sucker {
                     i++;
             }   
         }
-        System.out.println("WTF");
+        System.out.println("WTF in Sucker.findTagLen("+s+")");
         return -1;
     }
 
@@ -513,25 +638,11 @@ public class Sucker {
                     return true;
             }
         } catch (FileNotFoundException e) {
-            // TODO Auto-generated catch block
             e.printStackTrace();
         }
         return false;
     }
 
-    private static void usage() {
-        System.out.println("sucker --load $urlToFeed \n"
-                + "--proxyhost <host> \n" 
-                + "--proxyport <port> \n"
-                + "--importenclosures true \n" 
-                + "--importrefs true \n"
-                + "--tag feed \n" 
-                + "--outputdir ./sucker_out \n"
-                + "--exec pushscript.sh OUTPUTDIR UNIQUEID ENTRYTIMESTAMP \n"
-                + "--history ./sucker.history");
-        System.exit(1);
-    }
-
     private static String sha1(String s) {
         try {
             MessageDigest md = MessageDigest.getInstance("SHA");
@@ -552,7 +663,6 @@ public class Sucker {
             try {
                 c = in.read();
             } catch (IOException e) {
-                // TODO Auto-generated catch block
                 e.printStackTrace();
                 break;
             }
@@ -562,7 +672,6 @@ public class Sucker {
                 break;
             sb.append((char) c);
         }
-        // TODO Auto-generated method stub
         return sb.toString();
     }
 }

apps/syndie/java/src/net/i2p/syndie/Updater.java

@@ -1,6 +1,8 @@
 package net.i2p.syndie;
 
 import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
 
 import net.i2p.I2PAppContext;
 import net.i2p.util.Log;
@@ -23,6 +25,12 @@ public class Updater {
         for (int i = 0; i < archives.length; i++) {
             fetchArchive(archives[i]);
         }
+        List rssFeeds = bm.getRssFeeds();
+        Iterator iter = rssFeeds.iterator();
+        while(iter.hasNext()) {
+            Sucker sucker = new Sucker((String[])iter.next());
+            sucker.suck();
+        }
     }
     
     public void fetchArchive(String archive) {

apps/syndie/jsp/_topnav.jsp

@@ -5,6 +5,7 @@
 <span class="b_topnavHome"><a href="index.jsp" class="b_topnavHome">Home</a></span>
 <a href="admin.jsp" class="b_topnavAdmin">Syndie admin</a>
 <a href="remote.jsp" class="b_topnavRemote">Remote archives</a>
+<a href="rssimport.jsp" class="b_topnavRSSImport">RSS imports</a>
 <a href="import.jsp" class="b_topnavImport">Import</a>
 </td><td nowrap="nowrap" height="10" class="b_topnavUser"><%
 if ("true".equals(request.getParameter("logout"))) {

apps/syndie/jsp/rssimport.jsp

@@ -0,0 +1,143 @@
+<%@page contentType="text/html; charset=UTF-8" pageEncoding="UTF-8" import="net.i2p.data.Base64, net.i2p.syndie.web.*, net.i2p.syndie.sml.*, net.i2p.syndie.data.*, net.i2p.syndie.*, org.mortbay.servlet.MultiPartRequest, java.util.*, java.io.*" %><%
+request.setCharacterEncoding("UTF-8");  %><jsp:useBean scope="session" class="net.i2p.syndie.User" id="user" 
+/><!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 TRANSITIONAL//EN" "http://www.w3c.org/TR/1999/REC-html401-19991224/loose.dtd">
+<html>
+<head>
+<title>SyndieMedia rss import configuration</title>
+<link href="style.jsp" rel="stylesheet" type="text/css" >
+</head>
+<body>
+<table border="1" cellpadding="0" cellspacing="0" width="100%">
+<tr class="b_toplogo"><td colspan="5" valign="top" align="left" class="b_toplogo"><jsp:include page="_toplogo.jsp" /></td></tr>
+<tr><td valign="top" align="left" rowspan="2" class="b_leftnav"><jsp:include page="_leftnav.jsp" /></td>
+    <jsp:include page="_topnav.jsp" />
+    <td valign="top" align="left" rowspan="2" class="b_rightnav"><jsp:include page="_rightnav.jsp" /></td></tr>
+<tr class="b_content"><td valign="top" align="left" colspan="3" class="b_content"><%
+
+BlogManager bm = BlogManager.instance();
+if (!user.getAuthenticated()) {
+    %><span class="b_rssMsgErr">Please log in.</span><% 
+}
+else if(!bm.authorizeRemote(user)) { 
+    %><span class="b_rssMsgErr">You are not authorized for remote access.</span><% 
+} else {
+    String url=request.getParameter("url");
+    if(url!=null) url=url.trim();
+    String blog=request.getParameter("blog");
+    if(blog!=null) blog=blog.trim();
+    String tagPrefix=request.getParameter("tagprefix");
+    if(tagPrefix!=null) tagPrefix=tagPrefix.trim();
+    String action = request.getParameter("action");
+    if ( (action != null) && ("Add".equals(action)) ) {
+        if(url==null || blog==null || tagPrefix==null) {
+            %><span class="b_rssImportMsgErr">Please fill in all fields</span><br /><%
+        } else {
+            boolean ret=bm.addRssFeed(url,blog,tagPrefix);
+	    if(!ret) {
+                %><span class="b_rssImportMsgErr">addRssFeed failure.</span><% 
+            } else {
+                %><span class="b_rssImportMsgOk">RSS feed added.</span><% 
+            }
+        }
+    } else if ( (action != null) && ("Change".equals(action)) ) {
+        String lastUrl=request.getParameter("lasturl");
+        String lastBlog=request.getParameter("lastblog");
+        String lastTagPrefix=request.getParameter("lasttagprefix");
+        if(url==null || blog==null || tagPrefix==null || lastUrl==null || lastBlog==null || lastTagPrefix==null) {
+            %><span class="b_rssImportMsgErr">error, some fields were empty.</span><br /><%
+        } else {
+            boolean ret=bm.deleteRssFeed(lastUrl,lastBlog,lastTagPrefix);
+            if(!ret) {
+                %><span class="b_rssImportMsgErr">Could not delete while attempting to change.</span><% 
+            } else {
+                ret=bm.addRssFeed(url,blog,tagPrefix);
+                if(!ret) {
+                    %><span class="b_rssImportMsgErr">Could not add while attempting to change.</span><% 
+                } else {
+                    %><span class="b_rssImportMsgOk">Ok, changed successfully.</span><% 
+                }
+            }
+        }       
+    } else if ( (action != null) && ("Delete".equals(action)) ) {
+        %><span class="b_rssImportMsgErr">Delete some thing</span><br /><%
+        if(url==null || blog==null || tagPrefix==null) {
+            %><span class="b_rssImportMsgErr">error, some fields were empty.</span><br /><%
+        } else {
+            boolean ret=bm.instance().deleteRssFeed(url,blog,tagPrefix);
+            if(!ret) {
+                %><span class="b_rssImportMsgErr">error, could not delete.</span><% 
+            } else {
+                %><span class="b_rssImportMsgOk">ok, deleted successfully.</span><% 
+            }
+        }       
+    }
+    String blogStr=user.getBlogStr();
+    if(blogStr==null)
+        blogStr="";
+    
+    ///////////////////////////////////////////////
+%>
+<p>Here you can add RSS feeds that will be periodically polled and added to your syndie. </p>
+<form action="rssimport.jsp" method="POST"> 
+RSS URL. (e.g. http://tracker.postman.i2p/rss.php)<br />
+<em><span class="b_rssImportField">url:</span></em> <input class="b_rssImportField" type="text" size="50" name="url" /><br />
+Blog hash to which the RSS entries will get posted, defaults to the one you're logged in to.<br />
+<em><span class="b_rssImportField">blog:</span></em> <input class="b_rssImportField" type="text" <%="value=\""+blogStr+"\""%> size="20" name="blog" /><br />
+This will be prepended to any tags that the RSS feed contains. (e.g. feed.tracker)<br />
+<em><span class="b_rssImportField">tagprefix:</span></em> <input class="b_rssImportField" type="text" value="feed" size="20" name="tagprefix" /><br />
+<input class="b_rssImportSubmit" type="submit" name="action" value="Add" /> <input class="b_rssImportCancel" type="reset" value="Cancel" /></form>
+<%
+///////////////////////////////////////////////
+    List feedList = bm.getRssFeeds();
+    if(feedList.size()>0) {
+%>
+<hr /><h3>Subscriptions:</h3><br />
+<table border="0" width="100%" class="b_rss">
+<tr class="b_rssHeader">
+<td class="b_rssHeader"><em class="b_rssHeader">Url</em></td>
+<td class="b_rssHeader"><em class="b_rssHeader">Blog</em></td>
+<td class="b_rssHeader"><em class="b_rssHeader">TagPrefix</em></td>
+<td class="b_rssHeader">&nbsp;</td></tr>
+<%
+        Iterator iter = feedList.iterator();
+        while(iter.hasNext()) {
+            String fields[]=(String[])iter.next();
+            url=fields[0];
+            blog=fields[1];
+            tagPrefix=fields[2];
+            StringBuffer buf = new StringBuffer(128);
+            
+            buf.append("<tr class=\"b_rssDetail\"><form action=\"rssimport.jsp\" method=\"POST\">");
+            buf.append("<input type=\"hidden\" name=\"lasturl\" value=\"").append(url).append("\" />");
+            buf.append("<input type=\"hidden\" name=\"lastblog\" value=\"").append(blog).append("\" />");
+            buf.append("<input type=\"hidden\" name=\"lasttagprefix\" value=\"").append(tagPrefix).append("\" />");
+            
+            buf.append("<td class=\"b_rssUrl\"><input class=\"b_rssUrl\" type=\"text\" size=\"50\" name=\"url\" value=\"").append(url).append("\" /></td>");
+            buf.append("<td class=\"b_rssBlog\"><input class=\"b_rssBlog\" type=\"text\" size=\"20\" name=\"blog\" value=\"").append(blog).append("\" /></td>");
+            buf.append("<td class=\"b_rssPrefix\"><input class=\"b_rssPrefix\" type=\"text\" size=\"20\" name=\"tagprefix\" value=\"").append(tagPrefix).append("\" /></td>");
+            buf.append("<td class=\"b_rssDetail\" nowrap=\"nowrap\">");
+            
+            buf.append("<input class=\"b_rssChange\" type=\"submit\" name=\"action\" value=\"Change\" />");
+            buf.append("<input class=\"b_rssDelete\" type=\"submit\" name=\"action\" value=\"Delete\" />");
+            
+            buf.append("</td></form></tr>");
+            out.write(buf.toString());
+            buf.setLength(0);
+        }
+    }
+/*
+<p><h3>todo:</h3>
+<p>caching (eepget should do it)
+<p>enclosures support (requires cvs rome)
+<p>syndie.sucker.minHistory/maxHistory used to roll over the history file?
+<p>configurable update period
+*/
+%>
+</table>
+<hr />
+<% 
+} 
+%>
+</td></tr>
+</table>
+</body>

core/java/src/net/i2p/util/EepGet.java

@@ -51,6 +51,7 @@ public class EepGet {
     private String _etag;
     private boolean _encodingChunked;
     private boolean _notModified;
+    private String _contentType;
     
     public EepGet(I2PAppContext ctx, String proxyHost, int proxyPort, int numRetries, String outputFile, String url) {
         this(ctx, true, proxyHost, proxyPort, numRetries, outputFile, url);
@@ -209,11 +210,13 @@ public class EepGet {
                         if (timeToSend > 0) {
                             StringBuffer buf = new StringBuffer(50);
                             buf.append(" ");
+                            if ( bytesRemaining > 0 ) {
                                 double pct = ((double)alreadyTransferred + (double)_written) / ((double)alreadyTransferred + (double)bytesRemaining);
                                 synchronized (_pct) {
                                     buf.append(_pct.format(pct));
                                 }
                                 buf.append(": ");
+                            }
                             buf.append(_written+alreadyTransferred);
                             buf.append(" @ ");
                             double lineKBytes = ((double)_markSize * (double)_lineSize)/1024.0d;
@@ -243,9 +246,15 @@ public class EepGet {
             if (notModified) {
                 System.out.println("== Source not modified since last download");
             } else {
+                if ( bytesRemaining > 0 ) {
                     System.out.println("== Transfer of " + url + " completed with " + (alreadyTransferred+bytesTransferred)
                             + " and " + (bytesRemaining - bytesTransferred) + " remaining");
                     System.out.println("== Output saved to " + outputFile);
+                } else {
+                    System.out.println("== Transfer of " + url + " completed with " + (alreadyTransferred+bytesTransferred)
+                            + " bytes transferred");
+                    System.out.println("== Output saved to " + outputFile);
+                }
             }
             long timeToSend = _context.clock().now() - _startedOn;
             System.out.println("== Transfer time: " + DataHelper.formatDuration(timeToSend));
@@ -355,9 +364,19 @@ public class EepGet {
             _out.write(buf, 0, read);
             _bytesTransferred += read;
             remaining -= read;
+            if (remaining==0 && _encodingChunked) {
+                if(_proxyIn.read()=='\r' && _proxyIn.read()=='\n') {
+                    remaining = (int) readChunkLength();
+                }
+            }
             if (read > 0) 
                 for (int i = 0; i < _listeners.size(); i++) 
-                    ((StatusListener)_listeners.get(i)).bytesTransferred(_alreadyTransferred, read, _bytesTransferred, _bytesRemaining, _url);
+                    ((StatusListener)_listeners.get(i)).bytesTransferred(
+                            _alreadyTransferred, 
+                            read, 
+                            _bytesTransferred, 
+                            _encodingChunked?-1:_bytesRemaining, 
+                            _url);
         }
 
         if (_out != null)
@@ -369,7 +388,13 @@ public class EepGet {
 
         if ( (_bytesRemaining == -1) || (remaining == 0) ){
             for (int i = 0; i < _listeners.size(); i++) 
-                ((StatusListener)_listeners.get(i)).transferComplete(_alreadyTransferred, _bytesTransferred, _bytesRemaining, _url, _outputFile, _notModified);
+                ((StatusListener)_listeners.get(i)).transferComplete(
+                        _alreadyTransferred, 
+                        _bytesTransferred, 
+                        _encodingChunked?-1:_bytesRemaining, 
+                        _url, 
+                        _outputFile, 
+                        _notModified);
         } else {
             throw new IOException("Disconnection on attempt " + _currentAttempt + " after " + _bytesTransferred);
         }
@@ -387,6 +412,7 @@ public class EepGet {
         switch (responseCode) {
             case 200: // full
                 _out = new FileOutputStream(_outputFile, false);
+                _alreadyTransferred = 0;
                 rcOk = true;
                 break;
             case 206: // partial
@@ -405,7 +431,7 @@ public class EepGet {
             default:
                 rcOk = false;
         }
-
+        buf.setLength(0);
         byte lookahead[] = new byte[3];
         while (true) {
             int cur = _proxyIn.read();
@@ -435,7 +461,7 @@ public class EepGet {
                         if (!rcOk)
                             throw new IOException("Invalid HTTP response code: " + responseCode);
                         if (_encodingChunked) {
-                            readChunkLength();
+                            _bytesRemaining = readChunkLength();
                         }
                         return;
                     }
@@ -450,7 +476,7 @@ public class EepGet {
         }
     }
     
-    private void readChunkLength() throws IOException {
+    private long readChunkLength() throws IOException {
         StringBuffer buf = new StringBuffer(8);
         int nl = 0;
         while (true) {
@@ -472,9 +498,9 @@ public class EepGet {
         String len = buf.toString().trim();
         try {
             long bytes = Long.parseLong(len, 16);
-            _bytesRemaining = bytes;
             if (_log.shouldLog(Log.DEBUG))
                 _log.debug("Chunked length: " + bytes);
+            return bytes;
         } catch (NumberFormatException nfe) {
             throw new IOException("Invalid chunk length [" + len + "]");
         }
@@ -529,6 +555,8 @@ public class EepGet {
         } else if (key.equalsIgnoreCase("Transfer-encoding")) {
             if (val.indexOf("chunked") != -1)
                 _encodingChunked = true;
+        } else if (key.equalsIgnoreCase("Content-Type")) {
+            _contentType=val;
         } else {
             // ignore the rest
         }
@@ -631,4 +659,8 @@ public class EepGet {
         return _notModified;
     }
     
+    public String getContentType() {
+        return _contentType;
+    }
+
 }

history.txt

@@ -1,4 +1,10 @@
-$Id: history.txt,v 1.308 2005/10/29 16:35:27 jrandom Exp $
+$Id: history.txt,v 1.309 2005/10/29 18:20:04 jrandom Exp $
+
+2005-10-30  dust
+    * Merge sucker into syndie with a rssimport.jsp page.
+    * Add getContentType() to EepGet.
+    * Make chunked transfer work (better) with EepGet.
+    * Do replaceAll("<","&lt;") for logs.
 
 * 2005-10-29  0.6.1.4 released
 

router/java/src/net/i2p/router/RouterVersion.java

@@ -15,9 +15,9 @@ import net.i2p.CoreVersion;
  *
  */
 public class RouterVersion {
-    public final static String ID = "$Revision: 1.279 $ $Date: 2005/10/29 16:35:24 $";
+    public final static String ID = "$Revision: 1.280 $ $Date: 2005/10/29 18:20:05 $";
     public final static String VERSION = "0.6.1.4";
-    public final static long BUILD = 0;
+    public final static long BUILD = 1;
     public static void main(String args[]) {
         System.out.println("I2P Router version: " + VERSION + "-" + BUILD);
         System.out.println("Router ID: " + RouterVersion.ID);