I2P_Developers/i2p.itoopie
3
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

2005-10-30 dust

Browse Source 
* Merge sucker into syndie with a rssimport.jsp page.
    * Add getContentType() to EepGet.
    * Make chunked transfer work (better) with EepGet.
    * Do replaceAll("<","&lt;") for logs.
This commit is contained in:
dust
2005-10-30 05:47:55 +00:00
committed by zzz
parent f7c2ae9a3b
commit 10dde610dc
9 changed files with 554 additions and 175 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
apps/routerconsole/java/src/net/i2p/router/web/LogsHelper.java
View File

@ -30,6 +30,7 @@ public class LogsHelper {
buf.append("<code>\n");
for (int i = msgs.size(); i > 0; i--) {
String msg = (String)msgs.get(i - 1);
msg = msg.replaceAll("<","&lt;");
buf.append("<li>");
buf.append(msg);
buf.append("</li>\n");
@ -46,6 +47,7 @@ public class LogsHelper {
buf.append("<code>\n");
for (int i = msgs.size(); i > 0; i--) {
String msg = (String)msgs.get(i - 1);
msg = msg.replaceAll("<","&lt;");
buf.append("<li>");
buf.append(msg);
buf.append("</li>\n");
@ -59,8 +61,10 @@ public class LogsHelper {
String str = FileUtil.readTextFile("wrapper.log", 500, false);
if (str == null)
return "";
else
else {
str = str.replaceAll("<","&lt;");
return "<pre>" + str + "</pre>";
}
}
public String getConnectionLogs() {

86
apps/syndie/java/src/net/i2p/syndie/BlogManager.java
View File

@ -324,8 +324,10 @@ public class BlogManager {
}
public boolean isConfigured() {
File cfg = getConfigFile();
return (cfg.exists());
String p = _context.getProperty("syndie.configurationCheck");
if(p==null)
return false;
return true;
}
/**
@ -348,6 +350,73 @@ public class BlogManager {
return delay;
}
public List getRssFeeds() {
List feedList = new ArrayList();
int i=0;
while(true) {
String url = _context.getProperty("syndie.rssFeed."+i+".url");
String blog = _context.getProperty("syndie.rssFeed."+i+".blog");
String tagPrefix = _context.getProperty("syndie.rssFeed."+i+".tagPrefix");
if(url==null || blog==null || tagPrefix==null)
break;
String feed[] = new String[3];
feed[0]=url.trim();
feed[1]=blog.trim();
feed[2]=tagPrefix.trim();
feedList.add(feed);
i++;
}
return feedList;
}
public boolean addRssFeed(String url, String blog, String tagPrefix) {
List feedList = getRssFeeds();
int nextIdx=feedList.size();
String baseFeedProp="syndie.rssFeed."+nextIdx;
System.setProperty(baseFeedProp+".url",url);
System.setProperty(baseFeedProp+".blog",blog);
System.setProperty(baseFeedProp+".tagPrefix",tagPrefix);
_log.info("addRssFeed("+nextIdx+"): "+url);
writeConfig();
Updater.wakeup();
return true;
}
public boolean deleteRssFeed(String url, String blog, String tagPrefix) {
List feedList = getRssFeeds();
Iterator iter = feedList.iterator();
int idx=0;
while(iter.hasNext()) {
String fields[] = (String[])iter.next();
if(fields[0].equals(url) &&
fields[1].equals(blog) &&
fields[2].equals(tagPrefix)) {
break;
}
idx++;
}
// copy any remaining to idx-1
while(iter.hasNext()) {
String fields[] = (String[])iter.next();
String baseFeedProp="syndie.rssFeed."+idx;
System.setProperty(baseFeedProp+".url",fields[0]);
System.setProperty(baseFeedProp+".blog",fields[1]);
System.setProperty(baseFeedProp+".tagPrefix",fields[2]);
idx++;
}
// Delete last idx from properties
String baseFeedProp="syndie.rssFeed."+idx;
System.getProperties().remove(baseFeedProp+".url");
System.getProperties().remove(baseFeedProp+".blog");
System.getProperties().remove(baseFeedProp+".tagPrefix");
_log.info("deleteRssFeed("+idx+"): "+url);
writeConfig();
return true;
}
public boolean authorizeAdmin(String pass) {
if (isSingleUser()) return true;
String admin = getAdminPasswordHash();
@ -388,6 +457,7 @@ public class BlogManager {
if (defaultProxyPort > 0)
out.write("syndie.defaultProxyPort="+defaultProxyPort + "\n");
out.write("syndie.singleUser=" + isSingleUser + "\n");
out.write("syndie.configurationCheck=foo\n");
if (opts != null) {
for (Iterator iter = opts.keySet().iterator(); iter.hasNext(); ) {
String key = (String)iter.next();
@ -525,10 +595,13 @@ public class BlogManager {
}
public BlogURI createBlogEntry(User user, String subject, String tags, String entryHeaders, String sml) {
return createBlogEntry(user, subject, tags, entryHeaders, sml, null, null, null);
return createBlogEntry(user, true, subject, tags, entryHeaders, sml, null, null, null);
}
public BlogURI createBlogEntry(User user, String subject, String tags, String entryHeaders, String sml, List fileNames, List fileStreams, List fileTypes) {
if (!user.getAuthenticated()) return null;
return createBlogEntry(user, true, subject, tags, entryHeaders, sml, fileNames, fileStreams, fileTypes);
}
public BlogURI createBlogEntry(User user, boolean shouldAuthenticate, String subject, String tags, String entryHeaders, String sml, List fileNames, List fileStreams, List fileTypes) {
if (shouldAuthenticate && !user.getAuthenticated()) return null;
BlogInfo info = getArchive().getBlogInfo(user.getBlog());
if (info == null) return null;
SigningPrivateKey privkey = getMyPrivateKey(info);
@ -601,7 +674,10 @@ public class BlogManager {
if (ok) {
getArchive().regenerateIndex();
user.setMostRecentEntry(entryId);
saveUser(user);
if(shouldAuthenticate)
saveUser(user);
else
storeUser(user);
return uri;
} else {
return null;

415
apps/syndie/java/src/net/i2p/syndie/Sucker.java
View File

@ -6,16 +6,14 @@ import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.MalformedURLException;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.Date;
import java.util.Iterator;
import java.util.List;
import java.util.ListIterator;
//import sun.security.provider.SHA;
import java.util.Properties;
import com.sun.syndication.feed.synd.SyndCategory;
import com.sun.syndication.feed.synd.SyndContent;
@ -27,9 +25,15 @@ import com.sun.syndication.io.XmlReader;
import net.i2p.I2PAppContext;
import net.i2p.data.Base64;
import net.i2p.data.DataFormatException;
import net.i2p.data.DataHelper;
import net.i2p.data.Hash;
import net.i2p.syndie.data.BlogURI;
import net.i2p.util.EepGet;
import net.i2p.util.Log;
public class Sucker {
private static final Log _log = I2PAppContext.getGlobalContext().logManager().getLog(Sucker.class);
private String urlToLoad;
private String outputDir="./sucker_out";
private String historyPath="./sucker.history";
@ -46,9 +50,50 @@ public class Sucker {
private boolean pendingEndLink;
private boolean shouldProxy;
private int proxyPortNum;
private String blog;
private boolean pushToSyndie;
private long messageNumber=0;
private BlogManager bm;
private User user;
public Sucker() {}
//
private List fileNames;
private List fileStreams;
private List fileTypes;
public Sucker() {
}
/**
* Constructor for BlogManager.
*/
public Sucker(String[] strings) throws IllegalArgumentException {
pushToSyndie=true;
urlToLoad = strings[0];
blog = strings[1];
feedTag = strings[2];
outputDir = "blog-"+blog;
try {
historyPath=BlogManager.instance().getRootDir().getCanonicalPath()+"/rss.history";
} catch (IOException e) {
e.printStackTrace();
}
proxyPort="4444";
proxyHost="localhost";
bm = BlogManager.instance();
Hash blogHash = new Hash();
try {
blogHash.fromBase64(blog);
} catch (DataFormatException e1) {
throw new IllegalArgumentException("ooh, bad $blog");
}
user = bm.getUser(blogHash);
if(user==null)
throw new IllegalArgumentException("wtf, user==null? hash:"+blogHash);
}
public boolean parseArgs(String args[]) {
for (int i = 0; i < args.length; i++) {
if ("--load".equals(args[i]))
@ -77,58 +122,57 @@ public class Sucker {
if (urlToLoad == null)
return false;
// Find base url, gah HELP
int idx=urlToLoad.length();
int x=urlToLoad.indexOf('?');
if(x>0)
idx=x;
while(idx>0)
{
idx--;
if(urlToLoad.charAt(idx)=='/')
break;
}
if(idx==0)
idx=x;
baseUrl=urlToLoad.substring(0,idx);
System.out.println("Processing: "+urlToLoad);
return true;
}
/**
* Fetch urlToLoad and call convertToHtml() on any new entries.
*/
public void suck() {
SyndFeed feed;
// Find base url
int idx=urlToLoad.lastIndexOf('/');
if(idx>0)
baseUrl=urlToLoad.substring(0,idx);
else
baseUrl=urlToLoad;
debugLog("Processing: "+urlToLoad);
debugLog("Base url: "+baseUrl);
//
try {
// Create outputDir if missing
File f = new File(outputDir);
f.mkdirs();
File lastIdFile=null;
// Get next message number to use (for messageId in history only)
if(!pushToSyndie) {
lastIdFile = new File(historyPath + ".lastId");
if (!lastIdFile.exists())
lastIdFile.createNewFile();
FileInputStream fis = new FileInputStream(lastIdFile);
String number = readLine(fis);
try {
messageNumber = Integer.parseInt(number);
} catch (NumberFormatException e) {
messageNumber = 0;
}
// Create outputDir if missing
File f = new File(outputDir);
f.mkdirs();
} else {
messageNumber=bm.getNextBlogEntry(user);
}
// Create historyFile if missing
historyFile = new File(historyPath);
if (!historyFile.exists())
historyFile.createNewFile();
int messageNumber;
File lastIdFile = new File(historyPath + ".lastId");
if (!lastIdFile.exists())
lastIdFile.createNewFile();
FileInputStream fis = new FileInputStream(lastIdFile);
String number = readLine(fis);
try {
messageNumber = Integer.parseInt(number);
} catch (NumberFormatException e) {
messageNumber = 0;
}
SyndFeedInput input = new SyndFeedInput();
shouldProxy = false;
proxyPortNum = -1;
if ( (proxyHost != null) && (proxyPort != null) ) {
@ -140,11 +184,11 @@ public class Sucker {
nfe.printStackTrace();
}
}
// fetch
int numRetries = 2;
// perhaps specify a temp dir?
File fetched = File.createTempFile("sucker", ".fetch");
fetched.deleteOnExit();
// we use eepGet, since it retries and doesn't leak DNS requests like URL does
EepGet get = new EepGet(I2PAppContext.getGlobalContext(), shouldProxy, proxyHost, proxyPortNum,
numRetries, fetched.getAbsolutePath(), urlToLoad);
SuckerFetchListener lsnr = new SuckerFetchListener();
@ -155,89 +199,71 @@ public class Sucker {
System.err.println("Unable to retrieve the url after " + numRetries + " tries.");
return;
}
if(get.getNotModified()) {
debugLog("not modified, saving network bytes from useless fetch");
return;
}
// Build entry list from fetched rss file
SyndFeedInput input = new SyndFeedInput();
feed = input.build(new XmlReader(fetched));
List entries = feed.getEntries();
FileOutputStream hos = new FileOutputStream(historyFile, true);
ListIterator iter = entries.listIterator();
while (iter.hasNext()) {
// Process list backwards to get syndie to display the
// entries in the right order. (most recent at top)
for (int i = entries.size()-1; i >= 0; i--) {
SyndEntry e = (SyndEntry) entries.get(i);
attachmentCounter=0;
SyndEntry e = (SyndEntry) iter.next();
// Calculate messageId
String feedHash = sha1(urlToLoad);
String itemHash = sha1(e.getTitle() + e.getDescription());
Date d = e.getPublishedDate();
String time;
if(d!=null)
time = "" + d.getTime();
else
time = "" + new Date().getTime();
String outputFileName = outputDir + "/" + messageNumber;
/*
* $feedHash:$itemHash:$time:$outputfile. $feedHash would be the
* hash (md5? sha1? sha2?) of the $urlToFeed, $itemHash is some
* hash of the SyndEntry, $time would be the time that the entry
* was posted $outputfile would be the $outputdir/$uniqueid
*/
String messageId = feedHash + ":" + itemHash + ":" + time + ":"
+ outputFileName;
// Check if we already have this
if (!existsInHistory(messageId)) {
System.out.println("new: " + messageId);
if (convertToSml(e, ""+messageNumber)) {
if (pushScript != null) {
if (!execPushScript(""+messageNumber, time)) {
System.out.println("################## push failed");
}else {
System.out.println("push success");
hos.write(messageId.getBytes());
hos.write("\n".getBytes());
}
}
}
messageNumber++;
String messageId = convertToSml(e);
if (messageId!=null) {
hos.write(messageId.getBytes());
hos.write("\n".getBytes());
}
}
FileOutputStream fos = new FileOutputStream(lastIdFile);
fos.write(("" + messageNumber).getBytes());
if(!pushToSyndie) {
FileOutputStream fos = new FileOutputStream(lastIdFile);
fos.write(("" + messageNumber).getBytes());
}
} catch (MalformedURLException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IllegalArgumentException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (FeedException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
System.out.println("Done.");
debugLog("Done.");
}
public static void main(String[] args) {
Sucker sucker = new Sucker();
boolean ok = sucker.parseArgs(args);
if (!ok) {
usage();
return;
System.out.println("sucker --load $urlToFeed \n"
+ "--proxyhost <host> \n"
+ "--proxyport <port> \n"
+ "--importenclosures true \n"
+ "--importrefs true \n"
+ "--tag feed \n"
+ "--outputdir ./sucker_out \n"
+ "--exec pushscript.sh OUTPUTDIR UNIQUEID ENTRYTIMESTAMP \n"
+ "--history ./sucker.history");
System.exit(1);
}
sucker.suck();
}
/**
* Call the specified script with "$outputDir $id and $time".
*/
private boolean execPushScript(String id, String time) {
try {
String ls_str;
@ -251,7 +277,7 @@ public class Sucker {
try {
while ((ls_str = ls_in.readLine()) != null) {
System.out.println(pushScript + ": " + ls_str);
debugLog(pushScript + ": " + ls_str);
}
} catch (IOException e) {
return false;
@ -261,7 +287,6 @@ public class Sucker {
if(pushScript_proc.exitValue()==0)
return true;
} catch (InterruptedException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
return false;
@ -271,40 +296,57 @@ public class Sucker {
}
}
private boolean convertToSml(SyndEntry e, String messageName) {
// Create message
FileOutputStream fos;
messagePath=outputDir+"/"+messageName;
/**
* Converts the SyndEntry e to sml and fetches any images as attachments
*/
private String convertToSml(SyndEntry e) {
String subject;
// Calculate messageId, and check if we have got the message already
String feedHash = sha1(urlToLoad);
String itemHash = sha1(e.getTitle() + e.getDescription());
Date d = e.getPublishedDate();
String time;
if(d!=null)
time = "" + d.getTime();
else
time = "" + new Date().getTime();
String outputFileName = outputDir + "/" + messageNumber;
String messageId = feedHash + ":" + itemHash + ":" + time + ":" + outputFileName;
// Check if we already have this
if (existsInHistory(messageId))
return null;
debugLog("new: " + messageId);
try {
fos = new FileOutputStream(messagePath);
String sml;
sml = "Subject: " + e.getTitle() + "\n";
String sml="";
subject=e.getTitle();
List cats = e.getCategories();
Iterator iter = cats.iterator();
String tags = feedTag;
while (iter.hasNext()) {
SyndCategory c = (SyndCategory) iter.next();
debugLog("Name: "+c.getName());
debugLog("uri:"+c.getTaxonomyUri());
String tag=c.getName();
tag=tag.replaceAll("[^a-zA-z.-_:]","_");
tags += "\t" + feedTag + "." + tag;
}
sml += "Tags: " + tags + "\n";
sml += "\n";
SyndContent content;
List l = e.getContents();
if(l!=null)
{
System.out.println("There is content");
debugLog("There is content");
iter = l.iterator();
while(iter.hasNext())
{
content = (SyndContent)iter.next();
String c = content.getValue();
System.out.println("Content: "+c);
debugLog("Content: "+c);
sml += htmlToSml(c);
sml += "\n";
}
@ -314,17 +356,53 @@ public class Sucker {
source=baseUrl+source;
sml += "[link schema=\"web\" location=\""+source+"\"]source[/link]\n";
fos.write(sml.getBytes());
if(pushToSyndie) {
debugLog("user.blog: "+user.getBlogStr());
debugLog("user.id: "+bm.getNextBlogEntry(user));
debugLog("subject: "+subject);
debugLog("tags: "+tags);
debugLog("sml: "+sml);
debugLog("");
BlogURI uri = bm.createBlogEntry(
user,
false,
subject,
tags,
null,
sml,
fileNames,
fileStreams,
fileTypes);
return true;
if(uri==null) {
debugLog("pushToSyndie failure.");
return null;
}
else
debugLog("pushToSyndie success, uri: "+uri.toString());
}
else
{
FileOutputStream fos;
fos = new FileOutputStream(messagePath);
sml=subject + "\nTags: " + tags + "\n\n" + sml;
fos.write(sml.getBytes());
if (pushScript != null) {
if (!execPushScript(""+messageNumber, time)) {
debugLog("################## push failed");
} else {
debugLog("push success");
}
}
}
messageNumber++;
return messageId;
} catch (FileNotFoundException e1) {
// TODO Auto-generated catch block
e1.printStackTrace();
} catch (IOException e2) {
// TODO Auto-generated catch block
e2.printStackTrace();
}
return false;
return null;
}
private String htmlToSml(String html) {
@ -338,29 +416,32 @@ public class Sucker {
{
if(html.charAt(i)=='<')
{
//System.out.println("html: "+html.substring(i));
//log("html: "+html.substring(i));
int tagLen = findTagLen(html.substring(i));
if(tagLen<=0)
{
System.out.println("Bad html? ("+html+")");
debugLog("Bad html? ("+html+")");
break;
}
//
String htmlTag = html.substring(i,i+tagLen);
//System.out.println("htmlTag: "+htmlTag);
//log("htmlTag: "+htmlTag);
String smlTag = htmlTagToSmlTag(htmlTag);
if(smlTag!=null)
sml+=smlTag;
i+=tagLen;
//System.out.println("tagLen: "+tagLen);
//log("tagLen: "+tagLen);
sml+=" ";
}
else
{
sml+=html.charAt(i++);
char c=html.charAt(i++);
sml+=c;
if(c=='[' || c==']')
sml+=c;
}
}
@ -373,7 +454,7 @@ public class Sucker {
if(importEnclosures && htmlTagLowerCase.startsWith("<img"))
{
System.out.println("Found image tag: "+htmlTag);
debugLog("Found image tag: "+htmlTag);
int a,b;
a=htmlTagLowerCase.indexOf("src=\"")+5;
b=a+1;
@ -383,7 +464,7 @@ public class Sucker {
if(pendingEndLink) {
ret="[/link]";
pendingEndLink=false;
pendingEndLink=false;
}
ret += "[img attachment=\""+""+ attachmentCounter +"\"]";
@ -391,11 +472,13 @@ public class Sucker {
a=htmlTagLowerCase.indexOf("alt=\"")+5;
if(a>=5)
{
b=a+1;
while(htmlTagLowerCase.charAt(b)!='\"')
b++;
String altText=htmlTag.substring(a,b);
ret+=altText;
b=a;
if(htmlTagLowerCase.charAt(b)!='\"') {
while(htmlTagLowerCase.charAt(b)!='\"')
b++;
String altText=htmlTag.substring(a,b);
ret+=altText;
}
}
ret+="[/img]";
@ -405,14 +488,14 @@ public class Sucker {
fetchAttachment(imageLink);
System.out.println("Converted to: "+ret);
debugLog("Converted to: "+ret);
return ret;
}
if(importRefs && htmlTagLowerCase.startsWith("<a "))
{
System.out.println("Found link tag: "+htmlTag);
debugLog("Found link tag: "+htmlTag);
int a,b;
a=htmlTagLowerCase.indexOf("href=\"")+6;
@ -426,9 +509,12 @@ public class Sucker {
String schema="web";
ret += "[link schema=\""+schema+"\" location=\""+link+"\"]";
pendingEndLink=true;
if(htmlTagLowerCase.endsWith("/>"))
ret += "[/link]";
else
pendingEndLink=true;
System.out.println("Converted to: "+ret);
debugLog("Converted to: "+ret);
return ret;
}
@ -446,19 +532,36 @@ public class Sucker {
return "[i]";
if("</i>".equals(htmlTagLowerCase))
return "[/i]";
if("<em>".equals(htmlTagLowerCase))
return "[i]";
if("</em>".equals(htmlTagLowerCase))
return "[/i]";
return null;
}
private void fetchAttachment(String link) {
System.out.println("Fetch attachment from: "+link);
link=link.replaceAll("&amp;","&");
String attachmentPath = messagePath+"."+attachmentCounter;
link=link.replaceAll("&amp;","&");
debugLog("Fetch attachment from: "+link);
File fetched;
if(pushToSyndie) {
try {
// perhaps specify a temp dir?
fetched = File.createTempFile("sucker",".attachment");
fetched.deleteOnExit();
} catch (IOException e) {
e.printStackTrace();
return;
}
fetched.deleteOnExit();
} else {
String attachmentPath = messagePath+"."+attachmentCounter;
fetched = new File(attachmentPath);
}
int numRetries = 2;
File fetched = new File(attachmentPath);
// we use eepGet, since it retries and doesn't leak DNS requests like URL does
EepGet get = new EepGet(I2PAppContext.getGlobalContext(), shouldProxy, proxyHost, proxyPortNum,
numRetries, fetched.getAbsolutePath(), link);
@ -471,9 +574,31 @@ public class Sucker {
fetched.delete();
return;
}
String filename=EepGet.suggestName(link);
String contentType = get.getContentType();
if(contentType==null)
contentType="text/plain";
debugLog("successful fetch of filename "+filename);
if(fileNames==null) fileNames = new ArrayList();
if(fileTypes==null) fileTypes = new ArrayList();
if(fileStreams==null) fileStreams = new ArrayList();
fileNames.add(filename);
fileTypes.add(contentType);
try {
fileStreams.add(new FileInputStream(fetched));
} catch (FileNotFoundException e) {
e.printStackTrace();
}
attachmentCounter++;
}
private void debugLog(String string) {
if (_log.shouldLog(Log.DEBUG))
_log.debug(string);
if(!pushToSyndie)
System.out.println(string);
}
private static int findTagLen(String s) {
int i;
for(i=0;i<s.length();i++)
@ -487,7 +612,7 @@ public class Sucker {
i++;
}
}
System.out.println("WTF");
System.out.println("WTF in Sucker.findTagLen("+s+")");
return -1;
}
@ -513,25 +638,11 @@ public class Sucker {
return true;
}
} catch (FileNotFoundException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
return false;
}
private static void usage() {
System.out.println("sucker --load $urlToFeed \n"
+ "--proxyhost <host> \n"
+ "--proxyport <port> \n"
+ "--importenclosures true \n"
+ "--importrefs true \n"
+ "--tag feed \n"
+ "--outputdir ./sucker_out \n"
+ "--exec pushscript.sh OUTPUTDIR UNIQUEID ENTRYTIMESTAMP \n"
+ "--history ./sucker.history");
System.exit(1);
}
private static String sha1(String s) {
try {
MessageDigest md = MessageDigest.getInstance("SHA");
@ -552,7 +663,6 @@ public class Sucker {
try {
c = in.read();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
break;
}
@ -562,7 +672,6 @@ public class Sucker {
break;
sb.append((char) c);
}
// TODO Auto-generated method stub
return sb.toString();
}
}

8
apps/syndie/java/src/net/i2p/syndie/Updater.java
View File

@ -1,6 +1,8 @@
package net.i2p.syndie;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import net.i2p.I2PAppContext;
import net.i2p.util.Log;
@ -23,6 +25,12 @@ public class Updater {
for (int i = 0; i < archives.length; i++) {
fetchArchive(archives[i]);
}
List rssFeeds = bm.getRssFeeds();
Iterator iter = rssFeeds.iterator();
while(iter.hasNext()) {
Sucker sucker = new Sucker((String[])iter.next());
sucker.suck();
}
}
public void fetchArchive(String archive) {

1
apps/syndie/jsp/_topnav.jsp
View File

@ -5,6 +5,7 @@
<span class="b_topnavHome"><a href="index.jsp" class="b_topnavHome">Home</a></span>
<a href="admin.jsp" class="b_topnavAdmin">Syndie admin</a>
<a href="remote.jsp" class="b_topnavRemote">Remote archives</a>
<a href="rssimport.jsp" class="b_topnavRSSImport">RSS imports</a>
<a href="import.jsp" class="b_topnavImport">Import</a>
</td><td nowrap="nowrap" height="10" class="b_topnavUser"><%
if ("true".equals(request.getParameter("logout"))) {

143
apps/syndie/jsp/rssimport.jsp Normal file
View File

@ -0,0 +1,143 @@
<%@page contentType="text/html; charset=UTF-8" pageEncoding="UTF-8" import="net.i2p.data.Base64, net.i2p.syndie.web.*, net.i2p.syndie.sml.*, net.i2p.syndie.data.*, net.i2p.syndie.*, org.mortbay.servlet.MultiPartRequest, java.util.*, java.io.*" %><%
request.setCharacterEncoding("UTF-8"); %><jsp:useBean scope="session" class="net.i2p.syndie.User" id="user"
/><!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 TRANSITIONAL//EN" "http://www.w3c.org/TR/1999/REC-html401-19991224/loose.dtd">
<html>
<head>
<title>SyndieMedia rss import configuration</title>
<link href="style.jsp" rel="stylesheet" type="text/css" >
</head>
<body>
<table border="1" cellpadding="0" cellspacing="0" width="100%">
<tr class="b_toplogo"><td colspan="5" valign="top" align="left" class="b_toplogo"><jsp:include page="_toplogo.jsp" /></td></tr>
<tr><td valign="top" align="left" rowspan="2" class="b_leftnav"><jsp:include page="_leftnav.jsp" /></td>
<jsp:include page="_topnav.jsp" />
<td valign="top" align="left" rowspan="2" class="b_rightnav"><jsp:include page="_rightnav.jsp" /></td></tr>
<tr class="b_content"><td valign="top" align="left" colspan="3" class="b_content"><%
BlogManager bm = BlogManager.instance();
if (!user.getAuthenticated()) {
%><span class="b_rssMsgErr">Please log in.</span><%
}
else if(!bm.authorizeRemote(user)) {
%><span class="b_rssMsgErr">You are not authorized for remote access.</span><%
} else {
String url=request.getParameter("url");
if(url!=null) url=url.trim();
String blog=request.getParameter("blog");
if(blog!=null) blog=blog.trim();
String tagPrefix=request.getParameter("tagprefix");
if(tagPrefix!=null) tagPrefix=tagPrefix.trim();
String action = request.getParameter("action");
if ( (action != null) && ("Add".equals(action)) ) {
if(url==null || blog==null || tagPrefix==null) {
%><span class="b_rssImportMsgErr">Please fill in all fields</span><br /><%
} else {
boolean ret=bm.addRssFeed(url,blog,tagPrefix);
if(!ret) {
%><span class="b_rssImportMsgErr">addRssFeed failure.</span><%
} else {
%><span class="b_rssImportMsgOk">RSS feed added.</span><%
}
}
} else if ( (action != null) && ("Change".equals(action)) ) {
String lastUrl=request.getParameter("lasturl");
String lastBlog=request.getParameter("lastblog");
String lastTagPrefix=request.getParameter("lasttagprefix");
if(url==null || blog==null || tagPrefix==null || lastUrl==null || lastBlog==null || lastTagPrefix==null) {
%><span class="b_rssImportMsgErr">error, some fields were empty.</span><br /><%
} else {
boolean ret=bm.deleteRssFeed(lastUrl,lastBlog,lastTagPrefix);
if(!ret) {
%><span class="b_rssImportMsgErr">Could not delete while attempting to change.</span><%
} else {
ret=bm.addRssFeed(url,blog,tagPrefix);
if(!ret) {
%><span class="b_rssImportMsgErr">Could not add while attempting to change.</span><%
} else {
%><span class="b_rssImportMsgOk">Ok, changed successfully.</span><%
}
}
}
} else if ( (action != null) && ("Delete".equals(action)) ) {
%><span class="b_rssImportMsgErr">Delete some thing</span><br /><%
if(url==null || blog==null || tagPrefix==null) {
%><span class="b_rssImportMsgErr">error, some fields were empty.</span><br /><%
} else {
boolean ret=bm.instance().deleteRssFeed(url,blog,tagPrefix);
if(!ret) {
%><span class="b_rssImportMsgErr">error, could not delete.</span><%
} else {
%><span class="b_rssImportMsgOk">ok, deleted successfully.</span><%
}
}
}
String blogStr=user.getBlogStr();
if(blogStr==null)
blogStr="";
///////////////////////////////////////////////
%>
<p>Here you can add RSS feeds that will be periodically polled and added to your syndie. </p>
<form action="rssimport.jsp" method="POST">
RSS URL. (e.g. http://tracker.postman.i2p/rss.php)<br />
<em><span class="b_rssImportField">url:</span></em> <input class="b_rssImportField" type="text" size="50" name="url" /><br />
Blog hash to which the RSS entries will get posted, defaults to the one you're logged in to.<br />
<em><span class="b_rssImportField">blog:</span></em> <input class="b_rssImportField" type="text" <%="value=\""+blogStr+"\""%> size="20" name="blog" /><br />
This will be prepended to any tags that the RSS feed contains. (e.g. feed.tracker)<br />
<em><span class="b_rssImportField">tagprefix:</span></em> <input class="b_rssImportField" type="text" value="feed" size="20" name="tagprefix" /><br />
<input class="b_rssImportSubmit" type="submit" name="action" value="Add" /> <input class="b_rssImportCancel" type="reset" value="Cancel" /></form>
<%
///////////////////////////////////////////////
List feedList = bm.getRssFeeds();
if(feedList.size()>0) {
%>
<hr /><h3>Subscriptions:</h3><br />
<table border="0" width="100%" class="b_rss">
<tr class="b_rssHeader">
<td class="b_rssHeader"><em class="b_rssHeader">Url</em></td>
<td class="b_rssHeader"><em class="b_rssHeader">Blog</em></td>
<td class="b_rssHeader"><em class="b_rssHeader">TagPrefix</em></td>
<td class="b_rssHeader">&nbsp;</td></tr>
<%
Iterator iter = feedList.iterator();
while(iter.hasNext()) {
String fields[]=(String[])iter.next();
url=fields[0];
blog=fields[1];
tagPrefix=fields[2];
StringBuffer buf = new StringBuffer(128);
buf.append("<tr class=\"b_rssDetail\"><form action=\"rssimport.jsp\" method=\"POST\">");
buf.append("<input type=\"hidden\" name=\"lasturl\" value=\"").append(url).append("\" />");
buf.append("<input type=\"hidden\" name=\"lastblog\" value=\"").append(blog).append("\" />");
buf.append("<input type=\"hidden\" name=\"lasttagprefix\" value=\"").append(tagPrefix).append("\" />");
buf.append("<td class=\"b_rssUrl\"><input class=\"b_rssUrl\" type=\"text\" size=\"50\" name=\"url\" value=\"").append(url).append("\" /></td>");
buf.append("<td class=\"b_rssBlog\"><input class=\"b_rssBlog\" type=\"text\" size=\"20\" name=\"blog\" value=\"").append(blog).append("\" /></td>");
buf.append("<td class=\"b_rssPrefix\"><input class=\"b_rssPrefix\" type=\"text\" size=\"20\" name=\"tagprefix\" value=\"").append(tagPrefix).append("\" /></td>");
buf.append("<td class=\"b_rssDetail\" nowrap=\"nowrap\">");
buf.append("<input class=\"b_rssChange\" type=\"submit\" name=\"action\" value=\"Change\" />");
buf.append("<input class=\"b_rssDelete\" type=\"submit\" name=\"action\" value=\"Delete\" />");
buf.append("</td></form></tr>");
out.write(buf.toString());
buf.setLength(0);
}
}
/*
<p><h3>todo:</h3>
<p>caching (eepget should do it)
<p>enclosures support (requires cvs rome)
<p>syndie.sucker.minHistory/maxHistory used to roll over the history file?
<p>configurable update period
*/
%>
</table>
<hr />
<%
}
%>
</td></tr>
</table>
</body>

58
core/java/src/net/i2p/util/EepGet.java
View File

@ -51,6 +51,7 @@ public class EepGet {
private String _etag;
private boolean _encodingChunked;
private boolean _notModified;
private String _contentType;
public EepGet(I2PAppContext ctx, String proxyHost, int proxyPort, int numRetries, String outputFile, String url) {
this(ctx, true, proxyHost, proxyPort, numRetries, outputFile, url);
@ -209,11 +210,13 @@ public class EepGet {
if (timeToSend > 0) {
StringBuffer buf = new StringBuffer(50);
buf.append(" ");
double pct = ((double)alreadyTransferred + (double)_written) / ((double)alreadyTransferred + (double)bytesRemaining);
synchronized (_pct) {
buf.append(_pct.format(pct));
if ( bytesRemaining > 0 ) {
double pct = ((double)alreadyTransferred + (double)_written) / ((double)alreadyTransferred + (double)bytesRemaining);
synchronized (_pct) {
buf.append(_pct.format(pct));
}
buf.append(": ");
}
buf.append(": ");
buf.append(_written+alreadyTransferred);
buf.append(" @ ");
double lineKBytes = ((double)_markSize * (double)_lineSize)/1024.0d;
@ -243,9 +246,15 @@ public class EepGet {
if (notModified) {
System.out.println("== Source not modified since last download");
} else {
System.out.println("== Transfer of " + url + " completed with " + (alreadyTransferred+bytesTransferred)
+ " and " + (bytesRemaining - bytesTransferred) + " remaining");
System.out.println("== Output saved to " + outputFile);
if ( bytesRemaining > 0 ) {
System.out.println("== Transfer of " + url + " completed with " + (alreadyTransferred+bytesTransferred)
+ " and " + (bytesRemaining - bytesTransferred) + " remaining");
System.out.println("== Output saved to " + outputFile);
} else {
System.out.println("== Transfer of " + url + " completed with " + (alreadyTransferred+bytesTransferred)
+ " bytes transferred");
System.out.println("== Output saved to " + outputFile);
}
}
long timeToSend = _context.clock().now() - _startedOn;
System.out.println("== Transfer time: " + DataHelper.formatDuration(timeToSend));
@ -355,9 +364,19 @@ public class EepGet {
_out.write(buf, 0, read);
_bytesTransferred += read;
remaining -= read;
if (remaining==0 && _encodingChunked) {
if(_proxyIn.read()=='\r' && _proxyIn.read()=='\n') {
remaining = (int) readChunkLength();
}
}
if (read > 0)
for (int i = 0; i < _listeners.size(); i++)
((StatusListener)_listeners.get(i)).bytesTransferred(_alreadyTransferred, read, _bytesTransferred, _bytesRemaining, _url);
((StatusListener)_listeners.get(i)).bytesTransferred(
_alreadyTransferred,
read,
_bytesTransferred,
_encodingChunked?-1:_bytesRemaining,
_url);
}
if (_out != null)
@ -369,7 +388,13 @@ public class EepGet {
if ( (_bytesRemaining == -1) || (remaining == 0) ){
for (int i = 0; i < _listeners.size(); i++)
((StatusListener)_listeners.get(i)).transferComplete(_alreadyTransferred, _bytesTransferred, _bytesRemaining, _url, _outputFile, _notModified);
((StatusListener)_listeners.get(i)).transferComplete(
_alreadyTransferred,
_bytesTransferred,
_encodingChunked?-1:_bytesRemaining,
_url,
_outputFile,
_notModified);
} else {
throw new IOException("Disconnection on attempt " + _currentAttempt + " after " + _bytesTransferred);
}
@ -387,6 +412,7 @@ public class EepGet {
switch (responseCode) {
case 200: // full
_out = new FileOutputStream(_outputFile, false);
_alreadyTransferred = 0;
rcOk = true;
break;
case 206: // partial
@ -405,7 +431,7 @@ public class EepGet {
default:
rcOk = false;
}
buf.setLength(0);
byte lookahead[] = new byte[3];
while (true) {
int cur = _proxyIn.read();
@ -435,7 +461,7 @@ public class EepGet {
if (!rcOk)
throw new IOException("Invalid HTTP response code: " + responseCode);
if (_encodingChunked) {
readChunkLength();
_bytesRemaining = readChunkLength();
}
return;
}
@ -450,7 +476,7 @@ public class EepGet {
}
}
private void readChunkLength() throws IOException {
private long readChunkLength() throws IOException {
StringBuffer buf = new StringBuffer(8);
int nl = 0;
while (true) {
@ -472,9 +498,9 @@ public class EepGet {
String len = buf.toString().trim();
try {
long bytes = Long.parseLong(len, 16);
_bytesRemaining = bytes;
if (_log.shouldLog(Log.DEBUG))
_log.debug("Chunked length: " + bytes);
return bytes;
} catch (NumberFormatException nfe) {
throw new IOException("Invalid chunk length [" + len + "]");
}
@ -529,6 +555,8 @@ public class EepGet {
} else if (key.equalsIgnoreCase("Transfer-encoding")) {
if (val.indexOf("chunked") != -1)
_encodingChunked = true;
} else if (key.equalsIgnoreCase("Content-Type")) {
_contentType=val;
} else {
// ignore the rest
}
@ -630,5 +658,9 @@ public class EepGet {
public boolean getNotModified() {
return _notModified;
}
public String getContentType() {
return _contentType;
}
}

8
history.txt
View File

@ -1,4 +1,10 @@
$Id: history.txt,v 1.308 2005/10/29 16:35:27 jrandom Exp $
$Id: history.txt,v 1.309 2005/10/29 18:20:04 jrandom Exp $
2005-10-30 dust
* Merge sucker into syndie with a rssimport.jsp page.
* Add getContentType() to EepGet.
* Make chunked transfer work (better) with EepGet.
* Do replaceAll("<","&lt;") for logs.
* 2005-10-29 0.6.1.4 released

4
router/java/src/net/i2p/router/RouterVersion.java
View File

@ -15,9 +15,9 @@ import net.i2p.CoreVersion;
*
*/
public class RouterVersion {
public final static String ID = "$Revision: 1.279 $ $Date: 2005/10/29 16:35:24 $";
public final static String ID = "$Revision: 1.280 $ $Date: 2005/10/29 18:20:05 $";
public final static String VERSION = "0.6.1.4";
public final static long BUILD = 0;
public final static long BUILD = 1;
public static void main(String args[]) {
System.out.println("I2P Router version: " + VERSION + "-" + BUILD);
System.out.println("Router ID: " + RouterVersion.ID);