2005-10-13 dust
* Bundled dust's Sucker for pulling RSS/Atom content into SML, which can then be injected into Syndie with the Syndie CLI. * Bundled ROME and JDOM (BSD and Apache licensed, respectively) for RSS/Atom parsing. 2005-10-13 jrandom * SSU retransmission choke bugfix (== != !=) * Include initial transmissions in the retransmission choke, so that if we are already retransmitting a message, we won't send anything to that peer other than that message (or ACKs, if necessary)
This commit is contained in:
BIN
apps/jdom/jdom.jar
Normal file
BIN
apps/jdom/jdom.jar
Normal file
Binary file not shown.
1
apps/jdom/readme.txt
Normal file
1
apps/jdom/readme.txt
Normal file
@ -0,0 +1 @@
|
|||||||
|
This is JDOM 1.0 from http://jdom.org/, released under an Apache style license
|
1
apps/rome/readme.txt
Normal file
1
apps/rome/readme.txt
Normal file
@ -0,0 +1 @@
|
|||||||
|
This is ROME 0.7 from http://rome.dev.java.net/, released under a BSD license
|
BIN
apps/rome/rome-0.7.jar
Normal file
BIN
apps/rome/rome-0.7.jar
Normal file
Binary file not shown.
@ -14,7 +14,7 @@
|
|||||||
srcdir="./src"
|
srcdir="./src"
|
||||||
debug="true" deprecation="on" source="1.3" target="1.3"
|
debug="true" deprecation="on" source="1.3" target="1.3"
|
||||||
destdir="./build/obj"
|
destdir="./build/obj"
|
||||||
classpath="../../../core/java/build/i2p.jar:../../jetty/jettylib/org.mortbay.jetty.jar:../../jetty/jettylib/javax.servlet.jar" />
|
classpath="../../../core/java/build/i2p.jar:../../jetty/jettylib/org.mortbay.jetty.jar:../../jetty/jettylib/javax.servlet.jar:../../jdom/jdom.jar:../../rome/rome-0.7.jar" />
|
||||||
</target>
|
</target>
|
||||||
<target name="jar" depends="builddep, compile">
|
<target name="jar" depends="builddep, compile">
|
||||||
<jar destfile="./build/syndie.jar" basedir="./build/obj" includes="**/*.class">
|
<jar destfile="./build/syndie.jar" basedir="./build/obj" includes="**/*.class">
|
||||||
@ -23,6 +23,12 @@
|
|||||||
<attribute name="Class-Path" value="i2p.jar" />
|
<attribute name="Class-Path" value="i2p.jar" />
|
||||||
</manifest>
|
</manifest>
|
||||||
</jar>
|
</jar>
|
||||||
|
<jar destfile="./build/sucker.jar" basedir="./build/obj" includes="net/i2p/syndie/Sucker.class, net/i2p/syndie/SuckerFetchListener.class">
|
||||||
|
<manifest>
|
||||||
|
<attribute name="Main-Class" value="net.i2p.syndie.Sucker" />
|
||||||
|
<attribute name="Class-Path" value="i2p.jar jdom.jar rome-0.7.jar" />
|
||||||
|
</manifest>
|
||||||
|
</jar>
|
||||||
<ant target="war" />
|
<ant target="war" />
|
||||||
</target>
|
</target>
|
||||||
<target name="war" depends="builddep, compile, precompilejsp">
|
<target name="war" depends="builddep, compile, precompilejsp">
|
||||||
@ -48,6 +54,8 @@
|
|||||||
<pathelement location="../../jetty/jettylib/org.mortbay.jetty.jar" />
|
<pathelement location="../../jetty/jettylib/org.mortbay.jetty.jar" />
|
||||||
<pathelement location="../../jetty/jettylib/javax.servlet.jar" />
|
<pathelement location="../../jetty/jettylib/javax.servlet.jar" />
|
||||||
<pathelement location="../../jetty/jettylib/ant.jar" />
|
<pathelement location="../../jetty/jettylib/ant.jar" />
|
||||||
|
<pathelement location="../../jdom/jdom.jar" />
|
||||||
|
<pathelement location="../../rome/rome-0.7.jar" />
|
||||||
<pathelement location="build/obj" />
|
<pathelement location="build/obj" />
|
||||||
<pathelement location="../../../core/java/build/i2p.jar" />
|
<pathelement location="../../../core/java/build/i2p.jar" />
|
||||||
</classpath>
|
</classpath>
|
||||||
@ -69,6 +77,8 @@
|
|||||||
<pathelement location="../../jetty/jettylib/commons-el.jar" />
|
<pathelement location="../../jetty/jettylib/commons-el.jar" />
|
||||||
<pathelement location="../../jetty/jettylib/org.mortbay.jetty.jar" />
|
<pathelement location="../../jetty/jettylib/org.mortbay.jetty.jar" />
|
||||||
<pathelement location="../../jetty/jettylib/javax.servlet.jar" />
|
<pathelement location="../../jetty/jettylib/javax.servlet.jar" />
|
||||||
|
<pathelement location="../../jdom/jdom.jar" />
|
||||||
|
<pathelement location="../../rome/rome-0.7.jar" />
|
||||||
<pathelement location="build/obj" />
|
<pathelement location="build/obj" />
|
||||||
<pathelement location="../../../core/java/build/i2p.jar" />
|
<pathelement location="../../../core/java/build/i2p.jar" />
|
||||||
</classpath>
|
</classpath>
|
||||||
|
635
apps/syndie/java/src/net/i2p/syndie/Sucker.java
Normal file
635
apps/syndie/java/src/net/i2p/syndie/Sucker.java
Normal file
@ -0,0 +1,635 @@
|
|||||||
|
package net.i2p.syndie;
|
||||||
|
|
||||||
|
import java.io.DataInputStream;
|
||||||
|
import java.io.File;
|
||||||
|
import java.io.FileInputStream;
|
||||||
|
import java.io.FileNotFoundException;
|
||||||
|
import java.io.FileOutputStream;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.io.InputStream;
|
||||||
|
import java.io.OutputStream;
|
||||||
|
import java.net.MalformedURLException;
|
||||||
|
import java.net.Socket;
|
||||||
|
import java.net.SocketAddress;
|
||||||
|
import java.net.URL;
|
||||||
|
import java.net.URLConnection;
|
||||||
|
import java.security.MessageDigest;
|
||||||
|
import java.security.NoSuchAlgorithmException;
|
||||||
|
import java.text.SimpleDateFormat;
|
||||||
|
import java.util.Date;
|
||||||
|
import java.util.Iterator;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.ListIterator;
|
||||||
|
|
||||||
|
import sun.security.provider.SHA;
|
||||||
|
|
||||||
|
import com.sun.syndication.feed.atom.Entry;
|
||||||
|
import com.sun.syndication.feed.synd.SyndCategory;
|
||||||
|
import com.sun.syndication.feed.synd.SyndContent;
|
||||||
|
import com.sun.syndication.feed.synd.SyndEntry;
|
||||||
|
import com.sun.syndication.feed.synd.SyndFeed;
|
||||||
|
import com.sun.syndication.io.FeedException;
|
||||||
|
import com.sun.syndication.io.SyndFeedInput;
|
||||||
|
import com.sun.syndication.io.XmlReader;
|
||||||
|
|
||||||
|
import net.i2p.I2PAppContext;
|
||||||
|
import net.i2p.util.EepGet;
|
||||||
|
|
||||||
|
public class Sucker {
|
||||||
|
private String urlToLoad;
|
||||||
|
private String outputDir="./sucker_out";
|
||||||
|
private String historyPath="./sucker.history";
|
||||||
|
private String feedTag="feed";
|
||||||
|
private File historyFile;
|
||||||
|
private String proxyPort;
|
||||||
|
private String proxyHost;
|
||||||
|
private String pushScript;
|
||||||
|
private int attachmentCounter=0;
|
||||||
|
private String messagePath;
|
||||||
|
private String baseUrl;
|
||||||
|
private boolean importEnclosures=true;
|
||||||
|
private boolean importRefs=true;
|
||||||
|
private boolean pendingEndLink;
|
||||||
|
|
||||||
|
public Sucker() {}
|
||||||
|
|
||||||
|
public boolean parseArgs(String args[]) {
|
||||||
|
for (int i = 0; i < args.length; i++) {
|
||||||
|
if ("--load".equals(args[i]))
|
||||||
|
urlToLoad = args[++i];
|
||||||
|
if ("--outputdir".equals(args[i]))
|
||||||
|
outputDir = args[++i];
|
||||||
|
if ("--history".equals(args[i]))
|
||||||
|
historyPath = args[++i];
|
||||||
|
if ("--tag".equals(args[i]))
|
||||||
|
feedTag = args[++i];
|
||||||
|
if ("--proxyhost".equals(args[i]))
|
||||||
|
proxyHost = args[++i];
|
||||||
|
if ("--proxyport".equals(args[i]))
|
||||||
|
proxyPort = args[++i];
|
||||||
|
if ("--exec".equals(args[i]))
|
||||||
|
pushScript = args[++i];
|
||||||
|
if ("--importenclosures".equals(args[i]))
|
||||||
|
importEnclosures= args[++i].equals("true");
|
||||||
|
if ("--importenrefs".equals(args[i]))
|
||||||
|
importRefs= args[++i].equals("true");
|
||||||
|
}
|
||||||
|
|
||||||
|
// Cut ending '/' from outputDir
|
||||||
|
if (outputDir.endsWith("/"))
|
||||||
|
outputDir = outputDir.substring(0, outputDir.length() - 1);
|
||||||
|
|
||||||
|
if (urlToLoad == null)
|
||||||
|
return false;
|
||||||
|
|
||||||
|
// Find base url, gah HELP
|
||||||
|
int idx=urlToLoad.length();
|
||||||
|
int x=urlToLoad.indexOf('?');
|
||||||
|
if(x>0)
|
||||||
|
idx=x;
|
||||||
|
while(idx>0)
|
||||||
|
{
|
||||||
|
idx--;
|
||||||
|
if(urlToLoad.charAt(idx)=='/')
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
if(idx==0)
|
||||||
|
idx=x;
|
||||||
|
baseUrl=urlToLoad.substring(0,idx);
|
||||||
|
System.out.println("BaseUrl: "+baseUrl);
|
||||||
|
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void suck() {
|
||||||
|
URL feedUrl;
|
||||||
|
SyndFeed feed;
|
||||||
|
int i;
|
||||||
|
|
||||||
|
//if (proxyHost != null && proxyPort != null) {
|
||||||
|
// // Set proxy
|
||||||
|
// System.setProperty("http.proxyHost", proxyHost);
|
||||||
|
// System.setProperty("http.proxyPort", proxyPort);
|
||||||
|
//}
|
||||||
|
|
||||||
|
|
||||||
|
//
|
||||||
|
try {
|
||||||
|
|
||||||
|
// Create outputDir if missing
|
||||||
|
File f = new File(outputDir);
|
||||||
|
f.mkdirs();
|
||||||
|
|
||||||
|
// Create historyFile if missing
|
||||||
|
historyFile = new File(historyPath);
|
||||||
|
if (!historyFile.exists())
|
||||||
|
historyFile.createNewFile();
|
||||||
|
|
||||||
|
int messageNumber;
|
||||||
|
|
||||||
|
File lastIdFile = new File(historyPath + ".lastId");
|
||||||
|
if (!lastIdFile.exists())
|
||||||
|
lastIdFile.createNewFile();
|
||||||
|
|
||||||
|
FileInputStream fis = new FileInputStream(lastIdFile);
|
||||||
|
String number = readLine(fis);
|
||||||
|
try {
|
||||||
|
messageNumber = Integer.parseInt(number);
|
||||||
|
} catch (NumberFormatException e) {
|
||||||
|
messageNumber = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
SyndFeedInput input = new SyndFeedInput();
|
||||||
|
|
||||||
|
boolean shouldProxy = false;
|
||||||
|
int proxyPortNum = -1;
|
||||||
|
if ( (proxyHost != null) && (proxyPort != null) ) {
|
||||||
|
try {
|
||||||
|
proxyPortNum = Integer.parseInt(proxyPort);
|
||||||
|
if (proxyPortNum > 0)
|
||||||
|
shouldProxy = true;
|
||||||
|
} catch (NumberFormatException nfe) {
|
||||||
|
nfe.printStackTrace();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
int numRetries = 2;
|
||||||
|
// perhaps specify a temp dir?
|
||||||
|
File fetched = File.createTempFile("sucker", ".fetch");
|
||||||
|
fetched.deleteOnExit();
|
||||||
|
// we use eepGet, since it retries and doesn't leak DNS requests like URL does
|
||||||
|
EepGet get = new EepGet(I2PAppContext.getGlobalContext(), shouldProxy, proxyHost, proxyPortNum,
|
||||||
|
numRetries, fetched.getAbsolutePath(), urlToLoad);
|
||||||
|
SuckerFetchListener lsnr = new SuckerFetchListener();
|
||||||
|
get.addStatusListener(lsnr);
|
||||||
|
get.fetch();
|
||||||
|
boolean ok = lsnr.waitForSuccess();
|
||||||
|
if (!ok) {
|
||||||
|
System.err.println("Unable to retrieve the url after " + numRetries + " tries.");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
feed = input.build(new XmlReader(fetched));
|
||||||
|
|
||||||
|
List entries = feed.getEntries();
|
||||||
|
|
||||||
|
FileOutputStream hos = new FileOutputStream(historyFile, true);
|
||||||
|
|
||||||
|
ListIterator iter = entries.listIterator();
|
||||||
|
while (iter.hasNext()) {
|
||||||
|
|
||||||
|
attachmentCounter=0;
|
||||||
|
|
||||||
|
SyndEntry e = (SyndEntry) iter.next();
|
||||||
|
// Calculate messageId
|
||||||
|
String feedHash = sha1(urlToLoad);
|
||||||
|
String itemHash = sha1(e.getTitle() + e.getDescription());
|
||||||
|
Date d = e.getPublishedDate();
|
||||||
|
String time;
|
||||||
|
if(d!=null)
|
||||||
|
time = "" + d.getTime();
|
||||||
|
else
|
||||||
|
time = "" + new Date().getTime();
|
||||||
|
|
||||||
|
String outputFileName = outputDir + "/" + messageNumber;
|
||||||
|
|
||||||
|
/*
|
||||||
|
* $feedHash:$itemHash:$time:$outputfile. $feedHash would be the
|
||||||
|
* hash (md5? sha1? sha2?) of the $urlToFeed, $itemHash is some
|
||||||
|
* hash of the SyndEntry, $time would be the time that the entry
|
||||||
|
* was posted $outputfile would be the $outputdir/$uniqueid
|
||||||
|
*/
|
||||||
|
String messageId = feedHash + ":" + itemHash + ":" + time + ":"
|
||||||
|
+ outputFileName;
|
||||||
|
|
||||||
|
// Check if we already have this
|
||||||
|
if (!existsInHistory(messageId)) {
|
||||||
|
System.out.println("new: " + messageId);
|
||||||
|
|
||||||
|
if (convertToSml(e, ""+messageNumber)) {
|
||||||
|
hos.write(messageId.getBytes());
|
||||||
|
hos.write("\n".getBytes());
|
||||||
|
|
||||||
|
if (pushScript != null) {
|
||||||
|
if (!execPushScript(""+messageNumber, time))
|
||||||
|
System.out.println("push failed");
|
||||||
|
else
|
||||||
|
System.out.println("push success");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
messageNumber++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
FileOutputStream fos = new FileOutputStream(lastIdFile);
|
||||||
|
fos.write(("" + messageNumber).getBytes());
|
||||||
|
} catch (MalformedURLException e) {
|
||||||
|
// TODO Auto-generated catch block
|
||||||
|
e.printStackTrace();
|
||||||
|
} catch (IllegalArgumentException e) {
|
||||||
|
// TODO Auto-generated catch block
|
||||||
|
e.printStackTrace();
|
||||||
|
} catch (FeedException e) {
|
||||||
|
// TODO Auto-generated catch block
|
||||||
|
e.printStackTrace();
|
||||||
|
} catch (IOException e) {
|
||||||
|
// TODO Auto-generated catch block
|
||||||
|
e.printStackTrace();
|
||||||
|
}
|
||||||
|
System.out.println("Done.");
|
||||||
|
}
|
||||||
|
|
||||||
|
public static void main(String[] args) {
|
||||||
|
Sucker sucker = new Sucker();
|
||||||
|
boolean ok = sucker.parseArgs(args);
|
||||||
|
if (!ok) {
|
||||||
|
usage();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
sucker.suck();
|
||||||
|
}
|
||||||
|
|
||||||
|
private boolean execPushScript(String id, String time) {
|
||||||
|
try {
|
||||||
|
String ls_str;
|
||||||
|
|
||||||
|
String cli = pushScript + " " + outputDir + " " + id + " " + time;
|
||||||
|
Process pushScript_proc = Runtime.getRuntime().exec(cli);
|
||||||
|
|
||||||
|
// get its output (your input) stream
|
||||||
|
|
||||||
|
DataInputStream ls_in = new DataInputStream(pushScript_proc.getInputStream());
|
||||||
|
|
||||||
|
try {
|
||||||
|
while ((ls_str = ls_in.readLine()) != null) {
|
||||||
|
System.out.println(pushScript + ": " + ls_str);
|
||||||
|
}
|
||||||
|
} catch (IOException e) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
pushScript_proc.waitFor();
|
||||||
|
if(pushScript_proc.exitValue()==0)
|
||||||
|
return true;
|
||||||
|
} catch (InterruptedException e) {
|
||||||
|
// TODO Auto-generated catch block
|
||||||
|
e.printStackTrace();
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
} catch (IOException e1) {
|
||||||
|
System.err.println(e1);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private boolean convertToSml(SyndEntry e, String messageName) {
|
||||||
|
|
||||||
|
// Create message
|
||||||
|
File messageFile = new File(messageName);
|
||||||
|
FileOutputStream fos;
|
||||||
|
messagePath=outputDir+"/"+messageName;
|
||||||
|
try {
|
||||||
|
fos = new FileOutputStream(messagePath);
|
||||||
|
|
||||||
|
String sml;
|
||||||
|
sml = "Subject: " + e.getTitle() + "\n";
|
||||||
|
List cats = e.getCategories();
|
||||||
|
Iterator iter = cats.iterator();
|
||||||
|
String tags = feedTag;
|
||||||
|
while (iter.hasNext()) {
|
||||||
|
SyndCategory c = (SyndCategory) iter.next();
|
||||||
|
String tag=c.getName();
|
||||||
|
tag=tag.replaceAll("[^a-zA-z.-_:]","_");
|
||||||
|
tags += "\t" + feedTag + "." + tag;
|
||||||
|
}
|
||||||
|
sml += "Tags: " + tags + "\n";
|
||||||
|
sml += "\n";
|
||||||
|
|
||||||
|
SyndContent content;
|
||||||
|
|
||||||
|
List l = e.getContents();
|
||||||
|
if(l!=null)
|
||||||
|
{
|
||||||
|
System.out.println("There is content");
|
||||||
|
iter = l.iterator();
|
||||||
|
while(iter.hasNext())
|
||||||
|
{
|
||||||
|
content = (SyndContent)iter.next();
|
||||||
|
String c = content.getValue();
|
||||||
|
System.out.println("Content: "+c);
|
||||||
|
sml += htmlToSml(c);
|
||||||
|
sml += "\n";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
String source=e.getUri();
|
||||||
|
if(source.indexOf("http")<0)
|
||||||
|
source=baseUrl+source;
|
||||||
|
sml += "[link schema=\"web\" location=\""+source+"\"]source[/link]\n";
|
||||||
|
|
||||||
|
fos.write(sml.getBytes());
|
||||||
|
|
||||||
|
return true;
|
||||||
|
} catch (FileNotFoundException e1) {
|
||||||
|
// TODO Auto-generated catch block
|
||||||
|
e1.printStackTrace();
|
||||||
|
} catch (IOException e2) {
|
||||||
|
// TODO Auto-generated catch block
|
||||||
|
e2.printStackTrace();
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
private String htmlToSml(String html) {
|
||||||
|
|
||||||
|
String sml="";
|
||||||
|
int i;
|
||||||
|
|
||||||
|
pendingEndLink=false;
|
||||||
|
|
||||||
|
for(i=0;i<html.length();)
|
||||||
|
{
|
||||||
|
if(html.charAt(i)=='<')
|
||||||
|
{
|
||||||
|
//System.out.println("html: "+html.substring(i));
|
||||||
|
|
||||||
|
int tagLen = findTagLen(html.substring(i));
|
||||||
|
//
|
||||||
|
String htmlTag = html.substring(i,i+tagLen);
|
||||||
|
|
||||||
|
//System.out.println("htmlTag: "+htmlTag);
|
||||||
|
|
||||||
|
String smlTag = htmlTagToSmlTag(htmlTag);
|
||||||
|
if(smlTag!=null)
|
||||||
|
sml+=smlTag;
|
||||||
|
i+=tagLen;
|
||||||
|
//System.out.println("tagLen: "+tagLen);
|
||||||
|
sml+=" ";
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
sml+=html.charAt(i++);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return sml;
|
||||||
|
}
|
||||||
|
|
||||||
|
private String htmlTagToSmlTag(String htmlTag) {
|
||||||
|
String ret="";
|
||||||
|
|
||||||
|
if(importEnclosures && htmlTag.startsWith("<img"))
|
||||||
|
{
|
||||||
|
System.out.println("Found image tag: "+htmlTag);
|
||||||
|
int a,b;
|
||||||
|
a=htmlTag.indexOf("src=\"")+5;
|
||||||
|
b=a+1;
|
||||||
|
while(htmlTag.charAt(b)!='\"')
|
||||||
|
b++;
|
||||||
|
String imageLink=htmlTag.substring(a,b);
|
||||||
|
|
||||||
|
if(pendingEndLink) {
|
||||||
|
ret="[/link]";
|
||||||
|
pendingEndLink=false;
|
||||||
|
}
|
||||||
|
|
||||||
|
ret += "[img attachment=\""+""+ attachmentCounter +"\"]";
|
||||||
|
|
||||||
|
a=htmlTag.indexOf("alt=\"")+5;
|
||||||
|
if(a>=5)
|
||||||
|
{
|
||||||
|
b=a+1;
|
||||||
|
while(htmlTag.charAt(b)!='\"')
|
||||||
|
b++;
|
||||||
|
String altText=htmlTag.substring(a,b);
|
||||||
|
ret+=altText;
|
||||||
|
}
|
||||||
|
|
||||||
|
ret+="[/img]";
|
||||||
|
|
||||||
|
if(imageLink.indexOf("http")<0)
|
||||||
|
imageLink=baseUrl+"/"+imageLink;
|
||||||
|
|
||||||
|
fetchAttachment(imageLink);
|
||||||
|
|
||||||
|
System.out.println("Converted to: "+ret);
|
||||||
|
|
||||||
|
return ret;
|
||||||
|
|
||||||
|
}
|
||||||
|
if(importRefs && htmlTag.startsWith("<a "))
|
||||||
|
{
|
||||||
|
System.out.println("Found link tag: "+htmlTag);
|
||||||
|
int a,b;
|
||||||
|
|
||||||
|
a=htmlTag.indexOf("href=\"")+6;
|
||||||
|
b=a+1;
|
||||||
|
while(htmlTag.charAt(b)!='\"')
|
||||||
|
b++;
|
||||||
|
String link=htmlTag.substring(a,b);
|
||||||
|
if(link.indexOf("http")<0)
|
||||||
|
link=baseUrl+"/"+link;
|
||||||
|
|
||||||
|
String schema="web";
|
||||||
|
|
||||||
|
ret += "[link schema=\""+schema+"\" location=\""+link+"\"]";
|
||||||
|
pendingEndLink=true;
|
||||||
|
|
||||||
|
System.out.println("Converted to: "+ret);
|
||||||
|
|
||||||
|
return ret;
|
||||||
|
}
|
||||||
|
|
||||||
|
if ("</a>".equals(htmlTag)) {
|
||||||
|
if (pendingEndLink)
|
||||||
|
return "[/link]";
|
||||||
|
}
|
||||||
|
|
||||||
|
if("<b>".equals(htmlTag))
|
||||||
|
return "[b]";
|
||||||
|
if("</b>".equals(htmlTag))
|
||||||
|
return "[/b]";
|
||||||
|
if("<i>".equals(htmlTag))
|
||||||
|
return "[i]";
|
||||||
|
if("</i>".equals(htmlTag))
|
||||||
|
return "[/i]";
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void fetchAttachment(String link) {
|
||||||
|
System.out.println("Fetch attachment from: "+link);
|
||||||
|
String attachmentPath = messagePath+"."+attachmentCounter;
|
||||||
|
try {
|
||||||
|
link=link.replaceAll("&","&");
|
||||||
|
URL attachmentUrl = new URL(link);
|
||||||
|
InputStream is = attachmentUrl.openStream();
|
||||||
|
|
||||||
|
FileOutputStream fos = new FileOutputStream(attachmentPath);
|
||||||
|
|
||||||
|
while(true)
|
||||||
|
{
|
||||||
|
int i =is.read();
|
||||||
|
if(i<0)
|
||||||
|
break;
|
||||||
|
fos.write(i);
|
||||||
|
}
|
||||||
|
|
||||||
|
} catch (MalformedURLException e) {
|
||||||
|
// TODO Auto-generated catch block
|
||||||
|
e.printStackTrace();
|
||||||
|
} catch (IOException e) {
|
||||||
|
// TODO Auto-generated catch block
|
||||||
|
e.printStackTrace();
|
||||||
|
}
|
||||||
|
|
||||||
|
attachmentCounter++;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static int findTagLen(String s) {
|
||||||
|
int i;
|
||||||
|
for(i=0;i<s.length();i++)
|
||||||
|
{
|
||||||
|
if(s.charAt(i)=='>')
|
||||||
|
return i+1;
|
||||||
|
if(s.charAt(i)=='"')
|
||||||
|
{
|
||||||
|
i++;
|
||||||
|
while(s.charAt(i)!='"')
|
||||||
|
i++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
System.out.println("WTF");
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
private boolean existsInHistory(String messageId) {
|
||||||
|
int idx;
|
||||||
|
idx = messageId.lastIndexOf(":");
|
||||||
|
String lineToCompare = messageId.substring(0, idx-1);
|
||||||
|
idx = lineToCompare.lastIndexOf(":");
|
||||||
|
lineToCompare = lineToCompare.substring(0, idx-1);
|
||||||
|
try {
|
||||||
|
FileInputStream his = new FileInputStream(historyFile);
|
||||||
|
String line;
|
||||||
|
while ((line = readLine(his)) != null) {
|
||||||
|
idx = line.lastIndexOf(":");
|
||||||
|
if (idx < 0)
|
||||||
|
return false;
|
||||||
|
line = line.substring(0, idx-1);
|
||||||
|
idx = line.lastIndexOf(":");
|
||||||
|
if (idx < 0)
|
||||||
|
return false;
|
||||||
|
line = line.substring(0, idx-1);
|
||||||
|
if (line.equals(lineToCompare))
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
} catch (FileNotFoundException e) {
|
||||||
|
// TODO Auto-generated catch block
|
||||||
|
e.printStackTrace();
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void usage() {
|
||||||
|
System.out.println("sucker --load $urlToFeed \n"
|
||||||
|
+ "--proxyhost <host> \n"
|
||||||
|
+ "--proxyport <port> \n"
|
||||||
|
+ "--importenclosures true \n"
|
||||||
|
+ "--importrefs true \n"
|
||||||
|
+ "--tag feed \n"
|
||||||
|
+ "--outputdir ./sucker_out \n"
|
||||||
|
+ "--exec pushscript.sh OUTPUTDIR UNIQUEID ENTRYTIMESTAMP \n"
|
||||||
|
+ "--history ./sucker.history");
|
||||||
|
System.exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static String sha1(String s) {
|
||||||
|
try {
|
||||||
|
MessageDigest md = MessageDigest.getInstance("SHA");
|
||||||
|
md.update(s.getBytes());
|
||||||
|
byte[] buf = md.digest();
|
||||||
|
String ret = new sun.misc.BASE64Encoder().encode(buf);
|
||||||
|
return ret;
|
||||||
|
} catch (NoSuchAlgorithmException ex) {
|
||||||
|
ex.printStackTrace();
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static String readLine(FileInputStream in) {
|
||||||
|
StringBuffer sb = new StringBuffer();
|
||||||
|
int c = 0;
|
||||||
|
while (true) {
|
||||||
|
try {
|
||||||
|
c = in.read();
|
||||||
|
} catch (IOException e) {
|
||||||
|
// TODO Auto-generated catch block
|
||||||
|
e.printStackTrace();
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
if (c < 0)
|
||||||
|
break;
|
||||||
|
if (c == '\n')
|
||||||
|
break;
|
||||||
|
sb.append((char) c);
|
||||||
|
}
|
||||||
|
// TODO Auto-generated method stub
|
||||||
|
return sb.toString();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Simple blocking listener for eepget. block in waitForSuccess().
|
||||||
|
*/
|
||||||
|
class SuckerFetchListener implements EepGet.StatusListener {
|
||||||
|
private volatile boolean _complete;
|
||||||
|
private volatile boolean _successful;
|
||||||
|
|
||||||
|
public SuckerFetchListener() {
|
||||||
|
_complete = false;
|
||||||
|
_successful = false;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void transferComplete(long alreadyTransferred, long bytesTransferred, long bytesRemaining, String url, String outputFile, boolean notModified) {
|
||||||
|
notifyComplete(true);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void transferFailed(String url, long bytesTransferred, long bytesRemaining, int currentAttempt) {
|
||||||
|
notifyComplete(false);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void notifyComplete(boolean ok) {
|
||||||
|
synchronized (this) {
|
||||||
|
_complete = true;
|
||||||
|
_successful = ok;
|
||||||
|
notifyAll();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Block until the fetch is successful, returning true if it did fetch completely,
|
||||||
|
* false if it didn't.
|
||||||
|
*
|
||||||
|
*/
|
||||||
|
public boolean waitForSuccess() {
|
||||||
|
while (!_complete) {
|
||||||
|
try {
|
||||||
|
synchronized (this) {
|
||||||
|
wait();
|
||||||
|
}
|
||||||
|
} catch (InterruptedException ie) {}
|
||||||
|
}
|
||||||
|
return _successful;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void attemptFailed(String url, long bytesTransferred, long bytesRemaining, int currentAttempt, int numRetries, Exception cause) {
|
||||||
|
// noop, it may retry
|
||||||
|
}
|
||||||
|
public void bytesTransferred(long alreadyTransferred, int currentWrite, long bytesTransferred, long bytesRemaining, String url) {
|
||||||
|
// ignore this status update
|
||||||
|
}
|
||||||
|
public void headerReceived(String url, int currentAttempt, String key, String val) {
|
||||||
|
// ignore
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
14
build.xml
14
build.xml
@ -29,7 +29,7 @@
|
|||||||
<ant dir="apps/addressbook/" target="war" />
|
<ant dir="apps/addressbook/" target="war" />
|
||||||
<ant dir="apps/susimail/" target="war" />
|
<ant dir="apps/susimail/" target="war" />
|
||||||
<ant dir="apps/susidns/src" target="all" />
|
<ant dir="apps/susidns/src" target="all" />
|
||||||
<ant dir="apps/syndie/java/" target="jar" /> <!-- not pushed in the update... yet -->
|
<ant dir="apps/syndie/java/" target="jar" />
|
||||||
</target>
|
</target>
|
||||||
<target name="buildWEB">
|
<target name="buildWEB">
|
||||||
<ant dir="apps/jetty" target="fetchJettylib" />
|
<ant dir="apps/jetty" target="fetchJettylib" />
|
||||||
@ -89,7 +89,10 @@
|
|||||||
<copy file="apps/susidns/src/susidns.war" todir="build/" />
|
<copy file="apps/susidns/src/susidns.war" todir="build/" />
|
||||||
<copy file="apps/syndie/syndie.war" todir="build/" />
|
<copy file="apps/syndie/syndie.war" todir="build/" />
|
||||||
<copy file="apps/syndie/java/build/syndie.jar" todir="build/" />
|
<copy file="apps/syndie/java/build/syndie.jar" todir="build/" />
|
||||||
|
<copy file="apps/syndie/java/build/sucker.jar" todir="build/" />
|
||||||
<copy file="apps/syndie/syndie.war" todir="build/" />
|
<copy file="apps/syndie/syndie.war" todir="build/" />
|
||||||
|
<copy file="apps/jdom/jdom.jar" todir="build/" />
|
||||||
|
<copy file="apps/rome/rome-0.7.jar" todir="build/" />
|
||||||
</target>
|
</target>
|
||||||
<target name="javadoc">
|
<target name="javadoc">
|
||||||
<mkdir dir="./build" />
|
<mkdir dir="./build" />
|
||||||
@ -211,6 +214,9 @@
|
|||||||
<copy file="build/routerconsole.jar" todir="pkg-temp/lib/" />
|
<copy file="build/routerconsole.jar" todir="pkg-temp/lib/" />
|
||||||
<copy file="build/sam.jar" todir="pkg-temp/lib/" />
|
<copy file="build/sam.jar" todir="pkg-temp/lib/" />
|
||||||
<copy file="build/systray.jar" todir="pkg-temp/lib" />
|
<copy file="build/systray.jar" todir="pkg-temp/lib" />
|
||||||
|
<copy file="build/jdom.jar" todir="pkg-temp/lib" />
|
||||||
|
<copy file="build/rome-0.7.jar" todir="pkg-temp/lib" />
|
||||||
|
<copy file="build/sucker.jar" todir="pkg-temp/lib" />
|
||||||
<copy file="i2p.exe" todir="pkg-temp/" failonerror="false" />
|
<copy file="i2p.exe" todir="pkg-temp/" failonerror="false" />
|
||||||
<copy file="installer/resources/runplain.sh" todir="pkg-temp/" />
|
<copy file="installer/resources/runplain.sh" todir="pkg-temp/" />
|
||||||
<copy file="apps/systray/java/lib/systray4j.jar" todir="pkg-temp/lib" />
|
<copy file="apps/systray/java/lib/systray4j.jar" todir="pkg-temp/lib" />
|
||||||
@ -310,6 +316,12 @@
|
|||||||
<copy file="build/sam.jar" todir="pkg-temp/lib/" />
|
<copy file="build/sam.jar" todir="pkg-temp/lib/" />
|
||||||
<copy file="build/router.jar" todir="pkg-temp/lib/" />
|
<copy file="build/router.jar" todir="pkg-temp/lib/" />
|
||||||
<copy file="build/routerconsole.jar" todir="pkg-temp/lib/" />
|
<copy file="build/routerconsole.jar" todir="pkg-temp/lib/" />
|
||||||
|
<!-- for the 0.6.1.3 release, push JDOM and ROME. it can be
|
||||||
|
removed later updates (once people on earlier builds upgrade -->
|
||||||
|
<copy file="build/jdom.jar" todir="pkg-temp/lib" />
|
||||||
|
<copy file="build/rome-0.7.jar" todir="pkg-temp/lib" />
|
||||||
|
<copy file="build/sucker.jar" todir="pkg-temp/lib" />
|
||||||
|
|
||||||
<copy file="i2p.exe" todir="pkg-temp/" failonerror="false" />
|
<copy file="i2p.exe" todir="pkg-temp/" failonerror="false" />
|
||||||
<copy file="installer/resources/runplain.sh" todir="pkg-temp/" />
|
<copy file="installer/resources/runplain.sh" todir="pkg-temp/" />
|
||||||
|
|
||||||
|
@ -1,4 +1,10 @@
|
|||||||
$Id: history.txt,v 1.292 2005/10/12 22:54:54 jrandom Exp $
|
$Id: history.txt,v 1.293 2005/10/13 04:18:37 jrandom Exp $
|
||||||
|
|
||||||
|
2005-10-13 dust
|
||||||
|
* Bundled dust's Sucker for pulling RSS/Atom content into SML, which can
|
||||||
|
then be injected into Syndie with the Syndie CLI.
|
||||||
|
* Bundled ROME and JDOM (BSD and Apache licensed, respectively) for
|
||||||
|
RSS/Atom parsing.
|
||||||
|
|
||||||
2005-10-13 jrandom
|
2005-10-13 jrandom
|
||||||
* SSU retransmission choke bugfix (== != !=)
|
* SSU retransmission choke bugfix (== != !=)
|
||||||
|
@ -15,9 +15,9 @@ import net.i2p.CoreVersion;
|
|||||||
*
|
*
|
||||||
*/
|
*/
|
||||||
public class RouterVersion {
|
public class RouterVersion {
|
||||||
public final static String ID = "$Revision: 1.267 $ $Date: 2005/10/12 22:54:54 $";
|
public final static String ID = "$Revision: 1.268 $ $Date: 2005/10/13 04:18:36 $";
|
||||||
public final static String VERSION = "0.6.1.2";
|
public final static String VERSION = "0.6.1.2";
|
||||||
public final static long BUILD = 8;
|
public final static long BUILD = 9;
|
||||||
public static void main(String args[]) {
|
public static void main(String args[]) {
|
||||||
System.out.println("I2P Router version: " + VERSION + "-" + BUILD);
|
System.out.println("I2P Router version: " + VERSION + "-" + BUILD);
|
||||||
System.out.println("Router ID: " + RouterVersion.ID);
|
System.out.println("Router ID: " + RouterVersion.ID);
|
||||||
|
@ -1111,6 +1111,8 @@ public class UDPTransport extends TransportImpl implements TimedWeightedPriority
|
|||||||
buf.append("<tr><td colspan=\"14\" valign=\"top\" align=\"left\">");
|
buf.append("<tr><td colspan=\"14\" valign=\"top\" align=\"left\">");
|
||||||
long bytesTransmitted = _context.bandwidthLimiter().getTotalAllocatedOutboundBytes();
|
long bytesTransmitted = _context.bandwidthLimiter().getTotalAllocatedOutboundBytes();
|
||||||
double averagePacketSize = _context.statManager().getRate("udp.sendPacketSize").getLifetimeAverageValue();
|
double averagePacketSize = _context.statManager().getRate("udp.sendPacketSize").getLifetimeAverageValue();
|
||||||
|
// lifetime value, not just the retransmitted packets of current connections
|
||||||
|
resentTotal = (long)_context.statManager().getRate("udp.packetsRetransmitted").getLifetimeEventCount();
|
||||||
double nondupSent = ((double)bytesTransmitted - ((double)resentTotal)*averagePacketSize);
|
double nondupSent = ((double)bytesTransmitted - ((double)resentTotal)*averagePacketSize);
|
||||||
double bwResent = (nondupSent <= 0 ? 0d : ((((double)resentTotal)*averagePacketSize) / nondupSent));
|
double bwResent = (nondupSent <= 0 ? 0d : ((((double)resentTotal)*averagePacketSize) / nondupSent));
|
||||||
buf.append("Percentage of bytes retransmitted (lifetime): ").append(formatPct(bwResent));
|
buf.append("Percentage of bytes retransmitted (lifetime): ").append(formatPct(bwResent));
|
||||||
|
Reference in New Issue
Block a user