2005-10-13 dust

* Bundled dust's Sucker for pulling RSS/Atom content into SML, which can
      then be injected into Syndie with the Syndie CLI.
    * Bundled ROME and JDOM (BSD and Apache licensed, respectively) for
      RSS/Atom parsing.
2005-10-13  jrandom
    * SSU retransmission choke bugfix (== != !=)
    * Include initial transmissions in the retransmission choke, so that
      if we are already retransmitting a message, we won't send anything
      to that peer other than that message (or ACKs, if necessary)
This commit is contained in:
jrandom
2005-10-14 02:15:39 +00:00
committed by zzz
parent 00a4761b5e
commit cabb607211
10 changed files with 672 additions and 5 deletions

BIN
apps/jdom/jdom.jar Normal file

Binary file not shown.

1
apps/jdom/readme.txt Normal file
View File

@ -0,0 +1 @@
This is JDOM 1.0 from http://jdom.org/, released under an Apache style license

1
apps/rome/readme.txt Normal file
View File

@ -0,0 +1 @@
This is ROME 0.7 from http://rome.dev.java.net/, released under a BSD license

BIN
apps/rome/rome-0.7.jar Normal file

Binary file not shown.

View File

@ -14,7 +14,7 @@
srcdir="./src"
debug="true" deprecation="on" source="1.3" target="1.3"
destdir="./build/obj"
classpath="../../../core/java/build/i2p.jar:../../jetty/jettylib/org.mortbay.jetty.jar:../../jetty/jettylib/javax.servlet.jar" />
classpath="../../../core/java/build/i2p.jar:../../jetty/jettylib/org.mortbay.jetty.jar:../../jetty/jettylib/javax.servlet.jar:../../jdom/jdom.jar:../../rome/rome-0.7.jar" />
</target>
<target name="jar" depends="builddep, compile">
<jar destfile="./build/syndie.jar" basedir="./build/obj" includes="**/*.class">
@ -23,6 +23,12 @@
<attribute name="Class-Path" value="i2p.jar" />
</manifest>
</jar>
<jar destfile="./build/sucker.jar" basedir="./build/obj" includes="net/i2p/syndie/Sucker.class, net/i2p/syndie/SuckerFetchListener.class">
<manifest>
<attribute name="Main-Class" value="net.i2p.syndie.Sucker" />
<attribute name="Class-Path" value="i2p.jar jdom.jar rome-0.7.jar" />
</manifest>
</jar>
<ant target="war" />
</target>
<target name="war" depends="builddep, compile, precompilejsp">
@ -48,6 +54,8 @@
<pathelement location="../../jetty/jettylib/org.mortbay.jetty.jar" />
<pathelement location="../../jetty/jettylib/javax.servlet.jar" />
<pathelement location="../../jetty/jettylib/ant.jar" />
<pathelement location="../../jdom/jdom.jar" />
<pathelement location="../../rome/rome-0.7.jar" />
<pathelement location="build/obj" />
<pathelement location="../../../core/java/build/i2p.jar" />
</classpath>
@ -69,6 +77,8 @@
<pathelement location="../../jetty/jettylib/commons-el.jar" />
<pathelement location="../../jetty/jettylib/org.mortbay.jetty.jar" />
<pathelement location="../../jetty/jettylib/javax.servlet.jar" />
<pathelement location="../../jdom/jdom.jar" />
<pathelement location="../../rome/rome-0.7.jar" />
<pathelement location="build/obj" />
<pathelement location="../../../core/java/build/i2p.jar" />
</classpath>

View File

@ -0,0 +1,635 @@
package net.i2p.syndie;
import java.io.DataInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.MalformedURLException;
import java.net.Socket;
import java.net.SocketAddress;
import java.net.URL;
import java.net.URLConnection;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.Iterator;
import java.util.List;
import java.util.ListIterator;
import sun.security.provider.SHA;
import com.sun.syndication.feed.atom.Entry;
import com.sun.syndication.feed.synd.SyndCategory;
import com.sun.syndication.feed.synd.SyndContent;
import com.sun.syndication.feed.synd.SyndEntry;
import com.sun.syndication.feed.synd.SyndFeed;
import com.sun.syndication.io.FeedException;
import com.sun.syndication.io.SyndFeedInput;
import com.sun.syndication.io.XmlReader;
import net.i2p.I2PAppContext;
import net.i2p.util.EepGet;
public class Sucker {
private String urlToLoad;
private String outputDir="./sucker_out";
private String historyPath="./sucker.history";
private String feedTag="feed";
private File historyFile;
private String proxyPort;
private String proxyHost;
private String pushScript;
private int attachmentCounter=0;
private String messagePath;
private String baseUrl;
private boolean importEnclosures=true;
private boolean importRefs=true;
private boolean pendingEndLink;
public Sucker() {}
public boolean parseArgs(String args[]) {
for (int i = 0; i < args.length; i++) {
if ("--load".equals(args[i]))
urlToLoad = args[++i];
if ("--outputdir".equals(args[i]))
outputDir = args[++i];
if ("--history".equals(args[i]))
historyPath = args[++i];
if ("--tag".equals(args[i]))
feedTag = args[++i];
if ("--proxyhost".equals(args[i]))
proxyHost = args[++i];
if ("--proxyport".equals(args[i]))
proxyPort = args[++i];
if ("--exec".equals(args[i]))
pushScript = args[++i];
if ("--importenclosures".equals(args[i]))
importEnclosures= args[++i].equals("true");
if ("--importenrefs".equals(args[i]))
importRefs= args[++i].equals("true");
}
// Cut ending '/' from outputDir
if (outputDir.endsWith("/"))
outputDir = outputDir.substring(0, outputDir.length() - 1);
if (urlToLoad == null)
return false;
// Find base url, gah HELP
int idx=urlToLoad.length();
int x=urlToLoad.indexOf('?');
if(x>0)
idx=x;
while(idx>0)
{
idx--;
if(urlToLoad.charAt(idx)=='/')
break;
}
if(idx==0)
idx=x;
baseUrl=urlToLoad.substring(0,idx);
System.out.println("BaseUrl: "+baseUrl);
return true;
}
public void suck() {
URL feedUrl;
SyndFeed feed;
int i;
//if (proxyHost != null && proxyPort != null) {
// // Set proxy
// System.setProperty("http.proxyHost", proxyHost);
// System.setProperty("http.proxyPort", proxyPort);
//}
//
try {
// Create outputDir if missing
File f = new File(outputDir);
f.mkdirs();
// Create historyFile if missing
historyFile = new File(historyPath);
if (!historyFile.exists())
historyFile.createNewFile();
int messageNumber;
File lastIdFile = new File(historyPath + ".lastId");
if (!lastIdFile.exists())
lastIdFile.createNewFile();
FileInputStream fis = new FileInputStream(lastIdFile);
String number = readLine(fis);
try {
messageNumber = Integer.parseInt(number);
} catch (NumberFormatException e) {
messageNumber = 0;
}
SyndFeedInput input = new SyndFeedInput();
boolean shouldProxy = false;
int proxyPortNum = -1;
if ( (proxyHost != null) && (proxyPort != null) ) {
try {
proxyPortNum = Integer.parseInt(proxyPort);
if (proxyPortNum > 0)
shouldProxy = true;
} catch (NumberFormatException nfe) {
nfe.printStackTrace();
}
}
int numRetries = 2;
// perhaps specify a temp dir?
File fetched = File.createTempFile("sucker", ".fetch");
fetched.deleteOnExit();
// we use eepGet, since it retries and doesn't leak DNS requests like URL does
EepGet get = new EepGet(I2PAppContext.getGlobalContext(), shouldProxy, proxyHost, proxyPortNum,
numRetries, fetched.getAbsolutePath(), urlToLoad);
SuckerFetchListener lsnr = new SuckerFetchListener();
get.addStatusListener(lsnr);
get.fetch();
boolean ok = lsnr.waitForSuccess();
if (!ok) {
System.err.println("Unable to retrieve the url after " + numRetries + " tries.");
return;
}
feed = input.build(new XmlReader(fetched));
List entries = feed.getEntries();
FileOutputStream hos = new FileOutputStream(historyFile, true);
ListIterator iter = entries.listIterator();
while (iter.hasNext()) {
attachmentCounter=0;
SyndEntry e = (SyndEntry) iter.next();
// Calculate messageId
String feedHash = sha1(urlToLoad);
String itemHash = sha1(e.getTitle() + e.getDescription());
Date d = e.getPublishedDate();
String time;
if(d!=null)
time = "" + d.getTime();
else
time = "" + new Date().getTime();
String outputFileName = outputDir + "/" + messageNumber;
/*
* $feedHash:$itemHash:$time:$outputfile. $feedHash would be the
* hash (md5? sha1? sha2?) of the $urlToFeed, $itemHash is some
* hash of the SyndEntry, $time would be the time that the entry
* was posted $outputfile would be the $outputdir/$uniqueid
*/
String messageId = feedHash + ":" + itemHash + ":" + time + ":"
+ outputFileName;
// Check if we already have this
if (!existsInHistory(messageId)) {
System.out.println("new: " + messageId);
if (convertToSml(e, ""+messageNumber)) {
hos.write(messageId.getBytes());
hos.write("\n".getBytes());
if (pushScript != null) {
if (!execPushScript(""+messageNumber, time))
System.out.println("push failed");
else
System.out.println("push success");
}
}
messageNumber++;
}
}
FileOutputStream fos = new FileOutputStream(lastIdFile);
fos.write(("" + messageNumber).getBytes());
} catch (MalformedURLException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IllegalArgumentException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (FeedException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
System.out.println("Done.");
}
public static void main(String[] args) {
Sucker sucker = new Sucker();
boolean ok = sucker.parseArgs(args);
if (!ok) {
usage();
return;
}
sucker.suck();
}
private boolean execPushScript(String id, String time) {
try {
String ls_str;
String cli = pushScript + " " + outputDir + " " + id + " " + time;
Process pushScript_proc = Runtime.getRuntime().exec(cli);
// get its output (your input) stream
DataInputStream ls_in = new DataInputStream(pushScript_proc.getInputStream());
try {
while ((ls_str = ls_in.readLine()) != null) {
System.out.println(pushScript + ": " + ls_str);
}
} catch (IOException e) {
return false;
}
try {
pushScript_proc.waitFor();
if(pushScript_proc.exitValue()==0)
return true;
} catch (InterruptedException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
return false;
} catch (IOException e1) {
System.err.println(e1);
return false;
}
}
private boolean convertToSml(SyndEntry e, String messageName) {
// Create message
File messageFile = new File(messageName);
FileOutputStream fos;
messagePath=outputDir+"/"+messageName;
try {
fos = new FileOutputStream(messagePath);
String sml;
sml = "Subject: " + e.getTitle() + "\n";
List cats = e.getCategories();
Iterator iter = cats.iterator();
String tags = feedTag;
while (iter.hasNext()) {
SyndCategory c = (SyndCategory) iter.next();
String tag=c.getName();
tag=tag.replaceAll("[^a-zA-z.-_:]","_");
tags += "\t" + feedTag + "." + tag;
}
sml += "Tags: " + tags + "\n";
sml += "\n";
SyndContent content;
List l = e.getContents();
if(l!=null)
{
System.out.println("There is content");
iter = l.iterator();
while(iter.hasNext())
{
content = (SyndContent)iter.next();
String c = content.getValue();
System.out.println("Content: "+c);
sml += htmlToSml(c);
sml += "\n";
}
}
String source=e.getUri();
if(source.indexOf("http")<0)
source=baseUrl+source;
sml += "[link schema=\"web\" location=\""+source+"\"]source[/link]\n";
fos.write(sml.getBytes());
return true;
} catch (FileNotFoundException e1) {
// TODO Auto-generated catch block
e1.printStackTrace();
} catch (IOException e2) {
// TODO Auto-generated catch block
e2.printStackTrace();
}
return false;
}
private String htmlToSml(String html) {
String sml="";
int i;
pendingEndLink=false;
for(i=0;i<html.length();)
{
if(html.charAt(i)=='<')
{
//System.out.println("html: "+html.substring(i));
int tagLen = findTagLen(html.substring(i));
//
String htmlTag = html.substring(i,i+tagLen);
//System.out.println("htmlTag: "+htmlTag);
String smlTag = htmlTagToSmlTag(htmlTag);
if(smlTag!=null)
sml+=smlTag;
i+=tagLen;
//System.out.println("tagLen: "+tagLen);
sml+=" ";
}
else
{
sml+=html.charAt(i++);
}
}
return sml;
}
private String htmlTagToSmlTag(String htmlTag) {
String ret="";
if(importEnclosures && htmlTag.startsWith("<img"))
{
System.out.println("Found image tag: "+htmlTag);
int a,b;
a=htmlTag.indexOf("src=\"")+5;
b=a+1;
while(htmlTag.charAt(b)!='\"')
b++;
String imageLink=htmlTag.substring(a,b);
if(pendingEndLink) {
ret="[/link]";
pendingEndLink=false;
}
ret += "[img attachment=\""+""+ attachmentCounter +"\"]";
a=htmlTag.indexOf("alt=\"")+5;
if(a>=5)
{
b=a+1;
while(htmlTag.charAt(b)!='\"')
b++;
String altText=htmlTag.substring(a,b);
ret+=altText;
}
ret+="[/img]";
if(imageLink.indexOf("http")<0)
imageLink=baseUrl+"/"+imageLink;
fetchAttachment(imageLink);
System.out.println("Converted to: "+ret);
return ret;
}
if(importRefs && htmlTag.startsWith("<a "))
{
System.out.println("Found link tag: "+htmlTag);
int a,b;
a=htmlTag.indexOf("href=\"")+6;
b=a+1;
while(htmlTag.charAt(b)!='\"')
b++;
String link=htmlTag.substring(a,b);
if(link.indexOf("http")<0)
link=baseUrl+"/"+link;
String schema="web";
ret += "[link schema=\""+schema+"\" location=\""+link+"\"]";
pendingEndLink=true;
System.out.println("Converted to: "+ret);
return ret;
}
if ("</a>".equals(htmlTag)) {
if (pendingEndLink)
return "[/link]";
}
if("<b>".equals(htmlTag))
return "[b]";
if("</b>".equals(htmlTag))
return "[/b]";
if("<i>".equals(htmlTag))
return "[i]";
if("</i>".equals(htmlTag))
return "[/i]";
return null;
}
private void fetchAttachment(String link) {
System.out.println("Fetch attachment from: "+link);
String attachmentPath = messagePath+"."+attachmentCounter;
try {
link=link.replaceAll("&amp;","&");
URL attachmentUrl = new URL(link);
InputStream is = attachmentUrl.openStream();
FileOutputStream fos = new FileOutputStream(attachmentPath);
while(true)
{
int i =is.read();
if(i<0)
break;
fos.write(i);
}
} catch (MalformedURLException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
attachmentCounter++;
}
private static int findTagLen(String s) {
int i;
for(i=0;i<s.length();i++)
{
if(s.charAt(i)=='>')
return i+1;
if(s.charAt(i)=='"')
{
i++;
while(s.charAt(i)!='"')
i++;
}
}
System.out.println("WTF");
return 0;
}
private boolean existsInHistory(String messageId) {
int idx;
idx = messageId.lastIndexOf(":");
String lineToCompare = messageId.substring(0, idx-1);
idx = lineToCompare.lastIndexOf(":");
lineToCompare = lineToCompare.substring(0, idx-1);
try {
FileInputStream his = new FileInputStream(historyFile);
String line;
while ((line = readLine(his)) != null) {
idx = line.lastIndexOf(":");
if (idx < 0)
return false;
line = line.substring(0, idx-1);
idx = line.lastIndexOf(":");
if (idx < 0)
return false;
line = line.substring(0, idx-1);
if (line.equals(lineToCompare))
return true;
}
} catch (FileNotFoundException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
return false;
}
private static void usage() {
System.out.println("sucker --load $urlToFeed \n"
+ "--proxyhost <host> \n"
+ "--proxyport <port> \n"
+ "--importenclosures true \n"
+ "--importrefs true \n"
+ "--tag feed \n"
+ "--outputdir ./sucker_out \n"
+ "--exec pushscript.sh OUTPUTDIR UNIQUEID ENTRYTIMESTAMP \n"
+ "--history ./sucker.history");
System.exit(1);
}
private static String sha1(String s) {
try {
MessageDigest md = MessageDigest.getInstance("SHA");
md.update(s.getBytes());
byte[] buf = md.digest();
String ret = new sun.misc.BASE64Encoder().encode(buf);
return ret;
} catch (NoSuchAlgorithmException ex) {
ex.printStackTrace();
}
return null;
}
private static String readLine(FileInputStream in) {
StringBuffer sb = new StringBuffer();
int c = 0;
while (true) {
try {
c = in.read();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
break;
}
if (c < 0)
break;
if (c == '\n')
break;
sb.append((char) c);
}
// TODO Auto-generated method stub
return sb.toString();
}
}
/**
* Simple blocking listener for eepget. block in waitForSuccess().
*/
class SuckerFetchListener implements EepGet.StatusListener {
private volatile boolean _complete;
private volatile boolean _successful;
public SuckerFetchListener() {
_complete = false;
_successful = false;
}
public void transferComplete(long alreadyTransferred, long bytesTransferred, long bytesRemaining, String url, String outputFile, boolean notModified) {
notifyComplete(true);
}
public void transferFailed(String url, long bytesTransferred, long bytesRemaining, int currentAttempt) {
notifyComplete(false);
}
private void notifyComplete(boolean ok) {
synchronized (this) {
_complete = true;
_successful = ok;
notifyAll();
}
}
/**
* Block until the fetch is successful, returning true if it did fetch completely,
* false if it didn't.
*
*/
public boolean waitForSuccess() {
while (!_complete) {
try {
synchronized (this) {
wait();
}
} catch (InterruptedException ie) {}
}
return _successful;
}
public void attemptFailed(String url, long bytesTransferred, long bytesRemaining, int currentAttempt, int numRetries, Exception cause) {
// noop, it may retry
}
public void bytesTransferred(long alreadyTransferred, int currentWrite, long bytesTransferred, long bytesRemaining, String url) {
// ignore this status update
}
public void headerReceived(String url, int currentAttempt, String key, String val) {
// ignore
}
}

View File

@ -29,7 +29,7 @@
<ant dir="apps/addressbook/" target="war" />
<ant dir="apps/susimail/" target="war" />
<ant dir="apps/susidns/src" target="all" />
<ant dir="apps/syndie/java/" target="jar" /> <!-- not pushed in the update... yet -->
<ant dir="apps/syndie/java/" target="jar" />
</target>
<target name="buildWEB">
<ant dir="apps/jetty" target="fetchJettylib" />
@ -89,7 +89,10 @@
<copy file="apps/susidns/src/susidns.war" todir="build/" />
<copy file="apps/syndie/syndie.war" todir="build/" />
<copy file="apps/syndie/java/build/syndie.jar" todir="build/" />
<copy file="apps/syndie/java/build/sucker.jar" todir="build/" />
<copy file="apps/syndie/syndie.war" todir="build/" />
<copy file="apps/jdom/jdom.jar" todir="build/" />
<copy file="apps/rome/rome-0.7.jar" todir="build/" />
</target>
<target name="javadoc">
<mkdir dir="./build" />
@ -211,6 +214,9 @@
<copy file="build/routerconsole.jar" todir="pkg-temp/lib/" />
<copy file="build/sam.jar" todir="pkg-temp/lib/" />
<copy file="build/systray.jar" todir="pkg-temp/lib" />
<copy file="build/jdom.jar" todir="pkg-temp/lib" />
<copy file="build/rome-0.7.jar" todir="pkg-temp/lib" />
<copy file="build/sucker.jar" todir="pkg-temp/lib" />
<copy file="i2p.exe" todir="pkg-temp/" failonerror="false" />
<copy file="installer/resources/runplain.sh" todir="pkg-temp/" />
<copy file="apps/systray/java/lib/systray4j.jar" todir="pkg-temp/lib" />
@ -310,6 +316,12 @@
<copy file="build/sam.jar" todir="pkg-temp/lib/" />
<copy file="build/router.jar" todir="pkg-temp/lib/" />
<copy file="build/routerconsole.jar" todir="pkg-temp/lib/" />
<!-- for the 0.6.1.3 release, push JDOM and ROME. it can be
removed later updates (once people on earlier builds upgrade -->
<copy file="build/jdom.jar" todir="pkg-temp/lib" />
<copy file="build/rome-0.7.jar" todir="pkg-temp/lib" />
<copy file="build/sucker.jar" todir="pkg-temp/lib" />
<copy file="i2p.exe" todir="pkg-temp/" failonerror="false" />
<copy file="installer/resources/runplain.sh" todir="pkg-temp/" />

View File

@ -1,4 +1,10 @@
$Id: history.txt,v 1.292 2005/10/12 22:54:54 jrandom Exp $
$Id: history.txt,v 1.293 2005/10/13 04:18:37 jrandom Exp $
2005-10-13 dust
* Bundled dust's Sucker for pulling RSS/Atom content into SML, which can
then be injected into Syndie with the Syndie CLI.
* Bundled ROME and JDOM (BSD and Apache licensed, respectively) for
RSS/Atom parsing.
2005-10-13 jrandom
* SSU retransmission choke bugfix (== != !=)

View File

@ -15,9 +15,9 @@ import net.i2p.CoreVersion;
*
*/
public class RouterVersion {
public final static String ID = "$Revision: 1.267 $ $Date: 2005/10/12 22:54:54 $";
public final static String ID = "$Revision: 1.268 $ $Date: 2005/10/13 04:18:36 $";
public final static String VERSION = "0.6.1.2";
public final static long BUILD = 8;
public final static long BUILD = 9;
public static void main(String args[]) {
System.out.println("I2P Router version: " + VERSION + "-" + BUILD);
System.out.println("Router ID: " + RouterVersion.ID);

View File

@ -1111,6 +1111,8 @@ public class UDPTransport extends TransportImpl implements TimedWeightedPriority
buf.append("<tr><td colspan=\"14\" valign=\"top\" align=\"left\">");
long bytesTransmitted = _context.bandwidthLimiter().getTotalAllocatedOutboundBytes();
double averagePacketSize = _context.statManager().getRate("udp.sendPacketSize").getLifetimeAverageValue();
// lifetime value, not just the retransmitted packets of current connections
resentTotal = (long)_context.statManager().getRate("udp.packetsRetransmitted").getLifetimeEventCount();
double nondupSent = ((double)bytesTransmitted - ((double)resentTotal)*averagePacketSize);
double bwResent = (nondupSent <= 0 ? 0d : ((((double)resentTotal)*averagePacketSize) / nondupSent));
buf.append("Percentage of bytes retransmitted (lifetime): ").append(formatPct(bwResent));