2006-02-16 jrandom

* Bugfix to the I2PTunnel web config to properly accept i2cp port settings
    * Initial sucker refactoring to simplify reuse of the html parsing
    * Beginnings of hooks to push imported rss/atom out to remote syndie
      archives automatically (though not enabled currently)
    * Further SSU peer test cleanup
This commit is contained in:
jrandom
2006-02-16 08:24:07 +00:00
committed by zzz
parent 3d76df6af3
commit 79f934fe17
15 changed files with 245 additions and 193 deletions

View File

@ -448,9 +448,10 @@ public class IndexBean {
/** I2CP host the router is on */
public void setClientHost(String host) {
_i2cpHost = (host != null ? host.trim() : null);
System.out.println("set client host [" + host + "]");
}
/** I2CP port the router is on */
public void setClientPort(String port) {
public void setClientport(String port) {
_i2cpPort = (port != null ? port.trim() : null);
}
/** how many hops to use for inbound tunnels */
@ -636,10 +637,11 @@ public class IndexBean {
config.setProperty("description", _description);
if (_i2cpHost != null)
config.setProperty("i2cpHost", _i2cpHost);
if ( (_i2cpPort != null) && (_i2cpPort.trim().length() > 0) )
if ( (_i2cpPort != null) && (_i2cpPort.trim().length() > 0) ) {
config.setProperty("i2cpPort", _i2cpPort);
else
} else {
config.setProperty("i2cpPort", "7654");
}
if (_customOptions != null) {
StringTokenizer tok = new StringTokenizer(_customOptions);

View File

@ -254,7 +254,7 @@
<label for="clientPort" accesskey="r">
Po<span class="accessKey">r</span>t:
</label>
<input type="text" id="clientPort" name="clientPort" size="20" title="I2CP Port Number" value="<%=editBean.getI2CPPort(curTunnel)%>" class="freetext" />
<input type="text" id="clientPort" name="clientport" size="20" title="I2CP Port Number" value="<%=editBean.getI2CPPort(curTunnel)%>" class="freetext" />
</div>
<div class="subdivider">

View File

@ -226,7 +226,7 @@
<label for="clientPort" accesskey="r">
Po<span class="accessKey">r</span>t:
</label>
<input type="text" id="clientPort" name="clientPort" size="20" title="I2CP Port Number" value="<%=editBean.getI2CPPort(curTunnel)%>" class="freetext" />
<input type="text" id="clientPort" name="clientport" size="20" title="I2CP Port Number" value="<%=editBean.getI2CPPort(curTunnel)%>" class="freetext" />
</div>
<div class="subdivider">

View File

@ -33,97 +33,78 @@ import net.i2p.syndie.data.BlogURI;
import net.i2p.util.EepGet;
import net.i2p.util.Log;
/**
*
* todo:
* - factor out the parsing / formatting / posting to let the sucker pull in arbitrary HTML pages
* (importing the images and SMLizing some stuff)
* - push the posts out to a remote syndie instance too
*/
public class Sucker {
private static final Log _log = I2PAppContext.getGlobalContext().logManager().getLog(Sucker.class);
private String urlToLoad;
private String outputDir="./sucker_out";
private String historyPath="./sucker.history";
private String feedTag="feed";
private File historyFile;
private String proxyPort;
private String proxyHost;
private String pushScript;
private int attachmentCounter=0;
private String messagePath;
private String baseUrl;
private boolean importEnclosures=true;
private boolean importRefs=true;
private boolean pendingEndLink;
private boolean shouldProxy;
private int proxyPortNum;
private String blog;
private boolean pushToSyndie;
private long messageNumber=0;
private BlogManager bm;
private User user;
private SuckerState _state;
//
private List fileNames;
private List fileStreams;
private List fileTypes;
private List tempFiles; // deleted after finished push
private boolean stripNewlines;
public Sucker() {
}
public Sucker() {}
/**
* Constructor for BlogManager.
*/
public Sucker(String[] strings) throws IllegalArgumentException {
pushToSyndie=true;
urlToLoad = strings[0];
blog = strings[1];
feedTag = strings[2];
outputDir = "blog-"+blog;
SuckerState state = new SuckerState();
state.pushToSyndie=true;
state.urlToLoad = strings[0];
state.blog = strings[1];
state.feedTag = strings[2];
state.outputDir = "blog-"+state.blog;
try {
historyPath=BlogManager.instance().getRootDir().getCanonicalPath()+"/rss.history";
state.historyPath=BlogManager.instance().getRootDir().getCanonicalPath()+"/rss.history";
} catch (IOException e) {
e.printStackTrace();
}
proxyPort = BlogManager.instance().getDefaultProxyPort();
proxyHost = BlogManager.instance().getDefaultProxyHost();
state.proxyPort = BlogManager.instance().getDefaultProxyPort();
state.proxyHost = BlogManager.instance().getDefaultProxyHost();
bm = BlogManager.instance();
state.bm = BlogManager.instance();
Hash blogHash = new Hash();
try {
blogHash.fromBase64(blog);
blogHash.fromBase64(state.blog);
} catch (DataFormatException e1) {
throw new IllegalArgumentException("ooh, bad $blog");
}
user = bm.getUser(blogHash);
if(user==null)
state.user = state.bm.getUser(blogHash);
if(state.user==null)
throw new IllegalArgumentException("wtf, user==null? hash:"+blogHash);
_state = state;
}
public boolean parseArgs(String args[]) {
for (int i = 0; i < args.length; i++) {
if ("--load".equals(args[i]))
urlToLoad = args[++i];
_state.urlToLoad = args[++i];
if ("--outputdir".equals(args[i]))
outputDir = args[++i];
_state.outputDir = args[++i];
if ("--history".equals(args[i]))
historyPath = args[++i];
_state.historyPath = args[++i];
if ("--tag".equals(args[i]))
feedTag = args[++i];
_state.feedTag = args[++i];
if ("--proxyhost".equals(args[i]))
proxyHost = args[++i];
_state.proxyHost = args[++i];
if ("--proxyport".equals(args[i]))
proxyPort = args[++i];
_state.proxyPort = args[++i];
if ("--exec".equals(args[i]))
pushScript = args[++i];
_state.pushScript = args[++i];
if ("--importenclosures".equals(args[i]))
importEnclosures= args[++i].equals("true");
_state.importEnclosures= args[++i].equals("true");
if ("--importenrefs".equals(args[i]))
importRefs= args[++i].equals("true");
_state.importRefs= args[++i].equals("true");
}
// Cut ending '/' from outputDir
if (outputDir.endsWith("/"))
outputDir = outputDir.substring(0, outputDir.length() - 1);
if (_state.outputDir.endsWith("/"))
_state.outputDir = _state.outputDir.substring(0, _state.outputDir.length() - 1);
if (urlToLoad == null)
if (_state.urlToLoad == null)
return false;
return true;
@ -131,31 +112,33 @@ public class Sucker {
/**
* Fetch urlToLoad and call convertToHtml() on any new entries.
* @return list of BlogURI entries posted, if any
*/
public void suck() {
public List suck() {
_state.entriesPosted = new ArrayList();
SyndFeed feed;
File fetched=null;
tempFiles = new ArrayList();
_state.tempFiles = new ArrayList();
// Find base url
int idx=urlToLoad.lastIndexOf('/');
int idx=_state.urlToLoad.lastIndexOf('/');
if(idx>0)
baseUrl=urlToLoad.substring(0,idx);
_state.baseUrl=_state.urlToLoad.substring(0,idx);
else
baseUrl=urlToLoad;
_state.baseUrl=_state.urlToLoad;
infoLog("Processing: "+urlToLoad);
debugLog("Base url: "+baseUrl);
infoLog("Processing: "+_state.urlToLoad);
debugLog("Base url: "+_state.baseUrl);
//
try {
File lastIdFile=null;
// Get next message number to use (for messageId in history only)
if(!pushToSyndie) {
if(!_state.pushToSyndie) {
lastIdFile = new File(historyPath + ".lastId");
lastIdFile = new File(_state.historyPath + ".lastId");
if (!lastIdFile.exists())
lastIdFile.createNewFile();
@ -163,34 +146,34 @@ public class Sucker {
try {
fis = new FileInputStream(lastIdFile);
String number = readLine(fis);
messageNumber = Integer.parseInt(number);
_state.messageNumber = Integer.parseInt(number);
} catch (NumberFormatException e) {
messageNumber = 0;
_state.messageNumber = 0;
} finally {
if (fis != null) try { fis.close(); } catch (IOException ioe) {}
}
// Create outputDir if missing
File f = new File(outputDir);
File f = new File(_state.outputDir);
f.mkdirs();
} else {
messageNumber=bm.getNextBlogEntry(user);
_state.messageNumber=_state.bm.getNextBlogEntry(_state.user);
}
_log.debug("message number: " + messageNumber);
_log.debug("message number: " + _state.messageNumber);
// Create historyFile if missing
historyFile = new File(historyPath);
if (!historyFile.exists())
historyFile.createNewFile();
_state.historyFile = new File(_state.historyPath);
if (!_state.historyFile.exists())
_state.historyFile.createNewFile();
shouldProxy = false;
proxyPortNum = -1;
if ( (proxyHost != null) && (proxyPort != null) ) {
_state.shouldProxy = false;
_state.proxyPortNum = -1;
if ( (_state.proxyHost != null) && (_state.proxyPort != null) ) {
try {
proxyPortNum = Integer.parseInt(proxyPort);
if (proxyPortNum > 0)
shouldProxy = true;
_state.proxyPortNum = Integer.parseInt(_state.proxyPort);
if (_state.proxyPortNum > 0)
_state.shouldProxy = true;
} catch (NumberFormatException nfe) {
nfe.printStackTrace();
}
@ -199,12 +182,12 @@ public class Sucker {
// fetch
int numRetries = 2;
fetched = File.createTempFile("sucker", ".fetch");
EepGet get = new EepGet(I2PAppContext.getGlobalContext(), shouldProxy, proxyHost, proxyPortNum,
numRetries, fetched.getAbsolutePath(), urlToLoad);
EepGet get = new EepGet(I2PAppContext.getGlobalContext(), _state.shouldProxy, _state.proxyHost, _state.proxyPortNum,
numRetries, fetched.getAbsolutePath(), _state.urlToLoad);
SuckerFetchListener lsnr = new SuckerFetchListener();
get.addStatusListener(lsnr);
_log.debug("fetching [" + urlToLoad + "] / " + shouldProxy + "/" + proxyHost + "/" + proxyHost);
_log.debug("fetching [" + _state.urlToLoad + "] / " + _state.shouldProxy + "/" + _state.proxyHost + "/" + _state.proxyHost);
get.fetch();
_log.debug("fetched: " + get.getNotModified() + "/" + get.getETag());
@ -213,13 +196,13 @@ public class Sucker {
_log.debug("success? " + ok);
System.err.println("Unable to retrieve the url after " + numRetries + " tries.");
fetched.delete();
return;
return _state.entriesPosted;
}
_log.debug("fetched successfully? " + ok);
if(get.getNotModified()) {
debugLog("not modified, saving network bytes from useless fetch");
fetched.delete();
return;
return _state.entriesPosted;
}
// Build entry list from fetched rss file
@ -233,19 +216,19 @@ public class Sucker {
FileOutputStream hos = null;
try {
hos = new FileOutputStream(historyFile, true);
hos = new FileOutputStream(_state.historyFile, true);
// Process list backwards to get syndie to display the
// entries in the right order. (most recent at top)
for (int i = entries.size()-1; i >= 0; i--) {
SyndEntry e = (SyndEntry) entries.get(i);
attachmentCounter=0;
_state.attachmentCounter=0;
if (_log.shouldLog(Log.DEBUG))
_log.debug("Syndicate entry: " + e.getLink());
String messageId = convertToSml(e);
String messageId = convertToSml(_state, e);
if (messageId!=null) {
hos.write(messageId.getBytes());
hos.write("\n".getBytes());
@ -255,11 +238,11 @@ public class Sucker {
if (hos != null) try { hos.close(); } catch (IOException ioe) {}
}
if(!pushToSyndie) {
if(!_state.pushToSyndie) {
FileOutputStream fos = null;
try {
fos = new FileOutputStream(lastIdFile);
fos.write(("" + messageNumber).getBytes());
fos.write(("" + _state.messageNumber).getBytes());
} finally {
if (fos != null) try { fos.close(); } catch (IOException ioe) {}
}
@ -278,6 +261,7 @@ public class Sucker {
if(fetched!=null)
fetched.delete();
debugLog("Done.");
return _state.entriesPosted;
}
public static void main(String[] args) {
@ -302,11 +286,11 @@ public class Sucker {
/**
* Call the specified script with "$outputDir $id and $time".
*/
private boolean execPushScript(String id, String time) {
private static boolean execPushScript(SuckerState state, String id, String time) {
try {
String ls_str;
String cli = pushScript + " " + outputDir + " " + id + " " + time;
String cli = state.pushScript + " " + state.outputDir + " " + id + " " + time;
Process pushScript_proc = Runtime.getRuntime().exec(cli);
// get its output (your input) stream
@ -318,7 +302,7 @@ public class Sucker {
while (true) {
boolean eof = DataHelper.readLine(ls_in, buf);
if (buf.length() > 0)
infoLog(pushScript + ": " + buf.toString());
infoLog(state.pushScript + ": " + buf.toString());
buf.setLength(0);
if (eof)
break;
@ -343,13 +327,13 @@ public class Sucker {
/**
* Converts the SyndEntry e to sml and fetches any images as attachments
*/
private String convertToSml(SyndEntry e) {
private static String convertToSml(SuckerState state, SyndEntry e) {
String subject;
stripNewlines=false;
state.stripNewlines=false;
// Calculate messageId, and check if we have got the message already
String feedHash = sha1(urlToLoad);
String feedHash = sha1(state.urlToLoad);
String itemHash = sha1(e.getTitle() + e.getDescription());
Date d = e.getPublishedDate();
String time;
@ -357,10 +341,10 @@ public class Sucker {
time = "" + d.getTime();
else
time = "" + new Date().getTime();
String outputFileName = outputDir + "/" + messageNumber;
String outputFileName = state.outputDir + "/" + state.messageNumber;
String messageId = feedHash + ":" + itemHash + ":" + time + ":" + outputFileName;
// Check if we already have this
if (existsInHistory(messageId))
if (existsInHistory(state, messageId))
return null;
infoLog("new: " + messageId);
@ -371,14 +355,14 @@ public class Sucker {
subject=e.getTitle();
List cats = e.getCategories();
Iterator iter = cats.iterator();
String tags = feedTag;
String tags = state.feedTag;
while (iter.hasNext()) {
SyndCategory c = (SyndCategory) iter.next();
debugLog("Name: "+c.getName());
debugLog("uri:"+c.getTaxonomyUri());
String tag=c.getName();
tag=tag.replaceAll("[^a-zA-z.-_:]","_");
tags += "\t" + feedTag + "." + tag;
tags += "\t" + state.feedTag + "." + tag;
}
SyndContent content;
@ -393,7 +377,7 @@ public class Sucker {
content = (SyndContent)iter.next();
String c = content.getValue();
debugLog("Content: "+c);
sml += htmlToSml(c);
sml += htmlToSml(state, c);
sml += "\n";
}
}
@ -408,85 +392,86 @@ public class Sucker {
// e.g. postman's rss feed @ http://tracker.postman.i2p/rss.jsp has
// baseUrl = http://tracker.postman.i2p
// and enclosure URLs are /download.php?id=123&file=blah
if (enclosureURL.startsWith("/") || baseUrl.endsWith("/"))
enclosureURL = baseUrl + enclosureURL;
if (enclosureURL.startsWith("/") || state.baseUrl.endsWith("/"))
enclosureURL = state.baseUrl + enclosureURL;
else
enclosureURL = baseUrl + '/' + enclosureURL;
enclosureURL = state.baseUrl + '/' + enclosureURL;
}
fetchAttachment(enclosureURL, enc.getType()); // fetches and adds to our streams
fetchAttachment(state, enclosureURL, enc.getType()); // fetches and adds to our streams
}
}
String source=e.getLink(); //Uri();
if(source.indexOf("http")<0)
source=baseUrl+source;
source=state.baseUrl+source;
sml += "[link schema=\"web\" location=\""+source+"\"]source[/link]\n";
if(pushToSyndie) {
debugLog("user.blog: "+user.getBlogStr());
debugLog("user.id: "+bm.getNextBlogEntry(user));
if(state.pushToSyndie) {
debugLog("user.blog: "+state.user.getBlogStr());
debugLog("user.id: "+state.bm.getNextBlogEntry(state.user));
debugLog("subject: "+subject);
debugLog("tags: "+tags);
debugLog("sml: "+sml);
debugLog("");
BlogURI uri = bm.createBlogEntry(
user,
BlogURI uri = state.bm.createBlogEntry(
state.user,
false,
subject,
tags,
null,
sml,
fileNames,
fileStreams,
fileTypes);
state.fileNames,
state.fileStreams,
state.fileTypes);
if(uri==null) {
errorLog("pushToSyndie failure.");
return null;
}
else
} else {
state.entriesPosted.add(uri);
infoLog("pushToSyndie success, uri: "+uri.toString());
}
}
else
{
FileOutputStream fos;
fos = new FileOutputStream(messagePath);
fos = new FileOutputStream(state.messagePath);
sml=subject + "\nTags: " + tags + "\n\n" + sml;
fos.write(sml.getBytes());
if (pushScript != null) {
if (!execPushScript(""+messageNumber, time)) {
if (state.pushScript != null) {
if (!execPushScript(state, ""+state.messageNumber, time)) {
errorLog("push script failed");
} else {
infoLog("push script success: nr "+messageNumber);
infoLog("push script success: nr "+state.messageNumber);
}
}
}
messageNumber++;
deleteTempFiles();
state.messageNumber++;
deleteTempFiles(state);
return messageId;
} catch (FileNotFoundException e1) {
e1.printStackTrace();
} catch (IOException e2) {
e2.printStackTrace();
}
deleteTempFiles();
deleteTempFiles(state);
return null;
}
private void deleteTempFiles() {
Iterator iter = tempFiles.iterator();
private static void deleteTempFiles(SuckerState state) {
Iterator iter = state.tempFiles.iterator();
while(iter.hasNext()) {
File tempFile = (File)iter.next();
tempFile.delete();
}
}
private String htmlToSml(String html) {
private static String htmlToSml(SuckerState state, String html) {
String sml="";
int i=0;
pendingEndLink=false;
state.pendingEndLink=false;
while(i<html.length())
{
@ -506,7 +491,7 @@ public class Sucker {
//log("htmlTag: "+htmlTag);
String smlTag = htmlTagToSmlTag(htmlTag);
String smlTag = htmlTagToSmlTag(state, htmlTag);
if(smlTag!=null) {
sml+=smlTag;
i+=tagLen;
@ -517,11 +502,11 @@ public class Sucker {
sml+="&lt;";
break;
case '\r':
if(!stripNewlines)
if(!state.stripNewlines)
sml+='\r';
break;
case '\n':
if(!stripNewlines)
if(!state.stripNewlines)
sml+='\n';
break;
case '[':
@ -540,7 +525,7 @@ public class Sucker {
return sml;
}
private String htmlTagToSmlTag(String htmlTag) {
private static String htmlTagToSmlTag(SuckerState state, String htmlTag) {
final String ignoreTags[] = {
"span",
"tr",
@ -564,12 +549,12 @@ public class Sucker {
b++;
String imageLink=htmlTag.substring(a,b);
if(pendingEndLink) { // <a href="..."><img src="..."></a> -> [link][/link][img][/img]
if(state.pendingEndLink) { // <a href="..."><img src="..."></a> -> [link][/link][img][/img]
ret="[/link]";
pendingEndLink=false;
state.pendingEndLink=false;
}
ret += "[img attachment=\""+""+ attachmentCounter +"\"]";
ret += "[img attachment=\""+""+ state.attachmentCounter +"\"]";
a=htmlTagLowerCase.indexOf("alt=\"")+5;
if(a>=5)
@ -586,9 +571,9 @@ public class Sucker {
ret+="[/img]";
if(imageLink.indexOf("http")<0)
imageLink=baseUrl+"/"+imageLink;
imageLink=state.baseUrl+"/"+imageLink;
fetchAttachment(imageLink);
fetchAttachment(state, imageLink);
debugLog("Converted to: "+ret);
@ -608,7 +593,7 @@ public class Sucker {
return null; // abort the b0rked tag
String link=htmlTag.substring(a,b);
if(link.indexOf("http")<0)
link=baseUrl+"/"+link;
link=state.baseUrl+"/"+link;
String schema="web";
@ -616,7 +601,7 @@ public class Sucker {
if(htmlTagLowerCase.endsWith("/>"))
ret += "[/link]";
else
pendingEndLink=true;
state.pendingEndLink=true;
debugLog("Converted to: "+ret);
@ -624,8 +609,8 @@ public class Sucker {
}
if ("</a>".equals(htmlTagLowerCase)) {
if (pendingEndLink) {
pendingEndLink=false;
if (state.pendingEndLink) {
state.pendingEndLink=false;
return "[/link]";
}
}
@ -647,7 +632,7 @@ public class Sucker {
if("</strong>".equals(htmlTagLowerCase))
return "[/b]";
if(htmlTagLowerCase.startsWith("<br")) {
stripNewlines=true;
state.stripNewlines=true;
return "\n";
}
if("<p>".equals(htmlTagLowerCase))
@ -675,15 +660,15 @@ public class Sucker {
return null;
}
private void fetchAttachment(String link) { fetchAttachment(link, null); }
private void fetchAttachment(String link, String suggestedMimeType) {
private static void fetchAttachment(SuckerState state, String link) { fetchAttachment(state, link, null); }
private static void fetchAttachment(SuckerState state, String link, String suggestedMimeType) {
link=link.replaceAll("&amp;","&");
infoLog("Fetch attachment from: "+link);
File fetched;
if(pushToSyndie) {
if(state.pushToSyndie) {
try {
// perhaps specify a temp dir?
fetched = File.createTempFile("sucker",".attachment");
@ -692,14 +677,14 @@ public class Sucker {
e.printStackTrace();
return;
}
tempFiles.add(fetched);
state.tempFiles.add(fetched);
} else {
String attachmentPath = messagePath+"."+attachmentCounter;
String attachmentPath = state.messagePath+"."+state.attachmentCounter;
fetched = new File(attachmentPath);
}
int numRetries = 2;
// we use eepGet, since it retries and doesn't leak DNS requests like URL does
EepGet get = new EepGet(I2PAppContext.getGlobalContext(), shouldProxy, proxyHost, proxyPortNum,
EepGet get = new EepGet(I2PAppContext.getGlobalContext(), state.shouldProxy, state.proxyHost, state.proxyPortNum,
numRetries, fetched.getAbsolutePath(), link);
SuckerFetchListener lsnr = new SuckerFetchListener();
get.addStatusListener(lsnr);
@ -710,7 +695,7 @@ public class Sucker {
fetched.delete();
return;
}
tempFiles.add(fetched);
state.tempFiles.add(fetched);
String filename=EepGet.suggestName(link);
String contentType = suggestedMimeType;
if (contentType == null)
@ -719,38 +704,38 @@ public class Sucker {
contentType="text/plain";
debugLog("successful fetch of filename "+filename + " suggested mime type [" + suggestedMimeType
+ "], fetched mime type [" + get.getContentType() + "], final type [" + contentType + "]");
if(fileNames==null) fileNames = new ArrayList();
if(fileTypes==null) fileTypes = new ArrayList();
if(fileStreams==null) fileStreams = new ArrayList();
fileNames.add(filename);
fileTypes.add(contentType);
if(state.fileNames==null) state.fileNames = new ArrayList();
if(state.fileTypes==null) state.fileTypes = new ArrayList();
if(state.fileStreams==null) state.fileStreams = new ArrayList();
state.fileNames.add(filename);
state.fileTypes.add(contentType);
try {
fileStreams.add(new FileInputStream(fetched));
state.fileStreams.add(new FileInputStream(fetched));
} catch (FileNotFoundException e) {
e.printStackTrace();
}
attachmentCounter++;
state.attachmentCounter++;
}
private void errorLog(String string) {
private static void errorLog(String string) {
if (_log.shouldLog(Log.ERROR))
_log.error(string);
if(!pushToSyndie)
System.out.println(string);
//if(!pushToSyndie)
// System.out.println(string);
}
private void infoLog(String string) {
private static void infoLog(String string) {
if (_log.shouldLog(Log.INFO))
_log.info(string);
if(!pushToSyndie)
System.out.println(string);
//if(!pushToSyndie)
// System.out.println(string);
}
private void debugLog(String string) {
private static void debugLog(String string) {
if (_log.shouldLog(Log.DEBUG))
_log.debug(string);
if(!pushToSyndie)
System.out.println(string);
//if(!pushToSyndie)
// System.out.println(string);
}
private static int findTagLen(String s) {
@ -769,7 +754,7 @@ public class Sucker {
return -1;
}
private boolean existsInHistory(String messageId) {
private static boolean existsInHistory(SuckerState state, String messageId) {
int idx;
idx = messageId.lastIndexOf(":");
String lineToCompare = messageId.substring(0, idx-1);
@ -777,7 +762,7 @@ public class Sucker {
lineToCompare = lineToCompare.substring(0, idx-1);
FileInputStream his = null;
try {
his = new FileInputStream(historyFile);
his = new FileInputStream(state.historyFile);
String line;
while ((line = readLine(his)) != null) {
idx = line.lastIndexOf(":");
@ -885,5 +870,39 @@ class SuckerFetchListener implements EepGet.StatusListener {
public void headerReceived(String url, int currentAttempt, String key, String val) {
// ignore
}
}
class SuckerState {
String urlToLoad;
String outputDir="./sucker_out";
String historyPath="./sucker.history";
String feedTag="feed";
File historyFile;
String proxyPort;
String proxyHost;
String pushScript;
int attachmentCounter=0;
String messagePath;
String baseUrl;
boolean importEnclosures=true;
boolean importRefs=true;
boolean pendingEndLink;
boolean shouldProxy;
int proxyPortNum;
String blog;
boolean pushToSyndie;
long messageNumber=0;
BlogManager bm;
User user;
List entriesPosted;
//
List fileNames;
List fileStreams;
List fileTypes;
List tempFiles; // deleted after finished push
boolean stripNewlines;
}

View File

@ -583,6 +583,12 @@ public class RemoteArchiveBean {
List uris = new ArrayList(entries.length);
for (int i = 0; i < entries.length; i++)
uris.add(new BlogURI(entries[i]));
postSelectedEntries(user, uris, proxyHost, proxyPort, location);
}
public void postSelectedEntries(User user, List uris, String location) {
postSelectedEntries(user, uris, _proxyHost, _proxyPort, location);
}
public void postSelectedEntries(User user, List uris, String proxyHost, int proxyPort, String location) {
if ( (proxyPort > 0) && (proxyHost != null) && (proxyHost.trim().length() > 0) ) {
_proxyPort = proxyPort;
_proxyHost = proxyHost;

View File

@ -11,12 +11,14 @@ request.setCharacterEncoding("UTF-8");
<ul>
<li>newlines are newlines are newlines.</li>
<li>all &lt; and &gt; are replaced with their &amp;symbol;</li>
<li>the [ and ] characters delimit tags, or must be quoted by doubling them up ([[ displays as [, ]] displays as ])</li>
<li>[b][/b] = &lt;b&gt;bold&lt;/b&gt;</li>
<li>[i][/i] = &lt;i&gt;italics&lt;/i&gt;</li>
<li>[u][/u] = &lt;i&gt;underline&lt;/i&gt;</li>
<li>[pre]foo[/pre] = &lt;pre&gt;preformatted section&lt;/pre&gt;</li>
<li>[cut]more inside[/cut] = &lt;a href="#"&gt;more inside...&lt;/a&gt;</li>
<li>[quote][/quote] = Quoted text</li>
<li>[img attachment="1"]alt[/img] = use attachment 1 as an image with 'alt' as the alt text</li>
<li>[img attachment="1"]alt[/img] = use attachment 1 as an image with 'alt' as the alt text.</li>
<li>[attachment id="0"]text[/attachment] = offer attachment 0 as a link in your post</li>
<li>[attachment thumbnail="0" id="1"]text[/attachment] = offer attachment 1 as a link around a thumbnail image using attachment 0</li>
<li>[link schema="eep" location="http://forum.i2p"]text[/link] = offer a link to an external resource (accessible with the given schema)</li>
@ -25,6 +27,7 @@ request.setCharacterEncoding("UTF-8");
<li>[blog name="name" bloghash="base64hash" blogtag="tag"]description[/blog] = link to all posts in the blog with the specified tag</li>
<li>[blog name="name" blogtag="tag"]description[/blog] = link to all posts in all blogs with the specified tag</li>
<li>[archive name="name" description="they have good stuff" schema="eep" location="http://syndiemedia.i2p/archive/archive.txt"]foo![/archive] = offer an easy way to sync up with a new Syndie archive</li>
<li>[address name="www.i2p" location="Nf3ab-ZFkmI-LyMt7Gjg...vobM57UpqSAAAA" schema="i2p" proto="eep"]official website[/address] = share a pet name reference to the given eepsite (using fields from the addresses page)</li>
</ul>
SML headers are newline delimited key:value pairs. Example keys are:
<ul>

View File

@ -148,10 +148,18 @@ public class EepGet {
}
public static String suggestName(String url) {
int last = url.lastIndexOf('/');
if ((last < 0) || (url.lastIndexOf('#') > last))
last = url.lastIndexOf('#');
if ((last < 0) || (url.lastIndexOf('?') > last))
last = url.lastIndexOf('?');
if ((last < 0) || (url.lastIndexOf('=') > last))
last = url.lastIndexOf('=');
String name = null;
if (url.lastIndexOf('/') >= 0)
name = sanitize(url.substring(url.lastIndexOf('/')+1));
if (name != null)
if (last >= 0)
name = sanitize(url.substring(last+1));
if ( (name != null) && (name.length() > 0) )
return name;
else
return sanitize(url);

View File

@ -1,4 +1,11 @@
$Id: history.txt,v 1.397 2006/02/15 00:33:17 jrandom Exp $
$Id: history.txt,v 1.398 2006/02/15 08:36:29 jrandom Exp $
2006-02-16 jrandom
* Bugfix to the I2PTunnel web config to properly accept i2cp port settings
* Initial sucker refactoring to simplify reuse of the html parsing
* Beginnings of hooks to push imported rss/atom out to remote syndie
archives automatically (though not enabled currently)
* Further SSU peer test cleanup
2006-02-15 jrandom
* Add in per-blog RSS feeds to Syndie

View File

@ -15,9 +15,9 @@ import net.i2p.CoreVersion;
*
*/
public class RouterVersion {
public final static String ID = "$Revision: 1.341 $ $Date: 2006/02/15 00:33:33 $";
public final static String ID = "$Revision: 1.342 $ $Date: 2006/02/15 08:36:32 $";
public final static String VERSION = "0.6.1.9";
public final static long BUILD = 26;
public final static long BUILD = 27;
public static void main(String args[]) {
System.out.println("I2P Router version: " + VERSION + "-" + BUILD);
System.out.println("Router ID: " + RouterVersion.ID);

View File

@ -27,6 +27,7 @@ public class IntroductionManager {
_outbound = Collections.synchronizedMap(new HashMap(128));
_inbound = new ArrayList(128);
ctx.statManager().createRateStat("udp.receiveRelayIntro", "How often we get a relayed request for us to talk to someone?", "udp", new long[] { 60*1000, 5*60*1000, 10*60*1000 });
ctx.statManager().createRateStat("udp.receiveRelayRequest", "How often we receive a request to relay to someone else?", "udp", new long[] { 60*1000, 5*60*1000, 10*60*1000 });
}
public void reset() {
@ -91,6 +92,7 @@ public class IntroductionManager {
}
public void receiveRelayRequest(RemoteHostId alice, UDPPacketReader reader) {
_context.statManager().addRateData("udp.receiveRelayRequest", 1, 0);
if (_context.router().isHidden())
return;
long tag = reader.getRelayRequestReader().readTag();

View File

@ -47,6 +47,8 @@ class PeerTestManager {
_currentTest = null;
_currentTestComplete = false;
_context.statManager().createRateStat("udp.statusKnownCharlie", "How often the bob we pick passes us to a charlie we already have a session with?", "udp", new long[] { 60*1000, 20*60*1000, 60*60*1000 });
_context.statManager().createRateStat("udp.receiveTestReply", "How often we get a reply to our peer test?", "udp", new long[] { 60*1000, 20*60*1000, 60*60*1000 });
_context.statManager().createRateStat("udp.receiveTest", "How often we get a packet requesting us to participate in a peer test?", "udp", new long[] { 60*1000, 20*60*1000, 60*60*1000 });
}
private static final int RESEND_TIMEOUT = 5*1000;
@ -146,6 +148,7 @@ class PeerTestManager {
* test
*/
private void receiveTestReply(RemoteHostId from, UDPPacketReader.PeerTestReader testInfo) {
_context.statManager().addRateData("udp.receiveTestReply", 1, 0);
PeerTestState test = _currentTest;
if (expired())
return;
@ -161,7 +164,7 @@ class PeerTestManager {
test.setAlicePort(testInfo.readPort());
if (_log.shouldLog(Log.DEBUG))
_log.debug("Receive test reply from bob @ " + from.getIP() + " via our " + test.getAlicePort() + "/" + test.getAlicePortFromCharlie());
_log.debug("Receive test reply from bob @ " + from + " via our " + test.getAlicePort() + "/" + test.getAlicePortFromCharlie());
if (test.getAlicePortFromCharlie() > 0)
testComplete(false);
} catch (UnknownHostException uhe) {
@ -177,6 +180,7 @@ class PeerTestManager {
if (_log.shouldLog(Log.WARN))
_log.warn("Bob chose a charlie we already have a session to, cancelling the test and rerunning (bob: "
+ _currentTest + ", charlie: " + from + ")");
_currentTestComplete = true;
_context.statManager().addRateData("udp.statusKnownCharlie", 1, 0);
honorStatus(CommSystemFacade.STATUS_UNKNOWN);
return;
@ -286,6 +290,7 @@ class PeerTestManager {
*
*/
public void receiveTest(RemoteHostId from, UDPPacketReader reader) {
_context.statManager().addRateData("udp.receiveTest", 1, 0);
UDPPacketReader.PeerTestReader testInfo = reader.getPeerTestReader();
byte testIP[] = null;
int testPort = testInfo.readPort();
@ -318,7 +323,7 @@ class PeerTestManager {
// initiated test
} else {
if (_log.shouldLog(Log.DEBUG))
_log.debug("We are charlie, as te testIP/port is " + testIP + ":" + testPort + " and the state is unknown for " + nonce);
_log.debug("We are charlie, as te testIP/port is " + RemoteHostId.toString(testIP) + ":" + testPort + " and the state is unknown for " + nonce);
// we are charlie, since alice never sends us her IP and port, only bob does (and,
// erm, we're not alice, since it isn't our nonce)
receiveFromBobAsCharlie(from, testInfo, nonce, null);

View File

@ -113,7 +113,7 @@ public class UDPTransport extends TransportImpl implements TimedWeightedPriority
private static final int MAX_CONSECUTIVE_FAILED = 5;
private static final int TEST_FREQUENCY = 3*60*1000;
private static final int TEST_FREQUENCY = 13*60*1000;
public UDPTransport(RouterContext ctx) {
super(ctx);
@ -973,7 +973,7 @@ public class UDPTransport extends TransportImpl implements TimedWeightedPriority
buf.append(" pushes: ").append(pushCount);
buf.append(" expired? ").append(expired);
buf.append(" unacked: ").append(msg.getUnackedSize());
if (!successful) {
if ( (p != null) && (!successful) ) {
buf.append(" consec_failed: ").append(p.getConsecutiveFailedSends());
long timeSinceSend = _context.clock().now() - p.getLastSendFullyTime();
buf.append(" lastFullSend: ").append(timeSinceSend);

View File

@ -68,8 +68,8 @@ public class TunnelParticipant {
ok = _inboundEndpointProcessor.retrievePreprocessedData(msg.getData(), 0, msg.getData().length, recvFrom);
if (!ok) {
if (_log.shouldLog(Log.ERROR))
_log.error("Failed to dispatch " + msg + ": processor=" + _processor
if (_log.shouldLog(Log.WARN))
_log.warn("Failed to dispatch " + msg + ": processor=" + _processor
+ " inboundEndpoint=" + _inboundEndpointProcessor);
return;
}

View File

@ -82,8 +82,8 @@ class BuildExecutor implements Runnable {
PooledTunnelCreatorConfig cfg = (PooledTunnelCreatorConfig)expired.get(i);
// note the fact that this tunnel request timed out in the peers' profiles.
// or... not.
if (_log.shouldLog(Log.ERROR))
_log.error("Timed out waiting for reply asking for " + cfg);
if (_log.shouldLog(Log.INFO))
_log.info("Timed out waiting for reply asking for " + cfg);
TunnelPool pool = cfg.getTunnelPool();
if (pool != null)
pool.buildComplete(cfg);
@ -274,8 +274,8 @@ class BuildExecutor implements Runnable {
}
long expireBefore = _context.clock().now() + 10*60*1000 - BuildRequestor.REQUEST_TIMEOUT;
if (cfg.getExpiration() <= expireBefore) {
if (_log.shouldLog(Log.ERROR))
_log.error("Build complete for expired tunnel: " + cfg);
if (_log.shouldLog(Log.INFO))
_log.info("Build complete for expired tunnel: " + cfg);
}
}

View File

@ -221,8 +221,8 @@ class BuildHandler {
_context.statManager().addRateData("tunnel.buildClientReject", rtt, rtt);
}
} else {
if (_log.shouldLog(Log.ERROR))
_log.error(msg.getUniqueId() + ": Tunnel reply could not be decrypted for tunnel " + cfg);
if (_log.shouldLog(Log.WARN))
_log.warn(msg.getUniqueId() + ": Tunnel reply could not be decrypted for tunnel " + cfg);
}
}
@ -247,8 +247,8 @@ class BuildHandler {
_context.statManager().addRateData("tunnel.decryptRequestTime", decryptTime, decryptTime);
if (req == null) {
// no records matched, or the decryption failed. bah
if (_log.shouldLog(Log.ERROR))
_log.error("The request " + state.msg.getUniqueId() + " could not be decrypted");
if (_log.shouldLog(Log.WARN))
_log.warn("The request " + state.msg.getUniqueId() + " could not be decrypted");
return;
}
@ -309,8 +309,8 @@ class BuildHandler {
public String getName() { return "Timeout looking for next peer for tunnel join"; }
public void runJob() {
getContext().statManager().addRateData("tunnel.rejectTimeout", 1, 1);
if (_log.shouldLog(Log.ERROR))
_log.error("Request " + _state.msg.getUniqueId()
if (_log.shouldLog(Log.WARN))
_log.warn("Request " + _state.msg.getUniqueId()
+ " could no be satisfied, as the next peer could not be found: " + _nextPeer.toBase64());
getContext().messageHistory().tunnelRejected(_state.fromHash, new TunnelId(_req.readReceiveTunnelId()), _nextPeer,
"rejected because we couldn't find " + _nextPeer.toBase64() + ": " +