diff --git a/apps/syndie/java/src/net/i2p/syndie/Archive.java b/apps/syndie/java/src/net/i2p/syndie/Archive.java index 391a1e30c..7d080ab86 100644 --- a/apps/syndie/java/src/net/i2p/syndie/Archive.java +++ b/apps/syndie/java/src/net/i2p/syndie/Archive.java @@ -30,6 +30,7 @@ public class Archive { private Map _blogInfo; private ArchiveIndex _index; private EntryExtractor _extractor; + private String _defaultSelector; public static final String METADATA_FILE = "meta.snm"; public static final String INDEX_FILE = "archive.txt"; @@ -50,6 +51,8 @@ public class Archive { _blogInfo = new HashMap(); _index = null; _extractor = new EntryExtractor(ctx); + _defaultSelector = ctx.getProperty("syndie.defaultSelector"); + if (_defaultSelector == null) _defaultSelector = ""; reloadInfo(); } @@ -63,7 +66,12 @@ public class Archive { BlogInfo bi = new BlogInfo(); try { bi.load(new FileInputStream(meta)); - info.add(bi); + if (bi.verify(_context)) { + info.add(bi); + } else { + System.err.println("Invalid blog (but we're storing it anyway): " + bi); + info.add(bi); + } } catch (IOException ioe) { ioe.printStackTrace(); } @@ -79,8 +87,11 @@ public class Archive { } } } + + public String getDefaultSelector() { return _defaultSelector; } public BlogInfo getBlogInfo(BlogURI uri) { + if (uri == null) return null; synchronized (_blogInfo) { return (BlogInfo)_blogInfo.get(uri.getKeyHash()); } @@ -90,14 +101,20 @@ public class Archive { return (BlogInfo)_blogInfo.get(key); } } - public void storeBlogInfo(BlogInfo info) { + public boolean storeBlogInfo(BlogInfo info) { if (!info.verify(_context)) { System.err.println("Not storing the invalid blog " + info); - return; + return false; } + boolean isNew = true; synchronized (_blogInfo) { - _blogInfo.put(info.getKey().calculateHash(), info); + BlogInfo old = (BlogInfo)_blogInfo.get(info.getKey().calculateHash()); + if ( (old == null) || (old.getEdition() < info.getEdition()) ) + _blogInfo.put(info.getKey().calculateHash(), info); + else + isNew = false; } + if (!isNew) return true; // valid entry, but not stored, since its old try { File blogDir = new File(_rootDir, info.getKey().calculateHash().toBase64()); blogDir.mkdirs(); @@ -106,8 +123,10 @@ public class Archive { info.write(out); out.close(); System.out.println("Blog info written to " + blogFile.getPath()); + return true; } catch (IOException ioe) { ioe.printStackTrace(); + return false; } } @@ -262,7 +281,16 @@ public class Archive { } public boolean storeEntry(EntryContainer container) { + if (container == null) return false; BlogURI uri = container.getURI(); + if (uri == null) return false; + + File blogDir = new File(_rootDir, uri.getKeyHash().toBase64()); + blogDir.mkdirs(); + File entryFile = new File(blogDir, getEntryFilename(uri.getEntryId())); + if (entryFile.exists()) return true; + + BlogInfo info = getBlogInfo(uri); if (info == null) { System.out.println("no blog metadata for the uri " + uri); @@ -274,13 +302,10 @@ public class Archive { } else { //System.out.println("Signature is valid: " + container.getSignature() + " for info " + info); } - ByteArrayOutputStream baos = new ByteArrayOutputStream(); try { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); container.write(baos, true); - File blogDir = new File(_rootDir, uri.getKeyHash().toBase64()); - blogDir.mkdirs(); byte data[] = baos.toByteArray(); - File entryFile = new File(blogDir, getEntryFilename(uri.getEntryId())); FileOutputStream out = new FileOutputStream(entryFile); out.write(data); out.close(); diff --git a/apps/syndie/java/src/net/i2p/syndie/BlogManager.java b/apps/syndie/java/src/net/i2p/syndie/BlogManager.java index 31cc7a28c..b78a23712 100644 --- a/apps/syndie/java/src/net/i2p/syndie/BlogManager.java +++ b/apps/syndie/java/src/net/i2p/syndie/BlogManager.java @@ -18,6 +18,7 @@ public class BlogManager { private File _archiveDir; private File _userDir; private File _cacheDir; + private File _tempDir; private Archive _archive; static { @@ -42,11 +43,13 @@ public class BlogManager { _archiveDir = new File(root, "archive"); _userDir = new File(root, "users"); _cacheDir = new File(root, "cache"); + _tempDir = new File(root, "temp"); _blogKeyDir.mkdirs(); _privKeyDir.mkdirs(); _archiveDir.mkdirs(); _cacheDir.mkdirs(); _userDir.mkdirs(); + _tempDir.mkdirs(); _archive = new Archive(ctx, _archiveDir.getAbsolutePath(), _cacheDir.getAbsolutePath()); _archive.regenerateIndex(); } @@ -93,6 +96,7 @@ public class BlogManager { } public Archive getArchive() { return _archive; } + public File getTempDir() { return _tempDir; } public List listMyBlogs() { File files[] = _privKeyDir.listFiles(); @@ -175,45 +179,7 @@ public class BlogManager { FileWriter out = null; try { out = new FileWriter(userFile); - out.write("password=" + user.getHashedPassword() + "\n"); - out.write("blog=" + user.getBlog().toBase64() + "\n"); - out.write("lastid=" + user.getMostRecentEntry() + "\n"); - out.write("lastmetaedition=" + user.getLastMetaEntry() + "\n"); - out.write("lastlogin=" + user.getLastLogin() + "\n"); - out.write("addressbook=" + user.getAddressbookLocation() + "\n"); - out.write("showimages=" + user.getShowImages() + "\n"); - out.write("showexpanded=" + user.getShowExpanded() + "\n"); - StringBuffer buf = new StringBuffer(); - buf.append("groups="); - Map groups = user.getBlogGroups(); - for (Iterator iter = groups.keySet().iterator(); iter.hasNext(); ) { - String name = (String)iter.next(); - List selectors = (List)groups.get(name); - buf.append(name).append(':'); - for (int i = 0; i < selectors.size(); i++) { - buf.append(selectors.get(i)); - if (i + 1 < selectors.size()) - buf.append(","); - } - if (iter.hasNext()) - buf.append(' '); - } - buf.append('\n'); - out.write(buf.toString()); - // shitlist=hash,hash,hash - List shitlistedBlogs = user.getShitlistedBlogs(); - if (shitlistedBlogs.size() > 0) { - buf.setLength(0); - buf.append("shitlistedblogs="); - for (int i = 0; i < shitlistedBlogs.size(); i++) { - Hash blog = (Hash)shitlistedBlogs.get(i); - buf.append(blog.toBase64()); - if (i + 1 < shitlistedBlogs.size()) - buf.append(','); - } - buf.append('\n'); - out.write(buf.toString()); - } + out.write(user.export()); } catch (IOException ioe) { ioe.printStackTrace(); } finally { @@ -347,6 +313,39 @@ public class BlogManager { } } + /** + * read in the syndie blog metadata file from the stream, verifying it and adding it to + * the archive if necessary + * + */ + public boolean importBlogMetadata(InputStream metadataStream) throws IOException { + try { + BlogInfo info = new BlogInfo(); + info.load(metadataStream); + return _archive.storeBlogInfo(info); + } catch (IOException ioe) { + ioe.printStackTrace(); + return false; + } + } + + /** + * read in the syndie entry file from the stream, verifying it and adding it to + * the archive if necessary + * + */ + public boolean importBlogEntry(InputStream entryStream) throws IOException { + try { + EntryContainer c = new EntryContainer(); + c.load(entryStream); + return _archive.storeEntry(c); + } catch (IOException ioe) { + ioe.printStackTrace(); + return false; + } + } + + public String addAddress(User user, String name, String location, String schema) { if (!user.getAuthenticated()) return "Not logged in"; boolean ok = validateAddressName(name); diff --git a/apps/syndie/java/src/net/i2p/syndie/User.java b/apps/syndie/java/src/net/i2p/syndie/User.java index 37d031fd4..736dd6bdf 100644 --- a/apps/syndie/java/src/net/i2p/syndie/User.java +++ b/apps/syndie/java/src/net/i2p/syndie/User.java @@ -24,9 +24,17 @@ public class User { private String _addressbookLocation; private boolean _showImagesByDefault; private boolean _showExpandedByDefault; + private String _defaultSelector; private long _lastLogin; private long _lastMetaEntry; + private boolean _allowAccessRemote; private boolean _authenticated; + private String _eepProxyHost; + private int _eepProxyPort; + private String _webProxyHost; + private int _webProxyPort; + private String _torProxyHost; + private int _torProxyPort; public User() { _context = I2PAppContext.getGlobalContext(); @@ -40,9 +48,17 @@ public class User { _mostRecentEntry = -1; _blogGroups = new HashMap(); _shitlistedBlogs = new ArrayList(); + _defaultSelector = null; _addressbookLocation = "userhosts.txt"; _showImagesByDefault = false; _showExpandedByDefault = false; + _allowAccessRemote = false; + _eepProxyHost = null; + _webProxyHost = null; + _torProxyHost = null; + _eepProxyPort = -1; + _webProxyPort = -1; + _torProxyPort = -1; _lastLogin = -1; _lastMetaEntry = 0; } @@ -60,10 +76,21 @@ public class User { public long getLastLogin() { return _lastLogin; } public String getHashedPassword() { return _hashedPassword; } public long getLastMetaEntry() { return _lastMetaEntry; } + public String getDefaultSelector() { return _defaultSelector; } + public void setDefaultSelector(String sel) { _defaultSelector = sel; } + public boolean getAllowAccessRemote() { return _allowAccessRemote; } + public void setAllowAccessRemote(boolean allow) { _allowAccessRemote = true; } public void setMostRecentEntry(long id) { _mostRecentEntry = id; } public void setLastMetaEntry(long id) { _lastMetaEntry = id; } + public String getEepProxyHost() { return _eepProxyHost; } + public int getEepProxyPort() { return _eepProxyPort; } + public String getWebProxyHost() { return _webProxyHost; } + public int getWebProxyPort() { return _webProxyPort; } + public String getTorProxyHost() { return _torProxyHost; } + public int getTorProxyPort() { return _torProxyPort; } + public void invalidate() { BlogManager.instance().saveUser(this); init(); @@ -135,11 +162,75 @@ public class User { _showImagesByDefault = (show != null) && (show.equals("true")); show = props.getProperty("showexpanded", "false"); _showExpandedByDefault = (show != null) && (show.equals("true")); - + _defaultSelector = props.getProperty("defaultselector"); + String allow = props.getProperty("allowaccessremote", "false"); + _allowAccessRemote = (allow != null) && (allow.equals("true")); + _eepProxyPort = getInt(props.getProperty("eepproxyport")); + _webProxyPort = getInt(props.getProperty("webproxyport")); + _torProxyPort = getInt(props.getProperty("torproxyport")); + _eepProxyHost = props.getProperty("eepproxyhost"); + _webProxyHost = props.getProperty("webproxyhost"); + _torProxyHost = props.getProperty("torproxyhost"); _lastLogin = _context.clock().now(); _authenticated = true; return LOGIN_OK; } + private int getInt(String val) { + if (val == null) return -1; + try { return Integer.parseInt(val); } catch (NumberFormatException nfe) { return -1; } + } + public static final String LOGIN_OK = "Logged in"; + + public String export() { + StringBuffer buf = new StringBuffer(512); + buf.append("password=" + getHashedPassword() + "\n"); + buf.append("blog=" + getBlog().toBase64() + "\n"); + buf.append("lastid=" + getMostRecentEntry() + "\n"); + buf.append("lastmetaedition=" + getLastMetaEntry() + "\n"); + buf.append("lastlogin=" + getLastLogin() + "\n"); + buf.append("addressbook=" + getAddressbookLocation() + "\n"); + buf.append("showimages=" + getShowImages() + "\n"); + buf.append("showexpanded=" + getShowExpanded() + "\n"); + buf.append("defaultselector=" + getDefaultSelector() + "\n"); + buf.append("allowaccessremote=" + _allowAccessRemote + "\n"); + buf.append("eepproxyhost="+_eepProxyHost+"\n"); + buf.append("eepproxyport="+_eepProxyPort+"\n"); + buf.append("webproxyhost="+_webProxyHost+"\n"); + buf.append("webproxyport="+_webProxyPort+"\n"); + buf.append("torproxyhost="+_torProxyHost+"\n"); + buf.append("torproxyport="+_torProxyPort+"\n"); + + buf.append("groups="); + Map groups = getBlogGroups(); + for (Iterator iter = groups.keySet().iterator(); iter.hasNext(); ) { + String name = (String)iter.next(); + List selectors = (List)groups.get(name); + buf.append(name).append(':'); + for (int i = 0; i < selectors.size(); i++) { + buf.append(selectors.get(i)); + if (i + 1 < selectors.size()) + buf.append(","); + } + if (iter.hasNext()) + buf.append(' '); + } + buf.append('\n'); + // shitlist=hash,hash,hash + List shitlistedBlogs = getShitlistedBlogs(); + if (shitlistedBlogs.size() > 0) { + buf.setLength(0); + buf.append("shitlistedblogs="); + for (int i = 0; i < shitlistedBlogs.size(); i++) { + Hash blog = (Hash)shitlistedBlogs.get(i); + buf.append(blog.toBase64()); + if (i + 1 < shitlistedBlogs.size()) + buf.append(','); + } + buf.append('\n'); + } + + return buf.toString(); + } } diff --git a/apps/syndie/java/src/net/i2p/syndie/data/ArchiveIndex.java b/apps/syndie/java/src/net/i2p/syndie/data/ArchiveIndex.java index cf45436ee..819fe3e86 100644 --- a/apps/syndie/java/src/net/i2p/syndie/data/ArchiveIndex.java +++ b/apps/syndie/java/src/net/i2p/syndie/data/ArchiveIndex.java @@ -77,6 +77,44 @@ public class ArchiveIndex { /** get the raw entry size (including attachments) from the given blog/tag pair */ public long getBlogEntrySizeKB(int index, int entryIndex) { return ((EntrySummary)((BlogSummary)_blogs.get(index)).entries.get(entryIndex)).size; } + public boolean getEntryIsKnown(BlogURI uri) { return getEntry(uri) != null; } + public long getBlogEntrySizeKB(BlogURI uri) { + EntrySummary entry = getEntry(uri); + if (entry == null) return -1; + return entry.size; + } + private EntrySummary getEntry(BlogURI uri) { + if ( (uri == null) || (uri.getKeyHash() == null) || (uri.getEntryId() < 0) ) return null; + for (int i = 0; i < _blogs.size(); i++) { + BlogSummary summary = (BlogSummary)_blogs.get(i); + if (summary.blog.equals(uri.getKeyHash())) { + for (int j = 0; j < summary.entries.size(); j++) { + EntrySummary entry = (EntrySummary)summary.entries.get(j); + if (entry.entry.equals(uri)) + return entry; + } + } + } + return null; + } + public Set getBlogEntryTags(BlogURI uri) { + Set tags = new HashSet(); + if ( (uri == null) || (uri.getKeyHash() == null) || (uri.getEntryId() < 0) ) return tags; + for (int i = 0; i < _blogs.size(); i++) { + BlogSummary summary = (BlogSummary)_blogs.get(i); + if (summary.blog.equals(uri.getKeyHash())) { + for (int j = 0; j < summary.entries.size(); j++) { + EntrySummary entry = (EntrySummary)summary.entries.get(j); + if (entry.entry.equals(uri)) { + tags.add(summary.tag); + break; + } + } + } + } + return tags; + } + /** how many 'new' blogs are listed */ public int getNewestBlogCount() { return _newestBlogs.size(); } public Hash getNewestBlog(int index) { return (Hash)_newestBlogs.get(index); } diff --git a/apps/syndie/java/src/net/i2p/syndie/data/BlogInfo.java b/apps/syndie/java/src/net/i2p/syndie/data/BlogInfo.java index b6d491cf8..3ac0ecca5 100644 --- a/apps/syndie/java/src/net/i2p/syndie/data/BlogInfo.java +++ b/apps/syndie/java/src/net/i2p/syndie/data/BlogInfo.java @@ -13,6 +13,7 @@ import net.i2p.I2PAppContext; * Required keys: * Owner: base64 of their signing public key * Signature: base64 of the DSA signature of the rest of the ordered metadata + * Edition: base10 unique identifier for this metadata (higher clobbers lower) * * Optional keys: * Posters: comma delimited list of base64 signing public keys that @@ -53,6 +54,7 @@ public class BlogInfo { public static final String SIGNATURE = "Signature"; public static final String NAME = "Name"; public static final String DESCRIPTION = "Description"; + public static final String EDITION = "Edition"; public void load(InputStream in) throws IOException { BufferedReader reader = new BufferedReader(new InputStreamReader(in)); @@ -63,8 +65,13 @@ public class BlogInfo { line = line.trim(); int len = line.length(); int split = line.indexOf(':'); - if ( (len <= 0) || (split <= 0) || (split >= len - 2) ) + if ( (len <= 0) || (split <= 0) ) { continue; + } else if (split >= len - 1) { + names.add(line.substring(0, split).trim()); + vals.add(""); + continue; + } String key = line.substring(0, split).trim(); String val = line.substring(split+1).trim(); @@ -102,7 +109,8 @@ public class BlogInfo { if ( (includeRealSignature) || (!SIGNATURE.equals(_optionNames[i])) ) buf.append(_optionNames[i]).append(':').append(_optionValues[i]).append('\n'); } - out.write(buf.toString().getBytes()); + String s = buf.toString(); + out.write(s.getBytes()); } public String getProperty(String name) { @@ -133,6 +141,18 @@ public class BlogInfo { _optionValues = values; } + public int getEdition() { + String e = getProperty(EDITION); + if (e != null) { + try { + return Integer.parseInt(e); + } catch (NumberFormatException nfe) { + return 0; + } + } + return 0; + } + public String[] getProperties() { return _optionNames; } public SigningPublicKey[] getPosters() { return _posters; } @@ -151,7 +171,9 @@ public class BlogInfo { try { ByteArrayOutputStream out = new ByteArrayOutputStream(512); write(out, false); - return ctx.dsa().verifySignature(_signature, out.toByteArray(), _key); + out.close(); + byte data[] = out.toByteArray(); + return ctx.dsa().verifySignature(_signature, data, _key); } catch (IOException ioe) { return false; } @@ -192,4 +214,52 @@ public class BlogInfo { } return buf.toString(); } + + public static void main(String args[]) { + I2PAppContext ctx = I2PAppContext.getGlobalContext(); + /* + try { + Object keys[] = ctx.keyGenerator().generateSigningKeypair(); + SigningPublicKey pub = (SigningPublicKey)keys[0]; + SigningPrivateKey priv = (SigningPrivateKey)keys[1]; + + Properties opts = new Properties(); + opts.setProperty("Name", "n4m3"); + opts.setProperty("Description", "foo"); + opts.setProperty("Edition", "0"); + opts.setProperty("ContactURL", "u@h.org"); + + BlogInfo info = new BlogInfo(pub, null, opts); + System.err.println("\n"); + System.err.println("\n"); + info.sign(ctx, priv); + System.err.println("\n"); + boolean ok = info.verify(ctx); + System.err.println("\n"); + System.err.println("sign&verify: " + ok); + System.err.println("\n"); + System.err.println("\n"); + + FileOutputStream o = new FileOutputStream("bloginfo-test.dat"); + info.write(o, true); + o.close(); + FileInputStream i = new FileInputStream("bloginfo-test.dat"); + byte buf[] = new byte[4096]; + int sz = DataHelper.read(i, buf); + BlogInfo read = new BlogInfo(); + read.load(new ByteArrayInputStream(buf, 0, sz)); + ok = read.verify(ctx); + System.err.println("write to disk, verify read: " + ok); + System.err.println("Data: " + Base64.encode(buf, 0, sz)); + System.err.println("Str : " + new String(buf, 0, sz)); + } catch (Exception e) { e.printStackTrace(); } + */ + try { + FileInputStream in = new FileInputStream(args[0]); + BlogInfo info = new BlogInfo(); + info.load(in); + boolean ok = info.verify(I2PAppContext.getGlobalContext()); + System.out.println("OK? " + ok + " :" + info); + } catch (Exception e) { e.printStackTrace(); } + } } diff --git a/apps/syndie/java/src/net/i2p/syndie/data/EntryContainer.java b/apps/syndie/java/src/net/i2p/syndie/data/EntryContainer.java index 6afc3fa38..4aaa76dd8 100644 --- a/apps/syndie/java/src/net/i2p/syndie/data/EntryContainer.java +++ b/apps/syndie/java/src/net/i2p/syndie/data/EntryContainer.java @@ -72,7 +72,9 @@ public class EntryContainer { public int getFormat() { return _format; } public void load(InputStream source) throws IOException { - String fmt = DataHelper.readLine(source).trim(); + String line = DataHelper.readLine(source); + if (line == null) throw new IOException("No format line in the entry"); + String fmt = line.trim(); if (FORMAT_ZIP_UNENCRYPTED_STR.equals(fmt)) { _format = FORMAT_ZIP_UNENCRYPTED; } else if (FORMAT_ZIP_ENCRYPTED_STR.equals(fmt)) { @@ -81,7 +83,6 @@ public class EntryContainer { throw new IOException("Unsupported entry format: " + fmt); } - String line = null; while ( (line = DataHelper.readLine(source)) != null) { line = line.trim(); int len = line.length(); @@ -99,17 +100,24 @@ public class EntryContainer { parseHeaders(); String sigStr = DataHelper.readLine(source); + if ( (sigStr == null) || (sigStr.indexOf("Signature:") == -1) ) + throw new IOException("No signature line"); sigStr = sigStr.substring("Signature:".length()+1).trim(); _signature = new Signature(Base64.decode(sigStr)); //System.out.println("Sig: " + _signature.toBase64()); - line = DataHelper.readLine(source).trim(); + line = DataHelper.readLine(source); + if (line == null) + throw new IOException("No size line"); + line = line.trim(); int dataSize = -1; try { int index = line.indexOf("Size:"); if (index == 0) dataSize = Integer.parseInt(line.substring("Size:".length()+1).trim()); + else + throw new IOException("Invalid size line"); } catch (NumberFormatException nfe) { throw new IOException("Invalid entry size: " + line); } diff --git a/apps/syndie/java/src/net/i2p/syndie/sml/HTMLPreviewRenderer.java b/apps/syndie/java/src/net/i2p/syndie/sml/HTMLPreviewRenderer.java new file mode 100644 index 000000000..64d8d3bab --- /dev/null +++ b/apps/syndie/java/src/net/i2p/syndie/sml/HTMLPreviewRenderer.java @@ -0,0 +1,109 @@ +package net.i2p.syndie.sml; + +import java.io.*; +import java.text.*; +import java.util.*; +import net.i2p.data.*; +import net.i2p.syndie.*; +import net.i2p.syndie.data.*; +import net.i2p.syndie.web.*; + +/** + * + */ +public class HTMLPreviewRenderer extends HTMLRenderer { + private List _filenames; + private List _fileTypes; + private List _files; + + public HTMLPreviewRenderer(List filenames, List fileTypes, List files) { + super(); + _filenames = filenames; + _fileTypes = fileTypes; + _files = files; + } + + protected String getAttachmentURLBase() { return "viewtempattachment.jsp"; } + protected String getAttachmentURL(int id) { + return getAttachmentURLBase() + "?" + + ArchiveViewerBean.PARAM_ATTACHMENT + "=" + id; + } + + public void receiveAttachment(int id, String anchorText) { + if (!continueBody()) { return; } + if ( (id < 0) || (_files == null) || (id >= _files.size()) ) { + _bodyBuffer.append(sanitizeString(anchorText)); + } else { + File f = (File)_files.get(id); + String name = (String)_filenames.get(id); + String type = (String)_fileTypes.get(id); + _bodyBuffer.append(""); + _bodyBuffer.append(sanitizeString(anchorText)).append(""); + _bodyBuffer.append(" (").append(f.length()/1024).append("KB, "); + _bodyBuffer.append(" \"").append(sanitizeString(name)).append("\", "); + _bodyBuffer.append(sanitizeString(type)).append(")"); + } + } + + public void receiveEnd() { + _postBodyBuffer.append("\n"); + _postBodyBuffer.append("\n"); + _postBodyBuffer.append("
\n"); + _postBodyBuffer.append(" 0) { + _postBodyBuffer.append("Attachments: "); + _postBodyBuffer.append("\n"); + _postBodyBuffer.append("
\n"); + } + + if (_blogs.size() > 0) { + _postBodyBuffer.append("Blog references: "); + for (int i = 0; i < _blogs.size(); i++) { + Blog b = (Blog)_blogs.get(i); + _postBodyBuffer.append("").append(sanitizeString(b.name)).append(" "); + } + _postBodyBuffer.append("
\n"); + } + + if (_links.size() > 0) { + _postBodyBuffer.append("External links: "); + for (int i = 0; i < _links.size(); i++) { + Link l = (Link)_links.get(i); + _postBodyBuffer.append("").append(sanitizeString(l.location)); + _postBodyBuffer.append(" (").append(sanitizeString(l.schema)).append(") "); + } + _postBodyBuffer.append("
\n"); + } + + if (_addresses.size() > 0) { + _postBodyBuffer.append("Addresses: "); + for (int i = 0; i < _addresses.size(); i++) { + Address a = (Address)_addresses.get(i); + _postBodyBuffer.append("").append(sanitizeString(a.name)); + } + _postBodyBuffer.append("
\n"); + } + + _postBodyBuffer.append("\n\n\n"); + _postBodyBuffer.append("\n"); + } +} diff --git a/apps/syndie/java/src/net/i2p/syndie/sml/HTMLRenderer.java b/apps/syndie/java/src/net/i2p/syndie/sml/HTMLRenderer.java index 06580fe1c..622b2a4e6 100644 --- a/apps/syndie/java/src/net/i2p/syndie/sml/HTMLRenderer.java +++ b/apps/syndie/java/src/net/i2p/syndie/sml/HTMLRenderer.java @@ -12,23 +12,23 @@ import net.i2p.syndie.web.*; * */ public class HTMLRenderer extends EventReceiverImpl { - private SMLParser _parser; - private Writer _out; - private User _user; - private Archive _archive; - private EntryContainer _entry; - private boolean _showImages; - private boolean _cutBody; - private boolean _cutReached; - private int _cutSize; - private int _lastNewlineAt; - private Map _headers; - private List _addresses; - private List _links; - private List _blogs; - private StringBuffer _preBodyBuffer; - private StringBuffer _bodyBuffer; - private StringBuffer _postBodyBuffer; + protected SMLParser _parser; + protected Writer _out; + protected User _user; + protected Archive _archive; + protected EntryContainer _entry; + protected boolean _showImages; + protected boolean _cutBody; + protected boolean _cutReached; + protected int _cutSize; + protected int _lastNewlineAt; + protected Map _headers; + protected List _addresses; + protected List _links; + protected List _blogs; + protected StringBuffer _preBodyBuffer; + protected StringBuffer _bodyBuffer; + protected StringBuffer _postBodyBuffer; public HTMLRenderer() { _parser = new SMLParser(); @@ -190,7 +190,7 @@ public class HTMLRenderer extends EventReceiverImpl { } /** are we either before the cut or rendering without cutting? */ - private boolean continueBody() { + protected boolean continueBody() { boolean rv = ( (!_cutReached) && (_bodyBuffer.length() <= _cutSize) ) || (!_cutBody); //if (!rv) // System.out.println("rv: " + rv + " Cut reached: " + _cutReached + " bodyBufferSize: " + _bodyBuffer.length() + " cutBody? " + _cutBody); @@ -227,7 +227,7 @@ public class HTMLRenderer extends EventReceiverImpl { _bodyBuffer.append(']'); } - private static class Blog { + protected static class Blog { public String name; public String hash; public String tag; @@ -317,7 +317,7 @@ public class HTMLRenderer extends EventReceiverImpl { _bodyBuffer.append("] "); } - private static class Link { + protected static class Link { public String schema; public String location; public int hashCode() { return -1; } @@ -340,7 +340,7 @@ public class HTMLRenderer extends EventReceiverImpl { _bodyBuffer.append(sanitizeURL(text)).append("\">").append(sanitizeString(text)).append("
"); } - private static class Address { + protected static class Address { public String name; public String schema; public String location; @@ -381,79 +381,113 @@ public class HTMLRenderer extends EventReceiverImpl { public void receiveEnd() { _postBodyBuffer.append("\n"); - _postBodyBuffer.append("\n"); - _postBodyBuffer.append("
\n"); - _postBodyBuffer.append("\n"); - _postBodyBuffer.append("\n"); - _postBodyBuffer.append("\n"); - - if ( (_entry != null) && (_entry.getAttachments() != null) && (_entry.getAttachments().length > 0) ) { - _postBodyBuffer.append("Attachments: "); - _postBodyBuffer.append("\n"); + _postBodyBuffer.append("\n"); + _postBodyBuffer.append("\n"); + + if ( (_entry != null) && (_entry.getAttachments() != null) && (_entry.getAttachments().length > 0) ) { + _postBodyBuffer.append("Attachments: "); + _postBodyBuffer.append("\n"); + _postBodyBuffer.append("
\n"); } - _postBodyBuffer.append("\n"); - _postBodyBuffer.append("
\n"); - } - - if (_blogs.size() > 0) { - _postBodyBuffer.append("Blog references: "); - for (int i = 0; i < _blogs.size(); i++) { - Blog b = (Blog)_blogs.get(i); - _postBodyBuffer.append("").append(sanitizeString(b.name)).append(" "); + + if (_blogs.size() > 0) { + _postBodyBuffer.append("Blog references: "); + for (int i = 0; i < _blogs.size(); i++) { + Blog b = (Blog)_blogs.get(i); + _postBodyBuffer.append("").append(sanitizeString(b.name)).append(" "); + } + _postBodyBuffer.append("
\n"); } - _postBodyBuffer.append("
\n"); - } - - if (_links.size() > 0) { - _postBodyBuffer.append("External links: "); - for (int i = 0; i < _links.size(); i++) { - Link l = (Link)_links.get(i); - _postBodyBuffer.append("").append(sanitizeString(l.location)); - _postBodyBuffer.append(" (").append(sanitizeString(l.schema)).append(") "); + + if (_links.size() > 0) { + _postBodyBuffer.append("External links: "); + for (int i = 0; i < _links.size(); i++) { + Link l = (Link)_links.get(i); + _postBodyBuffer.append("").append(sanitizeString(l.location)); + _postBodyBuffer.append(" (").append(sanitizeString(l.schema)).append(") "); + } + _postBodyBuffer.append("
\n"); } - _postBodyBuffer.append("
\n"); - } - - if (_addresses.size() > 0) { - _postBodyBuffer.append("Addresses: "); - for (int i = 0; i < _addresses.size(); i++) { - Address a = (Address)_addresses.get(i); - _postBodyBuffer.append("").append(sanitizeString(a.name)); + + if (_addresses.size() > 0) { + _postBodyBuffer.append("Addresses: "); + for (int i = 0; i < _addresses.size(); i++) { + Address a = (Address)_addresses.get(i); + _postBodyBuffer.append("").append(sanitizeString(a.name)); + } + _postBodyBuffer.append("
\n"); } - _postBodyBuffer.append("
\n"); + + _postBodyBuffer.append("\n\n\n"); } - - _postBodyBuffer.append("\n\n\n"); _postBodyBuffer.append("\n"); } @@ -463,8 +497,9 @@ public class HTMLRenderer extends EventReceiverImpl { } public void receiveHeaderEnd() { - renderMetaCell(); + _preBodyBuffer.append("\n"); renderSubjectCell(); + renderMetaCell(); renderPreBodyCell(); } @@ -473,25 +508,24 @@ public class HTMLRenderer extends EventReceiverImpl { public static final String HEADER_IN_REPLY_TO = "InReplyTo"; private void renderSubjectCell() { - _preBodyBuffer.append("\n"); + _preBodyBuffer.append("\n"); } private void renderPreBodyCell() { String bgcolor = (String)_headers.get(HEADER_BGCOLOR); if (_cutBody) - _preBodyBuffer.append(" - - \ No newline at end of file + + + \ No newline at end of file diff --git a/apps/syndie/jsp/addaddress.jsp b/apps/syndie/jsp/addaddress.jsp index a252103af..6acf32933 100644 --- a/apps/syndie/jsp/addaddress.jsp +++ b/apps/syndie/jsp/addaddress.jsp @@ -3,6 +3,7 @@ SyndieMedia +
"); + _preBodyBuffer.append("
"); String subject = (String)_headers.get(HEADER_SUBJECT); if (subject == null) subject = "[no subject]"; _preBodyBuffer.append(sanitizeString(subject)); - _preBodyBuffer.append("
"); + _preBodyBuffer.append("
"); else - _preBodyBuffer.append("
"); + _preBodyBuffer.append("
"); } private void renderMetaCell() { - _preBodyBuffer.append("\n"); - _preBodyBuffer.append("\n"); + _preBodyBuffer.append(" Reply\n"); + _preBodyBuffer.append("\n\n"); } private final SimpleDateFormat _dateFormat = new SimpleDateFormat("yyyy/MM/dd"); @@ -539,7 +574,7 @@ public class HTMLRenderer extends EventReceiverImpl { try { String str = _dateFormat.format(new Date(when)); long dayBegin = _dateFormat.parse(str).getTime(); - return str + "
" + (when - dayBegin); + return str + "." + (when - dayBegin); } catch (ParseException pe) { pe.printStackTrace(); // wtf @@ -548,12 +583,26 @@ public class HTMLRenderer extends EventReceiverImpl { } } - public static final String sanitizeString(String str) { + public static final String sanitizeString(String str) { return sanitizeString(str, true); } + public static final String sanitizeString(String str, boolean allowNL) { if (str == null) return null; - if ( (str.indexOf('<') < 0) && (str.indexOf('>') < 0) ) - return str; + boolean unsafe = false; + unsafe = unsafe || str.indexOf('<') >= 0; + unsafe = unsafe || str.indexOf('>') >= 0; + if (!allowNL) { + unsafe = unsafe || str.indexOf('\n') >= 0; + unsafe = unsafe || str.indexOf('\r') >= 0; + unsafe = unsafe || str.indexOf('\f') >= 0; + } + if (!unsafe) return str; + str = str.replace('<', '_'); str = str.replace('>', '-'); + if (!allowNL) { + str = str.replace('\n', ' '); + str = str.replace('\r', ' '); + str = str.replace('\f', ' '); + } return str; } @@ -575,8 +624,8 @@ public class HTMLRenderer extends EventReceiverImpl { "&" + ArchiveViewerBean.PARAM_EXPAND_ENTRIES + "=true"; } - private String getAttachmentURLBase() { return "viewattachment.jsp"; } - private String getAttachmentURL(int id) { + protected String getAttachmentURLBase() { return "viewattachment.jsp"; } + protected String getAttachmentURL(int id) { if (_entry == null) return "unknown"; return getAttachmentURLBase() + "?" + ArchiveViewerBean.PARAM_BLOG + "=" + diff --git a/apps/syndie/java/src/net/i2p/syndie/web/ArchiveViewerBean.java b/apps/syndie/java/src/net/i2p/syndie/web/ArchiveViewerBean.java index 8c15f867f..b9d6a62b6 100644 --- a/apps/syndie/java/src/net/i2p/syndie/web/ArchiveViewerBean.java +++ b/apps/syndie/java/src/net/i2p/syndie/web/ArchiveViewerBean.java @@ -79,12 +79,26 @@ public class ArchiveViewerBean { public static final String SEL_BLOGTAG = "blogtag://"; public static final String SEL_ENTRY = "entry://"; public static final String SEL_GROUP = "group://"; + /** submit field for the selector form */ + public static final String PARAM_SELECTOR_ACTION = "action"; + public static final String SEL_ACTION_SET_AS_DEFAULT = "Set as default"; public static void renderBlogSelector(User user, Map parameters, Writer out) throws IOException { + String sel = getString(parameters, PARAM_SELECTOR); + String action = getString(parameters, PARAM_SELECTOR_ACTION); + if ( (sel != null) && (action != null) && (SEL_ACTION_SET_AS_DEFAULT.equals(action)) ) { + user.setDefaultSelector(HTMLRenderer.sanitizeString(sel, false)); + BlogManager.instance().saveUser(user); + } + out.write("\n"); + Set localBlogs = archive.getIndex().getUniqueBlogs(); + Set remoteBlogs = _remoteIndex.getUniqueBlogs(); + int newBlogs = 0; + for (Iterator iter = remoteBlogs.iterator(); iter.hasNext(); ) { + Hash blog = (Hash)iter.next(); + if (!localBlogs.contains(blog)) { + buf.append("\n"); + newBlogs++; + } + } + if (newBlogs > 0) { + out.write(buf.toString()); + out.write("
\n"); + } + + int newEntries = 0; + out.write("
\n"); + _preBodyBuffer.append("\n"); BlogInfo info = null; if (_entry != null) info = _archive.getBlogInfo(_entry.getURI()); @@ -506,31 +540,32 @@ public class HTMLRenderer extends EventReceiverImpl { } else { _preBodyBuffer.append("[unknown blog]"); } - _preBodyBuffer.append("
\n"); String tags[] = (_entry != null ? _entry.getTags() : null); - _preBodyBuffer.append(""); - for (int i = 0; tags != null && i < tags.length; i++) { - _preBodyBuffer.append(""); - _preBodyBuffer.append(sanitizeString(tags[i])); - _preBodyBuffer.append(""); - if (i + 1 < tags.length) - _preBodyBuffer.append(", "); + if ( (tags != null) && (tags.length > 0) ) { + _preBodyBuffer.append(" Tags: "); + _preBodyBuffer.append(""); + for (int i = 0; tags != null && i < tags.length; i++) { + _preBodyBuffer.append(""); + _preBodyBuffer.append(sanitizeString(tags[i])); + _preBodyBuffer.append(""); + if (i + 1 < tags.length) + _preBodyBuffer.append(", "); + } + _preBodyBuffer.append(""); } - _preBodyBuffer.append("
\n"); + _preBodyBuffer.append(" "); if (_entry != null) _preBodyBuffer.append(getEntryDate(_entry.getURI().getEntryId())); else _preBodyBuffer.append(getEntryDate(new Date().getTime())); - _preBodyBuffer.append("
"); String inReplyTo = (String)_headers.get(HEADER_IN_REPLY_TO); - System.err.println("In reply to: [" + inReplyTo + "]"); if ( (inReplyTo != null) && (inReplyTo.trim().length() > 0) ) - _preBodyBuffer.append("In reply to
\n"); + _preBodyBuffer.append(" In reply to\n"); if ( (_user != null) && (_user.getAuthenticated()) ) - _preBodyBuffer.append("Reply
\n"); - _preBodyBuffer.append("\n
\n"); + for (Iterator iter = remoteBlogs.iterator(); iter.hasNext(); ) { + Hash blog = (Hash)iter.next(); + buf = new StringBuffer(1024); + int shownEntries = 0; + buf.append("\n"); + buf.append("\n"); + List entries = new ArrayList(); + _remoteIndex.selectMatchesOrderByEntryId(entries, blog, null); + for (int i = 0; i < entries.size(); i++) { + BlogURI uri = (BlogURI)entries.get(i); + buf.append("\n"); + if (!archive.getIndex().getEntryIsKnown(uri)) { + buf.append("\n"); + newEntries++; + shownEntries++; + } else { + String page = HTMLRenderer.getPageURL(blog, null, uri.getEntryId(), -1, -1, + user.getShowExpanded(), user.getShowImages()); + buf.append("\n"); + } + buf.append("\n"); + buf.append("\n"); + buf.append("\n"); + buf.append("\n"); + buf.append("\n"); + } + if (shownEntries > 0) // skip blogs we have already syndicated + out.write(buf.toString()); + } + out.write("
\n"); + BlogInfo info = archive.getBlogInfo(blog); + if (info != null) { + buf.append("" + HTMLRenderer.sanitizeString(info.getProperty(BlogInfo.NAME)) + ": " + + HTMLRenderer.sanitizeString(info.getProperty(BlogInfo.DESCRIPTION)) + "\n"); + } else { + buf.append("" + blog.toBase64() + "\n"); + } + buf.append("
 Posted on#SizeTags
(local)" + getDate(uri.getEntryId()) + "" + getId(uri.getEntryId()) + "" + _remoteIndex.getBlogEntrySizeKB(uri) + "KB"); + for (Iterator titer = new TreeSet(_remoteIndex.getBlogEntryTags(uri)).iterator(); titer.hasNext(); ) { + String tag = (String)titer.next(); + buf.append("" + tag + " \n"); + } + buf.append("
\n"); + if (newEntries > 0) { + out.write(" \n"); + out.write(" \n"); + } else { + out.write(HTMLRenderer.sanitizeString(_remoteLocation) + " has no new posts to offer us\n"); + } + } + private final SimpleDateFormat _dateFormat = new SimpleDateFormat("yyyy/MM/dd"); + private String getDate(long when) { + synchronized (_dateFormat) { + return _dateFormat.format(new Date(when)); + } + } + + private long getId(long id) { + synchronized (_dateFormat) { + try { + String str = _dateFormat.format(new Date(id)); + long dayBegin = _dateFormat.parse(str).getTime(); + return (id - dayBegin); + } catch (ParseException pe) { + pe.printStackTrace(); + // wtf + return id; + } + } + } +} diff --git a/apps/syndie/jsp/_bodyindex.jsp b/apps/syndie/jsp/_bodyindex.jsp index d06e4cbc0..97fe883c8 100644 --- a/apps/syndie/jsp/_bodyindex.jsp +++ b/apps/syndie/jsp/_bodyindex.jsp @@ -2,7 +2,8 @@
Blogs: <%ArchiveViewerBean.renderBlogSelector(user, request.getParameterMap(), out);%> -
+ +
<%ArchiveViewerBean.renderBlogs(user, request.getParameterMap(), out); out.flush(); %> \ No newline at end of file diff --git a/apps/syndie/jsp/_topnav.jsp b/apps/syndie/jsp/_topnav.jsp index 0f403ebe5..151bd30ba 100644 --- a/apps/syndie/jsp/_topnav.jsp +++ b/apps/syndie/jsp/_topnav.jsp @@ -1,3 +1,3 @@ -		BlogsRemote archivesManageBlogsRemote archivesManage
diff --git a/apps/syndie/jsp/externallink.jsp b/apps/syndie/jsp/externallink.jsp index ed05f3c42..a3e390ac7 100644 --- a/apps/syndie/jsp/externallink.jsp +++ b/apps/syndie/jsp/externallink.jsp @@ -2,6 +2,7 @@ SyndieMedia +
diff --git a/apps/syndie/jsp/import.jsp b/apps/syndie/jsp/import.jsp new file mode 100644 index 000000000..4fce96933 --- /dev/null +++ b/apps/syndie/jsp/import.jsp @@ -0,0 +1,66 @@ +<%@page import="net.i2p.data.Base64, net.i2p.syndie.web.*, net.i2p.syndie.sml.*, net.i2p.syndie.data.*, net.i2p.syndie.*, org.mortbay.servlet.MultiPartRequest, java.util.*, java.io.*" %> + + + +SyndieMedia import + + + +
+ + + + + +
<% + +String contentType = request.getContentType(); +if ((contentType != null) && (contentType.indexOf("boundary=") != -1) ) { + MultiPartRequest req = new MultiPartRequest(request); + int metaId = 0; + while (true) { + InputStream meta = req.getInputStream("blogmeta" + metaId); + if (meta == null) + break; + if (!BlogManager.instance().importBlogMetadata(meta)) { + System.err.println("blog meta " + metaId + " failed to be imported"); + break; + } + metaId++; + } + int entryId = 0; + while (true) { + InputStream entry = req.getInputStream("blogpost" + entryId); + if (entry == null) + break; + if (!BlogManager.instance().importBlogEntry(entry)) { + System.err.println("blog entry " + entryId + " failed to be imported"); + break; + } + entryId++; + } + + if ( (entryId > 0) || (metaId > 0) ) { + BlogManager.instance().getArchive().regenerateIndex(); + session.setAttribute("index", BlogManager.instance().getArchive().getIndex()); + } +%>Imported <%=entryId%> posts and <%=metaId%> blog metadata files. +<% +} else { %>
+Blog metadata 0:
+Blog metadata 1:
+Post 0:
+Post 1:
+Post 2:
+Post 3:
+Post 4:
+Post 5:
+Post 6:
+Post 7:
+Post 8:
+Post 9:
+
+ +<% } %> +
+ diff --git a/apps/syndie/jsp/index.jsp b/apps/syndie/jsp/index.jsp index a2ec73d2f..54985c819 100644 --- a/apps/syndie/jsp/index.jsp +++ b/apps/syndie/jsp/index.jsp @@ -2,6 +2,7 @@ SyndieMedia + diff --git a/apps/syndie/jsp/post.jsp b/apps/syndie/jsp/post.jsp index 5ab5c8b79..1015d2531 100644 --- a/apps/syndie/jsp/post.jsp +++ b/apps/syndie/jsp/post.jsp @@ -1,8 +1,10 @@ <%@page import="net.i2p.data.Base64, net.i2p.syndie.web.*, net.i2p.syndie.sml.*, net.i2p.syndie.data.*, net.i2p.syndie.*, org.mortbay.servlet.MultiPartRequest, java.util.*" %> + SyndieMedia +
@@ -12,62 +14,76 @@ + +<% + } // end of the 'logged in, not confirmed, nothing posted' section + } // end of the 'logged in, not confirmed' section +} // end of the 'logged in' section +%>
<% -String contentType = request.getContentType(); -if ((contentType != null) && (contentType.indexOf("boundary=") != -1) ) { - if (!user.getAuthenticated()) { %>You must be logged in to post<% - } else { - MultiPartRequest req = new MultiPartRequest(request); - String entrySubject = req.getString("entrysubject"); - String entryTags = req.getString("entrytags"); - String entryText = req.getString("entrytext"); - String entryHeaders = req.getString("entryheaders"); - String replyTo = req.getString(ArchiveViewerBean.PARAM_IN_REPLY_TO); - if ( (replyTo != null) && (replyTo.trim().length() > 0) ) { - byte r[] = Base64.decode(replyTo); - if (r != null) { - if (entryHeaders == null) entryHeaders = HTMLRenderer.HEADER_IN_REPLY_TO + ": " + new String(r); - else entryHeaders = entryHeaders + '\n' + HTMLRenderer.HEADER_IN_REPLY_TO + ": " + new String(r); - } else { - replyTo = null; - } +if (!user.getAuthenticated()) { + %>You must be logged in to post<% +} else { + String confirm = request.getParameter("confirm"); + if ( (confirm != null) && (confirm.equalsIgnoreCase("true")) ) { + BlogURI uri = post.postEntry(); + if (uri != null) { + %>Blog entry posted!<% + } else { + %>There was an unknown error posting the entry...<% } - - List fileStreams = new ArrayList(); - List fileNames = new ArrayList(); - List fileTypes = new ArrayList(); - for (int i = 0; i < 32; i++) { - String filename = req.getFilename("entryfile" + i); - if ( (filename != null) && (filename.trim().length() > 0) ) { - fileNames.add(filename.trim()); - fileStreams.add(req.getInputStream("entryfile" + i)); - Hashtable params = req.getParams("entryfile" + i); - String type = "application/octet-stream"; - for (Iterator iter = params.keySet().iterator(); iter.hasNext(); ) { - String cur = (String)iter.next(); - if ("content-type".equalsIgnoreCase(cur)) { - type = (String)params.get(cur); - break; + post.reinitialize(); + post.setUser(user); + } else { + // logged in but not confirmed... + String contentType = request.getContentType(); + if ((contentType != null) && (contentType.indexOf("boundary=") != -1) ) { + // not confirmed but they posted stuff... gobble up what they give + // and display it as a preview (then we show the confirm form) + post.reinitialize(); + post.setUser(user); + + MultiPartRequest req = new MultiPartRequest(request); + String entrySubject = req.getString("entrysubject"); + String entryTags = req.getString("entrytags"); + String entryText = req.getString("entrytext"); + String entryHeaders = req.getString("entryheaders"); + String replyTo = req.getString(ArchiveViewerBean.PARAM_IN_REPLY_TO); + if ( (replyTo != null) && (replyTo.trim().length() > 0) ) { + byte r[] = Base64.decode(replyTo); + if (r != null) { + if (entryHeaders == null) entryHeaders = HTMLRenderer.HEADER_IN_REPLY_TO + ": " + new String(r); + else entryHeaders = entryHeaders + '\n' + HTMLRenderer.HEADER_IN_REPLY_TO + ": " + new String(r); + } else { + replyTo = null; } } - fileTypes.add(type); - } - } - - BlogURI entry = BlogManager.instance().createBlogEntry(user, entrySubject, entryTags, entryHeaders, entryText, fileNames, fileStreams, fileTypes); - if (entry != null) { - // it has been rebuilt... - request.setAttribute("index", BlogManager.instance().getArchive().getIndex()); -%> -Blog entry posted! -<% } else { %> -There was an error posting... dunno what it was... -<% } - } -} else { %> -Post subject:
-Post tags:
+ + post.setSubject(entrySubject); + post.setTags(entryTags); + post.setText(entryText); + post.setHeaders(entryHeaders); + + for (int i = 0; i < 32; i++) { + String filename = req.getFilename("entryfile" + i); + if ( (filename != null) && (filename.trim().length() > 0) ) { + Hashtable params = req.getParams("entryfile" + i); + String type = "application/octet-stream"; + for (Iterator iter = params.keySet().iterator(); iter.hasNext(); ) { + String cur = (String)iter.next(); + if ("content-type".equalsIgnoreCase(cur)) { + type = (String)params.get(cur); + break; + } + } + post.addAttachment(filename.trim(), req.getInputStream("entryfile" + i), type); + } + } + + post.renderPreview(out); + %>
Please confirm that this is ok. Otherwise, just go back and make changes.<% + } else { + // logged in and not confirmed because they didn't send us anything! + // give 'em a new form +%> +Post subject:
+Post tags:
Post content (in raw SML, no headers):
-
+
SML cheatsheet:

SML post headers:
-
<% +
<% String s = request.getParameter(ArchiveViewerBean.PARAM_IN_REPLY_TO); if ( (s != null) && (s.trim().length() > 0) ) {%> @@ -105,8 +121,11 @@ Attachment 7:
Attachment 8:
Attachment 9:
- -<% } %> -
diff --git a/apps/syndie/jsp/register.jsp b/apps/syndie/jsp/register.jsp index 4b514a167..9c4310984 100644 --- a/apps/syndie/jsp/register.jsp +++ b/apps/syndie/jsp/register.jsp @@ -3,6 +3,7 @@ SyndieMedia + diff --git a/apps/syndie/jsp/remote.jsp b/apps/syndie/jsp/remote.jsp new file mode 100644 index 000000000..8270c6418 --- /dev/null +++ b/apps/syndie/jsp/remote.jsp @@ -0,0 +1,58 @@ +<%@page contentType="text/html" import="net.i2p.syndie.web.*" %> + + + + + +SyndieMedia + + + +
+ + + + + +
+<% +if (!user.getAuthenticated() || !user.getAllowAccessRemote()) { +%>Sorry, you are not allowed to access remote archives from here. Perhaps you should install Syndie yourself?<% +} else { + %>Import from: + + +<% + String action = request.getParameter("action"); + if ("Continue...".equals(action)) { + remote.fetchIndex(user, request.getParameter("schema"), request.getParameter("location")); + } else if ("Fetch metadata".equals(action)) { + remote.fetchMetadata(user, request.getParameterMap()); + } else if ("Fetch selected entries".equals(action)) { + remote.fetchSelectedEntries(user, request.getParameterMap()); + } else if ("Fetch all new entries".equals(action)) { + remote.fetchAllEntries(user, request.getParameterMap()); + } + String msgs = remote.getStatus(); + if ( (msgs != null) && (msgs.length() > 0) ) { %>
<%=msgs%>
+Refresh

<% } + if (remote.getFetchIndexInProgress()) { %>Please wait while the index is being fetched +from <%=remote.getRemoteLocation()%>. <% + } else if (remote.getRemoteIndex() != null) { + // remote index is NOT null! + %><%=remote.getRemoteLocation()%>:
+<%remote.renderDeltaForm(user, archive, out);%> +<% + } +} +%> +
+ \ No newline at end of file diff --git a/apps/syndie/jsp/style.jsp b/apps/syndie/jsp/style.jsp new file mode 100644 index 000000000..43e5015d5 --- /dev/null +++ b/apps/syndie/jsp/style.jsp @@ -0,0 +1,2 @@ +<%@page contentType="text/css" %> +<%@include file="syndie.css" %> \ No newline at end of file diff --git a/apps/syndie/jsp/syndie.css b/apps/syndie/jsp/syndie.css new file mode 100644 index 000000000..b35a5d4a4 --- /dev/null +++ b/apps/syndie/jsp/syndie.css @@ -0,0 +1,67 @@ +.syndieEntrySubjectCell { + background-color: #999999; + font-size: 12px; + font-weight: bold; + margin: 0px 0px 0px 0px; + padding: 0px 0px 0px 0px; + border: 0px; +} +.syndieEntryMetaCell { + background-color: #888888; + font-size: 10px; + margin: 0px 0px 0px 0px; + padding: 0px 0px 0px 0px; + border: 0px; +} +.syndieEntryAttachmentsCell { + background-color: #aaaaaa; + font-size: 12px; + margin: 0px 0px 0px 0px; + padding: 0px 0px 0px 0px; + border: 0px; +} +.syndieEntrySummaryCell { + background-color: #eeeeee; + font-size: 12px; + margin: 0px 0px 0px 0px; + padding: 0px 0px 0px 0px; + border: 0px; +} +.syndieEntryBodyCell { + background-color: #eeeeee; + font-size: 12px; + margin: 0px 0px 0px 0px; + padding: 0px 0px 0px 0px; + border: 0px; +} +.syndieTopNavBlogsCell { + background-color: #888888; + font-size: 14px; + margin: 0px 0px 0px 0px; + padding: 0px 0px 0px 0px; + border: 0px; +} +.syndieTopNavRemoteCell { + background-color: #888888; + font-size: 14px; + margin: 0px 0px 0px 0px; + padding: 0px 0px 0px 0px; + border: 0px; +} +.syndieTopNavManageCell { + background-color: #888888; + font-size: 14px; + margin: 0px 0px 0px 0px; + padding: 0px 0px 0px 0px; + border: 0px; +} + +body { + margin : 0px; + padding : 0px; + text-align : center; + font-family: Arial, Helvetica, sans-serif; + background-color : #FFFFFF; + color: #000000; + font-size: 12px; +} diff --git a/apps/syndie/jsp/syndie/index.jsp b/apps/syndie/jsp/syndie/index.jsp new file mode 100644 index 000000000..5517346b6 --- /dev/null +++ b/apps/syndie/jsp/syndie/index.jsp @@ -0,0 +1 @@ +<%response.sendRedirect("../index.jsp");%> \ No newline at end of file diff --git a/apps/syndie/jsp/viewmetadata.jsp b/apps/syndie/jsp/viewmetadata.jsp index ede63cf52..7e97f99fe 100644 --- a/apps/syndie/jsp/viewmetadata.jsp +++ b/apps/syndie/jsp/viewmetadata.jsp @@ -2,6 +2,7 @@ SyndieMedia + diff --git a/apps/syndie/jsp/viewtempattachment.jsp b/apps/syndie/jsp/viewtempattachment.jsp new file mode 100644 index 000000000..f39b2e3fa --- /dev/null +++ b/apps/syndie/jsp/viewtempattachment.jsp @@ -0,0 +1,15 @@ +<%@page import="net.i2p.syndie.web.ArchiveViewerBean" %><% +String id = request.getParameter(ArchiveViewerBean.PARAM_ATTACHMENT); +if (id != null) { + try { + int attachmentId = Integer.parseInt(id); + if ( (attachmentId < 0) || (attachmentId >= post.getAttachmentCount()) ) { + %>Attachment <%=attachmentId%> does not exist<% + } else { + response.setContentType(post.getContentType(attachmentId)); + post.writeAttachmentData(attachmentId, response.getOutputStream()); + } + } catch (NumberFormatException nfe) {} +} +%> \ No newline at end of file diff --git a/build.xml b/build.xml index 24429ec04..a21a23aa5 100644 --- a/build.xml +++ b/build.xml @@ -59,6 +59,7 @@ + @@ -188,6 +189,7 @@ + @@ -286,6 +288,7 @@ + diff --git a/core/java/src/net/i2p/data/Base64.java b/core/java/src/net/i2p/data/Base64.java index b7da7ed51..a74f53a9e 100644 --- a/core/java/src/net/i2p/data/Base64.java +++ b/core/java/src/net/i2p/data/Base64.java @@ -152,6 +152,10 @@ public class Base64 { private static void runApp(String args[]) { try { + if ("encodestring".equalsIgnoreCase(args[0])) { + System.out.println(encode(args[1].getBytes())); + return; + } InputStream in = System.in; OutputStream out = System.out; if (args.length >= 3) { diff --git a/core/java/src/net/i2p/util/EepGetScheduler.java b/core/java/src/net/i2p/util/EepGetScheduler.java new file mode 100644 index 000000000..95a532eb3 --- /dev/null +++ b/core/java/src/net/i2p/util/EepGetScheduler.java @@ -0,0 +1,72 @@ +package net.i2p.util; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; +import net.i2p.I2PAppContext; + +/** + * + */ +public class EepGetScheduler implements EepGet.StatusListener { + private I2PAppContext _context; + private List _urls; + private List _localFiles; + private String _proxyHost; + private int _proxyPort; + private int _curURL; + private EepGet.StatusListener _listener; + + public EepGetScheduler(I2PAppContext ctx, List urls, List localFiles, String proxyHost, int proxyPort, EepGet.StatusListener lsnr) { + _context = ctx; + _urls = urls; + _localFiles = localFiles; + _proxyHost = proxyHost; + _proxyPort = proxyPort; + _curURL = -1; + _listener = lsnr; + } + + public void fetch() { + I2PThread t = new I2PThread(new Runnable() { public void run() { fetchNext(); } }, "EepGetScheduler"); + t.setDaemon(true); + t.start(); + } + + private void fetchNext() { + _curURL++; + if (_curURL >= _urls.size()) return; + String url = (String)_urls.get(_curURL); + String out = EepGet.suggestName(url); + if ( (_localFiles != null) && (_localFiles.size() > _curURL) ) { + File f = (File)_localFiles.get(_curURL); + out = f.getAbsolutePath(); + } else { + if (_localFiles == null) + _localFiles = new ArrayList(_urls.size()); + _localFiles.add(new File(out)); + } + EepGet get = new EepGet(_context, ((_proxyHost != null) && (_proxyPort > 0)), _proxyHost, _proxyPort, 0, out, url); + get.addStatusListener(this); + get.fetch(); + } + + public void attemptFailed(String url, long bytesTransferred, long bytesRemaining, int currentAttempt, int numRetries, Exception cause) { + _listener.attemptFailed(url, bytesTransferred, bytesRemaining, currentAttempt, numRetries, cause); + } + + public void bytesTransferred(long alreadyTransferred, int currentWrite, long bytesTransferred, long bytesRemaining, String url) { + _listener.bytesTransferred(alreadyTransferred, currentWrite, bytesTransferred, bytesRemaining, url); + } + + public void transferComplete(long alreadyTransferred, long bytesTransferred, long bytesRemaining, String url, String outputFile) { + _listener.transferComplete(alreadyTransferred, bytesTransferred, bytesRemaining, url, outputFile); + fetchNext(); + } + + public void transferFailed(String url, long bytesTransferred, long bytesRemaining, int currentAttempt) { + _listener.transferFailed(url, bytesTransferred, bytesRemaining, currentAttempt); + fetchNext(); + } + +} diff --git a/history.txt b/history.txt index 8dc717584..879e8c1bb 100644 --- a/history.txt +++ b/history.txt @@ -1,4 +1,14 @@ -$Id: history.txt,v 1.227 2005/08/17 15:05:03 jrandom Exp $ +$Id: history.txt,v 1.228 2005/08/21 13:39:06 jrandom Exp $ + +2005-08-23 jrandom + * Removed the concept of "no bandwidth limit" - if none is specified, its + 16KBps in/out. + * Include ack packets in the per-peer cwin throttle (they were part of the + bandwidth limit though). + * Tweak the SSU cwin operation to get more accurrate estimates under + congestions. + * SSU improvements to resend more efficiently. + * Added a basic scheduler to eepget to fetch multiple files sequentially. * 2005-08-21 0.6.0.3 released diff --git a/router/java/src/net/i2p/router/RouterVersion.java b/router/java/src/net/i2p/router/RouterVersion.java index babaaa0a2..41f84e589 100644 --- a/router/java/src/net/i2p/router/RouterVersion.java +++ b/router/java/src/net/i2p/router/RouterVersion.java @@ -15,9 +15,9 @@ import net.i2p.CoreVersion; * */ public class RouterVersion { - public final static String ID = "$Revision: 1.216 $ $Date: 2005/08/17 15:05:03 $"; + public final static String ID = "$Revision: 1.217 $ $Date: 2005/08/21 13:39:05 $"; public final static String VERSION = "0.6.0.3"; - public final static long BUILD = 0; + public final static long BUILD = 1; public static void main(String args[]) { System.out.println("I2P Router version: " + VERSION); System.out.println("Router ID: " + RouterVersion.ID); diff --git a/router/java/src/net/i2p/router/transport/FIFOBandwidthLimiter.java b/router/java/src/net/i2p/router/transport/FIFOBandwidthLimiter.java index f2f865486..c8a52d195 100644 --- a/router/java/src/net/i2p/router/transport/FIFOBandwidthLimiter.java +++ b/router/java/src/net/i2p/router/transport/FIFOBandwidthLimiter.java @@ -119,7 +119,9 @@ public class FIFOBandwidthLimiter { */ final void refillBandwidthQueues(long bytesInbound, long bytesOutbound) { if (_log.shouldLog(Log.DEBUG)) - _log.debug("Refilling the queues with " + bytesInbound + "/" + bytesOutbound); + _log.debug("Refilling the queues with " + bytesInbound + "/" + bytesOutbound + ", available " + + _availableInboundBytes + '/' + _availableOutboundBytes + ", max " + + _maxInboundBytes + '/' + _maxOutboundBytes); _availableInboundBytes += bytesInbound; _availableOutboundBytes += bytesOutbound; if (_availableInboundBytes > _maxInboundBytes) { diff --git a/router/java/src/net/i2p/router/transport/FIFOBandwidthRefiller.java b/router/java/src/net/i2p/router/transport/FIFOBandwidthRefiller.java index 2c648c572..6e2476548 100644 --- a/router/java/src/net/i2p/router/transport/FIFOBandwidthRefiller.java +++ b/router/java/src/net/i2p/router/transport/FIFOBandwidthRefiller.java @@ -25,15 +25,21 @@ class FIFOBandwidthRefiller implements Runnable { public static final String PROP_INBOUND_BANDWIDTH_PEAK = "i2np.bandwidth.inboundBurstKBytes"; public static final String PROP_OUTBOUND_BANDWIDTH_PEAK = "i2np.bandwidth.outboundBurstKBytes"; //public static final String PROP_REPLENISH_FREQUENCY = "i2np.bandwidth.replenishFrequencyMs"; - + + // no longer allow unlimited bandwidth - the user must specify a value, and if they do not, it is 16KBps + public static final int DEFAULT_INBOUND_BANDWIDTH = 16; + public static final int DEFAULT_OUTBOUND_BANDWIDTH = 16; + + public static final int DEFAULT_BURST_SECONDS = 60; + /** For now, until there is some tuning and safe throttling, we set the floor at 6KBps inbound */ - public static final int MIN_INBOUND_BANDWIDTH = 1; + public static final int MIN_INBOUND_BANDWIDTH = 5; /** For now, until there is some tuning and safe throttling, we set the floor at 6KBps outbound */ - public static final int MIN_OUTBOUND_BANDWIDTH = 1; + public static final int MIN_OUTBOUND_BANDWIDTH = 5; /** For now, until there is some tuning and safe throttling, we set the floor at a 10 second burst */ - public static final int MIN_INBOUND_BANDWIDTH_PEAK = 1; + public static final int MIN_INBOUND_BANDWIDTH_PEAK = 10; /** For now, until there is some tuning and safe throttling, we set the floor at a 10 second burst */ - public static final int MIN_OUTBOUND_BANDWIDTH_PEAK = 1; + public static final int MIN_OUTBOUND_BANDWIDTH_PEAK = 10; /** Updating the bandwidth more than once a second is silly. once every 2 or 5 seconds is less so. */ public static final long MIN_REPLENISH_FREQUENCY = 100; @@ -146,6 +152,8 @@ class FIFOBandwidthRefiller implements Runnable { _inboundKBytesPerSecond = in; else _inboundKBytesPerSecond = MIN_INBOUND_BANDWIDTH; + if (_log.shouldLog(Log.DEBUG)) + _log.debug("Updating inbound rate to " + _inboundKBytesPerSecond); } catch (NumberFormatException nfe) { if (_log.shouldLog(Log.WARN)) _log.warn("Invalid inbound bandwidth limit [" + inBwStr @@ -155,6 +163,9 @@ class FIFOBandwidthRefiller implements Runnable { if ( (inBwStr == null) && (_log.shouldLog(Log.DEBUG)) ) _log.debug("Inbound bandwidth limits not specified in the config via " + PROP_INBOUND_BANDWIDTH); } + + if (_inboundKBytesPerSecond <= 0) + _inboundKBytesPerSecond = DEFAULT_INBOUND_BANDWIDTH; } private void updateOutboundRate() { String outBwStr = _context.getProperty(PROP_OUTBOUND_BANDWIDTH); @@ -169,6 +180,8 @@ class FIFOBandwidthRefiller implements Runnable { _outboundKBytesPerSecond = out; else _outboundKBytesPerSecond = MIN_OUTBOUND_BANDWIDTH; + if (_log.shouldLog(Log.DEBUG)) + _log.debug("Updating outbound rate to " + _outboundKBytesPerSecond); } catch (NumberFormatException nfe) { if (_log.shouldLog(Log.WARN)) _log.warn("Invalid outbound bandwidth limit [" + outBwStr @@ -178,6 +191,9 @@ class FIFOBandwidthRefiller implements Runnable { if ( (outBwStr == null) && (_log.shouldLog(Log.DEBUG)) ) _log.debug("Outbound bandwidth limits not specified in the config via " + PROP_OUTBOUND_BANDWIDTH); } + + if (_outboundKBytesPerSecond <= 0) + _outboundKBytesPerSecond = DEFAULT_OUTBOUND_BANDWIDTH; } private void updateInboundPeak() { @@ -203,11 +219,13 @@ class FIFOBandwidthRefiller implements Runnable { if (_log.shouldLog(Log.WARN)) _log.warn("Invalid inbound bandwidth burst limit [" + inBwStr + "]"); + _limiter.setMaxInboundBytes(DEFAULT_BURST_SECONDS * _inboundKBytesPerSecond * 1024); } } else { if (_log.shouldLog(Log.DEBUG)) _log.debug("Inbound bandwidth burst limits not specified in the config via " + PROP_INBOUND_BANDWIDTH_PEAK); + _limiter.setMaxInboundBytes(DEFAULT_BURST_SECONDS * _inboundKBytesPerSecond * 1024); } } private void updateOutboundPeak() { @@ -233,11 +251,13 @@ class FIFOBandwidthRefiller implements Runnable { if (_log.shouldLog(Log.WARN)) _log.warn("Invalid outbound bandwidth burst limit [" + outBwStr + "]"); + _limiter.setMaxOutboundBytes(DEFAULT_BURST_SECONDS * _outboundKBytesPerSecond * 1024); } } else { if (_log.shouldLog(Log.DEBUG)) _log.debug("Outbound bandwidth burst limits not specified in the config via " + PROP_OUTBOUND_BANDWIDTH_PEAK); + _limiter.setMaxOutboundBytes(DEFAULT_BURST_SECONDS * _outboundKBytesPerSecond * 1024); } } diff --git a/router/java/src/net/i2p/router/transport/udp/EstablishmentManager.java b/router/java/src/net/i2p/router/transport/udp/EstablishmentManager.java index 31f432662..1efc91fa6 100644 --- a/router/java/src/net/i2p/router/transport/udp/EstablishmentManager.java +++ b/router/java/src/net/i2p/router/transport/udp/EstablishmentManager.java @@ -575,6 +575,8 @@ public class EstablishmentManager { if (outboundState != null) { if (outboundState.getLifetime() > MAX_ESTABLISH_TIME) { if (outboundState.getState() != OutboundEstablishState.STATE_CONFIRMED_COMPLETELY) { + if (_log.shouldLog(Log.WARN)) + _log.warn("Lifetime of expired outbound establish: " + outboundState.getLifetime()); while (true) { OutNetMessage msg = outboundState.getNextQueuedMessage(); if (msg == null) diff --git a/router/java/src/net/i2p/router/transport/udp/OutboundMessageFragments.java b/router/java/src/net/i2p/router/transport/udp/OutboundMessageFragments.java index 8b3d5bb01..8c7bc7846 100644 --- a/router/java/src/net/i2p/router/transport/udp/OutboundMessageFragments.java +++ b/router/java/src/net/i2p/router/transport/udp/OutboundMessageFragments.java @@ -326,7 +326,7 @@ public class OutboundMessageFragments { state.push(); - int rto = peer.getRTO() * state.getPushCount(); + int rto = peer.getRTO();// * state.getPushCount(); state.setNextSendTime(now + rto); if (peer.getSendWindowBytesRemaining() > 0) @@ -338,7 +338,7 @@ public class OutboundMessageFragments { _log.warn("Allocation of " + size + " rejected w/ wsize=" + peer.getSendWindowBytes() + " available=" + peer.getSendWindowBytesRemaining() + " for message " + state.getMessageId() + ": " + state); - state.setNextSendTime((now + 1024) & ~SECOND_MASK); + state.setNextSendTime(now+(_context.random().nextInt(2*ACKSender.ACK_FREQUENCY))); //(now + 1024) & ~SECOND_MASK); if (_log.shouldLog(Log.WARN)) _log.warn("Retransmit after choke for next send time in " + (state.getNextSendTime()-now) + "ms"); _throttle.choke(peer.getRemotePeer()); @@ -435,7 +435,7 @@ public class OutboundMessageFragments { PeerState peer = state.getPeer(); if (peer != null) { // this adjusts the rtt/rto/window/etc - peer.messageACKed(numFragments*state.getFragmentSize(), state.getLifetime(), state.getMaxSends()); + peer.messageACKed(numFragments*state.getFragmentSize(), state.getLifetime(), numSends); if (peer.getSendWindowBytesRemaining() > 0) _throttle.unchoke(peer.getRemotePeer()); } else { diff --git a/router/java/src/net/i2p/router/transport/udp/PeerState.java b/router/java/src/net/i2p/router/transport/udp/PeerState.java index 7aea8613f..67bb89221 100644 --- a/router/java/src/net/i2p/router/transport/udp/PeerState.java +++ b/router/java/src/net/i2p/router/transport/udp/PeerState.java @@ -93,6 +93,10 @@ public class PeerState { private int _sendBytes; private int _receiveBps; private int _receiveBytes; + private int _sendACKBps; + private int _sendACKBytes; + private int _receiveACKBps; + private int _receiveACKBytes; private long _receivePeriodBegin; private volatile long _lastCongestionOccurred; /** @@ -141,8 +145,11 @@ public class PeerState { private long _packetsTransmitted; /** how many packets were retransmitted within the last RETRANSMISSION_PERIOD_WIDTH packets */ private long _packetsRetransmitted; + /** how many packets were transmitted within the last RETRANSMISSION_PERIOD_WIDTH packets */ + private long _packetsPeriodTransmitted; + private int _packetsPeriodRetransmitted; private int _packetRetransmissionRate; - /** what was the $packetsTransmitted when the current RETRANSMISSION_PERIOD_WIDTH began */ + /** at what time did we last break off the retransmission counter period */ private long _retransmissionPeriodStart; /** how many dup packets were received within the last RETRANSMISSION_PERIOD_WIDTH packets */ private long _packetsReceivedDuplicate; @@ -163,7 +170,7 @@ public class PeerState { * of 608 */ private static final int DEFAULT_MTU = 608;//600; //1500; - private static final int MIN_RTO = 1000 + ACKSender.ACK_FREQUENCY; + private static final int MIN_RTO = 500 + ACKSender.ACK_FREQUENCY; private static final int MAX_RTO = 3000; // 5000; public PeerState(I2PAppContext ctx) { @@ -373,6 +380,10 @@ public class PeerState { return _consecutiveFailedSends; } + /** how fast we are sending *ack* packets */ + public int getSendACKBps() { return _sendACKBps; } + public int getReceiveACKBps() { return _receiveACKBps; } + /** * have all of the packets received in the current second requested that * the previous second's ACKs be sent? @@ -384,14 +395,20 @@ public class PeerState { * cannot. If it is not decremented, the window size remaining is * not adjusted at all. */ - public boolean allocateSendingBytes(int size) { + public boolean allocateSendingBytes(int size) { return allocateSendingBytes(size, false); } + public boolean allocateSendingBytes(int size, boolean isForACK) { long now = _context.clock().now(); long duration = now - _lastSendRefill; if (duration >= 1000) { _sendWindowBytesRemaining = _sendWindowBytes; _sendBytes += size; _sendBps = (int)(0.9f*(float)_sendBps + 0.1f*((float)_sendBytes * (1000f/(float)duration))); + if (isForACK) { + _sendACKBytes += size; + _sendACKBps = (int)(0.9f*(float)_sendACKBps + 0.1f*((float)_sendACKBytes * (1000f/(float)duration))); + } _sendBytes = 0; + _sendACKBytes = 0; _lastSendRefill = now; } //if (true) return true; @@ -399,6 +416,8 @@ public class PeerState { _sendWindowBytesRemaining -= size; _sendBytes += size; _lastSendTime = now; + if (isForACK) + _sendACKBytes += size; return true; } else { return false; @@ -432,14 +451,17 @@ public class PeerState { public int getSlowStartThreshold() { return _slowStartThreshold; } /** we received the message specified completely */ - public void messageFullyReceived(Long messageId, int bytes) { - if (bytes > 0) + public void messageFullyReceived(Long messageId, int bytes) { messageFullyReceived(messageId, bytes, false); } + public void messageFullyReceived(Long messageId, int bytes, boolean isForACK) { + if (bytes > 0) { _receiveBytes += bytes; - else { - if (_retransmissionPeriodStart + RETRANSMISSION_PERIOD_WIDTH < _packetsReceived) { + if (isForACK) + _receiveACKBytes += bytes; + } else { + if (_retransmissionPeriodStart + 1000 < _context.clock().now()) { _packetsReceivedDuplicate++; } else { - _retransmissionPeriodStart = _packetsReceived; + _retransmissionPeriodStart = _context.clock().now(); _packetsReceivedDuplicate = 1; } } @@ -448,6 +470,9 @@ public class PeerState { long duration = now - _receivePeriodBegin; if (duration >= 1000) { _receiveBps = (int)(0.9f*(float)_receiveBps + 0.1f*((float)_receiveBytes * (1000f/(float)duration))); + if (isForACK) + _receiveACKBps = (int)(0.9f*(float)_receiveACKBps + 0.1f*((float)_receiveACKBytes * (1000f/(float)duration))); + _receiveACKBytes = 0; _receiveBytes = 0; _receivePeriodBegin = now; _context.statManager().addRateData("udp.receiveBps", _receiveBps, 0); @@ -480,20 +505,21 @@ public class PeerState { */ private boolean congestionOccurred() { long now = _context.clock().now(); - if (_lastCongestionOccurred + 10*1000 > now) - return false; // only shrink once every 10 seconds + if (_lastCongestionOccurred + 5*1000 > now) + return false; // only shrink once every 5 seconds _lastCongestionOccurred = now; _context.statManager().addRateData("udp.congestionOccurred", _sendWindowBytes, _sendBps); + int congestionAt = _sendWindowBytes; //if (true) // _sendWindowBytes -= 10000; //else - _sendWindowBytes = (_sendWindowBytes*2) / 3; + _sendWindowBytes = _sendWindowBytes/4; //(_sendWindowBytes*2) / 3; if (_sendWindowBytes < MINIMUM_WINDOW_BYTES) _sendWindowBytes = MINIMUM_WINDOW_BYTES; - if (_sendWindowBytes < _slowStartThreshold) - _slowStartThreshold = _sendWindowBytes; + //if (congestionAt/2 < _slowStartThreshold) + _slowStartThreshold = congestionAt/2; return true; } @@ -595,24 +621,34 @@ public class PeerState { public void messageACKed(int bytesACKed, long lifetime, int numSends) { _consecutiveFailedSends = 0; _lastFailedSendPeriod = -1; - if (_sendWindowBytes <= _slowStartThreshold) { - _sendWindowBytes += bytesACKed; - } else { - double prob = ((double)bytesACKed) / ((double)_sendWindowBytes); - if (_context.random().nextDouble() <= prob) + if (numSends < 2) { + if (_sendWindowBytes <= _slowStartThreshold) { _sendWindowBytes += bytesACKed; + } else { + if (false) { + _sendWindowBytes += 16; // why 16? + } else { + float prob = ((float)bytesACKed) / ((float)_sendWindowBytes); + float v = _context.random().nextFloat(); + if (v < 0) v = 0-v; + if (v <= prob) + _sendWindowBytes += bytesACKed; + } + } } if (_sendWindowBytes > MAX_SEND_WINDOW_BYTES) _sendWindowBytes = MAX_SEND_WINDOW_BYTES; _lastReceiveTime = _context.clock().now(); - if (_sendWindowBytesRemaining + bytesACKed <= _sendWindowBytes) - _sendWindowBytesRemaining += bytesACKed; - else - _sendWindowBytesRemaining = _sendWindowBytes; + if (false) { + if (_sendWindowBytesRemaining + bytesACKed <= _sendWindowBytes) + _sendWindowBytesRemaining += bytesACKed; + else + _sendWindowBytesRemaining = _sendWindowBytes; + } _messagesSent++; - if (numSends <= 2) + if (numSends < 2) recalculateTimeouts(lifetime); else _log.warn("acked after numSends=" + numSends + " w/ lifetime=" + lifetime + " and size=" + bytesACKed); @@ -643,11 +679,14 @@ public class PeerState { /** we are resending a packet, so lets jack up the rto */ public void messageRetransmitted(int packets) { - if (_retransmissionPeriodStart + RETRANSMISSION_PERIOD_WIDTH < _packetsTransmitted) { + long now = _context.clock().now(); + if (_retransmissionPeriodStart + 1000 <= now) { _packetsRetransmitted += packets; } else { _packetRetransmissionRate = (int)((float)(0.9f*_packetRetransmissionRate) + (float)(0.1f*_packetsRetransmitted)); - _retransmissionPeriodStart = _packetsTransmitted; + //_packetsPeriodTransmitted = _packetsTransmitted - _retransmissionPeriodStart; + _packetsPeriodRetransmitted = (int)_packetsRetransmitted; + _retransmissionPeriodStart = now; _packetsRetransmitted = packets; } congestionOccurred(); @@ -655,10 +694,13 @@ public class PeerState { //_rto *= 2; } public void packetsTransmitted(int packets) { + long now = _context.clock().now(); _packetsTransmitted += packets; - if (_retransmissionPeriodStart + RETRANSMISSION_PERIOD_WIDTH > _packetsTransmitted) { + //_packetsPeriodTransmitted += packets; + if (_retransmissionPeriodStart + 1000 <= now) { _packetRetransmissionRate = (int)((float)(0.9f*_packetRetransmissionRate) + (float)(0.1f*_packetsRetransmitted)); - _retransmissionPeriodStart = _packetsTransmitted; + _retransmissionPeriodStart = 0; + _packetsPeriodRetransmitted = (int)_packetsRetransmitted; _packetsRetransmitted = 0; } } @@ -673,6 +715,8 @@ public class PeerState { public long getMessagesReceived() { return _messagesReceived; } public long getPacketsTransmitted() { return _packetsTransmitted; } public long getPacketsRetransmitted() { return _packetsRetransmitted; } + public long getPacketsPeriodTransmitted() { return _packetsPeriodTransmitted; } + public int getPacketsPeriodRetransmitted() { return _packetsPeriodRetransmitted; } /** avg number of packets retransmitted for every 100 packets */ public long getPacketRetransmissionRate() { return _packetRetransmissionRate; } public long getPacketsReceived() { return _packetsReceived; } diff --git a/router/java/src/net/i2p/router/transport/udp/PeerTestManager.java b/router/java/src/net/i2p/router/transport/udp/PeerTestManager.java index b57c36bc4..aa73ded77 100644 --- a/router/java/src/net/i2p/router/transport/udp/PeerTestManager.java +++ b/router/java/src/net/i2p/router/transport/udp/PeerTestManager.java @@ -132,6 +132,7 @@ class PeerTestManager { */ private void receiveTestReply(RemoteHostId from, UDPPacketReader.PeerTestReader testInfo) { PeerTestState test = _currentTest; + if (test == null) return; if ( (DataHelper.eq(from.getIP(), test.getBobIP().getAddress())) && (from.getPort() == test.getBobPort()) ) { byte ip[] = new byte[testInfo.readIPSize()]; testInfo.readIP(ip, 0); diff --git a/router/java/src/net/i2p/router/transport/udp/UDPReceiver.java b/router/java/src/net/i2p/router/transport/udp/UDPReceiver.java index b3efba8ba..c0dd5e4f6 100644 --- a/router/java/src/net/i2p/router/transport/udp/UDPReceiver.java +++ b/router/java/src/net/i2p/router/transport/udp/UDPReceiver.java @@ -40,9 +40,12 @@ public class UDPReceiver { _runner = new Runner(); _context.statManager().createRateStat("udp.receivePacketSize", "How large packets received are", "udp", new long[] { 60*1000, 10*60*1000, 60*60*1000 }); _context.statManager().createRateStat("udp.droppedInbound", "How many packet are queued up but not yet received when we drop", "udp", new long[] { 60*1000, 10*60*1000, 60*60*1000 }); + _context.statManager().createRateStat("udp.droppedInboundProbabalistically", "How many packet we drop probabalistically (to simulate failures)", "udp", new long[] { 60*1000, 5*60*1000, 10*60*1000, 60*60*1000 }); + _context.statManager().createRateStat("udp.acceptedInboundProbabalistically", "How many packet we accept probabalistically (to simulate failures)", "udp", new long[] { 60*1000, 5*60*1000, 10*60*1000, 60*60*1000 }); } public void startup() { + adjustDropProbability(); _keepRunning = true; I2PThread t = new I2PThread(_runner, _name); t.setDaemon(true); @@ -57,6 +60,18 @@ public class UDPReceiver { } } + private void adjustDropProbability() { + String p = _context.getProperty("i2np.udp.dropProbability"); + if (p != null) { + try { + ARTIFICIAL_DROP_PROBABILITY = Float.parseFloat(p); + } catch (NumberFormatException nfe) {} + if (ARTIFICIAL_DROP_PROBABILITY < 0) ARTIFICIAL_DROP_PROBABILITY = 0; + } else { + ARTIFICIAL_DROP_PROBABILITY = 0; + } + } + /** * Replace the old listen port with the new one, returning the old. * NOTE: this closes the old socket so that blocking calls unblock! @@ -69,17 +84,26 @@ public class UDPReceiver { /** if a packet been sitting in the queue for a full second (meaning the handlers are overwhelmed), drop subsequent packets */ private static final long MAX_QUEUE_PERIOD = 1*1000; - private static final float ARTIFICIAL_DROP_PROBABILITY = 0.0f; // 0.02f; // 0.0f; + private static float ARTIFICIAL_DROP_PROBABILITY = 0.0f; // 0.02f; // 0.0f; private static final int ARTIFICIAL_DELAY = 0; // 100; private static final int ARTIFICIAL_DELAY_BASE = 0; //100; private int receive(UDPPacket packet) { + //adjustDropProbability(); + if (ARTIFICIAL_DROP_PROBABILITY > 0) { // the first check is to let the compiler optimize away this // random block on the live system when the probability is == 0 - if (_context.random().nextFloat() <= ARTIFICIAL_DROP_PROBABILITY) + int v = _context.random().nextInt(1000); + if (v < ARTIFICIAL_DROP_PROBABILITY*1000) { + if (_log.shouldLog(Log.ERROR)) + _log.error("Drop with v=" + v + " p=" + ARTIFICIAL_DROP_PROBABILITY + " packet size: " + packet.getPacket().getLength()); + _context.statManager().addRateData("udp.droppedInboundProbabalistically", 1, 0); return -1; + } else { + _context.statManager().addRateData("udp.acceptedInboundProbabalistically", 1, 0); + } } if ( (ARTIFICIAL_DELAY > 0) || (ARTIFICIAL_DELAY_BASE > 0) ) {