echelon/i2p.i2p
1
0
Fork 0
forked from I2P_Developers/i2p.i2p
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SusiMail: Error handling fixes

Browse Source 
More tolerant parsing of Date headers
Set a date if we don't get a valid Date header
Fix parsing long Base64 encoded headers
Fix page count after changing page size
Make attribute name parsing case-insensitive
Import mail method for debugging
Debug and log tweaks
This commit is contained in:
zzz
2018-02-08 14:46:41 +00:00
parent 7da3de20aa
commit 8161f099d2
9 changed files with 216 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
apps/susimail/src/src/i2p/susi/util/FixCRLFOutputStream.java Normal file
View File

@ -0,0 +1,28 @@
package i2p.susi.util;
import java.io.FilterOutputStream;
import java.io.IOException;
import java.io.OutputStream;
/**
* Replace plain \n with \r\n on the fly.
* Used when importing .eml files.
*
* @since 0.9.34
*/
public class FixCRLFOutputStream extends FilterOutputStream {
private int previous = -1;
public FixCRLFOutputStream(OutputStream out) {
super(out);
}
@Override
public void write(int val) throws IOException {
if (val == '\n' && previous != '\r')
out.write('\r');
out.write(val);
previous = val;
}
}

45
apps/susimail/src/src/i2p/susi/webmail/Mail.java
View File

@ -28,6 +28,7 @@ import i2p.susi.util.Buffer;
import i2p.susi.util.Config;
import i2p.susi.util.CountingInputStream;
import i2p.susi.util.EOFOnMatchInputStream;
import i2p.susi.util.FileBuffer;
import i2p.susi.util.MemoryBuffer;
import i2p.susi.webmail.encoding.Encoding;
import i2p.susi.webmail.encoding.EncodingFactory;
@ -46,8 +47,10 @@ import java.util.Locale;
import java.util.TimeZone;
import java.util.regex.Pattern;
import net.i2p.I2PAppContext;
import net.i2p.data.DataHelper;
import net.i2p.servlet.util.ServletUtil;
import net.i2p.util.RFC822Date;
import net.i2p.util.SystemVersion;
/**
@ -129,6 +132,16 @@ class Mail {
String[] rv = parseHeaders(in);
if (closeIn)
rb.readComplete(true);
// set a date if we didn't get one in the headers
if (date == null) {
long dateLong;
if (rb instanceof FileBuffer) {
dateLong = ((FileBuffer) rb).getFile().lastModified();
} else {
dateLong = I2PAppContext.getGlobalContext().clock().now();
}
setDate(dateLong);
}
return rv;
}
@ -175,7 +188,7 @@ class Mail {
} catch (RuntimeException e) {
Debug.debug(Debug.ERROR, "Parse error", e);
} finally {
try { in.close(); } catch (IOException ioe) {}
if (in != null) try { in.close(); } catch (IOException ioe) {}
rb.readComplete(success);
}
}
@ -323,7 +336,6 @@ class Mail {
private static final DateFormat dateFormatter = new SimpleDateFormat("yyyy-MM-dd HH:mm");
private static final DateFormat localDateFormatter = DateFormat.getDateTimeInstance(DateFormat.SHORT, DateFormat.SHORT);
private static final DateFormat longLocalDateFormatter = DateFormat.getDateTimeInstance(DateFormat.MEDIUM, DateFormat.MEDIUM);
private static final DateFormat mailDateFormatter = new SimpleDateFormat("EEE, d MMM yyyy HH:mm:ss Z", Locale.ENGLISH );
static {
// the router sets the JVM time zone to UTC but saves the original here so we can get it
TimeZone tz = SystemVersion.getSystemTimeZone();
@ -331,6 +343,19 @@ class Mail {
longLocalDateFormatter.setTimeZone(tz);
}
/**
* @param dateLong non-negative
* @since 0.9.34 pulled from parseHeaders()
*/
private void setDate(long dateLong) {
date = new Date(dateLong);
synchronized(dateFormatter) {
formattedDate = dateFormatter.format( date );
localFormattedDate = localDateFormatter.format( date );
quotedDate = longLocalDateFormatter.format(date);
}
}
/**
* @return all headers, to pass to MailPart, or null on error
*/
@ -390,19 +415,9 @@ class Mail {
}
else if (hlc.startsWith("date:")) {
dateString = line.substring( 5 ).trim();
try {
synchronized(mailDateFormatter) {
date = mailDateFormatter.parse( dateString );
formattedDate = dateFormatter.format( date );
localFormattedDate = localDateFormatter.format( date );
//quotedDate = html.encode( dateString );
quotedDate = longLocalDateFormatter.format(date);
}
}
catch (ParseException e) {
date = null;
e.printStackTrace();
}
long dateLong = RFC822Date.parse822Date(dateString);
if (dateLong > 0)
setDate(dateLong);
}
else if (hlc.startsWith("subject:")) {
subject = line.substring( 8 ).trim();

2
apps/susimail/src/src/i2p/susi/webmail/MailCache.java
View File

@ -72,7 +72,7 @@ class MailCache {
mails = new Hashtable<String, Mail>();
PersistentMailCache pmc = null;
try {
pmc = new PersistentMailCache(host, port, user, pass, PersistentMailCache.DIR_FOLDER);
pmc = new PersistentMailCache(ctx, host, port, user, pass, PersistentMailCache.DIR_FOLDER);
// TODO Drafts, Sent, Trash
} catch (IOException ioe) {
Debug.debug(Debug.ERROR, "Error creating disk cache: " + ioe);

13
apps/susimail/src/src/i2p/susi/webmail/MailPart.java
View File

@ -202,10 +202,10 @@ class MailPart {
OutputStream dummy = new DummyOutputStream();
DataHelper.copy(eofin, dummy);
if (!eofin.wasFound())
Debug.debug(Debug.DEBUG, "EOF hit before first boundary " + boundary);
Debug.debug(Debug.DEBUG, "EOF hit before first boundary " + boundary + " UIDL: " + uidl);
if (readBoundaryTrailer(in)) {
if (!eofin.wasFound())
Debug.debug(Debug.DEBUG, "EOF hit before first part body " + boundary);
Debug.debug(Debug.DEBUG, "EOF hit before first part body " + boundary + " UIDL: " + uidl);
tmpEnd = (int) eofin.getRead();
break;
}
@ -220,7 +220,7 @@ class MailPart {
// if MailPart contains a MailPart, we may not have drained to the end
DataHelper.copy(eofin, DUMMY_OUTPUT);
if (!eofin.wasFound())
Debug.debug(Debug.DEBUG, "EOF hit before end of body " + i + " boundary: " + boundary);
Debug.debug(Debug.DEBUG, "EOF hit before end of body " + i + " boundary: " + boundary + " UIDL: " + uidl);
}
if (readBoundaryTrailer(in))
break;
@ -351,13 +351,18 @@ class MailPart {
return result;
}
/**
* @param attributeName must be lower case, will be matched case-insensitively
* @return as found, not necessarily lower case
*/
private static String getHeaderLineAttribute( String line, String attributeName )
{
String lineLC = line.toLowerCase(Locale.US);
String result = null;
int h = 0;
int l = attributeName.length();
while( true ) {
int i = line.indexOf( attributeName, h );
int i = lineLC.indexOf(attributeName, h);
// System.err.println( "i=" + i );
if( i == -1 )
break;

83
apps/susimail/src/src/i2p/susi/webmail/PersistentMailCache.java
View File

@ -4,6 +4,7 @@ import i2p.susi.debug.Debug;
import i2p.susi.webmail.Messages;
import i2p.susi.util.Buffer;
import i2p.susi.util.FileBuffer;
import i2p.susi.util.FixCRLFOutputStream;
import i2p.susi.util.GzipFileBuffer;
import i2p.susi.util.ReadBuffer;
@ -19,6 +20,7 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.Hashtable;
import java.util.List;
import java.util.Locale;
import java.util.concurrent.ConcurrentHashMap;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;
@ -60,6 +62,7 @@ class PersistentMailCache {
private final Object _lock;
private final File _cacheDir;
private final I2PAppContext _context;
private static final String DIR_SUSI = "susimail";
private static final String DIR_CACHE = "cache";
@ -68,6 +71,8 @@ class PersistentMailCache {
public static final String DIR_DRAFTS = "Drafts"; // MailDir-like
public static final String DIR_SENT = "Sent"; // MailDir-like
public static final String DIR_TRASH = "Trash"; // MailDir-like
public static final String DIR_SPAM = "Bulk Mail"; // MailDir-like
public static final String DIR_IMPORT = "import"; // Flat with .eml files, debug only for now
private static final String DIR_PREFIX = "s";
private static final String FILE_PREFIX = "mail-";
private static final String HDR_SUFFIX = ".hdr.txt.gz";
@ -79,10 +84,14 @@ class PersistentMailCache {
* @param pass ignored
* @param folder use DIR_FOLDER
*/
public PersistentMailCache(String host, int port, String user, String pass, String folder) throws IOException {
public PersistentMailCache(I2PAppContext ctx, String host, int port, String user, String pass, String folder) throws IOException {
_context = ctx;
_lock = getLock(host, port, user, pass);
synchronized(_lock) {
_cacheDir = makeCacheDirs(host, port, user, pass, folder);
// Debugging only for now.
if (folder.equals(DIR_FOLDER))
importMail();
}
}
@ -209,8 +218,8 @@ class PersistentMailCache {
* ~/.i2p/susimail/cache/cache-xxxxx/cur/s[b64char]/mail-xxxxx.full.txt.gz
* folder1 is the base.
*/
private static File makeCacheDirs(String host, int port, String user, String pass, String folder) throws IOException {
File f = new SecureDirectory(I2PAppContext.getGlobalContext().getConfigDir(), DIR_SUSI);
private File makeCacheDirs(String host, int port, String user, String pass, String folder) throws IOException {
File f = new SecureDirectory(_context.getConfigDir(), DIR_SUSI);
if (!f.exists() && !f.mkdir())
throw new IOException("Cannot create " + f);
f = new SecureDirectory(f, DIR_CACHE);
@ -307,4 +316,72 @@ class PersistentMailCache {
mail.setBody(rb);
return mail;
}
/**
* For debugging. Import .eml files from the import/ directory
* @since 0.9.34
*/
private void importMail() {
File importDir = new File(_cacheDir.getParentFile(), DIR_IMPORT);
if (importDir.exists() && importDir.isDirectory()) {
File[] files = importDir.listFiles();
if (files == null)
return;
for (int i = 0; i < files.length; i++) {
File f = files[i];
if (!f.isFile())
continue;
if (!f.getName().toLowerCase(Locale.US).endsWith(".eml"))
continue;
// Read in the headers to get the X-UIDL that Thunderbird stuck in there
String uidl = Long.toString(_context.random().nextLong());
InputStream in = null;
try {
in = new FileInputStream(f);
for (int j = 0; j < 20; j++) {
String line = DataHelper.readLine(in);
if (line.length() < 2)
break;
if (line.startsWith("X-UIDL:")) {
uidl = line.substring(7).trim();
break;
}
}
} catch (IOException ioe) {
Debug.debug(Debug.ERROR, "Import failed " + f, ioe);
continue;
} finally {
if (in != null)
try { in.close(); } catch (IOException ioe) {}
}
if (uidl == null)
uidl = Long.toString(_context.random().nextLong());
File to = getFullFile(uidl);
if (to.exists()) {
Debug.debug(Debug.DEBUG, "Already have " + f + " as UIDL " + uidl);
f.delete();
continue;
}
in = null;
OutputStream out = null;
try {
in = new FileInputStream(f);
GzipFileBuffer gb = new GzipFileBuffer(to);
// Thunderbird exports aren't CRLF terminated
out = new FixCRLFOutputStream(gb.getOutputStream());
DataHelper.copy(in, out);
} catch (IOException ioe) {
Debug.debug(Debug.ERROR, "Import failed " + f, ioe);
continue;
} finally {
if (in != null)
try { in.close(); } catch (IOException ioe) {}
if (out != null)
try { out.close(); } catch (IOException ioe) {}
}
f.delete();
Debug.debug(Debug.DEBUG, "Imported " + f + " as UIDL " + uidl);
}
}
}
}

37
apps/susimail/src/src/i2p/susi/webmail/WebMail.java
View File

@ -635,6 +635,20 @@ public class WebMail extends HttpServlet
}
if( chosen != null ) {
showPart( out, chosen, level + 1, html );
if (html) {
// DEBUG
for (MailPart subPart : mailPart.parts) {
if (chosen.equals(subPart))
continue;
out.println( "<!-- " );
out.println( "Debug: Not showing alternative Mail Part at level " + (level + 1) + " with hash code " + mailPart.hashCode());
out.println( "Debug: Mail Part headers follow");
for( int i = 0; i < subPart.headerLines.length; i++ ) {
out.println( subPart.headerLines[i].replace("--", "&#45;&#45;") );
}
out.println( "-->" );
}
}
return;
}
}
@ -1499,6 +1513,7 @@ public class WebMail extends HttpServlet
/*
* process paging buttons
*/
/**** not on the folder view any more, handled in processConfigButtons()
if (buttonPressed(request, SETPAGESIZE)) {
try {
int pageSize = Math.max(5, Integer.parseInt(request.getParameter(PAGESIZE)));
@ -1510,6 +1525,7 @@ public class WebMail extends HttpServlet
sessionObject.error += _t("Invalid pagesize number, resetting to default value.") + '\n';
}
}
****/
if( buttonPressed( request, PREVPAGE ) ) {
String sp = request.getParameter(PREV_PAGE_NUM);
if (sp != null) {
@ -1626,7 +1642,6 @@ public class WebMail extends HttpServlet
File cfg = new File(I2PAppContext.getGlobalContext().getConfigDir(), "susimail.config");
sessionObject.error += _t("Host unchanged. Edit configation file {0} to change host.", cfg.getAbsolutePath()) + '\n';
}
Config.saveConfiguration(props);
String ps = props.getProperty(Folder.PAGESIZE);
if (sessionObject.folder != null && ps != null) {
try {
@ -1636,6 +1651,7 @@ public class WebMail extends HttpServlet
sessionObject.folder.setPageSize( pageSize );
} catch( NumberFormatException nfe ) {}
}
Config.saveConfiguration(props);
boolean release = !Boolean.parseBoolean(props.getProperty(CONFIG_DEBUG));
Debug.setLevel( release ? Debug.ERROR : Debug.DEBUG );
state = sessionObject.folder != null ? State.LIST : State.AUTH;
@ -1646,9 +1662,6 @@ public class WebMail extends HttpServlet
} else if (buttonPressed(request, SETPAGESIZE)) {
try {
int pageSize = Math.max(5, Integer.parseInt(request.getParameter(PAGESIZE)));
Properties props = Config.getProperties();
props.setProperty(Folder.PAGESIZE, String.valueOf(pageSize));
Config.saveConfiguration(props);
if (sessionObject.folder != null) {
int oldPageSize = sessionObject.folder.getPageSize();
if( pageSize != oldPageSize )
@ -1657,6 +1670,9 @@ public class WebMail extends HttpServlet
} else {
state = State.AUTH;
}
Properties props = Config.getProperties();
props.setProperty(Folder.PAGESIZE, String.valueOf(pageSize));
Config.saveConfiguration(props);
} catch (IOException ioe) {
sessionObject.error = ioe.toString();
} catch( NumberFormatException nfe ) {
@ -1897,15 +1913,22 @@ public class WebMail extends HttpServlet
}
}
//// End state determination, state will not change after here
Debug.debug(Debug.DEBUG, "Final state is " + state);
/*
* update folder content
* We need a valid and sorted folder for SHOW also, for the previous/next buttons
*/
Folder<String> folder = sessionObject.folder;
// folder could be null after an error, we can't proceed if it is
if (folder == null && (state == State.LIST || state == State.SHOW)) {
sessionObject.error += "Internal error, no folder\n";
state = State.AUTH;
}
//// End state determination, state will not change after here
Debug.debug(Debug.DEBUG, "Final state is " + state);
if (state == State.LIST || state == State.SHOW) {
// sort buttons are GETs
String oldSort = folder.getCurrentSortBy();
SortOrder oldOrder = folder.getCurrentSortingDirection();

39
apps/susimail/src/src/i2p/susi/webmail/encoding/HeaderLine.java
View File

@ -29,6 +29,7 @@ import i2p.susi.util.Buffer;
import i2p.susi.util.ReadBuffer;
import i2p.susi.util.MemoryBuffer;
import java.io.EOFException;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
@ -210,6 +211,9 @@ public class HeaderLine extends Encoding {
return out.toString();
}
// could be 75 for quoted-printable only
private static final int DECODE_MAX = 256;
/**
* Decode all the header lines, up through \r\n\r\n,
* and puts them in the ReadBuffer, including the \r\n\r\n
@ -235,10 +239,11 @@ public class HeaderLine extends Encoding {
break;
}
if( c == '=' ) {
// An encoded-word is 75 chars max including the delimiters, and must be on a single line
// An encoded-word should be 75 chars max including the delimiters, and must be on a single line
// Store the full encoded word, including =? through ?=, in the buffer
// Sadly, base64 can be a lot longer
if (encodedWord == null)
encodedWord = new byte[75];
encodedWord = new byte[DECODE_MAX];
int offset = 0;
int f1 = 0, f2 = 0, f3 = 0, f4 = 0;
encodedWord[offset++] = (byte) c;
@ -246,7 +251,7 @@ public class HeaderLine extends Encoding {
// plus one char after the 4th '?', which should be '='
// We make a small attempt to pushback one char if it's not what we expect,
// but for the most part it gets thrown out, as RFC 2047 allows
for (; offset < 75; offset++) {
for (; offset < DECODE_MAX; offset++) {
c = in.read();
if (c == '?') {
if (f1 == 0)
@ -318,12 +323,20 @@ public class HeaderLine extends Encoding {
if (enc != null) {
Encoding e = EncodingFactory.getEncoding( enc );
if( e != null ) {
// System.err.println( "encoder found" );
try {
// System.err.println( "decode(" + (f3 + 1) + "," + ( f4 - f3 - 1 ) + ")" );
ReadBuffer tmpIn = new ReadBuffer(encodedWord, f3 + 1, f4 - f3 - 1);
MemoryBuffer tmp = new MemoryBuffer(75);
e.decode(tmpIn, tmp);
MemoryBuffer tmp = new MemoryBuffer(DECODE_MAX);
try {
e.decode(tmpIn, tmp);
} catch (EOFException eof) {
// probably Base64 exceeded DECODE_MAX
// Keep going and output what we got, if any
if (Debug.getLevel() >= Debug.DEBUG) {
Debug.debug(Debug.DEBUG, "q-w " + enc, eof);
Debug.debug(Debug.DEBUG, net.i2p.util.HexDump.dump(encodedWord));
}
}
tmp.writeComplete(true);
// get charset
String charset = new String(encodedWord, f1 + 1, f2 - f1 - 1, "ISO-8859-1");
@ -357,13 +370,15 @@ public class HeaderLine extends Encoding {
lastCharWasQuoted = true;
continue;
} catch (IOException e1) {
Debug.debug(Debug.DEBUG, "q-w", e1);
Debug.debug(Debug.DEBUG, "Decoder: " + enc + " Input:");
Debug.debug(Debug.DEBUG, net.i2p.util.HexDump.dump(encodedWord, f3 + 1, f4 - f3 - 1));
Debug.debug(Debug.ERROR, "q-w " + enc, e1);
if (Debug.getLevel() >= Debug.DEBUG) {
Debug.debug(Debug.DEBUG, net.i2p.util.HexDump.dump(encodedWord));
}
} catch (RuntimeException e1) {
Debug.debug(Debug.DEBUG, "q-w", e1);
Debug.debug(Debug.DEBUG, "Decoder: " + enc + " Input:");
Debug.debug(Debug.DEBUG, net.i2p.util.HexDump.dump(encodedWord, f3 + 1, f4 - f3 - 1));
Debug.debug(Debug.ERROR, "q-w " + enc, e1);
if (Debug.getLevel() >= Debug.DEBUG) {
Debug.debug(Debug.DEBUG, net.i2p.util.HexDump.dump(encodedWord));
}
}
} else {
// can't happen

10
history.txt
View File

@ -1,3 +1,13 @@
2018-02-08 zzz
* SusiMail:
- Error handling fixes
- More tolerant parsing of Date headers
- Set a date if we don't get a Date header
- Fix parsing long Base64 encoded headers
- Fix page count after changing page size
- Make attribute name parsing case-insensitive
- Import mail method for debugging
2018-02-07 zzz
* SusiMail: Use input streams for reading mail (ticket #2119)
- Rewrite Base64, HeaderLine, and QuotedPrintable decoders

2
router/java/src/net/i2p/router/RouterVersion.java
View File

@ -18,7 +18,7 @@ public class RouterVersion {
/** deprecated */
public final static String ID = "Monotone";
public final static String VERSION = CoreVersion.VERSION;
public final static long BUILD = 2;
public final static long BUILD = 3;
/** for example "-test" */
public final static String EXTRA = "";