SusiMail: Error handling fixes

More tolerant parsing of Date headers
Set a date if we don't get a valid Date header
Fix parsing long Base64 encoded headers
Fix page count after changing page size
Make attribute name parsing case-insensitive
Import mail method for debugging
Debug and log tweaks

apps/susimail/src/src/i2p/susi/util/FixCRLFOutputStream.java

@@ -0,0 +1,28 @@
+package i2p.susi.util;
+
+import java.io.FilterOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+
+/**
+ * Replace plain \n with \r\n on the fly.
+ * Used when importing .eml files.
+ *
+ * @since 0.9.34
+ */
+public class FixCRLFOutputStream extends FilterOutputStream {
+    
+    private int previous = -1;
+
+    public FixCRLFOutputStream(OutputStream out) {
+        super(out);
+    }
+    
+    @Override
+    public void write(int val) throws IOException {
+        if (val == '\n' && previous != '\r')
+            out.write('\r');
+        out.write(val);
+        previous = val;
+    }
+}

apps/susimail/src/src/i2p/susi/webmail/Mail.java

@@ -28,6 +28,7 @@ import i2p.susi.util.Buffer;
 import i2p.susi.util.Config;
 import i2p.susi.util.CountingInputStream;
 import i2p.susi.util.EOFOnMatchInputStream;
+import i2p.susi.util.FileBuffer;
 import i2p.susi.util.MemoryBuffer;
 import i2p.susi.webmail.encoding.Encoding;
 import i2p.susi.webmail.encoding.EncodingFactory;
@@ -46,8 +47,10 @@ import java.util.Locale;
 import java.util.TimeZone;
 import java.util.regex.Pattern;
 
+import net.i2p.I2PAppContext;
 import net.i2p.data.DataHelper;
 import net.i2p.servlet.util.ServletUtil;
+import net.i2p.util.RFC822Date;
 import net.i2p.util.SystemVersion;
 
 /**
@@ -129,6 +132,16 @@ class Mail {
 		String[] rv = parseHeaders(in);
 		if (closeIn)
 			rb.readComplete(true);
+		// set a date if we didn't get one in the headers
+		if (date == null) {
+			long dateLong;
+			if (rb instanceof FileBuffer) {
+				dateLong = ((FileBuffer) rb).getFile().lastModified();
+			} else {
+				dateLong = I2PAppContext.getGlobalContext().clock().now();
+			}
+			setDate(dateLong);
+		}
 		return rv;
 	}
 
@@ -175,7 +188,7 @@ class Mail {
 		} catch (RuntimeException e) {
 			Debug.debug(Debug.ERROR, "Parse error", e);
 		} finally {
-			try { in.close(); } catch (IOException ioe) {}
+			if (in != null) try { in.close(); } catch (IOException ioe) {}
 			rb.readComplete(success);
 		}
 	}
@@ -323,7 +336,6 @@ class Mail {
 	private static final DateFormat dateFormatter = new SimpleDateFormat("yyyy-MM-dd HH:mm");
 	private static final DateFormat localDateFormatter = DateFormat.getDateTimeInstance(DateFormat.SHORT, DateFormat.SHORT);
 	private static final DateFormat longLocalDateFormatter = DateFormat.getDateTimeInstance(DateFormat.MEDIUM, DateFormat.MEDIUM);
-	private static final DateFormat mailDateFormatter = new SimpleDateFormat("EEE, d MMM yyyy HH:mm:ss Z", Locale.ENGLISH );
 	static {
 		// the router sets the JVM time zone to UTC but saves the original here so we can get it
 		TimeZone tz = SystemVersion.getSystemTimeZone();
@@ -331,6 +343,19 @@ class Mail {
 		longLocalDateFormatter.setTimeZone(tz);
 	}
 
+	/**
+	 * @param dateLong non-negative
+	 * @since 0.9.34 pulled from parseHeaders()
+	 */
+	private void setDate(long dateLong) {
+		date = new Date(dateLong);
+		synchronized(dateFormatter) {
+			formattedDate = dateFormatter.format( date );
+			localFormattedDate = localDateFormatter.format( date );
+			quotedDate = longLocalDateFormatter.format(date);
+		}
+	}
+
 	/**
 	 * @return all headers, to pass to MailPart, or null on error
 	 */
@@ -390,19 +415,9 @@ class Mail {
 						}
 						else if (hlc.startsWith("date:")) {
 							dateString = line.substring( 5 ).trim();
-							try {
+							long dateLong = RFC822Date.parse822Date(dateString);
-								synchronized(mailDateFormatter) {
+							if (dateLong > 0)
-									date = mailDateFormatter.parse( dateString );
+								setDate(dateLong);
-									formattedDate = dateFormatter.format( date );
-									localFormattedDate = localDateFormatter.format( date );
-									//quotedDate = html.encode( dateString );
-									quotedDate = longLocalDateFormatter.format(date);
-								}
-							}
-							catch (ParseException e) {
-								date = null;
-								e.printStackTrace();
-							}
 						}
 						else if (hlc.startsWith("subject:")) {
 							subject = line.substring( 8 ).trim();

apps/susimail/src/src/i2p/susi/webmail/MailCache.java

@@ -72,7 +72,7 @@ class MailCache {
 		mails = new Hashtable<String, Mail>();
 		PersistentMailCache pmc = null;
 		try {
-			pmc = new PersistentMailCache(host, port, user, pass, PersistentMailCache.DIR_FOLDER);
+			pmc = new PersistentMailCache(ctx, host, port, user, pass, PersistentMailCache.DIR_FOLDER);
 			// TODO Drafts, Sent, Trash
 		} catch (IOException ioe) {
 			Debug.debug(Debug.ERROR, "Error creating disk cache: " + ioe);

apps/susimail/src/src/i2p/susi/webmail/MailPart.java

@@ -202,10 +202,10 @@ class MailPart {
 					OutputStream dummy = new DummyOutputStream();
 					DataHelper.copy(eofin, dummy);  
 					if (!eofin.wasFound())
-						Debug.debug(Debug.DEBUG, "EOF hit before first boundary " + boundary);
+						Debug.debug(Debug.DEBUG, "EOF hit before first boundary " + boundary + " UIDL: " + uidl);
 					if (readBoundaryTrailer(in)) {
 						if (!eofin.wasFound())
-							Debug.debug(Debug.DEBUG, "EOF hit before first part body " + boundary);
+							Debug.debug(Debug.DEBUG, "EOF hit before first part body " + boundary + " UIDL: " + uidl);
 						tmpEnd = (int) eofin.getRead();
 						break;
 					}
@@ -220,7 +220,7 @@ class MailPart {
 					// if MailPart contains a MailPart, we may not have drained to the end
 					DataHelper.copy(eofin, DUMMY_OUTPUT);  
 					if (!eofin.wasFound())
-						Debug.debug(Debug.DEBUG, "EOF hit before end of body " + i + " boundary: " + boundary);
+						Debug.debug(Debug.DEBUG, "EOF hit before end of body " + i + " boundary: " + boundary + " UIDL: " + uidl);
 				}
 				if (readBoundaryTrailer(in))
 					break;
@@ -351,13 +351,18 @@ class MailPart {
 		return result;
 	}
 
+	/**
+	 *  @param attributeName must be lower case, will be matched case-insensitively
+	 *  @return as found, not necessarily lower case
+	 */
 	private static String getHeaderLineAttribute( String line, String attributeName )
 	{
+		String lineLC = line.toLowerCase(Locale.US);
 		String result = null;
 		int h = 0;
 		int l = attributeName.length();
 		while( true ) {
-			int i = line.indexOf( attributeName, h );
+			int i = lineLC.indexOf(attributeName, h);
 			// System.err.println( "i=" + i );
 			if( i == -1 )
 				break;

apps/susimail/src/src/i2p/susi/webmail/PersistentMailCache.java

@@ -4,6 +4,7 @@ import i2p.susi.debug.Debug;
 import i2p.susi.webmail.Messages;
 import i2p.susi.util.Buffer;
 import i2p.susi.util.FileBuffer;
+import i2p.susi.util.FixCRLFOutputStream;
 import i2p.susi.util.GzipFileBuffer;
 import i2p.susi.util.ReadBuffer;
 
@@ -19,6 +20,7 @@ import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Hashtable;
 import java.util.List;
+import java.util.Locale;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.zip.GZIPInputStream;
 import java.util.zip.GZIPOutputStream;
@@ -60,6 +62,7 @@ class PersistentMailCache {
 
 	private final Object _lock;
 	private final File _cacheDir;
+	private final I2PAppContext _context;
 
 	private static final String DIR_SUSI = "susimail";
 	private static final String DIR_CACHE = "cache";
@@ -68,6 +71,8 @@ class PersistentMailCache {
 	public static final String DIR_DRAFTS = "Drafts"; // MailDir-like
 	public static final String DIR_SENT = "Sent"; // MailDir-like
 	public static final String DIR_TRASH = "Trash"; // MailDir-like
+	public static final String DIR_SPAM = "Bulk Mail"; // MailDir-like
+	public static final String DIR_IMPORT = "import"; // Flat with .eml files, debug only for now
 	private static final String DIR_PREFIX = "s";
 	private static final String FILE_PREFIX = "mail-";
 	private static final String HDR_SUFFIX = ".hdr.txt.gz";
@@ -79,10 +84,14 @@ class PersistentMailCache {
 	 *  @param pass ignored
 	 *  @param folder use DIR_FOLDER
 	 */
-	public PersistentMailCache(String host, int port, String user, String pass, String folder) throws IOException {
+	public PersistentMailCache(I2PAppContext ctx, String host, int port, String user, String pass, String folder) throws IOException {
+		_context = ctx;
 		_lock = getLock(host, port, user, pass);
 		synchronized(_lock) {
 			_cacheDir = makeCacheDirs(host, port, user, pass, folder);
+			// Debugging only for now.
+			if (folder.equals(DIR_FOLDER))
+				importMail();
 		}
 	}
 
@@ -209,8 +218,8 @@ class PersistentMailCache {
 	 *   ~/.i2p/susimail/cache/cache-xxxxx/cur/s[b64char]/mail-xxxxx.full.txt.gz
 	 *   folder1 is the base.
 	 */
-	private static File makeCacheDirs(String host, int port, String user, String pass, String folder) throws IOException {
+	private File makeCacheDirs(String host, int port, String user, String pass, String folder) throws IOException {
-		File f = new SecureDirectory(I2PAppContext.getGlobalContext().getConfigDir(), DIR_SUSI);
+		File f = new SecureDirectory(_context.getConfigDir(), DIR_SUSI);
 		if (!f.exists() && !f.mkdir())
 			throw new IOException("Cannot create " + f);
 		f = new SecureDirectory(f, DIR_CACHE);
@@ -307,4 +316,72 @@ class PersistentMailCache {
 			mail.setBody(rb);
 		return mail;
 	}
+
+	/**
+	 *  For debugging. Import .eml files from the import/ directory
+	 *  @since 0.9.34
+	 */
+	private void importMail() {
+		File importDir = new File(_cacheDir.getParentFile(), DIR_IMPORT);
+		if (importDir.exists() && importDir.isDirectory()) {
+			File[] files = importDir.listFiles();
+			if (files == null)
+				return;
+			for (int i = 0; i < files.length; i++) {
+				File f = files[i];
+				if (!f.isFile())
+					continue;
+				if (!f.getName().toLowerCase(Locale.US).endsWith(".eml"))
+					continue;
+				// Read in the headers to get the X-UIDL that Thunderbird stuck in there
+				String uidl = Long.toString(_context.random().nextLong());
+				InputStream in = null;
+				try {
+					in = new FileInputStream(f);
+					for (int j = 0; j < 20; j++) {
+						String line = DataHelper.readLine(in);
+						if (line.length() < 2)
+							break;
+						if (line.startsWith("X-UIDL:")) {
+							uidl = line.substring(7).trim();
+							break;
+						}
+					}
+				} catch (IOException ioe) {
+					Debug.debug(Debug.ERROR, "Import failed " + f, ioe);
+					continue;
+				} finally {
+					if (in != null) 
+						try { in.close(); } catch (IOException ioe) {}
+				}
+				if (uidl == null)
+					uidl = Long.toString(_context.random().nextLong());
+				File to = getFullFile(uidl);
+				if (to.exists()) {
+					Debug.debug(Debug.DEBUG, "Already have " + f + " as UIDL " + uidl);
+					f.delete();
+					continue;
+				}
+				in = null;
+				OutputStream out = null;
+				try {
+					in = new FileInputStream(f);
+					GzipFileBuffer gb = new GzipFileBuffer(to);
+					// Thunderbird exports aren't CRLF terminated
+					out = new FixCRLFOutputStream(gb.getOutputStream());
+					DataHelper.copy(in, out);
+				} catch (IOException ioe) {
+					Debug.debug(Debug.ERROR, "Import failed " + f, ioe);
+					continue;
+				} finally {
+					if (in != null) 
+						try { in.close(); } catch (IOException ioe) {}
+					if (out != null) 
+						try { out.close(); } catch (IOException ioe) {}
+				}
+				f.delete();
+				Debug.debug(Debug.DEBUG, "Imported " + f + " as UIDL " + uidl);
+	       		}
+		}
+	}
 }

apps/susimail/src/src/i2p/susi/webmail/WebMail.java

@@ -635,6 +635,20 @@ public class WebMail extends HttpServlet
 				}
 				if( chosen != null ) {
 					showPart( out, chosen, level + 1, html );
+					if (html) {
+						// DEBUG
+						for (MailPart subPart : mailPart.parts) {
+							if (chosen.equals(subPart))
+								continue;
+							out.println( "<!-- " );
+							out.println( "Debug: Not showing alternative Mail Part at level " + (level + 1) + " with hash code " + mailPart.hashCode());
+							out.println( "Debug: Mail Part headers follow");
+							for( int i = 0; i < subPart.headerLines.length; i++ ) {
+								out.println( subPart.headerLines[i].replace("--", "&#45;&#45;") );
+							}	
+							out.println( "-->" );
+						}
+					}
 					return;
 				}
 			}
@@ -1499,6 +1513,7 @@ public class WebMail extends HttpServlet
 		/*
 		 * process paging buttons
 		 */
+/**** not on the folder view any more, handled in processConfigButtons()
 		if (buttonPressed(request, SETPAGESIZE)) {
 			try {
 				int pageSize = Math.max(5, Integer.parseInt(request.getParameter(PAGESIZE)));
@@ -1510,6 +1525,7 @@ public class WebMail extends HttpServlet
 				sessionObject.error += _t("Invalid pagesize number, resetting to default value.") + '\n';
 			}
 		}
+****/
 		if( buttonPressed( request, PREVPAGE ) ) {
 			String sp = request.getParameter(PREV_PAGE_NUM);
 			if (sp != null) {
@@ -1626,7 +1642,6 @@ public class WebMail extends HttpServlet
 					File cfg = new File(I2PAppContext.getGlobalContext().getConfigDir(), "susimail.config");
 					sessionObject.error += _t("Host unchanged. Edit configation file {0} to change host.", cfg.getAbsolutePath()) + '\n';
 				}
-				Config.saveConfiguration(props);
 				String ps = props.getProperty(Folder.PAGESIZE);
 				if (sessionObject.folder != null && ps != null) {
 					try {
@@ -1636,6 +1651,7 @@ public class WebMail extends HttpServlet
 							sessionObject.folder.setPageSize( pageSize );
 					} catch( NumberFormatException nfe ) {}
 				}
+				Config.saveConfiguration(props);
 				boolean release = !Boolean.parseBoolean(props.getProperty(CONFIG_DEBUG));
 				Debug.setLevel( release ? Debug.ERROR : Debug.DEBUG );
 				state = sessionObject.folder != null ? State.LIST : State.AUTH;
@@ -1646,9 +1662,6 @@ public class WebMail extends HttpServlet
 		} else if (buttonPressed(request, SETPAGESIZE)) {
 			try {
 				int pageSize = Math.max(5, Integer.parseInt(request.getParameter(PAGESIZE)));
-				Properties props = Config.getProperties();
-				props.setProperty(Folder.PAGESIZE, String.valueOf(pageSize));
-				Config.saveConfiguration(props);
 				if (sessionObject.folder != null) {
 					int oldPageSize = sessionObject.folder.getPageSize();
 					if( pageSize != oldPageSize )
@@ -1657,6 +1670,9 @@ public class WebMail extends HttpServlet
 				} else {
 					state = State.AUTH;
 				}
+				Properties props = Config.getProperties();
+				props.setProperty(Folder.PAGESIZE, String.valueOf(pageSize));
+				Config.saveConfiguration(props);
 			} catch (IOException ioe) {
 				sessionObject.error = ioe.toString();
 			} catch( NumberFormatException nfe ) {
@@ -1897,15 +1913,22 @@ public class WebMail extends HttpServlet
 				}
 			}
 
-			//// End state determination, state will not change after here
-			Debug.debug(Debug.DEBUG, "Final state is " + state);
-
 			/*
 			 * update folder content
 			 * We need a valid and sorted folder for SHOW also, for the previous/next buttons
 			 */
 			Folder<String> folder = sessionObject.folder;
+			// folder could be null after an error, we can't proceed if it is
+			if (folder == null && (state == State.LIST || state == State.SHOW)) {
+				sessionObject.error += "Internal error, no folder\n";
+				state = State.AUTH;
+			}
+
+			//// End state determination, state will not change after here
+			Debug.debug(Debug.DEBUG, "Final state is " + state);
+
 			if (state == State.LIST || state == State.SHOW) {
+
 				// sort buttons are GETs
 				String oldSort = folder.getCurrentSortBy();
 				SortOrder oldOrder = folder.getCurrentSortingDirection();

apps/susimail/src/src/i2p/susi/webmail/encoding/HeaderLine.java

@@ -29,6 +29,7 @@ import i2p.susi.util.Buffer;
 import i2p.susi.util.ReadBuffer;
 import i2p.susi.util.MemoryBuffer;
 
+import java.io.EOFException;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
@@ -210,6 +211,9 @@ public class HeaderLine extends Encoding {
 		return out.toString();
 	}
 
+	// could be 75 for quoted-printable only
+	private static final int DECODE_MAX = 256;
+
 	/**
 	 *  Decode all the header lines, up through \r\n\r\n,
 	 *  and puts them in the ReadBuffer, including the \r\n\r\n
@@ -235,10 +239,11 @@ public class HeaderLine extends Encoding {
 					break;
 			}
 			if( c == '=' ) {
-				// An encoded-word is 75 chars max including the delimiters, and must be on a single line
+				// An encoded-word should be 75 chars max including the delimiters, and must be on a single line
 				// Store the full encoded word, including =? through ?=, in the buffer
+				// Sadly, base64 can be a lot longer
 				if (encodedWord == null)
-					encodedWord = new byte[75];
+					encodedWord = new byte[DECODE_MAX];
 				int offset = 0;
 				int f1 = 0, f2 = 0, f3 = 0, f4 = 0;
 				encodedWord[offset++] = (byte) c;
@@ -246,7 +251,7 @@ public class HeaderLine extends Encoding {
 				// plus one char after the 4th '?', which should be '='
 				// We make a small attempt to pushback one char if it's not what we expect,
 				// but for the most part it gets thrown out, as RFC 2047 allows
-				for (; offset < 75; offset++) {
+				for (; offset < DECODE_MAX; offset++) {
 					c = in.read();
 					if (c == '?') {
 						if (f1 == 0)
@@ -318,12 +323,20 @@ public class HeaderLine extends Encoding {
 				if (enc != null) {
 					Encoding e = EncodingFactory.getEncoding( enc );
 					if( e != null ) {
-						// System.err.println( "encoder found" );
 						try {
 							// System.err.println( "decode(" + (f3 + 1) + "," + ( f4 - f3 - 1 ) + ")" );
 							ReadBuffer tmpIn = new ReadBuffer(encodedWord, f3 + 1, f4 - f3 - 1);
-							MemoryBuffer tmp = new MemoryBuffer(75);
+							MemoryBuffer tmp = new MemoryBuffer(DECODE_MAX);
-							e.decode(tmpIn, tmp);
+							try {
+								e.decode(tmpIn, tmp);
+							} catch (EOFException eof) {
+								// probably Base64 exceeded DECODE_MAX
+								// Keep going and output what we got, if any
+								if (Debug.getLevel() >= Debug.DEBUG) {
+									Debug.debug(Debug.DEBUG, "q-w " + enc, eof);
+									Debug.debug(Debug.DEBUG, net.i2p.util.HexDump.dump(encodedWord));
+								}
+							}
 							tmp.writeComplete(true);
 							// get charset
 							String charset = new String(encodedWord, f1 + 1, f2 - f1 - 1, "ISO-8859-1");
@@ -357,13 +370,15 @@ public class HeaderLine extends Encoding {
 							lastCharWasQuoted = true;
 							continue;
 						} catch (IOException e1) {
-							Debug.debug(Debug.DEBUG, "q-w", e1);
+							Debug.debug(Debug.ERROR, "q-w " + enc, e1);
-							Debug.debug(Debug.DEBUG, "Decoder: " + enc + " Input:");
+							if (Debug.getLevel() >= Debug.DEBUG) {
-							Debug.debug(Debug.DEBUG, net.i2p.util.HexDump.dump(encodedWord, f3 + 1, f4 - f3 - 1));
+								Debug.debug(Debug.DEBUG, net.i2p.util.HexDump.dump(encodedWord));
+							}
 						} catch (RuntimeException e1) {
-							Debug.debug(Debug.DEBUG, "q-w", e1);
+							Debug.debug(Debug.ERROR, "q-w " + enc, e1);
-							Debug.debug(Debug.DEBUG, "Decoder: " + enc + " Input:");
+							if (Debug.getLevel() >= Debug.DEBUG) {
-							Debug.debug(Debug.DEBUG, net.i2p.util.HexDump.dump(encodedWord, f3 + 1, f4 - f3 - 1));
+								Debug.debug(Debug.DEBUG, net.i2p.util.HexDump.dump(encodedWord));
+							}
 						}
 					} else {
 						// can't happen

history.txt

@@ -1,3 +1,13 @@
+2018-02-08 zzz
+ * SusiMail:
+   - Error handling fixes
+   - More tolerant parsing of Date headers
+   - Set a date if we don't get a Date header
+   - Fix parsing long Base64 encoded headers
+   - Fix page count after changing page size
+   - Make attribute name parsing case-insensitive
+   - Import mail method for debugging
+
 2018-02-07 zzz
  * SusiMail: Use input streams for reading mail (ticket #2119)
    - Rewrite Base64, HeaderLine, and QuotedPrintable decoders

router/java/src/net/i2p/router/RouterVersion.java

@@ -18,7 +18,7 @@ public class RouterVersion {
     /** deprecated */
     public final static String ID = "Monotone";
     public final static String VERSION = CoreVersion.VERSION;
-    public final static long BUILD = 2;
+    public final static long BUILD = 3;
 
     /** for example "-test" */
     public final static String EXTRA = "";