* SusiMail:

- Add background mail checker, not yet enabled
   - Add idle timeout connection closer
   - Rely on idle checker for most delayed deletions
   - Cleanup resources better when shutting down session
   - Don't add deleted mails to folder, caused errors after deletions
   - Set socket soTimeouts so things don't hang forever
   - Display errors after check mail button pushed
   - More IOE debug logging

apps/susimail/src/src/i2p/susi/webmail/MailCache.java

@@ -86,13 +86,17 @@ class MailCache {
 
 	/**
 	 * The ones known locally, which will include any known on the server, if connected.
+	 * Will not include any marked for deletion.
 	 * 
 	 * @since 0.9.13
 	 */
 	public String[] getUIDLs() {
 		List<String> uidls = new ArrayList<String>(mails.size());
 		synchronized(mails) {
-			uidls.addAll(mails.keySet());
+			for (Mail mail : mails.values()) {
+				if (!mail.markForDeletion)
+					uidls.add(mail.uidl);
+			}
 		}
 		return uidls.toArray(new String[uidls.size()]);
 	}

apps/susimail/src/src/i2p/susi/webmail/NewMailListener.java

@@ -0,0 +1,12 @@
+package i2p.susi.webmail;
+
+
+/**
+ *  Listen for indication of new mail, maybe
+ *  @since 0.9.13
+ */
+public interface NewMailListener {
+    
+	public void foundNewMail();
+
+}

apps/susimail/src/src/i2p/susi/webmail/PersistentMailCache.java

@@ -64,6 +64,7 @@ class PersistentMailCache {
 	 */
 	public PersistentMailCache(String host, int port, String user, String pass) throws IOException {
 		_cacheDir = makeCacheDirs(host, port, user, pass);
+		// TODO static locking
 	}
 
 	/**

apps/susimail/src/src/i2p/susi/webmail/WebMail.java

@@ -175,6 +175,9 @@ public class WebMail extends HttpServlet
 
 	private static final String CONFIG_BCC_TO_SELF = "composer.bcc.to.self";
 	static final String CONFIG_LEAVE_ON_SERVER = "pop3.leave.on.server";
+	public static final String CONFIG_BACKGROUND_CHECK = "pop3.check.enable";
+	public static final String CONFIG_CHECK_MINUTES = "pop3.check.interval.minutes";
+	public static final String CONFIG_IDLE_SECONDS = "pop3.idle.timeout.seconds";
 	private static final String CONFIG_DEBUG = "debug";
 
 	private static final String RC_PROP_THEME = "routerconsole.theme";
@@ -346,7 +349,7 @@ public class WebMail extends HttpServlet
 	 * data structure to hold any persistent data (to store them in session dictionary)
 	 * @author susi
 	 */
-	private static class SessionObject implements HttpSessionBindingListener {
+	private static class SessionObject implements HttpSessionBindingListener, NewMailListener {
 		boolean pageChanged, markAll, clear, invert;
 		int state, smtpPort;
 		POP3MailBox mailbox;
@@ -380,10 +383,28 @@ public class WebMail extends HttpServlet
 			Debug.debug(Debug.DEBUG, "Session unbound: " + event.getSession().getId());
 			POP3MailBox mbox = mailbox;
 			if (mbox != null) {
-				mbox.close();
+				mbox.destroy();
 				mailbox = null;
 			}
 		}
+
+		/**
+		 *  Relay from the checker to the webmail session object,
+		 *  which relays to MailCache, which will fetch the mail from us
+		 *  in a big circle
+		 *
+		 *  @since 0.9.13
+		 */
+		public void foundNewMail() {
+			MailCache mc = mailCache;
+			Folder<String> f = folder;
+			if (mc != null && f != null) {
+				if (mc.getMail(true)) {
+					String[] uidls = mc.getUIDLs();
+					f.setElements(uidls);
+				}
+			}
+		}
 	}
 
 	/**
@@ -517,7 +538,9 @@ public class WebMail extends HttpServlet
 					String charset = mailPart.charset;
 					if( charset == null ) {
 						charset = "US-ASCII";
-						reason += _("Warning: no charset found, fallback to US-ASCII.") + br;
+						// don't show this in text mode which is used to include the mail in the reply or forward
+						if (html)
+							reason += _("Warning: no charset found, fallback to US-ASCII.") + br;
 					}
 					try {
 						ReadBuffer decoded = mailPart.decode(0);
@@ -698,11 +721,17 @@ public class WebMail extends HttpServlet
 							Debug.debug(Debug.DEBUG, "OFFLINE MODE");
 						else
 							Debug.debug(Debug.DEBUG, "CONNECTED, YAY");
-					}
-					else {
+						// we do this after the initial priming above
+						mailbox.setNewMailListener(sessionObject);
+					} else {
 						sessionObject.error += mailbox.lastError();
-						mailbox.close();
+						Debug.debug(Debug.DEBUG, "LOGIN FAIL, REMOVING SESSION");
+						HttpSession session = request.getSession();
+						session.removeAttribute( "sessionObject" );
+						session.invalidate();
+						mailbox.destroy();
 						sessionObject.mailbox = null;
+						sessionObject.mailCache = null;
 						Debug.debug(Debug.DEBUG, "NOT CONNECTED, BOO");
 					}
 				}
@@ -718,14 +747,15 @@ public class WebMail extends HttpServlet
 	private static void processLogout( SessionObject sessionObject, RequestWrapper request )
 	{
 		if( buttonPressed( request, LOGOUT ) ) {
-			Debug.debug(Debug.DEBUG, "REMOVING SESSION");
+			Debug.debug(Debug.DEBUG, "LOGOUT, REMOVING SESSION");
 			HttpSession session = request.getSession();
 			session.removeAttribute( "sessionObject" );
 			session.invalidate();
 			POP3MailBox mailbox = sessionObject.mailbox;
 			if (mailbox != null) {
-				mailbox.close();
+				mailbox.destroy();
 				sessionObject.mailbox = null;
+				sessionObject.mailCache = null;
 			}
 			sessionObject.info += _("User logged out.") + "<br>";
 			sessionObject.state = STATE_AUTH;
@@ -1008,6 +1038,7 @@ public class WebMail extends HttpServlet
 		}
 		if( buttonPressed( request, REFRESH ) ) {
 			sessionObject.mailbox.refresh();
+			sessionObject.error += sessionObject.mailbox.lastError();
 			sessionObject.mailCache.getMail(true);
 			// get through cache so we have the disk-only ones too
 			String[] uidls = sessionObject.mailCache.getUIDLs();

apps/susimail/src/src/i2p/susi/webmail/pop3/BackgroundChecker.java

@@ -0,0 +1,123 @@
+package i2p.susi.webmail.pop3;
+
+import i2p.susi.debug.Debug;
+import i2p.susi.webmail.WebMail;
+import i2p.susi.util.Config;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Set;
+
+import net.i2p.I2PAppContext;
+import net.i2p.util.I2PAppThread;
+import net.i2p.util.ConcurrentHashSet;
+import net.i2p.util.SimpleTimer2;
+
+
+/**
+ *  Check for new mail periodically
+ *
+ *  @since 0.9.13
+ */
+class BackgroundChecker {
+
+	private final POP3MailBox mailbox;
+	private final Set<String> toDelete;
+	private final SimpleTimer2.TimedEvent timer;
+	private volatile boolean isChecking;
+	private volatile boolean isDead;
+
+	private static final int DEFAULT_CHECK_MINUTES = 3*60;
+	private static final int MIN_CHECK_MINUTES = 15;
+	// short for testing
+	//private final long MIN_IDLE = 30*60*1000;
+	private static final long MIN_IDLE = 10*60*1000;
+	// short for testing
+	//private final long MIN_SINCE = 60*60*1000;
+	private static final long MIN_SINCE = 10*60*1000;
+
+	public BackgroundChecker(POP3MailBox mailbox) {
+		this.mailbox = mailbox;
+		toDelete = new ConcurrentHashSet<String>();
+		timer = new Checker();
+	}
+
+	public Collection<String> getQueued() {
+		List<String> rv = new ArrayList<String>(toDelete);
+		return rv;
+	}
+
+	public void cancel() {
+		isDead = true;
+		timer.cancel();
+	}
+
+	private static long getCheckTime() {
+		int minutes = DEFAULT_CHECK_MINUTES;
+		String con = Config.getProperty(WebMail.CONFIG_CHECK_MINUTES);
+		if (con != null) {
+			try {
+				int mins = Integer.parseInt(con);
+				// allow shorter for testing
+				if (mins < MIN_CHECK_MINUTES && Debug.getLevel() != Debug.DEBUG)
+					mins = MIN_CHECK_MINUTES;
+				minutes = mins;
+			} catch (NumberFormatException nfe) {}
+		}
+		return minutes * 60 * 1000L;
+	}
+
+	private class Checker extends SimpleTimer2.TimedEvent {
+
+		public Checker() {
+			super(I2PAppContext.getGlobalContext().simpleTimer2(), getCheckTime());
+		}
+
+	        public void timeReached() {
+			if (isDead)
+				return;
+			if (!mailbox.isConnected() && !isChecking) {
+				long idle = System.currentTimeMillis() - mailbox.getLastActivity();
+				long last = System.currentTimeMillis() - mailbox.getLastChecked();
+				if (idle >= MIN_IDLE && last >= MIN_SINCE) {
+					Debug.debug(Debug.DEBUG, "Threading check for mail after " +
+							idle + " ms idle and " + last + " since last check");
+					Thread t = new Getter();
+					isChecking = true;
+					t.start();
+				} else {
+					Debug.debug(Debug.DEBUG, "Not checking after " +
+							idle + " ms idle and " + last + " since last check");
+				}
+			} else {
+				Debug.debug(Debug.DEBUG, "Not checking, still connected");
+			}
+			schedule(getCheckTime());
+		}
+	}
+
+	private class Getter extends I2PAppThread {
+
+		public Getter() {
+			super("Susimail-Getter");
+		}
+
+	        public void run() {
+			try {
+				if (mailbox.connectToServer()) {
+					int found = mailbox.getNumMails();
+					if (found > 0) {
+						Debug.debug(Debug.DEBUG, "Found " + found + " mails, calling listener");
+						// may not really be new
+						mailbox.foundNewMail();
+					}
+				}
+			} finally {		
+				isChecking = false;
+				if (!isDead)
+					timer.schedule(getCheckTime());
+			}		
+		}
+	}
+}

apps/susimail/src/src/i2p/susi/webmail/pop3/DelayedDeleter.java

@@ -14,7 +14,10 @@ import net.i2p.util.SimpleTimer2;
 
 
 /**
- *  Queue UIDLs for later deletion
+ *  Queue UIDLs for later deletion.
+ *  We send deletions at close time but we don't wait around
+ *  for the answer. Also, the user may delete mails when offline.
+ *  So we queue them here and reconnect to delete.
  *
  *  @since 0.9.13
  */
@@ -26,8 +29,8 @@ class DelayedDeleter {
 	private volatile boolean isDeleting;
 	private volatile boolean isDead;
 
-	private final long CHECK_TIME = 5*60*1000;
-	private final long MIN_IDLE = 5*60*1000;
+	private static final long CHECK_TIME = 16*60*1000;
+	private static final long MIN_IDLE = 60*60*1000;
 
 	public DelayedDeleter(POP3MailBox mailbox) {
 		this.mailbox = mailbox;

apps/susimail/src/src/i2p/susi/webmail/pop3/IdleCloser.java

@@ -0,0 +1,116 @@
+package i2p.susi.webmail.pop3;
+
+import i2p.susi.debug.Debug;
+import i2p.susi.webmail.WebMail;
+import i2p.susi.util.Config;
+
+import net.i2p.I2PAppContext;
+import net.i2p.util.I2PAppThread;
+import net.i2p.util.SimpleTimer2;
+
+
+/**
+ *  Close the POP3 connection after a certain idle time
+ *
+ *  @since 0.9.13
+ */
+class IdleCloser {
+
+	private final POP3MailBox mailbox;
+	private final SimpleTimer2.TimedEvent timer;
+	private volatile boolean isClosing;
+	private volatile boolean isDead;
+
+	private static final long CHECK_TIME = 30*1000;
+	// POP3 RFC 1939 server minimum idle timeout is 10 minutes
+	// pop3.postman.i2p timeout is 5 minutes
+	// We want to be less than that.
+	private static final int DEFAULT_IDLE_SECONDS = 4*60;
+	private static final int MIN_IDLE_CONFIG = 60;
+
+	public IdleCloser(POP3MailBox mailbox) {
+		this.mailbox = mailbox;
+		timer = new Checker();
+	}
+
+	public void cancel() {
+		isDead = true;
+		timer.cancel();
+	}
+
+	private static long getMaxIdle() {
+		int seconds = DEFAULT_IDLE_SECONDS;
+		String con = Config.getProperty(WebMail.CONFIG_IDLE_SECONDS);
+		if (con != null) {
+			try {
+				int secs = Integer.parseInt(con);
+				if (secs < MIN_IDLE_CONFIG)
+					secs = MIN_IDLE_CONFIG;
+				seconds = secs;
+			} catch (NumberFormatException nfe) {}
+		}
+		return seconds * 1000L;
+	}
+
+	private class Checker extends SimpleTimer2.TimedEvent {
+
+		public Checker() {
+			super(I2PAppContext.getGlobalContext().simpleTimer2(), getMaxIdle() + 5*1000);
+		}
+
+	        public void timeReached() {
+			if (isDead)
+				return;
+			// unsynchronized here, synch in thread only
+			if (!mailbox.isConnected())
+				return;
+			if (!isClosing) {
+				long config = getMaxIdle();
+				long idle = System.currentTimeMillis() - mailbox.getLastActivity();
+				long remaining = config - idle;
+				if (remaining <= 0) {
+					Debug.debug(Debug.DEBUG, "Threading close after " +
+							idle + " ms idle");
+					Thread t = new Closer();
+					isClosing = true;
+					t.start();
+				} else {
+					Debug.debug(Debug.DEBUG, "Not closing after " +
+							idle + " ms idle");
+					schedule(remaining + 5000);
+				}
+			}
+		}
+	}
+
+	private class Closer extends I2PAppThread {
+
+		public Closer() {
+			super("Susimail-Closer");
+		}
+
+	        public void run() {
+			try {
+				synchronized (mailbox.getLock()) {
+					if (!mailbox.isConnected())
+						return;
+					long config = getMaxIdle();
+					long idle = System.currentTimeMillis() - mailbox.getLastActivity();
+					long remaining = config - idle;
+					if (remaining <= 0) {
+						// If we have items to delete, wait for the response code,
+						// otherwise the DelayedDeleter thread will have to run later.
+						// Since we are threaded we can do that here.
+						boolean shouldWait = !mailbox.hasQueuedDeletions();
+						mailbox.close(shouldWait);
+						isDead = true;
+					} else {
+						timer.schedule(remaining + 5000);
+					}
+				}
+			} finally {		
+				isClosing = false;
+			}		
+		}
+	}
+}

apps/susimail/src/src/i2p/susi/webmail/pop3/POP3MailBox.java

@@ -25,6 +25,9 @@ package i2p.susi.webmail.pop3;
 
 import i2p.susi.debug.Debug;
 import i2p.susi.webmail.Messages;
+import i2p.susi.webmail.NewMailListener;
+import i2p.susi.webmail.WebMail;
+import i2p.susi.util.Config;
 import i2p.susi.util.ReadBuffer;
 
 import java.io.ByteArrayOutputStream;
@@ -44,7 +47,7 @@ import net.i2p.data.DataHelper;
 /**
  * @author susi23
  */
-public class POP3MailBox {
+public class POP3MailBox implements NewMailListener {
 
 	private final String host, user, pass;
 
@@ -70,6 +73,11 @@ public class POP3MailBox {
 
 	private final Object synchronizer;
 	private final DelayedDeleter delayedDeleter;
+	// instantiated after first successful connection
+	private BackgroundChecker backgroundChecker;
+	// instantiated after every successful connection
+	private IdleCloser idleCloser;
+	private volatile NewMailListener newMailListener;
 
 	/**
 	 * Does not connect. Caller must call connectToServer() if desired.
@@ -109,7 +117,7 @@ public class POP3MailBox {
 				// we must be connected to know the UIDL to ID mapping
 				checkConnection();
 			} catch (IOException ioe) {
-				Debug.debug( Debug.DEBUG, "Error fetching: " + ioe);
+				Debug.debug( Debug.DEBUG, "Error fetching header: " + ioe);
 				return null;
 			}
 			int id = getIDfromUIDL(uidl);
@@ -161,7 +169,7 @@ public class POP3MailBox {
 				// we must be connected to know the UIDL to ID mapping
 				checkConnection();
 			} catch (IOException ioe) {
-				Debug.debug( Debug.DEBUG, "Error fetching: " + ioe);
+				Debug.debug( Debug.DEBUG, "Error fetching body: " + ioe);
 				return null;
 			}
 			int id = getIDfromUIDL(uidl);
@@ -205,6 +213,7 @@ public class POP3MailBox {
 			try {
 				sendCmds(srs);
 			} catch (IOException ioe) {
+				Debug.debug( Debug.DEBUG, "Error fetching bodies: " + ioe);
 				// todo maybe
 			}
 		}
@@ -602,6 +611,7 @@ public class POP3MailBox {
 			lastError = e.toString();
 			return;
 		} catch (IOException e) {
+			Debug.debug( Debug.DEBUG, "Error connecting: " + e);
 			lastError = e.toString();
 			return;
 		}
@@ -609,12 +619,14 @@ public class POP3MailBox {
 			try {
 				// pipeline 2 commands
 				lastError = "";
+				socket.setSoTimeout(120*1000);
 				boolean ok = doHandshake();
 				if (ok) {
 					// TODO APOP (unsupported by postman)
 					List<SendRecv> cmds = new ArrayList<SendRecv>(4);
 					cmds.add(new SendRecv("USER " + user, Mode.A1));
 					cmds.add(new SendRecv("PASS " + pass, Mode.A1));
+					socket.setSoTimeout(60*1000);
 					ok =  sendCmds(cmds);
 				}
 				if (ok) {
@@ -627,7 +639,8 @@ public class POP3MailBox {
 					SendRecv list = new SendRecv("LIST", Mode.LS);
 					cmds.add(list);
 					// check individual responses
-					sendCmds(cmds);
+					socket.setSoTimeout(120*1000);
+					ok = sendCmds(cmds);
 					if (stat.result)
 						updateMailCount(stat.response);
 					else
@@ -640,6 +653,12 @@ public class POP3MailBox {
 						updateSizes(list.ls);
 					else
 						Debug.debug(Debug.DEBUG, "LIST failed");
+					socket.setSoTimeout(300*1000);
+					if (ok && backgroundChecker == null &&
+						Boolean.parseBoolean(Config.getProperty(WebMail.CONFIG_BACKGROUND_CHECK)))
+						backgroundChecker = new BackgroundChecker(this);
+					if (ok && idleCloser == null)
+						idleCloser = new IdleCloser(this);
 				} else {
 					if (lastError.equals(""))
 						lastError = _("Error connecting to server");
@@ -790,6 +809,7 @@ public class POP3MailBox {
 						sr.rb = getResultNa();
 						sr.result = true;
 					} catch (IOException ioe) {
+						Debug.debug( Debug.DEBUG, "Error getting RB: " + ioe);
 						result = false;
 						sr.result = false;
 					}
@@ -800,6 +820,7 @@ public class POP3MailBox {
 						sr.ls = getResultNl();
 						sr.result = true;
 					} catch (IOException ioe) {
+						Debug.debug( Debug.DEBUG, "Error getting LS: " + ioe);
 						result = false;
 						sr.result = false;
 					}
@@ -975,7 +996,6 @@ public class POP3MailBox {
 	 * Warning - forces a connection.
 	 *
 	 * @return The amount of e-mails available.
-	 * @deprecated unused
 	 */
 	public int getNumMails() {
 		synchronized( synchronizer ) {
@@ -1002,13 +1022,58 @@ public class POP3MailBox {
 		return e;
 	}
 
+
+	/**
+	 *  Relay from the checker to the webmail session object,
+	 *  which relays to MailCache, which will fetch the mail from us
+	 *  in a big circle
+	 *
+	 *  @since 0.9.13
+	 */
+	public void setNewMailListener(NewMailListener nml) {
+		newMailListener = nml;
+	}
+
+	/**
+	 *  Relay from the checker to the webmail session object,
+	 *  which relays to MailCache, which will fetch the mail from us
+	 *  in a big circle
+	 *
+	 *  @since 0.9.13
+	 */
+	public void foundNewMail() {
+		NewMailListener  nml = newMailListener;
+		if (nml != null)
+			nml.foundNewMail();
+	}
+
 	/**
 	 *  Close without waiting for response,
 	 *  and remove any delayed tasks and resources.
 	 */
 	public void destroy() {
 		delayedDeleter.cancel();
-		close(false);
+		synchronized( synchronizer ) {
+			if (backgroundChecker != null)
+				backgroundChecker.cancel();
+			close(false);
+		}
+	}
+
+	/**
+	 *  For helper threads to lock
+	 *  @since 0.9.13
+	 */
+	Object getLock() {
+		return synchronizer;
+	}
+
+	/**
+	 *  Do we have UIDLs to delete?
+	 *  @since 0.9.13
+	 */
+	boolean hasQueuedDeletions() {
+		return !delayedDeleter.getQueued().isEmpty();
 	}
 
 	/**
@@ -1024,9 +1089,11 @@ public class POP3MailBox {
 	 *  Deletes all queued deletions.
 	 *  @since 0.9.13
 	 */
-	private void close(boolean shouldWait) {
+	void close(boolean shouldWait) {
 		synchronized( synchronizer ) {
 			Debug.debug(Debug.DEBUG, "close()");
+			if (idleCloser != null)
+				idleCloser.cancel();
 			if (socket != null && socket.isConnected()) {
 				try {
 					Collection<String> toDelete = delayedDeleter.getQueued();
@@ -1058,7 +1125,9 @@ public class POP3MailBox {
 						sendCmd1aNoWait("QUIT");
 					}
 					socket.close();
-				} catch (IOException e) {}
+				} catch (IOException e) {
+					Debug.debug( Debug.DEBUG, "error closing: " + e);
+				}
 			}
 			socket = null;
 			connected = false;

apps/susimail/src/src/i2p/susi/webmail/smtp/SMTPClient.java

@@ -219,6 +219,7 @@ public class SMTPClient {
 			// Pipelining ref: RFC 2920
 			// AUTH ref: RFC 4954
 			if (ok) {
+				socket.setSoTimeout(120*1000);
 				int result = sendCmd(null);
 				if (result != 220) {
 					error += _("Server refused connection") + " (" + result +  ")<br>";
@@ -228,6 +229,7 @@ public class SMTPClient {
 			if (ok) {
 				sendCmdNoWait( "EHLO localhost" );
 				socket.getOutputStream().flush();
+				socket.setSoTimeout(60*1000);
 				Result r = getFullResult();
 				if (r.result == 250) {
 					supportsPipelining = r.recv.contains("PIPELINING");
@@ -266,6 +268,7 @@ public class SMTPClient {
 					body = body.replace( "\r\n.\r\n", "\r\n..\r\n" );
 				socket.getOutputStream().write(DataHelper.getUTF8(body));
 				socket.getOutputStream().write(DataHelper.getASCII("\r\n.\r\n"));
+				socket.setSoTimeout(0);
 				int result = sendCmd(null);
 				if (result == 250)
 					mailSent = true;

history.txt

@@ -1,3 +1,13 @@
+2014-04-24 zzz
+ * SusiMail:
+   - Add background mail checker
+   - Add idle timeout connection closer
+   - Rely on idle checker for most delayed deletions
+   - Cleanup resources better when shutting down session
+   - Don't add deleted mails to folder
+   - Set socket soTimeouts
+   - Display errors after check mail button pushed
+
 2014-04-23 zzz
  * SusiMail:
    - Queue deletions for a later thread

router/java/src/net/i2p/router/RouterVersion.java

@@ -18,7 +18,7 @@ public class RouterVersion {
     /** deprecated */
     public final static String ID = "Monotone";
     public final static String VERSION = CoreVersion.VERSION;
-    public final static long BUILD = 10;
+    public final static long BUILD = 11;
 
     /** for example "-test" */
     public final static String EXTRA = "";