From 44d6e117d54b989a580539e2392fccb05a84d43f Mon Sep 17 00:00:00 2001 From: zzz Date: Mon, 20 Oct 2014 14:01:36 +0000 Subject: [PATCH] Console and Eepsite Jetty: Switch back to QueuedThreadPool (ticket #1395) In Jetty 5/6, the default QTP was not concurrent, so we switched to ThreadPoolExecutor with a fixed-size queue, a set maxThreads, and a RejectedExecutionPolicy of CallerRuns. Unfortunately, CallerRuns causes lockups in Jetty NIO. In addition, no flavor of TPE gives us what QTP does: - TPE direct handoff (which we were using) never queues. This doesn't provide any burst management when maxThreads is reached. CallerRuns was an attempt to work around that. - TPE unbounded queue does not adjust the number of threads. This doesn't provide automatic resource management. - TPE bounded queue does not add threads until the queue is full. This doesn't provide good responsiveness to even small bursts. QTP adds threads as soon as the queue is non-empty. QTP as of Jetty 7 uses concurrent. QTP unbounded queue is the default in Jetty. So switch back to QTP with a bounded queue, which does what we want, which is first expand the thread pool, then start queueing, then reject. ref: http://docs.oracle.com/javase/6/docs/api/java/util/concurrent/ThreadPoolExecutor.html https://wiki.eclipse.org/Jetty/Howto/High_Load --- .../i2p/router/web/RouterConsoleRunner.java | 62 ++++++++++++++----- history.txt | 4 ++ installer/resources/eepsite/jetty.xml | 43 ++++++++++--- .../src/net/i2p/router/RouterVersion.java | 4 +- 4 files changed, 87 insertions(+), 26 deletions(-) diff --git a/apps/routerconsole/java/src/net/i2p/router/web/RouterConsoleRunner.java b/apps/routerconsole/java/src/net/i2p/router/web/RouterConsoleRunner.java index 92fbd1a06b..fe62160f3b 100644 --- a/apps/routerconsole/java/src/net/i2p/router/web/RouterConsoleRunner.java +++ b/apps/routerconsole/java/src/net/i2p/router/web/RouterConsoleRunner.java @@ -15,11 +15,7 @@ import java.util.Map; import java.util.Properties; import java.util.SortedSet; import java.util.StringTokenizer; -import java.util.concurrent.Executors; -import java.util.concurrent.SynchronousQueue; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.LinkedBlockingQueue; import net.i2p.I2PAppContext; import net.i2p.app.ClientAppManager; @@ -122,7 +118,9 @@ public class RouterConsoleRunner implements RouterApp { private static final String USAGE = "Bad RouterConsoleRunner arguments, check clientApp.0.args in your clients.config file! " + "Usage: [[port host[,host]] [-s sslPort [host[,host]]] [webAppsDir]]"; + /** this is for the handlers only. We will adjust for the connectors and acceptors below. */ private static final int MIN_THREADS = 1; + /** this is for the handlers only. We will adjust for the connectors and acceptors below. */ private static final int MAX_THREADS = 24; private static final int MAX_IDLE_TIME = 90*1000; private static final String THREAD_NAME = "RouterConsole Jetty"; @@ -318,19 +316,45 @@ public class RouterConsoleRunner implements RouterApp { _server = new Server(); _server.setGracefulShutdown(1000); - try { - ThreadPool ctp = new CustomThreadPoolExecutor(); - // Gone in Jetty 7 - //ctp.prestartAllCoreThreads(); - _server.setThreadPool(ctp); - } catch (Throwable t) { + // In Jetty 6, QTP was not concurrent, so we switched to + // ThreadPoolExecutor with a fixed-size queue, a set maxThreads, + // and a RejectedExecutionPolicy of CallerRuns. + // Unfortunately, CallerRuns causes lockups in Jetty NIO (ticket #1395) + // In addition, no flavor of TPE gives us what QTP does: + // - TPE direct handoff (which we were using) never queues. + // This doesn't provide any burst management when maxThreads is reached. + // CallerRuns was an attempt to work around that. + // - TPE unbounded queue does not adjust the number of threads. + // This doesn't provide automatic resource management. + // - TPE bounded queue does not add threads until the queue is full. + // This doesn't provide good responsiveness to even small bursts. + // QTP adds threads as soon as the queue is non-empty. + // QTP as of Jetty 7 uses concurrent. + // QTP unbounded queue is the default in Jetty. + // So switch back to QTP with a bounded queue. + // + // ref: + // http://docs.oracle.com/javase/6/docs/api/java/util/concurrent/ThreadPoolExecutor.html + // https://wiki.eclipse.org/Jetty/Howto/High_Load + // + //try { + // ThreadPool ctp = new CustomThreadPoolExecutor(); + // // Gone in Jetty 7 + // //ctp.prestartAllCoreThreads(); + // _server.setThreadPool(ctp); + //} catch (Throwable t) { // class not found... - System.out.println("INFO: Jetty concurrent ThreadPool unavailable, using QueuedThreadPool"); - QueuedThreadPool qtp = new QueuedThreadPool(MAX_THREADS); - qtp.setMinThreads(MIN_THREADS); + //System.out.println("INFO: Jetty concurrent ThreadPool unavailable, using QueuedThreadPool"); + LinkedBlockingQueue lbq = new LinkedBlockingQueue(4*MAX_THREADS); + QueuedThreadPool qtp = new QueuedThreadPool(lbq); + // min and max threads will be set below + //qtp.setMinThreads(MIN_THREADS); + //qtp.setMaxThreads(MAX_THREADS); qtp.setMaxIdleTimeMs(MAX_IDLE_TIME); + qtp.setName(THREAD_NAME); + qtp.setDaemon(true); _server.setThreadPool(qtp); - } + //} HandlerCollection hColl = new HandlerCollection(); ContextHandlerCollection chColl = new ContextHandlerCollection(); @@ -527,6 +551,10 @@ public class RouterConsoleRunner implements RouterApp { System.err.println("Unable to bind routerconsole to any address on port " + _listenPort + (sslPort > 0 ? (" or SSL port " + sslPort) : "")); return; } + // Each address spawns a Connector and an Acceptor thread + // If the min is less than this, we have no thread for the handlers or the expiration thread. + qtp.setMinThreads(MIN_THREADS + (2 * boundAddresses)); + qtp.setMaxThreads(MAX_THREADS + (2 * boundAddresses)); File tmpdir = new SecureDirectory(workDir, ROUTERCONSOLE + "-" + (_listenPort != null ? _listenPort : _sslListenPort)); @@ -838,6 +866,7 @@ public class RouterConsoleRunner implements RouterApp { * Just to set the name and set Daemon * @since Jetty 6 */ +/***** private static class CustomThreadPoolExecutor extends ExecutorThreadPool { public CustomThreadPoolExecutor() { super(new ThreadPoolExecutor( @@ -848,11 +877,13 @@ public class RouterConsoleRunner implements RouterApp { ); } } +*****/ /** * Just to set the name and set Daemon * @since Jetty 6 */ +/***** private static class CustomThreadFactory implements ThreadFactory { public Thread newThread(Runnable r) { @@ -862,5 +893,6 @@ public class RouterConsoleRunner implements RouterApp { return rv; } } +*****/ } diff --git a/history.txt b/history.txt index 9c7e4e3332..aab955f8da 100644 --- a/history.txt +++ b/history.txt @@ -1,3 +1,7 @@ +2014-10-20 zzz + * Console: Fix lockups (ticket #1395) + * Eepsite Jetty: Switch back to QueuedThreadPool (ticket #1395) + 2014-10-17 zzz * NTCP: Deadlock fix 3rd try (ticket #1394) diff --git a/installer/resources/eepsite/jetty.xml b/installer/resources/eepsite/jetty.xml index 41ebe5b7d5..4ebecbb571 100644 --- a/installer/resources/eepsite/jetty.xml +++ b/installer/resources/eepsite/jetty.xml @@ -53,26 +53,50 @@ - - + 3 - 20 - 60000 @@ -83,6 +107,7 @@ + --> diff --git a/router/java/src/net/i2p/router/RouterVersion.java b/router/java/src/net/i2p/router/RouterVersion.java index 7f4edb9752..affa6f41b1 100644 --- a/router/java/src/net/i2p/router/RouterVersion.java +++ b/router/java/src/net/i2p/router/RouterVersion.java @@ -18,10 +18,10 @@ public class RouterVersion { /** deprecated */ public final static String ID = "Monotone"; public final static String VERSION = CoreVersion.VERSION; - public final static long BUILD = 13; + public final static long BUILD = 14; /** for example "-test" */ - public final static String EXTRA = ""; + public final static String EXTRA = "-rc"; public final static String FULL_VERSION = VERSION + "-" + BUILD + EXTRA; public static void main(String args[]) { System.out.println("I2P Router version: " + FULL_VERSION);