* adjust the algorithm to deal with IO bound requests:

if more tokens become available while the first pending request is still blocked on
  read/write (aka after allocation and before next .waitForAllocation()), give the tokens
  to the next request
* refactor the satisfy{In,Out}boundRequests methods into smaller logical units

router/java/src/net/i2p/router/transport/FIFOBandwidthLimiter.java

@@ -128,9 +128,38 @@ public class FIFOBandwidthLimiter {
     private final void satisfyInboundRequests() {
         List satisfied = null;
         synchronized (_pendingInboundRequests) {
-            while (_pendingInboundRequests.size() > 0) {
-                SimpleRequest req = (SimpleRequest)_pendingInboundRequests.get(0);
             if (_inboundUnlimited) {
+                satisfied = locked_satisfyInboundUnlimited();
+            } else {
+                if (_availableInboundBytes > 0) {
+                    satisfied = locked_satisfyInboundAvailable();
+                } else {
+                    // no bandwidth available
+                    if (_log.shouldLog(Log.WARN))
+                        _log.warn("Still denying the " + _pendingInboundRequests.size() 
+                                  + " pending inbound requests (available "
+                                  + _availableInboundBytes + "/" + _availableOutboundBytes + " in/out)");
+                }
+            }
+        }
+        
+        if (satisfied != null) {
+            for (int i = 0; i < satisfied.size(); i++) {
+                SimpleRequest req = (SimpleRequest)satisfied.get(i);
+                req.notifyAllocation();
+            }
+        }
+    }
+    
+    /**
+     * There are no limits, so just give every inbound request whatever they want
+     *
+     */
+    private final List locked_satisfyInboundUnlimited() {
+        List satisfied = null;
+
+        while (_pendingInboundRequests.size() > 0) {
+            SimpleRequest req = (SimpleRequest)_pendingInboundRequests.remove(0);
             int allocated = req.getPendingInboundRequested();
             _totalAllocatedInboundBytes += allocated;
             req.allocateBytes(allocated, 0);
@@ -143,11 +172,32 @@ public class FIFOBandwidthLimiter {
                             + req.getTotalInboundRequested() + " bytes (waited " 
                             + waited
                             + "ms) pending " + _pendingInboundRequests.size());
-                    // if we're unlimited, we always grant it fully, so there's no need to keep it around
-                    _pendingInboundRequests.remove(0);
             if (waited > 10)
                 _context.statManager().addRateData("bwLimiter.inboundDelayedTime", waited, waited);
-                } else if (_availableInboundBytes > 0) {
+        }
+        return satisfied;
+    }
+    
+    /**
+     * ok, we have limits, so lets iterate through the requests, allocating as much
+     * bandwidth as we can to those who have used what we have given them and are waiting
+     * for more (giving priority to the first ones who requested it)
+     * 
+     * @return list of requests that were completely satisfied
+     */
+    private final List locked_satisfyInboundAvailable() {
+        List satisfied = null;
+        
+        for (int i = 0; i < _pendingInboundRequests.size(); i++) {
+            if (_availableInboundBytes <= 0) break;
+            SimpleRequest req = (SimpleRequest)_pendingInboundRequests.get(i);
+            if (req.getAllocationsSinceWait() > 0) {
+                // we have already allocated some values to this request, but
+                // they haven't taken advantage of it yet (most likely they're
+                // IO bound)
+                continue;
+            }
+            // ok, they are really waiting for us to give them stuff
             int requested = req.getPendingInboundRequested();
             int allocated = 0;
             if (_availableInboundBytes > requested) 
@@ -175,20 +225,29 @@ public class FIFOBandwidthLimiter {
                                 + req.getTotalInboundRequested() + " bytes, waited " 
                                 + waited
                                 + "ms) pending " + _pendingInboundRequests.size());
-                        _pendingInboundRequests.remove(0);
+                _pendingInboundRequests.remove(i);
+                i--;
                 if (waited > 10)
                     _context.statManager().addRateData("bwLimiter.inboundDelayedTime", waited, waited);
             }
+        }
+        return satisfied;
+    }
+    
+    private final void satisfyOutboundRequests() {
+        List satisfied = null;
+        synchronized (_pendingOutboundRequests) {
+            if (_outboundUnlimited) {
+                satisfied = locked_satisfyOutboundUnlimited();
+            } else {
+                if (_availableOutboundBytes > 0) {
+                    satisfied = locked_satisfyOutboundAvailable();
                 } else {
                     // no bandwidth available
                     if (_log.shouldLog(Log.WARN))
-                        _log.warn("Still denying the first inbound request (" + req.getRequestName() 
+                        _log.warn("Still denying the " + _pendingOutboundRequests.size() 
-                                    + " for " 
+                                  + " pending outbound requests (available "
-                                    + req.getTotalInboundRequested() + " bytes (available "
+                                  + _availableInboundBytes + "/" + _availableOutboundBytes + " in/out)");
-                                    + _availableInboundBytes + "/" + _availableOutboundBytes + " in/out) (waited " 
-                                    + (_context.clock().now() - req.getRequestTime()) 
-                                    + "ms so far) pending " + (_pendingInboundRequests.size()));
-                    break;
                 }
             }
         }
@@ -201,12 +260,15 @@ public class FIFOBandwidthLimiter {
         }
     }
     
-    private final void satisfyOutboundRequests() {
+    /**
+     * There are no limits, so just give every outbound request whatever they want
+     *
+     */
+    private final List locked_satisfyOutboundUnlimited() {
         List satisfied = null;
-        synchronized (_pendingOutboundRequests) {
+
         while (_pendingOutboundRequests.size() > 0) {
-                SimpleRequest req = (SimpleRequest)_pendingOutboundRequests.get(0);
+            SimpleRequest req = (SimpleRequest)_pendingOutboundRequests.remove(0);
-                if (_outboundUnlimited) {
             int allocated = req.getPendingOutboundRequested();
             _totalAllocatedOutboundBytes += allocated;
             req.allocateBytes(0, allocated);
@@ -219,11 +281,32 @@ public class FIFOBandwidthLimiter {
                             + req.getTotalOutboundRequested() + " bytes (waited " 
                             + waited
                             + "ms) pending " + _pendingOutboundRequests.size());
-                    // if we're unlimited, we always grant it fully, so there's no need to keep it around
-                    _pendingOutboundRequests.remove(0);
             if (waited > 10)
                 _context.statManager().addRateData("bwLimiter.outboundDelayedTime", waited, waited);
-                } else if (_availableOutboundBytes > 0) {
+        }
+        return satisfied;
+    }
+    
+    /**
+     * ok, we have limits, so lets iterate through the requests, allocating as much
+     * bandwidth as we can to those who have used what we have given them and are waiting
+     * for more (giving priority to the first ones who requested it)
+     * 
+     * @return list of requests that were completely satisfied
+     */
+    private final List locked_satisfyOutboundAvailable() {
+        List satisfied = null;
+        
+        for (int i = 0; i < _pendingOutboundRequests.size(); i++) {
+            if (_availableOutboundBytes <= 0) break;
+            SimpleRequest req = (SimpleRequest)_pendingOutboundRequests.get(i);
+            if (req.getAllocationsSinceWait() > 0) {
+                // we have already allocated some values to this request, but
+                // they haven't taken advantage of it yet (most likely they're
+                // IO bound)
+                continue;
+            }
+            // ok, they are really waiting for us to give them stuff
             int requested = req.getPendingOutboundRequested();
             int allocated = 0;
             if (_availableOutboundBytes > requested) 
@@ -251,30 +334,13 @@ public class FIFOBandwidthLimiter {
                                 + req.getTotalOutboundRequested() + " bytes, waited " 
                                 + waited
                                 + "ms) pending " + _pendingOutboundRequests.size());
-                        _pendingOutboundRequests.remove(0);
+                _pendingOutboundRequests.remove(i);
+                i--;
                 if (waited > 10)
                     _context.statManager().addRateData("bwLimiter.outboundDelayedTime", waited, waited);
             }
-                } else {
-                    // no bandwidth available
-                    if (_log.shouldLog(Log.WARN))
-                        _log.warn("Still denying the first outbound request (" + req.getRequestName() 
-                                    + " for " 
-                                    + req.getTotalOutboundRequested() + " bytes (available "
-                                    + _availableInboundBytes + "/" + _availableOutboundBytes + " in/out) (waited " 
-                                    + (_context.clock().now() - req.getRequestTime()) 
-                                    + "ms so far) pending " + (_pendingOutboundRequests.size()));
-                    break;
-                }
-            }
-        }
-        
-        if (satisfied != null) {
-            for (int i = 0; i < satisfied.size(); i++) {
-                SimpleRequest req = (SimpleRequest)satisfied.get(i);
-                req.notifyAllocation();
-            }
         }
+        return satisfied;
     }
     
     public void renderStatusHTML(OutputStream out) throws IOException {
@@ -318,6 +384,7 @@ public class FIFOBandwidthLimiter {
         private long _requestId;
         private long _requestTime;
         private String _target;
+        private int _allocationsSinceWait;
         
         public SimpleRequest(int in, int out, String target) {
             _inTotal = in;
@@ -336,6 +403,7 @@ public class FIFOBandwidthLimiter {
         public int getTotalInboundRequested() { return _inTotal; }
         public int getPendingInboundRequested() { return _inTotal - _inAllocated; }
         public void waitForNextAllocation() {
+            _allocationsSinceWait = 0;
             if ( (_outAllocated >= _outTotal) && 
                  (_inAllocated >= _inTotal) ) 
                 return;
@@ -345,9 +413,11 @@ public class FIFOBandwidthLimiter {
                 }
             } catch (InterruptedException ie) {}
         }
+        int getAllocationsSinceWait() { return _allocationsSinceWait; }
         void allocateBytes(int in, int out) {
             _inAllocated += in;
             _outAllocated += out;
+            _allocationsSinceWait++;
         }
         void notifyAllocation() {
             synchronized (SimpleRequest.this) {