tchannel 3.6.19 → 3.6.20

package/benchmarks/Makefile

@@ -1,6 +1,10 @@
 torch-client-bench:
 	node index.js --relay --torch client --torchFile ./flame.raw --torchTime 10 \
-		-- --relay --skipPing -s 4096,16384 -p 1000,10000,20000
+		-- --relay --skipPing -s 4096,16384 -p 1000 -m 3
+
+torch-client-only-bench:
+	node index.js --torch client --torchFile ./flame.raw --torchTime 10 \
+		-- --skipPing -s 4096,16384 -p 1000 -m 3
 
 torch-server-bench:
 	node index.js --relay --torch server --torchFile ./flame.raw --torchTime 10 \

package/benchmarks/index.js

@@ -265,6 +265,7 @@ function startClient(clientPort) {
         '--benchPort', String(self.ports.serverPort),
         '--relayServerPort', String(self.ports.relayServerPort),
         '--clientPort', String(clientPort),
+        '--instances', String(self.instanceCount),
         '--instanceNumber', String(self.benchCounter)
     ]);
     var benchProc = self.run(bench, args);

package/benchmarks/multi_bench.js

@@ -63,13 +63,17 @@ argv.pipeline = parseIntList(argv.pipeline);
 argv.sizes = parseIntList(argv.sizes);
 
 var DESTINATION_SERVER;
+var DESTINATION_PORT;
 var TRACE_SERVER;
 var CLIENT_PORT = argv.clientPort;
+var INSTANCES = 0;
 
 if (argv.relay) {
     DESTINATION_SERVER = '127.0.0.1:' + argv.relayServerPort;
 } else {
     DESTINATION_SERVER = '127.0.0.1:' + argv.benchPort;
+    DESTINATION_PORT = argv.benchPort;
+    INSTANCES = parseInt(argv.instances, 10);
 }
 
 if (argv.trace) {
@@ -171,9 +175,19 @@ Test.prototype.newClient = function newClient(id, callback) {
         };
     }
 
+    var peers = [DESTINATION_SERVER];
+    if (INSTANCES > 0) {
+        peers = [];
+        var basePort = DESTINATION_PORT;
+        for (var i = 0; i < INSTANCES; i++) {
+            peers.push('127.0.0.1:' + (basePort + i));
+        }
+    }
+
     var client = clientChan.makeSubChannel({
         serviceName: 'benchmark',
-        peers: [DESTINATION_SERVER]
+        peers: peers,
+        minConnections: INSTANCES > 0 ? 10 : 1
     });
     client.createTime = Date.now();
     clientChan.listen(port, '127.0.0.1', function listened(err) {

package/package.json

@@ -2,7 +2,7 @@
   "name": "tchannel",
   "description": "network multiplexing and framing protocol for RPC or parser drag racing",
   "author": "mranney@uber.com",
-  "version": "3.6.19",
+  "version": "3.6.20",
   "scripts": {
     "lint": "eslint $(git ls-files | grep '.js$')",
     "travis": "npm run test",
@@ -10,7 +10,7 @@
     "benchmark": "echo '!!! DEPRECATED: Better to just run `node benchmarks` directly' >&2; node benchmarks",
     "hyperbahn-link-test": "./test/link_hyperbahn.sh",
     "tcurl-link-test": "./test/link_tcurl.sh",
-    "check-benchmark": "node benchmarks -- -r 10000 -p 100",
+    "check-benchmark": "node benchmarks -- -r 10000 -p 750",
     "take-benchmark": "make -C benchmarks take",
     "take-relay-benchmark": "make -C benchmarks take_relay",
     "take-trace-benchmark": "amke -C benchmarks take_trace",

package/peer.js

@@ -39,6 +39,8 @@ var PeerDrain = require('./drain.js').PeerDrain;
 var ObjectPool = require('./lib/object_pool');
 
 var DEFAULT_REPORT_INTERVAL = 1000;
+var INITIAL_CONN_ATTEMPT_DELAY = 5000;
+var CONN_ATTEMPT_DELAY_MULTIPLER = 2;
 
 /*eslint max-statements: [2, 40]*/
 function TChannelPeer(channel, hostPort, options) {
@@ -50,6 +52,8 @@ function TChannelPeer(channel, hostPort, options) {
     this.stateChangedEvent = this.defineEvent('stateChanged');
     this.allocConnectionEvent = this.defineEvent('allocConnection');
     this.removeConnectionEvent = this.defineEvent('removeConnection');
+    this.deltaOutConnectionEvent = this.defineEvent('deltaOutConnection');
+
     this.channel = channel;
     this.logger = this.channel.logger;
     this.timers = this.channel.timers;
@@ -66,6 +70,11 @@ function TChannelPeer(channel, hostPort, options) {
     this.boundOnPendingChange = onPendingChange;
     this.scoreRange = null;
 
+    // Timestamp when next conn attempt is allowed
+    this.nextConnAttemptTime = 0;
+    // How long to delay conn attempt by on failure (ms)
+    this.nextConnAttemptDelay = 0;
+
     this.waitForIdentifiedListeners = [];
 
     this.reportInterval = options.reportInterval || DEFAULT_REPORT_INTERVAL;
@@ -408,6 +417,47 @@ TChannelPeer.prototype.connectTo = function connectTo() {
     return self.connect(true);
 };
 
+TChannelPeer.prototype.tryConnect = function tryConnect() {
+    var self = this;
+
+    var connectTime = Date.now();
+    if (connectTime < self.nextConnAttemptTime) {
+        return;
+    }
+
+    var conn = this.getOutConnection();
+    if (!conn || conn.direction !== 'out') {
+        conn = this.connectTo();
+    }
+
+    this.waitForIdentified(conn, onIdentified);
+
+    function onIdentified(err) {
+        if (!err) {
+            self.nextConnAttemptDelay = 0;
+            self.nextConnAttemptTime = 0;
+            return;
+        }
+
+        if (self.nextConnAttemptDelay === 0) {
+            self.nextConnAttemptDelay = INITIAL_CONN_ATTEMPT_DELAY;
+        } else {
+            self.nextConnAttemptDelay *= CONN_ATTEMPT_DELAY_MULTIPLER;
+            // Add some amount of fuzz, +-100ms
+            self.nextConnAttemptDelay += Math.floor(100 * (Math.random() - 0.5));
+        }
+
+        var afterConnect = Date.now();
+        if (self.nextConnAttemptTime < afterConnect) {
+            // When in the past set next attempt to now + delay
+            self.nextConnAttemptTime = afterConnect + self.nextConnAttemptDelay;
+        } else {
+            // When in the future; go further in the future
+            self.nextConnAttemptTime += self.nextConnAttemptDelay;
+        }
+    }
+};
+
 TChannelPeer.prototype.waitForIdentified =
 function waitForIdentified(conn, callback) {
     var self = this;
@@ -522,9 +572,11 @@ TChannelPeer.prototype.addConnection = function addConnection(conn) {
     // TODO: second approx support pruning
     if (conn.direction === 'out') {
         self.connections.push(conn);
+        self.deltaOutConnectionEvent.emit(self, 1);
     } else {
         self.connections.unshift(conn);
     }
+
     conn.errorEvent.on(self.boundOnConnectionError);
     conn.closeEvent.on(self.boundOnConnectionClose);
     conn.ops.pendingChangeEvent.on(self.boundOnPendingChange);
@@ -606,15 +658,20 @@ TChannelPeer.prototype.removeConnection = function removeConnection(conn) {
     var self = this;
 
     var ret = null;
+    var isRemoved = false;
 
     var index = self.connections ? self.connections.indexOf(conn) : -1;
     if (index !== -1) {
         ret = self.connections.splice(index, 1)[0];
+        isRemoved = conn.direction === 'out';
     }
 
     self._maybeInvalidateScore('removeConnection');
 
     self.removeConnectionEvent.emit(self, conn);
+    if (isRemoved) {
+        self.deltaOutConnectionEvent.emit(self, -1);
+    }
     return ret;
 };
 

package/peer_heap.js

@@ -30,8 +30,10 @@ module.exports = PeerHeap;
 // peer list.
 var dfsStack = [0, 1, 2];
 
-function PeerHeap(random) {
+function PeerHeap(peers, random) {
     this.array = [];
+    this.peers = peers || null;
+    this.hasMinConnections = peers ? peers.hasMinConnections : false;
 
     this.random = random || Math.random;
     assert(typeof this.random === 'function', 'PeerHeap expected random fn');
@@ -44,25 +46,6 @@ function PeerHeap(random) {
     this.maxRangeStart = 0;
 }
 
-PeerHeap.prototype.choose1 = function choose1(threshold, filter) {
-    var self = this;
-    if (self.array[0].range.lo >= threshold && (!filter || filter(self.array[0].peer))) {
-        return self.array[0].peer;
-    }
-};
-
-PeerHeap.prototype.choose2 = function choose2(threshold, filter) {
-    var self = this;
-    var prob1 = self.array[0].peer.getScore();
-    var prob2 = self.array[1].peer.getScore();
-
-    if (prob1 > threshold && prob1 >= prob2 && (!filter || filter(self.array[0].peer))) {
-        return self.array[0].peer;
-    } else if (prob2 > threshold && (!filter || filter(self.array[1].peer))) {
-        return self.array[1].peer;
-    }
-};
-
 /*eslint-disable complexity */
 /*eslint-disable max-statements */
 PeerHeap.prototype.choose = function choose(threshold, filter) {
@@ -72,26 +55,36 @@ PeerHeap.prototype.choose = function choose(threshold, filter) {
         return null;
     }
 
-    if (self.array.length === 1) {
-        return self.choose1(threshold, filter);
-    }
-
-    if (self.array.length === 2) {
-        return self.choose2(threshold, filter);
-    }
-
+    var isSecondary = false;
     var chosenPeer = null;
+    var secondaryPeer = null;
     var highestProbability = 0;
+    var secondaryProbability = 0;
     var firstScore = self.array[0].peer.getScore();
 
+    var notEnoughPeers = false;
+    if (self.hasMinConnections) {
+        notEnoughPeers = self.peers.currentConnectedPeers < self.peers.minConnections;
+    }
+
     // Pointers into dfsStack
     var stackBegin = 0;
     var stackEnd = 0;
 
     if (firstScore > threshold && (!filter || filter(self.array[0].peer))) {
         // Don't check first peer if it looks good, check its children though
-        chosenPeer = self.array[0].peer;
-        highestProbability = firstScore;
+
+        var firstPeer = self.array[0].peer;
+        isSecondary = notEnoughPeers && firstPeer.isConnected('out');
+
+        if (isSecondary) {
+            secondaryPeer = firstPeer;
+            secondaryProbability = firstScore;
+        } else {
+            chosenPeer = firstPeer;
+            highestProbability = firstScore;
+        }
+
         // The array is seeded with 0, 1, 2 so we just have to advance the
         // stack pointers
         stackBegin = 1;
@@ -104,13 +97,20 @@ PeerHeap.prototype.choose = function choose(threshold, filter) {
 
         var el = self.array[i];
 
-        if (self.rangehis[i] <= self.maxRangeStart) {
+        if (!el || (
+            self.rangehis[i] <= self.maxRangeStart &&
+            (!filter && !notEnoughPeers)
+        )) {
             // This range ends before the range with the largest start begins,
             // so it can't possibly be chosen over any of the ranges we've
             // seen. All ranges below this one have a smaller end, so this
             // range and any below it can't be chosen.
             continue;
-        } else if (!filter || filter(el.peer)) {
+        }
+
+        if (!filter || filter(el.peer)) {
+            isSecondary = notEnoughPeers && el.peer.isConnected('out');
+
             // INLINE of TChannelPeer#getScore
             var lo = self.rangelos[i];
             var hi = self.rangehis[i];
@@ -120,9 +120,18 @@ PeerHeap.prototype.choose = function choose(threshold, filter) {
             }
             var probability = lo + ((hi - lo) * rand);
 
-            if ((probability > highestProbability) && (probability > threshold)) {
-                highestProbability = probability;
-                chosenPeer = el.peer;
+            var isBestChoice = !isSecondary ?
+                (probability > highestProbability) :
+                (probability > secondaryProbability);
+
+            if ((probability > threshold) && isBestChoice) {
+                if (isSecondary) {
+                    secondaryProbability = probability;
+                    secondaryPeer = el.peer;
+                } else {
+                    highestProbability = probability;
+                    chosenPeer = el.peer;
+                }
             }
         }
 
@@ -138,11 +147,18 @@ PeerHeap.prototype.choose = function choose(threshold, filter) {
         }
     }
 
-    return chosenPeer;
+    if (secondaryProbability > highestProbability && chosenPeer) {
+        chosenPeer.tryConnect(noop);
+        return secondaryPeer;
+    }
+
+    return chosenPeer || secondaryPeer;
 };
 /*eslint-enable complexity */
 /*eslint-enable max-statements */
 
+function noop() {}
+
 PeerHeap.prototype.clear = function clear() {
     var self = this;
 

package/sub_peers.js

@@ -27,13 +27,37 @@ var PeerHeap = require('./peer_heap.js');
 
 function TChannelSubPeers(channel, options) {
     TChannelPeersBase.call(this, channel, options);
+
+    var self = this;
     this.peerScoreThreshold = this.options.peerScoreThreshold || 0;
-    this._heap = new PeerHeap(channel.random);
     this.choosePeerWithHeap = channel.choosePeerWithHeap;
+
+    this.hasMinConnections = typeof options.minConnections === 'number';
+    this.minConnections = options.minConnections;
+
+    this.currentConnectedPeers = 0;
+    this._heap = new PeerHeap(this, channel.random);
+
+    this.boundOnOutConnectionDelta = boundOnOutConnectionDelta;
+
+    function boundOnOutConnectionDelta(delta, peer) {
+        self.onOutConnectionDelta(peer, delta);
+    }
 }
 
 inherits(TChannelSubPeers, TChannelPeersBase);
 
+TChannelSubPeers.prototype.onOutConnectionDelta =
+function onOutConnectionDelta(peer, delta) {
+    var connCount = peer.countConnections('out');
+
+    if (delta === 1 && connCount === 1) {
+        this.currentConnectedPeers++;
+    } else if (delta === -1 && connCount === 0) {
+        this.currentConnectedPeers--;
+    }
+};
+
 TChannelSubPeers.prototype.close = function close(callback) {
     var self = this;
 
@@ -55,6 +79,12 @@ TChannelSubPeers.prototype.add = function add(hostPort, options) {
     peer = topChannel.peers.add(hostPort, options);
     peer.setPreferConnectionDirection(self.preferConnectionDirection);
 
+    if (peer.countConnections('out') > 0) {
+        this.currentConnectedPeers++;
+    }
+
+    peer.deltaOutConnectionEvent.on(self.boundOnOutConnectionDelta);
+
     self._map[hostPort] = peer;
     self._keys.push(hostPort);
 
@@ -80,6 +110,13 @@ TChannelSubPeers.prototype._delete = function _del(peer) {
         return;
     }
 
+    if (peer.countConnections('out') > 0) {
+        this.currentConnectedPeers--;
+    }
+
+    peer.deltaOutConnectionEvent
+        .removeListener(self.boundOnOutConnectionDelta);
+
     delete self._map[peer.hostPort];
     popout(self._keys, index);
 
@@ -109,6 +146,7 @@ TChannelSubPeers.prototype.choosePeer = function choosePeer(req) {
     return self.chooseLinearPeer(req);
 };
 
+/*eslint max-statements: [2, 40]*/
 TChannelSubPeers.prototype.chooseLinearPeer = function chooseLinearPeer(req) {
     /* eslint complexity: [2, 15]*/
     var self = this;
@@ -121,11 +159,22 @@ TChannelSubPeers.prototype.chooseLinearPeer = function chooseLinearPeer(req) {
     var threshold = self.peerScoreThreshold;
 
     var selectedPeer = null;
+    var secondaryPeer = null;
     var selectedScore = 0;
+    var secondaryScore = 0;
+
+    var notEnoughPeers = false;
+    if (this.hasMinConnections) {
+        notEnoughPeers = this.currentConnectedPeers < this.minConnections;
+    }
+
     for (var i = 0; i < hosts.length; i++) {
         var hostPort = hosts[i];
         var peer = self._map[hostPort];
-        if (!req || !req.triedRemoteAddrs || !req.triedRemoteAddrs[hostPort]) {
+
+        var shouldSkip = req && req.triedRemoteAddrs && req.triedRemoteAddrs[hostPort];
+        if (!shouldSkip) {
+            var isSecondary = notEnoughPeers && peer.isConnected('out');
             var score = peer.getScore(req);
 
             if (self.channel.topChannel.peerScoredEvent) {
@@ -136,11 +185,24 @@ TChannelSubPeers.prototype.chooseLinearPeer = function chooseLinearPeer(req) {
                 });
             }
 
-            var want = score > threshold &&
-                       (selectedPeer === null || score > selectedScore);
+            var want;
+            if (isSecondary) {
+                want = score > threshold && (
+                    secondaryPeer === null || score > secondaryScore
+                );
+            } else {
+                want = score > threshold &&
+                    (selectedPeer === null || score > selectedScore);
+            }
+
             if (want) {
-                selectedPeer = peer;
-                selectedScore = score;
+                if (isSecondary) {
+                    secondaryPeer = peer;
+                    secondaryScore = score;
+                } else {
+                    selectedPeer = peer;
+                    selectedScore = score;
+                }
             }
         }
     }
@@ -152,14 +214,21 @@ TChannelSubPeers.prototype.chooseLinearPeer = function chooseLinearPeer(req) {
         });
     }
 
-    return selectedPeer;
+    if (secondaryScore > selectedScore && selectedPeer) {
+        selectedPeer.tryConnect(noop);
+        return secondaryPeer;
+    }
+
+    return selectedPeer || secondaryPeer;
 };
 
+function noop() {}
+
 TChannelSubPeers.prototype.chooseHeapPeer = function chooseHeapPeer(req) {
     var self = this;
 
     var peer;
-    if (req && req.triedRemoteAddrs) {
+    if ((req && req.triedRemoteAddrs)) {
         peer = self._choosePeerSkipTried(req);
     } else {
         peer = self._heap.choose(self.peerScoreThreshold);
@@ -182,7 +251,8 @@ function _choosePeerSkipTried(req) {
     return self._heap.choose(self.peerScoreThreshold, filterTriedPeers);
 
     function filterTriedPeers(peer) {
-        return !req.triedRemoteAddrs[peer.hostPort];
+        var shouldSkip = req.triedRemoteAddrs[peer.hostPort];
+        return !shouldSkip;
     }
 };
 

package/test/index.js

@@ -77,6 +77,7 @@ require('./relay-clamps-the-ttl.js');
 require('./relay-under-timeouts.js');
 require('./relay-kills-socket-after-timeouts.js');
 require('./object_pool.js');
+require('./peer-to-peer-load-balance.js');
 
 require('./trace/basic_server.js');
 require('./trace/server_2_requests.js');

package/test/lib/alloc-cluster.js

@@ -259,6 +259,10 @@ allocCluster.test.only = function testClusterOnly(desc, opts, t) {
     test.only(desc, clusterTester(opts, t));
 };
 
+allocCluster.test.skip = function testClusterSkip(desc, opts, t) {
+    test.skip(desc, clusterTester(opts, t));
+};
+
 function connectChannels(channels, callback) {
     return parallel(channels.map(function (channel) {
         return function connectChannelToHosts(callback) {

package/test/lib/batch-client.js

@@ -24,6 +24,19 @@ var parallel = require('run-parallel');
 var collectParallel = require('collect-parallel/array');
 var setTimeout = require('timers').setTimeout;
 
+/*
+    BatchClient takes a channel and a set of peers to talk
+    to.
+
+    It has a set of options including:
+        - totalRequests
+        - delay
+        - batchSize
+
+    If you want to do 100QPS you can set batchSize=1 and delay=10
+    If you want to do 1000QPS you can set batchSize=50 and delay=50
+
+*/
 module.exports = BatchClient;
 
 function BatchClient(channel, hosts, options) {
@@ -42,11 +55,12 @@ function BatchClient(channel, hosts, options) {
     self.serviceName = 'server';
     self.endpoint = options.endpoint || 'echo';
     self.body = 'foobar';
-    self.timeout = 500;
+    self.timeout = options.timeout || 500;
 
     self.subChannel = self.channel.makeSubChannel({
         serviceName: self.serviceName,
-        peers: self.hosts
+        peers: self.hosts,
+        minConnections: options.minConnections || null
     });
 
     self.requestOptions = {
@@ -54,6 +68,7 @@ function BatchClient(channel, hosts, options) {
         hasNoParent: true,
         timeout: self.timeout,
         retryFlags: options && options.retryFlags,
+        retryLimit: options && options.retryLimit ? options.retryLimit : 5,
         headers: {
             cn: 'client',
             as: 'raw'

package/test/peer-to-peer-load-balance.js

@@ -0,0 +1,459 @@
+// Copyright (c) 2015 Uber Technologies, Inc.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+'use strict';
+
+var collectParallel = require('collect-parallel/array');
+var metrics = require('metrics');
+
+var allocCluster = require('./lib/alloc-cluster.js');
+var BatchClient = require('./lib/batch-client.js');
+var CollapsedAssert = require('./lib/collapsed-assert.js');
+
+allocCluster.test('p2p requests from 40 -> 40', {
+    numPeers: 80
+}, function t(cluster, assert) {
+    setup(cluster);
+
+    collectParallel(cluster.batches, function runRequests(batch, _, cb) {
+        batch.sendRequests(cb);
+    }, onBatches);
+
+    /*eslint max-statements: [2, 40]*/
+    function onBatches(err, results) {
+        var cassert = CollapsedAssert();
+        cassert.ifError(err);
+
+        var statuses = [];
+        for (var i = 0; i < results.length; i++) {
+            cassert.ifError(results[i].err, 'expect no batch error');
+            cassert.ifError(results[i].value.errors.length > 0,
+                'expect zero errors in batch');
+            statuses.push(results[i].value);
+        }
+        cassert.report(assert, 'expected no errors');
+
+        var statusTable = findServerHostDistribution(statuses);
+
+        var uniqHosts = Object.keys(statusTable);
+        if (uniqHosts.length < 35) {
+            checkConnections();
+            checkDistributions(statusTable);
+        } else {
+            assert.ok(true, 'SKIP: suprisingly large number of peers reached');
+        }
+
+        assert.end();
+    }
+
+    function checkConnections() {
+        var cassert = CollapsedAssert();
+        var distribution = new metrics.Histogram();
+        for (var i = 0; i < cluster.batches.length; i++) {
+            var connCount = countConnections(cluster.batches[i]);
+            distribution.update(connCount);
+
+            cassert.ok(
+                connCount >= 1 &&
+                connCount <= 5,
+                'expected a small number of connections'
+            );
+        }
+
+        var info = distribution.printObj();
+        cassert.ok(info.min <= 2, 'expected low minimum');
+        cassert.ok(info.max <= 5, 'expected low maximum');
+        cassert.ok(info.sum <= 100, 'expected low total connections');
+        cassert.ok(info.p75 <= 2, 'expected low p75');
+        // console.log('conn distribution', info);
+
+        cassert.report(assert, 'expected batch connections to be fine');
+    }
+
+    function checkDistributions(statusTable) {
+        var uniqHosts = Object.keys(statusTable);
+        assert.ok(uniqHosts.length <= 35,
+            'Expected host reached (' + uniqHosts.length + ') to <= 35');
+
+        var distribution = new metrics.Histogram();
+        for (var i = 0; i < uniqHosts.length; i++) {
+            distribution.update(statusTable[uniqHosts[i]]);
+        }
+
+        var info = distribution.printObj();
+
+        var cassert = CollapsedAssert();
+        cassert.ok(info.min <= 50,
+            'expected minimum to be no more then 50'
+        );
+        cassert.equal(info.sum, 2000,
+            'expected 2000 requests to be made'
+        );
+        cassert.ok(info.median >= 48,
+            'expected median (' + info.median + ') to be larger then 49'
+        );
+        cassert.ok(info.max >= 100, 'expected maximum to be huge');
+        cassert.ok(info.p75 >= 50, 'expected P75 to be huge');
+        cassert.ok(info.p95 > 80, 'expected P95 to be huge');
+        cassert.ok(info.variance >= 500,
+            'expected variance (' + info.variance + ') to be huge'
+        );
+        // console.log('conn distribution', info);
+
+        cassert.report(assert, 'expected request distribution to be ok');
+    }
+});
+
+allocCluster.test('p2p requests from 40 -> 40 with minConnections', {
+    numPeers: 80,
+    channelOptions: {
+        choosePeerWithHeap: true
+    }
+}, function t(cluster, assert) {
+    setup(cluster, {
+        minConnections: 10
+    });
+
+    collectParallel(cluster.batches, function runRequests(batch, _, cb) {
+        batch.sendRequests(cb);
+    }, onBatches);
+
+    /*eslint max-statements: [2, 40]*/
+    function onBatches(err, results) {
+        var cassert = CollapsedAssert();
+        cassert.ifError(err);
+
+        var statuses = [];
+        for (var i = 0; i < results.length; i++) {
+            cassert.ifError(results[i].err, 'expect no batch error');
+            cassert.ifError(results[i].value.errors.length > 0,
+                'expect zero errors in batch');
+            statuses.push(results[i].value);
+        }
+        cassert.report(assert, 'expected no errors');
+
+        var statusTable = findServerHostDistribution(statuses);
+
+        cassert = verifyConnections(cluster, 10, 12);
+        cassert.report(assert, 'expected batch connections to be fine');
+
+        cassert = verifyDistributions(statusTable, {
+            min: 40,
+            sum: 2000,
+            median: [40, 60],
+            mean: [45, 55],
+            max: 120,
+            p75: [55, 70],
+            p95: 100,
+            variance: 500
+        });
+        cassert.report(assert, 'expected request distribution to be ok');
+
+        assert.end();
+    }
+});
+
+allocCluster.test('p2p requests where minConns > no of servers', {
+    numPeers: 45,
+    channelOptions: {
+        choosePeerWithHeap: true
+    }
+}, function t(cluster, assert) {
+    setup(cluster, {
+        minConnections: 6,
+        servers: 5
+    });
+
+    collectParallel(cluster.batches, function runRequests(batch, _, cb) {
+        batch.sendRequests(cb);
+    }, onBatches);
+
+    /*eslint max-statements: [2, 40]*/
+    function onBatches(err, results) {
+        var cassert = CollapsedAssert();
+        cassert.ifError(err);
+
+        var statuses = [];
+        for (var i = 0; i < results.length; i++) {
+            cassert.ifError(results[i].err, 'expect no batch error');
+            cassert.ifError(results[i].value.errors.length > 0,
+                'expect zero errors in batch');
+
+            statuses.push(results[i].value);
+        }
+        cassert.report(assert, 'expected no errors');
+
+        var statusTable = findServerHostDistribution(statuses);
+
+        cassert = verifyConnections(cluster, 5, 5);
+        cassert.report(assert, 'expected batch connections to be fine');
+
+        cassert = verifyDistributions(statusTable, {
+            min: 395,
+            sum: 2000,
+            median: [380, 420],
+            mean: [395, 405],
+            max: 500,
+            p75: [400, 450],
+            p95: 475
+        });
+        cassert.report(assert, 'expected request distribution to be ok');
+
+        assert.end();
+    }
+});
+
+allocCluster.test('p2p requests where half of servers down', {
+    numPeers: 48,
+    channelOptions: {
+        choosePeerWithHeap: true
+    }
+}, function t(cluster, assert) {
+    cluster.logger.whitelist('info', 'resetting connection');
+
+    setup(cluster, {
+        minConnections: 5,
+        servers: 8,
+        retryLimit: 2
+    });
+
+    // Close half the servers...
+    for (var j = 0; j < cluster.servers.length / 2; j++) {
+        cluster.servers[j * 2].close();
+    }
+
+    collectParallel(cluster.batches, function runRequests(batch, _, cb) {
+        batch.sendRequests(cb);
+    }, onBatches);
+
+    /*eslint max-statements: [2, 40]*/
+    function onBatches(err, results) {
+        var cassert = CollapsedAssert();
+        cassert.ifError(err);
+
+        var statuses = [];
+        for (var i = 0; i < results.length; i++) {
+            cassert.ifError(results[i].err, 'expect no batch error');
+            cassert.ifError(results[i].value.errors.length > 5,
+                'expect at most five error in batch(' +
+                    results[i].value.errors.length + ')');
+
+            statuses.push(results[i].value);
+        }
+        cassert.report(assert, 'expected no errors');
+
+        var statusTable = findServerHostDistribution(statuses);
+
+        cassert = verifyConnections(cluster, 4, 4);
+        cassert.report(assert, 'expected batch connections to be fine');
+
+        cassert = verifyDistributions(statusTable, {
+            min: 495,
+            sum: [1985, 2000],
+            median: [480, 520],
+            mean: [495, 505],
+            max: 600,
+            p75: [500, 575],
+            p95: 600
+        });
+        cassert.report(assert, 'expected request distribution to be ok');
+
+        var logs = cluster.logger.items();
+        assert.ok(logs.length <= 280 + 160,
+            'expected conn reset logs (' + logs.length + ') to be <= 420');
+
+        assert.end();
+    }
+});
+
+function findServerHostDistribution(statuses) {
+    var statusTable = {};
+    for (var i = 0; i < statuses.length; i++) {
+        var records = statuses[i].results;
+        for (var j = 0; j < records.length; j++) {
+            var record = records[j];
+            if (!statusTable[record.outReqHostPort]) {
+                statusTable[record.outReqHostPort] = 0;
+            }
+            statusTable[record.outReqHostPort]++;
+        }
+    }
+    return statusTable;
+}
+
+function countConnections(batchClient) {
+    var subChannel = batchClient.subChannel;
+    var peers = subChannel.peers.values();
+
+    var conns = [];
+    for (var i = 0; i < peers.length; i++) {
+        var peer = peers[i];
+        for (var j = 0; j < peer.connections.length; j++) {
+            conns.push(peer.connections[j]);
+        }
+    }
+
+    return conns.length;
+}
+
+function verifyConnections(cluster, min, max) {
+    var MIN = min;
+    var MAX = max;
+    var COUNT = cluster.batches.length;
+
+    var cassert = CollapsedAssert();
+    var distribution = new metrics.Histogram();
+    for (var i = 0; i < cluster.batches.length; i++) {
+        var connCount = countConnections(cluster.batches[i]);
+        distribution.update(connCount);
+
+        cassert.ok(
+            connCount >= MIN &&
+            connCount <= MAX,
+            'expected connections(' + connCount + ') to be ' +
+                '>= ' + MIN + ' and <= ' + MAX
+        );
+    }
+
+    var info = distribution.printObj();
+    cassert.ok(info.min <= MAX,
+        'expected min connections(' + info.min + ') to be <= ' + MAX);
+    cassert.ok(info.max >= MIN,
+        'expected max conns(' + info.max + ') to be >= ' + MIN);
+    cassert.ok(info.sum >= COUNT * MIN,
+        'expected sum of conns(' + info.sum + ') to be at ' +
+            COUNT * MIN + ' conns');
+    cassert.ok(info.p75 <= MAX,
+        'expected p75(' + info.p75 + ') to be <= ' + MAX);
+
+    return cassert;
+}
+
+function verifyDistributions(statusTable, opts) {
+    var uniqHosts = Object.keys(statusTable);
+
+    var distribution = new metrics.Histogram();
+    for (var i = 0; i < uniqHosts.length; i++) {
+        distribution.update(statusTable[uniqHosts[i]]);
+    }
+
+    var info = distribution.printObj();
+
+    var cassert = CollapsedAssert();
+    cassert.ok(info.min <= opts.min,
+        'expected minimum(' + info.min + ') to be no more then ' + opts.min
+    );
+
+    if (Array.isArray(opts.sum)) {
+        cassert.ok(
+            info.sum >= opts.sum[0] &&
+            info.sum <= opts.sum[1],
+            'expected sum(' + info.sum + ') to be within ' +
+                opts.sum[0] + ' & ' + opts.sum[1]
+        );
+    } else {
+        cassert.equal(info.sum, opts.sum,
+            'expected sum(' + info.sum + ') to be ' + opts.sum
+        );
+    }
+
+    cassert.ok(
+        info.median >= opts.median[0] &&
+        info.median <= opts.median[1],
+        'expected median(' + info.median + ') to be within ' +
+            opts.median[0] + ' & ' + opts.median[1]
+    );
+
+    cassert.ok(
+        info.mean >= opts.mean[0] &&
+        info.mean <= opts.mean[1],
+        'expected mean(' + info.mean + ') to be within ' +
+            opts.mean[0] + ' & ' + opts.mean[1]
+    );
+
+    cassert.ok(info.max <= opts.max,
+        'expected maximum(' + info.max + ') to no more then ' + opts.max
+    );
+    cassert.ok(
+        info.p75 >= opts.p75[0] &&
+        info.p75 <= opts.p75[1],
+        'expected P75(' + info.p75 + ') to be within ' +
+            opts.p75[0] + ' & ' + opts.p75[1]
+    );
+    cassert.ok(info.p95 <= opts.p95,
+        'expected P95 (' + info.p95 + ') to be less than ' + opts.p95
+    );
+
+    if (opts.variance) {
+        cassert.ok(info.variance <= opts.variance,
+            'expected variance(' + info.variance + ') to be less than ' + opts.variance
+        );
+    }
+
+    return cassert;
+}
+
+function setup(cluster, opts) {
+    opts = opts || {};
+    var NUM_CLIENTS = opts.clients || 40;
+    var NUM_SERVERS = opts.servers || cluster.channels.length - NUM_CLIENTS;
+
+    cluster.clients = cluster.channels.slice(0, NUM_CLIENTS);
+    cluster.servers = cluster.channels.slice(
+        NUM_CLIENTS, NUM_CLIENTS + NUM_SERVERS
+    );
+
+    var i;
+    for (i = 0; i < cluster.servers.length; i++) {
+        makeServer(cluster.servers[i], i);
+    }
+
+    cluster.serverHosts = [];
+    for (i = 0; i < cluster.servers.length; i++) {
+        cluster.serverHosts.push(cluster.servers[i].hostPort);
+    }
+
+    cluster.batches = [];
+    for (i = 0; i < cluster.clients.length; i++) {
+        cluster.batches.push(new BatchClient(
+            cluster.clients[i], cluster.serverHosts, {
+                delay: 40,
+                batchSize: 1,
+                timeout: 1000,
+                totalRequests: 50,
+                minConnections: opts.minConnections || null,
+                retryLimit: opts.retryLimit || null
+            }
+        ));
+    }
+}
+
+function makeServer(channel, index) {
+    var chanNum = index + 1;
+
+    var serverChan = channel.makeSubChannel({
+        serviceName: 'server'
+    });
+
+    serverChan.register('echo', function echo(req, res, arg2, arg3) {
+        res.headers.as = 'raw';
+        res.sendOk(arg2, arg3 + ' served by ' + chanNum);
+    });
+}