tchannel 3.6.14 → 3.6.24

package/lib/object_pool.js

@@ -0,0 +1,209 @@
+// Copyright (c) 2015 Uber Technologies, Inc.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+'use strict';
+
+var assert = require('assert');
+var stat = require('../stat-tags');
+
+var DEFAULT_MAX_SIZE = 1000;
+
+module.exports = ObjectPool;
+
+function ObjectPool(options) {
+    assert(typeof options === 'object', 'expected options object');
+
+    this.Type = options.Type;
+    assert(typeof this.Type === 'function', 'expected options.Type to be constructor function');
+    assert(typeof this.Type.prototype.reset === 'function', 'expected options.Type to have reset method');
+    assert(typeof this.Type.prototype.clear === 'function', 'expected options.Type to have clear method');
+
+    this.maxSize = options.maxSize || DEFAULT_MAX_SIZE;
+    assert(typeof this.maxSize === 'number', 'expected options.maxSize to be number');
+
+    this.name = options.name || this.Type.name;
+    assert(typeof this.name === 'string', 'expected options.name to be string');
+
+    this.freeList = [];
+    this.outstanding = 0;
+
+    this.freeListStatTags = new stat.ObjectPoolTags(
+        this.name,
+        'free'
+    );
+
+    this.outstandingStatTags = new stat.ObjectPoolTags(
+        this.name,
+        'outstanding'
+    );
+
+    // only used in debug mode
+    this.outstandingList = [];
+}
+
+ObjectPool.channel = null;
+ObjectPool.reportInterval = null;
+ObjectPool.timers = null;
+ObjectPool.pools = [];
+ObjectPool.timer = null;
+ObjectPool.refs = 0;
+ObjectPool.debug = false;
+
+ObjectPool.setup = function setup(options) {
+    var pool = new ObjectPool(options);
+    options.Type.alloc = alloc;
+
+    options.Type.prototype.free = function freeThisObj() {
+        pool.free(this);
+    };
+
+    ObjectPool.pools.push(pool);
+
+    return pool;
+
+    function alloc() {
+        var obj = pool.get();
+        return obj;
+    }
+};
+
+ObjectPool.bootstrap = function bootstrap(options) {
+    if (ObjectPool.refs >= 1) {
+        ObjectPool.refs += 1;
+        return;
+    }
+
+    ObjectPool.refs += 1;
+
+    assert(typeof options === 'object', 'expected options object');
+
+    assert(
+        typeof options.channel === 'object' &&
+        typeof options.channel.emitFastStat === 'function',
+        'expected options.channel to be TChannel instance'
+    );
+    ObjectPool.channel = options.channel;
+
+    assert(
+        typeof options.reportInterval === 'number',
+        'expected options.reportInterval to be number'
+    );
+    ObjectPool.reportInterval = options.reportInterval;
+
+    assert(
+        typeof options.timers === 'object' &&
+        typeof options.timers.setTimeout === 'function',
+        'expected options.timers to be timers object'
+    );
+    ObjectPool.timers = options.timers;
+
+    if (typeof options.debug === 'boolean') {
+        ObjectPool.debug = options.debug;
+    }
+
+    ObjectPool.timer = ObjectPool.timers.setTimeout(
+        ObjectPool.reportStats,
+        ObjectPool.reportInterval
+    );
+};
+
+ObjectPool.unref = function unref() {
+    ObjectPool.refs = Math.max(0, ObjectPool.refs - 1);
+    if (ObjectPool.refs === 0) {
+        ObjectPool.timers.clearTimeout(ObjectPool.timer);
+        ObjectPool.timer = null;
+    }
+};
+
+ObjectPool.reportStats = function reportStats() {
+    // Iterate over pools, report their current size
+
+    var i;
+    for (i = 0; i < ObjectPool.pools.length; i++) {
+        ObjectPool.pools[i].reportStats(ObjectPool.channel);
+    }
+
+    ObjectPool.timer = ObjectPool.timers.setTimeout(
+        ObjectPool.reportStats,
+        ObjectPool.reportInterval
+    );
+};
+
+ObjectPool.prototype.reportStats = function reportStats(channel) {
+    channel.emitFastStat(
+        'tchannel.object-pool',
+        'gauge',
+        this.freeList.length,
+        this.freeListStatTags
+    );
+
+    channel.emitFastStat(
+        'tchannel.object-pool',
+        'gauge',
+        this.outstanding,
+        this.outstandingStatTags
+    );
+};
+
+ObjectPool.prototype.get = function get() {
+    var inst;
+    this.outstanding += 1;
+    if (this.freeList.length) {
+        inst = this.freeList.pop();
+        assert(inst._objectPoolIsFreed, 'instance retreived from pool is free');
+        inst._objectPoolIsFreed = false;
+
+        if (ObjectPool.debug) {
+            this.outstandingList.push(inst);
+        }
+
+        return inst;
+    } else {
+        inst = new this.Type();
+        inst._objectPoolIsFreed = false;
+
+        if (ObjectPool.debug) {
+            this.outstandingList.push(inst);
+        }
+
+        return inst;
+    }
+};
+
+ObjectPool.prototype.free = function free(inst) {
+    assert(!inst._objectPoolIsFreed, 'object pool double free');
+    inst._objectPoolIsFreed = true;
+
+    inst.clear();
+    if (this.outstanding <= this.maxSize) {
+        this.freeList.push(inst);
+    }
+    this.outstanding -= 1;
+
+    var i;
+    if (ObjectPool.debug) {
+        for (i = 0; i < this.outstandingList.length; i++) {
+            if (this.outstandingList[i] === inst) {
+                this.outstandingList.splice(i, 1);
+                return;
+            }
+        }
+    }
+};

package/package.json

@@ -2,7 +2,7 @@
   "name": "tchannel",
   "description": "network multiplexing and framing protocol for RPC or parser drag racing",
   "author": "mranney@uber.com",
-  "version": "3.6.14",
+  "version": "3.6.24",
   "scripts": {
     "lint": "eslint $(git ls-files | grep '.js$')",
     "travis": "npm run test",
@@ -10,7 +10,7 @@
     "benchmark": "echo '!!! DEPRECATED: Better to just run `node benchmarks` directly' >&2; node benchmarks",
     "hyperbahn-link-test": "./test/link_hyperbahn.sh",
     "tcurl-link-test": "./test/link_tcurl.sh",
-    "check-benchmark": "node benchmarks -- -r 10000 -p 100",
+    "check-benchmark": "node benchmarks -- -r 10000 -p 750",
     "take-benchmark": "make -C benchmarks take",
     "take-relay-benchmark": "make -C benchmarks take_relay",
     "take-trace-benchmark": "amke -C benchmarks take_trace",
@@ -28,7 +28,7 @@
     "url": "git@github.com:uber/tchannel-node"
   },
   "dependencies": {
-    "bufrw": "^1.1.0",
+    "bufrw": "^1.2.1",
     "crc": "^3.2.1",
     "error": "^7.0.1",
     "farmhash": "1.1.0",
@@ -43,7 +43,7 @@
     "safe-json-parse": "^4.0.0",
     "sse4_crc32": "4.1.1",
     "tape-cluster": "2.1.0",
-    "thriftrw": "^3.1.0",
+    "thriftrw": "^3.4.3",
     "xorshift": "^0.2.0",
     "xtend": "^4.0.0"
   },

package/peer.js

@@ -36,8 +36,11 @@ var NoPreference = require('./peer_score_strategies.js').NoPreference;
 var PreferIncoming = require('./peer_score_strategies.js').PreferIncoming;
 var Range = require('./range');
 var PeerDrain = require('./drain.js').PeerDrain;
+var ObjectPool = require('./lib/object_pool');
 
 var DEFAULT_REPORT_INTERVAL = 1000;
+var INITIAL_CONN_ATTEMPT_DELAY = 5000;
+var CONN_ATTEMPT_DELAY_MULTIPLER = 2;
 
 /*eslint max-statements: [2, 40]*/
 function TChannelPeer(channel, hostPort, options) {
@@ -49,6 +52,8 @@ function TChannelPeer(channel, hostPort, options) {
     this.stateChangedEvent = this.defineEvent('stateChanged');
     this.allocConnectionEvent = this.defineEvent('allocConnection');
     this.removeConnectionEvent = this.defineEvent('removeConnection');
+    this.deltaOutConnectionEvent = this.defineEvent('deltaOutConnection');
+
     this.channel = channel;
     this.logger = this.channel.logger;
     this.timers = this.channel.timers;
@@ -65,6 +70,13 @@ function TChannelPeer(channel, hostPort, options) {
     this.boundOnPendingChange = onPendingChange;
     this.scoreRange = null;
 
+    // Timestamp when next conn attempt is allowed
+    this.nextConnAttemptTime = 0;
+    // How long to delay conn attempt by on failure (ms)
+    this.nextConnAttemptDelay = 0;
+
+    this.waitForIdentifiedListeners = [];
+
     this.reportInterval = options.reportInterval || DEFAULT_REPORT_INTERVAL;
     if (this.reportInterval > 0 && this.channel.emitConnectionMetrics) {
         this.reportTimer = this.timers.setTimeout(
@@ -405,6 +417,47 @@ TChannelPeer.prototype.connectTo = function connectTo() {
     return self.connect(true);
 };
 
+TChannelPeer.prototype.tryConnect = function tryConnect() {
+    var self = this;
+
+    var connectTime = Date.now();
+    if (connectTime < self.nextConnAttemptTime) {
+        return;
+    }
+
+    var conn = this.getOutConnection();
+    if (!conn || conn.direction !== 'out') {
+        conn = this.connectTo();
+    }
+
+    this.waitForIdentified(conn, onIdentified);
+
+    function onIdentified(err) {
+        if (!err) {
+            self.nextConnAttemptDelay = 0;
+            self.nextConnAttemptTime = 0;
+            return;
+        }
+
+        if (self.nextConnAttemptDelay === 0) {
+            self.nextConnAttemptDelay = INITIAL_CONN_ATTEMPT_DELAY;
+        } else {
+            self.nextConnAttemptDelay *= CONN_ATTEMPT_DELAY_MULTIPLER;
+            // Add some amount of fuzz, +-100ms
+            self.nextConnAttemptDelay += Math.floor(100 * (Math.random() - 0.5));
+        }
+
+        var afterConnect = Date.now();
+        if (self.nextConnAttemptTime < afterConnect) {
+            // When in the past set next attempt to now + delay
+            self.nextConnAttemptTime = afterConnect + self.nextConnAttemptDelay;
+        } else {
+            // When in the future; go further in the future
+            self.nextConnAttemptTime += self.nextConnAttemptDelay;
+        }
+    }
+};
+
 TChannelPeer.prototype.waitForIdentified =
 function waitForIdentified(conn, callback) {
     var self = this;
@@ -419,13 +472,27 @@ function waitForIdentified(conn, callback) {
     } else if (conn.remoteName) {
         callback(null);
     } else {
-        self._waitForIdentified(conn, callback);
+        return self._waitForIdentified(conn, callback);
     }
+
+    return -1;
 };
 
 TChannelPeer.prototype._waitForIdentified =
 function _waitForIdentified(conn, callback) {
     var self = this;
+    var called = false;
+
+    // Setup an ident descriptor so we can stop waiting for identified later
+    var slot = self.getIdentDescriptorSlot();
+    var descriptor = WaitForIdentifiedDescriptor.alloc();
+    descriptor.reset(
+        onConnectionClose,
+        onConnectionError,
+        onIdentified,
+        conn
+    );
+    self.waitForIdentifiedListeners[slot] = descriptor;
 
     self.pendingIdentified++;
     conn.errorEvent.on(onConnectionError);
@@ -433,6 +500,8 @@ function _waitForIdentified(conn, callback) {
     conn.identifiedEvent.on(onIdentified);
     self.invalidateScore('waitForIdentified');
 
+    return slot;
+
     function onConnectionError(err) {
         finish(err);
     }
@@ -446,15 +515,51 @@ function _waitForIdentified(conn, callback) {
     }
 
     function finish(err) {
-        self.pendingIdentified = 0;
-        conn.errorEvent.removeListener(onConnectionError);
-        conn.closeEvent.removeListener(onConnectionClose);
-        conn.identifiedEvent.removeListener(onIdentified);
-        self.invalidateScore('waitForIdentified > finish');
+        // Multiple events can trigger which causes double callback hilarity.
+        if (called) {
+            return;
+        }
+        called = true;
+
+        self.stopWaitingForIdentified(slot);
         callback(err);
     }
 };
 
+TChannelPeer.prototype.stopWaitingForIdentified =
+function stopWaitingForIdentified(slot) {
+    assert(typeof slot === 'number', 'stopWaitingForIdentified arg1 should be number');
+
+    if (slot === -1) {
+        // when connection was already identified, `waitForIdentified` will
+        // return -1
+        return;
+    }
+
+    var descriptor = this.waitForIdentifiedListeners[slot];
+    this.waitForIdentifiedListeners[slot] = null;
+    var conn = descriptor.conn;
+
+    conn.errorEvent.removeListener(descriptor.error);
+    conn.closeEvent.removeListener(descriptor.close);
+    conn.identifiedEvent.removeListener(descriptor.ident);
+    this.pendingIdentified = 0;
+    this.invalidateScore('waitForIdentified > finish');
+
+    descriptor.free();
+};
+
+TChannelPeer.prototype.getIdentDescriptorSlot =
+function getIdentDescriptorSlot() {
+    var i;
+    for (i = 0; i < this.waitForIdentifiedListeners.length; i++) {
+        if (this.waitForIdentifiedListeners[i] === null) {
+            return i;
+        }
+    }
+    return this.waitForIdentifiedListeners.length;
+};
+
 TChannelPeer.prototype.request = function peerRequest(options) {
     var self = this;
     options.timeout = options.timeout || Request.defaultTimeout;
@@ -467,9 +572,11 @@ TChannelPeer.prototype.addConnection = function addConnection(conn) {
     // TODO: second approx support pruning
     if (conn.direction === 'out') {
         self.connections.push(conn);
+        self.deltaOutConnectionEvent.emit(self, 1);
     } else {
         self.connections.unshift(conn);
     }
+
     conn.errorEvent.on(self.boundOnConnectionError);
     conn.closeEvent.on(self.boundOnConnectionClose);
     conn.ops.pendingChangeEvent.on(self.boundOnPendingChange);
@@ -551,15 +658,20 @@ TChannelPeer.prototype.removeConnection = function removeConnection(conn) {
     var self = this;
 
     var ret = null;
+    var isRemoved = false;
 
     var index = self.connections ? self.connections.indexOf(conn) : -1;
     if (index !== -1) {
         ret = self.connections.splice(index, 1)[0];
+        isRemoved = conn.direction === 'out';
     }
 
     self._maybeInvalidateScore('removeConnection');
 
     self.removeConnectionEvent.emit(self, conn);
+    if (isRemoved) {
+        self.deltaOutConnectionEvent.emit(self, -1);
+    }
     return ret;
 };
 
@@ -677,3 +789,28 @@ TChannelPeer.prototype.getScore = function getScore() {
 };
 
 module.exports = TChannelPeer;
+
+function WaitForIdentifiedDescriptor(close, error, ident, conn) {
+    this.close = null;
+    this.error = null;
+    this.ident = null;
+    this.conn = null;
+}
+
+WaitForIdentifiedDescriptor.prototype.reset =
+function reset(close, error, ident, conn) {
+    this.close = close;
+    this.error = error;
+    this.ident = ident;
+    this.conn = conn;
+};
+
+WaitForIdentifiedDescriptor.prototype.clear =
+function clear() {
+    this.close = null;
+    this.error = null;
+    this.ident = null;
+    this.conn = null;
+};
+
+ObjectPool.setup({Type: WaitForIdentifiedDescriptor, maxSize: 100});

package/peer_heap.js

@@ -30,8 +30,10 @@ module.exports = PeerHeap;
 // peer list.
 var dfsStack = [0, 1, 2];
 
-function PeerHeap(random) {
+function PeerHeap(peers, random) {
     this.array = [];
+    this.peers = peers || null;
+    this.hasMinConnections = peers ? peers.hasMinConnections : false;
 
     this.random = random || Math.random;
     assert(typeof this.random === 'function', 'PeerHeap expected random fn');
@@ -44,25 +46,6 @@ function PeerHeap(random) {
     this.maxRangeStart = 0;
 }
 
-PeerHeap.prototype.choose1 = function choose1(threshold, filter) {
-    var self = this;
-    if (self.array[0].range.lo >= threshold && (!filter || filter(self.array[0].peer))) {
-        return self.array[0].peer;
-    }
-};
-
-PeerHeap.prototype.choose2 = function choose2(threshold, filter) {
-    var self = this;
-    var prob1 = self.array[0].peer.getScore();
-    var prob2 = self.array[1].peer.getScore();
-
-    if (prob1 > threshold && prob1 >= prob2 && (!filter || filter(self.array[0].peer))) {
-        return self.array[0].peer;
-    } else if (prob2 > threshold && (!filter || filter(self.array[1].peer))) {
-        return self.array[1].peer;
-    }
-};
-
 /*eslint-disable complexity */
 /*eslint-disable max-statements */
 PeerHeap.prototype.choose = function choose(threshold, filter) {
@@ -72,26 +55,36 @@ PeerHeap.prototype.choose = function choose(threshold, filter) {
         return null;
     }
 
-    if (self.array.length === 1) {
-        return self.choose1(threshold, filter);
-    }
-
-    if (self.array.length === 2) {
-        return self.choose2(threshold, filter);
-    }
-
+    var isSecondary = false;
     var chosenPeer = null;
+    var secondaryPeer = null;
     var highestProbability = 0;
+    var secondaryProbability = 0;
     var firstScore = self.array[0].peer.getScore();
 
+    var notEnoughPeers = false;
+    if (self.hasMinConnections) {
+        notEnoughPeers = self.peers.currentConnectedPeers < self.peers.minConnections;
+    }
+
     // Pointers into dfsStack
     var stackBegin = 0;
     var stackEnd = 0;
 
     if (firstScore > threshold && (!filter || filter(self.array[0].peer))) {
         // Don't check first peer if it looks good, check its children though
-        chosenPeer = self.array[0].peer;
-        highestProbability = firstScore;
+
+        var firstPeer = self.array[0].peer;
+        isSecondary = notEnoughPeers && firstPeer.isConnected('out');
+
+        if (isSecondary) {
+            secondaryPeer = firstPeer;
+            secondaryProbability = firstScore;
+        } else {
+            chosenPeer = firstPeer;
+            highestProbability = firstScore;
+        }
+
         // The array is seeded with 0, 1, 2 so we just have to advance the
         // stack pointers
         stackBegin = 1;
@@ -104,13 +97,20 @@ PeerHeap.prototype.choose = function choose(threshold, filter) {
 
         var el = self.array[i];
 
-        if (self.rangehis[i] <= self.maxRangeStart) {
+        if (!el || (
+            self.rangehis[i] <= self.maxRangeStart &&
+            (!filter && !notEnoughPeers)
+        )) {
             // This range ends before the range with the largest start begins,
             // so it can't possibly be chosen over any of the ranges we've
             // seen. All ranges below this one have a smaller end, so this
             // range and any below it can't be chosen.
             continue;
-        } else if (!filter || filter(el.peer)) {
+        }
+
+        if (!filter || filter(el.peer)) {
+            isSecondary = notEnoughPeers && el.peer.isConnected('out');
+
             // INLINE of TChannelPeer#getScore
             var lo = self.rangelos[i];
             var hi = self.rangehis[i];
@@ -120,9 +120,18 @@ PeerHeap.prototype.choose = function choose(threshold, filter) {
             }
             var probability = lo + ((hi - lo) * rand);
 
-            if ((probability > highestProbability) && (probability > threshold)) {
-                highestProbability = probability;
-                chosenPeer = el.peer;
+            var isBestChoice = !isSecondary ?
+                (probability > highestProbability) :
+                (probability > secondaryProbability);
+
+            if ((probability > threshold) && isBestChoice) {
+                if (isSecondary) {
+                    secondaryProbability = probability;
+                    secondaryPeer = el.peer;
+                } else {
+                    highestProbability = probability;
+                    chosenPeer = el.peer;
+                }
             }
         }
 
@@ -138,11 +147,18 @@ PeerHeap.prototype.choose = function choose(threshold, filter) {
         }
     }
 
-    return chosenPeer;
+    if (secondaryProbability > highestProbability && chosenPeer) {
+        chosenPeer.tryConnect(noop);
+        return secondaryPeer;
+    }
+
+    return chosenPeer || secondaryPeer;
 };
 /*eslint-enable complexity */
 /*eslint-enable max-statements */
 
+function noop() {}
+
 PeerHeap.prototype.clear = function clear() {
     var self = this;
 

package/relay_handler.js

@@ -46,7 +46,9 @@ RelayHandler.prototype.handleLazily = function handleLazily(conn, reqFrame) {
         return false;
     }
 
-    var rereq = new LazyRelayInReq(conn, reqFrame);
+    var rereq = LazyRelayInReq.alloc();
+    rereq.reset(conn, reqFrame);
+
     var err = rereq.initRead();
     if (err) {
         rereq.onReadError(err);
@@ -57,6 +59,7 @@ RelayHandler.prototype.handleLazily = function handleLazily(conn, reqFrame) {
     if (!rereq.peer) {
         rereq.sendErrorFrame('Declined', 'no peer available for request');
         self.logger.info('no relay peer available', rereq.extendLogInfo({}));
+        rereq.free();
         return true;
     }
 

package/service-name-handler.js

@@ -22,6 +22,9 @@
 
 var errors = require('./errors');
 var assert = require('assert');
+var ReadResult = require('bufrw').ReadResult;
+
+var readRes = new ReadResult();
 
 function TChannelServiceNameHandler(options) {
     if (!(this instanceof TChannelServiceNameHandler)) {
@@ -45,7 +48,7 @@ TChannelServiceNameHandler.prototype.type = 'tchannel.service-name-handler';
 TChannelServiceNameHandler.prototype.handleLazily = function handleLazily(conn, reqFrame) {
     var self = this;
 
-    var res = reqFrame.bodyRW.lazy.readService(reqFrame);
+    var res = reqFrame.bodyRW.lazy.poolReadService(readRes, reqFrame);
     if (res.err) {
         // TODO: stat?
         self.channel.logger.warn('failed to lazy read frame serviceName', conn.extendLogInfo({