tchannel 3.6.17 → 3.6.18

package/channel.js

@@ -230,14 +230,6 @@ function TChannel(options) {
     this.statsd = this.options.statsd;
     this.batchStats = null;
 
-    if (this.statsd) {
-        ObjectPool.bootstrap({
-            statsd: this.statsd,
-            reportInterval: 1000,
-            timers: this.timers
-        });
-    }
-
     this.requestDefaults = this.options.requestDefaults ?
         new RequestDefaults(this.options.requestDefaults) : null;
 
@@ -262,6 +254,15 @@ function TChannel(options) {
         this.batchStats = this.topChannel.batchStats;
     }
 
+    if (this.batchStats) {
+        ObjectPool.bootstrap({
+            channel: this,
+            reportInterval: 5000,
+            timers: this.timers,
+            debug: this.options.objectPoolDebug ? true : false
+        });
+    }
+
     this.maximumRelayTTL = MAXIMUM_TTL_ALLOWED;
 
     function doSanitySweep() {
@@ -918,7 +919,7 @@ TChannel.prototype.close = function close(callback) {
 
     var counter = 1;
 
-    if (self.statsd) {
+    if (self.batchStats) {
         ObjectPool.unref();
     }
 

package/lazy_relay.js

@@ -20,6 +20,7 @@
 
 'use strict';
 
+var process = require('process');
 var errors = require('./errors');
 var v2 = require('./v2');
 var stat = require('./stat-tags.js');
@@ -61,6 +62,7 @@ function LazyRelayInReq(conn, reqFrame) {
     this.tracing = null;
     this.reqContFrames = [];
     this.hasRead = null;
+    this.waitForIdentSlot = null;
 
     this.boundExtendLogInfo = extendLogInfo;
     this.boundOnIdentified = onIdentified;
@@ -72,6 +74,10 @@ function LazyRelayInReq(conn, reqFrame) {
     }
 
     function onIdentified(err) {
+        // The ident descriptor will be cleared out of the peer when the ident
+        // comes back, so this slot id will be invalid once ident happens.
+        self.waitForIdentSlot = -1;
+
         if (err) {
             self.onError(err);
         } else {
@@ -102,6 +108,7 @@ LazyRelayInReq.prototype.reset = function reset(conn, reqFrame) {
     this.reqContFrames.length = 0;
     this.hasRead = false;
     this.circuit = reqFrame.circuit;
+    this.waitForIdentSlot = null;
 };
 
 LazyRelayInReq.prototype.clear = function clear() {
@@ -109,6 +116,10 @@ LazyRelayInReq.prototype.clear = function clear() {
         this.outreq.free();
     }
 
+    if (this.peer && this.waitForIdentSlot !== null) {
+        this.peer.stopWaitingForIdentified(this.waitForIdentSlot);
+    }
+
     this.channel = null;
     this.conn = null;
     this.start = null;
@@ -130,6 +141,7 @@ LazyRelayInReq.prototype.clear = function clear() {
     this.reqContFrames.length = 0;
     this.hasRead = null;
     this.circuit = null;
+    this.waitForIdentSlot = null;
 };
 
 ObjectPool.setup({Type: LazyRelayInReq, maxSize: 200});
@@ -273,7 +285,7 @@ function createOutRequest() {
     if (conn && conn.remoteName && !conn.closing) {
         self.forwardTo(conn);
     } else {
-        self.peer.waitForIdentified(self.boundOnIdentified);
+        self.waitForIdentSlot = self.peer.waitForIdentified(self.boundOnIdentified);
     }
 };
 
@@ -433,6 +445,8 @@ function onError(err) {
     }));
 
     self.reqContFrames.length = 0;
+
+    self.free();
 };
 
 LazyRelayInReq.prototype.sendErrorFrame =
@@ -698,7 +712,14 @@ function emitError(err) {
         self.inreq.circuit.state.onRequestError(err);
     }
 
-    self.inreq.onError(err);
+    // We need to defer the inreq onError work, because we might be
+    // *syncronously* processing an error after an attempt to write to a
+    // socket. Otherwise, we might end up freeing the in/outreq pair before a
+    // handleFrameLazily has completed.
+    process.nextTick(deferInReqOnError);
+    function deferInReqOnError() {
+        self.inreq.onError(err);
+    }
 };
 
 LazyRelayOutReq.prototype.logError =

package/lib/object_pool.js

@@ -21,6 +21,7 @@
 'use strict';
 
 var assert = require('assert');
+var stat = require('../stat-tags');
 
 var DEFAULT_MAX_SIZE = 1000;
 
@@ -42,16 +43,28 @@ function ObjectPool(options) {
 
     this.freeList = [];
     this.outstanding = 0;
-    this.freeStatName = 'object-pools.' + this.name + '.free';
-    this.outstandingStatName = 'object-pools.' + this.name + '.outstanding';
+
+    this.freeListStatTags = new stat.ObjectPoolTags(
+        this.name,
+        'free'
+    );
+
+    this.outstandingStatTags = new stat.ObjectPoolTags(
+        this.name,
+        'outstanding'
+    );
+
+    // only used in debug mode
+    this.outstandingList = [];
 }
 
-ObjectPool.statsd = null;
+ObjectPool.channel = null;
 ObjectPool.reportInterval = null;
 ObjectPool.timers = null;
 ObjectPool.pools = [];
 ObjectPool.timer = null;
 ObjectPool.refs = 0;
+ObjectPool.debug = false;
 
 ObjectPool.setup = function setup(options) {
     var pool = new ObjectPool(options);
@@ -82,11 +95,11 @@ ObjectPool.bootstrap = function bootstrap(options) {
     assert(typeof options === 'object', 'expected options object');
 
     assert(
-        typeof options.statsd === 'object' &&
-        typeof options.statsd.gauge === 'function',
-        'expected options.statsd to be statsd instance'
+        typeof options.channel === 'object' &&
+        typeof options.channel.emitFastStat === 'function',
+        'expected options.channel to be TChannel instance'
     );
-    ObjectPool.statsd = options.statsd;
+    ObjectPool.channel = options.channel;
 
     assert(
         typeof options.reportInterval === 'number',
@@ -101,6 +114,10 @@ ObjectPool.bootstrap = function bootstrap(options) {
     );
     ObjectPool.timers = options.timers;
 
+    if (typeof options.debug === 'boolean') {
+        ObjectPool.debug = options.debug;
+    }
+
     ObjectPool.timer = ObjectPool.timers.setTimeout(
         ObjectPool.reportStats,
         ObjectPool.reportInterval
@@ -120,7 +137,7 @@ ObjectPool.reportStats = function reportStats() {
 
     var i;
     for (i = 0; i < ObjectPool.pools.length; i++) {
-        ObjectPool.pools[i].reportStats(ObjectPool.statsd);
+        ObjectPool.pools[i].reportStats(ObjectPool.channel);
     }
 
     ObjectPool.timer = ObjectPool.timers.setTimeout(
@@ -129,16 +146,19 @@ ObjectPool.reportStats = function reportStats() {
     );
 };
 
-ObjectPool.prototype.reportStats = function reportStats(statsd) {
-    statsd.gauge(
-        this.freeStatName,
-        this.freeList.length
+ObjectPool.prototype.reportStats = function reportStats(channel) {
+    channel.emitFastStat(
+        'tchannel.object-pool',
+        'gauge',
+        this.freeList.length,
+        this.freeListStatTags
     );
 
-    // This stat is how we infer the maxSize property
-    statsd.gauge(
-        this.outstandingStatName,
-        this.outstanding
+    channel.emitFastStat(
+        'tchannel.object-pool',
+        'gauge',
+        this.outstanding,
+        this.outstandingStatTags
     );
 };
 
@@ -149,10 +169,20 @@ ObjectPool.prototype.get = function get() {
         inst = this.freeList.pop();
         assert(inst._objectPoolIsFreed, 'instance retreived from pool is free');
         inst._objectPoolIsFreed = false;
+
+        if (ObjectPool.debug) {
+            this.outstandingList.push(inst);
+        }
+
         return inst;
     } else {
         inst = new this.Type();
         inst._objectPoolIsFreed = false;
+
+        if (ObjectPool.debug) {
+            this.outstandingList.push(inst);
+        }
+
         return inst;
     }
 };
@@ -166,4 +196,14 @@ ObjectPool.prototype.free = function free(inst) {
         this.freeList.push(inst);
     }
     this.outstanding -= 1;
+
+    var i;
+    if (ObjectPool.debug) {
+        for (i = 0; i < this.outstandingList.length; i++) {
+            if (this.outstandingList[i] === inst) {
+                this.outstandingList.splice(i, 1);
+                return;
+            }
+        }
+    }
 };

package/package.json

@@ -2,7 +2,7 @@
   "name": "tchannel",
   "description": "network multiplexing and framing protocol for RPC or parser drag racing",
   "author": "mranney@uber.com",
-  "version": "3.6.17",
+  "version": "3.6.18",
   "scripts": {
     "lint": "eslint $(git ls-files | grep '.js$')",
     "travis": "npm run test",

package/peer.js

@@ -36,6 +36,7 @@ var NoPreference = require('./peer_score_strategies.js').NoPreference;
 var PreferIncoming = require('./peer_score_strategies.js').PreferIncoming;
 var Range = require('./range');
 var PeerDrain = require('./drain.js').PeerDrain;
+var ObjectPool = require('./lib/object_pool');
 
 var DEFAULT_REPORT_INTERVAL = 1000;
 
@@ -65,6 +66,8 @@ function TChannelPeer(channel, hostPort, options) {
     this.boundOnPendingChange = onPendingChange;
     this.scoreRange = null;
 
+    this.waitForIdentifiedListeners = [];
+
     this.reportInterval = options.reportInterval || DEFAULT_REPORT_INTERVAL;
     if (this.reportInterval > 0 && this.channel.emitConnectionMetrics) {
         this.reportTimer = this.timers.setTimeout(
@@ -419,20 +422,35 @@ function waitForIdentified(conn, callback) {
     } else if (conn.remoteName) {
         callback(null);
     } else {
-        self._waitForIdentified(conn, callback);
+        return self._waitForIdentified(conn, callback);
     }
+
+    return -1;
 };
 
 TChannelPeer.prototype._waitForIdentified =
 function _waitForIdentified(conn, callback) {
     var self = this;
 
+    // Setup an ident descriptor so we can stop waiting for identified later
+    var slot = self.getIdentDescriptorSlot();
+    var descriptor = WaitForIdentifiedDescriptor.alloc();
+    descriptor.reset(
+        onConnectionClose,
+        onConnectionError,
+        onIdentified,
+        conn
+    );
+    self.waitForIdentifiedListeners[slot] = descriptor;
+
     self.pendingIdentified++;
     conn.errorEvent.on(onConnectionError);
     conn.closeEvent.on(onConnectionClose);
     conn.identifiedEvent.on(onIdentified);
     self.invalidateScore('waitForIdentified');
 
+    return slot;
+
     function onConnectionError(err) {
         finish(err);
     }
@@ -446,15 +464,45 @@ function _waitForIdentified(conn, callback) {
     }
 
     function finish(err) {
-        self.pendingIdentified = 0;
-        conn.errorEvent.removeListener(onConnectionError);
-        conn.closeEvent.removeListener(onConnectionClose);
-        conn.identifiedEvent.removeListener(onIdentified);
-        self.invalidateScore('waitForIdentified > finish');
+        self.stopWaitingForIdentified(slot);
         callback(err);
     }
 };
 
+TChannelPeer.prototype.stopWaitingForIdentified =
+function stopWaitingForIdentified(slot) {
+    assert(typeof slot === 'number', 'stopWaitingForIdentified arg1 should be number');
+
+    if (slot === -1) {
+        // when connection was already identified, `waitForIdentified` will
+        // return -1
+        return;
+    }
+
+    var descriptor = this.waitForIdentifiedListeners[slot];
+    this.waitForIdentifiedListeners[slot] = null;
+    var conn = descriptor.conn;
+
+    conn.errorEvent.removeListener(descriptor.error);
+    conn.closeEvent.removeListener(descriptor.close);
+    conn.identifiedEvent.removeListener(descriptor.ident);
+    this.pendingIdentified = 0;
+    this.invalidateScore('waitForIdentified > finish');
+
+    descriptor.free();
+};
+
+TChannelPeer.prototype.getIdentDescriptorSlot =
+function getIdentDescriptorSlot() {
+    var i;
+    for (i = 0; i < this.waitForIdentifiedListeners.length; i++) {
+        if (this.waitForIdentifiedListeners[i] === null) {
+            return i;
+        }
+    }
+    return this.waitForIdentifiedListeners.length;
+};
+
 TChannelPeer.prototype.request = function peerRequest(options) {
     var self = this;
     options.timeout = options.timeout || Request.defaultTimeout;
@@ -677,3 +725,28 @@ TChannelPeer.prototype.getScore = function getScore() {
 };
 
 module.exports = TChannelPeer;
+
+function WaitForIdentifiedDescriptor(close, error, ident, conn) {
+    this.close = null;
+    this.error = null;
+    this.ident = null;
+    this.conn = null;
+}
+
+WaitForIdentifiedDescriptor.prototype.reset =
+function reset(close, error, ident, conn) {
+    this.close = close;
+    this.error = error;
+    this.ident = ident;
+    this.conn = conn;
+};
+
+WaitForIdentifiedDescriptor.prototype.clear =
+function clear() {
+    this.close = null;
+    this.error = null;
+    this.ident = null;
+    this.conn = null;
+};
+
+ObjectPool.setup({Type: WaitForIdentifiedDescriptor, maxSize: 100});

package/relay_handler.js

@@ -59,6 +59,7 @@ RelayHandler.prototype.handleLazily = function handleLazily(conn, reqFrame) {
     if (!rereq.peer) {
         rereq.sendErrorFrame('Declined', 'no peer available for request');
         self.logger.info('no relay peer available', rereq.extendLogInfo({}));
+        rereq.free();
         return true;
     }
 

package/stat-tags.js

@@ -58,7 +58,8 @@ module.exports = {
     ConnectionsErrorsTags: ConnectionsErrorsTags,
     ConnectionsClosedTags: ConnectionsClosedTags,
     RelayLatencyTags: RelayLatencyTags,
-    HTTPHanlderBuildLatencyTags: HTTPHanlderBuildLatencyTags
+    HTTPHanlderBuildLatencyTags: HTTPHanlderBuildLatencyTags,
+    ObjectPoolTags: ObjectPoolTags
 };
 
 function InboundCallsRecvdTags(cn, serviceName, endpoint) {
@@ -663,3 +664,25 @@ HTTPHanlderBuildLatencyTags.prototype.toStatKey = function toStatKey(prefix) {
         clean(this.targetService, 'no-target-service') + '.' +
         (this.streamed ? 'streamed' : 'unstreamed');
 };
+
+function ObjectPoolTags(poolName, statType) {
+    this.app = '';
+    this.host = '';
+    this.cluster = '';
+    this.version = '';
+
+    this.poolName = poolName;
+    this.statType = statType;
+
+    this._cachedPrefix = '';
+    this._key = '';
+}
+
+ObjectPoolTags.prototype.toStatKey = function toStatKey(prefix) {
+    // prefix should never change but we store it in _cachedPrefix just in case
+    if (!this._key || prefix !== this._cachedPrefix) {
+        this._key = prefix + '.' + this.poolName + '.' + this.statType;
+        this._cachedPrefix = prefix;
+    }
+    return this._key;
+};

package/test/lib/alloc-cluster.js

@@ -29,6 +29,7 @@ var debugLogtron = require('debug-logtron');
 
 var TChannel = require('../../channel.js');
 var CollapsedAssert = require('./collapsed-assert.js');
+var ObjectPool = require('../../lib/object_pool');
 
 var loadConfig = require('./load_config.js');
 
@@ -77,7 +78,8 @@ function allocCluster(opts) {
     var channelOptions = extend({
         logger: logger,
         timeoutFuzz: 0,
-        traceSample: 1
+        traceSample: 1,
+        objectPoolDebug: true
     }, defaultChannelOptions, opts.channelOptions || opts);
 
     for (var i = 0; i < opts.numPeers; i++) {
@@ -217,6 +219,8 @@ function clusterTester(opts, t) {
 
                     cluster.assertEmptyState(collapsedAssert);
                     collapsedAssert.report(assert, 'cluster has an empty state');
+
+                    checkObjectPools(assert);
                 }
                 cluster.destroy();
             });
@@ -225,6 +229,24 @@ function clusterTester(opts, t) {
     }
 }
 
+function checkObjectPools(assert) {
+    var i;
+    var pool;
+    var length;
+    for (i = 0; i < ObjectPool.pools.length; i++) {
+        pool = ObjectPool.pools[i];
+        length = pool.outstandingList.length;
+        assert.ok(
+            length === 0,
+            'expected object pool ' + pool.name + ' to have no outstanding ' +
+            'instances, found ' + length + ' instances'
+        );
+        if (length) {
+            assert.comment(util.inspect(pool.outstandingList));
+        }
+    }
+}
+
 allocCluster.test = function testCluster(desc, opts, t) {
     if (opts === undefined) {
         return test(desc);

package/test/object_pool.js

@@ -47,10 +47,14 @@ ObjectPool.setup({
 test('object pool happy', function t1(assert) {
     var timers = Timer(0);
     var stats = {};
-    var statsd = {gauge: function (k, v) { stats[k] = v; }};
+    var channel = {emitFastStat: fakeEmitFastStat};
+
+    function fakeEmitFastStat(name, type, val, tags) {
+        stats[name + tags.toStatKey('')] = val;
+    }
 
     ObjectPool.bootstrap({
-        statsd: statsd,
+        channel: channel,
         timers: timers,
         reportInterval: 500
     });
@@ -89,8 +93,8 @@ test('object pool happy', function t1(assert) {
     assert.equal(pool.outstanding, 1, '1 outstanding instace');
 
     timers.advance(500);
-    assert.equal(stats['object-pools.Widget.free'], 1);
-    assert.equal(stats['object-pools.Widget.outstanding'], 1);
+    assert.equal(stats['tchannel.object-pool.Widget.free'], 1);
+    assert.equal(stats['tchannel.object-pool.Widget.outstanding'], 1);
 
     w = Widget.alloc();
 
@@ -108,8 +112,8 @@ test('object pool happy', function t1(assert) {
     assert.equal(pool.outstanding, 0, '0 outstanding instace');
 
     timers.advance(500);
-    assert.equal(stats['object-pools.Widget.free'], 2);
-    assert.equal(stats['object-pools.Widget.outstanding'], 0);
+    assert.equal(stats['tchannel.object-pool.Widget.free'], 2);
+    assert.equal(stats['tchannel.object-pool.Widget.outstanding'], 0);
 
     assert.end();
 });

package/test/relay_lazy.js

@@ -301,6 +301,7 @@ allocCluster.test('lazy relay request times out', {
 
     var relayOutPeer = relayChan.peers.get(dest.hostPort);
     relayOutPeer.waitForIdentified = function punchWaitForIdentified() {
+        return -1;
     };
 
     sourceChan.request({
@@ -442,7 +443,7 @@ allocCluster.test('relay request handles channel close correctly', {
                      'tchannel.connection.reset',
                      'expected connection error');
         assert.notOk(res, 'expected no response');
-        finish();
+        process.nextTick(finish);
     }
 
     function finish() {