tchannel 3.6.14 → 3.6.24

package/as/http.js

@@ -261,7 +261,7 @@ TChannelHTTP.prototype.sendRequest = function send(treq, hreq, start, options, c
             self.logger.error('Buffer read for arg2 failed', {
                 error: arg2res.err
             });
-            var fromBufferErr = errors.HTTPReqArg2fromoBufferError(arg2res.err, {
+            var fromBufferErr = errors.HTTPReqArg2fromBufferError(arg2res.err, {
                 arg2: res1
             });
             callback(fromBufferErr, null, null, null);
@@ -557,7 +557,7 @@ AsHTTPHandler.prototype.handleRequest = function handleRequest(req, buildRespons
             self.logger.error('Buffer read for arg2 failed', {
                 error: arg2res.err
             });
-            var fromBufferErr = errors.HTTPResArg2fromoBufferError(arg2res.err, {
+            var fromBufferErr = errors.HTTPResArg2fromBufferError(arg2res.err, {
                 arg2: arg2
             });
             sendError(fromBufferErr);

package/benchmarks/Makefile

@@ -1,6 +1,10 @@
 torch-client-bench:
 	node index.js --relay --torch client --torchFile ./flame.raw --torchTime 10 \
-		-- --relay --skipPing -s 4096,16384 -p 1000,10000,20000
+		-- --relay --skipPing -s 4096,16384 -p 1000 -m 3
+
+torch-client-only-bench:
+	node index.js --torch client --torchFile ./flame.raw --torchTime 10 \
+		-- --skipPing -s 4096,16384 -p 1000 -m 3
 
 torch-server-bench:
 	node index.js --relay --torch server --torchFile ./flame.raw --torchTime 10 \

package/benchmarks/index.js

@@ -265,6 +265,7 @@ function startClient(clientPort) {
         '--benchPort', String(self.ports.serverPort),
         '--relayServerPort', String(self.ports.relayServerPort),
         '--clientPort', String(clientPort),
+        '--instances', String(self.instanceCount),
         '--instanceNumber', String(self.benchCounter)
     ]);
     var benchProc = self.run(bench, args);

package/benchmarks/multi_bench.js

@@ -63,13 +63,17 @@ argv.pipeline = parseIntList(argv.pipeline);
 argv.sizes = parseIntList(argv.sizes);
 
 var DESTINATION_SERVER;
+var DESTINATION_PORT;
 var TRACE_SERVER;
 var CLIENT_PORT = argv.clientPort;
+var INSTANCES = 0;
 
 if (argv.relay) {
     DESTINATION_SERVER = '127.0.0.1:' + argv.relayServerPort;
 } else {
     DESTINATION_SERVER = '127.0.0.1:' + argv.benchPort;
+    DESTINATION_PORT = argv.benchPort;
+    INSTANCES = parseInt(argv.instances, 10);
 }
 
 if (argv.trace) {
@@ -171,9 +175,19 @@ Test.prototype.newClient = function newClient(id, callback) {
         };
     }
 
+    var peers = [DESTINATION_SERVER];
+    if (INSTANCES > 0) {
+        peers = [];
+        var basePort = DESTINATION_PORT;
+        for (var i = 0; i < INSTANCES; i++) {
+            peers.push('127.0.0.1:' + (basePort + i));
+        }
+    }
+
     var client = clientChan.makeSubChannel({
         serviceName: 'benchmark',
-        peers: [DESTINATION_SERVER]
+        peers: peers,
+        minConnections: INSTANCES > 0 ? 10 : 1
     });
     client.createTime = Date.now();
     clientChan.listen(port, '127.0.0.1', function listened(err) {

package/channel.js

@@ -55,6 +55,7 @@ var TChannelRequest = require('./request');
 var TChannelServiceNameHandler = require('./service-name-handler');
 var errors = require('./errors');
 var EventEmitter = require('./lib/event_emitter.js');
+var ObjectPool = require('./lib/object_pool');
 
 var TChannelAsThrift = require('./as/thrift');
 var TChannelAsJSON = require('./as/json');
@@ -121,7 +122,8 @@ function TChannel(options) {
     this.emitConnectionMetrics =
         typeof this.options.emitConnectionMetrics === 'boolean' ?
         this.options.emitConnectionMetrics : false;
-    this.choosePeerWithHeap = this.options.choosePeerWithHeap || false;
+    this.choosePeerWithHeap = typeof this.options.choosePeerWithHeap === 'boolean' ?
+        this.options.choosePeerWithHeap : true;
 
     this.setObservePeerScoreEvents(this.options.observePeerScoreEvents);
 
@@ -253,6 +255,15 @@ function TChannel(options) {
         this.batchStats = this.topChannel.batchStats;
     }
 
+    if (this.batchStats) {
+        ObjectPool.bootstrap({
+            channel: this,
+            reportInterval: 5000,
+            timers: this.timers,
+            debug: this.options.objectPoolDebug ? true : false
+        });
+    }
+
     this.maximumRelayTTL = MAXIMUM_TTL_ALLOWED;
 
     function doSanitySweep() {
@@ -521,9 +532,6 @@ TChannel.prototype.onServerSocketConnection = function onServerSocketConnection(
 TChannel.prototype.onServerSocketListening = function onServerSocketListening() {
     var self = this;
 
-    var address = self.serverSocket.address();
-    var hostPort = self.host + ':' + address.port;
-
     if (self.destroyed) {
         self.logger.error('got serverSocket listen whilst destroyed', self.extendLogInfo({
             requestedPort: self.requestedPort
@@ -531,7 +539,8 @@ TChannel.prototype.onServerSocketListening = function onServerSocketListening()
         return;
     }
 
-    self.hostPort = hostPort;
+    var address = self.serverSocket.address();
+    self.hostPort = self.host + ':' + address.port;
     self.listening = true;
 
     if (self.subChannels) {
@@ -911,6 +920,10 @@ TChannel.prototype.close = function close(callback) {
 
     var counter = 1;
 
+    if (self.batchStats) {
+        ObjectPool.unref();
+    }
+
     if (self.sanityTimer) {
         self.timers.clearTimeout(self.sanityTimer);
         self.sanityTimer = null;

package/hyperbahn-client.js

@@ -454,7 +454,9 @@ function advertiseAgain(delay) {
 
     self._advertisementTimer = timers.setTimeout(
         function advertiseTimeout() {
-            self.advertise();
+            if (!self._destroyed) {
+                self.advertise();
+            }
         },
         delay
     );

package/lazy_relay.js

@@ -20,9 +20,16 @@
 
 'use strict';
 
+var process = require('process');
 var errors = require('./errors');
 var v2 = require('./v2');
 var stat = require('./stat-tags.js');
+var ObjectPool = require('./lib/object_pool');
+var ReadResult = require('bufrw').ReadResult;
+var WriteResult = require('bufrw').WriteResult;
+
+var readRes = new ReadResult();
+var writeRes = new WriteResult();
 
 module.exports = {
     LazyRelayInReq: LazyRelayInReq,
@@ -35,30 +42,30 @@ module.exports = {
 
 /*eslint max-statements: [2, 40]*/
 function LazyRelayInReq(conn, reqFrame) {
-    this.channel = conn.channel;
-    this.conn = conn;
-    this.start = conn.timers.now();
-    this.remoteAddr = conn.remoteName;
-    this.logger = conn.logger;
+    this.channel = null;
+    this.conn = null;
+    this.start = null;
+    this.remoteAddr = null;
+    this.logger = null;
     this.peer = null;
     this.outreq = null;
-    this.reqFrame = reqFrame;
-    this.id = this.reqFrame.id;
-    this.serviceName = '';
-    this.callerName = '';
-    this.timeout = 0;
-    this.alive = true;
+    this.reqFrame = null;
+    this.id = null;
+    this.serviceName = null;
+    this.callerName = null;
+    this.timeout = null;
+    this.alive = null;
     this.operations = null;
     this.timeHeapHandle = null;
-    this.endpoint = '';
+    this.endpoint = null;
     this.error = null;
     this.tracing = null;
     this.reqContFrames = [];
-    this.hasRead = false;
+    this.hasRead = null;
+    this.waitForIdentSlot = null;
 
     this.boundExtendLogInfo = extendLogInfo;
     this.boundOnIdentified = onIdentified;
-    this.circuit = reqFrame.circuit;
 
     var self = this;
 
@@ -67,6 +74,10 @@ function LazyRelayInReq(conn, reqFrame) {
     }
 
     function onIdentified(err) {
+        // The ident descriptor will be cleared out of the peer when the ident
+        // comes back, so this slot id will be invalid once ident happens.
+        self.waitForIdentSlot = -1;
+
         if (err) {
             self.onError(err);
         } else {
@@ -75,6 +86,66 @@ function LazyRelayInReq(conn, reqFrame) {
     }
 }
 
+LazyRelayInReq.prototype.reset = function reset(conn, reqFrame) {
+    this.channel = conn.channel;
+    this.conn = conn;
+    this.start = conn.timers.now();
+    this.remoteAddr = conn.remoteName;
+    this.logger = conn.logger;
+    this.peer = null;
+    this.outreq = null;
+    this.reqFrame = reqFrame;
+    this.id = this.reqFrame.id;
+    this.serviceName = '';
+    this.callerName = '';
+    this.timeout = 0;
+    this.alive = true;
+    this.operations = null;
+    this.timeHeapHandle = null;
+    this.endpoint = '';
+    this.error = null;
+    this.tracing = null;
+    this.reqContFrames.length = 0;
+    this.hasRead = false;
+    this.circuit = reqFrame.circuit;
+    this.waitForIdentSlot = null;
+};
+
+LazyRelayInReq.prototype.clear = function clear() {
+    if (this.outreq && !this.outreq._objectPoolIsFreed) {
+        this.outreq.free();
+    }
+
+    if (this.peer && this.waitForIdentSlot !== null) {
+        this.peer.stopWaitingForIdentified(this.waitForIdentSlot);
+    }
+
+    this.channel = null;
+    this.conn = null;
+    this.start = null;
+    this.remoteAddr = null;
+    this.logger = null;
+    this.peer = null;
+    this.outreq = null;
+    this.reqFrame = null;
+    this.id = null;
+    this.serviceName = null;
+    this.callerName = null;
+    this.timeout = null;
+    this.alive = null;
+    this.operations = null;
+    this.timeHeapHandle = null;
+    this.endpoint = null;
+    this.error = null;
+    this.tracing = null;
+    this.reqContFrames.length = 0;
+    this.hasRead = null;
+    this.circuit = null;
+    this.waitForIdentSlot = null;
+};
+
+ObjectPool.setup({Type: LazyRelayInReq, maxSize: 200});
+
 LazyRelayInReq.prototype.type = 'tchannel.lazy.incoming-request';
 
 LazyRelayInReq.prototype.initRead =
@@ -128,7 +199,7 @@ function initRead() {
     self.endpoint = endpoint;
 
     var tracing = self.reqFrame.bodyRW.lazy
-        .readTracingValue(self.reqFrame);
+        .poolReadTracingValue(readRes, self.reqFrame);
     self.tracing = tracing || v2.Tracing.emptyTracing;
 
     self.channel.emitFastStat(
@@ -214,7 +285,7 @@ function createOutRequest() {
     if (conn && conn.remoteName && !conn.closing) {
         self.forwardTo(conn);
     } else {
-        self.peer.waitForIdentified(self.boundOnIdentified);
+        self.waitForIdentSlot = self.peer.waitForIdentified(self.boundOnIdentified);
     }
 };
 
@@ -259,7 +330,8 @@ LazyRelayInReq.prototype.forwardTo =
 function forwardTo(conn) {
     var self = this;
 
-    self.outreq = new LazyRelayOutReq(conn, self);
+    self.outreq = LazyRelayOutReq.alloc();
+    self.outreq.reset(conn, self);
 
     var ttl = self.updateTTL(self.outreq.start);
     if (!ttl || ttl < 0) {
@@ -315,7 +387,7 @@ function updateTTL(now) {
         timeout = self.channel.maximumRelayTTL;
     }
 
-    var res = self.reqFrame.bodyRW.lazy.writeTTL(timeout, self.reqFrame);
+    var res = self.reqFrame.bodyRW.lazy.poolWriteTTL(writeRes, timeout, self.reqFrame);
     if (res.err) {
         // TODO: wrap? protocol write error?
         self.onError(res.err);
@@ -339,8 +411,10 @@ function onReadError(err) {
         );
     }
 
+    var conn = self.conn;
     self.onError(err);
-    self.conn.resetAll(err);
+
+    conn.resetAll(err);
 };
 
 LazyRelayInReq.prototype.onError =
@@ -351,6 +425,7 @@ function onError(err) {
         self.logger.warn('dropping error from dead relay request', self.extendLogInfo({
             error: err
         }));
+        self.free();
         return;
     }
 
@@ -370,6 +445,8 @@ function onError(err) {
     }));
 
     self.reqContFrames.length = 0;
+
+    self.free();
 };
 
 LazyRelayInReq.prototype.sendErrorFrame =
@@ -391,6 +468,7 @@ function handleFrameLazily(frame) {
 
     if (!self.alive) {
         self.logger.warn('dropping frame from dead relay request', self.extendLogInfo({}));
+        self.free();
         return;
     }
 
@@ -518,6 +596,22 @@ function _observeCallReqContFrame(frame) {
 };
 
 function LazyRelayOutReq(conn, inreq) {
+    this.channel = null;
+    this.conn = null;
+    this.start = null;
+    this.remoteAddr = null;
+    this.logger = null;
+    this.inreq = null;
+    this.id = null;
+    this.serviceName = null;
+    this.callerName = null;
+    this.timeout = null;
+    this.operations = null;
+    this.timeHeapHandle = null;
+    this.alive = null;
+}
+
+LazyRelayOutReq.prototype.reset = function reset(conn, inreq) {
     this.channel = conn.channel;
     this.conn = conn;
     this.start = conn.timers.now();
@@ -530,7 +624,26 @@ function LazyRelayOutReq(conn, inreq) {
     this.timeout = 0;
     this.operations = null;
     this.timeHeapHandle = null;
-}
+    this.alive = true;
+};
+
+LazyRelayOutReq.prototype.clear = function clear() {
+    this.channel = null;
+    this.conn = null;
+    this.start = null;
+    this.remoteAddr = null;
+    this.logger = null;
+    this.inreq = null;
+    this.id = null;
+    this.serviceName = null;
+    this.callerName = null;
+    this.timeout = null;
+    this.operations = null;
+    this.timeHeapHandle = null;
+    this.alive = null;
+};
+
+ObjectPool.setup({Type: LazyRelayOutReq, maxSize: 200});
 
 LazyRelayOutReq.prototype.type = 'tchannel.lazy.outgoing-request';
 
@@ -566,6 +679,11 @@ LazyRelayOutReq.prototype.emitError =
 function emitError(err) {
     var self = this;
 
+    // ObjectPool weird free() issue; bail early
+    if (!self.channel) {
+        return;
+    }
+
     var now = self.channel.timers.now();
     var elapsed = now - self.start;
 
@@ -599,7 +717,16 @@ function emitError(err) {
         self.inreq.circuit.state.onRequestError(err);
     }
 
-    self.inreq.onError(err);
+    // We need to defer the inreq onError work, because we might be
+    // *syncronously* processing an error after an attempt to write to a
+    // socket. Otherwise, we might end up freeing the in/outreq pair before a
+    // handleFrameLazily has completed.
+    process.nextTick(deferInReqOnError);
+    function deferInReqOnError() {
+        if (self.inreq) {
+            self.inreq.onError(err);
+        }
+    }
 };
 
 LazyRelayOutReq.prototype.logError =
@@ -613,6 +740,11 @@ LazyRelayOutReq.prototype.onTimeout =
 function onTimeout(now) {
     var self = this;
 
+    // ObjectPool weird free() issue; bail early
+    if (!self.conn) {
+        return;
+    }
+
     self.conn.ops.checkLastTimeoutTime(now);
     self.conn.ops.popOutReq(self.id, self.extendLogInfo({
         info: 'lazy out request timed out',
@@ -636,9 +768,15 @@ function handleFrameLazily(frame) {
     // - v2.Types.ErrorResponse
     var self = this;
 
+    // ObjectPool weird free() issue; bail early
+    if (!self.inreq) {
+        return;
+    }
+
     frame.setId(self.inreq.id);
     self.inreq.conn.writeToSocket(frame.buffer);
     if (frame.bodyRW.lazy.isFrameTerminal(frame)) {
+        self.alive = false;
         self.conn.ops.popOutReq(self.id, self.extendLogInfo({
             info: 'lazy relay request done',
             relayDirection: 'out'
@@ -655,6 +793,11 @@ function handleFrameLazily(frame) {
         self._observeErrorFrame(frame, now);
     // } else { TODO: log
     }
+
+    if (!self.alive && !self.inreq.alive) {
+        // Implicitly frees this because inreq.clear will free outreq
+        self.inreq.free();
+    }
 };
 
 LazyRelayOutReq.prototype._observeErrorFrame =
@@ -767,7 +910,7 @@ function _observeCallResFrame(frame, now) {
         self.inreq.circuit.state.onRequestHealthy();
     }
 
-    var res = frame.bodyRW.lazy.readFlags(frame);
+    var res = frame.bodyRW.lazy.poolReadCode(readRes, frame);
     if (res.err) {
         self.logger.error('failed to read error frame code', self.extendLogInfo({
             error: res.err
@@ -775,8 +918,8 @@ function _observeCallResFrame(frame, now) {
         return;
     }
 
-    var flags = res.value;
-    var ok = flags === 0;
+    var code = res.value;
+    var ok = code === 0;
 
     if (ok) {
         self.channel.emitFastStat(