whistle 2.10.9 → 2.10.11

package/lib/plugins/load-plugin.js

@@ -117,7 +117,7 @@ var appendTrailers = function (_res, res, newTrailers, req) {
     return;
   }
   common.addTrailerNames(_res, newTrailers, null, null, req);
-  common.onResEnd(_res, function () {
+  common.onReadEnd(_res, function () {
     var trailers = _res.trailers;
     if (
       !res.chunkedEncoding ||
@@ -786,15 +786,12 @@ var getSession = function (req, cb, isReq) {
   retryRequestSession();
 };
 
-var initWsReq = function (req, res) {
-  initReq(req, res, true);
-};
-var initConnectReq = function (req, res) {
+var setupConnect = function (req, socket, isServer) {
   if (req.originalReq && req.originalRes) {
     return;
   }
   var established;
-  initWsReq(req, res);
+  initReq(req, socket, isServer);
   req.sendEstablished = function (err, cb) {
     if (established) {
       return;
@@ -815,25 +812,25 @@ var initConnectReq = function (req, res) {
     ];
     if (err || !cb || !req.headers[common.ACK_HEADER]) {
       resCtn.push('\r\n', body);
-      res.write(resCtn.join('\r\n'));
+      socket.write(resCtn.join('\r\n'));
       return cb && cb();
     }
     resCtn.push('x-whistle-allow-tunnel-ack: 1');
     resCtn.push('\r\n', body);
-    res.once('data', function (chunk) {
+    socket.once('data', function (chunk) {
       if (!req._hasError) {
-        res.pause();
-        var on = res.on;
-        res.on = function () {
-          res.on = on;
-          res.resume();
+        socket.pause();
+        var on = socket.on;
+        socket.on = function () {
+          socket.on = on;
+          socket.resume();
           return on.apply(this, arguments);
         };
-        chunk.length > 1 && res.unshift(chunk.slice(1));
+        chunk.length > 1 && socket.unshift(chunk.slice(1));
         cb();
       }
     });
-    return res.write(resCtn.join('\r\n'));
+    return socket.write(resCtn.join('\r\n'));
   };
 };
 
@@ -1111,10 +1108,6 @@ function handleWsSignal(receiver, sender) {
   };
 }
 
-function destroySocket() {
-  this.destroy();
-}
-
 function formatRules(rules) {
   if (Array.isArray(rules)) {
     return rules.join('\n');
@@ -2240,7 +2233,7 @@ module.exports = async function (options, callback) {
           break;
         case PLUGIN_HOOKS.HTTP:
           if (httpServer) {
-            initWsReq(req, socket);
+            initReq(req, socket, true);
             var curUrl = req.originalReq.realUrl;
             httpServer.setMaxWsPayload = setMaxWsPayload;
             var svrRes = '';
@@ -2354,41 +2347,39 @@ module.exports = async function (options, callback) {
 
       var emitHttpPipe = function (httpServer, req, socket) {
         if (httpServer) {
-          initConnectReq(req, socket);
+          setupConnect(req, socket);
           req.sendEstablished(function () {
             var decoder = getDecodeTransform();
             var encoder = getEncodeTransform();
             var done;
-            var handleError = function (err) {
+            var destroy = function () {
               if (!done) {
                 done = true;
-                socket.emit('error', err || new Error('destroyed'));
+                socket.destroy();
               }
             };
             socket.on('error', function (err) {
               if (!done) {
-                done = true;
+                destroy();
                 decoder.emit('error', err);
                 encoder.emit('error', err);
               }
             });
-            encoder.once('finish', function () {
-              done = true;
-            });
-            socket.on('close', function () {
+            socket.once('close', function () {
               done = true;
               decoder.emit('close');
               encoder.emit('close');
             });
-            decoder.destroy = encoder.destroy = handleError;
-            decoder.on('error', handleError);
-            encoder.on('error', handleError);
+            decoder.destroy = encoder.destroy = destroy;
+            decoder.on('error', destroy);
+            encoder.on('error', destroy);
             ADDITIONAL_FIELDS.forEach(function (key) {
               decoder[key] = req[key];
             });
-            socket.pipe(decoder);
-            encoder.pipe(socket);
-            socket.on('end', destroySocket);
+            encoder.once('finish', function() {
+              socket.end();
+            });
+            encoder.pipe(socket).pipe(decoder);
             httpServer.emit('request', decoder, encoder);
           });
         }
@@ -2396,7 +2387,7 @@ module.exports = async function (options, callback) {
 
       function pipeUpgrade(pipeSvr, req, socket, head, fromClient, toServer) {
         if (pipeSvr) {
-          initConnectReq(req, socket);
+          setupConnect(req, socket);
           req.sendEstablished(function () {
             addFrameHandler(req, socket, maxWsPayload, fromClient, toServer);
             pipeSvr.emit('connect', req, socket, head);
@@ -2433,7 +2424,7 @@ module.exports = async function (options, callback) {
           break;
         case PLUGIN_HOOKS.TUNNEL:
           if (tunnelServer) {
-            initConnectReq(req, socket);
+            setupConnect(req, socket, true);
             req.sendEstablished(function () {
               var reqRules;
               req.setRules = req.setReqRules = socket.setReqRules = function (rules) {
@@ -2524,7 +2515,7 @@ module.exports = async function (options, callback) {
           }
           break;
         case PLUGIN_HOOKS.TUNNEL_REQ_READ:
-          initConnectReq(req, socket);
+          setupConnect(req, socket);
           pipeServer = req._isUpgrade ? wsReqRead : tunnelReqRead;
           if (pipeServer) {
             req.sendEstablished(function () {
@@ -2537,7 +2528,7 @@ module.exports = async function (options, callback) {
           }
           break;
         case PLUGIN_HOOKS.TUNNEL_REQ_WRITE:
-          initConnectReq(req, socket);
+          setupConnect(req, socket);
           pipeServer = req._isUpgrade ? wsReqWrite : tunnelReqWrite;
           if (pipeServer) {
             req.sendEstablished(function () {
@@ -2549,7 +2540,7 @@ module.exports = async function (options, callback) {
           }
           break;
         case PLUGIN_HOOKS.TUNNEL_RES_READ:
-          initConnectReq(req, socket);
+          setupConnect(req, socket);
           pipeServer = req._isUpgrade ? wsResRead : tunnelResRead;
           if (pipeServer) {
             req.sendEstablished(function () {
@@ -2562,7 +2553,7 @@ module.exports = async function (options, callback) {
           }
           break;
         case PLUGIN_HOOKS.TUNNEL_RES_WRITE:
-          initConnectReq(req, socket);
+          setupConnect(req, socket);
           pipeServer = req._isUpgrade ? wsResWrite : tunnelResWrite;
           if (pipeServer) {
             req.sendEstablished(function () {

package/lib/rules/dns.js

@@ -50,7 +50,7 @@ function lookDnsOverHttps(hostname, callback) {
     } catch (e) {
       err = data || e;
     }
-    return callback(err || 'DNS Over HTTPS Look Failed');
+    return callback(err || 'DNS over HTTPS Look Failed');
   });
 }
 

package/lib/rules/index.js

@@ -51,6 +51,7 @@ exports.getEnabledRules = function() {
 exports.getMFlag = function() {
   return rules._mflag;
 };
+exports.resolveFilter = rules.resolveFilter.bind(rules);
 exports.parse = rules.parse.bind(rules);
 exports.append = rules.append.bind(rules);
 exports.resolveSNICallback = rules.resolveSNICallback.bind(rules);
@@ -386,6 +387,7 @@ function getScriptContext(req, res, body, pattern, frameOpts) {
   }
   if (frameOpts) {
     ctx.ctx = {
+      destroy: frameOpts.destroy,
       sendToServer: frameOpts.sendToServer,
       sendToClient: frameOpts.sendToClient,
       handleSendToClientFrame: null,
@@ -434,7 +436,7 @@ function getReqPayload(req, res, cb) {
 
 function execRulesScript(script, req, res, body, pattern, frameOpts) {
   var context = getScriptContext(req, res, body, pattern, frameOpts);
-  if (!util.execScriptSync(script, context)) {
+  if (!util.runInContext(script, context)) {
     return '';
   }
   if (frameOpts) {

package/lib/rules/protocols.js

@@ -272,12 +272,10 @@ function isWebProtocol(protocol) {
 
 exports.isWebProtocol = isWebProtocol;
 
-function isWebsocketProtocol(protocol) {
+function isWsProtocol(protocol) {
   return protocol == 'ws:' || protocol == 'wss:';
 }
 
-exports.isWebsocketProtocol = isWebsocketProtocol;
-
 function isFileProxy(protocol) {
   return RULE_RE.test(protocol) || LOC_RE.test(protocol);
 }
@@ -298,7 +296,7 @@ function contains(name) {
   }
   name += ':';
   return (
-    isWebsocketProtocol(name) ||
+    isWsProtocol(name) ||
     isWebProtocol(name) ||
     isFileProxy(name) ||
     name == 'tunnel:'

package/lib/rules/rules.js

@@ -54,10 +54,8 @@ var REG_URL_SYMBOL_RE = /^(\^+)/;
 var PATTERN_FILTER_RE = /^(?:filter|ignore):\/\/(.+)\/(i)?$/;
 var LINE_PROPS_RE = /^lineProps:\/\/(.*)$/;
 var FILTER_RE = /^(?:excludeFilter|includeFilter):\/\/(.*)$/;
-var PROPS_FILTER_RE =
-  /^(?:filter|excludeFilter|includeFilter|ignore):\/\/(m(?:ethod)?|i(?:p)?|h(?:eader)?|env|s(?:tatusCode)?|from|b(?:ody)?|clientIp|clientIP|clientPort|remoteAddress|remotePort|serverIp|serverIP|serverPort|chance|probability|re[qs](?:H(?:eaders?)?)?):(.+)$/;
-var PURE_FILTER_RE =
-  /^(?:excludeFilter|includeFilter):\/\/(statusCode|from|env|clientIp|clientIP|clientPort|remoteAddress|remotePort|serverIp|serverIP|chance|probability|serverPort|host|re[qs](?:H(?:eaders?)?)?)[.=](.+)$/;
+var PROPS_FILTER_RE = /^(?:filter|excludeFilter|includeFilter|ignore):\/\/(m(?:ethod)?|i(?:p)?|h(?:eader)?|env|s(?:tatusCode)?|sni|from|b(?:ody)?|clientIp|clientIP|clientPort|remoteAddress|remotePort|serverIp|serverIP|serverPort|chance|probability|host|re[qs](?:H(?:eaders?)?)?)[:=](.+)$/;
+var PURE_FILTER_RE = /^(?:excludeFilter|includeFilter):\/\/(re[qs](?:H(?:eaders?)?)?)\.(.+)$/;
 var PATTERN_WILD_FILTER_RE = /^(?:filter|ignore):\/\/(!)?(\*+\/)/;
 var INLINE_RE = /^((?:filter|excludeFilter|includeFilter|ignore):\/\/)(\(.*\)|\<.*\>)$/;
 var CHROME_PATH_RE = /^file:\/\/\/[A-Z]:\//;
@@ -916,7 +914,7 @@ function checkInternal(req, rule) {
 }
 
 function resolveRuleList(req, list, vals, index, isFilter, isEnableProxy, host, isReq) {
-  var curUrl = util.formatUrl(req.curUrl);
+  var curUrl = util.formatUrl(req.curUrl || req.fullUrl);
   var notHttp = list.isRuleProto && curUrl[0] !== 'h';
   //支持域名匹配
   var domainUrl = removePort(curUrl);

package/lib/rules/util.js

@@ -19,12 +19,9 @@ var inlineValues = {};
 var proxy, pluginMgr;
 var serviceRules;
 var mockRules;
-var dnsOrder = propertiesStorage.getProperty('dnsOrder');
 
 config.ipv6Only = config.ipv6Only || !!propertiesStorage.getProperty('ipv6Only');
-if (!config.dnsOrder && !config.setDefaultResultOrder(dnsOrder)) {
-  config.dnsOrder = config.defaultDnsOrder;
-}
+config.setDefaultResultOrder(propertiesStorage.getProperty('dnsOrder'));
 
 function limitValueLen(name, len) {
   var value = propertiesStorage.getProperty(name);
@@ -535,9 +532,7 @@ exports.properties = {
   },
   setDnsOrder: function(order) {
     if (config.setDefaultResultOrder(order)) {
-      order = +order;
-      config.dnsOrder = order;
-      propertiesStorage.setProperty('dnsOrder', order);
+      propertiesStorage.setProperty('dnsOrder', +order);
     }
   },
   getLatestVersion: function (name) {

package/lib/service/compose-data.js

@@ -22,8 +22,9 @@ exports = module.exports = function(req, res) {
       } else {
         cache.set(reqId, '');
       }
+      curBuf = curBuf.toString('base64');
     }
-    result[reqId] = curBuf ? curBuf.toString('base64') : curBuf;
+    result[reqId] = curBuf;
   });
   res.json(result);
 };

package/lib/service/composer.js

@@ -154,10 +154,7 @@ function handleConnect(options, cb, count, testId) {
       proxyHost: PROXY_OPTS.host,
       proxyPort: PROXY_OPTS.port,
       headers: options.headers
-    }, function(socket, svrRes, err) {
-      if (err) {
-        return handleCallback(execCb, err, null, testId);
-      }
+    }, function(socket, svrRes) {
       if (TLS_PROTOS.indexOf(options.protocol) !== -1) {
         socket = tls.connect({
           rejectUnauthorized: config.rejectUnauthorized,
@@ -171,7 +168,7 @@ function handleConnect(options, cb, count, testId) {
         socket.write(data);
         options.body = data = null;
       }
-      handleCallback(execCb, err, svrRes, testId);
+      handleCallback(execCb, null, svrRes, testId);
     }, config).on('error', execCb || noop);
   }
 }
@@ -257,13 +254,12 @@ function handleHttp(options, cb, count, reqId, testId) {
       svrRes.on('error', execCb);
       var buffer;
       var enableStream;
-      var unzipStream = common.getUnzipStream(svrRes.headers);
       var timer = reqId && setTimeout(function() {
         composeData.setData(reqId, buffer, true);
         enableStream = true;
         timer = buffer = undefined;
         handleResponse();
-        common.onResEnd(svrRes, function() {
+        common.onReadEnd(svrRes, function() {
           var buf = composeData.getData(reqId);
           if (buf) {
             buf._hasW2End = true;
@@ -278,8 +274,9 @@ function handleHttp(options, cb, count, reqId, testId) {
         }
         clearTimeout(timer);
         var headers = svrRes.headers;
-        if (typeof headers.trailer === 'string' && headers.trailer.indexOf(',') !== -1) {
-          headers.trailer = headers.trailer.split(',');
+        var trailer = headers.trailer;
+        if (typeof trailer === 'string' && trailer.indexOf(',') !== -1) {
+          headers.trailer = trailer.split(',');
         }
         var result = {
           statusCode: svrRes.statusCode,
@@ -293,6 +290,11 @@ function handleHttp(options, cb, count, reqId, testId) {
         execCb(null, result);
         buffer = null;
       };
+      if (!common.hasBody(svrRes, options)) {
+        drain(svrRes);
+        return handleResponse();
+      }
+      var unzipStream = common.getUnzipStream(svrRes.headers);
       if (unzipStream) {
         unzipStream.on('error', function(err) {
           drain(svrRes);
@@ -320,7 +322,7 @@ function handleHttp(options, cb, count, reqId, testId) {
           }
         }
       });
-      unzipStream.on('end', handleResponse);
+      unzipStream.once('end', handleResponse);
     });
     client.on('error', execCb || noop);
     client.end(options.body);

package/lib/service/service.js

@@ -18,8 +18,6 @@ var Limiter = require('async-limiter');
 var common = require('../util/common');
 var logger = require('./console-log');
 
-process.env.NODE_TLS_REJECT_UNAUTHORIZED = '1';
-
 var saveData = dataCenter.saveData;
 var shareData = dataCenter.shareData;
 var requestData = dataCenter.requestData;
@@ -304,11 +302,11 @@ function installPlugins(data, clientId) {
 function sessionsHandler() {
   var app = express();
   app.use(function (req, res, next) {
-    req.on('error', abort);
-    res.on('error', abort);
-    function abort() {
+    function destroy() {
       res.destroy();
     }
+    req.on('error', destroy);
+    res.on('error', destroy);
     if (req.headers[common.WHISTLE_UID_HEADER] !== config.uid) {
       return res.status(403).end('Forbidden');
     }

package/lib/socket-mgr.js

@@ -379,7 +379,7 @@ function handleWsSend(ctx, reqTrans, sendStatus, handleTransform, frameCtx, even
   reqReceiver.onData = function (data, opts, drain) {
     var ignore;
     if (drain !== true && hasFilter) {
-      data = execHandleFrame(frameCtx, data, opts);
+      data = execHandleFrame(frameCtx, data, opts, true);
       if (!data) {
         return;
       }
@@ -657,6 +657,9 @@ function getFrameSender(options, toServer) {
 function getFrameCtx(options, callback) {
   var req = options.req;
   rules.getFrameScriptCtx(req.rules.frameScript, req, options.res, {
+    destroy: function() {
+      destroy(options.reqId);
+    },
     sendToClient: getFrameSender(options),
     sendToServer: getFrameSender(options, true)
   }, callback);
@@ -703,7 +706,7 @@ exports.handleUpgrade = function (req, res) {
   resTrans.on('error', emitError);
   res.headers = res.headers || {};
   req.wsExts = res.headers['sec-websocket-extensions'] || '';
-  var options = { req: req, res: res };
+  var options = { req: req, res: res, reqId: reqId };
   pluginMgr.getWsPipe(
     req,
     res,

package/lib/tunnel.js

@@ -206,10 +206,9 @@ function tunnelProxy(server, proxy, type) {
         if (!isIPHost || !hasCustomCerts()) {
           return false;
         }
-        reqSocket.useProxifier =
-          (config.proxifier2 || (config.proxifier && isWebPort)) &&
-          !disable.proxifier;
-        return reqSocket.useProxifier;
+        var useProxifier = (config.proxifier2 || (config.proxifier && isWebPort)) && !disable.proxifier;
+        reqSocket.useProxifier = useProxifier;
+        return useProxifier;
       };
       var resolvedPlugin;
       if (isIntercept()) {
@@ -248,14 +247,19 @@ function tunnelProxy(server, proxy, type) {
       pluginMgr.getTunnelRules(req, function (_rulesMgr) {
         handlePluginRules(_rulesMgr);
         policy = headers[config.WHISTLE_POLICY_HEADER];
+        var useSniAsTunnelDomain = util.isEnable(req, 'useSniAsTunnelDomain');
+        var noCapture;
         if (!reqSocket._hasError && !req._authForbidden &&
-          !req.fromComposer && (req._forceCapture || isIntercept())) {
+          !req.fromComposer && (req._forceCapture || isIntercept() || (noCapture = useSniAsTunnelDomain))) {
           reqSocket.rulesHeaders = req.rulesHeaders;
           reqSocket.enable = req.enable;
           reqSocket.disable = req.disable;
           reqSocket.tunnelHostname = hostname;
+          reqSocket.tunnelPort = options.port;
           reqSocket.rules = _rules;
           reqSocket._pluginVars = req._pluginVars;
+          reqSocket._noCaptureW2 = noCapture;
+          reqSocket._useSniAsTunnelDomain = useSniAsTunnelDomain;
           dispatch(
             reqSocket,
             function (chunk) {