whistle 2.10.3 → 2.10.5

package/lib/config.js

@@ -91,6 +91,7 @@ config.WEBUI_HEAD = 'x-forwarded-from-whistle-' + uid;
 config.RES_RULES_HEAD = 'x-whistle-res-rules-' + uid;
 config.COMPOSER_CLIENT_ID_HEADER = 'x-whistle-client-id-' + uid;
 config.FROM_COM_HEADER = 'x-whistle-composer-' + uid;
+config.TEST_HEADER = 'x-whistle-test-' + uid;
 config.PLUGIN_HOOKS = {
   SNI: 'sni-' + uid,
   AUTH: 'auth-' + uid,

package/lib/https/index.js

@@ -106,6 +106,7 @@ function resolveWebsocket(socket, wss) {
     _clientId: socket._clientId,
     id: socket.reqId,
     fc: socket.fromComposer ? 1 : undefined,
+    testId: socket._testId,
     url: fullUrl,
     startTime: now,
     sniPlugin: socket.sniPlugin,

package/lib/inspectors/data.js

@@ -158,6 +158,7 @@ function emitDataEvents(req, res, proxy) {
     h2Id: req._alpn,
     fc: req.fromComposer ? 1 : undefined,
     isPR: req.isPluginReq ? 1 : undefined,
+    testId: req._testId,
     _clientId: req._clientId,
     startTime: now,
     ttfb: getTTFB(),
@@ -417,7 +418,7 @@ function emitDataEvents(req, res, proxy) {
   req.once('dest', function (_req) {
     _req.once('finish', function () {
       if (!requestTime) {
-        setRequestTime(requestTime);
+        setRequestTime(Date.now());
       }
     });
     setReqStatus();

package/lib/rules/rules.js

@@ -7,6 +7,7 @@ var rulesUtil = require('./util');
 var lookup = require('./dns');
 var protoMgr = require('./protocols');
 var config = require('../config');
+var common = require('../util/common');
 
 var rules = rulesUtil.rules;
 var values = rulesUtil.values;
@@ -68,7 +69,6 @@ var NON_STAR_RE = /[^*]/;
 var DOMAIN_STAR_RE = /([*~]+)(\\.)?/g;
 var STAR_RE = /\*+/g;
 var PORT_PATTERN_RE = /^!?:\d{1,5}$/;
-var COMMENT_RE = /#[^\r\n]*/g;
 var TPL_RE = /^((?:[\w.-]+:)?\/\/)?(`.*`)$/;
 // url:  protocol, host, port, hostname, search, query, pathname, path, href, query.key
 // req|res: ip, method, statusCode, headers?.key, cookies?.key
@@ -101,6 +101,7 @@ var TOOL_RE = /^(?:log|weinre):\/\//;
 var mIndex = 0;
 var mNow = Date.now();
 var IS_JSON = Symbol('isJson');
+var removeComment = common.removeComment;
 
 function getMFlag() {
   if (mIndex === Number.MAX_SAFE_INTEGER) {
@@ -110,10 +111,6 @@ function getMFlag() {
   return mNow + '-' + (mIndex++);
 }
 
-function removeComment(text) {
-  return text.replace(COMMENT_RE, '').trim();
-}
-
 function domainToRegExp(all, star, dot) {
   var len = star.length;
   var result = len > 1 ? '([^/?]*)' : '([^/?.]*)';
@@ -1838,6 +1835,9 @@ function matchFilter(url, filter, req) {
     if (filter.from === 'tunnel') {
       return filter.not ? !req.fromTunnel : req.fromTunnel;
     }
+    if (filter.from === 'test') {
+      return filter.not ? !req._testId : req._testId;
+    }
     if (filter.from === 'composer') {
       return filter.not ? !req.fromComposer : req.fromComposer;
     }

package/lib/service/composer.js

@@ -29,6 +29,7 @@ var MAX_BODY_LEN = 260 * 1024;
 var MAX_BASE64_LEN = 360 * 1024;
 var MAX_METHOD_LEN = 64;
 var LEN_HEADER = 'content-length';
+var testIndex = 0;
 
 var composerTimer;
 function saveComposerHistory() {
@@ -125,7 +126,24 @@ function getReqCount(count) {
   return count > 0 ? Math.min(count, MAX_REQ_COUNT) : 1;
 }
 
-function handleConnect(options, cb, count) {
+function getResponse(res, testId) {
+  if (testId) {
+    return { testId: testId };
+  }
+  return { statusCode: res.statusCode, headers: res.headers || {} };
+}
+
+function handleCallback(cb, err, res, testId) {
+  if (!cb) {
+    return;
+  }
+  if (testId) {
+    return cb(null, getResponse(null, testId));
+  }
+  cb(err, !err && getResponse(res));
+}
+
+function handleConnect(options, cb, count, testId) {
   count = getReqCount(count);
   options.headers['x-whistle-policy'] = 'tunnel';
   var origOpts = options;
@@ -145,7 +163,7 @@ function handleConnect(options, cb, count) {
       headers: options.headers
     }, function(socket, svrRes, err) {
       if (err) {
-        return execCb && execCb(err);
+        return handleCallback(execCb, err, null, testId);
       }
       if (TLS_PROTOS.indexOf(options.protocol) !== -1) {
         socket = tls.connect({
@@ -160,10 +178,7 @@ function handleConnect(options, cb, count) {
         socket.write(data);
         options.body = data = null;
       }
-      execCb && execCb(null, {
-        statusCode: svrRes.statusCode,
-        headers: svrRes.headers
-      });
+      handleCallback(execCb, err, svrRes, testId);
     }, config).on('error', execCb || noop);
   }
 }
@@ -178,7 +193,7 @@ function getReqRaw(options) {
   return raw + '\r\n\r\n';
 }
 
-function handleWebSocket(options, cb, count) {
+function handleWebSocket(options, cb, count, testId) {
   count = getReqCount(count);
   if (options.protocol === 'https:' || options.protocol === 'wss:') {
     options.headers[common.HTTPS_FIELD] = 1;
@@ -196,7 +211,7 @@ function handleWebSocket(options, cb, count) {
     }
     common.connect(PROXY_OPTS, function(err, socket) {
       if (err) {
-        execCb && execCb(err);
+        handleCallback(execCb, err, null, testId);
       } else {
         socket.write(getReqRaw(options));
         var data = options.body;
@@ -206,7 +221,7 @@ function handleWebSocket(options, cb, count) {
         parseReq(socket, function(e) {
           if (e) {
             socket.destroy();
-            return execCb && execCb(e);
+            return handleCallback(execCb, e, null, testId);
           }
           var statusCode = socket.statusCode;
           if (statusCode == 101) {
@@ -226,17 +241,14 @@ function handleWebSocket(options, cb, count) {
           } else {
             socket.destroy();
           }
-          execCb && execCb(null, {
-            statusCode: statusCode,
-            headers: socket.headers || {}
-          });
+          handleCallback(execCb, err, socket, testId);
         }, true);
       }
     });
   }
 }
 
-function handleHttp(options, cb, count, reqId) {
+function handleHttp(options, cb, count, reqId, testId) {
   count = getReqCount(count);
   if (options.protocol === 'https:') {
     options.headers[common.HTTPS_FIELD] = 1;
@@ -255,7 +267,8 @@ function handleHttp(options, cb, count, reqId) {
       options = extend({}, origOpts);
     }
     var client = http.request(options, function(svrRes) {
-      if (!execCb) {
+      if (!execCb || testId) {
+        handleCallback(execCb, null, svrRes, testId);
         return drain(svrRes);
       }
       svrRes.on('error', execCb);
@@ -338,7 +351,8 @@ function getCharset(headers) {
 }
 
 exports.handleRequest = function(req, res) {
-  var fullUrl = req.body.url;
+  var reqBody = req.body;
+  var fullUrl = reqBody.url;
   if (!fullUrl || typeof fullUrl !== 'string') {
     return res.json({ec: 2, em: 'Invalid URL'});
   }
@@ -354,13 +368,41 @@ exports.handleRequest = function(req, res) {
   }
   var rawHeaderNames = {};
   var clientId = req.headers[config.CLIENT_ID_HEADER];
-  var headers = parseHeaders(req.body.headers, rawHeaderNames, clientId);
-  var method = common.getMethod(req.body.method);
+  var headers = parseHeaders(reqBody.headers, rawHeaderNames, clientId);
+  var method = common.getMethod(reqBody.method);
   var isWebSocket = method === 'WEBSOCKET';
+  var rules = common.trimStr(reqBody.rules);
+  if (rules) {
+    var curRules = headers['x-whistle-rule-value'];
+    if (Array.isArray(curRules)) {
+      curRules = curRules.join('\n');
+    }
+    curRules = common.trimStr(curRules);
+    if (curRules) {
+      try {
+        curRules = decodeURIComponent(curRules);
+        rules = common.removeComment(rules + '\n' + curRules);
+      } catch (e) {}
+    }
+    headers['x-whistle-rule-value'] = common.safeEncodeURIComponent(rules);
+  }
+  var type = common.trimStr(reqBody.contentType);
+  if (type) {
+    headers['content-type'] = type;
+  }
+  if (reqBody.contentLength >= 0) {
+    headers['content-length'] = reqBody.contentLength;
+  }
   delete headers[config.WEBUI_HEAD];
   headers[config.FROM_COM_HEADER] = '1';
-  if (req.body.enableProxyRules === false) {
+  var isTest = reqBody.isTest;
+  var testId;
+  if (reqBody.enableProxyRules === false || isTest) {
     headers[config.DISABLE_RULES_HEADER] = '1';
+    if (isTest) {
+      testId = ++testIndex + '';
+      headers[config.TEST_HEADER] = testId;
+    }
   }
   headers.host = options.host;
   options.clientId = clientId;
@@ -370,8 +412,8 @@ exports.handleRequest = function(req, res) {
 
   var isConn = common.isConnect(options);
   var isWs = !isConn && (isWebSocket || common.isUpgrade(options, headers));
-  var useH2 = req.body.useH2 || req.body.isH2;
-  req.body.useH2 = false;
+  var useH2 = reqBody.useH2 || reqBody.isH2;
+  reqBody.useH2 = false;
   if (isWs) {
     headers.connection = 'Upgrade';
     headers.upgrade = (!isWebSocket && headers.upgrade) || 'websocket';
@@ -382,7 +424,7 @@ exports.handleRequest = function(req, res) {
   } else {
     delete headers.upgrade;
     if (!isConn && ((useH2 && (protocol === 'https:' || protocol === 'http:')) || protocol === 'h2:' || protocol === 'http2:')) {
-      req.body.useH2 = true;
+      reqBody.useH2 = true;
       var isHttp = protocol === 'http:';
       options.protocol = isHttp ? 'http:' : 'https:';
       if (!headers[config.ALPN_PROTOCOL_HEADER]) {
@@ -390,8 +432,8 @@ exports.handleRequest = function(req, res) {
       }
     }
   }
-  if (!req.body.noStore && req.body.needResponse && common.checkHistory(req.body)) {
-    handleComposerHistory(req.body, isConn);
+  if (!reqBody.noStore && !testId && reqBody.needResponse && common.checkHistory(reqBody)) {
+    handleComposerHistory(reqBody, isConn);
     dataCenter.saveData({
       type: 'composer',
       history: composerHistory
@@ -399,15 +441,15 @@ exports.handleRequest = function(req, res) {
   }
 
   var getBody = function(cb) {
-    var base64 = req.body.base64;
-    var body = base64 || req.body.body;
+    var base64 = reqBody.base64;
+    var body = base64 || reqBody.body;
     if (!isWs) {
       delete headers.trailer;
     }
     if (isWs || isConn || common.hasRequestBody(options)) {
       body = body && common.toBuffer(body, base64 ? 'base64' : getCharset(headers));
       options.body = body;
-      if (!isWs && !isConn && body && req.body.isGzip) {
+      if (!isWs && !isConn && body && reqBody.isGzip) {
         gzip(body, function(err, gzipData) {
           if (err) {
             return cb(err);
@@ -439,7 +481,7 @@ exports.handleRequest = function(req, res) {
   getBody(function(err) {
     options.headers = formatHeaders(headers, rawHeaderNames);
     var done;
-    var needResponse = req.query.needResponse || req.body.needResponse;
+    var needResponse = req.query.needResponse || reqBody.needResponse;
     var handleResponse = needResponse ? function(err, data) {
       if (done) {
         return;
@@ -453,23 +495,23 @@ exports.handleRequest = function(req, res) {
         }});
         return;
       }
-      common.sendGzip(req, res, {ec: 0, em: 'success', res: data || ''});
+      common.sendGzip(req, res, {ec: 0, res: data || ''});
     } : null;
     if (err) {
       return handleResponse && handleResponse(err);
     }
-    var count = req.body.repeatCount;
-    count = count > 0 ? count : req.body.repeatTimes;
+    var count = reqBody.repeatCount;
+    count = count > 0 ? count : reqBody.repeatTimes;
     if (isWs) {
       options.method = 'GET';
-      handleWebSocket(options, handleResponse, count);
+      handleWebSocket(options, handleResponse, count, testId);
     } else if (isConn) {
-      handleConnect(options, handleResponse, count);
+      handleConnect(options, handleResponse, count, testId);
     } else  {
-      handleHttp(options, handleResponse, count, req.body.reqId);
+      handleHttp(options, handleResponse, count, reqBody.reqId, testId);
     }
     if (!handleResponse) {
-      res.json({ec: 0, em: 'success'});
+      res.json({ec: 0});
     }
   });
 };

package/lib/tunnel.js

@@ -91,6 +91,7 @@ function tunnelProxy(server, proxy, type) {
     reqSocket.headers = headers;
     reqSocket.fromTunnel = req.fromTunnel;
     reqSocket.fromComposer = req.fromComposer;
+    reqSocket._testId = req._testId;
     req.isPluginReq = reqSocket.isPluginReq = util.checkPluginReqOnce(req, true);
     util.onSocketEnd(reqSocket, function (err) {
       // 处理可能的证书校验出错
@@ -100,6 +101,7 @@ function tunnelProxy(server, proxy, type) {
           id: req.reqId,
           url: options.host,
           fc: req.fromComposer ? 1 : undefined,
+          testId: req._testId,
           startTime: now,
           dnsTime: now,
           captureError: true,
@@ -319,6 +321,7 @@ function tunnelProxy(server, proxy, type) {
             id: req.reqId,
             url: options.host,
             fc: req.fromComposer ? 1 : undefined,
+            testId: req._testId,
             startTime: Date.now(),
             rules: _rules,
             req: reqData,

package/lib/upgrade.js

@@ -167,6 +167,7 @@ function upgradeHandler(req, socket) {
     socket.url = req.url;
     socket.fromTunnel = req.fromTunnel;
     socket.fromComposer = req.fromComposer;
+    socket._testId = req._testId;
     socket.enableXFF = req.enableXFF;
     socket._isInternalReq = req._isInternalReq;
     socket.isInternalUrl = req.isInternalUrl;

package/lib/util/common.js

@@ -1127,6 +1127,10 @@ function isNum(n) {
   return typeof n === 'number';
 }
 
+exports.trimStr = function(str) {
+  return typeof str === 'string' ? str.trim() : '';
+};
+
 exports.parsePlainText = function (text, resolveKeys) {
   var result;
   text.split(LINE_END_RE).forEach(function(line) {
@@ -2034,4 +2038,10 @@ exports.writeTempFile = function (value, callback) {
   writeFile(path.join(TEMP_FILES_PATH, filename), value, function(err) {
     callback(err, filename);
   });
-};
+};
+
+var COMMENT_RE = /#[^\r\n]*/g;
+
+exports.removeComment = function(text) {
+  return text.replace(COMMENT_RE, '').trim();
+};

package/lib/util/data-server.js

@@ -577,6 +577,16 @@ module.exports = function init(_proxy) {
     }
     return item;
   };
+  proxy.getTestItem = function(id) {
+    var list = Object.values(reqData);
+    for (var i = 0, len = list.length; i < len; i++) {
+      var item = list[i];
+      if (item.testId === id) {
+        return item;
+      }
+    }
+  };
+
   proxy.abortRequest = function (id) {
     var item = id && reqData[id];
     item && item.abort && item.abort();

package/lib/util/index.js

@@ -29,6 +29,7 @@ var h2Consts = config.enableH2 ? require('http2').constants : {};
 var protocols = protoMgr.protocols;
 var resProtocols = protoMgr.resProtocols;
 var uid = config.uid;
+var trimStr = common.trimStr;
 var parseQuery = common.parseQuery;
 var supportsBr = common.supportsBr;
 var isLocalIp = common.isLocalIp;
@@ -126,6 +127,7 @@ workerIndex = workerIndex >= 0 ? common.padLeft(config.workerIndex, 3) : '';
 
 
 exports.sendRes = common.sendRes;
+exports.trimStr = trimStr;
 exports.THEME_STYLE = THEME_STYLE;
 exports.encodeNonLatin1Char = encodeNonLatin1Char;
 exports.isJson = common.isJson;
@@ -2359,15 +2361,6 @@ exports.transformReq = function(req, res, port, host, html) {
   return client;
 };
 
-function trimStr(str) {
-  if (typeof str !== 'string') {
-    return '';
-  }
-  return str.trim();
-}
-
-exports.trimStr = trimStr;
-
 function hasHeaderRules(headers) {
   return (
     headers['x-whistle-rule-key'] ||
@@ -3389,19 +3382,25 @@ exports.parseRange = function (req, size) {
 };
 
 exports.parseClientInfo = function (req) {
-  if (req.headers[config.FROM_COM_HEADER] === '1') {
+  var headers = req.headers;
+  var testId = headers[config.TEST_HEADER];
+  if (testId) {
+    delete headers[config.TEST_HEADER];
+    req._testId = testId;
+  }
+  if (headers[config.FROM_COM_HEADER] === '1') {
     req.fromComposer = true;
-    req._disabledProxyRules = !!req.headers[config.DISABLE_RULES_HEADER];
-    delete req.headers[config.FROM_COM_HEADER];
-    delete req.headers[config.DISABLE_RULES_HEADER];
+    req._disabledProxyRules = !!headers[config.DISABLE_RULES_HEADER];
+    delete headers[config.FROM_COM_HEADER];
+    delete headers[config.DISABLE_RULES_HEADER];
   }
   var socket = req.socket || '';
   if (socket.fromTunnel) {
     req.fromTunnel = true;
   }
-  var clientInfo = req.headers[clientInfoKey];
+  var clientInfo = headers[clientInfoKey];
   if (clientInfo) {
-    delete req.headers[clientInfoKey];
+    delete headers[clientInfoKey];
     clientInfo = String(clientInfo).split(',');
     if (!net.isIP(clientInfo[0]) || !(clientInfo[1] > 0)) {
       return '';

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "whistle",
   "description": "HTTP, HTTP2, HTTPS, Websocket debugging proxy",
-  "version": "2.10.3",
+  "version": "2.10.5",
   "dataDirname": ".whistle",
   "localUIHost": "local.whistlejs.com",
   "port": 8899,
@@ -53,7 +53,7 @@
     "parseurl": "^1.3.1",
     "pfork": "^0.6.3",
     "pipestream": "^0.7.5",
-    "set-global-proxy": "^0.4.0",
+    "set-global-proxy": "^0.4.2",
     "sni": "1.0.0",
     "sockx": "^0.2.3",
     "starting": "^8.0.3",