whistle 2.9.83 → 2.9.85-beta

package/biz/webui/lib/index.js

@@ -291,54 +291,60 @@ app.use(function(req, res, next) {
   if (req.headers.host !== 'rootca.pro') {
     return next();
   }
-  var type = 'crt';
-  if (!req.path.indexOf('/cer')) {
-    type = 'cer';
+  var type = 'cer';
+  if (!req.path.indexOf('/crt')) {
+    type = 'crt';
   } else if (!req.path.indexOf('/pem')) {
     type = 'pem';
   }
   res.download(getRootCAFile(), 'rootCA.' + type);
 });
 
-function checkAllowOrigin(req) {
-  if (config.allowAllOrigin ||
-    req.headers['sec-fetch-site'] === 'same-origin' ||
-    ALLOW_CROSS_URLS.indexOf(req.path) !== -1) {
+function isAllowHost(host) {
+  var list = config.allowOrigin;
+  if (list) {
+    for (var i = 0, len = list.length; i < len; i++) {
+      var h = list[i];
+      if (typeof h === 'string' ? host === h : h.test(host)) {
+        return true;
+      }
+    }
+  }
+}
+
+function checkInternalPath(req) {
+  if (config.allowAllOrigin || ALLOW_CROSS_URLS.indexOf(req.path) !== -1) {
     return true;
   }
   var host = req.headers['x-whistle-origin-host'];
-  var list = config.allowOrigin;
-  if (!host) {
-    host = req.headers.origin;
-    if (!host || typeof host !== 'string') {
-      return true;
-    }
-    if (!list) {
-      return false;
-    }
-    var index = host.indexOf('://');
-    if (index !== -1) {
-      host = host.substring(index + 3);
-    }
-    host = util.parseHost(host)[0] || '*';
+  return !host || isAllowHost(host);
+}
+
+function checkAllowOrigin(req) {
+  var host = req.headers.origin;
+  if (!host || typeof host !== 'string' ||
+    req.headers['sec-fetch-site'] === 'same-origin') {
+    return false;
+  }
+  if (config.allowAllOrigin) {
+    return true;
   }
-  if (host === '*') {
+  if (!config.allowOrigin) {
     return false;
   }
-  if (list) {
-    for (var i = 0, len = list.length; i < len; i++) {
-      var h = list[i];
-      return typeof h === 'string' ? host === h : h.test(host);
-    }
+  var index = host.indexOf('://');
+  if (index !== -1) {
+    host = host.substring(index + 3);
   }
-  return false;
+  host = util.parseHost(host)[0];
+  return host && isAllowHost(host);
 }
 
 function cgiHandler(req, res) {
-  if (UP_PATH_REGEXP.test(req.path) || !checkAllowOrigin(req)) {
+  if (UP_PATH_REGEXP.test(req.path) || !checkInternalPath(req)) {
     return res.status(403).end('Forbidden');
   }
-  if (req.headers.origin) {
+  if (checkAllowOrigin(req)) {
     res.setHeader('access-control-allow-origin', req.headers.origin);
     res.setHeader('access-control-allow-credentials', true);
   }

package/lib/https/index.js

@@ -31,6 +31,7 @@ var getSNIServer = ca.getSNIServer;
 var getHttp2Server = ca.getHttp2Server;
 var LOCALHOST = '127.0.0.1';
 var tunnelTmplData = new LRU({ max: 3000, maxAge: 30000 });
+var uaCache = new LRU({ max: 1024 });
 var TIMEOUT = 12000;
 var CONN_TIMEOUT = 60000;
 var X_RE = /^x/;
@@ -55,6 +56,16 @@ function handleWebsocket(socket, clientIp, clientPort) {
   });
 }
 
+function setCaptureUA(ua) {
+  if (ua && typeof ua === 'string' && ua.length < 1024) {
+    uaCache.set(ua, 1);
+  }
+}
+
+function isCaptureUA(ua) {
+  return ua && uaCache.get(ua);
+}
+
 function getTransProto(str) {
   if (!str || typeof str !== 'string') {
     return;
@@ -938,6 +949,7 @@ function addReqInfo(req) {
   }
   if (!req.isHttpH2) {
     headers[config.HTTPS_FIELD] = 1;
+    setCaptureUA(req.headers['user-agent']);
   }
 }
 
@@ -1122,10 +1134,16 @@ module.exports = function (socket, next, isWebPort) {
   var headersStr;
   var enable = socket.enable || '';
   var disable = socket.disable || '';
-  var isEnable = function(p1, p2) {
-    if (p2) {
-      return (enable[p1] || enable[p2]) && !disable[p1] && !disable[p2];
+  var isCaptureIp = function() {
+    if (disable.captureIp || disable.captureIP) {
+      return false;
+    }
+    if (enable.capture || enable.captureIp || enable.captureIP) {
+      return true;
     }
+    return isCaptureUA(socket.headers['user-agent']);
+  };
+  var isEnable = function(p1) {
     return enable[p1] && !disable[p1];
   };
   var destroy = function (err) {
@@ -1178,7 +1196,7 @@ module.exports = function (socket, next, isWebPort) {
         return isWebPort ? socket.destroy() : next(chunk);
       }
       if (isHttp) {
-        if (isEnable('forHttps') || isEnable('captureHttp')) {
+        if (isEnable('forHttps') || disable.captureHttp) {
           next(chunk);
         } else {
           socket.resume();
@@ -1188,7 +1206,7 @@ module.exports = function (socket, next, isWebPort) {
             addClientInfo(socket, chunk, statusLine, clientIp, clientPort)
           );
         }
-      } else if (isEnable('forHttp') || isEnable('captureHttps')) {
+      } else if (isEnable('forHttp') || disable.captureHttps) {
         return next(chunk);
       } else {
         var isHttpH2 = HTTP2_RE.test(headersStr);
@@ -1260,8 +1278,7 @@ module.exports = function (socket, next, isWebPort) {
         var servername = useSNI || socket.tunnelHostname;
         if (
           !servername ||
-          (useSNI ? disable.captureSNI : (disable.captureNoSNI ||
-          (net.isIP(servername) && !isEnable('captureIp', 'captureIP')))) ||
+          (useSNI ? disable.captureSNI : (disable.captureNoSNI || (net.isIP(servername) && !isCaptureIp()))) ||
           (socket.useProxifier && !ca.existsCustomCert(servername))
         ) {
           return next(chunk);

package/lib/init.js

@@ -192,15 +192,11 @@ module.exports = function (req, res, next) {
     delete headers[config.HTTPS_FIELD];
     delete headers[config.HTTPS_PROTO_HEADER];
   }
-  if (headers['proxy-connection']) {
-    headers.connection = headers['proxy-connection'];
-  }
   var sniPlugin = headers[config.SNI_PLUGIN_HEADER];
   if (sniPlugin) {
     req.sniPlugin = sniPlugin;
     delete headers[config.SNI_PLUGIN_HEADER];
   }
-  delete headers['proxy-connection'];
   if (!req.isHttps && HTTPS_RE.test(req.url)) {
     req.isHttps = true;
   }
@@ -216,6 +212,14 @@ module.exports = function (req, res, next) {
     }
     req.rawHeaders = [];
     delete headers[config.ALPN_PROTOCOL_HEADER];
+    delete headers.connection;
+  }
+  var conn = headers['proxy-connection'];
+  if (conn) {
+    if (headers.connection) {
+      headers.connection = conn;
+    }
+    delete headers['proxy-connection'];
   }
   res.response = function (_res) {
     if (req._hasRespond) {

package/lib/inspectors/data.js

@@ -5,9 +5,9 @@ var socketMgr = require('../socket-mgr');
 var config = require('../config');
 
 var MAX_BODY_SIZE = 360 * 1024;
-var MAX_SIZE = (config.strict ? 256 : 768) * 1024;
-var MAX_REQ_BODY_SIZE = (config.strict ? 256 : 1536) * 1024;
-var MAX_RES_BODY_SIZE = (config.strict ? 256 : 1536) * 1024;
+var MAX_SIZE = (config.strict ? 256 : 1024) * 1024;
+var MAX_REQ_BODY_SIZE = (config.strict ? 256 : 2048) * 1024;
+var MAX_RES_BODY_SIZE = (config.strict ? 256 : 2048) * 1024;
 var BIG_DATA_SIZE = 1024 * 1024 * 10;
 var LOCALHOST = '127.0.0.1';
 
@@ -45,11 +45,6 @@ function checkBodySize(data, useBigData) {
   }
 }
 
-function isUnzipJs(r) {
-  r = r.headers;
-  return !r['content-encoding'] && util.getContentType(r) === 'JS';
-}
-
 function getEventName(proxy) {
   if (util.listenerCount(proxy, '_request')) {
     return '_request';
@@ -113,9 +108,7 @@ function emitDataEvents(req, res, proxy) {
 
   var requestTime;
   var endTime;
-  var isStream;
   var updateEvent;
-  var useReqStream;
   var useFrames;
   var enable = req.enable;
   var disable = req.disable;
@@ -174,13 +167,14 @@ function emitDataEvents(req, res, proxy) {
       reqBody = null;
     }
     reqData.size = 0;
+    var isUnzip = !getZipType(info);
     var write = stream.write;
     var end = stream.end;
     var MAX_REQ_BODY = useBigData
       ? BIG_DATA_SIZE
-      : info.headers && info.headers['content-encoding']
-      ? MAX_SIZE
-      : MAX_REQ_BODY_SIZE;
+      : (isUnzip
+      ? MAX_REQ_BODY_SIZE
+      : MAX_SIZE);
     stream.write = function (chunk) {
       if (chunk) {
         if (reqBody || reqBody === null) {
@@ -188,7 +182,7 @@ function emitDataEvents(req, res, proxy) {
             reqBody = '';
           } else {
             reqBody = reqBody ? Buffer.concat([reqBody, chunk]) : chunk;
-            if (useReqStream) {
+            if (isUnzip) {
               reqData.body = reqBody;
             }
             if (reqBody.length > MAX_REQ_BODY) {
@@ -229,14 +223,14 @@ function emitDataEvents(req, res, proxy) {
     info =
       info ||
       (res._needGunzip ? { statusCode: res.statusCode, headers: '' } : res);
-    var useStream = isStream && !getZipType(info);
+    var isUnzip = !getZipType(info);
     var MAX_RES_BODY = useBigData
       ? BIG_DATA_SIZE
-      : isUnzipJs(info)
+      : (isUnzip
       ? MAX_RES_BODY_SIZE
-      : MAX_SIZE;
+      : MAX_SIZE);
     if (!cleared && util.hasBody(info, req) && checkType(info)) {
-      if (info.headers['content-length'] > MAX_RES_BODY) {
+      if (info.headers['content-length'] > MAX_RES_BODY && !util.isEnable(req, 'captureStream')) {
         resBody = false;
       } else {
         resBody = null;
@@ -252,7 +246,7 @@ function emitDataEvents(req, res, proxy) {
             resBody = '';
           } else {
             resBody = resBody ? Buffer.concat([resBody, chunk]) : chunk;
-            if (useStream) {
+            if (isUnzip) {
               resData.body = resBody;
             }
             if (resBody.length > MAX_RES_BODY) {
@@ -399,13 +393,7 @@ function emitDataEvents(req, res, proxy) {
     resData.statusMessage = _res.statusMessage;
     data.responseTime = Date.now();
     if (!requestTime && !data.requestTime) {
-      isStream = true;
-      data.isStream = true;
       updateEvent = eventName;
-      useReqStream = !getZipType(req);
-    }
-    if (useReqStream && reqBody) {
-      reqData.body = reqBody;
     }
     resData.headers = _res.headers;
   }

package/lib/inspectors/res.js

@@ -392,7 +392,7 @@ module.exports = function (req, res, next) {
                         } else if (!options.port) {
                           options.port = isHttps ? 443 : 80;
                         }
-                        proxyHeaders.host = util.join(options.host, options.port);
+                        proxyHeaders.host = util.joinIpPort(options.host, options.port);
                       } else {
                         options.host = options.hostname;
                       }
@@ -917,7 +917,7 @@ module.exports = function (req, res, next) {
                 newType[0] =
                   !type || type.indexOf('/') != -1
                     ? type
-                    : mime.lookup(type, type);
+                    : (type === 'sse' ? 'text/event-stream' : mime.lookup(type, type));
                 util.setHeader(data, 'content-type', newType.join(';'));
               }
 

package/lib/plugins/index.js

@@ -206,7 +206,7 @@ pluginMgr.on('updateRules', function (byParse) {
         });
       }
       if (rules) {
-        const root = plugin.path;
+        var root = plugin.path;
         var index = 0;
         rules = rules.replace(REMOTE_RULES_RE, function (_, apo, rulesUrl) {
           if (index >= MAX_REMOTE_RULES_COUNT) {

package/lib/util/data-server.js

@@ -162,6 +162,13 @@ function getIndex(startTime, start, end) {
   return getIndex(startTime, start, midIndex);
 }
 
+function toBase64String(data) {
+  if (Buffer.isBuffer(data.body)) {
+    data.base64 = data.body.toString('base64');
+    data.body = '';
+  }
+}
+
 function getIds(startTime, count, lastRowId) {
   startTime = startTime || lastRowId;
   if (!startTime) {
@@ -184,12 +191,77 @@ function getIds(startTime, count, lastRowId) {
   return ids.slice(index, index + count);
 }
 
-function getList(ids) {
+function removeBase64(data) {
+  var result = {};
+  Object.keys(data).forEach(function(key) {
+    if (key !== 'headers' && key !== 'base64' && key !== 'rawHeaderNames') {
+      result[key] = data[key];
+    }
+  });
+  return result;
+}
+
+function getBase64Len(base64) {
+  if (!base64) {
+    return 0;
+  }
+  var len = base64.length;
+  if (base64[len - 1] === '=') {
+    len -= 2;
+    if (base64[len] === '=') {
+      --len;
+    }
+  }
+  return len;
+}
+
+function setBody(item, len) {
+  if (!(len > 0)) {
+    return item;
+  }
+  var base64 = item.base64;
+  var curLen = getBase64Len(base64);
+  item = removeBase64(item);
+  if (len < curLen) {
+    item.preLen = len;
+    item.base64 = base64.substring(len);
+  }
+  return item;
+}
+
+function getList(ids, status) {
   if (!Array.isArray(ids)) {
     return [];
   }
   return ids.map(function (id, i) {
-    return reqData[id];
+    var data = reqData[id];
+    var opts = status && status[i];
+    if (data) {
+      toBase64String(data.req);
+      toBase64String(data.res);
+    }
+    if (!data || typeof opts !== 'string') {
+      return data;
+    }
+    opts = opts.split('-');
+    var result = {};
+    Object.keys(data).forEach(function(key) {
+      if (key === 'url' || key === 'rulesHeaders' || (key === 'rules' && opts[1] != '0')) {
+        return;
+      }
+      var item = data[key];
+      if (key === 'req') {
+        if (!opts[0]) {
+          item = removeBase64(item);
+        } else {
+          item = setBody(item, +opts[0]);
+        }
+      } else if (key === 'res') {
+        item = setBody(item, +opts[1]);
+      }
+      result[key] = item;
+    });
+    return result;
   });
 }
 
@@ -498,13 +570,6 @@ module.exports = function init(_proxy) {
     return result;
   };
 
-  function toBase64String(data) {
-    if (Buffer.isBuffer(data.body)) {
-      data.base64 = data.body.toString('base64');
-      data.body = '';
-    }
-  }
-
   proxy.getItem = function (id) {
     var item = reqData[id];
     if (item) {
@@ -557,10 +622,6 @@ module.exports = function init(_proxy) {
         : getIds(startTime, count, options.lastRowId);
     var setData = function (item) {
       if (item) {
-        var req = item.req;
-        var res = item.res;
-        toBase64String(req);
-        toBase64String(res);
         if (config.secureFilter) {
           try {
             item = config.secureFilter(item, clientIp, filter) || item;
@@ -583,6 +644,8 @@ module.exports = function init(_proxy) {
         while (id && count > 0) {
           var item = reqData[id];
           if (checkItem(item, filter)) {
+            toBase64String(item.req);
+            toBase64String(item.res);
             setData(item);
             newIds.push(id);
             --count;
@@ -593,7 +656,7 @@ module.exports = function init(_proxy) {
         getList(newIds).forEach(setData);
       }
     }
-    getList(options.ids).forEach(setData);
+    getList(options.ids, options.status).forEach(setData);
     var endId = ids[ids.length - 1];
     var lastFrameId, frames;
     if (options.lastFrameId == -3) {

package/lib/util/index.js

@@ -3165,7 +3165,7 @@ exports.disableReqProps = function (req) {
     delete headers['user-agent'];
   }
 
-  if (disable.gzip) {
+  if (disable.gzip || isEnable(req, 'captureStream')) {
     delete headers['accept-encoding'];
   }
 

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "whistle",
   "description": "HTTP, HTTP2, HTTPS, Websocket debugging proxy",
-  "version": "2.9.83",
+  "version": "2.9.85-beta",
   "dataDirname": ".whistle",
   "localUIHost": "local.whistlejs.com",
   "port": 8899,
@@ -57,7 +57,7 @@
     "sni": "1.0.0",
     "sockx": "^0.2.2",
     "starting": "^8.0.3",
-    "weinre2": "^1.3.4",
+    "weinre2": "^1.3.6",
     "ws-parser": "^0.6.4",
     "xml2js": "0.5.0"
   },