whistle 2.9.35 → 2.9.36-beta

package/biz/webui/lib/index.js

@@ -18,6 +18,7 @@ var setProxy = require('./proxy');
 var rulesUtil = require('../../../lib/rules/util');
 var getRootCAFile = require('../../../lib/https/ca').getRootCAFile;
 var config = require('../../../lib/config');
+var getWorker = require('../../../lib/plugins/util').getWorker;
 var loadAuthPlugins = require('../../../lib/plugins').loadAuthPlugins;
 
 var PARSE_CONF = { extended: true, limit: '3mb'};
@@ -422,6 +423,17 @@ app.get('/values', function(req, res) {
   sendText(res, rulesUtil.values.get(name));
 });
 
+app.get('/web-worker.js', urlencodedParser, function(req, res) {
+  var body = getWorker(req.query.id);
+  if (!body) {
+    return res.status(404).end('Not Found');
+  }
+  res.writeHead(200, {
+    'Content-Type': 'application/javascript; charset=utf-8'
+  });
+  res.end(body);
+});
+
 app.all('/cgi-bin/*', function(req, res, next) {
   req.isUploadReq = UPLOAD_URLS.indexOf(req.path) !== -1;
   return req.isUploadReq ? uploadUrlencodedParser(req, res, next) : urlencodedParser(req, res, next);

package/lib/https/index.js

@@ -676,11 +676,10 @@ function resolveWebsocket(socket, wss) {
         newUrl = util.parsePathReplace(newUrl, urlReplace) || newUrl;
         if (newUrl !== fullUrl) {
           fullUrl = newUrl;
-          options = util.parseUrl(fullUrl);
           socket._realUrl = newUrl;
-        } else {
-          options = util.parseUrl(fullUrl);
         }
+        options = util.parseUrl(fullUrl);
+        socket._w2hostname = options.hostname;
         data.realUrl = fullUrl;
         var host = (headers.host = options.host);
         socket._origin = headers.origin;
@@ -726,9 +725,8 @@ function resolveWebsocket(socket, wss) {
           if (socket._origin && origin && origin !== '*') {
             curResHeaders['access-control-allow-origin'] = socket._origin;
           }
-          var resRuleData = { headers: curResHeaders };
-          util.setResCors(resRuleData, cors, socket);
-          util.setResCookies(resRuleData, cookies, socket);
+          util.setResCors(res, cors, socket);
+          util.setResCookies(res, cookies, socket);
           util.setResponseFor(_rules, curResHeaders, socket, socket.hostIp);
           var delResHeaders = util.parseDelProps(socket).resHeaders;
           Object.keys(delResHeaders).forEach(function (name) {

package/lib/inspectors/res.js

@@ -860,8 +860,8 @@ module.exports = function (req, res, next) {
                 );
               }
 
-              util.setResCookies(data, cookies, req);
-              util.setResCors(data, cors, req);
+              util.setResCookies(_res, cookies, req);
+              util.setResCors(_res, cors, req);
 
               var cache = util.getMatcherValue(resRules.cache);
               var enable = req.enable;

package/lib/plugins/get-plugins-sync.js

@@ -113,6 +113,8 @@ module.exports = function () {
           tunnelKey: util.getTunnelKey(conf),
           staticDir: util.getStaticDir(conf),
           pluginVars: util.getPluginVarsConf(conf),
+          networkColumn: util.getNetworkColumn(conf),
+          webWorker: pluginUtil.readWorkerSync(dir, conf),
           networkMenus: util.getPluginMenu(
             conf.networkMenus || conf.networkMenu,
             simpleName

package/lib/plugins/index.js

@@ -375,6 +375,7 @@ function readPackages(obj, callback) {
             0;
           newPkg.rulesUrl = util.getCgiUrl(conf.rulesUrl);
           newPkg.valuesUrl = util.getCgiUrl(conf.valuesUrl);
+          newPkg.networkColumn = util.getNetworkColumn(conf);
           newPkg.networkMenus = util.getPluginMenu(
             conf.networkMenus || conf.networkMenu,
             simpleName
@@ -436,7 +437,10 @@ function readPackages(obj, callback) {
                           result,
                           simpleName
                         );
-                        callbackHandler();
+                        pluginUtil.readWorker(newPkg.path, conf, function(hash) {
+                          newPkg.webWorker = hash;
+                          callbackHandler();
+                        });
                       }
                     );
                   }
@@ -512,6 +516,7 @@ setTimeout(checkUpdate, 5000);
             uninstallPlugins && pluginMgr.emit('uninstall', uninstallPlugins);
             updatePlugins && pluginMgr.emit('update', updatePlugins);
             pluginMgr.emit('updateRules');
+            pluginUtil.resetWorkers(allPlugins);
           }
           update();
         });

package/lib/plugins/load-plugin.js

@@ -899,6 +899,19 @@ function toBinary(data) {
   return Buffer.from(data);
 }
 
+function getCustomBody(body, headers) {
+  if (body && typeof body.pipe === 'function') {
+    delete headers['content-length'];
+    return body;
+  }
+  body = toBinary(body) || '';
+  if (headers['content-length'] != null) {
+    headers['content-length'] = body ? body.length : 0;
+    delete headers['transfer-encoding'];
+  }
+  return body;
+}
+
 function wrapTunnelWriter(socket, toServer) {
   var write = socket.write;
   var end = socket.end;
@@ -1569,6 +1582,7 @@ module.exports = async function (options, callback) {
       wrapWsWriter: wrapWsWriter
     }, cb);
   };
+
   var initServers = function (_ctx) {
     ctx = _ctx || ctx;
     var execPlugin = require(options.value) || '';
@@ -2066,16 +2080,70 @@ module.exports = async function (options, callback) {
               return client;
             };
             req.passThrough = function (uri, newTrailers) {
-              var client = req.request(uri, function (_res) {
-                res.writeHead(
+              var handleReq;
+              var handleRes;
+              if (typeof uri === 'function') {
+                handleReq = uri;
+                uri = null;
+              }
+              if (typeof newTrailers === 'function') {
+                handleRes = newTrailers;
+                newTrailers = null;
+              }
+              var _request = function(reqBody, _uri) {
+                var client = req.request(_uri || uri, function (_res) {
+                  var _response = function(resBody, trailers) {
+                    res.writeHead(
                       _res.statusCode,
                       _res.statusMessage,
                       _res.headers
                     );
-                appendTrailers(_res, res, newTrailers, req);
-                _res.pipe(res);
-              });
-              req.pipe(client);
+                    appendTrailers(_res, res, trailers || newTrailers, req);
+                    if (resBody === undefined) {
+                      _res.pipe(res);
+                    } else if (resBody.pipe) {
+                      resBody.pipe(res);
+                    } else {
+                      res.end(resBody);
+                    }
+                  };
+                  if (handleRes) {
+                    common.readStream(_res, function(buffer) {
+                      handleRes(buffer, function(result) {
+                        if (result) {
+                          req.setResRules(result.rules);
+                          _response(getCustomBody(result.body, _res.headers), result.trailers);
+                        } else {
+                          _response('');
+                        }
+                      });
+                    });
+                  } else {
+                    _response();
+                  }
+                });
+                if (reqBody === undefined) {
+                  req.pipe(client);
+                } else if (reqBody.pipe) {
+                  reqBody.pipe(client);
+                } else {
+                  client.end(reqBody);
+                }
+              };
+              if (handleReq) {
+                common.readStream(req, function(buffer) {
+                  handleReq(buffer, function(result) {
+                    if (result) {
+                      req.setReqRules(result.rules);
+                      _request(getCustomBody(result.body, req.headers) || '', result.uri || result.url);
+                    } else {
+                      _request('');
+                    }
+                  });
+                });
+              } else {
+                _request();
+              }
             };
             httpServer.emit('request', req, res);
           }

package/lib/plugins/util.js

@@ -12,6 +12,51 @@ var PLUGIN_NAME_RE = /^(?:@[\w-]+\/)?(whistle\.[a-z\d_\-]+)$/;
 var DEV_PLUGINS_PATH = config.DEV_PLUGINS_PATH;
 var MAX_EXPIRE = 36000;
 var UTF8_OPTIONS = { encoding: 'utf8' };
+var workerScript = util.readFileSync(path.join(__dirname, '../../assets/js/worker.js'));
+var workers = {};
+
+function replaceScript(ctn) {
+  return workerScript.replace('/*sourcecode*/', ctn);
+} 
+
+exports.readWorkerSync = function(root, conf) {
+  var file = util.getWebWorker(conf);
+  file = file && util.readFileSync(path.join(root, file));
+  if (!file) {
+    return;
+  }
+  var id = util.createHash(file);
+  workers[id] = replaceScript(file);
+  return id;
+};
+
+exports.readWorker = function(root, conf, callback) {
+  var file = util.getWebWorker(conf);
+  if (!file) {
+    return callback();
+  }
+  readFile(path.join(root, file), function(err, ctn) {
+    if (err || !ctn) {
+      return callback();
+    }
+    var id = util.createHash(ctn);
+    workers[id] = replaceScript(ctn);
+    callback(id);
+  });
+};
+
+exports.getWorker = function(id) {
+  return workers[id];
+};
+
+exports.resetWorkers = function(plugins) {
+  var _workers = {};
+  Object.keys(plugins).forEach(function(key) {
+    var plugin = plugins[key];
+    _workers[plugin.webWorker] = workers[plugin.webWorker];
+  });
+  workers = _workers;
+};
 
 function isOrgModule(name) {
   return ORG_RE.test(name);

package/lib/util/common.js

@@ -324,3 +324,13 @@ exports.wrapRuleValue = function(key, value, size, policy) {
 exports.isGroup = function(name) {
   return name && name[0] === '\r';
 };
+
+exports.readStream = function(stream, callback) {
+  var buffer = null;
+  stream.on('data', function(chunk) {
+    buffer = buffer ? Buffer.concat([buffer, chunk]) : chunk;
+  });
+  stream.once('end', function() {
+    callback(buffer);
+  });
+};

package/lib/util/index.js

@@ -61,6 +61,11 @@ var HTTP_RE = /^https?:\/\/[^/?]/;
 var SEP_RE = /[|&]/;
 var ctxTimer;
 var END_RE = /[/\\]$/;
+var EXPIRED_SEC = -123456;
+var EXP_COOKIE = { maxAge: EXPIRED_SEC, path: '/' };
+var EXP_SECURE_COOKIE = { maxAge: EXPIRED_SEC, path: '/', secure: true };
+var EXP_COOKIE_D = { maxAge: EXPIRED_SEC, path: '/' };
+var EXP_SECURE_COOKIE_D = { maxAge: EXPIRED_SEC, path: '/', secure: true };
 var resetContext = function () {
   ctxTimer = null;
   CONTEXT = vm.createContext();
@@ -705,6 +710,12 @@ if (isString(hostname)) {
   simpleHostname = simpleHostname ? simpleHostname + '.' : '';
 }
 
+function createHash(str) {
+  return crypto.createHash('sha256').update(str).digest('hex');
+}
+
+exports.createHash = createHash;
+
 var clientId = [
   hostname,
   os.platform(),
@@ -1927,7 +1938,9 @@ exports.replaceUrlQueryString = replaceUrlQueryString;
 exports.decodeBuffer = fileMgr.decode;
 
 function setHeaders(data, obj) {
-  data.headers = data.headers || {};
+  if (!data.headers) {
+    data.headers = {};
+  }
   for (var i in obj) {
     data.headers[i] = obj[i];
   }
@@ -1937,7 +1950,9 @@ function setHeaders(data, obj) {
 exports.setHeaders = setHeaders;
 
 function setHeader(data, name, value) {
-  data.headers = data.headers || {};
+  if (!data.headers) {
+    data.headers = {};
+  }
   data.headers[name] = value;
   return data;
 }
@@ -2539,6 +2554,30 @@ exports.getPluginMenu = function (menus, pluginName) {
   return result;
 };
 
+exports.getNetworkColumn = function(conf) {
+  var column = conf.networkColumn;
+  var title = column && (column.title || column.name);
+  var key = title && column.key;
+  if (!isString(title) || !isString(key)) {
+    return;
+  }
+  return {
+    title: title.substring(0, 16),
+    key: key.substring(0, 72),
+    showTitle: (column.showTips || column.showTitle) ? 1 : undefined,
+    width: Math.min(360, Math.max(parseInt(column.width) || 120, 70))
+  };
+};
+
+exports.getWebWorker = function(conf) {
+  var webWorker = conf.webWorker;
+  if (!isString(webWorker)) {
+    return;
+  }
+  webWorker = toRelPath(webWorker);
+  return webWorker && webWorker.length <= 120 ? webWorker : undefined;
+};
+
 var MAX_HINT_LEN = 512;
 var MAX_VAR_LEN = 100;
 
@@ -2621,6 +2660,10 @@ exports.getPluginVarsConf = function (conf) {
   return true;
 };
 
+function toRelPath(str) {
+  return str.replace(/^\/+/, '');
+}
+
 exports.getStaticDir = function (conf) {
   var staticDir = conf.staticDir;
   if (
@@ -2631,7 +2674,7 @@ exports.getStaticDir = function (conf) {
   ) {
     return;
   }
-  return staticDir.replace(/^\/+/, '');
+  return toRelPath(staticDir.replace(/^\/+/, ''));
 };
 
 function toString(str) {
@@ -2700,7 +2743,7 @@ var RES_HEADER_RE = /^res\.?H(?:eaders?)?\.(.+)$/i;
 var TRAILER_RE = /trailer\.(.+)$/;
 var HEADER_RE = /^headers\.(.+)$/;
 var REQ_COOKIE_RE = /^req\.?C(?:ookies?)?\.(.+)$/i;
-var RES_COOKIE_RE = /^req\.?C(?:ookies?)?\.(.+)$/i;
+var RES_COOKIE_RE = /^res\.?C(?:ookies?)?\.(.+)$/i;
 var COOKIE_RE = /^cookies?\.(.+)$/i;
 var QUERY_RE = /^(?:query|urlParams?)\.(.+)$/i;
 
@@ -2727,14 +2770,30 @@ function parseDelReqCookies(req) {
     Object.keys(deleteRule).forEach(function (prop) {
       if (REQ_COOKIE_RE.test(prop) || COOKIE_RE.test(prop)) {
         reqCookies = reqCookies || {};
-        reqCookies[RegExp.$1.toLowerCase()] = 1;
+        reqCookies[RegExp.$1] = 1;
       }
     });
   }
   return reqCookies;
 }
 
-exports.parseDelReqCookies = parseDelReqCookies;
+function getDomain(req) {
+  var host = req._w2hostname;
+  if (typeof host !== 'string') {
+    return;
+  }
+  host = host.split('.');
+  var len = host.length;
+  if (len < 3) {
+    return;
+  }
+  if (len === 3) {
+    host[0] = '';
+  } else {
+    host.shift();
+  }
+  return host.join('.');
+}
 
 function parseDelResCookies(req) {
   var resCookies;
@@ -2743,15 +2802,20 @@ function parseDelResCookies(req) {
     Object.keys(deleteRule).forEach(function (prop) {
       if (RES_COOKIE_RE.test(prop) || COOKIE_RE.test(prop)) {
         resCookies = resCookies || {};
-        resCookies[RegExp.$1.toLowerCase()] = 1;
+        var list = [EXP_COOKIE, EXP_SECURE_COOKIE];
+        resCookies[RegExp.$1] = list;
+        var domain = getDomain(req);
+        if (domain) {
+          EXP_COOKIE_D.domain = domain;
+          EXP_SECURE_COOKIE_D.domain = domain;
+          list.push(EXP_COOKIE_D, EXP_SECURE_COOKIE_D);
+        }
       }
     });
   }
   return resCookies;
 }
 
-exports.parseDelResCookies = parseDelResCookies;
-
 function parseDelProps(req) {
   var deleteRule = req['delete'];
   var reqHeaders = {};
@@ -2994,79 +3058,87 @@ exports.setReqCookies = function (data, cookies, curCookies, req) {
   cookies = Object.keys(result)
     .map(function (name) {
       var value = result[name];
-      return name + (value == null ? '' : '=' + value);
+      return name + '=' + (value == null ? '' : value);
     })
     .join('; ');
   setHeader(data, 'cookie', cookies);
 };
 
+function getCookieItem(name, cookie) {
+  if (!cookie || typeof cookie != 'object') {
+    cookie = cookie ? escapeValue(cookie) : cookie;
+    return name + '=' + (cookie == null ? '' : cookie);
+  }
+  var attrs = [escapeValue(cookie.value)];
+  var maxAge = cookie.maxAge ||  cookie.maxage ||
+      cookie.MaxAge || cookie['Max-Age'] || cookie['max-age'];
+  maxAge = parseInt(cookie.maxAge, 10);
+  if (!Number.isNaN(maxAge)) {
+    attrs.push('Expires=' + new Date(Date.now() + maxAge * 1000).toGMTString());
+    attrs.push('Max-Age=' + (maxAge === EXPIRED_SEC ? 0 : maxAge));
+  }
+
+  (cookie.secure || cookie.Secure) && attrs.push('Secure');
+  (cookie.httpOnly || cookie.HttpOnly || cookie.httponly) && attrs.push('HttpOnly');
+  var path = cookie.path || cookie.Path;
+  var domain = cookie.domain || cookie.Domain;
+  var sameSite = cookie.sameSite || cookie.samesite || cookie.SameSite;
+  path && attrs.push('Path=' + path);
+  domain && attrs.push('Domain=' + domain);
+  sameSite && attrs.push('SameSite=' + sameSite);
+  return name + attrs.join('; ');
+}
+
+function addMapArr(obj, key, value) {
+  var arr = obj[key] || [];
+  arr.push(value);
+  obj[key] = arr;
+}
+
 exports.setResCookies = function (data, cookies, req) {
+  var delKeys = parseDelResCookies(req);
+  if (delKeys) {
+    cookies = cookies ? extend(cookies, delKeys) : delKeys;
+  }
   var list = cookies && Object.keys(cookies);
   var notEmpty = list && list.length;
-  var delKeys = parseDelResCookies(req);
-  var curCookies = data.headers && data.headers['set-cookie'];
-  if (!notEmpty && (!delKeys || !curCookies)) {
+  if (!notEmpty) {
     return;
   }
+  var curCookies = data.headers && data.headers['set-cookie'];
   if (!Array.isArray(curCookies)) {
     curCookies = curCookies ? [curCookies + ''] : [];
   }
 
-  var result = {};
+  var curData = {};
   curCookies.forEach(function (cookie) {
     var index = cookie.indexOf('=');
     var key = cookie;
-    var value = null;
     if (index !== -1) {
       key = cookie.substring(0, index);
-      value = cookie.substring(index + 1);
-    }
-    if (!delKeys || !delKeys[key]) {
-      result[key] = value;
     }
+    addMapArr(curData, key, cookie);
   });
-
+  var result = {};
   list.forEach(function (name) {
     var cookie = cookies[name];
-    var origName = name;
     name = escapeName(name);
-    if (delKeys && (delKeys[name] || delKeys[origName])) {
-      return;
-    }
-    if (!cookie || typeof cookie != 'object') {
-      result[name] = cookie ? escapeValue(cookie) : cookie;
+    if (Array.isArray(cookie)) {
+      cookie.forEach(function(value) {
+        addMapArr(result, name, getCookieItem(name, value));
+      });
     } else {
-      var attrs = [];
-      var value = cookie.value;
-      attrs.push(escapeValue(value));
-      var maxAge =
-        cookie.maxAge ||
-        cookie.maxage ||
-        cookie['Max-Age'] ||
-        cookie['max-age'];
-      maxAge = parseInt(cookie.maxAge, 10);
-      if (!Number.isNaN(maxAge)) {
-        attrs.push(
-          'Expires=' + new Date(Date.now() + maxAge * 1000).toGMTString()
-        );
-        attrs.push('Max-Age=' + maxAge);
-      }
-
-      cookie.secure && attrs.push('Secure');
-      cookie.path && attrs.push('Path=' + cookie.path);
-      cookie.domain && attrs.push('Domain=' + cookie.domain);
-      (cookie.httpOnly || cookie.httponly) && attrs.push('HttpOnly');
-      var sameSite = cookie.sameSite || cookie.samesite || cookie.SameSite;
-      sameSite && attrs.push('SameSite=' + sameSite);
-      result[name] = attrs.join('; ');
+      addMapArr(result, name, getCookieItem(name, cookie));
     }
   });
-
-  cookies = Object.keys(result).map(function (name) {
-    var value = result[name];
-    return name + (value == null ? '' : '=' + value);
+  extend(curData, result);
+  result = [];
+  Object.keys(curData).forEach(function (name) {
+    curData[name].forEach(function(value) {
+      result.push(value);
+    });
   });
-  setHeader(data, 'set-cookie', cookies);
+  setHeader(data, 'set-cookie', result);
 };
 
 var SPEC_CHAR_RE = /[|\\{}()[\]^$+?.]/g;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "whistle",
   "description": "HTTP, HTTP2, HTTPS, Websocket debugging proxy",
-  "version": "2.9.35",
+  "version": "2.9.36-beta",
   "dataDirname": ".whistle",
   "localUIHost": "local.whistlejs.com",
   "port": 8899,
@@ -59,7 +59,7 @@
     "safe-buffer": "^5.1.2",
     "set-global-proxy": "^0.1.7",
     "sni": "1.0.0",
-    "sockx": "^0.2.0",
+    "sockx": "^0.2.1",
     "starting": "^8.0.1",
     "weinre2": "^1.3.2",
     "ws-parser": "^0.6.2",