whistle 2.10.9 → 2.10.11

package/lib/util/common.js

@@ -74,6 +74,13 @@ var PROP_SEP_RE = /(\\*)([|&]|\\[stnrfv])/g;
 var TEMP_FILE_RE = /^(?:\/?temp\/)?([\da-f]{64})(?:\.[\w.-]+)?$/;
 var TEMP_FILES_PATH = path.join(getWhistlePath(), 'temp_files');
 var SAVED_SESSIONS_FILE_RE = /_([1-9]\d*)_(\d+)$/;
+var REG_EXP_RE = /^\/(.+)\/(i?u?|ui)$/;
+
+function isRegExp(regExp) {
+  return REG_EXP_RE.test(regExp);
+}
+
+exports.isRegExp = isRegExp;
 
 exports.getSavedFileItem = function(filename) {
   var result = SAVED_SESSIONS_FILE_RE.exec(filename);
@@ -469,13 +476,22 @@ exports.addTrailerNames = function (
   });
 };
 
-exports.onResEnd = function (res, callback) {
-  var state = res._readableState || '';
-  if (state.endEmitted) {
+function onReadEnd(stream, callback) {
+  if (stream.readableEnded) {
     return callback();
   }
-  res.on('end', callback);
-};
+  stream.once('end', callback);
+}
+
+function onWriteEnd(stream, callback) {
+  if (stream.writableFinished) {
+    return callback();
+  }
+  stream.once('finish', callback);
+}
+
+exports.onReadEnd = onReadEnd;
+exports.onWriteEnd = onWriteEnd;
 
 var UPGRADE_RE = /^\s*upgrade\s*$/i;
 var WS_RE = /^\s*websocket\s*$/i;
@@ -725,7 +741,7 @@ function readStream(stream, callback) {
   stream.on('data', function(chunk) {
     buffer.push(chunk);
   });
-  stream.once('end', function() {
+  onReadEnd(stream, function() {
     var buf;
     var text;
     var json;
@@ -899,12 +915,11 @@ exports.onSocketEnd = function(socket, callback) {
       callback = null;
     }
   };
-  if (socket.aborted || socket.destroyed || socket._hasError) {
+  if (socket.aborted || socket.destroyed || socket.closed || socket._hasError) {
     return execCallback();
   }
   socket.on('error', execCallback);
   socket.once('close', execCallback);
-  socket.once('end', execCallback);
   socket.once('timeout', execCallback);
 };
 
@@ -1579,6 +1594,14 @@ function safeEncodeURIComponent(ch) {
   return ch;
 }
 
+function safeDecodeURIComponent(str) {
+  try {
+    return decodeURIComponent(str);
+  } catch (e) {}
+
+  return str;
+}
+
 exports.safeEncodeURIComponent = safeEncodeURIComponent;
 
 /**
@@ -1889,36 +1912,46 @@ exports.connectInner = function (options, cb, config) {
     auth = 'Basic ' + auth.toString('base64');
     setRawHeader(headers, 'proxy-authorization', auth);
   }
+  var req;
   var timer = setTimeout(function () {
-    if (req) {
-      req.emit('error', TIMEOUT_ERR);
-      req.destroy();
-      req.socket && req.socket.destroy();
-    }
+    req.emit('error', TIMEOUT_ERR);
+    req.destroy();
   }, CONN_TIMEOUT);
-  var req = httpModule.request(proxyOptions);
-  req.on('error', noop);
-  req
-    .on('connect', function (res, socket) {
+  var clearTimer = function (err) {
+    if (timer) {
+      err && req.destroy();
       clearTimeout(timer);
-      socket.on('error', noop);
-      if (res.statusCode !== 200) {
+      timer = null;
+    }
+  };
+  req = httpModule.request(proxyOptions);
+  req.on('error', clearTimer);
+  req.on('connect', function (res, socket) {
+    if (!timer) {
+      return;
+    }
+    clearTimer();
+    socket.on('error', function() {
+      socket.destroy();
+    });
+    var status = res.statusCode;
+    if (status !== 200) {
+      socket.destroy();
+      process.nextTick(function () {
         var err = new Error(
           'Tunneling socket could not be established, statusCode=' +
-            res.statusCode
+            status
         );
-        err.statusCode = res.statusCode;
-        socket.destroy();
-        process.nextTick(function () {
-          req.emit('error', err);
-        });
-        return;
-      }
-      if (res.headers[ALLOW_ACK]) {
-        socket.write('1');
-      }
-      cb(socket, res);
-    })
+        err.statusCode = status;
+        req.emit('error', err);
+      });
+      return;
+    }
+    if (res.headers[ALLOW_ACK]) {
+      socket.write('1');
+    }
+    cb(socket, res);
+  })
     .end();
   return req;
 };
@@ -2052,11 +2085,7 @@ exports.getSessionInfo = function(headers) {
     var value = data[LAST_RAW_INDEX + index];
     if (value) {
       rawResult[key] = value;
-      try {
-        result[key] = decodeURIComponent(value);
-      } catch (e) {
-        result[key] = value;
-      }
+      result[key] = safeDecodeURIComponent(value);
     }
   });
   result._ = rawResult;
@@ -2112,3 +2141,127 @@ function getHttpMeta(f1, f2, f3, headers, rawHeaders) {
 }
 
 exports.getHttpMeta = getHttpMeta;
+
+
+var PATTERN_FIELDS = ['url', 'm', 'method', 'reqH', 'reqHeader', 'reqHeaders',
+  'b', 'body', 'reqBody', 's', 'statusCode', 'status', 'resH', 'resHeader', 'resHeaders'];
+
+function getHeaderKey(key) {
+  return !key || typeof key !== 'string' || /[^\w!#$%&'*+.^`|~-]/.test(key) ? '' : key.toLowerCase();
+}
+
+function createFilter(item, name, key) {
+  var protocol = (item.exclude ? 'ex' : 'in') + 'cludeFilter://';
+  name = name ? name + (key ? '.' : ':') : '';
+  key = key ? key + ':' : '';
+  return protocol + name + key + item.value;
+}
+
+function wrapRegexp(value, item) {
+  var flags = item.ignoreCase || item.matchCase === false ? 'i' : '';
+  return { value: '/' + value.replace(/\//g, '\\$&') + '/' + flags, exclude: item.exclude};
+}
+
+function formatPath(url) {
+  if (/^\^?\/\//.test(url)) {
+    return url;
+  }
+  return url.replace(/^(\^?(?:https?:\/\/|wss?:\/\/)?)([/?.])/, function(all, prefix, flag) {
+    return flag === '/' ? prefix + '**/' : (prefix[0] === '^' ? prefix : '^' + prefix) + '**/**' + flag;
+  });
+}
+
+function formatUrlPattern(item) {
+  var value = item.value;
+  var type = item.type;
+  var prefix;
+  if (/^\s*(keyword|contains?)\s*$/i.test(type)) {
+    return wrapRegexp(item.value.replace(/[|/\\{}()[\]^$+?*.]/g, '\\$&'), item);
+  }
+  if (/^\s*regexp?\s*$/i.test(type)) {
+    return wrapRegexp(item.value.replace(/\//g, '\\$&'), item);
+  }
+  if (/^\s*(wildcard|star)\s*$/i.test(type)) {
+    return { value: formatPath(item.value.replace(/^\^*/, '^')), exclude: item.exclude };
+  }
+  if (value[0] === '?') {
+    prefix = '^**/**';
+  } else if (value[0] !== '/' || value[1] === '/' || isRegExp(value)) {
+    return { value: formatPath(value) };
+  }
+  return { value: (prefix || '**') + value };
+}
+
+function createPattern(list) {
+  if (!list) {
+    return '';
+  }
+  if (!Array.isArray(list)) {
+    list = [list];
+  }
+  var len = list.length;
+  if (!len) {
+    return '';
+  }
+  var result = [];
+  var urlPattern;
+  var key;
+  var add = function(item, name, key) {
+    var p = createFilter(item, name, key);
+    if (result.indexOf(p) === -1) {
+      result.push(p);
+    }
+  };
+  list.forEach(function(item) {
+    if (!item || !item.value) {
+      return;
+    }
+    var name = item.name || 'url';
+    if (PATTERN_FIELDS.indexOf(name) === -1) {
+      return;
+    }
+    if (typeof item.value !== 'string') {
+      item.value = String(item.value);
+    }
+    if (/\s/.test(item.value)) {
+      return;
+    }
+    switch(name) {
+    case 'url':
+      item = formatUrlPattern(item);
+      if (urlPattern || item.exclude) {
+        add(item);
+      } else {
+        urlPattern = item.value;
+      }
+      return;
+    case 'm':
+    case 'method':
+      return add(item, 'm');
+    case 'reqH':
+    case 'reqHeader':
+    case 'reqHeaders':
+      key = getHeaderKey(item.key);
+      return key && add(item, 'reqH', key);
+    case 'b':
+    case 'body':
+    case 'reqBody':
+      return add(item, 'b');
+    case 's':
+    case 'statusCode':
+    case 'status':
+      return add(item, 's');
+    case 'resH':
+    case 'resHeader':
+    case 'resHeaders':
+      key = getHeaderKey(item.key);
+      return key && add(item, 'resH', key);
+    }
+  });
+  if (urlPattern || result.length) {
+    result.unshift(urlPattern || '*');
+  }
+  return result.join(' ');
+}
+
+exports.createPattern = createPattern;

package/lib/util/data-server.js

@@ -323,7 +323,7 @@ function handleRequest(req, data) {
       delete data.abort;
     }
   };
-  req.on('end', removeAbort);
+  util.onReadEnd(req, removeAbort);
   req.on('error', removeAbort);
   req.on('abort', removeAbort);
   data.version = version;
@@ -743,6 +743,10 @@ module.exports = function init(_proxy) {
       }
     }
 
+    if (filterType === 'SSE') {
+      return rawType === 'text/event-stream';
+    }
+
     var curType = util.getContentType(rawType);
     for (i = 0, len = TYPES.length; i < len; i++) {
       type = TYPES[i];

package/lib/util/drain.js

@@ -1,14 +1,13 @@
-var createTransform = require('./common').createTransform;
+var common = require('./common');
 
-var noop = function () {};
+var noop = common.noop;
+var onReadEnd = common.onReadEnd;
 
 module.exports = function (stream, endHandler) {
   if (stream._hasAlreadyDrain || (!stream.noReqBody && stream.useH2)) {
     return typeof endHandler == 'function' && endHandler();
   }
   stream._hasAlreadyDrain = true;
-  var emitEndStream = createTransform();
-  emitEndStream.on('data', noop).on('error', noop);
-  emitEndStream.on('end', endHandler);
-  stream.pipe(emitEndStream);
+  onReadEnd(stream, endHandler);
+  stream.on('data', noop).on('error', noop);
 };

package/lib/util/file-mgr.js

@@ -58,7 +58,7 @@ function readSingleFile(path, callback) {
     }
   });
   stream.on('error', execCallback);
-  stream.on('end', execCallback);
+  stream.once('end', execCallback);
 }
 
 function getFileMap(list) {

package/lib/util/http-mgr.js

@@ -143,7 +143,11 @@ function sendReq(options, callback) {
 
   var timer, client;
   var handleCallback = function (err, res) {
+    if (timer == null) {
+      return;
+    }
     clearTimeout(timer);
+    timer = null;
     if (httpModule) {
       httpModule = null;
       callback(err, res);
@@ -249,7 +253,7 @@ function request(options, callback) {
           handleCallback(err, res, getBody(body, len));
         }
       });
-      res.on('end', function() {
+      res.once('end', function() {
         handleCallback(null, res, getBody(body, len));
       });
     };
@@ -325,7 +329,7 @@ function readFile(url, callback) {
       }
     });
     stream.on('error', listener);
-    stream.on('end', listener);
+    stream.once('end', listener);
   });
 }
 

package/lib/util/index.js

@@ -24,7 +24,7 @@ var ReplacePatternTransform = require('./replace-pattern-transform');
 var common = require('./common');
 var proc = require('./process');
 var parseUrl = require('./parse-url');
-var h2Consts = config.enableH2 ? require('http2').constants : {};
+var h2Consts = require('http2').constants;
 
 var protocols = protoMgr.protocols;
 var resProtocols = protoMgr.resProtocols;
@@ -39,7 +39,6 @@ var pendingFiles = {};
 var localIpCache = new LRU({ max: 120 });
 var LOCALHOST = '127.0.0.1';
 var aliasProtocols = protoMgr.aliasProtocols;
-var CONTEXT = vm.createContext();
 var END_SLASH_RE = /[/\\]$/;
 var GEN_URL_RE = /^\s*(?:https?:)?\/\/\w[^\s]*\s*$/i;
 var CORS_KEY_RE = /^(?:enable|use-credentials|useCredentials|credentials)$/i;
@@ -50,17 +49,12 @@ var STYLE_START = toBuffer('<style>');
 var STYLE_END = toBuffer('</style>');
 var PROXY_RE = /^x?(?:socks|https?-proxy|proxy|internal(?:-https)?-proxy)$/;
 var SEP_RE = /[|&]/;
-var ctxTimer;
 var END_RE = /[/\\]$/;
 var EXPIRED_SEC = -123456;
 var EXP_COOKIE = { maxAge: EXPIRED_SEC, path: '/' };
 var EXP_SECURE_COOKIE = { maxAge: EXPIRED_SEC, path: '/', secure: true };
 var EXP_COOKIE_D = { maxAge: EXPIRED_SEC, path: '/' };
 var EXP_SECURE_COOKIE_D = { maxAge: EXPIRED_SEC, path: '/', secure: true };
-var resetContext = function () {
-  ctxTimer = null;
-  CONTEXT = vm.createContext();
-};
 var TIMEOUT_ERR = common.TIMEOUT_ERR;
 var SUB_MATCH_RE = /\$[&\d]/;
 var PROTO_NAME_RE = /^([\w.-]+):\/\//;
@@ -89,6 +83,7 @@ var getMatcher = common.getMatcher;
 var getMatcherValue = common.getMatcherValue;
 var getRemoteAddr = common.getRemoteAddr;
 var getRemotePort = common.getRemotePort;
+var isRegExp = common.isRegExp;
 var workerIndex = process.env && process.env.workerIndex;
 var INTERNAL_ID = Date.now().toString(16) +  Math.floor(Math.random() * 10000000).toString(16);
 var pluginMgr;
@@ -175,6 +170,7 @@ exports.PLUGIN_INSPECTOR_CONFIG = Symbol('inspectorConfig');
 exports.drain = require('./drain');
 exports.isWin = process.platform === 'win32';
 exports.isUtf8 = isUtf8;
+exports.isRegExp = isRegExp;
 exports.WhistleTransform = require('./whistle-transform');
 exports.ReplacePatternTransform = ReplacePatternTransform;
 exports.replacePattern = replacePattern;
@@ -425,7 +421,7 @@ exports.compare = compare;
 var scriptCache = {};
 var VM_OPTIONS = {
   displayErrors: false,
-  timeout: 60
+  timeout: 72
 };
 var MAX_SCRIPT_SIZE = 1024 * 256;
 var MAX_SCRIPT_CACHE_COUNT = 64;
@@ -471,34 +467,20 @@ function getScript(content) {
   return script.script;
 }
 
-function clearContext() {
-  Object.keys(CONTEXT).forEach(function (key) {
-    delete CONTEXT[key];
-  });
-  if (!ctxTimer) {
-    ctxTimer = setTimeout(resetContext, 30000);
-  }
-}
+var customConsole = {};
+['fatal', 'error', 'warn', 'info', 'log', 'debug'].forEach(function (level) {
+  customConsole[level] = logger[level];
+});
 
-exports.execScriptSync = function(script, context) {
+exports.runInContext = function(script, context) {
   try {
-    if ((script = getScript(script))) {
-      CONTEXT.console = {};
-      ['fatal', 'error', 'warn', 'info', 'log', 'debug'].forEach(function (
-        level
-      ) {
-        CONTEXT.console[level] = logger[level];
-      });
-      Object.keys(context).forEach(function (key) {
-        CONTEXT[key] = context[key];
-      });
-      script.runInContext(CONTEXT, VM_OPTIONS);
+    if (script = getScript(script)) {
+      context = Object.assign({ console: Object.create(customConsole) }, context);
+      script.runInNewContext(context, VM_OPTIONS);
     }
     return true;
   } catch (e) {
     logger.error(e);
-  } finally {
-    clearContext();
   }
 };
 
@@ -603,14 +585,6 @@ function formatDate(now) {
 
 exports.formatDate = formatDate;
 
-var REG_EXP_RE = /^\/(.+)\/(i?u?|ui)$/;
-
-function isRegExp(regExp) {
-  return REG_EXP_RE.test(regExp);
-}
-
-exports.isRegExp = isRegExp;
-
 var ORIG_REG_EXP = /^\/(.+)\/([igmu]{0,4})$/;
 var FLAGS_RE = /[igmu]{2}/;
 
@@ -723,14 +697,13 @@ function isAuthCapture(req) {
 
 exports.isAuthCapture = isAuthCapture;
 
-exports.toRegExp = function toRegExp(regExp, ignoreCase) {
-  regExp = REG_EXP_RE.test(regExp);
-  try {
-    regExp = regExp && new RegExp(RegExp.$1, ignoreCase ? 'i' : RegExp.$2);
-  } catch (e) {
-    regExp = null;
+exports.toRegExp = function(regExp, ignoreCase) {
+  if (isRegExp(regExp)) {
+    try {
+      return new RegExp(RegExp.$1, ignoreCase ? 'i' : RegExp.$2);
+    } catch (e) {}
   }
-  return regExp;
+  return null;
 };
 
 var isString = common.isString;
@@ -967,8 +940,6 @@ function getPath(url, noProtocol) {
   return url;
 }
 
-exports.getPath = getPath;
-
 function getFilename(url) {
   if (typeof url == 'string' && (url = getPath(url).trim())) {
     var index = url.lastIndexOf('/');
@@ -1489,48 +1460,47 @@ function getRawType(data) {
 
 exports.getRawType = getRawType;
 
-function getContentType(contentType) {
-  if (contentType && typeof contentType != 'string') {
-    contentType = contentType['content-type'] || contentType.contentType;
+function getContentType(type) {
+  if (type && typeof type != 'string') {
+    type = type['content-type'] || type.contentType;
   }
-  contentType = _getRawType(contentType);
-  if (!contentType) {
+  type = _getRawType(type);
+  if (!type) {
     return;
   }
-  if (contentType.indexOf('javascript') != -1) {
+  if (type.indexOf('image/') != -1) {
+    return 'IMG';
+  }
+  if (type.indexOf('javascript') != -1) {
     return 'JS';
   }
 
-  if (contentType.indexOf('css') != -1) {
+  if (type.indexOf('css') != -1) {
     return 'CSS';
   }
 
-  if (contentType.indexOf('html') != -1) {
+  if (type.indexOf('html') != -1) {
     return 'HTML';
   }
 
-  if (contentType.indexOf('json') != -1) {
+  if (type.indexOf('json') != -1) {
     return 'JSON';
   }
 
-  if (contentType.indexOf('xml') != -1) {
+  if (type.indexOf('xml') != -1) {
     return 'XML';
   }
 
-  if (contentType.indexOf('text/') != -1) {
+  if (type.indexOf('text/') != -1) {
     return 'TEXT';
   }
-
-  if (contentType.indexOf('image/') != -1) {
-    return 'IMG';
-  }
 }
 
 exports.getContentType = getContentType;
 
-function isText(contentType) {
-  contentType = contentType && getContentType(contentType);
-  return contentType && contentType !== 'IMG';
+function isText(type) {
+  type = type && getContentType(type);
+  return type && type !== 'IMG';
 }
 
 exports.isText = isText;
@@ -1654,7 +1624,7 @@ function getPipeIconvStream(headers) {
       };
 
       res.on('data', resolveCharset);
-      res.on('end', resolveCharset);
+      onReadEnd(res, resolveCharset);
     });
 
     pipeStream.addTail(function (src, next) {
@@ -2649,7 +2619,10 @@ exports.getReqId = function (now) {
 };
 
 exports.onSocketEnd = common.onSocketEnd;
-exports.onResEnd = common.onResEnd;
+
+var onReadEnd = common.onReadEnd;
+exports.onReadEnd = onReadEnd;
+exports.onWriteEnd = common.onWriteEnd;
 
 exports.getEmptyRes = function getRes() {
   var res = common.createTransform();
@@ -2983,7 +2956,7 @@ exports.disableReqProps = function (req) {
     delete headers['user-agent'];
   }
 
-  if (disable.gzip || isEnable(req, 'captureStream')) {
+  if (disable.gzip || disable.zip || isEnable(req, 'captureStream')) {
     delete headers['accept-encoding'];
   }
 
@@ -3623,7 +3596,7 @@ function readOneChunk(stream, callback, timeout) {
     timer = setTimeout(handler, timeout);
   }
   stream.on('data', handler);
-  stream.on('end', handler);
+  onReadEnd(stream, handler);
 }
 
 exports.readOneChunk = readOneChunk;
@@ -4031,3 +4004,9 @@ exports.joinBuffer = function(buf, chunk) {
 exports.formatBuffer = function(buf, len) {
   return Array.isArray(buf) ? Buffer.concat(buf, len) : buf;
 };
+
+exports.endSocket = function (socket) {
+  if (!socket.destroyed && !socket.writableEnded) {
+    socket.end();
+  }
+};

package/lib/util/speed-transform.js

@@ -6,11 +6,11 @@ function SpeedTransform(options, req) {
   Transform.call(this);
   options = options || {};
   this._req = req;
-  var value = parseInt((options.speed * 1000) / 8);
+  var value = parseInt((options.speed * 1000) / 8, 10);
   if (value > 0) {
     this._speed = value;
   }
-  if ((value = parseInt(options.delay)) > 0) {
+  if ((value = parseInt(options.delay, 10)) > 0) {
     this._delay = value;
   }
 }

package/lib/util/transproto.js

@@ -71,15 +71,15 @@ function Parser() {
     }
     var data = parseChunk();
     if (data === false) {
-      return self.onEnd && self.onEnd();
+      return self.onEnd();
     }
     if (!data) {
       if (chunk === false) {
         return;
       }
-      return self.onContinue && self.onContinue();
+      return self.onContinue();
     }
-    self.onData && self.onData(data);
+    self.onData(data);
     self.write(false);
   };
 }
@@ -91,6 +91,7 @@ exports.getDecodeTransform = function () {
   parser.onEnd = function () {
     data && trans.push(data);
     trans.push(null);
+    data = null;
   };
   parser.onContinue = function () {
     if (transCb) {

package/lib/util/whistle-transform.js

@@ -9,11 +9,11 @@ var DOCTYPE = Buffer.from('<!DOCTYPE html>\r\n');
 function WhistleTransform(options, req) {
   Transform.call(this);
   options = options || {};
-  var value = parseInt((options.speed * 1000) / 8);
+  var value = parseInt((options.speed * 1000) / 8, 10);
   if (value > 0) {
     this._speed = value;
   }
-  if ((value = parseInt(options.delay)) > 0) {
+  if ((value = parseInt(options.delay, 10)) > 0) {
     this._delay = value;
   }
   this._noDoctype = options.noDoctype;