whistle 2.10.6 → 2.10.7

package/lib/tunnel.js

@@ -23,7 +23,6 @@ var X_RE = /^x/;
 var STATUS_CODES = require('http').STATUS_CODES || {};
 var getRawHeaderNames = hparser.getRawHeaderNames;
 var formatHeaders = hparser.formatHeaders;
-var getRawHeaders = hparser.getRawHeaders;
 var CRONET_RE = /\bCronet\b/;
 var SYS_HOST_RE = /^(?:[^/:]+\.)?(?:apple|cdn-apple|icloud|office|office365|mzstatic|apple-cloudkit|microsoft|parallels)\.(?:com|cn|com\.cn)$/;
 var TUNNEL_RE = /^tunnel:\/\//;
@@ -104,6 +103,7 @@ function tunnelProxy(server, proxy, type) {
           testId: req._testId,
           startTime: now,
           dnsTime: now,
+          servername: reqSocket.useSNI === hostname ? undefined : reqSocket.useSNI,
           captureError: true,
           rules: req.rules,
           req: {
@@ -382,7 +382,7 @@ function tunnelProxy(server, proxy, type) {
                 var reqDelay = util.getMatcherValue(_rules.reqDelay);
                 data.requestTime = data.dnsTime = Date.now();
                 if (reqDelay > 0) {
-                  setTimeout(handleConnect, reqDelay);
+                  util.setDelayTimer(reqSocket, setTimeout(handleConnect, reqDelay));
                 } else {
                   handleConnect();
                 }
@@ -404,8 +404,7 @@ function tunnelProxy(server, proxy, type) {
                 var proxyUrl;
                 if (tunnelPort) {
                   proxyUrl = 'proxy://127.0.0.1:' + tunnelPort;
-                  reqSocket.customParser = req.customParser =
-                    util.getParserStatus(req);
+                  reqSocket.customParser = req.customParser = util.getParserStatus(req);
                   pluginMgr.addSessionInfo(req, _rules);
                   headers[config.PLUGIN_HOOK_NAME_HEADER] =
                     config.PLUGIN_HOOKS.TUNNEL;
@@ -553,7 +552,7 @@ function tunnelProxy(server, proxy, type) {
                           }
                         };
                         if (reqDelay > 0) {
-                          setTimeout(connectProxy, reqDelay);
+                          util.setDelayTimer(reqSocket, setTimeout(connectProxy, reqDelay));
                         } else {
                           connectProxy();
                         }
@@ -687,7 +686,7 @@ function tunnelProxy(server, proxy, type) {
             };
             var resDelay = util.getMatcherValue(_rules.resDelay);
             if (resDelay > 0) {
-              setTimeout(handleEstablished, resDelay);
+              util.setDelayTimer(reqSocket, setTimeout(handleEstablished, resDelay));
             } else {
               handleEstablished();
             }
@@ -770,23 +769,14 @@ function tunnelProxy(server, proxy, type) {
                   util.deleteResHeaders(req, resHeaders);
                   code = util.getMatcherValue(reqRules.replaceStatus) || code;
                   var isSuccess = code == 200;
-                  var message =
-                    isSuccess
-                      ? 'Connection Established'
-                      : STATUS_CODES[code] || 'unknown';
-                  var statusLine = ['HTTP/1.1', code, message].join(' ');
+                  var message = isSuccess ? 'Connection Established' : STATUS_CODES[code] || 'unknown';
                   var curHeaders = resHeaders;
-                  var rawData = statusLine;
                   if (req.fromComposer) {
                     curHeaders = extend({}, resHeaders);
                     curHeaders['x-whistle-req-id'] = req.reqId;
                     util.setFramesMode(curHeaders, isSuccess && inspect);
                   }
-                  curHeaders = getRawHeaders(formatHeaders(curHeaders, rawHeaderNames));
-                  if (curHeaders) {
-                    rawData += '\r\n' + curHeaders;
-                  }
-                  rawData += '\r\n\r\n';
+                  var rawData = common.getHttpMeta('HTTP/1.1', code, message, curHeaders, rawHeaderNames);
                   try {
                     if (code != 200) {
                       reqSocket.end(rawData, cb);

package/lib/upgrade.js

@@ -6,9 +6,7 @@ var util = require('./util');
 var config = require('./config');
 var common = require('./util/common');
 
-var formatHeaders = hparser.formatHeaders;
 var getRawHeaderNames = hparser.getRawHeaderNames;
-var getRawHeaders = hparser.getRawHeaders;
 var PLUGIN_PATH_RE =
   /^\/(\.\.\.whistle-path\.5b6af7b9884e1165\.\.\.\/\/\/)?(whistle|plugin)\.([^/?#]+)\/?/;
 
@@ -105,12 +103,10 @@ function upgradeHandler(req, socket) {
   util.addTunnelData(socket, headers);
   socket._clientId = util.getComposerClientId(headers);
   var getBuffer = function (method, newHeaders, path) {
-    var rawData = (method || 'GET') + ' ' + (path || socket.url || req.url) + ' HTTP/1.1';
-    newHeaders = formatHeaders(newHeaders || headers, req.rawHeaders);
-    if (newHeaders = getRawHeaders(newHeaders)) {
-      rawData += '\r\n' + newHeaders;
-    }
-    return Buffer.from(rawData + '\r\n\r\n');
+    method = method || 'GET';
+    path = path || socket.url || req.url;
+    newHeaders = newHeaders || headers;
+    return Buffer.from(common.getHttpMeta(method, path, 'HTTP/1.1', newHeaders, req.rawHeaders));
   };
   var sniPlugin = headers[util.SNI_PLUGIN_HEADER];
   if (sniPlugin) {

package/lib/util/common.js

@@ -12,6 +12,7 @@ var json5 = require('json5');
 var extend = require('extend');
 var tls = require('tls');
 var https = require('https');
+var hparser = require('hparser');
 var zlibx = require('./zlib');
 var pkgConf = require('../../package.json');
 var isUtf8 = require('./is-utf8');
@@ -72,10 +73,22 @@ var TOKEN = '\r\u0000\n\u0003\r';
 var PROP_SEP_RE = /(\\*)([|&]|\\[stnrfv])/g;
 var TEMP_FILE_RE = /^(?:\/?temp\/)?([\da-f]{64})(?:\.[\w.-]+)?$/;
 var TEMP_FILES_PATH = path.join(getWhistlePath(), 'temp_files');
+var SAVED_SESSIONS_FILE_RE = /_([1-9]\d*)_(\d+)$/;
+
+exports.getSavedFileItem = function(filename) {
+  var result = SAVED_SESSIONS_FILE_RE.exec(filename);
+  return result && {
+    count: +result[1],
+    time: +result[2],
+    filename: filename.slice(0,  - result[1].length - result[2].length - 2)
+  };
+};
 
 
 exports.WHISTLE_UID_HEADER = '_x-whistle-uid_';
 
+exports.isUtf8 = isUtf8;
+
 exports.getTempFile = function(str) {
   var match = TEMP_FILE_RE.exec(str);
   return match && match[1];
@@ -670,18 +683,29 @@ exports.isGroup = function(name) {
 };
 
 var UTF8_RE = /^utf-?8$/i;
+
+function getUtf8Buf(buffer, charset) {
+  charset = isString(charset) ? charset.trim() : '';
+  if (UTF8_RE.test(charset) || isUtf8(buffer)) {
+    return buffer;
+  }
+  try {
+    buffer = iconv.encode(buffer, charset || 'GB18030');
+  } catch (e) {}
+  return buffer;
+}
+
+exports.getUtf8Buf = getUtf8Buf;
+
+
 function bufferToString(buffer, encoding) {
-  if (typeof encoding === 'string') {
-    encoding = encoding.trim();
-  } else {
-    encoding = null;
+  if (!buffer) {
+    return '';
   }
-  if (Buffer.isBuffer(buffer) && !UTF8_RE.test(encoding) && !isUtf8(buffer)) {
-    try {
-      buffer = iconv.encode(buffer, encoding || 'GB18030');
-    } catch (e) {}
+  if (Buffer.isBuffer(buffer)) {
+    buffer = getUtf8Buf(buffer, encoding);
   }
-  return String(buffer || '');
+  return String(buffer);
 }
 
 exports.bufferToString = bufferToString;
@@ -852,9 +876,24 @@ exports.getHostPort = function(str) {
   }
 };
 
+function clearDelayTimer(socket) {
+  var timer = socket._w2Timer_;
+  if (timer) {
+    clearTimeout(timer);
+    socket._w2Timer_ = null;
+  }
+}
+
+exports.clearDelayTimer = clearDelayTimer;
+exports.setDelayTimer = function (socket, id) {
+  socket._w2Timer_ = id;
+};
+
+
 exports.onSocketEnd = function(socket, callback) {
   var execCallback = function (err) {
     socket._hasError = true;
+    clearDelayTimer(socket);
     if (callback) {
       callback(err);
       callback = null;
@@ -970,6 +1009,10 @@ function isString(str) {
 
 exports.isString = isString;
 
+exports.isStr = function(str) {
+  return typeof str === 'string';
+};
+
 function isSafeNumStr(str) {
   if (str == '0') {
     return true;
@@ -1429,6 +1472,18 @@ exports.sendGzip = function (req, res, data) {
   });
 };
 
+exports.toString = function (str) {
+  if (str != null) {
+    if (typeof str === 'string') {
+      return str;
+    }
+    try {
+      return JSON.stringify(str) || '';
+    } catch (e) {}
+  }
+  return '';
+};
+
 exports.sendGzipText = function(req, res, headers, text, gzipText) {
   if (!text || !canGzip(req) || (!gzipText && text.length < GZIP_THRESHOLD)) {
     headers && res.writeHead(200, headers);
@@ -2046,3 +2101,14 @@ var COMMENT_RE = /#[^\r\n]*/g;
 exports.removeComment = function(text) {
   return text.replace(COMMENT_RE, '').trim();
 };
+
+function getHttpMeta(f1, f2, f3, headers, rawHeaders) {
+  var statusLine = [f1, f2, f3].join(' ');
+  if (headers) {
+    headers = hparser.formatHeaders(headers, rawHeaders);
+    headers = hparser.getRawHeaders(headers);
+  }
+  return (headers ? statusLine + '\r\n' + headers : statusLine) + '\r\n\r\n';
+}
+
+exports.getHttpMeta = getHttpMeta;

package/lib/util/data-server.js

@@ -31,6 +31,58 @@ var framesCache = [];
 var framesMap = {};
 var proxy, binded;
 var clearCount = 0;
+var MAX_COUNT = 120;
+
+function formatApiOptions(options) {
+  var result = { count: MAX_COUNT };
+  if (options) {
+    var count = options.count;
+    if (count > 0 && count < MAX_COUNT) {
+      result.count = count;
+    }
+    var startTime = options.startTime > 0 ? options.startTime + '' : null;
+    var startId = util.isString(options.startId) ? options.startId : null;
+    if (startId && startTime && startId < startTime) {
+      startId = startTime;
+    }
+    result.startId = startId || startTime;
+    result.latest = options.latest;
+  }
+  return result;
+}
+
+function getItemId(item, name) {
+  return name ? item[name] : item;
+}
+
+function getIdIndex(arr, target, name) {
+  if (!target) {
+    return 0;
+  }
+  var left = 0;
+  var right = arr.length - 1;
+
+  while (left <= right) {
+    if (getItemId(arr[left], name) > target) {
+      return left;
+    }
+    var count = right - left;
+    if (!count) {
+      return -1;
+    }
+    if (count === 1) {
+      return getItemId(arr[right], name) > target ? right : -1;
+    }
+    var mid = left + Math.floor(count / 2);
+    var val = getItemId(arr[mid], name);
+    if (val > target) {
+      right = mid;
+    } else {
+      left = mid + 1;
+    }
+  }
+  return -1;
+}
 
 function enable() {
   if (binded) {
@@ -595,6 +647,227 @@ module.exports = function init(_proxy) {
     return getFrames(options.curReqId, options.lastFrameId);
   };
 
+  proxy.getFramesForApi = function (options) {
+    var reqId = options.reqId;
+    var isServer = options.from === 'server';
+    var isClient = options.from === 'client';
+    var isAll = !isServer && !isClient;
+    var params = formatApiOptions(options);
+    var index = getIdIndex(framesCache, params.startId, 'frameId');
+    var result = [];
+    if (index === -1) {
+      return result;
+    }
+    var count = params.count;
+    var len = framesCache.length;
+    var addFrame = function(i) {
+      var frame = framesCache[i];
+      if ((isAll || isClient === !!frame.isClient) && frame.reqId === reqId) {
+        result.push(decodeData(frame));
+        if (--count <= 0) {
+          return true;
+        }
+      }
+    };
+    if (params.latest) {
+      for (var i = len - 1; i >= index; i--) {
+        if (addFrame(i)) {
+          break;
+        }
+      }
+      result.reverse();
+    } else {
+      for (; index < len; index++) {
+        if (addFrame(index)) {
+          break;
+        }
+      }
+    }
+    return result;
+  };
+
+  function checkUrl(session, subUrl) {
+    if (!subUrl) {
+      return true;
+    }
+    var url = session.isHttps ? 'tunnel://' + session.url : session.url.toLowerCase();
+    return url.indexOf(subUrl) !== -1;
+  }
+
+  var WS_RE = /^wss?:\/\//;
+  var FONT_RE = /font\//i;
+  var MEDIA_RE = /audio\/|video\/|application\/vnd.apple.mpegurl|application\/dash+xml/i;
+  var WASM_RE = /application\/wasm/i;
+  var TYPES = ['IMG', 'HTML', 'CSS', 'JS', 'JSON'];
+  var ALIAS_TYPES = {
+    WEBSOCKET: 'WS',
+    WSS: 'WS',
+    IMAGE: 'IMG',
+    JAVASCRIPT: 'JS',
+    STYLE: 'CSS',
+    MATCHED: 'RULES',
+    EDIT: 'COMPOSER'
+  };
+  var RAW_TYPES = ['FONT', 'MEDIA', 'WASM'];
+
+  function hasError(item) {
+    var statusCode = item.res && item.res.statusCode;
+    return item.reqError || item.resError || (item.customData && item.customData.error) ||
+    (statusCode && (!/^\d+$/.test(statusCode) || statusCode >= 400));
+  }
+
+  function checkType(session, filterType) {
+    if (!filterType) {
+      return true;
+    }
+    filterType = ALIAS_TYPES[filterType] || filterType;
+    if (filterType === 'CAPTUREERROR') {
+      return session.captureError;
+    }
+    if (filterType === 'WS') {
+      return WS_RE.test(session.url);
+    }
+    if (filterType === 'TUNNEL') {
+      return session.isHttps;
+    }
+    if (filterType === 'RULES') {
+      return session.rules && Object.keys(session.rules).length > 0;
+    }
+    var rawType = util.getRawType(session.res);
+    var type;
+    for (var i = 0, len = RAW_TYPES.length; i < len; i++) {
+      type = RAW_TYPES[i];
+      if (filterType === type) {
+        var p = type === 'FONT' ? FONT_RE : type === 'MEDIA' ? MEDIA_RE : WASM_RE;
+        return p.test(rawType);
+      }
+    }
+
+    var curType = util.getContentType(rawType);
+    for (i = 0, len = TYPES.length; i < len; i++) {
+      type = TYPES[i];
+      if (filterType === type) {
+        return curType === (i ? type : 'IMG');
+      }
+    }
+    if (filterType === 'ERROR') {
+      return hasError(session);
+    }
+    if (filterType === 'MOCK') {
+      var rules = session.rules || '';
+      return rules.rule && rules.rule.isLoc;
+    }
+    if (filterType === 'IMPORT') {
+      return session.importedData;
+    }
+    if (filterType === 'COMPOSER') {
+      return session.fc;
+    }
+  }
+
+  function formatHeaderFilter(filter) {
+    if (!filter) {
+      return;
+    }
+    var name = filter.name || filter.key;
+    if (typeof name !== 'string') {
+      return;
+    }
+    return {
+      name: name.trim(),
+      subValue: filter.subValue == null ? null : String(filter.subValue).toLowerCase()
+    };
+  }
+
+  function filterHeader(headers, filter) {
+    if (!filter) {
+      return true;
+    }
+    if (!headers) {
+      return false;
+    }
+    var value = headers[filter.name];
+    if (value == null) {
+      return false;
+    }
+    var subValue = filter.subValue;
+    var noSubValue = !subValue;
+    if (noSubValue || !value) {
+      return noSubValue;
+    }
+    value = Array.isArray(value) ? value.join('\n') : String(value);
+    return value.toLowerCase().indexOf(subValue) !== -1;
+  }
+
+  function checkMethod(session, method) {
+    if (!method) {
+      return true;
+    }
+    return method.indexOf(session.req.method) !== -1;
+  }
+
+  function checkStatus(session, status) {
+    return !status || status.indexOf(String(session.res.statusCode).toUpperCase()) !== -1;
+  }
+
+
+  proxy.getSessionsForApi = function (options) {
+    var params = formatApiOptions(options);
+    var index = getIdIndex(ids, params.startId);
+    var result = [];
+    if (index === -1) {
+      return result;
+    }
+    var count = params.count;
+    var len = ids.length;
+    var subUrl = util.getString(options.subUrl).toLowerCase();
+    var type = util.getString(options.type).toUpperCase();
+    var method = options.method;
+    var status = options.statusCode || options.status;
+    var reqHeader = formatHeaderFilter(options.reqHeader);
+    var resHeader = formatHeaderFilter(options.resHeader);
+    if (Array.isArray(method)) {
+      method = method.join();
+    } else {
+      method = util.getString(method);
+    }
+    if (Array.isArray(status)) {
+      status = status.join();
+    } else if (status) {
+      status = String(status);
+    }
+    method = method && method.toUpperCase().split(',');
+    status = status && status.toUpperCase().split(',');
+    var addSession = function(i) {
+      var session = reqData[ids[i]];
+      if (checkUrl(session, subUrl) && checkMethod(session, method)
+        && checkStatus(session, status) && filterHeader(session.req.headers, reqHeader)
+        && filterHeader(session.res.headers, resHeader) && checkType(session, type)) {
+        toBase64String(session.req);
+        toBase64String(session.res);
+        result.push(session);
+        if (--count <= 0) {
+          return true;
+        }
+      }
+    };
+    if (params.latest) {
+      for (var i = len - 1; i >= index; i--) {
+        if (addSession(i)) {
+          break;
+        }
+      }
+      result.reverse();
+    } else {
+      for (; index < len; index++) {
+        if (addSession(index)) {
+          break;
+        }
+      }
+    }
+    return result;
+  };
+
   function getNetworkTime(id) {
     var item = id && reqData[id];
     return item && {

package/lib/util/index.js

@@ -126,7 +126,10 @@ exports.ADDITIONAL_HEAD = 'x-whistle-additional-headers';
 workerIndex = workerIndex >= 0 ? common.padLeft(config.workerIndex, 3) : '';
 
 
+exports.toString = common.toString;
 exports.sendRes = common.sendRes;
+exports.setDelayTimer = common.setDelayTimer;
+exports.clearDelayTimer = common.clearDelayTimer;
 exports.trimStr = trimStr;
 exports.THEME_STYLE = THEME_STYLE;
 exports.encodeNonLatin1Char = encodeNonLatin1Char;
@@ -2464,10 +2467,7 @@ exports.getPluginModal = function(conf) {
 };
 
 function getString(str) {
-  if (!isString(str)) {
-    return;
-  }
-  return str.trim();
+  return isString(str) ? str.trim() : '';
 }
 
 exports.getString = getString;
@@ -2637,19 +2637,6 @@ exports.getStaticDir = function (conf) {
   return toRelPath(staticDir.replace(/^\/+/, ''));
 };
 
-function toString(str) {
-  if (str != null) {
-    if (typeof str === 'string') {
-      return str;
-    }
-    try {
-      return JSON.stringify(str) || '';
-    } catch (e) {}
-  }
-  return '';
-}
-exports.toString = toString;
-
 var index = 0;
 
 exports.getReqId = function (now) {
@@ -2977,7 +2964,7 @@ exports.setResCors = function (data, cors, req) {
   } else if (autoComp) {
     var method = req.headers['access-control-request-method'];
     if (method) {
-      setHeader(data, 'access-control-allow-method', method);
+      setHeader(data, 'access-control-allow-methods', method);
     }
   }
 
@@ -3624,7 +3611,7 @@ function readOneChunk(stream, callback, timeout) {
   }
   var timer;
   var handler = function (chunk) {
-    timer && clearTimeout(timer);
+    clearTimeout(timer);
     stream.pause();
     stream.removeListener('data', handler);
     stream.removeListener('end', handler);
@@ -3698,10 +3685,9 @@ exports.getAuthBasic = function (auth) {
 
 exports.delay = function (time, callback) {
   if (time > 0) {
-    setTimeout(callback, time);
-  } else {
-    callback();
+    return setTimeout(callback, time);
   }
+  callback();
 };
 
 var F_HOST_RE = /\bhost\b/i;

package/lib/util/patch.js

@@ -4,6 +4,7 @@ var Socket = require('net').Socket;
 var http = require('http');
 var https = require('https');
 var hparser = require('hparser');
+var common = require('./common');
 
 var httpRequest = http.request;
 var httpsRequest = https.request;
@@ -133,12 +134,7 @@ http.request = function (options) {
         headers['x-whistle-policy'] = 'intercept';
         delete headers['X-Whistle-Policy'];
       }
-      headers = hparser.getRawHeaders(headers);
-      var rawData = 'CONNECT ' + tunnelPath + ' HTTP/1.1';
-      if (headers) {
-        rawData += '\r\n' + headers;
-      }
-      rawData += '\r\n\r\n';
+      var rawData = common.getHttpMeta('CONNECT', tunnelPath, 'HTTP/1.1', headers);
       if (res.statusCode === 200 && res.headers['x-whistle-allow-tunnel-ack']) {
         rawData = '1' + rawData;
       }

package/lib/util/speed-transform.js

@@ -1,9 +1,11 @@
 var Transform = require('pipestream').Transform;
-var util = require('util');
+var inherits = require('util').inherits;
+var setDelayTimer = require('./common').setDelayTimer;
 
-function SpeedTransform(options) {
+function SpeedTransform(options, req) {
   Transform.call(this);
   options = options || {};
+  this._req = req;
   var value = parseInt((options.speed * 1000) / 8);
   if (value > 0) {
     this._speed = value;
@@ -13,13 +15,22 @@ function SpeedTransform(options) {
   }
 }
 
-util.inherits(SpeedTransform, Transform);
+inherits(SpeedTransform, Transform);
 
-SpeedTransform.prototype._transform = function (chunk, encoding, callback) {
+var proto = SpeedTransform.prototype;
+
+proto._setTimeout = function (callback, delay) {
+  var timer = setTimeout(callback, delay);
+  if (this._req) {
+    setDelayTimer(this._req, timer);
+  }
+};
+
+proto._transform = function (chunk, encoding, callback) {
   var self = this;
   var cb = function () {
     if (chunk && self._speed) {
-      setTimeout(function () {
+      self._setTimeout(function () {
         callback(null, chunk);
       }, Math.round((chunk.length * 1000) / self._speed));
     } else {
@@ -30,7 +41,7 @@ SpeedTransform.prototype._transform = function (chunk, encoding, callback) {
   if (self._delay) {
     var delay = self._delay;
     self._delay = null;
-    return setTimeout(cb, delay);
+    return self._setTimeout(cb, delay);
   }
 
   cb();