whistle 2.10.5 → 2.10.7

package/lib/util/data-server.js

@@ -31,6 +31,58 @@ var framesCache = [];
 var framesMap = {};
 var proxy, binded;
 var clearCount = 0;
+var MAX_COUNT = 120;
+
+function formatApiOptions(options) {
+  var result = { count: MAX_COUNT };
+  if (options) {
+    var count = options.count;
+    if (count > 0 && count < MAX_COUNT) {
+      result.count = count;
+    }
+    var startTime = options.startTime > 0 ? options.startTime + '' : null;
+    var startId = util.isString(options.startId) ? options.startId : null;
+    if (startId && startTime && startId < startTime) {
+      startId = startTime;
+    }
+    result.startId = startId || startTime;
+    result.latest = options.latest;
+  }
+  return result;
+}
+
+function getItemId(item, name) {
+  return name ? item[name] : item;
+}
+
+function getIdIndex(arr, target, name) {
+  if (!target) {
+    return 0;
+  }
+  var left = 0;
+  var right = arr.length - 1;
+
+  while (left <= right) {
+    if (getItemId(arr[left], name) > target) {
+      return left;
+    }
+    var count = right - left;
+    if (!count) {
+      return -1;
+    }
+    if (count === 1) {
+      return getItemId(arr[right], name) > target ? right : -1;
+    }
+    var mid = left + Math.floor(count / 2);
+    var val = getItemId(arr[mid], name);
+    if (val > target) {
+      right = mid;
+    } else {
+      left = mid + 1;
+    }
+  }
+  return -1;
+}
 
 function enable() {
   if (binded) {
@@ -595,6 +647,227 @@ module.exports = function init(_proxy) {
     return getFrames(options.curReqId, options.lastFrameId);
   };
 
+  proxy.getFramesForApi = function (options) {
+    var reqId = options.reqId;
+    var isServer = options.from === 'server';
+    var isClient = options.from === 'client';
+    var isAll = !isServer && !isClient;
+    var params = formatApiOptions(options);
+    var index = getIdIndex(framesCache, params.startId, 'frameId');
+    var result = [];
+    if (index === -1) {
+      return result;
+    }
+    var count = params.count;
+    var len = framesCache.length;
+    var addFrame = function(i) {
+      var frame = framesCache[i];
+      if ((isAll || isClient === !!frame.isClient) && frame.reqId === reqId) {
+        result.push(decodeData(frame));
+        if (--count <= 0) {
+          return true;
+        }
+      }
+    };
+    if (params.latest) {
+      for (var i = len - 1; i >= index; i--) {
+        if (addFrame(i)) {
+          break;
+        }
+      }
+      result.reverse();
+    } else {
+      for (; index < len; index++) {
+        if (addFrame(index)) {
+          break;
+        }
+      }
+    }
+    return result;
+  };
+
+  function checkUrl(session, subUrl) {
+    if (!subUrl) {
+      return true;
+    }
+    var url = session.isHttps ? 'tunnel://' + session.url : session.url.toLowerCase();
+    return url.indexOf(subUrl) !== -1;
+  }
+
+  var WS_RE = /^wss?:\/\//;
+  var FONT_RE = /font\//i;
+  var MEDIA_RE = /audio\/|video\/|application\/vnd.apple.mpegurl|application\/dash+xml/i;
+  var WASM_RE = /application\/wasm/i;
+  var TYPES = ['IMG', 'HTML', 'CSS', 'JS', 'JSON'];
+  var ALIAS_TYPES = {
+    WEBSOCKET: 'WS',
+    WSS: 'WS',
+    IMAGE: 'IMG',
+    JAVASCRIPT: 'JS',
+    STYLE: 'CSS',
+    MATCHED: 'RULES',
+    EDIT: 'COMPOSER'
+  };
+  var RAW_TYPES = ['FONT', 'MEDIA', 'WASM'];
+
+  function hasError(item) {
+    var statusCode = item.res && item.res.statusCode;
+    return item.reqError || item.resError || (item.customData && item.customData.error) ||
+    (statusCode && (!/^\d+$/.test(statusCode) || statusCode >= 400));
+  }
+
+  function checkType(session, filterType) {
+    if (!filterType) {
+      return true;
+    }
+    filterType = ALIAS_TYPES[filterType] || filterType;
+    if (filterType === 'CAPTUREERROR') {
+      return session.captureError;
+    }
+    if (filterType === 'WS') {
+      return WS_RE.test(session.url);
+    }
+    if (filterType === 'TUNNEL') {
+      return session.isHttps;
+    }
+    if (filterType === 'RULES') {
+      return session.rules && Object.keys(session.rules).length > 0;
+    }
+    var rawType = util.getRawType(session.res);
+    var type;
+    for (var i = 0, len = RAW_TYPES.length; i < len; i++) {
+      type = RAW_TYPES[i];
+      if (filterType === type) {
+        var p = type === 'FONT' ? FONT_RE : type === 'MEDIA' ? MEDIA_RE : WASM_RE;
+        return p.test(rawType);
+      }
+    }
+
+    var curType = util.getContentType(rawType);
+    for (i = 0, len = TYPES.length; i < len; i++) {
+      type = TYPES[i];
+      if (filterType === type) {
+        return curType === (i ? type : 'IMG');
+      }
+    }
+    if (filterType === 'ERROR') {
+      return hasError(session);
+    }
+    if (filterType === 'MOCK') {
+      var rules = session.rules || '';
+      return rules.rule && rules.rule.isLoc;
+    }
+    if (filterType === 'IMPORT') {
+      return session.importedData;
+    }
+    if (filterType === 'COMPOSER') {
+      return session.fc;
+    }
+  }
+
+  function formatHeaderFilter(filter) {
+    if (!filter) {
+      return;
+    }
+    var name = filter.name || filter.key;
+    if (typeof name !== 'string') {
+      return;
+    }
+    return {
+      name: name.trim(),
+      subValue: filter.subValue == null ? null : String(filter.subValue).toLowerCase()
+    };
+  }
+
+  function filterHeader(headers, filter) {
+    if (!filter) {
+      return true;
+    }
+    if (!headers) {
+      return false;
+    }
+    var value = headers[filter.name];
+    if (value == null) {
+      return false;
+    }
+    var subValue = filter.subValue;
+    var noSubValue = !subValue;
+    if (noSubValue || !value) {
+      return noSubValue;
+    }
+    value = Array.isArray(value) ? value.join('\n') : String(value);
+    return value.toLowerCase().indexOf(subValue) !== -1;
+  }
+
+  function checkMethod(session, method) {
+    if (!method) {
+      return true;
+    }
+    return method.indexOf(session.req.method) !== -1;
+  }
+
+  function checkStatus(session, status) {
+    return !status || status.indexOf(String(session.res.statusCode).toUpperCase()) !== -1;
+  }
+
+
+  proxy.getSessionsForApi = function (options) {
+    var params = formatApiOptions(options);
+    var index = getIdIndex(ids, params.startId);
+    var result = [];
+    if (index === -1) {
+      return result;
+    }
+    var count = params.count;
+    var len = ids.length;
+    var subUrl = util.getString(options.subUrl).toLowerCase();
+    var type = util.getString(options.type).toUpperCase();
+    var method = options.method;
+    var status = options.statusCode || options.status;
+    var reqHeader = formatHeaderFilter(options.reqHeader);
+    var resHeader = formatHeaderFilter(options.resHeader);
+    if (Array.isArray(method)) {
+      method = method.join();
+    } else {
+      method = util.getString(method);
+    }
+    if (Array.isArray(status)) {
+      status = status.join();
+    } else if (status) {
+      status = String(status);
+    }
+    method = method && method.toUpperCase().split(',');
+    status = status && status.toUpperCase().split(',');
+    var addSession = function(i) {
+      var session = reqData[ids[i]];
+      if (checkUrl(session, subUrl) && checkMethod(session, method)
+        && checkStatus(session, status) && filterHeader(session.req.headers, reqHeader)
+        && filterHeader(session.res.headers, resHeader) && checkType(session, type)) {
+        toBase64String(session.req);
+        toBase64String(session.res);
+        result.push(session);
+        if (--count <= 0) {
+          return true;
+        }
+      }
+    };
+    if (params.latest) {
+      for (var i = len - 1; i >= index; i--) {
+        if (addSession(i)) {
+          break;
+        }
+      }
+      result.reverse();
+    } else {
+      for (; index < len; index++) {
+        if (addSession(index)) {
+          break;
+        }
+      }
+    }
+    return result;
+  };
+
   function getNetworkTime(id) {
     var item = id && reqData[id];
     return item && {

package/lib/util/file-mgr.js

@@ -36,21 +36,24 @@ function readSingleFile(path, callback) {
     return callback();
   }
   var stream = fs.createReadStream(convertSlash(path));
-  var done, buf;
+  var done;
+  var buf = [];
+  var len = 0;
   var execCallback = function (err) {
     if (done) {
       return;
     }
     done = true;
     stream.close();
-    callback(err ? null : buf);
+    callback(buf.length > 1 ? Buffer.concat(buf) : buf[0]);
   };
   stream.on('data', function (data) {
     if (done) {
       return;
     }
-    buf = buf ? Buffer.concat([buf, data]) : data;
-    if (buf.length > MAX_SIZE) {
+    len += data.length;
+    buf.push(data);
+    if (len > MAX_SIZE) {
       execCallback();
     }
   });

package/lib/util/http-mgr.js

@@ -182,6 +182,10 @@ function gunzip(err, res, body, callback) {
   }
 }
 
+function getBody(body, len) {
+  return body.length > 1 ? Buffer.concat(body, len) : (body[0] || '');
+}
+
 function request(options, callback) {
   loadPlugin(options, function (err) {
     if (err) {
@@ -230,21 +234,23 @@ function request(options, callback) {
         return callback(null, res, res);
       }
       addTimeout();
-      var body = '';
+      var body = [];
+      var len = 0;
       res.on('data', function (data) {
-        body = body ? Buffer.concat([body, data]) : data;
+        len += data.length;
+        body.push(data);
         addTimeout();
-        if (maxLength && body.length > maxLength) {
+        if (maxLength && len > maxLength) {
           var err;
           if (!options.ignoreExceedError) {
             err = new Error('The response body exceeded length limit');
             err.code = EXCEED;
           }
-          handleCallback(err, res, body);
+          handleCallback(err, res, getBody(body, len));
         }
       });
       res.on('end', function() {
-        handleCallback(null, res, body);
+        handleCallback(null, res, getBody(body, len));
       });
     };
     client = sendReq(options, function (err, res) {
@@ -293,7 +299,8 @@ function readFile(url, callback) {
 
     var stream = fs.createReadStream(filePath, OPTIONS);
     var done;
-    var body = '';
+    var body = [];
+    var len = 0;
     var listener = function (err) {
       if (done) {
         return;
@@ -305,14 +312,15 @@ function readFile(url, callback) {
       done = true;
       data.mtime = time;
       stream.close();
-      triggerChange(data, body);
+      triggerChange(data, body.join(''));
     };
     stream.on('data', function (text) {
       if (done) {
         return;
       }
-      body += text;
-      if (body.length > MAX_FILE_LEN) {
+      len += text.length;
+      body.push(text);
+      if (len > MAX_FILE_LEN) {
         listener();
       }
     });

package/lib/util/index.js

@@ -126,7 +126,10 @@ exports.ADDITIONAL_HEAD = 'x-whistle-additional-headers';
 workerIndex = workerIndex >= 0 ? common.padLeft(config.workerIndex, 3) : '';
 
 
+exports.toString = common.toString;
 exports.sendRes = common.sendRes;
+exports.setDelayTimer = common.setDelayTimer;
+exports.clearDelayTimer = common.clearDelayTimer;
 exports.trimStr = trimStr;
 exports.THEME_STYLE = THEME_STYLE;
 exports.encodeNonLatin1Char = encodeNonLatin1Char;
@@ -747,24 +750,38 @@ exports.disableCSP = disableCSP;
 
 var interfaces = os.networkInterfaces();
 var hostname = os.hostname();
+var serverInfo;
 var simpleHostname = '';
 var cpus = os.cpus();
 var addressList = [];
 var usiTimer;
 
-function updateSystyemInfo() {
+function updateServerInfo() {
+  serverInfo = {ipv4: [], ipv6: []};
+  Object.keys(interfaces).forEach(function(ifname) {
+    interfaces[ifname].forEach(function (iface) {
+      if (iface.internal) {
+        return;
+      }
+      serverInfo[iface.family == 'IPv4' || iface.family === 4 ? 'ipv4' : 'ipv6'].push(iface.address);
+    });
+  });
+}
+
+function updateSysInfo() {
   clearTimeout(usiTimer);
   interfaces = os.networkInterfaces();
+  updateServerInfo();
   hostname = os.hostname();
   addressList = [];
   common.walkInterfaces(function (info) {
     addressList.push(info.address.toLowerCase());
   });
-  usiTimer = setTimeout(updateSystyemInfo, 36000);
+  usiTimer = setTimeout(updateSysInfo, 36000);
 }
 
-updateSystyemInfo();
-process.on('w2NetworkInterfacesChange', updateSystyemInfo);
+updateSysInfo();
+process.on('w2NetworkInterfacesChange', updateSysInfo);
 
 if (isString(hostname)) {
   simpleHostname = hostname.replace(/[^\w.-]+/g, '').substring(0, 20);
@@ -874,15 +891,13 @@ exports.removeClientId = function (headers) {
   delete headers[config.CLIENT_ID_HEADER];
 };
 
-function networkInterfaces() {
-  return interfaces;
-}
-
 function getHostname() {
   return hostname;
 }
 
-exports.networkInterfaces = networkInterfaces;
+exports.getServerInfo = function() {
+  return serverInfo;
+};
 exports.hostname = getHostname;
 
 function getProxyTunnelPath(req, isHttps) {
@@ -1613,35 +1628,35 @@ function getPipeIconvStream(headers) {
     pipeStream.addTail(iconv.encodeStream(charset));
   } else {
     pipeStream.addHead(function (res, next) {
-      var buffer, iconvDecoder;
-
-      res.on('data', function (chunk) {
-        buffer = buffer ? Buffer.concat([buffer, chunk]) : chunk;
-        resolveCharset(buffer);
-      });
-      res.on('end', resolveCharset);
-
-      function resolveCharset(chunk) {
-        if (!charset) {
-          if (chunk && buffer.length < 25600) {
+      var buf = [];
+      var len = 0;
+      var resolveCharset = function(chunk) {
+        if (chunk) {
+          len += chunk.length;
+          buf.push(chunk);
+          if (len < 25600) {
             return;
           }
-          charset = !buffer || isUtf8(buffer) ? 'utf8' : 'GB18030';
         }
-        if (!iconvDecoder) {
-          iconvDecoder = iconv.decodeStream(charset);
-          next(iconvDecoder);
+        res.removeListener('data', resolveCharset);
+        res.removeListener('end', resolveCharset);
+        buf = buf.length > 1 ? Buffer.concat(buf, len) : buf[0];
+        if (!buf) {
+          return next(res);
         }
-        if (buffer) {
-          iconvDecoder.write(buffer);
-          buffer = null;
-        }
-        !chunk && iconvDecoder.end();
-      }
+        charset = isUtf8(buf) ? 'utf8' : 'GB18030';
+        var decoder = iconv.decodeStream(charset);
+        decoder.write(buf);
+        next(res.pipe(decoder));
+        buf = null;
+      };
+
+      res.on('data', resolveCharset);
+      res.on('end', resolveCharset);
     });
 
     pipeStream.addTail(function (src, next) {
-      next(src.pipe(iconv.encodeStream(charset)));
+      next(charset ? src.pipe(iconv.encodeStream(charset)) : src);
     });
   }
 
@@ -2452,10 +2467,7 @@ exports.getPluginModal = function(conf) {
 };
 
 function getString(str) {
-  if (!isString(str)) {
-    return;
-  }
-  return str.trim();
+  return isString(str) ? str.trim() : '';
 }
 
 exports.getString = getString;
@@ -2625,19 +2637,6 @@ exports.getStaticDir = function (conf) {
   return toRelPath(staticDir.replace(/^\/+/, ''));
 };
 
-function toString(str) {
-  if (str != null) {
-    if (typeof str === 'string') {
-      return str;
-    }
-    try {
-      return JSON.stringify(str) || '';
-    } catch (e) {}
-  }
-  return '';
-}
-exports.toString = toString;
-
 var index = 0;
 
 exports.getReqId = function (now) {
@@ -2965,7 +2964,7 @@ exports.setResCors = function (data, cors, req) {
   } else if (autoComp) {
     var method = req.headers['access-control-request-method'];
     if (method) {
-      setHeader(data, 'access-control-allow-method', method);
+      setHeader(data, 'access-control-allow-methods', method);
     }
   }
 
@@ -3357,7 +3356,7 @@ exports.parseRange = function (req, size) {
   }
   var start = size;
   var end = -1;
-  range = range.split(',').forEach(function (item) {
+  range.split(',').forEach(function (item) {
     item = item.split('-');
     var s = parseInt(item[0], 10);
     var e = parseInt(item[1], 10);
@@ -3612,7 +3611,7 @@ function readOneChunk(stream, callback, timeout) {
   }
   var timer;
   var handler = function (chunk) {
-    timer && clearTimeout(timer);
+    clearTimeout(timer);
     stream.pause();
     stream.removeListener('data', handler);
     stream.removeListener('end', handler);
@@ -3686,10 +3685,9 @@ exports.getAuthBasic = function (auth) {
 
 exports.delay = function (time, callback) {
   if (time > 0) {
-    setTimeout(callback, time);
-  } else {
-    callback();
+    return setTimeout(callback, time);
   }
+  callback();
 };
 
 var F_HOST_RE = /\bhost\b/i;
@@ -4016,3 +4014,18 @@ exports.getFrameId = function() {
   }
   return Date.now() + '-00' + frameIndex;
 };
+
+exports.joinBuffer = function(buf, chunk) {
+  if (!buf) {
+    return chunk;
+  }
+  if (!Array.isArray(buf)) {
+    return [buf, chunk];
+  }
+  buf.push(chunk);
+  return buf;
+};
+
+exports.formatBuffer = function(buf, len) {
+  return Array.isArray(buf) ? Buffer.concat(buf, len) : buf;
+};

package/lib/util/parse-url.js

@@ -5,19 +5,21 @@ var URL_RE = /^([a-z0-9.+-]+:)\/\/([^/?#]*)(\/[^?#]*)?(\?[^#]*)?(#[\s\S]*)?$/i;
 var HOST_RE = /^(.+)(?::(\d*))$/;
 
 module.exports = function (url) {
-  if (!URL_RE.test(url)) {
+  var urlMatch = URL_RE.exec(url);
+  if (!urlMatch) {
     return parseUrl(url);
   }
-  var protocol = RegExp.$1;
-  var host = RegExp.$2;
-  var pathname = RegExp.$3 || '/';
-  var search = RegExp.$4;
-  var hash = RegExp.$5 || null;
+  var protocol = urlMatch[1];
+  var host = urlMatch[2];
+  var pathname = urlMatch[3] || '/';
+  var search = urlMatch[4] || '';
+  var hash = urlMatch[5] || null;
   var port = null;
   var hostname = host;
-  if (HOST_RE.test(host)) {
-    hostname = RegExp.$1;
-    port = RegExp.$2;
+  var hostMatch = HOST_RE.exec(host);
+  if (hostMatch) {
+    hostname = hostMatch[1];
+    port = hostMatch[2];
   }
 
   return {

package/lib/util/patch.js

@@ -4,6 +4,7 @@ var Socket = require('net').Socket;
 var http = require('http');
 var https = require('https');
 var hparser = require('hparser');
+var common = require('./common');
 
 var httpRequest = http.request;
 var httpsRequest = https.request;
@@ -133,12 +134,7 @@ http.request = function (options) {
         headers['x-whistle-policy'] = 'intercept';
         delete headers['X-Whistle-Policy'];
       }
-      headers = hparser.getRawHeaders(headers);
-      var rawData = 'CONNECT ' + tunnelPath + ' HTTP/1.1';
-      if (headers) {
-        rawData += '\r\n' + headers;
-      }
-      rawData += '\r\n\r\n';
+      var rawData = common.getHttpMeta('CONNECT', tunnelPath, 'HTTP/1.1', headers);
       if (res.statusCode === 200 && res.headers['x-whistle-allow-tunnel-ack']) {
         rawData = '1' + rawData;
       }

package/lib/util/speed-transform.js

@@ -1,9 +1,11 @@
 var Transform = require('pipestream').Transform;
-var util = require('util');
+var inherits = require('util').inherits;
+var setDelayTimer = require('./common').setDelayTimer;
 
-function SpeedTransform(options) {
+function SpeedTransform(options, req) {
   Transform.call(this);
   options = options || {};
+  this._req = req;
   var value = parseInt((options.speed * 1000) / 8);
   if (value > 0) {
     this._speed = value;
@@ -13,13 +15,22 @@ function SpeedTransform(options) {
   }
 }
 
-util.inherits(SpeedTransform, Transform);
+inherits(SpeedTransform, Transform);
 
-SpeedTransform.prototype._transform = function (chunk, encoding, callback) {
+var proto = SpeedTransform.prototype;
+
+proto._setTimeout = function (callback, delay) {
+  var timer = setTimeout(callback, delay);
+  if (this._req) {
+    setDelayTimer(this._req, timer);
+  }
+};
+
+proto._transform = function (chunk, encoding, callback) {
   var self = this;
   var cb = function () {
     if (chunk && self._speed) {
-      setTimeout(function () {
+      self._setTimeout(function () {
         callback(null, chunk);
       }, Math.round((chunk.length * 1000) / self._speed));
     } else {
@@ -30,7 +41,7 @@ SpeedTransform.prototype._transform = function (chunk, encoding, callback) {
   if (self._delay) {
     var delay = self._delay;
     self._delay = null;
-    return setTimeout(cb, delay);
+    return self._setTimeout(cb, delay);
   }
 
   cb();