whistle 2.10.5 → 2.10.7

package/lib/service/data-center.js

@@ -1,7 +1,6 @@
 var common = require('../util/common');
 var request = require('../util/http-mgr').request;
 var util = require('./util');
-var extend = require('extend');
 
 var requestData;
 var BASE_URL = 'https://' + common.SERVICE_HOST;
@@ -71,17 +70,6 @@ var saveComposeData = function(data) {
   if (!config.whistleToken || !Array.isArray(historyList)) {
     return;
   }
-  historyList.map(function(item) {
-    var newItem = extend({}, item);
-    var rawData = item.headers + '\r\n\r\n' + (item.base64 || item.body || '');
-    if (!item.dataHash && !item.base64Hash) {
-      item[item.base64 ? 'base64Hash' : 'dataHash'] = util.getHexHash(rawData);
-    }
-    delete newItem.headers;
-    delete newItem.body;
-    delete newItem.base64;
-    return newItem;
-  });
 };
 
 var descSorter = function(a, b) {

package/lib/service/generate-saz.js

@@ -124,11 +124,11 @@ module.exports = function (body, callback) {
   var zip = new AdmZip();
   sessions.map(function (item, index) {
     item.req.url = item.url;
-    var req = util.getReqRaw(item.req);
+    var req = util.getRawReq(item.req);
     var res =
       !item.res || item.res.statusCode == null
         ? Buffer.from('')
-        : util.getResRaw(item.res);
+        : util.getRawRes(item.res);
     var name = getName();
     item.index = index;
     zip.addFile('raw/' + name + '_c.txt', req);

package/lib/service/service.js

@@ -96,12 +96,13 @@ function getFilename(filename, time, count) {
 }
 
 function saveSessions(data, cb) {
-  var count = Array.isArray(data.sessions) && data.sessions.length;
+  var sessions = data.sessions;
+  var count = Array.isArray(sessions) && sessions.length;
   if (!count) {
     return cb();
   }
   limiter.push(function (done) {
-    gzip(JSON.stringify(data.sessions), function(err, buf) {
+    gzip(JSON.stringify(sessions), function(err, buf) {
       done();
       if (err) {
         return cb(err);
@@ -109,7 +110,7 @@ function saveSessions(data, cb) {
       var filename = getFilename(data.filename, 0, count);
       common.writeFile(path.join(SAVED_SESSIONS_PATH, filename), buf, function(e) {
         !e && saveData({ type: 'savedData', filename: filename, buffer: buf } );
-        cb(e);
+        cb(e, filename.substring(filename.indexOf('/') + 1));
       });
     });
   });
@@ -165,7 +166,7 @@ function getTempFiles(list, cb) {
       cb(result);
     }
   };
-  list = list.forEach(function(filename) {
+  list.forEach(function(filename) {
     filename = common.getTempFile(filename);
     if (!filename) {
       return execCb();
@@ -415,8 +416,8 @@ function sessionsHandler() {
     res.json({ec: 0});
   });
   app.post('/cgi-bin/saved/save', jsonParser, function(req, res) {
-    saveSessions(req.body, function(err) {
-      res.json({ec: err ? 2 : 0, em: err ? err.message || 'Error' : undefined});
+    saveSessions(req.body, function(err, filename) {
+      res.json({ec: err ? 2 : 0, em: err ? err.message || 'Error' : undefined, filename: filename});
     });
   });
   app.post('/cgi-bin/composer', jsonParser, composer.handleRequest);
@@ -504,7 +505,7 @@ function sessionsHandler() {
           common.sendGzip(req, res, data);
         });
       } catch (e) {
-        common.sendRes(req, 500, e.stack);
+        common.sendRes(res, 500, e.stack);
       }
     }
   );

package/lib/service/util.js

@@ -77,7 +77,7 @@ function getBodyBuffer(data) {
   }
 }
 
-function getReqRaw(req) {
+function getRawReq(req) {
   removeEncodingFields(req.headers);
   var headers = getHeadersRaw(req.headers, req.rawHeaderNames);
   var url = String(req.url || '').replace(/^ws/, 'http');
@@ -85,9 +85,9 @@ function getReqRaw(req) {
   return decodeRaw(headers, req);
 }
 
-exports.getReqRaw = getReqRaw;
+exports.getRawReq = getRawReq;
 
-function getResRaw(res) {
+function getRawRes(res) {
   removeEncodingFields(res.headers);
   var headers = getHeadersRaw(res.headers, res.rawHeaderNames);
   var statusCode = res.statusCode === 'aborted' ? 502 : res.statusCode;
@@ -98,7 +98,7 @@ function getResRaw(res) {
   return decodeRaw(headers, res);
 }
 
-exports.getResRaw = getResRaw;
+exports.getRawRes = getRawRes;
 
 var BODY_SEP = Buffer.from('\r\n\r\n');
 
@@ -425,23 +425,13 @@ function parseFrames(res, content, callback) {
 exports.parseFrames = parseFrames;
 
 var MAX_SESSIONS_FILES = 2000;
-var SAVED_SESSIONS_FILE_RE = /_([1-9]\d*)_(\d+)$/;
 var DIR_RE = /^\d{6}$/;
 
-exports.SAVED_SESSIONS_FILE_RE = SAVED_SESSIONS_FILE_RE;
-
 function getFileList(files) {
   var result = [];
   files.forEach(function(file, i) {
-    if (SAVED_SESSIONS_FILE_RE.test(file)) {
-      var count = RegExp.$1;
-      var time = RegExp.$2;
-      result.push({
-        filename: file.slice(0,  - time.length - count.length - 2),
-        count: +count,
-        time: +time
-      });
-    }
+    file = common.getSavedFileItem(file);
+    file && result.push(file);
   });
   return result;
 }

package/lib/socket-mgr.js

@@ -584,7 +584,7 @@ function handleWsReceive(ctx, resTrans, receiveStatus, handleTransform, frameCtx
 
 function getContext(req, res, hasEvent, sendStatus, receiveStatus, noDecompress) {
   var reqId = req.reqId;
-  var ctx = (conns[reqId] = {
+  var ctx = conns[reqId] = {
     customParser: req.customParser,
     req: req,
     res: res,
@@ -601,7 +601,7 @@ function getContext(req, res, hasEvent, sendStatus, receiveStatus, noDecompress)
     setReceiveStatus: function (status) {
       setConnStatus(ctx, status, receiveStatus);
     }
-  });
+  };
   initStatus(ctx, req.enable);
   return ctx;
 }
@@ -718,9 +718,21 @@ exports.handleUpgrade = function (req, res) {
           hasEvent && proxy.emit('wsRequest', url);
           cb(null, chunk);
         };
-        if (hideFrame && !frameCtx) {
+        var ctx = false;
+        if (!hide || frameCtx) {
+          ctx = getContext(
+            req,
+            res,
+            hasEvent,
+            sendStatus,
+            receiveStatus,
+            noDecompress
+          );
+          ctx.hideFrame = req.hideFrame = hideFrame;
+        }
+        options.ctx = ctx;
+        if (!ctx || customParser) {
           reqTrans._transform = resTrans._transform = handleTransform;
-          options.ctx = false;
         } else {
           reqTrans.headers = resTrans.headers = res.headers;
           if (reqWrite) {
@@ -731,15 +743,6 @@ exports.handleUpgrade = function (req, res) {
             resWrite.headers = res.headers;
             req.pipeWriter = resWrite;
           }
-          var ctx = options.ctx = (conns[reqId] = getContext(
-            req,
-            res,
-            hasEvent,
-            sendStatus,
-            receiveStatus,
-            noDecompress
-          ));
-          ctx.hideFrame = req.hideFrame = hideFrame;
           handleWsSend(ctx, reqTrans, sendStatus, handleTransform, frameCtx);
           handleWsReceive(ctx, resTrans, receiveStatus, handleTransform, frameCtx);
           cleanCacheFrames(ctx, options);
@@ -798,9 +801,14 @@ exports.handleConnect = function (req, res, isUpgrade) {
         if (req._hasClosed) {
           return;
         }
-        if (hideFrame && !frameCtx) {
+        var ctx = false;
+        if (!hide || frameCtx) {
+          ctx = getContext(req, res, hasEvent, sendStatus, receiveStatus);
+          ctx.hideFrame = req.hideFrame = hideFrame;
+        }
+        options.ctx = ctx;
+        if (!ctx || customParser) {
           reqTrans._transform = resTrans._transform = handleTransform;
-          options.ctx = false;
         } else {
           req.wsExts = res.wsExts = reqTrans.wsExts = resTrans.wsExts = '';
           if (reqWrite) {
@@ -811,8 +819,6 @@ exports.handleConnect = function (req, res, isUpgrade) {
             resWrite.wsExts = '';
             req.pipeWriter = resWrite;
           }
-          var ctx = options.ctx = getContext(req, res, hasEvent, sendStatus, receiveStatus);
-          ctx.hideFrame = req.hideFrame = hideFrame;
           if (reqRead && reqWrite) {
             handleWsSend(ctx, reqTrans, sendStatus, handleTransform, frameCtx, eventName);
           } else {

package/lib/tunnel.js

@@ -23,7 +23,6 @@ var X_RE = /^x/;
 var STATUS_CODES = require('http').STATUS_CODES || {};
 var getRawHeaderNames = hparser.getRawHeaderNames;
 var formatHeaders = hparser.formatHeaders;
-var getRawHeaders = hparser.getRawHeaders;
 var CRONET_RE = /\bCronet\b/;
 var SYS_HOST_RE = /^(?:[^/:]+\.)?(?:apple|cdn-apple|icloud|office|office365|mzstatic|apple-cloudkit|microsoft|parallels)\.(?:com|cn|com\.cn)$/;
 var TUNNEL_RE = /^tunnel:\/\//;
@@ -104,6 +103,7 @@ function tunnelProxy(server, proxy, type) {
           testId: req._testId,
           startTime: now,
           dnsTime: now,
+          servername: reqSocket.useSNI === hostname ? undefined : reqSocket.useSNI,
           captureError: true,
           rules: req.rules,
           req: {
@@ -382,7 +382,7 @@ function tunnelProxy(server, proxy, type) {
                 var reqDelay = util.getMatcherValue(_rules.reqDelay);
                 data.requestTime = data.dnsTime = Date.now();
                 if (reqDelay > 0) {
-                  setTimeout(handleConnect, reqDelay);
+                  util.setDelayTimer(reqSocket, setTimeout(handleConnect, reqDelay));
                 } else {
                   handleConnect();
                 }
@@ -404,8 +404,7 @@ function tunnelProxy(server, proxy, type) {
                 var proxyUrl;
                 if (tunnelPort) {
                   proxyUrl = 'proxy://127.0.0.1:' + tunnelPort;
-                  reqSocket.customParser = req.customParser =
-                    util.getParserStatus(req);
+                  reqSocket.customParser = req.customParser = util.getParserStatus(req);
                   pluginMgr.addSessionInfo(req, _rules);
                   headers[config.PLUGIN_HOOK_NAME_HEADER] =
                     config.PLUGIN_HOOKS.TUNNEL;
@@ -553,7 +552,7 @@ function tunnelProxy(server, proxy, type) {
                           }
                         };
                         if (reqDelay > 0) {
-                          setTimeout(connectProxy, reqDelay);
+                          util.setDelayTimer(reqSocket, setTimeout(connectProxy, reqDelay));
                         } else {
                           connectProxy();
                         }
@@ -687,7 +686,7 @@ function tunnelProxy(server, proxy, type) {
             };
             var resDelay = util.getMatcherValue(_rules.resDelay);
             if (resDelay > 0) {
-              setTimeout(handleEstablished, resDelay);
+              util.setDelayTimer(reqSocket, setTimeout(handleEstablished, resDelay));
             } else {
               handleEstablished();
             }
@@ -770,23 +769,14 @@ function tunnelProxy(server, proxy, type) {
                   util.deleteResHeaders(req, resHeaders);
                   code = util.getMatcherValue(reqRules.replaceStatus) || code;
                   var isSuccess = code == 200;
-                  var message =
-                    isSuccess
-                      ? 'Connection Established'
-                      : STATUS_CODES[code] || 'unknown';
-                  var statusLine = ['HTTP/1.1', code, message].join(' ');
+                  var message = isSuccess ? 'Connection Established' : STATUS_CODES[code] || 'unknown';
                   var curHeaders = resHeaders;
-                  var rawData = statusLine;
                   if (req.fromComposer) {
                     curHeaders = extend({}, resHeaders);
                     curHeaders['x-whistle-req-id'] = req.reqId;
                     util.setFramesMode(curHeaders, isSuccess && inspect);
                   }
-                  curHeaders = getRawHeaders(formatHeaders(curHeaders, rawHeaderNames));
-                  if (curHeaders) {
-                    rawData += '\r\n' + curHeaders;
-                  }
-                  rawData += '\r\n\r\n';
+                  var rawData = common.getHttpMeta('HTTP/1.1', code, message, curHeaders, rawHeaderNames);
                   try {
                     if (code != 200) {
                       reqSocket.end(rawData, cb);

package/lib/upgrade.js

@@ -6,9 +6,7 @@ var util = require('./util');
 var config = require('./config');
 var common = require('./util/common');
 
-var formatHeaders = hparser.formatHeaders;
 var getRawHeaderNames = hparser.getRawHeaderNames;
-var getRawHeaders = hparser.getRawHeaders;
 var PLUGIN_PATH_RE =
   /^\/(\.\.\.whistle-path\.5b6af7b9884e1165\.\.\.\/\/\/)?(whistle|plugin)\.([^/?#]+)\/?/;
 
@@ -105,12 +103,10 @@ function upgradeHandler(req, socket) {
   util.addTunnelData(socket, headers);
   socket._clientId = util.getComposerClientId(headers);
   var getBuffer = function (method, newHeaders, path) {
-    var rawData = (method || 'GET') + ' ' + (path || socket.url || req.url) + ' HTTP/1.1';
-    newHeaders = formatHeaders(newHeaders || headers, req.rawHeaders);
-    if (newHeaders = getRawHeaders(newHeaders)) {
-      rawData += '\r\n' + newHeaders;
-    }
-    return Buffer.from(rawData + '\r\n\r\n');
+    method = method || 'GET';
+    path = path || socket.url || req.url;
+    newHeaders = newHeaders || headers;
+    return Buffer.from(common.getHttpMeta(method, path, 'HTTP/1.1', newHeaders, req.rawHeaders));
   };
   var sniPlugin = headers[util.SNI_PLUGIN_HEADER];
   if (sniPlugin) {

package/lib/util/common.js

@@ -12,6 +12,7 @@ var json5 = require('json5');
 var extend = require('extend');
 var tls = require('tls');
 var https = require('https');
+var hparser = require('hparser');
 var zlibx = require('./zlib');
 var pkgConf = require('../../package.json');
 var isUtf8 = require('./is-utf8');
@@ -72,10 +73,22 @@ var TOKEN = '\r\u0000\n\u0003\r';
 var PROP_SEP_RE = /(\\*)([|&]|\\[stnrfv])/g;
 var TEMP_FILE_RE = /^(?:\/?temp\/)?([\da-f]{64})(?:\.[\w.-]+)?$/;
 var TEMP_FILES_PATH = path.join(getWhistlePath(), 'temp_files');
+var SAVED_SESSIONS_FILE_RE = /_([1-9]\d*)_(\d+)$/;
+
+exports.getSavedFileItem = function(filename) {
+  var result = SAVED_SESSIONS_FILE_RE.exec(filename);
+  return result && {
+    count: +result[1],
+    time: +result[2],
+    filename: filename.slice(0,  - result[1].length - result[2].length - 2)
+  };
+};
 
 
 exports.WHISTLE_UID_HEADER = '_x-whistle-uid_';
 
+exports.isUtf8 = isUtf8;
+
 exports.getTempFile = function(str) {
   var match = TEMP_FILE_RE.exec(str);
   return match && match[1];
@@ -670,18 +683,29 @@ exports.isGroup = function(name) {
 };
 
 var UTF8_RE = /^utf-?8$/i;
+
+function getUtf8Buf(buffer, charset) {
+  charset = isString(charset) ? charset.trim() : '';
+  if (UTF8_RE.test(charset) || isUtf8(buffer)) {
+    return buffer;
+  }
+  try {
+    buffer = iconv.encode(buffer, charset || 'GB18030');
+  } catch (e) {}
+  return buffer;
+}
+
+exports.getUtf8Buf = getUtf8Buf;
+
+
 function bufferToString(buffer, encoding) {
-  if (typeof encoding === 'string') {
-    encoding = encoding.trim();
-  } else {
-    encoding = null;
+  if (!buffer) {
+    return '';
   }
-  if (Buffer.isBuffer(buffer) && !UTF8_RE.test(encoding) && !isUtf8(buffer)) {
-    try {
-      buffer = iconv.encode(buffer, encoding || 'GB18030');
-    } catch (e) {}
+  if (Buffer.isBuffer(buffer)) {
+    buffer = getUtf8Buf(buffer, encoding);
   }
-  return String(buffer || '');
+  return String(buffer);
 }
 
 exports.bufferToString = bufferToString;
@@ -697,9 +721,9 @@ function isUrlEncoded(req) {
 exports.isUrlEncoded = isUrlEncoded;
 
 function readStream(stream, callback) {
-  var buffer = null;
+  var buffer = [];
   stream.on('data', function(chunk) {
-    buffer = buffer ? Buffer.concat([buffer, chunk]) : chunk;
+    buffer.push(chunk);
   });
   stream.once('end', function() {
     var buf;
@@ -708,6 +732,7 @@ function readStream(stream, callback) {
     var err;
     var jsonErr;
     stream.zlib = zlibx;
+    buffer = buffer.length > 1 ? Buffer.concat(buffer) : (buffer[0] || null);
     var encoding = stream.headers && stream.headers['content-encoding'];
     var getBuffer = function(cb) {
       if (err || buf !== undefined) {
@@ -851,9 +876,24 @@ exports.getHostPort = function(str) {
   }
 };
 
+function clearDelayTimer(socket) {
+  var timer = socket._w2Timer_;
+  if (timer) {
+    clearTimeout(timer);
+    socket._w2Timer_ = null;
+  }
+}
+
+exports.clearDelayTimer = clearDelayTimer;
+exports.setDelayTimer = function (socket, id) {
+  socket._w2Timer_ = id;
+};
+
+
 exports.onSocketEnd = function(socket, callback) {
   var execCallback = function (err) {
     socket._hasError = true;
+    clearDelayTimer(socket);
     if (callback) {
       callback(err);
       callback = null;
@@ -969,6 +1009,10 @@ function isString(str) {
 
 exports.isString = isString;
 
+exports.isStr = function(str) {
+  return typeof str === 'string';
+};
+
 function isSafeNumStr(str) {
   if (str == '0') {
     return true;
@@ -1428,6 +1472,18 @@ exports.sendGzip = function (req, res, data) {
   });
 };
 
+exports.toString = function (str) {
+  if (str != null) {
+    if (typeof str === 'string') {
+      return str;
+    }
+    try {
+      return JSON.stringify(str) || '';
+    } catch (e) {}
+  }
+  return '';
+};
+
 exports.sendGzipText = function(req, res, headers, text, gzipText) {
   if (!text || !canGzip(req) || (!gzipText && text.length < GZIP_THRESHOLD)) {
     headers && res.writeHead(200, headers);
@@ -2045,3 +2101,14 @@ var COMMENT_RE = /#[^\r\n]*/g;
 exports.removeComment = function(text) {
   return text.replace(COMMENT_RE, '').trim();
 };
+
+function getHttpMeta(f1, f2, f3, headers, rawHeaders) {
+  var statusLine = [f1, f2, f3].join(' ');
+  if (headers) {
+    headers = hparser.formatHeaders(headers, rawHeaders);
+    headers = hparser.getRawHeaders(headers);
+  }
+  return (headers ? statusLine + '\r\n' + headers : statusLine) + '\r\n\r\n';
+}
+
+exports.getHttpMeta = getHttpMeta;