whistle 2.10.9 → 2.10.11

package/lib/https/index.js

@@ -8,6 +8,7 @@ var EventEmitter = require('events').EventEmitter;
 var LRU = require('lru-cache');
 var checkSNI = require('sni');
 var util = require('../util');
+var common = require('../util/common');
 var extend = require('extend');
 var config = require('../config');
 var rules = require('../rules');
@@ -17,7 +18,7 @@ var hparser = require('hparser');
 var properties = require('../rules/util').properties;
 var ca = require('./ca');
 var loadCert = require('./load-cert');
-var h2Consts = config.enableH2 ? require('http2').constants : {};
+var h2Consts = require('http2').constants;
 
 var STATUS_CODES = http.STATUS_CODES || {};
 var getRawHeaders = hparser.getRawHeaders;
@@ -1108,36 +1109,39 @@ var handlers = {
     server.emit('upgrade', req, socket);
   }
 };
-var h2Handlers = config.enableH2
-  ? {
-    request: toHttp1,
-    upgrade: handlers.upgrade
-  }
-  : handlers;
+var h2Handlers = {
+  request: toHttp1,
+  upgrade: handlers.upgrade
+};
 
 var HTTP_RE = /^(\w+)\s+(\S+)\s+HTTP\/1.\d$/im;
 var HTTP2_RE = /^PRI\s\*\s+HTTP\/2.0$/im;
 var CONNECT_RE = /^CONNECT$/i;
 
-function addClientInfo(socket, chunk, statusLine, clientIp, clientPort) {
-  var len = Buffer.byteLength(statusLine);
-  chunk = chunk.slice(len);
-  statusLine +=
-    '\r\n' +
-    config.CLIENT_INFO_HEADER +
-    ': ' +
-    clientIp +
-    ',' +
-    clientPort +
-    ',' +
-    socket._remoteAddr +
-    ',' +
-    socket._remotePort;
+function getTunnelHeaders(socket, statusLine, clientIp, clientPort, newHost) {
+  var clientInfo = clientIp + ',' + clientPort + ',' + socket._remoteAddr + ',' + socket._remotePort;
+  var headers = newHost ? {} : null;
+  if (headers) {
+    headers.host = newHost;
+    headers[config.CLIENT_INFO_HEADER] = clientInfo;
+  } else {
+    statusLine += '\r\n' + config.CLIENT_INFO_HEADER + ': ' + clientInfo;
+  }
   var tunnelData = getTunnelData(socket, clientIp, clientPort);
   if (tunnelData) {
-    statusLine += '\r\n' + util.TEMP_TUNNEL_DATA_HEADER + ': ' + encodeURIComponent(JSON.stringify(tunnelData));
+    tunnelData = encodeURIComponent(JSON.stringify(tunnelData));
+    if (headers) {
+      headers[util.TEMP_TUNNEL_DATA_HEADER] = tunnelData;
+    } else {
+      statusLine += '\r\n' + util.TEMP_TUNNEL_DATA_HEADER + ': ' + tunnelData;
+    }
   }
-  return Buffer.concat([Buffer.from(statusLine), chunk]);
+  return headers || Buffer.from(statusLine);
+}
+
+function addClientInfo(socket, chunk, statusLine, clientIp, clientPort) {
+  chunk = chunk.slice(Buffer.byteLength(statusLine));
+  return Buffer.concat([getTunnelHeaders(socket, statusLine, clientIp, clientPort), chunk]);
 }
 
 module.exports = function (socket, next, isWebPort) {
@@ -1177,6 +1181,12 @@ module.exports = function (socket, next, isWebPort) {
     headersStr = chunk && chunk.toString();
     var isHttp = chunk && HTTP_RE.test(headersStr);
     var statusLine = isHttp && RegExp['$&'];
+    var handlePipe = function() {
+      reqSocket.on('error', destroy);
+      reqSocket.write(chunk);
+      reqSocket.pipe(socket).pipe(reqSocket);
+      socket.resume();
+    };
     if (isHttp && CONNECT_RE.test(RegExp.$1)) {
       chunk = addClientInfo(socket, chunk, statusLine, clientIp, clientPort);
       util.connect(
@@ -1189,10 +1199,7 @@ module.exports = function (socket, next, isWebPort) {
             if (err || socket._hasError) {
               return destroy(err);
             }
-            reqSocket.on('error', destroy);
-            reqSocket.write(chunk);
-            reqSocket.pipe(socket).pipe(reqSocket);
-            socket.resume();
+            handlePipe();
           }
         );
       return;
@@ -1212,105 +1219,115 @@ module.exports = function (socket, next, isWebPort) {
             addClientInfo(socket, chunk, statusLine, clientIp, clientPort)
           );
       }
-    } else if (isEnable('forHttp') || disable.captureHttps) {
+      return;
+    }
+    if (isEnable('forHttp') || disable.captureHttps) {
       return next(chunk);
-    } else {
-      var isHttpH2 = HTTP2_RE.test(headersStr);
-      if (!isHttpH2 && chunk[0] != 22) {
-        return next(chunk);
-      }
-      var useSNI, domain, serverKey;
-      var handleConnect = function (port) {
-        var promise =
+    }
+    var isHttpH2 = HTTP2_RE.test(headersStr);
+    if (!isHttpH2 && chunk[0] != 22) {
+      return next(chunk);
+    }
+    var useSNI, domain, serverKey;
+    var handleConnect = function (port) {
+      var promise =
             !isHttpH2 && !useSNI && serverAgent.existsServer(serverKey);
-        var httpsServer = promise && promise.cert && promise.server;
-        if (httpsServer && httpsServer.setSecureContext) {
-          var cert = serverAgent.createCertificate(domain);
-          if (
-              cert.key !== promise.cert.key ||
-              cert.cert !== promise.cert.cert
-            ) {
-            try {
-              cert._ctx = cert._ctx || ca.createSecureContext(cert);
-              httpsServer.setSecureContext(cert._ctx);
-              promise.cert = cert;
-            } catch (e) {}
-          }
+      var oldCert = promise && promise.cert;
+      var httpsServer = oldCert && promise.server;
+      if (httpsServer && httpsServer.setSecureContext) {
+        var cert = serverAgent.createCertificate(domain);
+        if (cert.key !== oldCert.key || cert.cert !== oldCert.cert) {
+          try {
+            cert._ctx = cert._ctx || ca.createSecureContext(cert);
+            httpsServer.setSecureContext(cert._ctx);
+            promise.cert = cert;
+          } catch (e) {}
         }
-        util.connect(
-          {
-            port: port,
-            host: LOCALHOST,
-            localAddress: LOCALHOST
-          },
+      }
+      util.connect(
+        {
+          port: port,
+          host: LOCALHOST,
+          localAddress: LOCALHOST
+        },
             function (err, s) {
               reqSocket = s;
               if (err || socket._hasError) {
                 return destroy(err);
               }
-              reqSocket.on('error', destroy);
-              socket._w2TunnelKey = reqSocket.localPort + ':' + reqSocket.remotePort;
-              tunnelTmplData.set(socket._w2TunnelKey,  getTunnelData(socket, clientIp, clientPort, isHttpH2));
-              reqSocket.write(chunk);
-              reqSocket.pipe(socket).pipe(reqSocket);
-              socket.resume();
+              var key = reqSocket.localPort + ':' + reqSocket.remotePort;
+              socket._w2TunnelKey = key;
+              tunnelTmplData.set(key, getTunnelData(socket, clientIp, clientPort, isHttpH2));
+              handlePipe();
             }
           );
-      };
-      var useNoSNIServer = function () {
-        serverKey = requestCert ? ':' + domain : domain;
-        serverAgent.createServer(serverKey, handlers, handleConnect, 0, 0);
-      };
+    };
+    var useNoSNIServer = function () {
+      serverKey = requestCert ? ':' + domain : domain;
+      serverAgent.createServer(serverKey, handlers, handleConnect, 0, 0);
+    };
 
-      var handleRequest = function () {
-        if (useSNI) {
-          var disableH2 = !properties.isEnableHttp2();
-          if (disable.http2) {
-            disableH2 = true;
-          } else if (enable.http2) {
-            disableH2 = false;
-          }
-          getSNIServer(h2Handlers, handleConnect, disableH2, requestCert);
-        } else if (isHttpH2) {
-          getHttp2Server(h2Handlers, handleConnect);
-        } else {
-          useNoSNIServer();
+    var handleRequest = function () {
+      if (useSNI) {
+        var disableH2 = !properties.isEnableHttp2();
+        if (disable.http2) {
+          disableH2 = true;
+        } else if (enable.http2) {
+          disableH2 = false;
         }
-      };
-      if (isHttpH2) {
-        return handleRequest();
+        getSNIServer(h2Handlers, handleConnect, disableH2, requestCert);
+      } else if (isHttpH2) {
+        getHttp2Server(h2Handlers, handleConnect);
+      } else {
+        useNoSNIServer();
       }
-      useSNI = checkSNI(chunk);
-      var servername = useSNI || socket.tunnelHostname;
-      if (
-          !servername ||
+    };
+    if (isHttpH2) {
+      return handleRequest();
+    }
+    useSNI = checkSNI(chunk);
+    var servername = useSNI || socket.tunnelHostname;
+    if (useSNI && socket._useSniAsTunnelDomain && useSNI !== socket.tunnelHostname) {
+      var tunnelPort = socket.tunnelPort || 443;
+      return common.connectInner({
+        host: useSNI,
+        port: tunnelPort,
+        proxyHost: config.host || LOCALHOST,
+        proxyPort: config.port,
+        headers: getTunnelHeaders(socket, statusLine, clientIp, clientPort, useSNI + ':' + tunnelPort)
+      }, function(s) {
+        reqSocket = s;
+        handlePipe();
+      }, config).on('error', destroy);
+    }
+    if (
+          !servername || socket._noCaptureW2 ||
           (useSNI ? disable.captureSNI : (disable.captureNoSNI || (net.isIP(servername) && !isCaptureIp()))) ||
           (socket.useProxifier && !ca.existsCustomCert(servername))
         ) {
-        return next(chunk);
-      }
-      var requestCert =
+      return next(chunk);
+    }
+    var requestCert =
           (enable.clientCert || enable.requestCert) &&
           !disable.clientCert &&
           !disable.requestCert;
-      domain = getDomain(servername);
-      socket.curUrl = socket.fullUrl = 'https://' + servername;
-      socket.useSNI = useSNI;
-      socket.serverName = socket.servername = servername;
-      socket.commonName = domain;
-      loadCert(socket, function (cert) {
-        if (socket._hasError) {
-          return destroy();
-        }
-        if (cert === false) {
-          return next(chunk);
-        }
-        if (cert) {
-          domain = servername;
-        }
-        handleRequest();
-      });
-    }
+    domain = getDomain(servername);
+    socket.curUrl = socket.fullUrl = 'https://' + servername;
+    socket.useSNI = useSNI;
+    socket.serverName = socket.servername = servername;
+    socket.commonName = domain;
+    loadCert(socket, function (cert) {
+      if (socket._hasError) {
+        return destroy();
+      }
+      if (cert === false) {
+        return next(chunk);
+      }
+      if (cert) {
+        domain = servername;
+      }
+      handleRequest();
+    });
   },
     isWebPort ? 0 : TIMEOUT
   );

package/lib/inspectors/data.js

@@ -197,6 +197,7 @@ function emitDataEvents(req, res, proxy) {
   var setRequestTime = function (time) {
     data.requestTime = time;
     data.connectTime = req._connectTime;
+    reqData.size = reqData.size || 0;
   };
   var setEndTime = function () {
     if (data.requestTime || !requestTime) {
@@ -205,6 +206,7 @@ function emitDataEvents(req, res, proxy) {
     } else if (endTime == null) {
       endTime = endTime || Date.now();
     }
+    resData.size = resData.size || 0;
   };
   var updateVersion = function () {
     if (req.useH2 != null) {
@@ -419,7 +421,7 @@ function emitDataEvents(req, res, proxy) {
   }
 
   req.once('dest', function (_req) {
-    _req.once('finish', function () {
+    util.onWriteEnd(_req, function () {
       if (!requestTime) {
         setRequestTime(Date.now());
       }
@@ -441,15 +443,19 @@ function emitDataEvents(req, res, proxy) {
   req.once('error', handleError);
   res.once('error', handleError);
   res.once('close', handleError);
-  res.once('finish', setEndTime);
+  util.onWriteEnd(res, setEndTime);
 
   function handleError(err) {
     req._hasError = true;
     util.clearDelayTimer(req);
-    if (!data || endTime || data.endTime || (data.responseTime && !err)) {
+    if (!data || endTime || data.endTime) {
       return;
     }
+    // 触发 endTime
     !endTime && setEndTime();
+    if (data.responseTime && !err) {
+      return;
+    }
     delete data.abort;
     // 不能乱优化，跟 var msg = err && err.message; if (msg !== 'Aborted' && msg !== 'aborted') 不一样
     if (err && err.message !== 'Aborted' && err.message !== 'aborted') {

package/lib/inspectors/log.js

@@ -7,7 +7,6 @@ var logScriptFile = path.join(config.ASSESTS_PATH, 'js/log.js');
 var logScript = fs.readFileSync(logScriptFile, { encoding: 'utf8' });
 var logScriptTag = '<script>' + logScript + '</script>\r\n';
 var logHtmlScript = '<!DOCTYPE html>\r\n' + logScriptTag;
-var LOG_ID_RE = /^log:\/\/(\{[^\s]{1,36}\}|[^/\\{}()<>\s]{1,36})$/;
 
 function wrapScript(script, isHtml) {
   return isHtml
@@ -53,39 +52,21 @@ module.exports = function (req, res) {
         transform._transform = function (chunk, _, callback) {
           if (!added) {
             added = true;
-            var logId = '';
-            var isValue;
-            if (LOG_ID_RE.test(log.matcher)) {
-              logId = RegExp.$1;
-              if (logId[0] === '{') {
-                logId = logId.slice(1, -1);
-                isValue = true;
-              }
-              try {
-                logId = encodeURIComponent(logId);
-              } catch (e) {}
-            }
-            util.getRuleValue(
-              logId && !isValue ? null : log,
-              function (script) {
-                topScript = topScript.replace('$LOG_ID', logId);
-                var buf = [util.toBuffer(topScript)];
-                userScript = script || null;
-                if (userScript) {
-                  userScript = util.toBuffer(wrapScript(userScript, isHtml));
-                  if (isHtml || !chunk) {
-                    buf.push(userScript);
-                    userScript = null;
-                  }
+            var logId = util.removeProtocol(log.matcher, true);
+            util.getRuleValue(!logId || /^[^/\\{}()<>]+$/.test(logId) ? null : log, function (script) {
+              topScript = topScript.replace('$LOG_ID', logId && util.encodeURIComponent(logId));
+              var buf = [util.toBuffer(topScript)];
+              userScript = script || null;
+              if (userScript) {
+                userScript = util.toBuffer(wrapScript(userScript, isHtml));
+                if (isHtml || !chunk) {
+                  buf.push(userScript);
+                  userScript = null;
                 }
-                chunk && buf.push(chunk);
-                callback(null, Buffer.concat(buf));
-              },
-              null,
-              null,
-              null,
-              req
-            );
+              }
+              chunk && buf.push(chunk);
+              callback(null, Buffer.concat(buf));
+            }, null, null, null, req);
           } else {
             callback(null, chunk || userScript);
           }

package/lib/inspectors/req.js

@@ -502,8 +502,7 @@ module.exports = function (req, res, next) {
       }
 
       if (reqRules.method) {
-        var method = util.getMatcherValue(reqRules.method);
-        data.method = method;
+        data.method = util.getMatcherValue(reqRules.method);
       }
 
       if (reqRules.reqCharset) {
@@ -511,13 +510,11 @@ module.exports = function (req, res, next) {
       }
 
       if (reqRules.referer) {
-        var referer = util.getMatcherValue(reqRules.referer);
-        util.setHeader(data, 'referer', referer);
+        util.setHeader(data, 'referer', util.getMatcherValue(reqRules.referer));
       }
 
       if (reqRules.ua) {
-        var ua = util.getMatcherValue(reqRules.ua);
-        util.setHeader(data, 'user-agent', ua);
+        util.setHeader(data, 'user-agent', util.getMatcherValue(reqRules.ua));
       }
 
       var reqSpeed = util.getMatcherValue(reqRules.reqSpeed);

package/lib/inspectors/res.js

@@ -168,9 +168,6 @@ function readFirstChunk(req, res, src, cb) {
 }
 
 function checkH2(req, isHttps) {
-  if (!config.enableH2) {
-    return;
-  }
   req.useH2 = req.isH2;
   var d = req.disable;
   var e = req.enable;
@@ -798,7 +795,7 @@ module.exports = function (req, res, next) {
     var useFrames;
     res.headers = req.resHeaders = headers;
     res._originEncoding = headers['content-encoding'];
-    req.statusCode = _res.statusCode;
+    req.statusCode = res.statusCode = _res.statusCode;
     if (_res.rawHeaderNames) {
       res.rawHeaderNames = _res.rawHeaderNames;
     } else {
@@ -1243,7 +1240,7 @@ module.exports = function (req, res, next) {
                               _res.statusCode,
                               formatHeaders(curHeaders, rawNames)
                             );
-                            util.onResEnd(_res, function () {
+                            util.onReadEnd(_res, function () {
                               var trailers = _res.trailers;
                               if (
                                 !res.chunkedEncoding ||

package/lib/inspectors/rules.js

@@ -61,6 +61,15 @@ function setupRules(req, next) {
   });
 }
 
+function handleCodec(socket, enTrans, deTrans, handleError) {
+  enTrans.on('error', handleError);
+  deTrans.on('error', handleError);
+  deTrans.once('end', function() {
+    util.endSocket(socket);
+  });
+  enTrans.pipe(socket).pipe(deTrans);
+}
+
 function getDecoder(obj) {
   return function (socket, callback) {
     var encoding = obj._originEncoding;
@@ -97,9 +106,7 @@ function getDecoder(obj) {
         delete obj.headers['content-length'];
         var enTrans = getEncodeTransform();
         var deTrans = getDecodeTransform();
-        enTrans.pipe(socket).pipe(deTrans);
-        enTrans.on('error', handleError);
-        deTrans.on('error', handleError);
+        handleCodec(socket, enTrans, deTrans, handleError);
         if (decoder) {
           decoder.pipe(enTrans);
         } else {
@@ -127,9 +134,7 @@ function getEncoder(obj, req) {
       delete obj.headers['content-length'];
       var enTrans = getEncodeTransform();
       var deTrans = getDecodeTransform();
-      enTrans.on('error', handleError);
-      deTrans.on('error', handleError);
-      enTrans.pipe(socket).pipe(deTrans);
+      handleCodec(socket, enTrans, deTrans, handleError);
       socket = enTrans;
       if (encoder) {
         deTrans.pipe(encoder);

package/lib/inspectors/weinre.js

@@ -36,7 +36,7 @@ module.exports = function (req, res) {
       }
       !util.isEnable(req, 'keepAllCSP') && util.disableCSP(_res.headers);
       !req._customCache && util.disableResStore(_res.headers);
-      var name = util.getPath(util.rule.getMatcher(weinre));
+      var name = util.removeProtocol(weinre.matcher, true);
       var transform = new Transform();
       transform._transform = function (chunk, _, callback) {
         if (!chunk) {

package/lib/plugins/index.js

@@ -289,8 +289,8 @@ pluginMgr.on('uninstall', function (result) {
   });
 });
 
-function formatMTime(mtime) {
-  return '[' + (new Date(mtime).toLocaleTimeString()) + ']';
+function showUpdateInfo(action, name) {
+  return '[' + (new Date().toLocaleTimeString()) + '] ' + action + ' ' + name.slice(0, -1);
 }
 
 function showVerbose(oldData, newData) {
@@ -319,30 +319,15 @@ function showVerbose(oldData, newData) {
 
   uninstallData &&
     Object.keys(uninstallData).forEach(function (name) {
-      console.log(
-        colors.red(
-          formatMTime(uninstallData[name].mtime) +
-            ' UNINSTALL ' + name.slice(0, -1)
-        )
-      );
+      console.log(colors.red(showUpdateInfo('UNINSTALL', name)));
     });
   installData &&
     Object.keys(installData).forEach(function (name) {
-      console.log(
-        colors.green(
-          formatMTime(installData[name].mtime) +
-            ' INSTALL ' + name.slice(0, -1)
-        )
-      );
+      console.log(colors.green(showUpdateInfo('INSTALL', name)));
     });
   updateData &&
     Object.keys(updateData).forEach(function (name) {
-      console.log(
-        colors.yellow(
-          formatMTime(updateData[name].mtime) +
-            ' UPDATE ' + name.slice(0, -1)
-        )
-      );
+      console.log(colors.yellow(showUpdateInfo('UPDATE', name)));
     });
 }
 
@@ -1155,6 +1140,10 @@ function emptyFilter(val) {
   return !!val;
 }
 
+function isClosed(req) {
+  return req._hasClosed || req._hasError;
+}
+
 function getRulesMgr(type, req, res, callback) {
   var plugins = req.whistlePlugins;
   if (!plugins) {
@@ -1170,14 +1159,29 @@ function getPluginRulesCallback(req, callback) {
   };
 }
 
+function resolvePipe(req, m1, m2) {
+  if (req.isPluginReq) {
+    return;
+  }
+  var filter = m1 && m1.resolveFilter(req);
+  if (m2) {
+    var filter2 = m2.resolveFilter(req);
+    filter = filter ? extend(filter, filter2) : filter2;
+  }
+  if (util.isIgnored(filter, 'pipe')) {
+    return;
+  }
+  return (m1 && m1.resolvePipe(req)) || (m2 && m2.resolvePipe(req));
+}
+
 function resolvePipePlugin(req, callback) {
   if (req._pipePlugin == null) {
     var pipe;
     var hRules = req.headerRulesMgr;
     if (config.multiEnv) {
-      pipe = (hRules && hRules.resolvePipe(req)) || rulesMgr.resolvePipe(req);
+      pipe = resolvePipe(req, hRules, rulesMgr);
     } else {
-      pipe = rulesMgr.resolvePipe(req) || (hRules && hRules.resolvePipe(req));
+      pipe = resolvePipe(req, rulesMgr, hRules);
     }
     var plugin;
     req._pipeRule = pipe;
@@ -1205,7 +1209,7 @@ function getPipe(type, hookName) {
     }
     resolvePipePlugin(req, function (ports) {
       var port = ports && ports[type + 'Port'];
-      if (!port || req._hasClosed || req._hasError) {
+      if (!port || isClosed(req)) {
         return callback();
       }
       var options = getOptions(req, res, isRes && 'resRules');
@@ -1232,46 +1236,33 @@ function getPipe(type, hookName) {
           delete req.headers['sec-websocket-extensions'];
         }
       }
-      var client;
       var done;
-      var handleConnect = function (socket, _res) {
+      var handleConnect = function (socket) {
         if (!done) {
           done = true;
-          if (req._hasError) {
-            return socket.destroy();
+          if (isClosed(req)) {
+            return util.endSocket(socket);
           }
+          req.once('_closed', function () {
+            util.endSocket(socket);
+          });
           callback(socket);
+        } else if (socket) {
+          util.endSocket(socket);
         }
       };
-      var destroy = function () {
-        if (client) {
-          client.destroy();
-          client.socket && client.socket.destroy();
-          client = null;
-          handleConnect();
-        }
-      };
-      var handleError = function (err) {
-        if (client) {
-          destroy();
-          if (err) {
-            debugMode && console.log(req._pipeRule.matcher, type);
-            (res || req).emit('error', err);
-          }
-        }
-      };
-      client = config.connect(options, handleConnect);
-      client.on('error', function () {
-        if (client) {
-          destroy();
-          client = !req._hasError && config.connect(options, handleConnect);
-          client && util.onSocketEnd(client, handleError);
+      var client = config.connect(options, handleConnect);
+      client.once('error', function () {
+        if (!done && !req._hasError) {
+          client = config.connect(options, handleConnect);
+          client.once('error', function (err) {
+            if (!done && !isClosed(req)) {
+              debugMode && console.log(type, err);
+              (res || req).emit('error', err);
+            }
+          });
         }
       });
-      req.once('_closed', function() {
-        handleConnect();
-        destroy();
-      });
     });
   };
 }