whistle 2.10.8 → 2.10.10

package/lib/plugins/index.js

@@ -289,8 +289,8 @@ pluginMgr.on('uninstall', function (result) {
   });
 });
 
-function formatMTime(mtime) {
-  return '[' + (new Date(mtime).toLocaleTimeString()) + ']';
+function showUpdateInfo(action, name) {
+  return '[' + (new Date().toLocaleTimeString()) + '] ' + action + ' ' + name.slice(0, -1);
 }
 
 function showVerbose(oldData, newData) {
@@ -319,30 +319,15 @@ function showVerbose(oldData, newData) {
 
   uninstallData &&
     Object.keys(uninstallData).forEach(function (name) {
-      console.log(
-        colors.red(
-          formatMTime(uninstallData[name].mtime) +
-            ' UNINSTALL ' + name.slice(0, -1)
-        )
-      );
+      console.log(colors.red(showUpdateInfo('UNINSTALL', name)));
     });
   installData &&
     Object.keys(installData).forEach(function (name) {
-      console.log(
-        colors.green(
-          formatMTime(installData[name].mtime) +
-            ' INSTALL ' + name.slice(0, -1)
-        )
-      );
+      console.log(colors.green(showUpdateInfo('INSTALL', name)));
     });
   updateData &&
     Object.keys(updateData).forEach(function (name) {
-      console.log(
-        colors.yellow(
-          formatMTime(updateData[name].mtime) +
-            ' UPDATE ' + name.slice(0, -1)
-        )
-      );
+      console.log(colors.yellow(showUpdateInfo('UPDATE', name)));
     });
 }
 
@@ -1155,6 +1140,10 @@ function emptyFilter(val) {
   return !!val;
 }
 
+function isClosed(req) {
+  return req._hasClosed || req._hasError;
+}
+
 function getRulesMgr(type, req, res, callback) {
   var plugins = req.whistlePlugins;
   if (!plugins) {
@@ -1170,14 +1159,29 @@ function getPluginRulesCallback(req, callback) {
   };
 }
 
+function resolvePipe(req, m1, m2) {
+  if (req.isPluginReq) {
+    return;
+  }
+  var filter = m1 && m1.resolveFilter(req);
+  if (m2) {
+    var filter2 = m2.resolveFilter(req);
+    filter = filter ? extend(filter, filter2) : filter2;
+  }
+  if (util.isIgnored(filter, 'pipe')) {
+    return;
+  }
+  return (m1 && m1.resolvePipe(req)) || (m2 && m2.resolvePipe(req));
+}
+
 function resolvePipePlugin(req, callback) {
   if (req._pipePlugin == null) {
     var pipe;
     var hRules = req.headerRulesMgr;
     if (config.multiEnv) {
-      pipe = (hRules && hRules.resolvePipe(req)) || rulesMgr.resolvePipe(req);
+      pipe = resolvePipe(req, hRules, rulesMgr);
     } else {
-      pipe = rulesMgr.resolvePipe(req) || (hRules && hRules.resolvePipe(req));
+      pipe = resolvePipe(req, rulesMgr, hRules);
     }
     var plugin;
     req._pipeRule = pipe;
@@ -1205,7 +1209,7 @@ function getPipe(type, hookName) {
     }
     resolvePipePlugin(req, function (ports) {
       var port = ports && ports[type + 'Port'];
-      if (!port || req._hasClosed || req._hasError) {
+      if (!port || isClosed(req)) {
         return callback();
       }
       var options = getOptions(req, res, isRes && 'resRules');
@@ -1232,46 +1236,33 @@ function getPipe(type, hookName) {
           delete req.headers['sec-websocket-extensions'];
         }
       }
-      var client;
       var done;
       var handleConnect = function (socket, _res) {
         if (!done) {
           done = true;
-          if (req._hasError) {
-            return socket.destroy();
+          if (isClosed(req)) {
+            return util.endSocket(socket);
           }
+          req.once('_closed', function () {
+            util.endSocket(socket);
+          });
           callback(socket);
+        } else if (socket) {
+          util.endSocket(socket);
         }
       };
-      var destroy = function () {
-        if (client) {
-          client.destroy();
-          client.socket && client.socket.destroy();
-          client = null;
-          handleConnect();
-        }
-      };
-      var handleError = function (err) {
-        if (client) {
-          destroy();
-          if (err) {
-            debugMode && console.log(req._pipeRule.matcher, type);
-            (res || req).emit('error', err);
-          }
-        }
-      };
-      client = config.connect(options, handleConnect);
-      client.on('error', function () {
-        if (client) {
-          destroy();
-          client = !req._hasError && config.connect(options, handleConnect);
-          client && util.onSocketEnd(client, handleError);
+      var client = config.connect(options, handleConnect);
+      client.once('error', function () {
+        if (!done && !req._hasError) {
+          client = config.connect(options, handleConnect);
+          client.once('error', function (err) {
+            if (!done && !isClosed(req)) {
+              debugMode && console.log(type, err);
+              (res || req).emit('error', err);
+            }
+          });
         }
       });
-      req.once('_closed', function() {
-        handleConnect();
-        destroy();
-      });
     });
   };
 }

package/lib/plugins/load-plugin.js

@@ -117,7 +117,7 @@ var appendTrailers = function (_res, res, newTrailers, req) {
     return;
   }
   common.addTrailerNames(_res, newTrailers, null, null, req);
-  common.onResEnd(_res, function () {
+  common.onReadEnd(_res, function () {
     var trailers = _res.trailers;
     if (
       !res.chunkedEncoding ||
@@ -786,15 +786,12 @@ var getSession = function (req, cb, isReq) {
   retryRequestSession();
 };
 
-var initWsReq = function (req, res) {
-  initReq(req, res, true);
-};
-var initConnectReq = function (req, res) {
+var setupConnect = function (req, socket, isServer) {
   if (req.originalReq && req.originalRes) {
     return;
   }
   var established;
-  initWsReq(req, res);
+  initReq(req, socket, isServer);
   req.sendEstablished = function (err, cb) {
     if (established) {
       return;
@@ -815,25 +812,25 @@ var initConnectReq = function (req, res) {
     ];
     if (err || !cb || !req.headers[common.ACK_HEADER]) {
       resCtn.push('\r\n', body);
-      res.write(resCtn.join('\r\n'));
+      socket.write(resCtn.join('\r\n'));
       return cb && cb();
     }
     resCtn.push('x-whistle-allow-tunnel-ack: 1');
     resCtn.push('\r\n', body);
-    res.once('data', function (chunk) {
+    socket.once('data', function (chunk) {
       if (!req._hasError) {
-        res.pause();
-        var on = res.on;
-        res.on = function () {
-          res.on = on;
-          res.resume();
+        socket.pause();
+        var on = socket.on;
+        socket.on = function () {
+          socket.on = on;
+          socket.resume();
           return on.apply(this, arguments);
         };
-        chunk.length > 1 && res.unshift(chunk.slice(1));
+        chunk.length > 1 && socket.unshift(chunk.slice(1));
         cb();
       }
     });
-    return res.write(resCtn.join('\r\n'));
+    return socket.write(resCtn.join('\r\n'));
   };
 };
 
@@ -1111,10 +1108,6 @@ function handleWsSignal(receiver, sender) {
   };
 }
 
-function destroySocket() {
-  this.destroy();
-}
-
 function formatRules(rules) {
   if (Array.isArray(rules)) {
     return rules.join('\n');
@@ -2240,7 +2233,7 @@ module.exports = async function (options, callback) {
           break;
         case PLUGIN_HOOKS.HTTP:
           if (httpServer) {
-            initWsReq(req, socket);
+            initReq(req, socket, true);
             var curUrl = req.originalReq.realUrl;
             httpServer.setMaxWsPayload = setMaxWsPayload;
             var svrRes = '';
@@ -2354,41 +2347,39 @@ module.exports = async function (options, callback) {
 
       var emitHttpPipe = function (httpServer, req, socket) {
         if (httpServer) {
-          initConnectReq(req, socket);
+          setupConnect(req, socket);
           req.sendEstablished(function () {
             var decoder = getDecodeTransform();
             var encoder = getEncodeTransform();
             var done;
-            var handleError = function (err) {
+            var destroy = function () {
               if (!done) {
                 done = true;
-                socket.emit('error', err || new Error('destroyed'));
+                socket.destroy();
               }
             };
             socket.on('error', function (err) {
               if (!done) {
-                done = true;
+                destroy();
                 decoder.emit('error', err);
                 encoder.emit('error', err);
               }
             });
-            encoder.once('finish', function () {
-              done = true;
-            });
-            socket.on('close', function () {
+            socket.once('close', function () {
               done = true;
               decoder.emit('close');
               encoder.emit('close');
             });
-            decoder.destroy = encoder.destroy = handleError;
-            decoder.on('error', handleError);
-            encoder.on('error', handleError);
+            decoder.destroy = encoder.destroy = destroy;
+            decoder.on('error', destroy);
+            encoder.on('error', destroy);
             ADDITIONAL_FIELDS.forEach(function (key) {
               decoder[key] = req[key];
             });
-            socket.pipe(decoder);
-            encoder.pipe(socket);
-            socket.on('end', destroySocket);
+            encoder.once('finish', function() {
+              socket.end();
+            });
+            encoder.pipe(socket).pipe(decoder);
             httpServer.emit('request', decoder, encoder);
           });
         }
@@ -2396,7 +2387,7 @@ module.exports = async function (options, callback) {
 
       function pipeUpgrade(pipeSvr, req, socket, head, fromClient, toServer) {
         if (pipeSvr) {
-          initConnectReq(req, socket);
+          setupConnect(req, socket);
           req.sendEstablished(function () {
             addFrameHandler(req, socket, maxWsPayload, fromClient, toServer);
             pipeSvr.emit('connect', req, socket, head);
@@ -2433,7 +2424,7 @@ module.exports = async function (options, callback) {
           break;
         case PLUGIN_HOOKS.TUNNEL:
           if (tunnelServer) {
-            initConnectReq(req, socket);
+            setupConnect(req, socket, true);
             req.sendEstablished(function () {
               var reqRules;
               req.setRules = req.setReqRules = socket.setReqRules = function (rules) {
@@ -2524,7 +2515,7 @@ module.exports = async function (options, callback) {
           }
           break;
         case PLUGIN_HOOKS.TUNNEL_REQ_READ:
-          initConnectReq(req, socket);
+          setupConnect(req, socket);
           pipeServer = req._isUpgrade ? wsReqRead : tunnelReqRead;
           if (pipeServer) {
             req.sendEstablished(function () {
@@ -2537,7 +2528,7 @@ module.exports = async function (options, callback) {
           }
           break;
         case PLUGIN_HOOKS.TUNNEL_REQ_WRITE:
-          initConnectReq(req, socket);
+          setupConnect(req, socket);
           pipeServer = req._isUpgrade ? wsReqWrite : tunnelReqWrite;
           if (pipeServer) {
             req.sendEstablished(function () {
@@ -2549,7 +2540,7 @@ module.exports = async function (options, callback) {
           }
           break;
         case PLUGIN_HOOKS.TUNNEL_RES_READ:
-          initConnectReq(req, socket);
+          setupConnect(req, socket);
           pipeServer = req._isUpgrade ? wsResRead : tunnelResRead;
           if (pipeServer) {
             req.sendEstablished(function () {
@@ -2562,7 +2553,7 @@ module.exports = async function (options, callback) {
           }
           break;
         case PLUGIN_HOOKS.TUNNEL_RES_WRITE:
-          initConnectReq(req, socket);
+          setupConnect(req, socket);
           pipeServer = req._isUpgrade ? wsResWrite : tunnelResWrite;
           if (pipeServer) {
             req.sendEstablished(function () {

package/lib/rules/dns.js

@@ -50,7 +50,7 @@ function lookDnsOverHttps(hostname, callback) {
     } catch (e) {
       err = data || e;
     }
-    return callback(err || 'DNS Over HTTPS Look Failed');
+    return callback(err || 'DNS over HTTPS Look Failed');
   });
 }
 

package/lib/rules/index.js

@@ -51,6 +51,7 @@ exports.getEnabledRules = function() {
 exports.getMFlag = function() {
   return rules._mflag;
 };
+exports.resolveFilter = rules.resolveFilter.bind(rules);
 exports.parse = rules.parse.bind(rules);
 exports.append = rules.append.bind(rules);
 exports.resolveSNICallback = rules.resolveSNICallback.bind(rules);

package/lib/rules/rules.js

@@ -916,7 +916,7 @@ function checkInternal(req, rule) {
 }
 
 function resolveRuleList(req, list, vals, index, isFilter, isEnableProxy, host, isReq) {
-  var curUrl = util.formatUrl(req.curUrl);
+  var curUrl = util.formatUrl(req.curUrl || req.fullUrl);
   var notHttp = list.isRuleProto && curUrl[0] !== 'h';
   //支持域名匹配
   var domainUrl = removePort(curUrl);

package/lib/rules/util.js

@@ -19,12 +19,9 @@ var inlineValues = {};
 var proxy, pluginMgr;
 var serviceRules;
 var mockRules;
-var dnsOrder = propertiesStorage.getProperty('dnsOrder');
 
 config.ipv6Only = config.ipv6Only || !!propertiesStorage.getProperty('ipv6Only');
-if (!config.dnsOrder && !config.setDefaultResultOrder(dnsOrder)) {
-  config.dnsOrder = config.defaultDnsOrder;
-}
+config.setDefaultResultOrder(propertiesStorage.getProperty('dnsOrder'));
 
 function limitValueLen(name, len) {
   var value = propertiesStorage.getProperty(name);
@@ -535,9 +532,7 @@ exports.properties = {
   },
   setDnsOrder: function(order) {
     if (config.setDefaultResultOrder(order)) {
-      order = +order;
-      config.dnsOrder = order;
-      propertiesStorage.setProperty('dnsOrder', order);
+      propertiesStorage.setProperty('dnsOrder', +order);
     }
   },
   getLatestVersion: function (name) {

package/lib/service/composer.js

@@ -257,13 +257,12 @@ function handleHttp(options, cb, count, reqId, testId) {
       svrRes.on('error', execCb);
       var buffer;
       var enableStream;
-      var unzipStream = common.getUnzipStream(svrRes.headers);
       var timer = reqId && setTimeout(function() {
         composeData.setData(reqId, buffer, true);
         enableStream = true;
         timer = buffer = undefined;
         handleResponse();
-        common.onResEnd(svrRes, function() {
+        common.onReadEnd(svrRes, function() {
           var buf = composeData.getData(reqId);
           if (buf) {
             buf._hasW2End = true;
@@ -293,6 +292,11 @@ function handleHttp(options, cb, count, reqId, testId) {
         execCb(null, result);
         buffer = null;
       };
+      if (!common.hasBody(svrRes, options)) {
+        drain(svrRes);
+        return handleResponse();
+      }
+      var unzipStream = common.getUnzipStream(svrRes.headers);
       if (unzipStream) {
         unzipStream.on('error', function(err) {
           drain(svrRes);
@@ -320,7 +324,7 @@ function handleHttp(options, cb, count, reqId, testId) {
           }
         }
       });
-      unzipStream.on('end', handleResponse);
+      unzipStream.once('end', handleResponse);
     });
     client.on('error', execCb || noop);
     client.end(options.body);

package/lib/service/service.js

@@ -18,8 +18,6 @@ var Limiter = require('async-limiter');
 var common = require('../util/common');
 var logger = require('./console-log');
 
-process.env.NODE_TLS_REJECT_UNAUTHORIZED = '1';
-
 var saveData = dataCenter.saveData;
 var shareData = dataCenter.shareData;
 var requestData = dataCenter.requestData;
@@ -304,11 +302,11 @@ function installPlugins(data, clientId) {
 function sessionsHandler() {
   var app = express();
   app.use(function (req, res, next) {
-    req.on('error', abort);
-    res.on('error', abort);
-    function abort() {
+    function destroy() {
       res.destroy();
     }
+    req.on('error', destroy);
+    res.on('error', destroy);
     if (req.headers[common.WHISTLE_UID_HEADER] !== config.uid) {
       return res.status(403).end('Forbidden');
     }

package/lib/socket-mgr.js

@@ -379,7 +379,7 @@ function handleWsSend(ctx, reqTrans, sendStatus, handleTransform, frameCtx, even
   reqReceiver.onData = function (data, opts, drain) {
     var ignore;
     if (drain !== true && hasFilter) {
-      data = execHandleFrame(frameCtx, data, opts);
+      data = execHandleFrame(frameCtx, data, opts, true);
       if (!data) {
         return;
       }

package/lib/util/common.js

@@ -469,13 +469,22 @@ exports.addTrailerNames = function (
   });
 };
 
-exports.onResEnd = function (res, callback) {
-  var state = res._readableState || '';
-  if (state.endEmitted) {
+function onReadEnd(stream, callback) {
+  if (stream.readableEnded) {
     return callback();
   }
-  res.on('end', callback);
-};
+  stream.once('end', callback);
+}
+
+function onWriteEnd(stream, callback) {
+  if (stream.writableFinished) {
+    return callback();
+  }
+  stream.once('finish', callback);
+}
+
+exports.onReadEnd = onReadEnd;
+exports.onWriteEnd = onWriteEnd;
 
 var UPGRADE_RE = /^\s*upgrade\s*$/i;
 var WS_RE = /^\s*websocket\s*$/i;
@@ -725,7 +734,7 @@ function readStream(stream, callback) {
   stream.on('data', function(chunk) {
     buffer.push(chunk);
   });
-  stream.once('end', function() {
+  onReadEnd(stream, function() {
     var buf;
     var text;
     var json;
@@ -899,12 +908,11 @@ exports.onSocketEnd = function(socket, callback) {
       callback = null;
     }
   };
-  if (socket.aborted || socket.destroyed || socket._hasError) {
+  if (socket.aborted || socket.destroyed || socket.closed || socket._hasError) {
     return execCallback();
   }
   socket.on('error', execCallback);
   socket.once('close', execCallback);
-  socket.once('end', execCallback);
   socket.once('timeout', execCallback);
 };
 
@@ -1579,6 +1587,14 @@ function safeEncodeURIComponent(ch) {
   return ch;
 }
 
+function safeDecodeURIComponent(str) {
+  try {
+    return decodeURIComponent(str);
+  } catch (e) {}
+
+  return str;
+}
+
 exports.safeEncodeURIComponent = safeEncodeURIComponent;
 
 /**
@@ -1893,32 +1909,43 @@ exports.connectInner = function (options, cb, config) {
     if (req) {
       req.emit('error', TIMEOUT_ERR);
       req.destroy();
-      req.socket && req.socket.destroy();
     }
   }, CONN_TIMEOUT);
-  var req = httpModule.request(proxyOptions);
-  req.on('error', noop);
-  req
-    .on('connect', function (res, socket) {
+  var req;
+  var clearTimer = function (err) {
+    if (timer) {
+      err && req && req.destroy();
       clearTimeout(timer);
-      socket.on('error', noop);
-      if (res.statusCode !== 200) {
-        var err = new Error(
+      timer = null;
+    }
+  };
+  req = httpModule.request(proxyOptions);
+  req.on('error', clearTimer);
+  req.on('connect', function (res, socket) {
+    if (!timer) {
+      return;
+    }
+    clearTimer();
+    socket.on('error', function() {
+      socket.destroy();
+    });
+    if (res.statusCode !== 200) {
+      var err = new Error(
           'Tunneling socket could not be established, statusCode=' +
             res.statusCode
         );
-        err.statusCode = res.statusCode;
-        socket.destroy();
-        process.nextTick(function () {
-          req.emit('error', err);
-        });
-        return;
-      }
-      if (res.headers[ALLOW_ACK]) {
-        socket.write('1');
-      }
-      cb(socket, res);
-    })
+      err.statusCode = res.statusCode;
+      socket.destroy();
+      process.nextTick(function () {
+        req.emit('error', err);
+      });
+      return;
+    }
+    if (res.headers[ALLOW_ACK]) {
+      socket.write('1');
+    }
+    cb(socket, res);
+  })
     .end();
   return req;
 };
@@ -2052,11 +2079,7 @@ exports.getSessionInfo = function(headers) {
     var value = data[LAST_RAW_INDEX + index];
     if (value) {
       rawResult[key] = value;
-      try {
-        result[key] = decodeURIComponent(value);
-      } catch (e) {
-        result[key] = value;
-      }
+      result[key] = safeDecodeURIComponent(value);
     }
   });
   result._ = rawResult;

package/lib/util/data-server.js

@@ -323,7 +323,7 @@ function handleRequest(req, data) {
       delete data.abort;
     }
   };
-  req.on('end', removeAbort);
+  util.onReadEnd(req, removeAbort);
   req.on('error', removeAbort);
   req.on('abort', removeAbort);
   data.version = version;
@@ -743,6 +743,10 @@ module.exports = function init(_proxy) {
       }
     }
 
+    if (filterType === 'SSE') {
+      return rawType === 'text/event-stream';
+    }
+
     var curType = util.getContentType(rawType);
     for (i = 0, len = TYPES.length; i < len; i++) {
       type = TYPES[i];

package/lib/util/drain.js

@@ -1,14 +1,13 @@
-var createTransform = require('./common').createTransform;
+var common = require('./common');
 
-var noop = function () {};
+var noop = common.noop;
+var onReadEnd = common.onReadEnd;
 
 module.exports = function (stream, endHandler) {
   if (stream._hasAlreadyDrain || (!stream.noReqBody && stream.useH2)) {
     return typeof endHandler == 'function' && endHandler();
   }
   stream._hasAlreadyDrain = true;
-  var emitEndStream = createTransform();
-  emitEndStream.on('data', noop).on('error', noop);
-  emitEndStream.on('end', endHandler);
-  stream.pipe(emitEndStream);
+  onReadEnd(stream, endHandler);
+  stream.on('data', noop).on('error', noop);
 };

package/lib/util/file-mgr.js

@@ -58,7 +58,7 @@ function readSingleFile(path, callback) {
     }
   });
   stream.on('error', execCallback);
-  stream.on('end', execCallback);
+  stream.once('end', execCallback);
 }
 
 function getFileMap(list) {