whistle 2.10.1 → 2.10.3

package/biz/webui/lib/index.js

@@ -20,6 +20,7 @@ var common = require('../../../lib/util/common');
 var getWorker = require('../../../lib/plugins/util').getWorker;
 var loadAuthPlugins = require('../../../lib/plugins').loadAuthPlugins;
 var parseUrl = require('../../../lib/util/parse-url-safe');
+var loadService = require('../../../lib/service');
 
 var sendError = cgiUtil.sendError;
 var sendGzipText = cgiUtil.sendGzipText;
@@ -55,12 +56,32 @@ var MENU_URL = '???_WHISTLE_PLUGIN_EXT_CONTEXT_MENU_' + config.port + '???';
 var INSPECTOR_URL = '???_WHISTLE_PLUGIN_INSPECTOR_TAB_' + config.port + '???';
 var KEY_RE_G = /\${[^{}\s]+}|{\S+}/g;
 var COMMENT_RE = /#[^\r\n]*$/mg;
+var MAX_LEN = 1024 * 1024 * 6;
+var HTTP_RE = /https?:\/\/\S/i;
+
+function hasLogin() {
+  return config.username && config.password;
+}
+
+function isTempFile(query) {
+  var files = query.files;
+  if (files && typeof files === 'string') {
+    return false;
+  }
+  return common.getTempFile(query.filename);
+}
 
 function sendToService(req, res) {
-  proxyEvent.loadService(function(err, options) {
+  if (!hasLogin() && req.path === '/cgi-bin/temp/get' && !isTempFile(req.query)) {
+    res.setHeader('Content-Type', 'application/json; charset=utf-8');
+    res.end('{"ec":0,"value":"","forbidden":true}');
+    return;
+  }
+  loadService(function(err, options) {
     if (err) {
       common.sendRes(res, 500, err.stack || err);
     } else {
+      req.headers[common.WHISTLE_UID_HEADER] = config.uid;
       util.transformReq(req, res, options.port);
     }
   });
@@ -390,13 +411,37 @@ function cgiHandler(req, res) {
     handleResponse();
   });
 }
+app.get('/cgi-bin/import-remote', function(req, res) {
+  var url = req.query.url;
+  if (HTTP_RE.test(url)) {
+    util.request({
+      url: url,
+      maxLength: MAX_LEN
+    }, function(err, body, r) {
+      if (err) {
+        var msg = err.code === 'EEXCEED'  ? 'The size of response body exceeds 6MB' : err.message;
+        return res.json({ec: 2, em: msg});
+      }
+      var status = r.statusCode;
+      if (status !== 200) {
+        var em = status > 200 && status < 400 ? 'No data' : 'Request failed';
+        return res.json({ec: 2, em: em + ' (statusCode: ' + status + ')'});
+      }
+      return res.json({ec: 0, body: body});
+    });
+  } else if (util.isString(url)) {
+    req.url = '/cgi-bin/temp/get?filename=' + encodeURIComponent(url);
+    req.query.filename = url;
+    sendToService(req, res);
+  } else {
+    res.json({ec: 400, em: 'Bad url'});
+  }
+});
 app.all('/service/*', sendToService);
 app.all('/cgi-bin/service/*', sendToService);
 app.all('/cgi-bin/sessions/*', sendToService);
+app.all('/cgi-bin/log/*', sendToService);
 app.post('/cgi-bin/plugins/install', sendToService);
-app.all('/favicon.ico', function(req, res) {
-  res.sendFile(htdocs.getImgFile('favicon.ico'));
-});
 
 function readPluginPage(req, res, plugin, html, config) {
   res.type('html');

package/lib/config.js

@@ -148,15 +148,17 @@ function getDataDir(dirname) {
 }
 
 config.baseDir = getDataDir();
-config.TEMP_FILES_PATH = path.join(getWhistlePath(), 'temp_files');
 config.CUSTOM_PLUGIN_PATH = path.join(getWhistlePath(), 'custom_plugins');
 config.CUSTOM_CERTS_DIR = path.resolve(getWhistlePath(), 'custom_certs');
 config.SAVED_SESSIONS_PATH = path.resolve(getWhistlePath(), 'saved_sessions');
 config.DEV_PLUGINS_PATH = path.resolve(getWhistlePath(), 'dev_plugins');
+config.rulesDir = path.join(config.baseDir, 'rules');
+config.valuesDir = path.join(config.baseDir, 'values');
+config.propertiesDir = path.join(config.baseDir, 'properties');
 
 try {
-  fse.ensureDirSync(config.TEMP_FILES_PATH);
-  var blankFile = path.join(config.TEMP_FILES_PATH, 'blank');
+  fse.ensureDirSync(common.TEMP_FILES_PATH);
+  var blankFile = path.join(common.TEMP_FILES_PATH, 'blank');
   if (!fs.existsSync(blankFile)) {
     fs.writeFileSync(blankFile, '');
   }
@@ -793,7 +795,7 @@ exports.extend = function (newConf) {
         setDefaultResultOrder(m.toLowerCase());
       } else if (m === 'proxyOnly') {
         config.pureProxy = true;
-      } else if (m === 'useMultipleRules' || m === 'enableMultipleRules') {
+      } else if (m === 'useMultipleRules' || m === 'enableMultipleRules' || m === 'allowMultipleChoice') {
         config.allowMultipleChoice = true;
       } else if (m === 'disableMultipleRules') {
         config.allowMultipleChoice = false;

package/lib/handlers/file-proxy.js

@@ -66,7 +66,7 @@ function parseRes(str, rawHeaderNames, fromValue) {
     };
   }
   var headers = str.split(CRLF_RE);
-  var statusLine = headers.shift().trim().split(/\s+/g);
+  var statusLine = headers.shift().trim().split(/\s+/);
   headers = util.parseHeaders(headers, rawHeaderNames);
   if (fromValue) {
     delete headers['content-encoding'];

package/lib/https/ca.js

@@ -15,8 +15,6 @@ var common = require('../util/common');
 var rulesUtil = require('../rules/util');
 
 var pki = forge.pki;
-var CUR_VERSION = process.version;
-var requiredVersion = parseInt(CUR_VERSION.slice(1), 10) >= 6;
 var HTTPS_DIR = mkdir(path.join(config.getDataDir(), 'certs'));
 var ROOT_NEW_KEY_FILE = path.join(HTTPS_DIR, 'root_new.key');
 var ROOT_NEW_CRT_FILE = path.join(HTTPS_DIR, 'root_new.crt');
@@ -40,6 +38,7 @@ var customCertCount = 0;
 var cachePairs = new LRU({ max: 5120 });
 var certsCache = new LRU({ max: 256 });
 var remoteCerts = new LRU({ max: 1280 });
+var KEY_SIZE = 2048;
 var ILEGAL_CHAR_RE = /[^a-z\d-]/i;
 // https://cloud.tencent.com/developer/ask/sof/58155
 var RANDOM_SERIAL = '/' + Date.now() + '/' + process.pid + '/' + Math.floor(Math.random() * 100);
@@ -58,12 +57,12 @@ exports.remoteCerts = remoteCerts;
 exports.createSecureContext = createSecureContext;
 exports.CUSTOM_CERTS_DIR = CUSTOM_CERTS_DIR;
 
-function isRootCa(name) {
+function notRootCa(name) {
   return name !== 'root';
 }
 
 function checkFilename(name) {
-  return name && !ILLEGAL_PATH_RE.test(name) && isRootCa(name);
+  return name && !ILLEGAL_PATH_RE.test(name) && notRootCa(name);
 }
 
 function resetAllCerts() {
@@ -84,7 +83,7 @@ if (timer && typeof timer.unref === 'function') {
   timer.unref();
 }
 
-if (!useNewKey && requiredVersion && !checkCertificate()) {
+if (!useNewKey && !checkCertificate()) {
   try {
     fs.unlinkSync(ROOT_KEY_FILE);
     fs.unlinkSync(ROOT_CRT_FILE);
@@ -108,7 +107,7 @@ function isExpiredCert(crt) {
 function checkCertificate() {
   try {
     var crt = pki.certificateFromPem(fs.readFileSync(ROOT_CRT_FILE));
-    if (crt.publicKey.n.toString(2).length < 2048 || isExpiredCert(crt.validity)) {
+    if (crt.publicKey.n.toString(2).length < KEY_SIZE || isExpiredCert(crt.validity)) {
       return false;
     }
     return /^whistle\.\d+$/.test(getCommonName(crt));
@@ -178,6 +177,19 @@ exports.hasCustomCerts = function () {
   return customCertCount;
 };
 
+var rootSkiBytes;
+
+function getRootSKI() {
+  if (!rootSkiBytes) {
+    try {
+      rootSkiBytes = ROOT_CRT.generateSubjectKeyIdentifier().getBytes();
+    } catch (e) {
+      rootSkiBytes = null;
+    }
+  }
+  return rootSkiBytes;
+}
+
 function getCacheCert(hostname) {
   return (
     customPairs[hostname] ||
@@ -207,6 +219,20 @@ function createSelfCert(hostname) {
   ]);
   cert.setIssuer(ROOT_CRT.subject.attributes);
   cert.setExtensions([
+    {
+      name: 'basicConstraints',
+      cA: false
+    },
+    {
+      name: 'keyUsage',
+      digitalSignature: true,
+      keyEncipherment: true
+    },
+    {
+      name: 'extKeyUsage',
+      serverAuth: true,
+      clientAuth: true
+    },
     {
       name: 'subjectAltName',
       altNames: [
@@ -220,6 +246,10 @@ function createSelfCert(hostname) {
             value: hostname
           }
       ]
+    },
+    {
+      name: 'authorityKeyIdentifier',
+      keyIdentifier: getRootSKI() // Get SKI from ROOT_CRT for authorityKeyIdentifier extension
     }
   ]);
   cert.sign(ROOT_KEY, forge.md.sha256.create());
@@ -265,7 +295,7 @@ function parseAllCustomCerts() {
     var validity = info.validity;
     var altNames = info.altNames;
     var dnsName = [];
-    var disabled = (isRootCa(filename) && rulesUtil.isDisabledCertFile(filename)) || undefined;
+    var disabled = (notRootCa(filename) && rulesUtil.isDisabledCertFile(filename)) || undefined;
     altNames.forEach(function (item) {
       if ((item.type === 2 || item.type === 7) && !pairs[item.value]) {
         if (!disabled) {
@@ -439,7 +469,7 @@ function createRootCA() {
 
 function createCACert(opts) {
   opts = opts || {};
-  var keys = pki.rsa.generateKeyPair(requiredVersion ? 2048 : 1024);
+  var keys = pki.rsa.generateKeyPair(KEY_SIZE);
   var cert = createCert(keys.publicKey);
   var now = Date.now() + common.padLeft(Math.floor(Math.random() * 1000), 3);
   var attrs = [
@@ -757,7 +787,7 @@ exports.setActiveCert = function (opts) {
     return;
   }
   var filename = opts.filename;
-  if (isRootCa(filename) && allCustomCerts[filename]) {
+  if (notRootCa(filename) && allCustomCerts[filename]) {
     rulesUtil.setDisabledCertFile(filename, opts.disabled);
     parseAllCustomCerts();
   }

package/lib/https/index.js

@@ -207,7 +207,7 @@ function resolveWebsocket(socket, wss) {
   };
 
   var plugin = pluginMgr.resolveWhistlePlugins(socket);
-  abortIfUnavailable(socket);
+  handleEnd(socket);
   pluginMgr.getRules(socket, function (rulesMgr) {
     if (rulesMgr) {
       socket.pluginRules = rulesMgr;
@@ -394,7 +394,7 @@ function resolveWebsocket(socket, wss) {
                           proxySocket.on('error', handleProxyError);
                           reqSocket = proxySocket;
                           proxySocket.on('secureConnect', function () {
-                            abortIfUnavailable(reqSocket);
+                            handleEnd(reqSocket);
                             pipeData();
                           });
                         } catch (e) {
@@ -403,7 +403,7 @@ function resolveWebsocket(socket, wss) {
                         return;
                       }
                       reqSocket = proxySocket;
-                      abortIfUnavailable(reqSocket);
+                      handleEnd(reqSocket);
                       pipeData();
                     };
                     // 对应 internal-proxy 要用直接请求，方便用来穿透 nginx
@@ -557,7 +557,7 @@ function resolveWebsocket(socket, wss) {
           return execCallback(e);
         }
         if (retryConnect) {
-          abortIfUnavailable(reqSocket);
+          handleEnd(reqSocket);
         } else {
           retryConnect = function (e) {
             if (!newIp && (newIp = util.getLocalhostIP(e, socket, socket._w2hostname, socket.hostIp))) {
@@ -621,7 +621,7 @@ function resolveWebsocket(socket, wss) {
     var disable = socket.disable;
     if (retryConnect) {
       reqSocket.removeListener('error', retryConnect);
-      abortIfUnavailable(reqSocket);
+      handleEnd(reqSocket);
       retryConnect = null;
     }
     clearTimeout(timeout);
@@ -718,9 +718,10 @@ function resolveWebsocket(socket, wss) {
           extend(headers, reqHeaders);
           headers.host = headers.host || host;
         }
-        auth = util.getAuthBasic(auth || authObj);
+        authObj = auth || authObj;
+        auth = util.getAuthBasic(authObj);
         if (auth) {
-          headers['authorization'] = auth;
+          headers[(authObj.proxy ? 'proxy-' : '') + 'authorization'] = auth;
         }
         var reqRuleData = { headers: headers };
         util.setReqCors(reqRuleData, reqCors);
@@ -847,7 +848,7 @@ function resolveWebsocket(socket, wss) {
     }
   }
 
-  function abortIfUnavailable(socket) {
+  function handleEnd(socket) {
     return util.onSocketEnd(socket, destroy);
   }
   var destroyed, reqDestroyed, resDestroyed;
@@ -1080,11 +1081,10 @@ function toHttp1(req, res) {
         formatRawHeaders(svrRes, isH2)
       );
       var write = res.write;
-      var handleError = function (e) {
-        e && handleAbort();
-      };
       res.write = function (chunk) {
-        return write.call(res, chunk, handleError);
+        return write.call(res, chunk, function (e) {
+          e && handleAbort();
+        });
       };
       svrRes.pipe(res);
       res.flushHeaders && res.flushHeaders();
@@ -1168,152 +1168,147 @@ module.exports = function (socket, next, isWebPort) {
 
   util.onSocketEnd(socket, destroy);
 
-  function abortIfUnavailable(s) {
-    return s.on('error', destroy);
-  }
   var clientIp = socket.clientIp;
   var clientPort = socket.clientPort;
-  util.readOneChunk(
-    socket,
-    function (chunk) {
-      headersStr = chunk && chunk.toString();
-      var isHttp = chunk && HTTP_RE.test(headersStr);
-      var statusLine = isHttp && RegExp['$&'];
-      if (isHttp && CONNECT_RE.test(RegExp.$1)) {
-        chunk = addClientInfo(socket, chunk, statusLine, clientIp, clientPort);
-        util.connect(
-          {
-            port: config.port,
-            host: LOCALHOST
-          },
+  util.readOneChunk(socket, function (chunk) {
+    headersStr = chunk && chunk.toString();
+    var isHttp = chunk && HTTP_RE.test(headersStr);
+    var statusLine = isHttp && RegExp['$&'];
+    if (isHttp && CONNECT_RE.test(RegExp.$1)) {
+      chunk = addClientInfo(socket, chunk, statusLine, clientIp, clientPort);
+      util.connect(
+        {
+          port: config.port,
+          host: config.host || LOCALHOST
+        },
           function (err, s) {
             reqSocket = s;
             if (err || socket._hasError) {
               return destroy(err);
             }
+            reqSocket.on('error', destroy);
             reqSocket.write(chunk);
             reqSocket.pipe(socket).pipe(reqSocket);
-            abortIfUnavailable(reqSocket);
             socket.resume();
           }
         );
-        return;
-      }
-      if (!chunk) {
+      return;
+    }
+    if (!chunk) {
         //没有数据
-        return isWebPort ? socket.destroy() : next(chunk);
-      }
-      if (isHttp) {
-        if (isEnable('forHttps') || disable.captureHttp) {
-          next(chunk);
-        } else {
-          socket.resume();
-          server.emit('connection', socket);
-          socket.emit(
+      return isWebPort ? socket.destroy() : next(chunk);
+    }
+    if (isHttp) {
+      if (isEnable('forHttps') || disable.captureHttp) {
+        next(chunk);
+      } else {
+        socket.resume();
+        server.emit('connection', socket);
+        socket.emit(
             'data',
             addClientInfo(socket, chunk, statusLine, clientIp, clientPort)
           );
-        }
-      } else if (isEnable('forHttp') || disable.captureHttps) {
+      }
+    } else if (isEnable('forHttp') || disable.captureHttps) {
+      return next(chunk);
+    } else {
+      var isHttpH2 = HTTP2_RE.test(headersStr);
+      if (!isHttpH2 && chunk[0] != 22) {
         return next(chunk);
-      } else {
-        var isHttpH2 = HTTP2_RE.test(headersStr);
-        if (!isHttpH2 && chunk[0] != 22) {
-          return next(chunk);
-        }
-        var useSNI, domain, serverKey;
-        var handleConnect = function (port) {
-          var promise =
+      }
+      var useSNI, domain, serverKey;
+      var handleConnect = function (port) {
+        var promise =
             !isHttpH2 && !useSNI && serverAgent.existsServer(serverKey);
-          var httpsServer = promise && promise.cert && promise.server;
-          if (httpsServer && httpsServer.setSecureContext) {
-            var cert = serverAgent.createCertificate(domain);
-            if (
+        var httpsServer = promise && promise.cert && promise.server;
+        if (httpsServer && httpsServer.setSecureContext) {
+          var cert = serverAgent.createCertificate(domain);
+          if (
               cert.key !== promise.cert.key ||
               cert.cert !== promise.cert.cert
             ) {
-              try {
-                cert._ctx = cert._ctx || ca.createSecureContext(cert);
-                httpsServer.setSecureContext(cert._ctx);
-                promise.cert = cert;
-              } catch (e) {}
-            }
+            try {
+              cert._ctx = cert._ctx || ca.createSecureContext(cert);
+              httpsServer.setSecureContext(cert._ctx);
+              promise.cert = cert;
+            } catch (e) {}
           }
-          util.connect(
-            {
-              port: port,
-              host: LOCALHOST,
-              localAddress: LOCALHOST
-            },
+        }
+        util.connect(
+          {
+            port: port,
+            host: LOCALHOST,
+            localAddress: LOCALHOST
+          },
             function (err, s) {
               reqSocket = s;
               if (err || socket._hasError) {
                 return destroy(err);
               }
+              reqSocket.on('error', destroy);
               socket._w2TunnelKey = reqSocket.localPort + ':' + reqSocket.remotePort;
               tunnelTmplData.set(socket._w2TunnelKey,  getTunnelData(socket, clientIp, clientPort, isHttpH2));
               reqSocket.write(chunk);
               reqSocket.pipe(socket).pipe(reqSocket);
               socket.resume();
-              abortIfUnavailable(reqSocket);
             }
           );
-        };
-        var useNoSNIServer = function () {
-          serverKey = requestCert ? ':' + domain : domain;
-          serverAgent.createServer(serverKey, handlers, handleConnect, 0, 0);
-        };
+      };
+      var useNoSNIServer = function () {
+        serverKey = requestCert ? ':' + domain : domain;
+        serverAgent.createServer(serverKey, handlers, handleConnect, 0, 0);
+      };
 
-        var handleRequest = function () {
-          if (useSNI) {
-            var disableH2 = !properties.isEnableHttp2();
-            if (disable.http2) {
-              disableH2 = true;
-            } else if (enable.http2) {
-              disableH2 = false;
-            }
-            getSNIServer(h2Handlers, handleConnect, disableH2, requestCert);
-          } else if (isHttpH2) {
-            getHttp2Server(h2Handlers, handleConnect);
-          } else {
-            useNoSNIServer();
+      var handleRequest = function () {
+        if (useSNI) {
+          var disableH2 = !properties.isEnableHttp2();
+          if (disable.http2) {
+            disableH2 = true;
+          } else if (enable.http2) {
+            disableH2 = false;
           }
-        };
-        if (isHttpH2) {
-          return handleRequest();
+          getSNIServer(h2Handlers, handleConnect, disableH2, requestCert);
+        } else if (isHttpH2) {
+          getHttp2Server(h2Handlers, handleConnect);
+        } else {
+          useNoSNIServer();
         }
-        useSNI = checkSNI(chunk);
-        var servername = useSNI || socket.tunnelHostname;
-        if (
+      };
+      if (isHttpH2) {
+        return handleRequest();
+      }
+      useSNI = checkSNI(chunk);
+      var servername = useSNI || socket.tunnelHostname;
+      if (
           !servername ||
           (useSNI ? disable.captureSNI : (disable.captureNoSNI || (net.isIP(servername) && !isCaptureIp()))) ||
           (socket.useProxifier && !ca.existsCustomCert(servername))
         ) {
-          return next(chunk);
-        }
-        var requestCert =
+        return next(chunk);
+      }
+      var requestCert =
           (enable.clientCert || enable.requestCert) &&
           !disable.clientCert &&
           !disable.requestCert;
-        domain = getDomain(servername);
-        socket.curUrl = socket.fullUrl = 'https://' + servername;
-        socket.useSNI = useSNI;
-        socket.serverName = socket.servername = servername;
-        socket.commonName = domain;
-        loadCert(socket, function (cert) {
-          if (socket._hasError) {
-            return destroy();
-          }
-          if (cert === false) {
-            return next(chunk);
-          }
-          if (cert) {
-            domain = servername;
-          }
-          handleRequest();
-        });
-      }
-    },
+      domain = getDomain(servername);
+      socket.curUrl = socket.fullUrl = 'https://' + servername;
+      socket.useSNI = useSNI;
+      socket.serverName = socket.servername = servername;
+      socket.commonName = domain;
+      loadCert(socket, function (cert) {
+        if (socket._hasError) {
+          return destroy();
+        }
+        if (cert === false) {
+          return next(chunk);
+        }
+        if (cert) {
+          domain = servername;
+        }
+        handleRequest();
+      });
+    }
+  },
     isWebPort ? 0 : TIMEOUT
   );
 };

package/lib/index.js

@@ -11,10 +11,8 @@ var setupHttps = require('./https').setup;
 var httpsUtil = require('./https/ca');
 var rulesUtil = require('./rules/util');
 var initDataServer = require('./util/data-server');
-var initLogServer = require('./util/log-server');
 var pluginMgr = require('./plugins');
 var config = require('./config');
-var loadService = require('./service');
 var initSocketMgr = require('./socket-mgr');
 var tunnelProxy = require('./tunnel');
 var upgradeProxy = require('./upgrade');
@@ -61,7 +59,6 @@ function proxy(callback, _server) {
   tunnelProxy(server, proxyEvents);
   upgradeProxy(server);
   initDataServer(proxyEvents);
-  initLogServer(proxyEvents);
   rulesUtil.setup(proxyEvents);
   var properties = rulesUtil.properties;
   if (config.disableAllRules) {
@@ -224,7 +221,6 @@ function exportInterfaces(obj) {
   obj.httpsUtil = httpsUtil;
   obj.pluginMgr = pluginMgr;
   obj.logger = logger;
-  obj.loadService = loadService;
   obj.setAuth = config.setAuth;
   obj.setUIHost = config.setUIHost;
   obj.setPluginUIHost = config.setPluginUIHost;
@@ -254,7 +250,7 @@ function handleGlobalException(err) {
     code === 'ERR_HTTP_TRAILER_INVALID' ||
     code === 'ERR_INTERNAL_ASSERTION' ||
     code === 'ERR_INVALID_ARG_TYPE' ||
-    (err && /finishwrite/i.test(err.message)))
+    (err && /finishwrite|'_hadError'/i.test(err.message)))
   ) {
     return;
   }