whistle 2.9.14 → 2.9.16

package/bin/ca/cli.js

@@ -0,0 +1,95 @@
+var net = require('net');
+var fs = require('fs');
+var path = require('path');
+var installRootCA = require('./index');
+var util = require('../util');
+var fileMgr = require('../../lib/util/file-mgr');
+var httpMgr = require('../../lib/util/http-mgr');
+var commonUtil = require('../../lib/util/common');
+var config = require('../../lib/config');
+
+var NUM_RE = /^\d+$/;
+var HOST_SUFFIX_RE = /\:(\d+|auto)?$/;
+var HOST_RE = /^[a-z\d_-]+(?:\.[a-z\d_-]+)*$/i;
+var URL_RE = /^https?:\/\/./i;
+var MAX_LEN = 1024 * 1024;
+
+function installCert(certFile, url) {
+  try {
+    installRootCA(fileMgr.convertSlash(certFile));
+    util.info('Install root CA (' + (url || certFile) + ') successful.');
+  } catch (e) {
+    util.error(e.message);
+  }
+}
+
+function install(addr) {
+  if (addr.file) {
+    return installCert(addr.file);
+  }
+  addr.needRawData = true;
+  addr.maxLength = MAX_LEN;
+  addr.headers = {
+    'user-agent': 'whistle/' + config.name
+  };
+  httpMgr.request(addr, function(err, body, res) {
+    if (err) {
+      return util.error(err.message);
+    }
+    if (res.statusCode != 200) {
+      return util.error('Bad response (' + res.statusCode + ').');
+    }
+    if (!body || !body.length) {
+      return util.error('No content.');
+    }
+    var tempFile = path.join(commonUtil.getWhistlePath(), Date.now() + '-' + util.getHash(addr.url) + '.crt');
+    fs.writeFileSync(tempFile, body);
+    installCert(tempFile, addr.url);
+    fs.unlinkSync(tempFile);
+  });
+}
+
+module.exports = function(argv) {
+  var options = {};
+  argv.forEach(function(arg) {
+    if (NUM_RE.test(arg)) {
+      delete options.addr;
+      options.port = parseInt(arg, 10) || options.port;
+    } else if (net.isIP(arg)) {
+      delete options.addr;
+      options.host = arg || options.host;
+    } else if (HOST_SUFFIX_RE.test(arg)) {
+      delete options.port;
+      delete options.addr;
+      var port = RegExp.$1;
+      if (port > 0) {
+        options.port = parseInt(port, 10) || options.port;
+      }
+      var host = arg.slice(0, - port.length - 1);
+      if (host[0] === '[') {
+        host = host.substring(1);
+      }
+      var lastIndex = host.length - 1;
+      if (host[lastIndex] === ']') {
+        host = host.substring(0, lastIndex);
+      }
+      if (host && (net.isIP(host) || HOST_RE.test(host))) {
+        options.host = host || options.host;
+      }
+    } else {
+      delete options.port;
+      delete options.host;
+      if (URL_RE.test(arg)) {
+        options.addr = { url: arg };
+      } else {
+        options.addr = { file: arg };
+      }
+    }
+  });
+  if (!options.addr) {
+    var host = options.host || '127.0.0.1';
+    var port = options.port || util.getDefaultPort();
+    options.addr = { url: 'http://' + host + ':' + port + '/cgi-bin/rootca' };
+  }
+  install(options.addr);
+};

package/bin/ca/index.d.ts

@@ -0,0 +1 @@
+export default function (certFile: String): void;

package/bin/ca/index.js

@@ -0,0 +1,43 @@
+var spawnSync = require('child_process').spawnSync;
+
+function checkSuccess(result) {
+  var stderr = result.stderr;
+  if (stderr && stderr.length) {
+    throw new Error(stderr + '');
+  }
+}
+
+function getKeyChain() {
+  var result = spawnSync('security', ['default-keychain']);
+  checkSuccess(result);
+  return (result.stdout + '').split('"')[1];
+}
+
+function installMac(certPath) {
+  var result = spawnSync('security', ['add-trusted-cert', '-k', getKeyChain(), certPath]);
+  checkSuccess(result);
+  var msg = result.stdout + '';
+  if (/Error:/i.test(msg)) {
+    throw new Error(msg);
+  }
+}
+
+
+function installWin(certFile) {
+  var result = spawnSync('certutil', ['-addstore', '-user', 'Root', certFile]);
+  checkSuccess(result);
+  if (/ERROR_CANCELLED/i.test(result.stdout + '')) {
+    throw new Error('The authorization was canceled by the user.');
+  }
+}
+
+module.exports = function(certFile) {
+  var platform = process.platform;
+  if (platform === 'darwin') {
+    return installMac(certFile);
+  }
+  if (platform === 'win32') {
+    return installWin(certFile);
+  }
+  throw new Error('Platform ' + platform + ' is unsupported to install root CA for now.');
+};

package/bin/proxy/cli.js

@@ -2,33 +2,33 @@ var net = require('net');
 var proxy = require('./index');
 var util = require('../util');
 
-var readConfig = util.readConfig;
 var OFF_RE = /^(?:o|0|-{0,2}off)$/i;
 var BYPASS_RE = /^(?:-{0,2}bypass|-x|-b)$/i;
 var NUM_RE = /^\d+$/;
 var HOST_SUFFIX_RE = /\:(\d+|auto)?$/;
 var HOST_RE = /^[a-z\d_-]+(?:\.[a-z\d_-]+)*$/i;
 
-function getDefaultPort() {
-  var conf = readConfig();
-  conf = conf && conf.options;
-  var port = conf && conf.port;
-  return port > 0 ? port : 8899;
-}
-
 function enableProxy(options) {
-  if (proxy.enableProxy(options)) {
-    util.info('Setting global proxy (' + options.host + ':' + options.port + ') successful.');
-  } else {
-    util.error('Failed to set global proxy (' + options.host + ':' + options.port + ').');
+  try {
+    if (proxy.enableProxy(options)) {
+      util.info('Setting global proxy (' + options.host + ':' + options.port + ') successful.');
+    } else {
+      util.error('Failed to set global proxy (' + options.host + ':' + options.port + ').');
+    }
+  } catch (e) {
+    util.error(e.message);
   }
 }
 
 function disableProxy() {
-  if (proxy.disableProxy()) {
-    util.info('Turn off global proxy successful.');
-  } else {
-    util.error('Failed to turn off global proxy.');
+  try {
+    if (proxy.disableProxy()) {
+      util.info('Turn off global proxy successful.');
+    } else {
+      util.error('Failed to turn off global proxy.');
+    }
+  } catch (e) {
+    util.error(e.message);
   }
 }
 
@@ -69,7 +69,7 @@ module.exports = function(argv) {
     }
   });
   if (!options.port) {
-    options.port = getDefaultPort();
+    options.port = util.getDefaultPort();
   }
   options.host = options.host || '127.0.0.1';
   enableProxy(options);

package/bin/util.js

@@ -7,12 +7,13 @@ var config = require('../lib/config');
 var colors = require('colors/safe');
 var path = require('path');
 var createHmac = require('crypto').createHmac;
+
 /*eslint no-console: "off"*/
 var CHECK_RUNNING_CMD = process.platform === 'win32' ? 
   'tasklist /fi "PID eq %s" | findstr /i "node.exe"'
   : 'ps -f -p %s | grep "node"';
 var isWin = process.platform === 'win32';
-  
+
 function isRunning(pid, callback) {
   pid ? cp.exec(util.format(CHECK_RUNNING_CMD, pid), 
     function (err, stdout, stderr) {
@@ -88,6 +89,15 @@ function showUsage(isRunning, options, restart) {
   if (parseInt(process.version.slice(1), 10) < 6) {
     warn(colors.bold('\nWarning: The current Node version is too low, access https://nodejs.org to install the latest version, or may not be able to Capture HTTPS CONNECTs\n'));
   }
+  var bypass = options.init;
+  if (bypass == null) {
+    return;
+  }
+  return {
+    host: options.host || '127.0.0.1',
+    port: port,
+    bypass: typeof bypass === 'string' ? bypass : undefined
+  };
 }
 
 exports.showUsage = showUsage;
@@ -146,4 +156,11 @@ exports.readConfigList = readConfigList;
 exports.getHash = function(str) {
   var hmac = createHmac('sha256', 'a secret');
   return hmac.update(str).digest('hex');
-};
+};
+
+exports.getDefaultPort = function () {
+  var conf = readConfig();
+  conf = conf && conf.options;
+  var port = conf && conf.port;
+  return port > 0 ? port : 8899;
+};

package/bin/whistle.js

@@ -8,14 +8,29 @@ var showStatus = require('./status');
 var util = require('./util');
 var plugin = require('./plugin');
 var setProxy = require('./proxy/cli');
+var installCA = require('./ca/cli');
 
-var showUsage = util.showUsage;
 var error = util.error;
 var info = util.info;
+var hasInit;
+
+function handleEnd(err, options, restart) {
+  options = util.showUsage(err, options, restart);
+  if (!options || !hasInit) {
+    return;
+  }
+  var host = options.host + ':' + options.port;
+  var argv = [host];
+  if (options.bypass) {
+    argv.push('-x', options.bypass);
+  }
+  setProxy(argv);
+  installCA([host]);
+}
 
 function showStartupInfo(err, options, debugMode, restart) {
   if (!err || err === true) {
-    return showUsage(err, options, restart);
+    return handleEnd(err, options, restart);
   }
   if (/listen EADDRINUSE/.test(err)) {
     options = util.formatOptions(options);
@@ -55,7 +70,7 @@ program.setConfig({
       showStartupInfo(err, options, true);
       return;
     }
-    showUsage(false, options);
+    handleEnd(false, options);
     console.log('Press [Ctrl+C] to stop ' + config.name + '...');
   },
   startCallback: showStartupInfo,
@@ -85,6 +100,8 @@ program
   .description('Add rules from local js file (.whistle.js by default)');
 program.command('proxy')
   .description('Set global proxy');
+program.command('ca')
+  .description('Install root CA');
 program.command('install')
   .description('Install whistle plugin');
 program.command('uninstall')
@@ -117,6 +134,7 @@ program
   .option('-R, --reqCacheSize [reqCacheSize]', 'set the cache size of request data (600 by default)', String, undefined)
   .option('-F, --frameCacheSize [frameCacheSize]', 'set the cache size of webSocket and socket\'s frames (512 by default)', String, undefined)
   .option('-A, --addon [pluginPaths]', 'add custom plugin paths', String, undefined)
+  .option('--init [bypass]', 'auto set global proxy (and bypass) and install root CA')
   .option('--config [workers]', 'start the cluster server and set worker number (os.cpus().length by default)', String, undefined)
   .option('--cluster [config]', 'load the startup config from a local file', String, undefined)
   .option('--dnsServer [dnsServer]', 'set custom dns servers', String, undefined)
@@ -135,6 +153,10 @@ var removeItem = function(list, name) {
   var i = list.indexOf(name);
   i !== -1 && list.splice(i, 1);
 };
+if (argv.indexOf('--init') !== -1) {
+  hasInit = true;
+  process.env.WHISTLE_MODE = (process.env.WHISTLE_MODE || '') + '|persistentCapture';
+}
 if (cmd === 'status') {
   var all = argv[3] === '--all' || argv[3] === '-l';
   if (argv[3] === '-S') {
@@ -143,6 +165,8 @@ if (cmd === 'status') {
   showStatus(all, storage);
 } else if (cmd === 'proxy') {
   setProxy(Array.prototype.slice.call(argv, 3));
+} else if (cmd === 'ca') {
+  installCA(Array.prototype.slice.call(argv, 3));
 } else if (/^([a-z]{1,2})?uni(nstall)?$/.test(cmd)) {
   plugin.uninstall(Array.prototype.slice.call(argv, 3));
 } else if (/^([a-z]{1,2})?i(nstall)?$/.test(cmd)) {

package/biz/webui/cgi-bin/certs/remove.js

@@ -2,7 +2,7 @@ var ca = require('../../../../lib/https/ca');
 
 
 module.exports = function(req, res) {
-  ca.removeCert(req.body.filename);
+  ca.removeCert(req.body);
   var isparsed = req.query.dataType === 'parsed';
   res.json(isparsed ? ca.getCustomCertsInfo() : ca.getCustomCertsFiles());
 };

package/biz/webui/cgi-bin/composer.js

@@ -3,6 +3,7 @@ var gzip = require('zlib').gzip;
 var tls = require('tls');
 var crypto = require('crypto');
 var Buffer = require('safe-buffer').Buffer;
+var extend = require('extend');
 var common = require('../../../lib/util/common');
 var config = require('../../../lib/config');
 var util = require('../../../lib/util');
@@ -16,6 +17,7 @@ var getRawHeaders = hparser.getRawHeaders;
 var getRawHeaderNames = hparser.getRawHeaderNames;
 var parseReq = hparser.parse;
 var MAX_LENGTH = 1024 * 512;
+var MAX_REQ_COUNT = 100;
 var TLS_PROTOS = 'https:,wss:,tls:'.split(',');
 var PROXY_OPTS = {
   host: config.host || '127.0.0.1',
@@ -45,38 +47,53 @@ function drain(socket) {
   socket.on('data', util.noop);
 }
 
-function handleConnect(options, cb) {
+function getReqCount(count) {
+  return count > 0 ? Math.min(count, MAX_REQ_COUNT) : 1;
+}
+
+function handleConnect(options, cb, count) {
+  count = getReqCount(count);
   options.headers['x-whistle-policy'] = 'tunnel';
-  config.connect({
-    host: options.hostname,
-    port: options.port || 443,
-    proxyHost: PROXY_OPTS.host,
-    proxyPort: PROXY_OPTS.port,
-    headers: options.headers
-  }, function(socket, svrRes, err) {
-    if (err) {
-      return cb && cb(err);
+  var origOpts = options;
+  var lastIndex = count - 1;
+  for (var i = 0; i < count; i++) {
+    var execCb;
+    if (i === lastIndex) {
+      execCb = cb;
+    } else {
+      options = extend({}, origOpts);
     }
-    if (!err) {
-      if (TLS_PROTOS.indexOf(options.protocol) !== -1) {
-        socket = tls.connect({
-          rejectUnauthorized: config.rejectUnauthorized,
-          socket: socket,
-          servername: options.hostname
-        });
+    config.connect({
+      host: options.hostname,
+      port: options.port || 443,
+      proxyHost: PROXY_OPTS.host,
+      proxyPort: PROXY_OPTS.port,
+      headers: options.headers
+    }, function(socket, svrRes, err) {
+      if (err) {
+        return execCb && execCb(err);
       }
-      drain(socket);
-      var data = options.body;
-      if (data && data.length) {
-        socket.write(data);
-        options.body = data = null;
+      if (!err) {
+        if (TLS_PROTOS.indexOf(options.protocol) !== -1) {
+          socket = tls.connect({
+            rejectUnauthorized: config.rejectUnauthorized,
+            socket: socket,
+            servername: options.hostname
+          });
+        }
+        drain(socket);
+        var data = options.body;
+        if (data && data.length) {
+          socket.write(data);
+          options.body = data = null;
+        }
       }
-    }
-    cb && cb(null, {
-      statusCode: svrRes.statusCode,
-      headers: svrRes.headers
-    });
-  }).on('error', cb || util.noop);
+      execCb && execCb(null, {
+        statusCode: svrRes.statusCode,
+        headers: svrRes.headers
+      });
+    }).on('error', execCb || util.noop);
+  }
 }
 
 function getReqRaw(options) {
@@ -86,52 +103,64 @@ function getReqRaw(options) {
   return raw.join('\r\n') + '\r\n\r\n';
 }
 
-function handleWebSocket(options, cb) {
+function handleWebSocket(options, cb, count) {
+  count = getReqCount(count);
   if (options.protocol === 'https:' || options.protocol === 'wss:') {
     options.headers[config.HTTPS_FIELD] = 1;
   }
   var binary = !!options.headers['x-whistle-frame-binary'];
   delete options.headers['x-whistle-frame-binary'];
-  util.connect(PROXY_OPTS, function(err, socket) {
-    if (err) {
-      cb && cb(err);
+  var origOpts = options;
+  var lastIndex = count - 1;
+  for (var i = 0; i < count; i++) {
+    var execCb;
+    if (i === lastIndex) {
+      execCb = cb;
     } else {
-      socket.write(getReqRaw(options));
-      var data = options.body;
-      if ((!data || !data.length) && !cb) {
-        return drain(socket);
-      }
-      parseReq(socket, function(e) {
-        if (e) {
-          socket.destroy();
-          return cb && cb(e);
+      options = extend({}, origOpts);
+    }
+    util.connect(PROXY_OPTS, function(err, socket) {
+      if (err) {
+        execCb && execCb(err);
+      } else {
+        socket.write(getReqRaw(options));
+        var data = options.body;
+        if ((!data || !data.length) && !cb) {
+          return drain(socket);
         }
-        var statusCode = socket.statusCode;
-        if (statusCode == 101) {
-          var sender = getSender(socket);
-          if (data) {
-            sender.send(data, {
-              mask: true,
-              binary: binary
-            }, util.noop);
-            options.body = data = null;
+        parseReq(socket, function(e) {
+          if (e) {
+            socket.destroy();
+            return execCb && execCb(e);
           }
-          socket.body = '';
-          drain(socket);
-        } else {
-          socket.destroy();
-        }
-        cb && cb(null, {
-          statusCode: statusCode,
-          headers: socket.headers || {},
-          body: socket.body || ''
-        });
-      }, true);
-    }
-  });
+          var statusCode = socket.statusCode;
+          if (statusCode == 101) {
+            var sender = getSender(socket);
+            if (data) {
+              sender.send(data, {
+                mask: true,
+                binary: binary
+              }, util.noop);
+              options.body = data = null;
+            }
+            socket.body = '';
+            drain(socket);
+          } else {
+            socket.destroy();
+          }
+          execCb && execCb(null, {
+            statusCode: statusCode,
+            headers: socket.headers || {},
+            body: socket.body || ''
+          });
+        }, true);
+      }
+    });
+  }
 }
 
-function handleHttp(options, cb) {
+function handleHttp(options, cb, count) {
+  count = getReqCount(count);
   if (options.protocol === 'https:') {
     options.headers[config.HTTPS_FIELD] = 1;
   }
@@ -139,46 +168,56 @@ function handleHttp(options, cb) {
   options.hostname = null;
   options.host = PROXY_OPTS.host;
   options.port = PROXY_OPTS.port;
-  var client = http.request(options, function(res) {
-    if (cb) {
-      res.on('error', cb);
-      var buffer;
-      res.on('data', function(data) {
-        if (buffer !== null) {
-          buffer = buffer ? Buffer.concat([buffer, data]) : data;
-          if (buffer.length > MAX_LENGTH) {
-            buffer = null;
-          }
-        }
-      });
-      res.on('end', function() {
-        zlib.unzip(res.headers['content-encoding'], buffer, function(err, body) {
-          var headers = res.headers;
-          if (typeof headers.trailer === 'string' && headers.trailer.indexOf(',') !== -1) {
-            headers.trailer = headers.trailer.split(',');
-          }
-          var result = {
-            statusCode: res.statusCode,
-            headers: headers,
-            trailers: res.trailers,
-            rawHeaderNames: getRawHeaderNames(res.rawHeaders),
-            rawTrailerNames: getRawHeaderNames(res.rawTrailers)
-          };
-          if (err) {
-            result.body = err.stack;
-          } else if (body) {
-            result.base64 = body.toString('base64');
-          }
-          cb(null, result);
-        });
-      });
+  var origOpts = options;
+  var lastIndex = count - 1;
+  for (var i = 0; i < count; i++) {
+    var execCb;
+    if (i === lastIndex) {
+      execCb = cb;
     } else {
-      drain(res);
+      options = extend({}, origOpts);
     }
-  });
-  client.on('error', cb || util.noop);
-  client.end(options.body);
-  options.body = null;
+    var client = http.request(options, function(res) {
+      if (execCb) {
+        res.on('error', execCb);
+        var buffer;
+        res.on('data', function(data) {
+          if (buffer !== null) {
+            buffer = buffer ? Buffer.concat([buffer, data]) : data;
+            if (buffer.length > MAX_LENGTH) {
+              buffer = null;
+            }
+          }
+        });
+        res.on('end', function() {
+          zlib.unzip(res.headers['content-encoding'], buffer, function(err, body) {
+            var headers = res.headers;
+            if (typeof headers.trailer === 'string' && headers.trailer.indexOf(',') !== -1) {
+              headers.trailer = headers.trailer.split(',');
+            }
+            var result = {
+              statusCode: res.statusCode,
+              headers: headers,
+              trailers: res.trailers,
+              rawHeaderNames: getRawHeaderNames(res.rawHeaders),
+              rawTrailerNames: getRawHeaderNames(res.rawTrailers)
+            };
+            if (err) {
+              result.body = err.stack;
+            } else if (body) {
+              result.base64 = body.toString('base64');
+            }
+            execCb(null, result);
+          });
+        });
+      } else {
+        drain(res);
+      }
+    });
+    client.on('error', execCb || util.noop);
+    client.end(options.body);
+    options.body = null;
+  }
 }
 
 function getCharset(headers) {
@@ -296,13 +335,15 @@ module.exports = function(req, res) {
     if (err) {
       return handleResponse && handleResponse(err);
     }
+    var count = req.body.repeatCount;
+    count = count > 0 ? count : req.body.repeatTimes;
     if (isWs) {
       options.method = 'GET';
-      handleWebSocket(options, handleResponse);
+      handleWebSocket(options, handleResponse, count);
     } else if (isConn) {
-      handleConnect(options, handleResponse);
+      handleConnect(options, handleResponse, count);
     } else  {
-      handleHttp(options, handleResponse);
+      handleHttp(options, handleResponse, count);
     }
     if (!handleResponse) {
       res.json({ec: 0, em: 'success'});

package/biz/webui/cgi-bin/rootca.js

@@ -1,5 +1,9 @@
 var getRootCAFile = require('../../../lib/https/ca').getRootCAFile;
 
 module.exports = function(req, res) {
-  res.download(getRootCAFile(), 'rootCA.' + (req.query.type === 'cer' ? 'cer' : 'crt'));
+  var type = req.query.type;
+  if (type !== 'cer' && type !== 'pem') {
+    type = 'crt';
+  }
+  res.download(getRootCAFile(), 'rootCA.' + type);
 };

package/biz/webui/htdocs/index.html

@@ -8,6 +8,6 @@
 </head>
 <body style="overscroll-behavior-x: none;">
 <div id="container" class="main"></div>
-<script src="js/index.js?v=2.9.14"></script>
+<script src="js/index.js?v=2.9.16"></script>
 </body>
 </html>