whistle 2.10.1 → 2.10.3

package/lib/service/composer.js

@@ -38,7 +38,12 @@ function saveComposerHistory() {
   } catch (e) {}
 }
 
-function handleComposerHistory(data) {
+function getUrl(url, isConn) {
+  url = isConn ? url.replace(/[?#].*$/, '') : url;
+  return url.length > MAX_URL_LEN ? url.substring(0, MAX_URL_LEN) : url;
+}
+
+function handleComposerHistory(data, isConn) {
   var url = data.url;
   var method = data.method;
   var headers = data.headers;
@@ -50,7 +55,7 @@ function handleComposerHistory(data) {
   var result = {
     date: Date.now(),
     useH2: data.useH2,
-    url: url.length > MAX_URL_LEN ? url.substring(0, MAX_URL_LEN) : url,
+    url: getUrl(url, isConn),
     method:
         method.length > MAX_METHOD_LEN
           ? method.substring(0, MAX_METHOD_LEN)
@@ -335,13 +340,13 @@ function getCharset(headers) {
 exports.handleRequest = function(req, res) {
   var fullUrl = req.body.url;
   if (!fullUrl || typeof fullUrl !== 'string') {
-    return res.json({ec: 0});
+    return res.json({ec: 2, em: 'Invalid URL'});
   }
 
   fullUrl = common.encodeNonLatin1Char(fullUrl.replace(/#.*$/, ''));
   var options = common.parseUrl(common.setProtocol(fullUrl));
   if (!options.host) {
-    return res.json({ec: 0});
+    return res.json({ec: 2, em: 'Invalid URL'});
   }
   var protocol = options.protocol;
   if (protocol) {
@@ -386,7 +391,7 @@ exports.handleRequest = function(req, res) {
     }
   }
   if (!req.body.noStore && req.body.needResponse && common.checkHistory(req.body)) {
-    handleComposerHistory(req.body);
+    handleComposerHistory(req.body, isConn);
     dataCenter.saveData({
       type: 'composer',
       history: composerHistory

package/lib/{util/log-server.js → service/console-log.js}

@@ -1,6 +1,6 @@
 var MAX_LENGTH = 800;
 var MIN_LENGTH = 720;
-var SIZE = 1024 * 64;
+var SIZE = 1024 * 512;
 var COUNT = 100;
 var logIndex = 0;
 var logs = [];
@@ -45,40 +45,43 @@ function getLogs(startTime, count, logId) {
   return sliceLogs(0, count, logId);
 }
 
-module.exports = function init(proxy) {
-  proxy.addLog = function set(log) {
-    if (!log) {
-      return;
-    }
+function getLogIndex() {
+  ++logIndex;
+  if (logIndex > 999999) {
+    logIndex = 0;
+  }
+  return logIndex;
+}
 
-    var now = Date.now();
-    var text = log.text;
-    if (text == null) {
-      text = '';
-    } else if (typeof text != 'string') {
-      text += '';
-    }
-    var overflow = text.length - SIZE;
-    logs.push({
-      id: now + '-' + ++logIndex,
-      logId: log.id,
-      date: (log.t && parseInt(log.t, 10)) || now,
-      level: LEVEL_RE.test(log.level)
+exports.addLog = function set(log) {
+  if (!log) {
+    return;
+  }
+
+  var now = Date.now();
+  var text = log.text;
+  if (text == null) {
+    text = '';
+  } else if (typeof text != 'string') {
+    text += '';
+  }
+  var overflow = text.length - SIZE;
+  logs.push({
+    id: now + '-' + getLogIndex(),
+    logId: log.id,
+    date: (log.t && parseInt(log.t, 10)) || now,
+    level: LEVEL_RE.test(log.level)
         ? log.level
         : 'info',
-      text: overflow > 9 ? text.substring(0, SIZE) + '...(' + overflow + ')' : text
-    });
-    var len = logs.length;
-    if (len > MAX_LENGTH) {
-      logs = logs.slice(len - MIN_LENGTH, len);
-      if (logIndex > 100000) {
-        logIndex = 0;
-      }
-    }
-  };
-  proxy.getLogs = getLogs;
-  proxy.getLatestId = function () {
-    var last = logs[logs.length - 1];
-    return last && last.id;
-  };
+    text: overflow > 9 ? text.substring(0, SIZE) + '...(' + overflow + ')' : text
+  });
+  var len = logs.length;
+  if (len > MAX_LENGTH) {
+    logs = logs.slice(len - MIN_LENGTH, len);
+  }
+};
+exports.getLogs = getLogs;
+exports.getLatestId = function () {
+  var last = logs[logs.length - 1];
+  return last && last.id;
 };

package/lib/service/index.js

@@ -170,9 +170,7 @@ var saveData = function(type, data) {
       method: 'POST',
       url: 'http://127.0.0.1:' + curOptions.port + '/cgi-bin/service/save',
       strictMode: true,
-      headers: {
-        'content-type': 'application/json'
-      },
+      headers: { 'content-type': 'application/json' },
       body: body
     }, function (err) {
       if (err) {
@@ -213,7 +211,7 @@ function sendData() {
   });
 }
 
-process.nextTick(sendData);
+process.on('whistleStarted', sendData);
 
 module.exports = loadService;
 

package/lib/service/service.js

@@ -16,6 +16,7 @@ var composer = require('./composer');
 var handleComposeData = require('./compose-data');
 var Limiter = require('async-limiter');
 var common = require('../util/common');
+var logger = require('./console-log');
 
 process.env.NODE_TLS_REJECT_UNAUTHORIZED = '1';
 
@@ -26,21 +27,19 @@ var forwardRequest = dataCenter.forwardRequest;
 
 var INSTALL_SCRIPT = path.join(__dirname, 'install.js');
 var installChild;
-
+var LOG_RE = /^(\d+)\r(fatal|error|warn|info|debug|log)\r([^\s]*)\r/;
 var TEMP_PLUGINS_PATH = path.join(common.getWhistlePath(), '.temp_plugins');
 var SESSIONS_FILE_RE = /\.(txt|json|saz)$/i;
 var limiter = new Limiter({ concurrency: 10 });
 var MAX_PLUGINS = 10;
-var TEMP_FILES_PATH;
+var TEMP_FILES_PATH = common.TEMP_FILES_PATH;
 var SAVED_SESSIONS_PATH;
 var LIMIT_SIZE = 1024 * 1024 * 128;
 var MAX_TEMP_SIZE = 1024 * 1024 * 12;
 var MAX_PLUGIN_SIZE = MAX_TEMP_SIZE;
 var MAX_TEMP_PLUGINS_AGE = 1000 * 60 * 10; // 10 minutes
-var TEMP_FILE_RE = /^[\da-f]{64}$/;
 var CR = '\r';
 var jsonParser = bodyParser.json({ limit: LIMIT_SIZE });
-var urlencodedParser = bodyParser.urlencoded({ extended: true, limit: '1mb'});
 var PLUGIN_SEP = /\s*,\s*/;
 var config;
 var curWhistleId;
@@ -48,7 +47,6 @@ var hasWhistleToken;
 var whistleIdFile;
 var storage = multer.memoryStorage();
 var INVALID_NAME_RE = /[\u001e\u001f\u200e\u200f\u200d\u200c\u202a\u202d\u202e\u202c\u206e\u206f\u206b\u206a\u206d\u206c'<>:"\\/|?*]+/g;
-var SPACE_RE = /\s+/g;
 var upload = multer({
   storage: storage,
   fieldSize: LIMIT_SIZE
@@ -67,6 +65,19 @@ var ensureTempPluginsDir = function() {
 
 ensureTempPluginsDir();
 
+function parseLog(log) {
+  if (!log || typeof log !== 'string') {
+    return;
+  }
+  var match = LOG_RE.exec(log);
+  return match && {
+    t: match[1],
+    level: match[2],
+    id: match[3],
+    text: log.substring(match[0].length)
+  };
+}
+
 process.on('data', function(data) {
   if (data) {
     if (data.type === 'whistleId') {
@@ -79,7 +90,7 @@ process.on('data', function(data) {
 });
 
 function getFilename(filename, time, count) {
-  filename = common.isString(filename) ? filename.replace(INVALID_NAME_RE, '').replace(SPACE_RE, ' ').substring(0, 64).trim() : '';
+  filename = common.isString(filename) ? filename.replace(INVALID_NAME_RE, '').replace(/\s+/g, ' ').substring(0, 64).trim() : '';
   time = time || Date.now();
   return common.getMonth(time) + '/' + filename + '_' + count + '_' + time;
 }
@@ -96,7 +107,7 @@ function saveSessions(data, cb) {
         return cb(err);
       }
       var filename = getFilename(data.filename, 0, count);
-      util.writeFile(path.join(SAVED_SESSIONS_PATH, filename), buf, function(e) {
+      common.writeFile(path.join(SAVED_SESSIONS_PATH, filename), buf, function(e) {
         !e && saveData({ type: 'savedData', filename: filename, buffer: buf } );
         cb(e);
       });
@@ -155,7 +166,8 @@ function getTempFiles(list, cb) {
     }
   };
   list = list.forEach(function(filename) {
-    if (!TEMP_FILE_RE.test(filename)) {
+    filename = common.getTempFile(filename);
+    if (!filename) {
       return execCb();
     }
     filename = path.join(TEMP_FILES_PATH, filename);
@@ -182,7 +194,7 @@ function writeTempPlugin(plugin, data, cb) {
   plugin = plugin.split('/').pop();
   var tempPath = path.join(TEMP_PLUGINS_PATH, plugin);
   ensureTempPluginsDir();
-  util.writeFile(tempPath, data, function(e) {
+  common.writeFile(tempPath, data, function(e) {
     cb(e, e ? null : tempPath);
   });
 }
@@ -296,6 +308,10 @@ function sessionsHandler() {
     function abort() {
       res.destroy();
     }
+    if (req.headers[common.WHISTLE_UID_HEADER] !== config.uid) {
+      return res.status(403).end('Forbidden');
+    }
+    delete req.headers[common.WHISTLE_UID_HEADER];
     next();
   });
   app.get('/service/*', forwardRequest);
@@ -336,7 +352,7 @@ function sessionsHandler() {
       } catch (e) {}
     });
   });
-  app.post('/cgi-bin/plugins/install', urlencodedParser, function(req, res) {
+  app.post('/cgi-bin/plugins/install', bodyParser.urlencoded({ extended: true, limit: '1mb'}), function(req, res) {
     var plugins = req.body.plugins;
     var clientId = req.body.clientId;
     plugins = common.isString(plugins) ? plugins.trim().split(PLUGIN_SEP) : null;
@@ -444,8 +460,9 @@ function sessionsHandler() {
       });
     }
     var filename = req.query.filename;
-    if (TEMP_FILE_RE.test(filename)) {
-      filename = path.join(TEMP_FILES_PATH, filename);
+    var tempFile = common.getTempFile(filename);
+    if (tempFile) {
+      filename = path.join(TEMP_FILES_PATH, tempFile);
     }
     getFile(filename, function(em, data) {
       if (em) {
@@ -458,7 +475,7 @@ function sessionsHandler() {
   app.use('/cgi-bin/temp/create',
     jsonParser,
     function(req, res) {
-      util.writeTempFile(TEMP_FILES_PATH, getContent(req.body), function(e, filename) {
+      common.writeTempFile(getContent(req.body), function(e, filename) {
         if (e) {
           return res.json({ ec: 2, em: e.message });
         }
@@ -491,8 +508,12 @@ function sessionsHandler() {
       }
     }
   );
+  app.use('/cgi-bin/log/set', function(req, _, next) {
+    req.headers['content-type'] = 'application/json';
+    next();
+  });
   app.use(bodyParser.urlencoded({ extended: true, limit: LIMIT_SIZE }));
-  app.use(bodyParser.json());
+  app.use(jsonParser);
   app.use('/cgi-bin/sessions/export', function (req, res) {
     var body = req.body;
     var type = body.exportFileType;
@@ -509,6 +530,35 @@ function sessionsHandler() {
       download(body);
     });
   });
+
+  app.use('/cgi-bin/log/set', function(req, res) {
+    res.setHeader('content-type', 'image/png');
+    if (req.method === 'POST') {
+      var list = req.body.list;
+      if (Array.isArray(list)) {
+        var len = Math.min(list.length, 20);
+        for (var i = 0; i < len; i++) {
+          logger.addLog(parseLog(list[i]));
+        }
+      }
+    } else {
+      logger.addLog(req.query);
+    }
+    res.end();
+  });
+  app.use('/cgi-bin/log/get', function(req, res) {
+    var query = req.query;
+    var startLogTime = query.startLogTime;
+    var init = startLogTime == -4;
+    var stopRecordConsole = startLogTime == -3;
+    var curLogId = logger.getLatestId();
+    common.sendGzip(req, res, {
+      ec: 0,
+      log: init || stopRecordConsole ? [] : logger.getLogs(startLogTime, query.count, query.logId),
+      curLogId: stopRecordConsole ? undefined : curLogId,
+      lastLogId: init ? (curLogId || -2) : (stopRecordConsole ? curLogId : undefined)
+    });
+  });
   return app;
 }
 
@@ -564,7 +614,6 @@ function checkWhistleId(cb) {
 module.exports = function (options, callback) {
   config = options;
   whistleIdFile = path.join(options.baseDir, options.storage || '', '.whistle_id');
-  TEMP_FILES_PATH = options.TEMP_FILES_PATH;
   SAVED_SESSIONS_PATH = options.SAVED_SESSIONS_PATH;
   dataCenter.setup(options);
   composer.setup(options);

package/lib/service/util.js

@@ -1,7 +1,5 @@
-var crypto = require('crypto');
 var path = require('path');
 var fs = require('fs');
-var fse = require('fs-extra2');
 var zlib = require('../util/zlib');
 var common = require('../util/common');
 
@@ -426,16 +424,6 @@ function parseFrames(res, content, callback) {
 
 exports.parseFrames = parseFrames;
 
-function getHexHash(ctn, key) {
-  return crypto
-    .createHmac('sha256', key || 'whistle_temp_files')
-    .update(ctn || '').digest('hex').toLowerCase();
-}
-
-function descSorter(a, b) {
-  return a > b ? -1 : 1;
-}
-
 var MAX_SESSIONS_FILES = 2000;
 var SAVED_SESSIONS_FILE_RE = /_([1-9]\d*)_(\d+)$/;
 var DIR_RE = /^\d{6}$/;
@@ -492,7 +480,9 @@ exports.getSavedList = function(savedDir, cb) {
         list.push(dir);
       }
     });
-    list = list.sort(descSorter).slice(0, 12).map(function(dir) {
+    list = list.sort(function (a, b) {
+      return a > b ? -1 : 1;
+    }).slice(0, 12).map(function(dir) {
       return path.join(savedDir, dir);
     });
     readSessionsFiles(list, cb);
@@ -504,32 +494,3 @@ var SHARE_TYPE_RE = /Share$/;
 exports.isShareType = function(type) {
   return SHARE_TYPE_RE.test(type);
 };
-
-function writeRetry(filepath, data, callback, retry) {
-  fse.outputFile(filepath, data, function(e) {
-    if (!e || retry) {
-      return callback(e);
-    }
-    writeRetry(filepath, data, callback, true);
-  });
-}
-
-function writeFile(filepath, data, callback) {
-  common.getStat(filepath, function(_, stat) {
-    if (!stat || !stat.isFile()) {
-      return writeRetry(filepath, data, callback);
-    }
-    callback();
-  });
-}
-
-exports.writeFile = writeFile;
-
-function writeTempFile(dir, value, callback) {
-  var filename = getHexHash(value);
-  writeFile(path.join(dir, filename), value, function(err) {
-    callback(err, filename);
-  });
-}
-
-exports.writeTempFile = writeTempFile;

package/lib/socket-mgr.js

@@ -7,28 +7,14 @@ var extend = require('extend');
 var pendingReqList = [];
 var INTERVAL = 22 * 1000;
 var proxy;
-var index = 0;
-var MAX_PAYLOAD = 1024 * 1024;
 var conns = {};
 var PING = Buffer.from('iQA=', 'base64');
 var PONG = Buffer.from('ioAn6ubf', 'base64');
 var PAUSE_STATUS = 1;
 var IGNORE_STATUS = 2;
 var MAX_COMPOSE_FRAME_COUNT = 5;
-
-function getFrameId() {
-  ++index;
-  if (index > 999) {
-    index = 0;
-  }
-  if (index > 99) {
-    return Date.now() + '-' + index;
-  }
-  if (index > 9) {
-    return Date.now() + '-0' + index;
-  }
-  return Date.now() + '-00' + index;
-}
+var getFrameId = util.getFrameId;
+var MAX_PAYLOAD = util.MAX_FRAME_PAYLOAD;
 
 exports = module.exports = function (p) {
   proxy = p;

package/lib/tunnel.js

@@ -47,8 +47,7 @@ function tunnelProxy(server, proxy, type) {
     var headers = req.headers;
     if (headers[config.WEBUI_HEAD]) {
       delete headers[config.WEBUI_HEAD];
-      reqSocket.destroy();
-      return;
+      return reqSocket.destroy();
     }
     req.fromHttpServer = reqSocket.fromHttpServer = fromHttpServer;
     req.fromHttpsServer = reqSocket.fromHttpsServer = fromHttpsServer;
@@ -83,8 +82,7 @@ function tunnelProxy(server, proxy, type) {
     var clientInfo = util.parseClientInfo(req);
     reqSocket._disabledProxyRules = req._disabledProxyRules;
     reqSocket.clientIp = req.clientIp = util.getClientIp(req, clientInfo[0]);
-    reqSocket.clientPort = req.clientPort =
-      clientInfo[1] || util.getClientPort(req);
+    reqSocket.clientPort = req.clientPort = clientInfo[1] || util.getClientPort(req);
     req._remoteAddr = clientInfo[2] || util.getRemoteAddr(req);
     req._remotePort = clientInfo[3] || util.getRemotePort(req);
     delete headers[config.CLIENT_PORT_HEADER];

package/lib/upgrade.js

@@ -105,7 +105,7 @@ function upgradeHandler(req, socket) {
   util.addTunnelData(socket, headers);
   socket._clientId = util.getComposerClientId(headers);
   var getBuffer = function (method, newHeaders, path) {
-    var rawData = (method || 'GET') + ' ' + (path || socket.url || req.url) + ' ' + 'HTTP/1.1';
+    var rawData = (method || 'GET') + ' ' + (path || socket.url || req.url) + ' HTTP/1.1';
     newHeaders = formatHeaders(newHeaders || headers, req.rawHeaders);
     if (newHeaders = getRawHeaders(newHeaders)) {
       rawData += '\r\n' + newHeaders;

package/lib/util/common.js

@@ -67,7 +67,81 @@ var PATH_SEP_RE = /[\\/]/;
 var CUR_PATH_RE = /^\.[\\/]/;
 var CLIENT_PORT_HEADER = 'x-whistle-client-port';
 var ALLOW_ACK = 'x-whistle-allow-tunnel-ack';
+var PLUS_RE = /\+/g;
+var TOKEN = '\r\u0000\n\u0003\r';
+var PROP_SEP_RE = /(\\*)([|&]|\\[stnrfv])/g;
+var TEMP_FILE_RE = /^(?:\/?temp\/)?([\da-f]{64})(?:\.[\w.-]+)?$/;
+var TEMP_FILES_PATH = path.join(getWhistlePath(), 'temp_files');
 
+
+exports.WHISTLE_UID_HEADER = '_x-whistle-uid_';
+
+exports.getTempFile = function(str) {
+  var match = TEMP_FILE_RE.exec(str);
+  return match && match[1];
+
+};
+
+var PROPS_CHAR = {
+  s: ' ',
+  t: '\t',
+  n: '\n',
+  r: '\r',
+  f: '\f',
+  v: '\v'
+};
+
+function replaceToken(s, val) {
+  return s.split(TOKEN).join(val);
+}
+
+exports.parseProps = function(str) {
+  return str.replace(PROP_SEP_RE, function(_, slash, val) {
+    var isChar = val.length > 1;
+    if (isChar) {
+      slash += '/';
+      val = val.substring(1);
+    } else if (!slash) {
+      return TOKEN;
+    }
+    var len = slash.length;
+    slash = slash.substring(0, Math.floor(len / 2));
+    if (isChar) {
+      return slash + (len % 2 ? PROPS_CHAR[val] : val);
+    }
+    return slash + (len % 2 ? val : TOKEN);
+  }).split(TOKEN);
+};
+
+var decoder = {
+  decodeURIComponent: function (s) {
+    s = replaceToken(s, '+');
+    return qs.unescape(s);
+  }
+};
+var rawDecoder = {
+  decodeURIComponent: function (s) {
+    return replaceToken(s, '+');
+  }
+};
+var rawDecoder2 = {
+  decodeURIComponent: function (s) {
+    return s;
+  }
+};
+
+exports.parseQuery = function (str, sep, eq, escape) {
+  try {
+    if (str.indexOf('+') === -1 || str.indexOf(TOKEN) !== -1) {
+      return qs.parse(str, sep, eq, escape ? rawDecoder2 : undefined);
+    }
+    str = str.replace(PLUS_RE, TOKEN);
+    return qs.parse(str, sep, eq, escape ? rawDecoder : decoder);
+  } catch (e) {}
+  return '';
+};
+
+exports.TEMP_FILES_PATH = TEMP_FILES_PATH;
 exports.ALLOW_ACK = ALLOW_ACK;
 exports.CONN_TIMEOUT = CONN_TIMEOUT;
 exports.TGZ_FILE_NAME_RE = /^(?:[\w.~-]+-)?whistle\.[a-z\d_-]+-\d+\.\d+\.\d+[\w.-]*\.(?:tgz|tar(?:\.gz)?)$/;
@@ -414,14 +488,8 @@ exports.isConnect = function(options) {
   );
 };
 
-var NO_PROTO_RE = /[^a-zA-Z0-9.-]/;
-
 function hasProtocol(url) {
-  var index = isString(url) ? url.indexOf('://') : -1;
-  if (index <= 0) {
-    return false;
-  }
-  return !NO_PROTO_RE.test(url.substring(0, index));
+  return /^[a-zA-Z0-9.-]+:\/\//.test(url);
 }
 
 function setProtocol(url, isHttps) {
@@ -894,7 +962,6 @@ var DIG_RE = /^([+-]?)([1-9]\d{0,15})$/;
 var NUM_RE = /^(?:0|[1-9]\d*)$/;
 var ARR_RE = /\[(0|[1-8]\d{0,15}|9\d{0,14})\]$/;
 var DOT_RE = /(\\+)\./g;
-var CR_RE = /\r/g;
 
 function isString(str) {
   return str && typeof str === 'string';
@@ -963,12 +1030,6 @@ function parseKey(key) {
   return list;
 }
 
-function replaceDot(_, slash) {
-  var len = slash.length;
-  slash = slash.substring(0, Math.floor(len / 2));
-  return slash + (len % 2 ? '\r' : '.');
-}
-
 function parseKeys(key) {
   if (!isString(key)) {
     return key;
@@ -977,13 +1038,17 @@ function parseKeys(key) {
   if (key.indexOf('.') === -1) {
     return parseKey(key);
   }
-  key = key.replace(DOT_RE, replaceDot);
+  key = key.replace(DOT_RE, function (_, slash) {
+    var len = slash.length;
+    slash = slash.substring(0, Math.floor(len / 2));
+    return slash + (len % 2 ? TOKEN : '.');
+  });
   if (key.indexOf('.') === -1) {
-    return parseKey(key.replace(CR_RE, '.'));
+    return parseKey(replaceToken(key, '.'));
   }
   var result = [];
   key.split('.').forEach(function(k) {
-    k = parseKey(k.trim().replace(CR_RE, '.'));
+    k = parseKey(replaceToken(k.trim(), '.'));
     if (Array.isArray(k)) {
       result = result.concat(k);
     } else {
@@ -1937,3 +2002,36 @@ exports.getSessionInfo = function(headers) {
   result._ = rawResult;
   return result;
 };
+
+function writeRetry(filepath, data, callback, retry) {
+  fse.outputFile(filepath, data, function(e) {
+    if (!e || retry) {
+      return callback(e);
+    }
+    writeRetry(filepath, data, callback, true);
+  });
+}
+
+function writeFile(filepath, data, callback) {
+  getStat(filepath, function(_, stat) {
+    if (!stat || !stat.isFile()) {
+      return writeRetry(filepath, data, callback);
+    }
+    callback();
+  });
+}
+
+exports.writeFile = writeFile;
+
+function getHexHash(ctn, key) {
+  return crypto
+    .createHmac('sha256', key || 'whistle_temp_files')
+    .update(ctn || '').digest('hex').toLowerCase();
+}
+
+exports.writeTempFile = function (value, callback) {
+  var filename = getHexHash(value);
+  writeFile(path.join(TEMP_FILES_PATH, filename), value, function(err) {
+    callback(err, filename);
+  });
+};