t880216t-server 1.5.20

package/index.js

@@ -0,0 +1,89 @@
+/**
+ * Tencent is pleased to support the open source community by making nohost-环境配置与抓包调试平台 available.
+ * Copyright (C) 2020 THL A29 Limited, a Tencent company.  All rights reserved. The below software in
+ * this distribution may have been modified by THL A29 Limited ("Tencent Modifications").
+ * All Tencent Modifications are Copyright (C) THL A29 Limited.
+ * nohost-环境配置与抓包调试平台 is licensed under the MIT License except for the third-party components listed below.
+ */
+
+// 避免第三方模块没处理好异常导致程序crash
+require('whistle/lib/util/patch');
+const fse = require('fs-extra');
+const fs = require('fs');
+const path = require('path');
+const { getWhistlePath } = require('whistle/lib/config');
+const pkg = require('./package.json');
+const initConfig = require('./lib/config');
+
+const PURE_URL_RE = /^(#?(?:https?:)?\/\/[\w.-]+[^?#]*)/;
+
+// 设置存储路径
+const WHISTLE_PATH = process.env.NOHOST_PATH || getWhistlePath();
+process.env.WHISTLE_PATH = WHISTLE_PATH;
+fse.ensureDirSync(process.env.WHISTLE_PATH); // eslint-disable-line
+
+
+const getPureUrl = (url) => {
+  if (!url || !PURE_URL_RE.test(url)) {
+    return;
+  }
+  return RegExp.$1.replace(/\/+$/, '');
+};
+
+const getErrorStack = (err) => {
+  if (!err) {
+    return '';
+  }
+  let stack;
+  try {
+    stack = err.stack;
+  } catch (e) {}
+  stack = stack || err.message || err;
+  const result = [
+    `From: nohost@${pkg.version}`,
+    `Node: ${process.version}`,
+    `Date: ${new Date().toLocaleString()}`,
+    stack];
+  return result.join('\r\n');
+};
+
+const handleUncaughtException = (err) => {
+  if (!err || err.code !== 'ERR_IPC_CHANNEL_CLOSED') {
+    const stack = getErrorStack(err);
+    fs.writeFileSync(path.join(WHISTLE_PATH, 'nohost.log'), `\r\n${stack}\r\n`, { flag: 'a' }); // eslint-disable-line
+    console.error(stack); // eslint-disable-line
+  }
+  process.exit(1);
+};
+
+process.on('unhandledRejection', handleUncaughtException);
+process.on('uncaughtException', handleUncaughtException);
+
+module.exports = (options, cb) => {
+  if (typeof options === 'function') {
+    cb = options;
+    options = {};
+  } else if (!options) {
+    options = {};
+  }
+  if (options.__maxHttpHeaderSize > 0) {
+    process.env.PFORK_MAX_HTTP_HEADER_SIZE = options.__maxHttpHeaderSize;
+  }
+  if (options.debugMode) {
+    const mode = typeof options.mode === 'string' ? options.mode.trim().split(/\s*[|,&]\s*/) : [];
+    if (mode.includes('prod') || mode.includes('production')) {
+      options.debugMode = false;
+    } else {
+      process.env.PFORK_MODE = 'bind';
+    }
+  }
+  const redirect = getPureUrl(options.redirect);
+  if (redirect && redirect[0] === '#') {
+    options.redirect = redirect.substring(1);
+    options.deprecated = true;
+  } else {
+    options.redirect = redirect;
+  }
+  initConfig(options);
+  require('./lib')(options, cb); // eslint-disable-line
+};

package/lib/config.js

@@ -0,0 +1,42 @@
+const { shasum } = require('./util/login');
+
+const getPaths = (paths) => {
+  if (typeof paths === 'string') {
+    paths = paths.trim().split(/\s*[|,;]\s*/);
+  } else if (!Array.isArray(paths)) {
+    return;
+  }
+  paths = paths.filter((path) => {
+    return path && typeof path === 'string';
+  });
+  return paths.length ? paths : undefined;
+};
+
+const getDefaultKey = () => {
+  return shasum(`${Math.random()}\n${Date.now()}\n${Math.random()}`);
+};
+
+module.exports = (options) => {
+  options.totalReqs = 0;
+  options.uiReqs = 0;
+  options.upgradeReqs = 0;
+  options.tunnelReqs = 0;
+  options.totalQps = 0;
+  options.uiQps = 0;
+  options.globalPluginPath = getPaths(options.globalPluginPath);
+  options.accountPluginPath = getPaths(options.accountPluginPath || options.workerPluginPath);
+  options.mainAuthKey = getDefaultKey();
+  let totalReqs = 0;
+  let uiReqs = 0;
+  let now = Date.now();
+  setInterval(() => {
+    const cur = Date.now();
+    const cost = cur - now || 1;
+    options.totalQps = ((options.totalReqs - totalReqs) * 1000 / cost).toFixed(2);
+    options.uiQps = ((options.uiReqs - uiReqs) * 1000 / cost).toFixed(2);
+    totalReqs = options.totalReqs;
+    uiReqs = options.uiReqs;
+    now = cur;
+  }, 1000);
+  module.exports = options;
+};

package/lib/index.js

@@ -0,0 +1,205 @@
+const assert = require('assert');
+const { createServer } = require('http');
+const { onClose } = require('@nohost/router');
+const handleUIRequest = require('./main');
+const { passToWhistle } = require('./main/util');
+const { isUIRequest, setAdmin, checkDomain } = require('./main/storage');
+
+const NOHOST_ENV = 'x-whistle-nohost-env';
+const NOHOST_RULE = 'x-whistle-nohost-rule';
+const NOHOST_VALUE = 'x-whistle-nohost-value';
+const WHISTLE_RULE = 'x-whistle-rule-value';
+const WHISTLE_VALUE = 'x-whistle-key-value';
+const WORKER_RE = /^\$(\d+)?$/;
+const DEFAULT_PORT = 8080;
+const LOCALHOST = '127.0.0.1';
+const HOST_LIST_RE = /^(?:([\w.-]+):)?([1-9]\d{0,4}(?:[-~][1-9]\d{0,4})?(?:,[1-9]\d{0,4}(?:[-~][1-9]\d{0,4})?)*)$/;
+const HTTP_RE = /^\w+\s+\S+\s+HTTP\/1.\d$/mi;
+const SPEC_PATH_RE = /^(?:https?:\/\/[^/?]+)?(\/[_-]\/)/;
+const resolveHost = (host) => {
+  if (/^(?:([\w.-]+):)?([1-9]\d{0,4})$/.test(host) || /^\[([\w.:]+)\]:([1-9]\d{0,4})$/.test(host)) {
+    return {
+      host: RegExp.$1,
+      port: parseInt(RegExp.$2, 10),
+    };
+  }
+  return {};
+};
+
+const resolveHostList = (str) => {
+  if (!str || typeof str !== 'string') {
+    return;
+  }
+  str = str.split(/\||;/);
+  let list;
+  str.forEach((host) => {
+    if (!HOST_LIST_RE.test(host)) {
+      return;
+    }
+    host = RegExp.$1 || LOCALHOST;
+    RegExp.$2.split(',').forEach((port) => {
+      if (/^([1-9]\d{0,4})[-~]([1-9]\d{0,4})$/.test(port)) {
+        const p1 = parseInt(RegExp.$1, 10);
+        const p2 = parseInt(RegExp.$2, 10);
+        const max = Math.min(128, p2 - p1 + 1);
+        for (let i = 0; i < max; ++i) {
+          list = list || [];
+          list.push({ host, port: p1 + i });
+        }
+      } else {
+        list = list || [];
+        list.push({ host, port: parseInt(port, 10) });
+      }
+    });
+  });
+  return list;
+};
+
+const sendEstablished = (socket, err) => {
+  const msg = err ? 'Bad Gateway' : 'Connection Established';
+  const body = String((err && err.stack) || '');
+  socket.write([
+    `HTTP/1.1 ${err ? 502 : 200} ${msg}`,
+    `Content-Length: ${Buffer.byteLength(body)}`,
+    'Proxy-Agent: nohost/server',
+    '\r\n',
+  ].join('\r\n'));
+};
+
+
+const CLIENT_ID_HEAD = 'x-whistle-client-id';
+const CLIENT_PORT_HEAD = 'x-whistle-client-port';
+const CLIENT_IP_HEAD = 'x-forwarded-for';
+const PROXY_AUTH_HEAD = 'proxy-authorization';
+const ID_HEAD = `x-whistle-.-${Math.random()}`;
+
+const restoreClientInfo = function(req) {
+  const { headers, socket } = req;
+  let { clientInfo } = socket;
+  if (!clientInfo) {
+    clientInfo = headers[ID_HEAD];
+    if (clientInfo) {
+      delete headers[ID_HEAD];
+      try {
+        clientInfo = clientInfo && JSON.parse(decodeURIComponent(clientInfo));
+        req.socket.clientInfo = clientInfo;
+      } catch (e) {
+        return;
+      }
+    }
+  }
+  if (clientInfo) {
+    const {
+      clientIp,
+      clientPort,
+      clientId,
+      auth,
+    } = clientInfo;
+    if (clientIp) {
+      headers[CLIENT_IP_HEAD] = clientIp;
+    }
+    if (clientPort) {
+      headers[CLIENT_PORT_HEAD] = clientPort;
+    }
+    if (clientId) {
+      headers[CLIENT_ID_HEAD] = clientId;
+    }
+    if (auth) {
+      headers[PROXY_AUTH_HEAD] = auth;
+    }
+  }
+};
+
+const addClientInfo = (socket, chunk, statusLine) => {
+  chunk = chunk.slice(Buffer.byteLength(statusLine));
+  const { remoteAddress, remotePort, headers } = socket;
+  const auth = headers[PROXY_AUTH_HEAD];
+  const clientIp = headers[CLIENT_IP_HEAD] || remoteAddress;
+  const clientPort = headers[CLIENT_PORT_HEAD] || remotePort;
+  const clientId = headers[CLIENT_ID_HEAD];
+  const clientInfo = JSON.stringify({ auth, clientIp, clientPort, clientId });
+  statusLine += `\r\n${ID_HEAD}: ${encodeURIComponent(clientInfo)}`;
+  socket.emit('data', Buffer.concat([Buffer.from(statusLine), chunk]));
+};
+
+
+module.exports = (options, cb) => {
+  const { username, password } = options;
+  if (username || password) {
+    if (username !== '+') {
+      assert(username && typeof username === 'string', 'username is required.');
+      assert(/^[\w.-]{1,32}$/.test(username), 'username is incorrect (/^[\\w.-]{1,32}$/).');
+      assert(password && typeof password === 'string', 'password is required.');
+      setAdmin({ username, password });
+    }
+  }
+  const { host, port } = resolveHost(options.port);
+  options.host = host;
+  options.port = port || DEFAULT_PORT;
+  options.storage = resolveHostList(options.storage);
+  options.portStr = `${options.port}`;
+  const { nohostDomain } = options;
+  delete options.nohostDomain;
+  options.domain = checkDomain(nohostDomain) ? nohostDomain.toLowerCase() : '';
+
+  const handleRequest = (req, res) => {
+    const { headers } = req;
+    if (WORKER_RE.test(headers[NOHOST_ENV])) {
+      if (headers[NOHOST_RULE]) {
+        headers[WHISTLE_RULE] = headers[NOHOST_RULE];
+      }
+      if (headers[NOHOST_VALUE]) {
+        headers[WHISTLE_VALUE] = headers[NOHOST_VALUE];
+      }
+      passToWhistle(req, res, RegExp.$1);
+    } else {
+      passToWhistle(req, res);
+    }
+  };
+
+  const server = createServer((req, res) => {
+    onClose(res);
+    ++options.totalReqs;
+    if (!SPEC_PATH_RE.test(req.url) && isUIRequest(req)) {
+      ++options.uiReqs;
+      restoreClientInfo(req);
+      const { headers } = req;
+      delete headers.referer;
+      // 强制替换域名
+      headers['x-whistle-real-host'] = 'local.whistlejs.com';
+      headers.host = 'local.whistlejs.com';
+      handleUIRequest(req, res);
+    } else {
+      handleRequest(req, res);
+    }
+  });
+  server.timeout = 360000;
+  server.on('upgrade', (req, socket) => {
+    onClose(socket);
+    ++options.upgradeReqs;
+    ++options.totalReqs;
+    handleRequest(req, socket);
+  });
+  server.on('connect', async (req, socket) => {
+    onClose(socket);
+    ++options.tunnelReqs;
+    ++options.totalReqs;
+    if (isUIRequest(req, true)) {
+      sendEstablished(socket);
+      socket.headers = req.headers;
+      socket.once('data', (chunk) => {
+        if (HTTP_RE.test(`${chunk}`)) {
+          const statusLine = RegExp['$&'];
+          server.emit('connection', socket);
+          addClientInfo(socket, chunk, statusLine);
+        } else {
+          socket.destroy();
+        }
+      });
+    } else {
+      handleRequest(req, socket);
+    }
+  });
+  server.listen(options.port, host, cb);
+  return server;
+};

package/lib/main/cgi/getSettings.js

@@ -0,0 +1,11 @@
+
+module.exports = async (ctx) => {
+  const { storage } = ctx;
+  ctx.body = {
+    ec: 0,
+    admin: {
+      username: storage.getAdmin().username,
+    },
+    domain: storage.getDomain(),
+  };
+};

package/lib/main/cgi/getVersion.js

@@ -0,0 +1,12 @@
+const pkg = require('../../../package.json');
+
+const { pid } = process;
+
+module.exports = (ctx) => {
+  ctx.body = {
+    ec: 0,
+    pid,
+    version: pkg.version,
+    latestVersion: ctx.storage.latestVersion,
+  };
+};

package/lib/main/cgi/login.js

@@ -0,0 +1,14 @@
+const { checkLogin } = require('../../util/login');
+
+module.exports = async (ctx, next) => {
+  const { username, password } = ctx.storage.getAdmin();
+  const accept = checkLogin(ctx, {
+    username,
+    password,
+    nameKey: 'nohost_admin_name',
+    authKey: 'nohost_admin_key',
+  });
+  if (accept) {
+    await next();
+  }
+};

package/lib/main/cgi/restart.js

@@ -0,0 +1,5 @@
+
+module.exports = (ctx) => {
+  ctx.whistleMgr.restart();
+  ctx.body = { ec: 0 };
+};

package/lib/main/cgi/setAdmin.js

@@ -0,0 +1,6 @@
+module.exports = async (ctx) => {
+  ctx.body = { ec: 0 };
+  if (ctx.storage.setAdmin(ctx.request.body)) {
+    ctx.whistleMgr.restart();
+  }
+};

package/lib/main/cgi/setDomain.js

@@ -0,0 +1,13 @@
+
+module.exports = async (ctx) => {
+  const { domain } = ctx.request.body;
+  const { checkDomain, getDomain } = ctx.storage;
+  if (checkDomain(domain)) {
+    const curDomain = getDomain();
+    ctx.storage.setDomain(domain);
+    if (curDomain !== domain) {
+      ctx.whistleMgr.restart();
+    }
+  }
+  ctx.body = { ec: 0 };
+};

package/lib/main/cgi/status.js

@@ -0,0 +1,122 @@
+const WORKER_NUM = require('../workerNum');
+const config = require('../../config');
+
+const STATUS = { workerNum: WORKER_NUM };
+let allWorkers = [];
+let envWorkerMap = {};
+let curWorkerInfo;
+const MAX_CACHE_TIME = 1000 * 60 * 8;
+const WORKER_INFO_RE = /^\d{1,3}(\.\d{1,3}){3}:\d{1,5}\/[1-9]\d*(,\d{1,3}(\.\d{1,3}){3}:\d{1,5}\/[1-9]\d*)*$/;
+
+const parseWorker = (item) => {
+  item = item.split('/');
+  const index = parseInt(item[1], 10);
+  item = item[0].split(':');
+  return {
+    index,
+    envNum: 0,
+    host: item[0],
+    port: parseInt(item[1], 10),
+  };
+};
+
+setInterval(() => {
+  const now = Date.now();
+  let isChanged;
+  Object.keys(envWorkerMap).forEach((env) => {
+    const { updateTime, worker } = envWorkerMap[env];
+    if (now - updateTime > MAX_CACHE_TIME) {
+      delete envWorkerMap[env];
+      --worker.envNum;
+      isChanged = true;
+    }
+  });
+  if (isChanged) {
+    allWorkers.sort((w1, w2) => {
+      return w1.envNum > w2.envNum ? 1 : -1;
+    });
+  }
+}, 30000);
+
+const flattenWorkers = (workerList) => {
+  const result = [];
+  let len = workerList.length;
+  while (workerList.length) {
+    for (let i = len - 1; i >= 0; i--) {
+      const worker = workerList[i];
+      --worker.index;
+      if (!worker.index) {
+        workerList.splice(i, 1);
+      }
+      result.push(Object.assign({}, worker, STATUS));
+    }
+    len = workerList.length;
+  }
+  return result;
+};
+
+const update = (options) => {
+  if (!options || curWorkerInfo === options) {
+    return allWorkers;
+  }
+  let workerList;
+  try {
+    workerList = Buffer.from(options, 'base64').toString();
+    if (!WORKER_INFO_RE.test(workerList)) {
+      return allWorkers;
+    }
+  } catch (e) {
+    return allWorkers;
+  }
+  curWorkerInfo = options;
+  workerList = workerList.split(',').map(parseWorker);
+  allWorkers = flattenWorkers(workerList);
+  envWorkerMap = {};
+  return allWorkers;
+};
+
+const notEmptyStr = (str) => {
+  return str && typeof str === 'string';
+};
+
+const getWorker = ({ space, group, env }, servers) => {
+  if (!notEmptyStr(space) || !notEmptyStr(group) || !update(servers).length) {
+    return STATUS;
+  }
+  env = [space, group, env || ''].join('/');
+  const info = envWorkerMap[env];
+  if (info) {
+    info.updateTime = Date.now();
+    return info.worker;
+  }
+  const worker = allWorkers[0];
+  ++worker.envNum;
+  envWorkerMap[env] = {
+    updateTime: Date.now(),
+    worker,
+  };
+  const index = allWorkers.indexOf(worker);
+  for (let i = allWorkers.length - 1; i > index; i--) {
+    const next = allWorkers[i];
+    if (!next || worker.envNum > next.envNum) {
+      allWorkers[i] = worker;
+      allWorkers[index] = next;
+      return worker;
+    }
+  }
+  return worker;
+};
+
+
+module.exports = (ctx) => {
+  const servers = ctx.get('x-nohost-servers');
+  const { query } = ctx.request;
+  const status = Object.assign({}, getWorker(query, servers));
+  status.totalReqs = config.totalReqs;
+  status.uiReqs = config.uiReqs;
+  status.upgradeReqs = config.upgradeReqs;
+  status.tunnelReqs = config.tunnelReqs;
+  status.totalQps = config.totalQps;
+  status.uiQps = config.uiQps;
+  ctx.body = status;
+};

package/lib/main/index.js

@@ -0,0 +1,94 @@
+
+const Koa = require('koa');
+const onerror = require('koa-onerror');
+const serve = require('koa-static');
+const { join } = require('path');
+const { Z_SYNC_FLUSH } = require('zlib');
+const router = require('koa-router')();
+const compress = require('koa-compress');
+const setupRouter = require('./router');
+const whistleMgr = require('./whistleMgr');
+const storage = require('./storage');
+const { getRedirectUrl } = require('./util');
+const config = require('../config');
+
+const MAX_AGE = 1000 * 60 * 5;
+const HEADLESS_RE = /^\/account\/[^/]+\/share\//;
+const EXPORT_RE = /\/export_sessions$/;
+const SPECIAL_PATH = '/nohost/';
+const aliasPages = {
+  '/': '/select.html',
+  '/index.html': '/select.html',
+  '/data.html': '/capture.html',
+  '/share.html': '/network.html',
+};
+
+const startApp = () => {
+  const app = new Koa();
+  app.proxy = true;
+  app.silent = true;
+  onerror(app);
+  app.use(async (ctx, next) => {
+    if (config.deprecated) {
+      return ctx.redirect(config.redirect);
+    }
+    let { path, req } = ctx;
+    if (!path.indexOf(SPECIAL_PATH)) {
+      req.url = req.url.replace(SPECIAL_PATH, '/');
+      path = ctx.path;
+    }
+    const newPath = aliasPages[path];
+    if (newPath) {
+      ctx.pageName = path;
+      req.url = newPath;
+    } else if (HEADLESS_RE.test(path)) {
+      req.url = req.url.replace(RegExp['$&'], '/');
+      path = ctx.path;
+      if (path === '/' || path === '/share.html') {
+        req.url = '/network.html';
+        ctx.pageName = '/share.html';
+      }
+    } else if (EXPORT_RE.test(path)) {
+      path = '/cgi-bin/sessions/export';
+      let query = req.url.indexOf('?');
+      query = query === -1 ? '' : req.url.substring(query);
+      req.url = path + query;
+    } else {
+      const index = path.indexOf('/nohost_share/');
+      if (index !== -1) {
+        const accountName = path.substring(0, index);
+        ctx.accountName = accountName.substring(accountName.lastIndexOf('/') + 1);
+        req.url = req.url.substring(req.url.indexOf('/nohost_share/') + 13);
+        path = ctx.path;
+        if (path === '/' || path === '/share.html') {
+          req.url = '/network.html';
+          ctx.pageName = '/share.html';
+        }
+      }
+    }
+    const redirectUrl = getRedirectUrl(ctx);
+    if (redirectUrl) {
+      return ctx.redirect(redirectUrl);
+    }
+    ctx.whistleMgr = whistleMgr;
+    ctx.storage = storage;
+    await next();
+  });
+  setupRouter(router);
+  app.use(router.routes());
+  app.use(router.allowedMethods());
+  app.use(compress({
+    threshold: 2048,
+    gzip: {
+      flush: Z_SYNC_FLUSH,
+    },
+    deflate: {
+      flush: Z_SYNC_FLUSH,
+    },
+    br: false, // disable brotli
+  }));
+  app.use(serve(join(__dirname, '../../public'), { maxage: MAX_AGE }));
+  return app.callback();
+};
+
+module.exports = startApp();