tt-help-cli-ycl 1.4.113 → 1.4.115

package/src/main.js

@@ -13,6 +13,7 @@ import { handleVideoStats } from "./cli/videostats.js";
 import { handleDbImport } from "./cli/db-import.js";
 import { handleWebserver } from "./cli/webserver.js";
 import { handleRefresh } from "./cli/refresh.js";
+import { handleRefreshMini } from "./cli/refresh-mini.js";
 import { handlePrecheck } from "./cli/precheck.js";
 import {
   handleTag,
@@ -54,6 +55,8 @@ async function main() {
       return handleDbImport(parsed);
     case "refresh":
       return handleRefresh(parsed);
+    case "refresh-mini":
+      return handleRefreshMini(parsed);
     case "precheck":
       return handlePrecheck(parsed);
     case "tag":

package/src/npm-main.js

@@ -9,6 +9,7 @@ import { handleConfig, showConfig, showUsage, version } from "./cli/config.js";
 import { handleOpen } from "./cli/open.js";
 import { handleComments } from "./cli/comments.js";
 import { handleRefresh } from "./cli/refresh.js";
+import { handleRefreshMini } from "./cli/refresh-mini.js";
 import { handlePrecheck } from "./cli/precheck.js";
 import { handleWebserver } from "./cli/webserver.js";
 import {
@@ -55,6 +56,8 @@ async function main() {
       return handleComments(parsed);
     case "refresh":
       return handleRefresh(parsed);
+    case "refresh-mini":
+      return handleRefreshMini(parsed);
     case "precheck":
       return handlePrecheck(parsed);
     case "tag":

package/src/scraper/explore-core.js

@@ -28,6 +28,7 @@ async function processExplore(page, username, options, log) {
     proxyServer = null,
     webserverUrl = null,
     noProduct = false,
+    mini = false, // 精简模式：仅采集视频数据，跳过用户信息/国家采样/关注粉丝/商品详情
   } = options;
 
   const result = {
@@ -63,13 +64,17 @@ async function processExplore(page, username, options, log) {
     log(`  访问 @${username} 主页...`);
     const videoList = await collectVideos(page, username, maxVideos, log);
 
-    log("  获取用户信息...");
-    const info = await getUserInfo(page);
-    if (info) {
-      result.userInfo = info;
-      log(
-        `  用户: ${info.nickname || username} | 粉丝: ${info.followerCount || "-"} | 视频: ${info.videoCount || "-"}`,
-      );
+    if (!mini) {
+      log("  获取用户信息...");
+      const info = await getUserInfo(page);
+      if (info) {
+        result.userInfo = info;
+        log(
+          `  用户: ${info.nickname || username} | 粉丝: ${info.followerCount || "-"} | 视频: ${info.videoCount || "-"}`,
+        );
+      }
+    } else {
+      log("  [mini] 跳过用户信息");
     }
 
     const captcha = await detectCaptcha(page);
@@ -132,8 +137,8 @@ async function processExplore(page, username, options, log) {
       };
 
       // 如果有商品链接且未禁用，通过 WebService API 抓取商品详情页（评分、销量等）
-      // --no-product 时跳过（由独立商品程序异步补抓，避免阻塞 explore/refresh 主流程）
-      if (topVideo.seoUrl && options.webserverUrl && !noProduct) {
+      // --no-product / mini 模式时跳过（由独立商品程序异步补抓，避免阻塞主流程）
+      if (topVideo.seoUrl && options.webserverUrl && !noProduct && !mini) {
         try {
           const resp = await fetch(
             `${options.webserverUrl}/api/tiktok/product`,
@@ -166,6 +171,16 @@ async function processExplore(page, username, options, log) {
       return result;
     }
 
+    // mini 模式：跳过国家采样与关注/粉丝，仅登记视频列表，大幅提速
+    if (mini) {
+      result.keepFollow = false;
+      result.hasFollowData = false;
+      result.videoList = videoArray;
+      result.processed = true;
+      log(`  [mini] 仅采集视频数据，跳过国家采样与关注/粉丝`);
+      return result;
+    }
+
     // 国家采样判断
     // explore 模式：采样 5 个，优先命中目标国家，不匹配则回退众数 → 写 locationCreated
     // refresh 模式：采样 7 个，纯众数逻辑 → 写 confirmedLocation（二次确认）

package/src/watch/llm-config.js

@@ -177,6 +177,20 @@ export function getLlmConfig() {
   };
 }
 
+/**
+ * 获取完整 LLM 配置（含完整 API Key）— 仅供本机内部服务使用
+ * 由 /api/llm-config/raw 端点暴露，该端点仅允许 127.0.0.1 访问
+ */
+export function getLlmConfigRaw() {
+  if (!_loaded) loadLlmConfig();
+  return {
+    provider: _cachedProvider,
+    model: _cachedModel,
+    url: _cachedUrl,
+    apiKey: _cachedApiKey,
+  };
+}
+
 /** 获取当前 LLM URL（内部调用） */
 export function getLlmUrl() {
   if (!_loaded) loadLlmConfig();

package/src/watch/server.js

@@ -447,6 +447,23 @@ export function startWatchServer(
         }
       }
 
+      // ── LLM 完整配置端点（仅本机可访问，供看门狗独立服务动态拉取） ──
+      if (req.method === "GET" && routePath === "/api/llm-config/raw") {
+        const remote = req.socket.remoteAddress || "";
+        const isLocal =
+          remote === "127.0.0.1" ||
+          remote === "::1" ||
+          remote === "::ffff:127.0.0.1";
+        if (!isLocal) {
+          console.error(`[llm-raw] 拒绝非本机访问: ${remote}`);
+          sendJSON(res, 403, { error: "forbidden" });
+          return;
+        }
+        const { getLlmConfigRaw } = await import("./llm-config.js");
+        sendJSON(res, 200, getLlmConfigRaw());
+        return;
+      }
+
       if (req.method === "POST" && routePath === "/api/users") {
         try {
           const { usernames, sources, guessedLocation } = await readBody(req);

package/src/watchdog/process-manager.js

@@ -24,6 +24,60 @@ const IS_WIN = process.platform === "win32";
 const SPAWN_STAGGER_MS = 30000; // 批量启动时每个进程间隔 30 秒，避免抢网络
 const STALE_LOG_TIMEOUT_MS = 15 * 60 * 1000; // 15 分钟无日志输出视为假死
 
+// === 控制台日志格式化（仅显示层增强，不影响任何功能链路） ===
+// 目标：多进程日志交错打印时，通过时间戳 + 固定宽度彩色 [cmdId] 前缀方便人眼分流。
+// 兼容：Windows（Windows Terminal 彩色 / 旧 conhost 自动降级纯文本）与 macOS（原生 ANSI 彩色）。
+
+// 每进程固定一种颜色（按 cmdId 哈希取色），新增进程自动分配且不重复
+const PROC_COLORS = [32, 36, 33, 35, 34, 92, 96, 93, 95, 94];
+const ID_PAD_WIDTH = 10; // cmdId 前缀固定宽度，短 ID 补齐保持消息列对齐
+
+function getProcColor(id) {
+  let sum = 0;
+  const s = String(id);
+  for (let i = 0; i < s.length; i++) sum = (sum + s.charCodeAt(i)) % 997;
+  return PROC_COLORS[sum % PROC_COLORS.length];
+}
+
+function canUseColor() {
+  if (!process.stderr.isTTY) return false;
+  if (process.env.NO_COLOR) return false;
+  if (process.env.FORCE_COLOR === "1" || process.env.FORCE_COLOR === "true")
+    return true;
+  // macOS 终端恒 ≥4；Windows Terminal ≥4；旧 conhost 未开启 VT 时为 1 → 自动降级纯文本
+  const depth =
+    typeof process.stderr.getColorDepth === "function"
+      ? process.stderr.getColorDepth()
+      : 1;
+  return depth >= 4;
+}
+
+function fmtClock(d = new Date()) {
+  const p = (n) => String(n).padStart(2, "0");
+  return `${p(d.getHours())}:${p(d.getMinutes())}:${p(d.getSeconds())}`;
+}
+
+/**
+ * 打印子进程日志行（替代裸 console.error）。
+ * 只改显示形态，msg 内容一行不少；多行消息逐行加前缀。
+ */
+function printProcessLine(id, msg) {
+  const lines = String(msg).split(/\r?\n/);
+  const prefixId = `[${String(id).padEnd(ID_PAD_WIDTH)}]`;
+  const color = getProcColor(id);
+  const useColor = canUseColor();
+  for (const line of lines) {
+    const clock = fmtClock();
+    if (useColor) {
+      process.stderr.write(
+        `\x1b[90m${clock}\x1b[0m \x1b[${color}m${prefixId}\x1b[0m ${line}\n`,
+      );
+    } else {
+      process.stderr.write(`${clock} ${prefixId} ${line}\n`);
+    }
+  }
+}
+
 // 从命令行中提取 CDP 端口列表（解析 --base-port 和 --port-count）
 function extractCdpPorts(cmdLine) {
   const match = cmdLine.match(/--base-port\s+(\d+)/);
@@ -219,7 +273,10 @@ export class ProcessManager extends EventEmitter {
       const msg = data.toString().trim();
       if (msg) {
         const short = msg.substring(0, 200);
-        console.error(`[${id}] ${short}`);
+        // 仅显示层格式化：时间戳 + 彩色固定宽度前缀，内容不变。
+        // 前缀优先用命令名（如 explore-F1），未命名时退回 cmdId。
+        const displayId = (cmdDef && cmdDef.name) || id;
+        printProcessLine(displayId, short);
         // 环形缓冲：保留最近 N 行供 LLM 分析
         const info = this.processes.get(id);
         if (info) {

package/src/watchdog/reporter.js

@@ -39,6 +39,7 @@ export function getClientConfig() {
       const cfg = JSON.parse(readFileSync(CONFIG_PATH, "utf-8"));
       return {
         server: cfg.server || "",
+        watchdogServer: cfg.watchdogServer || "",
         proxy: cfg.proxy || "",
         browser: cfg.browser || "",
         userId: cfg.userId || "",

package/src/watchdog/standalone-server.js

@@ -0,0 +1,228 @@
+/**
+ * 看门狗独立服务 — 与 watch 完全隔离的独立进程
+ *
+ * 复用 src/watch/watchdog-server.js 的全部看门狗逻辑
+ * （内存状态 + ~/.tt-help-watchdog.json 持久化），
+ * 不接触 watch 的 SQLite 数据库、不依赖 watch 的 http 服务。
+ *
+ * 用法:
+ *   node src/watchdog/standalone-server.js [--port 17302]
+ *   WATCHDOG_PORT=17302 node src/watchdog/standalone-server.js
+ *
+ * LLM 配置（可选，仅影响 POST /api/watchdog/analyze-logs）:
+ *   环境变量 LLM_URL / LLM_MODEL / APIKEY，缺省时用 llm-config 内置默认值。
+ *   本进程永不打开 SQLite（getDb() 恒为 null），LLM 配置自动走环境变量兜底。
+ */
+
+import http from "http";
+import {
+  existsSync,
+  createReadStream,
+  readFileSync,
+  writeFileSync,
+} from "fs";
+import { extname, join, normalize, dirname, sep } from "path";
+import { fileURLToPath } from "url";
+import { homedir } from "os";
+import {
+  registerWatchdogRoutes,
+  tryHandleWatchdogRoute,
+} from "../watch/watchdog-server.js";
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const PUBLIC_DIR = join(__dirname, "..", "watch", "public");
+
+// ── 端口解析：--port 参数 > WATCHDOG_PORT 环境变量 > 默认 17302 ──
+function resolvePort() {
+  const argv = process.argv.slice(2);
+  const idx = argv.indexOf("--port");
+  if (idx !== -1 && argv[idx + 1]) {
+    const p = parseInt(argv[idx + 1], 10);
+    if (p > 0 && p < 65536) return p;
+  }
+  const env = parseInt(process.env.WATCHDOG_PORT, 10);
+  if (env > 0 && env < 65536) return env;
+  return 17302;
+}
+
+// ── watch 端口（本机 127.0.0.1）：--watch-port 参数 > WATCHDOG_WATCH_PORT 环境变量 > 默认 3011 ──
+function resolveWatchPort() {
+  const argv = process.argv.slice(2);
+  const idx = argv.indexOf("--watch-port");
+  if (idx !== -1 && argv[idx + 1]) {
+    const p = parseInt(argv[idx + 1], 10);
+    if (p > 0 && p < 65536) return p;
+  }
+  const env = parseInt(process.env.WATCHDOG_WATCH_PORT, 10);
+  if (env > 0 && env < 65536) return env;
+  return 3011;
+}
+
+// LLM 配置本地缓存（“以前的”配置，watch 不可达时兜底）
+const LLM_CACHE_PATH = join(homedir(), ".tt-help-watchdog-llm.json");
+const WATCH_PORT = resolveWatchPort();
+
+/**
+ * 动态刷新 LLM 配置：
+ * 1. 从 watch 的受限端点 /api/llm-config/raw（仅 127.0.0.1）拉取完整配置
+ * 2. 拉取成功 → 注入环境变量 + 更新本地缓存
+ * 3. 拉取失败 → 读本地缓存（“以前的”）注入
+ * 4. reloadLlmConfig() 强制 llm-config 重读 env（绕开其 _loaded 缓存）
+ */
+async function refreshLlmConfig() {
+  let cfg = null;
+  try {
+    const resp = await fetch(
+      `http://127.0.0.1:${WATCH_PORT}/api/llm-config/raw`,
+      { signal: AbortSignal.timeout(3000) },
+    );
+    if (resp.ok) {
+      cfg = await resp.json();
+      if (cfg && cfg.apiKey) {
+        try {
+          writeFileSync(LLM_CACHE_PATH, JSON.stringify(cfg, null, 2), "utf-8");
+        } catch {}
+        console.error(
+          `[watchdog-standalone] LLM 配置已从 watch 刷新并更新本地缓存 (model=${cfg.model})`,
+        );
+      }
+    }
+  } catch {}
+
+  if (!cfg || !cfg.apiKey) {
+    try {
+      if (existsSync(LLM_CACHE_PATH)) {
+        cfg = JSON.parse(readFileSync(LLM_CACHE_PATH, "utf-8"));
+        if (cfg && cfg.apiKey) {
+          console.error(
+            "[watchdog-standalone] watch LLM 端点不可达，使用本地缓存配置",
+          );
+        }
+      }
+    } catch {}
+  }
+
+  if (cfg) {
+    if (cfg.apiKey) process.env.APIKEY = cfg.apiKey;
+    if (cfg.url) process.env.LLM_URL = cfg.url;
+    if (cfg.model) process.env.LLM_MODEL = cfg.model;
+  }
+  try {
+    const { reloadLlmConfig } = await import("../watch/llm-config.js");
+    reloadLlmConfig();
+  } catch {}
+}
+
+const MIME_TYPES = {
+  ".html": "text/html; charset=utf-8",
+  ".js": "application/javascript; charset=utf-8",
+  ".css": "text/css; charset=utf-8",
+  ".json": "application/json; charset=utf-8",
+  ".svg": "image/svg+xml",
+  ".png": "image/png",
+  ".ico": "image/x-icon",
+};
+
+function readBody(req) {
+  return new Promise((resolve, reject) => {
+    let body = "";
+    req.on("data", (chunk) => (body += chunk));
+    req.on("end", () => {
+      try {
+        resolve(body ? JSON.parse(body) : {});
+      } catch (e) {
+        reject(e);
+      }
+    });
+    req.on("error", reject);
+  });
+}
+
+function sendJSON(res, code, data) {
+  res.writeHead(code, { "Content-Type": "application/json; charset=utf-8" });
+  res.end(JSON.stringify(data));
+}
+
+// 静态文件（仅限 public 目录内，防路径穿越）
+function serveStatic(res, pathname) {
+  let decoded;
+  try {
+    decoded = decodeURIComponent(pathname);
+  } catch {
+    sendJSON(res, 400, { error: "bad request" });
+    return;
+  }
+  const filePath =
+    decoded === "/"
+      ? join(PUBLIC_DIR, "index.html")
+      : join(PUBLIC_DIR, decoded);
+  const normalized = normalize(filePath);
+  const base = normalize(PUBLIC_DIR);
+  if (normalized !== base && !normalized.startsWith(base + sep)) {
+    sendJSON(res, 403, { error: "forbidden" });
+    return;
+  }
+  if (!existsSync(normalized)) {
+    sendJSON(res, 404, { error: "not found" });
+    return;
+  }
+  const ext = extname(normalized).toLowerCase();
+  res.writeHead(200, {
+    "Content-Type": MIME_TYPES[ext] || "application/octet-stream",
+  });
+  createReadStream(normalized).pipe(res);
+}
+
+const server = http.createServer(async (req, res) => {
+  const url = new URL(req.url, "http://127.0.0.1");
+  const routePath = url.pathname;
+  const params = Object.fromEntries(url.searchParams.entries());
+
+  // analyze-logs 前动态刷新 LLM 配置（watch 换 key 无需重启本服务）
+  if (req.method === "POST" && routePath === "/api/watchdog/analyze-logs") {
+    await refreshLlmConfig();
+  }
+
+  // 看门狗 API：复用 watchdog-server.js 的完整路由
+  if (routePath.startsWith("/api/watchdog/")) {
+    try {
+      let body = {};
+      if (req.method === "POST") body = await readBody(req);
+      const handled = await tryHandleWatchdogRoute(
+        routePath,
+        req.method,
+        params,
+        body,
+        res,
+      );
+      if (handled) return;
+      sendJSON(res, 404, { error: "watchdog route not found" });
+      return;
+    } catch (e) {
+      sendJSON(res, 400, { error: e.message });
+      return;
+    }
+  }
+
+  // 其余 GET：serve watch 的前端静态页（看门狗页签同源可用）
+  if (req.method === "GET" || req.method === "HEAD") {
+    serveStatic(res, routePath);
+    return;
+  }
+
+  sendJSON(res, 404, { error: "not found" });
+});
+
+const port = resolvePort();
+registerWatchdogRoutes(server);
+server.listen(port, "0.0.0.0", () => {
+  console.error(
+    `[watchdog-standalone] 看门狗独立服务已启动: http://0.0.0.0:${port}`,
+  );
+  console.error(`[watchdog-standalone] 前端页面: http://<本机IP>:${port}/`);
+  console.error(
+    `[watchdog-standalone] LLM 配置源: http://127.0.0.1:${WATCH_PORT}/api/llm-config/raw（本地缓存兜底: ${LLM_CACHE_PATH}）`,
+  );
+  // 启动后异步预热 LLM 配置（不阻塞启动，失败静默）
+  refreshLlmConfig().catch(() => {});
+});

package/src/webserver/server.mjs

@@ -80,6 +80,28 @@ function apiLog(msg) {
   if (SCRAPER_CONFIG.enableLog) console.error(msg);
 }
 
+// 限频错误日志：同类错误在窗口内只打首条，其余计数汇总，避免 CDN 限流等高频错误刷屏
+const ERROR_LOG_THROTTLE_MS = 10000;
+let lastThrottledLogTime = 0;
+let throttledLogCount = 0;
+let throttledLogLabel = "";
+function logErrorThrottled(label, msg) {
+  const now = Date.now();
+  if (now - lastThrottledLogTime >= ERROR_LOG_THROTTLE_MS) {
+    if (throttledLogCount > 0) {
+      console.error(
+        `[webserver] ${throttledLogLabel} | 窗口内共 ${throttledLogCount} 条同类错误（已限频）`,
+      );
+    }
+    console.error(`[webserver] ${label} | ${msg}`);
+    lastThrottledLogTime = now;
+    throttledLogCount = 0;
+    throttledLogLabel = label;
+  } else {
+    throttledLogCount++;
+  }
+}
+
 let scraperInstance = null;
 let scraperClosing = false;
 let scraperInitPromise = null;
@@ -257,16 +279,19 @@ async function handleTiktokUser(req, res) {
 
   if (result.error && result.statusCode) {
     // TikTok 明确返回的状态码（209002 验证码、10202 被封、10221 不存在等）：
-    // 属业务结果而非 webserver 故障，不计入连续错误、不触发 scraper 重启
+    // 属业务结果而非 webserver 故障，不计入连续错误、不触发 scraper 重启；
+    // 返回 200 而非 503：避免 nginx 将其当作节点故障去 failover/熔断
+    //（账号/IP 级风控，切换后端无意义，只会放大对 TikTok 的访问）
     const retryable = TIKTOK_RETRYABLE_NORMAL_CODES.has(result.statusCode);
     apiLog(
       `[webserver] user business | ${clientIP} | @${body.uniqueId} | ${elapsed}ms | statusCode=${result.statusCode}${retryable ? " (可重试)" : ""}`,
     );
     recordRequest(true);
-    return sendJSON(res, 503, {
+    return sendJSON(res, 200, {
       error: `TikTok statusCode=${result.statusCode}`,
       retryable,
-      tiktokStatusCode: result.statusCode,
+      statusCode: result.statusCode, // 后端 data-store / attach 外层读取此字段
+      tiktokStatusCode: result.statusCode, // 兼容保留
     });
   }
   if (result.error) {
@@ -560,22 +585,26 @@ export async function startWebserver(
     try {
       return await route.handler(req, res);
     } catch (e) {
-      // 自动恢复：连续错误超过阈值时重启 scraper
-      consecutiveErrors++;
+      // 错误分类：CDN 限流是环境问题（代理/IP 被 TikTok CDN 拦截），重启 scraper 无法解决，
+      // 故不累计 consecutiveErrors、不触发重启；但仍计入 recentErrors（/health 展示错误率）。
+      const isCdnBlocked = e.name === "CDNBlockedError";
+      if (!isCdnBlocked) consecutiveErrors++;
       recordRequest(false);
 
       // 区分错误类型，返回正确的 HTTP 状态码供 nginx 判断
       let status = 500;
       let retryable = false;
 
-      if (e.name === "CDNBlockedError") {
-        // CDN 限流（Akamai Access Denied）→ 503 临时不可用
+      if (isCdnBlocked) {
+        // CDN 限流（Akamai Access Denied）→ 503 临时不可用（节点级，nginx 可 failover 到其他节点）
         status = 503;
         retryable = true;
+        logErrorThrottled("request CDN限流", e.message);
       } else if (e.message?.includes("Server is shutting down")) {
         // 服务正在关闭 → 503
         status = 503;
         retryable = true;
+        console.error(`[webserver] request 服务关闭中: ${e.message}`);
       } else if (
         e.message?.includes("Timeout") ||
         e.message?.includes("timeout") ||
@@ -584,6 +613,7 @@ export async function startWebserver(
         // 超时 → 504
         status = 504;
         retryable = true;
+        console.error(`[webserver] request 超时: ${e.message}`);
       } else if (
         e.message?.includes("Protocol error") ||
         e.message?.includes("Target closed") ||
@@ -593,6 +623,9 @@ export async function startWebserver(
         // 浏览器/页面崩溃 → 503
         status = 503;
         retryable = true;
+        console.error(`[webserver] request 浏览器异常: ${e.message}`);
+      } else {
+        console.error(`[webserver] request 未知错误: ${e.message}`);
       }
 
       if (consecutiveErrors >= MAX_CONSECUTIVE_ERRORS) {