tt-help-cli-ycl 1.4.18 → 1.4.20

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tt-help-cli-ycl",
-  "version": "1.4.18",
+  "version": "1.4.20",
   "description": "TikTok user & video data scraper - extract ttSeller, verified, locationCreated from HTML source",
   "type": "module",
   "bin": {

package/src/cli/recheck.js

@@ -0,0 +1,381 @@
+/**
+ * recheck — 重检 jobs 表中已完成的非商家用户
+ *
+ * 基于 attach.js 的核心循环，但任务源和提交逻辑不同：
+ * - 拉任务：GET /api/recheck-jobs（只取 status='done' 且 tt_seller!=1 且 7 天未更新的用户）
+ * - 提交结果：POST /api/recheck-jobs-batch（直接 UPDATE jobs 表，不做表迁移）
+ */
+
+import { TikTokScraper } from "../lib/tiktok-scraper.mjs";
+import { CDNBlockedError } from "../lib/parse-ssr.mjs";
+import { proxy as configuredProxy } from "../lib/constants.js";
+import { createApiClient } from "../lib/api-client.js";
+import v8 from "node:v8";
+
+const HEAP_RESTART_RATIO = 0.72;
+const MAX_TASK_BATCHES_BEFORE_RESTART = 200;
+
+function formatNow() {
+  return new Date().toLocaleString("zh-CN", {
+    hour12: false,
+  });
+}
+
+function recheckLog(message = "") {
+  console.error(`[${formatNow()}] ${message}`);
+}
+
+function isBrowserClosedError(err) {
+  if (!err) return false;
+  const msg = err.message || err.toString() || "";
+  return (
+    msg.includes("Target page, context or browser has been closed") ||
+    msg.includes("browser has been closed") ||
+    msg.includes("browserContext.newPage") ||
+    msg.includes("Protocol error")
+  );
+}
+
+function getHeapUsage() {
+  const heapUsed = process.memoryUsage().heapUsed;
+  const heapLimit = v8.getHeapStatistics().heap_size_limit;
+  return {
+    heapUsed,
+    heapLimit,
+    ratio: heapLimit > 0 ? heapUsed / heapLimit : 0,
+  };
+}
+
+async function recycleScraper(scraper, reason) {
+  recheckLog(`[Recheck] ${reason}，正在重启浏览器实例...`);
+  await scraper.restart();
+  recheckLog("[Recheck] 浏览器实例重启完成");
+}
+
+export async function handleRecheck(options) {
+  const {
+    attachParallel,
+    attachInterval,
+    serverUrl,
+    attachCountries,
+    customProxy,
+    attachPoolSize,
+    showHelp,
+  } = options;
+  const effectiveProxy = customProxy || configuredProxy;
+
+  if (effectiveProxy) {
+    recheckLog(`[Recheck] 使用代理: ${effectiveProxy}`);
+  }
+  let shuttingDown = false;
+  let forceExitTimer = null;
+
+  if (showHelp) {
+    recheckLog(
+      "用法: tt-help recheck [-p 并行数] [-i 间隔秒数] [-s 服务端地址] [-c 国家列表]",
+    );
+    recheckLog("");
+    recheckLog("参数:");
+    recheckLog("  -p, --parallel <N>     并行抓取数（默认: 1）");
+    recheckLog("  -i, --interval <N>     无任务时轮询间隔，单位秒（默认: 10）");
+    recheckLog(
+      "  -s, --server <URL>     服务端地址（默认: http://127.0.0.1:3001）",
+    );
+    recheckLog(
+      "  -c, --countries <A,B,C>  目标国家列表（逗号分隔，如 PL,DE,FR），默认全部目标国家",
+    );
+    recheckLog(
+      "  --proxy <代理地址>    HTTP 代理地址（如 http://127.0.0.1:7890），不配置则从 ~/.tt-help.json 读取",
+    );
+    recheckLog(
+      "  --size <N>            浏览器页面池大小（默认: 3，不建议超过 3）",
+    );
+    recheckLog("");
+    recheckLog("说明:");
+    recheckLog(
+      "  从 jobs 表中拉取 status='done' 且非商家（tt_seller!=1）且 7 天未更新的用户，",
+    );
+    recheckLog(
+      "  重新抓取 TikTok 用户基本信息（重点检查 tt_seller），更新回 jobs 表。",
+    );
+    recheckLog(
+      "  每 10 轮打印汇总：已跑轮数、发现 ttSeller 数量、累计检测数。",
+    );
+    recheckLog("  一轮跑完后自动空转，7 天后可重新运行。");
+    recheckLog("  支持多客户端并行，服务端自动分配不同用户。");
+    recheckLog("");
+    recheckLog("示例:");
+    recheckLog("  tt-help recheck");
+    recheckLog("  tt-help recheck -p 5 -i 10");
+    recheckLog("  tt-help recheck -c PL,DE,FR -p 5");
+    recheckLog("  tt-help recheck -p 5 --size 5");
+    return;
+  }
+
+  const countryStr =
+    attachCountries && attachCountries.length > 0
+      ? `, 目标国家: ${attachCountries.join(", ")}`
+      : ", 目标国家: 全部";
+  recheckLog(
+    `[Recheck] 并行数: ${attachParallel}, 空闲间隔: ${attachInterval}秒, 服务端: ${serverUrl}${countryStr}`,
+  );
+
+  const { apiGet, apiPost } = createApiClient({ log: false });
+
+  const scraper = new TikTokScraper({
+    poolSize: attachPoolSize || 3,
+    proxyServer: effectiveProxy || null,
+  });
+
+  const shutdown = async (signal) => {
+    if (shuttingDown) return;
+    shuttingDown = true;
+    forceExitTimer = setTimeout(() => {
+      recheckLog("[Recheck] 关闭超时，强制退出");
+      process.exit(0);
+    }, 30000);
+    forceExitTimer.unref?.();
+    recheckLog(`\n[Recheck] 收到 ${signal}，等待当前任务完成后退出...`);
+  };
+
+  const onSigint = () => {
+    void shutdown("SIGINT");
+  };
+  const onSigterm = () => {
+    void shutdown("SIGTERM");
+  };
+
+  process.on("SIGINT", onSigint);
+  process.on("SIGTERM", onSigterm);
+
+  try {
+    await scraper.init();
+    recheckLog("[Recheck] 浏览器初始化完成，开始循环接收任务...");
+
+    let loopCount = 0;
+    let browserRestartCount = 0;
+    let taskBatchCount = 0;
+    let totalChecked = 0;
+    let totalSellerFound = 0;
+
+    while (!shuttingDown) {
+      loopCount++;
+
+      // 检查浏览器是否存活
+      if (!scraper.isAlive()) {
+        recheckLog(
+          `[Recheck] 浏览器已关闭，正在重启 (${++browserRestartCount})...`,
+        );
+        await scraper.restart();
+        recheckLog("[Recheck] 浏览器重启完成");
+      }
+
+      // 收到退出信号且当前无任务正在处理，则退出
+      if (shuttingDown) {
+        recheckLog("[Recheck] 当前任务已完成，正在关闭浏览器...");
+        await scraper.close().catch(() => {});
+        if (forceExitTimer) {
+          clearTimeout(forceExitTimer);
+          forceExitTimer = null;
+        }
+        recheckLog(
+          `[Recheck] 已退出，共检测 ${totalChecked} 个用户，发现 ttSeller: ${totalSellerFound}`,
+        );
+        process.exit(0);
+      }
+
+      const countryParam =
+        attachCountries && attachCountries.length > 0
+          ? `&countries=${encodeURIComponent(attachCountries.join(","))}`
+          : "";
+      const { total, tasks } = await apiGet(
+        `${serverUrl}/api/recheck-jobs?limit=${attachParallel}${countryParam}`,
+      );
+
+      if (!tasks || tasks.length === 0) {
+        const heap = getHeapUsage();
+        if (heap.ratio >= HEAP_RESTART_RATIO) {
+          await recycleScraper(
+            scraper,
+            `空闲时堆占用 ${(heap.ratio * 100).toFixed(1)}%`,
+          );
+        }
+        if (loopCount === 1) {
+          recheckLog(
+            `[Recheck] 当前无待重检任务（7 天内已全部检测过或无非商家用户），${attachInterval}秒后重试...`,
+          );
+        }
+        await new Promise((r) => setTimeout(r, attachInterval * 1000));
+        continue;
+      }
+
+      recheckLog(`\n[Recheck] 获取到 ${tasks.length} 个待重检任务...`);
+      taskBatchCount++;
+
+      if (shuttingDown) {
+        recheckLog("[Recheck] 正在处理当前批次任务，完成后退出...");
+      }
+
+      const results = await Promise.allSettled(
+        tasks.map(async (task) => {
+          const uniqueId = task.uniqueId;
+          const countryTag = task.guessedLocation
+            ? ` [国家: ${task.guessedLocation}]`
+            : "";
+          recheckLog(`  → 重检 @${uniqueId}...${countryTag}`);
+          try {
+            const info = await scraper.getUserInfo(uniqueId);
+            return { uniqueId, info, error: null };
+          } catch (err) {
+            return { uniqueId, info: null, error: err };
+          }
+        }),
+      );
+
+      let successCount = 0;
+      let failCount = 0;
+      let cdnBlockedCount = 0;
+      let needRestart = false;
+
+      const successTasks = [];
+      for (const result of results) {
+        if (result.status === "fulfilled") {
+          const { uniqueId, info, error } = result.value;
+          if (error) {
+            if (error instanceof CDNBlockedError) {
+              recheckLog(`  ⚠ @${uniqueId} CDN限流 (Access Denied)`);
+              cdnBlockedCount++;
+              failCount++;
+            } else if (isBrowserClosedError(error)) {
+              needRestart = true;
+              recheckLog(`  ✗ @${uniqueId} 浏览器断开: ${error.message}`);
+              failCount++;
+            } else {
+              recheckLog(
+                `  ✗ @${uniqueId} 获取失败: ${error.message || "未知错误"}`,
+              );
+              failCount++;
+            }
+          } else if (info) {
+            if (info.error) {
+              // TikTok 返回了明确状态码（如 404），也提交记录
+              successTasks.push({ uniqueId, info });
+            } else {
+              successTasks.push({ uniqueId, info });
+            }
+          } else {
+            recheckLog(`  ✗ @${uniqueId} 未获取到用户信息`);
+            failCount++;
+          }
+        } else {
+          recheckLog(
+            `  ✗ 任务执行异常: ${result.reason?.message || "未知错误"}`,
+          );
+          failCount++;
+        }
+      }
+
+      // 批量提交结果
+      if (successTasks.length > 0) {
+        try {
+          const batchRet = await apiPost(
+            `${serverUrl}/api/recheck-jobs-batch`,
+            { updates: successTasks },
+          );
+          if (batchRet && batchRet.results && Array.isArray(batchRet.results)) {
+            for (const r of batchRet.results) {
+              if (r.ok) {
+                successCount++;
+                totalChecked++;
+                const task = successTasks.find(
+                  (t) => t.uniqueId === r.uniqueId,
+                );
+                const info = task?.info;
+                const isSeller = info?.ttSeller;
+                const sellerTag = isSeller ? " [ttSeller=true ✓]" : "";
+                if (isSeller) totalSellerFound++;
+                if (info && info.error) {
+                  recheckLog(
+                    `  ⚠ @${r.uniqueId} 已跳过 (statusCode=${info.statusCode})`,
+                  );
+                } else {
+                  recheckLog(`  ✓${sellerTag} @${r.uniqueId} 已更新`);
+                }
+              } else {
+                failCount++;
+                recheckLog(`  ✗ @${r.uniqueId} 更新失败: ${r.error}`);
+              }
+            }
+          } else {
+            recheckLog(`  ⚠ 后端响应格式异常，降级为批量成功`);
+            successCount = successTasks.length;
+            totalChecked += successTasks.length;
+          }
+        } catch (err) {
+          failCount += successTasks.length;
+          recheckLog(`  ✗ 批量提交失败: ${err.message}`);
+        }
+      }
+
+      recheckLog(
+        `  本批结果: ${successCount} 成功, ${failCount} 失败 (累计: ${totalChecked})`,
+      );
+
+      // 每 10 轮打印汇总
+      if (taskBatchCount % 10 === 0) {
+        recheckLog(
+          `  ── 已跑 ${taskBatchCount} 轮，发现 ttSeller: ${totalSellerFound}，累计检测: ${totalChecked}`,
+        );
+      }
+
+      // CDN 限流比例超过 30% 时，冷却 + 重启浏览器
+      const cdnRatio = tasks.length > 0 ? cdnBlockedCount / tasks.length : 0;
+      if (cdnRatio > 0.3) {
+        const coolDownSeconds = cdnRatio > 0.8 ? 120 : 60;
+        recheckLog(
+          `  [Recheck] CDN限流比例 ${(cdnRatio * 100).toFixed(0)}% (${cdnBlockedCount}/${tasks.length})，冷却 ${coolDownSeconds} 秒后重启浏览器...`,
+        );
+        await new Promise((r) => setTimeout(r, coolDownSeconds * 1000));
+        await recycleScraper(
+          scraper,
+          `CDN限流比例过高 (${cdnBlockedCount}/${tasks.length})`,
+        );
+        browserRestartCount++;
+        taskBatchCount = 0;
+      }
+
+      const heap = getHeapUsage();
+      if (heap.ratio >= HEAP_RESTART_RATIO) {
+        await recycleScraper(
+          scraper,
+          `本批后堆占用 ${(heap.ratio * 100).toFixed(1)}%`,
+        );
+        browserRestartCount++;
+        taskBatchCount = 0;
+      } else if (taskBatchCount >= MAX_TASK_BATCHES_BEFORE_RESTART) {
+        await recycleScraper(scraper, `已连续处理 ${taskBatchCount} 批任务`);
+        browserRestartCount++;
+        taskBatchCount = 0;
+      }
+
+      if (needRestart) {
+        recheckLog("[Recheck] 检测到浏览器异常，将在下一轮重启...");
+      }
+
+      await new Promise((r) => setTimeout(r, 500));
+    }
+  } catch (err) {
+    recheckLog(`[Recheck] 运行异常: ${err.message}`);
+    throw err;
+  } finally {
+    if (forceExitTimer) {
+      clearTimeout(forceExitTimer);
+      forceExitTimer = null;
+    }
+    process.removeListener("SIGINT", onSigint);
+    process.removeListener("SIGTERM", onSigterm);
+    if (!shuttingDown) {
+      await scraper.close().catch(() => {});
+    }
+  }
+}

package/src/cli/tag.js

@@ -1424,7 +1424,7 @@ export async function handleTagRefresh(parsed) {
       cdpPort +
       (effectiveProxy ? "  代理: " + effectiveProxy : ""),
   );
-  log("║     只抓取不重新打分，score>70 的标签");
+  log("║     只抓取不重新打分，score>60 的标签");
   log("╚══════════════════════════════════════════╝");
   log("");
 

package/src/lib/args.js

@@ -604,6 +604,54 @@ function parseAttachArgs(args) {
   };
 }
 
+function parseRecheckArgs(args) {
+  let parallel = 1;
+  let interval = 10;
+  let serverUrl = defaultServer;
+  let countries = [];
+  let customProxy = null;
+  let poolSize = 3;
+
+  for (let i = 0; i < args.length; i++) {
+    const arg = args[i];
+    if (arg === "-p" || arg === "--parallel") {
+      parallel = parseInt(args[++i], 10) || 1;
+    } else if (arg === "-i" || arg === "--interval") {
+      interval = parseInt(args[++i], 10) || 10;
+    } else if (arg === "-s" || arg === "--server") {
+      serverUrl = args[++i];
+    } else if (arg === "-c" || arg === "--countries") {
+      countries = args[++i]
+        .split(",")
+        .map((c) => c.trim().toUpperCase())
+        .filter(Boolean);
+    } else if (arg === "--proxy") {
+      customProxy = args[++i];
+    } else if (arg === "--size") {
+      poolSize = parseInt(args[++i], 10) || 3;
+    }
+  }
+
+  return {
+    subcommand: "recheck",
+    attachParallel: parallel,
+    attachInterval: interval,
+    serverUrl,
+    attachCountries: countries,
+    customProxy,
+    attachPoolSize: poolSize,
+    urls: [],
+    outputFormat: "json",
+    exploreCount: 0,
+    showConfig: false,
+    showHelp: false,
+    configAction: null,
+    configValue: null,
+    pipeMode: false,
+    filterStr: null,
+  };
+}
+
 function parseOpenArgs(args) {
   let openPort = null;
   let openList = false;
@@ -767,6 +815,8 @@ function parseTagArgs(args) {
   let scoreTag = null;
   let scoreCountries = null;
   let scorePort = 9222;
+  // 延迟路由：--port 的值先暂存，等循环结束后根据已确定的 subcommand 分配
+  let rawTagPort = null;
 
   for (let i = 0; i < args.length; i++) {
     const arg = args[i];
@@ -808,13 +858,9 @@ function parseTagArgs(args) {
         .map((s) => s.trim().toUpperCase())
         .filter(Boolean);
     } else if (arg === "--port") {
-      if (isScoreAll) {
-        scoreAllPort = parseInt(args[++i]) || 7222;
-      } else if (isRefresh) {
-        refreshPort = parseInt(args[++i]) || 8222;
-      } else {
-        scorePort = parseInt(args[++i]) || 9222;
-      }
+      // 延迟路由：暂存端口值，等循环结束后根据 subcommand 分配
+      // 避免 --port 出现在 refresh/score-all 之前时被错误分配到其他变量
+      rawTagPort = parseInt(args[++i]);
     } else if (arg === "--proxy") {
       scoreProxy = args[++i];
     } else if (arg === "-p" || arg === "--prompt") {
@@ -852,6 +898,13 @@ function parseTagArgs(args) {
     }
   }
 
+  // ── 延迟路由：将 --port 值分配到正确的 subcommand 变量 ──
+  if (rawTagPort !== null && !isNaN(rawTagPort)) {
+    if (isScoreAll) scoreAllPort = rawTagPort;
+    else if (isRefresh) refreshPort = rawTagPort;
+    else scorePort = rawTagPort;
+  }
+
   if (isDiscover) {
     return {
       subcommand: "tag-discover",
@@ -1024,6 +1077,10 @@ export function parseArgs() {
     return parseAttachArgs(args.slice(1));
   }
 
+  if (args.length > 0 && args[0] === "recheck") {
+    return parseRecheckArgs(args.slice(1));
+  }
+
   if (args.length > 0 && args[0] === "open") {
     return parseOpenArgs(args.slice(1));
   }

package/src/lib/constants.js

@@ -177,6 +177,17 @@ const HELP_TEXT = [
   "    示例: tt-help attach -p 5 -i 10",
   "          tt-help attach -c PL,DE,FR -p 5",
   "",
+  "  tt-help recheck [-p 并行数] [-i 间隔秒数] [-s 服务端地址] [-c 国家列表]",
+  "    重检 jobs 表中 done 非商家用户（tt_seller!=1），7 天未更新才检测",
+  "    重新抓取用户基本信息（重点检查 tt_seller），更新回 jobs 表",
+  "    -p, --parallel <N>     并行抓取数（默认: 1）",
+  "    -i, --interval <N>     无任务时轮询间隔，单位秒（默认: 10）",
+  "    -s, --server <URL>     服务端地址（默认: http://127.0.0.1:3001）",
+  "    -c, --countries <A,B>  目标国家列表（逗号分隔，如 PL,DE,FR），默认全部",
+  "    示例: tt-help recheck -p 5 -i 10",
+  "          tt-help recheck -c PL,DE,FR -p 5",
+  "          tt-help recheck -p 5 --size 5",
+  "",
   "  tt-help watch -o <db路径> [-p 端口] [--no-llm-refill] [--no-llm-discover]",
   "    启动监控服务；运行期主数据仅来自 SQLite",
   "    选项:",

package/src/main.js

@@ -3,6 +3,7 @@ import { proxy, HELP_TEXT, getConfigText } from "./lib/constants.js";
 import { handleInfo } from "./cli/info.js";
 import { handleExplore } from "./cli/explore.js";
 import { handleAttach } from "./cli/attach.js";
+import { handleRecheck } from "./cli/recheck.js";
 import { handleWatch } from "./cli/watch.js";
 import { handleConfig, showConfig, showUsage, version } from "./cli/config.js";
 import { handleOpen } from "./cli/open.js";
@@ -29,6 +30,8 @@ async function main() {
       return handleInfo(parsed);
     case "attach":
       return handleAttach(parsed);
+    case "recheck":
+      return handleRecheck(parsed);
     case "watch":
       return handleWatch(parsed);
     case "webserver":