tt-help-cli-ycl 1.4.3 → 1.4.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tt-help-cli-ycl",
-  "version": "1.4.3",
+  "version": "1.4.4",
   "description": "TikTok user & video data scraper - extract ttSeller, verified, locationCreated from HTML source",
   "type": "module",
   "bin": {

package/src/cli/tag.js

@@ -1,5 +1,7 @@
 import { writeFileSync } from "fs";
 import { randomUUID } from "crypto";
+import { execSync } from "child_process";
+import os from "os";
 import readline from "readline";
 import { fetchTagData, enrichVideosWithLocation } from "../lib/tag-fetcher.js";
 import { killEdgeProcesses, ensureBrowserReady } from "../lib/browser/cdp.js";
@@ -42,6 +44,46 @@ function formatMemoryUsage(mem = process.memoryUsage()) {
   return `rss:${(mem.rss / 1024 / 1024).toFixed(0)}MB heap:${(mem.heapUsed / 1024 / 1024).toFixed(0)}MB ext:${(mem.external / 1024 / 1024).toFixed(0)}MB ab:${(mem.arrayBuffers / 1024 / 1024).toFixed(0)}MB`;
 }
 
+/**
+ * 获取 Edge 浏览器进程的内存占用（macOS/Linux）
+ */
+function getBrowserMemory(port = null) {
+  try {
+    const platform = os.platform();
+    let cmd;
+    if (platform === "darwin") {
+      // 按端口筛选：匹配 --remote-debugging-port=PORT 的进程
+      if (port) {
+        cmd = `ps aux | grep "[M]icrosoft Edge" | grep "remote-debugging-port=${port}" | awk '{sum += $6} END {printf "%d", sum/1024}'`;
+      } else {
+        cmd =
+          'ps aux | grep "[M]icrosoft Edge" | awk \'{sum += $6} END {printf "%d", sum/1024}\'';
+      }
+    } else if (platform === "linux") {
+      if (port) {
+        cmd = `ps aux | grep "[m]sedge" | grep "remote-debugging-port=${port}" | awk '{sum += $6} END {printf "%d", sum/1024}'`;
+      } else {
+        cmd =
+          'ps aux | grep "[m]sedge" | awk \'{sum += $6} END {printf "%d", sum/1024}\'';
+      }
+    } else {
+      return null;
+    }
+    const result = execSync(cmd, { timeout: 3000 }).toString().trim();
+    const mb = parseInt(result, 10);
+    return mb > 0 ? mb : null;
+  } catch {
+    return null;
+  }
+}
+
+function formatMemoryWithBrowser(port = null) {
+  const nodeMem = formatMemoryUsage();
+  const browserMB = getBrowserMemory(port);
+  const portLabel = port ? `:${port}` : "";
+  return browserMB ? `${nodeMem} edge${portLabel}:${browserMB}MB` : nodeMem;
+}
+
 // 构建带客户端追踪 header 的 fetch 封装
 function buildClientHeaders(clientId, meta, extra = {}) {
   return {
@@ -576,14 +618,7 @@ async function reportToServer(baseUrl, result, clientId, meta) {
 
 export async function handleScoreAll(parsed) {
   const { tagScoreAll } = parsed;
-  let {
-    countries,
-    serverUrl,
-    autoDiscover,
-    discoverCount = 10,
-    port,
-    proxy: cliProxy,
-  } = tagScoreAll || {};
+  let { countries, serverUrl, port, proxy: cliProxy } = tagScoreAll || {};
 
   const baseUrl = serverUrl || DEFAULT_SERVER;
   const defaultScoreAllPort =
@@ -604,10 +639,14 @@ export async function handleScoreAll(parsed) {
       cdpPort +
       (effectiveProxy ? "  代理: " + effectiveProxy : ""),
   );
-  if (autoDiscover) log("║  自动发现: 开启");
+  log("║  发现: 服务端定时自动（无需 --auto-discover）");
   log("╚══════════════════════════════════════════╝");
   log("");
 
+  // 启动时显示内存状态
+  log(`📊 启动内存: ${formatMemoryWithBrowser(cdpPort)}`);
+  log("");
+
   // 连接 CDP 浏览器
   const cdpOpts = { port: cdpPort };
   if (effectiveProxy) cdpOpts.proxyServer = effectiveProxy;
@@ -636,8 +675,6 @@ export async function handleScoreAll(parsed) {
     2,
     parseInt(process.env.TAG_SCOREALL_PORT_POOL_SIZE || "10", 10) || 10,
   );
-  const switchPortOnRecycle =
-    String(process.env.TAG_SCOREALL_SWITCH_PORT_ON_RECYCLE || "1") !== "0";
 
   function pickNextPort(currentPort) {
     const candidates = [];
@@ -649,26 +686,35 @@ export async function handleScoreAll(parsed) {
     return candidates[Math.floor(Math.random() * candidates.length)];
   }
 
-  const memRssRecycleMb = Math.max(
-    256,
-    parseInt(process.env.TAG_SCOREALL_RECYCLE_RSS_MB || "900", 10) || 900,
-  );
-  const memHeapRecycleMb = Math.max(
-    128,
-    parseInt(process.env.TAG_SCOREALL_RECYCLE_HEAP_MB || "512", 10) || 512,
-  );
-  const recycleCooldownMs = Math.max(
-    0,
-    parseInt(process.env.TAG_SCOREALL_RECYCLE_COOLDOWN_MS || "180000", 10) ||
-      180000,
-  );
-  // 默认关闭按固定轮次重建，仅在高内存时触发；需要可通过环境变量开启。
-  const periodicRecycleEvery = Math.max(
-    0,
-    parseInt(process.env.TAG_SCOREALL_PERIODIC_RECYCLE_EVERY || "0", 10) || 0,
-  );
+  // ── 内存回收配置 ──
+  const RECYCLE_CONFIG = {
+    nodeRssMb: Math.max(
+      256,
+      parseInt(process.env.TAG_SCOREALL_RECYCLE_RSS_MB || "900", 10) || 900,
+    ),
+    nodeHeapMb: Math.max(
+      128,
+      parseInt(process.env.TAG_SCOREALL_RECYCLE_HEAP_MB || "512", 10) || 512,
+    ),
+    edgeMb: Math.max(
+      500,
+      parseInt(process.env.TAG_SCOREALL_RECYCLE_EDGE_MB || "2000", 10) || 2000,
+    ),
+    cooldownMs: Math.max(
+      0,
+      parseInt(process.env.TAG_SCOREALL_RECYCLE_COOLDOWN_MS || "180000", 10) ||
+        180000,
+    ),
+    periodicEvery: Math.max(
+      0,
+      parseInt(process.env.TAG_SCOREALL_PERIODIC_RECYCLE_EVERY || "0", 10) || 0,
+    ),
+    switchPortOnRecycle:
+      String(process.env.TAG_SCOREALL_SWITCH_PORT_ON_RECYCLE || "1") !== "0",
+  };
   let lastRecycleAt = 0;
 
+  // ── 轻量回收：只清空当前 tab ──
   async function recyclePage() {
     if (!page || page.isClosed()) {
       page = await getOrCreatePage(browser);
@@ -681,23 +727,24 @@ export async function handleScoreAll(parsed) {
         timeout: 5000,
       });
     } catch {
-      // 页面状态异常时回退到重建 tab
       await page.close().catch(() => {});
       page = await getOrCreatePage(browser);
       await setupPageRequestBlocking(page);
     }
   }
 
+  // ── 深度回收：关闭浏览器 + 可选切换端口 ──
   async function recycleCdpSession(reason, { switchPort = true } = {}) {
     const oldPort = cdpPort;
-    if (switchPort && switchPortOnRecycle) {
+    const shouldSwitch = switchPort && RECYCLE_CONFIG.switchPortOnRecycle;
+    if (shouldSwitch) {
       cdpPort = pickNextPort(cdpPort);
       cdpOpts.port = cdpPort;
       clientMeta.port = cdpPort;
     }
-    const switchHint =
-      oldPort === cdpPort ? `port=${cdpPort}` : `port ${oldPort} -> ${cdpPort}`;
-    log(`  ♻️ 重建 CDP 会话 (${reason}; ${switchHint})...`);
+    const portHint =
+      oldPort === cdpPort ? `port=${cdpPort}` : `port ${oldPort}→${cdpPort}`;
+    log(`  ♻️ 重建 CDP 会话 (${reason}; ${portHint})...`);
     if (page) {
       await page.close().catch(() => {});
       page = null;
@@ -712,27 +759,49 @@ export async function handleScoreAll(parsed) {
     lastRecycleAt = Date.now();
   }
 
+  // ── 内存检查 + 自动回收 ──
+  // 策略：Node.js 内存超阈值 → 不换端口（只重建连接）
+  //        Edge 内存超阈值 → 换端口（杀旧浏览器 + 启新实例，彻底释放）
+  //        定期轮次         → 不换端口
   async function maybeRecycleForMemory() {
     const mem = process.memoryUsage();
     const rssMB = mem.rss / 1024 / 1024;
     const heapMB = mem.heapUsed / 1024 / 1024;
-    if (rssMB >= memRssRecycleMb || heapMB >= memHeapRecycleMb) {
-      const now = Date.now();
-      if (recycleCooldownMs > 0 && now - lastRecycleAt < recycleCooldownMs) {
+    const edgeMB = getBrowserMemory(cdpPort);
+
+    const nodeOver =
+      rssMB >= RECYCLE_CONFIG.nodeRssMb || heapMB >= RECYCLE_CONFIG.nodeHeapMb;
+    const edgeOver = edgeMB !== null && edgeMB >= RECYCLE_CONFIG.edgeMb;
+
+    if (nodeOver || edgeOver) {
+      // 冷却期内跳过
+      if (
+        RECYCLE_CONFIG.cooldownMs > 0 &&
+        Date.now() - lastRecycleAt < RECYCLE_CONFIG.cooldownMs
+      )
         return;
-      }
-      await recycleCdpSession(
-        `mem rss=${rssMB.toFixed(0)}MB heap=${heapMB.toFixed(0)}MB (threshold rss=${memRssRecycleMb} heap=${memHeapRecycleMb})`,
-        { switchPort: false },
-      );
+
+      const parts = [];
+      if (rssMB >= RECYCLE_CONFIG.nodeRssMb)
+        parts.push(`rss=${rssMB.toFixed(0)}≥${RECYCLE_CONFIG.nodeRssMb}`);
+      if (heapMB >= RECYCLE_CONFIG.nodeHeapMb)
+        parts.push(`heap=${heapMB.toFixed(0)}≥${RECYCLE_CONFIG.nodeHeapMb}`);
+      if (edgeOver)
+        parts.push(`edge:${cdpPort}=${edgeMB}≥${RECYCLE_CONFIG.edgeMb}`);
+
+      await recycleCdpSession(`mem ${parts.join("+")}`, {
+        switchPort: edgeOver,
+      });
       return;
     }
+
+    // 定期轮次回收（默认关闭）
     if (
-      periodicRecycleEvery > 0 &&
+      RECYCLE_CONFIG.periodicEvery > 0 &&
       totalScored > 0 &&
-      totalScored % periodicRecycleEvery === 0
+      totalScored % RECYCLE_CONFIG.periodicEvery === 0
     ) {
-      await recycleCdpSession(`periodic every ${totalScored} tasks`, {
+      await recycleCdpSession(`periodic #${totalScored}`, {
         switchPort: false,
       });
     }
@@ -741,12 +810,7 @@ export async function handleScoreAll(parsed) {
   let totalScored = 0;
   let emptyRounds = 0; // 连续无任务的轮数
   let consecutive403 = 0; // 连续 HTTP 403 次数，达 3 次认为被封
-  const DISCOVER_AFTER_EMPTY = 3; // 连续 3 轮无任务时触发 discover（兜底）
   const CONSECUTIVE_403_BLOCKED = 3; // 连续 3 次 403 认为账号被封
-  const DISCOVER_LOW_THRESHOLD = 5; // 剩余 new 标签 < 5 时提前触发发现
-  const DISCOVER_COOLDOWN_MS = 5 * 60 * 1000; // 发现冷却时间 5 分钟，防止频繁调用 LLM
-  let discoverPromise = null; // 正在后台执行的发现 Promise
-  let lastDiscoverTime = 0; // 上次发现完成的时间戳
 
   // 生成客户端 ID，用于服务端追踪
   const clientId = randomUUID();
@@ -771,39 +835,6 @@ export async function handleScoreAll(parsed) {
   };
   process.on("SIGINT", cleanup);
 
-  // 异步发现函数：后台为所有目标国家生成标签，不阻塞打分循环
-  async function runDiscoverInBackground() {
-    if (discoverPromise) return; // 已在执行中，不重复触发
-    const elapsed = Date.now() - lastDiscoverTime;
-    if (elapsed < DISCOVER_COOLDOWN_MS) {
-      const remainMin = Math.ceil((DISCOVER_COOLDOWN_MS - elapsed) / 60000);
-      log(`🔍 发现冷却中，还需 ${remainMin} 分钟后才能再次触发`);
-      return;
-    }
-    log(`🔍 剩余标签不足，后台为 ${targetCountries.length} 个国家生成标签...`);
-    discoverPromise = (async () => {
-      let totalInserted = 0;
-      for (const country of targetCountries) {
-        try {
-          const discRes = await fetch(
-            `${baseUrl}/api/tags/discover?country=${country}&count=${discoverCount}`,
-            { headers: buildClientHeaders(clientId, clientMeta) },
-          );
-          const discData = await discRes.json();
-          if (discData.added) {
-            totalInserted += discData.added;
-            log(`  ${country}: 新增 ${discData.added} 个`);
-          }
-        } catch (e) {
-          log(`  ${country}: 请求失败 (${e.message})`);
-        }
-      }
-      log(`  ✅ 发现完成，共入库 ${totalInserted} 个新标签`);
-      lastDiscoverTime = Date.now(); // 记录完成时间，开始冷却
-      discoverPromise = null;
-    })();
-  }
-
   try {
     while (true) {
       try {
@@ -813,41 +844,16 @@ export async function handleScoreAll(parsed) {
         });
         const tagsData = await tagsRes.json();
 
-        // 提前触发发现：剩余 new 标签低于阈值且不在冷却期时后台生成
-        if (
-          autoDiscover &&
-          !discoverPromise &&
-          Date.now() - lastDiscoverTime >= DISCOVER_COOLDOWN_MS &&
-          (tagsData.total || 0) < DISCOVER_LOW_THRESHOLD
-        ) {
-          runDiscoverInBackground();
-        }
-
         if (!tagsData.tags || tagsData.tags.length === 0) {
           emptyRounds++;
 
-          // 兜底发现：连续 N 轮无任务（提前发现可能没生效）
-          if (autoDiscover && emptyRounds >= DISCOVER_AFTER_EMPTY) {
-            // 等后台发现完成（如果正在跑）
-            if (discoverPromise) {
-              log(`  ⏳ 等待后台发现完成...`);
-              await discoverPromise;
-            } else if (Date.now() - lastDiscoverTime >= DISCOVER_COOLDOWN_MS) {
-              // 冷却期已过，触发一次同步发现
-              log(`🔍 连续 ${emptyRounds} 轮无待打分标签，同步生成标签...`);
-              await runDiscoverInBackground();
-              await discoverPromise;
-            } else {
-              const remainMin = Math.ceil(
-                (DISCOVER_COOLDOWN_MS - (Date.now() - lastDiscoverTime)) /
-                  60000,
-              );
-              log(
-                `🔍 连续 ${emptyRounds} 轮空，但发现冷却中（还需 ${remainMin} 分钟），继续等待...`,
-              );
-            }
+          // 服务端定时发现会自动补充标签，客户端只需等待
+          if (emptyRounds >= 5) {
+            log(
+              `⏳ 连续 ${emptyRounds} 轮无标签，等待服务端发现补充（30 秒后重试）...`,
+            );
+            await new Promise((r) => setTimeout(r, 30000));
             emptyRounds = 0;
-            await new Promise((r) => setTimeout(r, 3000));
             continue;
           }
           log(`⏳ 暂无待打分标签（连续 ${emptyRounds} 轮），10 秒后重试...`);
@@ -926,7 +932,7 @@ export async function handleScoreAll(parsed) {
         if (!videos || videos.length === 0) {
           const deadSec = ((Date.now() - fetchStart) / 1000).toFixed(1);
           log(
-            `  ⚠️ 无视频 (${deadSec}s) mem=${formatMemoryUsage()}，标记 dead`,
+            `  ⚠️ 无视频 (${deadSec}s) mem=${formatMemoryWithBrowser(cdpPort)}，标记 dead`,
           );
           result.status = "dead";
           result.error = "no videos found";
@@ -1004,12 +1010,18 @@ export async function handleScoreAll(parsed) {
         const mc = result.matchedCountries
           .map((c) => `${c.c}:${c.n}`)
           .join(" ");
-        const memStr = ` mem=${formatMemoryUsage()}`;
+        const memStr = ` mem=${formatMemoryWithBrowser(cdpPort)}`;
         log(
           `  ${icon} ${result.status} score=${result.score} authors=${result.authorCount} matched=${result.matchedAuthors} (${elapsed}s)${mc ? "  " + mc : ""}${memStr}`,
         );
         log("");
         consecutive403 = 0; // 成功完成，重置 403 计数器
+        // 每 10 个标签输出一次内存汇总
+        if (totalScored % 10 === 0) {
+          log(
+            `📊 [${totalScored}] 内存汇总: ${formatMemoryWithBrowser(cdpPort)}`,
+          );
+        }
         await recyclePage();
         await maybeRecycleForMemory();
         await randomDelay(3000, 7000);

package/src/lib/args.js

@@ -735,12 +735,10 @@ function parseTagArgs(args) {
   let discoverCountries = [];
   let discoverCount = 4;
   let discoverPrompt = null;
-  let autoDiscover = false;
   let isDiscover = false;
   let isScore = false;
   let isScoreAll = false;
   let scoreAllPort = 7222;
-  let discoverCountScoreAll = 10;
   let scoreProxy = null;
   let scoreTag = null;
   let scoreCountries = null;
@@ -780,10 +778,6 @@ function parseTagArgs(args) {
       }
     } else if (arg === "--count") {
       discoverCount = parseInt(args[++i]) || 4;
-    } else if (arg === "--auto-discover") {
-      autoDiscover = true;
-    } else if (arg === "--discover-count") {
-      discoverCountScoreAll = parseInt(args[++i]) || 10;
     } else if (arg === "--countries") {
       scoreCountries = args[++i]
         .split(",")
@@ -881,8 +875,6 @@ function parseTagArgs(args) {
       tagScoreAll: {
         countries: scoreCountries,
         serverUrl,
-        autoDiscover,
-        discoverCount: discoverCountScoreAll,
         port: scoreAllPort,
         proxy: scoreProxy,
       },