tt-help-cli-ycl 1.3.98 → 1.3.99

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tt-help-cli-ycl",
-  "version": "1.3.98",
+  "version": "1.3.99",
   "description": "TikTok user & video data scraper - extract ttSeller, verified, locationCreated from HTML source",
   "type": "module",
   "bin": {

package/src/cli/tag.js

@@ -18,6 +18,29 @@ import {
 
 const ALL_COUNTRIES = DEFAULT_TARGET_LOCATIONS;
 const DEFAULT_SERVER = cfgServer || "http://127.0.0.1:3000";
+const DEFAULT_SCORE_COUNTRIES = [
+  "ES",
+  "FR",
+  "DE",
+  "PT",
+  "IT",
+  "NL",
+  "BE",
+  "AT",
+  "IE",
+  "PL",
+  "CZ",
+  "GR",
+  "HU",
+];
+
+function resolveTargetCountries(countries) {
+  return countries || DEFAULT_SCORE_COUNTRIES;
+}
+
+function formatMemoryUsage(mem = process.memoryUsage()) {
+  return `rss:${(mem.rss / 1024 / 1024).toFixed(0)}MB heap:${(mem.heapUsed / 1024 / 1024).toFixed(0)}MB ext:${(mem.external / 1024 / 1024).toFixed(0)}MB ab:${(mem.arrayBuffers / 1024 / 1024).toFixed(0)}MB`;
+}
 
 // 构建带客户端追踪 header 的 fetch 封装
 function buildClientHeaders(clientId, meta, extra = {}) {
@@ -426,21 +449,7 @@ export async function handleScore(parsed) {
   const baseUrl = serverUrl || DEFAULT_SERVER;
   const cdpPort = port || 9222;
   const effectiveProxy = cliProxy || configuredProxy;
-  const targetCountries = countries || [
-    "ES",
-    "FR",
-    "DE",
-    "PT",
-    "IT",
-    "NL",
-    "BE",
-    "AT",
-    "IE",
-    "PL",
-    "CZ",
-    "GR",
-    "HU",
-  ];
+  const targetCountries = resolveTargetCountries(countries);
 
   const log = (...args) => process.stderr.write(args.join(" ") + "\n");
 
@@ -576,23 +585,11 @@ export async function handleScoreAll(parsed) {
   } = tagScoreAll || {};
 
   const baseUrl = serverUrl || DEFAULT_SERVER;
-  const cdpPort = port || 9222;
+  const defaultScoreAllPort =
+    parseInt(process.env.TAG_SCOREALL_PORT_POOL_START || "7222", 10) || 7222;
+  let cdpPort = port || defaultScoreAllPort;
   const effectiveProxy = cliProxy || configuredProxy;
-  const targetCountries = countries || [
-    "ES",
-    "FR",
-    "DE",
-    "PT",
-    "IT",
-    "NL",
-    "BE",
-    "AT",
-    "IE",
-    "PL",
-    "CZ",
-    "GR",
-    "HU",
-  ];
+  const targetCountries = resolveTargetCountries(countries);
 
   const log = (...args) => process.stderr.write(args.join(" ") + "\n");
 
@@ -613,8 +610,131 @@ export async function handleScoreAll(parsed) {
   // 连接 CDP 浏览器
   const cdpOpts = { port: cdpPort };
   if (effectiveProxy) cdpOpts.proxyServer = effectiveProxy;
-  const browser = await ensureBrowserReady(cdpOpts);
+  let browser = await ensureBrowserReady(cdpOpts);
   let page = await getOrCreatePage(browser);
+  const blockedRoutePages = new WeakSet();
+
+  async function setupPageRequestBlocking(targetPage) {
+    if (!targetPage || blockedRoutePages.has(targetPage)) return;
+    await targetPage.route("**/*", (route) => {
+      const resourceType = route.request().resourceType();
+      if (resourceType === "image" || resourceType === "stylesheet") {
+        route.abort();
+      } else {
+        route.continue();
+      }
+    });
+    blockedRoutePages.add(targetPage);
+  }
+
+  await setupPageRequestBlocking(page);
+
+  const portPoolStart = Math.max(
+    1,
+    parseInt(process.env.TAG_SCOREALL_PORT_POOL_START || "7222", 10) || 7222,
+  );
+  const portPoolSize = Math.max(
+    2,
+    parseInt(process.env.TAG_SCOREALL_PORT_POOL_SIZE || "10", 10) || 10,
+  );
+  const switchPortOnRecycle =
+    String(process.env.TAG_SCOREALL_SWITCH_PORT_ON_RECYCLE || "1") !== "0";
+
+  function pickNextPort(currentPort) {
+    const candidates = [];
+    for (let i = 0; i < portPoolSize; i++) {
+      const p = portPoolStart + i;
+      if (p !== currentPort) candidates.push(p);
+    }
+    if (candidates.length === 0) return currentPort;
+    return candidates[Math.floor(Math.random() * candidates.length)];
+  }
+
+  const memRssRecycleMb = Math.max(
+    256,
+    parseInt(process.env.TAG_SCOREALL_RECYCLE_RSS_MB || "900", 10) || 900,
+  );
+  const memHeapRecycleMb = Math.max(
+    128,
+    parseInt(process.env.TAG_SCOREALL_RECYCLE_HEAP_MB || "320", 10) || 320,
+  );
+  const recycleCooldownMs = Math.max(
+    0,
+    parseInt(process.env.TAG_SCOREALL_RECYCLE_COOLDOWN_MS || "180000", 10) ||
+      180000,
+  );
+  // 默认关闭按固定轮次重建，仅在高内存时触发；需要可通过环境变量开启。
+  const periodicRecycleEvery = Math.max(
+    0,
+    parseInt(process.env.TAG_SCOREALL_PERIODIC_RECYCLE_EVERY || "0", 10) || 0,
+  );
+  let lastRecycleAt = 0;
+
+  async function recyclePage() {
+    if (!page || page.isClosed()) {
+      page = await getOrCreatePage(browser);
+      await setupPageRequestBlocking(page);
+      return;
+    }
+    try {
+      await page.goto("about:blank", {
+        waitUntil: "domcontentloaded",
+        timeout: 5000,
+      });
+    } catch {
+      // 页面状态异常时回退到重建 tab
+      await page.close().catch(() => {});
+      page = await getOrCreatePage(browser);
+      await setupPageRequestBlocking(page);
+    }
+  }
+
+  async function recycleCdpSession(reason) {
+    const oldPort = cdpPort;
+    if (switchPortOnRecycle) {
+      cdpPort = pickNextPort(cdpPort);
+      cdpOpts.port = cdpPort;
+      clientMeta.port = cdpPort;
+    }
+    const switchHint =
+      oldPort === cdpPort ? `port=${cdpPort}` : `port ${oldPort} -> ${cdpPort}`;
+    log(`  ♻️ 重建 CDP 会话 (${reason}; ${switchHint})...`);
+    if (page) {
+      await page.close().catch(() => {});
+      page = null;
+    }
+    await browser.close().catch(() => {});
+    if (oldPort !== cdpPort) {
+      await killEdgeProcesses(null, oldPort).catch(() => {});
+    }
+    browser = await ensureBrowserReady(cdpOpts);
+    page = await getOrCreatePage(browser);
+    await setupPageRequestBlocking(page);
+    lastRecycleAt = Date.now();
+  }
+
+  async function maybeRecycleForMemory() {
+    const mem = process.memoryUsage();
+    const rssMB = mem.rss / 1024 / 1024;
+    const heapMB = mem.heapUsed / 1024 / 1024;
+    if (rssMB >= memRssRecycleMb || heapMB >= memHeapRecycleMb) {
+      const now = Date.now();
+      if (recycleCooldownMs > 0 && now - lastRecycleAt < recycleCooldownMs) {
+        return;
+      }
+      await recycleCdpSession(
+        `mem rss=${rssMB.toFixed(0)}MB heap=${heapMB.toFixed(0)}MB (threshold rss=${memRssRecycleMb} heap=${memHeapRecycleMb})`,
+      );
+      return;
+    }
+    if (
+      periodicRecycleEvery > 0 &&
+      totalScored > 0 &&
+      totalScored % periodicRecycleEvery === 0
+    ) {
+      await recycleCdpSession(`periodic every ${totalScored} tasks`);
+    }
+  }
 
   let totalScored = 0;
   let emptyRounds = 0; // 连续无任务的轮数
@@ -622,7 +742,7 @@ export async function handleScoreAll(parsed) {
 
   // 生成客户端 ID，用于服务端追踪
   const clientId = randomUUID();
-  const clientMeta = { type: "scoring" };
+  const clientMeta = { type: "scoring", port: cdpPort };
 
   // 复用 TikTokScraper 实例，避免每次 enrich 都启动/关闭 headless 浏览器
   const enrichScraper = new TikTokScraper({
@@ -719,6 +839,7 @@ export async function handleScoreAll(parsed) {
           // already claimed: 其他机器抢先了，跳过不标 dead
           if (claimData.error && claimData.error.includes("already claimed")) {
             log(`  ⏭️ 已被其他客户端锁定，跳过`);
+            await recyclePage();
             continue;
           }
           log(`  ⚠️ 无法锁定 (${claimData.error})，标记为 dead 并跳过`);
@@ -726,6 +847,7 @@ export async function handleScoreAll(parsed) {
           result.status = "dead";
           await reportToServer(baseUrl, result, clientId, clientMeta);
           totalScored++;
+          await recyclePage();
           continue;
         }
 
@@ -733,7 +855,7 @@ export async function handleScoreAll(parsed) {
         const fetchStart = Date.now();
         log(`  抓取 TikTok 标签页...`);
         const tagResult = await fetchTagData(tag, {
-          port: cdpPort,
+          page,
           onProgress: ({ videos, authors }) => {
             process.stderr.write(
               `\r  抓取中: ${videos} 视频, ${authors} 作者\x1b[K`,
@@ -751,23 +873,17 @@ export async function handleScoreAll(parsed) {
 
         if (!videos || videos.length === 0) {
           const deadSec = ((Date.now() - fetchStart) / 1000).toFixed(1);
-          const memMB = (process.memoryUsage().heapUsed / 1024 / 1024).toFixed(
-            0,
+          log(
+            `  ⚠️ 无视频 (${deadSec}s) mem=${formatMemoryUsage()}，标记 dead`,
           );
-          log(`  ⚠️ 无视频 (${deadSec}s) mem=${memMB}MB，标记 dead`);
           result.status = "dead";
           result.error = "no videos found";
           await reportToServer(baseUrl, result, clientId, clientMeta);
           totalScored++;
           // 随机等待 3-7 秒，避免连续访问 TikTok 触发风控
           await randomDelay(0, 5000);
-          // 导航到 about:blank 释放页面状态再跳过
-          await page
-            .goto("about:blank", {
-              waitUntil: "domcontentloaded",
-              timeout: 5000,
-            })
-            .catch(() => {});
+          await recyclePage();
+          await maybeRecycleForMemory();
           continue;
         }
 
@@ -836,20 +952,13 @@ export async function handleScoreAll(parsed) {
         const mc = result.matchedCountries
           .map((c) => `${c.c}:${c.n}`)
           .join(" ");
-        // Node.js 进程内存占用
-        const memMB = (process.memoryUsage().heapUsed / 1024 / 1024).toFixed(0);
-        const memStr = ` mem=${memMB}MB`;
+        const memStr = ` mem=${formatMemoryUsage()}`;
         log(
           `  ${icon} ${result.status} score=${result.score} authors=${result.authorCount} matched=${result.matchedAuthors} (${elapsed}s)${mc ? "  " + mc : ""}${memStr}`,
         );
         log("");
-
-        // 导航到 about:blank 卸载页面，状态清零，下次 goto 重新初始化
-        await page
-          .goto("about:blank", { waitUntil: "domcontentloaded", timeout: 5000 })
-          .catch((e) => {
-            log(`  ⚠️ about:blank 跳转失败: ${e.message}`);
-          });
+        await recyclePage();
+        await maybeRecycleForMemory();
         await randomDelay(3000, 7000);
       } catch (e) {
         // 区分网络错误和业务错误
@@ -881,6 +990,8 @@ export async function handleScoreAll(parsed) {
           );
         } catch {}
         totalScored++;
+        await recyclePage();
+        await maybeRecycleForMemory();
       }
     }
   } finally {

package/src/lib/args.js

@@ -739,7 +739,7 @@ function parseTagArgs(args) {
   let isDiscover = false;
   let isScore = false;
   let isScoreAll = false;
-  let scoreAllPort = 9222;
+  let scoreAllPort = 7222;
   let scoreProxy = null;
   let scoreTag = null;
   let scoreCountries = null;
@@ -788,7 +788,7 @@ function parseTagArgs(args) {
         .filter(Boolean);
     } else if (arg === "--port") {
       if (isScoreAll) {
-        scoreAllPort = parseInt(args[++i]) || 9222;
+        scoreAllPort = parseInt(args[++i]) || 7222;
       } else {
         scorePort = parseInt(args[++i]) || 9222;
       }

package/src/lib/tag-fetcher.js

@@ -48,14 +48,19 @@ export async function fetchTagData(tag, options = {}) {
     userDataDir,
     proxyServer,
     onProgress,
+    browser: existingBrowser,
+    page: existingPage,
   } = options;
 
   const cdpOptions = { port };
   if (userDataDir) cdpOptions.userDataDir = userDataDir;
   if (proxyServer) cdpOptions.proxyServer = proxyServer;
 
-  const browser = await ensureBrowserReady(cdpOptions);
-  const page = await getOrCreatePage(browser);
+  const browser =
+    existingBrowser ||
+    (existingPage ? existingPage.context().browser() : null) ||
+    (await ensureBrowserReady(cdpOptions));
+  const page = existingPage || (await getOrCreatePage(browser));
 
   let challengeInfo = null;
   const rawVideos = [];