tt-help-cli-ycl 1.3.98 → 1.3.100

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tt-help-cli-ycl",
-  "version": "1.3.98",
+  "version": "1.3.100",
   "description": "TikTok user & video data scraper - extract ttSeller, verified, locationCreated from HTML source",
   "type": "module",
   "bin": {

package/src/cli/tag.js

@@ -5,6 +5,7 @@ import { fetchTagData, enrichVideosWithLocation } from "../lib/tag-fetcher.js";
 import { killEdgeProcesses, ensureBrowserReady } from "../lib/browser/cdp.js";
 import { getOrCreatePage } from "../lib/browser/page.js";
 import { TikTokScraper } from "../lib/tiktok-scraper.mjs";
+import { CDNBlockedError } from "../lib/parse-ssr.mjs";
 import {
   DEFAULT_TARGET_LOCATIONS,
   isLocationInList,
@@ -18,6 +19,41 @@ import {
 
 const ALL_COUNTRIES = DEFAULT_TARGET_LOCATIONS;
 const DEFAULT_SERVER = cfgServer || "http://127.0.0.1:3000";
+const DEFAULT_SCORE_COUNTRIES = [
+  "ES",
+  "FR",
+  "DE",
+  "PT",
+  "IT",
+  "NL",
+  "BE",
+  "AT",
+  "IE",
+  "PL",
+  "CZ",
+  "GR",
+  "HU",
+];
+
+function resolveTargetCountries(countries) {
+  return countries || DEFAULT_SCORE_COUNTRIES;
+}
+
+function formatMemoryUsage(mem = process.memoryUsage()) {
+  return `rss:${(mem.rss / 1024 / 1024).toFixed(0)}MB heap:${(mem.heapUsed / 1024 / 1024).toFixed(0)}MB ext:${(mem.external / 1024 / 1024).toFixed(0)}MB ab:${(mem.arrayBuffers / 1024 / 1024).toFixed(0)}MB`;
+}
+
+function getCdnCooldownSeconds(blockedCount, totalCount, isTooManyRequests = false) {
+  if (isTooManyRequests) return 120;
+  const ratio = blockedCount / Math.max(totalCount, 1);
+  return ratio > 0.3 ? 120 : 60;
+}
+
+async function cooldownAndRecycle(cooldownSec, recyclePage, maybeRecycleForMemory) {
+  await new Promise((r) => setTimeout(r, cooldownSec * 1000));
+  await recyclePage();
+  await maybeRecycleForMemory();
+}
 
 // 构建带客户端追踪 header 的 fetch 封装
 function buildClientHeaders(clientId, meta, extra = {}) {
@@ -426,21 +462,7 @@ export async function handleScore(parsed) {
   const baseUrl = serverUrl || DEFAULT_SERVER;
   const cdpPort = port || 9222;
   const effectiveProxy = cliProxy || configuredProxy;
-  const targetCountries = countries || [
-    "ES",
-    "FR",
-    "DE",
-    "PT",
-    "IT",
-    "NL",
-    "BE",
-    "AT",
-    "IE",
-    "PL",
-    "CZ",
-    "GR",
-    "HU",
-  ];
+  const targetCountries = resolveTargetCountries(countries);
 
   const log = (...args) => process.stderr.write(args.join(" ") + "\n");
 
@@ -576,23 +598,11 @@ export async function handleScoreAll(parsed) {
   } = tagScoreAll || {};
 
   const baseUrl = serverUrl || DEFAULT_SERVER;
-  const cdpPort = port || 9222;
+  const defaultScoreAllPort =
+    parseInt(process.env.TAG_SCOREALL_PORT_POOL_START || "7222", 10) || 7222;
+  let cdpPort = port || defaultScoreAllPort;
   const effectiveProxy = cliProxy || configuredProxy;
-  const targetCountries = countries || [
-    "ES",
-    "FR",
-    "DE",
-    "PT",
-    "IT",
-    "NL",
-    "BE",
-    "AT",
-    "IE",
-    "PL",
-    "CZ",
-    "GR",
-    "HU",
-  ];
+  const targetCountries = resolveTargetCountries(countries);
 
   const log = (...args) => process.stderr.write(args.join(" ") + "\n");
 
@@ -613,8 +623,131 @@ export async function handleScoreAll(parsed) {
   // 连接 CDP 浏览器
   const cdpOpts = { port: cdpPort };
   if (effectiveProxy) cdpOpts.proxyServer = effectiveProxy;
-  const browser = await ensureBrowserReady(cdpOpts);
+  let browser = await ensureBrowserReady(cdpOpts);
   let page = await getOrCreatePage(browser);
+  const blockedRoutePages = new WeakSet();
+
+  async function setupPageRequestBlocking(targetPage) {
+    if (!targetPage || blockedRoutePages.has(targetPage)) return;
+    await targetPage.route("**/*", (route) => {
+      const resourceType = route.request().resourceType();
+      if (resourceType === "image" || resourceType === "stylesheet") {
+        route.abort();
+      } else {
+        route.continue();
+      }
+    });
+    blockedRoutePages.add(targetPage);
+  }
+
+  await setupPageRequestBlocking(page);
+
+  const portPoolStart = Math.max(
+    1,
+    parseInt(process.env.TAG_SCOREALL_PORT_POOL_START || "7222", 10) || 7222,
+  );
+  const portPoolSize = Math.max(
+    2,
+    parseInt(process.env.TAG_SCOREALL_PORT_POOL_SIZE || "10", 10) || 10,
+  );
+  const switchPortOnRecycle =
+    String(process.env.TAG_SCOREALL_SWITCH_PORT_ON_RECYCLE || "1") !== "0";
+
+  function pickNextPort(currentPort) {
+    const candidates = [];
+    for (let i = 0; i < portPoolSize; i++) {
+      const p = portPoolStart + i;
+      if (p !== currentPort) candidates.push(p);
+    }
+    if (candidates.length === 0) return currentPort;
+    return candidates[Math.floor(Math.random() * candidates.length)];
+  }
+
+  const memRssRecycleMb = Math.max(
+    256,
+    parseInt(process.env.TAG_SCOREALL_RECYCLE_RSS_MB || "900", 10) || 900,
+  );
+  const memHeapRecycleMb = Math.max(
+    128,
+    parseInt(process.env.TAG_SCOREALL_RECYCLE_HEAP_MB || "320", 10) || 320,
+  );
+  const recycleCooldownMs = Math.max(
+    0,
+    parseInt(process.env.TAG_SCOREALL_RECYCLE_COOLDOWN_MS || "180000", 10) ||
+      180000,
+  );
+  // 默认关闭按固定轮次重建，仅在高内存时触发；需要可通过环境变量开启。
+  const periodicRecycleEvery = Math.max(
+    0,
+    parseInt(process.env.TAG_SCOREALL_PERIODIC_RECYCLE_EVERY || "0", 10) || 0,
+  );
+  let lastRecycleAt = 0;
+
+  async function recyclePage() {
+    if (!page || page.isClosed()) {
+      page = await getOrCreatePage(browser);
+      await setupPageRequestBlocking(page);
+      return;
+    }
+    try {
+      await page.goto("about:blank", {
+        waitUntil: "domcontentloaded",
+        timeout: 5000,
+      });
+    } catch {
+      // 页面状态异常时回退到重建 tab
+      await page.close().catch(() => {});
+      page = await getOrCreatePage(browser);
+      await setupPageRequestBlocking(page);
+    }
+  }
+
+  async function recycleCdpSession(reason) {
+    const oldPort = cdpPort;
+    if (switchPortOnRecycle) {
+      cdpPort = pickNextPort(cdpPort);
+      cdpOpts.port = cdpPort;
+      clientMeta.port = cdpPort;
+    }
+    const switchHint =
+      oldPort === cdpPort ? `port=${cdpPort}` : `port ${oldPort} -> ${cdpPort}`;
+    log(`  ♻️ 重建 CDP 会话 (${reason}; ${switchHint})...`);
+    if (page) {
+      await page.close().catch(() => {});
+      page = null;
+    }
+    await browser.close().catch(() => {});
+    if (oldPort !== cdpPort) {
+      await killEdgeProcesses(null, oldPort).catch(() => {});
+    }
+    browser = await ensureBrowserReady(cdpOpts);
+    page = await getOrCreatePage(browser);
+    await setupPageRequestBlocking(page);
+    lastRecycleAt = Date.now();
+  }
+
+  async function maybeRecycleForMemory() {
+    const mem = process.memoryUsage();
+    const rssMB = mem.rss / 1024 / 1024;
+    const heapMB = mem.heapUsed / 1024 / 1024;
+    if (rssMB >= memRssRecycleMb || heapMB >= memHeapRecycleMb) {
+      const now = Date.now();
+      if (recycleCooldownMs > 0 && now - lastRecycleAt < recycleCooldownMs) {
+        return;
+      }
+      await recycleCdpSession(
+        `mem rss=${rssMB.toFixed(0)}MB heap=${heapMB.toFixed(0)}MB (threshold rss=${memRssRecycleMb} heap=${memHeapRecycleMb})`,
+      );
+      return;
+    }
+    if (
+      periodicRecycleEvery > 0 &&
+      totalScored > 0 &&
+      totalScored % periodicRecycleEvery === 0
+    ) {
+      await recycleCdpSession(`periodic every ${totalScored} tasks`);
+    }
+  }
 
   let totalScored = 0;
   let emptyRounds = 0; // 连续无任务的轮数
@@ -622,7 +755,7 @@ export async function handleScoreAll(parsed) {
 
   // 生成客户端 ID，用于服务端追踪
   const clientId = randomUUID();
-  const clientMeta = { type: "scoring" };
+  const clientMeta = { type: "scoring", port: cdpPort };
 
   // 复用 TikTokScraper 实例，避免每次 enrich 都启动/关闭 headless 浏览器
   const enrichScraper = new TikTokScraper({
@@ -719,6 +852,7 @@ export async function handleScoreAll(parsed) {
           // already claimed: 其他机器抢先了，跳过不标 dead
           if (claimData.error && claimData.error.includes("already claimed")) {
             log(`  ⏭️ 已被其他客户端锁定，跳过`);
+            await recyclePage();
             continue;
           }
           log(`  ⚠️ 无法锁定 (${claimData.error})，标记为 dead 并跳过`);
@@ -726,6 +860,7 @@ export async function handleScoreAll(parsed) {
           result.status = "dead";
           await reportToServer(baseUrl, result, clientId, clientMeta);
           totalScored++;
+          await recyclePage();
           continue;
         }
 
@@ -733,7 +868,7 @@ export async function handleScoreAll(parsed) {
         const fetchStart = Date.now();
         log(`  抓取 TikTok 标签页...`);
         const tagResult = await fetchTagData(tag, {
-          port: cdpPort,
+          page,
           onProgress: ({ videos, authors }) => {
             process.stderr.write(
               `\r  抓取中: ${videos} 视频, ${authors} 作者\x1b[K`,
@@ -751,23 +886,17 @@ export async function handleScoreAll(parsed) {
 
         if (!videos || videos.length === 0) {
           const deadSec = ((Date.now() - fetchStart) / 1000).toFixed(1);
-          const memMB = (process.memoryUsage().heapUsed / 1024 / 1024).toFixed(
-            0,
+          log(
+            `  ⚠️ 无视频 (${deadSec}s) mem=${formatMemoryUsage()}，标记 dead`,
           );
-          log(`  ⚠️ 无视频 (${deadSec}s) mem=${memMB}MB，标记 dead`);
           result.status = "dead";
           result.error = "no videos found";
           await reportToServer(baseUrl, result, clientId, clientMeta);
           totalScored++;
           // 随机等待 3-7 秒，避免连续访问 TikTok 触发风控
           await randomDelay(0, 5000);
-          // 导航到 about:blank 释放页面状态再跳过
-          await page
-            .goto("about:blank", {
-              waitUntil: "domcontentloaded",
-              timeout: 5000,
-            })
-            .catch(() => {});
+          await recyclePage();
+          await maybeRecycleForMemory();
           continue;
         }
 
@@ -790,12 +919,11 @@ export async function handleScoreAll(parsed) {
         // CDN 限流检测：有拦截则冷却 + 重启 scraper
         const cdnBlocked = enriched.cdnBlockedCount || 0;
         if (cdnBlocked > 0) {
-          const cdnRatio = cdnBlocked / (videos.length || 1);
-          const coolSec = cdnRatio > 0.3 ? 120 : 60;
+          const coolSec = getCdnCooldownSeconds(cdnBlocked, videos.length);
           log(
-            `  ⚠️ CDN 限流: ${cdnBlocked}/${videos.length} (${(cdnRatio * 100).toFixed(0)}%)，冷却 ${coolSec} 秒后重启 scraper`,
+            `  ⚠️ CDN 限流: ${cdnBlocked}/${videos.length} (${((cdnBlocked / Math.max(videos.length, 1)) * 100).toFixed(0)}%)，冷却 ${coolSec} 秒后重启 scraper`,
           );
-          await new Promise((r) => setTimeout(r, coolSec * 1000));
+          await cooldownAndRecycle(coolSec, recyclePage, maybeRecycleForMemory);
           log(`  正在重启 TikTokScraper...`);
           await enrichScraper.restart();
           log(`  ✅ TikTokScraper 已重启`);
@@ -836,22 +964,26 @@ export async function handleScoreAll(parsed) {
         const mc = result.matchedCountries
           .map((c) => `${c.c}:${c.n}`)
           .join(" ");
-        // Node.js 进程内存占用
-        const memMB = (process.memoryUsage().heapUsed / 1024 / 1024).toFixed(0);
-        const memStr = ` mem=${memMB}MB`;
+        const memStr = ` mem=${formatMemoryUsage()}`;
         log(
           `  ${icon} ${result.status} score=${result.score} authors=${result.authorCount} matched=${result.matchedAuthors} (${elapsed}s)${mc ? "  " + mc : ""}${memStr}`,
         );
         log("");
-
-        // 导航到 about:blank 卸载页面，状态清零，下次 goto 重新初始化
-        await page
-          .goto("about:blank", { waitUntil: "domcontentloaded", timeout: 5000 })
-          .catch((e) => {
-            log(`  ⚠️ about:blank 跳转失败: ${e.message}`);
-          });
+        await recyclePage();
+        await maybeRecycleForMemory();
         await randomDelay(3000, 7000);
       } catch (e) {
+        if (e instanceof CDNBlockedError || /HTTP\s+(403|429)/.test(e.message)) {
+          log(`  ⚠️ CDN 被封: ${e.message}`);
+          result.status = "dead";
+          result.error = "cdn_blocked";
+          await reportToServer(baseUrl, result, clientId, clientMeta);
+          totalScored++;
+          const cooldownSec = getCdnCooldownSeconds(1, 1, /429/.test(e.message));
+          log(`  冷却 ${cooldownSec} 秒后再继续...`);
+          await cooldownAndRecycle(cooldownSec, recyclePage, maybeRecycleForMemory);
+          continue;
+        }
         // 区分网络错误和业务错误
         const isNetworkError =
           e.code === "ECONNREFUSED" ||
@@ -881,6 +1013,8 @@ export async function handleScoreAll(parsed) {
           );
         } catch {}
         totalScored++;
+        await recyclePage();
+        await maybeRecycleForMemory();
       }
     }
   } finally {

package/src/lib/args.js

@@ -739,7 +739,7 @@ function parseTagArgs(args) {
   let isDiscover = false;
   let isScore = false;
   let isScoreAll = false;
-  let scoreAllPort = 9222;
+  let scoreAllPort = 7222;
   let scoreProxy = null;
   let scoreTag = null;
   let scoreCountries = null;
@@ -788,7 +788,7 @@ function parseTagArgs(args) {
         .filter(Boolean);
     } else if (arg === "--port") {
       if (isScoreAll) {
-        scoreAllPort = parseInt(args[++i]) || 9222;
+        scoreAllPort = parseInt(args[++i]) || 7222;
       } else {
         scorePort = parseInt(args[++i]) || 9222;
       }

package/src/lib/tag-fetcher.js

@@ -48,14 +48,19 @@ export async function fetchTagData(tag, options = {}) {
     userDataDir,
     proxyServer,
     onProgress,
+    browser: existingBrowser,
+    page: existingPage,
   } = options;
 
   const cdpOptions = { port };
   if (userDataDir) cdpOptions.userDataDir = userDataDir;
   if (proxyServer) cdpOptions.proxyServer = proxyServer;
 
-  const browser = await ensureBrowserReady(cdpOptions);
-  const page = await getOrCreatePage(browser);
+  const browser =
+    existingBrowser ||
+    (existingPage ? existingPage.context().browser() : null) ||
+    (await ensureBrowserReady(cdpOptions));
+  const page = existingPage || (await getOrCreatePage(browser));
 
   let challengeInfo = null;
   const rawVideos = [];
@@ -102,6 +107,9 @@ export async function fetchTagData(tag, options = {}) {
       timeout: 30000,
     });
 
+    if (resp.status() === 403 || resp.status() === 429) {
+      throw new CDNBlockedError(`标签页返回 HTTP ${resp.status()}`);
+    }
     if (resp.status() !== 200) {
       throw new Error(`标签页返回 HTTP ${resp.status()}`);
     }

package/src/watch/data-store.js

@@ -1160,59 +1160,112 @@ export function createStore(filePath, options = {}) {
       })();
     }
 
-    // 常规移动：INSERT + DELETE 事务
-    const moveTxn = getDb().transaction(() => {
-      getDb()
-        .prepare(
-          `
-        INSERT OR IGNORE INTO jobs (
-          unique_id, nickname, status, sources, pinned,
-          tt_seller, verified, video_count, comment_count,
-          guessed_location, location_created, confirmed_location,
-          follower_count, following_count, heart_count,
-          created_at, updated_at, region, signature, bio_link, sec_uid,
-          status_code, latest_video_time, user_create_time
-        )
-        SELECT
-          unique_id, nickname, 'pending', sources, pinned,
-          tt_seller, verified, video_count, comment_count,
-          guessed_location, location_created, confirmed_location,
-          follower_count, following_count, heart_count,
-          created_at, updated_at, region, signature, bio_link, sec_uid,
-          status_code, latest_video_time, user_create_time
-        FROM raw_jobs
-        WHERE ${whereSql}
+    // 常规移动：多国家时先按国家均衡补充，再用全局兜底补齐剩余额度
+    const insertFromRawSql = `
+      INSERT OR IGNORE INTO jobs (
+        unique_id, nickname, status, sources, pinned,
+        tt_seller, verified, video_count, comment_count,
+        guessed_location, location_created, confirmed_location,
+        follower_count, following_count, heart_count,
+        created_at, updated_at, region, signature, bio_link, sec_uid,
+        status_code, latest_video_time, user_create_time
+      )
+      SELECT
+        unique_id, nickname, 'pending', sources, pinned,
+        tt_seller, verified, video_count, comment_count,
+        guessed_location, location_created, confirmed_location,
+        follower_count, following_count, heart_count,
+        created_at, updated_at, region, signature, bio_link, sec_uid,
+        status_code, latest_video_time, user_create_time
+      FROM raw_jobs
+      WHERE __WHERE__
+      ORDER BY
+        CASE WHEN sources LIKE '%tag%' THEN 0 ELSE 1 END,
+        COALESCE(video_count, 0) DESC, created_at DESC
+      LIMIT ?
+    `;
+
+    const deleteFromRawSql = `
+      DELETE FROM raw_jobs
+      WHERE unique_id IN (
+        SELECT unique_id FROM raw_jobs
+        WHERE __WHERE__
         ORDER BY
           CASE WHEN sources LIKE '%tag%' THEN 0 ELSE 1 END,
           COALESCE(video_count, 0) DESC, created_at DESC
         LIMIT ?
-      `,
-        )
-        .run(...args, safeLimit);
+      )
+    `;
 
-      // 删除已移动的记录：用子查询匹配刚 INSERT 的 unique_id
-      getDb()
-        .prepare(
-          `
-        DELETE FROM raw_jobs
-        WHERE unique_id IN (
-          SELECT unique_id FROM raw_jobs
-          WHERE ${whereSql}
-          ORDER BY
-            CASE WHEN sources LIKE '%tag%' THEN 0 ELSE 1 END,
-            COALESCE(video_count, 0) DESC, created_at DESC
-          LIMIT ?
-        )
-      `,
-        )
-        .run(...args, safeLimit);
+    const uniqueLocations = normalizedLocations
+      ? Array.from(new Set(normalizedLocations))
+      : [];
+    const shouldBalanceByCountry = uniqueLocations.length > 1;
+
+    const moveTxn = getDb().transaction(() => {
+      let moved = 0;
+      const movedByCountry = {};
+
+      if (shouldBalanceByCountry) {
+        const baseQuota = Math.floor(safeLimit / uniqueLocations.length);
+        const remainder = safeLimit % uniqueLocations.length;
+
+        for (let i = 0; i < uniqueLocations.length; i++) {
+          if (moved >= safeLimit) break;
+
+          const location = uniqueLocations[i];
+          const quota = baseQuota + (i < remainder ? 1 : 0);
+          const currentLimit = Math.max(0, Math.min(quota, safeLimit - moved));
+          if (!currentLimit) continue;
+
+          const locationWhere = `${whereSql} AND UPPER(COALESCE(guessed_location, '')) = ?`;
+          const locationArgs = [...args, location];
+
+          getDb()
+            .prepare(insertFromRawSql.replace("__WHERE__", locationWhere))
+            .run(...locationArgs, currentLimit);
+          const del = getDb()
+            .prepare(deleteFromRawSql.replace("__WHERE__", locationWhere))
+            .run(...locationArgs, currentLimit);
+
+          const movedThisCountry = del?.changes || 0;
+          moved += movedThisCountry;
+          movedByCountry[location] = movedThisCountry;
+        }
+
+        // 某些国家库存不足时，用全局查询补齐剩余额度（仍受 whereSql 国家范围约束）
+        const remaining = safeLimit - moved;
+        if (remaining > 0) {
+          getDb()
+            .prepare(insertFromRawSql.replace("__WHERE__", whereSql))
+            .run(...args, remaining);
+          const del = getDb()
+            .prepare(deleteFromRawSql.replace("__WHERE__", whereSql))
+            .run(...args, remaining);
+          moved += del?.changes || 0;
+        }
+
+        console.error(
+          `[data-store] refill 国家均衡: ${uniqueLocations
+            .map((loc) => `${loc}:${movedByCountry[loc] || 0}`)
+            .join(", ")} | total=${moved}`,
+        );
+      } else {
+        getDb()
+          .prepare(insertFromRawSql.replace("__WHERE__", whereSql))
+          .run(...args, safeLimit);
+        const del = getDb()
+          .prepare(deleteFromRawSql.replace("__WHERE__", whereSql))
+          .run(...args, safeLimit);
+        moved = del?.changes || 0;
+      }
+
+      return moved;
     });
 
-    moveTxn();
+    const moved = moveTxn();
     markStatsDirty();
-
-    const actualMoved = Math.min(count, safeLimit);
-    return { moved: actualMoved };
+    return { moved };
   }
 
   async function claimNextJob(
@@ -1512,8 +1565,7 @@ export function createStore(filePath, options = {}) {
         }
         return null;
       }
-      const refillResult = (async () => {
-        refillLock = Promise.resolve(); // 占位
+      const refillPromise = (async () => {
         const result = refillJobsFromRaw(
           normalizedLocations.length ? normalizedLocations : null,
           500,
@@ -1521,30 +1573,17 @@ export function createStore(filePath, options = {}) {
         );
         // refillJobsFromRaw 在 LLM 模式下返回 Promise
         if (result && typeof result.then === "function") {
-          return result.finally(() => {
-            refillLock = null;
-          });
+          return await result;
         }
         return result;
       })();
-      if (refillResult && typeof refillResult.then === "function") {
-        const awaited = await refillResult;
-        if (awaited.moved > 0) {
-          console.error(
-            `[data-store] 从 raw_jobs 补充了 ${awaited.moved} 条任务到 jobs`,
-          );
-          for (const requireVideo of [true, false]) {
-            const pinned = findPinnedPending(requireVideo);
-            if (pinned) {
-              return claimRow(pinned);
-            }
-            const ranked = findPrioritizedPending(requireVideo);
-            if (ranked) {
-              return claimRow(ranked);
-            }
-          }
-        }
-      } else if (refillResult.moved > 0) {
+      // 让并发请求等待同一个 refill，并且无论成功/失败都释放锁
+      refillLock = refillPromise.finally(() => {
+        refillLock = null;
+      });
+
+      const refillResult = await refillLock;
+      if (refillResult.moved > 0) {
         console.error(
           `[data-store] 从 raw_jobs 补充了 ${refillResult.moved} 条任务到 jobs`,
         );