tt-help-cli-ycl 1.3.99 → 1.3.100

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tt-help-cli-ycl",
-  "version": "1.3.99",
+  "version": "1.3.100",
   "description": "TikTok user & video data scraper - extract ttSeller, verified, locationCreated from HTML source",
   "type": "module",
   "bin": {

package/src/cli/tag.js

@@ -5,6 +5,7 @@ import { fetchTagData, enrichVideosWithLocation } from "../lib/tag-fetcher.js";
 import { killEdgeProcesses, ensureBrowserReady } from "../lib/browser/cdp.js";
 import { getOrCreatePage } from "../lib/browser/page.js";
 import { TikTokScraper } from "../lib/tiktok-scraper.mjs";
+import { CDNBlockedError } from "../lib/parse-ssr.mjs";
 import {
   DEFAULT_TARGET_LOCATIONS,
   isLocationInList,
@@ -42,6 +43,18 @@ function formatMemoryUsage(mem = process.memoryUsage()) {
   return `rss:${(mem.rss / 1024 / 1024).toFixed(0)}MB heap:${(mem.heapUsed / 1024 / 1024).toFixed(0)}MB ext:${(mem.external / 1024 / 1024).toFixed(0)}MB ab:${(mem.arrayBuffers / 1024 / 1024).toFixed(0)}MB`;
 }
 
+function getCdnCooldownSeconds(blockedCount, totalCount, isTooManyRequests = false) {
+  if (isTooManyRequests) return 120;
+  const ratio = blockedCount / Math.max(totalCount, 1);
+  return ratio > 0.3 ? 120 : 60;
+}
+
+async function cooldownAndRecycle(cooldownSec, recyclePage, maybeRecycleForMemory) {
+  await new Promise((r) => setTimeout(r, cooldownSec * 1000));
+  await recyclePage();
+  await maybeRecycleForMemory();
+}
+
 // 构建带客户端追踪 header 的 fetch 封装
 function buildClientHeaders(clientId, meta, extra = {}) {
   return {
@@ -906,12 +919,11 @@ export async function handleScoreAll(parsed) {
         // CDN 限流检测：有拦截则冷却 + 重启 scraper
         const cdnBlocked = enriched.cdnBlockedCount || 0;
         if (cdnBlocked > 0) {
-          const cdnRatio = cdnBlocked / (videos.length || 1);
-          const coolSec = cdnRatio > 0.3 ? 120 : 60;
+          const coolSec = getCdnCooldownSeconds(cdnBlocked, videos.length);
           log(
-            `  ⚠️ CDN 限流: ${cdnBlocked}/${videos.length} (${(cdnRatio * 100).toFixed(0)}%)，冷却 ${coolSec} 秒后重启 scraper`,
+            `  ⚠️ CDN 限流: ${cdnBlocked}/${videos.length} (${((cdnBlocked / Math.max(videos.length, 1)) * 100).toFixed(0)}%)，冷却 ${coolSec} 秒后重启 scraper`,
           );
-          await new Promise((r) => setTimeout(r, coolSec * 1000));
+          await cooldownAndRecycle(coolSec, recyclePage, maybeRecycleForMemory);
           log(`  正在重启 TikTokScraper...`);
           await enrichScraper.restart();
           log(`  ✅ TikTokScraper 已重启`);
@@ -961,6 +973,17 @@ export async function handleScoreAll(parsed) {
         await maybeRecycleForMemory();
         await randomDelay(3000, 7000);
       } catch (e) {
+        if (e instanceof CDNBlockedError || /HTTP\s+(403|429)/.test(e.message)) {
+          log(`  ⚠️ CDN 被封: ${e.message}`);
+          result.status = "dead";
+          result.error = "cdn_blocked";
+          await reportToServer(baseUrl, result, clientId, clientMeta);
+          totalScored++;
+          const cooldownSec = getCdnCooldownSeconds(1, 1, /429/.test(e.message));
+          log(`  冷却 ${cooldownSec} 秒后再继续...`);
+          await cooldownAndRecycle(cooldownSec, recyclePage, maybeRecycleForMemory);
+          continue;
+        }
         // 区分网络错误和业务错误
         const isNetworkError =
           e.code === "ECONNREFUSED" ||

package/src/lib/tag-fetcher.js

@@ -107,6 +107,9 @@ export async function fetchTagData(tag, options = {}) {
       timeout: 30000,
     });
 
+    if (resp.status() === 403 || resp.status() === 429) {
+      throw new CDNBlockedError(`标签页返回 HTTP ${resp.status()}`);
+    }
     if (resp.status() !== 200) {
       throw new Error(`标签页返回 HTTP ${resp.status()}`);
     }

package/src/watch/data-store.js

@@ -1160,59 +1160,112 @@ export function createStore(filePath, options = {}) {
       })();
     }
 
-    // 常规移动：INSERT + DELETE 事务
-    const moveTxn = getDb().transaction(() => {
-      getDb()
-        .prepare(
-          `
-        INSERT OR IGNORE INTO jobs (
-          unique_id, nickname, status, sources, pinned,
-          tt_seller, verified, video_count, comment_count,
-          guessed_location, location_created, confirmed_location,
-          follower_count, following_count, heart_count,
-          created_at, updated_at, region, signature, bio_link, sec_uid,
-          status_code, latest_video_time, user_create_time
-        )
-        SELECT
-          unique_id, nickname, 'pending', sources, pinned,
-          tt_seller, verified, video_count, comment_count,
-          guessed_location, location_created, confirmed_location,
-          follower_count, following_count, heart_count,
-          created_at, updated_at, region, signature, bio_link, sec_uid,
-          status_code, latest_video_time, user_create_time
-        FROM raw_jobs
-        WHERE ${whereSql}
+    // 常规移动：多国家时先按国家均衡补充，再用全局兜底补齐剩余额度
+    const insertFromRawSql = `
+      INSERT OR IGNORE INTO jobs (
+        unique_id, nickname, status, sources, pinned,
+        tt_seller, verified, video_count, comment_count,
+        guessed_location, location_created, confirmed_location,
+        follower_count, following_count, heart_count,
+        created_at, updated_at, region, signature, bio_link, sec_uid,
+        status_code, latest_video_time, user_create_time
+      )
+      SELECT
+        unique_id, nickname, 'pending', sources, pinned,
+        tt_seller, verified, video_count, comment_count,
+        guessed_location, location_created, confirmed_location,
+        follower_count, following_count, heart_count,
+        created_at, updated_at, region, signature, bio_link, sec_uid,
+        status_code, latest_video_time, user_create_time
+      FROM raw_jobs
+      WHERE __WHERE__
+      ORDER BY
+        CASE WHEN sources LIKE '%tag%' THEN 0 ELSE 1 END,
+        COALESCE(video_count, 0) DESC, created_at DESC
+      LIMIT ?
+    `;
+
+    const deleteFromRawSql = `
+      DELETE FROM raw_jobs
+      WHERE unique_id IN (
+        SELECT unique_id FROM raw_jobs
+        WHERE __WHERE__
         ORDER BY
           CASE WHEN sources LIKE '%tag%' THEN 0 ELSE 1 END,
           COALESCE(video_count, 0) DESC, created_at DESC
         LIMIT ?
-      `,
-        )
-        .run(...args, safeLimit);
+      )
+    `;
 
-      // 删除已移动的记录：用子查询匹配刚 INSERT 的 unique_id
-      getDb()
-        .prepare(
-          `
-        DELETE FROM raw_jobs
-        WHERE unique_id IN (
-          SELECT unique_id FROM raw_jobs
-          WHERE ${whereSql}
-          ORDER BY
-            CASE WHEN sources LIKE '%tag%' THEN 0 ELSE 1 END,
-            COALESCE(video_count, 0) DESC, created_at DESC
-          LIMIT ?
-        )
-      `,
-        )
-        .run(...args, safeLimit);
+    const uniqueLocations = normalizedLocations
+      ? Array.from(new Set(normalizedLocations))
+      : [];
+    const shouldBalanceByCountry = uniqueLocations.length > 1;
+
+    const moveTxn = getDb().transaction(() => {
+      let moved = 0;
+      const movedByCountry = {};
+
+      if (shouldBalanceByCountry) {
+        const baseQuota = Math.floor(safeLimit / uniqueLocations.length);
+        const remainder = safeLimit % uniqueLocations.length;
+
+        for (let i = 0; i < uniqueLocations.length; i++) {
+          if (moved >= safeLimit) break;
+
+          const location = uniqueLocations[i];
+          const quota = baseQuota + (i < remainder ? 1 : 0);
+          const currentLimit = Math.max(0, Math.min(quota, safeLimit - moved));
+          if (!currentLimit) continue;
+
+          const locationWhere = `${whereSql} AND UPPER(COALESCE(guessed_location, '')) = ?`;
+          const locationArgs = [...args, location];
+
+          getDb()
+            .prepare(insertFromRawSql.replace("__WHERE__", locationWhere))
+            .run(...locationArgs, currentLimit);
+          const del = getDb()
+            .prepare(deleteFromRawSql.replace("__WHERE__", locationWhere))
+            .run(...locationArgs, currentLimit);
+
+          const movedThisCountry = del?.changes || 0;
+          moved += movedThisCountry;
+          movedByCountry[location] = movedThisCountry;
+        }
+
+        // 某些国家库存不足时，用全局查询补齐剩余额度（仍受 whereSql 国家范围约束）
+        const remaining = safeLimit - moved;
+        if (remaining > 0) {
+          getDb()
+            .prepare(insertFromRawSql.replace("__WHERE__", whereSql))
+            .run(...args, remaining);
+          const del = getDb()
+            .prepare(deleteFromRawSql.replace("__WHERE__", whereSql))
+            .run(...args, remaining);
+          moved += del?.changes || 0;
+        }
+
+        console.error(
+          `[data-store] refill 国家均衡: ${uniqueLocations
+            .map((loc) => `${loc}:${movedByCountry[loc] || 0}`)
+            .join(", ")} | total=${moved}`,
+        );
+      } else {
+        getDb()
+          .prepare(insertFromRawSql.replace("__WHERE__", whereSql))
+          .run(...args, safeLimit);
+        const del = getDb()
+          .prepare(deleteFromRawSql.replace("__WHERE__", whereSql))
+          .run(...args, safeLimit);
+        moved = del?.changes || 0;
+      }
+
+      return moved;
     });
 
-    moveTxn();
+    const moved = moveTxn();
     markStatsDirty();
-
-    const actualMoved = Math.min(count, safeLimit);
-    return { moved: actualMoved };
+    return { moved };
   }
 
   async function claimNextJob(
@@ -1512,8 +1565,7 @@ export function createStore(filePath, options = {}) {
         }
         return null;
       }
-      const refillResult = (async () => {
-        refillLock = Promise.resolve(); // 占位
+      const refillPromise = (async () => {
         const result = refillJobsFromRaw(
           normalizedLocations.length ? normalizedLocations : null,
           500,
@@ -1521,30 +1573,17 @@ export function createStore(filePath, options = {}) {
         );
         // refillJobsFromRaw 在 LLM 模式下返回 Promise
         if (result && typeof result.then === "function") {
-          return result.finally(() => {
-            refillLock = null;
-          });
+          return await result;
         }
         return result;
       })();
-      if (refillResult && typeof refillResult.then === "function") {
-        const awaited = await refillResult;
-        if (awaited.moved > 0) {
-          console.error(
-            `[data-store] 从 raw_jobs 补充了 ${awaited.moved} 条任务到 jobs`,
-          );
-          for (const requireVideo of [true, false]) {
-            const pinned = findPinnedPending(requireVideo);
-            if (pinned) {
-              return claimRow(pinned);
-            }
-            const ranked = findPrioritizedPending(requireVideo);
-            if (ranked) {
-              return claimRow(ranked);
-            }
-          }
-        }
-      } else if (refillResult.moved > 0) {
+      // 让并发请求等待同一个 refill，并且无论成功/失败都释放锁
+      refillLock = refillPromise.finally(() => {
+        refillLock = null;
+      });
+
+      const refillResult = await refillLock;
+      if (refillResult.moved > 0) {
         console.error(
           `[data-store] 从 raw_jobs 补充了 ${refillResult.moved} 条任务到 jobs`,
         );