tt-help-cli-ycl 1.4.114 → 1.4.115

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tt-help-cli-ycl",
-  "version": "1.4.114",
+  "version": "1.4.115",
   "description": "TikTok user & video data scraper - extract ttSeller, verified, locationCreated from HTML source",
   "type": "module",
   "bin": {

package/scripts/precheck-monitor.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+# 远程服务器 precheck / pending_refresh 持续监控
+# 每 15 分钟采样一次，追加写入日志（按天一个文件）
+# 用法: nohup ./precheck-monitor.sh [日志文件] >/dev/null 2>&1 &
+
+LOG="${1:-/Users/yinchengliang/soft/tiktok/newdata/precheck-monitor-$(date +%Y%m%d).log}"
+TARGETS="'CZ','GR','HU','PT','ES','PL','NL','BE','DE','FR','IT','IE','AT','US'"
+SERVER="http://117.71.53.99:17301"
+
+count_sql() { # $1 = 已 URL 编码的 SQL
+  curl -s --max-time 20 "$SERVER/api/db-query?sql=$1" | python3 -c '
+import json, sys
+try:
+  print(json.load(sys.stdin)["rows"][0]["cnt"])
+except Exception:
+  print("ERR")
+'
+}
+
+while true; do
+  TS=$(date "+%Y-%m-%d %H:%M:%S")
+  THRESHOLD=$(( $(date +%s)000 - 172800000 ))
+
+  SQL_PC=$(python3 -c "
+import urllib.parse
+print(urllib.parse.quote(\"SELECT COUNT(*) AS cnt FROM jobs WHERE tt_seller=1 AND verified=0 AND (pending_refresh IS NULL OR pending_refresh=0) AND COALESCE(refresh_time, 1452000000000) < $THRESHOLD AND location_created IN ($TARGETS)\"))
+")
+  SQL_PR=$(python3 -c "
+import urllib.parse
+print(urllib.parse.quote(\"SELECT COUNT(*) AS cnt FROM jobs WHERE tt_seller=1 AND verified=0 AND pending_refresh=1 AND location_created IN ($TARGETS)\"))
+")
+
+  PC=$(count_sql "$SQL_PC")
+  PR=$(count_sql "$SQL_PR")
+  echo "$TS precheck=$PC pending_refresh=$PR" >> "$LOG"
+  sleep 900
+done

package/src/cli/attach.js

@@ -1,5 +1,6 @@
 import { TikTokScraper } from "../lib/tiktok-scraper.mjs";
 import { CDNBlockedError } from "../lib/parse-ssr.mjs";
+import { writeCommand } from "../clash/ipc.js";
 import {
   proxy as configuredProxy,
   userId as configuredUserId,
@@ -11,6 +12,8 @@ import v8 from "node:v8";
 
 const HEAP_RESTART_RATIO = 0.72;
 const MAX_TASK_BATCHES_BEFORE_RESTART = 200;
+const MAX_CDN_SWITCH_ATTEMPTS = 3; // 连续 CDN 限流最多切换几次节点，之后切换无效则改纯等待（账号/整体风控时切换无用）
+const MAX_CDN_COOLDOWN_SEC = 300; // CDN 冷却上限（指数退避：60s→120s→240s→300s）
 
 function formatNow() {
   return new Date().toLocaleString("zh-CN", {
@@ -147,8 +150,27 @@ export async function handleAttach(options) {
         body: JSON.stringify({ uniqueId }),
         signal: AbortSignal.timeout(30000),
       });
-      const data = await res.json();
-      if (!res.ok || data.error) return null;
+      let data = null;
+      try {
+        data = await res.json();
+      } catch {}
+      if (!res.ok) {
+        // CDN 限流：抛 CDNBlockedError，外层据此冷却（webserver 模式同样生效）
+        if (
+          data &&
+          typeof data.error === "string" &&
+          data.error.includes("CDN限流")
+        ) {
+          throw new CDNBlockedError(data.error);
+        }
+        // 其余失败（瞬时 503/5xx）返回 null，由服务端后续重新排队
+        return null;
+      }
+      // 业务结果：TikTok 明确 statusCode（209002 验证码等）→ 返回上层走"已记录"分支
+      if (data && data.error && data.tiktokStatusCode) {
+        return data;
+      }
+      if (data && data.error) return null;
       return data;
     };
   } else {
@@ -185,6 +207,7 @@ export async function handleAttach(options) {
     let loopCount = 0;
     let browserRestartCount = 0;
     let taskBatchCount = 0;
+    let consecutiveCdnBlocks = 0; // 连续 CDN 限流轮数（自适应：前几次切换节点，之后长等待）
 
     while (!shuttingDown) {
       loopCount++;
@@ -348,23 +371,47 @@ export async function handleAttach(options) {
 
       attachLog(`  本批结果: ${successCount} 成功, ${failCount} 失败\n`);
 
-      // CDN 限流比例超过 30% 时，冷却 + 重启浏览器（仅本地浏览器模式）
-      if (scraper) {
-        const cdnRatio = cdnBlockedCount / tasks.length;
-        if (cdnRatio > 0.3) {
-          const coolDownSeconds = cdnRatio > 0.8 ? 120 : 60;
+      // CDN 限流比例超过 30% 时：自适应策略（前几次切节点，连续限流则长等待）
+      const cdnRatio = tasks.length > 0 ? cdnBlockedCount / tasks.length : 0;
+      if (cdnRatio > 0.3) {
+        consecutiveCdnBlocks++;
+        // 冷却指数退避：60s→120s→240s→…上限 300s（切换后短等，持续限流则长等窗口过去）
+        const coolDownSeconds = Math.min(
+          60 * Math.pow(2, consecutiveCdnBlocks - 1),
+          MAX_CDN_COOLDOWN_SEC,
+        );
+        const shouldSwitch = consecutiveCdnBlocks <= MAX_CDN_SWITCH_ATTEMPTS;
+        if (shouldSwitch) {
+          // 用现有 IPC 接口通知 daemon 从候选集切下一个健康节点（换 IP 绕过 CDN 限流）
+          try {
+            writeCommand({
+              action: "failover",
+              reason: `attach CDN限流 连续${consecutiveCdnBlocks}次 (${(cdnRatio * 100).toFixed(0)}%)`,
+            });
+          } catch {}
+          attachLog(
+            `  [Attach] CDN限流比例 ${(cdnRatio * 100).toFixed(0)}% (${cdnBlockedCount}/${tasks.length})，第${consecutiveCdnBlocks}次连续限流，已通知 daemon 切换节点，冷却 ${coolDownSeconds} 秒...`,
+          );
+        } else {
+          // 连续多次切换仍限流 → 大概率账号/整体风控，切换无用，纯等待窗口过去
           attachLog(
-            `  [Attach] CDN限流比例 ${(cdnRatio * 100).toFixed(0)}% (${cdnBlockedCount}/${tasks.length})，冷却 ${coolDownSeconds} 秒后重启浏览器...`,
+            `  [Attach] CDN限流比例 ${(cdnRatio * 100).toFixed(0)}% (${cdnBlockedCount}/${tasks.length})，已连续 ${consecutiveCdnBlocks} 次切换仍限流（可能账号/整体风控），停止切换，长冷却 ${coolDownSeconds} 秒等待窗口过去...`,
           );
-          await new Promise((r) => setTimeout(r, coolDownSeconds * 1000));
+        }
+        await new Promise((r) => setTimeout(r, coolDownSeconds * 1000));
+        if (scraper) {
           await recycleScraper(
             scraper,
             `CDN限流比例过高 (${cdnBlockedCount}/${tasks.length})`,
           );
           browserRestartCount++;
-          taskBatchCount = 0;
         }
+        taskBatchCount = 0;
+      } else {
+        consecutiveCdnBlocks = 0; // 正常批次重置连续限流计数
+      }
 
+      if (scraper) {
         const heap = getHeapUsage();
         if (heap.ratio >= HEAP_RESTART_RATIO) {
           await recycleScraper(

package/src/cli/precheck.js

@@ -1,5 +1,6 @@
 import { TikTokScraper } from "../lib/tiktok-scraper.mjs";
 import { CDNBlockedError } from "../lib/parse-ssr.mjs";
+import { writeCommand } from "../clash/ipc.js";
 import {
   proxy as configuredProxy,
   userId as configuredUserId,
@@ -163,8 +164,27 @@ export async function handlePrecheck(options) {
           body: JSON.stringify({ uniqueId }),
           signal: AbortSignal.timeout(30000),
         });
-        const data = await res.json();
-        if (!res.ok || data.error) return null;
+        let data = null;
+        try {
+          data = await res.json();
+        } catch {}
+        if (!res.ok) {
+          // CDN 限流：抛 CDNBlockedError，外层据此冷却（webserver 模式同样生效）
+          if (
+            data &&
+            typeof data.error === "string" &&
+            data.error.includes("CDN限流")
+          ) {
+            throw new CDNBlockedError(data.error);
+          }
+          // 其余失败（瞬时 503/5xx）返回 null，由服务端后续重新排队
+          return null;
+        }
+        // 业务结果：TikTok 明确 statusCode（209002 验证码等）→ 返回上层走"已记录"分支
+        if (data && data.error && data.tiktokStatusCode) {
+          return data;
+        }
+        if (data && data.error) return null;
         return data;
       };
     } else {
@@ -397,24 +417,32 @@ export async function handlePrecheck(options) {
         `  本轮结果: ${needRefreshCount} 需刷新, ${skipCount} 跳过, ${failCount} 失败`,
       );
 
-      // CDN 限流/堆内存/批次重启（仅本地浏览器模式）
-      if (scraper) {
-        const cdnRatio =
-          candidates.length > 0 ? cdnBlockedCount / candidates.length : 0;
-        if (cdnRatio > 0.3) {
-          const coolDownSeconds = cdnRatio > 0.8 ? 120 : 60;
-          precheckLog(
-            `  [Precheck] CDN限流比例 ${(cdnRatio * 100).toFixed(0)}% (${cdnBlockedCount}/${candidates.length})，冷却 ${coolDownSeconds} 秒后重启浏览器...`,
-          );
-          await new Promise((r) => setTimeout(r, coolDownSeconds * 1000));
+      // CDN 限流比例超过 30% 时：通知 clash daemon 故障切换（换出口 IP 绕过限流）+ 冷却
+      const cdnRatio =
+        candidates.length > 0 ? cdnBlockedCount / candidates.length : 0;
+      if (cdnRatio > 0.3) {
+        // 用现有 IPC 接口通知 daemon 从候选集切下一个健康节点（换 IP 绕过 CDN 限流）
+        try {
+          writeCommand({
+            action: "failover",
+            reason: `precheck CDN限流 (${(cdnRatio * 100).toFixed(0)}%)`,
+          });
+        } catch {}
+        // 切换节点后新 IP 生效需短暂等待，冷却可缩短（替代纯硬等 120s）
+        const coolDownSeconds = cdnRatio > 0.8 ? 90 : 60;
+        precheckLog(
+          `  [Precheck] CDN限流比例 ${(cdnRatio * 100).toFixed(0)}% (${cdnBlockedCount}/${candidates.length})，已通知 daemon 切换节点，冷却 ${coolDownSeconds} 秒...`,
+        );
+        await new Promise((r) => setTimeout(r, coolDownSeconds * 1000));
+        if (scraper) {
           await recycleScraper(
             scraper,
             `CDN限流比例过高 (${cdnBlockedCount}/${candidates.length})`,
           );
           browserRestartCount++;
-          taskBatchCount = 0;
         }
-
+        taskBatchCount = 0;
+      } else if (scraper) {
         const heap = getHeapUsage();
         if (heap.ratio >= HEAP_RESTART_RATIO) {
           await recycleScraper(

package/src/webserver/server.mjs

@@ -80,6 +80,28 @@ function apiLog(msg) {
   if (SCRAPER_CONFIG.enableLog) console.error(msg);
 }
 
+// 限频错误日志：同类错误在窗口内只打首条，其余计数汇总，避免 CDN 限流等高频错误刷屏
+const ERROR_LOG_THROTTLE_MS = 10000;
+let lastThrottledLogTime = 0;
+let throttledLogCount = 0;
+let throttledLogLabel = "";
+function logErrorThrottled(label, msg) {
+  const now = Date.now();
+  if (now - lastThrottledLogTime >= ERROR_LOG_THROTTLE_MS) {
+    if (throttledLogCount > 0) {
+      console.error(
+        `[webserver] ${throttledLogLabel} | 窗口内共 ${throttledLogCount} 条同类错误（已限频）`,
+      );
+    }
+    console.error(`[webserver] ${label} | ${msg}`);
+    lastThrottledLogTime = now;
+    throttledLogCount = 0;
+    throttledLogLabel = label;
+  } else {
+    throttledLogCount++;
+  }
+}
+
 let scraperInstance = null;
 let scraperClosing = false;
 let scraperInitPromise = null;
@@ -257,16 +279,19 @@ async function handleTiktokUser(req, res) {
 
   if (result.error && result.statusCode) {
     // TikTok 明确返回的状态码（209002 验证码、10202 被封、10221 不存在等）：
-    // 属业务结果而非 webserver 故障，不计入连续错误、不触发 scraper 重启
+    // 属业务结果而非 webserver 故障，不计入连续错误、不触发 scraper 重启；
+    // 返回 200 而非 503：避免 nginx 将其当作节点故障去 failover/熔断
+    //（账号/IP 级风控，切换后端无意义，只会放大对 TikTok 的访问）
     const retryable = TIKTOK_RETRYABLE_NORMAL_CODES.has(result.statusCode);
     apiLog(
       `[webserver] user business | ${clientIP} | @${body.uniqueId} | ${elapsed}ms | statusCode=${result.statusCode}${retryable ? " (可重试)" : ""}`,
     );
     recordRequest(true);
-    return sendJSON(res, 503, {
+    return sendJSON(res, 200, {
       error: `TikTok statusCode=${result.statusCode}`,
       retryable,
-      tiktokStatusCode: result.statusCode,
+      statusCode: result.statusCode, // 后端 data-store / attach 外层读取此字段
+      tiktokStatusCode: result.statusCode, // 兼容保留
     });
   }
   if (result.error) {
@@ -560,22 +585,26 @@ export async function startWebserver(
     try {
       return await route.handler(req, res);
     } catch (e) {
-      // 自动恢复：连续错误超过阈值时重启 scraper
-      consecutiveErrors++;
+      // 错误分类：CDN 限流是环境问题（代理/IP 被 TikTok CDN 拦截），重启 scraper 无法解决，
+      // 故不累计 consecutiveErrors、不触发重启；但仍计入 recentErrors（/health 展示错误率）。
+      const isCdnBlocked = e.name === "CDNBlockedError";
+      if (!isCdnBlocked) consecutiveErrors++;
       recordRequest(false);
 
       // 区分错误类型，返回正确的 HTTP 状态码供 nginx 判断
       let status = 500;
       let retryable = false;
 
-      if (e.name === "CDNBlockedError") {
-        // CDN 限流（Akamai Access Denied）→ 503 临时不可用
+      if (isCdnBlocked) {
+        // CDN 限流（Akamai Access Denied）→ 503 临时不可用（节点级，nginx 可 failover 到其他节点）
         status = 503;
         retryable = true;
+        logErrorThrottled("request CDN限流", e.message);
       } else if (e.message?.includes("Server is shutting down")) {
         // 服务正在关闭 → 503
         status = 503;
         retryable = true;
+        console.error(`[webserver] request 服务关闭中: ${e.message}`);
       } else if (
         e.message?.includes("Timeout") ||
         e.message?.includes("timeout") ||
@@ -584,6 +613,7 @@ export async function startWebserver(
         // 超时 → 504
         status = 504;
         retryable = true;
+        console.error(`[webserver] request 超时: ${e.message}`);
       } else if (
         e.message?.includes("Protocol error") ||
         e.message?.includes("Target closed") ||
@@ -593,6 +623,9 @@ export async function startWebserver(
         // 浏览器/页面崩溃 → 503
         status = 503;
         retryable = true;
+        console.error(`[webserver] request 浏览器异常: ${e.message}`);
+      } else {
+        console.error(`[webserver] request 未知错误: ${e.message}`);
       }
 
       if (consecutiveErrors >= MAX_CONSECUTIVE_ERRORS) {