tt-help-cli-ycl 1.4.113 → 1.4.115

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tt-help-cli-ycl",
-  "version": "1.4.113",
+  "version": "1.4.115",
   "description": "TikTok user & video data scraper - extract ttSeller, verified, locationCreated from HTML source",
   "type": "module",
   "bin": {

package/scripts/monitor-refresh-mini.mjs

@@ -0,0 +1,161 @@
+/**
+ * refresh-mini 运行状况监控（4222 / 5222）
+ *
+ * - 每 5 分钟采样：各端口 refresh-mini 进程 RSS + Edge 浏览器 RSS/进程数
+ * - 采样日志：../newdata/refresh-mini-monitor.log（JSONL）
+ * - 每天 08:00~08:10 自动生成报告（md 文件 + macOS 系统通知）
+ */
+import { execSync } from "child_process";
+import fs from "fs";
+
+const LOG_FILE =
+  "/Users/yinchengliang/soft/tiktok/newdata/refresh-mini-monitor.log";
+const REPORT_DIR = "/Users/yinchengliang/soft/tiktok/newdata";
+const INTERVAL_MS = 5 * 60 * 1000;
+const PORTS = [4222, 5222];
+
+function execSilent(cmd) {
+  try {
+    return execSync(cmd, { encoding: "utf8" }).trim();
+  } catch {
+    return "";
+  }
+}
+
+/** 采样一次：refresh-mini 进程 + Edge 浏览器进程 */
+function sample() {
+  const ts = new Date();
+  const entry = { time: ts.toISOString(), ports: {} };
+
+  // ps ax -o pid=,rss=,command= ：一行一个进程（macOS/Linux 兼容）
+  const psOut = execSilent("ps ax -o pid=,rss=,command=");
+  const lines = psOut ? psOut.split("\n") : [];
+
+  for (const port of PORTS) {
+    const rmList = [];
+    const edgeList = [];
+    for (const line of lines) {
+      const m = line.match(/^\s*(\d+)\s+(\d+)\s+(.*)$/);
+      if (!m) continue;
+      const [, pid, rssKb, cmd] = m;
+      const is5222 = /--port(=|\s+)5222/.test(cmd);
+      // 排除监控脚本自身（文件名含 refresh-mini）
+      if (
+        cmd.includes("refresh-mini") &&
+        !cmd.includes("monitor-refresh-mini")
+      ) {
+        if ((port === 5222) === is5222) {
+          rmList.push({ pid: Number(pid), rssKb: Number(rssKb) });
+        }
+      }
+      if (cmd.includes(`remote-debugging-port=${port}`)) {
+        edgeList.push({ pid: Number(pid), rssKb: Number(rssKb) });
+      }
+    }
+    const sum = (list) => list.reduce((a, b) => a + b.rssKb, 0);
+    entry.ports[port] = {
+      refreshMini: {
+        processes: rmList.length,
+        rssMb: +(sum(rmList) / 1024).toFixed(1),
+        pids: rmList.map((p) => p.pid),
+      },
+      edge: {
+        processes: edgeList.length,
+        rssMb: +(sum(edgeList) / 1024).toFixed(1),
+      },
+    };
+  }
+
+  fs.appendFileSync(LOG_FILE, JSON.stringify(entry) + "\n");
+  const parts = Object.entries(entry.ports).map(
+    ([p, d]) =>
+      `${p}: rm ${d.refreshMini.processes}个/${d.refreshMini.rssMb}MB, Edge ${d.edge.processes}进程/${d.edge.rssMb}MB`,
+  );
+  console.log(`[${ts.toLocaleString("zh-CN")}] ${parts.join(" | ")}`);
+  return entry;
+}
+
+/** 统计日志生成报告 */
+function generateReport() {
+  let rows = [];
+  try {
+    rows = fs
+      .readFileSync(LOG_FILE, "utf8")
+      .split("\n")
+      .filter(Boolean)
+      .map((l) => JSON.parse(l));
+  } catch {
+    /* 无日志 */
+  }
+  if (rows.length === 0) return;
+
+  const fmt = (v) => (v == null ? "-" : `${v} MB`);
+  const stats = (arr) => {
+    const vals = arr.filter((v) => v != null);
+    if (vals.length === 0) return { min: "-", max: "-", avg: "-" };
+    return {
+      min: Math.min(...vals).toFixed(1),
+      max: Math.max(...vals).toFixed(1),
+      avg: (vals.reduce((a, b) => a + b, 0) / vals.length).toFixed(1),
+    };
+  };
+
+  const lines = [
+    `# refresh-mini 运行状况报告`,
+    ``,
+    `生成时间: ${new Date().toLocaleString("zh-CN")}`,
+    `采样频率: 5 分钟 | 采样点: ${rows.length} 个`,
+    ``,
+  ];
+  for (const port of PORTS) {
+    lines.push(`## 端口 ${port}`, ``);
+    const rmRss = rows.map((r) => r.ports?.[port]?.refreshMini?.rssMb ?? null);
+    const edgeRss = rows.map((r) => r.ports?.[port]?.edge?.rssMb ?? null);
+    const rmAlive = rows.filter(
+      (r) => (r.ports?.[port]?.refreshMini?.processes ?? 0) > 0,
+    ).length;
+    const edgeAlive = rows.filter(
+      (r) => (r.ports?.[port]?.edge?.processes ?? 0) > 0,
+    ).length;
+    const rmS = stats(rmRss);
+    const edgeS = stats(edgeRss);
+    const last = rows[rows.length - 1]?.ports?.[port] ?? {};
+    lines.push(
+      `- refresh-mini 进程：存活采样 ${rmAlive}/${rows.length} 次，内存 min/max/avg = ${fmt(rmS.min)} / ${fmt(rmS.max)} / ${fmt(rmS.avg)}`,
+      `- 当前：${last.refreshMini?.processes ?? 0} 个进程，${fmt(last.refreshMini?.rssMb)}（PID: ${(last.refreshMini?.pids ?? []).join(", ") || "无"}）`,
+      `- Edge 浏览器：存活采样 ${edgeAlive}/${rows.length} 次，内存 min/max/avg = ${fmt(edgeS.min)} / ${fmt(edgeS.max)} / ${fmt(edgeS.avg)}`,
+      `- 当前：${last.edge?.processes ?? 0} 个进程，${fmt(last.edge?.rssMb)}`,
+      ``,
+    );
+  }
+
+  const now = new Date();
+  const file = `${REPORT_DIR}/refresh-mini-report-${now.toISOString().slice(0, 10)}.md`;
+  fs.writeFileSync(file, lines.join("\n"));
+  console.log(`\n=== 8 点报告已生成: ${file} ===`);
+  execSilent(
+    `osascript -e 'display notification "refresh-mini 运行报告已生成" with title "8点监控报告" subtitle "${file.split("/").pop()}"'`,
+  );
+}
+
+// 每天 08:00~08:10 窗口内报告一次
+let reportedToday = false;
+function maybeReport() {
+  const now = new Date();
+  const key = now.toISOString().slice(0, 10);
+  if (now.getHours() === 8 && now.getMinutes() < 10) {
+    if (reportedToday !== key) {
+      reportedToday = key;
+      generateReport();
+    }
+  }
+}
+
+console.log(
+  `[监控] 启动于 ${new Date().toLocaleString("zh-CN")}，每 5 分钟采样，每天 08:00 报告`,
+);
+sample();
+setInterval(() => {
+  sample();
+  maybeReport();
+}, INTERVAL_MS);

package/scripts/precheck-monitor.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+# 远程服务器 precheck / pending_refresh 持续监控
+# 每 15 分钟采样一次，追加写入日志（按天一个文件）
+# 用法: nohup ./precheck-monitor.sh [日志文件] >/dev/null 2>&1 &
+
+LOG="${1:-/Users/yinchengliang/soft/tiktok/newdata/precheck-monitor-$(date +%Y%m%d).log}"
+TARGETS="'CZ','GR','HU','PT','ES','PL','NL','BE','DE','FR','IT','IE','AT','US'"
+SERVER="http://117.71.53.99:17301"
+
+count_sql() { # $1 = 已 URL 编码的 SQL
+  curl -s --max-time 20 "$SERVER/api/db-query?sql=$1" | python3 -c '
+import json, sys
+try:
+  print(json.load(sys.stdin)["rows"][0]["cnt"])
+except Exception:
+  print("ERR")
+'
+}
+
+while true; do
+  TS=$(date "+%Y-%m-%d %H:%M:%S")
+  THRESHOLD=$(( $(date +%s)000 - 172800000 ))
+
+  SQL_PC=$(python3 -c "
+import urllib.parse
+print(urllib.parse.quote(\"SELECT COUNT(*) AS cnt FROM jobs WHERE tt_seller=1 AND verified=0 AND (pending_refresh IS NULL OR pending_refresh=0) AND COALESCE(refresh_time, 1452000000000) < $THRESHOLD AND location_created IN ($TARGETS)\"))
+")
+  SQL_PR=$(python3 -c "
+import urllib.parse
+print(urllib.parse.quote(\"SELECT COUNT(*) AS cnt FROM jobs WHERE tt_seller=1 AND verified=0 AND pending_refresh=1 AND location_created IN ($TARGETS)\"))
+")
+
+  PC=$(count_sql "$SQL_PC")
+  PR=$(count_sql "$SQL_PR")
+  echo "$TS precheck=$PC pending_refresh=$PR" >> "$LOG"
+  sleep 900
+done

package/src/clash/checker.js

@@ -173,13 +173,22 @@ export async function requireClashOrExit(label = "") {
 
 export function checkNodeHealth(url = CONFIG.testUrl) {
   const { body, metaLine, error } = _curlViaProxy(url);
-  const { httpCode, timeTotal, sizeDownload } = _parseCurlMeta(metaLine);
+  const { httpCode, timeTotal, sizeDownload, numRedirects } =
+    _parseCurlMeta(metaLine);
   const title = _extractTitle(body);
   return {
-    healthy: _isHealthyPage(error, httpCode, sizeDownload, title, body),
+    healthy: _isHealthyPage(
+      error,
+      httpCode,
+      sizeDownload,
+      title,
+      body,
+      numRedirects,
+    ),
     httpCode,
     timeTotal,
     sizeDownload,
+    numRedirects,
     title,
     error,
   };
@@ -189,7 +198,7 @@ function _curlViaProxy(url) {
   const args = [
     "-s",
     "-w",
-    "\n__CURL_META__%{http_code}|%{time_total}|%{size_download}",
+    "\n__CURL_META__%{http_code}|%{time_total}|%{size_download}|%{num_redirects}",
     "--proxy",
     CONFIG.proxyUrl,
     "--max-time",
@@ -224,6 +233,7 @@ function _parseCurlMeta(line) {
     httpCode: parseInt(p[0]) || 0,
     timeTotal: parseFloat(p[1]) || 0,
     sizeDownload: parseInt(p[2]) || 0,
+    numRedirects: parseInt(p[3]) || 0,
   };
 }
 
@@ -232,10 +242,14 @@ function _extractTitle(body) {
   return m ? m[1].trim() : "";
 }
 
-function _isHealthyPage(error, httpCode, size, title, body) {
+function _isHealthyPage(error, httpCode, size, title, body, numRedirects = 0) {
   return (
     !error &&
     httpCode === 200 &&
+    // 发生跳转即不健康：探测 URL 正常时应直接 200。
+    // 跳转意味着出口地区/节点异常（如香港停运 302 → /hk/about，
+    // 该页 title 仍是 TikTok，旧版 title 规则无法识别）
+    numRedirects === 0 &&
     size >= CONFIG.minPageSize &&
     title.includes("TikTok") &&
     !title.startsWith("About |") &&

package/src/clash/daemon.js

@@ -212,6 +212,7 @@ export class ClashDaemon {
       if (!target) {
         // 全部候选不可用，恢复原节点
         await checker.switchToNode(node).catch(() => {});
+        await api.closeAllConnections().catch(() => {});
         this._log(`✗ 恢复原节点 ${node}（原因: ${reason} — 候选全部不可用）`);
         ipc.writeHealth({
           node,
@@ -226,6 +227,9 @@ export class ClashDaemon {
       this._log(
         `⚠ 节点切换 [巡检异常: ${reason}]: ${node} → ${target} (${h2.timeTotal.toFixed(1)}s ${(h2.sizeDownload / 1024).toFixed(0)}KB)`,
       );
+      // 切换后断开所有旧连接：mihomo 不主动掐旧连接，浏览器 keep-alive 会继续走旧节点
+      // 导致"时好时坏"（实测旧连接可存活 8+ 分钟），必须强制重建才即时生效
+      await api.closeAllConnections().catch(() => {});
       ipc.writeHealth({
         node: target,
         healthy: h2.healthy,
@@ -312,6 +316,8 @@ export class ClashDaemon {
         });
       }
       const h = checker.checkNodeHealth();
+      // 切换后断开所有旧连接，浏览器立即重建到新节点（同巡检切换）
+      await api.closeAllConnections().catch(() => {});
       this._log(
         `🔄 看门狗指令: 切换完成 (${h.timeTotal.toFixed(1)}s ${(h.sizeDownload / 1024).toFixed(0)}KB, ${h.healthy ? "健康" : "异常"})`,
       );
@@ -366,6 +372,7 @@ export class ClashDaemon {
       }
       if (!target) {
         await checker.switchToNode(node).catch(() => {});
+        await api.closeAllConnections().catch(() => {});
         this._log(
           `✗ failover [${reason}] — 候选全部不可用，恢复原节点 ${node}`,
         );
@@ -378,6 +385,8 @@ export class ClashDaemon {
         });
       }
       this._log(`⚠ 节点切换 [${reason}]: ${node} → ${target}`);
+      // 切换后断开所有旧连接，浏览器立即重建到新节点（同巡检切换）
+      await api.closeAllConnections().catch(() => {});
       ipc.writeHealth({
         node: target,
         healthy: h2.healthy,

package/src/cli/attach.js

@@ -1,5 +1,6 @@
 import { TikTokScraper } from "../lib/tiktok-scraper.mjs";
 import { CDNBlockedError } from "../lib/parse-ssr.mjs";
+import { writeCommand } from "../clash/ipc.js";
 import {
   proxy as configuredProxy,
   userId as configuredUserId,
@@ -11,6 +12,8 @@ import v8 from "node:v8";
 
 const HEAP_RESTART_RATIO = 0.72;
 const MAX_TASK_BATCHES_BEFORE_RESTART = 200;
+const MAX_CDN_SWITCH_ATTEMPTS = 3; // 连续 CDN 限流最多切换几次节点，之后切换无效则改纯等待（账号/整体风控时切换无用）
+const MAX_CDN_COOLDOWN_SEC = 300; // CDN 冷却上限（指数退避：60s→120s→240s→300s）
 
 function formatNow() {
   return new Date().toLocaleString("zh-CN", {
@@ -147,8 +150,27 @@ export async function handleAttach(options) {
         body: JSON.stringify({ uniqueId }),
         signal: AbortSignal.timeout(30000),
       });
-      const data = await res.json();
-      if (!res.ok || data.error) return null;
+      let data = null;
+      try {
+        data = await res.json();
+      } catch {}
+      if (!res.ok) {
+        // CDN 限流：抛 CDNBlockedError，外层据此冷却（webserver 模式同样生效）
+        if (
+          data &&
+          typeof data.error === "string" &&
+          data.error.includes("CDN限流")
+        ) {
+          throw new CDNBlockedError(data.error);
+        }
+        // 其余失败（瞬时 503/5xx）返回 null，由服务端后续重新排队
+        return null;
+      }
+      // 业务结果：TikTok 明确 statusCode（209002 验证码等）→ 返回上层走"已记录"分支
+      if (data && data.error && data.tiktokStatusCode) {
+        return data;
+      }
+      if (data && data.error) return null;
       return data;
     };
   } else {
@@ -185,6 +207,7 @@ export async function handleAttach(options) {
     let loopCount = 0;
     let browserRestartCount = 0;
     let taskBatchCount = 0;
+    let consecutiveCdnBlocks = 0; // 连续 CDN 限流轮数（自适应：前几次切换节点，之后长等待）
 
     while (!shuttingDown) {
       loopCount++;
@@ -348,23 +371,47 @@ export async function handleAttach(options) {
 
       attachLog(`  本批结果: ${successCount} 成功, ${failCount} 失败\n`);
 
-      // CDN 限流比例超过 30% 时，冷却 + 重启浏览器（仅本地浏览器模式）
-      if (scraper) {
-        const cdnRatio = cdnBlockedCount / tasks.length;
-        if (cdnRatio > 0.3) {
-          const coolDownSeconds = cdnRatio > 0.8 ? 120 : 60;
+      // CDN 限流比例超过 30% 时：自适应策略（前几次切节点，连续限流则长等待）
+      const cdnRatio = tasks.length > 0 ? cdnBlockedCount / tasks.length : 0;
+      if (cdnRatio > 0.3) {
+        consecutiveCdnBlocks++;
+        // 冷却指数退避：60s→120s→240s→…上限 300s（切换后短等，持续限流则长等窗口过去）
+        const coolDownSeconds = Math.min(
+          60 * Math.pow(2, consecutiveCdnBlocks - 1),
+          MAX_CDN_COOLDOWN_SEC,
+        );
+        const shouldSwitch = consecutiveCdnBlocks <= MAX_CDN_SWITCH_ATTEMPTS;
+        if (shouldSwitch) {
+          // 用现有 IPC 接口通知 daemon 从候选集切下一个健康节点（换 IP 绕过 CDN 限流）
+          try {
+            writeCommand({
+              action: "failover",
+              reason: `attach CDN限流 连续${consecutiveCdnBlocks}次 (${(cdnRatio * 100).toFixed(0)}%)`,
+            });
+          } catch {}
+          attachLog(
+            `  [Attach] CDN限流比例 ${(cdnRatio * 100).toFixed(0)}% (${cdnBlockedCount}/${tasks.length})，第${consecutiveCdnBlocks}次连续限流，已通知 daemon 切换节点，冷却 ${coolDownSeconds} 秒...`,
+          );
+        } else {
+          // 连续多次切换仍限流 → 大概率账号/整体风控，切换无用，纯等待窗口过去
           attachLog(
-            `  [Attach] CDN限流比例 ${(cdnRatio * 100).toFixed(0)}% (${cdnBlockedCount}/${tasks.length})，冷却 ${coolDownSeconds} 秒后重启浏览器...`,
+            `  [Attach] CDN限流比例 ${(cdnRatio * 100).toFixed(0)}% (${cdnBlockedCount}/${tasks.length})，已连续 ${consecutiveCdnBlocks} 次切换仍限流（可能账号/整体风控），停止切换，长冷却 ${coolDownSeconds} 秒等待窗口过去...`,
           );
-          await new Promise((r) => setTimeout(r, coolDownSeconds * 1000));
+        }
+        await new Promise((r) => setTimeout(r, coolDownSeconds * 1000));
+        if (scraper) {
           await recycleScraper(
             scraper,
             `CDN限流比例过高 (${cdnBlockedCount}/${tasks.length})`,
           );
           browserRestartCount++;
-          taskBatchCount = 0;
         }
+        taskBatchCount = 0;
+      } else {
+        consecutiveCdnBlocks = 0; // 正常批次重置连续限流计数
+      }
 
+      if (scraper) {
         const heap = getHeapUsage();
         if (heap.ratio >= HEAP_RESTART_RATIO) {
           await recycleScraper(

package/src/cli/precheck.js

@@ -1,5 +1,6 @@
 import { TikTokScraper } from "../lib/tiktok-scraper.mjs";
 import { CDNBlockedError } from "../lib/parse-ssr.mjs";
+import { writeCommand } from "../clash/ipc.js";
 import {
   proxy as configuredProxy,
   userId as configuredUserId,
@@ -163,8 +164,27 @@ export async function handlePrecheck(options) {
           body: JSON.stringify({ uniqueId }),
           signal: AbortSignal.timeout(30000),
         });
-        const data = await res.json();
-        if (!res.ok || data.error) return null;
+        let data = null;
+        try {
+          data = await res.json();
+        } catch {}
+        if (!res.ok) {
+          // CDN 限流：抛 CDNBlockedError，外层据此冷却（webserver 模式同样生效）
+          if (
+            data &&
+            typeof data.error === "string" &&
+            data.error.includes("CDN限流")
+          ) {
+            throw new CDNBlockedError(data.error);
+          }
+          // 其余失败（瞬时 503/5xx）返回 null，由服务端后续重新排队
+          return null;
+        }
+        // 业务结果：TikTok 明确 statusCode（209002 验证码等）→ 返回上层走"已记录"分支
+        if (data && data.error && data.tiktokStatusCode) {
+          return data;
+        }
+        if (data && data.error) return null;
         return data;
       };
     } else {
@@ -397,24 +417,32 @@ export async function handlePrecheck(options) {
         `  本轮结果: ${needRefreshCount} 需刷新, ${skipCount} 跳过, ${failCount} 失败`,
       );
 
-      // CDN 限流/堆内存/批次重启（仅本地浏览器模式）
-      if (scraper) {
-        const cdnRatio =
-          candidates.length > 0 ? cdnBlockedCount / candidates.length : 0;
-        if (cdnRatio > 0.3) {
-          const coolDownSeconds = cdnRatio > 0.8 ? 120 : 60;
-          precheckLog(
-            `  [Precheck] CDN限流比例 ${(cdnRatio * 100).toFixed(0)}% (${cdnBlockedCount}/${candidates.length})，冷却 ${coolDownSeconds} 秒后重启浏览器...`,
-          );
-          await new Promise((r) => setTimeout(r, coolDownSeconds * 1000));
+      // CDN 限流比例超过 30% 时：通知 clash daemon 故障切换（换出口 IP 绕过限流）+ 冷却
+      const cdnRatio =
+        candidates.length > 0 ? cdnBlockedCount / candidates.length : 0;
+      if (cdnRatio > 0.3) {
+        // 用现有 IPC 接口通知 daemon 从候选集切下一个健康节点（换 IP 绕过 CDN 限流）
+        try {
+          writeCommand({
+            action: "failover",
+            reason: `precheck CDN限流 (${(cdnRatio * 100).toFixed(0)}%)`,
+          });
+        } catch {}
+        // 切换节点后新 IP 生效需短暂等待，冷却可缩短（替代纯硬等 120s）
+        const coolDownSeconds = cdnRatio > 0.8 ? 90 : 60;
+        precheckLog(
+          `  [Precheck] CDN限流比例 ${(cdnRatio * 100).toFixed(0)}% (${cdnBlockedCount}/${candidates.length})，已通知 daemon 切换节点，冷却 ${coolDownSeconds} 秒...`,
+        );
+        await new Promise((r) => setTimeout(r, coolDownSeconds * 1000));
+        if (scraper) {
           await recycleScraper(
             scraper,
             `CDN限流比例过高 (${cdnBlockedCount}/${candidates.length})`,
           );
           browserRestartCount++;
-          taskBatchCount = 0;
         }
-
+        taskBatchCount = 0;
+      } else if (scraper) {
         const heap = getHeapUsage();
         if (heap.ratio >= HEAP_RESTART_RATIO) {
           await recycleScraper(