tt-help-cli-ycl 1.4.112 → 1.4.114

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tt-help-cli-ycl",
-  "version": "1.4.112",
+  "version": "1.4.114",
   "description": "TikTok user & video data scraper - extract ttSeller, verified, locationCreated from HTML source",
   "type": "module",
   "bin": {

package/scripts/monitor-refresh-mini.mjs

@@ -0,0 +1,161 @@
+/**
+ * refresh-mini 运行状况监控（4222 / 5222）
+ *
+ * - 每 5 分钟采样：各端口 refresh-mini 进程 RSS + Edge 浏览器 RSS/进程数
+ * - 采样日志：../newdata/refresh-mini-monitor.log（JSONL）
+ * - 每天 08:00~08:10 自动生成报告（md 文件 + macOS 系统通知）
+ */
+import { execSync } from "child_process";
+import fs from "fs";
+
+const LOG_FILE =
+  "/Users/yinchengliang/soft/tiktok/newdata/refresh-mini-monitor.log";
+const REPORT_DIR = "/Users/yinchengliang/soft/tiktok/newdata";
+const INTERVAL_MS = 5 * 60 * 1000;
+const PORTS = [4222, 5222];
+
+function execSilent(cmd) {
+  try {
+    return execSync(cmd, { encoding: "utf8" }).trim();
+  } catch {
+    return "";
+  }
+}
+
+/** 采样一次：refresh-mini 进程 + Edge 浏览器进程 */
+function sample() {
+  const ts = new Date();
+  const entry = { time: ts.toISOString(), ports: {} };
+
+  // ps ax -o pid=,rss=,command= ：一行一个进程（macOS/Linux 兼容）
+  const psOut = execSilent("ps ax -o pid=,rss=,command=");
+  const lines = psOut ? psOut.split("\n") : [];
+
+  for (const port of PORTS) {
+    const rmList = [];
+    const edgeList = [];
+    for (const line of lines) {
+      const m = line.match(/^\s*(\d+)\s+(\d+)\s+(.*)$/);
+      if (!m) continue;
+      const [, pid, rssKb, cmd] = m;
+      const is5222 = /--port(=|\s+)5222/.test(cmd);
+      // 排除监控脚本自身（文件名含 refresh-mini）
+      if (
+        cmd.includes("refresh-mini") &&
+        !cmd.includes("monitor-refresh-mini")
+      ) {
+        if ((port === 5222) === is5222) {
+          rmList.push({ pid: Number(pid), rssKb: Number(rssKb) });
+        }
+      }
+      if (cmd.includes(`remote-debugging-port=${port}`)) {
+        edgeList.push({ pid: Number(pid), rssKb: Number(rssKb) });
+      }
+    }
+    const sum = (list) => list.reduce((a, b) => a + b.rssKb, 0);
+    entry.ports[port] = {
+      refreshMini: {
+        processes: rmList.length,
+        rssMb: +(sum(rmList) / 1024).toFixed(1),
+        pids: rmList.map((p) => p.pid),
+      },
+      edge: {
+        processes: edgeList.length,
+        rssMb: +(sum(edgeList) / 1024).toFixed(1),
+      },
+    };
+  }
+
+  fs.appendFileSync(LOG_FILE, JSON.stringify(entry) + "\n");
+  const parts = Object.entries(entry.ports).map(
+    ([p, d]) =>
+      `${p}: rm ${d.refreshMini.processes}个/${d.refreshMini.rssMb}MB, Edge ${d.edge.processes}进程/${d.edge.rssMb}MB`,
+  );
+  console.log(`[${ts.toLocaleString("zh-CN")}] ${parts.join(" | ")}`);
+  return entry;
+}
+
+/** 统计日志生成报告 */
+function generateReport() {
+  let rows = [];
+  try {
+    rows = fs
+      .readFileSync(LOG_FILE, "utf8")
+      .split("\n")
+      .filter(Boolean)
+      .map((l) => JSON.parse(l));
+  } catch {
+    /* 无日志 */
+  }
+  if (rows.length === 0) return;
+
+  const fmt = (v) => (v == null ? "-" : `${v} MB`);
+  const stats = (arr) => {
+    const vals = arr.filter((v) => v != null);
+    if (vals.length === 0) return { min: "-", max: "-", avg: "-" };
+    return {
+      min: Math.min(...vals).toFixed(1),
+      max: Math.max(...vals).toFixed(1),
+      avg: (vals.reduce((a, b) => a + b, 0) / vals.length).toFixed(1),
+    };
+  };
+
+  const lines = [
+    `# refresh-mini 运行状况报告`,
+    ``,
+    `生成时间: ${new Date().toLocaleString("zh-CN")}`,
+    `采样频率: 5 分钟 | 采样点: ${rows.length} 个`,
+    ``,
+  ];
+  for (const port of PORTS) {
+    lines.push(`## 端口 ${port}`, ``);
+    const rmRss = rows.map((r) => r.ports?.[port]?.refreshMini?.rssMb ?? null);
+    const edgeRss = rows.map((r) => r.ports?.[port]?.edge?.rssMb ?? null);
+    const rmAlive = rows.filter(
+      (r) => (r.ports?.[port]?.refreshMini?.processes ?? 0) > 0,
+    ).length;
+    const edgeAlive = rows.filter(
+      (r) => (r.ports?.[port]?.edge?.processes ?? 0) > 0,
+    ).length;
+    const rmS = stats(rmRss);
+    const edgeS = stats(edgeRss);
+    const last = rows[rows.length - 1]?.ports?.[port] ?? {};
+    lines.push(
+      `- refresh-mini 进程：存活采样 ${rmAlive}/${rows.length} 次，内存 min/max/avg = ${fmt(rmS.min)} / ${fmt(rmS.max)} / ${fmt(rmS.avg)}`,
+      `- 当前：${last.refreshMini?.processes ?? 0} 个进程，${fmt(last.refreshMini?.rssMb)}（PID: ${(last.refreshMini?.pids ?? []).join(", ") || "无"}）`,
+      `- Edge 浏览器：存活采样 ${edgeAlive}/${rows.length} 次，内存 min/max/avg = ${fmt(edgeS.min)} / ${fmt(edgeS.max)} / ${fmt(edgeS.avg)}`,
+      `- 当前：${last.edge?.processes ?? 0} 个进程，${fmt(last.edge?.rssMb)}`,
+      ``,
+    );
+  }
+
+  const now = new Date();
+  const file = `${REPORT_DIR}/refresh-mini-report-${now.toISOString().slice(0, 10)}.md`;
+  fs.writeFileSync(file, lines.join("\n"));
+  console.log(`\n=== 8 点报告已生成: ${file} ===`);
+  execSilent(
+    `osascript -e 'display notification "refresh-mini 运行报告已生成" with title "8点监控报告" subtitle "${file.split("/").pop()}"'`,
+  );
+}
+
+// 每天 08:00~08:10 窗口内报告一次
+let reportedToday = false;
+function maybeReport() {
+  const now = new Date();
+  const key = now.toISOString().slice(0, 10);
+  if (now.getHours() === 8 && now.getMinutes() < 10) {
+    if (reportedToday !== key) {
+      reportedToday = key;
+      generateReport();
+    }
+  }
+}
+
+console.log(
+  `[监控] 启动于 ${new Date().toLocaleString("zh-CN")}，每 5 分钟采样，每天 08:00 报告`,
+);
+sample();
+setInterval(() => {
+  sample();
+  maybeReport();
+}, INTERVAL_MS);

package/src/clash/checker.js

@@ -173,13 +173,22 @@ export async function requireClashOrExit(label = "") {
 
 export function checkNodeHealth(url = CONFIG.testUrl) {
   const { body, metaLine, error } = _curlViaProxy(url);
-  const { httpCode, timeTotal, sizeDownload } = _parseCurlMeta(metaLine);
+  const { httpCode, timeTotal, sizeDownload, numRedirects } =
+    _parseCurlMeta(metaLine);
   const title = _extractTitle(body);
   return {
-    healthy: _isHealthyPage(error, httpCode, sizeDownload, title, body),
+    healthy: _isHealthyPage(
+      error,
+      httpCode,
+      sizeDownload,
+      title,
+      body,
+      numRedirects,
+    ),
     httpCode,
     timeTotal,
     sizeDownload,
+    numRedirects,
     title,
     error,
   };
@@ -189,7 +198,7 @@ function _curlViaProxy(url) {
   const args = [
     "-s",
     "-w",
-    "\n__CURL_META__%{http_code}|%{time_total}|%{size_download}",
+    "\n__CURL_META__%{http_code}|%{time_total}|%{size_download}|%{num_redirects}",
     "--proxy",
     CONFIG.proxyUrl,
     "--max-time",
@@ -224,6 +233,7 @@ function _parseCurlMeta(line) {
     httpCode: parseInt(p[0]) || 0,
     timeTotal: parseFloat(p[1]) || 0,
     sizeDownload: parseInt(p[2]) || 0,
+    numRedirects: parseInt(p[3]) || 0,
   };
 }
 
@@ -232,10 +242,14 @@ function _extractTitle(body) {
   return m ? m[1].trim() : "";
 }
 
-function _isHealthyPage(error, httpCode, size, title, body) {
+function _isHealthyPage(error, httpCode, size, title, body, numRedirects = 0) {
   return (
     !error &&
     httpCode === 200 &&
+    // 发生跳转即不健康：探测 URL 正常时应直接 200。
+    // 跳转意味着出口地区/节点异常（如香港停运 302 → /hk/about，
+    // 该页 title 仍是 TikTok，旧版 title 规则无法识别）
+    numRedirects === 0 &&
     size >= CONFIG.minPageSize &&
     title.includes("TikTok") &&
     !title.startsWith("About |") &&

package/src/clash/daemon.js

@@ -212,6 +212,7 @@ export class ClashDaemon {
       if (!target) {
         // 全部候选不可用，恢复原节点
         await checker.switchToNode(node).catch(() => {});
+        await api.closeAllConnections().catch(() => {});
         this._log(`✗ 恢复原节点 ${node}（原因: ${reason} — 候选全部不可用）`);
         ipc.writeHealth({
           node,
@@ -226,6 +227,9 @@ export class ClashDaemon {
       this._log(
         `⚠ 节点切换 [巡检异常: ${reason}]: ${node} → ${target} (${h2.timeTotal.toFixed(1)}s ${(h2.sizeDownload / 1024).toFixed(0)}KB)`,
       );
+      // 切换后断开所有旧连接：mihomo 不主动掐旧连接，浏览器 keep-alive 会继续走旧节点
+      // 导致"时好时坏"（实测旧连接可存活 8+ 分钟），必须强制重建才即时生效
+      await api.closeAllConnections().catch(() => {});
       ipc.writeHealth({
         node: target,
         healthy: h2.healthy,
@@ -312,6 +316,8 @@ export class ClashDaemon {
         });
       }
       const h = checker.checkNodeHealth();
+      // 切换后断开所有旧连接，浏览器立即重建到新节点（同巡检切换）
+      await api.closeAllConnections().catch(() => {});
       this._log(
         `🔄 看门狗指令: 切换完成 (${h.timeTotal.toFixed(1)}s ${(h.sizeDownload / 1024).toFixed(0)}KB, ${h.healthy ? "健康" : "异常"})`,
       );
@@ -366,6 +372,7 @@ export class ClashDaemon {
       }
       if (!target) {
         await checker.switchToNode(node).catch(() => {});
+        await api.closeAllConnections().catch(() => {});
         this._log(
           `✗ failover [${reason}] — 候选全部不可用，恢复原节点 ${node}`,
         );
@@ -378,6 +385,8 @@ export class ClashDaemon {
         });
       }
       this._log(`⚠ 节点切换 [${reason}]: ${node} → ${target}`);
+      // 切换后断开所有旧连接，浏览器立即重建到新节点（同巡检切换）
+      await api.closeAllConnections().catch(() => {});
       ipc.writeHealth({
         node: target,
         healthy: h2.healthy,

package/src/cli/product-fill.js

@@ -9,11 +9,26 @@
  *
  * 用法: tt-help product-fill [-s 服务端] [-w webserver] [-i 间隔秒] [-l 每批上限]
  */
+import { spawn } from "node:child_process";
+import { fileURLToPath } from "node:url";
 import { getMacOrUuid } from "../lib/mac-or-uuid.js";
 import { createApiClient } from "../lib/api-client.js";
 import { userId as configuredUserId, saveUserId } from "../lib/constants.js";
 
 const CAPTCHA_BACKOFF_MS = 30_000; // 验证码后冷却时长
+const WEBSERVER_READY_TIMEOUT_MS = 60_000; // 等待自动拉起的 webserver 就绪上限
+
+/** 探测 webserver /health 是否可用 */
+async function probeWebserverHealth(url, timeoutMs = 1500) {
+  try {
+    const res = await fetch(`${url}/health`, {
+      signal: AbortSignal.timeout(timeoutMs),
+    });
+    return res.ok;
+  } catch {
+    return false;
+  }
+}
 
 function formatNow() {
   return new Date().toLocaleString("zh-CN", { hour12: false });
@@ -51,9 +66,16 @@ export async function handleProductFill(options) {
     productFillInterval,
     serverUrl,
     productFillWebserver,
+    productFillProxy,
     showHelp,
   } = options;
-  const webserverUrl = productFillWebserver || null;
+  // 不指定 -w 时默认使用本地 webserver（http://127.0.0.1:3000），
+  // 且本地无响应时自动拉起（适合单独机器：验证码时弹浏览器人工处理）
+  const spawnLocalWebserver = !productFillWebserver;
+  const webserverUrl = productFillWebserver || "http://127.0.0.1:3000";
+  // 自动拉起 webserver 时透传的代理（默认本机 clash 7897）
+  const localWebserverProxy = productFillProxy || "http://127.0.0.1:7897";
+  let localWebserverChild = null;
 
   if (showHelp) {
     log(
@@ -63,7 +85,10 @@ export async function handleProductFill(options) {
     log("参数:");
     log("  -s, --server <URL>      服务端地址（默认: http://127.0.0.1:3001）");
     log(
-      "  -w, --webserver <URL>   商品详情 webserver 地址（必填，如 http://127.0.0.1:3000）",
+      "  -w, --webserver <URL>   商品详情 webserver 地址（默认: http://127.0.0.1:3000，本地未运行会自动拉起）",
+    );
+    log(
+      "  --proxy <URL>           自动拉起 webserver 时使用的代理（默认: http://127.0.0.1:7897）",
     );
     log("  -i, --interval <N>      无任务时轮询间隔，单位秒（默认: 10）");
     log("  -l, --limit <N>         每批领取任务数上限（默认: 5）");
@@ -73,6 +98,9 @@ export async function handleProductFill(options) {
     log("  对每个任务调用 webserver /api/tiktok/product 抓取评分/销量/评论；");
     log("  成功后回传 /api/product-fill/:uniqueId 写入 jobs 表。");
     log("  商品详情验证码(503)自动冷却 30 秒后重试，其他失败计数后继续。");
+    log(
+      "  不传 -w 时本地无 webserver 会自动拉起（验证码时弹浏览器，人工解决后继续）。",
+    );
     log("");
     log("示例:");
     log(
@@ -82,13 +110,6 @@ export async function handleProductFill(options) {
     return;
   }
 
-  if (!webserverUrl) {
-    log(
-      "错误: 缺少 webserver 地址，请用 -w/--webserver 指定（如 http://127.0.0.1:3000）",
-    );
-    return;
-  }
-
   // ── 设备标识 ──
   let userId = configuredUserId;
   if (!userId) {
@@ -106,14 +127,101 @@ export async function handleProductFill(options) {
   const { apiGet, apiPost } = createApiClient({ log: false });
 
   let shuttingDown = false;
+  let forceExitTimer = null;
+  /** 强制退出路径清理自动拉起的 webserver（防止孤儿进程） */
+  const killLocalWebserver = () => {
+    if (localWebserverChild && !localWebserverChild.killed) {
+      localWebserverChild.kill("SIGKILL");
+    }
+  };
   const shutdown = () => {
-    if (shuttingDown) return;
+    if (shuttingDown) {
+      // 第二次信号：立即强制退出（网络重试等场景可能无法优雅退出）
+      log(`\n[ProductFill] 再次收到信号，强制退出...`);
+      killLocalWebserver();
+      process.exit(1);
+      return;
+    }
     shuttingDown = true;
-    log(`\n[ProductFill] 收到退出信号，完成当前批次后退出...`);
+    log(
+      `\n[ProductFill] 收到退出信号，完成当前批次后退出（再次按 Ctrl+C 强制退出）...`,
+    );
+    // 兜底：卡在网络重试/等待中时 10 秒后强制退出
+    forceExitTimer = setTimeout(() => {
+      log("[ProductFill] 等待超时，强制退出");
+      killLocalWebserver();
+      process.exit(0);
+    }, 10_000);
+    forceExitTimer.unref?.();
   };
   process.on("SIGINT", () => shutdown());
   process.on("SIGTERM", () => shutdown());
 
+  // ── 确保 webserver 可用：未指定 -w 且本地无响应时自动拉起 ──
+  if (await probeWebserverHealth(webserverUrl)) {
+    log(`[ProductFill] webserver 已就绪: ${webserverUrl}`);
+  } else {
+    if (!spawnLocalWebserver) {
+      log(
+        `[ProductFill] 警告: webserver ${webserverUrl} 无响应，任务将按失败重试`,
+      );
+    } else {
+      const portMatch = webserverUrl.match(/:(\d+)\/?$/);
+      const port = portMatch ? parseInt(portMatch[1], 10) : 3000;
+      // cli.js 在包根（源码与 npm 包结构一致），webserver 命令两者都支持
+      const cliJs = fileURLToPath(new URL("../../cli.js", import.meta.url));
+      log(
+        `[ProductFill] 本地 webserver 未运行，自动拉起（端口 ${port}，代理 ${localWebserverProxy}）...`,
+      );
+      localWebserverChild = spawn(
+        process.execPath,
+        [
+          cliJs,
+          "webserver",
+          "-p",
+          String(port),
+          "--proxy",
+          localWebserverProxy,
+        ],
+        {
+          stdio: ["ignore", "ignore", "inherit"], // webserver 日志（含验证码提示）透传到本终端
+          windowsHide: true,
+        },
+      );
+      localWebserverChild.on("exit", (code) => {
+        if (!shuttingDown && code !== null && code !== 0) {
+          log(`[ProductFill] ⚠ 本地 webserver 已退出 (code=${code})`);
+        }
+      });
+      // 等待就绪（最多 60 秒）
+      const deadline = Date.now() + WEBSERVER_READY_TIMEOUT_MS;
+      while (!shuttingDown && Date.now() < deadline) {
+        if (await probeWebserverHealth(webserverUrl, 1000)) break;
+        await new Promise((r) => setTimeout(r, 1000));
+      }
+      if (await probeWebserverHealth(webserverUrl, 1000)) {
+        log(`[ProductFill] webserver 已就绪: ${webserverUrl}`);
+      } else {
+        log(
+          `[ProductFill] 警告: webserver 启动超时，继续运行（抓取失败将自动重试）`,
+        );
+      }
+    }
+  }
+
+  /** 可被退出信号中断的等待（500ms 粒度检查 shuttingDown） */
+  const interruptibleSleep = (ms) =>
+    new Promise((resolve) => {
+      const start = Date.now();
+      const tick = () => {
+        if (shuttingDown) return resolve();
+        const left = ms - (Date.now() - start);
+        if (left <= 0) return resolve();
+        setTimeout(tick, Math.min(500, left));
+      };
+      tick();
+    });
+
   let loopCount = 0;
   let totalSuccess = 0;
   let totalFailed = 0;
@@ -132,7 +240,7 @@ export async function handleProductFill(options) {
         log(
           `[ProductFill] ⏳ 验证码冷却中（剩余 ${Math.ceil(waitMs / 1000)}s），暂不领取任务...`,
         );
-        await new Promise((r) => setTimeout(r, waitMs));
+        await interruptibleSleep(waitMs);
         continue;
       }
 
@@ -146,7 +254,7 @@ export async function handleProductFill(options) {
             `[ProductFill] 当前无缺商品任务，${productFillInterval}秒后重试...`,
           );
         }
-        await new Promise((r) => setTimeout(r, productFillInterval * 1000));
+        await interruptibleSleep(productFillInterval * 1000);
         continue;
       }
 
@@ -207,12 +315,17 @@ export async function handleProductFill(options) {
       log(
         `  本批: 成功 ${batchSuccess} | 失败 ${batchFailed} | 跳过 ${batchSkipped}（累计: 成功 ${totalSuccess} | 失败 ${totalFailed} | 跳过 ${totalSkipped}）\n`,
       );
-      await new Promise((r) => setTimeout(r, 300));
+      await interruptibleSleep(300);
     }
   } catch (err) {
     log(`[ProductFill] 运行异常: ${err.message}`);
     throw err;
   } finally {
+    if (forceExitTimer) clearTimeout(forceExitTimer);
+    if (localWebserverChild && !localWebserverChild.killed) {
+      log("[ProductFill] 关闭自动拉起的本地 webserver...");
+      localWebserverChild.kill("SIGTERM");
+    }
     log(
       `[ProductFill] 已退出。总计: 成功 ${totalSuccess} | 失败 ${totalFailed} | 跳过 ${totalSkipped}`,
     );