tt-help-cli-ycl 1.3.39 → 1.3.41

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tt-help-cli-ycl",
-  "version": "1.3.39",
+  "version": "1.3.41",
   "description": "TikTok user & video data scraper - extract ttSeller, verified, locationCreated from HTML source",
   "type": "module",
   "bin": {

package/src/cli/attach.js

@@ -1,6 +1,19 @@
 import { TikTokScraper } from "../lib/tiktok-scraper.mjs";
+import v8 from "node:v8";
 
 const MAX_RETRY_WAIT = 5 * 60 * 1000;
+const HEAP_RESTART_RATIO = 0.72;
+const MAX_TASK_BATCHES_BEFORE_RESTART = 200;
+
+function formatNow() {
+  return new Date().toLocaleString("zh-CN", {
+    hour12: false,
+  });
+}
+
+function attachLog(message = "") {
+  console.error(`[${formatNow()}] ${message}`);
+}
 
 async function withRetry(label, fn) {
   let backoff = 1000;
@@ -8,7 +21,7 @@ async function withRetry(label, fn) {
     try {
       return await fn();
     } catch (err) {
-      console.error(
+      attachLog(
         `[连接] ${label} 失败: ${err.message}，${backoff / 1000}秒后重试...`,
       );
       await new Promise((r) => setTimeout(r, backoff));
@@ -46,40 +59,52 @@ function isBrowserClosedError(err) {
   );
 }
 
+function getHeapUsage() {
+  const heapUsed = process.memoryUsage().heapUsed;
+  const heapLimit = v8.getHeapStatistics().heap_size_limit;
+  return {
+    heapUsed,
+    heapLimit,
+    ratio: heapLimit > 0 ? heapUsed / heapLimit : 0,
+  };
+}
+
+async function recycleScraper(scraper, reason) {
+  attachLog(`[Attach] ${reason}，正在重启浏览器实例...`);
+  await scraper.restart();
+  attachLog("[Attach] 浏览器实例重启完成");
+}
+
 export async function handleAttach(options) {
   const { attachParallel, attachInterval, serverUrl, showHelp } = options;
   let shuttingDown = false;
   let forceExitTimer = null;
 
   if (showHelp) {
-    console.error(
-      "用法: tt-help attach [-p 并行数] [-i 间隔秒数] [-s 服务端地址]",
-    );
-    console.error("");
-    console.error("参数:");
-    console.error("  -p, --parallel <N>     并行抓取数（默认: 1）");
-    console.error(
-      "  -i, --interval <N>     无任务时轮询间隔，单位秒（默认: 10）",
-    );
-    console.error(
+    attachLog("用法: tt-help attach [-p 并行数] [-i 间隔秒数] [-s 服务端地址]");
+    attachLog("");
+    attachLog("参数:");
+    attachLog("  -p, --parallel <N>     并行抓取数（默认: 1）");
+    attachLog("  -i, --interval <N>     无任务时轮询间隔，单位秒（默认: 10）");
+    attachLog(
       "  -s, --server <URL>     服务端地址（默认: http://127.0.0.1:3001）",
     );
-    console.error("");
-    console.error("说明:");
-    console.error(
+    attachLog("");
+    attachLog("说明:");
+    attachLog(
       "  后台轮询服务端 /api/user-update-tasks 接口，自动抓取 TikTok 用户信息",
     );
-    console.error("  抓取完成后通过 POST /api/user-info-batch 批量回传结果");
-    console.error("  浏览器崩溃时自动重启，支持长时间无人值守运行");
-    console.error("");
-    console.error("示例:");
-    console.error("  tt-help attach");
-    console.error("  tt-help attach -p 5 -i 10");
-    console.error("  tt-help attach -p 3 -i 5 -s http://127.0.0.1:3001");
+    attachLog("  抓取完成后通过 POST /api/user-info-batch 批量回传结果");
+    attachLog("  浏览器崩溃时自动重启，支持长时间无人值守运行");
+    attachLog("");
+    attachLog("示例:");
+    attachLog("  tt-help attach");
+    attachLog("  tt-help attach -p 5 -i 10");
+    attachLog("  tt-help attach -p 3 -i 5 -s http://127.0.0.1:3001");
     return;
   }
 
-  console.error(
+  attachLog(
     `[Attach] 并行数: ${attachParallel}, 空闲间隔: ${attachInterval}秒, 服务端: ${serverUrl}`,
   );
 
@@ -88,18 +113,11 @@ export async function handleAttach(options) {
     if (shuttingDown) return;
     shuttingDown = true;
     forceExitTimer = setTimeout(() => {
-      console.error("[Attach] 关闭超时，强制退出");
+      attachLog("[Attach] 关闭超时，强制退出");
       process.exit(0);
-    }, 8000);
+    }, 30000);
     forceExitTimer.unref?.();
-    console.error(`\n[Attach] 收到 ${signal}，正在关闭浏览器...`);
-    await scraper.close().catch(() => {});
-    if (forceExitTimer) {
-      clearTimeout(forceExitTimer);
-      forceExitTimer = null;
-    }
-    console.error("[Attach] 已退出");
-    process.exit(0);
+    attachLog(`\n[Attach] 收到 ${signal}，等待当前任务完成后退出...`);
   };
 
   const onSigint = () => {
@@ -114,21 +132,34 @@ export async function handleAttach(options) {
 
   try {
     await scraper.init();
-    console.error("[Attach] 浏览器初始化完成，开始循环接收任务...");
+    attachLog("[Attach] 浏览器初始化完成，开始循环接收任务...");
 
     let loopCount = 0;
     let browserRestartCount = 0;
+    let taskBatchCount = 0;
 
     while (!shuttingDown) {
       loopCount++;
 
       // 检查浏览器是否存活，不存活则重启
       if (!scraper.isAlive()) {
-        console.error(
+        attachLog(
           `[Attach] 浏览器已关闭，正在重启 (${++browserRestartCount})...`,
         );
         await scraper.restart();
-        console.error("[Attach] 浏览器重启完成");
+        attachLog("[Attach] 浏览器重启完成");
+      }
+
+      // 收到退出信号且当前无任务正在处理，则退出
+      if (shuttingDown) {
+        attachLog("[Attach] 当前任务已完成，正在关闭浏览器...");
+        await scraper.close().catch(() => {});
+        if (forceExitTimer) {
+          clearTimeout(forceExitTimer);
+          forceExitTimer = null;
+        }
+        attachLog("[Attach] 已退出");
+        process.exit(0);
       }
 
       const { total, tasks } = await apiGet(
@@ -136,21 +167,33 @@ export async function handleAttach(options) {
       );
 
       if (!tasks || tasks.length === 0) {
-        if (loopCount === 1) {
-          console.error(
-            `[Attach] 当前无待更新任务，${attachInterval}秒后重试...`,
+        const heap = getHeapUsage();
+        if (heap.ratio >= HEAP_RESTART_RATIO) {
+          await recycleScraper(
+            scraper,
+            `空闲时堆占用 ${(heap.ratio * 100).toFixed(1)}%`,
           );
         }
+        if (loopCount === 1) {
+          attachLog(`[Attach] 当前无待更新任务，${attachInterval}秒后重试...`);
+        }
+        // 空闲等待时可中断
         await new Promise((r) => setTimeout(r, attachInterval * 1000));
         continue;
       }
 
-      console.error(`\n[Attach] 获取到 ${tasks.length} 个待更新任务...`);
+      attachLog(`\n[Attach] 获取到 ${tasks.length} 个待更新任务...`);
+      taskBatchCount++;
+
+      // 收到退出信号但已有任务在跑，继续完成当前批次
+      if (shuttingDown) {
+        attachLog("[Attach] 正在处理当前批次任务，完成后退出...");
+      }
 
       const results = await Promise.allSettled(
         tasks.map(async (task) => {
           const uniqueId = task.uniqueId;
-          console.error(`  → 获取 @${uniqueId} 的用户信息...`);
+          attachLog(`  → 获取 @${uniqueId} 的用户信息...`);
           try {
             const info = await scraper.getUserInfo(uniqueId);
             return { uniqueId, info, error: null };
@@ -173,18 +216,18 @@ export async function handleAttach(options) {
             if (isBrowserClosedError(error)) {
               needRestart = true;
             }
-            console.error(
+            attachLog(
               `  ✗ @${uniqueId} 获取失败: ${error.message || "未知错误"}`,
             );
             failCount++;
           } else if (info) {
             successTasks.push({ uniqueId, info });
           } else {
-            console.error(`  ✗ @${uniqueId} 未获取到用户信息`);
+            attachLog(`  ✗ @${uniqueId} 未获取到用户信息`);
             failCount++;
           }
         } else {
-          console.error(
+          attachLog(
             `  ✗ 任务执行异常: ${result.reason?.message || "未知错误"}`,
           );
           failCount++;
@@ -201,40 +244,58 @@ export async function handleAttach(options) {
             for (const r of batchRet.results) {
               if (r.ok) {
                 successCount++;
-                console.error(`  ✓ @${r.uniqueId} 已提交更新`);
+                attachLog(`  ✓ @${r.uniqueId} 已提交更新`);
               } else {
                 failCount++;
-                console.error(`  ✗ @${r.uniqueId} 提交失败: ${r.error}`);
+                attachLog(`  ✗ @${r.uniqueId} 提交失败: ${r.error}`);
               }
             }
           } else {
             successCount = successTasks.length;
-            console.error(`  ✓ 批量提交完成 (${successTasks.length} 条)`);
+            attachLog(`  ✓ 批量提交完成 (${successTasks.length} 条)`);
           }
         } catch (err) {
           failCount += successTasks.length;
-          console.error(`  ✗ 批量提交失败: ${err.message}`);
+          attachLog(`  ✗ 批量提交失败: ${err.message}`);
         }
       }
 
-      console.error(`  本批结果: ${successCount} 成功, ${failCount} 失败\n`);
+      attachLog(`  本批结果: ${successCount} 成功, ${failCount} 失败\n`);
+
+      const heap = getHeapUsage();
+      if (heap.ratio >= HEAP_RESTART_RATIO) {
+        await recycleScraper(
+          scraper,
+          `本批后堆占用 ${(heap.ratio * 100).toFixed(1)}%`,
+        );
+        browserRestartCount++;
+        taskBatchCount = 0;
+      } else if (taskBatchCount >= MAX_TASK_BATCHES_BEFORE_RESTART) {
+        await recycleScraper(scraper, `已连续处理 ${taskBatchCount} 批任务`);
+        browserRestartCount++;
+        taskBatchCount = 0;
+      }
 
       if (needRestart) {
-        console.error("[Attach] 检测到浏览器异常，将在下一轮重启...");
+        attachLog("[Attach] 检测到浏览器异常，将在下一轮重启...");
       }
 
       await new Promise((r) => setTimeout(r, 500));
     }
   } catch (err) {
-    console.error(`[Attach] 运行异常: ${err.message}`);
+    attachLog(`[Attach] 运行异常: ${err.message}`);
     throw err;
   } finally {
+    // 正常退出（非信号触发）或超时强制退出时的清理
     if (forceExitTimer) {
       clearTimeout(forceExitTimer);
       forceExitTimer = null;
     }
     process.removeListener("SIGINT", onSigint);
     process.removeListener("SIGTERM", onSigterm);
-    await scraper.close().catch(() => {});
+    // 只有在非信号退出时才关闭浏览器（信号退出已在循环内关闭）
+    if (!shuttingDown) {
+      await scraper.close().catch(() => {});
+    }
   }
 }

package/src/cli/explore.js

@@ -62,6 +62,7 @@ export async function handleExplore(options) {
   const {
     exploreUsernames,
     explorePreset,
+    exploreInterval,
     exploreMaxComments,
     exploreMaxGuess,
     exploreEnableFollow,
@@ -123,6 +124,7 @@ export async function handleExplore(options) {
     if (exploreJobLocations) console.error(`任务国家: ${exploreJobLocations}`);
     console.error(`视频采集: ${exploreMaxVideos || 1}`);
     console.error(`关注/粉丝: ${exploreEnableFollow ? "启用" : "禁用"}`);
+    console.error(`空闲间隔: ${exploreInterval} 秒`);
     console.error(`服务器: ${serverUrl}（断开会自动重连）`);
     if (exploreMaxUsers > 0) console.error(`上限: ${exploreMaxUsers} 个用户`);
 
@@ -239,7 +241,11 @@ export async function handleExplore(options) {
         ? `${serverUrl}/api/job?userId=${encodeURIComponent(userId)}&locations=${encodeURIComponent(exploreJobLocations)}&loggedIn=${loggedIn}`
         : `${serverUrl}/api/job?userId=${encodeURIComponent(userId)}&loggedIn=${loggedIn}`;
       const job = await apiGet(jobQuery);
-      if (!job.hasJob) break;
+      if (!job.hasJob) {
+        console.error(`\n[Explore] 当前无任务，${exploreInterval} 秒后重试...`);
+        await new Promise((r) => setTimeout(r, exploreInterval * 1000));
+        continue;
+      }
 
       const username = job.user.uniqueId;
       processedCount++;
@@ -393,38 +399,67 @@ export async function handleExplore(options) {
       if (result.error) {
         consecutiveNetworkErrors++;
         errorCount++;
-        await apiPost(`${serverUrl}/api/job/${username}`, {
-          error: result.error,
-        });
-        const errorType = result.error.startsWith("被封:")
-          ? "被封"
-          : consecutiveNetworkErrors > 1
-            ? "network"
-            : "other";
-        await withRetry("report error", () =>
-          apiPost(`${serverUrl}/api/error-report`, {
-            userId,
+
+        // 临时性错误：自动重试一次
+        if (result.retryable) {
+          console.error(`  [临时错误] 等待 5 秒后重试 @${username}...`);
+          await new Promise((r) => setTimeout(r, 5000));
+          result = await processExplore(
+            page,
             username,
-            errorType,
-            errorMessage: result.error,
-            stage: "process",
-            errorStack: result.errorStack || "",
-          }),
-        ).catch(() => {});
-        if (errorType === "被封") {
-          blockedCount++;
-          console.error(`  [被封] 累计 ${blockedCount} 次`);
-          if (blockedCount >= 3) {
-            await handleAccountSwitch(`账号被封累计 ${blockedCount} 次`);
-            blockedCount = 0;
+            {
+              maxVideos: exploreMaxVideos,
+              enableFollow: exploreEnableFollow,
+              loggedIn,
+              maxFollowing: exploreMaxFollowing,
+              maxFollowers: exploreMaxFollowers,
+              location: exploreLocation,
+              browser,
+            },
+            console.error,
+          );
+          // 重试成功后继续正常流程
+          if (!result.error) {
+            consecutiveNetworkErrors = 0;
+            errorCount--;
           }
-          continue;
         }
-        if (exploreMaxUsers > 0 && processedCount >= exploreMaxUsers) {
-          console.error(`\n已达上限 ${exploreMaxUsers} 个用户，停止处理`);
-          break;
+
+        if (result.error) {
+          // 上报错误（重试后仍有错误才上报）
+          await apiPost(`${serverUrl}/api/job/${username}`, {
+            error: result.error,
+          });
+          const errorType = result.error.startsWith("被封:")
+            ? "被封"
+            : consecutiveNetworkErrors > 1
+              ? "network"
+              : "other";
+          await withRetry("report error", () =>
+            apiPost(`${serverUrl}/api/error-report`, {
+              userId,
+              username,
+              errorType,
+              errorMessage: result.error,
+              stage: "process",
+              errorStack: result.errorStack || "",
+            }),
+          ).catch(() => {});
+          if (errorType === "被封") {
+            blockedCount++;
+            console.error(`  [被封] 累计 ${blockedCount} 次`);
+            if (blockedCount >= 3) {
+              await handleAccountSwitch(`账号被封累计 ${blockedCount} 次`);
+              blockedCount = 0;
+            }
+            continue;
+          }
+          if (exploreMaxUsers > 0 && processedCount >= exploreMaxUsers) {
+            console.error(`\n已达上限 ${exploreMaxUsers} 个用户，停止处理`);
+            break;
+          }
+          continue;
         }
-        continue;
       }
 
       if (result.captchaDetected) {

package/src/lib/api-interceptor.js

@@ -42,7 +42,7 @@ async function processAPIResponse(
               }, newUrl);
             } catch (e) {
               if (
-                e.message.includes('Execution context was destroyed') &&
+                e.message.includes("Execution context was destroyed") &&
                 i < retries - 1
               ) {
                 await delay(500 * (i + 1), 500 * (i + 1));
@@ -102,10 +102,10 @@ async function fetchUserVideosAPI(page, username, maxVideos, log) {
   let interceptionError = null;
 
   try {
-    await page.goto(
-      `https://www.tiktok.com/@${username}`,
-      { waitUntil: "domcontentloaded", timeout: 30000 },
-    );
+    await page.goto(`https://www.tiktok.com/@${username}`, {
+      waitUntil: "domcontentloaded",
+      timeout: 30000,
+    });
     await assertPageUrl(page, `@${username}`);
 
     const response = await page.waitForResponse(
@@ -158,4 +158,4 @@ async function fetchUserVideosAPI(page, username, maxVideos, log) {
   return new Map();
 }
 
-export { fetchUserVideosAPI };
+export { fetchUserVideosAPI };

package/src/lib/args.js

@@ -165,6 +165,7 @@ function parseAutoArgs(args) {
 function parseExploreArgs(args) {
   let serverUrl = defaultServer;
   let explorePreset = "normal";
+  let exploreInterval = 10;
   let exploreMaxComments = 10;
   let exploreMaxGuess = 0;
   let exploreEnableFollow = true;
@@ -186,6 +187,8 @@ function parseExploreArgs(args) {
     const arg = args[i];
     if (arg === "--server") {
       serverUrl = args[++i];
+    } else if (arg === "-i" || arg === "--interval") {
+      exploreInterval = parseInt(args[++i], 10) || 10;
     } else if (arg === "--max-comments") {
       exploreMaxComments = parseInt(args[++i]) || 0;
     } else if (arg === "--max-guess") {
@@ -239,6 +242,7 @@ function parseExploreArgs(args) {
     subcommand: "explore",
     exploreUsernames: usernames,
     explorePreset,
+    exploreInterval,
     exploreMaxComments,
     exploreMaxGuess,
     exploreEnableFollow,

package/src/lib/constants.js

@@ -131,6 +131,7 @@ const HELP_TEXT = [
   "      --max-following <数量>    最大获取关注数，默认 50（同时设置粉丝数）",
   "      --max-followers <数量>    最大获取粉丝数，默认 50",
   "      --max-users <数量>        最大处理用户数，默认无限制",
+  "      -i, --interval <秒数>     无任务时轮询间隔，默认 10 秒",
   "      --port <端口号>           固定 CDP 端口（调试用，关闭自动轮换）",
   "      --base-port <端口号>      起始端口，默认 9222",
   "      --port-count <数量>       端口数量（账户数），默认 10",

package/src/lib/page-error-detector.js

@@ -56,7 +56,9 @@ const PATTERNS = {
 
 export async function detectPageError(page) {
   return page.evaluate((patterns) => {
-    const bodyText = document.body.innerText;
+    const body = document.body;
+    if (!body) return null;
+    const bodyText = body.innerText;
     const lower = bodyText.toLowerCase();
 
     for (const [type, phrases] of Object.entries(patterns)) {
@@ -81,7 +83,9 @@ export async function detectPageErrorWithWait(page, timeout = 8000) {
   try {
     const handle = await page.waitForFunction(
       (patterns) => {
-        const bodyText = document.body.innerText;
+        const body = document.body;
+        if (!body) return null;
+        const bodyText = body.innerText;
         const lower = bodyText.toLowerCase();
 
         for (const [type, phrases] of Object.entries(patterns)) {

package/src/lib/tiktok-scraper.mjs

@@ -6,6 +6,7 @@ const DEFAULT_POOL_SIZE = 3;
 const DEFAULT_WAF_TTL = 120000;
 const DEFAULT_WARM_URL = "https://www.tiktok.com/@nike";
 const BROWSER_CLOSE_TIMEOUT = 5000;
+const DEFAULT_MAX_REQUESTS_PER_PAGE = 50;
 
 function delay(ms) {
   return new Promise((r) => setTimeout(r, ms));
@@ -15,6 +16,7 @@ class PageSlot {
   constructor(page) {
     this.page = page;
     this.lock = new PromiseQueue();
+    this.requestCount = 0;
   }
 }
 
@@ -50,10 +52,12 @@ export class TikTokScraper {
     poolSize = DEFAULT_POOL_SIZE,
     wafTtl = DEFAULT_WAF_TTL,
     warmUrl = DEFAULT_WARM_URL,
+    maxRequestsPerPage = DEFAULT_MAX_REQUESTS_PER_PAGE,
   } = {}) {
     this.poolSize = poolSize;
     this.wafTtl = wafTtl;
     this.warmUrl = warmUrl;
+    this.maxRequestsPerPage = maxRequestsPerPage;
     this.browser = null;
     this.context = null;
     this.slots = [];
@@ -160,9 +164,16 @@ export class TikTokScraper {
 
   async _ensurePage(slot) {
     try {
-      if (!slot.page.isClosed()) return slot.page;
+      if (
+        !slot.page.isClosed() &&
+        slot.requestCount < this.maxRequestsPerPage
+      ) {
+        return slot.page;
+      }
     } catch {}
+    await slot.page?.close().catch(() => {});
     slot.page = await this.context.newPage();
+    slot.requestCount = 0;
     return slot.page;
   }
 
@@ -174,7 +185,7 @@ export class TikTokScraper {
       timeout: 15000,
     });
 
-    return await page.evaluate(() => {
+    const content = await page.evaluate(() => {
       const rows = document.querySelectorAll("tr");
       let content = "";
       rows.forEach((r) => {
@@ -183,6 +194,18 @@ export class TikTokScraper {
       });
       return content;
     });
+
+    // 导航到 about:blank 释放当前页面的 DOM 和 JS 堆
+    await page
+      .goto("about:blank", {
+        waitUntil: "domcontentloaded",
+        timeout: 5000,
+      })
+      .catch(() => {});
+
+    slot.requestCount += 1;
+
+    return content;
   }
 
   async getUserInfo(uniqueId) {

package/src/scraper/explore-core.js

@@ -173,6 +173,15 @@ async function processExplore(page, username, options, log) {
       result.noVideo = false;
       log(`  @${username} 认定为被封`);
     }
+
+    // 临时性错误：超时、执行上下文销毁、网络异常 — 标记为可重试
+    const retryablePatterns = [
+      "Timeout",
+      "Execution context was destroyed",
+      "ERR_CONNECTION",
+      "net::",
+    ];
+    result.retryable = retryablePatterns.some((p) => e.message.includes(p));
   }
 
   return result;

package/src/scraper/modules/follow-extractor.js

@@ -98,10 +98,37 @@ async function openFollowModal(page) {
       "未找到关注入口元素，请确认当前页面为用户主页或页面结构已变化",
     );
   }
-  await page.waitForSelector("[class*=DivUserListContainer]", {
-    timeout: 30000,
-  });
-  await waitForListContent(page, 1, 5000);
+
+  // 等待用户列表容器，超时说明可能被弹窗遮挡
+  let containerReady = false;
+  for (let attempt = 1; attempt <= 2; attempt++) {
+    try {
+      await page.waitForSelector("[class*=DivUserListContainer]", {
+        timeout: 30000,
+      });
+      containerReady = true;
+      break;
+    } catch (e) {
+      if (attempt === 1) {
+        // 第一次超时，刷新页面重试
+        await page.reload({ waitUntil: "domcontentloaded" });
+        await delay(2000, 3000);
+        // 重新点击 follow 入口
+        opened = await tryOpen();
+        if (!opened) {
+          throw new Error(
+            "刷新后仍未找到关注入口元素，请确认当前页面为用户主页",
+          );
+        }
+      } else {
+        throw e;
+      }
+    }
+  }
+
+  if (containerReady) {
+    await waitForListContent(page, 1, 5000);
+  }
 }
 
 async function switchToFollowersTab(page) {

package/src/videos/core.js

@@ -14,7 +14,7 @@ async function getUserInfo(page) {
         return await page.evaluate(fn);
       } catch (e) {
         if (
-          e.message.includes('Execution context was destroyed') &&
+          e.message.includes("Execution context was destroyed") &&
           i < retries - 1
         ) {
           await new Promise((r) => setTimeout(r, 500 * (i + 1)));