tt-help-cli-ycl 1.4.27 → 1.4.29

package/src/cli/comments.js

@@ -732,6 +732,28 @@ async function runAutoMode(options) {
             await session.ensurePage();
             if (!session.page) throw new Error("未找到可用页面");
 
+            // ── 登录弹窗检测（任务前）──
+            try {
+              const modals = await detectModals(session.page);
+              const loginModal = modals.find(
+                (m) =>
+                  m.type === "login-modal" || m.type === "modal-container",
+              );
+              if (loginModal) {
+                console.error(`        🔔 检测到登录弹窗，正在关闭...`);
+                const { closed, failed } = await closeAllModals(
+                  session.page,
+                  { waitMs: 300 },
+                );
+                console.error(
+                  `        🔔 弹窗关闭: closed=${JSON.stringify(closed)} failed=${JSON.stringify(failed)}`,
+                );
+                await new Promise((r) => setTimeout(r, 2000));
+              }
+            } catch (e) {
+              // 忽略弹窗检测异常
+            }
+
             // 导航到视频页
             await prepareVideoPage(session.page, href);
 

package/src/cli/raw-sync.js

@@ -1,7 +1,7 @@
 /**
  * raw-sync CLI 入口
  *
- * 启动 raw_jobs 同步程序（独立子进程）。
+ * 启动 raw_jobs 同步与任务补充程序（独立子进程）。
  */
 
 import { spawn } from "child_process";
@@ -16,10 +16,7 @@ export function handleRawSync(parsed) {
 
   if (parsed.rawSyncInterval != null)
     args.push("--interval", String(parsed.rawSyncInterval));
-  if (parsed.rawSyncMax != null)
-    args.push("--max", String(parsed.rawSyncMax));
-  if (parsed.rawSyncBatch != null)
-    args.push("--batch", String(parsed.rawSyncBatch));
+  if (parsed.rawSyncLlm) args.push("--llm");
   if (parsed.rawSyncOnce) args.push("--once");
 
   console.error(`[raw-sync] 启动同步程序: ${SCRIPT}`);

package/src/cli/search.js

@@ -0,0 +1,583 @@
+/**
+ * handleSearch — 基于 productive 标签搜索 TikTok 作者
+ *
+ * 流程:
+ *   1. GET /api/tags/search-task   → 领取标签
+ *   2. POST /api/tags/search-claim → 锁定标签
+ *   3. collectSearchAuthors()      → 并行搜索 5 个 tab
+ *   4. POST /api/tags/search-result → 提交结果
+ *
+ * 监控检测流程（对齐 explore）:
+ *   - 验证码检测（切换任务前 + 搜索完成后）
+ *   - 浏览器关闭恢复（isBrowserClosedError + relaunchBrowser）
+ *   - 网络错误退避（consecutiveNetworkErrors 累进等待）
+ *   - 临时错误重试（Timeout/ExecutionContext）
+ *   - 错误上报（POST /api/error-report）
+ *
+ * 健康检测流程（对齐 explore）:
+ *   - HealthChecker 监控内存（>95%）+ 运行时间（>48分钟）
+ *   - switchAccount 自动切换账户
+ */
+
+import { randomUUID } from "crypto";
+import { ensureBrowserReady as ensureBrowserReadyCDP } from "../lib/browser/cdp.js";
+import { switchAccount, killEdgeProcesses } from "../lib/browser/cdp.js";
+import {
+  getOrCreatePage,
+  safeCheckLogin,
+  isBrowserClosedError,
+  relaunchBrowser,
+} from "../lib/browser/page.js";
+import { collectSearchAuthors } from "../lib/search-collector.js";
+import {
+  delay,
+  getDelayConfig,
+  setDelayConfig,
+} from "../scraper/modules/page-helpers.js";
+import {
+  detectCaptcha,
+  closeCaptcha,
+} from "../scraper/modules/captcha-handler.js";
+import { createApiClient } from "../lib/api-client.js";
+import { HealthChecker } from "../lib/browser/health-checker.js";
+import {
+  server as cfgServer,
+  proxy as configuredProxy,
+} from "../lib/constants.js";
+
+const DEFAULT_SERVER = cfgServer || "http://127.0.0.1:3000";
+const STARTUP_TIKTOK_URL = "https://www.tiktok.com/@ycl5007";
+const PAGE_GOTO_TIMEOUT = 60000;
+const MAX_RETRY_WAIT = 5 * 60 * 1000;
+
+async function withRetry(label, fn) {
+  let backoff = 1000;
+  while (true) {
+    try {
+      return await fn();
+    } catch (err) {
+      console.error(
+        `[连接] ${label} 失败: ${err.message}，${backoff / 1000}秒后重试...`,
+      );
+      await new Promise((r) => setTimeout(r, backoff));
+      if (backoff < MAX_RETRY_WAIT) backoff *= 2;
+    }
+  }
+}
+
+export async function handleSearch(parsed) {
+  const { search: searchOpts } = parsed;
+  const {
+    serverUrl,
+    basePort = 3333,
+    portCount = 10,
+    proxy: cliProxy,
+  } = searchOpts || {};
+
+  const effectiveProxy = cliProxy || configuredProxy;
+
+  const log = (...args) => process.stderr.write(args.join(" ") + "\n");
+
+  log("");
+  log("╔════════════════════════════════════╗");
+  log("║    TikTok 搜索作者采集（标签驱动）");
+  log("║    服务端: " + (serverUrl || DEFAULT_SERVER));
+  log(
+    "║    端口: " +
+      basePort +
+      "-" +
+      (basePort + portCount - 1) +
+      "（共 " +
+      portCount +
+      " 个）",
+  );
+  log("║    代理: " + (effectiveProxy || "无"));
+  log("╚════════════════════════════════════╝");
+  log("");
+
+  // ── 初始化 ──
+  const baseUrl = serverUrl || DEFAULT_SERVER;
+  const healthChecker = new HealthChecker({
+    basePort,
+    totalAccounts: portCount,
+  });
+  let currentAccount = healthChecker.getCurrentAccount();
+
+  const cdpOptions = {};
+  cdpOptions.port = currentAccount.port;
+  cdpOptions.userDataDir = currentAccount.userDataDir;
+  if (effectiveProxy) {
+    cdpOptions.proxyServer = effectiveProxy;
+  }
+
+  let browser = null;
+  let shuttingDown = false;
+
+  const cleanup = () => {
+    if (shuttingDown) return;
+    shuttingDown = true;
+    log("\n正在清理...");
+    browser?.close().catch(() => {});
+    killEdgeProcesses(cdpOptions.userDataDir, cdpOptions.port).catch(() => {});
+    process.exit(0);
+  };
+  process.on("SIGINT", cleanup);
+  process.on("SIGTERM", cleanup);
+
+  const { apiGet, apiPost } = createApiClient({
+    meta: { port: cdpOptions.port, type: "tag-search" },
+  });
+
+  setDelayConfig("normal");
+
+  try {
+    // ── 启动：逐个尝试账户，直到找到一个已登录的 ──
+    let page = null;
+    let loggedIn = false;
+    let triedAccounts = 0;
+    const maxAccounts = portCount;
+
+    while (!loggedIn && triedAccounts < maxAccounts) {
+      if (triedAccounts > 0) {
+        // 切换到下一个账户
+        const prevAccount = currentAccount;
+        currentAccount = healthChecker.getNextAccount();
+        log(
+          `\n[启动] 尝试下一个账户: 端口 ${prevAccount.port} → ${currentAccount.port}`,
+        );
+        Object.assign(cdpOptions, {
+          port: currentAccount.port,
+          userDataDir: currentAccount.userDataDir,
+          ...(effectiveProxy ? { proxyServer: effectiveProxy } : {}),
+        });
+      }
+
+      triedAccounts++;
+      log(
+        `\n[启动] 尝试连接端口 ${cdpOptions.port} (${triedAccounts}/${maxAccounts})...`,
+      );
+
+      try {
+        // 连接 CDP 浏览器
+        browser = await ensureBrowserReadyCDP(cdpOptions);
+        page = await getOrCreatePage(browser);
+
+        // 导航到 TikTok 并检测登录
+        await withRetry("启动页面导航", async () => {
+          await page.goto(STARTUP_TIKTOK_URL, {
+            waitUntil: "domcontentloaded",
+            timeout: PAGE_GOTO_TIMEOUT,
+          });
+        });
+
+        loggedIn = await safeCheckLogin(page);
+        if (!loggedIn) {
+          log(`  ⚠️ 端口 ${cdpOptions.port} 未登录 TikTok`);
+          await browser.close().catch(() => {});
+          browser = null;
+        }
+      } catch (e) {
+        log(`  ❌ 端口 ${cdpOptions.port} 连接失败: ${e.message}`);
+        await browser?.close().catch(() => {});
+        browser = null;
+      }
+    }
+
+    if (!loggedIn || !browser || !page) {
+      log(
+        `❌ 所有 ${triedAccounts} 个账户均未登录或连接失败，请至少在一个端口的浏览器中登录 TikTok 后重试`,
+      );
+      process.exit(1);
+    }
+    log(`✅ 端口 ${cdpOptions.port} 已登录 TikTok`);
+
+    // 提取当前登录用户名（通过 TikTok 内部 API）
+    async function extractMyUsername(p) {
+      try {
+        return await p.evaluate(async () => {
+          try {
+            const res = await fetch("https://www.tiktok.com/passport/web/account/info/");
+            const data = await res.json();
+            return data?.data?.unique_id || null;
+          } catch {}
+          return null;
+        });
+      } catch {
+        return null;
+      }
+    }
+
+    let myUsername = await extractMyUsername(page);
+    if (myUsername) {
+      log(`  当前用户: @${myUsername}`);
+    } else {
+      log(`  ⚠️ 无法提取当前用户名，将不会排除自身`);
+    }
+
+    // 全局拦截图片/样式资源，减少内存和带宽
+    await page.route("**/*", (route) => {
+      const resourceType = route.request().resourceType();
+      if (resourceType === "image" || resourceType === "stylesheet") {
+        route.abort();
+      } else {
+        route.continue();
+      }
+    });
+
+    // ── 账户切换后重建 page ──
+    async function setupNewPage(newBrowser) {
+      const newPage = await getOrCreatePage(newBrowser);
+      await newPage.route("**/*", (route) => {
+        const resourceType = route.request().resourceType();
+        if (resourceType === "image" || resourceType === "stylesheet") {
+          route.abort();
+        } else {
+          route.continue();
+        }
+      });
+      return newPage;
+    }
+
+    // ── 账户切换处理 ──
+    async function handleAccountSwitch(reason) {
+      log(`\n[健康检查] ${reason}，切换到下一个账户...`);
+      const oldAccount = currentAccount;
+      const nextAccount = healthChecker.getNextAccount();
+      currentAccount = nextAccount;
+      // 先关闭旧浏览器连接，再切换
+      await browser.close().catch(() => {});
+      const newBrowser = await switchAccount(
+        { port: oldAccount.port, userDataDir: oldAccount.userDataDir },
+        { port: nextAccount.port, userDataDir: nextAccount.userDataDir },
+        effectiveProxy,
+      );
+      browser = newBrowser;
+      const newPage = await setupNewPage(browser);
+      page = newPage;
+      Object.assign(cdpOptions, {
+        port: nextAccount.port,
+        userDataDir: nextAccount.userDataDir,
+        ...(effectiveProxy ? { proxyServer: effectiveProxy } : {}),
+      });
+      log(
+        `[健康检查] 已切换到端口 ${nextAccount.port}${effectiveProxy ? ", 代理: " + effectiveProxy : ""}`,
+      );
+      // 重新导航并检测登录
+      await withRetry("账户切换后页面导航", async () => {
+        await page.goto(STARTUP_TIKTOK_URL, {
+          waitUntil: "domcontentloaded",
+          timeout: PAGE_GOTO_TIMEOUT,
+        });
+      });
+      loggedIn = await safeCheckLogin(page);
+      log(`[健康检查] 新账户登录状态: ${loggedIn ? "已登录" : "未登录"}`);
+      // 重新提取用户名
+      myUsername = await extractMyUsername(page);
+      if (myUsername) {
+        log(`[健康检查] 新账户用户名: @${myUsername}`);
+      }
+      captchaCount = 0;
+      staleRounds = 0;
+      consecutiveNetworkErrors = 0;
+    }
+
+    const clientId = randomUUID();
+    const headers = (extra = {}) => ({
+      "X-Client-Id": clientId,
+      "X-Client-Info": JSON.stringify({
+        type: "tag-search",
+        port: cdpOptions.port,
+      }),
+      ...extra,
+    });
+    log(`客户端 ID: ${clientId.substring(0, 8)}...`);
+    log("");
+
+    let totalSearched = 0;
+    let emptyRounds = 0;
+    let staleRounds = 0;
+    let captchaCount = 0;
+    let consecutiveNetworkErrors = 0;
+
+    // ── 主循环 ──
+    while (!shuttingDown) {
+      try {
+        // ── 健康检查 ──
+        const checkResult = healthChecker.check();
+        if (checkResult.shouldSwitch) {
+          await handleAccountSwitch(checkResult.reason);
+        } else if (checkResult.info && totalSearched % 10 === 0) {
+          log(`\n[健康检查] ${checkResult.info}`);
+        }
+
+        // ── 验证码检测（切换任务前） ──
+        const preCaptcha = await detectCaptcha(page);
+        if (preCaptcha && preCaptcha.visible) {
+          log(`\n[验证码] 切换任务前检测到验证码，等待3分钟...`);
+          captchaCount++;
+          log(`  [验证码] 累计 ${captchaCount} 次`);
+
+          await new Promise((r) => setTimeout(r, 3 * 60 * 1000));
+
+          const closeResult = await closeCaptcha(page);
+          if (closeResult.success) {
+            log(`  [验证码] 已关闭`);
+          } else {
+            log(`  [验证码] 关闭失败: ${closeResult.reason}`);
+          }
+
+          // 上报
+          await withRetry("report captcha", () =>
+            apiPost(`${baseUrl}/api/error-report`, {
+              userId: clientId,
+              username: "unknown",
+              errorType: "captcha",
+              errorMessage: "切换任务前检测到验证码",
+              stage: "between-tasks",
+              errorStack: "",
+            }),
+          ).catch(() => {});
+
+          // 累计2次切换账户
+          if (captchaCount >= 2) {
+            await handleAccountSwitch(`验证码累计 ${captchaCount} 次`);
+            captchaCount = 0;
+          }
+        }
+
+        // ── 网络错误退避 ──
+        if (consecutiveNetworkErrors > 0) {
+          const waitTime =
+            consecutiveNetworkErrors <= 2
+              ? 0
+              : consecutiveNetworkErrors <= 5
+                ? 30000
+                : 300000;
+          if (waitTime > 0) {
+            log(
+              `  [网络] 连续 ${consecutiveNetworkErrors} 次网络异常，等待 ${waitTime / 1000}s 后重试...`,
+            );
+            await new Promise((r) => setTimeout(r, waitTime));
+          }
+        }
+
+        // ── 任务间延迟 ──
+        const { switchMax } = getDelayConfig();
+        await delay(switchMax, switchMax * 3);
+
+        // ── 1. 获取搜索任务 ──
+        const taskRes = await fetch(`${baseUrl}/api/tags/search-task`, {
+          headers: headers(),
+        });
+        const task = await taskRes.json();
+
+        if (!task.tag) {
+          emptyRounds++;
+          if (emptyRounds >= 5) {
+            log(`⏳ 连续 ${emptyRounds} 轮无标签可搜索（30 秒后重试）...`);
+            await new Promise((r) => setTimeout(r, 30000));
+            emptyRounds = 0;
+            await handleAccountSwitch("连续5轮无标签可搜索");
+            continue;
+          }
+          log(`⏳ 暂无标签可搜索（连续 ${emptyRounds} 轮），10 秒后重试...`);
+          await new Promise((r) => setTimeout(r, 10000));
+          continue;
+        }
+        emptyRounds = 0;
+
+        const tag = task.tag;
+        const country = task.country;
+        log(
+          `[${totalSearched + 1}] 搜索: "${tag}" (score: ${task.score}, 国家: ${country})`,
+        );
+
+        // ── 2. 锁定标签 ──
+        const claimRes = await fetch(`${baseUrl}/api/tags/search-claim`, {
+          method: "POST",
+          headers: headers({ "Content-Type": "application/json" }),
+          body: JSON.stringify({ tag }),
+        });
+        const claimData = await claimRes.json();
+        if (!claimData.ok) {
+          log(`  ⏭️ ${claimData.error}，跳过`);
+          continue;
+        }
+
+        // ── 3. 并行搜索 ──
+        const startTime = Date.now();
+        let ids = [];
+        let searchError = null;
+        try {
+          ids = await collectSearchAuthors(browser, tag, {
+            log,
+            exclude: myUsername ? [myUsername] : [],
+          });
+        } catch (e) {
+          searchError = e;
+        }
+        const elapsed = ((Date.now() - startTime) / 1000).toFixed(1);
+
+        // ── 浏览器关闭恢复 ──
+        if (searchError && isBrowserClosedError(searchError)) {
+          log(
+            `\n[浏览器] 检测到浏览器关闭 (${searchError.message})，正在重建...`,
+          );
+          const newBrowser = await relaunchBrowser(cdpOptions, cdpOptions.port);
+          browser = newBrowser;
+          page = await setupNewPage(browser);
+          await new Promise((r) => setTimeout(r, 3000));
+          log(`[浏览器] 已重建，重试当前搜索...`);
+
+          // 重试搜索
+          try {
+            ids = await collectSearchAuthors(browser, tag, {
+              log,
+              exclude: myUsername ? [myUsername] : [],
+            });
+            searchError = null;
+          } catch (e2) {
+            searchError = e2;
+          }
+        }
+
+        // ── 临时错误重试 ──
+        if (searchError) {
+          const retryablePatterns = [
+            "Timeout",
+            "Execution context was destroyed",
+            "ERR_CONNECTION",
+            "net::",
+          ];
+          const isRetryable = retryablePatterns.some((p) =>
+            searchError.message.includes(p),
+          );
+          if (isRetryable) {
+            log(`  [临时错误] 等待 5 秒后重试...`);
+            await new Promise((r) => setTimeout(r, 5000));
+            try {
+              ids = await collectSearchAuthors(browser, tag, {
+                log,
+                exclude: myUsername ? [myUsername] : [],
+              });
+              searchError = null;
+            } catch (e2) {
+              searchError = e2;
+            }
+          }
+        }
+
+        // ── 错误上报 ──
+        if (searchError) {
+          consecutiveNetworkErrors++;
+          log(`  ❌ 搜索异常: ${searchError.message}`);
+
+          const errorType = consecutiveNetworkErrors > 1 ? "network" : "other";
+          await withRetry("report error", () =>
+            apiPost(`${baseUrl}/api/error-report`, {
+              userId: clientId,
+              username: tag,
+              errorType,
+              errorMessage: searchError.message,
+              stage: "search",
+              errorStack: searchError.stack || "",
+            }),
+          ).catch(() => {});
+
+          // 仍提交空结果，释放标签锁定
+          await fetch(`${baseUrl}/api/tags/search-result`, {
+            method: "POST",
+            headers: headers({ "Content-Type": "application/json" }),
+            body: JSON.stringify({ tag, authors: [], country }),
+          }).catch(() => {});
+          log(`  提交: 0 个作者（搜索异常）`);
+          totalSearched++;
+          continue;
+        }
+
+        log(`  搜索完成: ${ids.length} 个作者 (${elapsed}s)`);
+
+        // ── 搜索完成后验证码检测 ──
+        const postCaptcha = await detectCaptcha(page);
+        if (postCaptcha && postCaptcha.visible) {
+          captchaCount++;
+          log(`  [验证码] 搜索后检测到验证码，累计 ${captchaCount} 次`);
+
+          await withRetry("report captcha", () =>
+            apiPost(`${baseUrl}/api/error-report`, {
+              userId: clientId,
+              username: tag,
+              errorType: "captcha",
+              errorMessage: "搜索后页面出现验证码",
+              stage: "after-search",
+              errorStack: "",
+            }),
+          ).catch(() => {});
+
+          if (captchaCount >= 2) {
+            await handleAccountSwitch(`验证码累计 ${captchaCount} 次`);
+            captchaCount = 0;
+          }
+        }
+
+        consecutiveNetworkErrors = 0;
+
+        // ── 4. 提交结果 ──
+        let pushed = 0;
+        if (ids.length > 0) {
+          const resultRes = await fetch(`${baseUrl}/api/tags/search-result`, {
+            method: "POST",
+            headers: headers({ "Content-Type": "application/json" }),
+            body: JSON.stringify({ tag, authors: ids, country }),
+          });
+          const result = await resultRes.json();
+          pushed = result.pushed || 0;
+          log(`  提交: 新增 ${pushed} 个, 跳过 ${result.skipped} 个`);
+        } else {
+          await fetch(`${baseUrl}/api/tags/search-result`, {
+            method: "POST",
+            headers: headers({ "Content-Type": "application/json" }),
+            body: JSON.stringify({ tag, authors: [], country }),
+          });
+          log(`  提交: 0 个作者`);
+        }
+
+        // 无新作者计数 & 切换
+        if (pushed > 0) {
+          staleRounds = 0;
+        } else {
+          staleRounds++;
+          log(`  [无新增] 连续 ${staleRounds} 轮`);
+          if (staleRounds >= 5) {
+            staleRounds = 0;
+            await handleAccountSwitch("连续5轮无新作者");
+            continue;
+          }
+        }
+
+        totalSearched++;
+      } catch (e) {
+        // 浏览器关闭错误：自动重建
+        if (isBrowserClosedError(e)) {
+          log(`\n[浏览器] 检测到浏览器关闭 (${e.message})，正在重建...`);
+          const newBrowser = await relaunchBrowser(cdpOptions, cdpOptions.port);
+          browser = newBrowser;
+          page = await setupNewPage(browser);
+          await new Promise((r) => setTimeout(r, 3000));
+          log(`[浏览器] 已重建，继续处理...`);
+          continue;
+        }
+        // 其他未预期错误
+        log(`❌ 异常: ${e.message}`);
+        log(e.stack || "");
+        await new Promise((r) => setTimeout(r, 5000));
+      }
+    }
+  } finally {
+    process.removeListener("SIGINT", cleanup);
+    process.removeListener("SIGTERM", cleanup);
+    await browser?.close().catch(() => {});
+    killEdgeProcesses(cdpOptions.userDataDir, cdpOptions.port).catch(() => {});
+    log("\n搜索采集已停止");
+  }
+}