tt-help-cli-ycl 1.3.93 → 1.3.95

package/src/lib/args.js

@@ -668,6 +668,8 @@ function parseCommentsArgs(args) {
   let commentsParallel = 1;
   let commentsInterval = 10;
   let commentsServer = defaultServer;
+  let commentsBasePort = 9222;
+  let commentsProxy = null;
 
   const positional = [];
 
@@ -683,6 +685,10 @@ function parseCommentsArgs(args) {
       commentsServer = args[++i];
     } else if (arg === "-m" || arg === "--max-comments") {
       commentsMax = parseInt(args[++i]) || 20;
+    } else if (arg === "--base-port") {
+      commentsBasePort = parseInt(args[++i]) || 9222;
+    } else if (arg === "--proxy") {
+      commentsProxy = args[++i];
     } else {
       positional.push(arg);
     }
@@ -701,6 +707,8 @@ function parseCommentsArgs(args) {
     commentsParallel,
     commentsInterval,
     commentsServer,
+    commentsBasePort,
+    commentsProxy,
     urls: [],
     outputFormat: "json",
     exploreCount: 0,
@@ -731,8 +739,11 @@ function parseTagArgs(args) {
   let isDiscover = false;
   let isScore = false;
   let isScoreAll = false;
+  let scoreAllPort = 9222;
+  let scoreProxy = null;
   let scoreTag = null;
   let scoreCountries = null;
+  let scorePort = 9222;
 
   for (let i = 0; i < args.length; i++) {
     const arg = args[i];
@@ -775,6 +786,14 @@ function parseTagArgs(args) {
         .split(",")
         .map((s) => s.trim().toUpperCase())
         .filter(Boolean);
+    } else if (arg === "--port") {
+      if (isScoreAll) {
+        scoreAllPort = parseInt(args[++i]) || 9222;
+      } else {
+        scorePort = parseInt(args[++i]) || 9222;
+      }
+    } else if (arg === "--proxy") {
+      scoreProxy = args[++i];
     } else if (arg === "-p" || arg === "--prompt") {
       discoverPrompt = args[++i];
     } else if (!arg.startsWith("-")) {
@@ -837,6 +856,8 @@ function parseTagArgs(args) {
         tag: scoreTag,
         countries: scoreCountries,
         serverUrl,
+        port: scorePort,
+        proxy: scoreProxy,
       },
       urls: [],
       outputFormat: "json",
@@ -858,6 +879,8 @@ function parseTagArgs(args) {
         countries: scoreCountries,
         serverUrl,
         autoDiscover,
+        port: scoreAllPort,
+        proxy: scoreProxy,
       },
       urls: [],
       outputFormat: "json",

package/src/lib/browser/cdp.js

@@ -208,7 +208,11 @@ export { killEdgeProcesses };
 
 export async function ensureBrowserReady(options = {}) {
   const port = options.port || DEFAULT_CDP_PORT;
-  const userDataDir = options.userDataDir || DEFAULT_USER_DATA_DIR;
+  const userDataDir =
+    options.userDataDir ||
+    (port !== DEFAULT_CDP_PORT
+      ? `${DEFAULT_USER_DATA_DIR}_p${port}`
+      : DEFAULT_USER_DATA_DIR);
   const proxyServer = options.proxyServer || null;
   const isCustom = port !== DEFAULT_CDP_PORT || !!options.userDataDir;
 

package/src/lib/constants.js

@@ -208,6 +208,21 @@ const HELP_TEXT = [
   "      POST /api/tiktok/lookup 同时获取视频和作者信息 { videoUrl: string }",
   "    示例: tt-help webserver -p 3000",
   "",
+  "  comments [选项]",
+  "    评论采集：从视频评论中发现新用户，来源标记为 comment",
+  "    自动模式：循环从服务端取视频任务，抓评论，提交新用户",
+  "    手动模式：指定视频URL，采集评论后保存",
+  "    选项:",
+  "      -p, --parallel       并行数（默认: 1）",
+  "      -i, --interval       空闲间隔上限秒（默认: 10，实际 1~N 随机）",
+  "      -s, --server         服务端地址（默认: http://127.0.0.1:3001）",
+  "      -m, --max-comments   每视频最大评论数（默认: 200）",
+  "      --base-port          浏览器CDP端口（默认: 9222）",
+  "      --save               去重后保存到服务端，来源标记为 comment",
+  "    示例: tt-help comments",
+  "          tt-help comments -p 2 -i 15 --base-port 9223",
+  "          tt-help comments <视频URL> 50 --save",
+  "",
   "  tag  <标签名> [...] [选项]",
   "    抓取标签页视频和作者（旧版 CLI 模式）",
   "    选项:",

package/src/lib/tag-fetcher.js

@@ -1,11 +1,9 @@
 import { chromium } from "playwright";
-import { detectBrowser } from "./browser/launch.js";
-import { getAntiDetectScript } from "./browser/anti-detect.js";
+import { ensureBrowserReady } from "./browser/cdp.js";
+import { getOrCreatePage } from "./browser/page.js";
 import { TikTokScraper } from "./tiktok-scraper.mjs";
 
 const TAG_URL = "https://www.tiktok.com/tag";
-const USER_AGENT =
-  "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36";
 const SCROLL_INTERVAL = 3000;
 const MAX_STALE_ROUNDS = 3;
 
@@ -13,33 +11,7 @@ function sleep(ms) {
   return new Promise((r) => setTimeout(r, ms));
 }
 
-function findBrowser() {
-  return detectBrowser();
-}
-
-async function launchBrowser(browserPath) {
-  const opts = {
-    headless: true,
-    args: [
-      "--no-sandbox",
-      "--disable-blink-features=AutomationControlled",
-      "--disable-dev-shm-usage",
-    ],
-  };
-  if (browserPath) opts.executablePath = browserPath;
-
-  try {
-    return await chromium.launch(opts);
-  } catch {
-    if (browserPath) {
-      opts.executablePath = undefined;
-      return await chromium.launch(opts);
-    }
-    throw new Error("无法启动浏览器");
-  }
-}
-
-async function extractItemData(item) {
+function extractItemData(item) {
   return {
     id: item.id || "",
     desc: (item.desc || "").trim(),
@@ -59,43 +31,35 @@ async function extractItemData(item) {
 }
 
 /**
- * 获取 TikTok 标签页下的所有视频和作者
+ * 获取 TikTok 标签页下的所有视频和作者（通过 CDP 连接已登录的 Edge 浏览器）
  * @param {string} tag - 标签名称（不含 # 号）
  * @param {object} [options]
  * @param {number} [options.timeout=300000] - 最大等待时间 (ms)
- * @param {string} [options.browserPath] - 浏览器可执行文件路径，不传则自动探测
- * @param {string} [options.locale='en-US'] - 页面语言
+ * @param {number} [options.port=9222] - CDP 端口
+ * @param {string} [options.userDataDir] - 用户数据目录
  * @param {Function} [options.onProgress] - 进度回调 ({ videos, authors })
  * @returns {Promise<{ tag: string, challengeId: string, totalPosts: number, videos: Array, uniqueAuthors: string[] }>}
  */
 export async function fetchTagData(tag, options = {}) {
   const {
     timeout = 300000,
-    browserPath: customBrowserPath,
-    locale = "en-US",
+    port = 9222,
+    userDataDir,
+    proxyServer,
     onProgress,
   } = options;
 
-  const browserPath = customBrowserPath || findBrowser();
-  if (!browserPath) {
-    throw new Error(
-      "未找到可用的浏览器，请设置 browserPath 或安装 Chrome/Edge",
-    );
-  }
+  const cdpOptions = { port };
+  if (userDataDir) cdpOptions.userDataDir = userDataDir;
+  if (proxyServer) cdpOptions.proxyServer = proxyServer;
 
-  const browser = await launchBrowser(browserPath);
+  const browser = await ensureBrowserReady(cdpOptions);
+  const page = await getOrCreatePage(browser);
 
   try {
-    const context = await browser.newContext({
-      viewport: { width: 1280, height: 900 },
-      userAgent: USER_AGENT,
-      locale,
-    });
-    await context.addInitScript(getAntiDetectScript());
-    const page = await context.newPage();
-
     let challengeInfo = null;
     const rawVideos = [];
+    const seenVideoIds = new Set();
     const authors = new Set();
 
     page.on("response", async (resp) => {
@@ -114,9 +78,13 @@ export async function fetchTagData(tag, options = {}) {
           const body = await resp.json();
           if (!body?.itemList) return;
           for (const item of body.itemList) {
-            const uid = item.author?.uniqueId || "";
-            if (uid) authors.add(uid);
-            rawVideos.push(await extractItemData(item));
+            const vid = item.id || "";
+            if (vid && !seenVideoIds.has(vid)) {
+              seenVideoIds.add(vid);
+              const uid = item.author?.uniqueId || "";
+              if (uid) authors.add(uid);
+              rawVideos.push(extractItemData(item));
+            }
           }
           if (onProgress) {
             onProgress({ videos: rawVideos.length, authors: authors.size });
@@ -137,11 +105,49 @@ export async function fetchTagData(tag, options = {}) {
 
     await page.waitForTimeout(3000);
 
+    // 检测页面异常：验证码、网络错误、标签不存在
     const pageError = await page.evaluate(() => {
       const text = document.body?.innerText || "";
-      if (text.includes("Something went wrong")) return "page_error";
+      if (
+        document.querySelector(
+          ".captcha-verify-container, #captcha_close_button",
+        )
+      )
+        return "captcha";
+      if (
+        text.includes("Verify you are a robot") ||
+        text.includes("Please rotate")
+      )
+        return "captcha";
+      if (
+        text.includes("找不到此话题标签") ||
+        text.includes("This hashtag is unavailable")
+      )
+        return "tag_not_found";
+      if (
+        text.includes("Something went wrong") ||
+        text.includes("Network error")
+      )
+        return "page_error";
       return null;
     });
+
+    if (pageError === "captcha") {
+      throw new Error("⚠️ 遇到验证码，请手动处理后重试");
+    }
+    if (pageError === "tag_not_found") {
+      // 标签不存在 → dead
+      return {
+        tag,
+        challengeId: "",
+        totalPosts: 0,
+        videoCount: 0,
+        uniqueAuthorCount: 0,
+        videos: [],
+        uniqueAuthors: [],
+        error: "tag_not_found",
+      };
+    }
     if (pageError) {
       throw new Error("标签页加载失败，TikTok 返回了错误页面");
     }
@@ -156,18 +162,16 @@ export async function fetchTagData(tag, options = {}) {
       await page.evaluate(() => window.scrollBy(0, 3000));
       await sleep(SCROLL_INTERVAL);
 
-      if (rawVideos.length === lastCount) {
+      if (seenVideoIds.size === lastCount) {
         staleRounds++;
       } else {
         staleRounds = 0;
-        lastCount = rawVideos.length;
+        lastCount = seenVideoIds.size;
       }
     }
 
-    const seen = new Set();
-    const uniqueVideos = rawVideos.filter((v) =>
-      seen.has(v.id) ? false : (seen.add(v.id), true),
-    );
+    // 已在采集时去重，rawVideos 即为唯一视频列表
+    const uniqueVideos = rawVideos;
 
     const totalPosts = challengeInfo?.stats?.videoCount || 0;
 
@@ -181,7 +185,7 @@ export async function fetchTagData(tag, options = {}) {
       uniqueAuthors: [...authors],
     };
   } finally {
-    await browser.close();
+    // 不关闭 page 和 browser，由用户自行关闭
   }
 }
 
@@ -200,11 +204,13 @@ export async function enrichVideosWithLocation(videos, options = {}) {
     mode = "videos",
     poolSize = 3,
     maxRetries = 3,
+    proxyServer,
     onProgress,
     existingScraper,
   } = options;
 
-  const scraper = existingScraper || new TikTokScraper({ poolSize });
+  const scraper =
+    existingScraper || new TikTokScraper({ poolSize, proxyServer });
   const ownsScraper = !existingScraper;
   if (ownsScraper) await scraper.init();