tt-help-cli-ycl 1.3.0 → 1.3.1

package/src/scraper/{core.cjs → core.mjs}

@@ -1,4 +1,4 @@
-const {
+import {
   closeCommentPanel,
   delay,
   ensureBrowserReady,
@@ -6,101 +6,78 @@ const {
   setDelayConfig,
   getDelayConfig,
   retryWithBackoff,
-} = require("./modules/page-helpers.cjs");
-const { extractCommentAuthors } = require("./modules/comment-extractor.cjs");
-const { extractGuessVideos } = require("./modules/guess-extractor.cjs");
+} from './modules/page-helpers.mjs';
+import { extractCommentAuthors } from './modules/comment-extractor.mjs';
+import { extractGuessVideos } from './modules/guess-extractor.mjs';
 
 async function scrapeSingleVideo(page, maxComments, maxGuess, log, location = 'ES') {
   const config = getDelayConfig();
 
-  await page
-    .waitForSelector('[class*="VideoMeta"]', { timeout: 10000 })
-    .catch(() => {});
+  await page.waitForSelector('[class*="VideoMeta"]', { timeout: 10000 }).catch(() => {});
   await delay(Math.round(config.commentMax * 0.3), config.commentMax);
 
   const userData = await page.evaluate(() => {
     const result = {};
-
-    const m = window.location.href.match(/\/@([^\/]+)\/video/);
+    const m = window.location.href.match(/\/@([^/]+)\/video/);
     if (m) result.uniqueId = m[1];
-
     const authorEls = document.querySelectorAll('[class*="Author"]');
     for (const el of authorEls) {
-      const text = (el.textContent || "").trim();
-      if (text && !text.includes("TikTok") && !text.includes("Share")) {
+      const text = (el.textContent || '').trim();
+      if (text && !text.includes('TikTok') && !text.includes('Share')) {
         result.nickname = text;
         break;
       }
     }
-
     const html = document.documentElement.outerHTML;
     const locMatch = html.match(/"locationCreated":"([^"]*)/);
     if (locMatch) result.locationCreated = locMatch[1];
-
     return result;
   });
 
-  const videoAuthor = userData.uniqueId ? "@" + userData.uniqueId : null;
-  if (!videoAuthor) {
-    throw new Error("无法获取视频作者");
-  }
+  const videoAuthor = userData.uniqueId ? '@' + userData.uniqueId : null;
+  if (!videoAuthor) throw new Error('无法获取视频作者');
 
   let guessVideos = [];
   let commentUsers = [];
 
   if (userData.locationCreated === location) {
     guessVideos = await extractGuessVideos(page, maxGuess);
-
     commentUsers = await extractCommentAuthors(page, maxComments);
     await closeCommentPanel(page);
     await delay(Math.round(config.commentMax * 0.3), config.commentMax);
   }
 
-  const uniqueUsers = [...new Set(commentUsers)];
-
   return {
     videoAuthor,
     uniqueId: userData.uniqueId,
     nickname: userData.nickname,
     locationCreated: userData.locationCreated,
-    commentUsers: uniqueUsers,
+    commentUsers: [...new Set(commentUsers)],
     guessVideos,
   };
 }
 
 async function runScrape(options) {
   const {
-    videoUrl,
-    maxVideos = 20,
-    maxComments = 999,
-    maxGuess = 10,
-    preset = null,
-    switchMax = null,
-    commentMax = null,
+    videoUrl, maxVideos = 20, maxComments = 999, maxGuess = 10,
+    preset = null, switchMax = null, commentMax = null,
     log = console.error,
-    browser: externalBrowser = null,
-    page: externalPage = null,
+    browser: externalBrowser = null, page: externalPage = null,
   } = options;
 
   if (preset) {
     setDelayConfig(preset);
   } else if (switchMax || commentMax) {
-    setDelayConfig({
-      switchMax: switchMax || 5000,
-      commentMax: commentMax || 3000,
-    });
+    setDelayConfig({ switchMax: switchMax || 5000, commentMax: commentMax || 3000 });
   }
 
   const config = getDelayConfig();
-
   let browser, page;
   const isExternal = !!(externalBrowser && externalPage);
 
   if (!isExternal) {
     log(`视频地址: ${videoUrl}`);
-    log(
-      `视频数: ${maxVideos}, 评论数: ${maxComments}, 猜你喜欢: ${maxGuess}, 切换延迟: ${config.switchMax}ms, 评论延迟: ${config.commentMax}ms`,
-    );
+    log(`视频数: ${maxVideos}, 评论数: ${maxComments}, 猜你喜欢: ${maxGuess}, 切换延迟: ${config.switchMax}ms, 评论延迟: ${config.commentMax}ms`);
   }
 
   if (isExternal) {
@@ -116,7 +93,7 @@ async function runScrape(options) {
     }
   }
 
-  await retryWithBackoff(() => page.goto(videoUrl, { waitUntil: "load", timeout: 30000 }), { log });
+  await retryWithBackoff(() => page.goto(videoUrl, { waitUntil: 'load', timeout: 30000 }), { log });
   await delay(Math.round(config.switchMax * 0.5), config.switchMax);
   await closeCommentPanel(page);
   await delay(Math.round(config.commentMax * 0.5), config.commentMax);
@@ -138,9 +115,7 @@ async function runScrape(options) {
       log(`[${i + 1}/${maxVideos}] 跳过: ${e.message}`);
       if (i < maxVideos - 1) {
         await page.evaluate(() => {
-          const container = document.querySelector(
-            '[class*="ColumnListContainer"]',
-          );
+          const container = document.querySelector('[class*="ColumnListContainer"]');
           if (container) container.scrollTop += 700;
           else window.scrollBy(0, 700);
         });
@@ -148,37 +123,30 @@ async function runScrape(options) {
       }
       continue;
     }
+
     allResults.push(result);
     videoAuthors.add(result.videoAuthor);
-    result.commentUsers.forEach((u) => commentUsers.add(u));
+    result.commentUsers.forEach(u => commentUsers.add(u));
     allCommentAuthorsList.push(...result.commentUsers);
     if (result.guessVideos) {
       allGuessVideos.push(...result.guessVideos);
-      result.guessVideos.forEach((v) => {
-        if (v.author) allGuessAuthors.add(v.author);
-      });
+      result.guessVideos.forEach(v => { if (v.author) allGuessAuthors.add(v.author); });
     }
 
     if ((i + 1) % 5 === 0 || i === 0) {
-      log(
-        `[${i + 1}/${maxVideos}] ${result.videoAuthor} | 昵称: ${result.nickname || "-"} | 评论用户: ${result.commentUsers.length} | 猜你喜欢: ${result.guessVideos ? result.guessVideos.length : 0}`,
-      );
+      log(`[${i + 1}/${maxVideos}] ${result.videoAuthor} | 昵称: ${result.nickname || '-'} | 评论用户: ${result.commentUsers.length} | 猜你喜欢: ${result.guessVideos ? result.guessVideos.length : 0}`);
     }
 
     if (i < maxVideos - 1) {
       await page.evaluate(() => {
-        const container = document.querySelector(
-          '[class*="ColumnListContainer"]',
-        );
+        const container = document.querySelector('[class*="ColumnListContainer"]');
         if (container) container.scrollTop += 700;
       });
       await delay(2000, config.switchMax);
     }
   }
 
-  log(
-    `\n结果: 视频作者 ${videoAuthors.size} | 评论用户 ${commentUsers.size} | 总评论 ${allCommentAuthorsList.length} | 猜你喜欢作者 ${allGuessAuthors.size} | 总猜中视频 ${allGuessVideos.length}`,
-  );
+  log(`\n结果: 视频作者 ${videoAuthors.size} | 评论用户 ${commentUsers.size} | 总评论 ${allCommentAuthorsList.length} | 猜你喜欢作者 ${allGuessAuthors.size} | 总猜中视频 ${allGuessVideos.length}`);
 
   const videoDetails = {};
   for (const r of allResults) {
@@ -211,4 +179,4 @@ async function runScrape(options) {
   return { output, browser, isExternal };
 }
 
-module.exports = { scrapeSingleVideo, runScrape };
+export { scrapeSingleVideo, runScrape };

package/src/scraper/modules/{comment-extractor.cjs → comment-extractor.mjs}

@@ -1,13 +1,22 @@
-const { delay, getDelayConfig, closeCommentPanel } = require('./page-helpers.cjs');
-const { scrollAndCollect } = require('./scroll-collector.cjs');
+import { delay, getDelayConfig, closeCommentPanel } from "./page-helpers.mjs";
+import { scrollAndCollect } from "./scroll-collector.mjs";
 
 async function openCommentPanel(page) {
   const tabs = page.locator('[class*="tabbar-item"]');
-  const commentTab = tabs.filter({ hasText: '评论' }).first();
+  const commentTab = tabs.filter({ hasText: "评论" }).first();
   await commentTab.click();
-  const config = getDelayConfig();
-  await delay(Math.round(config.commentMax * 0.5), config.commentMax);
-  await page.waitForSelector('[class*="CommentListContainer"]', { timeout: 5000 }).catch(() => {});
+  await page
+    .waitForSelector('[class*="CommentListContainer"]', { timeout: 5000 })
+    .catch(() => {});
+  await page
+    .waitForFunction(
+      () => {
+        const list = document.querySelector('[class*="CommentListContainer"]');
+        return list && list.children.length > 0;
+      },
+      { timeout: 10000 },
+    )
+    .catch(() => {});
 }
 
 async function extractCommentAuthors(page, maxComments = 10) {
@@ -20,14 +29,15 @@ async function extractCommentAuthors(page, maxComments = 10) {
     collectFn: (container) => {
       const list = document.querySelector('[class*="CommentListContainer"]');
       if (!list) return { items: [] };
-
       const authors = [];
-      Array.from(list.children).forEach(wrapper => {
-        const link = wrapper.querySelector('[class*="UsernameContentWrapper"] a');
+      Array.from(list.children).forEach((wrapper) => {
+        const link = wrapper.querySelector(
+          '[class*="UsernameContentWrapper"] a',
+        );
         if (link) {
-          const href = link.href || link.getAttribute('href');
+          const href = link.href || link.getAttribute("href");
           const m = href && href.match(/@([^/]+)/);
-          if (m) authors.push('@' + m[1]);
+          if (m) authors.push("@" + m[1]);
         }
       });
       return { items: authors };
@@ -35,7 +45,7 @@ async function extractCommentAuthors(page, maxComments = 10) {
     uniqueKey: (a) => a,
     maxItems: maxComments,
     delayRange: [Math.round(config.commentMax * 0.3), config.commentMax],
-    staleThreshold: 3,
+    staleThreshold: 2,
   });
 
   await closeCommentPanel(page);
@@ -44,6 +54,4 @@ async function extractCommentAuthors(page, maxComments = 10) {
   return allAuthors.slice(0, maxComments);
 }
 
-module.exports = {
-  extractCommentAuthors,
-};
+export { extractCommentAuthors };

package/src/scraper/modules/follow-extractor.mjs

@@ -0,0 +1,121 @@
+import { delay, getDelayConfig } from "./page-helpers.mjs";
+import { scrollAndCollect } from "./scroll-collector.mjs";
+
+const FILTER_WORDS = ["主页", "已关注", "粉丝", "推荐"];
+
+async function waitForListContent(page, minChildren = 1, timeout = 15000) {
+  await page
+    .waitForFunction(
+      (min) => {
+        const container = document.querySelector(
+          "[class*=DivUserListContainer]",
+        );
+        return container && container.children.length >= min;
+      },
+      minChildren,
+      { timeout },
+    )
+    .catch(() => {});
+}
+
+async function openFollowModal(page) {
+  const el = await page.$("[data-e2e=following]");
+  if (!el) {
+    throw new Error(
+      "未找到 [data-e2e=following] 元素，请确认当前页面为用户主页",
+    );
+  }
+  await el.evaluate((el) => el.parentElement.click());
+  await page
+    .waitForSelector("[class*=DivUserListContainer]", { timeout: 5000 })
+    .catch(() => {
+      throw new Error("关注弹窗未出现 DivUserListContainer");
+    });
+  await waitForListContent(page, 1, 3000);
+}
+
+async function switchToFollowersTab(page) {
+  await page.evaluate(() => {
+    const tabs = document.querySelectorAll("[class*=DivTabItem]");
+    for (const tab of tabs) {
+      if (tab.textContent?.includes("粉丝")) {
+        tab.click();
+        return;
+      }
+    }
+    throw new Error("未找到粉丝 Tab");
+  });
+  await waitForListContent(page, 1, 3000);
+}
+
+async function closeFollowModal(page) {
+  await page.evaluate(() => {
+    const closeBtn = document.querySelector("[data-e2e=follow-popup-close]");
+    if (closeBtn) closeBtn.click();
+  });
+  await page.waitForTimeout(500);
+}
+
+function createUserCollectFn() {
+  return (container) => {
+    const FILTER_WORDS = ["主页", "已关注", "粉丝", "推荐"];
+    const modal = document.querySelector("[class*=eyhy6180]");
+    const root = modal || document;
+    const users = [];
+    const seen = new Set();
+    const links = root.querySelectorAll('a[href*="/@"]');
+    for (const link of links) {
+      const match = link.href.match(/@([^/?]+)/);
+      if (!match) continue;
+      const handle = "@" + decodeURIComponent(match[1]);
+      const text = (link.textContent || "").trim();
+      if (text.length <= 2) continue;
+      if (FILTER_WORDS.includes(text)) continue;
+      if (seen.has(handle)) continue;
+      seen.add(handle);
+      users.push({ handle, displayName: text });
+    }
+    return { items: users };
+  };
+}
+
+async function extractUsersFromModal(page, maxUsers) {
+  const config = getDelayConfig();
+  const minDelay = Math.max(300, Math.round(config.commentMax * 0.3));
+  const maxDelay = Math.max(800, config.commentMax);
+
+  const allUsers = await scrollAndCollect(page, {
+    container: "[class*=DivUserListContainer]",
+    findScrollable: false,
+    collectFn: createUserCollectFn(),
+    uniqueKey: (u) => u.handle,
+    maxItems: maxUsers,
+    delayRange: [minDelay, maxDelay],
+    staleThreshold: 2,
+  });
+
+  return allUsers.slice(0, maxUsers);
+}
+
+async function extractFollowAndFollowers(page, options = {}) {
+  const { maxFollowing = 999, maxFollowers = 999, log = () => {} } = options;
+
+  await openFollowModal(page);
+
+  const following = await extractUsersFromModal(page, maxFollowing);
+  log(`  已关注: ${following.length}`);
+
+  await switchToFollowersTab(page);
+
+  const followers = await extractUsersFromModal(page, maxFollowers);
+  log(`  粉丝: ${followers.length}`);
+
+  await closeFollowModal(page);
+
+  return {
+    following: following.map((u) => [u.handle, u.displayName]),
+    followers: followers.map((u) => [u.handle, u.displayName]),
+  };
+}
+
+export { extractFollowAndFollowers };

package/src/scraper/modules/{guess-extractor.cjs → guess-extractor.mjs}

@@ -1,5 +1,5 @@
-const { delay, getDelayConfig, closeCommentPanel } = require('./page-helpers.cjs');
-const { scrollAndCollect } = require('./scroll-collector.cjs');
+import { delay, getDelayConfig, closeCommentPanel } from './page-helpers.mjs';
+import { scrollAndCollect } from './scroll-collector.mjs';
 
 async function openGuessTab(page) {
   const tabs = page.locator('[class*="tabbar-item"]');
@@ -48,6 +48,4 @@ async function extractGuessVideos(page, maxVideos = 10) {
   return allVideos.slice(0, maxVideos);
 }
 
-module.exports = {
-  extractGuessVideos,
-};
+export { extractGuessVideos };

package/src/scraper/modules/page-error-detector.mjs

@@ -0,0 +1,68 @@
+const PATTERNS = {
+  login_required: [
+    "登录 TikTok",
+    "登录后查看",
+    "查看需登录",
+    "Log in to TikTok",
+    "Login to TikTok",
+    "观众管理功能",
+    "Viewer management",
+    "私密账号",
+    "私密状态",
+  ],
+  captcha: [
+    "captcha",
+    "verify",
+    "验证码",
+    "点击下一步",
+    "Press and hold",
+    "slide to verify",
+    "滑动验证",
+    "人机验证",
+    "安全验证",
+  ],
+  rate_limited: [
+    "访问过于频繁",
+    "操作过于频繁",
+    "too many requests",
+    "rate limit",
+    "稍后再试",
+    "try again later",
+    "请稍后再来",
+  ],
+  region_blocked: [
+    "地区限制",
+    "not available in your",
+    "此内容不可用",
+    "content not available",
+    "currently unavailable",
+    "抱歉，此内容",
+    "此页面不可用",
+  ],
+  not_found: [
+    "页面不存在",
+    "page not found",
+    "找不到",
+    "Couldn't find this",
+    "nothing here",
+    "此页面不存在",
+    "没有内容",
+  ],
+};
+
+export async function detectPageError(page) {
+  return page.evaluate((patterns) => {
+    const bodyText = document.body.innerText;
+    const lower = bodyText.toLowerCase();
+
+    for (const [type, phrases] of Object.entries(patterns)) {
+      for (const phrase of phrases) {
+        if (lower.includes(phrase.toLowerCase())) {
+          return type;
+        }
+      }
+    }
+
+    return null;
+  }, PATTERNS);
+}

package/src/scraper/modules/page-helpers.mjs

@@ -0,0 +1,44 @@
+import {
+  delay,
+  getDelayConfig,
+  setDelayConfig,
+  listDelayPresets,
+  DELAY_PRESETS,
+} from '../../lib/delay.js';
+import { ensureBrowserReady } from '../../lib/browser/cdp.js';
+import {
+  ensureTikTokPage,
+  closeCommentPanel,
+  findTikTokPage,
+  getOrCreatePage,
+} from '../../lib/browser/page.js';
+import { retryWithBackoff, isRetryableError } from '../../lib/retry.js';
+import {
+  extractUserSection,
+  parseUserSection,
+  extractLocationCreated,
+  USER_SECTION_SIZE,
+} from '../../lib/parser.js';
+import { detectPageError } from './page-error-detector.mjs';
+
+export {
+  delay,
+  setDelayConfig,
+  getDelayConfig,
+  listDelayPresets,
+  DELAY_PRESETS,
+  ensureBrowserReady,
+  ensureTikTokPage,
+  closeCommentPanel,
+  findTikTokPage,
+  getOrCreatePage,
+  retryWithBackoff,
+  isRetryableError,
+  extractUserSection,
+  parseUserSection,
+  extractLocationCreated,
+  USER_SECTION_SIZE,
+  detectPageError,
+};
+
+export const CDP_PORT = 9222;

package/src/scraper/modules/scroll-collector.mjs

@@ -0,0 +1,189 @@
+import { delay } from "../../lib/delay.js";
+import { detectPageError } from "./page-error-detector.mjs";
+
+async function doCollect(
+  page,
+  { container, findScrollable, fnStr, extraArgs },
+) {
+  return page.evaluate(
+    ({ fn: fnStr, containerSelector, findScrollableFlag, args }) => {
+      let el;
+      if (!containerSelector) {
+        el = window;
+      } else {
+        el = document.querySelector(containerSelector);
+        if (!el) {
+          el = window;
+        } else if (findScrollableFlag) {
+          let current = el;
+          let found = false;
+          while (current && current !== document.body) {
+            if (current.scrollHeight > current.clientHeight + 10) {
+              el = current;
+              found = true;
+              break;
+            }
+            current = current.parentElement;
+          }
+          if (!found) el = document.body;
+        }
+      }
+      const fn = eval("(" + fnStr + ")");
+      return fn(el, args);
+    },
+    {
+      fn: fnStr,
+      containerSelector: container,
+      findScrollableFlag: findScrollable,
+      args: extraArgs,
+    },
+  );
+}
+
+const LOADING_SELECTORS = [
+  '[class*="loading"]',
+  '[class*="Loading"]',
+  '[class*="spinner"]',
+  '[class*="Spinner"]',
+  '[class*="skeleton"]',
+  '[class*="Skeleton"]',
+  '[aria-busy="true"]',
+];
+
+async function waitForLoading(page) {
+  const maxWait = 5000;
+  const startTime = Date.now();
+  while (Date.now() - startTime < maxWait) {
+    const isLoading = await page.evaluate((sels) => {
+      if (document.readyState !== "complete") return true;
+      for (const sel of sels) {
+        const el = document.querySelector(sel);
+        if (el && el.offsetParent !== null) return true;
+      }
+      return false;
+    }, LOADING_SELECTORS);
+    if (!isLoading) return;
+    await delay(300, 600);
+  }
+}
+
+export async function scrollAndCollect(page, options) {
+  const {
+    container,
+    findScrollable = false,
+    collectFn,
+    extraArgs,
+    delayRange = [800, 1500],
+    maxItems,
+    maxRounds = 200,
+    staleThreshold = 3,
+    uniqueKey,
+    onRound,
+  } = options;
+
+  if (!collectFn) throw new Error("collectFn is required");
+
+  const fnStr =
+    typeof collectFn === "function" ? collectFn.toString() : collectFn;
+  const allItems = [];
+  const seenKeys = uniqueKey ? new Set() : null;
+  let staleCount = 0;
+
+  const processItems = (result) => {
+    const raw = result.items || [];
+    const newItems = uniqueKey
+      ? raw.filter((item) => {
+          const key = uniqueKey(item);
+          if (seenKeys.has(key)) return false;
+          seenKeys.add(key);
+          return true;
+        })
+      : raw;
+    allItems.push(...newItems);
+    return newItems;
+  };
+
+  const isDone = (newItems) => {
+    if (maxItems !== undefined && allItems.length >= maxItems) return true;
+    if (newItems.length === 0) {
+      staleCount++;
+      if (staleCount >= staleThreshold) return true;
+    } else {
+      staleCount = 0;
+    }
+    return false;
+  };
+
+  const collectCtx = { container, findScrollable, fnStr, extraArgs };
+
+  const pageError = await detectPageError(page);
+
+  if (pageError) return [];
+
+  await waitForLoading(page);
+  let result = await doCollect(page, collectCtx);
+  let newItems = processItems(result);
+  if (onRound) onRound(0, newItems, allItems);
+  if (isDone(newItems)) return allItems;
+
+  for (let round = 1; round < maxRounds; round++) {
+    await threePhaseScroll(page, { container, findScrollable });
+    await delay(delayRange[0], delayRange[1]);
+    await waitForLoading(page);
+
+    result = await doCollect(page, collectCtx);
+    newItems = processItems(result);
+
+    if (onRound) onRound(round, newItems, allItems);
+
+    if (isDone(newItems)) break;
+  }
+
+  return allItems;
+}
+
+async function threePhaseScroll(page, { container, findScrollable }) {
+  await page.evaluate(
+    async (opts) => {
+      let el;
+      if (!opts.container) {
+        el = window;
+      } else {
+        el = document.querySelector(opts.container);
+        if (!el) {
+          el = window;
+        } else if (opts.findScrollable) {
+          let current = el;
+          let found = false;
+          while (current && current !== document.body) {
+            if (current.scrollHeight > current.clientHeight + 10) {
+              el = current;
+              found = true;
+              break;
+            }
+            current = current.parentElement;
+          }
+          if (!found) el = document.body;
+        }
+      }
+
+      const randDelay = (min, max) =>
+        new Promise((r) => setTimeout(r, min + Math.random() * (max - min)));
+
+      if (el === window) {
+        window.scrollBy(0, window.innerHeight);
+        await randDelay(400, 800);
+        window.scrollBy(0, -200);
+        await randDelay(200, 400);
+        window.scrollBy(0, window.innerHeight);
+      } else {
+        el.scrollTop = el.scrollHeight;
+        await randDelay(400, 800);
+        el.scrollTop -= 100 + Math.random() * 100;
+        await randDelay(200, 400);
+        el.scrollTop = el.scrollHeight;
+      }
+    },
+    { container, findScrollable },
+  );
+}