tt-help-cli-ycl 1.3.0 → 1.3.2

package/src/{explore-core.cjs → scraper/explore-core.mjs}

@@ -1,27 +1,29 @@
-const {
+import {
   delay,
   ensureBrowserReady,
   setDelayConfig,
   closeCommentPanel,
   retryWithBackoff,
-} = require('./scraper/modules/page-helpers.cjs');
-const {
+  detectPageError,
+  isLoggedIn,
+} from './modules/page-helpers.mjs';
+export { ensureBrowserReady };
+import {
   getUserInfo,
   collectVideos,
-  isPageRestricted,
-} = require('./get-user-videos-core.cjs');
-const { scrapeSingleVideo } = require('./scraper/core.cjs');
-const { extractFollowAndFollowers } = require('./scraper/modules/follow-extractor.cjs');
-const { extractCommentAuthors } = require('./scraper/modules/comment-extractor.cjs');
-const { extractGuessVideos } = require('./scraper/modules/guess-extractor.cjs');
+} from '../videos/core.mjs';
+import { scrapeSingleVideo } from './core.mjs';
+import { extractFollowAndFollowers } from './modules/follow-extractor.mjs';
+import { extractCommentAuthors } from './modules/comment-extractor.mjs';
+import { extractGuessVideos } from './modules/guess-extractor.mjs';
 
 async function processExplore(page, username, options, log) {
   const {
     maxComments = 0,
     maxGuess = 0,
     enableFollow = true,
-    maxFollowing = 200,
-    maxFollowers = 200,
+    maxFollowing = 5,
+    maxFollowers = 5,
     location = 'ES',
   } = options;
 
@@ -48,35 +50,13 @@ async function processExplore(page, username, options, log) {
     await page.waitForSelector('[class*="DivVideoList"]', { timeout: 10000 }).catch(() => {});
     await delay(1000, 2000);
 
-    // 1. 获取用户信息
-    log(`  获取用户信息...`);
+    log('  获取用户信息...');
     const info = await getUserInfo(page);
     if (info) {
       result.userInfo = info;
       log(`  用户: ${info.nickname || username} | 粉丝: ${info.followerCount || '-'} | 视频: ${info.videoCount || '-'}`);
     }
 
-    // 2. 获取关注+粉丝（在滚动前执行，避免按钮被滚出视口）
-    if (enableFollow) {
-      try {
-        log(`  获取关注/粉丝...`);
-        const { following, followers } = await extractFollowAndFollowers(
-          page,
-          { maxFollowing, maxFollowers, log }
-        );
-        result.discoveredFollowing = following || [];
-        result.discoveredFollowers = followers || [];
-        result.hasFollowData = true;
-        log(`  关注: ${result.discoveredFollowing.length}, 粉丝: ${result.discoveredFollowers.length}`);
-      } catch (e) {
-        log(`  关注/粉丝提取失败: ${e.message}`);
-        result.hasFollowData = false;
-        result.discoveredFollowing = [];
-        result.discoveredFollowers = [];
-      }
-    }
-
-    // 3. 获取视频列表
     const videoList = await collectVideos(page, username, 1, log);
     const videoArray = videoList ? [...videoList.values()] : [];
     result.collectedVideos = videoArray.length;
@@ -84,17 +64,42 @@ async function processExplore(page, username, options, log) {
     if (videoArray.length <= 0) {
       result.processed = true;
       result.noVideo = true;
-      const restricted = await isPageRestricted(page);
-      if (restricted) {
+      const pageError = await detectPageError(page);
+      if (pageError) {
         result.restricted = true;
-        log(`  @${username} 页面受限（需登录），标记跳过`);
+        log(`  @${username} 页面受限（${pageError}），标记跳过`);
       } else {
         log(`  @${username} 没有视频，标记已处理`);
       }
       return result;
     }
 
-    // 4. 进入第一个视频
+    if (enableFollow) {
+      const loggedIn = await isLoggedIn(page);
+      if (!loggedIn) {
+        log('  [跳过] 获取关注/粉丝：未登录，请先登录 TikTok');
+        result.hasFollowData = false;
+        result.discoveredFollowing = [];
+        result.discoveredFollowers = [];
+      } else {
+        try {
+          log('  获取关注/粉丝...');
+          const { following, followers } = await extractFollowAndFollowers(
+            page, { maxFollowing, maxFollowers, log }
+          );
+          result.discoveredFollowing = following || [];
+          result.discoveredFollowers = followers || [];
+          result.hasFollowData = true;
+          log(`  关注: ${result.discoveredFollowing.length}, 粉丝: ${result.discoveredFollowers.length}`);
+        } catch (e) {
+          log(`  关注/粉丝提取失败: ${e.message}`);
+          result.hasFollowData = false;
+          result.discoveredFollowing = [];
+          result.discoveredFollowers = [];
+        }
+      }
+    }
+
     const firstVideo = videoArray[0];
     const videoUrl = firstVideo.href.startsWith('http')
       ? firstVideo.href
@@ -104,12 +109,10 @@ async function processExplore(page, username, options, log) {
     await retryWithBackoff(() => page.goto(videoUrl, { waitUntil: 'domcontentloaded', timeout: 30000 }), { log });
     await delay(1500, 2500);
 
-    // 5. 获取视频信息（含 locationCreated）
     const videoData = await scrapeSingleVideo(page, 0, 0, log, 'NEVER_MATCH');
     result.locationCreated = videoData.locationCreated || null;
     log(`  视频作者: ${videoData.videoAuthor} | 国家: ${result.locationCreated || '未知'}`);
 
-    // 6. 判断是否为目标国家
     const isTargetLocation = result.locationCreated === location;
 
     if (isTargetLocation) {
@@ -145,7 +148,6 @@ async function processExplore(page, username, options, log) {
     }
 
     result.processed = true;
-
   } catch (e) {
     result.error = e.message;
     log(`  [错误] ${e.message}`);
@@ -154,4 +156,4 @@ async function processExplore(page, username, options, log) {
   return result;
 }
 
-module.exports = { processExplore };
+export { processExplore };

package/src/scraper/modules/captcha-handler.mjs

@@ -0,0 +1,114 @@
+export async function detectCaptcha(page) {
+  return page.evaluate(() => {
+    const container = document.querySelector('.captcha-verify-container');
+    if (!container) return null;
+
+    const r = container.getBoundingClientRect();
+    return {
+      exists: true,
+      visible: container.offsetParent !== null,
+      rect: {
+        x: Math.round(r.x),
+        y: Math.round(r.y),
+        w: Math.round(r.width),
+        h: Math.round(r.height),
+      },
+    };
+  });
+}
+
+export async function closeCaptcha(page) {
+  return page.evaluate(() => {
+    const closeBtn = document.getElementById('captcha_close_button');
+    if (!closeBtn) return { success: false, reason: 'close button not found' };
+
+    closeBtn.dispatchEvent(new MouseEvent('mousedown', { bubbles: true }));
+    closeBtn.dispatchEvent(new MouseEvent('mouseup', { bubbles: true }));
+    closeBtn.dispatchEvent(new MouseEvent('click', { bubbles: true }));
+
+    return { success: true };
+  });
+}
+
+export async function handleCaptcha(page, options = {}) {
+  const { waitMs = 2000 } = options;
+
+  const captcha = await detectCaptcha(page);
+  if (!captcha) return { detected: false, closed: false };
+
+  await new Promise(r => setTimeout(r, waitMs));
+
+  const result = await closeCaptcha(page);
+  if (!result.success) return { detected: true, closed: false, reason: result.reason };
+
+  await new Promise(r => setTimeout(r, 1000));
+
+  const stillThere = await detectCaptcha(page);
+  return { detected: true, closed: !stillThere };
+}
+
+export async function getIncognitoPage(browser, url, options = {}) {
+  const { waitMs = 3000 } = options;
+  const context = await browser.newContext();
+  const page = await context.newPage();
+  await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 30000 });
+  await new Promise(r => setTimeout(r, waitMs));
+  return { page, context };
+}
+
+export async function waitAndGetCaptcha(page, options = {}) {
+  const { waitMs = 180000, pollInterval = 5000, log } = options;
+
+  const captcha = await detectCaptcha(page);
+  if (!captcha) return { detected: false, resolved: false, waited: 0 };
+
+  if (log) log('  检测到验证码，等待用户手动输入...');
+
+  const startTime = Date.now();
+  const deadline = startTime + waitMs;
+
+  while (Date.now() < deadline) {
+    await new Promise(r => setTimeout(r, pollInterval));
+    const remaining = await detectCaptcha(page);
+    if (!remaining) {
+      const waited = Math.round((Date.now() - startTime) / 1000);
+      if (log) log(`  验证码已解决（等待 ${waited}s）`);
+      return { detected: true, resolved: true, waited };
+    }
+  }
+
+  const waited = Math.round(waitMs / 1000);
+  if (log) log(`  验证码等待超时（${waited}s），继续执行`);
+
+  // 超时后尝试关闭验证码弹窗
+  await closeCaptcha(page);
+  await new Promise(r => setTimeout(r, 1000));
+
+  return { detected: true, resolved: false, waited };
+}
+
+export async function safeClickComment(page, options = {}) {
+  const { waitMs = 3000 } = options;
+
+  // 点击评论
+  await page.evaluate(() => {
+    const all = document.querySelectorAll('button');
+    for (const el of all) {
+      if (/^评论$/.test(el.textContent?.trim()) && el.offsetParent !== null && el.getBoundingClientRect().width > 0) {
+        el.click();
+        break;
+      }
+    }
+  });
+
+  await new Promise(r => setTimeout(r, waitMs));
+
+  // 检测并关闭验证码
+  const captcha = await detectCaptcha(page);
+  if (captcha) {
+    const result = await handleCaptcha(page);
+    return { clicked: true, captchaDetected: true, captchaClosed: result.closed };
+  }
+
+  return { clicked: true, captchaDetected: false, captchaClosed: false };
+}

package/src/scraper/modules/comment-extractor.mjs

@@ -0,0 +1,69 @@
+import { delay, getDelayConfig, closeCommentPanel } from "./page-helpers.mjs";
+import { scrollAndCollect } from "./scroll-collector.mjs";
+import { waitAndGetCaptcha } from "./captcha-handler.mjs";
+
+async function openCommentPanel(page) {
+  const tabs = page.locator('[class*="tabbar-item"]');
+  const commentTab = tabs.filter({ hasText: "评论" }).first();
+  await commentTab.click();
+
+  // 等待短暂时间让页面渲染
+  await new Promise(r => setTimeout(r, 2000));
+
+  // 检测验证码
+  await waitAndGetCaptcha(page, {
+    waitMs: 180000,
+    pollInterval: 5000,
+    log: console.error,
+  });
+
+  await page
+    .waitForSelector('[class*="CommentListContainer"]', { timeout: 5000 })
+    .catch(() => {});
+  await page
+    .waitForFunction(
+      () => {
+        const list = document.querySelector('[class*="CommentListContainer"]');
+        return list && list.children.length > 0;
+      },
+      { timeout: 10000 },
+    )
+    .catch(() => {});
+}
+
+async function extractCommentAuthors(page, maxComments = 10) {
+  await openCommentPanel(page);
+
+  const config = getDelayConfig();
+  const allAuthors = await scrollAndCollect(page, {
+    container: '[class*="CommentMain"]',
+    findScrollable: true,
+    collectFn: (container) => {
+      const list = document.querySelector('[class*="CommentListContainer"]');
+      if (!list) return { items: [] };
+      const authors = [];
+      Array.from(list.children).forEach((wrapper) => {
+        const link = wrapper.querySelector(
+          '[class*="UsernameContentWrapper"] a',
+        );
+        if (link) {
+          const href = link.href || link.getAttribute("href");
+          const m = href && href.match(/@([^/]+)/);
+          if (m) authors.push("@" + m[1]);
+        }
+      });
+      return { items: authors };
+    },
+    uniqueKey: (a) => a,
+    maxItems: maxComments,
+    delayRange: [Math.round(config.commentMax * 0.3), config.commentMax],
+    staleThreshold: 2,
+  });
+
+  await closeCommentPanel(page);
+  await delay(Math.round(config.commentMax * 0.3), config.commentMax);
+
+  return allAuthors.slice(0, maxComments);
+}
+
+export { extractCommentAuthors };

package/src/scraper/modules/follow-extractor.mjs

@@ -0,0 +1,121 @@
+import { delay, getDelayConfig } from "./page-helpers.mjs";
+import { scrollAndCollect } from "./scroll-collector.mjs";
+
+const FILTER_WORDS = ["主页", "已关注", "粉丝", "推荐"];
+
+async function waitForListContent(page, minChildren = 1, timeout = 15000) {
+  await page
+    .waitForFunction(
+      (min) => {
+        const container = document.querySelector(
+          "[class*=DivUserListContainer]",
+        );
+        return container && container.children.length >= min;
+      },
+      minChildren,
+      { timeout },
+    )
+    .catch(() => {});
+}
+
+async function openFollowModal(page) {
+  const el = await page.$("[data-e2e=following]");
+  if (!el) {
+    throw new Error(
+      "未找到 [data-e2e=following] 元素，请确认当前页面为用户主页",
+    );
+  }
+  await el.evaluate((el) => el.parentElement.click());
+  await page
+    .waitForSelector("[class*=DivUserListContainer]", { timeout: 5000 })
+    .catch(() => {
+      throw new Error("关注弹窗未出现 DivUserListContainer");
+    });
+  await waitForListContent(page, 1, 3000);
+}
+
+async function switchToFollowersTab(page) {
+  await page.evaluate(() => {
+    const tabs = document.querySelectorAll("[class*=DivTabItem]");
+    for (const tab of tabs) {
+      if (tab.textContent?.includes("粉丝")) {
+        tab.click();
+        return;
+      }
+    }
+    throw new Error("未找到粉丝 Tab");
+  });
+  await waitForListContent(page, 1, 3000);
+}
+
+async function closeFollowModal(page) {
+  await page.evaluate(() => {
+    const closeBtn = document.querySelector("[data-e2e=follow-popup-close]");
+    if (closeBtn) closeBtn.click();
+  });
+  await page.waitForTimeout(500);
+}
+
+function createUserCollectFn() {
+  return (container) => {
+    const FILTER_WORDS = ["主页", "已关注", "粉丝", "推荐"];
+    const modal = document.querySelector("[class*=eyhy6180]");
+    const root = modal || document;
+    const users = [];
+    const seen = new Set();
+    const links = root.querySelectorAll('a[href*="/@"]');
+    for (const link of links) {
+      const match = link.href.match(/@([^/?]+)/);
+      if (!match) continue;
+      const handle = "@" + decodeURIComponent(match[1]);
+      const text = (link.textContent || "").trim();
+      if (text.length <= 2) continue;
+      if (FILTER_WORDS.includes(text)) continue;
+      if (seen.has(handle)) continue;
+      seen.add(handle);
+      users.push({ handle, displayName: text });
+    }
+    return { items: users };
+  };
+}
+
+async function extractUsersFromModal(page, maxUsers) {
+  const config = getDelayConfig();
+  const minDelay = Math.max(300, Math.round(config.commentMax * 0.3));
+  const maxDelay = Math.max(800, config.commentMax);
+
+  const allUsers = await scrollAndCollect(page, {
+    container: "[class*=DivUserListContainer]",
+    findScrollable: false,
+    collectFn: createUserCollectFn(),
+    uniqueKey: (u) => u.handle,
+    maxItems: maxUsers,
+    delayRange: [minDelay, maxDelay],
+    staleThreshold: 2,
+  });
+
+  return allUsers.slice(0, maxUsers);
+}
+
+async function extractFollowAndFollowers(page, options = {}) {
+  const { maxFollowing = 999, maxFollowers = 999, log = () => {} } = options;
+
+  await openFollowModal(page);
+
+  const following = await extractUsersFromModal(page, maxFollowing);
+  log(`  已关注: ${following.length}`);
+
+  await switchToFollowersTab(page);
+
+  const followers = await extractUsersFromModal(page, maxFollowers);
+  log(`  粉丝: ${followers.length}`);
+
+  await closeFollowModal(page);
+
+  return {
+    following: following.map((u) => [u.handle, u.displayName]),
+    followers: followers.map((u) => [u.handle, u.displayName]),
+  };
+}
+
+export { extractFollowAndFollowers };

package/src/scraper/modules/{guess-extractor.cjs → guess-extractor.mjs}

@@ -1,53 +1,51 @@
-const { delay, getDelayConfig, closeCommentPanel } = require('./page-helpers.cjs');
-const { scrollAndCollect } = require('./scroll-collector.cjs');
-
-async function openGuessTab(page) {
-  const tabs = page.locator('[class*="tabbar-item"]');
-  const guessTab = tabs.filter({ hasText: /猜你喜欢/i }).first();
-  await guessTab.click();
-  const config = getDelayConfig();
-  await delay(Math.round(config.commentMax * 0.5), config.commentMax);
-  await page.waitForSelector('[class*="Related"]', { timeout: 5000 }).catch(() => {});
-}
-
-async function extractGuessVideos(page, maxVideos = 10) {
-  await openGuessTab(page);
-
-  const config = getDelayConfig();
-  const allVideos = await scrollAndCollect(page, {
-    container: '[class*="Related"]',
-    findScrollable: true,
-    collectFn: (container) => {
-      const items = [];
-      Array.from(container.querySelectorAll('[class*="DivItemContainer"]')).forEach(item => {
-        const link = item.querySelector('a[href*="/video/"]');
-        if (link) {
-          const href = link.href || link.getAttribute('href');
-          const m = href && href.match(/@([^/]+)\/video\/(\d+)/);
-          if (m) {
-            items.push({
-              author: '@' + m[1],
-              videoId: m[2],
-              url: href,
-              title: '',
-            });
-          }
-        }
-      });
-      return { items };
-    },
-    uniqueKey: (v) => v.videoId,
-    maxItems: maxVideos,
-    delayRange: [Math.round(config.commentMax * 0.3), config.commentMax],
-    staleThreshold: 3,
-  });
-
-  await closeCommentPanel(page);
-  await delay(Math.round(config.commentMax * 0.3), config.commentMax);
-
-  return allVideos.slice(0, maxVideos);
-}
-
-module.exports = {
-  extractGuessVideos,
-};
+import { delay, getDelayConfig, closeCommentPanel } from './page-helpers.mjs';
+import { scrollAndCollect } from './scroll-collector.mjs';
+
+async function openGuessTab(page) {
+  const tabs = page.locator('[class*="tabbar-item"]');
+  const guessTab = tabs.filter({ hasText: /猜你喜欢/i }).first();
+  await guessTab.click();
+  const config = getDelayConfig();
+  await delay(Math.round(config.commentMax * 0.5), config.commentMax);
+  await page.waitForSelector('[class*="Related"]', { timeout: 5000 }).catch(() => {});
+}
+
+async function extractGuessVideos(page, maxVideos = 10) {
+  await openGuessTab(page);
+
+  const config = getDelayConfig();
+  const allVideos = await scrollAndCollect(page, {
+    container: '[class*="Related"]',
+    findScrollable: true,
+    collectFn: (container) => {
+      const items = [];
+      Array.from(container.querySelectorAll('[class*="DivItemContainer"]')).forEach(item => {
+        const link = item.querySelector('a[href*="/video/"]');
+        if (link) {
+          const href = link.href || link.getAttribute('href');
+          const m = href && href.match(/@([^/]+)\/video\/(\d+)/);
+          if (m) {
+            items.push({
+              author: '@' + m[1],
+              videoId: m[2],
+              url: href,
+              title: '',
+            });
+          }
+        }
+      });
+      return { items };
+    },
+    uniqueKey: (v) => v.videoId,
+    maxItems: maxVideos,
+    delayRange: [Math.round(config.commentMax * 0.3), config.commentMax],
+    staleThreshold: 3,
+  });
+
+  await closeCommentPanel(page);
+  await delay(Math.round(config.commentMax * 0.3), config.commentMax);
+
+  return allVideos.slice(0, maxVideos);
+}
+
+export { extractGuessVideos };

package/src/scraper/modules/page-error-detector.mjs

@@ -0,0 +1,70 @@
+const PATTERNS = {
+  login_required: [
+    "登录 TikTok",
+    "登录后查看",
+    "查看需登录",
+    "Log in to TikTok",
+    "Login to TikTok",
+    "观众管理功能",
+    "Viewer management",
+    "私密账号",
+    "私密状态",
+  ],
+  captcha: [
+    "captcha",
+    "verify",
+    "验证码",
+    "点击下一步",
+    "Press and hold",
+    "slide to verify",
+    "滑动验证",
+    "人机验证",
+    "安全验证",
+  ],
+  rate_limited: [
+    "访问过于频繁",
+    "操作过于频繁",
+    "too many requests",
+    "rate limit",
+    "稍后再试",
+    "try again later",
+    "请稍后再来",
+  ],
+  region_blocked: [
+    "地区限制",
+    "not available in your",
+    "此内容不可用",
+    "content not available",
+    "currently unavailable",
+    "抱歉，此内容",
+    "此页面不可用",
+  ],
+  not_found: [
+    "页面不存在",
+    "page not found",
+    "找不到",
+    "Couldn't find this",
+    "nothing here",
+    "此页面不存在",
+    "没有内容",
+    "发起对话",
+    "0 条评论",
+  ],
+};
+
+export async function detectPageError(page) {
+  return page.evaluate((patterns) => {
+    const bodyText = document.body.innerText;
+    const lower = bodyText.toLowerCase();
+
+    for (const [type, phrases] of Object.entries(patterns)) {
+      for (const phrase of phrases) {
+        if (lower.includes(phrase.toLowerCase())) {
+          return type;
+        }
+      }
+    }
+
+    return null;
+  }, PATTERNS);
+}

package/src/scraper/modules/page-helpers.mjs

@@ -0,0 +1,46 @@
+import {
+  delay,
+  getDelayConfig,
+  setDelayConfig,
+  listDelayPresets,
+  DELAY_PRESETS,
+} from '../../lib/delay.js';
+import { ensureBrowserReady } from '../../lib/browser/cdp.js';
+import {
+  ensureTikTokPage,
+  closeCommentPanel,
+  findTikTokPage,
+  getOrCreatePage,
+  isLoggedIn,
+} from '../../lib/browser/page.js';
+import { retryWithBackoff, isRetryableError } from '../../lib/retry.js';
+import {
+  extractUserSection,
+  parseUserSection,
+  extractLocationCreated,
+  USER_SECTION_SIZE,
+} from '../../lib/parser.js';
+import { detectPageError } from './page-error-detector.mjs';
+
+export {
+  delay,
+  setDelayConfig,
+  getDelayConfig,
+  listDelayPresets,
+  DELAY_PRESETS,
+  ensureBrowserReady,
+  ensureTikTokPage,
+  closeCommentPanel,
+  findTikTokPage,
+  getOrCreatePage,
+  isLoggedIn,
+  retryWithBackoff,
+  isRetryableError,
+  extractUserSection,
+  parseUserSection,
+  extractLocationCreated,
+  USER_SECTION_SIZE,
+  detectPageError,
+};
+
+export const CDP_PORT = 9222;