tt-help-cli-ycl 1.3.12 → 1.3.14

package/src/scraper/auto-core.js

@@ -1,194 +1,203 @@
-import {
-  delay,
-  ensureBrowserReady,
-  ensureTikTokPage,
-  setDelayConfig,
-  getDelayConfig,
-  closeCommentPanel,
-  retryWithBackoff,
-  detectPageError,
-  isLoggedIn,
-  assertPageUrl,
-} from './modules/page-helpers.js';
-import { detectCaptcha } from './modules/captcha-handler.js';
-export { ensureBrowserReady };
-import {
-  getUserInfo,
-  collectVideos,
-} from '../videos/core.js';
-import { runScrape } from './core.js';
-import { extractFollowAndFollowers } from './modules/follow-extractor.js';
-
-function mergeUserInfo(existing, incoming, source) {
-  const merged = { ...existing };
-  for (const [key, value] of Object.entries(incoming)) {
-    if (key === '_sources') continue;
-    if (value === undefined || value === null || value === '') continue;
-    if (typeof value === 'number' && typeof merged[key] === 'number') {
-      merged[key] = Math.max(merged[key], value);
-    } else if (merged[key] === undefined || merged[key] === null || merged[key] === '') {
-      merged[key] = value;
-    }
-  }
-  if (source) {
-    if (!merged._sources) merged._sources = [];
-    if (!merged._sources.includes(source)) merged._sources.push(source);
-  }
-  return merged;
-}
-
-async function processUser(page, username, options, log) {
-  const {
-    collectMax = 1,
-    scrapeDepth = 50,
-    maxComments = 200,
-    maxGuess = 10,
-    preset = 'fast',
-    switchMax = null,
-    commentMax = null,
-    enableFollow = false,
-    maxFollowing = 200,
-    maxFollowers = 200,
-    browser = null,
-  } = options;
-
-  const result = {
-    userInfo: null,
-    collectedVideos: [],
-    discoveredVideoAuthors: [],
-    discoveredCommentAuthors: [],
-    discoveredGuessAuthors: [],
-    discoveredFollowing: [],
-    discoveredFollowers: [],
-    error: null,
-  };
-
-  try {
-    log(`\n[processUser] 访问 @${username}...`);
-    await retryWithBackoff(async () => {
-      await page.goto(`https://www.tiktok.com/@${username}`, {
-        waitUntil: 'load', timeout: 30000,
-      });
-      assertPageUrl(page, `@${username}`);
-    }, { log });
-    await page.waitForSelector('[class*="DivVideoList"]', { timeout: 10000 }).catch(() => {});
-    await delay(1000, 2000);
-
-    const info = await getUserInfo(page);
-    result.userInfo = info;
-    if (!info.uniqueId) info.uniqueId = username;
-    log(`  昵称: ${info.nickname || '-'} | 粉丝: ${info.followerCount || 0}`);
-
-    if (options.enableFollow) {
-      const loggedIn = await isLoggedIn(page);
-      if (!loggedIn) {
-        log('  [跳过] 提取关注/粉丝：未登录，请先登录 TikTok');
-        result.discoveredFollowing = [];
-        result.discoveredFollowers = [];
-      } else {
-        try {
-          log('  提取关注/粉丝列表...');
-          const { following, followers } = await extractFollowAndFollowers(page, {
-            maxFollowing: options.maxFollowing || 200,
-            maxFollowers: options.maxFollowers || 200,
-            log,
-          });
-          result.discoveredFollowing = following;
-          result.discoveredFollowers = followers;
-          log(`  关注: ${following.length} | 粉丝: ${followers.length}`);
-        } catch (e) {
-          log(`  关注/粉丝提取失败: ${e.message}`);
-          result.discoveredFollowing = [];
-          result.discoveredFollowers = [];
-        }
-      }
-    }
-
-    const captcha = await detectCaptcha(page);
-    if (captcha && captcha.visible) {
-      log(`[验证码] @${username} 页面出现验证码`);
-      result.captchaDetected = true;
-    }
-
-    const videos = await collectVideos(page, username, collectMax, log);
-    const videoList = Array.from(videos.values()).slice(0, collectMax);
-    result.collectedVideos = videoList.map(v => ({
-      videoId: v.id,
-      videoUrl: v.href,
-    }));
-
-    if (videoList.length > 0) {
-      const allVideoAuthors = new Map();
-      const allCommentAuthors = new Set();
-      const allGuessAuthors = new Set();
-
-      for (let i = 0; i < videoList.length; i++) {
-        const video = videoList[i];
-        const videoUrl = video.href.startsWith('http')
-          ? video.href
-          : `https://www.tiktok.com${video.href}`;
-        log(`  [${i + 1}/${videoList.length}] 开始 scrape: ${videoUrl} (深度 ${scrapeDepth})`);
-
-        const scrapeResult = await runScrape({
-          videoUrl,
-          maxVideos: scrapeDepth,
-          maxComments,
-          maxGuess,
-          preset,
-          switchMax,
-          commentMax,
-          browser,
-          page,
-          log,
-        });
-
-        const scrapeOutput = scrapeResult.output;
-
-        if (scrapeOutput && scrapeOutput.videoDetails) {
-          for (const vd of scrapeOutput.videoDetails) {
-            if (!allVideoAuthors.has(vd.uniqueId)) {
-              allVideoAuthors.set(vd.uniqueId, {
-                uniqueId: vd.uniqueId,
-                nickname: vd.nickname,
-                locationCreated: vd.locationCreated,
-              });
-            }
-          }
-        }
-
-        if (scrapeOutput && scrapeOutput.commentUsers) {
-          for (const cu of scrapeOutput.commentUsers) {
-            allCommentAuthors.add(cu);
-          }
-        }
-
-        if (scrapeOutput && scrapeOutput.guessAuthors) {
-          for (const ga of scrapeOutput.guessAuthors) {
-            allGuessAuthors.add(ga);
-          }
-        }
-      }
-
-      result.discoveredVideoAuthors = [...allVideoAuthors.values()];
-      result.discoveredCommentAuthors = [...allCommentAuthors];
-      result.discoveredGuessAuthors = [...allGuessAuthors];
-
-      log(`  发现: ${result.discoveredVideoAuthors.length} 个视频作者, ${result.discoveredCommentAuthors.length} 个评论作者, ${result.discoveredGuessAuthors.length} 个猜你喜欢作者`);
-    } else {
-      const pageError = await detectPageError(page);
-      result.restricted = !!pageError;
-      if (pageError) {
-        log(`  @${username} 页面受限（${pageError}），标记跳过`);
-      } else {
-        log(`  @${username} 没有视频，跳过 scrape`);
-      }
-    }
-  } catch (e) {
-    result.error = e.message;
-    log(`  [错误] ${e.message}`);
-  }
-
-  return result;
-}
-
-export { processUser, mergeUserInfo };
+import {
+  delay,
+  ensureBrowserReady,
+  ensureTikTokPage,
+  setDelayConfig,
+  getDelayConfig,
+  closeCommentPanel,
+  retryWithBackoff,
+  detectPageError,
+  isLoggedIn,
+  assertPageUrl,
+} from './modules/page-helpers.js';
+import { detectCaptcha } from './modules/captcha-handler.js';
+export { ensureBrowserReady };
+import {
+  getUserInfo,
+  collectVideos,
+} from '../videos/core.js';
+import { runScrape } from './core.js';
+import { extractFollowAndFollowers } from './modules/follow-extractor.js';
+
+function mergeUserInfo(existing, incoming, source) {
+  const merged = { ...existing };
+  for (const [key, value] of Object.entries(incoming)) {
+    if (key === '_sources') continue;
+    if (value === undefined || value === null || value === '') continue;
+    if (typeof value === 'number' && typeof merged[key] === 'number') {
+      merged[key] = Math.max(merged[key], value);
+    } else if (merged[key] === undefined || merged[key] === null || merged[key] === '') {
+      merged[key] = value;
+    }
+  }
+  if (source) {
+    if (!merged._sources) merged._sources = [];
+    if (!merged._sources.includes(source)) merged._sources.push(source);
+  }
+  return merged;
+}
+
+async function processUser(page, username, options, log) {
+  const {
+    collectMax = 1,
+    scrapeDepth = 50,
+    maxComments = 200,
+    maxGuess = 10,
+    preset = 'fast',
+    switchMax = null,
+    commentMax = null,
+    enableFollow = false,
+    maxFollowing = 200,
+    maxFollowers = 200,
+    browser = null,
+  } = options;
+
+  const result = {
+    userInfo: null,
+    collectedVideos: [],
+    discoveredVideoAuthors: [],
+    discoveredCommentAuthors: [],
+    discoveredGuessAuthors: [],
+    discoveredFollowing: [],
+    discoveredFollowers: [],
+    error: null,
+  };
+
+  try {
+    log(`\n[processUser] 访问 @${username}...`);
+    await retryWithBackoff(async () => {
+      await page.goto(`https://www.tiktok.com/@${username}`, {
+        waitUntil: 'load', timeout: 30000,
+      });
+      assertPageUrl(page, `@${username}`);
+    }, { log });
+    await page.waitForSelector('[class*="DivVideoList"]', { timeout: 10000 }).catch(() => {});
+    await delay(1000, 2000);
+
+    const info = await getUserInfo(page);
+    result.userInfo = info;
+    if (!info.uniqueId) info.uniqueId = username;
+    log(`  昵称: ${info.nickname || '-'} | 粉丝: ${info.followerCount || 0}`);
+
+    if (options.enableFollow) {
+      const loggedIn = await isLoggedIn(page);
+      if (!loggedIn) {
+        log('  [跳过] 提取关注/粉丝：未登录，请先登录 TikTok');
+        result.discoveredFollowing = [];
+        result.discoveredFollowers = [];
+      } else {
+        try {
+          log('  提取关注/粉丝列表...');
+          const { following, followers } = await extractFollowAndFollowers(page, {
+            maxFollowing: options.maxFollowing || 200,
+            maxFollowers: options.maxFollowers || 200,
+            log,
+          });
+          result.discoveredFollowing = following;
+          result.discoveredFollowers = followers;
+          log(`  关注: ${following.length} | 粉丝: ${followers.length}`);
+        } catch (e) {
+          log(`  关注/粉丝提取失败: ${e.message}`);
+          result.discoveredFollowing = [];
+          result.discoveredFollowers = [];
+        }
+      }
+    }
+
+    const captcha = await detectCaptcha(page);
+    if (captcha && captcha.visible) {
+      log(`[验证码] @${username} 页面出现验证码`);
+      result.captchaDetected = true;
+      result.captchaStage = result.captchaStage || 'video-page';
+      result.captchaMessage = result.captchaMessage || '视频页出现验证码';
+    }
+
+    const videos = await collectVideos(page, username, collectMax, log);
+    const videoList = Array.from(videos.values()).slice(0, collectMax);
+    result.collectedVideos = videoList.map(v => ({
+      videoId: v.id,
+      videoUrl: v.href,
+    }));
+
+    if (videoList.length > 0) {
+      const allVideoAuthors = new Map();
+      const allCommentAuthors = new Set();
+      const allGuessAuthors = new Set();
+
+      for (let i = 0; i < videoList.length; i++) {
+        const video = videoList[i];
+        const videoUrl = video.href.startsWith('http')
+          ? video.href
+          : `https://www.tiktok.com${video.href}`;
+        log(`  [${i + 1}/${videoList.length}] 开始 scrape: ${videoUrl} (深度 ${scrapeDepth})`);
+
+        const scrapeResult = await runScrape({
+          videoUrl,
+          maxVideos: scrapeDepth,
+          maxComments,
+          maxGuess,
+          preset,
+          switchMax,
+          commentMax,
+          browser,
+          page,
+          log,
+        });
+
+        if (scrapeResult.captchaDetected) {
+          result.captchaDetected = true;
+          result.captchaStage = scrapeResult.captchaStage || 'scrape';
+          result.captchaMessage = scrapeResult.captchaMessage || 'scrape阶段出现验证码';
+        }
+
+        const scrapeOutput = scrapeResult.output;
+
+        if (scrapeOutput && scrapeOutput.videoDetails) {
+          for (const vd of scrapeOutput.videoDetails) {
+            if (!allVideoAuthors.has(vd.uniqueId)) {
+              allVideoAuthors.set(vd.uniqueId, {
+                uniqueId: vd.uniqueId,
+                nickname: vd.nickname,
+                locationCreated: vd.locationCreated,
+              });
+            }
+          }
+        }
+
+        if (scrapeOutput && scrapeOutput.commentUsers) {
+          for (const cu of scrapeOutput.commentUsers) {
+            allCommentAuthors.add(cu);
+          }
+        }
+
+        if (scrapeOutput && scrapeOutput.guessAuthors) {
+          for (const ga of scrapeOutput.guessAuthors) {
+            allGuessAuthors.add(ga);
+          }
+        }
+      }
+
+      result.discoveredVideoAuthors = [...allVideoAuthors.values()];
+      result.discoveredCommentAuthors = [...allCommentAuthors];
+      result.discoveredGuessAuthors = [...allGuessAuthors];
+
+      log(`  发现: ${result.discoveredVideoAuthors.length} 个视频作者, ${result.discoveredCommentAuthors.length} 个评论作者, ${result.discoveredGuessAuthors.length} 个猜你喜欢作者`);
+    } else {
+      const pageError = await detectPageError(page);
+      result.restricted = !!pageError;
+      if (pageError) {
+        log(`  @${username} 页面受限（${pageError}），标记跳过`);
+      } else {
+        log(`  @${username} 没有视频，跳过 scrape`);
+      }
+    }
+  } catch (e) {
+    result.error = e.message;
+    result.errorStack = e.stack || '';
+    log(`  [错误] ${e.message}`);
+  }
+
+  return result;
+}
+
+export { processUser, mergeUserInfo };