tt-help-cli-ycl 1.0.8 → 1.1.0

package/src/auto-core.cjs

@@ -1,310 +1,367 @@
-const {
-  delay,
-  ensureBrowserReady,
-  ensureTikTokPage,
-  setDelayConfig,
-  getDelayConfig,
-  closeCommentPanel,
-  retryWithBackoff,
-} = require('./scraper/modules/page-helpers.cjs');
-const {
-  getUserInfo,
-  collectVideos,
-  isPageRestricted,
-} = require('./get-user-videos-core.cjs');
-const { runScrape } = require('./scraper/core.cjs');
-
-function mergeUserInfo(existing, incoming, source) {
-  const merged = { ...existing };
-  for (const [key, value] of Object.entries(incoming)) {
-    if (key === '_sources') continue;
-    if (value === undefined || value === null || value === '') continue;
-    if (typeof value === 'number' && typeof merged[key] === 'number') {
-      merged[key] = Math.max(merged[key], value);
-    } else if (merged[key] === undefined || merged[key] === null || merged[key] === '') {
-      merged[key] = value;
-    }
-  }
-  if (source) {
-    if (!merged._sources) merged._sources = [];
-    if (!merged._sources.includes(source)) merged._sources.push(source);
-  }
-  return merged;
-}
-
-async function runAuto(options) {
-  const {
-    username,
-    collectMax = 1,
-    scrapeDepth = 50,
-    maxComments = 200,
-    maxGuess = 10,
-    preset = null,
-    switchMax = null,
-    commentMax = null,
-    log = console.error,
-  } = options;
-
-  if (preset) {
-    setDelayConfig(preset);
-  } else if (switchMax || commentMax) {
-    setDelayConfig({
-      switchMax: switchMax || 5000,
-      commentMax: commentMax || 3000,
-    });
-  }
-
-  const config = getDelayConfig();
-  const cleanUsername = username.replace('@', '');
-
-  log(`auto 模式: @${cleanUsername}`);
-  log(`收集视频数: ${collectMax}, 每个滑动: ${scrapeDepth}次, 每视频评论数: ${maxComments}`);
-
-  const browser = await ensureBrowserReady();
-  let page;
-  try {
-    page = await ensureTikTokPage(browser, `https://www.tiktok.com/@${cleanUsername}`);
-  } catch (e) {
-    await browser.close().catch(() => {});
-    throw e;
-  }
-
-  // [1/3] 获取种子用户信息
-  const profileUrl = `https://www.tiktok.com/@${cleanUsername}`;
-  log(`\n[1/3] 获取 @${cleanUsername} 的用户信息和视频列表...`);
-  await retryWithBackoff(() => page.goto(profileUrl, { waitUntil: 'load', timeout: 30000 }), { log });
-  await page.waitForSelector('[class*="DivVideoList"]', { timeout: 10000 }).catch(() => {});
-  await delay(1000, 2000);
-
-  const seedUserInfo = await getUserInfo(page);
-  if (!seedUserInfo.uniqueId) {
-    seedUserInfo.uniqueId = cleanUsername;
-  }
-  log(`种子用户: ${seedUserInfo.nickname || seedUserInfo.uniqueId} (粉丝: ${seedUserInfo.followerCount || '-'})`);
-
-  // [2/3] 收集视频列表
-  const videos = await collectVideos(page, cleanUsername, collectMax, log);
-  const videoList = Array.from(videos.values()).slice(0, collectMax);
-  log(`获取到 ${videoList.length} 个视频`);
-
-  if (videoList.length === 0) {
-    const restricted = await isPageRestricted(page);
-    if (restricted) {
-      log('种子用户页面受限（需登录），结束');
-    } else {
-      log('没有获取到视频，结束');
-    }
-    const output = {
-      seedUser: { ...seedUserInfo, sources: ['seed'], restricted },
-      users: [{ ...seedUserInfo, sources: ['seed'], restricted }],
-      stats: {
-        totalVideos: 0,
-        totalUsers: 1,
-        fromSeed: 1,
-        fromVideo: 0,
-        fromComment: 0,
-      },
-    };
-    return { output, browser };
-  }
-
-  // [3/3] 循环每个视频，执行 runScrape
-  log(`\n[3/3] 开始循环抓取（${videoList.length} 个视频，每个滑动 ${scrapeDepth} 次）...`);
-
-  const users = new Map();
-  users.set(seedUserInfo.uniqueId, mergeUserInfo({}, seedUserInfo, 'seed'));
-
-  const restrictedUsers = new Set();
-  let totalVideosScraped = 0;
-
-  for (let i = 0; i < videoList.length; i++) {
-    const videoUrl = videoList[i].href.startsWith('http')
-      ? videoList[i].href
-      : `https://www.tiktok.com${videoList[i].href}`;
-
-    log(`\n[${i + 1}/${videoList.length}] ${videoUrl}`);
-
-    const { output: scrapeOutput } = await runScrape({
-      videoUrl,
-      maxVideos: scrapeDepth,
-      maxComments,
-      maxGuess,
-      preset,
-      switchMax,
-      commentMax,
-      log,
-      browser,
-      page,
-    });
-
-    totalVideosScraped += (scrapeOutput && scrapeOutput.stats) ? scrapeOutput.stats.totalVideos : 0;
-
-    // 合并视频作者信息
-    for (const vd of scrapeOutput.videoDetails) {
-      if (restrictedUsers.has(vd.uniqueId)) continue;
-      const existing = users.get(vd.uniqueId);
-      users.set(vd.uniqueId, mergeUserInfo(existing || {}, vd, 'video'));
-    }
-
-    // 添加评论者
-    for (const cu of scrapeOutput.commentUsers) {
-      if (restrictedUsers.has(cu)) continue;
-      if (!users.has(cu)) {
-        users.set(cu, mergeUserInfo({}, { uniqueId: cu }, 'comment'));
-      }
-    }
-
-    // 添加猜你喜欢作者
-    for (const ga of (scrapeOutput.guessAuthors || [])) {
-      const gaId = ga.replace(/^@/, '');
-      if (restrictedUsers.has(gaId)) continue;
-      if (!users.has(gaId)) {
-        users.set(gaId, mergeUserInfo({}, { uniqueId: gaId }, 'guess'));
-      }
-    }
-  }
-
-  // 构建输出
-  const usersList = [...users.values()].map(u => {
-    const { _sources, ...rest } = u;
-    return { ...rest, sources: _sources || [] };
-  });
-
-  usersList.sort((a, b) => {
-    const aIsSeed = a._sources && a._sources.includes('seed');
-    const bIsSeed = b._sources && b._sources.includes('seed');
-    if (aIsSeed && !bIsSeed) return -1;
-    if (!aIsSeed && bIsSeed) return 1;
-    const aHasInfo = a.nickname || a.followerCount;
-    const bHasInfo = b.nickname || b.followerCount;
-    if (aHasInfo && !bHasInfo) return -1;
-    if (!aHasInfo && bHasInfo) return 1;
-    return 0;
-  });
-
-  const output = usersList;
-
-  log(`\n结果: ${usersList.length} 个用户`);
-
-  return { output, browser };
-}
-
-async function processUser(page, username, options, log) {
-  const {
-    collectMax = 1,
-    scrapeDepth = 50,
-    maxComments = 200,
-    maxGuess = 10,
-    preset = 'fast',
-    switchMax = null,
-    commentMax = null,
-    browser = null,
-  } = options;
-
-  const result = {
-    userInfo: null,
-    collectedVideos: [],
-    discoveredVideoAuthors: [],
-    discoveredCommentAuthors: [],
-    discoveredGuessAuthors: [],
-    error: null,
-  };
-
-  try {
-    log(`\n[processUser] 访问 @${username}...`);
-    await retryWithBackoff(() => page.goto(`https://www.tiktok.com/@${username}`, {
-      waitUntil: 'load',
-      timeout: 30000,
-    }), { log });
-    await page.waitForSelector('[class*="DivVideoList"]', { timeout: 10000 }).catch(() => {});
-    await delay(1000, 2000);
-
-    const info = await getUserInfo(page);
-    result.userInfo = info;
-    if (!info.uniqueId) {
-      info.uniqueId = username;
-    }
-    log(`  昵称: ${info.nickname || '-'} | 粉丝: ${info.followerCount || 0}`);
-
-    const videos = await collectVideos(page, username, collectMax, log);
-    const videoList = Array.from(videos.values()).slice(0, collectMax);
-    result.collectedVideos = videoList.map(v => ({
-      videoId: v.id,
-      videoUrl: v.href,
-    }));
-
-      if (videoList.length > 0) {
-      const allVideoAuthors = new Map();
-      const allCommentAuthors = new Set();
-      const allGuessAuthors = new Set();
-
-      for (let i = 0; i < videoList.length; i++) {
-        const video = videoList[i];
-        const videoUrl = video.href.startsWith('http')
-          ? video.href
-          : `https://www.tiktok.com${video.href}`;
-        log(`  [${i + 1}/${videoList.length}] 开始 scrape: ${videoUrl} (深度 ${scrapeDepth})`);
-
-        const scrapeResult = await runScrape({
-          videoUrl,
-          maxVideos: scrapeDepth,
-          maxComments,
-          maxGuess,
-          preset,
-          switchMax,
-          commentMax,
-          browser,
-          page,
-          log,
-        });
-
-        const scrapeOutput = scrapeResult.output;
-
-        if (scrapeOutput && scrapeOutput.videoDetails) {
-          for (const vd of scrapeOutput.videoDetails) {
-            if (!allVideoAuthors.has(vd.uniqueId)) {
-              allVideoAuthors.set(vd.uniqueId, {
-                uniqueId: vd.uniqueId,
-                nickname: vd.nickname,
-                locationCreated: vd.locationCreated,
-              });
-            }
-          }
-        }
-
-        if (scrapeOutput && scrapeOutput.commentUsers) {
-          for (const cu of scrapeOutput.commentUsers) {
-            allCommentAuthors.add(cu);
-          }
-        }
-
-        if (scrapeOutput && scrapeOutput.guessAuthors) {
-          for (const ga of scrapeOutput.guessAuthors) {
-            allGuessAuthors.add(ga);
-          }
-        }
-      }
-
-      result.discoveredVideoAuthors = [...allVideoAuthors.values()];
-      result.discoveredCommentAuthors = [...allCommentAuthors];
-      result.discoveredGuessAuthors = [...allGuessAuthors];
-
-      log(`  发现: ${result.discoveredVideoAuthors.length} 个视频作者, ${result.discoveredCommentAuthors.length} 个评论作者, ${result.discoveredGuessAuthors.length} 个猜你喜欢作者`);
-    } else {
-      const restricted = await isPageRestricted(page);
-      result.restricted = restricted;
-      if (restricted) {
-        log(`  @${username} 页面受限（需登录），标记跳过`);
-      } else {
-        log(`  @${username} 没有视频，跳过 scrape`);
-      }
-    }
-  } catch (e) {
-    result.error = e.message;
-    log(`  [错误] ${e.message}`);
-  }
-
-  return result;
-}
-
-module.exports = { runAuto, processUser };
+const {
+  delay,
+  ensureBrowserReady,
+  ensureTikTokPage,
+  setDelayConfig,
+  getDelayConfig,
+  closeCommentPanel,
+  retryWithBackoff,
+} = require('./scraper/modules/page-helpers.cjs');
+const {
+  getUserInfo,
+  collectVideos,
+  isPageRestricted,
+} = require('./get-user-videos-core.cjs');
+const { runScrape } = require('./scraper/core.cjs');
+const { extractFollowAndFollowers } = require('./scraper/modules/follow-extractor.cjs');
+
+function mergeUserInfo(existing, incoming, source) {
+  const merged = { ...existing };
+  for (const [key, value] of Object.entries(incoming)) {
+    if (key === '_sources') continue;
+    if (value === undefined || value === null || value === '') continue;
+    if (typeof value === 'number' && typeof merged[key] === 'number') {
+      merged[key] = Math.max(merged[key], value);
+    } else if (merged[key] === undefined || merged[key] === null || merged[key] === '') {
+      merged[key] = value;
+    }
+  }
+  if (source) {
+    if (!merged._sources) merged._sources = [];
+    if (!merged._sources.includes(source)) merged._sources.push(source);
+  }
+  return merged;
+}
+
+async function runAuto(options) {
+  const {
+    username,
+    collectMax = 1,
+    scrapeDepth = 50,
+    maxComments = 200,
+    maxGuess = 10,
+    preset = null,
+    switchMax = null,
+    commentMax = null,
+    enableFollow = false,
+    maxFollowing = 200,
+    maxFollowers = 200,
+    log = console.error,
+  } = options;
+
+  if (preset) {
+    setDelayConfig(preset);
+  } else if (switchMax || commentMax) {
+    setDelayConfig({
+      switchMax: switchMax || 5000,
+      commentMax: commentMax || 3000,
+    });
+  }
+
+  const config = getDelayConfig();
+  const cleanUsername = username.replace('@', '');
+
+  log(`auto 模式: @${cleanUsername}`);
+  log(`收集视频数: ${collectMax}, 每个滑动: ${scrapeDepth}次, 每视频评论数: ${maxComments}`);
+
+  const browser = await ensureBrowserReady();
+  let page;
+  try {
+    page = await ensureTikTokPage(browser, `https://www.tiktok.com/@${cleanUsername}`);
+  } catch (e) {
+    await browser.close().catch(() => {});
+    throw e;
+  }
+
+  // [1/3] 获取种子用户信息
+  const profileUrl = `https://www.tiktok.com/@${cleanUsername}`;
+  log(`\n[1/3] 获取 @${cleanUsername} 的用户信息和视频列表...`);
+  await retryWithBackoff(() => page.goto(profileUrl, { waitUntil: 'load', timeout: 30000 }), { log });
+  await page.waitForSelector('[class*="DivVideoList"]', { timeout: 10000 }).catch(() => {});
+  await delay(1000, 2000);
+
+  const seedUserInfo = await getUserInfo(page);
+  if (!seedUserInfo.uniqueId) {
+    seedUserInfo.uniqueId = cleanUsername;
+  }
+  log(`种子用户: ${seedUserInfo.nickname || seedUserInfo.uniqueId} (粉丝: ${seedUserInfo.followerCount || '-'})`);
+
+  if (options.enableFollow) {
+    try {
+      log(`  提取关注/粉丝列表...`);
+      const { following, followers } = await extractFollowAndFollowers(page, {
+        maxFollowing: options.maxFollowing || 200,
+        maxFollowers: options.maxFollowers || 200,
+        log,
+      });
+      log(`  关注: ${following.length} | 粉丝: ${followers.length}`);
+      following.forEach(([handle, name]) => {
+        const uid = handle.replace(/^@/, '');
+        users.set(uid, mergeUserInfo(
+          users.get(uid) || {},
+          { uniqueId: uid, nickname: name },
+          'following'
+        ));
+      });
+      followers.forEach(([handle, name]) => {
+        const uid = handle.replace(/^@/, '');
+        users.set(uid, mergeUserInfo(
+          users.get(uid) || {},
+          { uniqueId: uid, nickname: name },
+          'follower'
+        ));
+      });
+    } catch (e) {
+      log(`  关注/粉丝提取失败: ${e.message}`);
+    }
+  }
+
+  // [2/3] 收集视频列表
+  const videos = await collectVideos(page, cleanUsername, collectMax, log);
+  const videoList = Array.from(videos.values()).slice(0, collectMax);
+  log(`获取到 ${videoList.length} 个视频`);
+
+  if (videoList.length === 0) {
+    const restricted = await isPageRestricted(page);
+    if (restricted) {
+      log('种子用户页面受限（需登录），结束');
+    } else {
+      log('没有获取到视频，结束');
+    }
+    const output = {
+      seedUser: { ...seedUserInfo, sources: ['seed'], restricted },
+      users: [{ ...seedUserInfo, sources: ['seed'], restricted }],
+      stats: {
+        totalVideos: 0,
+        totalUsers: 1,
+        fromSeed: 1,
+        fromVideo: 0,
+        fromComment: 0,
+      },
+    };
+    return { output, browser };
+  }
+
+  // [3/3] 循环每个视频，执行 runScrape
+  log(`\n[3/3] 开始循环抓取（${videoList.length} 个视频，每个滑动 ${scrapeDepth} 次）...`);
+
+  const users = new Map();
+  users.set(seedUserInfo.uniqueId, mergeUserInfo({}, seedUserInfo, 'seed'));
+
+  const restrictedUsers = new Set();
+  let totalVideosScraped = 0;
+
+  for (let i = 0; i < videoList.length; i++) {
+    const videoUrl = videoList[i].href.startsWith('http')
+      ? videoList[i].href
+      : `https://www.tiktok.com${videoList[i].href}`;
+
+    log(`\n[${i + 1}/${videoList.length}] ${videoUrl}`);
+
+    const { output: scrapeOutput } = await runScrape({
+      videoUrl,
+      maxVideos: scrapeDepth,
+      maxComments,
+      maxGuess,
+      preset,
+      switchMax,
+      commentMax,
+      log,
+      browser,
+      page,
+    });
+
+    totalVideosScraped += (scrapeOutput && scrapeOutput.stats) ? scrapeOutput.stats.totalVideos : 0;
+
+    // 合并视频作者信息
+    for (const vd of scrapeOutput.videoDetails) {
+      if (restrictedUsers.has(vd.uniqueId)) continue;
+      const existing = users.get(vd.uniqueId);
+      users.set(vd.uniqueId, mergeUserInfo(existing || {}, vd, 'video'));
+    }
+
+    // 添加评论者
+    for (const cu of scrapeOutput.commentUsers) {
+      if (restrictedUsers.has(cu)) continue;
+      if (!users.has(cu)) {
+        users.set(cu, mergeUserInfo({}, { uniqueId: cu }, 'comment'));
+      }
+    }
+
+    // 添加猜你喜欢作者
+    for (const ga of (scrapeOutput.guessAuthors || [])) {
+      const gaId = ga.replace(/^@/, '');
+      if (restrictedUsers.has(gaId)) continue;
+      if (!users.has(gaId)) {
+        users.set(gaId, mergeUserInfo({}, { uniqueId: gaId }, 'guess'));
+      }
+    }
+  }
+
+  // 构建输出
+  const usersList = [...users.values()].map(u => {
+    const { _sources, ...rest } = u;
+    return { ...rest, sources: _sources || [] };
+  });
+
+  usersList.sort((a, b) => {
+    const aIsSeed = a._sources && a._sources.includes('seed');
+    const bIsSeed = b._sources && b._sources.includes('seed');
+    if (aIsSeed && !bIsSeed) return -1;
+    if (!aIsSeed && bIsSeed) return 1;
+    const aHasInfo = a.nickname || a.followerCount;
+    const bHasInfo = b.nickname || b.followerCount;
+    if (aHasInfo && !bHasInfo) return -1;
+    if (!aHasInfo && bHasInfo) return 1;
+    return 0;
+  });
+
+  const output = usersList;
+
+  log(`\n结果: ${usersList.length} 个用户`);
+
+  return { output, browser };
+}
+
+async function processUser(page, username, options, log) {
+  const {
+    collectMax = 1,
+    scrapeDepth = 50,
+    maxComments = 200,
+    maxGuess = 10,
+    preset = 'fast',
+    switchMax = null,
+    commentMax = null,
+    enableFollow = false,
+    maxFollowing = 200,
+    maxFollowers = 200,
+    browser = null,
+  } = options;
+
+  const result = {
+    userInfo: null,
+    collectedVideos: [],
+    discoveredVideoAuthors: [],
+    discoveredCommentAuthors: [],
+    discoveredGuessAuthors: [],
+    discoveredFollowing: [],
+    discoveredFollowers: [],
+    error: null,
+  };
+
+  try {
+    log(`\n[processUser] 访问 @${username}...`);
+    await retryWithBackoff(() => page.goto(`https://www.tiktok.com/@${username}`, {
+      waitUntil: 'load',
+      timeout: 30000,
+    }), { log });
+    await page.waitForSelector('[class*="DivVideoList"]', { timeout: 10000 }).catch(() => {});
+    await delay(1000, 2000);
+
+    const info = await getUserInfo(page);
+    result.userInfo = info;
+    if (!info.uniqueId) {
+      info.uniqueId = username;
+    }
+    log(`  昵称: ${info.nickname || '-'} | 粉丝: ${info.followerCount || 0}`);
+
+    if (options.enableFollow) {
+      try {
+        log(`  提取关注/粉丝列表...`);
+        const { following, followers } = await extractFollowAndFollowers(page, {
+          maxFollowing: options.maxFollowing || 200,
+          maxFollowers: options.maxFollowers || 200,
+          log,
+        });
+        result.discoveredFollowing = following;
+        result.discoveredFollowers = followers;
+        log(`  关注: ${following.length} | 粉丝: ${followers.length}`);
+      } catch (e) {
+        log(`  关注/粉丝提取失败: ${e.message}`);
+        result.discoveredFollowing = [];
+        result.discoveredFollowers = [];
+      }
+    }
+
+    const videos = await collectVideos(page, username, collectMax, log);
+    const videoList = Array.from(videos.values()).slice(0, collectMax);
+    result.collectedVideos = videoList.map(v => ({
+      videoId: v.id,
+      videoUrl: v.href,
+    }));
+
+      if (videoList.length > 0) {
+      const allVideoAuthors = new Map();
+      const allCommentAuthors = new Set();
+      const allGuessAuthors = new Set();
+
+      for (let i = 0; i < videoList.length; i++) {
+        const video = videoList[i];
+        const videoUrl = video.href.startsWith('http')
+          ? video.href
+          : `https://www.tiktok.com${video.href}`;
+        log(`  [${i + 1}/${videoList.length}] 开始 scrape: ${videoUrl} (深度 ${scrapeDepth})`);
+
+        const scrapeResult = await runScrape({
+          videoUrl,
+          maxVideos: scrapeDepth,
+          maxComments,
+          maxGuess,
+          preset,
+          switchMax,
+          commentMax,
+          browser,
+          page,
+          log,
+        });
+
+        const scrapeOutput = scrapeResult.output;
+
+        if (scrapeOutput && scrapeOutput.videoDetails) {
+          for (const vd of scrapeOutput.videoDetails) {
+            if (!allVideoAuthors.has(vd.uniqueId)) {
+              allVideoAuthors.set(vd.uniqueId, {
+                uniqueId: vd.uniqueId,
+                nickname: vd.nickname,
+                locationCreated: vd.locationCreated,
+              });
+            }
+          }
+        }
+
+        if (scrapeOutput && scrapeOutput.commentUsers) {
+          for (const cu of scrapeOutput.commentUsers) {
+            allCommentAuthors.add(cu);
+          }
+        }
+
+        if (scrapeOutput && scrapeOutput.guessAuthors) {
+          for (const ga of scrapeOutput.guessAuthors) {
+            allGuessAuthors.add(ga);
+          }
+        }
+      }
+
+      result.discoveredVideoAuthors = [...allVideoAuthors.values()];
+      result.discoveredCommentAuthors = [...allCommentAuthors];
+      result.discoveredGuessAuthors = [...allGuessAuthors];
+
+      log(`  发现: ${result.discoveredVideoAuthors.length} 个视频作者, ${result.discoveredCommentAuthors.length} 个评论作者, ${result.discoveredGuessAuthors.length} 个猜你喜欢作者`);
+    } else {
+      const restricted = await isPageRestricted(page);
+      result.restricted = restricted;
+      if (restricted) {
+        log(`  @${username} 页面受限（需登录），标记跳过`);
+      } else {
+        log(`  @${username} 没有视频，跳过 scrape`);
+      }
+    }
+  } catch (e) {
+    result.error = e.message;
+    log(`  [错误] ${e.message}`);
+  }
+
+  return result;
+}
+
+module.exports = { runAuto, processUser };