tt-help-cli-ycl 1.3.0 → 1.3.2

package/src/results/user-videos-bar.lar.lar.moeta.json

@@ -0,0 +1,37 @@
+{
+  "user": {
+    "uniqueId": "bar.lar.lar.moeta",
+    "secUid": "MS4wLjABAAAA3cgKTWvKfga0JAWeakAzx3zQ-aFAC8RuQvxD4HQFraKKsc_TbOIyMo3_ofVlXofV",
+    "nickname": "Bar Lar Lar Moetain",
+    "ttSeller": false,
+    "verified": false,
+    "followerCount": 24000,
+    "videoCount": 749,
+    "followingCount": 4293,
+    "heartCount": 254300,
+    "signature": ""
+  },
+  "totalVideos": 5,
+  "videos": [
+    {
+      "id": "7638231799084158228",
+      "url": "https://www.tiktok.com/@bar.lar.lar.moeta/video/7638231799084158228"
+    },
+    {
+      "id": "7638162444698914068",
+      "url": "https://www.tiktok.com/@bar.lar.lar.moeta/video/7638162444698914068"
+    },
+    {
+      "id": "7638116251767819541",
+      "url": "https://www.tiktok.com/@bar.lar.lar.moeta/video/7638116251767819541"
+    },
+    {
+      "id": "7638069637321690388",
+      "url": "https://www.tiktok.com/@bar.lar.lar.moeta/video/7638069637321690388"
+    },
+    {
+      "id": "7637927171025112341",
+      "url": "https://www.tiktok.com/@bar.lar.lar.moeta/video/7637927171025112341"
+    }
+  ]
+}

package/src/scraper/auto-core.mjs

@@ -0,0 +1,183 @@
+import {
+  delay,
+  ensureBrowserReady,
+  ensureTikTokPage,
+  setDelayConfig,
+  getDelayConfig,
+  closeCommentPanel,
+  retryWithBackoff,
+  detectPageError,
+  isLoggedIn,
+} from './modules/page-helpers.mjs';
+export { ensureBrowserReady };
+import {
+  getUserInfo,
+  collectVideos,
+} from '../videos/core.mjs';
+import { runScrape } from './core.mjs';
+import { extractFollowAndFollowers } from './modules/follow-extractor.mjs';
+
+function mergeUserInfo(existing, incoming, source) {
+  const merged = { ...existing };
+  for (const [key, value] of Object.entries(incoming)) {
+    if (key === '_sources') continue;
+    if (value === undefined || value === null || value === '') continue;
+    if (typeof value === 'number' && typeof merged[key] === 'number') {
+      merged[key] = Math.max(merged[key], value);
+    } else if (merged[key] === undefined || merged[key] === null || merged[key] === '') {
+      merged[key] = value;
+    }
+  }
+  if (source) {
+    if (!merged._sources) merged._sources = [];
+    if (!merged._sources.includes(source)) merged._sources.push(source);
+  }
+  return merged;
+}
+
+async function processUser(page, username, options, log) {
+  const {
+    collectMax = 1,
+    scrapeDepth = 50,
+    maxComments = 200,
+    maxGuess = 10,
+    preset = 'fast',
+    switchMax = null,
+    commentMax = null,
+    enableFollow = false,
+    maxFollowing = 200,
+    maxFollowers = 200,
+    browser = null,
+  } = options;
+
+  const result = {
+    userInfo: null,
+    collectedVideos: [],
+    discoveredVideoAuthors: [],
+    discoveredCommentAuthors: [],
+    discoveredGuessAuthors: [],
+    discoveredFollowing: [],
+    discoveredFollowers: [],
+    error: null,
+  };
+
+  try {
+    log(`\n[processUser] 访问 @${username}...`);
+    await retryWithBackoff(() => page.goto(`https://www.tiktok.com/@${username}`, {
+      waitUntil: 'load', timeout: 30000,
+    }), { log });
+    await page.waitForSelector('[class*="DivVideoList"]', { timeout: 10000 }).catch(() => {});
+    await delay(1000, 2000);
+
+    const info = await getUserInfo(page);
+    result.userInfo = info;
+    if (!info.uniqueId) info.uniqueId = username;
+    log(`  昵称: ${info.nickname || '-'} | 粉丝: ${info.followerCount || 0}`);
+
+    if (options.enableFollow) {
+      const loggedIn = await isLoggedIn(page);
+      if (!loggedIn) {
+        log('  [跳过] 提取关注/粉丝：未登录，请先登录 TikTok');
+        result.discoveredFollowing = [];
+        result.discoveredFollowers = [];
+      } else {
+        try {
+          log('  提取关注/粉丝列表...');
+          const { following, followers } = await extractFollowAndFollowers(page, {
+            maxFollowing: options.maxFollowing || 200,
+            maxFollowers: options.maxFollowers || 200,
+            log,
+          });
+          result.discoveredFollowing = following;
+          result.discoveredFollowers = followers;
+          log(`  关注: ${following.length} | 粉丝: ${followers.length}`);
+        } catch (e) {
+          log(`  关注/粉丝提取失败: ${e.message}`);
+          result.discoveredFollowing = [];
+          result.discoveredFollowers = [];
+        }
+      }
+    }
+
+    const videos = await collectVideos(page, username, collectMax, log);
+    const videoList = Array.from(videos.values()).slice(0, collectMax);
+    result.collectedVideos = videoList.map(v => ({
+      videoId: v.id,
+      videoUrl: v.href,
+    }));
+
+    if (videoList.length > 0) {
+      const allVideoAuthors = new Map();
+      const allCommentAuthors = new Set();
+      const allGuessAuthors = new Set();
+
+      for (let i = 0; i < videoList.length; i++) {
+        const video = videoList[i];
+        const videoUrl = video.href.startsWith('http')
+          ? video.href
+          : `https://www.tiktok.com${video.href}`;
+        log(`  [${i + 1}/${videoList.length}] 开始 scrape: ${videoUrl} (深度 ${scrapeDepth})`);
+
+        const scrapeResult = await runScrape({
+          videoUrl,
+          maxVideos: scrapeDepth,
+          maxComments,
+          maxGuess,
+          preset,
+          switchMax,
+          commentMax,
+          browser,
+          page,
+          log,
+        });
+
+        const scrapeOutput = scrapeResult.output;
+
+        if (scrapeOutput && scrapeOutput.videoDetails) {
+          for (const vd of scrapeOutput.videoDetails) {
+            if (!allVideoAuthors.has(vd.uniqueId)) {
+              allVideoAuthors.set(vd.uniqueId, {
+                uniqueId: vd.uniqueId,
+                nickname: vd.nickname,
+                locationCreated: vd.locationCreated,
+              });
+            }
+          }
+        }
+
+        if (scrapeOutput && scrapeOutput.commentUsers) {
+          for (const cu of scrapeOutput.commentUsers) {
+            allCommentAuthors.add(cu);
+          }
+        }
+
+        if (scrapeOutput && scrapeOutput.guessAuthors) {
+          for (const ga of scrapeOutput.guessAuthors) {
+            allGuessAuthors.add(ga);
+          }
+        }
+      }
+
+      result.discoveredVideoAuthors = [...allVideoAuthors.values()];
+      result.discoveredCommentAuthors = [...allCommentAuthors];
+      result.discoveredGuessAuthors = [...allGuessAuthors];
+
+      log(`  发现: ${result.discoveredVideoAuthors.length} 个视频作者, ${result.discoveredCommentAuthors.length} 个评论作者, ${result.discoveredGuessAuthors.length} 个猜你喜欢作者`);
+    } else {
+      const pageError = await detectPageError(page);
+      result.restricted = !!pageError;
+      if (pageError) {
+        log(`  @${username} 页面受限（${pageError}），标记跳过`);
+      } else {
+        log(`  @${username} 没有视频，跳过 scrape`);
+      }
+    }
+  } catch (e) {
+    result.error = e.message;
+    log(`  [错误] ${e.message}`);
+  }
+
+  return result;
+}
+
+export { processUser, mergeUserInfo };

package/src/scraper/{core.cjs → core.mjs}

@@ -1,214 +1,188 @@
-const {
-  closeCommentPanel,
-  delay,
-  ensureBrowserReady,
-  ensureTikTokPage,
-  setDelayConfig,
-  getDelayConfig,
-  retryWithBackoff,
-} = require("./modules/page-helpers.cjs");
-const { extractCommentAuthors } = require("./modules/comment-extractor.cjs");
-const { extractGuessVideos } = require("./modules/guess-extractor.cjs");
-
-async function scrapeSingleVideo(page, maxComments, maxGuess, log, location = 'ES') {
-  const config = getDelayConfig();
-
-  await page
-    .waitForSelector('[class*="VideoMeta"]', { timeout: 10000 })
-    .catch(() => {});
-  await delay(Math.round(config.commentMax * 0.3), config.commentMax);
-
-  const userData = await page.evaluate(() => {
-    const result = {};
-
-    const m = window.location.href.match(/\/@([^\/]+)\/video/);
-    if (m) result.uniqueId = m[1];
-
-    const authorEls = document.querySelectorAll('[class*="Author"]');
-    for (const el of authorEls) {
-      const text = (el.textContent || "").trim();
-      if (text && !text.includes("TikTok") && !text.includes("Share")) {
-        result.nickname = text;
-        break;
-      }
-    }
-
-    const html = document.documentElement.outerHTML;
-    const locMatch = html.match(/"locationCreated":"([^"]*)/);
-    if (locMatch) result.locationCreated = locMatch[1];
-
-    return result;
-  });
-
-  const videoAuthor = userData.uniqueId ? "@" + userData.uniqueId : null;
-  if (!videoAuthor) {
-    throw new Error("无法获取视频作者");
-  }
-
-  let guessVideos = [];
-  let commentUsers = [];
-
-  if (userData.locationCreated === location) {
-    guessVideos = await extractGuessVideos(page, maxGuess);
-
-    commentUsers = await extractCommentAuthors(page, maxComments);
-    await closeCommentPanel(page);
-    await delay(Math.round(config.commentMax * 0.3), config.commentMax);
-  }
-
-  const uniqueUsers = [...new Set(commentUsers)];
-
-  return {
-    videoAuthor,
-    uniqueId: userData.uniqueId,
-    nickname: userData.nickname,
-    locationCreated: userData.locationCreated,
-    commentUsers: uniqueUsers,
-    guessVideos,
-  };
-}
-
-async function runScrape(options) {
-  const {
-    videoUrl,
-    maxVideos = 20,
-    maxComments = 999,
-    maxGuess = 10,
-    preset = null,
-    switchMax = null,
-    commentMax = null,
-    log = console.error,
-    browser: externalBrowser = null,
-    page: externalPage = null,
-  } = options;
-
-  if (preset) {
-    setDelayConfig(preset);
-  } else if (switchMax || commentMax) {
-    setDelayConfig({
-      switchMax: switchMax || 5000,
-      commentMax: commentMax || 3000,
-    });
-  }
-
-  const config = getDelayConfig();
-
-  let browser, page;
-  const isExternal = !!(externalBrowser && externalPage);
-
-  if (!isExternal) {
-    log(`视频地址: ${videoUrl}`);
-    log(
-      `视频数: ${maxVideos}, 评论数: ${maxComments}, 猜你喜欢: ${maxGuess}, 切换延迟: ${config.switchMax}ms, 评论延迟: ${config.commentMax}ms`,
-    );
-  }
-
-  if (isExternal) {
-    browser = externalBrowser;
-    page = externalPage;
-  } else {
-    browser = await ensureBrowserReady();
-    try {
-      page = await ensureTikTokPage(browser, videoUrl);
-    } catch (e) {
-      await browser.close().catch(() => {});
-      throw e;
-    }
-  }
-
-  await retryWithBackoff(() => page.goto(videoUrl, { waitUntil: "load", timeout: 30000 }), { log });
-  await delay(Math.round(config.switchMax * 0.5), config.switchMax);
-  await closeCommentPanel(page);
-  await delay(Math.round(config.commentMax * 0.5), config.commentMax);
-
-  const allResults = [];
-  const videoAuthors = new Set();
-  const commentUsers = new Set();
-  const allCommentAuthorsList = [];
-  const allGuessAuthors = new Set();
-  const allGuessVideos = [];
-
-  for (let i = 0; i < maxVideos; i++) {
-    await delay(Math.round(config.commentMax * 0.3), config.commentMax);
-
-    let result;
-    try {
-      result = await scrapeSingleVideo(page, maxComments, maxGuess, log);
-    } catch (e) {
-      log(`[${i + 1}/${maxVideos}] 跳过: ${e.message}`);
-      if (i < maxVideos - 1) {
-        await page.evaluate(() => {
-          const container = document.querySelector(
-            '[class*="ColumnListContainer"]',
-          );
-          if (container) container.scrollTop += 700;
-          else window.scrollBy(0, 700);
-        });
-        await delay(Math.round(config.switchMax * 0.5), config.switchMax);
-      }
-      continue;
-    }
-    allResults.push(result);
-    videoAuthors.add(result.videoAuthor);
-    result.commentUsers.forEach((u) => commentUsers.add(u));
-    allCommentAuthorsList.push(...result.commentUsers);
-    if (result.guessVideos) {
-      allGuessVideos.push(...result.guessVideos);
-      result.guessVideos.forEach((v) => {
-        if (v.author) allGuessAuthors.add(v.author);
-      });
-    }
-
-    if ((i + 1) % 5 === 0 || i === 0) {
-      log(
-        `[${i + 1}/${maxVideos}] ${result.videoAuthor} | 昵称: ${result.nickname || "-"} | 评论用户: ${result.commentUsers.length} | 猜你喜欢: ${result.guessVideos ? result.guessVideos.length : 0}`,
-      );
-    }
-
-    if (i < maxVideos - 1) {
-      await page.evaluate(() => {
-        const container = document.querySelector(
-          '[class*="ColumnListContainer"]',
-        );
-        if (container) container.scrollTop += 700;
-      });
-      await delay(2000, config.switchMax);
-    }
-  }
-
-  log(
-    `\n结果: 视频作者 ${videoAuthors.size} | 评论用户 ${commentUsers.size} | 总评论 ${allCommentAuthorsList.length} | 猜你喜欢作者 ${allGuessAuthors.size} | 总猜中视频 ${allGuessVideos.length}`,
-  );
-
-  const videoDetails = {};
-  for (const r of allResults) {
-    const key = r.videoAuthor;
-    if (!videoDetails[key]) {
-      videoDetails[key] = {
-        videoAuthor: r.videoAuthor,
-        uniqueId: r.uniqueId,
-        nickname: r.nickname,
-        locationCreated: r.locationCreated,
-      };
-    }
-  }
-
-  const output = {
-    videoDetails: Object.values(videoDetails),
-    commentUsers: [...commentUsers].sort(),
-    allCommentAuthorsList,
-    guessVideos: allGuessVideos,
-    guessAuthors: [...allGuessAuthors].sort(),
-    stats: {
-      totalVideos: allResults.length,
-      uniqueVideoAuthors: videoAuthors.size,
-      uniqueCommentAuthors: commentUsers.size,
-      uniqueGuessAuthors: allGuessAuthors.size,
-      totalGuessVideos: allGuessVideos.length,
-    },
-  };
-
-  return { output, browser, isExternal };
-}
-
-module.exports = { scrapeSingleVideo, runScrape };
+import {
+  closeCommentPanel,
+  delay,
+  ensureBrowserReady,
+  ensureTikTokPage,
+  setDelayConfig,
+  getDelayConfig,
+  retryWithBackoff,
+} from './modules/page-helpers.mjs';
+import { extractCommentAuthors } from './modules/comment-extractor.mjs';
+import { extractGuessVideos } from './modules/guess-extractor.mjs';
+
+async function scrapeSingleVideo(page, maxComments, maxGuess, log, location = 'ES') {
+  const config = getDelayConfig();
+
+  await page.waitForSelector('[class*="VideoMeta"]', { timeout: 10000 }).catch(() => {});
+  await delay(Math.round(config.commentMax * 0.3), config.commentMax);
+
+  const userData = await page.evaluate(() => {
+    const result = {};
+    const m = window.location.href.match(/\/@([^/]+)\/video/);
+    if (m) result.uniqueId = m[1];
+    const authorEls = document.querySelectorAll('[class*="Author"]');
+    for (const el of authorEls) {
+      const text = (el.textContent || '').trim();
+      if (text && !text.includes('TikTok') && !text.includes('Share')) {
+        result.nickname = text;
+        break;
+      }
+    }
+    const html = document.documentElement.outerHTML;
+    const locMatch = html.match(/"locationCreated":"([^"]*)/);
+    if (locMatch) result.locationCreated = locMatch[1];
+    return result;
+  });
+
+  const videoAuthor = userData.uniqueId ? '@' + userData.uniqueId : null;
+  if (!videoAuthor) throw new Error('无法获取视频作者');
+
+    let guessVideos = [];
+    let commentUsers = [];
+
+    if (userData.locationCreated === location) {
+      if (maxGuess > 0) {
+        guessVideos = await extractGuessVideos(page, maxGuess);
+      }
+      if (maxComments > 0) {
+        commentUsers = await extractCommentAuthors(page, maxComments);
+      }
+      await closeCommentPanel(page);
+      if (maxGuess > 0 || maxComments > 0) {
+        await delay(Math.round(config.commentMax * 0.3), config.commentMax);
+      }
+    }
+
+  return {
+    videoAuthor,
+    uniqueId: userData.uniqueId,
+    nickname: userData.nickname,
+    locationCreated: userData.locationCreated,
+    commentUsers: [...new Set(commentUsers)],
+    guessVideos,
+  };
+}
+
+async function runScrape(options) {
+  const {
+    videoUrl, maxVideos = 20, maxComments = 999, maxGuess = 10,
+    preset = null, switchMax = null, commentMax = null,
+    log = console.error,
+    browser: externalBrowser = null, page: externalPage = null,
+  } = options;
+
+  if (preset) {
+    setDelayConfig(preset);
+  } else if (switchMax || commentMax) {
+    setDelayConfig({ switchMax: switchMax || 5000, commentMax: commentMax || 3000 });
+  }
+
+  const config = getDelayConfig();
+  let browser, page;
+  const isExternal = !!(externalBrowser && externalPage);
+
+  if (!isExternal) {
+    log(`视频地址: ${videoUrl}`);
+    log(`视频数: ${maxVideos}, 评论数: ${maxComments}, 猜你喜欢: ${maxGuess}, 切换延迟: ${config.switchMax}ms, 评论延迟: ${config.commentMax}ms`);
+  }
+
+  if (isExternal) {
+    browser = externalBrowser;
+    page = externalPage;
+  } else {
+    browser = await ensureBrowserReady();
+    try {
+      page = await ensureTikTokPage(browser, videoUrl);
+    } catch (e) {
+      await browser.close().catch(() => {});
+      throw e;
+    }
+  }
+
+  await retryWithBackoff(() => page.goto(videoUrl, { waitUntil: 'load', timeout: 30000 }), { log });
+  await delay(Math.round(config.switchMax * 0.5), config.switchMax);
+  await closeCommentPanel(page);
+  await delay(Math.round(config.commentMax * 0.5), config.commentMax);
+
+  const allResults = [];
+  const videoAuthors = new Set();
+  const commentUsers = new Set();
+  const allCommentAuthorsList = [];
+  const allGuessAuthors = new Set();
+  const allGuessVideos = [];
+
+  for (let i = 0; i < maxVideos; i++) {
+    await delay(Math.round(config.commentMax * 0.3), config.commentMax);
+
+    let result;
+    try {
+      result = await scrapeSingleVideo(page, maxComments, maxGuess, log);
+    } catch (e) {
+      log(`[${i + 1}/${maxVideos}] 跳过: ${e.message}`);
+      if (i < maxVideos - 1) {
+        await page.evaluate(() => {
+          const container = document.querySelector('[class*="ColumnListContainer"]');
+          if (container) container.scrollTop += 700;
+          else window.scrollBy(0, 700);
+        });
+        await delay(Math.round(config.switchMax * 0.5), config.switchMax);
+      }
+      continue;
+    }
+
+    allResults.push(result);
+    videoAuthors.add(result.videoAuthor);
+    result.commentUsers.forEach(u => commentUsers.add(u));
+    allCommentAuthorsList.push(...result.commentUsers);
+    if (result.guessVideos) {
+      allGuessVideos.push(...result.guessVideos);
+      result.guessVideos.forEach(v => { if (v.author) allGuessAuthors.add(v.author); });
+    }
+
+    if ((i + 1) % 5 === 0 || i === 0) {
+      log(`[${i + 1}/${maxVideos}] ${result.videoAuthor} | 昵称: ${result.nickname || '-'} | 评论用户: ${result.commentUsers.length} | 猜你喜欢: ${result.guessVideos ? result.guessVideos.length : 0}`);
+    }
+
+    if (i < maxVideos - 1) {
+      await page.evaluate(() => {
+        const container = document.querySelector('[class*="ColumnListContainer"]');
+        if (container) container.scrollTop += 700;
+      });
+      await delay(2000, config.switchMax);
+    }
+  }
+
+  log(`\n结果: 视频作者 ${videoAuthors.size} | 评论用户 ${commentUsers.size} | 总评论 ${allCommentAuthorsList.length} | 猜你喜欢作者 ${allGuessAuthors.size} | 总猜中视频 ${allGuessVideos.length}`);
+
+  const videoDetails = {};
+  for (const r of allResults) {
+    const key = r.videoAuthor;
+    if (!videoDetails[key]) {
+      videoDetails[key] = {
+        videoAuthor: r.videoAuthor,
+        uniqueId: r.uniqueId,
+        nickname: r.nickname,
+        locationCreated: r.locationCreated,
+      };
+    }
+  }
+
+  const output = {
+    videoDetails: Object.values(videoDetails),
+    commentUsers: [...commentUsers].sort(),
+    allCommentAuthorsList,
+    guessVideos: allGuessVideos,
+    guessAuthors: [...allGuessAuthors].sort(),
+    stats: {
+      totalVideos: allResults.length,
+      uniqueVideoAuthors: videoAuthors.size,
+      uniqueCommentAuthors: commentUsers.size,
+      uniqueGuessAuthors: allGuessAuthors.size,
+      totalGuessVideos: allGuessVideos.length,
+    },
+  };
+
+  return { output, browser, isExternal };
+}
+
+export { scrapeSingleVideo, runScrape };