tt-help-cli-ycl 1.3.45 → 1.3.46

package/src/scraper/core.js

@@ -1,255 +1,255 @@
-import {
-  closeCommentPanel,
-  delay,
-  ensureBrowserReady,
-  ensureTikTokPage,
-  setDelayConfig,
-  getDelayConfig,
-  retryWithBackoff,
-  assertPageUrl,
-} from "./modules/page-helpers.js";
-import { extractCommentAuthors } from "./modules/comment-extractor.js";
-import { extractGuessVideos } from "./modules/guess-extractor.js";
-import {
-  DEFAULT_TARGET_LOCATIONS_CSV,
-  isLocationInList,
-  normalizeLocationList,
-} from "../lib/target-locations.js";
-
-async function scrapeSingleVideo(
-  page,
-  maxComments,
-  maxGuess,
-  log,
-  location = DEFAULT_TARGET_LOCATIONS_CSV,
-) {
-  const config = getDelayConfig();
-
-  await page
-    .waitForSelector('[class*="VideoMeta"]', { timeout: 10000 })
-    .catch(() => {});
-  await delay(Math.round(config.commentMax * 0.3), config.commentMax);
-
-  const userData = await page.evaluate(() => {
-    const result = {};
-    const m = window.location.href.match(/\/@([^/]+)\/video/);
-    if (m) result.uniqueId = m[1];
-    const authorEls = document.querySelectorAll('[class*="Author"]');
-    for (const el of authorEls) {
-      const text = (el.textContent || "").trim();
-      if (text && !text.includes("TikTok") && !text.includes("Share")) {
-        result.nickname = text;
-        break;
-      }
-    }
-    const html = document.documentElement.outerHTML;
-    const locMatch = html.match(/"locationCreated":"([^"]*)/);
-    if (locMatch) result.locationCreated = locMatch[1];
-    return result;
-  });
-
-  const videoAuthor = userData.uniqueId ? "@" + userData.uniqueId : null;
-  if (!videoAuthor) throw new Error("无法获取视频作者");
-
-  let guessVideos = [];
-  let commentUsers = [];
-  let captchaDetected = false;
-  let captchaStage = "";
-  let captchaMessage = "";
-
-  const locationList = normalizeLocationList(location);
-  if (isLocationInList(userData.locationCreated, locationList)) {
-    if (maxGuess > 0) {
-      guessVideos = await extractGuessVideos(page, maxGuess);
-    }
-    if (maxComments > 0) {
-      const commentResult = await extractCommentAuthors(page, maxComments);
-      commentUsers = commentResult.authors || [];
-      if (commentResult.captchaDetected) {
-        captchaDetected = true;
-        captchaStage = "comment";
-        captchaMessage = "评论阶段出现验证码";
-      }
-    }
-    await closeCommentPanel(page);
-    if (maxGuess > 0 || maxComments > 0) {
-      await delay(Math.round(config.commentMax * 0.3), config.commentMax);
-    }
-  }
-
-  return {
-    videoAuthor,
-    uniqueId: userData.uniqueId,
-    nickname: userData.nickname,
-    locationCreated: userData.locationCreated,
-    commentUsers: [...new Set(commentUsers)],
-    guessVideos,
-    captchaDetected,
-    captchaStage,
-    captchaMessage,
-  };
-}
-
-async function runScrape(options) {
-  const {
-    videoUrl,
-    maxVideos = 20,
-    maxComments = 999,
-    maxGuess = 10,
-    preset = null,
-    switchMax = null,
-    commentMax = null,
-    log = console.error,
-    browser: externalBrowser = null,
-    page: externalPage = null,
-  } = options;
-
-  if (preset) {
-    setDelayConfig(preset);
-  } else if (switchMax || commentMax) {
-    setDelayConfig({
-      switchMax: switchMax || 5000,
-      commentMax: commentMax || 3000,
-    });
-  }
-
-  const config = getDelayConfig();
-  let browser, page;
-  const isExternal = !!(externalBrowser && externalPage);
-
-  if (!isExternal) {
-    log(`视频地址: ${videoUrl}`);
-    log(
-      `视频数: ${maxVideos}, 评论数: ${maxComments}, 猜你喜欢: ${maxGuess}, 切换延迟: ${config.switchMax}ms, 评论延迟: ${config.commentMax}ms`,
-    );
-  }
-
-  if (isExternal) {
-    browser = externalBrowser;
-    page = externalPage;
-  } else {
-    browser = await ensureBrowserReady();
-    try {
-      page = await ensureTikTokPage(browser, videoUrl);
-    } catch (e) {
-      await browser.close().catch(() => {});
-      throw e;
-    }
-  }
-
-  await retryWithBackoff(
-    () => page.goto(videoUrl, { waitUntil: "load", timeout: 30000 }),
-    { log },
-  );
-  assertPageUrl(page, videoUrl.split("/video/")[0]);
-  await delay(Math.round(config.switchMax * 0.5), config.switchMax);
-  await closeCommentPanel(page);
-  await delay(Math.round(config.commentMax * 0.5), config.commentMax);
-
-  const allResults = [];
-  let anyCaptchaDetected = false;
-  let anyCaptchaStage = "";
-  let anyCaptchaMessage = "";
-  const videoAuthors = new Set();
-  const commentUsers = new Set();
-  const allCommentAuthorsList = [];
-  const allGuessAuthors = new Set();
-  const allGuessVideos = [];
-
-  for (let i = 0; i < maxVideos; i++) {
-    await delay(Math.round(config.commentMax * 0.3), config.commentMax);
-
-    let result;
-    try {
-      result = await scrapeSingleVideo(page, maxComments, maxGuess, log);
-    } catch (e) {
-      log(`[${i + 1}/${maxVideos}] 跳过: ${e.message}`);
-      if (i < maxVideos - 1) {
-        await page.evaluate(() => {
-          const container = document.querySelector(
-            '[class*="ColumnListContainer"]',
-          );
-          if (container) container.scrollTop += 700;
-          else window.scrollBy(0, 700);
-        });
-        await delay(Math.round(config.switchMax * 0.5), config.switchMax);
-      }
-      continue;
-    }
-
-    allResults.push(result);
-    if (result.captchaDetected) {
-      anyCaptchaDetected = true;
-      anyCaptchaStage = result.captchaStage || "";
-      anyCaptchaMessage = result.captchaMessage || "";
-    }
-    videoAuthors.add(result.videoAuthor);
-    result.commentUsers.forEach((u) => commentUsers.add(u));
-    allCommentAuthorsList.push(...result.commentUsers);
-    if (result.guessVideos) {
-      allGuessVideos.push(...result.guessVideos);
-      result.guessVideos.forEach((v) => {
-        if (v.author) allGuessAuthors.add(v.author);
-      });
-    }
-
-    if ((i + 1) % 5 === 0 || i === 0) {
-      log(
-        `[${i + 1}/${maxVideos}] ${result.videoAuthor} | 昵称: ${result.nickname || "-"} | 评论用户: ${result.commentUsers.length} | 猜你喜欢: ${result.guessVideos ? result.guessVideos.length : 0}`,
-      );
-    }
-
-    if (i < maxVideos - 1) {
-      await page.evaluate(() => {
-        const container = document.querySelector(
-          '[class*="ColumnListContainer"]',
-        );
-        if (container) container.scrollTop += 700;
-      });
-      await delay(2000, config.switchMax);
-    }
-  }
-
-  log(
-    `\n结果: 视频作者 ${videoAuthors.size} | 评论用户 ${commentUsers.size} | 总评论 ${allCommentAuthorsList.length} | 猜你喜欢作者 ${allGuessAuthors.size} | 总猜中视频 ${allGuessVideos.length}`,
-  );
-
-  const videoDetails = {};
-  for (const r of allResults) {
-    const key = r.videoAuthor;
-    if (!videoDetails[key]) {
-      videoDetails[key] = {
-        videoAuthor: r.videoAuthor,
-        uniqueId: r.uniqueId,
-        nickname: r.nickname,
-        locationCreated: r.locationCreated,
-      };
-    }
-  }
-
-  const output = {
-    videoDetails: Object.values(videoDetails),
-    commentUsers: [...commentUsers].sort(),
-    allCommentAuthorsList,
-    guessVideos: allGuessVideos,
-    guessAuthors: [...allGuessAuthors].sort(),
-    stats: {
-      totalVideos: allResults.length,
-      uniqueVideoAuthors: videoAuthors.size,
-      uniqueCommentAuthors: commentUsers.size,
-      uniqueGuessAuthors: allGuessAuthors.size,
-      totalGuessVideos: allGuessVideos.length,
-    },
-  };
-
-  return {
-    output,
-    browser,
-    isExternal,
-    captchaDetected: anyCaptchaDetected,
-    captchaStage: anyCaptchaStage,
-    captchaMessage: anyCaptchaMessage,
-  };
-}
-
-export { scrapeSingleVideo, runScrape };
+import {
+  closeCommentPanel,
+  delay,
+  ensureBrowserReady,
+  ensureTikTokPage,
+  setDelayConfig,
+  getDelayConfig,
+  retryWithBackoff,
+  assertPageUrl,
+} from "./modules/page-helpers.js";
+import { extractCommentAuthors } from "./modules/comment-extractor.js";
+import { extractGuessVideos } from "./modules/guess-extractor.js";
+import {
+  DEFAULT_TARGET_LOCATIONS_CSV,
+  isLocationInList,
+  normalizeLocationList,
+} from "../lib/target-locations.js";
+
+async function scrapeSingleVideo(
+  page,
+  maxComments,
+  maxGuess,
+  log,
+  location = DEFAULT_TARGET_LOCATIONS_CSV,
+) {
+  const config = getDelayConfig();
+
+  await page
+    .waitForSelector('[class*="VideoMeta"]', { timeout: 10000 })
+    .catch(() => {});
+  await delay(Math.round(config.commentMax * 0.3), config.commentMax);
+
+  const userData = await page.evaluate(() => {
+    const result = {};
+    const m = window.location.href.match(/\/@([^/]+)\/video/);
+    if (m) result.uniqueId = m[1];
+    const authorEls = document.querySelectorAll('[class*="Author"]');
+    for (const el of authorEls) {
+      const text = (el.textContent || "").trim();
+      if (text && !text.includes("TikTok") && !text.includes("Share")) {
+        result.nickname = text;
+        break;
+      }
+    }
+    const html = document.documentElement.outerHTML;
+    const locMatch = html.match(/"locationCreated":"([^"]*)/);
+    if (locMatch) result.locationCreated = locMatch[1];
+    return result;
+  });
+
+  const videoAuthor = userData.uniqueId ? "@" + userData.uniqueId : null;
+  if (!videoAuthor) throw new Error("无法获取视频作者");
+
+  let guessVideos = [];
+  let commentUsers = [];
+  let captchaDetected = false;
+  let captchaStage = "";
+  let captchaMessage = "";
+
+  const locationList = normalizeLocationList(location);
+  if (isLocationInList(userData.locationCreated, locationList)) {
+    if (maxGuess > 0) {
+      guessVideos = await extractGuessVideos(page, maxGuess);
+    }
+    if (maxComments > 0) {
+      const commentResult = await extractCommentAuthors(page, maxComments);
+      commentUsers = commentResult.authors || [];
+      if (commentResult.captchaDetected) {
+        captchaDetected = true;
+        captchaStage = "comment";
+        captchaMessage = "评论阶段出现验证码";
+      }
+    }
+    await closeCommentPanel(page);
+    if (maxGuess > 0 || maxComments > 0) {
+      await delay(Math.round(config.commentMax * 0.3), config.commentMax);
+    }
+  }
+
+  return {
+    videoAuthor,
+    uniqueId: userData.uniqueId,
+    nickname: userData.nickname,
+    locationCreated: userData.locationCreated,
+    commentUsers: [...new Set(commentUsers)],
+    guessVideos,
+    captchaDetected,
+    captchaStage,
+    captchaMessage,
+  };
+}
+
+async function runScrape(options) {
+  const {
+    videoUrl,
+    maxVideos = 20,
+    maxComments = 999,
+    maxGuess = 10,
+    preset = null,
+    switchMax = null,
+    commentMax = null,
+    log = console.error,
+    browser: externalBrowser = null,
+    page: externalPage = null,
+  } = options;
+
+  if (preset) {
+    setDelayConfig(preset);
+  } else if (switchMax || commentMax) {
+    setDelayConfig({
+      switchMax: switchMax || 5000,
+      commentMax: commentMax || 3000,
+    });
+  }
+
+  const config = getDelayConfig();
+  let browser, page;
+  const isExternal = !!(externalBrowser && externalPage);
+
+  if (!isExternal) {
+    log(`视频地址: ${videoUrl}`);
+    log(
+      `视频数: ${maxVideos}, 评论数: ${maxComments}, 猜你喜欢: ${maxGuess}, 切换延迟: ${config.switchMax}ms, 评论延迟: ${config.commentMax}ms`,
+    );
+  }
+
+  if (isExternal) {
+    browser = externalBrowser;
+    page = externalPage;
+  } else {
+    browser = await ensureBrowserReady();
+    try {
+      page = await ensureTikTokPage(browser, videoUrl);
+    } catch (e) {
+      await browser.close().catch(() => {});
+      throw e;
+    }
+  }
+
+  await retryWithBackoff(
+    () => page.goto(videoUrl, { waitUntil: "load", timeout: 30000 }),
+    { log },
+  );
+  assertPageUrl(page, videoUrl.split("/video/")[0]);
+  await delay(Math.round(config.switchMax * 0.5), config.switchMax);
+  await closeCommentPanel(page);
+  await delay(Math.round(config.commentMax * 0.5), config.commentMax);
+
+  const allResults = [];
+  let anyCaptchaDetected = false;
+  let anyCaptchaStage = "";
+  let anyCaptchaMessage = "";
+  const videoAuthors = new Set();
+  const commentUsers = new Set();
+  const allCommentAuthorsList = [];
+  const allGuessAuthors = new Set();
+  const allGuessVideos = [];
+
+  for (let i = 0; i < maxVideos; i++) {
+    await delay(Math.round(config.commentMax * 0.3), config.commentMax);
+
+    let result;
+    try {
+      result = await scrapeSingleVideo(page, maxComments, maxGuess, log);
+    } catch (e) {
+      log(`[${i + 1}/${maxVideos}] 跳过: ${e.message}`);
+      if (i < maxVideos - 1) {
+        await page.evaluate(() => {
+          const container = document.querySelector(
+            '[class*="ColumnListContainer"]',
+          );
+          if (container) container.scrollTop += 700;
+          else window.scrollBy(0, 700);
+        });
+        await delay(Math.round(config.switchMax * 0.5), config.switchMax);
+      }
+      continue;
+    }
+
+    allResults.push(result);
+    if (result.captchaDetected) {
+      anyCaptchaDetected = true;
+      anyCaptchaStage = result.captchaStage || "";
+      anyCaptchaMessage = result.captchaMessage || "";
+    }
+    videoAuthors.add(result.videoAuthor);
+    result.commentUsers.forEach((u) => commentUsers.add(u));
+    allCommentAuthorsList.push(...result.commentUsers);
+    if (result.guessVideos) {
+      allGuessVideos.push(...result.guessVideos);
+      result.guessVideos.forEach((v) => {
+        if (v.author) allGuessAuthors.add(v.author);
+      });
+    }
+
+    if ((i + 1) % 5 === 0 || i === 0) {
+      log(
+        `[${i + 1}/${maxVideos}] ${result.videoAuthor} | 昵称: ${result.nickname || "-"} | 评论用户: ${result.commentUsers.length} | 猜你喜欢: ${result.guessVideos ? result.guessVideos.length : 0}`,
+      );
+    }
+
+    if (i < maxVideos - 1) {
+      await page.evaluate(() => {
+        const container = document.querySelector(
+          '[class*="ColumnListContainer"]',
+        );
+        if (container) container.scrollTop += 700;
+      });
+      await delay(2000, config.switchMax);
+    }
+  }
+
+  log(
+    `\n结果: 视频作者 ${videoAuthors.size} | 评论用户 ${commentUsers.size} | 总评论 ${allCommentAuthorsList.length} | 猜你喜欢作者 ${allGuessAuthors.size} | 总猜中视频 ${allGuessVideos.length}`,
+  );
+
+  const videoDetails = {};
+  for (const r of allResults) {
+    const key = r.videoAuthor;
+    if (!videoDetails[key]) {
+      videoDetails[key] = {
+        videoAuthor: r.videoAuthor,
+        uniqueId: r.uniqueId,
+        nickname: r.nickname,
+        locationCreated: r.locationCreated,
+      };
+    }
+  }
+
+  const output = {
+    videoDetails: Object.values(videoDetails),
+    commentUsers: [...commentUsers].sort(),
+    allCommentAuthorsList,
+    guessVideos: allGuessVideos,
+    guessAuthors: [...allGuessAuthors].sort(),
+    stats: {
+      totalVideos: allResults.length,
+      uniqueVideoAuthors: videoAuthors.size,
+      uniqueCommentAuthors: commentUsers.size,
+      uniqueGuessAuthors: allGuessAuthors.size,
+      totalGuessVideos: allGuessVideos.length,
+    },
+  };
+
+  return {
+    output,
+    browser,
+    isExternal,
+    captchaDetected: anyCaptchaDetected,
+    captchaStage: anyCaptchaStage,
+    captchaMessage: anyCaptchaMessage,
+  };
+}
+
+export { scrapeSingleVideo, runScrape };