tt-help-cli-ycl 1.3.34 → 1.3.36

package/src/scraper/core.js

@@ -1,211 +1,211 @@
-import {
-  closeCommentPanel,
-  delay,
-  ensureBrowserReady,
-  ensureTikTokPage,
-  setDelayConfig,
-  getDelayConfig,
-  retryWithBackoff,
-  assertPageUrl,
-} from './modules/page-helpers.js';
-import { extractCommentAuthors } from './modules/comment-extractor.js';
-import { extractGuessVideos } from './modules/guess-extractor.js';
-
-async function scrapeSingleVideo(page, maxComments, maxGuess, log, location = 'PL,NL,BE,DE,FR,IT,ES,IE') {
-  const config = getDelayConfig();
-
-  await page.waitForSelector('[class*="VideoMeta"]', { timeout: 10000 }).catch(() => {});
-  await delay(Math.round(config.commentMax * 0.3), config.commentMax);
-
-  const userData = await page.evaluate(() => {
-    const result = {};
-    const m = window.location.href.match(/\/@([^/]+)\/video/);
-    if (m) result.uniqueId = m[1];
-    const authorEls = document.querySelectorAll('[class*="Author"]');
-    for (const el of authorEls) {
-      const text = (el.textContent || '').trim();
-      if (text && !text.includes('TikTok') && !text.includes('Share')) {
-        result.nickname = text;
-        break;
-      }
-    }
-    const html = document.documentElement.outerHTML;
-    const locMatch = html.match(/"locationCreated":"([^"]*)/);
-    if (locMatch) result.locationCreated = locMatch[1];
-    return result;
-  });
-
-  const videoAuthor = userData.uniqueId ? '@' + userData.uniqueId : null;
-  if (!videoAuthor) throw new Error('无法获取视频作者');
-
-    let guessVideos = [];
-    let commentUsers = [];
-    let captchaDetected = false;
-    let captchaStage = '';
-    let captchaMessage = '';
-
-    const locationList = (location || 'ES').split(',').map(s => s.trim().toUpperCase());
-    if (locationList.includes(userData.locationCreated?.toUpperCase?.() || userData.locationCreated)) {
-      if (maxGuess > 0) {
-        guessVideos = await extractGuessVideos(page, maxGuess);
-      }
-      if (maxComments > 0) {
-        const commentResult = await extractCommentAuthors(page, maxComments);
-        commentUsers = commentResult.authors || [];
-        if (commentResult.captchaDetected) {
-          captchaDetected = true;
-          captchaStage = 'comment';
-          captchaMessage = '评论阶段出现验证码';
-        }
-      }
-      await closeCommentPanel(page);
-      if (maxGuess > 0 || maxComments > 0) {
-        await delay(Math.round(config.commentMax * 0.3), config.commentMax);
-      }
-    }
-
-  return {
-    videoAuthor,
-    uniqueId: userData.uniqueId,
-    nickname: userData.nickname,
-    locationCreated: userData.locationCreated,
-    commentUsers: [...new Set(commentUsers)],
-    guessVideos,
-    captchaDetected,
-    captchaStage,
-    captchaMessage,
-  };
-}
-
-async function runScrape(options) {
-  const {
-    videoUrl, maxVideos = 20, maxComments = 999, maxGuess = 10,
-    preset = null, switchMax = null, commentMax = null,
-    log = console.error,
-    browser: externalBrowser = null, page: externalPage = null,
-  } = options;
-
-  if (preset) {
-    setDelayConfig(preset);
-  } else if (switchMax || commentMax) {
-    setDelayConfig({ switchMax: switchMax || 5000, commentMax: commentMax || 3000 });
-  }
-
-  const config = getDelayConfig();
-  let browser, page;
-  const isExternal = !!(externalBrowser && externalPage);
-
-  if (!isExternal) {
-    log(`视频地址: ${videoUrl}`);
-    log(`视频数: ${maxVideos}, 评论数: ${maxComments}, 猜你喜欢: ${maxGuess}, 切换延迟: ${config.switchMax}ms, 评论延迟: ${config.commentMax}ms`);
-  }
-
-  if (isExternal) {
-    browser = externalBrowser;
-    page = externalPage;
-  } else {
-    browser = await ensureBrowserReady();
-    try {
-      page = await ensureTikTokPage(browser, videoUrl);
-    } catch (e) {
-      await browser.close().catch(() => {});
-      throw e;
-    }
-  }
-
-  await retryWithBackoff(() => page.goto(videoUrl, { waitUntil: 'load', timeout: 30000 }), { log });
-  assertPageUrl(page, videoUrl.split('/video/')[0]);
-  await delay(Math.round(config.switchMax * 0.5), config.switchMax);
-  await closeCommentPanel(page);
-  await delay(Math.round(config.commentMax * 0.5), config.commentMax);
-
-  const allResults = [];
-  let anyCaptchaDetected = false;
-  let anyCaptchaStage = '';
-  let anyCaptchaMessage = '';
-  const videoAuthors = new Set();
-  const commentUsers = new Set();
-  const allCommentAuthorsList = [];
-  const allGuessAuthors = new Set();
-  const allGuessVideos = [];
-
-  for (let i = 0; i < maxVideos; i++) {
-    await delay(Math.round(config.commentMax * 0.3), config.commentMax);
-
-    let result;
-    try {
-      result = await scrapeSingleVideo(page, maxComments, maxGuess, log);
-    } catch (e) {
-      log(`[${i + 1}/${maxVideos}] 跳过: ${e.message}`);
-      if (i < maxVideos - 1) {
-        await page.evaluate(() => {
-          const container = document.querySelector('[class*="ColumnListContainer"]');
-          if (container) container.scrollTop += 700;
-          else window.scrollBy(0, 700);
-        });
-        await delay(Math.round(config.switchMax * 0.5), config.switchMax);
-      }
-      continue;
-    }
-
-    allResults.push(result);
-    if (result.captchaDetected) {
-      anyCaptchaDetected = true;
-      anyCaptchaStage = result.captchaStage || '';
-      anyCaptchaMessage = result.captchaMessage || '';
-    }
-    videoAuthors.add(result.videoAuthor);
-    result.commentUsers.forEach(u => commentUsers.add(u));
-    allCommentAuthorsList.push(...result.commentUsers);
-    if (result.guessVideos) {
-      allGuessVideos.push(...result.guessVideos);
-      result.guessVideos.forEach(v => { if (v.author) allGuessAuthors.add(v.author); });
-    }
-
-    if ((i + 1) % 5 === 0 || i === 0) {
-      log(`[${i + 1}/${maxVideos}] ${result.videoAuthor} | 昵称: ${result.nickname || '-'} | 评论用户: ${result.commentUsers.length} | 猜你喜欢: ${result.guessVideos ? result.guessVideos.length : 0}`);
-    }
-
-    if (i < maxVideos - 1) {
-      await page.evaluate(() => {
-        const container = document.querySelector('[class*="ColumnListContainer"]');
-        if (container) container.scrollTop += 700;
-      });
-      await delay(2000, config.switchMax);
-    }
-  }
-
-  log(`\n结果: 视频作者 ${videoAuthors.size} | 评论用户 ${commentUsers.size} | 总评论 ${allCommentAuthorsList.length} | 猜你喜欢作者 ${allGuessAuthors.size} | 总猜中视频 ${allGuessVideos.length}`);
-
-  const videoDetails = {};
-  for (const r of allResults) {
-    const key = r.videoAuthor;
-    if (!videoDetails[key]) {
-      videoDetails[key] = {
-        videoAuthor: r.videoAuthor,
-        uniqueId: r.uniqueId,
-        nickname: r.nickname,
-        locationCreated: r.locationCreated,
-      };
-    }
-  }
-
-  const output = {
-    videoDetails: Object.values(videoDetails),
-    commentUsers: [...commentUsers].sort(),
-    allCommentAuthorsList,
-    guessVideos: allGuessVideos,
-    guessAuthors: [...allGuessAuthors].sort(),
-    stats: {
-      totalVideos: allResults.length,
-      uniqueVideoAuthors: videoAuthors.size,
-      uniqueCommentAuthors: commentUsers.size,
-      uniqueGuessAuthors: allGuessAuthors.size,
-      totalGuessVideos: allGuessVideos.length,
-    },
-  };
-
-  return { output, browser, isExternal, captchaDetected: anyCaptchaDetected, captchaStage: anyCaptchaStage, captchaMessage: anyCaptchaMessage };
-}
-
-export { scrapeSingleVideo, runScrape };
+import {
+  closeCommentPanel,
+  delay,
+  ensureBrowserReady,
+  ensureTikTokPage,
+  setDelayConfig,
+  getDelayConfig,
+  retryWithBackoff,
+  assertPageUrl,
+} from './modules/page-helpers.js';
+import { extractCommentAuthors } from './modules/comment-extractor.js';
+import { extractGuessVideos } from './modules/guess-extractor.js';
+
+async function scrapeSingleVideo(page, maxComments, maxGuess, log, location = 'PL,NL,BE,DE,FR,IT,ES,IE') {
+  const config = getDelayConfig();
+
+  await page.waitForSelector('[class*="VideoMeta"]', { timeout: 10000 }).catch(() => {});
+  await delay(Math.round(config.commentMax * 0.3), config.commentMax);
+
+  const userData = await page.evaluate(() => {
+    const result = {};
+    const m = window.location.href.match(/\/@([^/]+)\/video/);
+    if (m) result.uniqueId = m[1];
+    const authorEls = document.querySelectorAll('[class*="Author"]');
+    for (const el of authorEls) {
+      const text = (el.textContent || '').trim();
+      if (text && !text.includes('TikTok') && !text.includes('Share')) {
+        result.nickname = text;
+        break;
+      }
+    }
+    const html = document.documentElement.outerHTML;
+    const locMatch = html.match(/"locationCreated":"([^"]*)/);
+    if (locMatch) result.locationCreated = locMatch[1];
+    return result;
+  });
+
+  const videoAuthor = userData.uniqueId ? '@' + userData.uniqueId : null;
+  if (!videoAuthor) throw new Error('无法获取视频作者');
+
+    let guessVideos = [];
+    let commentUsers = [];
+    let captchaDetected = false;
+    let captchaStage = '';
+    let captchaMessage = '';
+
+    const locationList = (location || 'ES').split(',').map(s => s.trim().toUpperCase());
+    if (locationList.includes(userData.locationCreated?.toUpperCase?.() || userData.locationCreated)) {
+      if (maxGuess > 0) {
+        guessVideos = await extractGuessVideos(page, maxGuess);
+      }
+      if (maxComments > 0) {
+        const commentResult = await extractCommentAuthors(page, maxComments);
+        commentUsers = commentResult.authors || [];
+        if (commentResult.captchaDetected) {
+          captchaDetected = true;
+          captchaStage = 'comment';
+          captchaMessage = '评论阶段出现验证码';
+        }
+      }
+      await closeCommentPanel(page);
+      if (maxGuess > 0 || maxComments > 0) {
+        await delay(Math.round(config.commentMax * 0.3), config.commentMax);
+      }
+    }
+
+  return {
+    videoAuthor,
+    uniqueId: userData.uniqueId,
+    nickname: userData.nickname,
+    locationCreated: userData.locationCreated,
+    commentUsers: [...new Set(commentUsers)],
+    guessVideos,
+    captchaDetected,
+    captchaStage,
+    captchaMessage,
+  };
+}
+
+async function runScrape(options) {
+  const {
+    videoUrl, maxVideos = 20, maxComments = 999, maxGuess = 10,
+    preset = null, switchMax = null, commentMax = null,
+    log = console.error,
+    browser: externalBrowser = null, page: externalPage = null,
+  } = options;
+
+  if (preset) {
+    setDelayConfig(preset);
+  } else if (switchMax || commentMax) {
+    setDelayConfig({ switchMax: switchMax || 5000, commentMax: commentMax || 3000 });
+  }
+
+  const config = getDelayConfig();
+  let browser, page;
+  const isExternal = !!(externalBrowser && externalPage);
+
+  if (!isExternal) {
+    log(`视频地址: ${videoUrl}`);
+    log(`视频数: ${maxVideos}, 评论数: ${maxComments}, 猜你喜欢: ${maxGuess}, 切换延迟: ${config.switchMax}ms, 评论延迟: ${config.commentMax}ms`);
+  }
+
+  if (isExternal) {
+    browser = externalBrowser;
+    page = externalPage;
+  } else {
+    browser = await ensureBrowserReady();
+    try {
+      page = await ensureTikTokPage(browser, videoUrl);
+    } catch (e) {
+      await browser.close().catch(() => {});
+      throw e;
+    }
+  }
+
+  await retryWithBackoff(() => page.goto(videoUrl, { waitUntil: 'load', timeout: 30000 }), { log });
+  assertPageUrl(page, videoUrl.split('/video/')[0]);
+  await delay(Math.round(config.switchMax * 0.5), config.switchMax);
+  await closeCommentPanel(page);
+  await delay(Math.round(config.commentMax * 0.5), config.commentMax);
+
+  const allResults = [];
+  let anyCaptchaDetected = false;
+  let anyCaptchaStage = '';
+  let anyCaptchaMessage = '';
+  const videoAuthors = new Set();
+  const commentUsers = new Set();
+  const allCommentAuthorsList = [];
+  const allGuessAuthors = new Set();
+  const allGuessVideos = [];
+
+  for (let i = 0; i < maxVideos; i++) {
+    await delay(Math.round(config.commentMax * 0.3), config.commentMax);
+
+    let result;
+    try {
+      result = await scrapeSingleVideo(page, maxComments, maxGuess, log);
+    } catch (e) {
+      log(`[${i + 1}/${maxVideos}] 跳过: ${e.message}`);
+      if (i < maxVideos - 1) {
+        await page.evaluate(() => {
+          const container = document.querySelector('[class*="ColumnListContainer"]');
+          if (container) container.scrollTop += 700;
+          else window.scrollBy(0, 700);
+        });
+        await delay(Math.round(config.switchMax * 0.5), config.switchMax);
+      }
+      continue;
+    }
+
+    allResults.push(result);
+    if (result.captchaDetected) {
+      anyCaptchaDetected = true;
+      anyCaptchaStage = result.captchaStage || '';
+      anyCaptchaMessage = result.captchaMessage || '';
+    }
+    videoAuthors.add(result.videoAuthor);
+    result.commentUsers.forEach(u => commentUsers.add(u));
+    allCommentAuthorsList.push(...result.commentUsers);
+    if (result.guessVideos) {
+      allGuessVideos.push(...result.guessVideos);
+      result.guessVideos.forEach(v => { if (v.author) allGuessAuthors.add(v.author); });
+    }
+
+    if ((i + 1) % 5 === 0 || i === 0) {
+      log(`[${i + 1}/${maxVideos}] ${result.videoAuthor} | 昵称: ${result.nickname || '-'} | 评论用户: ${result.commentUsers.length} | 猜你喜欢: ${result.guessVideos ? result.guessVideos.length : 0}`);
+    }
+
+    if (i < maxVideos - 1) {
+      await page.evaluate(() => {
+        const container = document.querySelector('[class*="ColumnListContainer"]');
+        if (container) container.scrollTop += 700;
+      });
+      await delay(2000, config.switchMax);
+    }
+  }
+
+  log(`\n结果: 视频作者 ${videoAuthors.size} | 评论用户 ${commentUsers.size} | 总评论 ${allCommentAuthorsList.length} | 猜你喜欢作者 ${allGuessAuthors.size} | 总猜中视频 ${allGuessVideos.length}`);
+
+  const videoDetails = {};
+  for (const r of allResults) {
+    const key = r.videoAuthor;
+    if (!videoDetails[key]) {
+      videoDetails[key] = {
+        videoAuthor: r.videoAuthor,
+        uniqueId: r.uniqueId,
+        nickname: r.nickname,
+        locationCreated: r.locationCreated,
+      };
+    }
+  }
+
+  const output = {
+    videoDetails: Object.values(videoDetails),
+    commentUsers: [...commentUsers].sort(),
+    allCommentAuthorsList,
+    guessVideos: allGuessVideos,
+    guessAuthors: [...allGuessAuthors].sort(),
+    stats: {
+      totalVideos: allResults.length,
+      uniqueVideoAuthors: videoAuthors.size,
+      uniqueCommentAuthors: commentUsers.size,
+      uniqueGuessAuthors: allGuessAuthors.size,
+      totalGuessVideos: allGuessVideos.length,
+    },
+  };
+
+  return { output, browser, isExternal, captchaDetected: anyCaptchaDetected, captchaStage: anyCaptchaStage, captchaMessage: anyCaptchaMessage };
+}
+
+export { scrapeSingleVideo, runScrape };