tt-help-cli-ycl 1.3.0 → 1.3.1

package/src/scraper/modules/page-helpers.cjs

@@ -1,422 +0,0 @@
-const { chromium } = require('playwright');
-const { exec } = require('child_process');
-const http = require('http');
-const os = require('os');
-const path = require('path');
-
-const USER_SECTION_SIZE = 12000;
-const CDP_PORT = 9222;
-
-// --- Retry utilities ---
-
-const RETRYABLE_PATTERNS = [
-  'interrupted',
-  'Navigation.*interrupted',
-  'net::',
-  'ECONN',
-  'ETIMEDOUT',
-  'ENOTFOUND',
-  'EAI_AGAIN',
-  'ESOCKETRESET',
-  'connection.*refused',
-  'connection.*reset',
-  'failed.*navigate',
-  'target.*closed',
-  'crash',
-];
-
-function isRetryableError(error) {
-  if (!error) return false;
-  const msg = (error.message || error.toString() || '').toLowerCase();
-  return RETRYABLE_PATTERNS.some(p => new RegExp(p, 'i').test(msg));
-}
-
-async function retryWithBackoff(fn, { maxRetries = 3, baseDelay = 3000, log } = {}) {
-  let lastError;
-  for (let attempt = 0; attempt <= maxRetries; attempt++) {
-    try {
-      return await fn();
-    } catch (error) {
-      lastError = error;
-      if (attempt >= maxRetries || !isRetryableError(error)) {
-        throw error;
-      }
-      const jitter = Math.random() * 2000;
-      const waitTime = baseDelay * Math.pow(2, attempt) + jitter;
-      if (log) {
-        log(`  [重试] ${attempt + 1}/${maxRetries}，${Math.round(waitTime / 1000)}s 后重试...`);
-      }
-      await delay(Math.round(waitTime), Math.round(waitTime));
-    }
-  }
-  throw lastError;
-}
-
-// 预设延迟配置（毫秒）
-const DELAY_PRESETS = {
-  fast:   { switchMax: 300, commentMax: 200, fast: true },
-  normal: { switchMax: 1500, commentMax: 800 },
-  slow:   { switchMax: 3000, commentMax: 2000 },
-  stealth: { switchMax: 5000, commentMax: 3500 },
-};
-
-// 当前延迟配置
-const delayConfig = {
-  switchMax: 2500,
-  commentMax: 1500,
-  fast: false,
-};
-
-function setDelayConfig(config) {
-  if (typeof config === 'string') {
-    const preset = DELAY_PRESETS[config.toLowerCase()];
-    if (!preset) {
-      throw new Error(
-        `未知的延迟预设: ${config}\n` +
-        `可用预设: ${Object.keys(DELAY_PRESETS).join(', ')}`
-      );
-    }
-    delayConfig.switchMax = preset.switchMax;
-    delayConfig.commentMax = preset.commentMax;
-    delayConfig.fast = preset.fast || false;
-  } else if (typeof config === 'object') {
-    if (config.switchMax) delayConfig.switchMax = config.switchMax;
-    if (config.commentMax) delayConfig.commentMax = config.commentMax;
-    delayConfig.fast = config.fast || false;
-  }
-}
-
-function getDelayConfig() {
-  return { ...delayConfig };
-}
-
-function listDelayPresets() {
-  return DELAY_PRESETS;
-}
-
-function delay(min, max) {
-  const lo = Math.min(min, max);
-  const hi = Math.max(min, max);
-  let ms;
-  if (delayConfig.fast) {
-    ms = 0;
-  } else {
-    ms = Math.floor(Math.random() * (hi - lo + 1)) + lo;
-  }
-  return new Promise(r => setTimeout(r, ms));
-}
-
-async function openCommentPanel(page) {
-  await page.evaluate(() => {
-    const tabs = document.querySelectorAll('[class*="tabbar-item"]');
-    for (const tab of tabs) {
-      if (tab.textContent?.includes('评论')) {
-        tab.click();
-        break;
-      }
-    }
-  });
-  const c = delayConfig.commentMax;
-  await delay(Math.round(c * 0.5), c);
-}
-
-async function closeCommentPanel(page) {
-  await page.evaluate(() => {
-    const rightPanel = document.querySelector('[class*="RightPanelContainer"]');
-    if (rightPanel) {
-      const tabContainer = rightPanel.querySelector('[class*="TabContainer"]');
-      if (tabContainer) {
-        const closeOverlay = tabContainer.querySelector('div:last-child');
-        if (closeOverlay) closeOverlay.click();
-      }
-    }
-  });
-}
-
-async function getVideoAuthor(page) {
-  return await page.evaluate(() => {
-    const m = window.location.href.match(/@([^/]+)\/video/);
-    return m ? '@' + m[1] : null;
-  });
-}
-
-async function swipeNextVideo(page) {
-  await page.evaluate(() => {
-    const container = document.querySelector('[class*="ColumnListContainer"]');
-    if (container) {
-      container.scrollTop += 700;
-    }
-  });
-  const s = delayConfig.switchMax;
-  await delay(Math.round(s * 0.5), s);
-}
-
-function extractUserSection(html) {
-  const idx = html.indexOf('"uniqueId"');
-  if (idx < 0) return null;
-  return html.substring(idx, idx + USER_SECTION_SIZE);
-}
-
-function parseUserSection(section) {
-  const data = {};
-
-  for (const key of ['uniqueId', 'uid', 'secUid']) {
-    const m = section.match(new RegExp(`"${key}":"([^"]*)`));
-    if (m) data[key] = m[1];
-  }
-
-  for (const key of ['nickname', 'signature']) {
-    const m = section.match(new RegExp(`"${key}":"((?:[^"\\\\]|\\\\.)*)"`, 'g'));
-    if (m) {
-      const raw = m[0].replace(`"${key}":"`, '').replace(/"$/, '');
-      data[key] = raw.replace(/\\n/g, '\n').replace(/\\\\/g, '\\');
-    }
-  }
-
-  for (const key of ['ttSeller', 'verified']) {
-    const m = section.match(new RegExp(`"${key}":\\s*(true|false)`));
-    data[key] = m ? m[1] === 'true' : undefined;
-  }
-
-  for (const key of ['followerCount', 'followingCount', 'heartCount', 'videoCount', 'diggCount']) {
-    const m = section.match(new RegExp(`"${key}":(\\d+)`));
-    if (m) data[key] = parseInt(m[1], 10);
-  }
-
-  const mt = section.match(/"createTime":(\d+)/);
-  if (mt) data.createTime = parseInt(mt[1], 10);
-
-  const ma = section.match(/"avatarLarger":"([^"]*)/);
-  if (ma) data.avatarLarger = ma[1].replace(/\\u002F/g, '/');
-
-  return data;
-}
-
-function extractLocationCreated(html) {
-  const m = html.match(/"locationCreated":"([^"]*)/);
-  return m ? m[1] : null;
-}
-
-/**
- * 检查 CDP 端口是否已就绪
- */
-function checkCDPPort() {
-  return new Promise(resolve => {
-    const req = http.get(`http://127.0.0.1:${CDP_PORT}/json`, res => {
-      res.on('data', () => {});
-      res.on('end', () => resolve(res.statusCode === 200));
-    });
-    req.on('error', () => resolve(false));
-    req.setTimeout(3000, () => { resolve(false); req.destroy(); });
-  });
-}
-
-/**
- * 检查 Edge 主进程是否带有 --user-data-dir 参数
- */
-function checkEdgeArgs() {
-  return new Promise(resolve => {
-    const platform = os.platform();
-    let command;
-    if (platform === 'darwin') {
-      command = 'ps aux | grep "[M]icrosoft Edge" | grep -v "Helper\\|crashpad" | grep "user-data-dir"';
-    } else if (platform === 'win32') {
-      command = 'wmic process where "name like \"%msedge%\"" get commandline | findstr "user-data-dir"';
-    } else {
-      command = 'ps aux | grep "[m]sedge" | grep -v "Helper\\|crashpad" | grep "user-data-dir"';
-    }
-    exec(command, (err, stdout) => {
-      resolve(!err && stdout.trim().length > 0);
-    });
-  });
-}
-
-/**
- * 获取 Edge 浏览器可执行文件路径
- */
-function getEdgePath() {
-  const platform = os.platform();
-  if (platform === 'darwin') {
-    return '"Microsoft Edge"';
-  }
-  if (platform === 'win32') {
-    return 'msedge.exe';
-  }
-  return 'msedge';
-}
-
-/**
- * 杀掉已运行的 Edge 浏览器进程
- */
-function killEdgeProcesses() {
-  return new Promise((resolve, reject) => {
-    const platform = os.platform();
-    let command;
-
-    if (platform === 'darwin') {
-      command = 'killall -9 "Microsoft Edge" 2>/dev/null; rm -f ~/Library/Caches/Microsoft\\ Edge/Singleton*; true';
-    } else if (platform === 'win32') {
-      command = 'taskkill /F /IM msedge.exe 2>nul || exit 0';
-    } else {
-      command = 'pkill -9 -f msedge 2>/dev/null; true';
-    }
-
-    exec(command, () => {
-      resolve();
-    });
-  });
-}
-
-/**
- * 启动 Edge 浏览器并启用 CDP 调试端口
- */
-function launchEdgeWithCDP() {
-  return new Promise((resolve, reject) => {
-    const platform = os.platform();
-    const edgePath = getEdgePath();
-    let command;
-
-    const userDataDir = path.join(os.homedir(), 'Library', 'Application Support', 'Microsoft Edge For Testing');
-    if (platform === 'darwin') {
-      command = `open -a ${edgePath} --args --remote-debugging-port=${CDP_PORT} --user-data-dir="${userDataDir}"`;
-    } else if (platform === 'win32') {
-      command = `start msedge --remote-debugging-port=${CDP_PORT} --user-data-dir="${userDataDir}"`;
-    } else {
-      command = `msedge --remote-debugging-port=${CDP_PORT} --user-data-dir="${userDataDir}" &`;
-    }
-
-    exec(command, (err, stdout, stderr) => {
-      if (err) {
-        reject(new Error(`启动 Edge 浏览器失败: ${err.message}`));
-        return;
-      }
-      resolve();
-    });
-  });
-}
-
-/**
- * 等待 CDP 端口就绪，最多等待 timeout 毫秒
- */
-async function waitForCDP(timeout = 30000, interval = 1000) {
-  const start = Date.now();
-  while (Date.now() - start < timeout) {
-    const ready = await checkCDPPort();
-    if (ready) return true;
-    await new Promise(r => setTimeout(r, interval));
-  }
-  return false;
-}
-
-/**
- * 确保浏览器已启动且 CDP 端口可用。
- * 如果端口未就绪，先杀掉已运行的 Edge，再以调试模式重启。
- * 返回 playwright 的 browser 实例。
- */
-async function ensureBrowserReady() {
-  const isReady = await checkCDPPort();
-  let needLaunch = !isReady;
-
-  if (!needLaunch) {
-    const edgeArgsValid = await checkEdgeArgs();
-    if (!edgeArgsValid) {
-      console.error(`Edge 已运行但启动参数不完整，正在重启...`);
-      await killEdgeProcesses();
-      await new Promise(r => setTimeout(r, 3000));
-      needLaunch = true;
-    }
-  }
-
-  if (needLaunch) {
-    const edgeRunning = await isEdgeRunning();
-    if (edgeRunning) {
-      console.error(`Edge 已运行但 CDP 端口 ${CDP_PORT} 未启用，正在重启...`);
-      await killEdgeProcesses();
-      await new Promise(r => setTimeout(r, 3000));
-    } else {
-      console.error(`CDP 端口 ${CDP_PORT} 未就绪，正在启动 Edge 浏览器...`);
-    }
-    await launchEdgeWithCDP();
-
-    console.error('等待浏览器启动...');
-    const launched = await waitForCDP();
-    if (!launched) {
-      throw new Error(
-        `等待 CDP 端口 ${CDP_PORT} 超时。请确认 Edge 浏览器已安装，\n` +
-        '或手动启动: Microsoft Edge --remote-debugging-port=9222'
-      );
-    }
-    console.error('浏览器启动成功');
-  }
-
-  const browser = await chromium.connectOverCDP(`http://127.0.0.1:${CDP_PORT}`);
-  return browser;
-}
-
-/**
- * 检查 Edge 浏览器是否正在运行
- */
-function isEdgeRunning() {
-  return new Promise(resolve => {
-    const platform = os.platform();
-    let command;
-    if (platform === 'darwin') {
-      command = 'ps aux | grep -q "[M]icrosoft Edge.app/Contents/MacOS/Microsoft Edge" 2>/dev/null';
-    } else if (platform === 'win32') {
-      command = 'tasklist /FI "IMAGENAME eq msedge.exe" 2>nul | findstr /I msedge';
-    } else {
-      command = 'pgrep -f msedge > /dev/null 2>&1';
-    }
-    exec(command, (err) => {
-      resolve(!err);
-    });
-  });
-}
-
-/**
- * 在已连接的 browser 中找到或打开 TikTok 页面
- */
-async function ensureTikTokPage(browser, url) {
-  const contexts = browser.contexts();
-  let page = null;
-
-  for (const ctx of contexts) {
-    for (const p of ctx.pages()) {
-      if (p.url().includes('tiktok.com')) {
-        page = p;
-        break;
-      }
-    }
-    if (page) break;
-  }
-
-  if (!page) {
-    console.error('未找到 TikTok 页面，正在打开...');
-    const defaultCtx = browser.contexts()[0];
-    page = await defaultCtx.newPage();
-    await retryWithBackoff(() => page.goto(url, { waitUntil: 'load', timeout: 30000 }));
-    await delay(Math.round(delayConfig.switchMax * 0.5), delayConfig.switchMax);
-    console.error('TikTok 页面已打开');
-  }
-
-  return page;
-}
-
-module.exports = {
-  delay,
-  openCommentPanel,
-  closeCommentPanel,
-  getVideoAuthor,
-  swipeNextVideo,
-  extractUserSection,
-  parseUserSection,
-  extractLocationCreated,
-  ensureBrowserReady,
-  ensureTikTokPage,
-  isEdgeRunning,
-  setDelayConfig,
-  getDelayConfig,
-  listDelayPresets,
-  retryWithBackoff,
-  isRetryableError,
-};

package/src/scraper/modules/scroll-collector.cjs

@@ -1,173 +0,0 @@
-const { delay } = require('./page-helpers.cjs');
-
-/**
- * 通用滚动收集器 - 三段式滚动策略
- *
- * 滚动逻辑由框架负责，收集逻辑由调用方通过 collectFn 提供。
- *
- * @param {Page} page - Playwright page 对象
- * @param {Object} options
- * @param {string} [options.container] - 滚动容器选择器，不传则用 window
- * @param {boolean} [options.findScrollable] - 找到容器后向上查可滚动祖先
- * @param {Function} options.collectFn - 在 page context 执行，接收 (containerEl, extraArgs)，
- *   返回 { items: any[] }，items 为本轮收集到的原始数据
- * @param {*} [options.extraArgs] - 额外参数，透传给 collectFn 的第二个参数
- * @param {number[]} [options.delayRange] - 每轮滚动后等待时间 [min, max]，默认 [800, 1500]
- * @param {number} [options.maxItems] - 最大去重后数据数量，不传则滚动到底（靠 stale 停）
- * @param {number} [options.maxRounds] - 最大滚动轮次（安全保险），默认 200
- * @param {number} [options.staleThreshold] - 连续 N 次无新增则停止，默认 3
- * @param {Function} [options.uniqueKey] - 去重键提取函数 (item) => key，不传则不做去重
- * @param {Function} [options.onRound] - 每轮回调 (round, newItems, allItems)
- * @returns {Promise<any[]>} 所有收集到的数据（已去重如果传了 uniqueKey）
- */
-async function scrollAndCollect(page, options) {
-  const {
-    container,
-    findScrollable = false,
-    collectFn,
-    extraArgs,
-    delayRange = [800, 1500],
-    maxItems,
-    maxRounds = 200,
-    staleThreshold = 3,
-    uniqueKey,
-    onRound,
-  } = options;
-
-  if (!collectFn) {
-    throw new Error('collectFn is required');
-  }
-
-  const fnStr = typeof collectFn === 'function' ? collectFn.toString() : collectFn;
-  const allItems = [];
-  const seenKeys = uniqueKey ? new Set() : null;
-  let staleCount = 0;
-
-  for (let round = 0; round < maxRounds; round++) {
-    // 1. 三段式滚动
-    await threePhaseScroll(page, { container, findScrollable });
-
-    // 2. 等待内容加载
-    await delay(delayRange[0], delayRange[1]);
-
-    // 3. 收集数据
-    const result = await page.evaluate(({ fn: fnStr, containerSelector, findScrollableFlag, args }) => {
-      let el;
-      if (!containerSelector) {
-        el = window;
-      } else {
-        el = document.querySelector(containerSelector);
-        if (!el) {
-          el = window;
-        } else if (findScrollableFlag) {
-          let current = el;
-          let found = false;
-          while (current && current !== document.body) {
-            if (current.scrollHeight > current.clientHeight + 10) {
-              el = current;
-              found = true;
-              break;
-            }
-            current = current.parentElement;
-          }
-          if (!found) {
-            el = document.body;
-          }
-        }
-      }
-
-      const fn = eval('(' + fnStr + ')');
-      return fn(el, args);
-    }, { fn: fnStr, containerSelector: container, findScrollableFlag: findScrollable, args: extraArgs });
-
-    const raw = result.items || [];
-
-    // 4. 去重：只保留本轮新增的
-    const newItems = uniqueKey
-      ? raw.filter(item => {
-          const key = uniqueKey(item);
-          if (seenKeys.has(key)) return false;
-          seenKeys.add(key);
-          return true;
-        })
-      : raw;
-
-    allItems.push(...newItems);
-
-    // 5. 回调通知
-    if (onRound) {
-      onRound(round, newItems, allItems);
-    }
-
-    // 6. 判断是否达到 maxItems
-    if (maxItems !== undefined && allItems.length >= maxItems) {
-      break;
-    }
-
-    // 7. stale 判断（基于本轮新增数）
-    if (newItems.length === 0) {
-      staleCount++;
-      if (staleCount >= staleThreshold) {
-        break;
-      }
-    } else {
-      staleCount = 0;
-    }
-  }
-
-  return allItems;
-}
-
-/**
- * 三段式滚动：滚到底 → 回退一点 → 再滚到底
- * 用于触发 TikTok 的 IntersectionObserver 懒加载
- */
-async function threePhaseScroll(page, { container, findScrollable }) {
-  await page.evaluate(async (opts) => {
-    let el;
-
-    if (!opts.container) {
-      el = window;
-    } else {
-      el = document.querySelector(opts.container);
-      if (!el) {
-        el = window;
-      } else if (opts.findScrollable) {
-        let current = el;
-        let found = false;
-        while (current && current !== document.body) {
-          if (current.scrollHeight > current.clientHeight + 10) {
-            el = current;
-            found = true;
-            break;
-          }
-          current = current.parentElement;
-        }
-        if (!found) {
-          el = document.body;
-        }
-      }
-    }
-
-    const randDelay = (min, max) =>
-      new Promise(r => setTimeout(r, min + Math.random() * (max - min)));
-
-    if (el === window) {
-      window.scrollBy(0, window.innerHeight);
-      await randDelay(400, 800);
-      window.scrollBy(0, -200);
-      await randDelay(200, 400);
-      window.scrollBy(0, window.innerHeight);
-    } else {
-      el.scrollTop = el.scrollHeight;
-      await randDelay(400, 800);
-      el.scrollTop -= 100 + Math.random() * 100;
-      await randDelay(200, 400);
-      el.scrollTop = el.scrollHeight;
-    }
-  }, { container, findScrollable });
-}
-
-module.exports = {
-  scrollAndCollect,
-};

package/src/scraper/modules/video-scanner.cjs

@@ -1,43 +0,0 @@
-const { swipeNextVideo, getVideoAuthor, closeCommentPanel, getDelayConfig } = require('./page-helpers.cjs');
-const { extractCommentAuthors } = require('./comment-extractor');
-const { delay } = require('./page-helpers.cjs');
-
-async function scrapeSingleVideo(page, maxComments = 10) {
-  const videoAuthor = await getVideoAuthor(page);
-  if (!videoAuthor) {
-    throw new Error('无法获取视频作者');
-  }
-
-  const commentAuthors = await extractCommentAuthors(page, maxComments);
-  await closeCommentPanel(page);
-  const config = getDelayConfig();
-  await delay(Math.round(config.commentMax * 0.3), config.commentMax);
-
-  const uniqueAuthors = [...new Set(commentAuthors.map(c => c.author))];
-
-  return {
-    videoAuthor,
-    commentAuthors: uniqueAuthors,
-  };
-}
-
-async function scanAndScrape(page, maxComments = 10) {
-  const results = [];
-
-  for (let i = 0; i < maxComments * 5; i++) {
-    const result = await scrapeSingleVideo(page, maxComments);
-    results.push(result);
-    console.log(`[${results.length}] ${result.videoAuthor} | 评论作者: ${result.commentAuthors.length}`);
-
-    if (i < maxComments * 5 - 1) {
-      await swipeNextVideo(page);
-    }
-  }
-
-  return results;
-}
-
-module.exports = {
-  scrapeSingleVideo,
-  scanAndScrape,
-};