tt-help-cli-ycl 1.0.8 → 1.1.0

package/src/scraper/modules/scroll-collector.cjs

@@ -0,0 +1,173 @@
+const { delay } = require('./page-helpers.cjs');
+
+/**
+ * 通用滚动收集器 - 三段式滚动策略
+ *
+ * 滚动逻辑由框架负责，收集逻辑由调用方通过 collectFn 提供。
+ *
+ * @param {Page} page - Playwright page 对象
+ * @param {Object} options
+ * @param {string} [options.container] - 滚动容器选择器，不传则用 window
+ * @param {boolean} [options.findScrollable] - 找到容器后向上查可滚动祖先
+ * @param {Function} options.collectFn - 在 page context 执行，接收 (containerEl, extraArgs)，
+ *   返回 { items: any[] }，items 为本轮收集到的原始数据
+ * @param {*} [options.extraArgs] - 额外参数，透传给 collectFn 的第二个参数
+ * @param {number[]} [options.delayRange] - 每轮滚动后等待时间 [min, max]，默认 [800, 1500]
+ * @param {number} [options.maxItems] - 最大去重后数据数量，不传则滚动到底（靠 stale 停）
+ * @param {number} [options.maxRounds] - 最大滚动轮次（安全保险），默认 200
+ * @param {number} [options.staleThreshold] - 连续 N 次无新增则停止，默认 3
+ * @param {Function} [options.uniqueKey] - 去重键提取函数 (item) => key，不传则不做去重
+ * @param {Function} [options.onRound] - 每轮回调 (round, newItems, allItems)
+ * @returns {Promise<any[]>} 所有收集到的数据（已去重如果传了 uniqueKey）
+ */
+async function scrollAndCollect(page, options) {
+  const {
+    container,
+    findScrollable = false,
+    collectFn,
+    extraArgs,
+    delayRange = [800, 1500],
+    maxItems,
+    maxRounds = 200,
+    staleThreshold = 3,
+    uniqueKey,
+    onRound,
+  } = options;
+
+  if (!collectFn) {
+    throw new Error('collectFn is required');
+  }
+
+  const fnStr = typeof collectFn === 'function' ? collectFn.toString() : collectFn;
+  const allItems = [];
+  const seenKeys = uniqueKey ? new Set() : null;
+  let staleCount = 0;
+
+  for (let round = 0; round < maxRounds; round++) {
+    // 1. 三段式滚动
+    await threePhaseScroll(page, { container, findScrollable });
+
+    // 2. 等待内容加载
+    await delay(delayRange[0], delayRange[1]);
+
+    // 3. 收集数据
+    const result = await page.evaluate(({ fn: fnStr, containerSelector, findScrollableFlag, args }) => {
+      let el;
+      if (!containerSelector) {
+        el = window;
+      } else {
+        el = document.querySelector(containerSelector);
+        if (!el) {
+          el = window;
+        } else if (findScrollableFlag) {
+          let current = el;
+          let found = false;
+          while (current && current !== document.body) {
+            if (current.scrollHeight > current.clientHeight + 10) {
+              el = current;
+              found = true;
+              break;
+            }
+            current = current.parentElement;
+          }
+          if (!found) {
+            el = document.body;
+          }
+        }
+      }
+
+      const fn = eval('(' + fnStr + ')');
+      return fn(el, args);
+    }, { fn: fnStr, containerSelector: container, findScrollableFlag: findScrollable, args: extraArgs });
+
+    const raw = result.items || [];
+
+    // 4. 去重：只保留本轮新增的
+    const newItems = uniqueKey
+      ? raw.filter(item => {
+          const key = uniqueKey(item);
+          if (seenKeys.has(key)) return false;
+          seenKeys.add(key);
+          return true;
+        })
+      : raw;
+
+    allItems.push(...newItems);
+
+    // 5. 回调通知
+    if (onRound) {
+      onRound(round, newItems, allItems);
+    }
+
+    // 6. 判断是否达到 maxItems
+    if (maxItems !== undefined && allItems.length >= maxItems) {
+      break;
+    }
+
+    // 7. stale 判断（基于本轮新增数）
+    if (newItems.length === 0) {
+      staleCount++;
+      if (staleCount >= staleThreshold) {
+        break;
+      }
+    } else {
+      staleCount = 0;
+    }
+  }
+
+  return allItems;
+}
+
+/**
+ * 三段式滚动：滚到底 → 回退一点 → 再滚到底
+ * 用于触发 TikTok 的 IntersectionObserver 懒加载
+ */
+async function threePhaseScroll(page, { container, findScrollable }) {
+  await page.evaluate(async (opts) => {
+    let el;
+
+    if (!opts.container) {
+      el = window;
+    } else {
+      el = document.querySelector(opts.container);
+      if (!el) {
+        el = window;
+      } else if (opts.findScrollable) {
+        let current = el;
+        let found = false;
+        while (current && current !== document.body) {
+          if (current.scrollHeight > current.clientHeight + 10) {
+            el = current;
+            found = true;
+            break;
+          }
+          current = current.parentElement;
+        }
+        if (!found) {
+          el = document.body;
+        }
+      }
+    }
+
+    const randDelay = (min, max) =>
+      new Promise(r => setTimeout(r, min + Math.random() * (max - min)));
+
+    if (el === window) {
+      window.scrollBy(0, window.innerHeight);
+      await randDelay(400, 800);
+      window.scrollBy(0, -200);
+      await randDelay(200, 400);
+      window.scrollBy(0, window.innerHeight);
+    } else {
+      el.scrollTop = el.scrollHeight;
+      await randDelay(400, 800);
+      el.scrollTop -= 100 + Math.random() * 100;
+      await randDelay(200, 400);
+      el.scrollTop = el.scrollHeight;
+    }
+  }, { container, findScrollable });
+}
+
+module.exports = {
+  scrollAndCollect,
+};

package/src/scraper/modules/video-scanner.cjs

@@ -1,43 +1,43 @@
-const { swipeNextVideo, getVideoAuthor, closeCommentPanel, getDelayConfig } = require('./page-helpers.cjs');
-const { extractCommentAuthors } = require('./comment-extractor');
-const { delay } = require('./page-helpers.cjs');
-
-async function scrapeSingleVideo(page, maxComments = 10) {
-  const videoAuthor = await getVideoAuthor(page);
-  if (!videoAuthor) {
-    throw new Error('无法获取视频作者');
-  }
-
-  const commentAuthors = await extractCommentAuthors(page, maxComments);
-  await closeCommentPanel(page);
-  const config = getDelayConfig();
-  await delay(Math.round(config.commentMax * 0.3), config.commentMax);
-
-  const uniqueAuthors = [...new Set(commentAuthors.map(c => c.author))];
-
-  return {
-    videoAuthor,
-    commentAuthors: uniqueAuthors,
-  };
-}
-
-async function scanAndScrape(page, maxComments = 10) {
-  const results = [];
-
-  for (let i = 0; i < maxComments * 5; i++) {
-    const result = await scrapeSingleVideo(page, maxComments);
-    results.push(result);
-    console.log(`[${results.length}] ${result.videoAuthor} | 评论作者: ${result.commentAuthors.length}`);
-
-    if (i < maxComments * 5 - 1) {
-      await swipeNextVideo(page);
-    }
-  }
-
-  return results;
-}
-
-module.exports = {
-  scrapeSingleVideo,
-  scanAndScrape,
-};
+const { swipeNextVideo, getVideoAuthor, closeCommentPanel, getDelayConfig } = require('./page-helpers.cjs');
+const { extractCommentAuthors } = require('./comment-extractor');
+const { delay } = require('./page-helpers.cjs');
+
+async function scrapeSingleVideo(page, maxComments = 10) {
+  const videoAuthor = await getVideoAuthor(page);
+  if (!videoAuthor) {
+    throw new Error('无法获取视频作者');
+  }
+
+  const commentAuthors = await extractCommentAuthors(page, maxComments);
+  await closeCommentPanel(page);
+  const config = getDelayConfig();
+  await delay(Math.round(config.commentMax * 0.3), config.commentMax);
+
+  const uniqueAuthors = [...new Set(commentAuthors.map(c => c.author))];
+
+  return {
+    videoAuthor,
+    commentAuthors: uniqueAuthors,
+  };
+}
+
+async function scanAndScrape(page, maxComments = 10) {
+  const results = [];
+
+  for (let i = 0; i < maxComments * 5; i++) {
+    const result = await scrapeSingleVideo(page, maxComments);
+    results.push(result);
+    console.log(`[${results.length}] ${result.videoAuthor} | 评论作者: ${result.commentAuthors.length}`);
+
+    if (i < maxComments * 5 - 1) {
+      await swipeNextVideo(page);
+    }
+  }
+
+  return results;
+}
+
+module.exports = {
+  scrapeSingleVideo,
+  scanAndScrape,
+};

package/src/test-auto-follow.cjs

@@ -0,0 +1,109 @@
+const path = require('path');
+const fs = require('fs');
+const { ensureBrowserReady, setDelayConfig } = require('./scraper/modules/page-helpers.cjs');
+const { processUser } = require('./auto-core.cjs');
+const { createStore } = require('./data-store.cjs');
+
+async function main() {
+  const outFile = path.join(__dirname, '..', 'results', 'auto-test.json');
+  const store = createStore(outFile);
+
+  setDelayConfig('fast');
+
+  const browser = await ensureBrowserReady();
+  let page;
+  try {
+    const contexts = browser.contexts();
+    page = null;
+    for (const ctx of contexts) {
+      for (const p of ctx.pages()) {
+        if (p.url().includes('tiktok.com')) { page = p; break; }
+      }
+      if (page) break;
+    }
+    if (!page) page = await contexts[0].newPage();
+
+    console.error('========== 测试 processUser + enableFollow ==========');
+    console.error('用户: @qiqi23280\n');
+
+    const result = await processUser(page, 'qiqi23280', {
+      collectMax: 1,
+      scrapeDepth: 1,
+      maxComments: 10,
+      maxGuess: 5,
+      preset: 'fast',
+      enableFollow: true,
+      maxFollowing: 50,
+      maxFollowers: 50,
+      browser,
+    }, console.error);
+
+    console.error('\n========== 结果验证 ==========');
+    let allPassed = true;
+
+    const checks = [
+      { label: '用户信息', ok: result.userInfo && result.userInfo.uniqueId, detail: result.userInfo?.uniqueId },
+      { label: '关注列表', ok: Array.isArray(result.discoveredFollowing) && result.discoveredFollowing.length > 0, detail: `${result.discoveredFollowing?.length || 0} 人` },
+      { label: '粉丝列表', ok: Array.isArray(result.discoveredFollowers) && result.discoveredFollowers.length > 0, detail: `${result.discoveredFollowers?.length || 0} 人` },
+      { label: '关注格式', ok: result.discoveredFollowing?.every(p => Array.isArray(p) && p.length === 2 && p[0].startsWith('@')), detail: null },
+      { label: '粉丝格式', ok: result.discoveredFollowers?.every(p => Array.isArray(p) && p.length === 2 && p[0].startsWith('@')), detail: null },
+      { label: '无错误', ok: !result.error, detail: result.error },
+    ];
+
+    for (const c of checks) {
+      const status = c.ok ? 'PASS' : 'FAIL';
+      const detailStr = c.detail !== null ? ` (${c.detail})` : '';
+      console.error(`  ${status}: ${c.label}${detailStr}`);
+      if (!c.ok) allPassed = false;
+    }
+
+    // 模拟入队逻辑
+    const queue = ['qiqi23280'];
+    const followingIds = (result.discoveredFollowing || []).map(([h]) => h.replace(/^@/, ''));
+    const followerIds = (result.discoveredFollowers || []).map(([h]) => h.replace(/^@/, ''));
+
+    for (const uid of followingIds) queue.push(uid);
+    for (const uid of followerIds) queue.push(uid);
+    const uniqueQueue = [...new Set(queue)];
+
+    console.error(`\n  队列长度: ${uniqueQueue.length}（关注 ${followingIds.length} + 粉丝 ${followerIds.length} + 种子 1）`);
+
+    // 写入 store 验证
+    store.addUser({
+      uniqueId: 'qiqi23280',
+      ...result.userInfo,
+      sources: ['seed'],
+    });
+    for (const [handle, name] of (result.discoveredFollowing || [])) {
+      store.addUser({ uniqueId: handle.replace(/^@/, ''), nickname: name, sources: ['following'] });
+    }
+    for (const [handle, name] of (result.discoveredFollowers || [])) {
+      store.addUser({ uniqueId: handle.replace(/^@/, ''), nickname: name, sources: ['follower'] });
+    }
+    store.save();
+
+    const allUsers = store.getAllUsers();
+    console.error(`  Store 用户数: ${allUsers.length}`);
+    
+    // 验证 source 标记
+    const followingUsers = allUsers.filter(u => u.sources?.includes('following'));
+    const followerUsers = allUsers.filter(u => u.sources?.includes('follower'));
+    console.error(`  关注来源: ${followingUsers.length} | 粉丝来源: ${followerUsers.length}`);
+    
+    if (followingUsers.length === 0 || followerUsers.length === 0) {
+      console.error('  FAIL: 缺少 following 或 follower 来源标记');
+      allPassed = false;
+    } else {
+      console.error('  PASS: 来源标记正确');
+    }
+
+    console.error(`\n${allPassed ? 'ALL PASSED' : 'SOME FAILED'}`);
+    console.error(`数据保存到: ${outFile}`);
+    process.exit(allPassed ? 0 : 1);
+
+  } finally {
+    await browser.close().catch(() => {});
+  }
+}
+
+main().catch(err => { console.error('FATAL:', err.message); process.exit(1); });

package/src/test-extractors.cjs

@@ -0,0 +1,75 @@
+const { ensureBrowserReady, delay, setDelayConfig } = require('./scraper/modules/page-helpers.cjs');
+const { extractCommentAuthors } = require('./scraper/modules/comment-extractor.cjs');
+const { extractGuessVideos } = require('./scraper/modules/guess-extractor.cjs');
+
+async function main() {
+  setDelayConfig('fast');
+
+  const videoUrl = process.argv[2] || 'https://www.tiktok.com/@porfirio.fructuoso/video/7615853535955111198';
+  console.error(`目标: ${videoUrl}`);
+
+  const browser = await ensureBrowserReady();
+  let page;
+  try {
+    const contexts = browser.contexts();
+    page = null;
+    for (const ctx of contexts) {
+      for (const p of ctx.pages()) {
+        if (p.url().includes('tiktok.com')) { page = p; break; }
+      }
+      if (page) break;
+    }
+    if (!page) {
+      page = await contexts[0].newPage();
+    }
+
+    await page.goto(videoUrl, { waitUntil: 'networkidle', timeout: 60000 });
+    await delay(5000, 8000);
+
+    console.error(`当前URL: ${page.url()}`);
+
+    let allPassed = true;
+
+    // ========== 评论提取 ==========
+    console.error('\n--- 评论提取 (max=30) ---');
+    const t1 = Date.now();
+    let commentUsers = [];
+    try { commentUsers = await extractCommentAuthors(page, 30); }
+    catch (e) { console.error(`  异常: ${e.message}`); }
+    console.error(`  耗时: ${((Date.now()-t1)/1000).toFixed(1)}s, 结果: ${commentUsers.length} 个`);
+
+    if (commentUsers.length > 0) {
+      const s = new Set(commentUsers);
+      const ok = s.size === commentUsers.length;
+      console.error(`  ${ok ? 'PASS' : 'FAIL'}: 唯一${s.size}/总数${commentUsers.length}`);
+      if (!ok) allPassed = false;
+    }
+
+    // ========== 猜你喜欢提取 ==========
+    console.error('\n--- 猜你喜欢提取 (max=20) ---');
+    const t2 = Date.now();
+    let guessVideos = [];
+    try { guessVideos = await extractGuessVideos(page, 20); }
+    catch (e) { console.error(`  异常: ${e.message}`); }
+    console.error(`  耗时: ${((Date.now()-t2)/1000).toFixed(1)}s, 结果: ${guessVideos.length} 个`);
+
+    if (guessVideos.length > 0) {
+      const ids = guessVideos.map(v => v.videoId);
+      const s = new Set(ids);
+      const ok = s.size === ids.length;
+      console.error(`  ${ok ? 'PASS' : 'FAIL'}: 唯一${s.size}/总数${ids.length}`);
+      if (!ok) allPassed = false;
+      const ok2 = guessVideos.every(v => v.author && v.videoId && v.url);
+      console.error(`  ${ok2 ? 'PASS' : 'FAIL'}: 结构完整`);
+      if (!ok2) allPassed = false;
+    }
+
+    console.error(`\n${allPassed ? 'ALL PASSED' : 'SOME FAILED'}`);
+    process.exit(allPassed ? 0 : 1);
+
+  } finally {
+    await browser.close().catch(() => {});
+  }
+}
+
+main().catch(err => { console.error('FATAL:', err.message); process.exit(1); });

package/src/test-follow.cjs

@@ -0,0 +1,41 @@
+const path = require('path');
+const { ensureBrowserReady, ensureTikTokPage, delay } = require('./scraper/modules/page-helpers.cjs');
+const { extractFollowAndFollowers } = require('./scraper/modules/follow-extractor.cjs');
+
+async function main() {
+  const url = process.argv[2] || 'https://www.tiktok.com/@qiqi23280';
+  console.error(`目标: ${url}`);
+
+  const browser = await ensureBrowserReady();
+  try {
+    const page = await ensureTikTokPage(browser, url);
+    await page.goto(url, { waitUntil: 'load', timeout: 30000 });
+    console.error('等待页面加载...');
+    await delay(3000, 5000);
+
+    console.error('开始提取关注和粉丝...\n');
+    const result = await extractFollowAndFollowers(page, {
+      log: console.error,
+    });
+
+    console.error('\n--- 提取完成 ---');
+    console.error(`关注: ${result.following.length} 人`);
+    console.error(`粉丝: ${result.followers.length} 人`);
+
+    const outDir = path.join(__dirname, '..', 'results');
+    const fs = require('fs');
+    fs.mkdirSync(outDir, { recursive: true });
+    const outPath = path.join(outDir, 'follow-result.json');
+    fs.writeFileSync(outPath, JSON.stringify(result, null, 2));
+    console.error(`已保存到 ${outPath}`);
+
+    console.log(JSON.stringify(result, null, 2));
+  } finally {
+    await browser.close().catch(() => {});
+  }
+}
+
+main().catch(err => {
+  console.error('错误:', err.message);
+  process.exit(1);
+});