tt-help-cli-ycl 1.3.6 → 1.3.8

package/src/scraper/modules/page-helpers.mjs

@@ -1,48 +1,48 @@
-import {
-  delay,
-  getDelayConfig,
-  setDelayConfig,
-  listDelayPresets,
-  DELAY_PRESETS,
-} from '../../lib/delay.js';
-import { ensureBrowserReady } from '../../lib/browser/cdp.js';
-import {
-  ensureTikTokPage,
-  closeCommentPanel,
-  findTikTokPage,
-  getOrCreatePage,
-  isLoggedIn,
-  assertPageUrl,
-} from '../../lib/browser/page.js';
-import { retryWithBackoff, isRetryableError } from '../../lib/retry.js';
-import {
-  extractUserSection,
-  parseUserSection,
-  extractLocationCreated,
-  USER_SECTION_SIZE,
-} from '../../lib/parser.js';
-import { detectPageError } from './page-error-detector.mjs';
-
-export {
-  delay,
-  setDelayConfig,
-  getDelayConfig,
-  listDelayPresets,
-  DELAY_PRESETS,
-  ensureBrowserReady,
-  ensureTikTokPage,
-  closeCommentPanel,
-  findTikTokPage,
-  getOrCreatePage,
-  isLoggedIn,
-  assertPageUrl,
-  retryWithBackoff,
-  isRetryableError,
-  extractUserSection,
-  parseUserSection,
-  extractLocationCreated,
-  USER_SECTION_SIZE,
-  detectPageError,
-};
-
-export const CDP_PORT = 9222;
+import {
+  delay,
+  getDelayConfig,
+  setDelayConfig,
+  listDelayPresets,
+  DELAY_PRESETS,
+} from '../../lib/delay.js';
+import { ensureBrowserReady } from '../../lib/browser/cdp.js';
+import {
+  ensureTikTokPage,
+  closeCommentPanel,
+  findTikTokPage,
+  getOrCreatePage,
+  isLoggedIn,
+  assertPageUrl,
+} from '../../lib/browser/page.js';
+import { retryWithBackoff, isRetryableError } from '../../lib/retry.js';
+import {
+  extractUserSection,
+  parseUserSection,
+  extractLocationCreated,
+  USER_SECTION_SIZE,
+} from '../../lib/parser.js';
+import { detectPageError } from './page-error-detector.mjs';
+
+export {
+  delay,
+  setDelayConfig,
+  getDelayConfig,
+  listDelayPresets,
+  DELAY_PRESETS,
+  ensureBrowserReady,
+  ensureTikTokPage,
+  closeCommentPanel,
+  findTikTokPage,
+  getOrCreatePage,
+  isLoggedIn,
+  assertPageUrl,
+  retryWithBackoff,
+  isRetryableError,
+  extractUserSection,
+  parseUserSection,
+  extractLocationCreated,
+  USER_SECTION_SIZE,
+  detectPageError,
+};
+
+export const CDP_PORT = 9222;

package/src/scraper/modules/scroll-collector.mjs

@@ -1,189 +1,189 @@
-import { delay } from "../../lib/delay.js";
-import { detectPageError } from "./page-error-detector.mjs";
-
-async function doCollect(
-  page,
-  { container, findScrollable, fnStr, extraArgs },
-) {
-  return page.evaluate(
-    ({ fn: fnStr, containerSelector, findScrollableFlag, args }) => {
-      let el;
-      if (!containerSelector) {
-        el = window;
-      } else {
-        el = document.querySelector(containerSelector);
-        if (!el) {
-          el = window;
-        } else if (findScrollableFlag) {
-          let current = el;
-          let found = false;
-          while (current && current !== document.body) {
-            if (current.scrollHeight > current.clientHeight + 10) {
-              el = current;
-              found = true;
-              break;
-            }
-            current = current.parentElement;
-          }
-          if (!found) el = document.body;
-        }
-      }
-      const fn = eval("(" + fnStr + ")");
-      return fn(el, args);
-    },
-    {
-      fn: fnStr,
-      containerSelector: container,
-      findScrollableFlag: findScrollable,
-      args: extraArgs,
-    },
-  );
-}
-
-const LOADING_SELECTORS = [
-  '[class*="loading"]',
-  '[class*="Loading"]',
-  '[class*="spinner"]',
-  '[class*="Spinner"]',
-  '[class*="skeleton"]',
-  '[class*="Skeleton"]',
-  '[aria-busy="true"]',
-];
-
-async function waitForLoading(page) {
-  const maxWait = 5000;
-  const startTime = Date.now();
-  while (Date.now() - startTime < maxWait) {
-    const isLoading = await page.evaluate((sels) => {
-      if (document.readyState !== "complete") return true;
-      for (const sel of sels) {
-        const el = document.querySelector(sel);
-        if (el && el.offsetParent !== null) return true;
-      }
-      return false;
-    }, LOADING_SELECTORS);
-    if (!isLoading) return;
-    await delay(300, 600);
-  }
-}
-
-export async function scrollAndCollect(page, options) {
-  const {
-    container,
-    findScrollable = false,
-    collectFn,
-    extraArgs,
-    delayRange = [800, 1500],
-    maxItems,
-    maxRounds = 200,
-    staleThreshold = 3,
-    uniqueKey,
-    onRound,
-  } = options;
-
-  if (!collectFn) throw new Error("collectFn is required");
-
-  const fnStr =
-    typeof collectFn === "function" ? collectFn.toString() : collectFn;
-  const allItems = [];
-  const seenKeys = uniqueKey ? new Set() : null;
-  let staleCount = 0;
-
-  const processItems = (result) => {
-    const raw = result.items || [];
-    const newItems = uniqueKey
-      ? raw.filter((item) => {
-          const key = uniqueKey(item);
-          if (seenKeys.has(key)) return false;
-          seenKeys.add(key);
-          return true;
-        })
-      : raw;
-    allItems.push(...newItems);
-    return newItems;
-  };
-
-  const isDone = (newItems) => {
-    if (maxItems !== undefined && allItems.length >= maxItems) return true;
-    if (newItems.length === 0) {
-      staleCount++;
-      if (staleCount >= staleThreshold) return true;
-    } else {
-      staleCount = 0;
-    }
-    return false;
-  };
-
-  const collectCtx = { container, findScrollable, fnStr, extraArgs };
-
-  const pageError = await detectPageError(page);
-
-  if (pageError) return [];
-
-  await waitForLoading(page);
-  let result = await doCollect(page, collectCtx);
-  let newItems = processItems(result);
-  if (onRound) onRound(0, newItems, allItems);
-  if (isDone(newItems)) return allItems;
-
-  for (let round = 1; round < maxRounds; round++) {
-    await threePhaseScroll(page, { container, findScrollable });
-    await delay(delayRange[0], delayRange[1]);
-    await waitForLoading(page);
-
-    result = await doCollect(page, collectCtx);
-    newItems = processItems(result);
-
-    if (onRound) onRound(round, newItems, allItems);
-
-    if (isDone(newItems)) break;
-  }
-
-  return allItems;
-}
-
-async function threePhaseScroll(page, { container, findScrollable }) {
-  await page.evaluate(
-    async (opts) => {
-      let el;
-      if (!opts.container) {
-        el = window;
-      } else {
-        el = document.querySelector(opts.container);
-        if (!el) {
-          el = window;
-        } else if (opts.findScrollable) {
-          let current = el;
-          let found = false;
-          while (current && current !== document.body) {
-            if (current.scrollHeight > current.clientHeight + 10) {
-              el = current;
-              found = true;
-              break;
-            }
-            current = current.parentElement;
-          }
-          if (!found) el = document.body;
-        }
-      }
-
-      const randDelay = (min, max) =>
-        new Promise((r) => setTimeout(r, min + Math.random() * (max - min)));
-
-      if (el === window) {
-        window.scrollBy(0, window.innerHeight);
-        await randDelay(400, 800);
-        window.scrollBy(0, -200);
-        await randDelay(200, 400);
-        window.scrollBy(0, window.innerHeight);
-      } else {
-        el.scrollTop = el.scrollHeight;
-        await randDelay(400, 800);
-        el.scrollTop -= 100 + Math.random() * 100;
-        await randDelay(200, 400);
-        el.scrollTop = el.scrollHeight;
-      }
-    },
-    { container, findScrollable },
-  );
-}
+import { delay } from "../../lib/delay.js";
+import { detectPageError } from "./page-error-detector.mjs";
+
+async function doCollect(
+  page,
+  { container, findScrollable, fnStr, extraArgs },
+) {
+  return page.evaluate(
+    ({ fn: fnStr, containerSelector, findScrollableFlag, args }) => {
+      let el;
+      if (!containerSelector) {
+        el = window;
+      } else {
+        el = document.querySelector(containerSelector);
+        if (!el) {
+          el = window;
+        } else if (findScrollableFlag) {
+          let current = el;
+          let found = false;
+          while (current && current !== document.body) {
+            if (current.scrollHeight > current.clientHeight + 10) {
+              el = current;
+              found = true;
+              break;
+            }
+            current = current.parentElement;
+          }
+          if (!found) el = document.body;
+        }
+      }
+      const fn = eval("(" + fnStr + ")");
+      return fn(el, args);
+    },
+    {
+      fn: fnStr,
+      containerSelector: container,
+      findScrollableFlag: findScrollable,
+      args: extraArgs,
+    },
+  );
+}
+
+const LOADING_SELECTORS = [
+  '[class*="loading"]',
+  '[class*="Loading"]',
+  '[class*="spinner"]',
+  '[class*="Spinner"]',
+  '[class*="skeleton"]',
+  '[class*="Skeleton"]',
+  '[aria-busy="true"]',
+];
+
+async function waitForLoading(page) {
+  const maxWait = 5000;
+  const startTime = Date.now();
+  while (Date.now() - startTime < maxWait) {
+    const isLoading = await page.evaluate((sels) => {
+      if (document.readyState !== "complete") return true;
+      for (const sel of sels) {
+        const el = document.querySelector(sel);
+        if (el && el.offsetParent !== null) return true;
+      }
+      return false;
+    }, LOADING_SELECTORS);
+    if (!isLoading) return;
+    await delay(300, 600);
+  }
+}
+
+export async function scrollAndCollect(page, options) {
+  const {
+    container,
+    findScrollable = false,
+    collectFn,
+    extraArgs,
+    delayRange = [800, 1500],
+    maxItems,
+    maxRounds = 200,
+    staleThreshold = 3,
+    uniqueKey,
+    onRound,
+  } = options;
+
+  if (!collectFn) throw new Error("collectFn is required");
+
+  const fnStr =
+    typeof collectFn === "function" ? collectFn.toString() : collectFn;
+  const allItems = [];
+  const seenKeys = uniqueKey ? new Set() : null;
+  let staleCount = 0;
+
+  const processItems = (result) => {
+    const raw = result.items || [];
+    const newItems = uniqueKey
+      ? raw.filter((item) => {
+          const key = uniqueKey(item);
+          if (seenKeys.has(key)) return false;
+          seenKeys.add(key);
+          return true;
+        })
+      : raw;
+    allItems.push(...newItems);
+    return newItems;
+  };
+
+  const isDone = (newItems) => {
+    if (maxItems !== undefined && allItems.length >= maxItems) return true;
+    if (newItems.length === 0) {
+      staleCount++;
+      if (staleCount >= staleThreshold) return true;
+    } else {
+      staleCount = 0;
+    }
+    return false;
+  };
+
+  const collectCtx = { container, findScrollable, fnStr, extraArgs };
+
+  const pageError = await detectPageError(page);
+
+  if (pageError) return [];
+
+  await waitForLoading(page);
+  let result = await doCollect(page, collectCtx);
+  let newItems = processItems(result);
+  if (onRound) onRound(0, newItems, allItems);
+  if (isDone(newItems)) return allItems;
+
+  for (let round = 1; round < maxRounds; round++) {
+    await threePhaseScroll(page, { container, findScrollable });
+    await delay(delayRange[0], delayRange[1]);
+    await waitForLoading(page);
+
+    result = await doCollect(page, collectCtx);
+    newItems = processItems(result);
+
+    if (onRound) onRound(round, newItems, allItems);
+
+    if (isDone(newItems)) break;
+  }
+
+  return allItems;
+}
+
+async function threePhaseScroll(page, { container, findScrollable }) {
+  await page.evaluate(
+    async (opts) => {
+      let el;
+      if (!opts.container) {
+        el = window;
+      } else {
+        el = document.querySelector(opts.container);
+        if (!el) {
+          el = window;
+        } else if (opts.findScrollable) {
+          let current = el;
+          let found = false;
+          while (current && current !== document.body) {
+            if (current.scrollHeight > current.clientHeight + 10) {
+              el = current;
+              found = true;
+              break;
+            }
+            current = current.parentElement;
+          }
+          if (!found) el = document.body;
+        }
+      }
+
+      const randDelay = (min, max) =>
+        new Promise((r) => setTimeout(r, min + Math.random() * (max - min)));
+
+      if (el === window) {
+        window.scrollBy(0, window.innerHeight);
+        await randDelay(400, 800);
+        window.scrollBy(0, -200);
+        await randDelay(200, 400);
+        window.scrollBy(0, window.innerHeight);
+      } else {
+        el.scrollTop = el.scrollHeight;
+        await randDelay(400, 800);
+        el.scrollTop -= 100 + Math.random() * 100;
+        await randDelay(200, 400);
+        el.scrollTop = el.scrollHeight;
+      }
+    },
+    { container, findScrollable },
+  );
+}

package/src/test-auto-follow.cjs

@@ -0,0 +1,109 @@
+const path = require('path');
+const fs = require('fs');
+const { ensureBrowserReady, setDelayConfig } = require('./scraper/modules/page-helpers.cjs');
+const { processUser } = require('./auto-core.cjs');
+const { createStore } = require('./data-store.cjs');
+
+async function main() {
+  const outFile = path.join(__dirname, '..', 'results', 'auto-test.json');
+  const store = createStore(outFile);
+
+  setDelayConfig('fast');
+
+  const browser = await ensureBrowserReady();
+  let page;
+  try {
+    const contexts = browser.contexts();
+    page = null;
+    for (const ctx of contexts) {
+      for (const p of ctx.pages()) {
+        if (p.url().includes('tiktok.com')) { page = p; break; }
+      }
+      if (page) break;
+    }
+    if (!page) page = await contexts[0].newPage();
+
+    console.error('========== 测试 processUser + enableFollow ==========');
+    console.error('用户: @qiqi23280\n');
+
+    const result = await processUser(page, 'qiqi23280', {
+      collectMax: 1,
+      scrapeDepth: 1,
+      maxComments: 10,
+      maxGuess: 5,
+      preset: 'fast',
+      enableFollow: true,
+      maxFollowing: 50,
+      maxFollowers: 50,
+      browser,
+    }, console.error);
+
+    console.error('\n========== 结果验证 ==========');
+    let allPassed = true;
+
+    const checks = [
+      { label: '用户信息', ok: result.userInfo && result.userInfo.uniqueId, detail: result.userInfo?.uniqueId },
+      { label: '关注列表', ok: Array.isArray(result.discoveredFollowing) && result.discoveredFollowing.length > 0, detail: `${result.discoveredFollowing?.length || 0} 人` },
+      { label: '粉丝列表', ok: Array.isArray(result.discoveredFollowers) && result.discoveredFollowers.length > 0, detail: `${result.discoveredFollowers?.length || 0} 人` },
+      { label: '关注格式', ok: result.discoveredFollowing?.every(p => Array.isArray(p) && p.length === 2 && p[0].startsWith('@')), detail: null },
+      { label: '粉丝格式', ok: result.discoveredFollowers?.every(p => Array.isArray(p) && p.length === 2 && p[0].startsWith('@')), detail: null },
+      { label: '无错误', ok: !result.error, detail: result.error },
+    ];
+
+    for (const c of checks) {
+      const status = c.ok ? 'PASS' : 'FAIL';
+      const detailStr = c.detail !== null ? ` (${c.detail})` : '';
+      console.error(`  ${status}: ${c.label}${detailStr}`);
+      if (!c.ok) allPassed = false;
+    }
+
+    // 模拟入队逻辑
+    const queue = ['qiqi23280'];
+    const followingIds = (result.discoveredFollowing || []).map(([h]) => h.replace(/^@/, ''));
+    const followerIds = (result.discoveredFollowers || []).map(([h]) => h.replace(/^@/, ''));
+
+    for (const uid of followingIds) queue.push(uid);
+    for (const uid of followerIds) queue.push(uid);
+    const uniqueQueue = [...new Set(queue)];
+
+    console.error(`\n  队列长度: ${uniqueQueue.length}（关注 ${followingIds.length} + 粉丝 ${followerIds.length} + 种子 1）`);
+
+    // 写入 store 验证
+    store.addUser({
+      uniqueId: 'qiqi23280',
+      ...result.userInfo,
+      sources: ['seed'],
+    });
+    for (const [handle, name] of (result.discoveredFollowing || [])) {
+      store.addUser({ uniqueId: handle.replace(/^@/, ''), nickname: name, sources: ['following'] });
+    }
+    for (const [handle, name] of (result.discoveredFollowers || [])) {
+      store.addUser({ uniqueId: handle.replace(/^@/, ''), nickname: name, sources: ['follower'] });
+    }
+    store.save();
+
+    const allUsers = store.getAllUsers();
+    console.error(`  Store 用户数: ${allUsers.length}`);
+    
+    // 验证 source 标记
+    const followingUsers = allUsers.filter(u => u.sources?.includes('following'));
+    const followerUsers = allUsers.filter(u => u.sources?.includes('follower'));
+    console.error(`  关注来源: ${followingUsers.length} | 粉丝来源: ${followerUsers.length}`);
+    
+    if (followingUsers.length === 0 || followerUsers.length === 0) {
+      console.error('  FAIL: 缺少 following 或 follower 来源标记');
+      allPassed = false;
+    } else {
+      console.error('  PASS: 来源标记正确');
+    }
+
+    console.error(`\n${allPassed ? 'ALL PASSED' : 'SOME FAILED'}`);
+    console.error(`数据保存到: ${outFile}`);
+    process.exit(allPassed ? 0 : 1);
+
+  } finally {
+    await browser.close().catch(() => {});
+  }
+}
+
+main().catch(err => { console.error('FATAL:', err.message); process.exit(1); });

package/src/test-extractors.cjs

@@ -0,0 +1,75 @@
+const { ensureBrowserReady, delay, setDelayConfig } = require('./scraper/modules/page-helpers.cjs');
+const { extractCommentAuthors } = require('./scraper/modules/comment-extractor.cjs');
+const { extractGuessVideos } = require('./scraper/modules/guess-extractor.cjs');
+
+async function main() {
+  setDelayConfig('fast');
+
+  const videoUrl = process.argv[2] || 'https://www.tiktok.com/@porfirio.fructuoso/video/7615853535955111198';
+  console.error(`目标: ${videoUrl}`);
+
+  const browser = await ensureBrowserReady();
+  let page;
+  try {
+    const contexts = browser.contexts();
+    page = null;
+    for (const ctx of contexts) {
+      for (const p of ctx.pages()) {
+        if (p.url().includes('tiktok.com')) { page = p; break; }
+      }
+      if (page) break;
+    }
+    if (!page) {
+      page = await contexts[0].newPage();
+    }
+
+    await page.goto(videoUrl, { waitUntil: 'networkidle', timeout: 60000 });
+    await delay(5000, 8000);
+
+    console.error(`当前URL: ${page.url()}`);
+
+    let allPassed = true;
+
+    // ========== 评论提取 ==========
+    console.error('\n--- 评论提取 (max=30) ---');
+    const t1 = Date.now();
+    let commentUsers = [];
+    try { commentUsers = await extractCommentAuthors(page, 30); }
+    catch (e) { console.error(`  异常: ${e.message}`); }
+    console.error(`  耗时: ${((Date.now()-t1)/1000).toFixed(1)}s, 结果: ${commentUsers.length} 个`);
+
+    if (commentUsers.length > 0) {
+      const s = new Set(commentUsers);
+      const ok = s.size === commentUsers.length;
+      console.error(`  ${ok ? 'PASS' : 'FAIL'}: 唯一${s.size}/总数${commentUsers.length}`);
+      if (!ok) allPassed = false;
+    }
+
+    // ========== 猜你喜欢提取 ==========
+    console.error('\n--- 猜你喜欢提取 (max=20) ---');
+    const t2 = Date.now();
+    let guessVideos = [];
+    try { guessVideos = await extractGuessVideos(page, 20); }
+    catch (e) { console.error(`  异常: ${e.message}`); }
+    console.error(`  耗时: ${((Date.now()-t2)/1000).toFixed(1)}s, 结果: ${guessVideos.length} 个`);
+
+    if (guessVideos.length > 0) {
+      const ids = guessVideos.map(v => v.videoId);
+      const s = new Set(ids);
+      const ok = s.size === ids.length;
+      console.error(`  ${ok ? 'PASS' : 'FAIL'}: 唯一${s.size}/总数${ids.length}`);
+      if (!ok) allPassed = false;
+      const ok2 = guessVideos.every(v => v.author && v.videoId && v.url);
+      console.error(`  ${ok2 ? 'PASS' : 'FAIL'}: 结构完整`);
+      if (!ok2) allPassed = false;
+    }
+
+    console.error(`\n${allPassed ? 'ALL PASSED' : 'SOME FAILED'}`);
+    process.exit(allPassed ? 0 : 1);
+
+  } finally {
+    await browser.close().catch(() => {});
+  }
+}
+
+main().catch(err => { console.error('FATAL:', err.message); process.exit(1); });