tt-help-cli-ycl 1.3.0 → 1.3.2

package/src/scraper/modules/scroll-collector.mjs

@@ -0,0 +1,189 @@
+import { delay } from "../../lib/delay.js";
+import { detectPageError } from "./page-error-detector.mjs";
+
+async function doCollect(
+  page,
+  { container, findScrollable, fnStr, extraArgs },
+) {
+  return page.evaluate(
+    ({ fn: fnStr, containerSelector, findScrollableFlag, args }) => {
+      let el;
+      if (!containerSelector) {
+        el = window;
+      } else {
+        el = document.querySelector(containerSelector);
+        if (!el) {
+          el = window;
+        } else if (findScrollableFlag) {
+          let current = el;
+          let found = false;
+          while (current && current !== document.body) {
+            if (current.scrollHeight > current.clientHeight + 10) {
+              el = current;
+              found = true;
+              break;
+            }
+            current = current.parentElement;
+          }
+          if (!found) el = document.body;
+        }
+      }
+      const fn = eval("(" + fnStr + ")");
+      return fn(el, args);
+    },
+    {
+      fn: fnStr,
+      containerSelector: container,
+      findScrollableFlag: findScrollable,
+      args: extraArgs,
+    },
+  );
+}
+
+const LOADING_SELECTORS = [
+  '[class*="loading"]',
+  '[class*="Loading"]',
+  '[class*="spinner"]',
+  '[class*="Spinner"]',
+  '[class*="skeleton"]',
+  '[class*="Skeleton"]',
+  '[aria-busy="true"]',
+];
+
+async function waitForLoading(page) {
+  const maxWait = 5000;
+  const startTime = Date.now();
+  while (Date.now() - startTime < maxWait) {
+    const isLoading = await page.evaluate((sels) => {
+      if (document.readyState !== "complete") return true;
+      for (const sel of sels) {
+        const el = document.querySelector(sel);
+        if (el && el.offsetParent !== null) return true;
+      }
+      return false;
+    }, LOADING_SELECTORS);
+    if (!isLoading) return;
+    await delay(300, 600);
+  }
+}
+
+export async function scrollAndCollect(page, options) {
+  const {
+    container,
+    findScrollable = false,
+    collectFn,
+    extraArgs,
+    delayRange = [800, 1500],
+    maxItems,
+    maxRounds = 200,
+    staleThreshold = 3,
+    uniqueKey,
+    onRound,
+  } = options;
+
+  if (!collectFn) throw new Error("collectFn is required");
+
+  const fnStr =
+    typeof collectFn === "function" ? collectFn.toString() : collectFn;
+  const allItems = [];
+  const seenKeys = uniqueKey ? new Set() : null;
+  let staleCount = 0;
+
+  const processItems = (result) => {
+    const raw = result.items || [];
+    const newItems = uniqueKey
+      ? raw.filter((item) => {
+          const key = uniqueKey(item);
+          if (seenKeys.has(key)) return false;
+          seenKeys.add(key);
+          return true;
+        })
+      : raw;
+    allItems.push(...newItems);
+    return newItems;
+  };
+
+  const isDone = (newItems) => {
+    if (maxItems !== undefined && allItems.length >= maxItems) return true;
+    if (newItems.length === 0) {
+      staleCount++;
+      if (staleCount >= staleThreshold) return true;
+    } else {
+      staleCount = 0;
+    }
+    return false;
+  };
+
+  const collectCtx = { container, findScrollable, fnStr, extraArgs };
+
+  const pageError = await detectPageError(page);
+
+  if (pageError) return [];
+
+  await waitForLoading(page);
+  let result = await doCollect(page, collectCtx);
+  let newItems = processItems(result);
+  if (onRound) onRound(0, newItems, allItems);
+  if (isDone(newItems)) return allItems;
+
+  for (let round = 1; round < maxRounds; round++) {
+    await threePhaseScroll(page, { container, findScrollable });
+    await delay(delayRange[0], delayRange[1]);
+    await waitForLoading(page);
+
+    result = await doCollect(page, collectCtx);
+    newItems = processItems(result);
+
+    if (onRound) onRound(round, newItems, allItems);
+
+    if (isDone(newItems)) break;
+  }
+
+  return allItems;
+}
+
+async function threePhaseScroll(page, { container, findScrollable }) {
+  await page.evaluate(
+    async (opts) => {
+      let el;
+      if (!opts.container) {
+        el = window;
+      } else {
+        el = document.querySelector(opts.container);
+        if (!el) {
+          el = window;
+        } else if (opts.findScrollable) {
+          let current = el;
+          let found = false;
+          while (current && current !== document.body) {
+            if (current.scrollHeight > current.clientHeight + 10) {
+              el = current;
+              found = true;
+              break;
+            }
+            current = current.parentElement;
+          }
+          if (!found) el = document.body;
+        }
+      }
+
+      const randDelay = (min, max) =>
+        new Promise((r) => setTimeout(r, min + Math.random() * (max - min)));
+
+      if (el === window) {
+        window.scrollBy(0, window.innerHeight);
+        await randDelay(400, 800);
+        window.scrollBy(0, -200);
+        await randDelay(200, 400);
+        window.scrollBy(0, window.innerHeight);
+      } else {
+        el.scrollTop = el.scrollHeight;
+        await randDelay(400, 800);
+        el.scrollTop -= 100 + Math.random() * 100;
+        await randDelay(200, 400);
+        el.scrollTop = el.scrollHeight;
+      }
+    },
+    { container, findScrollable },
+  );
+}

package/src/{get-user-videos-core.cjs → videos/core.mjs}

@@ -1,143 +1,126 @@
-const { delay, ensureBrowserReady, ensureTikTokPage, retryWithBackoff } = require('./scraper/modules/page-helpers.cjs');
-const { scrollAndCollect } = require('./scraper/modules/scroll-collector.cjs');
-
-async function getUserInfo(page) {
-  return await page.evaluate(() => {
-    const html = document.documentElement.outerHTML;
-    const result = {};
-
-    const m = window.location.href.match(/\/@([^\/]+)/);
-    if (m) result.uniqueId = m[1];
-
-    const patterns = {
-      secUid: /"secUid":"([^"]+)"/,
-      nickname: /"nickname":"((?:[^"\\]|\\.)*)"/,
-      ttSeller: /"ttSeller":\s*(true|false)/,
-      verified: /"verified":\s*(true|false)/,
-      followerCount: /"followerCount":(\d+)/,
-      videoCount: /"videoCount":(\d+)/,
-      followingCount: /"followingCount":(\d+)/,
-      heartCount: /"heartCount":(\d+)/,
-      signature: /"signature":"((?:[^"\\]|\\.)*)"/,
-      locationCreated: /"locationCreated":"([^"]*)/,
-      region: /"region":"([^"]*)/,
-    };
-
-    const boolKeys = ['ttSeller', 'verified'];
-    const numKeys = ['followerCount', 'videoCount', 'followingCount', 'heartCount'];
-
-    for (const [key, pat] of Object.entries(patterns)) {
-      const match = html.match(pat);
-      if (match) {
-        if (boolKeys.includes(key)) result[key] = match[1] === 'true';
-        else if (numKeys.includes(key)) result[key] = parseInt(match[1], 10);
-        else if (key === 'signature') result[key] = match[1].replace(/\\n/g, '\n').replace(/\\\\/g, '\\');
-        else result[key] = match[1];
-      }
-    }
-
-    return result;
-  });
-}
-
-async function collectVideos(page, username, maxVideos, log) {
-  const allLinks = await scrollAndCollect(page, {
-    container: '[class*="ColumnListContainer"]',
-    extraArgs: { handle: username },
-    collectFn: (container, args) => {
-      const pattern = '/@' + args.handle + '/video/';
-      return {
-        items: Array.from(document.querySelectorAll('a'))
-          .filter(el => (el.getAttribute('href') || '').includes(pattern))
-          .map(el => {
-            const href = el.getAttribute('href') || '';
-            const idMatch = href.match(/\/video\/(\d+)/);
-            return { id: idMatch ? idMatch[1] : null, href };
-          })
-          .filter(v => v.id),
-      };
-    },
-    maxItems: maxVideos,
-    delayRange: [2000, 3000],
-    staleThreshold: 5,
-    maxRounds: 500,
-    onRound: (round, items, allItems) => {
-      const uniqueCount = new Set(allItems.map(v => v.id)).size;
-      if (uniqueCount > 0 && (uniqueCount % 10 === 0 || items.length > 0)) {
-        log(`滚动 ${round + 1}: ${uniqueCount} 个视频 (本轮 ${items.length} 条)`);
-      }
-    },
-  });
-
-  // 去重
-  const uniqueVideos = new Map();
-  allLinks.forEach(v => {
-    if (!uniqueVideos.has(v.id)) {
-      uniqueVideos.set(v.id, v);
-    }
-  });
-
-  log(`收集完成: ${uniqueVideos.size} 个视频`);
-  return uniqueVideos;
-}
-
-async function runGetUserVideos(options) {
-  const {
-    username,
-    maxVideos = 5,
-    log = console.error,
-  } = options;
-
-  const url = `https://www.tiktok.com/@${username}`;
-
-  log(`用户: @${username}`);
-  log(`URL: ${url}`);
-  log(`最大视频数: ${maxVideos}\n`);
-
-  log('连接浏览器...');
-  const browser = await ensureBrowserReady();
-
-  let page;
-  try {
-    page = await ensureTikTokPage(browser, url);
-  } catch (e) {
-    await browser.close().catch(() => {});
-    throw e;
-  }
-
-  await retryWithBackoff(() => page.goto(url, { waitUntil: 'load', timeout: 30000 }), { log });
-  await delay(3000, 5000);
-  await page.waitForSelector('[class*="DivVideoList"]', { timeout: 10000 }).catch(() => {});
-
-  log('获取用户信息...');
-  const userInfo = await getUserInfo(page);
-  log('用户信息: ' + JSON.stringify(userInfo, null, 2));
-
-  log('\n开始滚动收集视频...');
-  const videos = await collectVideos(page, username, maxVideos, log);
-  const allVideos = Array.from(videos.values());
-
-  log(`\n总计: ${allVideos.length} 个视频`);
-
-  const output = {
-    user: userInfo,
-    totalVideos: Math.min(allVideos.length, maxVideos),
-    videos: allVideos.slice(0, maxVideos).map(v => ({
-      id: v.id,
-      url: v.href.startsWith('http') ? v.href : `https://www.tiktok.com${v.href}`,
-    })),
-  };
-
-  return { output, browser };
-}
-
-async function isPageRestricted(page) {
-  return await page.evaluate(() => {
-    const bodyText = document.body.innerText;
-    return !!(bodyText.includes('登录 TikTok') ||
-             bodyText.includes('观众管理功能') ||
-             bodyText.includes('Login to TikTok'));
-  });
-}
-
-module.exports = { getUserInfo, collectVideos, runGetUserVideos, isPageRestricted };
+import { delay, ensureBrowserReady, ensureTikTokPage, retryWithBackoff } from '../scraper/modules/page-helpers.mjs';
+import { scrollAndCollect } from '../scraper/modules/scroll-collector.mjs';
+
+async function getUserInfo(page) {
+  return await page.evaluate(() => {
+    const html = document.documentElement.outerHTML;
+    const result = {};
+
+    const m = window.location.href.match(/\/@([^/]+)/);
+    if (m) result.uniqueId = m[1];
+
+    const patterns = {
+      secUid: /"secUid":"([^"]+)"/,
+      nickname: /"nickname":"((?:[^"\\]|\\.)*)"/,
+      ttSeller: /"ttSeller":\s*(true|false)/,
+      verified: /"verified":\s*(true|false)/,
+      followerCount: /"followerCount":(\d+)/,
+      videoCount: /"videoCount":(\d+)/,
+      followingCount: /"followingCount":(\d+)/,
+      heartCount: /"heartCount":(\d+)/,
+      signature: /"signature":"((?:[^"\\]|\\.)*)"/,
+      locationCreated: /"locationCreated":"([^"]*)/,
+      region: /"region":"([^"]*)/,
+    };
+
+    const boolKeys = ['ttSeller', 'verified'];
+    const numKeys = ['followerCount', 'videoCount', 'followingCount', 'heartCount'];
+
+    for (const [key, pat] of Object.entries(patterns)) {
+      const match = html.match(pat);
+      if (match) {
+        if (boolKeys.includes(key)) result[key] = match[1] === 'true';
+        else if (numKeys.includes(key)) result[key] = parseInt(match[1], 10);
+        else if (key === 'signature') result[key] = match[1].replace(/\\n/g, '\n').replace(/\\\\/g, '\\');
+        else result[key] = match[1];
+      }
+    }
+
+    return result;
+  });
+}
+
+async function collectVideos(page, username, maxVideos, log) {
+  const allLinks = await scrollAndCollect(page, {
+    container: '[class*="ColumnListContainer"]',
+    extraArgs: { handle: username },
+    collectFn: (container, args) => {
+      const pattern = '/@' + args.handle + '/video/';
+      return {
+        items: Array.from(document.querySelectorAll('a'))
+          .filter(el => (el.getAttribute('href') || '').includes(pattern))
+          .map(el => {
+            const href = el.getAttribute('href') || '';
+            const idMatch = href.match(/\/video\/(\d+)/);
+            return { id: idMatch ? idMatch[1] : null, href };
+          })
+          .filter(v => v.id),
+      };
+    },
+    maxItems: maxVideos,
+    delayRange: [2000, 3000],
+    staleThreshold: 5,
+    maxRounds: 500,
+    onRound: (round, items, allItems) => {
+      const uniqueCount = new Set(allItems.map(v => v.id)).size;
+      if (uniqueCount > 0 && (uniqueCount % 10 === 0 || items.length > 0)) {
+        log(`滚动 ${round + 1}: ${uniqueCount} 个视频 (本轮 ${items.length} 条)`);
+      }
+    },
+  });
+
+  const uniqueVideos = new Map();
+  allLinks.forEach(v => {
+    if (!uniqueVideos.has(v.id)) uniqueVideos.set(v.id, v);
+  });
+
+  log(`收集完成: ${uniqueVideos.size} 个视频`);
+  return uniqueVideos;
+}
+
+async function runGetUserVideos(options) {
+  const { username, maxVideos = 5, log = console.error } = options;
+  const url = `https://www.tiktok.com/@${username}`;
+
+  log(`用户: @${username}`);
+  log(`URL: ${url}`);
+  log(`最大视频数: ${maxVideos}\n`);
+
+  log('连接浏览器...');
+  const browser = await ensureBrowserReady();
+
+  let page;
+  try {
+    page = await ensureTikTokPage(browser, url);
+  } catch (e) {
+    await browser.close().catch(() => {});
+    throw e;
+  }
+
+  await retryWithBackoff(() => page.goto(url, { waitUntil: 'load', timeout: 30000 }), { log });
+  await delay(3000, 5000);
+  await page.waitForSelector('[class*="DivVideoList"]', { timeout: 10000 }).catch(() => {});
+
+  log('获取用户信息...');
+  const userInfo = await getUserInfo(page);
+  log('用户信息: ' + JSON.stringify(userInfo, null, 2));
+
+  log('\n开始滚动收集视频...');
+  const videos = await collectVideos(page, username, maxVideos, log);
+  const allVideos = Array.from(videos.values());
+
+  log(`\n总计: ${allVideos.length} 个视频`);
+
+  const output = {
+    user: userInfo,
+    totalVideos: Math.min(allVideos.length, maxVideos),
+    videos: allVideos.slice(0, maxVideos).map(v => ({
+      id: v.id,
+      url: v.href.startsWith('http') ? v.href : `https://www.tiktok.com${v.href}`,
+    })),
+  };
+
+  return { output, browser };
+}
+
+export { getUserInfo, collectVideos, runGetUserVideos };