tt-help-cli-ycl 1.3.14 → 1.3.16

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tt-help-cli-ycl",
-  "version": "1.3.14",
+  "version": "1.3.16",
   "description": "TikTok user & video data scraper - extract ttSeller, verified, locationCreated from HTML source",
   "type": "module",
   "bin": {

package/src/lib/api-interceptor.js

@@ -0,0 +1,124 @@
+import { delay } from './delay.js';
+
+/**
+ * 通过拦截 TikTok 内部 API 获取用户视频列表
+ * 比 DOM 滚动解析快 5-10 倍
+ *
+ * @param {import('playwright').Page} page - Playwright page (CDP 连接)
+ * @param {string} username - TikTok 用户名
+ * @param {number} maxVideos - 最大视频数
+ * @param {Function} log - 日志函数
+ * @returns {Map<string, {id: string, href: string}>} 与 collectVideos 返回格式一致
+ */
+async function fetchUserVideosAPI(page, username, maxVideos, log) {
+  const url = `https://www.tiktok.com/@${username}`;
+  const items = [];
+
+  // 1. 注册 response 拦截器 + request URL 捕获
+  let apiResolve = null;
+  const apiPromise = new Promise(r => { apiResolve = r; });
+
+  let apiRequestUrl = null;
+
+  const responseHandler = async (response) => {
+    if (response.url().includes('/api/post/item_list/')) {
+      try {
+        apiResolve(await response.json());
+      } catch (e) {
+        apiResolve(null);
+      }
+    }
+  };
+
+  const requestHandler = (request) => {
+    if (request.url().includes('/api/post/item_list/') && !apiRequestUrl) {
+      apiRequestUrl = request.url();
+    }
+  };
+
+  page.on('response', responseHandler);
+  page.on('request', requestHandler);
+
+  try {
+    // 2. 导航并等待 API 响应
+    log('  [API拦截] 导航到用户页，等待 /api/post/item_list/ ...');
+    const t0 = Date.now();
+
+    await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 30000 });
+
+    const data = await Promise.race([
+      apiPromise,
+      new Promise(r => setTimeout(() => r(null), 8000)),
+    ]);
+
+    const elapsed = Date.now() - t0;
+
+    if (!data || !data.itemList) {
+      log(`  [API拦截] ${elapsed}ms 后未拿到 API 数据`);
+      return null;
+    }
+
+    // 3. 提取首页视频
+    const firstPageItems = data.itemList || [];
+    for (const item of firstPageItems) {
+      if (items.length >= maxVideos) break;
+      const href = `https://www.tiktok.com/@${username}/video/${item.id}`;
+      items.push({ id: item.id, href });
+    }
+
+    log(`  [API拦截] ${elapsed}ms 获取首页 ${firstPageItems.length} 条视频`);
+
+    // 4. 翻页获取后续视频
+    let cursor = data.cursor;
+    let hasMore = data.hasMore;
+
+    while (hasMore && cursor && items.length < maxVideos) {
+      if (!apiRequestUrl) {
+        log('  [API拦截] 未捕获到 API 请求 URL，无法翻页');
+        break;
+      }
+
+      const newUrl = apiRequestUrl.replace(/cursor=\d+/, `cursor=${cursor}`);
+
+      try {
+        const pageData = await page.evaluate(async (u) => {
+          const res = await fetch(u);
+          return await res.json();
+        }, newUrl);
+
+        if (pageData && pageData.itemList) {
+          for (const item of pageData.itemList) {
+            if (items.length >= maxVideos) break;
+            const href = `https://www.tiktok.com/@${username}/video/${item.id}`;
+            items.push({ id: item.id, href });
+          }
+          log(`  [API拦截] 翻页 cursor=${cursor}，获取 ${pageData.itemList.length} 条，累计 ${items.length}`);
+        }
+
+        cursor = pageData.cursor;
+        hasMore = pageData.hasMore;
+      } catch (e) {
+        log(`  [API拦截] 翻页失败: ${e.message}`);
+        break;
+      }
+
+      await delay(300, 600);
+    }
+
+    log(`  [API拦截] 总计获取 ${items.length} 条视频`);
+
+    // 转成 Map 返回，与 collectVideos 一致
+    const videoMap = new Map();
+    for (const v of items) {
+      if (!videoMap.has(v.id)) videoMap.set(v.id, v);
+    }
+
+    return videoMap;
+  } finally {
+    // 5. 必须清理拦截器，防止累积
+    page.off('response', responseHandler);
+    page.off('request', requestHandler);
+  }
+}
+
+export { fetchUserVideosAPI };

package/src/lib/args.js

@@ -164,7 +164,7 @@ function parseExploreArgs(args) {
   let explorePort = null;
   let exploreProfile = null;
   let exploreUserId = null;
-  let exploreMaxVideos = 1;
+  let exploreMaxVideos = 16;
 
   const positional = [];
   const PRESETS = ['fast', 'normal', 'slow', 'stealth'];
@@ -196,7 +196,7 @@ function parseExploreArgs(args) {
     } else if (arg === '--user-id') {
       exploreUserId = args[++i];
     } else if (arg === '--max-videos') {
-      exploreMaxVideos = parseInt(args[++i]) || 1;
+      exploreMaxVideos = parseInt(args[++i]) || 16;
     } else {
       positional.push(arg);
     }

package/src/lib/constants.js

@@ -17,7 +17,7 @@ let browser = null;
 let userId = null;
 let maxFollowing = 5;
 let maxFollowers = 5;
-let maxVideos = 1;
+let maxVideos = 16;
 let maxComments = 10;
 
 try {
@@ -87,7 +87,7 @@ function saveMaxFollowers(val) {
 
 function saveMaxVideos(val) {
   const cfg = existsSync(configPath) ? JSON.parse(readFileSync(configPath, 'utf-8')) : {};
-  cfg.maxVideos = parseInt(val) || 1;
+  cfg.maxVideos = parseInt(val) || 16;
   writeFileSync(configPath, JSON.stringify(cfg, null, 2), 'utf-8');
   maxVideos = cfg.maxVideos;
   configFile = configPath;

package/src/lib/tiktok-scraper.mjs

@@ -99,11 +99,16 @@ export class TikTokScraper {
   async warmWaf() {
     if (this.warmPromise) return this.warmPromise;
     this.warmPromise = (async () => {
-      const page = this.slots[0].page;
-      await page.goto(this.warmUrl, { waitUntil: 'domcontentloaded', timeout: 15000 });
-      await delay(1500);
-      this.lastWarmTime = Date.now();
-      this.warmPromise = null;
+      try {
+        const page = this.slots[0].page;
+        await page.goto(this.warmUrl, { waitUntil: 'domcontentloaded', timeout: 15000 });
+        await delay(1500);
+        this.lastWarmTime = Date.now();
+      } catch (e) {
+        console.error(`[warmWaf] failed: ${e.message}`);
+      } finally {
+        this.warmPromise = null;
+      }
     })();
     return this.warmPromise;
   }
@@ -146,28 +151,40 @@ export class TikTokScraper {
   }
 
   async getUserInfo(uniqueId) {
-    if (this._needWarm()) await this.warmWaf();
     const slot = this._pickSlot();
     return slot.lock.run(async () => {
-      const rawHtml = await this._fetchViewSource(
+      let rawHtml = await this._fetchViewSource(
         `https://www.tiktok.com/@${uniqueId}`,
         slot
       );
-      return parseUserInfo(rawHtml);
+      let result = parseUserInfo(rawHtml);
+      if (!result) {
+        try { await this.warmWaf(); } catch {}
+        rawHtml = await this._fetchViewSource(
+          `https://www.tiktok.com/@${uniqueId}`,
+          slot
+        );
+        result = parseUserInfo(rawHtml);
+      }
+      return result || null;
     });
   }
 
   async getVideoInfo(videoUrl) {
-    if (this._needWarm()) await this.warmWaf();
     const slot = this._pickSlot();
     return slot.lock.run(async () => {
-      const rawHtml = await this._fetchViewSource(videoUrl, slot);
-      return parseVideoInfo(rawHtml);
+      let rawHtml = await this._fetchViewSource(videoUrl, slot);
+      let result = parseVideoInfo(rawHtml);
+      if (!result) {
+        try { await this.warmWaf(); } catch {}
+        rawHtml = await this._fetchViewSource(videoUrl, slot);
+        result = parseVideoInfo(rawHtml);
+      }
+      return result || null;
     });
   }
 
   async getUserAndVideo(videoUrl) {
-    if (this._needWarm()) await this.warmWaf();
     const video = await this.getVideoInfo(videoUrl);
     if (!video) return null;
     const user = await this.getUserInfo(video.author.uniqueId);

package/src/scraper/explore-core.js

@@ -15,7 +15,7 @@ import { maxFollowing as globalMaxFollowing, maxFollowers as globalMaxFollowers,
 
 async function processExplore(page, username, options, log) {
   const {
-    maxVideos = 1,
+    maxVideos = 16,
     enableFollow = true,
     maxFollowing = 5,
     maxFollowers = 5,

package/src/videos/core.js

@@ -1,5 +1,5 @@
 import { delay, ensureBrowserReady, ensureTikTokPage, retryWithBackoff } from '../scraper/modules/page-helpers.js';
-import { scrollAndCollect } from '../scraper/modules/scroll-collector.js';
+import { fetchUserVideosAPI } from '../lib/api-interceptor.js';
 
 async function getUserInfo(page) {
   return await page.evaluate(() => {
@@ -41,41 +41,12 @@ async function getUserInfo(page) {
 }
 
 async function collectVideos(page, username, maxVideos, log) {
-  const allLinks = await scrollAndCollect(page, {
-    container: '[class*="ColumnListContainer"]',
-    extraArgs: { handle: username },
-    collectFn: (container, args) => {
-      const pattern = '/@' + args.handle + '/video/';
-      return {
-        items: Array.from(document.querySelectorAll('a'))
-          .filter(el => (el.getAttribute('href') || '').includes(pattern))
-          .map(el => {
-            const href = el.getAttribute('href') || '';
-            const idMatch = href.match(/\/video\/(\d+)/);
-            return { id: idMatch ? idMatch[1] : null, href };
-          })
-          .filter(v => v.id),
-      };
-    },
-    maxItems: maxVideos,
-    delayRange: [2000, 3000],
-    staleThreshold: 5,
-    maxRounds: 500,
-    onRound: (round, items, allItems) => {
-      const uniqueCount = new Set(allItems.map(v => v.id)).size;
-      if (uniqueCount > 0 && (uniqueCount % 10 === 0 || items.length > 0)) {
-        log(`滚动 ${round + 1}: ${uniqueCount} 个视频 (本轮 ${items.length} 条)`);
-      }
-    },
-  });
-
-  const uniqueVideos = new Map();
-  allLinks.forEach(v => {
-    if (!uniqueVideos.has(v.id)) uniqueVideos.set(v.id, v);
-  });
-
-  log(`收集完成: ${uniqueVideos.size} 个视频`);
-  return uniqueVideos;
+  const apiResult = await fetchUserVideosAPI(page, username, maxVideos, log);
+  if (apiResult && apiResult.size > 0) {
+    log(`收集完成: ${apiResult.size} 个视频`);
+    return apiResult;
+  }
+  return new Map();
 }
 
 async function runGetUserVideos(options) {