tt-help-cli-ycl 1.3.23 → 1.3.24

package/src/scraper/refresh-core.js

@@ -1,179 +1,179 @@
-import {
-  delay,
-  retryWithBackoff,
-  detectPageError,
-  assertPageUrl,
-} from './modules/page-helpers.js';
-import { detectCaptcha } from './modules/captcha-handler.js';
-import {
-  getUserInfo,
-  collectVideos,
-} from '../videos/core.js';
-import { extractFollowAndFollowers } from './modules/follow-extractor.js';
-import { processExplore } from './explore-core.js';
-
-export async function processRefresh(page, username, serverUrl, options, log) {
-  const {
-    maxFollowing = 100,
-    maxFollowers = 100,
-    maxVideos = 100,
-  } = options;
-
-  const result = {
-    userInfo: null,
-    discoveredVideoAuthors: [],
-    discoveredFollowing: [],
-    discoveredFollowers: [],
-    newUsersAdded: 0,
-    collectedVideos: 0,
-    error: null,
-  };
-
-  try {
-    log(`  访问 @${username} 主页...`);
-    const homeUrl = `https://www.tiktok.com/@${username}`;
-    await retryWithBackoff(async () => {
-      await page.goto(homeUrl, { waitUntil: 'domcontentloaded', timeout: 30000 });
-      assertPageUrl(page, `@${username}`);
-    }, { log });
-    await page.waitForSelector('[class*="DivVideoList"]', { timeout: 10000 }).catch(() => {});
-    await delay(1000, 2000);
-
-    log('  获取用户信息...');
-    const info = await getUserInfo(page);
-    if (info) {
-      result.userInfo = info;
-      log(`  用户: ${info.nickname || username} | 粉丝: ${info.followerCount || '-'} | 视频: ${info.videoCount || '-'}`);
-    }
-
-    const captcha = await detectCaptcha(page);
-    if (captcha && captcha.visible) {
-      log(`[验证码] @${username} 页面出现验证码`);
-      result.captchaDetected = true;
-      result.captchaStage = result.captchaStage || 'video-page';
-      result.captchaMessage = result.captchaMessage || '视频页出现验证码';
-    }
-
-    // 采集视频
-    log(`  采集视频 (最多 ${maxVideos} 个)...`);
-    const videoList = await collectVideos(page, username, maxVideos, log);
-    const videoArray = videoList ? [...videoList.values()] : [];
-    result.collectedVideos = videoArray.length;
-    result.discoveredVideoAuthors = videoArray.map(v => v.author);
-
-    if (videoArray.length <= 0) {
-      result.noVideo = true;
-      const pageError = await detectPageError(page);
-      if (pageError) {
-        result.restricted = true;
-        log(`  @${username} 页面受限（${pageError}），标记跳过`);
-      }
-      return result;
-    }
-
-    // 采集关注和粉丝
-    log(`  采集关注 (最多 ${maxFollowing}) + 粉丝 (最多 ${maxFollowers})...`);
-    try {
-      const followResult = await extractFollowAndFollowers(page, {
-        maxFollowing,
-        maxFollowers,
-      });
-      result.discoveredFollowing = followResult.following || [];
-      result.discoveredFollowers = followResult.followers || [];
-      log(`  关注: ${result.discoveredFollowing.length}, 粉丝: ${result.discoveredFollowers.length}`);
-    } catch (e) {
-      log(`  [关注/粉丝采集失败] ${e.message}`);
-      result.discoveredFollowing = [];
-      result.discoveredFollowers = [];
-    }
-
-    // 处理新发现的用户（关注 + 粉丝），循环执行完整 explore
-    // follow-extractor 返回 [handle, displayName] 数组
-    const allDiscovered = [
-      ...result.discoveredFollowing.map(h => ({ handle: Array.isArray(h) ? h[0] : h, source: 'refresh-following' })),
-      ...result.discoveredFollowers.map(h => ({ handle: Array.isArray(h) ? h[0] : h, source: 'refresh-follower' })),
-    ];
-
-    for (const { handle, source } of allDiscovered) {
-      const uniqueId = handle.replace('@', '');
-
-      // 检查用户是否已存在
-      const existsResp = await fetch(`${serverUrl}/api/user-exists/${encodeURIComponent(uniqueId)}`);
-      const existsData = await existsResp.json();
-
-      if (existsData.exists) {
-        continue;
-      }
-
-      log(`  [新用户] @${uniqueId} 不存在，开始探索 (来源: ${source})...`);
-      await delay(1000, 2000);
-
-      // 对新用户做完整 explore（与 explore 命令逻辑一致）
-      const exploreResult = await processExplore(page, uniqueId, {
-        maxComments: 10,
-        maxGuess: 0,
-        enableFollow: true,
-        maxFollowing: 5,
-        maxFollowers: 5,
-        location: 'PL,NL,BE,DE,FR,IT,ES,IE',
-      }, log);
-
-      // 提交 explore 结果到服务端（和 explore 命令的 commitJob 一致）
-      if (exploreResult.userInfo) {
-        const guessedLocation = exploreResult.locationCreated || null;
-
-        const payload = {
-          userInfo: exploreResult.userInfo || {},
-          discoveredVideoAuthors: (exploreResult.discoveredVideoAuthors || []).map(item =>
-            typeof item === 'object' ? { ...item, guessedLocation } : item
-          ),
-          discoveredCommentAuthors: (exploreResult.discoveredCommentAuthors || []).map(author => ({ author, guessedLocation })),
-          discoveredGuessAuthors: (exploreResult.discoveredGuessAuthors || []).map(author => ({ author, guessedLocation })),
-          discoveredFollowing: (exploreResult.discoveredFollowing || []).map(f => ({
-            handle: Array.isArray(f) ? f[0] : f,
-            displayName: Array.isArray(f) ? f[1] : null,
-            guessedLocation,
-          })),
-          discoveredFollowers: (exploreResult.discoveredFollowers || []).map(f => ({
-            handle: Array.isArray(f) ? f[0] : f,
-            displayName: Array.isArray(f) ? f[1] : null,
-            guessedLocation,
-          })),
-          processed: exploreResult.processed,
-          hasFollowData: exploreResult.hasFollowData,
-          keepFollow: exploreResult.keepFollow,
-          locationCreated: exploreResult.locationCreated,
-          noVideo: exploreResult.noVideo,
-          restricted: exploreResult.restricted,
-          error: exploreResult.error,
-        };
-
-        const addResp = await fetch(`${serverUrl}/api/explore-new/${uniqueId}`, {
-          method: 'POST',
-          headers: { 'Content-Type': 'application/json' },
-          body: JSON.stringify(payload),
-        });
-        const addResult = await addResp.json();
-
-        if (!addResult.saved) {
-          log(`  [跳过] @${uniqueId} 提交失败`);
-          continue;
-        }
-
-        result.newUsersAdded++;
-        if (exploreResult.captchaDetected) {
-          result.captchaDetected = true;
-        }
-        log(`  [已提交] @${uniqueId} ${addResult.created ? '(新用户)' : '(已存在)'} | 发现: ${addResult.newUsers?.length || 0} 个`);
-      }
-
-      await delay(2000, 4000);
-    }
-  } catch (e) {
-    log(`  [错误] ${e.message}`);
-    result.error = e.message;
-    result.errorStack = e.stack || '';
-  }
-
-  return result;
-}
+import {
+  delay,
+  retryWithBackoff,
+  detectPageError,
+  assertPageUrl,
+} from './modules/page-helpers.js';
+import { detectCaptcha } from './modules/captcha-handler.js';
+import {
+  getUserInfo,
+  collectVideos,
+} from '../videos/core.js';
+import { extractFollowAndFollowers } from './modules/follow-extractor.js';
+import { processExplore } from './explore-core.js';
+
+export async function processRefresh(page, username, serverUrl, options, log) {
+  const {
+    maxFollowing = 100,
+    maxFollowers = 100,
+    maxVideos = 100,
+  } = options;
+
+  const result = {
+    userInfo: null,
+    discoveredVideoAuthors: [],
+    discoveredFollowing: [],
+    discoveredFollowers: [],
+    newUsersAdded: 0,
+    collectedVideos: 0,
+    error: null,
+  };
+
+  try {
+    log(`  访问 @${username} 主页...`);
+    const homeUrl = `https://www.tiktok.com/@${username}`;
+    await retryWithBackoff(async () => {
+      await page.goto(homeUrl, { waitUntil: 'domcontentloaded', timeout: 30000 });
+      assertPageUrl(page, `@${username}`);
+    }, { log });
+    await page.waitForSelector('[class*="DivVideoList"]', { timeout: 10000 }).catch(() => {});
+    await delay(1000, 2000);
+
+    log('  获取用户信息...');
+    const info = await getUserInfo(page);
+    if (info) {
+      result.userInfo = info;
+      log(`  用户: ${info.nickname || username} | 粉丝: ${info.followerCount || '-'} | 视频: ${info.videoCount || '-'}`);
+    }
+
+    const captcha = await detectCaptcha(page);
+    if (captcha && captcha.visible) {
+      log(`[验证码] @${username} 页面出现验证码`);
+      result.captchaDetected = true;
+      result.captchaStage = result.captchaStage || 'video-page';
+      result.captchaMessage = result.captchaMessage || '视频页出现验证码';
+    }
+
+    // 采集视频
+    log(`  采集视频 (最多 ${maxVideos} 个)...`);
+    const videoList = await collectVideos(page, username, maxVideos, log);
+    const videoArray = videoList ? [...videoList.values()] : [];
+    result.collectedVideos = videoArray.length;
+    result.discoveredVideoAuthors = videoArray.map(v => v.author);
+
+    if (videoArray.length <= 0) {
+      result.noVideo = true;
+      const pageError = await detectPageError(page);
+      if (pageError) {
+        result.restricted = true;
+        log(`  @${username} 页面受限（${pageError}），标记跳过`);
+      }
+      return result;
+    }
+
+    // 采集关注和粉丝
+    log(`  采集关注 (最多 ${maxFollowing}) + 粉丝 (最多 ${maxFollowers})...`);
+    try {
+      const followResult = await extractFollowAndFollowers(page, {
+        maxFollowing,
+        maxFollowers,
+      });
+      result.discoveredFollowing = followResult.following || [];
+      result.discoveredFollowers = followResult.followers || [];
+      log(`  关注: ${result.discoveredFollowing.length}, 粉丝: ${result.discoveredFollowers.length}`);
+    } catch (e) {
+      log(`  [关注/粉丝采集失败] ${e.message}`);
+      result.discoveredFollowing = [];
+      result.discoveredFollowers = [];
+    }
+
+    // 处理新发现的用户（关注 + 粉丝），循环执行完整 explore
+    // follow-extractor 返回 [handle, displayName] 数组
+    const allDiscovered = [
+      ...result.discoveredFollowing.map(h => ({ handle: Array.isArray(h) ? h[0] : h, source: 'refresh-following' })),
+      ...result.discoveredFollowers.map(h => ({ handle: Array.isArray(h) ? h[0] : h, source: 'refresh-follower' })),
+    ];
+
+    for (const { handle, source } of allDiscovered) {
+      const uniqueId = handle.replace('@', '');
+
+      // 检查用户是否已存在
+      const existsResp = await fetch(`${serverUrl}/api/user-exists/${encodeURIComponent(uniqueId)}`);
+      const existsData = await existsResp.json();
+
+      if (existsData.exists) {
+        continue;
+      }
+
+      log(`  [新用户] @${uniqueId} 不存在，开始探索 (来源: ${source})...`);
+      await delay(1000, 2000);
+
+      // 对新用户做完整 explore（与 explore 命令逻辑一致）
+      const exploreResult = await processExplore(page, uniqueId, {
+        maxComments: 10,
+        maxGuess: 0,
+        enableFollow: true,
+        maxFollowing: 5,
+        maxFollowers: 5,
+        location: 'PL,NL,BE,DE,FR,IT,ES,IE',
+      }, log);
+
+      // 提交 explore 结果到服务端（和 explore 命令的 commitJob 一致）
+      if (exploreResult.userInfo) {
+        const guessedLocation = exploreResult.locationCreated || null;
+
+        const payload = {
+          userInfo: exploreResult.userInfo || {},
+          discoveredVideoAuthors: (exploreResult.discoveredVideoAuthors || []).map(item =>
+            typeof item === 'object' ? { ...item, guessedLocation } : item
+          ),
+          discoveredCommentAuthors: (exploreResult.discoveredCommentAuthors || []).map(author => ({ author, guessedLocation })),
+          discoveredGuessAuthors: (exploreResult.discoveredGuessAuthors || []).map(author => ({ author, guessedLocation })),
+          discoveredFollowing: (exploreResult.discoveredFollowing || []).map(f => ({
+            handle: Array.isArray(f) ? f[0] : f,
+            displayName: Array.isArray(f) ? f[1] : null,
+            guessedLocation,
+          })),
+          discoveredFollowers: (exploreResult.discoveredFollowers || []).map(f => ({
+            handle: Array.isArray(f) ? f[0] : f,
+            displayName: Array.isArray(f) ? f[1] : null,
+            guessedLocation,
+          })),
+          processed: exploreResult.processed,
+          hasFollowData: exploreResult.hasFollowData,
+          keepFollow: exploreResult.keepFollow,
+          locationCreated: exploreResult.locationCreated,
+          noVideo: exploreResult.noVideo,
+          restricted: exploreResult.restricted,
+          error: exploreResult.error,
+        };
+
+        const addResp = await fetch(`${serverUrl}/api/explore-new/${uniqueId}`, {
+          method: 'POST',
+          headers: { 'Content-Type': 'application/json' },
+          body: JSON.stringify(payload),
+        });
+        const addResult = await addResp.json();
+
+        if (!addResult.saved) {
+          log(`  [跳过] @${uniqueId} 提交失败`);
+          continue;
+        }
+
+        result.newUsersAdded++;
+        if (exploreResult.captchaDetected) {
+          result.captchaDetected = true;
+        }
+        log(`  [已提交] @${uniqueId} ${addResult.created ? '(新用户)' : '(已存在)'} | 发现: ${addResult.newUsers?.length || 0} 个`);
+      }
+
+      await delay(2000, 4000);
+    }
+  } catch (e) {
+    log(`  [错误] ${e.message}`);
+    result.error = e.message;
+    result.errorStack = e.stack || '';
+  }
+
+  return result;
+}

package/src/videos/core.js

@@ -1,97 +1,101 @@
-import { delay, ensureBrowserReady, ensureTikTokPage, retryWithBackoff } from '../scraper/modules/page-helpers.js';
-import { fetchUserVideosAPI } from '../lib/api-interceptor.js';
-
-async function getUserInfo(page) {
-  return await page.evaluate(() => {
-    const html = document.documentElement.outerHTML;
-    const result = {};
-
-    const m = window.location.href.match(/\/@([^/]+)/);
-    if (m) result.uniqueId = m[1];
-
-    const patterns = {
-      secUid: /"secUid":"([^"]+)"/,
-      nickname: /"nickname":"((?:[^"\\]|\\.)*)"/,
-      ttSeller: /"ttSeller":\s*(true|false)/,
-      verified: /"verified":\s*(true|false)/,
-      followerCount: /"followerCount":(\d+)/,
-      videoCount: /"videoCount":(\d+)/,
-      followingCount: /"followingCount":(\d+)/,
-      heartCount: /"heartCount":(\d+)/,
-      signature: /"signature":"((?:[^"\\]|\\.)*)"/,
-      locationCreated: /"locationCreated":"([^"]*)/,
-      region: /"region":"([^"]*)/,
-    };
-
-    const boolKeys = ['ttSeller', 'verified'];
-    const numKeys = ['followerCount', 'videoCount', 'followingCount', 'heartCount'];
-
-    for (const [key, pat] of Object.entries(patterns)) {
-      const match = html.match(pat);
-      if (match) {
-        if (boolKeys.includes(key)) result[key] = match[1] === 'true';
-        else if (numKeys.includes(key)) result[key] = parseInt(match[1], 10);
-        else if (key === 'signature') result[key] = match[1].replace(/\\n/g, '\n').replace(/\\\\/g, '\\');
-        else result[key] = match[1];
-      }
-    }
-
-    return result;
-  });
-}
-
-async function collectVideos(page, username, maxVideos, log) {
-  const apiResult = await fetchUserVideosAPI(page, username, maxVideos, log);
-  if (apiResult && apiResult.size > 0) {
-    log(`收集完成: ${apiResult.size} 个视频`);
-    return apiResult;
-  }
-  return new Map();
-}
-
-async function runGetUserVideos(options) {
-  const { username, maxVideos = 5, log = console.error } = options;
-  const url = `https://www.tiktok.com/@${username}`;
-
-  log(`用户: @${username}`);
-  log(`URL: ${url}`);
-  log(`最大视频数: ${maxVideos}\n`);
-
-  log('连接浏览器...');
-  const browser = await ensureBrowserReady();
-
-  let page;
-  try {
-    page = await ensureTikTokPage(browser, url);
-  } catch (e) {
-    await browser.close().catch(() => {});
-    throw e;
-  }
-
-  await retryWithBackoff(() => page.goto(url, { waitUntil: 'load', timeout: 30000 }), { log });
-  await delay(3000, 5000);
-  await page.waitForSelector('[class*="DivVideoList"]', { timeout: 10000 }).catch(() => {});
-
-  log('获取用户信息...');
-  const userInfo = await getUserInfo(page);
-  log('用户信息: ' + JSON.stringify(userInfo, null, 2));
-
-  log('\n开始滚动收集视频...');
-  const videos = await collectVideos(page, username, maxVideos, log);
-  const allVideos = Array.from(videos.values());
-
-  log(`\n总计: ${allVideos.length} 个视频`);
-
-  const output = {
-    user: userInfo,
-    totalVideos: Math.min(allVideos.length, maxVideos),
-    videos: allVideos.slice(0, maxVideos).map(v => ({
-      id: v.id,
-      url: v.href.startsWith('http') ? v.href : `https://www.tiktok.com${v.href}`,
-    })),
-  };
-
-  return { output, browser };
-}
-
-export { getUserInfo, collectVideos, runGetUserVideos };
+import { delay, ensureBrowserReady, ensureTikTokPage, retryWithBackoff } from '../scraper/modules/page-helpers.js';
+import { fetchUserVideosAPI } from '../lib/api-interceptor.js';
+
+async function getUserInfo(page) {
+  return await page.evaluate(() => {
+    const html = document.documentElement.outerHTML;
+    const result = {};
+
+    const m = window.location.href.match(/\/@([^/]+)/);
+    if (m) result.uniqueId = m[1];
+
+    const patterns = {
+      secUid: /"secUid":"([^"]+)"/,
+      nickname: /"nickname":"((?:[^"\\]|\\.)*)"/,
+      ttSeller: /"ttSeller":\s*(true|false)/,
+      verified: /"verified":\s*(true|false)/,
+      followerCount: /"followerCount":(\d+)/,
+      videoCount: /"videoCount":(\d+)/,
+      followingCount: /"followingCount":(\d+)/,
+      heartCount: /"heartCount":(\d+)/,
+      signature: /"signature":"((?:[^"\\]|\\.)*)"/,
+      locationCreated: /"locationCreated":"([^"]*)/,
+      region: /"region":"([^"]*)/,
+    };
+
+    const boolKeys = ['ttSeller', 'verified'];
+    const numKeys = ['followerCount', 'videoCount', 'followingCount', 'heartCount'];
+
+    for (const [key, pat] of Object.entries(patterns)) {
+      const match = html.match(pat);
+      if (match) {
+        if (boolKeys.includes(key)) result[key] = match[1] === 'true';
+        else if (numKeys.includes(key)) result[key] = parseInt(match[1], 10);
+        else if (key === 'signature') result[key] = match[1].replace(/\\n/g, '\n').replace(/\\\\/g, '\\');
+        else result[key] = match[1];
+      }
+    }
+
+    return result;
+  });
+}
+
+async function collectVideos(page, username, maxVideos, log) {
+  const apiResult = await fetchUserVideosAPI(page, username, maxVideos, log);
+  if (apiResult && apiResult.size > 0) {
+    log(`收集完成: ${apiResult.size} 个视频`);
+    return apiResult;
+  }
+  if (apiResult && apiResult.size === 0) {
+    return new Map();
+  }
+  // apiResult is null means timeout before refactor; now it should throw instead
+  throw new Error(`API 拦截失败：@${username} 获取视频列表超时，请检查网络`);
+}
+
+async function runGetUserVideos(options) {
+  const { username, maxVideos = 5, log = console.error } = options;
+  const url = `https://www.tiktok.com/@${username}`;
+
+  log(`用户: @${username}`);
+  log(`URL: ${url}`);
+  log(`最大视频数: ${maxVideos}\n`);
+
+  log('连接浏览器...');
+  const browser = await ensureBrowserReady();
+
+  let page;
+  try {
+    page = await ensureTikTokPage(browser, url);
+  } catch (e) {
+    await browser.close().catch(() => {});
+    throw e;
+  }
+
+  await retryWithBackoff(() => page.goto(url, { waitUntil: 'load', timeout: 30000 }), { log });
+  await delay(3000, 5000);
+  await page.waitForSelector('[class*="DivVideoList"]', { timeout: 10000 }).catch(() => {});
+
+  log('获取用户信息...');
+  const userInfo = await getUserInfo(page);
+  log('用户信息: ' + JSON.stringify(userInfo, null, 2));
+
+  log('\n开始滚动收集视频...');
+  const videos = await collectVideos(page, username, maxVideos, log);
+  const allVideos = Array.from(videos.values());
+
+  log(`\n总计: ${allVideos.length} 个视频`);
+
+  const output = {
+    user: userInfo,
+    totalVideos: Math.min(allVideos.length, maxVideos),
+    videos: allVideos.slice(0, maxVideos).map(v => ({
+      id: v.id,
+      url: v.href.startsWith('http') ? v.href : `https://www.tiktok.com${v.href}`,
+    })),
+  };
+
+  return { output, browser };
+}
+
+export { getUserInfo, collectVideos, runGetUserVideos };