tt-help-cli-ycl 1.3.11 → 1.3.13

package/src/scraper/refresh-core.js

@@ -0,0 +1,179 @@
+import {
+  delay,
+  retryWithBackoff,
+  detectPageError,
+  assertPageUrl,
+} from './modules/page-helpers.js';
+import { detectCaptcha } from './modules/captcha-handler.js';
+import {
+  getUserInfo,
+  collectVideos,
+} from '../videos/core.js';
+import { extractFollowAndFollowers } from './modules/follow-extractor.js';
+import { processExplore } from './explore-core.js';
+
+export async function processRefresh(page, username, serverUrl, options, log) {
+  const {
+    maxFollowing = 100,
+    maxFollowers = 100,
+    maxVideos = 100,
+  } = options;
+
+  const result = {
+    userInfo: null,
+    discoveredVideoAuthors: [],
+    discoveredFollowing: [],
+    discoveredFollowers: [],
+    newUsersAdded: 0,
+    collectedVideos: 0,
+    error: null,
+  };
+
+  try {
+    log(`  访问 @${username} 主页...`);
+    const homeUrl = `https://www.tiktok.com/@${username}`;
+    await retryWithBackoff(async () => {
+      await page.goto(homeUrl, { waitUntil: 'domcontentloaded', timeout: 30000 });
+      assertPageUrl(page, `@${username}`);
+    }, { log });
+    await page.waitForSelector('[class*="DivVideoList"]', { timeout: 10000 }).catch(() => {});
+    await delay(1000, 2000);
+
+    log('  获取用户信息...');
+    const info = await getUserInfo(page);
+    if (info) {
+      result.userInfo = info;
+      log(`  用户: ${info.nickname || username} | 粉丝: ${info.followerCount || '-'} | 视频: ${info.videoCount || '-'}`);
+    }
+
+    const captcha = await detectCaptcha(page);
+    if (captcha && captcha.visible) {
+      log(`[验证码] @${username} 页面出现验证码`);
+      result.captchaDetected = true;
+      result.captchaStage = result.captchaStage || 'video-page';
+      result.captchaMessage = result.captchaMessage || '视频页出现验证码';
+    }
+
+    // 采集视频
+    log(`  采集视频 (最多 ${maxVideos} 个)...`);
+    const videoList = await collectVideos(page, username, maxVideos, log);
+    const videoArray = videoList ? [...videoList.values()] : [];
+    result.collectedVideos = videoArray.length;
+    result.discoveredVideoAuthors = videoArray.map(v => v.author);
+
+    if (videoArray.length <= 0) {
+      result.noVideo = true;
+      const pageError = await detectPageError(page);
+      if (pageError) {
+        result.restricted = true;
+        log(`  @${username} 页面受限（${pageError}），标记跳过`);
+      }
+      return result;
+    }
+
+    // 采集关注和粉丝
+    log(`  采集关注 (最多 ${maxFollowing}) + 粉丝 (最多 ${maxFollowers})...`);
+    try {
+      const followResult = await extractFollowAndFollowers(page, {
+        maxFollowing,
+        maxFollowers,
+      });
+      result.discoveredFollowing = followResult.following || [];
+      result.discoveredFollowers = followResult.followers || [];
+      log(`  关注: ${result.discoveredFollowing.length}, 粉丝: ${result.discoveredFollowers.length}`);
+    } catch (e) {
+      log(`  [关注/粉丝采集失败] ${e.message}`);
+      result.discoveredFollowing = [];
+      result.discoveredFollowers = [];
+    }
+
+    // 处理新发现的用户（关注 + 粉丝），循环执行完整 explore
+    // follow-extractor 返回 [handle, displayName] 数组
+    const allDiscovered = [
+      ...result.discoveredFollowing.map(h => ({ handle: Array.isArray(h) ? h[0] : h, source: 'refresh-following' })),
+      ...result.discoveredFollowers.map(h => ({ handle: Array.isArray(h) ? h[0] : h, source: 'refresh-follower' })),
+    ];
+
+    for (const { handle, source } of allDiscovered) {
+      const uniqueId = handle.replace('@', '');
+
+      // 检查用户是否已存在
+      const existsResp = await fetch(`${serverUrl}/api/user-exists/${encodeURIComponent(uniqueId)}`);
+      const existsData = await existsResp.json();
+
+      if (existsData.exists) {
+        continue;
+      }
+
+      log(`  [新用户] @${uniqueId} 不存在，开始探索 (来源: ${source})...`);
+      await delay(1000, 2000);
+
+      // 对新用户做完整 explore（与 explore 命令逻辑一致）
+      const exploreResult = await processExplore(page, uniqueId, {
+        maxComments: 10,
+        maxGuess: 0,
+        enableFollow: true,
+        maxFollowing: 5,
+        maxFollowers: 5,
+        location: 'PL,NL,BE,DE,FR,IT,ES,IE',
+      }, log);
+
+      // 提交 explore 结果到服务端（和 explore 命令的 commitJob 一致）
+      if (exploreResult.userInfo) {
+        const guessedLocation = exploreResult.locationCreated || null;
+
+        const payload = {
+          userInfo: exploreResult.userInfo || {},
+          discoveredVideoAuthors: (exploreResult.discoveredVideoAuthors || []).map(item =>
+            typeof item === 'object' ? { ...item, guessedLocation } : item
+          ),
+          discoveredCommentAuthors: (exploreResult.discoveredCommentAuthors || []).map(author => ({ author, guessedLocation })),
+          discoveredGuessAuthors: (exploreResult.discoveredGuessAuthors || []).map(author => ({ author, guessedLocation })),
+          discoveredFollowing: (exploreResult.discoveredFollowing || []).map(f => ({
+            handle: Array.isArray(f) ? f[0] : f,
+            displayName: Array.isArray(f) ? f[1] : null,
+            guessedLocation,
+          })),
+          discoveredFollowers: (exploreResult.discoveredFollowers || []).map(f => ({
+            handle: Array.isArray(f) ? f[0] : f,
+            displayName: Array.isArray(f) ? f[1] : null,
+            guessedLocation,
+          })),
+          processed: exploreResult.processed,
+          hasFollowData: exploreResult.hasFollowData,
+          keepFollow: exploreResult.keepFollow,
+          locationCreated: exploreResult.locationCreated,
+          noVideo: exploreResult.noVideo,
+          restricted: exploreResult.restricted,
+          error: exploreResult.error,
+        };
+
+        const addResp = await fetch(`${serverUrl}/api/explore-new/${uniqueId}`, {
+          method: 'POST',
+          headers: { 'Content-Type': 'application/json' },
+          body: JSON.stringify(payload),
+        });
+        const addResult = await addResp.json();
+
+        if (!addResult.saved) {
+          log(`  [跳过] @${uniqueId} 提交失败`);
+          continue;
+        }
+
+        result.newUsersAdded++;
+        if (exploreResult.captchaDetected) {
+          result.captchaDetected = true;
+        }
+        log(`  [已提交] @${uniqueId} ${addResult.created ? '(新用户)' : '(已存在)'} | 发现: ${addResult.newUsers?.length || 0} 个`);
+      }
+
+      await delay(2000, 4000);
+    }
+  } catch (e) {
+    log(`  [错误] ${e.message}`);
+    result.error = e.message;
+    result.errorStack = e.stack || '';
+  }
+
+  return result;
+}

package/src/videos/{core.mjs → core.js}

@@ -1,126 +1,126 @@
-import { delay, ensureBrowserReady, ensureTikTokPage, retryWithBackoff } from '../scraper/modules/page-helpers.mjs';
-import { scrollAndCollect } from '../scraper/modules/scroll-collector.mjs';
-
-async function getUserInfo(page) {
-  return await page.evaluate(() => {
-    const html = document.documentElement.outerHTML;
-    const result = {};
-
-    const m = window.location.href.match(/\/@([^/]+)/);
-    if (m) result.uniqueId = m[1];
-
-    const patterns = {
-      secUid: /"secUid":"([^"]+)"/,
-      nickname: /"nickname":"((?:[^"\\]|\\.)*)"/,
-      ttSeller: /"ttSeller":\s*(true|false)/,
-      verified: /"verified":\s*(true|false)/,
-      followerCount: /"followerCount":(\d+)/,
-      videoCount: /"videoCount":(\d+)/,
-      followingCount: /"followingCount":(\d+)/,
-      heartCount: /"heartCount":(\d+)/,
-      signature: /"signature":"((?:[^"\\]|\\.)*)"/,
-      locationCreated: /"locationCreated":"([^"]*)/,
-      region: /"region":"([^"]*)/,
-    };
-
-    const boolKeys = ['ttSeller', 'verified'];
-    const numKeys = ['followerCount', 'videoCount', 'followingCount', 'heartCount'];
-
-    for (const [key, pat] of Object.entries(patterns)) {
-      const match = html.match(pat);
-      if (match) {
-        if (boolKeys.includes(key)) result[key] = match[1] === 'true';
-        else if (numKeys.includes(key)) result[key] = parseInt(match[1], 10);
-        else if (key === 'signature') result[key] = match[1].replace(/\\n/g, '\n').replace(/\\\\/g, '\\');
-        else result[key] = match[1];
-      }
-    }
-
-    return result;
-  });
-}
-
-async function collectVideos(page, username, maxVideos, log) {
-  const allLinks = await scrollAndCollect(page, {
-    container: '[class*="ColumnListContainer"]',
-    extraArgs: { handle: username },
-    collectFn: (container, args) => {
-      const pattern = '/@' + args.handle + '/video/';
-      return {
-        items: Array.from(document.querySelectorAll('a'))
-          .filter(el => (el.getAttribute('href') || '').includes(pattern))
-          .map(el => {
-            const href = el.getAttribute('href') || '';
-            const idMatch = href.match(/\/video\/(\d+)/);
-            return { id: idMatch ? idMatch[1] : null, href };
-          })
-          .filter(v => v.id),
-      };
-    },
-    maxItems: maxVideos,
-    delayRange: [2000, 3000],
-    staleThreshold: 5,
-    maxRounds: 500,
-    onRound: (round, items, allItems) => {
-      const uniqueCount = new Set(allItems.map(v => v.id)).size;
-      if (uniqueCount > 0 && (uniqueCount % 10 === 0 || items.length > 0)) {
-        log(`滚动 ${round + 1}: ${uniqueCount} 个视频 (本轮 ${items.length} 条)`);
-      }
-    },
-  });
-
-  const uniqueVideos = new Map();
-  allLinks.forEach(v => {
-    if (!uniqueVideos.has(v.id)) uniqueVideos.set(v.id, v);
-  });
-
-  log(`收集完成: ${uniqueVideos.size} 个视频`);
-  return uniqueVideos;
-}
-
-async function runGetUserVideos(options) {
-  const { username, maxVideos = 5, log = console.error } = options;
-  const url = `https://www.tiktok.com/@${username}`;
-
-  log(`用户: @${username}`);
-  log(`URL: ${url}`);
-  log(`最大视频数: ${maxVideos}\n`);
-
-  log('连接浏览器...');
-  const browser = await ensureBrowserReady();
-
-  let page;
-  try {
-    page = await ensureTikTokPage(browser, url);
-  } catch (e) {
-    await browser.close().catch(() => {});
-    throw e;
-  }
-
-  await retryWithBackoff(() => page.goto(url, { waitUntil: 'load', timeout: 30000 }), { log });
-  await delay(3000, 5000);
-  await page.waitForSelector('[class*="DivVideoList"]', { timeout: 10000 }).catch(() => {});
-
-  log('获取用户信息...');
-  const userInfo = await getUserInfo(page);
-  log('用户信息: ' + JSON.stringify(userInfo, null, 2));
-
-  log('\n开始滚动收集视频...');
-  const videos = await collectVideos(page, username, maxVideos, log);
-  const allVideos = Array.from(videos.values());
-
-  log(`\n总计: ${allVideos.length} 个视频`);
-
-  const output = {
-    user: userInfo,
-    totalVideos: Math.min(allVideos.length, maxVideos),
-    videos: allVideos.slice(0, maxVideos).map(v => ({
-      id: v.id,
-      url: v.href.startsWith('http') ? v.href : `https://www.tiktok.com${v.href}`,
-    })),
-  };
-
-  return { output, browser };
-}
-
-export { getUserInfo, collectVideos, runGetUserVideos };
+import { delay, ensureBrowserReady, ensureTikTokPage, retryWithBackoff } from '../scraper/modules/page-helpers.js';
+import { scrollAndCollect } from '../scraper/modules/scroll-collector.js';
+
+async function getUserInfo(page) {
+  return await page.evaluate(() => {
+    const html = document.documentElement.outerHTML;
+    const result = {};
+
+    const m = window.location.href.match(/\/@([^/]+)/);
+    if (m) result.uniqueId = m[1];
+
+    const patterns = {
+      secUid: /"secUid":"([^"]+)"/,
+      nickname: /"nickname":"((?:[^"\\]|\\.)*)"/,
+      ttSeller: /"ttSeller":\s*(true|false)/,
+      verified: /"verified":\s*(true|false)/,
+      followerCount: /"followerCount":(\d+)/,
+      videoCount: /"videoCount":(\d+)/,
+      followingCount: /"followingCount":(\d+)/,
+      heartCount: /"heartCount":(\d+)/,
+      signature: /"signature":"((?:[^"\\]|\\.)*)"/,
+      locationCreated: /"locationCreated":"([^"]*)/,
+      region: /"region":"([^"]*)/,
+    };
+
+    const boolKeys = ['ttSeller', 'verified'];
+    const numKeys = ['followerCount', 'videoCount', 'followingCount', 'heartCount'];
+
+    for (const [key, pat] of Object.entries(patterns)) {
+      const match = html.match(pat);
+      if (match) {
+        if (boolKeys.includes(key)) result[key] = match[1] === 'true';
+        else if (numKeys.includes(key)) result[key] = parseInt(match[1], 10);
+        else if (key === 'signature') result[key] = match[1].replace(/\\n/g, '\n').replace(/\\\\/g, '\\');
+        else result[key] = match[1];
+      }
+    }
+
+    return result;
+  });
+}
+
+async function collectVideos(page, username, maxVideos, log) {
+  const allLinks = await scrollAndCollect(page, {
+    container: '[class*="ColumnListContainer"]',
+    extraArgs: { handle: username },
+    collectFn: (container, args) => {
+      const pattern = '/@' + args.handle + '/video/';
+      return {
+        items: Array.from(document.querySelectorAll('a'))
+          .filter(el => (el.getAttribute('href') || '').includes(pattern))
+          .map(el => {
+            const href = el.getAttribute('href') || '';
+            const idMatch = href.match(/\/video\/(\d+)/);
+            return { id: idMatch ? idMatch[1] : null, href };
+          })
+          .filter(v => v.id),
+      };
+    },
+    maxItems: maxVideos,
+    delayRange: [2000, 3000],
+    staleThreshold: 5,
+    maxRounds: 500,
+    onRound: (round, items, allItems) => {
+      const uniqueCount = new Set(allItems.map(v => v.id)).size;
+      if (uniqueCount > 0 && (uniqueCount % 10 === 0 || items.length > 0)) {
+        log(`滚动 ${round + 1}: ${uniqueCount} 个视频 (本轮 ${items.length} 条)`);
+      }
+    },
+  });
+
+  const uniqueVideos = new Map();
+  allLinks.forEach(v => {
+    if (!uniqueVideos.has(v.id)) uniqueVideos.set(v.id, v);
+  });
+
+  log(`收集完成: ${uniqueVideos.size} 个视频`);
+  return uniqueVideos;
+}
+
+async function runGetUserVideos(options) {
+  const { username, maxVideos = 5, log = console.error } = options;
+  const url = `https://www.tiktok.com/@${username}`;
+
+  log(`用户: @${username}`);
+  log(`URL: ${url}`);
+  log(`最大视频数: ${maxVideos}\n`);
+
+  log('连接浏览器...');
+  const browser = await ensureBrowserReady();
+
+  let page;
+  try {
+    page = await ensureTikTokPage(browser, url);
+  } catch (e) {
+    await browser.close().catch(() => {});
+    throw e;
+  }
+
+  await retryWithBackoff(() => page.goto(url, { waitUntil: 'load', timeout: 30000 }), { log });
+  await delay(3000, 5000);
+  await page.waitForSelector('[class*="DivVideoList"]', { timeout: 10000 }).catch(() => {});
+
+  log('获取用户信息...');
+  const userInfo = await getUserInfo(page);
+  log('用户信息: ' + JSON.stringify(userInfo, null, 2));
+
+  log('\n开始滚动收集视频...');
+  const videos = await collectVideos(page, username, maxVideos, log);
+  const allVideos = Array.from(videos.values());
+
+  log(`\n总计: ${allVideos.length} 个视频`);
+
+  const output = {
+    user: userInfo,
+    totalVideos: Math.min(allVideos.length, maxVideos),
+    videos: allVideos.slice(0, maxVideos).map(v => ({
+      id: v.id,
+      url: v.href.startsWith('http') ? v.href : `https://www.tiktok.com${v.href}`,
+    })),
+  };
+
+  return { output, browser };
+}
+
+export { getUserInfo, collectVideos, runGetUserVideos };