tt-help-cli-ycl 1.3.45 → 1.3.46

package/src/scraper/modules/page-helpers.js

@@ -1,48 +1,48 @@
-import {
-  delay,
-  getDelayConfig,
-  setDelayConfig,
-  listDelayPresets,
-  DELAY_PRESETS,
-} from '../../lib/delay.js';
-import { ensureBrowserReady } from '../../lib/browser/cdp.js';
-import {
-  ensureTikTokPage,
-  closeCommentPanel,
-  findTikTokPage,
-  getOrCreatePage,
-  isLoggedIn,
-  assertPageUrl,
-} from '../../lib/browser/page.js';
-import { retryWithBackoff, isRetryableError } from '../../lib/retry.js';
-import {
-  extractUserSection,
-  parseUserSection,
-  extractLocationCreated,
-  USER_SECTION_SIZE,
-} from '../../lib/parser.js';
-import { detectPageError } from './page-error-detector.js';
-
-export {
-  delay,
-  setDelayConfig,
-  getDelayConfig,
-  listDelayPresets,
-  DELAY_PRESETS,
-  ensureBrowserReady,
-  ensureTikTokPage,
-  closeCommentPanel,
-  findTikTokPage,
-  getOrCreatePage,
-  isLoggedIn,
-  assertPageUrl,
-  retryWithBackoff,
-  isRetryableError,
-  extractUserSection,
-  parseUserSection,
-  extractLocationCreated,
-  USER_SECTION_SIZE,
-  detectPageError,
-};
-
-export const CDP_PORT = 9222;
+import {
+  delay,
+  getDelayConfig,
+  setDelayConfig,
+  listDelayPresets,
+  DELAY_PRESETS,
+} from '../../lib/delay.js';
+import { ensureBrowserReady } from '../../lib/browser/cdp.js';
+import {
+  ensureTikTokPage,
+  closeCommentPanel,
+  findTikTokPage,
+  getOrCreatePage,
+  isLoggedIn,
+  assertPageUrl,
+} from '../../lib/browser/page.js';
+import { retryWithBackoff, isRetryableError } from '../../lib/retry.js';
+import {
+  extractUserSection,
+  parseUserSection,
+  extractLocationCreated,
+  USER_SECTION_SIZE,
+} from '../../lib/parser.js';
+import { detectPageError } from './page-error-detector.js';
+
+export {
+  delay,
+  setDelayConfig,
+  getDelayConfig,
+  listDelayPresets,
+  DELAY_PRESETS,
+  ensureBrowserReady,
+  ensureTikTokPage,
+  closeCommentPanel,
+  findTikTokPage,
+  getOrCreatePage,
+  isLoggedIn,
+  assertPageUrl,
+  retryWithBackoff,
+  isRetryableError,
+  extractUserSection,
+  parseUserSection,
+  extractLocationCreated,
+  USER_SECTION_SIZE,
+  detectPageError,
+};
+
+export const CDP_PORT = 9222;

package/src/scraper/refresh-core.js

@@ -1,213 +1,213 @@
-import {
-  delay,
-  retryWithBackoff,
-  detectPageError,
-  assertPageUrl,
-} from "./modules/page-helpers.js";
-import { detectCaptcha } from "./modules/captcha-handler.js";
-import { getUserInfo, collectVideos } from "../videos/core.js";
-import { extractFollowAndFollowers } from "./modules/follow-extractor.js";
-import { processExplore } from "./explore-core.js";
-import { DEFAULT_TARGET_LOCATIONS_CSV } from "../lib/target-locations.js";
-
-export async function processRefresh(page, username, serverUrl, options, log) {
-  const { maxFollowing = 100, maxFollowers = 100, maxVideos = 100 } = options;
-
-  const result = {
-    userInfo: null,
-    discoveredVideoAuthors: [],
-    discoveredFollowing: [],
-    discoveredFollowers: [],
-    newUsersAdded: 0,
-    collectedVideos: 0,
-    error: null,
-  };
-
-  try {
-    log(`  访问 @${username} 主页...`);
-    const homeUrl = `https://www.tiktok.com/@${username}`;
-    await retryWithBackoff(
-      async () => {
-        await page.goto(homeUrl, {
-          waitUntil: "domcontentloaded",
-          timeout: 30000,
-        });
-        assertPageUrl(page, `@${username}`);
-      },
-      { log },
-    );
-    await page
-      .waitForSelector('[class*="DivVideoList"]', { timeout: 10000 })
-      .catch(() => {});
-    await delay(1000, 2000);
-
-    log("  获取用户信息...");
-    const info = await getUserInfo(page);
-    if (info) {
-      result.userInfo = info;
-      log(
-        `  用户: ${info.nickname || username} | 粉丝: ${info.followerCount || "-"} | 视频: ${info.videoCount || "-"}`,
-      );
-    }
-
-    const captcha = await detectCaptcha(page);
-    if (captcha && captcha.visible) {
-      log(`[验证码] @${username} 页面出现验证码`);
-      result.captchaDetected = true;
-      result.captchaStage = result.captchaStage || "video-page";
-      result.captchaMessage = result.captchaMessage || "视频页出现验证码";
-    }
-
-    // 采集视频
-    log(`  采集视频 (最多 ${maxVideos} 个)...`);
-    const videoList = await collectVideos(page, username, maxVideos, log);
-    const videoArray = videoList ? [...videoList.values()] : [];
-    result.collectedVideos = videoArray.length;
-    result.discoveredVideoAuthors = videoArray.map((v) => v.author);
-
-    if (videoArray.length <= 0) {
-      result.noVideo = true;
-      const pageError = await detectPageError(page);
-      if (pageError) {
-        result.restricted = true;
-        log(`  @${username} 页面受限（${pageError}），标记跳过`);
-      }
-      return result;
-    }
-
-    // 采集关注和粉丝
-    log(`  采集关注 (最多 ${maxFollowing}) + 粉丝 (最多 ${maxFollowers})...`);
-    try {
-      const followResult = await extractFollowAndFollowers(page, {
-        maxFollowing,
-        maxFollowers,
-      });
-      result.discoveredFollowing = followResult.following || [];
-      result.discoveredFollowers = followResult.followers || [];
-      log(
-        `  关注: ${result.discoveredFollowing.length}, 粉丝: ${result.discoveredFollowers.length}`,
-      );
-    } catch (e) {
-      log(`  [关注/粉丝采集失败] ${e.message}`);
-      result.discoveredFollowing = [];
-      result.discoveredFollowers = [];
-    }
-
-    // 处理新发现的用户（关注 + 粉丝），循环执行完整 explore
-    // follow-extractor 返回 [handle, displayName] 数组
-    const allDiscovered = [
-      ...result.discoveredFollowing.map((h) => ({
-        handle: Array.isArray(h) ? h[0] : h,
-        source: "refresh-following",
-      })),
-      ...result.discoveredFollowers.map((h) => ({
-        handle: Array.isArray(h) ? h[0] : h,
-        source: "refresh-follower",
-      })),
-    ];
-
-    for (const { handle, source } of allDiscovered) {
-      const uniqueId = handle.replace("@", "");
-
-      // 检查用户是否已存在
-      const existsResp = await fetch(
-        `${serverUrl}/api/user-exists/${encodeURIComponent(uniqueId)}`,
-      );
-      const existsData = await existsResp.json();
-
-      if (existsData.exists) {
-        continue;
-      }
-
-      log(`  [新用户] @${uniqueId} 不存在，开始探索 (来源: ${source})...`);
-      await delay(1000, 2000);
-
-      // 对新用户做完整 explore（与 explore 命令逻辑一致）
-      const exploreResult = await processExplore(
-        page,
-        uniqueId,
-        {
-          maxComments: 10,
-          maxGuess: 0,
-          enableFollow: true,
-          maxFollowing: 5,
-          maxFollowers: 5,
-          location: DEFAULT_TARGET_LOCATIONS_CSV,
-        },
-        log,
-      );
-
-      // 提交 explore 结果到服务端（和 explore 命令的 commitJob 一致）
-      if (exploreResult.userInfo) {
-        const guessedLocation = exploreResult.locationCreated || null;
-
-        const payload = {
-          userInfo: exploreResult.userInfo || {},
-          discoveredVideoAuthors: (
-            exploreResult.discoveredVideoAuthors || []
-          ).map((item) =>
-            typeof item === "object" ? { ...item, guessedLocation } : item,
-          ),
-          discoveredCommentAuthors: (
-            exploreResult.discoveredCommentAuthors || []
-          ).map((author) => ({ author, guessedLocation })),
-          discoveredGuessAuthors: (
-            exploreResult.discoveredGuessAuthors || []
-          ).map((author) => ({ author, guessedLocation })),
-          discoveredFollowing: (exploreResult.discoveredFollowing || []).map(
-            (f) => ({
-              handle: Array.isArray(f) ? f[0] : f,
-              displayName: Array.isArray(f) ? f[1] : null,
-              guessedLocation,
-            }),
-          ),
-          discoveredFollowers: (exploreResult.discoveredFollowers || []).map(
-            (f) => ({
-              handle: Array.isArray(f) ? f[0] : f,
-              displayName: Array.isArray(f) ? f[1] : null,
-              guessedLocation,
-            }),
-          ),
-          processed: exploreResult.processed,
-          hasFollowData: exploreResult.hasFollowData,
-          keepFollow: exploreResult.keepFollow,
-          locationCreated: exploreResult.locationCreated,
-          noVideo: exploreResult.noVideo,
-          restricted: exploreResult.restricted,
-          error: exploreResult.error,
-        };
-
-        const addResp = await fetch(
-          `${serverUrl}/api/explore-new/${uniqueId}`,
-          {
-            method: "POST",
-            headers: { "Content-Type": "application/json" },
-            body: JSON.stringify(payload),
-          },
-        );
-        const addResult = await addResp.json();
-
-        if (!addResult.saved) {
-          log(`  [跳过] @${uniqueId} 提交失败`);
-          continue;
-        }
-
-        result.newUsersAdded++;
-        if (exploreResult.captchaDetected) {
-          result.captchaDetected = true;
-        }
-        log(
-          `  [已提交] @${uniqueId} ${addResult.created ? "(新用户)" : "(已存在)"} | 发现: ${addResult.newUsers?.length || 0} 个`,
-        );
-      }
-
-      await delay(2000, 4000);
-    }
-  } catch (e) {
-    log(`  [错误] ${e.message}`);
-    result.error = e.message;
-    result.errorStack = e.stack || "";
-  }
-
-  return result;
-}
+import {
+  delay,
+  retryWithBackoff,
+  detectPageError,
+  assertPageUrl,
+} from "./modules/page-helpers.js";
+import { detectCaptcha } from "./modules/captcha-handler.js";
+import { getUserInfo, collectVideos } from "../videos/core.js";
+import { extractFollowAndFollowers } from "./modules/follow-extractor.js";
+import { processExplore } from "./explore-core.js";
+import { DEFAULT_TARGET_LOCATIONS_CSV } from "../lib/target-locations.js";
+
+export async function processRefresh(page, username, serverUrl, options, log) {
+  const { maxFollowing = 100, maxFollowers = 100, maxVideos = 100 } = options;
+
+  const result = {
+    userInfo: null,
+    discoveredVideoAuthors: [],
+    discoveredFollowing: [],
+    discoveredFollowers: [],
+    newUsersAdded: 0,
+    collectedVideos: 0,
+    error: null,
+  };
+
+  try {
+    log(`  访问 @${username} 主页...`);
+    const homeUrl = `https://www.tiktok.com/@${username}`;
+    await retryWithBackoff(
+      async () => {
+        await page.goto(homeUrl, {
+          waitUntil: "domcontentloaded",
+          timeout: 30000,
+        });
+        assertPageUrl(page, `@${username}`);
+      },
+      { log },
+    );
+    await page
+      .waitForSelector('[class*="DivVideoList"]', { timeout: 10000 })
+      .catch(() => {});
+    await delay(1000, 2000);
+
+    log("  获取用户信息...");
+    const info = await getUserInfo(page);
+    if (info) {
+      result.userInfo = info;
+      log(
+        `  用户: ${info.nickname || username} | 粉丝: ${info.followerCount || "-"} | 视频: ${info.videoCount || "-"}`,
+      );
+    }
+
+    const captcha = await detectCaptcha(page);
+    if (captcha && captcha.visible) {
+      log(`[验证码] @${username} 页面出现验证码`);
+      result.captchaDetected = true;
+      result.captchaStage = result.captchaStage || "video-page";
+      result.captchaMessage = result.captchaMessage || "视频页出现验证码";
+    }
+
+    // 采集视频
+    log(`  采集视频 (最多 ${maxVideos} 个)...`);
+    const videoList = await collectVideos(page, username, maxVideos, log);
+    const videoArray = videoList ? [...videoList.values()] : [];
+    result.collectedVideos = videoArray.length;
+    result.discoveredVideoAuthors = videoArray.map((v) => v.author);
+
+    if (videoArray.length <= 0) {
+      result.noVideo = true;
+      const pageError = await detectPageError(page);
+      if (pageError) {
+        result.restricted = true;
+        log(`  @${username} 页面受限（${pageError}），标记跳过`);
+      }
+      return result;
+    }
+
+    // 采集关注和粉丝
+    log(`  采集关注 (最多 ${maxFollowing}) + 粉丝 (最多 ${maxFollowers})...`);
+    try {
+      const followResult = await extractFollowAndFollowers(page, {
+        maxFollowing,
+        maxFollowers,
+      });
+      result.discoveredFollowing = followResult.following || [];
+      result.discoveredFollowers = followResult.followers || [];
+      log(
+        `  关注: ${result.discoveredFollowing.length}, 粉丝: ${result.discoveredFollowers.length}`,
+      );
+    } catch (e) {
+      log(`  [关注/粉丝采集失败] ${e.message}`);
+      result.discoveredFollowing = [];
+      result.discoveredFollowers = [];
+    }
+
+    // 处理新发现的用户（关注 + 粉丝），循环执行完整 explore
+    // follow-extractor 返回 [handle, displayName] 数组
+    const allDiscovered = [
+      ...result.discoveredFollowing.map((h) => ({
+        handle: Array.isArray(h) ? h[0] : h,
+        source: "refresh-following",
+      })),
+      ...result.discoveredFollowers.map((h) => ({
+        handle: Array.isArray(h) ? h[0] : h,
+        source: "refresh-follower",
+      })),
+    ];
+
+    for (const { handle, source } of allDiscovered) {
+      const uniqueId = handle.replace("@", "");
+
+      // 检查用户是否已存在
+      const existsResp = await fetch(
+        `${serverUrl}/api/user-exists/${encodeURIComponent(uniqueId)}`,
+      );
+      const existsData = await existsResp.json();
+
+      if (existsData.exists) {
+        continue;
+      }
+
+      log(`  [新用户] @${uniqueId} 不存在，开始探索 (来源: ${source})...`);
+      await delay(1000, 2000);
+
+      // 对新用户做完整 explore（与 explore 命令逻辑一致）
+      const exploreResult = await processExplore(
+        page,
+        uniqueId,
+        {
+          maxComments: 10,
+          maxGuess: 0,
+          enableFollow: true,
+          maxFollowing: 5,
+          maxFollowers: 5,
+          location: DEFAULT_TARGET_LOCATIONS_CSV,
+        },
+        log,
+      );
+
+      // 提交 explore 结果到服务端（和 explore 命令的 commitJob 一致）
+      if (exploreResult.userInfo) {
+        const guessedLocation = exploreResult.locationCreated || null;
+
+        const payload = {
+          userInfo: exploreResult.userInfo || {},
+          discoveredVideoAuthors: (
+            exploreResult.discoveredVideoAuthors || []
+          ).map((item) =>
+            typeof item === "object" ? { ...item, guessedLocation } : item,
+          ),
+          discoveredCommentAuthors: (
+            exploreResult.discoveredCommentAuthors || []
+          ).map((author) => ({ author, guessedLocation })),
+          discoveredGuessAuthors: (
+            exploreResult.discoveredGuessAuthors || []
+          ).map((author) => ({ author, guessedLocation })),
+          discoveredFollowing: (exploreResult.discoveredFollowing || []).map(
+            (f) => ({
+              handle: Array.isArray(f) ? f[0] : f,
+              displayName: Array.isArray(f) ? f[1] : null,
+              guessedLocation,
+            }),
+          ),
+          discoveredFollowers: (exploreResult.discoveredFollowers || []).map(
+            (f) => ({
+              handle: Array.isArray(f) ? f[0] : f,
+              displayName: Array.isArray(f) ? f[1] : null,
+              guessedLocation,
+            }),
+          ),
+          processed: exploreResult.processed,
+          hasFollowData: exploreResult.hasFollowData,
+          keepFollow: exploreResult.keepFollow,
+          locationCreated: exploreResult.locationCreated,
+          noVideo: exploreResult.noVideo,
+          restricted: exploreResult.restricted,
+          error: exploreResult.error,
+        };
+
+        const addResp = await fetch(
+          `${serverUrl}/api/explore-new/${uniqueId}`,
+          {
+            method: "POST",
+            headers: { "Content-Type": "application/json" },
+            body: JSON.stringify(payload),
+          },
+        );
+        const addResult = await addResp.json();
+
+        if (!addResult.saved) {
+          log(`  [跳过] @${uniqueId} 提交失败`);
+          continue;
+        }
+
+        result.newUsersAdded++;
+        if (exploreResult.captchaDetected) {
+          result.captchaDetected = true;
+        }
+        log(
+          `  [已提交] @${uniqueId} ${addResult.created ? "(新用户)" : "(已存在)"} | 发现: ${addResult.newUsers?.length || 0} 个`,
+        );
+      }
+
+      await delay(2000, 4000);
+    }
+  } catch (e) {
+    log(`  [错误] ${e.message}`);
+    result.error = e.message;
+    result.errorStack = e.stack || "";
+  }
+
+  return result;
+}