tt-help-cli-ycl 1.3.81 → 1.3.83

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tt-help-cli-ycl",
-  "version": "1.3.81",
+  "version": "1.3.83",
   "description": "TikTok user & video data scraper - extract ttSeller, verified, locationCreated from HTML source",
   "type": "module",
   "bin": {

package/src/cli/explore.js

@@ -264,10 +264,11 @@ export async function handleExplore(options) {
     const FOLLOW_BLOCK_THRESHOLD = 10 * 60 * 1000; // 10分钟
 
     while (!shuttingDown) {
-      const jobQuery = exploreJobLocations
-        ? `${serverUrl}/api/job?userId=${encodeURIComponent(userId)}&locations=${encodeURIComponent(exploreJobLocations)}&loggedIn=${loggedIn}`
-        : `${serverUrl}/api/job?userId=${encodeURIComponent(userId)}&loggedIn=${loggedIn}`;
-      const job = await apiGet(jobQuery);
+      try {
+        const jobQuery = exploreJobLocations
+          ? `${serverUrl}/api/job?userId=${encodeURIComponent(userId)}&locations=${encodeURIComponent(exploreJobLocations)}&loggedIn=${loggedIn}`
+          : `${serverUrl}/api/job?userId=${encodeURIComponent(userId)}&loggedIn=${loggedIn}`;
+        const job = await apiGet(jobQuery);
       if (!job.hasJob) {
         console.error(`\n[Explore] 当前无任务，${exploreInterval} 秒后重试...`);
         await new Promise((r) => setTimeout(r, exploreInterval * 1000));
@@ -569,6 +570,28 @@ export async function handleExplore(options) {
         console.error(`\n已达上限 ${exploreMaxUsers} 个用户，停止处理`);
         break;
       }
+      } catch (e) {
+        // 浏览器关闭错误：自动重建 browser + page，然后重试当前轮次
+        if (isBrowserClosedError(e)) {
+          console.error(
+            `\n[浏览器] 检测到浏览器关闭 (${e.message})，正在重建...`,
+          );
+          const newBrowser = await relaunchBrowser(
+            cdpOptions,
+            cdpOptions.port || 9222,
+          );
+          browser = newBrowser;
+          const newPage = await setupNewPage(browser);
+          Object.assign(page, newPage);
+          // 重建后等待页面稳定
+          await new Promise((r) => setTimeout(r, 3000));
+          console.error(`[浏览器] 已重建，继续处理...`);
+          continue;
+        }
+        // 其他未预期错误：打印堆栈并跳过本轮
+        console.error(`\n[未捕获错误] ${e.message}`);
+        console.error(e.stack || "");
+      }
     }
 
     const stats = await apiGet(`${serverUrl}/api/stats`);

package/src/cli/refresh.js

@@ -549,6 +549,7 @@ export async function handleRefresh(options) {
 
       const payload = {
         userInfo: refreshUserInfo,
+        topRecentVideo: result.topRecentVideo || null,
         discoveredFollowing: (result.discoveredFollowing || []).map((f) => ({
           handle: Array.isArray(f) ? f[0] : f,
           displayName: Array.isArray(f) ? f[1] : null,

package/src/cli/watch.js

@@ -5,19 +5,40 @@ import { startWatchServer, openBrowser } from "../watch/server.js";
 
 export async function handleWatch(options) {
   const dataAnchor = options.dataAnchor || options.outputFile;
-  const { watchPort } = options;
+  const { watchPort, llmRefill, llmRefillMinScore, llmRefillSampleSize } =
+    options;
 
   if (!dataAnchor) {
-    console.error("用法: tt-help watch -o <db路径> [-p 端口]");
+    console.error("用法: tt-help watch -o <db路径> [-p 端口] [--llm-refill]");
     console.error("      tt-help watch -o data/result.db");
     console.error("      tt-help watch -o data/result.db -p 8080");
+    console.error(
+      "      tt-help watch -o data/result.db --llm-refill --llm-refill-min 60",
+    );
     process.exit(1);
   }
 
   mkdirSync(path.dirname(path.resolve(dataAnchor)), { recursive: true });
 
-  const store = createStore(dataAnchor);
-  const { server, port } = await startWatchServer(dataAnchor, watchPort, store);
+  const storeOptions = {};
+  if (llmRefill) {
+    storeOptions.refillLlm = {
+      llmScore: true,
+      llmMinScore: llmRefillMinScore ?? 60,
+      llmSampleSize: llmRefillSampleSize ?? 100,
+    };
+    console.error(
+      `[watch] LLM 自动打分已启用: 最低分 ${storeOptions.refillLlm.llmMinScore}, 采样 ${storeOptions.refillLlm.llmSampleSize}`,
+    );
+  }
+
+  const store = createStore(dataAnchor, storeOptions);
+  const { server, port } = await startWatchServer(
+    dataAnchor,
+    watchPort,
+    store,
+    storeOptions,
+  );
   openBrowser(port);
 
   process.once("SIGINT", () => {

package/src/lib/api-interceptor-comment.js

@@ -12,35 +12,61 @@ import { delay } from "./delay.js";
  * @param {function} options.onCaptcha - 验证码检测回调 (page) => Promise<{detected: boolean}>
  * @returns {Promise<{comments: Array, total: number, captchaDetected: boolean, error: string|null}>}
  */
-async function fetchUserCommentsAPI(page, { maxComments = 100, log = console.log, onCaptcha } = {}) {
+async function fetchUserCommentsAPI(
+  page,
+  { maxComments = 100, log = console.log, onCaptcha } = {},
+) {
   // 先注册 API 拦截器，再点 tab（顺序不能反）
   let apiResolve = null;
   let apiRequestUrl = null;
-  const apiPromise = new Promise(r => { apiResolve = r; });
+  const apiPromise = new Promise((r) => {
+    apiResolve = r;
+  });
 
   const handler = async (response) => {
     const url = response.url();
-    if (response.status() === 200 && url.includes('/api/comment/list/') && !apiRequestUrl) {
+    if (
+      response.status() === 200 &&
+      url.includes("/api/comment/list/") &&
+      !apiRequestUrl
+    ) {
       apiRequestUrl = url;
       try {
-        apiResolve(await response.json());
+        // 超时保护：response.json() 内部调用 CDP Network.getResponseBody，
+        // 当页面刷新/验证码导致响应资源丢失时会挂起，需独立超时控制
+        apiResolve(
+          await Promise.race([
+            response.json(),
+            new Promise((_, reject) =>
+              setTimeout(
+                () => reject(new Error("Response body fetch timeout (60s)")),
+                60000,
+              ),
+            ),
+          ]),
+        );
       } catch (e) {
         apiResolve(null);
       }
     }
   };
 
-  page.on('response', handler);
+  page.on("response", handler);
 
   try {
     // 点击评论 tab 触发 API
-    log('  [API拦截] 点击评论 tab...');
+    log("  [API拦截] 点击评论 tab...");
     const tabs = page.locator('[class*="tabbar-item"]');
     const commentTab = tabs.filter({ hasText: /评论|Comment/ });
     const count = await commentTab.count();
 
     if (count === 0) {
-      return { comments: [], total: 0, captchaDetected: false, error: '未找到评论 tab' };
+      return {
+        comments: [],
+        total: 0,
+        captchaDetected: false,
+        error: "未找到评论 tab",
+      };
     }
 
     await commentTab.first().click({ force: true });
@@ -55,7 +81,12 @@ async function fetchUserCommentsAPI(page, { maxComments = 100, log = console.log
 
     if (!data || !apiRequestUrl) {
       log(`  [API拦截] 点击评论 tab 后 ${elapsed}ms 未拿到 API 响应`);
-      return { comments: [], total: 0, captchaDetected: false, error: 'API 超时或未响应' };
+      return {
+        comments: [],
+        total: 0,
+        captchaDetected: false,
+        error: "API 超时或未响应",
+      };
     }
 
     // 验证码检测（API 拿完后检测）
@@ -65,7 +96,7 @@ async function fetchUserCommentsAPI(page, { maxComments = 100, log = console.log
         const captchaResult = await onCaptcha(page);
         captchaDetected = !!captchaResult.detected;
         if (captchaDetected) {
-          log('  [API拦截] 检测到验证码');
+          log("  [API拦截] 检测到验证码");
         }
       } catch (e) {
         log(`  [API拦截] 验证码检测异常: ${e.message}`);
@@ -73,10 +104,16 @@ async function fetchUserCommentsAPI(page, { maxComments = 100, log = console.log
     }
 
     const items = data.comments || [];
-    log(`  [API拦截] ${elapsed}ms 后拿到 ${items.length} 条评论 (total: ${data.total || '?'})`);
+    log(
+      `  [API拦截] ${elapsed}ms 后拿到 ${items.length} 条评论 (total: ${data.total || "?"})`,
+    );
 
     if (items.length >= maxComments) {
-      return { comments: items.slice(0, maxComments), total: data.total || 0, captchaDetected };
+      return {
+        comments: items.slice(0, maxComments),
+        total: data.total || 0,
+        captchaDetected,
+      };
     }
 
     // 翻页
@@ -86,7 +123,10 @@ async function fetchUserCommentsAPI(page, { maxComments = 100, log = console.log
 
     while (hasMore && cursor && items.length < maxComments) {
       pageNum++;
-      const pageUrl = apiRequestUrl.replace(/cursor=([^&]+)/, `cursor=${cursor}`);
+      const pageUrl = apiRequestUrl.replace(
+        /cursor=([^&]+)/,
+        `cursor=${cursor}`,
+      );
 
       const pageData = await page.evaluate(async (u) => {
         try {
@@ -103,7 +143,9 @@ async function fetchUserCommentsAPI(page, { maxComments = 100, log = console.log
       }
 
       const pageComments = pageData.comments || [];
-      log(`  [API拦截] 翻页 ${pageNum}: ${pageComments.length} 条 (累计: ${items.length + pageComments.length})`);
+      log(
+        `  [API拦截] 翻页 ${pageNum}: ${pageComments.length} 条 (累计: ${items.length + pageComments.length})`,
+      );
 
       items.push(...pageComments);
       cursor = pageData.cursor;
@@ -119,7 +161,7 @@ async function fetchUserCommentsAPI(page, { maxComments = 100, log = console.log
 
     return { comments: result, total: data.total || 0, captchaDetected };
   } finally {
-    page.off('response', handler);
+    page.off("response", handler);
   }
 }
 

package/src/lib/api-interceptor.js

@@ -18,7 +18,12 @@ async function processAPIResponse(
   for (const item of firstPageItems) {
     if (items.length >= maxVideos) break;
     const href = `https://www.tiktok.com/@${username}/video/${item.id}`;
-    items.push({ id: item.id, href, createTime: item.createTime || null });
+    items.push({
+      id: item.id,
+      href,
+      createTime: item.createTime || null,
+      playCount: item.stats?.playCount || 0,
+    });
   }
 
   let cursor = data.cursor;
@@ -66,6 +71,7 @@ async function processAPIResponse(
             id: item.id,
             href,
             createTime: item.createTime || null,
+            playCount: item.stats?.playCount || 0,
           });
         }
       }
@@ -128,7 +134,17 @@ async function fetchUserVideosAPI(page, username, maxVideos, log) {
       { timeout: 30000 },
     );
 
-    data = await response.json();
+    // 超时保护：response.json() 内部调用 CDP Network.getResponseBody，
+    // 当页面刷新/验证码导致响应资源丢失时会挂起，需独立超时控制
+    data = await Promise.race([
+      response.json(),
+      new Promise((_, reject) =>
+        setTimeout(
+          () => reject(new Error("Response body fetch timeout (60s)")),
+          60000,
+        ),
+      ),
+    ]);
   } catch (e) {
     interceptionError = e.message;
   } finally {

package/src/lib/args.js

@@ -347,6 +347,9 @@ function parseInfoArgs(args) {
 function parseWatchArgs(args) {
   let dataAnchor = "./result.db";
   let watchPort = 3001;
+  let llmRefill = true;
+  let llmRefillMinScore = null;
+  let llmRefillSampleSize = null;
 
   for (let i = 0; i < args.length; i++) {
     const arg = args[i];
@@ -354,6 +357,14 @@ function parseWatchArgs(args) {
       dataAnchor = args[++i];
     } else if (arg === "-p") {
       watchPort = parseInt(args[++i]) || 3001;
+    } else if (arg === "--llm-refill") {
+      llmRefill = true;
+    } else if (arg === "--no-llm-refill") {
+      llmRefill = false;
+    } else if (arg === "--llm-refill-min") {
+      llmRefillMinScore = parseInt(args[++i]) || 60;
+    } else if (arg === "--llm-refill-sample") {
+      llmRefillSampleSize = parseInt(args[++i]) || 100;
     }
   }
 
@@ -362,6 +373,9 @@ function parseWatchArgs(args) {
     outputFile: dataAnchor,
     dataAnchor,
     watchPort,
+    llmRefill,
+    llmRefillMinScore,
+    llmRefillSampleSize,
     urls: [],
     outputFormat: "json",
     exploreCount: 0,

package/src/scraper/explore-core.js

@@ -8,6 +8,7 @@ import {
   DEFAULT_TARGET_LOCATIONS_CSV,
   findFirstMatchingLocation,
   isLocationInList,
+  normalizeLocation,
   normalizeLocationList,
 } from "../lib/target-locations.js";
 import {
@@ -40,6 +41,7 @@ async function processExplore(page, username, options, log) {
     keepFollow: false,
     locationCreated: null,
     latestVideoTime: null,
+    topRecentVideo: null,
     noVideo: false,
     restricted: false,
     error: null,
@@ -84,6 +86,27 @@ async function processExplore(page, username, options, log) {
       if (result.userInfo) result.userInfo.latestVideoTime = latestCreateTime;
     }
 
+    // 找出 7 天内发布且播放量最大的视频
+    const SEVEN_DAYS_SECONDS = 7 * 24 * 60 * 60;
+    const nowSeconds = Math.floor(Date.now() / 1000);
+    const recentVideos = videoArray.filter(
+      (v) => v.createTime && nowSeconds - v.createTime <= SEVEN_DAYS_SECONDS,
+    );
+    if (recentVideos.length > 0) {
+      const topVideo = recentVideos.reduce((max, v) =>
+        v.playCount > max.playCount ? v : max,
+      );
+      result.topRecentVideo = {
+        id: topVideo.id,
+        href: topVideo.href,
+        playCount: topVideo.playCount,
+        createTime: topVideo.createTime,
+      };
+      log(
+        `  7天内最高播放视频: ${topVideo.playCount} 次播放 (${recentVideos.length} 个候选)`,
+      );
+    }
+
     if (videoArray.length <= 0) {
       // 视频为空：可能是页面受限或用户真的没有视频
       result.processed = true;
@@ -110,7 +133,10 @@ async function processExplore(page, username, options, log) {
       log(
         `  国家采样(${locations.length}个): [${locations.filter(Boolean).join(", ") || "无数据"}]`,
       );
-      const normalizedLocations = normalizeLocationList(locations, []);
+      // 直接标准化，不去重（保留重复值用于频率统计）
+      const normalizedLocations = locations
+        .map((loc) => normalizeLocation(loc))
+        .filter(Boolean);
 
       // 统计频率
       const freq = {};