tt-help-cli-ycl 1.3.35 → 1.3.37

package/src/cli/refresh.js

@@ -1,11 +1,15 @@
-import { getOrCreatePage, isBrowserClosedError, relaunchBrowser } from '../lib/browser/page.js';
-import { delay, setDelayConfig } from '../scraper/modules/page-helpers.js';
-import { userId as configuredUserId, saveUserId } from '../lib/constants.js';
-import { getMacOrUuid } from '../lib/mac-or-uuid.js';
-import { ensureBrowserReady as ensureBrowserReadyCDP } from '../lib/browser/cdp.js';
-import { processRefresh } from '../scraper/refresh-core.js';
-import path from 'path';
-import os from 'os';
+import {
+  getOrCreatePage,
+  isBrowserClosedError,
+  relaunchBrowser,
+} from "../lib/browser/page.js";
+import { delay, setDelayConfig } from "../scraper/modules/page-helpers.js";
+import { userId as configuredUserId, saveUserId } from "../lib/constants.js";
+import { getMacOrUuid } from "../lib/mac-or-uuid.js";
+import { ensureBrowserReady as ensureBrowserReadyCDP } from "../lib/browser/cdp.js";
+import { processRefresh } from "../scraper/refresh-core.js";
+import path from "path";
+import os from "os";
 
 async function withRetry(fn, maxRetries = 5) {
   for (let attempt = 1; attempt <= maxRetries; attempt++) {
@@ -13,12 +17,15 @@ async function withRetry(fn, maxRetries = 5) {
       return await fn();
     } catch (e) {
       if (attempt < maxRetries) {
-        const waitTime = attempt <= 2
-          ? 5000 + Math.random() * 5000
-          : attempt <= 4
-            ? 10000 + Math.random() * 10000
-            : 20000 + Math.random() * 10000;
-        console.error(`  [网络] 请求失败 (${attempt}/${maxRetries})，${Math.round(waitTime / 1000)}s 后重试...`);
+        const waitTime =
+          attempt <= 2
+            ? 5000 + Math.random() * 5000
+            : attempt <= 4
+              ? 10000 + Math.random() * 10000
+              : 20000 + Math.random() * 10000;
+        console.error(
+          `  [网络] 请求失败 (${attempt}/${maxRetries})，${Math.round(waitTime / 1000)}s 后重试...`,
+        );
         await delay(waitTime / 1000, waitTime / 1000);
       } else {
         throw e;
@@ -35,8 +42,8 @@ async function apiGet(url) {
 
 async function apiPost(url, body) {
   const resp = await fetch(url, {
-    method: 'POST',
-    headers: { 'Content-Type': 'application/json' },
+    method: "POST",
+    headers: { "Content-Type": "application/json" },
     body: JSON.stringify(body),
   });
   if (!resp.ok) throw new Error(`HTTP ${resp.status}: ${resp.statusText}`);
@@ -46,171 +53,236 @@ async function apiPost(url, body) {
 export async function handleRefresh(options) {
   const {
     explorePreset,
-    explorePort, exploreProfile, exploreUserId, serverUrl,
+    explorePort,
+    exploreProfile,
+    exploreUserId,
+    serverUrl,
   } = options;
+  let browser = null;
+  let shuttingDown = false;
 
-  let userId = exploreUserId || configuredUserId;
-  if (!userId) {
-    userId = await getMacOrUuid();
-    saveUserId(userId);
-    console.error(`[初始化] 未检测到本地用户编号，已生成并使用: ${userId}`);
-  }
-
-  setDelayConfig(explorePreset);
+  const shutdown = async (signal) => {
+    if (shuttingDown) return;
+    shuttingDown = true;
+    console.error(`\n[Refresh] 收到 ${signal}，正在关闭浏览器...`);
+    await browser?.close().catch(() => {});
+    console.error("[Refresh] 已退出");
+    process.exit(0);
+  };
 
-  console.error(`\n=== Refresh 模式 ===`);
-  console.error(`服务器: ${serverUrl}`);
-  console.error(`CDP 端口: ${explorePort || 9222}, 用户编号: ${userId}`);
-  if (exploreProfile) console.error(`浏览器配置: ${exploreProfile}`);
-  console.error(`刷新: 视频 100 + 关注 100 + 粉丝 100`);
-  console.error(`新用户探索: 评论 + 猜你喜欢 + 关注/粉丝`);
+  const onSigint = () => {
+    void shutdown("SIGINT");
+  };
+  const onSigterm = () => {
+    void shutdown("SIGTERM");
+  };
 
-  const cdpOptions = {};
-  if (explorePort) cdpOptions.port = explorePort;
-  if (exploreProfile) {
-    cdpOptions.userDataDir = path.join(os.homedir(), 'Library', 'Application Support', `Microsoft Edge For Testing_${exploreProfile}`);
-  }
+  process.once("SIGINT", onSigint);
+  process.once("SIGTERM", onSigterm);
 
-  let browser = await ensureBrowserReadyCDP(cdpOptions);
-  const page = await getOrCreatePage(browser);
+  try {
+    let userId = exploreUserId || configuredUserId;
+    if (!userId) {
+      userId = await getMacOrUuid();
+      saveUserId(userId);
+      console.error(`[初始化] 未检测到本地用户编号，已生成并使用: ${userId}`);
+    }
 
-  let processedCount = 0;
-  let errorCount = 0;
-  let consecutiveNetworkErrors = 0;
+    setDelayConfig(explorePreset);
 
-  console.error(`\n开始循环刷新任务...\n`);
+    console.error(`\n=== Refresh 模式 ===`);
+    console.error(`服务器: ${serverUrl}`);
+    console.error(`CDP 端口: ${explorePort || 9222}, 用户编号: ${userId}`);
+    if (exploreProfile) console.error(`浏览器配置: ${exploreProfile}`);
+    console.error(`刷新: 视频 100 + 关注 100 + 粉丝 100`);
+    console.error(`新用户探索: 评论 + 猜你喜欢 + 关注/粉丝`);
 
-  while (true) {
-    try {
-      const jobData = await withRetry(() =>
-        apiGet(`${serverUrl}/api/redo-job?userId=${userId}`)
+    const cdpOptions = {};
+    if (explorePort) cdpOptions.port = explorePort;
+    if (exploreProfile) {
+      cdpOptions.userDataDir = path.join(
+        os.homedir(),
+        "Library",
+        "Application Support",
+        `Microsoft Edge For Testing_${exploreProfile}`,
       );
+    }
 
-      if (!jobData.hasJob) {
-        console.error(`\n[空闲] 暂无 redo 任务，30s 后重试...`);
-        await delay(30000, 30000);
-        continue;
-      }
+    browser = await ensureBrowserReadyCDP(cdpOptions);
+    const page = await getOrCreatePage(browser);
 
-      const { uniqueId, nickname } = jobData.user;
-      consecutiveNetworkErrors = 0;
-      processedCount++;
+    let processedCount = 0;
+    let errorCount = 0;
+    let consecutiveNetworkErrors = 0;
 
-      console.error(`\n[${processedCount}] 刷新 @${uniqueId} (${nickname || '未知'})...`);
+    console.error(`\n开始循环刷新任务...\n`);
 
-      const result = await processRefresh(page, uniqueId, serverUrl, {
-        maxFollowing: 100,
-        maxFollowers: 100,
-        maxVideos: 100,
-      }, console.error);
+    while (!shuttingDown) {
+      try {
+        const jobData = await withRetry(() =>
+          apiGet(`${serverUrl}/api/redo-job?userId=${userId}`),
+        );
 
-      if (result.restricted) {
-        console.error(`  @${uniqueId} 页面受限，跳过`);
-        await apiPost(`${serverUrl}/api/redo-job/${uniqueId}`, {
-          restricted: true,
-          userInfo: result.userInfo || {},
-        });
-        continue;
-      }
+        if (!jobData.hasJob) {
+          console.error(`\n[空闲] 暂无 redo 任务，30s 后重试...`);
+          await delay(30000, 30000);
+          continue;
+        }
+
+        const { uniqueId, nickname } = jobData.user;
+        consecutiveNetworkErrors = 0;
+        processedCount++;
+
+        console.error(
+          `\n[${processedCount}] 刷新 @${uniqueId} (${nickname || "未知"})...`,
+        );
 
-      if (result.error) {
-        // 浏览器关闭检测
-        if (isBrowserClosedError(new Error(result.error))) {
-          const newBrowser = await relaunchBrowser(cdpOptions, explorePort || 9222);
-          browser = newBrowser;
-          const newPage = await getOrCreatePage(browser);
-          Object.assign(page, newPage);
-          // 重试当前用户
-          const retryResult = await processRefresh(page, uniqueId, serverUrl, {
+        const result = await processRefresh(
+          page,
+          uniqueId,
+          serverUrl,
+          {
             maxFollowing: 100,
             maxFollowers: 100,
             maxVideos: 100,
-          }, console.error);
-          Object.assign(result, retryResult);
-          // 继续下方逻辑，检查重试后的 result
-        } else {
-          consecutiveNetworkErrors++;
-          errorCount++;
-          console.error(`  [错误] ${result.error}`);
-
-          if (consecutiveNetworkErrors >= 3) {
-            console.error(`  [警告] 连续 ${consecutiveNetworkErrors} 次错误，等待 60s 后重试...`);
-            await delay(60000, 60000);
-            consecutiveNetworkErrors = 0;
-          }
+          },
+          console.error,
+        );
 
+        if (result.restricted) {
+          console.error(`  @${uniqueId} 页面受限，跳过`);
           await apiPost(`${serverUrl}/api/redo-job/${uniqueId}`, {
-            error: result.error,
+            restricted: true,
             userInfo: result.userInfo || {},
           });
-          const errorType = consecutiveNetworkErrors > 1 ? 'network' : 'other';
-          apiPost(`${serverUrl}/api/error-report`, {
+          continue;
+        }
+
+        if (result.error) {
+          // 浏览器关闭检测
+          if (isBrowserClosedError(new Error(result.error))) {
+            const newBrowser = await relaunchBrowser(
+              cdpOptions,
+              explorePort || 9222,
+            );
+            browser = newBrowser;
+            const newPage = await getOrCreatePage(browser);
+            Object.assign(page, newPage);
+            // 重试当前用户
+            const retryResult = await processRefresh(
+              page,
+              uniqueId,
+              serverUrl,
+              {
+                maxFollowing: 100,
+                maxFollowers: 100,
+                maxVideos: 100,
+              },
+              console.error,
+            );
+            Object.assign(result, retryResult);
+            // 继续下方逻辑，检查重试后的 result
+          } else {
+            consecutiveNetworkErrors++;
+            errorCount++;
+            console.error(`  [错误] ${result.error}`);
+
+            if (consecutiveNetworkErrors >= 3) {
+              console.error(
+                `  [警告] 连续 ${consecutiveNetworkErrors} 次错误，等待 60s 后重试...`,
+              );
+              await delay(60000, 60000);
+              consecutiveNetworkErrors = 0;
+            }
+
+            await apiPost(`${serverUrl}/api/redo-job/${uniqueId}`, {
+              error: result.error,
+              userInfo: result.userInfo || {},
+            });
+            const errorType =
+              consecutiveNetworkErrors > 1 ? "network" : "other";
+            apiPost(`${serverUrl}/api/error-report`, {
+              userId,
+              username: uniqueId,
+              errorType,
+              errorMessage: result.error,
+              stage: "process",
+              errorStack: result.errorStack || "",
+            }).catch(() => {});
+            continue;
+          }
+        }
+
+        if (result.captchaDetected) {
+          await apiPost(`${serverUrl}/api/error-report`, {
             userId,
             username: uniqueId,
-            errorType,
-            errorMessage: result.error,
-            stage: 'process',
-            errorStack: result.errorStack || '',
-          }).catch(() => {});
-          continue;
+            errorType: "captcha",
+            errorMessage: result.captchaMessage || "页面出现验证码",
+            stage: result.captchaStage || "video-page",
+            errorStack: "",
+          });
         }
-      }
 
-      if (result.captchaDetected) {
-        await apiPost(`${serverUrl}/api/error-report`, {
-          userId,
-          username: uniqueId,
-          errorType: 'captcha',
-          errorMessage: result.captchaMessage || '页面出现验证码',
-          stage: result.captchaStage || 'video-page',
-          errorStack: '',
+        consecutiveNetworkErrors = 0;
+
+        const guessedLocation = result.locationCreated || null;
+
+        await apiPost(`${serverUrl}/api/redo-job/${uniqueId}`, {
+          userInfo: result.userInfo || {},
+          discoveredVideoAuthors: (result.discoveredVideoAuthors || []).map(
+            (item) =>
+              typeof item === "object" ? { ...item, guessedLocation } : item,
+          ),
+          discoveredCommentAuthors: (result.discoveredCommentAuthors || []).map(
+            (author) => ({ author, guessedLocation }),
+          ),
+          discoveredGuessAuthors: (result.discoveredGuessAuthors || []).map(
+            (author) => ({ author, guessedLocation }),
+          ),
+          discoveredFollowing: (result.discoveredFollowing || []).map((f) => ({
+            handle: Array.isArray(f) ? f[0] : f,
+            displayName: Array.isArray(f) ? f[1] : null,
+            guessedLocation,
+          })),
+          discoveredFollowers: (result.discoveredFollowers || []).map((f) => ({
+            handle: Array.isArray(f) ? f[0] : f,
+            displayName: Array.isArray(f) ? f[1] : null,
+            guessedLocation,
+          })),
+          newUsersAdded: result.newUsersAdded || 0,
+          collectedVideos: result.collectedVideos || 0,
         });
-      }
 
-      consecutiveNetworkErrors = 0;
-
-      const guessedLocation = result.locationCreated || null;
-
-      await apiPost(`${serverUrl}/api/redo-job/${uniqueId}`, {
-        userInfo: result.userInfo || {},
-        discoveredVideoAuthors: (result.discoveredVideoAuthors || []).map(item =>
-          typeof item === 'object' ? { ...item, guessedLocation } : item
-        ),
-        discoveredCommentAuthors: (result.discoveredCommentAuthors || []).map(author => ({ author, guessedLocation })),
-        discoveredGuessAuthors: (result.discoveredGuessAuthors || []).map(author => ({ author, guessedLocation })),
-        discoveredFollowing: (result.discoveredFollowing || []).map(f => ({
-          handle: Array.isArray(f) ? f[0] : f,
-          displayName: Array.isArray(f) ? f[1] : null,
-          guessedLocation,
-        })),
-        discoveredFollowers: (result.discoveredFollowers || []).map(f => ({
-          handle: Array.isArray(f) ? f[0] : f,
-          displayName: Array.isArray(f) ? f[1] : null,
-          guessedLocation,
-        })),
-        newUsersAdded: result.newUsersAdded || 0,
-        collectedVideos: result.collectedVideos || 0,
-      });
-
-      console.error(`  [完成] 视频: ${result.collectedVideos}, 评论作者: ${result.discoveredCommentAuthors?.length || 0}, 关注: ${result.discoveredFollowing?.length || 0}, 粉丝: ${result.discoveredFollowers?.length || 0}, 新增用户: ${result.newUsersAdded}`);
-
-      await delay(3000, 5000);
-    } catch (e) {
-      consecutiveNetworkErrors++;
-      errorCount++;
-      console.error(`\n[错误] ${e.message}`);
+        console.error(
+          `  [完成] 视频: ${result.collectedVideos}, 评论作者: ${result.discoveredCommentAuthors?.length || 0}, 关注: ${result.discoveredFollowing?.length || 0}, 粉丝: ${result.discoveredFollowers?.length || 0}, 新增用户: ${result.newUsersAdded}`,
+        );
 
-      if (consecutiveNetworkErrors >= 3) {
-        console.error(`[警告] 连续 ${consecutiveNetworkErrors} 次网络异常，等待 60s 后重试...`);
-        await delay(60000, 60000);
-        consecutiveNetworkErrors = 0;
-      } else {
-        const waitTime = consecutiveNetworkErrors <= 2
-          ? 5000 + Math.random() * 5000
-          : 10000 + Math.random() * 10000;
-        console.error(`  等待 ${Math.round(waitTime / 1000)}s 后重试...`);
-        await delay(waitTime / 1000, waitTime / 1000);
+        await delay(3000, 5000);
+      } catch (e) {
+        consecutiveNetworkErrors++;
+        errorCount++;
+        console.error(`\n[错误] ${e.message}`);
+
+        if (consecutiveNetworkErrors >= 3) {
+          console.error(
+            `[警告] 连续 ${consecutiveNetworkErrors} 次网络异常，等待 60s 后重试...`,
+          );
+          await delay(60000, 60000);
+          consecutiveNetworkErrors = 0;
+        } else {
+          const waitTime =
+            consecutiveNetworkErrors <= 2
+              ? 5000 + Math.random() * 5000
+              : 10000 + Math.random() * 10000;
+          console.error(`  等待 ${Math.round(waitTime / 1000)}s 后重试...`);
+          await delay(waitTime / 1000, waitTime / 1000);
+        }
       }
     }
+  } finally {
+    process.removeListener("SIGINT", onSigint);
+    process.removeListener("SIGTERM", onSigterm);
+    await browser?.close().catch(() => {});
   }
 }

package/src/cli/videostats.js

@@ -1,33 +1,144 @@
-import { readFileSync, writeFileSync } from 'fs';
-import { TikTokScraper } from '../lib/tiktok-scraper.mjs';
+import path from "path";
+import Database from "better-sqlite3";
+import { TikTokScraper } from "../lib/tiktok-scraper.mjs";
+
+function resolveVideoSource(inputPath) {
+  if (!inputPath) return null;
+
+  const resolved = path.resolve(inputPath);
+  const ext = path.extname(resolved).toLowerCase();
+
+  if (ext === ".db") {
+    return { kind: "db", dbPath: resolved, label: resolved };
+  }
+
+  return null;
+}
+
+function ensureVideoStatsColumns(db) {
+  const existing = new Set(
+    db
+      .prepare("PRAGMA table_info(videos)")
+      .all()
+      .map((column) => column.name),
+  );
+  const required = {
+    play_count: "INTEGER",
+    digg_count: "INTEGER",
+    comment_count: "INTEGER",
+    share_count: "INTEGER",
+    collect_count: "INTEGER",
+    stats_updated_at: "INTEGER",
+  };
+  for (const [column, type] of Object.entries(required)) {
+    if (!existing.has(column)) {
+      db.exec(`ALTER TABLE videos ADD COLUMN ${column} ${type}`);
+    }
+  }
+}
+
+function loadVideosFromDb(dbPath) {
+  const db = new Database(dbPath);
+  ensureVideoStatsColumns(db);
+  const rows = db
+    .prepare("SELECT * FROM videos ORDER BY registered_at ASC")
+    .all();
+  const videos = rows.map((row) => ({
+    id: row.id,
+    href: row.href,
+    authorUniqueId: row.author_unique_id,
+    locationCreated: row.location_created,
+    ttSeller: !!row.tt_seller,
+    registeredAt: row.registered_at,
+    userUpdateCount: row.user_update_count,
+    stats:
+      row.play_count === null &&
+      row.digg_count === null &&
+      row.comment_count === null &&
+      row.share_count === null &&
+      row.collect_count === null
+        ? undefined
+        : {
+            playCount: row.play_count,
+            diggCount: row.digg_count,
+            commentCount: row.comment_count,
+            shareCount: row.share_count,
+            collectCount: row.collect_count,
+          },
+  }));
+  return { db, videos };
+}
+
+function writeVideosToDb(db, videos) {
+  const stmt = db.prepare(`
+    UPDATE videos
+    SET play_count = ?,
+        digg_count = ?,
+        comment_count = ?,
+        share_count = ?,
+        collect_count = ?,
+        stats_updated_at = ?
+    WHERE id = ?
+  `);
+  const now = Date.now();
+  const txn = db.transaction((items) => {
+    for (const video of items) {
+      if (!video.stats) continue;
+      stmt.run(
+        video.stats.playCount ?? null,
+        video.stats.diggCount ?? null,
+        video.stats.commentCount ?? null,
+        video.stats.shareCount ?? null,
+        video.stats.collectCount ?? null,
+        now,
+        video.id,
+      );
+    }
+  });
+  txn(videos);
+}
 
 export async function handleVideoStats(options) {
-  const { statsFile, statsParallel } = options;
+  const statsSource = options.statsSource || options.statsFile;
+  const { statsParallel } = options;
 
-  if (!statsFile) {
-    console.error('用法: tt-help videostats <文件路径> -p <并发数>');
-    console.error('示例: tt-help videostats data/result-videos.json -p 3');
-    console.error('');
-    console.error('选项: -p, --parallel <N>  并发数（默认: 3）');
+  if (!statsSource) {
+    console.error("用法: tt-help videostats <db路径> -p <并发数>");
+    console.error("      tt-help videostats data/result.db -p 3");
+    console.error("");
+    console.error("选项: -p, --parallel <N>  并发数（默认: 3）");
     process.exit(1);
   }
 
-  // 读取文件
-  const videos = JSON.parse(readFileSync(statsFile, 'utf-8'));
+  const source = resolveVideoSource(statsSource);
+  if (!source) {
+    console.error("[videostats] 仅支持 .db 路径");
+    process.exit(1);
+  }
+
+  let db = null;
+  let videos = [];
+  ({ db, videos } = loadVideosFromDb(source.dbPath));
+
   if (!Array.isArray(videos) || videos.length === 0) {
-    console.error('[videostats] 文件为空或格式错误');
+    console.error("[videostats] 视频源为空或格式错误");
+    db?.close();
     process.exit(1);
   }
 
   const poolSize = Math.min(statsParallel, videos.length);
-  console.error(`[videostats] 读取 ${statsFile}: ${videos.length} 个视频，并发: ${poolSize}`);
+  console.error(
+    `[videostats] 读取 ${source.label}: ${videos.length} 个视频，并发: ${poolSize}`,
+  );
 
   // 初始化 scraper
   const scraper = new TikTokScraper({ poolSize });
   await scraper.init();
-  console.error('[videostats] 浏览器初始化完成，开始刷新...');
+  console.error("[videostats] 浏览器初始化完成，开始刷新...");
 
-  let success = 0, failed = 0, skipped = 0;
+  let success = 0,
+    failed = 0,
+    skipped = 0;
 
   try {
     for (let i = 0; i < videos.length; i += poolSize) {
@@ -35,27 +146,29 @@ export async function handleVideoStats(options) {
 
       const results = await Promise.allSettled(
         batch.map(async (v) => {
-          if (!v.href) return { video: v, error: 'no href' };
+          if (!v.href) return { video: v, error: "no href" };
           try {
             const info = await scraper.getVideoInfo(v.href);
-            if (!info) return { video: v, error: 'no info' };
+            if (!info) return { video: v, error: "no info" };
             return { video: v, info };
           } catch (err) {
             return { video: v, error: err.message };
           }
-        })
+        }),
       );
 
       for (const result of results) {
-        if (result.status === 'fulfilled') {
+        if (result.status === "fulfilled") {
           const { video, info, error } = result.value;
           if (info && info.stats) {
             video.stats = info.stats;
             success++;
           } else {
             failed++;
-            const id = video.id || video.href || '?';
-            console.error(`  [失败] ${id.substring(0, 30)} - ${error || '获取失败'}`);
+            const id = video.id || video.href || "?";
+            console.error(
+              `  [失败] ${id.substring(0, 30)} - ${error || "获取失败"}`,
+            );
           }
         } else {
           failed++;
@@ -63,19 +176,21 @@ export async function handleVideoStats(options) {
       }
 
       const processed = Math.min(i + poolSize, videos.length);
-      console.error(`[videostats] 进度: ${processed}/${videos.length} (成功: ${success}, 失败: ${failed})`);
+      console.error(
+        `[videostats] 进度: ${processed}/${videos.length} (成功: ${success}, 失败: ${failed})`,
+      );
     }
 
-    // 写回文件
-    writeFileSync(statsFile, JSON.stringify(videos, null, 2), 'utf-8');
+    writeVideosToDb(db, videos);
     console.error(``);
     console.error(`[videostats] 完成: 成功 ${success}, 失败 ${failed}`);
-    console.error(`[videostats] 已更新 ${statsFile}`);
-
+    console.error(`[videostats] 已更新 ${source.dbPath}`);
   } catch (err) {
     console.error(`[videostats] 异常: ${err.message}`);
+    db?.close();
     process.exit(1);
   } finally {
+    db?.close();
     await scraper.close();
   }
 }