tt-help-cli-ycl 1.3.35 → 1.3.37

package/README.md

@@ -14,4 +14,20 @@ npm i -g tt-help
 tt-help [options] <urls...>
 ```
 
-示例、代理配置、和更多用法请查看仓库或 `src/main.mjs` 中的帮助文本。
+## Current Recommended Entrypoints
+
+```bash
+tt-help db-import --db data/result.db --users data/result.json --done data/result-done.json --videos data/result-videos.json
+tt-help watch -o data/result.db
+tt-help attach -p 5 --server http://127.0.0.1:3001
+tt-help explore --base-port 9223 --server http://127.0.0.1:3001
+tt-help videostats data/result.db -p 3
+```
+
+说明:
+
+- `watch` / `attach` / `explore` / `comments` / `videostats` 运行期只依赖 SQLite
+- legacy JSON 的初始化/迁移通过 `db-import` 显式执行，不再在运行时自动导入
+- 推荐把运行期主库固定为 `data/result.db`
+
+更多命令说明请查看 [FEATURES.md](FEATURES.md) 或帮助文本。

package/cli.js

@@ -1,9 +1,9 @@
 #!/usr/bin/env node
-import { fileURLToPath } from 'url';
-import { dirname, resolve } from 'path';
+import { fileURLToPath } from "url";
+import { dirname, resolve } from "path";
 
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = dirname(__filename);
 
-const mainPath = resolve(__dirname, 'src', 'main.js');
+const mainPath = resolve(__dirname, "src", "npm-main.js");
 await import(`file://${mainPath}`);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tt-help-cli-ycl",
-  "version": "1.3.35",
+  "version": "1.3.37",
   "description": "TikTok user & video data scraper - extract ttSeller, verified, locationCreated from HTML source",
   "type": "module",
   "bin": {
@@ -13,7 +13,8 @@
     "scripts/"
   ],
   "scripts": {
-    "start": "node src/main.js"
+    "start": "node src/main.js",
+    "test:watch-db-smoke": "node scripts/test-watch-db-smoke.mjs"
   },
   "keywords": [
     "tiktok",
@@ -42,6 +43,10 @@
     "axios": "^1.16.1",
     "https-proxy-agent": "^9.0.0",
     "playwright": "^1.59.1",
+    "tt-help-cli-ycl": "^1.3.36",
     "undici": "^8.1.0"
+  },
+  "devDependencies": {
+    "better-sqlite3": "^12.10.0"
   }
 }

package/scripts/test-watch-db-smoke.mjs

@@ -0,0 +1,246 @@
+import assert from "node:assert/strict";
+import { mkdtempSync, rmSync } from "node:fs";
+import os from "node:os";
+import path from "node:path";
+
+import { createStore, closeStoreDb } from "../src/watch/data-store.js";
+import { startWatchServer } from "../src/watch/server.js";
+
+function createTempDbPath(prefix) {
+  const dir = mkdtempSync(path.join(os.tmpdir(), `${prefix}-`));
+  return {
+    dir,
+    dbPath: path.join(dir, "result.db"),
+  };
+}
+
+function cleanupTempDir(dir) {
+  rmSync(dir, { recursive: true, force: true });
+}
+
+function seedClaimStore(store) {
+  store.addUser({
+    uniqueId: "follow-tier2",
+    status: "pending",
+    followerCount: 100,
+    videoCount: 1,
+    guessedLocation: "DE",
+    sources: ["following"],
+  });
+  store.addUser({
+    uniqueId: "seller-tier2",
+    status: "pending",
+    followerCount: 50,
+    videoCount: 1,
+    guessedLocation: "DE",
+    ttSeller: true,
+    verified: false,
+    sources: ["comment"],
+  });
+  store.addUser({
+    uniqueId: "seed-tier1",
+    status: "pending",
+    followerCount: 10,
+    videoCount: 1,
+    guessedLocation: "PL",
+    sources: ["seed"],
+  });
+}
+
+function seedWatchStore(store) {
+  store.addUser({
+    uniqueId: "seed-tier1",
+    nickname: "Seed User",
+    status: "pending",
+    followerCount: 10,
+    videoCount: 1,
+    guessedLocation: "PL",
+    ttSeller: false,
+    verified: false,
+    sources: ["seed"],
+  });
+  store.addUser({
+    uniqueId: "seller-target",
+    nickname: "Seller Target",
+    status: "pending",
+    followerCount: 500,
+    videoCount: 1,
+    guessedLocation: "DE",
+    locationCreated: "DE",
+    ttSeller: true,
+    verified: false,
+    sources: ["comment"],
+  });
+  store.addUser({
+    uniqueId: "follow-tier2",
+    nickname: "Follow User",
+    status: "pending",
+    followerCount: 300,
+    videoCount: 1,
+    guessedLocation: "FR",
+    ttSeller: false,
+    verified: false,
+    sources: ["following"],
+  });
+  store.addUser({
+    uniqueId: "pending-update",
+    nickname: "Needs Update",
+    status: "pending",
+    guessedLocation: "US",
+    ttSeller: null,
+    verified: false,
+    userUpdateCount: 0,
+    sources: ["comment"],
+  });
+  store.addUser({
+    uniqueId: "done-es",
+    nickname: "Done ES",
+    status: "done",
+    followerCount: 800,
+    videoCount: 3,
+    guessedLocation: "ES",
+    locationCreated: "ES",
+    ttSeller: true,
+    verified: false,
+    processedAt: Date.now(),
+    processed: true,
+    sources: ["processed"],
+  });
+  store.addUser({
+    uniqueId: "restricted-it",
+    nickname: "Restricted IT",
+    status: "restricted",
+    locationCreated: "IT",
+    ttSeller: false,
+    verified: false,
+    sources: ["comment"],
+  });
+  store.addUser({
+    uniqueId: "error-user",
+    nickname: "Error User",
+    status: "error",
+    locationCreated: "US",
+    ttSeller: false,
+    verified: false,
+    sources: ["comment"],
+  });
+}
+
+async function testClaimPriorityAndRenewal() {
+  const { dir, dbPath } = createTempDbPath("tt-watch-claim");
+  try {
+    const store = createStore(dbPath);
+    seedClaimStore(store);
+
+    const first = store.claimNextJob("worker-a", 5 * 60 * 1000, null, true);
+    assert.ok(first, "expected a claim for logged-in worker");
+    assert.equal(first.uniqueId, "seed-tier1");
+
+    const renewed = store.claimNextJob("worker-a", 5 * 60 * 1000, null, true);
+    assert.ok(renewed, "expected renewal claim for same worker");
+    assert.equal(renewed.uniqueId, "seed-tier1");
+    assert.ok(
+      renewed.claimedAt >= first.claimedAt,
+      "expected renewal claimedAt to advance or stay equal",
+    );
+
+    const loggedOut = store.claimNextJob(
+      "worker-b",
+      5 * 60 * 1000,
+      null,
+      false,
+    );
+    assert.ok(loggedOut, "expected a claim for logged-out worker");
+    assert.equal(loggedOut.uniqueId, "follow-tier2");
+  } finally {
+    closeStoreDb();
+    cleanupTempDir(dir);
+  }
+}
+
+async function testWatchHttpEndpoints() {
+  const { dir, dbPath } = createTempDbPath("tt-watch-http");
+  let server;
+  try {
+    const store = createStore(dbPath);
+    seedWatchStore(store);
+
+    const started = await startWatchServer(dbPath, 0, store);
+    server = started.server;
+    const actualPort = server.address().port;
+    const baseUrl = `http://127.0.0.1:${actualPort}`;
+
+    const [statsRes, usersRes, targetRes, lightUsersRes] = await Promise.all([
+      fetch(`${baseUrl}/api/stats`),
+      fetch(`${baseUrl}/api/users?limit=3`),
+      fetch(`${baseUrl}/api/target-users`),
+      fetch(`${baseUrl}/api/users?limit=2&view=light`),
+    ]);
+
+    assert.equal(statsRes.status, 200);
+    assert.equal(usersRes.status, 200);
+    assert.equal(targetRes.status, 200);
+    assert.equal(lightUsersRes.status, 200);
+
+    const [stats, users, targets, lightUsers] = await Promise.all([
+      statsRes.json(),
+      usersRes.json(),
+      targetRes.json(),
+      lightUsersRes.json(),
+    ]);
+
+    assert.equal(stats.totalUsers, 7);
+    assert.equal(stats.pendingUsers, 4);
+    assert.equal(stats.processedUsers, 1);
+    assert.equal(stats.restrictedUsers, 1);
+    assert.equal(stats.errorUsers, 1);
+    assert.equal(stats.targetUsers, 2);
+    assert.equal(stats.userUpdateTasks, 1);
+    assert.ok(Array.isArray(stats.countryStats));
+    assert.ok(stats.countryStats.some((item) => item.country === "ES"));
+
+    assert.equal(users.total, 7);
+    assert.equal(users.users.length, 3);
+    assert.equal(users.users[0].uniqueId, "seller-target");
+
+    assert.equal(targets.total, 2);
+    assert.deepEqual(
+      targets.users.map((item) => item.uniqueId),
+      ["done-es", "seller-target"],
+    );
+
+    assert.equal(lightUsers.total, 7);
+    assert.equal(lightUsers.users.length, 2);
+    assert.deepEqual(Object.keys(lightUsers.users[0]).sort(), [
+      "followerCount",
+      "guessedLocation",
+      "locationCreated",
+      "nickname",
+      "pinned",
+      "processedAt",
+      "sources",
+      "status",
+      "ttSeller",
+      "uniqueId",
+      "verified",
+    ]);
+  } finally {
+    if (server) {
+      await new Promise((resolve) => server.close(resolve));
+    }
+    closeStoreDb();
+    cleanupTempDir(dir);
+  }
+}
+
+async function main() {
+  await testClaimPriorityAndRenewal();
+  await testWatchHttpEndpoints();
+  console.log("watch db smoke test passed");
+}
+
+main().catch((error) => {
+  console.error("watch db smoke test failed");
+  console.error(error);
+  process.exitCode = 1;
+});

package/src/cli/attach.js

@@ -1,4 +1,4 @@
-import { TikTokScraper } from '../lib/tiktok-scraper.mjs';
+import { TikTokScraper } from "../lib/tiktok-scraper.mjs";
 
 const MAX_RETRY_WAIT = 5 * 60 * 1000;
 
@@ -8,8 +8,10 @@ async function withRetry(label, fn) {
     try {
       return await fn();
     } catch (err) {
-      console.error(`[连接] ${label} 失败: ${err.message}，${backoff / 1000}秒后重试...`);
-      await new Promise(r => setTimeout(r, backoff));
+      console.error(
+        `[连接] ${label} 失败: ${err.message}，${backoff / 1000}秒后重试...`,
+      );
+      await new Promise((r) => setTimeout(r, backoff));
       if (backoff < MAX_RETRY_WAIT) backoff *= 2;
     }
   }
@@ -22,12 +24,11 @@ async function apiGet(url) {
   });
 }
 
-
 async function apiPost(url, body) {
   return withRetry(`POST ${url}`, async () => {
     const res = await fetch(url, {
-      method: 'POST',
-      headers: { 'Content-Type': 'application/json' },
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
       body: JSON.stringify(body),
     });
     return res.json();
@@ -36,64 +37,111 @@ async function apiPost(url, body) {
 
 function isBrowserClosedError(err) {
   if (!err) return false;
-  const msg = err.message || err.toString() || '';
-  return msg.includes('Target page, context or browser has been closed') ||
-         msg.includes('browser has been closed') ||
-         msg.includes('browserContext.newPage') ||
-         msg.includes('Protocol error');
+  const msg = err.message || err.toString() || "";
+  return (
+    msg.includes("Target page, context or browser has been closed") ||
+    msg.includes("browser has been closed") ||
+    msg.includes("browserContext.newPage") ||
+    msg.includes("Protocol error")
+  );
 }
 
 export async function handleAttach(options) {
   const { attachParallel, attachInterval, serverUrl, showHelp } = options;
+  let shuttingDown = false;
+  let forceExitTimer = null;
 
   if (showHelp) {
-    console.error('用法: tt-help attach [-p 并行数] [-i 间隔秒数] [-s 服务端地址]');
-    console.error('');
-    console.error('参数:');
-    console.error('  -p, --parallel <N>     并行抓取数（默认: 1）');
-    console.error('  -i, --interval <N>     无任务时轮询间隔，单位秒（默认: 10）');
-    console.error('  -s, --server <URL>     服务端地址（默认: http://127.0.0.1:3001）');
-    console.error('');
-    console.error('说明:');
-    console.error('  后台轮询服务端 /api/user-update-tasks 接口，自动抓取 TikTok 用户信息');
-    console.error('  抓取完成后通过 PUT /api/user-update-result/{uniqueId} 回传结果');
-    console.error('  浏览器崩溃时自动重启，支持长时间无人值守运行');
-    console.error('');
-    console.error('示例:');
-    console.error('  tt-help attach');
-    console.error('  tt-help attach -p 5 -i 10');
-    console.error('  tt-help attach -p 3 -i 5 -s http://127.0.0.1:3001');
+    console.error(
+      "用法: tt-help attach [-p 并行数] [-i 间隔秒数] [-s 服务端地址]",
+    );
+    console.error("");
+    console.error("参数:");
+    console.error("  -p, --parallel <N>     并行抓取数（默认: 1）");
+    console.error(
+      "  -i, --interval <N>     无任务时轮询间隔，单位秒（默认: 10）",
+    );
+    console.error(
+      "  -s, --server <URL>     服务端地址（默认: http://127.0.0.1:3001）",
+    );
+    console.error("");
+    console.error("说明:");
+    console.error(
+      "  后台轮询服务端 /api/user-update-tasks 接口，自动抓取 TikTok 用户信息",
+    );
+    console.error("  抓取完成后通过 POST /api/user-info-batch 批量回传结果");
+    console.error("  浏览器崩溃时自动重启，支持长时间无人值守运行");
+    console.error("");
+    console.error("示例:");
+    console.error("  tt-help attach");
+    console.error("  tt-help attach -p 5 -i 10");
+    console.error("  tt-help attach -p 3 -i 5 -s http://127.0.0.1:3001");
     return;
   }
 
-  console.error(`[Attach] 并行数: ${attachParallel}, 空闲间隔: ${attachInterval}秒, 服务端: ${serverUrl}`);
+  console.error(
+    `[Attach] 并行数: ${attachParallel}, 空闲间隔: ${attachInterval}秒, 服务端: ${serverUrl}`,
+  );
 
   const scraper = new TikTokScraper();
+  const shutdown = async (signal) => {
+    if (shuttingDown) return;
+    shuttingDown = true;
+    forceExitTimer = setTimeout(() => {
+      console.error("[Attach] 关闭超时，强制退出");
+      process.exit(0);
+    }, 8000);
+    forceExitTimer.unref?.();
+    console.error(`\n[Attach] 收到 ${signal}，正在关闭浏览器...`);
+    await scraper.close().catch(() => {});
+    if (forceExitTimer) {
+      clearTimeout(forceExitTimer);
+      forceExitTimer = null;
+    }
+    console.error("[Attach] 已退出");
+    process.exit(0);
+  };
+
+  const onSigint = () => {
+    void shutdown("SIGINT");
+  };
+  const onSigterm = () => {
+    void shutdown("SIGTERM");
+  };
+
+  process.on("SIGINT", onSigint);
+  process.on("SIGTERM", onSigterm);
 
   try {
     await scraper.init();
-    console.error('[Attach] 浏览器初始化完成，开始循环接收任务...');
+    console.error("[Attach] 浏览器初始化完成，开始循环接收任务...");
 
     let loopCount = 0;
     let browserRestartCount = 0;
 
-    while (true) {
+    while (!shuttingDown) {
       loopCount++;
 
       // 检查浏览器是否存活，不存活则重启
       if (!scraper.isAlive()) {
-        console.error(`[Attach] 浏览器已关闭，正在重启 (${++browserRestartCount})...`);
+        console.error(
+          `[Attach] 浏览器已关闭，正在重启 (${++browserRestartCount})...`,
+        );
         await scraper.restart();
-        console.error('[Attach] 浏览器重启完成');
+        console.error("[Attach] 浏览器重启完成");
       }
 
-      const { total, tasks } = await apiGet(`${serverUrl}/api/user-update-tasks?limit=${attachParallel}`);
+      const { total, tasks } = await apiGet(
+        `${serverUrl}/api/user-update-tasks?limit=${attachParallel}`,
+      );
 
       if (!tasks || tasks.length === 0) {
         if (loopCount === 1) {
-          console.error(`[Attach] 当前无待更新任务，${attachInterval}秒后重试...`);
+          console.error(
+            `[Attach] 当前无待更新任务，${attachInterval}秒后重试...`,
+          );
         }
-        await new Promise(r => setTimeout(r, attachInterval * 1000));
+        await new Promise((r) => setTimeout(r, attachInterval * 1000));
         continue;
       }
 
@@ -109,72 +157,84 @@ export async function handleAttach(options) {
           } catch (err) {
             return { uniqueId, info: null, error: err };
           }
-        })
+        }),
       );
 
-       let successCount = 0;
-       let failCount = 0;
-       let needRestart = false;
-
-       // 收集抓取成功的任务，记录抓取失败的
-       const successTasks = [];
-       for (const result of results) {
-         if (result.status === 'fulfilled') {
-           const { uniqueId, info, error } = result.value;
-           if (error) {
-             if (isBrowserClosedError(error)) {
-               needRestart = true;
-             }
-             console.error(`  ✗ @${uniqueId} 获取失败: ${error.message || '未知错误'}`);
-             failCount++;
-           } else if (info) {
-             successTasks.push({ uniqueId, info });
-           } else {
-             console.error(`  ✗ @${uniqueId} 未获取到用户信息`);
-             failCount++;
-           }
-         } else {
-           console.error(`  ✗ 任务执行异常: ${result.reason?.message || '未知错误'}`);
-           failCount++;
-         }
-       }
-
-       // 批量提交成功的结果
-       if (successTasks.length > 0) {
-         try {
-           const batchRet = await apiPost(`${serverUrl}/api/user-info-batch`, { updates: successTasks });
-           if (batchRet && batchRet.results) {
-             for (const r of batchRet.results) {
-               if (r.ok) {
-                 successCount++;
-                 console.error(`  ✓ @${r.uniqueId} 已提交更新`);
-               } else {
-                 failCount++;
-                 console.error(`  ✗ @${r.uniqueId} 提交失败: ${r.error}`);
-               }
-             }
-           } else {
-             successCount = successTasks.length;
-             console.error(`  ✓ 批量提交完成 (${successTasks.length} 条)`);
-           }
-         } catch (err) {
-           failCount += successTasks.length;
-           console.error(`  ✗ 批量提交失败: ${err.message}`);
-         }
-       }
+      let successCount = 0;
+      let failCount = 0;
+      let needRestart = false;
+
+      // 收集抓取成功的任务，记录抓取失败的
+      const successTasks = [];
+      for (const result of results) {
+        if (result.status === "fulfilled") {
+          const { uniqueId, info, error } = result.value;
+          if (error) {
+            if (isBrowserClosedError(error)) {
+              needRestart = true;
+            }
+            console.error(
+              `  ✗ @${uniqueId} 获取失败: ${error.message || "未知错误"}`,
+            );
+            failCount++;
+          } else if (info) {
+            successTasks.push({ uniqueId, info });
+          } else {
+            console.error(`  ✗ @${uniqueId} 未获取到用户信息`);
+            failCount++;
+          }
+        } else {
+          console.error(
+            `  ✗ 任务执行异常: ${result.reason?.message || "未知错误"}`,
+          );
+          failCount++;
+        }
+      }
+
+      // 批量提交成功的结果
+      if (successTasks.length > 0) {
+        try {
+          const batchRet = await apiPost(`${serverUrl}/api/user-info-batch`, {
+            updates: successTasks,
+          });
+          if (batchRet && batchRet.results) {
+            for (const r of batchRet.results) {
+              if (r.ok) {
+                successCount++;
+                console.error(`  ✓ @${r.uniqueId} 已提交更新`);
+              } else {
+                failCount++;
+                console.error(`  ✗ @${r.uniqueId} 提交失败: ${r.error}`);
+              }
+            }
+          } else {
+            successCount = successTasks.length;
+            console.error(`  ✓ 批量提交完成 (${successTasks.length} 条)`);
+          }
+        } catch (err) {
+          failCount += successTasks.length;
+          console.error(`  ✗ 批量提交失败: ${err.message}`);
+        }
+      }
 
       console.error(`  本批结果: ${successCount} 成功, ${failCount} 失败\n`);
 
       if (needRestart) {
-        console.error('[Attach] 检测到浏览器异常，将在下一轮重启...');
+        console.error("[Attach] 检测到浏览器异常，将在下一轮重启...");
       }
 
-      await new Promise(r => setTimeout(r, 500));
+      await new Promise((r) => setTimeout(r, 500));
     }
   } catch (err) {
     console.error(`[Attach] 运行异常: ${err.message}`);
     throw err;
   } finally {
-    await scraper.close();
+    if (forceExitTimer) {
+      clearTimeout(forceExitTimer);
+      forceExitTimer = null;
+    }
+    process.removeListener("SIGINT", onSigint);
+    process.removeListener("SIGTERM", onSigterm);
+    await scraper.close().catch(() => {});
   }
 }