tt-help-cli-ycl 1.3.78 → 1.3.80

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tt-help-cli-ycl",
-  "version": "1.3.78",
+  "version": "1.3.80",
   "description": "TikTok user & video data scraper - extract ttSeller, verified, locationCreated from HTML source",
   "type": "module",
   "bin": {

package/src/cli/attach.js

@@ -1,4 +1,5 @@
 import { TikTokScraper } from "../lib/tiktok-scraper.mjs";
+import { CDNBlockedError } from "../lib/parse-ssr.mjs";
 import { proxy as configuredProxy } from "../lib/constants.js";
 import v8 from "node:v8";
 
@@ -83,6 +84,7 @@ export async function handleAttach(options) {
     serverUrl,
     attachCountries,
     customProxy,
+    attachPoolSize,
     showHelp,
   } = options;
   const effectiveProxy = customProxy || configuredProxy;
@@ -110,6 +112,9 @@ export async function handleAttach(options) {
     attachLog(
       "  --proxy <代理地址>    HTTP 代理地址（如 http://127.0.0.1:7890），不配置则从 ~/.tt-help.json 读取",
     );
+    attachLog(
+      "  --size <N>            浏览器页面池大小（默认: 3，不建议超过 3）",
+    );
     attachLog("");
     attachLog("说明:");
     attachLog(
@@ -117,12 +122,16 @@ export async function handleAttach(options) {
     );
     attachLog("  抓取完成后通过 POST /api/user-info-batch 批量回传结果");
     attachLog("  浏览器崩溃时自动重启，支持长时间无人值守运行");
+    attachLog(
+      "  --size 控制单个浏览器内并发的 page 数，增大可提升吞吐但可能触发 TikTok 限流",
+    );
     attachLog("");
     attachLog("示例:");
     attachLog("  tt-help attach");
     attachLog("  tt-help attach -p 5 -i 10");
     attachLog("  tt-help attach -p 3 -i 5 -s http://127.0.0.1:3001");
     attachLog("  tt-help attach -c PL,DE,FR -p 5");
+    attachLog("  tt-help attach -p 5 --size 5");
     return;
   }
 
@@ -135,6 +144,7 @@ export async function handleAttach(options) {
   );
 
   const scraper = new TikTokScraper({
+    poolSize: attachPoolSize || 3,
     proxyServer: effectiveProxy || null,
   });
   const shutdown = async (signal) => {
@@ -240,6 +250,7 @@ export async function handleAttach(options) {
 
       let successCount = 0;
       let failCount = 0;
+      let cdnBlockedCount = 0;
       let needRestart = false;
 
       // 收集抓取成功的任务，记录抓取失败的
@@ -248,13 +259,20 @@ export async function handleAttach(options) {
         if (result.status === "fulfilled") {
           const { uniqueId, info, error } = result.value;
           if (error) {
-            if (isBrowserClosedError(error)) {
+            if (error instanceof CDNBlockedError) {
+              attachLog(`  ⚠ @${uniqueId} CDN限流 (Access Denied)`);
+              cdnBlockedCount++;
+              failCount++;
+            } else if (isBrowserClosedError(error)) {
               needRestart = true;
+              attachLog(`  ✗ @${uniqueId} 浏览器断开: ${error.message}`);
+              failCount++;
+            } else {
+              attachLog(
+                `  ✗ @${uniqueId} 获取失败: ${error.message || "未知错误"}`,
+              );
+              failCount++;
             }
-            attachLog(
-              `  ✗ @${uniqueId} 获取失败: ${error.message || "未知错误"}`,
-            );
-            failCount++;
           } else if (info) {
             // info 可能是 { error: true, statusCode: xxx } 表示 TikTok 给了明确响应
             if (info.error) {
@@ -318,6 +336,22 @@ export async function handleAttach(options) {
 
       attachLog(`  本批结果: ${successCount} 成功, ${failCount} 失败\n`);
 
+      // CDN 限流比例超过 30% 时，冷却 + 重启浏览器
+      const cdnRatio = cdnBlockedCount / tasks.length;
+      if (cdnRatio > 0.3) {
+        const coolDownSeconds = cdnRatio > 0.8 ? 120 : 60;
+        attachLog(
+          `  [Attach] CDN限流比例 ${(cdnRatio * 100).toFixed(0)}% (${cdnBlockedCount}/${tasks.length})，冷却 ${coolDownSeconds} 秒后重启浏览器...`,
+        );
+        await new Promise((r) => setTimeout(r, coolDownSeconds * 1000));
+        await recycleScraper(
+          scraper,
+          `CDN限流比例过高 (${cdnBlockedCount}/${tasks.length})`,
+        );
+        browserRestartCount++;
+        taskBatchCount = 0;
+      }
+
       const heap = getHeapUsage();
       if (heap.ratio >= HEAP_RESTART_RATIO) {
         await recycleScraper(

package/src/cli/explore.js

@@ -29,6 +29,7 @@ import os from "os";
 
 const MAX_RETRY_WAIT = 5 * 60 * 1000;
 const STARTUP_TIKTOK_URL = "https://www.tiktok.com/@ycl5007";
+const PAGE_GOTO_TIMEOUT = 60000; // 页面导航超时 60 秒（账户切换后需要更长时间）
 
 async function withRetry(label, fn) {
   let backoff = 1000;
@@ -181,8 +182,11 @@ export async function handleExplore(options) {
     const page = await getOrCreatePage(browser);
 
     // 先导航到 TikTok 页面，再检测登录状态
-    await page.goto(STARTUP_TIKTOK_URL, {
-      waitUntil: "domcontentloaded",
+    await withRetry("启动页面导航", async () => {
+      await page.goto(STARTUP_TIKTOK_URL, {
+        waitUntil: "domcontentloaded",
+        timeout: PAGE_GOTO_TIMEOUT,
+      });
     });
 
     // 检测登录状态（启动时只检测一次）
@@ -235,8 +239,11 @@ export async function handleExplore(options) {
         `[健康检查] 已切换到端口 ${nextAccount.port}${effectiveProxy ? ", 代理: " + effectiveProxy : ""}`,
       );
       // 切换账户后先导航到 TikTok 页面，再重新检测登录状态
-      await page.goto(STARTUP_TIKTOK_URL, {
-        waitUntil: "domcontentloaded",
+      await withRetry("账户切换后页面导航", async () => {
+        await page.goto(STARTUP_TIKTOK_URL, {
+          waitUntil: "domcontentloaded",
+          timeout: PAGE_GOTO_TIMEOUT,
+        });
       });
       loggedIn = await safeCheckLogin(page);
       console.error(

package/src/cli/refresh.js

@@ -28,6 +28,7 @@ import os from "os";
 
 const MAX_RETRY_WAIT = 5 * 60 * 1000;
 const STARTUP_TIKTOK_URL = "https://www.tiktok.com/@ycl5007";
+const PAGE_GOTO_TIMEOUT = 60000; // 页面导航超时 60 秒（账户切换后需要更长时间）
 
 async function withRetry(label, fn) {
   let backoff = 1000;
@@ -181,8 +182,11 @@ export async function handleRefresh(options) {
     const page = await getOrCreatePage(browser);
 
     // 导航到 TikTok 页面
-    await page.goto(STARTUP_TIKTOK_URL, {
-      waitUntil: "domcontentloaded",
+    await withRetry("启动页面导航", async () => {
+      await page.goto(STARTUP_TIKTOK_URL, {
+        waitUntil: "domcontentloaded",
+        timeout: PAGE_GOTO_TIMEOUT,
+      });
     });
 
     // 检测登录状态
@@ -233,8 +237,11 @@ export async function handleRefresh(options) {
         cdpOptions.proxyServer = effectiveProxy;
       }
       console.error(`[健康检查] 已切换到端口 ${nextAccount.port}`);
-      await page.goto(STARTUP_TIKTOK_URL, {
-        waitUntil: "domcontentloaded",
+      await withRetry("账户切换后页面导航", async () => {
+        await page.goto(STARTUP_TIKTOK_URL, {
+          waitUntil: "domcontentloaded",
+          timeout: PAGE_GOTO_TIMEOUT,
+        });
       });
       loggedIn = await isLoggedIn(page);
       console.error(

package/src/lib/args.js

@@ -534,6 +534,7 @@ function parseAttachArgs(args) {
   let serverUrl = defaultServer;
   let countries = [];
   let customProxy = null;
+  let poolSize = 3;
 
   for (let i = 0; i < args.length; i++) {
     const arg = args[i];
@@ -550,6 +551,8 @@ function parseAttachArgs(args) {
         .filter(Boolean);
     } else if (arg === "--proxy") {
       customProxy = args[++i];
+    } else if (arg === "--size") {
+      poolSize = parseInt(args[++i], 10) || 3;
     }
   }
 
@@ -560,6 +563,7 @@ function parseAttachArgs(args) {
     serverUrl,
     attachCountries: countries,
     customProxy,
+    attachPoolSize: poolSize,
     urls: [],
     outputFormat: "json",
     exploreCount: 0,

package/src/lib/browser/cdp.js

@@ -277,7 +277,8 @@ export async function switchAccount(oldAccount, newAccount, proxyServer) {
 
   const browser = await ensureBrowserReady(newCdpOptions);
 
-  await new Promise((r) => setTimeout(r, 10000));
+  // 等待浏览器完全稳定（Windows 下 Edge 启动后需要更长时间）
+  await new Promise((r) => setTimeout(r, 15000));
 
   return browser;
 }

package/src/lib/parse-ssr.mjs

@@ -1,3 +1,25 @@
+/**
+ * CDN 限流错误（Akamai Access Denied）
+ */
+export class CDNBlockedError extends Error {
+  constructor(message = "CDN限流 (Access Denied)", reference) {
+    super(message);
+    this.name = "CDNBlockedError";
+    this.reference = reference;
+  }
+}
+
+/**
+ * 检测 HTML 是否为 CDN Access Denied
+ * 返回 { isBlocked: true, reference: "xxx" } 或 null
+ */
+export function detectAccessDenied(rawHtml) {
+  if (!rawHtml || typeof rawHtml !== "string") return null;
+  if (!rawHtml.includes("Access Denied")) return null;
+  const refMatch = rawHtml.match(/Reference\s*#\s*([\w.]+)/);
+  return { isBlocked: true, reference: refMatch ? refMatch[1] : null };
+}
+
 /**
  * 判断失败是否可重试
  * - 有 statusCode（无论值是多少）：TikTok 给了明确响应，不可重试
@@ -5,6 +27,10 @@
  */
 export function isRetryableFailure(rawHtml) {
   if (!rawHtml || typeof rawHtml !== "string") return false;
+  // Access Denied = CDN 限流 = 可重试
+  if (detectAccessDenied(rawHtml)) {
+    return true;
+  }
   // 没有 SSR 标记 = 空壳 HTML = 可重试
   if (!rawHtml.includes("__UNIVERSAL_DATA_FOR_REHYDRATION__")) {
     return true;
@@ -49,6 +75,15 @@ function parseSSR(rawHtml) {
 }
 
 export function parseUserInfo(rawHtml) {
+  // 先检查 CDN 限流
+  const denied = detectAccessDenied(rawHtml);
+  if (denied) {
+    throw new CDNBlockedError(
+      `CDN限流 (Access Denied, ref:${denied.reference || "N/A"})`,
+      denied.reference,
+    );
+  }
+
   const data = parseSSR(rawHtml);
   if (!data) return null;
   const scopeKeys = data.__DEFAULT_SCOPE__

package/src/lib/tiktok-scraper.mjs

@@ -4,6 +4,8 @@ import {
   parseUserInfo,
   parseVideoInfo,
   isRetryableFailure,
+  CDNBlockedError,
+  detectAccessDenied,
 } from "./parse-ssr.mjs";
 
 const DEFAULT_POOL_SIZE = 3;
@@ -225,6 +227,14 @@ export class TikTokScraper {
         `https://www.tiktok.com/@${uniqueId}`,
         slot,
       );
+      // CDN 限流立即抛出，不重试
+      if (detectAccessDenied(rawHtml)) {
+        const denied = detectAccessDenied(rawHtml);
+        throw new CDNBlockedError(
+          `CDN限流 (Access Denied, ref:${denied.reference || "N/A"})`,
+          denied.reference,
+        );
+      }
       let result = parseUserInfo(rawHtml);
       for (let attempt = 1; !result && attempt <= maxRetries; attempt++) {
         // 检查是否值得重试：用户异常/不存在则跳过重试
@@ -239,6 +249,14 @@ export class TikTokScraper {
           `https://www.tiktok.com/@${uniqueId}`,
           slot,
         );
+        // 重试中也检查 CDN 限流
+        if (detectAccessDenied(rawHtml)) {
+          const denied = detectAccessDenied(rawHtml);
+          throw new CDNBlockedError(
+            `CDN限流 (Access Denied, ref:${denied.reference || "N/A"})`,
+            denied.reference,
+          );
+        }
         result = parseUserInfo(rawHtml);
       }
       return result || null;

package/src/watch/data-store.js

@@ -143,6 +143,66 @@ function initUserDb(filePath) {
   if (!existingJobColumns.has("bio_link")) {
     db.exec(`ALTER TABLE jobs ADD COLUMN bio_link TEXT`);
   }
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS jobs_base (
+      unique_id TEXT PRIMARY KEY,
+      nickname TEXT,
+      status TEXT DEFAULT 'pending',
+      sources TEXT,
+      claimed_by TEXT,
+      claimed_at INTEGER,
+      error TEXT,
+      pinned INTEGER DEFAULT 0,
+      no_video INTEGER DEFAULT 0,
+      restricted INTEGER DEFAULT 0,
+      user_update_count INTEGER DEFAULT 0,
+      tt_seller INTEGER,
+      verified INTEGER,
+      video_count INTEGER DEFAULT 0,
+      comment_count INTEGER DEFAULT 0,
+      guessed_location TEXT,
+      location_created TEXT,
+      confirmed_location TEXT,
+      modified_at INTEGER,
+      follower_count INTEGER DEFAULT 0,
+      following_count INTEGER DEFAULT 0,
+      heart_count INTEGER DEFAULT 0,
+      refresh_time INTEGER,
+      processed INTEGER DEFAULT 0,
+      processed_at INTEGER,
+      created_at INTEGER,
+      updated_at INTEGER,
+      region TEXT,
+      signature TEXT,
+      sec_uid TEXT,
+      status_code INTEGER,
+      latest_video_time INTEGER,
+      bio_link TEXT
+    )
+  `);
+
+  // 迁移：为已存在的 jobs_base 表补全列
+  const existingJobBaseColumns = new Set(
+    db
+      .prepare("PRAGMA table_info(jobs_base)")
+      .all()
+      .map((c) => c.name),
+  );
+  if (!existingJobBaseColumns.has("status_code")) {
+    db.exec(`ALTER TABLE jobs_base ADD COLUMN status_code INTEGER`);
+  }
+  if (!existingJobBaseColumns.has("latest_video_time")) {
+    db.exec(`ALTER TABLE jobs_base ADD COLUMN latest_video_time INTEGER`);
+  }
+  if (!existingJobBaseColumns.has("confirmed_location")) {
+    db.exec(`ALTER TABLE jobs_base ADD COLUMN confirmed_location TEXT`);
+  }
+  if (!existingJobBaseColumns.has("modified_at")) {
+    db.exec(`ALTER TABLE jobs_base ADD COLUMN modified_at INTEGER`);
+  }
+  if (!existingJobBaseColumns.has("bio_link")) {
+    db.exec(`ALTER TABLE jobs_base ADD COLUMN bio_link TEXT`);
+  }
   db.exec(`
     CREATE TABLE IF NOT EXISTS raw_jobs (
       unique_id TEXT PRIMARY KEY,
@@ -635,11 +695,16 @@ function getDashboardStatsFromDb(targetLocations = []) {
         .all(...targetLocations)
     : [];
 
+  const jobsBaseCount = db
+    .prepare("SELECT COUNT(*) as total FROM jobs_base")
+    .get().total;
+
   return {
     totalUsers: aggregateRow.total,
     rawJobs: getRawJobsCount(),
     dbTotalUsers: getUserDbCount(),
     jobsTotal: aggregateRow.total,
+    jobsBaseTotal: jobsBaseCount,
     jobsPending: aggregateRow.pending,
     processedUsers: aggregateRow.done,
     pendingUsers: aggregateRow.pending,
@@ -1730,6 +1795,91 @@ function inferStatus(u) {
   return "pending";
 }
 
+function addJobBaseToDb(user) {
+  if (!db) return;
+  const now = Date.now();
+  db.prepare(
+    `
+    INSERT OR IGNORE INTO jobs_base (
+      unique_id,
+      nickname,
+      status,
+      sources,
+      claimed_by,
+      claimed_at,
+      error,
+      pinned,
+      no_video,
+      restricted,
+      user_update_count,
+      tt_seller,
+      verified,
+      video_count,
+      comment_count,
+      guessed_location,
+      location_created,
+      follower_count,
+      following_count,
+      heart_count,
+      refresh_time,
+      processed,
+      processed_at,
+      created_at,
+      updated_at,
+      region,
+      signature,
+      bio_link,
+      sec_uid
+    )
+    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+  `,
+  ).run(
+    user.uniqueId,
+    user.nickname || null,
+    user.status || inferStatus(user),
+    JSON.stringify(
+      Array.isArray(user.sources) ? [...new Set(user.sources)] : [],
+    ),
+    user.claimedBy || null,
+    user.claimedAt || null,
+    user.error || null,
+    user.pinned ? 1 : 0,
+    user.noVideo ? 1 : 0,
+    user.restricted ? 1 : 0,
+    user.userUpdateCount || 0,
+    user.ttSeller === undefined ||
+      user.ttSeller === null ||
+      user.ttSeller === ""
+      ? null
+      : user.ttSeller
+        ? 1
+        : 0,
+    user.verified === undefined ||
+      user.verified === null ||
+      user.verified === ""
+      ? null
+      : user.verified
+        ? 1
+        : 0,
+    user.videoCount || 0,
+    user.commentCount || 0,
+    user.guessedLocation || null,
+    user.locationCreated || null,
+    user.followerCount || 0,
+    user.followingCount || 0,
+    user.heartCount || 0,
+    user.refreshTime || null,
+    user.processed ? 1 : 0,
+    user.processedAt || null,
+    user.createdAt || now,
+    user.updatedAt || now,
+    user.region || null,
+    user.signature || null,
+    user.bioLink?.link || user.bioLink?.url || user.bioLink || null,
+    user.secUid || null,
+  );
+}
+
 function addJob(user) {
   if (!db) {
     addUserToDb(user);
@@ -1951,9 +2101,9 @@ export function createStore(filePath) {
   function addUser(user, append) {
     const memoryIdx = uidIndex.get(user.uniqueId);
     if (db && memoryIdx === undefined) {
-      const existingJob = getJobRow(user.uniqueId);
-      if (existingJob) {
-        return updateJobInfo(user.uniqueId, user, false);
+      // 用 users 表判重（所有发现过的用户合集），而不是 jobs 表
+      if (hasUserInDb(user.uniqueId)) {
+        return;
       }
       addJob(user);
       return;
@@ -2802,7 +2952,8 @@ export function createStore(filePath) {
     const newUsers = [];
     for (const d of unique) {
       if (!hasUser(d.uniqueId)) {
-        addJob(d);
+        addUserToDb(d);
+        addJobBaseToDb(d);
         newUsers.push(d.uniqueId);
       }
     }
@@ -3423,37 +3574,37 @@ export function createStore(filePath) {
       });
       txn(updates);
 
-      // 在事务外执行移动操作（避免嵌套 transaction 问题）
+      // 批量移动非商家用户到 raw_jobs（优化：一次 SQL 搞定）
       if (moveList.length > 0) {
-        const moveTxn = db.transaction((ids) => {
-          ids.forEach((uid) => {
-            db.prepare(
-              `
-              INSERT OR REPLACE INTO raw_jobs (
-                unique_id, nickname, status, sources, claimed_by, claimed_at,
-                error, pinned, no_video, restricted, user_update_count,
-                tt_seller, verified, video_count, comment_count,
-                guessed_location, location_created, confirmed_location, modified_at,
-                follower_count, following_count, heart_count, refresh_time,
-                processed, processed_at, created_at, updated_at,
-                region, signature, bio_link, sec_uid, status_code, latest_video_time
-              )
-              SELECT
-                unique_id, nickname, status, sources, claimed_by, claimed_at,
-                error, pinned, no_video, restricted, user_update_count,
-                tt_seller, verified, video_count, comment_count,
-                guessed_location, location_created, confirmed_location, modified_at,
-                follower_count, following_count, heart_count, refresh_time,
-                processed, processed_at, created_at, updated_at,
-                region, signature, bio_link, sec_uid, status_code, latest_video_time
-              FROM jobs WHERE unique_id = ?
-            `,
-            ).run(uid);
-
-            db.prepare("DELETE FROM jobs WHERE unique_id = ?").run(uid);
-          });
-        });
-        moveTxn(moveList);
+        const placeholders = moveList.map(() => "?").join(",");
+        // 批量 INSERT 到 raw_jobs
+        db.prepare(
+          `
+          INSERT OR REPLACE INTO raw_jobs (
+            unique_id, nickname, status, sources, claimed_by, claimed_at,
+            error, pinned, no_video, restricted, user_update_count,
+            tt_seller, verified, video_count, comment_count,
+            guessed_location, location_created, confirmed_location, modified_at,
+            follower_count, following_count, heart_count, refresh_time,
+            processed, processed_at, created_at, updated_at,
+            region, signature, bio_link, sec_uid, status_code, latest_video_time
+          )
+          SELECT
+            unique_id, nickname, status, sources, claimed_by, claimed_at,
+            error, pinned, no_video, restricted, user_update_count,
+            tt_seller, verified, video_count, comment_count,
+            guessed_location, location_created, confirmed_location, modified_at,
+            follower_count, following_count, heart_count, refresh_time,
+            processed, processed_at, created_at, updated_at,
+            region, signature, bio_link, sec_uid, status_code, latest_video_time
+          FROM jobs WHERE unique_id IN (${placeholders})
+        `,
+        ).run(...moveList);
+
+        // 批量 DELETE 从 jobs
+        db.prepare(`DELETE FROM jobs WHERE unique_id IN (${placeholders})`).run(
+          ...moveList,
+        );
       }
 
       // 清理内部标记

package/src/watch/public/app.js

@@ -159,6 +159,9 @@ function renderStats() {
   flashEl("statProcessing", d.processingUsers || 0);
   flashEl("statDone", d.processedUsers);
   flashEl("statPending", d.pendingUsers);
+  const statJobsBase = document.getElementById("statJobsBase");
+  if (statJobsBase)
+    statJobsBase.textContent = `jobs_base: ${formatStatNum(d.jobsBaseTotal || 0)}`;
   flashEl("statError", d.errorUsers);
   flashEl("statRestricted", d.restrictedUsers);
   flashEl("statTarget", d.targetUsers, { full: true });

package/src/watch/public/index.html

@@ -35,6 +35,7 @@
       <div class="stat-card clickable pending-card" id="statPendingCard" onclick="navigateToPending()">
         <div class="label">待处理</div>
         <div class="value pending" id="statPending">0</div>
+        <div class="value-sub" id="statJobsBase">jobs_base: 0</div>
       </div>
       <div class="stat-card">
         <div class="label">错误</div>

package/src/watch/server.js

@@ -217,6 +217,17 @@ export function startWatchServer(dataAnchor, port = 3000, existingStore) {
       }
 
       // 调试接口：直接查询数据库原始数据
+      // 临时 API：执行 VACUUM 优化数据库
+      if (req.method === "POST" && routePath === "/api/db-vacuum") {
+        try {
+          store.rawQuery("VACUUM");
+          sendJSON(res, 200, { message: "VACUUM completed" });
+        } catch (e) {
+          sendJSON(res, 400, { error: e.message });
+        }
+        return;
+      }
+
       if (req.method === "GET" && routePath === "/api/db-query") {
         const sql = params.sql || "SELECT * FROM jobs LIMIT 10";
         const limit = Math.min(parseInt(params.limit) || 100, 1000);