tt-help-cli-ycl 1.4.147 → 1.4.148

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tt-help-cli-ycl",
-  "version": "1.4.147",
+  "version": "1.4.148",
   "description": "TikTok user & video data scraper - extract ttSeller, verified, locationCreated from HTML source",
   "type": "module",
   "bin": {

package/src/cli/attach.js

@@ -143,12 +143,17 @@ export async function handleAttach(options) {
 
   if (webserverUrl) {
     attachLog(`[Attach] webserver 模式: ${webserverUrl}`);
+    // webserver 端共享 scraper（poolSize 默认 8，每 slot 串行），当 -p 并发数大于
+    // poolSize 时，请求会在 webserver 内排队，端到端耗时可达数分钟（实测 270~310s）。
+    // 30s 超时太短会让整批任务被误判失败（aborted due to timeout），故放宽到 10 分钟
+    // 作为兜底——webserver 进程若真的挂了，连接会立即断开报错，不会真的空等 10 分钟。
+    const WEBSERVER_FETCH_TIMEOUT_MS = 600000;
     getUserInfo = async (uniqueId) => {
       const res = await fetch(`${webserverUrl}/api/tiktok/user`, {
         method: "POST",
         headers: { "Content-Type": "application/json" },
         body: JSON.stringify({ uniqueId }),
-        signal: AbortSignal.timeout(30000),
+        signal: AbortSignal.timeout(WEBSERVER_FETCH_TIMEOUT_MS),
       });
       let data = null;
       try {

package/src/cli/product-fill.js

@@ -342,29 +342,68 @@ export async function handleProductFill(options) {
   };
 
   /** CDP 直抓：验证码在已登录浏览器中弹出时，等待人工解决后自动重试 */
-  const fetchViaCDP = async (seoUrl) => {
+  const fetchViaCDP = async (seoUrl, uniqueId) => {
     try {
       return await getProductInfoCDP(page, seoUrl);
     } catch (e) {
       if (!e.message?.includes("Blocked by TikTok")) throw e;
-      // 验证码：优先检测可见容器并等待人工解决（浏览器就在本机屏幕上）
-      const cap = await detectCaptcha(page);
-      if (cap?.exists) {
-        log("  ⏳ 检测到验证码，请在已登录的浏览器窗口中手动解决...");
+      // 撞码即向服务端报一次（errorType: captcha），之后无论人工解决、
+      // 自动放行还是超时冷却，本任务流程不再重复上报
+      await apiPost(`${serverUrl}/api/error-report`, {
+        userId,
+        username: uniqueId,
+        errorType: "captcha",
+        errorMessage: e.message,
+        stage: "product-fill",
+      }).catch(() => {});
+      // 验证码拦截统一等待（上限 3 分钟）：容器可能稍后才渲染（HTML 已含
+      // 标记、DOM 未挂载），也可能全程不出现（风控软拦/结构变化）。
+      // 轮询期间容器出现 → 交人工解决；容器未出现但页面恢复出商品数据 →
+      // 直接重试；到点仍拦截 → 抛 captcha 错误走 30s 冷却
+      log(
+        `  ⏳ 验证码拦截，等待处理（上限 ${CDP_CAPTCHA_WAIT_MS / 1000}s，可在浏览器中人工解决）...`,
+      );
+      const deadline = Date.now() + CDP_CAPTCHA_WAIT_MS;
+      let cap = await detectCaptcha(page);
+      while (!cap && Date.now() < deadline) {
+        await new Promise((r) => setTimeout(r, 5000));
+        cap = await detectCaptcha(page);
+        if (!cap) {
+          // 容器始终没出现：探测页面是否已自动放行
+          const html = await page.content().catch(() => "");
+          if (
+            html.includes('"product_model"') ||
+            html.includes('"review_ratings"')
+          ) {
+            log("  ✓ 拦截已自动解除，重新抓取...");
+            return getProductInfoCDP(page, seoUrl);
+          }
+        }
+      }
+      if (cap) {
+        log("  ⏳ 请在已登录的浏览器窗口中手动解决验证码...");
         const r = await waitAndGetCaptcha(page, {
-          waitMs: CDP_CAPTCHA_WAIT_MS,
+          waitMs: Math.max(0, deadline - Date.now()),
           pollInterval: 5000,
           log,
         });
         if (r.resolved) {
           log("  ✓ 验证码已解决，重新抓取...");
-          return getProductInfoCDP(page, seoUrl);
+          try {
+            return await getProductInfoCDP(page, seoUrl);
+          } catch (retryErr) {
+            // 重试又撞码：仍标记 captcha 走冷却，避免误计 noteFail 哨兵
+            if (retryErr.message?.includes("Blocked by TikTok")) {
+              retryErr.captcha = true;
+            }
+            throw retryErr;
+          }
         }
       }
       const err = new Error(
-        cap?.exists
+        cap
           ? "验证码等待人工解决超时"
-          : "验证码拦截（无可见验证码容器）",
+          : "验证码拦截（3 分钟内未出现容器且未自动解除）",
       );
       err.captcha = true;
       throw err;
@@ -372,10 +411,10 @@ export async function handleProductFill(options) {
   };
 
   /** 抓取商品详情，带浏览器断连恢复与空数据防御 */
-  const getTaskDetail = async (seoUrl, country) => {
+  const getTaskDetail = async (seoUrl, country, uniqueId) => {
     let detail;
     try {
-      detail = await fetchViaCDP(seoUrl);
+      detail = await fetchViaCDP(seoUrl, uniqueId);
     } catch (e) {
       if (isBrowserDisconnected(browser, e)) {
         // 确凿断连（CDP 断开 / 明确关闭消息）才重启浏览器
@@ -399,7 +438,7 @@ export async function handleProductFill(options) {
           err.retryable = true;
           throw err;
         }
-        detail = await fetchViaCDP(seoUrl);
+        detail = await fetchViaCDP(seoUrl, uniqueId);
       } else if (isNetworkError(e)) {
         // 网络层错误：代理断连/Clash 切节点，浏览器没死，不杀浏览器，
         // 记为可重试错误跳过本轮，下轮代理探测自然跟上
@@ -492,7 +531,7 @@ export async function handleProductFill(options) {
         }
         log(`  → 补商品 @${uniqueId} ...`);
         try {
-          const detail = await getTaskDetail(seoUrl, profile.country);
+          const detail = await getTaskDetail(seoUrl, profile.country, uniqueId);
           const post = await apiPost(
             `${serverUrl}/api/product-fill/${encodeURIComponent(uniqueId)}`,
             { productDetail: detail },
@@ -518,6 +557,9 @@ export async function handleProductFill(options) {
             log(
               `  ⏳ @${uniqueId} 验证码拦截，冷却 ${CAPTCHA_BACKOFF_MS / 1000} 秒`,
             );
+            // 验证码是全局风控：立即终止本批次剩余任务，避免批次内
+            // 继续 goto 顶着风控连刷；30s 冷却在 while 顶部领取前生效
+            break;
           } else if (e.delisted) {
             // 壳页/下架：seller_id 为空且无销售数据 → 立即哨兵放弃
             batchSkipped++;

package/src/watch/db-schema.js

@@ -103,6 +103,9 @@ export function initDb(filePath) {
   db = new Database(dbPath);
   db.pragma("journal_mode = WAL");
   db.pragma("busy_timeout = 5000");
+  // 调大 SQLite 页缓存：主库 7.3GB，task_queue 等热表的回表若走磁盘 I/O 会非常慢
+  // （实测同规模数据 16MB 缓存 1.5s → 256MB 1.1s）。512MB 足以让 task_queue 全量驻留内存。
+  db.pragma("cache_size = -524288");
 
   // ===== users 表 =====
   db.exec(`
@@ -423,10 +426,11 @@ export function initDb(filePath) {
     `CREATE INDEX IF NOT EXISTS idx_jobs_list_done ON jobs(pinned, processed_at, follower_count, unique_id) WHERE status = 'done'`,
     `CREATE INDEX IF NOT EXISTS idx_jobs_list_error ON jobs(pinned, processed_at, follower_count, unique_id) WHERE status = 'error'`,
     `CREATE INDEX IF NOT EXISTS idx_jobs_list_restricted ON jobs(pinned, processed_at, follower_count, unique_id) WHERE status = 'restricted'`,
-    // jobs_done 冷归档表索引（done 列表 / 国家统计 / 重检）
+    // jobs_done 冷归档表索引（done 列表 / 国家统计 / 重检 / no_video 计数）
     `CREATE INDEX IF NOT EXISTS idx_jobs_done_list ON jobs_done(pinned, processed_at, follower_count, unique_id) WHERE status = 'done'`,
     `CREATE INDEX IF NOT EXISTS idx_jobs_done_loc ON jobs_done(location_created)`,
     `CREATE INDEX IF NOT EXISTS idx_jobs_done_status_updated ON jobs_done(status, updated_at) WHERE status = 'done' AND (tt_seller IS NULL OR tt_seller != 1)`,
+    `CREATE INDEX IF NOT EXISTS idx_jobs_done_novideo ON jobs_done(no_video) WHERE no_video = 1`,
   ];
   for (const sql of indexes) {
     db.exec(sql);

package/src/watch/public/app.js

@@ -5286,21 +5286,20 @@ function renderCmdRows(myCmds, userId, procs, isOnline, llmAnalysis, clientId) {
         styled(
           "span",
           "display:flex;gap:2px",
-          (isOnline
-            ? tag(
-                "button",
-                {
-                  onclick:
-                    "wdToggleClientCmd('" + esc(userId) + "'," + idx + ")",
-                  style:
-                    "padding:1px 5px;font-size:11px;background:" +
-                    (c.enabled ? "#f59e0b" : "#4ade80") +
-                    ";color:#000;border:none;border-radius:2px;cursor:pointer",
-                  title: c.enabled ? "停止" : "启动",
-                },
-                c.enabled ? "⏹" : "▶",
-              )
-            : "") +
+          tag(
+            "button",
+            {
+              onclick: "wdToggleClientCmd('" + esc(userId) + "'," + idx + ")",
+              style:
+                "padding:1px 5px;font-size:11px;background:" +
+                (c.enabled ? "#f59e0b" : "#4ade80") +
+                ";color:#000;border:none;border-radius:2px;cursor:pointer",
+              title:
+                (c.enabled ? "停止" : "启动") +
+                (isOnline ? "" : "（设备离线，上线后生效）"),
+            },
+            c.enabled ? "⏹" : "▶",
+          ) +
             tag(
               "button",
               {

package/src/watch/server.js

@@ -104,14 +104,16 @@ const STAT_GROUPS = {
             SUM(CASE WHEN no_video = 1 THEN 1 ELSE 0 END) as noVideo
           FROM jobs
           UNION ALL
+          -- jobs_done 冷归档表：全部是 done 且只增不改。改用覆盖索引/部分索引计数，
+          -- 避免对 147 万行全表回表（原 SUM(CASE WHEN no_video=1) 无法匹配索引）
           SELECT
-            COUNT(*) as jobsTotal,
-            COUNT(*) as done,
+            c.cnt as jobsTotal,
+            c.cnt as done,
             0 as error,
             0 as restricted,
             0 as targetUsers,
-            SUM(CASE WHEN no_video = 1 THEN 1 ELSE 0 END) as noVideo
-          FROM jobs_done
+            (SELECT COUNT(*) FROM jobs_done WHERE no_video = 1) as noVideo
+          FROM (SELECT COUNT(*) as cnt FROM jobs_done) c
         )`,
       },
       {
@@ -389,8 +391,15 @@ function computeStatsIncremental(st) {
   };
 }
 
-// 快项失效：任务领取/提交后调用，下一次 /api/stats 请求即触发 fast 组后台刷新
+// 快项失效：任务领取/提交后调用，下一次 /api/stats 请求即触发 fast 组后台刷新。
+// 节流：53 个采集进程高频领/交任务，若每次都失效会导致 fast 组几乎每 10s 全量重算，
+// 把失效限制为至少间隔 FAST_DIRTY_MIN_INTERVAL，重算节奏由 fast TTL(30s) 主导。
+let lastFastDirtyTime = 0;
+const FAST_DIRTY_MIN_INTERVAL = 30 * 1000;
 function markFastStatsDirty() {
+  const now = Date.now();
+  if (now - lastFastDirtyTime < FAST_DIRTY_MIN_INTERVAL) return;
+  lastFastDirtyTime = now;
   groupState.fast.cacheTime = 0;
 }
 
@@ -579,6 +588,7 @@ export function startWatchServer(
         "CREATE INDEX IF NOT EXISTS idx_task_queue_status ON task_queue(status)",
         "CREATE INDEX IF NOT EXISTS idx_jobs_done_list ON jobs_done(pinned, processed_at, follower_count, unique_id) WHERE status = 'done'",
         "CREATE INDEX IF NOT EXISTS idx_jobs_done_loc ON jobs_done(location_created)",
+        "CREATE INDEX IF NOT EXISTS idx_jobs_done_novideo ON jobs_done(no_video) WHERE no_video = 1",
         "CREATE INDEX IF NOT EXISTS idx_videos_user_update ON videos(user_update_count)",
         // product-fill 领任务查询的部分索引：只索引候选行，避免全表扫描 164 万行
         "CREATE INDEX IF NOT EXISTS idx_jobs_product_fill ON jobs(claimed_at, updated_at) WHERE top_video_seo_url IS NOT NULL AND top_video_seo_url != '' AND (product_name IS NULL OR product_name = '') AND product_rating IS NULL AND product_sold_count IS NULL",