tt-help-cli-ycl 1.4.110 → 1.4.112

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tt-help-cli-ycl",
-  "version": "1.4.110",
+  "version": "1.4.112",
   "description": "TikTok user & video data scraper - extract ttSeller, verified, locationCreated from HTML source",
   "type": "module",
   "bin": {

package/scripts/test-product-fill-db.mjs

@@ -37,6 +37,21 @@ db.prepare(
   `INSERT INTO jobs (unique_id, status, tt_seller, top_video_seo_url, product_name, updated_at)
    VALUES ('sellerC', 'done', 1, NULL, NULL, ?)`,
 ).run(now);
+// D: 有 seoUrl、name 空但 rating 有值 → 已抓到数据，不应被领取（防死循环）
+db.prepare(
+  `INSERT INTO jobs (unique_id, status, tt_seller, top_video_seo_url, product_name, product_rating, updated_at)
+   VALUES ('sellerD', 'done', 1, 'https://www.tiktok.com/shop/pdp/d1', NULL, 4.5, ?)`,
+).run(now);
+// E: 有 seoUrl、name 空但 sold_count 有值 → 已抓到数据，不应被领取（防死循环）
+db.prepare(
+  `INSERT INTO jobs (unique_id, status, tt_seller, top_video_seo_url, product_name, product_sold_count, updated_at)
+   VALUES ('sellerE', 'done', 1, 'https://www.tiktok.com/shop/pdp/e1', '', 120, ?)`,
+).run(now);
+// F: 有 seoUrl、无商品 → 与 A 一起验证 claim 防重复领取
+db.prepare(
+  `INSERT INTO jobs (unique_id, status, tt_seller, top_video_seo_url, product_name, updated_at)
+   VALUES ('sellerF', 'done', 1, 'https://www.tiktok.com/shop/pdp/f1', NULL, ?)`,
+).run(now + 1);
 
 let passed = 0;
 function check(name, fn) {
@@ -54,16 +69,61 @@ console.log("===== product-fill 服务端数据层测试 =====");
 
 console.log("\n[1] getPendingProductTasks：只领缺商品的");
 const tasks = store.getPendingProductTasks(5);
-check("返回 1 条（仅 sellerA）", () => {
+check("只返回缺商品的（sellerA/sellerF，不含 B/C/D/E）", () => {
   const ids = tasks.map((t) => t.uniqueId);
-  assert(ids.length === 1 && ids[0] === "sellerA", `实际: ${ids.join(",")}`);
+  assert(ids.length === 2, `实际: ${ids.join(",")}`);
+  assert(
+    ids.includes("sellerA") && ids.includes("sellerF"),
+    `实际: ${ids.join(",")}`,
+  );
 });
 check("任务带 seoUrl / productId / productTitle", () => {
-  assert(tasks[0].seoUrl === "https://www.tiktok.com/shop/pdp/a1");
-  assert("productId" in tasks[0] && "productTitle" in tasks[0]);
+  const a = tasks.find((t) => t.uniqueId === "sellerA");
+  assert(a.seoUrl === "https://www.tiktok.com/shop/pdp/a1");
+  assert("productId" in a && "productTitle" in a);
 });
 check("limit=1 只取 1 条", () => {
+  db.prepare(
+    "UPDATE jobs SET claimed_at = NULL, claimed_by = NULL WHERE unique_id IN ('sellerA','sellerF')",
+  ).run();
   assert(store.getPendingProductTasks(1).length === 1);
+  db.prepare(
+    "UPDATE jobs SET claimed_at = NULL, claimed_by = NULL WHERE unique_id IN ('sellerA','sellerF')",
+  ).run();
+});
+check("领取即锁定：写入 claimed_by/claimed_at", () => {
+  const claimed = store.getPendingProductTasks(5, "testerA");
+  assert(claimed.length === 2, `实际: ${claimed.length}`);
+  const row = db
+    .prepare(
+      "SELECT claimed_by, claimed_at FROM jobs WHERE unique_id = 'sellerA'",
+    )
+    .get();
+  assert(row.claimed_by === "testerA", JSON.stringify(row));
+  assert(row.claimed_at !== null && row.claimed_at !== undefined);
+});
+check("锁定期间不重复领取（多客户端防重复）", () => {
+  const again = store.getPendingProductTasks(5, "testerB");
+  assert(
+    again.length === 0,
+    `实际领到: ${again.map((t) => t.uniqueId).join(",")}`,
+  );
+});
+check("TTL 过期后自动回收", () => {
+  db.prepare("UPDATE jobs SET claimed_at = ? WHERE unique_id = 'sellerF'").run(
+    Date.now() - 6 * 60 * 1000,
+  );
+  const recycled = store.getPendingProductTasks(5, "testerC");
+  assert(
+    recycled.some((t) => t.uniqueId === "sellerF"),
+    `实际: ${recycled.map((t) => t.uniqueId).join(",")}`,
+  );
+});
+check("name 空但 rating 有值 → 不进候选", () => {
+  assert(!tasks.some((t) => t.uniqueId === "sellerD"));
+});
+check("name 空但 soldCount 有值 → 不进候选", () => {
+  assert(!tasks.some((t) => t.uniqueId === "sellerE"));
 });
 
 console.log("\n[2] updateProductInfo：回写商品详情");

package/src/cli/product-fill.js

@@ -31,16 +31,15 @@ async function fetchProductDetail(webserverUrl, seoUrl) {
     body: JSON.stringify({ url: seoUrl }),
     signal: AbortSignal.timeout(30_000),
   });
-  if (res.status === 503) {
+  if (!res.ok) {
+    const body = await res.json().catch(() => ({}));
     const err = new Error(
-      `captcha: ${(await res.json().catch(() => ({}))).error || "Blocked"}`,
+      `${body.error || `HTTP ${res.status}`}${res.status === 503 ? " (captcha)" : ""}`,
     );
-    err.retryable = true;
-    throw err;
-  }
-  if (!res.ok) {
-    const err = new Error(`HTTP ${res.status}`);
     err.status = res.status;
+    // webserver 约定：503=验证码；504/502 响应体带 retryable=true 表示可重试
+    err.captcha = res.status === 503;
+    err.retryable = !!body.retryable || res.status === 503;
     throw err;
   }
   return res.json();
@@ -126,6 +125,17 @@ export async function handleProductFill(options) {
       loopCount++;
       if (shuttingDown) break;
 
+      // 验证码冷却期间不领任务（避免反复触发 webserver 限流）
+      const now = Date.now();
+      if (now < captchaBackoffUntil) {
+        const waitMs = captchaBackoffUntil - now;
+        log(
+          `[ProductFill] ⏳ 验证码冷却中（剩余 ${Math.ceil(waitMs / 1000)}s），暂不领取任务...`,
+        );
+        await new Promise((r) => setTimeout(r, waitMs));
+        continue;
+      }
+
       const { tasks = [] } = await apiGet(
         `${serverUrl}/api/product-tasks?limit=${productFillLimit}&userId=${encodeURIComponent(userId)}`,
       );
@@ -142,20 +152,16 @@ export async function handleProductFill(options) {
 
       log(`\n[ProductFill] 获取到 ${tasks.length} 个补商品任务...`);
 
-      // 验证码冷却期间整批跳过（避免反复触发 webserver 限流）
-      const now = Date.now();
-      if (now < captchaBackoffUntil) {
-        const waitSec = Math.ceil((captchaBackoffUntil - now) / 1000);
-        log(`  ⏳ 验证码冷却中（剩余 ${waitSec}s），本批跳过`);
-        await new Promise((r) => setTimeout(r, captchaBackoffUntil - now));
-        continue;
-      }
+      let batchSuccess = 0;
+      let batchFailed = 0;
+      let batchSkipped = 0;
 
       for (const task of tasks) {
         const uniqueId = task.uniqueId;
         const seoUrl = task.seoUrl;
         if (!seoUrl) {
           log(`  ✗ @${uniqueId} 无 seoUrl，跳过`);
+          batchSkipped++;
           totalSkipped++;
           continue;
         }
@@ -167,22 +173,31 @@ export async function handleProductFill(options) {
             { productDetail: detail },
           );
           if (post && post.ok) {
+            batchSuccess++;
             totalSuccess++;
             log(
               `  ✓ @${uniqueId} 商品已补全 (评分${detail.rating ?? "-"} | 销量${detail.soldCount ?? "-"} | 评论${detail.reviewCount ?? "-"})`,
             );
           } else {
+            batchFailed++;
             totalFailed++;
             log(`  ✗ @${uniqueId} 回传失败: ${post?.error || "未知"}`);
           }
         } catch (e) {
-          if (e.retryable) {
+          if (e.captcha) {
+            batchSkipped++;
             totalSkipped++;
             captchaBackoffUntil = Date.now() + CAPTCHA_BACKOFF_MS;
             log(
               `  ⏳ @${uniqueId} 验证码拦截，冷却 ${CAPTCHA_BACKOFF_MS / 1000} 秒`,
             );
+          } else if (e.retryable) {
+            // 504 超时等可重试错误：未回写，下轮自然重试，不触发全局冷却
+            batchSkipped++;
+            totalSkipped++;
+            log(`  ⏳ @${uniqueId} 临时失败（可重试）: ${e.message}`);
           } else {
+            batchFailed++;
             totalFailed++;
             log(`  ✗ @${uniqueId} 商品详情失败: ${e.message || "未知错误"}`);
           }
@@ -190,7 +205,7 @@ export async function handleProductFill(options) {
       }
 
       log(
-        `  本批结果: 成功 ${totalSuccess} | 失败 ${totalFailed} | 跳过 ${totalSkipped}\n`,
+        `  本批: 成功 ${batchSuccess} | 失败 ${batchFailed} | 跳过 ${batchSkipped}（累计: 成功 ${totalSuccess} | 失败 ${totalFailed} | 跳过 ${totalSkipped}）\n`,
       );
       await new Promise((r) => setTimeout(r, 300));
     }

package/src/watch/data-store.js

@@ -2029,7 +2029,14 @@ export function createStore(filePath, options = {}) {
       const existing = getJob(uniqueId);
       if (existing) {
         updateUserFromResult(existing, result);
-        const persistRet = updateJobInfo(uniqueId, existing, false);
+        // 用户在哪个表就更新哪个表（task_queue 热表 / jobs 归档），
+        // 避免固定写 jobs 导致的假成功（UPDATE 0 行仍返回 ok）
+        const persistRet = updateJobInfo(
+          uniqueId,
+          existing,
+          false,
+          getUserTable(uniqueId) || "jobs",
+        );
         if (persistRet.error) {
           return { saved: false, error: persistRet.error };
         }
@@ -2049,6 +2056,13 @@ export function createStore(filePath, options = {}) {
       };
       updateUserFromResult(userObj, result);
       addJob(userObj);
+      // addJobToDb 的 INSERT 列不含商品列（top_video_* / product_*），
+      // 新用户行需显式补写 task_queue，否则探索发现的商品线索丢失，
+      // 且与 product-fill 领取/回写的位置（jobs 归档表）断链
+      const persistRet = updateJobInfo(uniqueId, userObj, false, "task_queue");
+      if (persistRet.error) {
+        return { saved: false, error: persistRet.error };
+      }
       const newUsers = processDiscoveredUsers(result);
       return { saved: true, created: true, status: userObj.status, newUsers };
     }
@@ -2601,14 +2615,28 @@ export function createStore(filePath, options = {}) {
   // ===== product-fill：给缺商品详情的商家补商品信息（jobs 表） =====
 
   /**
-   * 领取缺商品详情任务：jobs 表中有 top_video_seo_url 但未补过商品信息（product_name 为空）
-   * 补全是幂等的（同一 seoUrl 补出相同详情），成功回写后自动移出候选，无需额外锁定
+   * 领取缺商品详情任务：jobs 表中有 top_video_seo_url 但未补过商品信息
+   * 完成判定为「名称/评分/销量三者都空」：商品页可能解析不出 name，
+   * 若只以 product_name 为空作候选条件，这类任务会永远留在候选池重复抓取
+   *
+   * 多客户端防重复领取（claim 机制）：
+   *  - 领取时写入 claimed_at/claimed_by，锁定期间不再返回同一任务
+   *  - 回写成功（product_* 有值）后自然移出候选；回写失败等 TTL 过期重新可领
+   *  - 客户端崩溃/断连 → claimed_at 超过 TTL 自动回收（与 claimNextJob 的 5 分钟一致）
    * @param {number|string} limit
+   * @param {string|null} claimBy 领取者标识（userId），仅作标记/日志
+   * @param {number} claimTtlMs 锁定超时（默认 5 分钟）
    * @returns {{uniqueId:string, nickname:string|null, seoUrl:string, productId:string|null, productTitle:string|null}[]}
    */
-  function getPendingProductTasks(limit) {
+  function getPendingProductTasks(
+    limit,
+    claimBy = null,
+    claimTtlMs = 5 * 60 * 1000,
+  ) {
     if (getDb()) {
       const l = Math.max(1, parseInt(limit) || 5);
+      const now = Date.now();
+      const claimExpireBefore = now - claimTtlMs;
       const rows = getDb()
         .prepare(
           `
@@ -2617,11 +2645,24 @@ export function createStore(filePath, options = {}) {
           FROM jobs
           WHERE top_video_seo_url IS NOT NULL AND top_video_seo_url != ''
             AND (product_name IS NULL OR product_name = '')
-          ORDER BY updated_at DESC
+            AND product_rating IS NULL
+            AND product_sold_count IS NULL
+            AND (claimed_at IS NULL OR claimed_at < ?)
+          ORDER BY COALESCE(claimed_at, 0) ASC, updated_at DESC
           LIMIT ?
         `,
         )
-        .all(l);
+        .all(claimExpireBefore, l);
+      if (rows.length > 0) {
+        // 领取即锁定：写入 claimed_at/claimed_by，避免多客户端重复抓取同一商品
+        const ids = rows.map((r) => r.unique_id);
+        const ph = ids.map(() => "?").join(", ");
+        getDb()
+          .prepare(
+            `UPDATE jobs SET claimed_at = ?, claimed_by = ? WHERE unique_id IN (${ph})`,
+          )
+          .run(now, claimBy || null, ...ids);
+      }
       return rows.map((row) => ({
         uniqueId: row.unique_id,
         nickname: row.nickname || null,

package/src/watch/db-columns.js

@@ -114,6 +114,7 @@ export const JOB_BOOLEAN_COLUMNS = new Set([
   "verified",
   "pending_refresh",
   "error",
+  "top_video_is_ad",
 ]);
 
 export const VIDEO_BOOLEAN_COLUMNS = new Set(["tt_seller"]);

package/src/watch/public/app.js

@@ -296,6 +296,7 @@ function renderActiveClients(clients) {
     "attach",
     "recheck",
     "comments",
+    "product-fill",
     "scoring",
     "tag-refresh",
     "tag-search",
@@ -307,6 +308,7 @@ function renderActiveClients(clients) {
     attach: "Attach",
     recheck: "Recheck",
     comments: "Comments",
+    "product-fill": "ProductFill",
     scoring: "Scoring",
     "tag-refresh": "TagRefresh",
     "tag-search": "Search",

package/src/watch/server.js

@@ -957,9 +957,15 @@ export function startWatchServer(
       // ── product-fill 端点：补商品信息（缺商品详情的商家） ──
       if (req.method === "GET" && routePath === "/api/product-tasks") {
         const limit = params.limit;
-        const tasks = store.getPendingProductTasks(limit);
+        const clientId = params.userId || "?";
+        const tasks = store.getPendingProductTasks(
+          limit,
+          params.userId || null,
+        );
         const ts = new Date().toLocaleTimeString("zh-CN", { hour12: false });
-        console.error(`[JOB ${ts}] PRODUCT-TASKS: ${tasks.length} tasks`);
+        console.error(
+          `[JOB ${ts}] PRODUCT-TASKS: ${tasks.length} tasks (client=${clientId})`,
+        );
         sendJSON(res, 200, { total: tasks.length, tasks });
         return;
       }