tt-help-cli-ycl 1.4.150 → 1.4.151

package/scripts/test-seller-any-country.mjs

@@ -0,0 +1,167 @@
+/**
+ * 商家任务国家放开领取测试（claimNextJob 分支顺序）
+ *
+ * 验证（临时 SQLite 库）：
+ *   1. 传了国家子集：先领对应国家的 ttseller，领完再领任意国家 ttseller
+ *   2. 未传国家：领商家（任意国家均可）
+ *   3. 其他来源级顺序不变：seed 仍排商家前，非商家任务仍按国家分层
+ *   4. 未登录客户端不领商家任务（保持原逻辑）
+ *
+ * 运行: TMPDIR=$(mktemp -d) node scripts/test-seller-any-country.mjs
+ */
+import { createStore } from "../src/watch/data-store.js";
+import { getDb } from "../src/watch/db-schema.js";
+import { join } from "node:path";
+import { mkdtempSync } from "node:fs";
+import { tmpdir } from "node:os";
+
+const tmp = mkdtempSync(join(tmpdir(), "seller-any-"));
+const store = createStore(join(tmp, "test.db"));
+const db = getDb();
+if (!db) {
+  console.error("❌ db 未初始化");
+  process.exit(1);
+}
+
+let passed = 0;
+function check(name, fn) {
+  try {
+    fn();
+    passed++;
+    console.log(`  ✅ ${name}`);
+  } catch (e) {
+    console.error(`  ❌ ${name}\n     ${e.message}`);
+    process.exitCode = 1;
+  }
+}
+function assert(cond, msg = "断言失败") {
+  if (!cond) throw new Error(msg);
+}
+
+const now = Date.now();
+// 直接插 task_queue（refill 后形态）
+const insert = (id, loc, seller, sources, opts = {}) =>
+  db
+    .prepare(
+      `INSERT INTO task_queue (unique_id, status, pinned, guessed_location, tt_seller, verified, sources, video_count, follower_count, created_at, updated_at)
+       VALUES (?, 'pending', 0, ?, ?, 0, ?, ?, ?, ?, ?)`,
+    )
+    .run(
+      id,
+      loc,
+      seller,
+      JSON.stringify(sources),
+      opts.videoCount ?? 3,
+      opts.followers ?? 100,
+      now,
+      now,
+    );
+
+console.log("===== 商家任务国家放开领取测试 =====");
+
+// ── 场景 1：传 US，队列里有 US 商家 + DE 商家 ──
+console.log("\n[1] 传国家子集 US：US 商家优先于 DE 商家");
+insert("seller_de", "DE", 1, ["following"]);
+insert("seller_us", "US", 1, ["following"]);
+{
+  const job = await store.claimNextJob("t1", 5 * 60 * 1000, ["US"], true);
+  check("先领到 US 商家", () =>
+    assert(job?.uniqueId === "seller_us", `实际: ${job?.uniqueId}`),
+  );
+}
+
+// ── 场景 2：US 商家已不在（被领走），传 US → 兜任意国家 DE 商家 ──
+console.log("\n[2] 无 US 商家可领时：兜到任意国家商家（DE）");
+{
+  // seller_us 已被 t1 领走（processing），队列只剩 DE 商家 pending
+  const job = await store.claimNextJob("t2", 5 * 60 * 1000, ["US"], true);
+  check("兜到 DE 商家", () =>
+    assert(job?.uniqueId === "seller_de", `实际: ${job?.uniqueId}`),
+  );
+}
+
+// ── 场景 3：传 US 但队列只有 FR 商家 → 领 FR ──
+console.log("\n[3] 传 US 但无 US 商家 → 领 FR 商家（任意国家兜底）");
+db.prepare(`DELETE FROM task_queue`).run();
+insert("seller_fr", "FR", 1, ["following"]);
+{
+  const job = await store.claimNextJob("t3", 5 * 60 * 1000, ["US"], true);
+  check("领到 FR 商家", () =>
+    assert(job?.uniqueId === "seller_fr", `实际: ${job?.uniqueId}`),
+  );
+}
+
+// ── 场景 4：空位置商家（guessed_location=''）→ 任意国家分支领到 ──
+console.log("\n[4] 空位置商家可被任意国家分支领到");
+db.prepare(`DELETE FROM task_queue`).run();
+insert("seller_noloc", "", 1, ["following"]);
+{
+  const job = await store.claimNextJob("t4", 5 * 60 * 1000, ["US"], true);
+  check("领到空位置商家", () =>
+    assert(job?.uniqueId === "seller_noloc", `实际: ${job?.uniqueId}`),
+  );
+}
+
+// ── 场景 5：未传国家，队列只有配置外国商家 → 直接领到 ──
+console.log("\n[5] 未传国家：配置外国商家（FR）也能领到");
+db.prepare(`DELETE FROM task_queue`).run();
+insert("seller_fr2", "FR", 1, ["following"]);
+{
+  const job = await store.claimNextJob("t5", 5 * 60 * 1000, null, true);
+  check("领到 FR 商家（未传国家不再被挡）", () =>
+    assert(job?.uniqueId === "seller_fr2", `实际: ${job?.uniqueId}`),
+  );
+}
+
+// ── 场景 6：seed 仍优先于商家（顺序不变）──
+console.log("\n[6] seed 优先级仍高于商家");
+db.prepare(`DELETE FROM task_queue`).run();
+insert("seed_pl", "PL", 0, ["seed"]);
+insert("seller_xx", "XX", 1, ["following"]);
+{
+  const job = await store.claimNextJob("t6", 5 * 60 * 1000, null, true);
+  check("先领 seed", () =>
+    assert(job?.uniqueId === "seed_pl", `实际: ${job?.uniqueId}`),
+  );
+}
+
+// ── 场景 7：非商家任务仍按国家分层（PL 配置内优先于 ZZ 配置外）──
+console.log("\n[7] 商家清空后：非商家按国家顺序领取");
+db.prepare(`DELETE FROM task_queue`).run();
+insert("other_pl", "PL", 0, ["search"]);
+insert("other_zz", "ZZ", 0, ["search"]);
+{
+  const job = await store.claimNextJob("t7", 5 * 60 * 1000, null, true);
+  check("PL 非商家先于 ZZ（配置内优先）", () =>
+    assert(job?.uniqueId === "other_pl", `实际: ${job?.uniqueId}`),
+  );
+}
+
+// ── 场景 8：未登录客户端不领商家（原逻辑不变）──
+console.log("\n[8] 未登录不领商家任务");
+db.prepare(`DELETE FROM task_queue`).run();
+insert("seller_only", "US", 1, ["following"]);
+insert("other_task", "PL", 0, ["search"]);
+{
+  const job = await store.claimNextJob("t8", 5 * 60 * 1000, null, false);
+  check("未登录跳过商家领普通任务", () =>
+    assert(job?.uniqueId === "other_task", `实际: ${job?.uniqueId}`),
+  );
+}
+
+// ── 场景 9：非商家任务不享受任意国家兜底（配置内先于配置外）──
+console.log("\n[9] 非商家配置外国家不越级");
+db.prepare(`DELETE FROM task_queue`).run();
+insert("plain_de", "DE", 0, ["search"]); // DE 在默认配置内
+insert("plain_zz", "ZZ", 0, ["search"]); // ZZ 不在配置
+{
+  const job = await store.claimNextJob("t9", 5 * 60 * 1000, null, true);
+  check("DE（配置内）先于 ZZ（配置外）", () =>
+    assert(job?.uniqueId === "plain_de", `实际: ${job?.uniqueId}`),
+  );
+}
+
+store.stopBackup?.();
+console.log(
+  `\n结果: ${passed} 通过${process.exitCode ? ", 有失败" : ", 0 失败"}`,
+);

package/src/cli/product-fill.js

@@ -540,9 +540,21 @@ export async function handleProductFill(options) {
             batchSuccess++;
             totalSuccess++;
             failCount.delete(uniqueId);
+            const skus = detail.skus;
+            const skuInfo = skus
+              ? ` | SKU ${skus.length}个 总库存 ${skus.reduce((sum, s) => sum + (s.stock || 0), 0)}`
+              : "";
             log(
-              `  ✓ @${uniqueId} 商品已补全 (评分${detail.rating ?? "-"} | 销量${detail.soldCount ?? "-"} | 评论${detail.reviewCount ?? "-"})`,
+              `  ✓ @${uniqueId} 商品已补全 (评分${detail.rating ?? "-"} | 销量${detail.soldCount ?? "-"} | 评论${detail.reviewCount ?? "-"}${skuInfo})`,
             );
+            if (skus) {
+              // SKU 明细日志：规格 | 库存 | 状态（先观察数据质量，暂不入库）
+              for (const s of skus) {
+                log(
+                  `    · ${s.skuId} | ${s.specs || "(无规格名)"} | 库存${s.stock ?? "?"} | 状态${s.stockStatus ?? "?"}`,
+                );
+              }
+            }
           } else {
             batchFailed++;
             totalFailed++;

package/src/lib/api-interceptor.js

@@ -46,9 +46,17 @@ function extractProductInfo(item) {
 
 /**
  * 构建统一的视频对象
+ *
+ * 图文帖（photo post）与视频混在 item_list 响应里，用 imagePost 字段区分
+ * （实测 @zoecole.fr：图文 item 有 imagePost={cover,images,...}，
+ *   video.duration=0 且无 PlayAddrStruct/codecType；视频 duration>0）。
+ * - isPhoto 标记贯穿全链路：EC 筛选/入库照常保留图文，仅国家采样排除
+ * - href 按真实路径生成：图文 /photo/{id}，视频 /video/{id}
  */
 function buildVideoObject(item, username) {
-  const href = `https://www.tiktok.com/@${username}/video/${item.id}`;
+  // 双保险：优先官方 imagePost 字段，duration=0 兜底（防字段偶发缺失）
+  const isPhoto = item.imagePost !== undefined || item.video?.duration === 0;
+  const href = `https://www.tiktok.com/@${username}/${isPhoto ? "photo" : "video"}/${item.id}`;
   const productInfo = extractProductInfo(item);
   return {
     id: item.id,
@@ -57,6 +65,7 @@ function buildVideoObject(item, username) {
     playCount: item.stats?.playCount || 0,
     isECVideo: item.isECVideo ? 1 : 0,
     isAd: item.isAd ? 1 : 0,
+    isPhoto: isPhoto ? 1 : 0,
     productId: productInfo?.productId || null,
     productTitle: productInfo?.productTitle || null,
     seoUrl: productInfo?.seoUrl || null,

package/src/lib/shop-scraper.mjs

@@ -97,6 +97,81 @@ function extractJsonByKey(html, keyPattern) {
   return null;
 }
 
+/**
+ * Extract a JSON array by its key name (counterpart of extractJsonByKey for `[`).
+ * 用于 PDP SSR HTML 中的 "skus":[...] 段。
+ */
+function extractJsonArrayByKey(html, keyPattern) {
+  const match = html.match(keyPattern);
+  if (!match) return null;
+
+  const bracketIdx = html.indexOf("[", match.index);
+  if (bracketIdx === -1) return null;
+
+  let depth = 0;
+  let inString = false;
+  let escaped = false;
+
+  for (let i = bracketIdx; i < html.length; i++) {
+    const ch = html[i];
+    if (escaped) {
+      escaped = false;
+      continue;
+    }
+    if (ch === "\\") {
+      escaped = true;
+      continue;
+    }
+    if (ch === '"') {
+      inString = !inString;
+      continue;
+    }
+    if (inString) continue;
+    if (ch === "[") depth++;
+    else if (ch === "]") {
+      depth--;
+      if (depth === 0) {
+        try {
+          return JSON.parse(html.substring(bracketIdx, i + 1));
+        } catch {
+          return null;
+        }
+      }
+    }
+  }
+  return null;
+}
+
+/**
+ * 从 PDP SSR HTML 的 "skus":[...] 提取多 SKU 概要（原有商品级字段不动）。
+ * sku_id 为 19 位数字，必须按字符串保留（同 product_id 的精度坑）。
+ * @returns {Array<{skuId, specs, stock, stockStatus, warehouseId}>|null}
+ *   无 skus 段或解析失败返回 null（与现有字段缺失口径一致）
+ */
+function extractSkus(html) {
+  const skus = extractJsonArrayByKey(html, /"skus"\s*:/);
+  if (!Array.isArray(skus) || skus.length === 0) return null;
+  const out = [];
+  for (const s of skus) {
+    try {
+      out.push({
+        skuId: String(s.sku_id ?? ""),
+        specs: (s.property_pairs || [])
+          .map((p) => `${p.sku_property_name}: ${p.sku_property_value_name}`)
+          .join(" | "),
+        stock: s.sku_quantity?.available_quantity ?? null,
+        stockStatus: s.sku_stock_status ?? null,
+        warehouseId: s.sku_quantity?.warehouse_id
+          ? String(s.sku_quantity.warehouse_id)
+          : null,
+      });
+    } catch {
+      // 单条损坏跳过，不影响其余 SKU
+    }
+  }
+  return out.length > 0 ? out : null;
+}
+
 /**
  * Open a headed browser for manual captcha solving, extract cookies
  * so undici can reuse them for a while.
@@ -278,6 +353,7 @@ export function parseProductHtml(html) {
     ratingBreakdown: null,
     productId: null,
     sellerId: null,
+    skus: null, // 多 SKU 概要（无 skus 段或解析失败为 null，不影响原有字段）
   };
 
   // sold_count from product_model (string value, not shop-level integer)
@@ -308,6 +384,9 @@ export function parseProductHtml(html) {
     result.ratingBreakdown = ratings.rating_result ?? null;
   }
 
+  // skus（多规格概要：规格名/库存/仓库，sku_id 字符串保精度）
+  result.skus = extractSkus(html);
+
   // 壳页识别：有 product_id 骨架但 seller_id 为空且无任何销售/名称数据
   // → 商品已下架/店铺异常（200 空壳页），抛 delisted 供调用方标记放弃
   const hasSaleData =

package/src/scraper/explore-core.js

@@ -181,18 +181,24 @@ async function processExplore(page, username, options, log) {
       return result;
     }
 
-    // 国家采样判断
+    // 国家采样判断（只用真视频：图文帖 /photo/ 链接查不到 locationCreated，
+    // 会浪费采样位并污染众数统计；图文 EC 帖仍在上方 EC 筛选中正常保留）
     // explore 模式：采样 5 个，优先命中目标国家，不匹配则回退众数 → 写 locationCreated
     // refresh 模式：采样 7 个，纯众数逻辑 → 写 confirmedLocation（二次确认）
     const SAMPLE_SIZE = locationMode === "refresh" ? 7 : 5;
     let locationCreated = null;
     let confirmedLocation = null;
     let locationDecision = null;
-    const sampleVideos = videoArray.slice(0, SAMPLE_SIZE);
+    const sampleVideos = videoArray
+      .filter((v) => !v.isPhoto)
+      .slice(0, SAMPLE_SIZE);
     if (sampleVideos.length > 0) {
       const sampleUrls = sampleVideos.map((v) =>
         v.href.startsWith("http") ? v.href : `https://www.tiktok.com${v.href}`,
       );
+      log(
+        `  国家采样(${sampleUrls.length}个, 已排除图文${videoArray.length - videoArray.filter((v) => !v.isPhoto).length}条): 开始查询`,
+      );
       let locations;
       if (webserverUrl) {
         // 走集中式 webserver，不再启动本地 TikTokScraper

package/src/watch/data-store.js

@@ -683,7 +683,13 @@ function getPreciseMerchantsFromDb(targetLocations = [], options = {}) {
       product_review_count,
       product_sold_count,
       product_name,
+      product_skus,
       recent_ec_videos,
+      (CASE WHEN json_valid(product_skus) THEN
+        (SELECT COUNT(*) FROM json_each(product_skus)) ELSE 0 END) AS skuCount,
+      (CASE WHEN json_valid(product_skus) THEN
+        (SELECT COALESCE(SUM(CAST(json_extract(value, '$.stock') AS INTEGER)), 0)
+          FROM json_each(product_skus)) ELSE 0 END) AS skuTotalStock,
       (SELECT COUNT(*) FROM json_each(recent_ec_videos)
         WHERE CAST(json_extract(value, '$.playCount') AS INTEGER) > 1000) AS ecOver1k,
       (SELECT MAX(CAST(json_extract(value, '$.playCount') AS INTEGER))
@@ -1388,7 +1394,11 @@ export function createStore(filePath, options = {}) {
           filter: `instr(COALESCE(sources, ''), '"seed"') > 0`,
           loggedInOnly: false,
         },
-        { filter: `tt_seller = 1 AND verified = 0`, loggedInOnly: true },
+        {
+          filter: `tt_seller = 1 AND verified = 0`,
+          loggedInOnly: true,
+          sellerAnyCountry: true,
+        },
         {
           filter: `(instr(COALESCE(sources, ''), '"following"') > 0 OR instr(COALESCE(sources, ''), '"follower"') > 0 OR instr(COALESCE(sources, ''), '"comment"') > 0)`,
           loggedInOnly: false,
@@ -1407,8 +1417,28 @@ export function createStore(filePath, options = {}) {
         );
         if (normalizedLocations.length === 0)
           locationBranches.push(locNotIn(orderedCountries));
+        // 商家级专用（sellerAnyCountry）：国家对商家放开——末尾追加"任意国家"分支。
+        // 传了国家子集时对应国家的商家仍优先，领完再兜任意国家（含空位置商家）；
+        // 未传国家时同理（配置内各国 → 配置外 → 任意）。其他来源级分支不受影响。
+        if (p.sellerAnyCountry) locationBranches.push(null); // null → 任意国家分支
         let subPrio = 0;
         for (const loc of locationBranches) {
+          // 商家任意国家分支：不限 guessed_location（含空位置的商家）
+          if (loc === null) {
+            // hasVideo
+            addBranch(
+              `status = 'pending' AND pinned = 0 AND ${p.filter} AND video_count > 0 AND ${statusCodeWhere} ${notSellerWhere}`,
+              [],
+            );
+            subPrio++;
+            // noVideoReq
+            addBranch(
+              `status = 'pending' AND pinned = 0 AND ${p.filter} AND ${statusCodeWhere} ${notSellerWhere}`,
+              [],
+            );
+            subPrio++;
+            continue;
+          }
           if (!loc) continue;
           // hasVideo
           addBranch(
@@ -2452,28 +2482,40 @@ export function createStore(filePath, options = {}) {
     const placeholders = targetLocations.map(() => "?").join(",");
 
     // SELECT 候选用户
-    const selected = getDb()
-      .prepare(
-        `
-        SELECT unique_id, nickname, video_count, COALESCE(refresh_time, ?) AS refresh_time
-        FROM jobs
+    // 性能三件套（76 万行实测 1~3s → ~80ms）：
+    // 1. WHERE/ORDER BY 不用 COALESCE(refresh_time, ?)（表达式挡索引）；
+    //    ASC 排序 NULL 天然最前，与 COALESCE 2016 默认值语义等价
+    // 2. INDEXED BY 强制走 idx_jobs_precheck 部分索引（优化器无统计时会
+    //    误选 idx_jobs_seller_verified_loc 走 TEMP B-TREE 全排序）
+    // 3. ORDER BY jobs.refresh_time 限定表名——裸列名会被 SELECT 输出别名
+    //    refresh_time（COALESCE 表达式）遮蔽，又退回表达式排序
+    // 索引不存在的环境（老库未重启）抛 "no such index"，自动回退普通查询。
+    const baseWhere = `
         WHERE tt_seller = 1
           AND verified = 0
           AND (pending_refresh IS NULL OR pending_refresh = 0)
           AND location_created IN (${placeholders})
-          AND COALESCE(refresh_time, ?) < ?
-        ORDER BY COALESCE(refresh_time, ?) ASC
-        LIMIT ?
-      `,
-      )
-      .all(
-        defaultTime,
-        ...targetLocations,
-        defaultTime,
-        threshold,
-        defaultTime,
-        limit,
-      );
+          AND (refresh_time IS NULL OR refresh_time < ?)`;
+    const orderLimit = `
+        ORDER BY jobs.refresh_time ASC
+        LIMIT ?`;
+    const selectSql = `SELECT unique_id, nickname, video_count, COALESCE(refresh_time, ?) AS refresh_time
+        FROM jobs`;
+    const args = [defaultTime, ...targetLocations, threshold, limit];
+
+    let selected;
+    try {
+      selected = getDb()
+        .prepare(
+          `${selectSql} INDEXED BY idx_jobs_precheck${baseWhere}${orderLimit}`,
+        )
+        .all(...args);
+    } catch (e) {
+      if (!/no such index/i.test(e.message)) throw e;
+      selected = getDb()
+        .prepare(`${selectSql}${baseWhere}${orderLimit}`)
+        .all(...args);
+    }
 
     // 立即更新 refresh_time 防重复领取（非事务，小概率并发重复可接受）
     if (selected.length > 0) {
@@ -2799,8 +2841,9 @@ export function createStore(filePath, options = {}) {
   /**
    * 回写商品详情到 jobs 表（product_* 列）
    * @param {string} uniqueId
-   * @param {{rating?:number, reviewCount?:number, soldCount?:number, productName?:string}} productDetail
+   * @param {{rating?:number, reviewCount?:number, soldCount?:number, productName?:string, skus?:Array}} productDetail
    *   rating = -1 为哨兵值：商品不可售/已下架，写入后任务移出候选（正常评分 0-5）
+   *   skus = 多 SKU 概要数组（skuId/specs/stock/stockStatus/warehouseId），存 product_skus JSON
    */
   function updateProductInfo(uniqueId, productDetail) {
     if (getDb()) {
@@ -2814,6 +2857,10 @@ export function createStore(filePath, options = {}) {
           productReviewCount: productDetail.reviewCount ?? null,
           productSoldCount: productDetail.soldCount ?? null,
           productName: productDetail.productName || null,
+          // 多 SKU 概要：数组经 normalizeJobValue 自动序列化为 JSON（无 skus 存 null）
+          productSkus: Array.isArray(productDetail.skus)
+            ? productDetail.skus
+            : null,
         },
         false, // 商品补全与用户刷新计数无关，不增加 user_update_count
         "jobs",

package/src/watch/db-columns.js

@@ -99,6 +99,7 @@ export const WRITABLE_JOB_COLUMNS = new Set([
   "product_review_count",
   "product_sold_count",
   "product_name",
+  "product_skus",
   "user_create_time",
   "recent_ec_videos",
   "pending_refresh",

package/src/watch/db-schema.js

@@ -173,6 +173,7 @@ export function initDb(filePath) {
     product_review_count: "INTEGER",
     product_sold_count: "INTEGER",
     product_name: "TEXT",
+    product_skus: "TEXT",
     user_create_time: "INTEGER",
     recent_ec_videos: "TEXT",
   });
@@ -209,6 +210,7 @@ export function initDb(filePath) {
     product_review_count: "INTEGER",
     product_sold_count: "INTEGER",
     product_name: "TEXT",
+    product_skus: "TEXT",
     user_create_time: "INTEGER",
     recent_ec_videos: "TEXT",
   });
@@ -246,6 +248,7 @@ export function initDb(filePath) {
     product_review_count: "INTEGER",
     product_sold_count: "INTEGER",
     product_name: "TEXT",
+    product_skus: "TEXT",
     user_create_time: "INTEGER",
     recent_ec_videos: "TEXT",
   });

package/src/watch/db-tags.js

@@ -375,15 +375,25 @@ export function getAllTags(
 }
 
 export function rawQuery(sql, params = []) {
-  // 通用查询器：默认查主库（stats/db-query 复用），勿切到 tag 库
+  // 通用查询器：默认查主库（stats/db-query 复用），勿切到 tag 库。
+  // 注意：原实现用 prepare().all() 跑一切语句，DDL（CREATE INDEX 等）会抛
+  // "This statement does not return data" 且被 catch 吞成 {error} 返回——
+  // server.js 启动清单 ensureIdx 曾因此静默失效（生产库从未建过启动索引）。
+  // 现按语句类型分流：DDL/写语句走 exec，查询走 all；失败时抛错让调用方感知。
   const db = getMainDb();
-  if (!db) return { error: "db not ready" };
-  try {
+  if (!db) throw new Error("db not ready");
+  const trimmed = sql.trim().toUpperCase();
+  const isQuery =
+    trimmed.startsWith("SELECT") ||
+    trimmed.startsWith("PRAGMA") ||
+    trimmed.startsWith("EXPLAIN") ||
+    trimmed.startsWith("WITH");
+  if (isQuery) {
     const rows = db.prepare(sql).all(...params);
     return { rows };
-  } catch (e) {
-    return { error: e.message };
   }
+  db.exec(sql);
+  return { rows: [] };
 }
 
 export function normalizeTags() {