tt-help-cli-ycl 1.4.144 → 1.4.146

package/src/lib/args.js

@@ -894,15 +894,25 @@ function parseProductFillArgs(args) {
   let limit = 5;
   let interval = 10;
   let serverUrl = defaultServer;
-  let webserverUrl = null;
   let proxyUrl = null;
+  /** 端口/国家配置：--profiles US/9223,JP/1222 → [{country:"US",port:9223},...] */
+  let profiles = [];
 
   for (let i = 0; i < args.length; i++) {
     const arg = args[i];
     if (arg === "-s" || arg === "--server") {
       serverUrl = args[++i];
-    } else if (arg === "-w" || arg === "--webserver") {
-      webserverUrl = args[++i];
+    } else if (arg === "--profiles") {
+      for (const seg of String(args[++i] || "").split(",")) {
+        const [country, portStr] = seg.trim().split("/");
+        const port = parseInt(portStr, 10);
+        const cc = String(country || "")
+          .trim()
+          .toUpperCase();
+        if (port && /^[A-Z]{2}$/.test(cc)) {
+          profiles.push({ country: cc, port });
+        }
+      }
     } else if (arg === "--proxy") {
       proxyUrl = args[++i] || null;
     } else if (arg === "-i" || arg === "--interval") {
@@ -917,8 +927,8 @@ function parseProductFillArgs(args) {
     productFillLimit: limit,
     productFillInterval: interval,
     serverUrl,
-    productFillWebserver: webserverUrl,
     productFillProxy: proxyUrl,
+    productFillProfiles: profiles,
     urls: [],
     outputFormat: "json",
     exploreCount: 0,
@@ -1564,7 +1574,7 @@ export function parseArgs() {
         productFillLimit: 5,
         productFillInterval: 10,
         serverUrl: defaultServer,
-        productFillWebserver: null,
+        productFillProfiles: [],
         showHelp: true,
         showVersion: false,
         urls: [],

package/src/lib/constants.js

@@ -222,14 +222,17 @@ const HELP_TEXT = [
   "    示例: tt-help attach -p 5 -i 10",
   "          tt-help attach -c PL,DE,FR -p 5",
   "",
-  "  tt-help product-fill [-s 服务端地址] [-w webserver地址] [-i 间隔秒数] [-l 每批上限]",
-  "    循环领取缺商品详情任务，通过 webserver 抓取评分/销量/评论并回写 jobs 表",
+  "  tt-help product-fill --profiles <XX/port>[,<XX/port>...] [-s 服务端地址] [-i 间隔秒数] [-l 每批上限] [--proxy URL]",
+  "    循环领取缺商品详情任务，抓取评分/销量/评论并回写 jobs 表",
+  "    --profiles 国家/端口对绑定（端口 profile 区域需与商品国家一致，如 US/9223,JP/1222）；",
+  "    每轮探测代理出口，在配置中匹配端口领取对应国家任务，出口不在配置范围则跳过",
   "    -s, --server <URL>     服务端地址（默认: http://127.0.0.1:3001）",
-  "    -w, --webserver <URL>  商品详情 webserver 地址（必填）",
+  "    --proxy <URL>          浏览器启动代理及出口探测（默认: http://127.0.0.1:7897）",
   "    -i, --interval <N>     无任务时轮询间隔，单位秒（默认: 10）",
   "    -l, --limit <N>        每批领取任务数上限（默认: 5）",
-  "    示例: tt-help product-fill -w http://127.0.0.1:3000",
-  "          tt-help product-fill -s http://127.0.0.1:3001 -w http://117.71.53.99:17300 -i 5 -l 10",
+  "    下架壳页/连续失败3次/无效URL → 回写 rating=-1 哨兵永久移出候选",
+  "    示例: tt-help product-fill --profiles US/9223",
+  "          tt-help product-fill --profiles US/9223,JP/1222 -s http://117.71.53.99:17301 -i 5",
   "",
   "  tt-help recheck [-p 并行数] [-i 间隔秒数] [-s 服务端地址] [-c 国家列表]",
   "    重检 jobs 表中 done 非商家用户（tt_seller!=1），7 天未更新才检测",
@@ -360,15 +363,16 @@ const HELP_TEXT = [
   "    选项:",
   "      --countries <CSV>         目标国家，逗号分隔（默认 13 个欧洲国家）",
   "      -s, --server <URL>        服务端地址（默认 http://127.0.0.1:3000）",
-  "      --port <N>                起始 CDP 端口，端口池默认为 N~N+9（默认 7222）",
+  "      --port <N>                固定 CDP 端口（默认 7222，全程不切换）",
   "      --discover                开启自动发现：剩余 new 标签 < 5 时后台异步生成，连续 3 轮空兜底",
   "      --discover-count <N>      每次发现每个国家生成标签数（默认 10，上限 20）",
   "    行为:",
-  "      连续 3 次 HTTP 403 → 判断账号被封 → 等 2 分钟后自动切换到池中另一端口",
+  "      错误只打印不自愈：403 连续 3 次 → 上报服务端并触发 daemon 换节点；",
+  "      网络错误/其他错误连续 11 次 → 退出；验证码连续 4 次未人工解决 → 退出",
+  "      成功一个 tag 全部计数清零；浏览器窗口被手动关闭 → 自动退出",
   "    环境变量:",
-  "      TAG_SCOREALL_PORT_POOL_SIZE=N  端口池大小（默认 10）",
-  "      TAG_SCOREALL_RECYCLE_RSS_MB=N  内存回收阈值 RSS MB（默认 2000）",
-  "      TAG_SCOREALL_RECYCLE_HEAP_MB=N 内存回收阈值 Heap MB（默认 2000）",
+  "      TAG_SCOREALL_MIN_COUNTRY_RATE=0.8  国家识别率下限（低于则重试）",
+  "      TAG_SCOREALL_COUNTRY_MAX_RETRIES=2 国家识别率不足最大重试次数",
   "    示例: tt-help tag score-all",
   "          tt-help tag score-all --countries ES -s http://127.0.0.1:3001",
   "          tt-help tag score-all --port 9222 -s http://127.0.0.1:3001 --discover",

package/src/lib/shop-scraper.mjs

@@ -190,6 +190,8 @@ export async function getProductInfo(url, proxyUrl, options = {}) {
   const agent = proxy ? new ProxyAgent(proxy) : null;
 
   let lastError;
+  let captchaRetries = 0; // 验证码弹浏览器重试次数（防无限重试）
+  const MAX_CAPTCHA_RETRIES = 2;
   for (let attempt = 1; attempt <= 3; attempt++) {
     try {
       const headers = { ...PDP_HEADERS };
@@ -216,8 +218,15 @@ export async function getProductInfo(url, proxyUrl, options = {}) {
         err.message?.includes("Blocked by TikTok") &&
         options.captchaFallback
       ) {
+        if (captchaRetries >= MAX_CAPTCHA_RETRIES) {
+          console.error(
+            `[shop-scraper] Captcha persists after ${captchaRetries} browser attempts, giving up`,
+          );
+          throw err;
+        }
+        captchaRetries++;
         console.error(
-          `[shop-scraper] Captcha on ${proxy}, clearing cached cookies...`,
+          `[shop-scraper] Captcha on ${proxy}, clearing cached cookies (browser attempt ${captchaRetries}/${MAX_CAPTCHA_RETRIES})...`,
         );
         clearCachedCookies(proxy);
         console.error("[shop-scraper] Opening browser for manual captcha...");
@@ -255,6 +264,12 @@ export function parseProductHtml(html) {
     throw new Error("Blocked by TikTok security check (captcha)");
   }
 
+  // 页面不含任何商品数据标识 → 空壳页（风控降级/跳转/空白），
+  // 抛可重试错误，避免把全 null 数据当成功回写
+  if (!html.includes('"product_model"') && !html.includes('"review_ratings"')) {
+    throw new Error("Empty product page (no product data markers)");
+  }
+
   const result = {
     rating: null,
     reviewCount: null,
@@ -293,6 +308,19 @@ export function parseProductHtml(html) {
     result.ratingBreakdown = ratings.rating_result ?? null;
   }
 
+  // 壳页识别：有 product_id 骨架但 seller_id 为空且无任何销售/名称数据
+  // → 商品已下架/店铺异常（200 空壳页），抛 delisted 供调用方标记放弃
+  const hasSaleData =
+    result.soldCount !== null ||
+    result.rating !== null ||
+    result.reviewCount !== null ||
+    result.productName !== null;
+  if (result.productId && !result.sellerId && !hasSaleData) {
+    const err = new Error("Product delisted (seller_id empty, no sale data)");
+    err.delisted = true;
+    throw err;
+  }
+
   return result;
 }
 
@@ -303,11 +331,29 @@ export function parseProductHtml(html) {
  * @param {string} url - TikTok Shop PDP URL
  * @returns {Promise<{rating, reviewCount, soldCount, productName, ratingBreakdown, productId, sellerId}>}
  */
-export async function getProductInfoCDP(page, url) {
-  await page.goto(url, { waitUntil: "networkidle", timeout: 60000 });
-  // 等待 JS 渲染完成（CSR 页面需要）
-  await new Promise((r) => setTimeout(r, 3000));
-  const html = await page.content();
+export async function getProductInfoCDP(page, url, options = {}) {
+  const { gotoTimeout = 60000, renderWaitMs = 15000 } = options;
+  await page.goto(url, {
+    waitUntil: "domcontentloaded",
+    timeout: gotoTimeout,
+  });
+  // 轮询等待 CSR 渲染出商品数据标识（或出现验证码标记），
+  // 不用 networkidle：TikTok 页面持续请求，networkidle 极易超时
+  const deadline = Date.now() + renderWaitMs;
+  let html = "";
+  while (Date.now() < deadline) {
+    html = await page.content();
+    if (
+      html.includes('"product_model"') ||
+      html.includes('"review_ratings"') ||
+      html.includes("captcha-verify-container") ||
+      html.includes("<title>Security Check</title>")
+    ) {
+      break;
+    }
+    await new Promise((r) => setTimeout(r, 1000));
+  }
+  if (!html) html = await page.content();
   return parseProductHtml(html);
 }
 
@@ -320,3 +366,20 @@ export function formatCount(n) {
   if (n >= 1e3) return (n / 1e3).toFixed(1).replace(/\.0$/, "") + "K";
   return String(n);
 }
+
+/**
+ * 判断商品详情是否全空（页面被拦截/结构变化时解析结果全为 null）。
+ * 只判实际回写的四个字段：rating/soldCount/reviewCount/productName。
+ * productId/sellerId 不回写，有值也不能视为"抓到了数据"，
+ * 否则壳页（仅含 product_id 骨架）会被误报成功、任务永久留在候选池。
+ * @param {object} detail - parseProductHtml 的返回值
+ * @returns {boolean}
+ */
+export function isEmptyProductDetail(detail) {
+  return (
+    (detail?.rating ?? null) === null &&
+    (detail?.reviewCount ?? null) === null &&
+    (detail?.soldCount ?? null) === null &&
+    !detail?.productName
+  );
+}

package/src/watch/data-store.js

@@ -608,6 +608,92 @@ function getTargetUsersByCountryFromDb(targetLocations = [], options = {}) {
   };
 }
 
+/**
+ * 精确商家：目标商家（tt_seller=1 且 verified=0 且 目标国家）中，
+ * 最近 7 天有 isECVideo 播放 > 1000 的视频，且有商品、商品评论数=0、销量 > 10。
+ * 支持分页与搜索（unique_id/nickname），返回 total + users。
+ */
+function getPreciseMerchantsFromDb(targetLocations = [], options = {}) {
+  if (!getDb()) return null;
+  if (!targetLocations.length) return { total: 0, users: [] };
+
+  const { limit, offset, search } = options;
+  const placeholders = targetLocations.map(() => "?").join(", ");
+
+  const where = `
+    tt_seller = 1
+    AND verified = 0
+    AND location_created IN (${placeholders})
+    AND top_video_seo_url IS NOT NULL AND top_video_seo_url != ''
+    AND product_review_count = 0
+    AND product_sold_count > 10
+    AND recent_ec_videos IS NOT NULL AND recent_ec_videos != ''
+    AND EXISTS (
+      SELECT 1 FROM json_each(recent_ec_videos)
+      WHERE CAST(json_extract(value, '$.playCount') AS INTEGER) > 1000
+    )
+  `;
+
+  let searchSql = "";
+  const searchParams = [];
+  if (search) {
+    searchSql = ` AND (unique_id LIKE ? OR nickname LIKE ?)`;
+    searchParams.push(`%${search}%`, `%${search}%`);
+  }
+
+  const totalRow = getDb()
+    .prepare(`SELECT COUNT(*) AS n FROM jobs WHERE ${where}${searchSql}`)
+    .get(...targetLocations, ...searchParams);
+  const total = totalRow?.n || 0;
+
+  const selectSql = `
+    SELECT
+      unique_id,
+      nickname,
+      sec_uid,
+      status,
+      follower_count,
+      video_count,
+      location_created,
+      latest_video_time,
+      refresh_time,
+      top_video_play_count,
+      top_video_href,
+      top_video_is_ad,
+      top_video_product_id,
+      top_video_product_title,
+      top_video_seo_url,
+      product_rating,
+      product_review_count,
+      product_sold_count,
+      product_name,
+      recent_ec_videos,
+      (SELECT COUNT(*) FROM json_each(recent_ec_videos)
+        WHERE CAST(json_extract(value, '$.playCount') AS INTEGER) > 1000) AS ecOver1k,
+      (SELECT MAX(CAST(json_extract(value, '$.playCount') AS INTEGER))
+        FROM json_each(recent_ec_videos)) AS maxEcPlay
+    FROM jobs
+    WHERE ${where}${searchSql}
+    ORDER BY product_sold_count DESC, unique_id ASC
+  `;
+
+  if (limit !== undefined) {
+    const safeLimit = Math.max(Math.min(Math.floor(limit), 1000), 1);
+    const safeOffset = Math.max(Math.floor(offset || 0), 0);
+    const rows = getDb()
+      .prepare(`${selectSql} LIMIT ? OFFSET ?`)
+      .all(...targetLocations, ...searchParams, safeLimit, safeOffset)
+      .map(mapJobRow);
+    return { total, limit: safeLimit, offset: safeOffset, users: rows };
+  }
+
+  const rows = getDb()
+    .prepare(selectSql)
+    .all(...targetLocations, ...searchParams)
+    .map(mapJobRow);
+  return { total, users: rows };
+}
+
 /**
  * 查询已修改（modified_at IS NOT NULL）且 location_created 不在 targetLocations 中的目标商家，
  * 按国家分组返回。
@@ -2686,20 +2772,27 @@ export function createStore(filePath, options = {}) {
    * @param {number|string} limit
    * @param {string|null} claimBy 领取者标识（userId），仅作标记/日志
    * @param {number} claimTtlMs 锁定超时（默认 5 分钟）
+   * @param {string|null} country 国家过滤（大写两位，如 US/JP）；非空时只返回
+   *   location_created 匹配的任务，location_created 为 NULL 的任务不分发；
+   *   传 null/空串表示不限（兼容旧客户端）
    * @returns {{uniqueId:string, nickname:string|null, seoUrl:string, productId:string|null, productTitle:string|null}[]}
    */
   function getPendingProductTasks(
     limit,
     claimBy = null,
     claimTtlMs = 5 * 60 * 1000,
+    country = null,
   ) {
     if (getDb()) {
       const l = Math.max(1, parseInt(limit) || 5);
       const now = Date.now();
       const claimExpireBefore = now - claimTtlMs;
-      const rows = getDb()
-        .prepare(
-          `
+      // 国家过滤：只认大写两位字母，避免异常值影响；无国家段的任务
+      // 不分发给任何按国家领取的客户端（后续人工回填处理）
+      const countryFilter = country
+        ? String(country).trim().toUpperCase().slice(0, 2)
+        : "";
+      let sql = `
           SELECT unique_id, nickname, top_video_seo_url, top_video_product_id,
                  top_video_product_title
           FROM jobs
@@ -2707,12 +2800,21 @@ export function createStore(filePath, options = {}) {
             AND (product_name IS NULL OR product_name = '')
             AND product_rating IS NULL
             AND product_sold_count IS NULL
+      `;
+      const params = [];
+      if (countryFilter.length === 2) {
+        sql += ` AND location_created = ?`;
+        params.push(countryFilter);
+      }
+      sql += `
             AND (claimed_at IS NULL OR claimed_at < ?)
           ORDER BY COALESCE(claimed_at, 0) ASC, updated_at DESC
           LIMIT ?
-        `,
-        )
-        .all(claimExpireBefore, l);
+        `;
+      params.push(claimExpireBefore, l);
+      const rows = getDb()
+        .prepare(sql)
+        .all(...params);
       if (rows.length > 0) {
         // 领取即锁定：写入 claimed_at/claimed_by，避免多客户端重复抓取同一商品
         const ids = rows.map((r) => r.unique_id);
@@ -2739,6 +2841,7 @@ export function createStore(filePath, options = {}) {
    * 回写商品详情到 jobs 表（product_* 列）
    * @param {string} uniqueId
    * @param {{rating?:number, reviewCount?:number, soldCount?:number, productName?:string}} productDetail
+   *   rating = -1 为哨兵值：商品不可售/已下架，写入后任务移出候选（正常评分 0-5）
    */
   function updateProductInfo(uniqueId, productDetail) {
     if (getDb()) {
@@ -2757,6 +2860,24 @@ export function createStore(filePath, options = {}) {
         "jobs",
       );
       if (ret.error) return ret;
+      // 校验：写入后 product_* 仍全空 → 回写无效（壳页假成功防御），
+      // 防止任务在候选池里无限轮转
+      const row = getDb()
+        .prepare(
+          `SELECT product_name, product_rating, product_sold_count,
+                  product_review_count
+           FROM jobs WHERE unique_id = ?`,
+        )
+        .get(uniqueId);
+      if (
+        row &&
+        (row.product_name === null || row.product_name === "") &&
+        row.product_rating === null &&
+        row.product_sold_count === null &&
+        row.product_review_count === null
+      ) {
+        return { ok: false, error: "product 字段仍全空，回写无效" };
+      }
       return { ok: true };
     }
     return { ok: false, error: "db not initialized" };
@@ -3549,6 +3670,8 @@ export function createStore(filePath, options = {}) {
     getRawJobsPage: getRawJobsPageFromDb,
     getTargetUsers: getTargetUsersFromDb,
     getTargetUsersByCountry: getTargetUsersByCountryFromDb,
+    getPreciseMerchants: (targetLocations, options) =>
+      getPreciseMerchantsFromDb(targetLocations, options),
     getModifiedTargetUsersOutsideLocations: (loc) =>
       getModifiedTargetUsersOutsideLocations(loc),
     getStats,