tt-help-cli-ycl 1.4.145 → 1.4.146

package/src/lib/args.js

@@ -894,15 +894,25 @@ function parseProductFillArgs(args) {
   let limit = 5;
   let interval = 10;
   let serverUrl = defaultServer;
-  let webserverUrl = null;
   let proxyUrl = null;
+  /** 端口/国家配置：--profiles US/9223,JP/1222 → [{country:"US",port:9223},...] */
+  let profiles = [];
 
   for (let i = 0; i < args.length; i++) {
     const arg = args[i];
     if (arg === "-s" || arg === "--server") {
       serverUrl = args[++i];
-    } else if (arg === "-w" || arg === "--webserver") {
-      webserverUrl = args[++i];
+    } else if (arg === "--profiles") {
+      for (const seg of String(args[++i] || "").split(",")) {
+        const [country, portStr] = seg.trim().split("/");
+        const port = parseInt(portStr, 10);
+        const cc = String(country || "")
+          .trim()
+          .toUpperCase();
+        if (port && /^[A-Z]{2}$/.test(cc)) {
+          profiles.push({ country: cc, port });
+        }
+      }
     } else if (arg === "--proxy") {
       proxyUrl = args[++i] || null;
     } else if (arg === "-i" || arg === "--interval") {
@@ -917,8 +927,8 @@ function parseProductFillArgs(args) {
     productFillLimit: limit,
     productFillInterval: interval,
     serverUrl,
-    productFillWebserver: webserverUrl,
     productFillProxy: proxyUrl,
+    productFillProfiles: profiles,
     urls: [],
     outputFormat: "json",
     exploreCount: 0,
@@ -1564,7 +1574,7 @@ export function parseArgs() {
         productFillLimit: 5,
         productFillInterval: 10,
         serverUrl: defaultServer,
-        productFillWebserver: null,
+        productFillProfiles: [],
         showHelp: true,
         showVersion: false,
         urls: [],

package/src/lib/constants.js

@@ -222,14 +222,17 @@ const HELP_TEXT = [
   "    示例: tt-help attach -p 5 -i 10",
   "          tt-help attach -c PL,DE,FR -p 5",
   "",
-  "  tt-help product-fill [-s 服务端地址] [-w webserver地址] [-i 间隔秒数] [-l 每批上限]",
-  "    循环领取缺商品详情任务，通过 webserver 抓取评分/销量/评论并回写 jobs 表",
+  "  tt-help product-fill --profiles <XX/port>[,<XX/port>...] [-s 服务端地址] [-i 间隔秒数] [-l 每批上限] [--proxy URL]",
+  "    循环领取缺商品详情任务，抓取评分/销量/评论并回写 jobs 表",
+  "    --profiles 国家/端口对绑定（端口 profile 区域需与商品国家一致，如 US/9223,JP/1222）；",
+  "    每轮探测代理出口，在配置中匹配端口领取对应国家任务，出口不在配置范围则跳过",
   "    -s, --server <URL>     服务端地址（默认: http://127.0.0.1:3001）",
-  "    -w, --webserver <URL>  商品详情 webserver 地址（必填）",
+  "    --proxy <URL>          浏览器启动代理及出口探测（默认: http://127.0.0.1:7897）",
   "    -i, --interval <N>     无任务时轮询间隔，单位秒（默认: 10）",
   "    -l, --limit <N>        每批领取任务数上限（默认: 5）",
-  "    示例: tt-help product-fill -w http://127.0.0.1:3000",
-  "          tt-help product-fill -s http://127.0.0.1:3001 -w http://117.71.53.99:17300 -i 5 -l 10",
+  "    下架壳页/连续失败3次/无效URL → 回写 rating=-1 哨兵永久移出候选",
+  "    示例: tt-help product-fill --profiles US/9223",
+  "          tt-help product-fill --profiles US/9223,JP/1222 -s http://117.71.53.99:17301 -i 5",
   "",
   "  tt-help recheck [-p 并行数] [-i 间隔秒数] [-s 服务端地址] [-c 国家列表]",
   "    重检 jobs 表中 done 非商家用户（tt_seller!=1），7 天未更新才检测",

package/src/lib/shop-scraper.mjs

@@ -190,6 +190,8 @@ export async function getProductInfo(url, proxyUrl, options = {}) {
   const agent = proxy ? new ProxyAgent(proxy) : null;
 
   let lastError;
+  let captchaRetries = 0; // 验证码弹浏览器重试次数（防无限重试）
+  const MAX_CAPTCHA_RETRIES = 2;
   for (let attempt = 1; attempt <= 3; attempt++) {
     try {
       const headers = { ...PDP_HEADERS };
@@ -216,8 +218,15 @@ export async function getProductInfo(url, proxyUrl, options = {}) {
         err.message?.includes("Blocked by TikTok") &&
         options.captchaFallback
       ) {
+        if (captchaRetries >= MAX_CAPTCHA_RETRIES) {
+          console.error(
+            `[shop-scraper] Captcha persists after ${captchaRetries} browser attempts, giving up`,
+          );
+          throw err;
+        }
+        captchaRetries++;
         console.error(
-          `[shop-scraper] Captcha on ${proxy}, clearing cached cookies...`,
+          `[shop-scraper] Captcha on ${proxy}, clearing cached cookies (browser attempt ${captchaRetries}/${MAX_CAPTCHA_RETRIES})...`,
         );
         clearCachedCookies(proxy);
         console.error("[shop-scraper] Opening browser for manual captcha...");
@@ -255,6 +264,12 @@ export function parseProductHtml(html) {
     throw new Error("Blocked by TikTok security check (captcha)");
   }
 
+  // 页面不含任何商品数据标识 → 空壳页（风控降级/跳转/空白），
+  // 抛可重试错误，避免把全 null 数据当成功回写
+  if (!html.includes('"product_model"') && !html.includes('"review_ratings"')) {
+    throw new Error("Empty product page (no product data markers)");
+  }
+
   const result = {
     rating: null,
     reviewCount: null,
@@ -293,6 +308,19 @@ export function parseProductHtml(html) {
     result.ratingBreakdown = ratings.rating_result ?? null;
   }
 
+  // 壳页识别：有 product_id 骨架但 seller_id 为空且无任何销售/名称数据
+  // → 商品已下架/店铺异常（200 空壳页），抛 delisted 供调用方标记放弃
+  const hasSaleData =
+    result.soldCount !== null ||
+    result.rating !== null ||
+    result.reviewCount !== null ||
+    result.productName !== null;
+  if (result.productId && !result.sellerId && !hasSaleData) {
+    const err = new Error("Product delisted (seller_id empty, no sale data)");
+    err.delisted = true;
+    throw err;
+  }
+
   return result;
 }
 
@@ -303,11 +331,29 @@ export function parseProductHtml(html) {
  * @param {string} url - TikTok Shop PDP URL
  * @returns {Promise<{rating, reviewCount, soldCount, productName, ratingBreakdown, productId, sellerId}>}
  */
-export async function getProductInfoCDP(page, url) {
-  await page.goto(url, { waitUntil: "networkidle", timeout: 60000 });
-  // 等待 JS 渲染完成（CSR 页面需要）
-  await new Promise((r) => setTimeout(r, 3000));
-  const html = await page.content();
+export async function getProductInfoCDP(page, url, options = {}) {
+  const { gotoTimeout = 60000, renderWaitMs = 15000 } = options;
+  await page.goto(url, {
+    waitUntil: "domcontentloaded",
+    timeout: gotoTimeout,
+  });
+  // 轮询等待 CSR 渲染出商品数据标识（或出现验证码标记），
+  // 不用 networkidle：TikTok 页面持续请求，networkidle 极易超时
+  const deadline = Date.now() + renderWaitMs;
+  let html = "";
+  while (Date.now() < deadline) {
+    html = await page.content();
+    if (
+      html.includes('"product_model"') ||
+      html.includes('"review_ratings"') ||
+      html.includes("captcha-verify-container") ||
+      html.includes("<title>Security Check</title>")
+    ) {
+      break;
+    }
+    await new Promise((r) => setTimeout(r, 1000));
+  }
+  if (!html) html = await page.content();
   return parseProductHtml(html);
 }
 
@@ -320,3 +366,20 @@ export function formatCount(n) {
   if (n >= 1e3) return (n / 1e3).toFixed(1).replace(/\.0$/, "") + "K";
   return String(n);
 }
+
+/**
+ * 判断商品详情是否全空（页面被拦截/结构变化时解析结果全为 null）。
+ * 只判实际回写的四个字段：rating/soldCount/reviewCount/productName。
+ * productId/sellerId 不回写，有值也不能视为"抓到了数据"，
+ * 否则壳页（仅含 product_id 骨架）会被误报成功、任务永久留在候选池。
+ * @param {object} detail - parseProductHtml 的返回值
+ * @returns {boolean}
+ */
+export function isEmptyProductDetail(detail) {
+  return (
+    (detail?.rating ?? null) === null &&
+    (detail?.reviewCount ?? null) === null &&
+    (detail?.soldCount ?? null) === null &&
+    !detail?.productName
+  );
+}

package/src/watch/data-store.js

@@ -2772,20 +2772,27 @@ export function createStore(filePath, options = {}) {
    * @param {number|string} limit
    * @param {string|null} claimBy 领取者标识（userId），仅作标记/日志
    * @param {number} claimTtlMs 锁定超时（默认 5 分钟）
+   * @param {string|null} country 国家过滤（大写两位，如 US/JP）；非空时只返回
+   *   location_created 匹配的任务，location_created 为 NULL 的任务不分发；
+   *   传 null/空串表示不限（兼容旧客户端）
    * @returns {{uniqueId:string, nickname:string|null, seoUrl:string, productId:string|null, productTitle:string|null}[]}
    */
   function getPendingProductTasks(
     limit,
     claimBy = null,
     claimTtlMs = 5 * 60 * 1000,
+    country = null,
   ) {
     if (getDb()) {
       const l = Math.max(1, parseInt(limit) || 5);
       const now = Date.now();
       const claimExpireBefore = now - claimTtlMs;
-      const rows = getDb()
-        .prepare(
-          `
+      // 国家过滤：只认大写两位字母，避免异常值影响；无国家段的任务
+      // 不分发给任何按国家领取的客户端（后续人工回填处理）
+      const countryFilter = country
+        ? String(country).trim().toUpperCase().slice(0, 2)
+        : "";
+      let sql = `
           SELECT unique_id, nickname, top_video_seo_url, top_video_product_id,
                  top_video_product_title
           FROM jobs
@@ -2793,12 +2800,21 @@ export function createStore(filePath, options = {}) {
             AND (product_name IS NULL OR product_name = '')
             AND product_rating IS NULL
             AND product_sold_count IS NULL
+      `;
+      const params = [];
+      if (countryFilter.length === 2) {
+        sql += ` AND location_created = ?`;
+        params.push(countryFilter);
+      }
+      sql += `
             AND (claimed_at IS NULL OR claimed_at < ?)
           ORDER BY COALESCE(claimed_at, 0) ASC, updated_at DESC
           LIMIT ?
-        `,
-        )
-        .all(claimExpireBefore, l);
+        `;
+      params.push(claimExpireBefore, l);
+      const rows = getDb()
+        .prepare(sql)
+        .all(...params);
       if (rows.length > 0) {
         // 领取即锁定：写入 claimed_at/claimed_by，避免多客户端重复抓取同一商品
         const ids = rows.map((r) => r.unique_id);
@@ -2825,6 +2841,7 @@ export function createStore(filePath, options = {}) {
    * 回写商品详情到 jobs 表（product_* 列）
    * @param {string} uniqueId
    * @param {{rating?:number, reviewCount?:number, soldCount?:number, productName?:string}} productDetail
+   *   rating = -1 为哨兵值：商品不可售/已下架，写入后任务移出候选（正常评分 0-5）
    */
   function updateProductInfo(uniqueId, productDetail) {
     if (getDb()) {
@@ -2843,6 +2860,24 @@ export function createStore(filePath, options = {}) {
         "jobs",
       );
       if (ret.error) return ret;
+      // 校验：写入后 product_* 仍全空 → 回写无效（壳页假成功防御），
+      // 防止任务在候选池里无限轮转
+      const row = getDb()
+        .prepare(
+          `SELECT product_name, product_rating, product_sold_count,
+                  product_review_count
+           FROM jobs WHERE unique_id = ?`,
+        )
+        .get(uniqueId);
+      if (
+        row &&
+        (row.product_name === null || row.product_name === "") &&
+        row.product_rating === null &&
+        row.product_sold_count === null &&
+        row.product_review_count === null
+      ) {
+        return { ok: false, error: "product 字段仍全空，回写无效" };
+      }
       return { ok: true };
     }
     return { ok: false, error: "db not initialized" };

package/src/watch/public/app.js

@@ -1394,6 +1394,16 @@ function navigateToPrecise() {
   window.location.hash = "#precise";
 }
 
+// product-fill 任务查询条件表单：折叠 / 展开
+function togglePreciseCriteria() {
+  const body = document.getElementById("preciseCriteriaBody");
+  const arrow = document.getElementById("preciseCriteriaArrow");
+  if (!body) return;
+  const collapsed = body.style.display === "none";
+  body.style.display = collapsed ? "" : "none";
+  if (arrow) arrow.textContent = collapsed ? "▼" : "▶";
+}
+
 function navigateToTags() {
   window.location.hash = "#tags";
 }
@@ -3801,12 +3811,14 @@ fetchUsers();
 fetchClientErrors();
 initTableSorting();
 initTargetTableSorting();
+fetchWatchdogBadge();
 
 // 错峰轮询（服务端已缓存，瞬时响应无压力）
 setInterval(fetchStats, 10000);
 setTimeout(() => setInterval(fetchUsers, 10000), 2500);
 setTimeout(() => setInterval(fetchClientErrors, 10000), 5000);
 setTimeout(() => setInterval(refreshUserUpdateIfActive, 10000), 7500);
+setTimeout(() => setInterval(fetchWatchdogBadge, 30000), 9000);
 
 // ========== 看门狗页面 ==========
 
@@ -3830,6 +3842,7 @@ async function fetchWatchdogData() {
   try {
     const res = await fetch("/api/watchdog/clients");
     _wdData = await res.json();
+    updateWatchdogBadge(_wdData.clients || [], _wdData.clientCommands || {});
     renderWatchdogDashboard(
       _wdData.clients || [],
       _wdData.clientCommands || {},
@@ -3845,6 +3858,51 @@ async function fetchWatchdogData() {
   }
 }
 
+// ── 导航栏"看门狗"离线 badge ──
+// 离线数口径与看门狗页面卡片一致：
+//   = 已注册但离线的客户端数 + 配置了命令但从未注册（renderOfflineCard）的 userId 数
+function updateWatchdogBadge(clients, clientCommands) {
+  const badge = document.getElementById("wdOfflineBadge");
+  if (!badge) return;
+  const offlineClients = clients.filter(function (c) {
+    return !c.online;
+  }).length;
+  const registeredUids = {};
+  clients.forEach(function (c) {
+    if (c.config && c.config.userId) {
+      registeredUids[c.config.userId] = true;
+    }
+  });
+  let neverRegistered = 0;
+  for (var uid in clientCommands) {
+    if (
+      clientCommands[uid] &&
+      clientCommands[uid].length > 0 &&
+      !registeredUids[uid]
+    ) {
+      neverRegistered++;
+    }
+  }
+  const total = offlineClients + neverRegistered;
+  if (total > 0) {
+    badge.textContent = total;
+    badge.style.display = "";
+  } else {
+    badge.style.display = "none";
+  }
+}
+
+// 独立轻量轮询：导航栏 badge 在任何页面都实时显示离线数
+async function fetchWatchdogBadge() {
+  try {
+    const res = await fetch("/api/watchdog/clients");
+    const data = await res.json();
+    updateWatchdogBadge(data.clients || [], data.clientCommands || {});
+  } catch (e) {
+    // 静默失败，保留旧值
+  }
+}
+
 // ── 渲染辅助 ──
 
 function tag(tagName, attrs, content) {

package/src/watch/public/index.html

@@ -21,7 +21,9 @@
         <span>🎆</span>
       </label>
       <a class="script-link" onclick="navigateToSettings()" style="cursor:pointer">⚙️ 配置</a>
-      <a class="script-link" onclick="navigateToWatchdog()" style="cursor:pointer">🐕 看门狗</a>
+      <a class="script-link" onclick="navigateToWatchdog()" style="cursor:pointer;position:relative"
+        title="看门狗离线客户端数">🐕 看门狗<span id="wdOfflineBadge"
+          style="display:none;position:absolute;top:-9px;right:-8px;background:#ef4444;color:#fff;border-radius:10px;font-size:10px;font-weight:700;line-height:1;padding:3px 6px;box-shadow:0 0 6px rgba(239,68,68,0.6);white-space:nowrap">0</span></a>
       <span class="status" id="lastUpdate">--</span>
     </div>
   </div>
@@ -393,6 +395,49 @@
         <div class="label">← 返回主页面</div>
       </div>
     </div>
+    <!-- 精准商家检索条件（静态表单展示，与 src/watch/data-store.js getPreciseMerchantsFromDb 保持一致） -->
+    <div style="margin-bottom:16px;background:#1a1a24;border:1px solid #333;border-radius:8px;overflow:hidden">
+      <div
+        style="display:flex;align-items:center;justify-content:space-between;padding:10px 14px;cursor:pointer;user-select:none"
+        onclick="togglePreciseCriteria()" title="点击展开/收起">
+        <div style="display:flex;align-items:center;gap:8px;flex-wrap:wrap">
+          <span style="color:#f59e0b;font-size:14px">🎯</span>
+          <span style="font-size:13px;color:#e0e0e0;font-weight:600">精准商家检索条件</span>
+          <span style="font-size:11px;color:#666">（阈值参数 · jobs 表）</span>
+        </div>
+        <span id="preciseCriteriaArrow" style="color:#888;font-size:12px">▼</span>
+      </div>
+      <div id="preciseCriteriaBody" style="padding:2px 14px 14px">
+        <div style="font-size:11px;color:#888;margin-bottom:10px">以下阈值条件全部满足（AND），其余条件为固有规则：</div>
+        <div style="display:grid;grid-template-columns:repeat(auto-fill,minmax(260px,1fr));gap:10px">
+          <div style="background:#14141d;border:1px solid #2a2a3a;border-radius:8px;padding:10px 12px">
+            <div style="display:flex;justify-content:space-between;align-items:center;gap:8px">
+              <span style="font-size:12px;color:#60a5fa;font-weight:600">商品评论数</span>
+              <span style="font-size:10px;color:#666;font-family:monospace">product_review_count</span>
+            </div>
+            <div style="margin-top:6px;font-size:16px;color:#4ade80;font-weight:700">= 0</div>
+            <div style="margin-top:4px;font-size:11px;color:#888;line-height:1.5">商品评论数为 0</div>
+          </div>
+          <div style="background:#14141d;border:1px solid #2a2a3a;border-radius:8px;padding:10px 12px">
+            <div style="display:flex;justify-content:space-between;align-items:center;gap:8px">
+              <span style="font-size:12px;color:#60a5fa;font-weight:600">商品销量</span>
+              <span style="font-size:10px;color:#666;font-family:monospace">product_sold_count</span>
+            </div>
+            <div style="margin-top:6px;font-size:16px;color:#4ade80;font-weight:700">&gt; 10</div>
+            <div style="margin-top:4px;font-size:11px;color:#888;line-height:1.5">商品销量大于 10</div>
+          </div>
+          <div style="background:#14141d;border:1px solid #2a2a3a;border-radius:8px;padding:10px 12px">
+            <div style="display:flex;justify-content:space-between;align-items:center;gap:8px">
+              <span style="font-size:12px;color:#60a5fa;font-weight:600">EC 视频播放量</span>
+              <span style="font-size:10px;color:#666;font-family:monospace">playCount</span>
+            </div>
+            <div style="margin-top:6px;font-size:16px;color:#4ade80;font-weight:700">&gt; 1000</div>
+            <div style="margin-top:4px;font-size:11px;color:#888;line-height:1.5">最近 7 天发布的 EC 视频（isECVideo）中，存在播放量超过
+              1000 的视频</div>
+          </div>
+        </div>
+      </div>
+    </div>
     <div class="table-wrap">
       <div class="precise-list-header">
         <h3>精准商家列表</h3>

package/src/watch/server.js

@@ -1289,13 +1289,16 @@ export function startWatchServer(
       if (req.method === "GET" && routePath === "/api/product-tasks") {
         const limit = params.limit;
         const clientId = params.userId || "?";
+        const country = params.country || null;
         const tasks = store.getPendingProductTasks(
           limit,
           params.userId || null,
+          undefined, // claimTtlMs 用默认 5 分钟
+          country,
         );
         const ts = new Date().toLocaleTimeString("zh-CN", { hour12: false });
         console.error(
-          `[JOB ${ts}] PRODUCT-TASKS: ${tasks.length} tasks (client=${clientId})`,
+          `[JOB ${ts}] PRODUCT-TASKS: ${tasks.length} tasks (client=${clientId}${country ? `, country=${country}` : ""})`,
         );
         sendJSON(res, 200, { total: tasks.length, tasks });
         return;