tt-help-cli-ycl 1.4.23 → 1.4.24

package/src/watch/data-store.js

@@ -1563,291 +1563,211 @@ export function createStore(filePath, options = {}) {
             .filter(Boolean)
         : [];
 
-      function getLocationGroups() {
-        const selected = normalizedLocations.length
-          ? normalizedLocations
-          : null;
-        const tier1List = selected
-          ? selected.filter((loc) => tier1.has(loc))
-          : [...tier1];
-        const tier2List = selected
-          ? selected.filter((loc) => tier2.has(loc))
-          : [...tier2];
-        const otherList = selected
-          ? selected.filter((loc) => !tier1.has(loc) && !tier2.has(loc))
-          : null;
-        const groups = [];
-        if (tier1List.length > 0)
-          groups.push({ type: "include", values: tier1List });
-        if (tier2List.length > 0)
-          groups.push({ type: "include", values: tier2List });
-        if (selected) {
-          if (otherList.length > 0)
-            groups.push({ type: "include", values: otherList });
-        } else {
-          groups.push({ type: "exclude", values: [...tier1, ...tier2] });
-        }
-        return groups;
+      // ---- 过期 processing 回收 ----
+      const expiredRow = getDb()
+        .prepare(
+          `SELECT * FROM jobs WHERE status = 'processing'
+           AND claimed_at IS NOT NULL AND ? - claimed_at > ?
+           ORDER BY claimed_at ASC LIMIT 1`,
+        )
+        .get(now, expireMs);
+      let expiredCandidate = null;
+      if (expiredRow) {
+        getDb()
+          .prepare(
+            "UPDATE jobs SET status = 'pending', claimed_at = NULL WHERE unique_id = ?",
+          )
+          .run(expiredRow.unique_id);
+        expiredCandidate = expiredRow;
       }
 
-      const locationGroups = getLocationGroups();
+      // ---- 构建统一优先级查询（合并原 20+ 次独立 SELECT 为 1 次 UNION） ----
+      const statusCodeWhere = loggedIn
+        ? "(status_code IS NULL OR status_code = 0 OR status_code = 209002)"
+        : "(status_code IS NULL OR status_code = 0)";
+      const notSellerWhere = loggedIn ? "" : "AND COALESCE(tt_seller, 0) != 1";
+
+      const tier1List = normalizedLocations.length
+        ? normalizedLocations.filter((loc) => tier1.has(loc))
+        : [...tier1];
+      const tier2List = normalizedLocations.length
+        ? normalizedLocations.filter((loc) => tier2.has(loc))
+        : [...tier2];
+      const otherFiltered = normalizedLocations.length
+        ? normalizedLocations.filter(
+            (loc) => !tier1.has(loc) && !tier2.has(loc),
+          )
+        : null;
 
-      function applyLocationGroup(where, args, group) {
-        if (!group) return;
-        if (group.type === "include") {
-          where.push(
-            `(UPPER(COALESCE(guessed_location, '')) IN (${group.values.map(() => "?").join(", ")}) OR COALESCE(guessed_location, '') = '')`,
-          );
-          args.push(...group.values);
-          return;
-        }
-        where.push(
-          `UPPER(COALESCE(guessed_location, '')) NOT IN (${group.values.map(() => "?").join(", ")})`,
-        );
-        args.push(...group.values);
-      }
+      const locIn = (list) => {
+        if (!list || list.length === 0) return null;
+        const ph = list.map(() => "?").join(", ");
+        return {
+          sql: `(UPPER(COALESCE(guessed_location, '')) IN (${ph}) OR COALESCE(guessed_location, '') = '')`,
+          params: [...list],
+        };
+      };
+      const locNotIn = (list) => {
+        if (!list || list.length === 0) return null;
+        const ph = list.map(() => "?").join(", ");
+        return {
+          sql: `UPPER(COALESCE(guessed_location, '')) NOT IN (${ph})`,
+          params: [...list],
+        };
+      };
+      const locOther = () => {
+        if (otherFiltered && otherFiltered.length > 0)
+          return locIn(otherFiltered);
+        if (normalizedLocations.length === 0)
+          return locNotIn([...tier1, ...tier2]);
+        return null;
+      };
 
-      function queryPendingOne({ requireVideo, group, filters = [] }) {
-        const where = ["status = 'pending'"];
-        const args = [];
-        if (!loggedIn) {
-          where.push("COALESCE(tt_seller, 0) != 1");
-          // 未登录：只能领取 status_code 为空或 0 的任务
-          where.push("(status_code IS NULL OR status_code = 0)");
-        } else {
-          // 登录：可以领取 status_code 为空、0、或 209002 的任务
-          where.push(
-            "(status_code IS NULL OR status_code = 0 OR status_code = 209002)",
-          );
-        }
-        // 其他 status_code 值的任务不被领取
-        if (requireVideo) {
-          where.push("COALESCE(video_count, 0) > 0");
-        }
-        applyLocationGroup(where, args, group);
-        for (const filter of filters) {
-          where.push(filter);
-        }
-        return getDb()
-          .prepare(
-            `
-            SELECT *
-            FROM jobs
-            WHERE ${where.join(" AND ")}
-            ORDER BY follower_count DESC, created_at ASC, unique_id ASC
-            LIMIT 1
-          `,
-          )
-          .get(...args);
+      // 全局 location 过滤（pinned 查询用）
+      let pinnedLocSql = "";
+      const pinnedLocParams = [];
+      if (normalizedLocations.length > 0) {
+        const ph = normalizedLocations.map(() => "?").join(", ");
+        pinnedLocSql = `AND (UPPER(COALESCE(guessed_location, '')) IN (${ph}) OR COALESCE(guessed_location, '') = '')`;
+        pinnedLocParams.push(...normalizedLocations);
       }
 
-      function queryPendingByGroup({ requireVideo, group, filters = [] }) {
-        if (group?.type === "include" && group.values.length > 1) {
-          for (const location of group.values) {
-            const row = queryPendingOne({
-              requireVideo,
-              group: { type: "include", values: [location] },
-              filters,
-            });
-            if (row) return row;
-          }
-          return null;
-        }
-        return queryPendingOne({ requireVideo, group, filters });
-      }
+      const sqlParts = [];
+      const allParams = [];
 
-      function findPinnedPending(requireVideo) {
-        const where = ["status = 'pending'", "COALESCE(pinned, 0) = 1"];
-        const args = [];
-        if (!loggedIn) {
-          where.push("COALESCE(tt_seller, 0) != 1");
-        }
-        if (requireVideo) {
-          where.push("COALESCE(video_count, 0) > 0");
-        }
-        if (normalizedLocations.length > 0) {
-          where.push(
-            `(UPPER(COALESCE(guessed_location, '')) IN (${normalizedLocations.map(() => "?").join(", ")}) OR COALESCE(guessed_location, '') = '')`,
-          );
-          args.push(...normalizedLocations);
-        }
-        return getDb()
-          .prepare(
-            `
-            SELECT *
-            FROM jobs
-            WHERE ${where.join(" AND ")}
-            ORDER BY created_at ASC, unique_id ASC
-            LIMIT 1
-          `,
-          )
-          .get(...args);
-      }
+      const addPart = (prio, subPrio, conditions, params) => {
+        sqlParts.push(
+          `SELECT *, ${prio} as _prio, ${subPrio} as _sub_prio FROM jobs WHERE ${conditions}`,
+        );
+        allParams.push(...params);
+      };
 
-      function findPrioritizedPending(requireVideo) {
-        for (const group of locationGroups) {
-          const seed = queryPendingByGroup({
-            requireVideo,
-            group,
-            filters: [
-              "COALESCE(pinned, 0) = 0",
-              `instr(COALESCE(sources, ''), '"seed"') > 0`,
-            ],
-          });
-          if (seed) return seed;
-        }
+      // prio 0: pinned + hasVideo
+      addPart(
+        0,
+        0,
+        `status = 'pending' AND pinned = 1 AND video_count > 0 ${pinnedLocSql} AND ${statusCodeWhere} ${notSellerWhere}`,
+        pinnedLocParams,
+      );
+      // prio 1: pinned (no video req)
+      addPart(
+        1,
+        0,
+        `status = 'pending' AND pinned = 1 ${pinnedLocSql} AND ${statusCodeWhere} ${notSellerWhere}`,
+        pinnedLocParams,
+      );
 
-        if (loggedIn) {
-          for (const group of locationGroups) {
-            const seller = queryPendingByGroup({
-              requireVideo,
-              group,
-              filters: [
-                "COALESCE(pinned, 0) = 0",
-                "tt_seller = 1",
-                "verified = 0",
-              ],
-            });
-            if (seller) return seller;
-          }
-        }
+      // prio 2-3: expired candidate（回收的超时任务优先）
+      if (expiredCandidate) {
+        addPart(
+          2,
+          0,
+          `unique_id = ? AND video_count > 0 AND ${statusCodeWhere} ${notSellerWhere}`,
+          [expiredCandidate.unique_id],
+        );
+        addPart(
+          3,
+          0,
+          `unique_id = ? AND ${statusCodeWhere} ${notSellerWhere}`,
+          [expiredCandidate.unique_id],
+        );
+      }
 
-        for (const group of locationGroups) {
-          const follow = queryPendingByGroup({
-            requireVideo,
-            group,
-            filters: [
-              "COALESCE(pinned, 0) = 0",
-              `(
-                instr(COALESCE(sources, ''), '"following"') > 0
-                OR instr(COALESCE(sources, ''), '"follower"') > 0
-                OR instr(COALESCE(sources, ''), '"comment"') > 0
-              )`,
-            ],
-          });
-          if (follow) return follow;
-        }
+      // 非 pinned 优先级：seed → seller → follow → other，每级 × 3 个 location tier × (hasVideo / noVideoReq)
+      const priorities = [
+        {
+          filter: `instr(COALESCE(sources, ''), '"seed"') > 0`,
+          loggedInOnly: false,
+        },
+        { filter: `tt_seller = 1 AND verified = 0`, loggedInOnly: true },
+        {
+          filter: `(instr(COALESCE(sources, ''), '"following"') > 0 OR instr(COALESCE(sources, ''), '"follower"') > 0 OR instr(COALESCE(sources, ''), '"comment"') > 0)`,
+          loggedInOnly: false,
+        },
+        { filter: `1 = 1`, loggedInOnly: false },
+      ];
 
-        for (const group of locationGroups) {
-          const other = queryPendingByGroup({
-            requireVideo,
-            group,
-            filters: ["COALESCE(pinned, 0) = 0"],
-          });
-          if (other) return other;
+      let basePrio = 4;
+      for (const p of priorities) {
+        if (p.loggedInOnly && !loggedIn) continue;
+        const tierLocations = [
+          { list: tier1List, fn: locIn },
+          { list: tier2List, fn: locIn },
+          { list: null, fn: locOther },
+        ];
+        let subPrio = 0;
+        for (const tl of tierLocations) {
+          const loc = tl.fn(tl.list);
+          if (!loc) continue;
+          // hasVideo
+          addPart(
+            basePrio,
+            subPrio,
+            `status = 'pending' AND pinned = 0 AND ${p.filter} AND video_count > 0 AND ${loc.sql} AND ${statusCodeWhere} ${notSellerWhere}`,
+            loc.params,
+          );
+          subPrio++;
+          // noVideoReq
+          addPart(
+            basePrio + 1,
+            subPrio,
+            `status = 'pending' AND pinned = 0 AND ${p.filter} AND ${loc.sql} AND ${statusCodeWhere} ${notSellerWhere}`,
+            loc.params,
+          );
+          subPrio++;
         }
+        basePrio += 2;
+      }
 
-        return null;
+      let selectedRow = null;
+      if (sqlParts.length > 0) {
+        const unionSql =
+          sqlParts.join(" UNION ALL ") +
+          " ORDER BY _prio, _sub_prio, follower_count DESC, created_at ASC, unique_id ASC LIMIT 1";
+        selectedRow = getDb()
+          .prepare(unionSql)
+          .get(...allParams);
       }
 
-      function claimRow(row) {
-        if (!row) return null;
+      if (selectedRow) {
         getDb()
           .prepare(
             "UPDATE jobs SET status = 'processing', claimed_at = ?, claimed_by = ? WHERE unique_id = ?",
           )
-          .run(now, userId, row.unique_id);
+          .run(now, userId, selectedRow.unique_id);
         markStatsDirty();
         return {
-          uniqueId: row.unique_id,
-          nickname: row.nickname,
+          uniqueId: selectedRow.unique_id,
+          nickname: selectedRow.nickname,
           claimedAt: now,
           claimedBy: userId,
         };
       }
 
-      const expiredRow = getDb()
-        .prepare(
-          `
-          SELECT *
-          FROM jobs
-          WHERE status = 'processing'
-            AND claimed_at IS NOT NULL
-            AND ? - claimed_at > ?
-          ORDER BY claimed_at ASC
-          LIMIT 1
-        `,
-        )
-        .get(now, expireMs);
-      let expiredCandidate = null;
-      if (expiredRow) {
-        getDb()
-          .prepare(
-            "UPDATE jobs SET status = 'pending', claimed_at = NULL WHERE unique_id = ?",
-          )
-          .run(expiredRow.unique_id);
-        expiredCandidate = mapJobRow({
-          ...expiredRow,
-          status: "pending",
-          claimed_at: null,
-        });
-      }
-
-      for (const requireVideo of [true, false]) {
-        const pinned = findPinnedPending(requireVideo);
-        if (pinned) {
-          return claimRow(pinned);
-        }
-        if (expiredCandidate) {
-          return claimRow({
-            unique_id: expiredCandidate.uniqueId,
-            nickname: expiredCandidate.nickname,
-          });
-        }
-        const ranked = findPrioritizedPending(requireVideo);
-        if (ranked) {
-          return claimRow(ranked);
-        }
-      }
-      // 尝试从 raw_jobs 毛料库补充任务（使用 createStore 时配置的 LLM 打分）
-      // 使用锁防止多个请求同时触发 LLM refill
-      if (refillLock) {
-        // 已有 refill 在进行中，等待完成后重新尝试领取
-        await refillLock;
-        for (const requireVideo of [true, false]) {
-          const pinned = findPinnedPending(requireVideo);
-          if (pinned) {
-            return claimRow(pinned);
-          }
-          const ranked = findPrioritizedPending(requireVideo);
-          if (ranked) {
-            return claimRow(ranked);
-          }
-        }
-        return null;
-      }
-      const refillPromise = (async () => {
-        const result = refillJobsFromRaw(
-          normalizedLocations.length ? normalizedLocations : null,
-          500,
-          refillLlmConfig,
-        );
-        // refillJobsFromRaw 在 LLM 模式下返回 Promise
-        if (result && typeof result.then === "function") {
-          return await result;
-        }
-        return result;
-      })();
-      // 让并发请求等待同一个 refill，并且无论成功/失败都释放锁
-      refillLock = refillPromise.finally(() => {
-        refillLock = null;
-      });
-
-      const refillResult = await refillLock;
-      if (refillResult.moved > 0) {
-        console.error(
-          `[data-store] 从 raw_jobs 补充了 ${refillResult.moved} 条任务到 jobs`,
-        );
-        for (const requireVideo of [true, false]) {
-          const pinned = findPinnedPending(requireVideo);
-          if (pinned) {
-            return claimRow(pinned);
-          }
-          const ranked = findPrioritizedPending(requireVideo);
-          if (ranked) {
-            return claimRow(ranked);
+      // refill 异步化：不阻塞当前请求
+      if (!refillLock) {
+        refillLock = (async () => {
+          try {
+            const result = refillJobsFromRaw(
+              normalizedLocations.length ? normalizedLocations : null,
+              500,
+              refillLlmConfig,
+            );
+            const r =
+              result && typeof result.then === "function"
+                ? await result
+                : result;
+            if (r.moved > 0) {
+              console.error(
+                `[data-store] 从 raw_jobs 补充了 ${r.moved} 条任务到 jobs`,
+              );
+            }
+          } catch (e) {
+            console.error(`[data-store] refill 异常: ${e.message}`);
+          } finally {
+            refillLock = null;
           }
-        }
+        })();
       }
 
       return null;

package/src/watch/db-schema.js

@@ -425,6 +425,7 @@ export function initDb(filePath) {
     `CREATE INDEX IF NOT EXISTS idx_cross_country_source ON cross_country_learnings(source_country, result_status)`,
     `CREATE INDEX IF NOT EXISTS idx_cross_country_target ON cross_country_learnings(target_country)`,
     `CREATE INDEX IF NOT EXISTS idx_tags_dead_reason ON tags(dead_reason)`,
+    `CREATE INDEX IF NOT EXISTS idx_jobs_recheck ON jobs(status, updated_at) WHERE status = 'done' AND (tt_seller IS NULL OR tt_seller != 1)`,
   ];
   for (const sql of indexes) {
     db.exec(sql);

package/src/watch/db-stats.js

@@ -7,10 +7,11 @@
 import { getDb } from "./db-schema.js";
 import { getRawJobsCount, getUserDbCount } from "./db-crud.js";
 
-export function getDashboardStatsFromDb(targetLocations = []) {
+export function getDashboardStatsFromDb(targetLocations = [], options = {}) {
   const db = getDb();
   if (!db) return null;
 
+  const skipSlow = !!options.skipSlowCounts;
   const targetPlaceholders = targetLocations.map(() => "?").join(", ");
   const targetParams = targetLocations.length ? targetLocations : [];
 
@@ -89,21 +90,28 @@ export function getDashboardStatsFromDb(targetLocations = []) {
     .prepare("SELECT COUNT(*) as total FROM jobs_base")
     .get().total;
 
-  const tagCount = db.prepare("SELECT COUNT(*) as total FROM tags").get().total;
+  const tagCount = skipSlow
+    ? 0
+    : db.prepare("SELECT COUNT(*) as total FROM tags").get().total;
 
-  const commentTaskCount = db
-    .prepare(
-      `SELECT COUNT(*) as total FROM videos
-       WHERE user_update_count IS NULL OR user_update_count <= 0`,
-    )
-    .get().total;
+  const commentTaskCount = skipSlow
+    ? 0
+    : db
+        .prepare(
+          `SELECT COUNT(*) as total FROM videos
+           WHERE user_update_count IS NULL OR user_update_count <= 0`,
+        )
+        .get().total;
+
+  const rawJobsCount = skipSlow ? 0 : getRawJobsCount();
+  const dbUsersCount = skipSlow ? 0 : getUserDbCount();
 
   return {
     totalUsers: aggregateRow.total,
     tagCount,
     commentTaskCount,
-    rawJobs: getRawJobsCount(),
-    dbTotalUsers: getUserDbCount(),
+    rawJobs: rawJobsCount,
+    dbTotalUsers: dbUsersCount,
     jobsTotal: aggregateRow.total,
     jobsBaseTotal: jobsBaseCount,
     jobsPending: aggregateRow.pending,

package/src/watch/public/app.js

@@ -3356,7 +3356,7 @@ fetchClientErrors();
 initTableSorting();
 initTargetTableSorting();
 
-// 错峰轮询，避免同时发 4 个请求阻塞服务端
+// 错峰轮询（服务端已缓存，瞬时响应无压力）
 setInterval(fetchStats, 10000);
 setTimeout(() => setInterval(fetchUsers, 10000), 2500);
 setTimeout(() => setInterval(fetchClientErrors, 10000), 5000);