tt-help-cli-ycl 1.4.22 → 1.4.24

package/src/scraper/explore-core.js

@@ -42,7 +42,7 @@ async function processExplore(page, username, options, log) {
     keepFollow: false,
     locationCreated: null,
     latestVideoTime: null,
-    topRecentVideo: null,
+    recentEcVideos: null,
     noVideo: false,
     restricted: false,
     error: null,
@@ -97,6 +97,19 @@ async function processExplore(page, username, options, log) {
         nowSeconds - v.createTime <= SEVEN_DAYS_SECONDS,
     );
     if (recentVideos.length > 0) {
+      result.recentEcVideos = recentVideos.map((v) => ({
+        id: v.id,
+        href: v.href,
+        playCount: v.playCount,
+        createTime: v.createTime,
+        isAd: v.isAd,
+      }));
+      const maxPlay = recentVideos.reduce(
+        (max, v) => (v.playCount > max ? v.playCount : max),
+        0,
+      );
+      log(`  7天内 EC视频: ${recentVideos.length} 个, 最高播放 ${maxPlay}`);
+      // 兼容旧字段 topRecentVideo
       const topVideo = recentVideos.reduce((max, v) =>
         v.playCount > max.playCount ? v : max,
       );
@@ -107,9 +120,6 @@ async function processExplore(page, username, options, log) {
         createTime: topVideo.createTime,
         isAd: topVideo.isAd,
       };
-      log(
-        `  7天内 EC视频最高播放: ${topVideo.playCount} 次播放 (${recentVideos.length} 个EC候选)`,
-      );
     }
 
     if (videoArray.length <= 0) {

package/src/watch/data-store.js

@@ -320,7 +320,9 @@ function getTargetUsersFromDb(targetLocations = []) {
           tt_seller, verified, follower_count, following_count,
           location_created, latest_video_time, refresh_time,
           guessed_location, pinned, processed_at, video_count,
-          no_video, claimed_by, claimed_at, created_at, updated_at
+          no_video, claimed_by, claimed_at, created_at, updated_at,
+          top_video_play_count, top_video_href, top_video_is_ad,
+          recent_ec_videos
         FROM jobs
         WHERE tt_seller = 1
         AND verified = 0
@@ -349,6 +351,8 @@ function getTargetUsersByCountryFromDb(targetLocations = [], options = {}) {
     search,
     limit,
     offset,
+    sort,
+    sortOrder,
   } = options;
   const placeholders = targetLocations.map(() => "?").join(", ");
   const baseParams = [...targetLocations];
@@ -401,6 +405,7 @@ function getTargetUsersByCountryFromDb(targetLocations = [], options = {}) {
         top_video_play_count,
         top_video_href,
         top_video_is_ad,
+        recent_ec_videos,
         status,
         sources
       FROM jobs
@@ -441,6 +446,50 @@ function getTargetUsersByCountryFromDb(targetLocations = [], options = {}) {
         .prepare(countSql)
         .get(...params)?.cnt || 0;
 
+    // 服务端排序：当 sort=topVideoPlayCount 时，全量取出后按 EC 最大播放量排序再分页
+    if (sort === "topVideoPlayCount") {
+      const allRows = getDb()
+        .prepare(sql)
+        .all(...params)
+        .map(mapJobRow);
+
+      const getMaxPlay = (u) => {
+        try {
+          const ec =
+            typeof u.recentEcVideos === "string"
+              ? JSON.parse(u.recentEcVideos)
+              : Array.isArray(u.recentEcVideos)
+                ? u.recentEcVideos
+                : [];
+          if (ec.length > 0) {
+            return ec.reduce(
+              (max, v) => (v.playCount > max ? v.playCount : max),
+              0,
+            );
+          }
+        } catch (e) {}
+        return 0;
+      };
+
+      const isAsc = sortOrder === "asc";
+      allRows.sort((a, b) => {
+        const va = getMaxPlay(a);
+        const vb = getMaxPlay(b);
+        return isAsc ? va - vb : vb - va;
+      });
+
+      const safeLimit = Math.min(Math.floor(limit), 10000);
+      const safeOffset = Math.max(Math.floor(offset), 0);
+      const rows = allRows.slice(safeOffset, safeOffset + safeLimit);
+
+      return {
+        total,
+        limit: safeLimit,
+        offset: safeOffset,
+        users: rows,
+      };
+    }
+
     sql += ` LIMIT ? OFFSET ?`;
     const safeLimit = Math.min(Math.floor(limit), 10000);
     const safeOffset = Math.max(Math.floor(offset), 0);
@@ -1514,291 +1563,211 @@ export function createStore(filePath, options = {}) {
             .filter(Boolean)
         : [];
 
-      function getLocationGroups() {
-        const selected = normalizedLocations.length
-          ? normalizedLocations
-          : null;
-        const tier1List = selected
-          ? selected.filter((loc) => tier1.has(loc))
-          : [...tier1];
-        const tier2List = selected
-          ? selected.filter((loc) => tier2.has(loc))
-          : [...tier2];
-        const otherList = selected
-          ? selected.filter((loc) => !tier1.has(loc) && !tier2.has(loc))
-          : null;
-        const groups = [];
-        if (tier1List.length > 0)
-          groups.push({ type: "include", values: tier1List });
-        if (tier2List.length > 0)
-          groups.push({ type: "include", values: tier2List });
-        if (selected) {
-          if (otherList.length > 0)
-            groups.push({ type: "include", values: otherList });
-        } else {
-          groups.push({ type: "exclude", values: [...tier1, ...tier2] });
-        }
-        return groups;
+      // ---- 过期 processing 回收 ----
+      const expiredRow = getDb()
+        .prepare(
+          `SELECT * FROM jobs WHERE status = 'processing'
+           AND claimed_at IS NOT NULL AND ? - claimed_at > ?
+           ORDER BY claimed_at ASC LIMIT 1`,
+        )
+        .get(now, expireMs);
+      let expiredCandidate = null;
+      if (expiredRow) {
+        getDb()
+          .prepare(
+            "UPDATE jobs SET status = 'pending', claimed_at = NULL WHERE unique_id = ?",
+          )
+          .run(expiredRow.unique_id);
+        expiredCandidate = expiredRow;
       }
 
-      const locationGroups = getLocationGroups();
+      // ---- 构建统一优先级查询（合并原 20+ 次独立 SELECT 为 1 次 UNION） ----
+      const statusCodeWhere = loggedIn
+        ? "(status_code IS NULL OR status_code = 0 OR status_code = 209002)"
+        : "(status_code IS NULL OR status_code = 0)";
+      const notSellerWhere = loggedIn ? "" : "AND COALESCE(tt_seller, 0) != 1";
+
+      const tier1List = normalizedLocations.length
+        ? normalizedLocations.filter((loc) => tier1.has(loc))
+        : [...tier1];
+      const tier2List = normalizedLocations.length
+        ? normalizedLocations.filter((loc) => tier2.has(loc))
+        : [...tier2];
+      const otherFiltered = normalizedLocations.length
+        ? normalizedLocations.filter(
+            (loc) => !tier1.has(loc) && !tier2.has(loc),
+          )
+        : null;
 
-      function applyLocationGroup(where, args, group) {
-        if (!group) return;
-        if (group.type === "include") {
-          where.push(
-            `(UPPER(COALESCE(guessed_location, '')) IN (${group.values.map(() => "?").join(", ")}) OR COALESCE(guessed_location, '') = '')`,
-          );
-          args.push(...group.values);
-          return;
-        }
-        where.push(
-          `UPPER(COALESCE(guessed_location, '')) NOT IN (${group.values.map(() => "?").join(", ")})`,
-        );
-        args.push(...group.values);
-      }
+      const locIn = (list) => {
+        if (!list || list.length === 0) return null;
+        const ph = list.map(() => "?").join(", ");
+        return {
+          sql: `(UPPER(COALESCE(guessed_location, '')) IN (${ph}) OR COALESCE(guessed_location, '') = '')`,
+          params: [...list],
+        };
+      };
+      const locNotIn = (list) => {
+        if (!list || list.length === 0) return null;
+        const ph = list.map(() => "?").join(", ");
+        return {
+          sql: `UPPER(COALESCE(guessed_location, '')) NOT IN (${ph})`,
+          params: [...list],
+        };
+      };
+      const locOther = () => {
+        if (otherFiltered && otherFiltered.length > 0)
+          return locIn(otherFiltered);
+        if (normalizedLocations.length === 0)
+          return locNotIn([...tier1, ...tier2]);
+        return null;
+      };
 
-      function queryPendingOne({ requireVideo, group, filters = [] }) {
-        const where = ["status = 'pending'"];
-        const args = [];
-        if (!loggedIn) {
-          where.push("COALESCE(tt_seller, 0) != 1");
-          // 未登录：只能领取 status_code 为空或 0 的任务
-          where.push("(status_code IS NULL OR status_code = 0)");
-        } else {
-          // 登录：可以领取 status_code 为空、0、或 209002 的任务
-          where.push(
-            "(status_code IS NULL OR status_code = 0 OR status_code = 209002)",
-          );
-        }
-        // 其他 status_code 值的任务不被领取
-        if (requireVideo) {
-          where.push("COALESCE(video_count, 0) > 0");
-        }
-        applyLocationGroup(where, args, group);
-        for (const filter of filters) {
-          where.push(filter);
-        }
-        return getDb()
-          .prepare(
-            `
-            SELECT *
-            FROM jobs
-            WHERE ${where.join(" AND ")}
-            ORDER BY follower_count DESC, created_at ASC, unique_id ASC
-            LIMIT 1
-          `,
-          )
-          .get(...args);
+      // 全局 location 过滤（pinned 查询用）
+      let pinnedLocSql = "";
+      const pinnedLocParams = [];
+      if (normalizedLocations.length > 0) {
+        const ph = normalizedLocations.map(() => "?").join(", ");
+        pinnedLocSql = `AND (UPPER(COALESCE(guessed_location, '')) IN (${ph}) OR COALESCE(guessed_location, '') = '')`;
+        pinnedLocParams.push(...normalizedLocations);
       }
 
-      function queryPendingByGroup({ requireVideo, group, filters = [] }) {
-        if (group?.type === "include" && group.values.length > 1) {
-          for (const location of group.values) {
-            const row = queryPendingOne({
-              requireVideo,
-              group: { type: "include", values: [location] },
-              filters,
-            });
-            if (row) return row;
-          }
-          return null;
-        }
-        return queryPendingOne({ requireVideo, group, filters });
-      }
+      const sqlParts = [];
+      const allParams = [];
 
-      function findPinnedPending(requireVideo) {
-        const where = ["status = 'pending'", "COALESCE(pinned, 0) = 1"];
-        const args = [];
-        if (!loggedIn) {
-          where.push("COALESCE(tt_seller, 0) != 1");
-        }
-        if (requireVideo) {
-          where.push("COALESCE(video_count, 0) > 0");
-        }
-        if (normalizedLocations.length > 0) {
-          where.push(
-            `(UPPER(COALESCE(guessed_location, '')) IN (${normalizedLocations.map(() => "?").join(", ")}) OR COALESCE(guessed_location, '') = '')`,
-          );
-          args.push(...normalizedLocations);
-        }
-        return getDb()
-          .prepare(
-            `
-            SELECT *
-            FROM jobs
-            WHERE ${where.join(" AND ")}
-            ORDER BY created_at ASC, unique_id ASC
-            LIMIT 1
-          `,
-          )
-          .get(...args);
-      }
+      const addPart = (prio, subPrio, conditions, params) => {
+        sqlParts.push(
+          `SELECT *, ${prio} as _prio, ${subPrio} as _sub_prio FROM jobs WHERE ${conditions}`,
+        );
+        allParams.push(...params);
+      };
 
-      function findPrioritizedPending(requireVideo) {
-        for (const group of locationGroups) {
-          const seed = queryPendingByGroup({
-            requireVideo,
-            group,
-            filters: [
-              "COALESCE(pinned, 0) = 0",
-              `instr(COALESCE(sources, ''), '"seed"') > 0`,
-            ],
-          });
-          if (seed) return seed;
-        }
+      // prio 0: pinned + hasVideo
+      addPart(
+        0,
+        0,
+        `status = 'pending' AND pinned = 1 AND video_count > 0 ${pinnedLocSql} AND ${statusCodeWhere} ${notSellerWhere}`,
+        pinnedLocParams,
+      );
+      // prio 1: pinned (no video req)
+      addPart(
+        1,
+        0,
+        `status = 'pending' AND pinned = 1 ${pinnedLocSql} AND ${statusCodeWhere} ${notSellerWhere}`,
+        pinnedLocParams,
+      );
 
-        if (loggedIn) {
-          for (const group of locationGroups) {
-            const seller = queryPendingByGroup({
-              requireVideo,
-              group,
-              filters: [
-                "COALESCE(pinned, 0) = 0",
-                "tt_seller = 1",
-                "verified = 0",
-              ],
-            });
-            if (seller) return seller;
-          }
-        }
+      // prio 2-3: expired candidate（回收的超时任务优先）
+      if (expiredCandidate) {
+        addPart(
+          2,
+          0,
+          `unique_id = ? AND video_count > 0 AND ${statusCodeWhere} ${notSellerWhere}`,
+          [expiredCandidate.unique_id],
+        );
+        addPart(
+          3,
+          0,
+          `unique_id = ? AND ${statusCodeWhere} ${notSellerWhere}`,
+          [expiredCandidate.unique_id],
+        );
+      }
 
-        for (const group of locationGroups) {
-          const follow = queryPendingByGroup({
-            requireVideo,
-            group,
-            filters: [
-              "COALESCE(pinned, 0) = 0",
-              `(
-                instr(COALESCE(sources, ''), '"following"') > 0
-                OR instr(COALESCE(sources, ''), '"follower"') > 0
-                OR instr(COALESCE(sources, ''), '"comment"') > 0
-              )`,
-            ],
-          });
-          if (follow) return follow;
-        }
+      // 非 pinned 优先级：seed → seller → follow → other，每级 × 3 个 location tier × (hasVideo / noVideoReq)
+      const priorities = [
+        {
+          filter: `instr(COALESCE(sources, ''), '"seed"') > 0`,
+          loggedInOnly: false,
+        },
+        { filter: `tt_seller = 1 AND verified = 0`, loggedInOnly: true },
+        {
+          filter: `(instr(COALESCE(sources, ''), '"following"') > 0 OR instr(COALESCE(sources, ''), '"follower"') > 0 OR instr(COALESCE(sources, ''), '"comment"') > 0)`,
+          loggedInOnly: false,
+        },
+        { filter: `1 = 1`, loggedInOnly: false },
+      ];
 
-        for (const group of locationGroups) {
-          const other = queryPendingByGroup({
-            requireVideo,
-            group,
-            filters: ["COALESCE(pinned, 0) = 0"],
-          });
-          if (other) return other;
+      let basePrio = 4;
+      for (const p of priorities) {
+        if (p.loggedInOnly && !loggedIn) continue;
+        const tierLocations = [
+          { list: tier1List, fn: locIn },
+          { list: tier2List, fn: locIn },
+          { list: null, fn: locOther },
+        ];
+        let subPrio = 0;
+        for (const tl of tierLocations) {
+          const loc = tl.fn(tl.list);
+          if (!loc) continue;
+          // hasVideo
+          addPart(
+            basePrio,
+            subPrio,
+            `status = 'pending' AND pinned = 0 AND ${p.filter} AND video_count > 0 AND ${loc.sql} AND ${statusCodeWhere} ${notSellerWhere}`,
+            loc.params,
+          );
+          subPrio++;
+          // noVideoReq
+          addPart(
+            basePrio + 1,
+            subPrio,
+            `status = 'pending' AND pinned = 0 AND ${p.filter} AND ${loc.sql} AND ${statusCodeWhere} ${notSellerWhere}`,
+            loc.params,
+          );
+          subPrio++;
         }
+        basePrio += 2;
+      }
 
-        return null;
+      let selectedRow = null;
+      if (sqlParts.length > 0) {
+        const unionSql =
+          sqlParts.join(" UNION ALL ") +
+          " ORDER BY _prio, _sub_prio, follower_count DESC, created_at ASC, unique_id ASC LIMIT 1";
+        selectedRow = getDb()
+          .prepare(unionSql)
+          .get(...allParams);
       }
 
-      function claimRow(row) {
-        if (!row) return null;
+      if (selectedRow) {
         getDb()
           .prepare(
             "UPDATE jobs SET status = 'processing', claimed_at = ?, claimed_by = ? WHERE unique_id = ?",
           )
-          .run(now, userId, row.unique_id);
+          .run(now, userId, selectedRow.unique_id);
         markStatsDirty();
         return {
-          uniqueId: row.unique_id,
-          nickname: row.nickname,
+          uniqueId: selectedRow.unique_id,
+          nickname: selectedRow.nickname,
           claimedAt: now,
           claimedBy: userId,
         };
       }
 
-      const expiredRow = getDb()
-        .prepare(
-          `
-          SELECT *
-          FROM jobs
-          WHERE status = 'processing'
-            AND claimed_at IS NOT NULL
-            AND ? - claimed_at > ?
-          ORDER BY claimed_at ASC
-          LIMIT 1
-        `,
-        )
-        .get(now, expireMs);
-      let expiredCandidate = null;
-      if (expiredRow) {
-        getDb()
-          .prepare(
-            "UPDATE jobs SET status = 'pending', claimed_at = NULL WHERE unique_id = ?",
-          )
-          .run(expiredRow.unique_id);
-        expiredCandidate = mapJobRow({
-          ...expiredRow,
-          status: "pending",
-          claimed_at: null,
-        });
-      }
-
-      for (const requireVideo of [true, false]) {
-        const pinned = findPinnedPending(requireVideo);
-        if (pinned) {
-          return claimRow(pinned);
-        }
-        if (expiredCandidate) {
-          return claimRow({
-            unique_id: expiredCandidate.uniqueId,
-            nickname: expiredCandidate.nickname,
-          });
-        }
-        const ranked = findPrioritizedPending(requireVideo);
-        if (ranked) {
-          return claimRow(ranked);
-        }
-      }
-      // 尝试从 raw_jobs 毛料库补充任务（使用 createStore 时配置的 LLM 打分）
-      // 使用锁防止多个请求同时触发 LLM refill
-      if (refillLock) {
-        // 已有 refill 在进行中，等待完成后重新尝试领取
-        await refillLock;
-        for (const requireVideo of [true, false]) {
-          const pinned = findPinnedPending(requireVideo);
-          if (pinned) {
-            return claimRow(pinned);
-          }
-          const ranked = findPrioritizedPending(requireVideo);
-          if (ranked) {
-            return claimRow(ranked);
-          }
-        }
-        return null;
-      }
-      const refillPromise = (async () => {
-        const result = refillJobsFromRaw(
-          normalizedLocations.length ? normalizedLocations : null,
-          500,
-          refillLlmConfig,
-        );
-        // refillJobsFromRaw 在 LLM 模式下返回 Promise
-        if (result && typeof result.then === "function") {
-          return await result;
-        }
-        return result;
-      })();
-      // 让并发请求等待同一个 refill，并且无论成功/失败都释放锁
-      refillLock = refillPromise.finally(() => {
-        refillLock = null;
-      });
-
-      const refillResult = await refillLock;
-      if (refillResult.moved > 0) {
-        console.error(
-          `[data-store] 从 raw_jobs 补充了 ${refillResult.moved} 条任务到 jobs`,
-        );
-        for (const requireVideo of [true, false]) {
-          const pinned = findPinnedPending(requireVideo);
-          if (pinned) {
-            return claimRow(pinned);
-          }
-          const ranked = findPrioritizedPending(requireVideo);
-          if (ranked) {
-            return claimRow(ranked);
+      // refill 异步化：不阻塞当前请求
+      if (!refillLock) {
+        refillLock = (async () => {
+          try {
+            const result = refillJobsFromRaw(
+              normalizedLocations.length ? normalizedLocations : null,
+              500,
+              refillLlmConfig,
+            );
+            const r =
+              result && typeof result.then === "function"
+                ? await result
+                : result;
+            if (r.moved > 0) {
+              console.error(
+                `[data-store] 从 raw_jobs 补充了 ${r.moved} 条任务到 jobs`,
+              );
+            }
+          } catch (e) {
+            console.error(`[data-store] refill 异常: ${e.message}`);
+          } finally {
+            refillLock = null;
           }
-        }
+        })();
       }
 
       return null;
@@ -2414,6 +2383,7 @@ export function createStore(filePath, options = {}) {
         "uniqueId",
         "sources",
         "topRecentVideo", // 单独处理，不进入通用循环
+        "recentEcVideos", // 单独处理，不进入通用循环
       ];
       for (const key of Object.keys(result)) {
         if (extraFields.includes(key)) continue;
@@ -2431,6 +2401,15 @@ export function createStore(filePath, options = {}) {
         user.topVideoHref = result.topRecentVideo.href || null;
         user.topVideoIsAd = result.topRecentVideo.isAd ?? null;
       }
+      // 将 recentEcVideos 数组序列化为 JSON 字符串
+      if (
+        Array.isArray(result.recentEcVideos) &&
+        result.recentEcVideos.length > 0
+      ) {
+        user.recentEcVideos = JSON.stringify(result.recentEcVideos);
+      } else if (result.recentEcVideos === null) {
+        user.recentEcVideos = null;
+      }
       user.sources = [...new Set([...(user.sources || []), "processed"])];
     }
     if (user.status !== oldStatus) markStatsDirty();
@@ -2658,6 +2637,15 @@ export function createStore(filePath, options = {}) {
         user.topVideoHref = result.topRecentVideo.href || null;
         user.topVideoIsAd = result.topRecentVideo.isAd ?? null;
       }
+      // 将 recentEcVideos 数组序列化为 JSON 字符串
+      if (
+        Array.isArray(result.recentEcVideos) &&
+        result.recentEcVideos.length > 0
+      ) {
+        user.recentEcVideos = JSON.stringify(result.recentEcVideos);
+      } else if (result.recentEcVideos === null) {
+        user.recentEcVideos = null;
+      }
       const newUsers = processDiscoveredUsers(result);
       const ret = updateJobInfo(uniqueId, user, false);
       if (ret.error) return { saved: false, error: ret.error };
@@ -2684,6 +2672,15 @@ export function createStore(filePath, options = {}) {
       user.topVideoHref = result.topRecentVideo.href || null;
       user.topVideoIsAd = result.topRecentVideo.isAd ?? null;
     }
+    // 将 recentEcVideos 数组序列化为 JSON 字符串
+    if (
+      Array.isArray(result.recentEcVideos) &&
+      result.recentEcVideos.length > 0
+    ) {
+      user.recentEcVideos = JSON.stringify(result.recentEcVideos);
+    } else if (result.recentEcVideos === null) {
+      user.recentEcVideos = null;
+    }
     const newUsers = processDiscoveredUsers(result);
     return { saved: true, newUsers };
   }

package/src/watch/db-columns.js

@@ -49,6 +49,7 @@ export const JOB_EXT_COLUMNS = [
   "top_video_play_count",
   "top_video_href",
   "top_video_is_ad",
+  "recent_ec_videos",
 ];
 
 // 完整的 jobs 列 = 核心列 + 扩展列
@@ -91,6 +92,7 @@ export const WRITABLE_JOB_COLUMNS = new Set([
   "top_video_href",
   "top_video_is_ad",
   "user_create_time",
+  "recent_ec_videos",
 ]);
 
 // ===== Boolean 列（存储时 0/1，读取时转为 true/false/null） =====

package/src/watch/db-schema.js

@@ -129,6 +129,7 @@ export function initDb(filePath) {
     top_video_href: "TEXT",
     top_video_is_ad: "INTEGER",
     user_create_time: "INTEGER",
+    recent_ec_videos: "TEXT",
   });
 
   // ===== jobs_base 表 =====
@@ -424,6 +425,7 @@ export function initDb(filePath) {
     `CREATE INDEX IF NOT EXISTS idx_cross_country_source ON cross_country_learnings(source_country, result_status)`,
     `CREATE INDEX IF NOT EXISTS idx_cross_country_target ON cross_country_learnings(target_country)`,
     `CREATE INDEX IF NOT EXISTS idx_tags_dead_reason ON tags(dead_reason)`,
+    `CREATE INDEX IF NOT EXISTS idx_jobs_recheck ON jobs(status, updated_at) WHERE status = 'done' AND (tt_seller IS NULL OR tt_seller != 1)`,
   ];
   for (const sql of indexes) {
     db.exec(sql);