tt-help-cli-ycl 1.3.80 → 1.3.82

package/src/watch/data-store.js

@@ -1,7 +1,10 @@
 import fs from "fs";
 import path from "path";
 import Database from "better-sqlite3";
-import { isLocationInList } from "../lib/target-locations.js";
+import {
+  isLocationInList,
+  DEFAULT_TARGET_LOCATIONS,
+} from "../lib/target-locations.js";
 
 // SQLite 用户表（用于判重）
 let db = null;
@@ -143,6 +146,12 @@ function initUserDb(filePath) {
   if (!existingJobColumns.has("bio_link")) {
     db.exec(`ALTER TABLE jobs ADD COLUMN bio_link TEXT`);
   }
+  if (!existingJobColumns.has("top_video_play_count")) {
+    db.exec(`ALTER TABLE jobs ADD COLUMN top_video_play_count INTEGER`);
+  }
+  if (!existingJobColumns.has("top_video_href")) {
+    db.exec(`ALTER TABLE jobs ADD COLUMN top_video_href TEXT`);
+  }
   db.exec(`
     CREATE TABLE IF NOT EXISTS jobs_base (
       unique_id TEXT PRIMARY KEY,
@@ -650,13 +659,24 @@ function getDashboardStatsFromDb(targetLocations = []) {
             AND instr(COALESCE(sources, ''), '"guess"') = 0
             AND instr(COALESCE(sources, ''), '"following"') = 0
             AND instr(COALESCE(sources, ''), '"follower"') = 0
-          THEN 1 ELSE 0 END) as seed,
-        SUM(CASE WHEN COALESCE(tt_seller, '') = '' AND COALESCE(user_update_count, 0) <= 0 THEN 1 ELSE 0 END) as userUpdateTasks
+          THEN 1 ELSE 0 END) as seed
       FROM jobs
     `,
     )
     .get(...targetParams);
 
+  // userUpdateTasks 单独从 jobs_base 统计
+  const userUpdateTasksRow = db
+    .prepare(
+      `
+      SELECT COUNT(*) as userUpdateTasks
+      FROM jobs_base
+      WHERE COALESCE(tt_seller, '') = ''
+        AND COALESCE(user_update_count, 0) <= 0
+    `,
+    )
+    .get();
+
   // countryStats 和 targetCountryStats 需要 GROUP BY，保留为独立查询
   const countryStats = db
     .prepare(
@@ -712,7 +732,7 @@ function getDashboardStatsFromDb(targetLocations = []) {
     restrictedUsers: aggregateRow.restricted,
     errorUsers: aggregateRow.error,
     targetUsers: aggregateRow.targetUsers,
-    userUpdateTasks: aggregateRow.userUpdateTasks,
+    userUpdateTasks: userUpdateTasksRow.userUpdateTasks,
     targetCountryStats,
     countryStats,
     sourceStats: {
@@ -761,8 +781,8 @@ function getUserUpdateByCountryFromDb() {
       SELECT
         COALESCE(guessed_location, '未知') as country,
         COUNT(*) as count
-      FROM jobs
-      WHERE COALESCE(tt_seller, '') = ''
+      FROM jobs_base
+      WHERE tt_seller IS NULL
         AND COALESCE(user_update_count, 0) <= 0
       GROUP BY COALESCE(guessed_location, '未知')
       ORDER BY count DESC
@@ -782,8 +802,8 @@ function getAttachStuckByCountryFromDb() {
       SELECT
         COALESCE(guessed_location, '未知') as country,
         COUNT(*) as count
-      FROM jobs
-      WHERE COALESCE(tt_seller, '') = ''
+      FROM jobs_base
+      WHERE tt_seller IS NULL
         AND COALESCE(user_update_count, 0) = 1
       GROUP BY COALESCE(guessed_location, '未知')
       ORDER BY count DESC
@@ -816,7 +836,7 @@ function restoreAttachStuckByCountry(country) {
       .prepare(
         `
         SELECT COUNT(*) as c
-        FROM jobs
+        FROM jobs_base
         WHERE ${whereSql}
       `,
       )
@@ -828,7 +848,7 @@ function restoreAttachStuckByCountry(country) {
 
   db.prepare(
     `
-    UPDATE jobs
+    UPDATE jobs_base
     SET user_update_count = 0,
         updated_at = ?,
         claimed_by = NULL,
@@ -920,11 +940,36 @@ function moveJobsToRawByCountry(scope, country) {
     };
   }
 
+  // pending 操作 jobs 表（与 getPendingByCountryFromDb 数据源一致）
+  // userUpdate 操作 jobs_base 表（与 getUserUpdateByCountryFromDb 数据源一致）
+  let sourceTable = "";
   let scopeWhere = "";
+  let columns = "";
+
   if (normalizedScope === "pending") {
-    scopeWhere = `status = 'pending' AND COALESCE(user_update_count, 0) >= 2`;
+    sourceTable = "jobs";
+    scopeWhere = `status = 'pending'`;
+    columns = `
+      unique_id, nickname, status, sources, claimed_by, claimed_at,
+      error, pinned, no_video, restricted, user_update_count,
+      tt_seller, verified, video_count, comment_count,
+      guessed_location, location_created, follower_count,
+      following_count, heart_count, refresh_time, processed,
+      processed_at, created_at, updated_at, region, signature,
+      sec_uid, latest_video_time
+    `;
   } else if (normalizedScope === "userUpdate") {
-    scopeWhere = `COALESCE(tt_seller, '') = '' AND COALESCE(user_update_count, 0) <= 0`;
+    sourceTable = "jobs_base";
+    scopeWhere = `tt_seller IS NULL AND COALESCE(user_update_count, 0) <= 0`;
+    columns = `
+      unique_id, nickname, status, sources, claimed_by, claimed_at,
+      error, pinned, no_video, restricted, user_update_count,
+      tt_seller, verified, video_count, comment_count,
+      guessed_location, location_created, follower_count,
+      following_count, heart_count, refresh_time, processed,
+      processed_at, created_at, updated_at, region, signature,
+      sec_uid, latest_video_time
+    `;
   } else {
     return {
       moved: 0,
@@ -943,7 +988,7 @@ function moveJobsToRawByCountry(scope, country) {
       .prepare(
         `
         SELECT COUNT(*) as c
-        FROM jobs
+        FROM ${sourceTable}
         WHERE ${whereSql}
       `,
       )
@@ -957,74 +1002,18 @@ function moveJobsToRawByCountry(scope, country) {
     db.prepare(
       `
       INSERT OR REPLACE INTO raw_jobs (
-        unique_id,
-        nickname,
-        status,
-        sources,
-        claimed_by,
-        claimed_at,
-        error,
-        pinned,
-        no_video,
-        restricted,
-        user_update_count,
-        tt_seller,
-        verified,
-        video_count,
-        comment_count,
-        guessed_location,
-        location_created,
-        follower_count,
-        following_count,
-        heart_count,
-        refresh_time,
-        processed,
-        processed_at,
-        created_at,
-        updated_at,
-        region,
-        signature,
-        sec_uid,
-        latest_video_time
+        ${columns}
       )
       SELECT
-        unique_id,
-        nickname,
-        status,
-        sources,
-        claimed_by,
-        claimed_at,
-        error,
-        pinned,
-        no_video,
-        restricted,
-        user_update_count,
-        tt_seller,
-        verified,
-        video_count,
-        comment_count,
-        guessed_location,
-        location_created,
-        follower_count,
-        following_count,
-        heart_count,
-        refresh_time,
-        processed,
-        processed_at,
-        created_at,
-        updated_at,
-        region,
-        signature,
-        sec_uid,
-        latest_video_time
-      FROM jobs
+        ${columns}
+      FROM ${sourceTable}
       WHERE ${whereSql}
     `,
     ).run(targetCountry);
 
     db.prepare(
       `
-      DELETE FROM jobs
+      DELETE FROM ${sourceTable}
       WHERE ${whereSql}
     `,
     ).run(targetCountry);
@@ -1527,6 +1516,8 @@ function getTargetUsersByCountryFromDb(targetLocations = [], options = {}) {
         modified_at,
         latest_video_time,
         refresh_time,
+        top_video_play_count,
+        top_video_href,
         status,
         sources
       FROM jobs
@@ -1675,6 +1666,8 @@ const writableJobColumns = new Set([
   "sec_uid",
   "status_code",
   "latest_video_time",
+  "top_video_play_count",
+  "top_video_href",
 ]);
 
 function normalizeJobValue(column, value) {
@@ -1718,6 +1711,13 @@ function getJobRow(uniqueId) {
   return db.prepare("SELECT * FROM jobs WHERE unique_id = ?").get(uniqueId);
 }
 
+function getJobBaseRow(uniqueId) {
+  if (!db) return null;
+  return db
+    .prepare("SELECT * FROM jobs_base WHERE unique_id = ?")
+    .get(uniqueId);
+}
+
 function getJob(uniqueId) {
   return mapJobRow(getJobRow(uniqueId));
 }
@@ -1795,6 +1795,43 @@ function inferStatus(u) {
   return "pending";
 }
 
+function updateJobBaseInfo(uniqueId, info, incrementCount = true) {
+  if (!db) return { error: "db not initialized" };
+  const existing = getJobBaseRow(uniqueId);
+  if (!existing) return { error: "user not found" };
+
+  const nextValues = {};
+  for (const [key, value] of Object.entries(info || {})) {
+    if (key === "uniqueId" || key === "unique_id") continue;
+    if (value === undefined || value === "") continue;
+    let column = camelToSnake(key);
+    // 字段别名：bio → signature
+    if (column === "bio") column = "signature";
+    if (!writableJobColumns.has(column)) continue;
+    nextValues[column] = normalizeJobValue(column, value);
+  }
+
+  nextValues.updated_at = Date.now();
+  if (incrementCount) {
+    nextValues.user_update_count = (existing.user_update_count || 0) + 1;
+  }
+
+  const columns = Object.keys(nextValues);
+  if (columns.length > 0) {
+    const sql = `UPDATE jobs_base SET ${columns.map((column) => `${column} = ?`).join(", ")} WHERE unique_id = ?`;
+    db.prepare(sql).run(
+      ...columns.map((column) => nextValues[column]),
+      uniqueId,
+    );
+  }
+
+  return {
+    ok: true,
+    userUpdateCount:
+      nextValues.user_update_count ?? existing.user_update_count ?? 0,
+  };
+}
+
 function addJobBaseToDb(user) {
   if (!db) return;
   const now = Date.now();
@@ -1895,16 +1932,30 @@ function addJob(user) {
   writeTxn(user);
 }
 
-export function createStore(filePath) {
+export function createStore(filePath, options = {}) {
   if (!filePath) {
     throw new Error("createStore requires an explicit .db path");
   }
+
+  // refillJobsFromRaw 的 LLM 打分配置（自动补充任务时使用）
+  const refillLlmConfig = {
+    llmScore: false,
+    llmMinScore: 60,
+    llmSampleSize: 100,
+    ...options.refillLlm,
+  };
+
   let data = [];
   // uniqueId → index 内存索引，O(1) 查找
   let uidIndex = new Map();
   let clientErrors = new Map();
   // 客户端登录状态：userId → boolean
   let clientLoginStatus = new Map();
+  // refill 锁：防止多个 claimNextJob 同时触发 LLM refill
+  let refillLock = null; // Promise | null
+  // LLM 采样偏移量记忆：按猜测国家记录上次查询位置，避免重复采样
+  // 格式: { "ES": 300, "PL": 500, "NL": 400 }
+  let llmSampleOffsets = new Map();
   if (filePath) {
     // 初始化 SQLite 用户表（用于判重）
     initUserDb(filePath);
@@ -2162,7 +2213,439 @@ export function createStore(filePath) {
     return data;
   }
 
-  function claimNextJob(
+  /**
+   * 使用 LLM 对单个 job 的国家匹配度打分（0-100）
+   * @param {Object} job - raw_jobs 中的一条记录
+   * @param {string[]} targetLocations - 目标国家列表
+   * @returns {Promise<{ uniqueId: string, score: number, reason: string }>}
+   */
+  async function scoreJobLocation(job, targetLocations) {
+    const { fetch: undiciFetch } = await import("undici");
+
+    const prompt = `
+你是一个 TikTok 用户数据分析助手。请根据以下用户信息，判断该用户是否来自以下**任意一个**目标国家。
+
+目标国家列表: ${targetLocations.join(", ")}
+
+重要:
+- 用户只要来自上述**任意一个**国家就算匹配。
+- guessed_location 是系统初步猜测的结果，**仅供参考**，不要完全依赖它。
+- 请综合用户名、昵称、签名、位置等信息做判断。
+
+用户信息:
+- 用户名: ${job.unique_id || "未知"}
+- 昵称: ${job.nickname || "未知"}
+- 签名: ${job.signature || "未知"}
+- 地区: ${job.region || "未知"}
+- 猜测国家(参考): ${job.guessed_location || "未知"}
+- 位置信息: ${job.location_created || "未知"}
+- 主页链接: ${job.bio_link || "未知"}
+
+返回 JSON（仅返回 JSON，无其他内容）:
+{"score": 0-100, "reason": "English only, under 50 chars, no quotes/brackets"}
+
+Standards: 90-100=clear match, 70-89=likely, 50-69=possible, 20-49=low, 0-19=unlikely
+`;
+
+    try {
+      const apiKey = process.env.APIKEY || "";
+      const response = await undiciFetch(
+        "http://82.156.52.214:18000/v1/chat/completions",
+        {
+          method: "POST",
+          headers: {
+            "Content-Type": "application/json",
+            Authorization: `Bearer ${apiKey}`,
+          },
+          body: JSON.stringify({
+            model: "zc-fast",
+            messages: [{ role: "user", content: prompt }],
+            max_tokens: 512,
+            temperature: 0.1,
+          }),
+        },
+      );
+
+      const result = await response.json();
+      const content = result.choices?.[0]?.message?.content || "";
+
+      // 解析 JSON 响应（多层容错）
+      let parsed = null;
+
+      // 尝试 1: 直接解析
+      try {
+        parsed = JSON.parse(content);
+      } catch {
+        // 尝试 2: 提取 {} 包裹的内容
+        const match = content.match(/\{[\s\S]*\}/);
+        if (match) {
+          try {
+            parsed = JSON.parse(match[0]);
+          } catch {
+            // 尝试 3: 清理常见问题后解析
+            const cleaned = match[0]
+              .replace(/"/g, '"') // 弯引号 → 直引号
+              .replace(/\s+/g, " ") // 多余空白
+              .trim();
+            try {
+              parsed = JSON.parse(cleaned);
+            } catch {
+              // 尝试 4: 从文本中提取 score 和 reason（reason 可能包含引号等特殊字符）
+              const scoreMatch = content.match(/"?score"?\s*:\s*(\d+)/i);
+              if (scoreMatch) {
+                let reason = "解析降级";
+                // 找 "reason": 的位置，取到最后一个 } 前的内容
+                const reasonKeyPos = content.search(/"?reason"?\s*:\s*"/i);
+                if (reasonKeyPos !== -1) {
+                  const afterKey = content.substring(reasonKeyPos);
+                  const colonPos = afterKey.indexOf(":");
+                  const valueStart = afterKey.indexOf('"', colonPos + 1) + 1;
+                  const rawValue = afterKey.substring(valueStart);
+                  // 取到原始 content 最后一个 } 前
+                  const lastBrace = content.lastIndexOf("}");
+                  const reasonEnd = lastBrace - reasonKeyPos - valueStart;
+                  if (reasonEnd > 0) {
+                    reason = rawValue.substring(0, reasonEnd).trim();
+                    // 去掉首尾的引号
+                    if (reason.startsWith('"')) reason = reason.substring(1);
+                    if (reason.endsWith('"'))
+                      reason = reason.substring(0, reason.length - 1);
+                  }
+                }
+                parsed = {
+                  score: parseInt(scoreMatch[1]) || 50,
+                  reason,
+                };
+              }
+            }
+          }
+        }
+
+        // 尝试 5: 如果以上都失败，用更宽松的正则提取
+        if (!parsed) {
+          const scoreMatch = content.match(/"score"\s*:\s*(\d+)/);
+          const reasonMatch = content.match(/"reason"\s*:\s*"([^"]*)"/);
+          if (scoreMatch) {
+            parsed = {
+              score: parseInt(scoreMatch[1]) || 50,
+              reason: reasonMatch ? reasonMatch[1] : "解析降级 - 宽松模式",
+            };
+          }
+        }
+      }
+
+      if (parsed && typeof parsed.score === "number") {
+        return {
+          uniqueId: job.unique_id,
+          score: Math.max(0, Math.min(100, parsed.score)),
+          reason: parsed.reason || "",
+        };
+      }
+
+      // 所有解析都失败，返回默认分
+      console.error(
+        `[scoreJobLocation] JSON 解析失败 (${job.unique_id}): ${content.substring(0, 100)}`,
+      );
+      return {
+        uniqueId: job.unique_id,
+        score: 50,
+        reason: "LLM 响应解析失败，使用默认分",
+      };
+    } catch (e) {
+      console.error(
+        `[scoreJobLocation] LLM 调用失败 (${job.unique_id}): ${e.message}`,
+      );
+      return {
+        uniqueId: job.unique_id,
+        score: 50,
+        reason: `LLM 调用异常: ${e.message}`,
+      };
+    }
+  }
+
+  /**
+   * 批量对 jobs 进行 LLM 国家匹配度打分
+   * @param {Object[]} jobs - raw_jobs 记录数组
+   * @param {string[]} targetLocations - 目标国家列表
+   * @param {number} batchSize - 每批处理数量（并发），默认 10
+   * @returns {Promise<Array<{ uniqueId: string, score: number, reason: string }>>}
+   */
+  async function scoreJobsBatch(jobs, targetLocations, batchSize = 10) {
+    const results = [];
+    for (let i = 0; i < jobs.length; i += batchSize) {
+      const batch = jobs.slice(i, i + batchSize);
+      const batchResults = await Promise.all(
+        batch.map((job) => scoreJobLocation(job, targetLocations)),
+      );
+      results.push(...batchResults);
+    }
+    return results;
+  }
+
+  /**
+   * 从 raw_jobs 中移动一批符合条件的任务到 jobs 表
+   * @param {string[]} locations - 目标国家列表（null 表示不限制）
+   * @param {number} limit - 每次移动的最大数量，默认 500
+   * @param {Object} options - 可选配置
+   * @param {boolean} options.llmScore - 是否启用 LLM 打分过滤，默认 false
+   * @param {number} options.llmMinScore - LLM 最低分数阈值，默认 60
+   * @param {number} options.llmSampleSize - LLM 打分的采样数量，默认 100
+   * @returns {{ moved: number }} 实际移动的数量
+   */
+  function refillJobsFromRaw(locations = null, limit = 500, options = {}) {
+    if (!db) {
+      return { moved: 0, error: "db not ready" };
+    }
+
+    const safeLimit = Math.max(1, Math.min(2000, parseInt(limit) || 500));
+    const normalizedLocations = locations
+      ? locations.map((loc) => String(loc).trim().toUpperCase()).filter(Boolean)
+      : null;
+
+    const useLlm = !!options.llmScore;
+    const llmMinScore = options.llmMinScore ?? 60;
+    const llmSampleSize = options.llmSampleSize ?? 100;
+
+    // 构建 WHERE 条件
+    const conditions = [
+      "COALESCE(video_count, 0) > 0",
+      "COALESCE(follower_count, 0) > 0",
+      "COALESCE(following_count, 0) > 0",
+    ];
+    const args = [];
+
+    if (normalizedLocations && normalizedLocations.length > 0) {
+      conditions.push(
+        `UPPER(COALESCE(guessed_location, '')) IN (${normalizedLocations.map(() => "?").join(", ")})`,
+      );
+      args.push(...normalizedLocations);
+    }
+
+    const whereSql = conditions.join(" AND ");
+
+    // 统计符合条件的数量
+    const count =
+      db
+        .prepare(`SELECT COUNT(*) as c FROM raw_jobs WHERE ${whereSql}`)
+        .get(...args)?.c || 0;
+
+    if (!count) {
+      return { moved: 0 };
+    }
+
+    // 如果启用 LLM 打分，先采样一批进行评分（累积模式：按猜测国家分组，使用偏移量记忆避免重复采样）
+    if (useLlm && normalizedLocations && normalizedLocations.length > 0) {
+      const llmMinReturn = options.llmMinReturn ?? 60; // 最少返回合格数
+      const maxBatches = options.llmMaxBatches ?? 10; // 最多采样轮次，防止无限循环
+
+      // 打印当前偏移量状态
+      const offsetSummary = Array.from(llmSampleOffsets.entries())
+        .map(([k, v]) => `${k}:${v}`)
+        .join(", ");
+      console.error(
+        `[data-store] LLM 打分开始: 符合条件 ${count} 条，每批 ${llmSampleSize} 条，最低分 ${llmMinScore}，最少返回 ${llmMinReturn} 条`,
+      );
+      if (offsetSummary) {
+        console.error(`[data-store] 偏移量记忆: ${offsetSummary}`);
+      }
+
+      // 返回 Promise，调用方需要 await
+      return (async () => {
+        const allQualified = [];
+        const allScores = [];
+
+        // 按猜测国家分组处理，每个国家使用独立的偏移量
+        const locationGroups = normalizedLocations;
+        let totalBatches = 0;
+
+        for (const location of locationGroups) {
+          // 获取该国家上次的偏移量
+          let offset = llmSampleOffsets.get(location) || 0;
+
+          // 查询该国家的总数量
+          const locationCountSql = `SELECT COUNT(*) as c FROM raw_jobs WHERE ${whereSql} AND guessed_location = ?`;
+          const locationArgs = [...args, location];
+          const locationCount =
+            db.prepare(locationCountSql).get(...locationArgs)?.c || 0;
+
+          if (locationCount === 0) {
+            console.error(
+              `[data-store] 国家 ${location}: raw_jobs 中无数据，跳过`,
+            );
+            continue;
+          }
+
+          // 如果偏移量超过总数，重置为 0（一轮结束，重新开始）
+          if (offset >= locationCount) {
+            offset = 0;
+            llmSampleOffsets.set(location, 0);
+          }
+
+          console.error(
+            `[data-store] 国家 ${location}: 共 ${locationCount} 条，从偏移量 ${offset} 开始`,
+          );
+
+          for (let batch = 0; batch < maxBatches; batch++) {
+            const remaining = locationCount - offset;
+            if (remaining <= 0) break;
+
+            const sampleLimit = Math.min(llmSampleSize, remaining);
+            const samples = db
+              .prepare(
+                `
+                SELECT * FROM raw_jobs WHERE ${whereSql} AND guessed_location = ?
+                ORDER BY created_at DESC
+                LIMIT ? OFFSET ?
+              `,
+              )
+              .all(...locationArgs, sampleLimit, offset);
+
+            if (samples.length === 0) break;
+
+            const scores = await scoreJobsBatch(
+              samples,
+              DEFAULT_TARGET_LOCATIONS,
+            );
+            const batchQualified = scores.filter((s) => s.score >= llmMinScore);
+
+            allScores.push(...scores);
+            allQualified.push(...batchQualified.map((s) => s.uniqueId));
+
+            totalBatches++;
+            console.error(
+              `[data-store] ${location} 第 ${batch + 1} 批: 采样 ${samples.length} 条，本批合格 ${batchQualified.length} 条，累计合格 ${allQualified.length} 条`,
+            );
+
+            // 更新偏移量记忆
+            offset += samples.length;
+            llmSampleOffsets.set(location, offset);
+
+            // 合格数已达到最小返回阈值，停止采样
+            if (allQualified.length >= llmMinReturn) break;
+          }
+
+          // 合格数已达到最小返回阈值，停止所有国家的采样
+          if (allQualified.length >= llmMinReturn) break;
+        }
+
+        // 按分数降序排序，取前 safeLimit 条
+        const qualifiedScores = allScores
+          .filter((s) => s.score >= llmMinScore)
+          .sort((a, b) => b.score - a.score)
+          .slice(0, safeLimit);
+        const qualified = qualifiedScores.map((s) => s.uniqueId);
+
+        if (!qualified.length) {
+          console.error(
+            `[data-store] LLM 打分后无符合条件的任务（阈值: ${llmMinScore}，共采样 ${allScores.length} 条）`,
+          );
+          return {
+            moved: 0,
+            scored: allScores.length,
+            qualified: 0,
+            scores: allScores,
+          };
+        }
+
+        // 移动符合条件的记录
+        const placeholders = qualified.map(() => "?").join(", ");
+        const moveTxn = db.transaction(() => {
+          db.prepare(
+            `
+            INSERT OR IGNORE INTO jobs (
+              unique_id, nickname, status, sources, pinned,
+              tt_seller, verified, video_count, comment_count,
+              guessed_location, location_created, confirmed_location,
+              follower_count, following_count, heart_count,
+              created_at, updated_at, region, signature, bio_link, sec_uid,
+              status_code, latest_video_time
+            )
+            SELECT
+              unique_id, nickname, 'pending', sources, pinned,
+              tt_seller, verified, video_count, comment_count,
+              guessed_location, location_created, confirmed_location,
+              follower_count, following_count, heart_count,
+              created_at, updated_at, region, signature, bio_link, sec_uid,
+              status_code, latest_video_time
+            FROM raw_jobs
+            WHERE unique_id IN (${placeholders})
+          `,
+          ).run(...qualified);
+
+          db.prepare(
+            `DELETE FROM raw_jobs WHERE unique_id IN (${placeholders})`,
+          ).run(...qualified);
+        });
+
+        moveTxn();
+        markStatsDirty();
+
+        // 打印最终偏移量状态
+        const finalOffsetSummary = Array.from(llmSampleOffsets.entries())
+          .map(([k, v]) => `${k}:${v}`)
+          .join(", ");
+        console.error(
+          `[data-store] LLM 打分完成: 共采样 ${allScores.length} 条，合格 ${qualified.length} 条，已移动到 jobs`,
+        );
+        console.error(`[data-store] 偏移量记忆更新: ${finalOffsetSummary}`);
+        const scoresDetail = allScores.map((s) => s);
+        return {
+          moved: qualified.length,
+          scored: allScores.length,
+          qualified: qualified.length,
+          scores: scoresDetail,
+        };
+      })();
+    }
+
+    // 常规移动：INSERT + DELETE 事务
+    const moveTxn = db.transaction(() => {
+      db.prepare(
+        `
+        INSERT OR IGNORE INTO jobs (
+          unique_id, nickname, status, sources, pinned,
+          tt_seller, verified, video_count, comment_count,
+          guessed_location, location_created, confirmed_location,
+          follower_count, following_count, heart_count,
+          created_at, updated_at, region, signature, bio_link, sec_uid,
+          status_code, latest_video_time
+        )
+        SELECT
+          unique_id, nickname, 'pending', sources, pinned,
+          tt_seller, verified, video_count, comment_count,
+          guessed_location, location_created, confirmed_location,
+          follower_count, following_count, heart_count,
+          created_at, updated_at, region, signature, bio_link, sec_uid,
+          status_code, latest_video_time
+        FROM raw_jobs
+        WHERE ${whereSql}
+        ORDER BY created_at DESC
+        LIMIT ?
+      `,
+      ).run(...args, safeLimit);
+
+      // 删除已移动的记录：用子查询匹配刚 INSERT 的 unique_id
+      db.prepare(
+        `
+        DELETE FROM raw_jobs
+        WHERE unique_id IN (
+          SELECT unique_id FROM raw_jobs
+          WHERE ${whereSql}
+          ORDER BY created_at DESC
+          LIMIT ?
+        )
+      `,
+      ).run(...args, safeLimit);
+    });
+
+    moveTxn();
+    markStatsDirty();
+
+    const actualMoved = Math.min(count, safeLimit);
+    return { moved: actualMoved };
+  }
+
+  async function claimNextJob(
     userId,
     expireMs = 5 * 60 * 1000,
     locations = null,
@@ -2438,6 +2921,71 @@ export function createStore(filePath) {
           return claimRow(ranked);
         }
       }
+      // 尝试从 raw_jobs 毛料库补充任务（使用 createStore 时配置的 LLM 打分）
+      // 使用锁防止多个请求同时触发 LLM refill
+      if (refillLock) {
+        // 已有 refill 在进行中，等待完成后重新尝试领取
+        await refillLock;
+        for (const requireVideo of [true, false]) {
+          const pinned = findPinnedPending(requireVideo);
+          if (pinned) {
+            return claimRow(pinned);
+          }
+          const ranked = findPrioritizedPending(requireVideo);
+          if (ranked) {
+            return claimRow(ranked);
+          }
+        }
+        return null;
+      }
+      const refillResult = (async () => {
+        refillLock = Promise.resolve(); // 占位
+        const result = refillJobsFromRaw(
+          normalizedLocations.length ? normalizedLocations : null,
+          500,
+          refillLlmConfig,
+        );
+        // refillJobsFromRaw 在 LLM 模式下返回 Promise
+        if (result && typeof result.then === "function") {
+          return result.finally(() => {
+            refillLock = null;
+          });
+        }
+        return result;
+      })();
+      if (refillResult && typeof refillResult.then === "function") {
+        const awaited = await refillResult;
+        if (awaited.moved > 0) {
+          console.error(
+            `[data-store] 从 raw_jobs 补充了 ${awaited.moved} 条任务到 jobs`,
+          );
+          for (const requireVideo of [true, false]) {
+            const pinned = findPinnedPending(requireVideo);
+            if (pinned) {
+              return claimRow(pinned);
+            }
+            const ranked = findPrioritizedPending(requireVideo);
+            if (ranked) {
+              return claimRow(ranked);
+            }
+          }
+        }
+      } else if (refillResult.moved > 0) {
+        console.error(
+          `[data-store] 从 raw_jobs 补充了 ${refillResult.moved} 条任务到 jobs`,
+        );
+        for (const requireVideo of [true, false]) {
+          const pinned = findPinnedPending(requireVideo);
+          if (pinned) {
+            return claimRow(pinned);
+          }
+          const ranked = findPrioritizedPending(requireVideo);
+          if (ranked) {
+            return claimRow(ranked);
+          }
+        }
+      }
+
       return null;
     }
 
@@ -3033,6 +3581,7 @@ export function createStore(filePath) {
         "discoveredFollowers",
         "uniqueId",
         "sources",
+        "topRecentVideo", // 单独处理，不进入通用循环
       ];
       for (const key of Object.keys(result)) {
         if (extraFields.includes(key)) continue;
@@ -3044,6 +3593,11 @@ export function createStore(filePath) {
           user[key] = result[key];
         }
       }
+      // 将 topRecentVideo 对象展开为扁平字段
+      if (result.topRecentVideo && typeof result.topRecentVideo === "object") {
+        user.topVideoPlayCount = result.topRecentVideo.playCount || null;
+        user.topVideoHref = result.topRecentVideo.href || null;
+      }
       user.sources = [...new Set([...(user.sources || []), "processed"])];
     }
     if (user.status !== oldStatus) markStatsDirty();
@@ -3285,6 +3839,11 @@ export function createStore(filePath) {
           }
         }
       }
+      // 将 topRecentVideo 对象展开为扁平字段
+      if (result.topRecentVideo && typeof result.topRecentVideo === "object") {
+        user.topVideoPlayCount = result.topRecentVideo.playCount || null;
+        user.topVideoHref = result.topRecentVideo.href || null;
+      }
       const newUsers = processDiscoveredUsers(result);
       const ret = updateJobInfo(uniqueId, user, false);
       if (ret.error) return { saved: false, error: ret.error };
@@ -3305,6 +3864,11 @@ export function createStore(filePath) {
         }
       }
     }
+    // 将 topRecentVideo 对象展开为扁平字段
+    if (result.topRecentVideo && typeof result.topRecentVideo === "object") {
+      user.topVideoPlayCount = result.topRecentVideo.playCount || null;
+      user.topVideoHref = result.topRecentVideo.href || null;
+    }
     const newUsers = processDiscoveredUsers(result);
     return { saved: true, newUsers };
   }
@@ -3375,7 +3939,7 @@ export function createStore(filePath) {
 
       let sql = `
         SELECT *
-        FROM jobs
+        FROM jobs_base
         WHERE COALESCE(tt_seller, '') = ''
           AND COALESCE(user_update_count, 0) <= 0
       `;
@@ -3395,7 +3959,7 @@ export function createStore(filePath) {
       const now = Date.now();
       const bumpStmt = db.prepare(
         `
-        UPDATE jobs
+        UPDATE jobs_base
         SET user_update_count = COALESCE(user_update_count, 0) + 1,
             updated_at = ?
         WHERE unique_id = ?
@@ -3526,7 +4090,8 @@ export function createStore(filePath) {
   function batchUpdateUserInfo(updates) {
     if (db) {
       const results = [];
-      const moveList = [];
+      const rawMoveList = [];
+      const sellerMoveList = [];
 
       const txn = db.transaction((items) => {
         items.forEach((item) => {
@@ -3536,13 +4101,13 @@ export function createStore(filePath) {
           let updateResult;
           if (info && info.error && info.statusCode !== undefined) {
             // 只更新 status_code，不更新其他字段
-            updateResult = updateJobInfo(
+            updateResult = updateJobBaseInfo(
               uniqueId,
               { statusCode: info.statusCode },
               true,
             );
           } else {
-            updateResult = updateJobInfo(uniqueId, info, true);
+            updateResult = updateJobBaseInfo(uniqueId, info, true);
           }
 
           if (updateResult.error) {
@@ -3550,34 +4115,66 @@ export function createStore(filePath) {
             return;
           }
 
-          // 检查 tt_seller：非商家则标记为需要移动到毛料表
-          const row = getJobRow(uniqueId);
+          // 检查 tt_seller：商家移到 jobs，非商家移到 raw_jobs
+          const row = getJobBaseRow(uniqueId);
           const ttSeller = row ? row.tt_seller : null;
           if (ttSeller) {
-            // 商家：保持当前逻辑
+            // 商家：标记移动到 jobs
             results.push({
               uniqueId,
               ok: true,
               userUpdateCount: updateResult.userUpdateCount,
+              _movedToJobs: true,
             });
+            sellerMoveList.push(uniqueId);
           } else {
-            // 非商家：标记移动
+            // 非商家：标记移动到 raw_jobs
             results.push({
               uniqueId,
               ok: true,
               userUpdateCount: updateResult.userUpdateCount,
               _movedToRaw: true,
             });
-            moveList.push(uniqueId);
+            rawMoveList.push(uniqueId);
           }
         });
       });
       txn(updates);
 
-      // 批量移动非商家用户到 raw_jobs（优化：一次 SQL 搞定）
-      if (moveList.length > 0) {
-        const placeholders = moveList.map(() => "?").join(",");
-        // 批量 INSERT 到 raw_jobs
+      // 批量移动商家用户到 jobs
+      if (sellerMoveList.length > 0) {
+        const placeholders = sellerMoveList.map(() => "?").join(",");
+        db.prepare(
+          `
+          INSERT OR REPLACE INTO jobs (
+            unique_id, nickname, status, sources, claimed_by, claimed_at,
+            error, pinned, no_video, restricted, user_update_count,
+            tt_seller, verified, video_count, comment_count,
+            guessed_location, location_created, confirmed_location, modified_at,
+            follower_count, following_count, heart_count, refresh_time,
+            processed, processed_at, created_at, updated_at,
+            region, signature, bio_link, sec_uid, status_code, latest_video_time
+          )
+          SELECT
+            unique_id, nickname, status, sources, claimed_by, claimed_at,
+            error, pinned, no_video, restricted, user_update_count,
+            tt_seller, verified, video_count, comment_count,
+            guessed_location, location_created, confirmed_location, modified_at,
+            follower_count, following_count, heart_count, refresh_time,
+            processed, processed_at, created_at, updated_at,
+            region, signature, bio_link, sec_uid, status_code, latest_video_time
+          FROM jobs_base WHERE unique_id IN (${placeholders})
+        `,
+        ).run(...sellerMoveList);
+
+        db.prepare(
+          `DELETE FROM jobs_base WHERE unique_id IN (${placeholders})`,
+        ).run(...sellerMoveList);
+      }
+
+      // 批量移动非商家用户到 raw_jobs
+      if (rawMoveList.length > 0) {
+        const placeholders = rawMoveList.map(() => "?").join(",");
         db.prepare(
           `
           INSERT OR REPLACE INTO raw_jobs (
@@ -3597,19 +4194,18 @@ export function createStore(filePath) {
             follower_count, following_count, heart_count, refresh_time,
             processed, processed_at, created_at, updated_at,
             region, signature, bio_link, sec_uid, status_code, latest_video_time
-          FROM jobs WHERE unique_id IN (${placeholders})
+          FROM jobs_base WHERE unique_id IN (${placeholders})
         `,
-        ).run(...moveList);
+        ).run(...rawMoveList);
 
-        // 批量 DELETE 从 jobs
-        db.prepare(`DELETE FROM jobs WHERE unique_id IN (${placeholders})`).run(
-          ...moveList,
-        );
+        db.prepare(
+          `DELETE FROM jobs_base WHERE unique_id IN (${placeholders})`,
+        ).run(...rawMoveList);
       }
 
       // 清理内部标记
       return results.map((r) => {
-        const { _movedToRaw, ...rest } = r;
+        const { _movedToRaw, _movedToJobs, ...rest } = r;
         return rest;
       });
     }
@@ -3879,6 +4475,9 @@ export function createStore(filePath) {
     getStats,
     getStatusGroups,
     markGroupsDirty,
+    refillJobsFromRaw,
+    scoreJobLocation,
+    scoreJobsBatch,
     claimNextJob,
     commitJob,
     commitNewExplore,
@@ -3904,6 +4503,12 @@ export function createStore(filePath) {
     debugClaimNextJob,
     stopBackup,
     rawQuery,
+    getLlmSampleOffsets, // 获取 LLM 采样偏移量状态
     data,
   };
+
+  // 辅助函数：获取 LLM 采样偏移量
+  function getLlmSampleOffsets() {
+    return Object.fromEntries(llmSampleOffsets);
+  }
 }