tt-help-cli-ycl 1.3.93 → 1.3.95

package/src/watch/db-tags.js

@@ -0,0 +1,348 @@
+/**
+ * Tag 发现与打分 CRUD
+ *
+ * 管理 tags 表：插入、查询、认领、打分、清理。
+ */
+
+import { getDb } from "./db-schema.js";
+
+function parseTagRow(r) {
+  return {
+    ...r,
+    countries: JSON.parse(r.countries || "[]"),
+    matched_countries: JSON.parse(r.matched_countries || "[]"),
+  };
+}
+
+export function insertTag(tag, countries, source = "llm") {
+  const db = getDb();
+  if (!db) return { inserted: false, error: "db not ready" };
+  const normalized = tag.replace(/^#+/, "").trim().toLowerCase();
+  if (!normalized || normalized.length < 2) {
+    return { inserted: false, error: "invalid tag" };
+  }
+  try {
+    const result = db
+      .prepare(
+        "INSERT OR IGNORE INTO tags (tag, countries, source) VALUES (?, ?, ?)",
+      )
+      .run(normalized, JSON.stringify(countries), source);
+    return { inserted: result.changes > 0, tag: normalized };
+  } catch (e) {
+    return { inserted: false, error: e.message };
+  }
+}
+
+export function getTagsByStatus(status, limit = 100) {
+  const db = getDb();
+  if (!db) return [];
+  const rows = db
+    .prepare(
+      "SELECT * FROM tags WHERE status = ? ORDER BY score ASC, created_at ASC LIMIT ?",
+    )
+    .all(status, limit);
+  return rows.map(parseTagRow);
+}
+
+export function getTagsByCountry(country, minScore = 0) {
+  const db = getDb();
+  if (!db) return [];
+  const rows = db
+    .prepare("SELECT * FROM tags WHERE status != 'dead' ORDER BY score DESC")
+    .all();
+  return rows
+    .map(parseTagRow)
+    .filter((r) => r.countries.includes(country) && r.score >= minScore);
+}
+
+export function getDeadTags(country) {
+  const db = getDb();
+  if (!db) return [];
+  const rows = db
+    .prepare("SELECT * FROM tags WHERE status = 'dead' ORDER BY score ASC")
+    .all();
+  return rows.map(parseTagRow).filter((r) => r.countries.includes(country));
+}
+
+export function claimTag(tag) {
+  const db = getDb();
+  if (!db) return { ok: false, error: "db not ready" };
+  const result = db
+    .prepare(
+      "UPDATE tags SET status = 'scoring' WHERE tag = ? AND status = 'new'",
+    )
+    .run(tag);
+  if (result.changes === 0) {
+    const row = db.prepare("SELECT status FROM tags WHERE tag = ?").get(tag);
+    if (!row) return { ok: false, error: "tag not found" };
+    return { ok: false, error: `tag status is ${row.status}, already claimed` };
+  }
+  return { ok: true, tag };
+}
+
+export function reportTagScore(tag, fields) {
+  const db = getDb();
+  if (!db) return { ok: false, error: "db not ready" };
+  const {
+    score,
+    status,
+    totalPosts,
+    authorCount,
+    matchedAuthors,
+    matchedCountries,
+    pushedUsers,
+    error,
+  } = fields;
+  const matchedCountriesJson = matchedCountries
+    ? JSON.stringify(matchedCountries)
+    : null;
+  const now = new Date().toISOString();
+
+  try {
+    const result = db
+      .prepare(
+        `UPDATE tags SET
+          score = COALESCE(?, score),
+          status = COALESCE(?, status),
+          total_posts = COALESCE(?, total_posts),
+          author_count = COALESCE(?, author_count),
+          matched_authors = COALESCE(?, matched_authors),
+          matched_countries = COALESCE(?, matched_countries),
+          pushed_users = COALESCE(?, pushed_users),
+          last_error = COALESCE(?, last_error),
+          scored_at = ?,
+          score_count = score_count + 1
+        WHERE tag = ?`,
+      )
+      .run(
+        score ?? null,
+        status ?? null,
+        totalPosts ?? null,
+        authorCount ?? null,
+        matchedAuthors ?? null,
+        matchedCountriesJson,
+        pushedUsers ?? null,
+        error ?? null,
+        now,
+        tag,
+      );
+    return { ok: result.changes > 0, tag };
+  } catch (e) {
+    return { ok: false, error: e.message };
+  }
+}
+
+export function getAllTags(limit = 200) {
+  const db = getDb();
+  if (!db) return [];
+  const rows = db
+    .prepare("SELECT * FROM tags ORDER BY score DESC, created_at DESC LIMIT ?")
+    .all(limit);
+  return rows.map(parseTagRow);
+}
+
+export function rawQuery(sql, params = []) {
+  const db = getDb();
+  if (!db) return { error: "db not ready" };
+  try {
+    const rows = db.prepare(sql).all(...params);
+    return { rows };
+  } catch (e) {
+    return { error: e.message };
+  }
+}
+
+export function normalizeTags() {
+  const db = getDb();
+  if (!db) return { ok: false, error: "db not ready" };
+  const dirtyRows = db
+    .prepare("SELECT id, tag, countries FROM tags WHERE tag LIKE '#%'")
+    .all();
+  const fixed = [];
+  const merged = [];
+  const skipped = [];
+
+  for (const row of dirtyRows) {
+    const cleanTag = row.tag.replace(/^#+/, "").trim().toLowerCase();
+    if (!cleanTag || cleanTag.length < 2) {
+      db.prepare("DELETE FROM tags WHERE id = ?").run(row.id);
+      skipped.push({
+        dirty: row.tag,
+        reason: "empty after normalize, deleted",
+      });
+      continue;
+    }
+
+    const existing = db
+      .prepare("SELECT * FROM tags WHERE tag = ?")
+      .get(cleanTag);
+    if (existing) {
+      const oldCountries = JSON.parse(row.countries || "[]");
+      const existCountries = JSON.parse(existing.countries || "[]");
+      const mergedCountries = [
+        ...new Set([...existCountries, ...oldCountries]),
+      ];
+      db.prepare("UPDATE tags SET countries = ? WHERE tag = ?").run(
+        JSON.stringify(mergedCountries),
+        cleanTag,
+      );
+      db.prepare("DELETE FROM tags WHERE id = ?").run(row.id);
+      merged.push({ dirty: row.tag, clean: cleanTag, id: row.id });
+    } else {
+      db.prepare("UPDATE tags SET tag = ? WHERE id = ?").run(cleanTag, row.id);
+      fixed.push({ dirty: row.tag, clean: cleanTag, id: row.id });
+    }
+  }
+
+  return {
+    ok: true,
+    fixed: fixed.length,
+    merged: merged.length,
+    skipped: skipped.length,
+    details: { fixed, merged, skipped },
+  };
+}
+
+export function clearTags() {
+  const db = getDb();
+  if (!db) return { ok: false, error: "db not ready" };
+  const count = db.prepare("SELECT COUNT(*) as c FROM tags").get().c;
+  db.exec("DELETE FROM tags");
+  return { ok: true, deleted: count };
+}
+
+// ===== discover_log CRUD =====
+
+/**
+ * 记录一次 LLM discover 的完整过程
+ */
+export function insertDiscoverLog({
+  country,
+  round,
+  strategy,
+  tagsGenerated,
+  tagsAdded,
+  productiveSample,
+  deadSample,
+  avgProductiveScore,
+  avgDeadScore,
+}) {
+  const db = getDb();
+  if (!db) return { ok: false, error: "db not ready" };
+
+  try {
+    const result = db
+      .prepare(
+        `INSERT INTO discover_log
+         (country, round, strategy, tags_generated, tags_added,
+          productive_sample, dead_sample, avg_productive_score, avg_dead_score)
+         VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+      )
+      .run(
+        country,
+        round,
+        strategy || null,
+        JSON.stringify(tagsGenerated || []),
+        tagsAdded || 0,
+        JSON.stringify(productiveSample || []),
+        JSON.stringify(deadSample || []),
+        avgProductiveScore || 0,
+        avgDeadScore || 0,
+      );
+    return { ok: true, id: result.lastInsertRowid };
+  } catch (e) {
+    return { ok: false, error: e.message };
+  }
+}
+
+/**
+ * 获取某国家最近的 discover 记录（用于自学习）
+ */
+export function getRecentDiscoverLogs(country, limit = 10) {
+  const db = getDb();
+  if (!db) return [];
+
+  return db
+    .prepare(
+      `SELECT * FROM discover_log
+       WHERE country = ?
+       ORDER BY round DESC
+       LIMIT ?`,
+    )
+    .all(country, limit)
+    .map((row) => ({
+      ...row,
+      tags_generated: JSON.parse(row.tags_generated || "[]"),
+      productive_sample: JSON.parse(row.productive_sample || "[]"),
+      dead_sample: JSON.parse(row.dead_sample || "[]"),
+    }));
+}
+
+/**
+ * 获取某国家的 discover 轮次（用于递增）
+ */
+export function getDiscoverRound(country) {
+  const db = getDb();
+  if (!db) return 1;
+
+  const row = db
+    .prepare(
+      `SELECT MAX(round) as maxRound FROM discover_log WHERE country = ?`,
+    )
+    .get(country);
+  return (row?.maxRound || 0) + 1;
+}
+
+/**
+ * 获取某国家 discover 产出的 tag 平均效果（用于策略评估）
+ */
+export function getDiscoverEffectiveness(country) {
+  const db = getDb();
+  if (!db) return null;
+
+  // 先获取 discover_log 记录，再逐轮计算 tag 效果
+  const logs = db
+    .prepare(
+      `SELECT round, strategy, tags_added, tags_generated FROM discover_log
+       WHERE country = ?
+       ORDER BY round DESC
+       LIMIT 5`,
+    )
+    .all(country);
+
+  if (!logs || logs.length === 0) return [];
+
+  // 逐轮计算效果
+  return logs.map((dl) => {
+    const tagsGenerated = JSON.parse(dl.tags_generated || "[]");
+    if (tagsGenerated.length === 0) {
+      return {
+        round: dl.round,
+        strategy: dl.strategy,
+        tags_added: dl.tags_added,
+        avg_score: 0,
+        productive_count: 0,
+        dead_count: 0,
+      };
+    }
+
+    // 用 IN 子句查询这些 tag 的得分
+    const placeholders = tagsGenerated.map(() => "?").join(",");
+    const tags = db
+      .prepare(`SELECT score, status FROM tags WHERE tag IN (${placeholders})`)
+      .all(...tagsGenerated);
+
+    const scores = tags.map((t) => t.score || 0);
+    return {
+      round: dl.round,
+      strategy: dl.strategy,
+      tags_added: dl.tags_added,
+      avg_score:
+        scores.length > 0
+          ? scores.reduce((a, b) => a + b, 0) / scores.length
+          : 0,
+      productive_count: tags.filter((t) => t.status === "productive").length,
+      dead_count: tags.filter((t) => t.status === "dead").length,
+    };
+  });
+}

package/src/watch/llm-scoring.js

@@ -0,0 +1,235 @@
+/**
+ * LLM 打分模块
+ *
+ * 使用外部 LLM API 对 TikTok 用户的国家匹配度打分。
+ * 包含：单条打分、批量打分、采样偏移量持久化。
+ */
+
+import { getDb } from "./db-schema.js";
+
+/**
+ * 使用 LLM 对单个 job 的国家匹配度打分（0-100）
+ * @param {Object} job - raw_jobs 中的一条记录
+ * @param {string[]} targetLocations - 目标国家列表
+ * @returns {Promise<{ uniqueId: string, score: number, reason: string }>}
+ */
+export async function scoreJobLocation(job, targetLocations) {
+  const { fetch: undiciFetch } = await import("undici");
+
+  const prompt = `
+你是一个 TikTok 用户数据分析助手。请根据以下用户信息，判断该用户是否来自以下**任意一个**目标国家。
+
+目标国家列表: ${targetLocations.join(", ")}
+
+重要:
+- 用户只要来自上述**任意一个**国家就算匹配。
+- guessed_location 是系统初步猜测的结果，**仅供参考**，不要完全依赖它。
+- 请综合用户名、昵称、签名、位置等信息做判断。
+
+用户信息:
+- 用户名: ${job.unique_id || "未知"}
+- 昵称: ${job.nickname || "未知"}
+- 签名: ${job.signature || "未知"}
+- 地区: ${job.region || "未知"}
+- 猜测国家(参考): ${job.guessed_location || "未知"}
+- 位置信息: ${job.location_created || "未知"}
+- 主页链接: ${job.bio_link || "未知"}
+
+返回 JSON（仅返回 JSON，无其他内容）:
+{"score": 0-100, "reason": "English only, under 50 chars, no quotes/brackets"}
+
+Standards: 90-100=clear match, 70-89=likely, 50-69=possible, 20-49=low, 0-19=unlikely
+`;
+
+  try {
+    const apiKey = process.env.APIKEY || "";
+    const response = await undiciFetch(
+      "http://82.156.52.214:18000/v1/chat/completions",
+      {
+        method: "POST",
+        headers: {
+          "Content-Type": "application/json",
+          Authorization: `Bearer ${apiKey}`,
+        },
+        body: JSON.stringify({
+          model: "zc-fast",
+          messages: [{ role: "user", content: prompt }],
+          max_tokens: 512,
+          temperature: 0.1,
+        }),
+      },
+    );
+
+    const result = await response.json();
+    const content = result.choices?.[0]?.message?.content || "";
+
+    let parsed = null;
+
+    // 尝试 1: 直接解析
+    try {
+      parsed = JSON.parse(content);
+    } catch {
+      // 尝试 2: 提取 {} 包裹的内容
+      const match = content.match(/\{[\s\S]*\}/);
+      if (match) {
+        try {
+          parsed = JSON.parse(match[0]);
+        } catch {
+          // 尝试 3: 清理常见问题后解析
+          const cleaned = match[0]
+            .replace(/"/g, '"')
+            .replace(/\s+/g, " ")
+            .trim();
+          try {
+            parsed = JSON.parse(cleaned);
+          } catch {
+            // 尝试 4: 从文本中提取 score 和 reason
+            const scoreMatch = content.match(/"?score"?\s*:\s*(\d+)/i);
+            if (scoreMatch) {
+              let reason = "解析降级";
+              const reasonKeyPos = content.search(/"?reason"?\s*:\s*"/i);
+              if (reasonKeyPos !== -1) {
+                const afterKey = content.substring(reasonKeyPos);
+                const colonPos = afterKey.indexOf(":");
+                const valueStart = afterKey.indexOf('"', colonPos + 1) + 1;
+                const rawValue = afterKey.substring(valueStart);
+                const lastBrace = content.lastIndexOf("}");
+                const reasonEnd = lastBrace - reasonKeyPos - valueStart;
+                if (reasonEnd > 0) {
+                  reason = rawValue.substring(0, reasonEnd).trim();
+                  if (reason.startsWith('"')) reason = reason.substring(1);
+                  if (reason.endsWith('"'))
+                    reason = reason.substring(0, reason.length - 1);
+                }
+              }
+              parsed = { score: parseInt(scoreMatch[1]) || 50, reason };
+            }
+          }
+        }
+      }
+
+      // 尝试 5: 更宽松的正则提取
+      if (!parsed) {
+        const scoreMatch = content.match(/"score"\s*:\s*(\d+)/);
+        const reasonMatch = content.match(/"reason"\s*:\s*"([^"]*)"/);
+        if (scoreMatch) {
+          parsed = {
+            score: parseInt(scoreMatch[1]) || 50,
+            reason: reasonMatch ? reasonMatch[1] : "解析降级 - 宽松模式",
+          };
+        }
+      }
+    }
+
+    if (parsed && typeof parsed.score === "number") {
+      return {
+        uniqueId: job.unique_id,
+        score: Math.max(0, Math.min(100, parsed.score)),
+        reason: parsed.reason || "",
+      };
+    }
+
+    console.error(
+      `[scoreJobLocation] JSON 解析失败 (${job.unique_id}): ${content.substring(0, 100)}`,
+    );
+    return {
+      uniqueId: job.unique_id,
+      score: 50,
+      reason: "LLM 响应解析失败，使用默认分",
+    };
+  } catch (e) {
+    console.error(
+      `[scoreJobLocation] LLM 调用失败 (${job.unique_id}): ${e.message}`,
+    );
+    return {
+      uniqueId: job.unique_id,
+      score: 50,
+      reason: `LLM 调用异常: ${e.message}`,
+    };
+  }
+}
+
+/**
+ * 批量对 jobs 进行 LLM 国家匹配度打分
+ * @param {Object[]} jobs - raw_jobs 记录数组
+ * @param {string[]} targetLocations - 目标国家列表
+ * @param {number} batchSize - 每批处理数量（并发），默认 10
+ * @returns {Promise<Array<{ uniqueId: string, score: number, reason: string }>>}
+ */
+export async function scoreJobsBatch(jobs, targetLocations, batchSize = 10) {
+  const results = [];
+  for (let i = 0; i < jobs.length; i += batchSize) {
+    const batch = jobs.slice(i, i + batchSize);
+    const batchResults = await Promise.all(
+      batch.map((job) => scoreJobLocation(job, targetLocations)),
+    );
+    results.push(...batchResults);
+  }
+  return results;
+}
+
+/**
+ * 创建 LLM 采样偏移量存储器
+ * 管理按国家记录的查询偏移量，支持持久化到数据库。
+ * @returns {{ load(): void, save(): void, get(key: string): number, set(key: string, val: number): void, entries(): Iterator }}
+ */
+export function createLlmOffsetStore() {
+  const offsets = new Map();
+
+  return {
+    load() {
+      const db = getDb();
+      if (!db) return;
+      try {
+        const row = db
+          .prepare("SELECT offsets FROM _llm_sample_offsets LIMIT 1")
+          .get();
+        if (row && row.offsets) {
+          const parsed = JSON.parse(row.offsets);
+          if (parsed && typeof parsed === "object") {
+            Object.entries(parsed).forEach(([k, v]) => offsets.set(k, v));
+            console.error(
+              `[data-store] 已恢复 LLM 采样偏移量: ${Array.from(
+                offsets.entries(),
+              )
+                .map(([k, v]) => `${k}:${v}`)
+                .join(", ")}`,
+            );
+          }
+        }
+      } catch (e) {
+        console.error(
+          `[data-store] 加载 LLM 采样偏移量失败，使用空偏移量: ${e.message}`,
+        );
+      }
+    },
+
+    save() {
+      const db = getDb();
+      if (!db) return;
+      try {
+        const offsetsJson = JSON.stringify(Object.fromEntries(offsets));
+        db.prepare(
+          `CREATE TABLE IF NOT EXISTS _llm_sample_offsets (id INTEGER PRIMARY KEY CHECK (id = 1), offsets TEXT)`,
+        ).run();
+        db.prepare(
+          `INSERT INTO _llm_sample_offsets (id, offsets) VALUES (1, ?) ON CONFLICT(id) DO UPDATE SET offsets = excluded.offsets`,
+        ).run(offsetsJson);
+      } catch (e) {
+        console.error(`[data-store] 保存 LLM 采样偏移量失败: ${e.message}`);
+      }
+    },
+
+    get(key) {
+      return offsets.get(key) || 0;
+    },
+
+    set(key, val) {
+      offsets.set(key, val);
+    },
+
+    entries() {
+      return offsets.entries();
+    },
+  };
+}

package/src/watch/public/app.js

@@ -1521,6 +1521,53 @@ function renderUserUpdateCountryGrid(countries) {
     .join("");
 }
 
+async function moveSellerJobsToBase() {
+  // 确认提示
+  if (
+    !confirm(
+      "确定要重处理商家吗？\n\n这将把 jobs 和 raw_jobs 中 ttSeller=1 且视频数=0 的记录移动到 jobs_base，并重置 user_update_count=0，使其可以被 attach 重新领取。",
+    )
+  ) {
+    return;
+  }
+
+  const card = document.getElementById("statSellerResetCard");
+  if (card) {
+    card.style.pointerEvents = "none";
+    card.style.opacity = "0.6";
+  }
+
+  try {
+    showLoading("正在移动商家数据到 jobs_base...");
+
+    const res = await fetch("/api/move-seller-jobs-to-base", {
+      method: "POST",
+    });
+    const data = await res.json();
+
+    if (data.ok) {
+      hideLoading();
+      showNotification(
+        `✅ 完成：从 jobs 移动 ${data.fromJobs} 条，从 raw_jobs 移动 ${data.fromRawJobs} 条，重置 ${data.resetCount} 条，jobs_base 中可领取 ${data.availableInBase} 条`,
+        "success",
+      );
+      // 刷新页面统计
+      await fetchStats();
+    } else {
+      hideLoading();
+      showNotification(`❌ 移动失败：${data.error || "未知错误"}`, "error");
+    }
+  } catch (e) {
+    hideLoading();
+    showNotification(`❌ 请求失败：${e.message}`, "error");
+  } finally {
+    if (card) {
+      card.style.pointerEvents = "";
+      card.style.opacity = "";
+    }
+  }
+}
+
 async function fetchAttachStuckByCountry() {
   try {
     const res = await fetch("/api/attach-stuck-by-country");

package/src/watch/public/index.html

@@ -210,6 +210,12 @@
         <div class="label">待补资料</div>
         <div class="value target" id="userUpdateStatUserUpdateTasks">0</div>
       </div>
+      <div class="stat-card clickable" id="statSellerResetCard" onclick="moveSellerJobsToBase()"
+        style="background:rgba(236,72,153,0.12);border:1px solid rgba(236,72,153,0.25)">
+        <div class="label">🔄 重处理商家</div>
+        <div class="value-sub" style="font-size:10px;color:#ec4899">将 jobs/raw_jobs 中 ttSeller=1 且视频数=0 的记录移到 jobs_base
+          重新处理</div>
+      </div>
       <div class="stat-card clickable" onclick="navigateToRaw()">
         <div class="label">毛料库</div>
         <div class="value target" id="userUpdateStatRawJobs">0</div>

package/src/watch/server.js

@@ -274,6 +274,30 @@ export function startWatchServer(
         return;
       }
 
+      // 将 jobs/raw_jobs 中 tt_seller=1 且 video_count=0 的记录移动到 jobs_base
+      if (
+        req.method === "POST" &&
+        routePath === "/api/move-seller-jobs-to-base"
+      ) {
+        try {
+          const result = store.moveSellerJobsToBase();
+          const ts = new Date().toISOString().slice(11, 19);
+          if (result.ok) {
+            console.error(
+              `[JOB ${ts}] MOVE-SELLER-JOBS: jobs=${result.fromJobs}, raw_jobs=${result.fromRawJobs}, reset=${result.resetCount}, available=${result.availableInBase}`,
+            );
+          } else {
+            console.error(
+              `[JOB ${ts}] MOVE-SELLER-JOBS ERROR: ${result.error}`,
+            );
+          }
+          sendJSON(res, result.ok ? 200 : 400, result);
+        } catch (e) {
+          sendJSON(res, 500, { error: e.message });
+        }
+        return;
+      }
+
       if (req.method === "GET" && routePath === "/api/db-query") {
         const sql = params.sql || "SELECT * FROM jobs LIMIT 10";
         const limit = Math.min(parseInt(params.limit) || 100, 1000);
@@ -1245,14 +1269,23 @@ export function startWatchServer(
       _resolve({ server, port });
     });
 
+    // 标记是否正在关闭，避免重复处理
+    let isShuttingDown = false;
+
     async function gracefulShutdown(signal) {
+      if (isShuttingDown) {
+        console.error("[server] 正在关闭中，请稍候...");
+        return;
+      }
+      isShuttingDown = true;
+
       console.error(`\n[server] 收到 ${signal}，正在保存数据...`);
       server.close(() => {
         console.error("[server] HTTP 服务已关闭");
       });
       await store.flushSave();
       console.error("[server] 数据已保存");
-      // 备份数据库
+      // 备份数据库（同步操作，不会被 SIGINT 中断）
       store.stopBackup();
       console.error("[server] 退出");
       process.exit(0);