tt-help-cli-ycl 1.3.84 → 1.3.86

package/src/watch/data-store.js

@@ -152,6 +152,9 @@ function initUserDb(filePath) {
   if (!existingJobColumns.has("top_video_href")) {
     db.exec(`ALTER TABLE jobs ADD COLUMN top_video_href TEXT`);
   }
+  if (!existingJobColumns.has("user_create_time")) {
+    db.exec(`ALTER TABLE jobs ADD COLUMN user_create_time INTEGER`);
+  }
   db.exec(`
     CREATE TABLE IF NOT EXISTS jobs_base (
       unique_id TEXT PRIMARY KEY,
@@ -212,6 +215,9 @@ function initUserDb(filePath) {
   if (!existingJobBaseColumns.has("bio_link")) {
     db.exec(`ALTER TABLE jobs_base ADD COLUMN bio_link TEXT`);
   }
+  if (!existingJobBaseColumns.has("user_create_time")) {
+    db.exec(`ALTER TABLE jobs_base ADD COLUMN user_create_time INTEGER`);
+  }
   db.exec(`
     CREATE TABLE IF NOT EXISTS raw_jobs (
       unique_id TEXT PRIMARY KEY,
@@ -271,6 +277,9 @@ function initUserDb(filePath) {
   if (!existingRawJobColumns.has("bio_link")) {
     db.exec(`ALTER TABLE raw_jobs ADD COLUMN bio_link TEXT`);
   }
+  if (!existingRawJobColumns.has("user_create_time")) {
+    db.exec(`ALTER TABLE raw_jobs ADD COLUMN user_create_time INTEGER`);
+  }
   db.exec(`
     CREATE TABLE IF NOT EXISTS videos (
       id TEXT PRIMARY KEY,
@@ -384,6 +393,30 @@ function initUserDb(filePath) {
     db.exec(`ALTER TABLE videos ADD COLUMN create_time INTEGER`);
   }
 
+  // tags 表：标签发现与打分系统
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS tags (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      tag TEXT NOT NULL UNIQUE,
+      status TEXT NOT NULL DEFAULT 'new',
+      score REAL NOT NULL DEFAULT 0,
+      created_at TEXT NOT NULL DEFAULT (datetime('now')),
+      scored_at TEXT,
+      score_count INTEGER NOT NULL DEFAULT 0,
+      countries TEXT NOT NULL DEFAULT '[]',
+      matched_countries TEXT DEFAULT '[]',
+      total_posts INTEGER DEFAULT 0,
+      author_count INTEGER DEFAULT 0,
+      matched_authors INTEGER DEFAULT 0,
+      pushed_users INTEGER DEFAULT 0,
+      source TEXT NOT NULL DEFAULT 'llm',
+      user_prompt TEXT,
+      last_error TEXT
+    )
+  `);
+  db.exec(`CREATE INDEX IF NOT EXISTS idx_tags_status ON tags(status)`);
+  db.exec(`CREATE INDEX IF NOT EXISTS idx_tags_score ON tags(score DESC)`);
+
   const count = db.prepare("SELECT COUNT(*) as c FROM users").get().c;
   console.log(`[data-store] SQLite users 表初始化完成: ${count} 条`);
 }
@@ -956,7 +989,7 @@ function moveJobsToRawByCountry(scope, country) {
       guessed_location, location_created, follower_count,
       following_count, heart_count, refresh_time, processed,
       processed_at, created_at, updated_at, region, signature,
-      sec_uid, latest_video_time
+      sec_uid, latest_video_time, user_create_time
     `;
   } else if (normalizedScope === "userUpdate") {
     sourceTable = "jobs_base";
@@ -968,7 +1001,7 @@ function moveJobsToRawByCountry(scope, country) {
       guessed_location, location_created, follower_count,
       following_count, heart_count, refresh_time, processed,
       processed_at, created_at, updated_at, region, signature,
-      sec_uid, latest_video_time
+      sec_uid, latest_video_time, user_create_time
     `;
   } else {
     return {
@@ -1307,6 +1340,170 @@ function getRawJobsPageFromDb({
   };
 }
 
+// ====== Tag 发现与打分 CRUD ======
+
+function insertTag(tag, countries, source = "llm") {
+  if (!db) return { inserted: false, error: "db not ready" };
+  // 防止存入带 # 前缀的 tag
+  const normalized = tag.replace(/^#+/, "").trim().toLowerCase();
+  if (!normalized || normalized.length < 2) {
+    return { inserted: false, error: "invalid tag" };
+  }
+  try {
+    const result = db
+      .prepare(
+        `
+      INSERT OR IGNORE INTO tags (tag, countries, source)
+      VALUES (?, ?, ?)
+    `,
+      )
+      .run(normalized, JSON.stringify(countries), source);
+    return { inserted: result.changes > 0, tag: normalized };
+  } catch (e) {
+    return { inserted: false, error: e.message };
+  }
+}
+
+function getTagsByStatus(status, limit = 100) {
+  if (!db) return [];
+  const rows = db
+    .prepare(
+      `
+    SELECT * FROM tags WHERE status = ? ORDER BY score ASC, created_at ASC LIMIT ?
+  `,
+    )
+    .all(status, limit);
+  return rows.map((r) => ({
+    ...r,
+    countries: JSON.parse(r.countries || "[]"),
+    matched_countries: JSON.parse(r.matched_countries || "[]"),
+  }));
+}
+
+function getTagsByCountry(country, minScore = 0) {
+  if (!db) return [];
+  const rows = db
+    .prepare(
+      `
+    SELECT * FROM tags WHERE status != 'dead'
+    ORDER BY score DESC
+  `,
+    )
+    .all();
+  // Filter in JS since countries is JSON
+  return rows
+    .map((r) => ({
+      ...r,
+      countries: JSON.parse(r.countries || "[]"),
+      matched_countries: JSON.parse(r.matched_countries || "[]"),
+    }))
+    .filter((r) => r.countries.includes(country) && r.score >= minScore);
+}
+
+function getDeadTags(country) {
+  if (!db) return [];
+  const rows = db
+    .prepare(
+      `
+    SELECT * FROM tags WHERE status = 'dead' ORDER BY score ASC
+  `,
+    )
+    .all();
+  return rows
+    .map((r) => ({
+      ...r,
+      countries: JSON.parse(r.countries || "[]"),
+      matched_countries: JSON.parse(r.matched_countries || "[]"),
+    }))
+    .filter((r) => r.countries.includes(country));
+}
+
+function claimTag(tag) {
+  if (!db) return { ok: false, error: "db not ready" };
+  // 原子操作：只有 status='new' 时才更新为 'scoring'，避免竞态
+  const result = db
+    .prepare(
+      "UPDATE tags SET status = 'scoring' WHERE tag = ? AND status = 'new'",
+    )
+    .run(tag);
+  if (result.changes === 0) {
+    // 检查是否不存在 vs 已被别人锁定
+    const row = db.prepare("SELECT status FROM tags WHERE tag = ?").get(tag);
+    if (!row) return { ok: false, error: "tag not found" };
+    return { ok: false, error: `tag status is ${row.status}, already claimed` };
+  }
+  return { ok: true, tag };
+}
+
+function reportTagScore(tag, fields) {
+  if (!db) return { ok: false, error: "db not ready" };
+  const {
+    score,
+    status,
+    totalPosts,
+    authorCount,
+    matchedAuthors,
+    matchedCountries,
+    pushedUsers,
+    error,
+  } = fields;
+  const matchedCountriesJson = matchedCountries
+    ? JSON.stringify(matchedCountries)
+    : null;
+  const now = new Date().toISOString();
+
+  try {
+    const result = db
+      .prepare(
+        `
+      UPDATE tags SET
+        score = COALESCE(?, score),
+        status = COALESCE(?, status),
+        total_posts = COALESCE(?, total_posts),
+        author_count = COALESCE(?, author_count),
+        matched_authors = COALESCE(?, matched_authors),
+        matched_countries = COALESCE(?, matched_countries),
+        pushed_users = COALESCE(?, pushed_users),
+        last_error = COALESCE(?, last_error),
+        scored_at = ?,
+        score_count = score_count + 1
+      WHERE tag = ?
+    `,
+      )
+      .run(
+        score ?? null,
+        status ?? null,
+        totalPosts ?? null,
+        authorCount ?? null,
+        matchedAuthors ?? null,
+        matchedCountriesJson,
+        pushedUsers ?? null,
+        error ?? null,
+        now,
+        tag,
+      );
+    return { ok: result.changes > 0, tag };
+  } catch (e) {
+    return { ok: false, error: e.message };
+  }
+}
+
+function getAllTags(limit = 200) {
+  if (!db) return [];
+  const rows = db
+    .prepare(
+      `
+    SELECT * FROM tags ORDER BY score DESC, created_at DESC LIMIT ?
+  `,
+    )
+    .all(limit);
+  return rows.map((r) => ({
+    ...r,
+    countries: JSON.parse(r.countries || "[]"),
+    matched_countries: JSON.parse(r.matched_countries || "[]"),
+  }));
+}
+
 // 调试接口：直接执行 SQL 查询，返回原始数据
 function rawQuery(sql, params = []) {
   if (!db) return { error: "db not ready" };
@@ -1318,6 +1515,68 @@ function rawQuery(sql, params = []) {
   }
 }
 
+// 清理 tags 表中以 # 开头的脏数据
+function normalizeTags() {
+  if (!db) return { ok: false, error: "db not ready" };
+  const dirtyRows = db
+    .prepare("SELECT id, tag, countries FROM tags WHERE tag LIKE '#%'")
+    .all();
+  const fixed = [];
+  const merged = [];
+  const skipped = [];
+
+  for (const row of dirtyRows) {
+    const cleanTag = row.tag.replace(/^#+/, "").trim().toLowerCase();
+    if (!cleanTag || cleanTag.length < 2) {
+      db.prepare("DELETE FROM tags WHERE id = ?").run(row.id);
+      skipped.push({
+        dirty: row.tag,
+        reason: "empty after normalize, deleted",
+      });
+      continue;
+    }
+
+    // 检查 cleanTag 是否已存在
+    const existing = db
+      .prepare("SELECT * FROM tags WHERE tag = ?")
+      .get(cleanTag);
+    if (existing) {
+      // 合并：保留已有 clean 版本，合并 countries
+      const oldCountries = JSON.parse(row.countries || "[]");
+      const existCountries = JSON.parse(existing.countries || "[]");
+      const mergedCountries = [
+        ...new Set([...existCountries, ...oldCountries]),
+      ];
+      db.prepare("UPDATE tags SET countries = ? WHERE tag = ?").run(
+        JSON.stringify(mergedCountries),
+        cleanTag,
+      );
+      // 删除脏数据
+      db.prepare("DELETE FROM tags WHERE id = ?").run(row.id);
+      merged.push({ dirty: row.tag, clean: cleanTag, id: row.id });
+    } else {
+      // 直接重命名
+      db.prepare("UPDATE tags SET tag = ? WHERE id = ?").run(cleanTag, row.id);
+      fixed.push({ dirty: row.tag, clean: cleanTag, id: row.id });
+    }
+  }
+
+  return {
+    ok: true,
+    fixed: fixed.length,
+    merged: merged.length,
+    skipped: skipped.length,
+    details: { fixed, merged, skipped },
+  };
+}
+
+function clearTags() {
+  if (!db) return { ok: false, error: "db not ready" };
+  const count = db.prepare("SELECT COUNT(*) as c FROM tags").get().c;
+  db.exec("DELETE FROM tags");
+  return { ok: true, deleted: count };
+}
+
 function getUsersPageFromDb({
   status,
   search,
@@ -1668,6 +1927,7 @@ const writableJobColumns = new Set([
   "latest_video_time",
   "top_video_play_count",
   "top_video_href",
+  "user_create_time",
 ]);
 
 function normalizeJobValue(column, value) {
@@ -1761,8 +2021,9 @@ function updateJobInfo(uniqueId, info, incrementCount = true) {
     if (key === "uniqueId" || key === "unique_id") continue;
     if (value === undefined || value === "") continue;
     let column = camelToSnake(key);
-    // 字段别名：bio → signature
+    // 字段别名：bio → signature, createTime → user_create_time
     if (column === "bio") column = "signature";
+    if (column === "create_time") column = "user_create_time";
     if (!writableJobColumns.has(column)) continue;
     nextValues[column] = normalizeJobValue(column, value);
   }
@@ -1805,8 +2066,9 @@ function updateJobBaseInfo(uniqueId, info, incrementCount = true) {
     if (key === "uniqueId" || key === "unique_id") continue;
     if (value === undefined || value === "") continue;
     let column = camelToSnake(key);
-    // 字段别名：bio → signature
+    // 字段别名：bio → signature, createTime → user_create_time
     if (column === "bio") column = "signature";
+    if (column === "create_time") column = "user_create_time";
     if (!writableJobColumns.has(column)) continue;
     nextValues[column] = normalizeJobValue(column, value);
   }
@@ -2194,6 +2456,43 @@ export function createStore(filePath, options = {}) {
     }
   }
 
+  function addRawUsers(users) {
+    if (!Array.isArray(users)) return { added: 0, skipped: 0 };
+    const now = Date.now();
+    let added = 0;
+    let skipped = 0;
+
+    for (const u of users) {
+      const uniqueId = (u.uniqueId || "").replace(/^@/, "").trim();
+      if (!uniqueId) continue;
+      if (hasUser(uniqueId)) {
+        skipped++;
+        continue;
+      }
+      const userObj = {
+        uniqueId,
+        status: "pending",
+        sources: Array.isArray(u.sources)
+          ? u.sources
+          : u.sources
+            ? [u.sources]
+            : ["tag"],
+        guessedLocation: u.guessedLocation || u.locationCreated || null,
+        locationCreated: u.locationCreated || null,
+        createdAt: now,
+        updatedAt: now,
+      };
+      const writeTxn = db.transaction((job) => {
+        addUserToDb(job);
+        addJobBaseToDb(job);
+      });
+      writeTxn(userObj);
+      added++;
+    }
+
+    return { added, skipped };
+  }
+
   function getPendingUsers() {
     if (db) {
       return getAllJobs().filter((u) => u.status === "pending");
@@ -2410,7 +2709,7 @@ Standards: 90-100=clear match, 70-89=likely, 50-69=possible, 20-49=low, 0-19=unl
 
     // 构建 WHERE 条件
     const conditions = [
-      "COALESCE(video_count, 0) > 0",
+      "COALESCE(video_count, 0) > 6",
       "COALESCE(follower_count, 0) > 0",
       "COALESCE(following_count, 0) > 0",
     ];
@@ -2496,7 +2795,7 @@ Standards: 90-100=clear match, 70-89=likely, 50-69=possible, 20-49=low, 0-19=unl
               .prepare(
                 `
                 SELECT * FROM raw_jobs WHERE ${whereSql} AND guessed_location = ?
-                ORDER BY created_at DESC
+                ORDER BY COALESCE(video_count, 0) DESC, created_at DESC
                 LIMIT ? OFFSET ?
               `,
               )
@@ -2610,7 +2909,7 @@ Standards: 90-100=clear match, 70-89=likely, 50-69=possible, 20-49=low, 0-19=unl
           guessed_location, location_created, confirmed_location,
           follower_count, following_count, heart_count,
           created_at, updated_at, region, signature, bio_link, sec_uid,
-          status_code, latest_video_time
+          status_code, latest_video_time, user_create_time
         )
         SELECT
           unique_id, nickname, 'pending', sources, pinned,
@@ -2618,10 +2917,10 @@ Standards: 90-100=clear match, 70-89=likely, 50-69=possible, 20-49=low, 0-19=unl
           guessed_location, location_created, confirmed_location,
           follower_count, following_count, heart_count,
           created_at, updated_at, region, signature, bio_link, sec_uid,
-          status_code, latest_video_time
+          status_code, latest_video_time, user_create_time
         FROM raw_jobs
         WHERE ${whereSql}
-        ORDER BY created_at DESC
+        ORDER BY COALESCE(video_count, 0) DESC, created_at DESC
         LIMIT ?
       `,
       ).run(...args, safeLimit);
@@ -2633,7 +2932,7 @@ Standards: 90-100=clear match, 70-89=likely, 50-69=possible, 20-49=low, 0-19=unl
         WHERE unique_id IN (
           SELECT unique_id FROM raw_jobs
           WHERE ${whereSql}
-          ORDER BY created_at DESC
+          ORDER BY COALESCE(video_count, 0) DESC, created_at DESC
           LIMIT ?
         )
       `,
@@ -3734,20 +4033,7 @@ Standards: 90-100=clear match, 70-89=likely, 50-69=possible, 20-49=low, 0-19=unl
       const now = Date.now();
       const threshold = now - maxAgeSeconds * 1000;
       const defaultTime = new Date("2016-01-01T00:00:00Z").getTime();
-      const targetLocations = [
-        "CZ",
-        "GR",
-        "HU",
-        "PT",
-        "ES",
-        "PL",
-        "NL",
-        "BE",
-        "DE",
-        "FR",
-        "IT",
-        "IE",
-      ];
+      const targetLocations = DEFAULT_TARGET_LOCATIONS;
       const placeholders = targetLocations.map(() => "?").join(",");
       const row = db
         .prepare(
@@ -3758,7 +4044,7 @@ Standards: 90-100=clear match, 70-89=likely, 50-69=possible, 20-49=low, 0-19=unl
             AND verified = 0
             AND location_created IN (${placeholders})
             AND COALESCE(refresh_time, ?) < ?
-          ORDER BY COALESCE(refresh_time, ?) ASC
+          ORDER BY COALESCE(pinned, 0) DESC, COALESCE(refresh_time, ?) ASC
           LIMIT 1
         `,
         )
@@ -3779,20 +4065,7 @@ Standards: 90-100=clear match, 70-89=likely, 50-69=possible, 20-49=low, 0-19=unl
     const defaultTime = new Date("2016-01-01T00:00:00Z").getTime();
 
     // 筛选目标国家用户，按 refreshTime 升序取最远的（没有则默认 2016-01-01）
-    const targetLocations = [
-      "CZ",
-      "GR",
-      "HU",
-      "PT",
-      "ES",
-      "PL",
-      "NL",
-      "BE",
-      "DE",
-      "FR",
-      "IT",
-      "IE",
-    ];
+    const targetLocations = DEFAULT_TARGET_LOCATIONS;
     const targetUsers = data.filter(
       (u) =>
         u.ttSeller &&
@@ -3808,6 +4081,10 @@ Standards: 90-100=clear match, 70-89=likely, 50-69=possible, 20-49=low, 0-19=unl
     if (recentEnough.length === 0) return null;
 
     recentEnough.sort((a, b) => {
+      // pinned 优先，其次按 refreshTime 升序
+      if ((a.pinned ? 1 : 0) !== (b.pinned ? 1 : 0)) {
+        return (b.pinned ? 1 : 0) - (a.pinned ? 1 : 0);
+      }
       const ta = a.refreshTime || defaultTime;
       const tb = b.refreshTime || defaultTime;
       return ta - tb;
@@ -4101,7 +4378,8 @@ Standards: 90-100=clear match, 70-89=likely, 50-69=possible, 20-49=low, 0-19=unl
           guessed_location, location_created, confirmed_location, modified_at,
           follower_count, following_count, heart_count, refresh_time,
           processed, processed_at, created_at, updated_at,
-          region, signature, bio_link, sec_uid, status_code, latest_video_time
+          region, signature, bio_link, sec_uid, status_code, latest_video_time,
+          user_create_time
         )
         SELECT
           unique_id, nickname, status, sources, claimed_by, claimed_at,
@@ -4110,7 +4388,8 @@ Standards: 90-100=clear match, 70-89=likely, 50-69=possible, 20-49=low, 0-19=unl
           guessed_location, location_created, confirmed_location, modified_at,
           follower_count, following_count, heart_count, refresh_time,
           processed, processed_at, created_at, updated_at,
-          region, signature, bio_link, sec_uid, status_code, latest_video_time
+          region, signature, bio_link, sec_uid, status_code, latest_video_time,
+          user_create_time
         FROM jobs WHERE unique_id = ?
       `,
       ).run(safeId);
@@ -4149,11 +4428,12 @@ Standards: 90-100=clear match, 70-89=likely, 50-69=possible, 20-49=low, 0-19=unl
             return;
           }
 
-          // 检查 tt_seller：商家移到 jobs，非商家移到 raw_jobs
+          // 检查 tt_seller：商家且视频数>0移到 jobs，否则移到 raw_jobs
           const row = getJobBaseRow(uniqueId);
           const ttSeller = row ? row.tt_seller : null;
-          if (ttSeller) {
-            // 商家：标记移动到 jobs
+          const videoCount = row ? row.video_count || 0 : 0;
+          if (ttSeller && videoCount > 0) {
+            // 商家且有视频：标记移动到 jobs
             results.push({
               uniqueId,
               ok: true,
@@ -4162,7 +4442,7 @@ Standards: 90-100=clear match, 70-89=likely, 50-69=possible, 20-49=low, 0-19=unl
             });
             sellerMoveList.push(uniqueId);
           } else {
-            // 非商家：标记移动到 raw_jobs
+            // 非商家或无视频：标记移动到 raw_jobs
             results.push({
               uniqueId,
               ok: true,
@@ -4187,7 +4467,8 @@ Standards: 90-100=clear match, 70-89=likely, 50-69=possible, 20-49=low, 0-19=unl
             guessed_location, location_created, confirmed_location, modified_at,
             follower_count, following_count, heart_count, refresh_time,
             processed, processed_at, created_at, updated_at,
-            region, signature, bio_link, sec_uid, status_code, latest_video_time
+            region, signature, bio_link, sec_uid, status_code, latest_video_time,
+            user_create_time
           )
           SELECT
             unique_id, nickname, status, sources, claimed_by, claimed_at,
@@ -4196,7 +4477,8 @@ Standards: 90-100=clear match, 70-89=likely, 50-69=possible, 20-49=low, 0-19=unl
             guessed_location, location_created, confirmed_location, modified_at,
             follower_count, following_count, heart_count, refresh_time,
             processed, processed_at, created_at, updated_at,
-            region, signature, bio_link, sec_uid, status_code, latest_video_time
+            region, signature, bio_link, sec_uid, status_code, latest_video_time,
+            user_create_time
           FROM jobs_base WHERE unique_id IN (${placeholders})
         `,
         ).run(...sellerMoveList);
@@ -4218,7 +4500,8 @@ Standards: 90-100=clear match, 70-89=likely, 50-69=possible, 20-49=low, 0-19=unl
             guessed_location, location_created, confirmed_location, modified_at,
             follower_count, following_count, heart_count, refresh_time,
             processed, processed_at, created_at, updated_at,
-            region, signature, bio_link, sec_uid, status_code, latest_video_time
+            region, signature, bio_link, sec_uid, status_code, latest_video_time,
+            user_create_time
           )
           SELECT
             unique_id, nickname, status, sources, claimed_by, claimed_at,
@@ -4227,7 +4510,8 @@ Standards: 90-100=clear match, 70-89=likely, 50-69=possible, 20-49=low, 0-19=unl
             guessed_location, location_created, confirmed_location, modified_at,
             follower_count, following_count, heart_count, refresh_time,
             processed, processed_at, created_at, updated_at,
-            region, signature, bio_link, sec_uid, status_code, latest_video_time
+            region, signature, bio_link, sec_uid, status_code, latest_video_time,
+            user_create_time
           FROM jobs_base WHERE unique_id IN (${placeholders})
         `,
         ).run(...rawMoveList);
@@ -4483,6 +4767,7 @@ Standards: 90-100=clear match, 70-89=likely, 50-69=possible, 20-49=low, 0-19=unl
     hasUser,
     userExists,
     addUser,
+    addRawUsers,
     getPendingUsers,
     getProcessedUsers,
     getAllUsers,
@@ -4540,6 +4825,16 @@ Standards: 90-100=clear match, 70-89=likely, 50-69=possible, 20-49=low, 0-19=unl
     stopBackup,
     rawQuery,
     getLlmSampleOffsets, // 获取 LLM 采样偏移量状态
+    // Tag 发现与打分
+    insertTag,
+    getTagsByStatus,
+    getTagsByCountry,
+    getDeadTags,
+    claimTag,
+    reportTagScore,
+    getAllTags,
+    normalizeTags,
+    clearTags,
     data,
   };