tt-help-cli-ycl 1.3.92 → 1.3.94

package/src/watch/db-schema.js

@@ -0,0 +1,367 @@
+/**
+ * 数据库 Schema 初始化与迁移
+ *
+ * 负责建表、索引、列迁移。所有 DDL 操作集中在此文件，
+ * 其他模块只做 CRUD，不关心表结构。
+ */
+
+import fs from "fs";
+import path from "path";
+import Database from "better-sqlite3";
+import { columnsList, VIDEO_COLUMNS } from "./db-columns.js";
+
+// ===== 全局数据库连接（模块级共享） =====
+let db = null;
+let dbPath = null;
+
+export function getDb() {
+  return db;
+}
+
+export function getDbPath() {
+  return dbPath;
+}
+
+export function normalizeDbFilePath(filePath) {
+  if (!filePath) {
+    throw new Error("db path is required");
+  }
+  const resolved = path.resolve(filePath);
+  if (path.extname(resolved).toLowerCase() !== ".db") {
+    throw new Error(`仅支持 .db 路径，当前为: ${filePath}`);
+  }
+  return resolved;
+}
+
+export function resetDbConnection() {
+  if (db) {
+    db.close();
+    db = null;
+  }
+  dbPath = null;
+}
+
+// ===== 建表与迁移 =====
+
+/**
+ * 为一张表执行列迁移：检查已有列，补全缺失列
+ * @param {string} tableName - 表名
+ * @param {Object} requiredColumns - { columnName: columnType }
+ */
+function migrateColumns(tableName, requiredColumns) {
+  const existing = new Set(
+    db
+      .prepare(`PRAGMA table_info(${tableName})`)
+      .all()
+      .map((c) => c.name),
+  );
+  for (const [column, type] of Object.entries(requiredColumns)) {
+    if (!existing.has(column)) {
+      db.exec(`ALTER TABLE ${tableName} ADD COLUMN ${column} ${type}`);
+    }
+  }
+}
+
+/**
+ * 初始化所有表、索引、迁移
+ */
+export function initDb(filePath) {
+  dbPath = normalizeDbFilePath(filePath);
+  fs.mkdirSync(path.dirname(dbPath), { recursive: true });
+  db = new Database(dbPath);
+  db.pragma("journal_mode = WAL");
+
+  // ===== users 表 =====
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS users (
+      unique_id TEXT PRIMARY KEY,
+      tt_seller TEXT,
+      verified INTEGER,
+      location_created TEXT,
+      created_at TEXT,
+      updated_at TEXT
+    )
+  `);
+
+  // ===== jobs 表 =====
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS jobs (
+      unique_id TEXT PRIMARY KEY,
+      nickname TEXT,
+      status TEXT DEFAULT 'pending',
+      sources TEXT,
+      claimed_by TEXT,
+      claimed_at INTEGER,
+      error TEXT,
+      pinned INTEGER DEFAULT 0,
+      no_video INTEGER DEFAULT 0,
+      restricted INTEGER DEFAULT 0,
+      user_update_count INTEGER DEFAULT 0,
+      tt_seller INTEGER,
+      verified INTEGER,
+      video_count INTEGER DEFAULT 0,
+      comment_count INTEGER DEFAULT 0,
+      guessed_location TEXT,
+      location_created TEXT,
+      confirmed_location TEXT,
+      modified_at INTEGER,
+      follower_count INTEGER DEFAULT 0,
+      following_count INTEGER DEFAULT 0,
+      heart_count INTEGER DEFAULT 0,
+      refresh_time INTEGER,
+      processed INTEGER DEFAULT 0,
+      processed_at INTEGER,
+      created_at INTEGER,
+      updated_at INTEGER,
+      region TEXT,
+      signature TEXT,
+      sec_uid TEXT,
+      status_code INTEGER
+    )
+  `);
+  migrateColumns("jobs", {
+    status_code: "INTEGER",
+    latest_video_time: "INTEGER",
+    confirmed_location: "TEXT",
+    modified_at: "INTEGER",
+    bio_link: "TEXT",
+    top_video_play_count: "INTEGER",
+    top_video_href: "TEXT",
+    user_create_time: "INTEGER",
+  });
+
+  // ===== jobs_base 表 =====
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS jobs_base (
+      unique_id TEXT PRIMARY KEY,
+      nickname TEXT,
+      status TEXT DEFAULT 'pending',
+      sources TEXT,
+      claimed_by TEXT,
+      claimed_at INTEGER,
+      error TEXT,
+      pinned INTEGER DEFAULT 0,
+      no_video INTEGER DEFAULT 0,
+      restricted INTEGER DEFAULT 0,
+      user_update_count INTEGER DEFAULT 0,
+      tt_seller INTEGER,
+      verified INTEGER,
+      video_count INTEGER DEFAULT 0,
+      comment_count INTEGER DEFAULT 0,
+      guessed_location TEXT,
+      location_created TEXT,
+      confirmed_location TEXT,
+      modified_at INTEGER,
+      follower_count INTEGER DEFAULT 0,
+      following_count INTEGER DEFAULT 0,
+      heart_count INTEGER DEFAULT 0,
+      refresh_time INTEGER,
+      processed INTEGER DEFAULT 0,
+      processed_at INTEGER,
+      created_at INTEGER,
+      updated_at INTEGER,
+      region TEXT,
+      signature TEXT,
+      sec_uid TEXT,
+      status_code INTEGER,
+      latest_video_time INTEGER,
+      bio_link TEXT
+    )
+  `);
+  migrateColumns("jobs_base", {
+    status_code: "INTEGER",
+    latest_video_time: "INTEGER",
+    confirmed_location: "TEXT",
+    modified_at: "INTEGER",
+    bio_link: "TEXT",
+    user_create_time: "INTEGER",
+  });
+
+  // ===== raw_jobs 表 =====
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS raw_jobs (
+      unique_id TEXT PRIMARY KEY,
+      nickname TEXT,
+      status TEXT DEFAULT 'pending',
+      sources TEXT,
+      claimed_by TEXT,
+      claimed_at INTEGER,
+      error TEXT,
+      pinned INTEGER DEFAULT 0,
+      no_video INTEGER DEFAULT 0,
+      restricted INTEGER DEFAULT 0,
+      user_update_count INTEGER DEFAULT 0,
+      tt_seller INTEGER,
+      verified INTEGER,
+      video_count INTEGER DEFAULT 0,
+      comment_count INTEGER DEFAULT 0,
+      guessed_location TEXT,
+      location_created TEXT,
+      confirmed_location TEXT,
+      modified_at INTEGER,
+      follower_count INTEGER DEFAULT 0,
+      following_count INTEGER DEFAULT 0,
+      heart_count INTEGER DEFAULT 0,
+      refresh_time INTEGER,
+      processed INTEGER DEFAULT 0,
+      processed_at INTEGER,
+      created_at INTEGER,
+      updated_at INTEGER,
+      region TEXT,
+      signature TEXT,
+      sec_uid TEXT,
+      status_code INTEGER,
+      latest_video_time INTEGER
+    )
+  `);
+  migrateColumns("raw_jobs", {
+    status_code: "INTEGER",
+    latest_video_time: "INTEGER",
+    confirmed_location: "TEXT",
+    modified_at: "INTEGER",
+    bio_link: "TEXT",
+    user_create_time: "INTEGER",
+  });
+
+  // ===== videos 表 =====
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS videos (
+      id TEXT PRIMARY KEY,
+      href TEXT,
+      author_unique_id TEXT,
+      location_created TEXT,
+      tt_seller INTEGER DEFAULT 0,
+      registered_at INTEGER,
+      user_update_count INTEGER DEFAULT 0,
+      play_count INTEGER,
+      digg_count INTEGER,
+      comment_count INTEGER,
+      share_count INTEGER,
+      collect_count INTEGER,
+      stats_updated_at INTEGER,
+      create_time INTEGER
+    )
+  `);
+  migrateColumns("videos", {
+    play_count: "INTEGER",
+    digg_count: "INTEGER",
+    comment_count: "INTEGER",
+    share_count: "INTEGER",
+    collect_count: "INTEGER",
+    stats_updated_at: "INTEGER",
+    create_time: "INTEGER",
+  });
+
+  // ===== tags 表 =====
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS tags (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      tag TEXT NOT NULL UNIQUE,
+      status TEXT NOT NULL DEFAULT 'new',
+      score REAL NOT NULL DEFAULT 0,
+      created_at TEXT NOT NULL DEFAULT (datetime('now')),
+      scored_at TEXT,
+      score_count INTEGER NOT NULL DEFAULT 0,
+      countries TEXT NOT NULL DEFAULT '[]',
+      matched_countries TEXT DEFAULT '[]',
+      total_posts INTEGER DEFAULT 0,
+      author_count INTEGER DEFAULT 0,
+      matched_authors INTEGER DEFAULT 0,
+      pushed_users INTEGER DEFAULT 0,
+      source TEXT NOT NULL DEFAULT 'llm',
+      user_prompt TEXT,
+      last_error TEXT
+    )
+  `);
+
+  // ===== tags 表新增字段（兼容已存在的表） =====
+  migrateColumns("tags", {
+    discover_round: "INTEGER DEFAULT 0",
+    strategy: "TEXT",
+  });
+
+  // ===== discover_log 表（记录每次 LLM discover 的策略和思考） =====
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS discover_log (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      country TEXT NOT NULL,
+      round INTEGER NOT NULL DEFAULT 1,
+      strategy TEXT,
+      tags_generated TEXT NOT NULL DEFAULT '[]',
+      tags_added INTEGER DEFAULT 0,
+      created_at TEXT NOT NULL DEFAULT (datetime('now')),
+      productive_sample TEXT,
+      dead_sample TEXT,
+      avg_productive_score REAL DEFAULT 0,
+      avg_dead_score REAL DEFAULT 0
+    )
+  `);
+
+  // ===== 索引 =====
+  const indexes = [
+    `CREATE INDEX IF NOT EXISTS idx_jobs_status_video ON jobs(status, video_count DESC)`,
+    `CREATE INDEX IF NOT EXISTS idx_jobs_claimed_by_status ON jobs(claimed_by, status, claimed_at)`,
+    `CREATE INDEX IF NOT EXISTS idx_jobs_status_claimed_at ON jobs(status, claimed_at)`,
+    `CREATE INDEX IF NOT EXISTS idx_jobs_redo_target ON jobs(tt_seller, verified, location_created, refresh_time)`,
+    `CREATE INDEX IF NOT EXISTS idx_jobs_pending_priority ON jobs(status, pinned DESC, guessed_location, follower_count DESC)`,
+    `CREATE INDEX IF NOT EXISTS idx_jobs_claim_pending_pinned ON jobs(created_at ASC, unique_id ASC) WHERE status = 'pending' AND COALESCE(pinned, 0) = 1`,
+    `CREATE INDEX IF NOT EXISTS idx_jobs_claim_pending_seller ON jobs(UPPER(COALESCE(guessed_location, '')), follower_count DESC, created_at ASC, unique_id ASC) WHERE status = 'pending' AND COALESCE(pinned, 0) = 0 AND tt_seller = 1 AND verified = 0`,
+    `CREATE INDEX IF NOT EXISTS idx_jobs_claim_pending_follow ON jobs(UPPER(COALESCE(guessed_location, '')), follower_count DESC, created_at ASC, unique_id ASC) WHERE status = 'pending' AND COALESCE(pinned, 0) = 0 AND (instr(COALESCE(sources, ''), '"following"') > 0 OR instr(COALESCE(sources, ''), '"follower"') > 0)`,
+    `CREATE INDEX IF NOT EXISTS idx_jobs_claim_pending_other ON jobs(UPPER(COALESCE(guessed_location, '')), follower_count DESC, created_at ASC, unique_id ASC) WHERE status = 'pending' AND COALESCE(pinned, 0) = 0`,
+    `CREATE INDEX IF NOT EXISTS idx_jobs_user_update_queue ON jobs(created_at ASC, unique_id ASC) WHERE (tt_seller IS NULL OR tt_seller = '') AND (user_update_count IS NULL OR user_update_count <= 0)`,
+    `CREATE INDEX IF NOT EXISTS idx_jobs_user_update_queue_expr ON jobs(created_at ASC, unique_id ASC) WHERE COALESCE(tt_seller, '') = '' AND COALESCE(user_update_count, 0) <= 0`,
+    `CREATE INDEX IF NOT EXISTS idx_videos_comment_queue ON videos(user_update_count, tt_seller DESC, registered_at ASC)`,
+    `CREATE INDEX IF NOT EXISTS idx_videos_comment_queue_pending ON videos(tt_seller DESC, registered_at ASC, id) WHERE user_update_count IS NULL OR user_update_count <= 0`,
+    `CREATE INDEX IF NOT EXISTS idx_tags_status ON tags(status)`,
+    `CREATE INDEX IF NOT EXISTS idx_tags_score ON tags(score DESC)`,
+    `CREATE INDEX IF NOT EXISTS idx_discover_log_country ON discover_log(country, round DESC)`,
+  ];
+  for (const sql of indexes) {
+    db.exec(sql);
+  }
+
+  const count = db.prepare("SELECT COUNT(*) as c FROM users").get().c;
+  console.log(`[data-store] SQLite users 表初始化完成: ${count} 条`);
+}
+
+// ===== 历史数据加载 =====
+
+export function loadLegacyUsersFromFiles(userFilePath, doneFilePath) {
+  const merged = new Map();
+
+  const tryLoad = (targetPath, label) => {
+    if (!targetPath) return;
+    if (!fs.existsSync(targetPath)) return;
+    try {
+      const parsed = JSON.parse(fs.readFileSync(targetPath, "utf-8"));
+      if (!Array.isArray(parsed)) return;
+      for (const item of parsed) {
+        const uniqueId = item?.uniqueId || item?.unique_id;
+        if (!uniqueId) continue;
+        merged.set(uniqueId, { ...merged.get(uniqueId), ...item, uniqueId });
+      }
+    } catch (e) {
+      console.error(`[data-store] SQLite 导入 ${label} 失败: ${e.message}`);
+    }
+  };
+
+  tryLoad(userFilePath, "result.json");
+  tryLoad(doneFilePath, "result-done.json");
+
+  return [...merged.values()];
+}
+
+export function loadLegacyVideosFromFile(videoPath) {
+  if (!videoPath) return [];
+  if (!fs.existsSync(videoPath)) return [];
+
+  try {
+    const parsed = JSON.parse(fs.readFileSync(videoPath, "utf-8"));
+    return Array.isArray(parsed) ? parsed : [];
+  } catch (e) {
+    console.error(
+      `[data-store] SQLite 导入 result-videos.json 失败: ${e.message}`,
+    );
+    return [];
+  }
+}

package/src/watch/db-stats.js

@@ -0,0 +1,235 @@
+/**
+ * 数据库统计查询
+ *
+ * 仪表盘聚合统计、按国家分组的待处理/补资料/卡住/原始任务统计。
+ */
+
+import { getDb } from "./db-schema.js";
+import { getRawJobsCount, getUserDbCount } from "./db-crud.js";
+
+export function getDashboardStatsFromDb(targetLocations = []) {
+  const db = getDb();
+  if (!db) return null;
+
+  const targetPlaceholders = targetLocations.map(() => "?").join(", ");
+  const targetParams = targetLocations.length ? targetLocations : [];
+
+  const aggregateRow = db
+    .prepare(
+      `
+      SELECT
+        COUNT(*) as total,
+        SUM(CASE WHEN status = 'pending' THEN 1 ELSE 0 END) as pending,
+        SUM(CASE WHEN status = 'processing' THEN 1 ELSE 0 END) as processing,
+        SUM(CASE WHEN status = 'done' THEN 1 ELSE 0 END) as done,
+        SUM(CASE WHEN status = 'error' THEN 1 ELSE 0 END) as error,
+        SUM(CASE WHEN status = 'restricted' THEN 1 ELSE 0 END) as restricted,
+        SUM(CASE WHEN tt_seller = 1 AND verified = 0 ${
+          targetLocations.length
+            ? `AND location_created IN (${targetPlaceholders})`
+            : "AND 1 = 0"
+        } THEN 1 ELSE 0 END) as targetUsers,
+        SUM(CASE WHEN no_video = 1 THEN 1 ELSE 0 END) as noVideo,
+        SUM(CASE WHEN status != 'done' AND instr(COALESCE(sources, ''), '"video"') > 0 THEN 1 ELSE 0 END) as video,
+        SUM(CASE WHEN status != 'done' AND instr(COALESCE(sources, ''), '"comment"') > 0 THEN 1 ELSE 0 END) as comment,
+        SUM(CASE WHEN status != 'done' AND instr(COALESCE(sources, ''), '"guess"') > 0 THEN 1 ELSE 0 END) as guess,
+        SUM(CASE WHEN status != 'done' AND instr(COALESCE(sources, ''), '"following"') > 0 THEN 1 ELSE 0 END) as following,
+        SUM(CASE WHEN status != 'done' AND instr(COALESCE(sources, ''), '"follower"') > 0 THEN 1 ELSE 0 END) as follower,
+        SUM(CASE
+          WHEN status != 'done'
+            AND instr(COALESCE(sources, ''), '"video"') = 0
+            AND instr(COALESCE(sources, ''), '"comment"') = 0
+            AND instr(COALESCE(sources, ''), '"guess"') = 0
+            AND instr(COALESCE(sources, ''), '"following"') = 0
+            AND instr(COALESCE(sources, ''), '"follower"') = 0
+          THEN 1 ELSE 0 END) as seed
+      FROM jobs
+    `,
+    )
+    .get(...targetParams);
+
+  const userUpdateTasksRow = db
+    .prepare(
+      `SELECT COUNT(*) as userUpdateTasks FROM jobs_base WHERE COALESCE(tt_seller, '') = '' AND COALESCE(user_update_count, 0) <= 0`,
+    )
+    .get();
+
+  const countryStats = db
+    .prepare(
+      `
+      SELECT
+        COALESCE(location_created, '未知') as country,
+        COUNT(*) as count,
+        SUM(CASE
+          WHEN tt_seller = 1 AND verified = 0 ${
+            targetLocations.length
+              ? `AND location_created IN (${targetPlaceholders})`
+              : "AND 1 = 0"
+          }
+          THEN 1 ELSE 0 END) as targetCount
+      FROM jobs
+      WHERE status = 'done'
+      GROUP BY COALESCE(location_created, '未知')
+      ORDER BY count DESC
+    `,
+    )
+    .all(...targetParams);
+
+  const targetCountryStats = targetLocations.length
+    ? db
+        .prepare(
+          `
+          SELECT location_created as country, COUNT(*) as count
+          FROM jobs
+          WHERE tt_seller = 1 AND verified = 0 AND location_created IN (${targetPlaceholders})
+          GROUP BY location_created
+          ORDER BY count DESC
+        `,
+        )
+        .all(...targetLocations)
+    : [];
+
+  const jobsBaseCount = db
+    .prepare("SELECT COUNT(*) as total FROM jobs_base")
+    .get().total;
+
+  return {
+    totalUsers: aggregateRow.total,
+    rawJobs: getRawJobsCount(),
+    dbTotalUsers: getUserDbCount(),
+    jobsTotal: aggregateRow.total,
+    jobsBaseTotal: jobsBaseCount,
+    jobsPending: aggregateRow.pending,
+    processedUsers: aggregateRow.done,
+    pendingUsers: aggregateRow.pending,
+    processingUsers: aggregateRow.processing,
+    restrictedUsers: aggregateRow.restricted,
+    errorUsers: aggregateRow.error,
+    targetUsers: aggregateRow.targetUsers,
+    userUpdateTasks: userUpdateTasksRow.userUpdateTasks,
+    targetCountryStats,
+    countryStats,
+    sourceStats: {
+      seed: aggregateRow.seed || 0,
+      video: aggregateRow.video || 0,
+      comment: aggregateRow.comment || 0,
+      guess: aggregateRow.guess || 0,
+      following: aggregateRow.following || 0,
+      follower: aggregateRow.follower || 0,
+      processed: aggregateRow.done,
+      restricted: aggregateRow.restricted,
+      error: aggregateRow.error,
+      noVideo: aggregateRow.noVideo || 0,
+    },
+  };
+}
+
+export function getPendingByCountryFromDb() {
+  const db = getDb();
+  if (!db) return [];
+  return db
+    .prepare(
+      `SELECT COALESCE(guessed_location, '未知') as country, COUNT(*) as count
+       FROM jobs WHERE status = 'pending'
+       GROUP BY COALESCE(guessed_location, '未知')
+       ORDER BY count DESC`,
+    )
+    .all();
+}
+
+export function getUserUpdateByCountryFromDb() {
+  const db = getDb();
+  if (!db) return [];
+  return db
+    .prepare(
+      `SELECT COALESCE(guessed_location, '未知') as country, COUNT(*) as count
+       FROM jobs_base
+       WHERE tt_seller IS NULL AND COALESCE(user_update_count, 0) <= 0
+       GROUP BY COALESCE(guessed_location, '未知')
+       ORDER BY count DESC`,
+    )
+    .all();
+}
+
+export function getAttachStuckByCountryFromDb() {
+  const db = getDb();
+  if (!db) return [];
+  return db
+    .prepare(
+      `SELECT COALESCE(guessed_location, '未知') as country, COUNT(*) as count
+       FROM jobs_base
+       WHERE tt_seller IS NULL AND COALESCE(user_update_count, 0) = 1
+       GROUP BY COALESCE(guessed_location, '未知')
+       ORDER BY count DESC`,
+    )
+    .all();
+}
+
+export function getRawByCountryFromDb() {
+  const db = getDb();
+  if (!db) return [];
+  return db
+    .prepare(
+      `SELECT COALESCE(guessed_location, '未知') as country, COUNT(*) as count
+       FROM raw_jobs
+       GROUP BY COALESCE(guessed_location, '未知')
+       ORDER BY count DESC`,
+    )
+    .all();
+}
+
+export function restoreAttachStuckByCountry(country) {
+  const db = getDb();
+  if (!db) return { restored: 0, country, error: "db not ready" };
+
+  const normalizedCountry = String(country == null ? "未知" : country).trim();
+  if (!normalizedCountry) {
+    return {
+      restored: 0,
+      country: normalizedCountry,
+      error: "country is required",
+    };
+  }
+
+  const whereSql = `COALESCE(tt_seller, '') = '' AND COALESCE(user_update_count, 0) = 1 AND COALESCE(guessed_location, '未知') = ?`;
+  const count =
+    db
+      .prepare(`SELECT COUNT(*) as c FROM jobs_base WHERE ${whereSql}`)
+      .get(normalizedCountry)?.c || 0;
+
+  if (!count) return { restored: 0, country: normalizedCountry };
+
+  db.prepare(
+    `UPDATE jobs_base SET user_update_count = 0, updated_at = ?, claimed_by = NULL, claimed_at = NULL WHERE ${whereSql}`,
+  ).run(Date.now(), normalizedCountry);
+
+  return { restored: count, country: normalizedCountry };
+}
+
+export function resetPendingByCountry(country) {
+  const db = getDb();
+  if (!db) return { reset: 0, country, error: "db not ready" };
+
+  const normalizedCountry = String(country == null ? "未知" : country).trim();
+  if (!normalizedCountry) {
+    return {
+      reset: 0,
+      country: normalizedCountry,
+      error: "country is required",
+    };
+  }
+
+  const whereSql = `status = 'pending' AND COALESCE(guessed_location, '未知') = ?`;
+  const count =
+    db
+      .prepare(`SELECT COUNT(*) as c FROM jobs WHERE ${whereSql}`)
+      .get(normalizedCountry)?.c || 0;
+
+  if (!count) return { reset: 0, country: normalizedCountry };
+
+  db.prepare(
+    `UPDATE jobs SET user_update_count = 0, updated_at = ?, claimed_by = NULL, claimed_at = NULL WHERE ${whereSql}`,
+  ).run(Date.now(), normalizedCountry);
+
+  return { reset: count, country: normalizedCountry };
+}