tt-help-cli-ycl 1.3.92 → 1.3.94

package/src/watch/db-columns.js

@@ -0,0 +1,160 @@
+/**
+ * 共享列名常量 — 消除 SQL 中重复的列名定义
+ *
+ * jobs / raw_jobs / jobs_base 三张表共享大部分列，
+ * 之前每次 INSERT 都手写 30+ 列名，改一个字段要改 7 处。
+ * 现在统一在这里定义，所有模块引用即可。
+ */
+
+// ===== jobs / raw_jobs / jobs_base 共享的核心列 =====
+export const JOB_CORE_COLUMNS = [
+  "unique_id",
+  "nickname",
+  "status",
+  "sources",
+  "claimed_by",
+  "claimed_at",
+  "error",
+  "pinned",
+  "no_video",
+  "restricted",
+  "user_update_count",
+  "tt_seller",
+  "verified",
+  "video_count",
+  "comment_count",
+  "guessed_location",
+  "location_created",
+  "confirmed_location",
+  "modified_at",
+  "follower_count",
+  "following_count",
+  "heart_count",
+  "refresh_time",
+  "processed",
+  "processed_at",
+  "created_at",
+  "updated_at",
+  "region",
+  "signature",
+  "bio_link",
+  "sec_uid",
+];
+
+// jobs 表独有的扩展列（raw_jobs / jobs_base 也逐步迁移过来）
+export const JOB_EXT_COLUMNS = [
+  "status_code",
+  "latest_video_time",
+  "user_create_time",
+  "top_video_play_count",
+  "top_video_href",
+];
+
+// 完整的 jobs 列 = 核心列 + 扩展列
+export const JOB_FULL_COLUMNS = [...JOB_CORE_COLUMNS, ...JOB_EXT_COLUMNS];
+
+// ===== 可写入的列集合（updateJobInfo / updateJobBaseInfo 使用） =====
+export const WRITABLE_JOB_COLUMNS = new Set([
+  "nickname",
+  "status",
+  "sources",
+  "claimed_by",
+  "claimed_at",
+  "error",
+  "pinned",
+  "no_video",
+  "restricted",
+  "user_update_count",
+  "tt_seller",
+  "verified",
+  "video_count",
+  "comment_count",
+  "guessed_location",
+  "location_created",
+  "confirmed_location",
+  "modified_at",
+  "follower_count",
+  "following_count",
+  "heart_count",
+  "refresh_time",
+  "processed",
+  "processed_at",
+  "updated_at",
+  "region",
+  "signature",
+  "bio_link",
+  "sec_uid",
+  "status_code",
+  "latest_video_time",
+  "top_video_play_count",
+  "top_video_href",
+  "user_create_time",
+]);
+
+// ===== Boolean 列（存储时 0/1，读取时转为 true/false/null） =====
+export const JOB_BOOLEAN_COLUMNS = new Set([
+  "pinned",
+  "no_video",
+  "restricted",
+  "processed",
+  "tt_seller",
+  "verified",
+  "error",
+]);
+
+export const VIDEO_BOOLEAN_COLUMNS = new Set(["tt_seller"]);
+
+// ===== videos 表列 =====
+export const VIDEO_COLUMNS = [
+  "id",
+  "href",
+  "author_unique_id",
+  "location_created",
+  "tt_seller",
+  "registered_at",
+  "user_update_count",
+  "play_count",
+  "digg_count",
+  "comment_count",
+  "share_count",
+  "collect_count",
+  "stats_updated_at",
+  "create_time",
+];
+
+// ===== SQL 生成辅助 =====
+
+/** 生成列名列表字符串，如 "unique_id, nickname, status, ..." */
+export function columnsList(columns) {
+  return columns.join(", ");
+}
+
+/** 生成 VALUES 占位符，如 "?, ?, ?, ..." */
+export function valuesPlaceholders(columns) {
+  return columns.map(() => "?").join(", ");
+}
+
+/** 生成 INSERT OR IGNORE INTO 语句 */
+export function insertIgnoreSql(table, columns) {
+  return `INSERT OR IGNORE INTO ${table} (${columnsList(columns)}) VALUES (${valuesPlaceholders(columns)})`;
+}
+
+/** 生成 INSERT OR REPLACE INTO 语句 */
+export function insertReplaceSql(table, columns) {
+  return `INSERT OR REPLACE INTO ${table} (${columnsList(columns)}) VALUES (${valuesPlaceholders(columns)})`;
+}
+
+/** 生成 INSERT ... SELECT 语句（从 sourceTable 复制到 targetTable） */
+export function insertSelectSql(
+  targetTable,
+  sourceTable,
+  columns,
+  whereSql = "",
+) {
+  return `INSERT OR IGNORE INTO ${targetTable} (${columnsList(columns)}) SELECT ${columnsList(columns)} FROM ${sourceTable}${whereSql ? ` WHERE ${whereSql}` : ""}`;
+}
+
+/** 生成 UPDATE SET 子句，如 "nickname = ?, status = ?, ..." */
+export function updateSetClause(columns) {
+  return columns.map((c) => `${c} = ?`).join(", ");
+}

package/src/watch/db-crud.js

@@ -0,0 +1,458 @@
+/**
+ * 数据库 CRUD 操作
+ *
+ * 基础的增删改查：用户判重、添加、更新、行映射等。
+ * 所有函数依赖 db-schema.js 的 getDb() 获取数据库连接。
+ */
+
+import {
+  JOB_BOOLEAN_COLUMNS,
+  VIDEO_BOOLEAN_COLUMNS,
+  WRITABLE_JOB_COLUMNS,
+  JOB_CORE_COLUMNS,
+  columnsList,
+  valuesPlaceholders,
+  insertIgnoreSql,
+  updateSetClause,
+} from "./db-columns.js";
+import { getDb } from "./db-schema.js";
+
+// ===== 行映射工具 =====
+
+export function snakeToCamel(key) {
+  return key.replace(/_([a-z])/g, (_, ch) => ch.toUpperCase());
+}
+
+export function camelToSnake(key) {
+  return key.replace(/[A-Z]/g, (ch) => `_${ch.toLowerCase()}`);
+}
+
+export function normalizeJobValue(column, value) {
+  if (value === undefined || value === null) return null;
+  if (column === "sources") {
+    if (!Array.isArray(value)) return JSON.stringify([]);
+    return JSON.stringify([...new Set(value)]);
+  }
+  if (JOB_BOOLEAN_COLUMNS.has(column)) {
+    return value ? 1 : 0;
+  }
+  if (typeof value === "object") return JSON.stringify(value);
+  return value;
+}
+
+export function mapJobRow(row) {
+  if (!row) return undefined;
+  const mapped = {};
+  for (const [key, value] of Object.entries(row)) {
+    const camelKey = snakeToCamel(key);
+    if (key === "sources") {
+      try {
+        mapped[camelKey] = value ? JSON.parse(value) : [];
+      } catch {
+        mapped[camelKey] = [];
+      }
+      continue;
+    }
+    if (JOB_BOOLEAN_COLUMNS.has(key)) {
+      mapped[camelKey] = value === null || value === undefined ? null : !!value;
+      continue;
+    }
+    mapped[camelKey] = value;
+  }
+  return mapped;
+}
+
+export function mapVideoRow(row) {
+  if (!row) return undefined;
+  const mapped = {};
+  for (const [key, value] of Object.entries(row)) {
+    const camelKey = snakeToCamel(key);
+    if (VIDEO_BOOLEAN_COLUMNS.has(key)) {
+      mapped[camelKey] = value === null || value === undefined ? null : !!value;
+      continue;
+    }
+    mapped[camelKey] = value;
+  }
+  return mapped;
+}
+
+export function inferStatus(u) {
+  if (u.restricted) return "restricted";
+  if (u.error) return "error";
+  if (u.processed) return "done";
+  return "pending";
+}
+
+// ===== 用户判重 =====
+
+export function hasUserInDb(uid) {
+  const db = getDb();
+  if (!db) return false;
+  const row = db.prepare("SELECT 1 FROM users WHERE unique_id = ?").get(uid);
+  return !!row;
+}
+
+// ===== 添加用户 =====
+
+export function addUserToDb(user) {
+  const db = getDb();
+  if (!db) return;
+  db.prepare(
+    `INSERT OR IGNORE INTO users (unique_id, tt_seller, verified, location_created, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?)`,
+  ).run(
+    user.uniqueId,
+    user.ttSeller === undefined ||
+      user.ttSeller === null ||
+      user.ttSeller === ""
+      ? null
+      : user.ttSeller
+        ? 1
+        : 0,
+    user.verified === undefined ||
+      user.verified === null ||
+      user.verified === ""
+      ? null
+      : user.verified
+        ? 1
+        : 0,
+    user.locationCreated || null,
+    new Date().toISOString(),
+    new Date().toISOString(),
+  );
+}
+
+// ===== 添加 Job =====
+
+export function addJobToDb(user) {
+  const db = getDb();
+  if (!db) return;
+  const now = Date.now();
+  const columns = [
+    "unique_id",
+    "nickname",
+    "status",
+    "sources",
+    "claimed_by",
+    "claimed_at",
+    "error",
+    "pinned",
+    "no_video",
+    "restricted",
+    "user_update_count",
+    "tt_seller",
+    "verified",
+    "video_count",
+    "comment_count",
+    "guessed_location",
+    "location_created",
+    "follower_count",
+    "following_count",
+    "heart_count",
+    "refresh_time",
+    "processed",
+    "processed_at",
+    "created_at",
+    "updated_at",
+    "region",
+    "signature",
+    "bio_link",
+    "sec_uid",
+  ];
+  db.prepare(insertIgnoreSql("jobs", columns)).run(
+    user.uniqueId,
+    user.nickname || null,
+    user.status || inferStatus(user),
+    JSON.stringify(
+      Array.isArray(user.sources) ? [...new Set(user.sources)] : [],
+    ),
+    user.claimedBy || null,
+    user.claimedAt || null,
+    user.error || null,
+    user.pinned ? 1 : 0,
+    user.noVideo ? 1 : 0,
+    user.restricted ? 1 : 0,
+    user.userUpdateCount || 0,
+    user.ttSeller === undefined ||
+      user.ttSeller === null ||
+      user.ttSeller === ""
+      ? null
+      : user.ttSeller
+        ? 1
+        : 0,
+    user.verified === undefined ||
+      user.verified === null ||
+      user.verified === ""
+      ? null
+      : user.verified
+        ? 1
+        : 0,
+    user.videoCount || 0,
+    user.commentCount || 0,
+    user.guessedLocation || null,
+    user.locationCreated || null,
+    user.followerCount || 0,
+    user.followingCount || 0,
+    user.heartCount || 0,
+    user.refreshTime || null,
+    user.processed ? 1 : 0,
+    user.processedAt || null,
+    user.createdAt || now,
+    user.updatedAt || now,
+    user.region || null,
+    user.signature || null,
+    user.bioLink?.link || user.bioLink?.url || user.bioLink || null,
+    user.secUid || null,
+  );
+}
+
+// ===== 添加 JobBase =====
+
+export function addJobBaseToDb(user) {
+  const db = getDb();
+  if (!db) return;
+  const now = Date.now();
+  const columns = [
+    "unique_id",
+    "nickname",
+    "status",
+    "sources",
+    "claimed_by",
+    "claimed_at",
+    "error",
+    "pinned",
+    "no_video",
+    "restricted",
+    "user_update_count",
+    "tt_seller",
+    "verified",
+    "video_count",
+    "comment_count",
+    "guessed_location",
+    "location_created",
+    "follower_count",
+    "following_count",
+    "heart_count",
+    "refresh_time",
+    "processed",
+    "processed_at",
+    "created_at",
+    "updated_at",
+    "region",
+    "signature",
+    "bio_link",
+    "sec_uid",
+  ];
+  db.prepare(insertIgnoreSql("jobs_base", columns)).run(
+    user.uniqueId,
+    user.nickname || null,
+    user.status || inferStatus(user),
+    JSON.stringify(
+      Array.isArray(user.sources) ? [...new Set(user.sources)] : [],
+    ),
+    user.claimedBy || null,
+    user.claimedAt || null,
+    user.error || null,
+    user.pinned ? 1 : 0,
+    user.noVideo ? 1 : 0,
+    user.restricted ? 1 : 0,
+    user.userUpdateCount || 0,
+    user.ttSeller === undefined ||
+      user.ttSeller === null ||
+      user.ttSeller === ""
+      ? null
+      : user.ttSeller
+        ? 1
+        : 0,
+    user.verified === undefined ||
+      user.verified === null ||
+      user.verified === ""
+      ? null
+      : user.verified
+        ? 1
+        : 0,
+    user.videoCount || 0,
+    user.commentCount || 0,
+    user.guessedLocation || null,
+    user.locationCreated || null,
+    user.followerCount || 0,
+    user.followingCount || 0,
+    user.heartCount || 0,
+    user.refreshTime || null,
+    user.processed ? 1 : 0,
+    user.processedAt || null,
+    user.createdAt || now,
+    user.updatedAt || now,
+    user.region || null,
+    user.signature || null,
+    user.bioLink?.link || user.bioLink?.url || user.bioLink || null,
+    user.secUid || null,
+  );
+}
+
+// ===== 添加 Job（事务：users + jobs） =====
+
+export function addJob(user) {
+  const db = getDb();
+  if (!db) {
+    addUserToDb(user);
+    return;
+  }
+  if (!user.status) user.status = inferStatus(user);
+  if (!user.createdAt) user.createdAt = Date.now();
+  if (!user.updatedAt) user.updatedAt = user.createdAt;
+  const writeTxn = db.transaction((job) => {
+    addUserToDb(job);
+    addJobToDb(job);
+  });
+  writeTxn(user);
+}
+
+// ===== 查询 Job =====
+
+export function getJobRow(uniqueId) {
+  const db = getDb();
+  if (!db) return null;
+  return db.prepare("SELECT * FROM jobs WHERE unique_id = ?").get(uniqueId);
+}
+
+export function getJobBaseRow(uniqueId) {
+  const db = getDb();
+  if (!db) return null;
+  return db
+    .prepare("SELECT * FROM jobs_base WHERE unique_id = ?")
+    .get(uniqueId);
+}
+
+export function getJob(uniqueId) {
+  return mapJobRow(getJobRow(uniqueId));
+}
+
+export function getAllJobs() {
+  const db = getDb();
+  if (!db) return [];
+  return db.prepare("SELECT * FROM jobs").all().map(mapJobRow);
+}
+
+// ===== 查询 Video =====
+
+export function getVideoRow(videoId) {
+  const db = getDb();
+  if (!db) return null;
+  return db.prepare("SELECT * FROM videos WHERE id = ?").get(videoId);
+}
+
+export function getAllVideoRows() {
+  const db = getDb();
+  if (!db) return [];
+  return db.prepare("SELECT * FROM videos").all();
+}
+
+// ===== 更新 Job 信息 =====
+
+export function updateJobInfo(uniqueId, info, incrementCount = true) {
+  const db = getDb();
+  if (!db) return { error: "db not initialized" };
+  const existing = getJobRow(uniqueId);
+  if (!existing) return { error: "user not found" };
+
+  const nextValues = {};
+  for (const [key, value] of Object.entries(info || {})) {
+    if (key === "uniqueId" || key === "unique_id") continue;
+    if (value === undefined || value === "") continue;
+    let column = camelToSnake(key);
+    if (column === "bio") column = "signature";
+    if (column === "create_time") column = "user_create_time";
+    if (!WRITABLE_JOB_COLUMNS.has(column)) continue;
+    nextValues[column] = normalizeJobValue(column, value);
+  }
+
+  nextValues.updated_at = Date.now();
+  if (incrementCount) {
+    nextValues.user_update_count = (existing.user_update_count || 0) + 1;
+  }
+
+  const columns = Object.keys(nextValues);
+  if (columns.length > 0) {
+    const sql = `UPDATE jobs SET ${updateSetClause(columns)} WHERE unique_id = ?`;
+    db.prepare(sql).run(...columns.map((c) => nextValues[c]), uniqueId);
+  }
+
+  return {
+    ok: true,
+    userUpdateCount:
+      nextValues.user_update_count ?? existing.user_update_count ?? 0,
+  };
+}
+
+export function updateJobBaseInfo(uniqueId, info, incrementCount = true) {
+  const db = getDb();
+  if (!db) return { error: "db not initialized" };
+  const existing = getJobBaseRow(uniqueId);
+  if (!existing) return { error: "user not found" };
+
+  const nextValues = {};
+  for (const [key, value] of Object.entries(info || {})) {
+    if (key === "uniqueId" || key === "unique_id") continue;
+    if (value === undefined || value === "") continue;
+    let column = camelToSnake(key);
+    if (column === "bio") column = "signature";
+    if (column === "create_time") column = "user_create_time";
+    if (!WRITABLE_JOB_COLUMNS.has(column)) continue;
+    nextValues[column] = normalizeJobValue(column, value);
+  }
+
+  nextValues.updated_at = Date.now();
+  if (incrementCount) {
+    nextValues.user_update_count = (existing.user_update_count || 0) + 1;
+  }
+
+  const columns = Object.keys(nextValues);
+  if (columns.length > 0) {
+    const sql = `UPDATE jobs_base SET ${updateSetClause(columns)} WHERE unique_id = ?`;
+    db.prepare(sql).run(...columns.map((c) => nextValues[c]), uniqueId);
+  }
+
+  return {
+    ok: true,
+    userUpdateCount:
+      nextValues.user_update_count ?? existing.user_update_count ?? 0,
+  };
+}
+
+// ===== 计数 =====
+
+export function getUserDbCount() {
+  const db = getDb();
+  if (!db) return 0;
+  return db.prepare("SELECT COUNT(*) as c FROM users").get().c;
+}
+
+export function getJobsCount() {
+  const db = getDb();
+  if (!db) return 0;
+  return db.prepare("SELECT COUNT(*) as c FROM jobs").get().c;
+}
+
+export function getPendingJobsCount() {
+  const db = getDb();
+  if (!db) return 0;
+  return db
+    .prepare("SELECT COUNT(*) as c FROM jobs WHERE status = 'pending'")
+    .get().c;
+}
+
+export function getPendingJobsUserUpdateCount() {
+  const db = getDb();
+  if (!db) return 0;
+  return db
+    .prepare(
+      `SELECT COUNT(*) as c FROM jobs WHERE COALESCE(tt_seller, '') = '' AND COALESCE(user_update_count, 0) <= 0`,
+    )
+    .get().c;
+}
+
+export function getRawJobsCount() {
+  const db = getDb();
+  if (!db) return 0;
+  return db.prepare("SELECT COUNT(*) as c FROM raw_jobs").get().c;
+}