tt-help-cli-ycl 1.4.26 → 1.4.27

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tt-help-cli-ycl",
-  "version": "1.4.26",
+  "version": "1.4.27",
   "description": "TikTok user & video data scraper - extract ttSeller, verified, locationCreated from HTML source",
   "type": "module",
   "bin": {

package/scripts/sync-raw-jobs.mjs

@@ -13,22 +13,6 @@
 
 import Database from "better-sqlite3";
 
-// ===== 目标国家 =====
-const TARGET_LOCATIONS = [
-  "CZ",
-  "GR",
-  "HU",
-  "PT",
-  "ES",
-  "PL",
-  "NL",
-  "BE",
-  "DE",
-  "FR",
-  "IT",
-  "IE",
-  "AT",
-];
 
 // ===== 命令行参数 =====
 function parseArgs() {
@@ -37,7 +21,7 @@ function parseArgs() {
     db: null,
     archive: null,
     interval: 30,
-    threshold: 1000,
+    max: 10000,
     batch: 500,
     once: false,
   };
@@ -52,8 +36,8 @@ function parseArgs() {
       case "--interval":
         opts.interval = parseInt(args[++i]) || 30;
         break;
-      case "--threshold":
-        opts.threshold = parseInt(args[++i]) || 1000;
+      case "--max":
+        opts.max = parseInt(args[++i]) || 10000;
         break;
       case "--batch":
         opts.batch = parseInt(args[++i]) || 500;
@@ -77,24 +61,25 @@ if (!opts.db || !opts.archive) {
   console.error("  --db <path>       主库路径（必填）");
   console.error("  --archive <path>  归档库路径（必填）");
   console.error("  --interval <秒>   检测间隔，默认 30");
-  console.error("  --threshold <条>  每个国家最低保有量，默认 1000");
+  console.error("  --max <条>        主库 raw_jobs 保有量上限，默认 10000");
   console.error("  --batch <条>      每次搬移上限，默认 500");
   console.error("  --once            只执行一次后退出");
   process.exit(1);
 }
 
 // ===== 过滤与排序 =====
-const FILTER_CONDITIONS = [
-  "COALESCE(video_count, 0) > 6",
-  "COALESCE(follower_count, 0) > 0",
-  "COALESCE(following_count, 0) > 0",
+// 注意: 索引依赖的 WHERE 用原始列名（不用 COALESCE），与 idx_raw_archive_sync 匹配
+const FILTER_INDEXED = [
+  "video_count > 6",
+  "follower_count > 0",
+  "following_count > 0",
 ];
 
 const ORDER_CLAUSE = `
   ORDER BY
     CASE WHEN sources LIKE '%tag%' THEN 0 ELSE 1 END,
     CASE WHEN sources LIKE '%comment%' THEN 0 ELSE 1 END,
-    COALESCE(video_count, 0) DESC,
+    video_count DESC,
     created_at DESC
 `;
 
@@ -136,23 +121,17 @@ const RAW_JOB_COLUMNS = [
   "user_create_time",
 ];
 
-// ===== 核心：从归档库搬移一批到主库 =====
-function moveBatch(mainDb, archiveDb, country, limit) {
-  const whereSql = [
-    ...FILTER_CONDITIONS,
-    `(UPPER(COALESCE(guessed_location, '')) = ? OR COALESCE(guessed_location, '') = '')`,
-  ].join(" AND ");
-
+// ===== 核心：从归档库搬移一批到主库（全局排序，不区分国家）=====
+function moveBatch(mainDb, archiveDb, limit) {
+  const whereSql = FILTER_INDEXED.join(" AND ");
   const cols = RAW_JOB_COLUMNS.join(", ");
 
-  // 1. INSERT OR IGNORE 到主库
   const insertSql = `
     INSERT OR IGNORE INTO raw_jobs (${cols})
     SELECT ${cols}
     FROM (SELECT * FROM raw_jobs WHERE ${whereSql} ${ORDER_CLAUSE} LIMIT ?)
   `;
 
-  // 2. DELETE 归档库中已搬走的数据（匹配同一批次的 unique_id）
   const deleteSql = `
     DELETE FROM raw_jobs
     WHERE unique_id IN (
@@ -162,10 +141,8 @@ function moveBatch(mainDb, archiveDb, country, limit) {
   `;
 
   const moved = archiveDb.transaction(() => {
-    // 先删后插？不——先插后删，防止 INSERT OR IGNORE 冲突导致删了没插
-    // INSERT OR IGNORE: 如果主库已有该 unique_id，跳过但不报错
-    mainDb.prepare(insertSql).run(country, limit);
-    const del = archiveDb.prepare(deleteSql).run(country, limit);
+    mainDb.prepare(insertSql).run(limit);
+    const del = archiveDb.prepare(deleteSql).run(limit);
     return del.changes;
   })();
 
@@ -173,52 +150,35 @@ function moveBatch(mainDb, archiveDb, country, limit) {
 }
 
 // ===== 单次同步 =====
-function syncOnce(mainDb, archiveDb) {
+function syncOnce(mainDb, archiveDb, maxTotal, batch) {
   const start = Date.now();
-  const results = {};
-
-  for (const country of TARGET_LOCATIONS) {
-    // 检查主库中该国存量
-    const countRow = mainDb
-      .prepare(
-        `SELECT COUNT(*) as c FROM raw_jobs
-         WHERE UPPER(COALESCE(guessed_location, '')) = ?`,
-      )
-      .get(country);
-
-    const current = countRow?.c || 0;
-
-    if (current >= opts.threshold) {
-      continue;
-    }
-
-    const need = opts.threshold - current;
-    const limit = Math.min(need, opts.batch);
 
-    const moved = moveBatch(mainDb, archiveDb, country, limit);
+  // 主库 raw_jobs 总数够了就不搬
+  const currentTotal =
+    mainDb.prepare("SELECT COUNT(*) as c FROM raw_jobs").get()?.c || 0;
 
-    results[country] = { before: current, moved, after: current + moved };
-
-    console.error(
-      `[sync] ${country}: ${current} → ${current + moved} (搬入 ${moved} 条)`,
-    );
+  if (currentTotal >= maxTotal) {
+    return { moved: 0, total: currentTotal };
   }
 
-  const totalMoved = Object.values(results).reduce((s, r) => s + r.moved, 0);
+  const need = maxTotal - currentTotal;
+  const limit = Math.min(need, batch);
+
+  const moved = moveBatch(mainDb, archiveDb, limit);
   const elapsed = ((Date.now() - start) / 1000).toFixed(1);
 
-  if (totalMoved > 0) {
-    console.error(`[sync] 本轮共搬入 ${totalMoved} 条，耗时 ${elapsed}s`);
-  }
+  console.error(
+    `[sync] ${currentTotal} → ${currentTotal + moved} (搬入 ${moved} 条)，耗时 ${elapsed}s`,
+  );
 
-  return results;
+  return { moved, total: currentTotal + moved };
 }
 
 // ===== 主流程 =====
 console.error("[sync] raw_jobs 同步程序启动");
 console.error(`[sync] 主库: ${opts.db}`);
 console.error(`[sync] 归档库: ${opts.archive}`);
-console.error(`[sync] 最低保有量: ${opts.threshold} 条/国家`);
+console.error(`[sync] 主库上限: ${opts.max} 条，每次搬 ${opts.batch} 条`);
 console.error(`[sync] 检测间隔: ${opts.once ? "仅一次" : opts.interval + "s"}`);
 console.error("");
 
@@ -255,10 +215,23 @@ const archiveTotal =
 console.error(
   `[sync] 归档库 raw_jobs 总量: ${archiveTotal.toLocaleString()} 条`,
 );
+
+// 确保归档库有加速索引（首次运行自动创建）
+const startIdx = Date.now();
+archiveDb.exec(`
+  CREATE INDEX IF NOT EXISTS idx_raw_archive_sync
+  ON raw_jobs(video_count DESC, created_at DESC)
+  WHERE video_count > 6 AND follower_count > 0 AND following_count > 0;
+`);
+const idxElapsed = ((Date.now() - startIdx) / 1000).toFixed(1);
+if (idxElapsed > 0.1) {
+  console.error(`[sync] 归档库索引就绪，耗时 ${idxElapsed}s`);
+}
 console.error("");
 
 // 执行同步
-syncOnce(mainDb, archiveDb);
+const MAX_TOTAL = opts.max ?? 10000;
+syncOnce(mainDb, archiveDb, MAX_TOTAL, opts.batch);
 
 if (opts.once) {
   mainDb.close();
@@ -269,7 +242,7 @@ if (opts.once) {
 
 // 定时循环
 setInterval(() => {
-  syncOnce(mainDb, archiveDb);
+  syncOnce(mainDb, archiveDb, MAX_TOTAL, opts.batch);
 }, opts.interval * 1000);
 
 // 优雅退出

package/src/cli/raw-sync.js

@@ -16,8 +16,8 @@ export function handleRawSync(parsed) {
 
   if (parsed.rawSyncInterval != null)
     args.push("--interval", String(parsed.rawSyncInterval));
-  if (parsed.rawSyncThreshold != null)
-    args.push("--threshold", String(parsed.rawSyncThreshold));
+  if (parsed.rawSyncMax != null)
+    args.push("--max", String(parsed.rawSyncMax));
   if (parsed.rawSyncBatch != null)
     args.push("--batch", String(parsed.rawSyncBatch));
   if (parsed.rawSyncOnce) args.push("--once");

package/src/lib/args.js

@@ -690,7 +690,7 @@ function parseRawSyncArgs(args) {
   let rawSyncDb = null;
   let rawSyncArchive = null;
   let rawSyncInterval = null;
-  let rawSyncThreshold = null;
+  let rawSyncMax = null;
   let rawSyncBatch = null;
   let rawSyncOnce = false;
 
@@ -702,8 +702,8 @@ function parseRawSyncArgs(args) {
       rawSyncArchive = args[++i];
     } else if (arg === "--interval" && i + 1 < args.length) {
       rawSyncInterval = parseInt(args[++i]) || null;
-    } else if (arg === "--threshold" && i + 1 < args.length) {
-      rawSyncThreshold = parseInt(args[++i]) || null;
+    } else if (arg === "--max" && i + 1 < args.length) {
+      rawSyncMax = parseInt(args[++i]) || null;
     } else if (arg === "--batch" && i + 1 < args.length) {
       rawSyncBatch = parseInt(args[++i]) || null;
     } else if (arg === "--once") {
@@ -716,7 +716,7 @@ function parseRawSyncArgs(args) {
     rawSyncDb,
     rawSyncArchive,
     rawSyncInterval,
-    rawSyncThreshold,
+    rawSyncMax,
     rawSyncBatch,
     rawSyncOnce,
     urls: [],

package/src/lib/constants.js

@@ -315,17 +315,17 @@ const HELP_TEXT = [
   "          tt-help tag refresh --port 8222 -s http://127.0.0.1:3001",
   "",
   "  raw-sync --db <主库路径> --archive <归档库路径> [选项]",
-  "    从归档库向主库 raw_jobs 同步数据，按国家保持最低保有量",
+  "    从归档库向主库 raw_jobs 同步数据，保持主库总量不超上限",
   "    选项:",
   "      --db <路径>            主库路径（必填）",
   "      --archive <路径>       归档库路径（必填）",
   "      --interval <秒>        检测间隔，默认 30",
-  "      --threshold <条>       每个国家最低保有量，默认 1000",
-  "      --batch <条>           每次搬移上限，默认 500",
+  "      --max <条>             主库 raw_jobs 保有量上限，默认 10000",
+  "      --batch <条>           每次搬移条数，默认 500",
   "      --once                 只执行一次后退出",
   "    示例: tt-help raw-sync --db data/result.db --archive data/result-archive.db",
   "          tt-help raw-sync --db data/result.db --archive data/result-archive.db --once",
-  "          tt-help raw-sync --db data/result.db --archive data/result-archive.db --interval 60 --threshold 2000",
+  "          tt-help raw-sync --db data/result.db --archive data/result-archive.db --interval 60 --max 20000",
   "",
   "  config  [show|set|unset|reset]",
   "    config                    查看当前配置",