tt-help-cli-ycl 1.4.26 → 1.4.28

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,105 +1,57 @@
1
1
  /**
2
- * raw_jobs 同步程序
2
+ * raw_jobs 同步与任务补充程序
3
3
  *
4
- * 定时检测主库中各目标国家的 raw_jobs 数量,不足时从归档库搬移补充。
5
- * 搬移逻辑复用 refillJobsFromRaw 非 LLM 模式的排序规则。
4
+ * 职责:
5
+ * 1. 入库 — main.raw_jobs → archive.raw_jobs
6
+ * 2. 补任务 — archive.raw_jobs → main.jobs(每国 pending<50 时触发)
7
+ *
8
+ * 模式:LLM 多轮打分凑够 50 条,不可用时回退常规(tag > comment > video_count)
6
9
  *
7
10
  * 用法:
8
- * node scripts/sync-raw-jobs.mjs --db <主库> --archive <归档库>
9
- * node scripts/sync-raw-jobs.mjs --db data/result.db --archive data/result-archive.db
10
- * node scripts/sync-raw-jobs.mjs --db data/result.db --archive data/result-archive.db --once
11
- * node scripts/sync-raw-jobs.mjs --db data/result.db --archive data/result-archive.db --interval 60 --threshold 2000 --batch 300
11
+ * node scripts/sync-raw-jobs.mjs --db <主库> --archive <归档库> [--llm] [--once] [--interval 120]
12
12
  */
13
13
 
14
14
  import Database from "better-sqlite3";
15
+ import { initDb, getDb } from "../src/watch/db-schema.js";
16
+ import { loadLlmConfig, reloadLlmConfig } from "../src/watch/llm-config.js";
17
+ import { scoreJobsBatch } from "../src/watch/llm-scoring.js";
18
+ import { DEFAULT_TARGET_LOCATIONS } from "../src/lib/target-locations.js";
15
19
 
16
- // ===== 目标国家 =====
17
- const TARGET_LOCATIONS = [
18
- "CZ",
19
- "GR",
20
- "HU",
21
- "PT",
22
- "ES",
23
- "PL",
24
- "NL",
25
- "BE",
26
- "DE",
27
- "FR",
28
- "IT",
29
- "IE",
30
- "AT",
31
- ];
32
-
33
- // ===== 命令行参数 =====
34
- function parseArgs() {
35
- const args = process.argv.slice(2);
36
- const opts = {
37
- db: null,
38
- archive: null,
39
- interval: 30,
40
- threshold: 1000,
41
- batch: 500,
42
- once: false,
43
- };
44
- for (let i = 0; i < args.length; i++) {
45
- switch (args[i]) {
46
- case "--db":
47
- opts.db = args[++i];
48
- break;
49
- case "--archive":
50
- opts.archive = args[++i];
51
- break;
52
- case "--interval":
53
- opts.interval = parseInt(args[++i]) || 30;
54
- break;
55
- case "--threshold":
56
- opts.threshold = parseInt(args[++i]) || 1000;
57
- break;
58
- case "--batch":
59
- opts.batch = parseInt(args[++i]) || 500;
60
- break;
61
- case "--once":
62
- opts.once = true;
63
- break;
64
- }
65
- }
66
- return opts;
67
- }
20
+ // ═══════════════════════════════════════════════════════════════════
21
+ // 常量
22
+ // ═══════════════════════════════════════════════════════════════════
68
23
 
69
- const opts = parseArgs();
24
+ const LOCATIONS = DEFAULT_TARGET_LOCATIONS;
70
25
 
71
- if (!opts.db || !opts.archive) {
72
- console.error(
73
- "用法: node scripts/sync-raw-jobs.mjs --db <主库路径> --archive <归档库路径> [选项]",
74
- );
75
- console.error("");
76
- console.error("选项:");
77
- console.error(" --db <path> 主库路径(必填)");
78
- console.error(" --archive <path> 归档库路径(必填)");
79
- console.error(" --interval <秒> 检测间隔,默认 30");
80
- console.error(" --threshold <条> 每个国家最低保有量,默认 1000");
81
- console.error(" --batch <条> 每次搬移上限,默认 500");
82
- console.error(" --once 只执行一次后退出");
83
- process.exit(1);
84
- }
26
+ const FILTER = "video_count>6 AND follower_count>0 AND following_count>0";
85
27
 
86
- // ===== 过滤与排序 =====
87
- const FILTER_CONDITIONS = [
88
- "COALESCE(video_count, 0) > 6",
89
- "COALESCE(follower_count, 0) > 0",
90
- "COALESCE(following_count, 0) > 0",
28
+ const REFILL_COLS = [
29
+ "unique_id",
30
+ "nickname",
31
+ "sources",
32
+ "pinned",
33
+ "tt_seller",
34
+ "verified",
35
+ "video_count",
36
+ "comment_count",
37
+ "guessed_location",
38
+ "location_created",
39
+ "confirmed_location",
40
+ "follower_count",
41
+ "following_count",
42
+ "heart_count",
43
+ "created_at",
44
+ "updated_at",
45
+ "region",
46
+ "signature",
47
+ "bio_link",
48
+ "sec_uid",
49
+ "status_code",
50
+ "latest_video_time",
51
+ "user_create_time",
91
52
  ];
92
53
 
93
- const ORDER_CLAUSE = `
94
- ORDER BY
95
- CASE WHEN sources LIKE '%tag%' THEN 0 ELSE 1 END,
96
- CASE WHEN sources LIKE '%comment%' THEN 0 ELSE 1 END,
97
- COALESCE(video_count, 0) DESC,
98
- created_at DESC
99
- `;
100
-
101
- // raw_jobs 表完整列(与 db-schema.js 对齐)
102
- const RAW_JOB_COLUMNS = [
54
+ const ARCHIVE_COLS = [
103
55
  "unique_id",
104
56
  "nickname",
105
57
  "status",
@@ -136,146 +88,436 @@ const RAW_JOB_COLUMNS = [
136
88
  "user_create_time",
137
89
  ];
138
90
 
139
- // ===== 核心:从归档库搬移一批到主库 =====
140
- function moveBatch(mainDb, archiveDb, country, limit) {
141
- const whereSql = [
142
- ...FILTER_CONDITIONS,
143
- `(UPPER(COALESCE(guessed_location, '')) = ? OR COALESCE(guessed_location, '') = '')`,
144
- ].join(" AND ");
145
-
146
- const cols = RAW_JOB_COLUMNS.join(", ");
147
-
148
- // 1. INSERT OR IGNORE 到主库
149
- const insertSql = `
150
- INSERT OR IGNORE INTO raw_jobs (${cols})
151
- SELECT ${cols}
152
- FROM (SELECT * FROM raw_jobs WHERE ${whereSql} ${ORDER_CLAUSE} LIMIT ?)
153
- `;
154
-
155
- // 2. DELETE 归档库中已搬走的数据(匹配同一批次的 unique_id)
156
- const deleteSql = `
157
- DELETE FROM raw_jobs
158
- WHERE unique_id IN (
159
- SELECT unique_id FROM raw_jobs
160
- WHERE ${whereSql} ${ORDER_CLAUSE} LIMIT ?
91
+ const LLM_MIN = 60;
92
+ const LLM_FALLBACK_KW = [
93
+ "全部模型失败",
94
+ "全部模型不可用",
95
+ "解析失败",
96
+ "调用异常",
97
+ "使用默认分",
98
+ ];
99
+
100
+ const PER_COUNTRY = 50;
101
+
102
+ function parseArgs() {
103
+ const a = process.argv.slice(2);
104
+ const o = { db: null, archive: null, interval: 120, once: false, llm: false };
105
+ for (let i = 0; i < a.length; i++) {
106
+ const v = a[i + 1];
107
+ switch (a[i]) {
108
+ case "--db":
109
+ o.db = v;
110
+ i++;
111
+ break;
112
+ case "--archive":
113
+ o.archive = v;
114
+ i++;
115
+ break;
116
+ case "--interval":
117
+ o.interval = parseInt(v) || 120;
118
+ i++;
119
+ break;
120
+ case "--llm":
121
+ o.llm = true;
122
+ break;
123
+ case "--once":
124
+ o.once = true;
125
+ break;
126
+ }
127
+ }
128
+ return o;
129
+ }
130
+
131
+ // ═══════════════════════════════════════════════════════════════════
132
+ // 工具
133
+ // ═══════════════════════════════════════════════════════════════════
134
+
135
+ const log = (tag, msg) =>
136
+ console.error(
137
+ `[sync ${new Date().toLocaleTimeString("zh-CN", { hour12: false })}] [${tag}] ${msg}`,
138
+ );
139
+ /** 按 tag > comment > other 分拣 unique_id,返回去重数组 */
140
+ function pickBySource(rows, limit) {
141
+ const ids = [
142
+ ...rows.filter((r) => r.sources?.includes('"tag"')).map((r) => r.unique_id),
143
+ ...rows
144
+ .filter(
145
+ (r) =>
146
+ r.sources?.includes('"comment"') && !r.sources?.includes('"tag"'),
147
+ )
148
+ .map((r) => r.unique_id),
149
+ ...rows
150
+ .filter(
151
+ (r) =>
152
+ !r.sources?.includes('"tag"') && !r.sources?.includes('"comment"'),
153
+ )
154
+ .map((r) => r.unique_id),
155
+ ];
156
+ ids.length = Math.min(ids.length, limit);
157
+ return ids;
158
+ }
159
+
160
+ const vals = (row, cols) => cols.map((c) => row[c] ?? null);
161
+ const ph = (n) => Array(n).fill("?").join(", ");
162
+
163
+ /** 构建 WHERE 子句(含国家过滤)。strict=true 时仅匹配明确国家,不含 NULL/空 */
164
+ function buildWhere(locations = LOCATIONS, strict = false) {
165
+ const locs = [
166
+ ...new Set(
167
+ locations.map((l) => String(l).trim().toUpperCase()).filter(Boolean),
168
+ ),
169
+ ];
170
+ const nullClause = strict
171
+ ? ""
172
+ : " OR guessed_location IS NULL OR guessed_location=''";
173
+ const sql =
174
+ locs.length === 1
175
+ ? ` AND (guessed_location = ?${nullClause})`
176
+ : locs.length > 1
177
+ ? ` AND (guessed_location IN (${ph(locs.length)})${nullClause})`
178
+ : "";
179
+ return { sql: FILTER + sql, args: locs.length ? locs : [] };
180
+ }
181
+
182
+ // ═══════════════════════════════════════════════════════════════════
183
+ // 共享:写入 jobs 并清理归档库
184
+ // ═══════════════════════════════════════════════════════════════════
185
+
186
+ function moveToJobs(archiveDb, uniqueIds, targetLocation = null) {
187
+ if (!uniqueIds.length) return 0;
188
+ const cols = REFILL_COLS.join(", ");
189
+
190
+ const rows = archiveDb
191
+ .prepare(
192
+ `SELECT ${cols} FROM raw_jobs WHERE unique_id IN (${ph(uniqueIds.length)})`,
161
193
  )
162
- `;
163
-
164
- const moved = archiveDb.transaction(() => {
165
- // 先删后插?不——先插后删,防止 INSERT OR IGNORE 冲突导致删了没插
166
- // INSERT OR IGNORE: 如果主库已有该 unique_id,跳过但不报错
167
- mainDb.prepare(insertSql).run(country, limit);
168
- const del = archiveDb.prepare(deleteSql).run(country, limit);
169
- return del.changes;
194
+ .all(...uniqueIds);
195
+ if (!rows.length) return 0;
196
+
197
+ const inserted = [];
198
+ archiveDb.transaction(() => {
199
+ const insert = getDb().prepare(
200
+ `INSERT OR IGNORE INTO task_queue (${cols}, status) VALUES (${ph(REFILL_COLS.length)}, 'pending')`,
201
+ );
202
+ for (const row of rows) {
203
+ const v = vals(row, REFILL_COLS);
204
+ if (targetLocation && !v[REFILL_COLS.indexOf("guessed_location")]) {
205
+ v[REFILL_COLS.indexOf("guessed_location")] = targetLocation;
206
+ }
207
+ if (insert.run(...v).changes) inserted.push(row.unique_id);
208
+ }
209
+ if (inserted.length) {
210
+ archiveDb
211
+ .prepare(
212
+ `DELETE FROM raw_jobs WHERE unique_id IN (${ph(inserted.length)})`,
213
+ )
214
+ .run(...inserted);
215
+ }
170
216
  })();
217
+ return inserted.length;
218
+ }
219
+
220
+ // ═══════════════════════════════════════════════════════════════════
221
+ // 入库(分批,防止大表 OOM)
222
+ // ═══════════════════════════════════════════════════════════════════
171
223
 
224
+ function archiveNewUsers(archiveDb) {
225
+ const mainDb = getDb();
226
+ const BATCH = 1000;
227
+ const cols = ARCHIVE_COLS.join(", ");
228
+ const insert = archiveDb.prepare(
229
+ `INSERT OR IGNORE INTO raw_jobs (${cols}) VALUES (${ph(ARCHIVE_COLS.length)})`,
230
+ );
231
+ let moved = 0;
232
+
233
+ while (true) {
234
+ const rows = mainDb
235
+ .prepare(`SELECT ${cols} FROM raw_jobs LIMIT ?`)
236
+ .all(BATCH);
237
+ if (!rows.length) break;
238
+
239
+ const ids = rows.map((r) => r.unique_id);
240
+
241
+ archiveDb.transaction(() => {
242
+ for (const r of rows) {
243
+ if (insert.run(...vals(r, ARCHIVE_COLS)).changes) moved++;
244
+ }
245
+ })();
246
+
247
+ mainDb
248
+ .prepare(`DELETE FROM raw_jobs WHERE unique_id IN (${ph(ids.length)})`)
249
+ .run(...ids);
250
+ }
251
+
252
+ if (moved) log(" ", `归档 +${moved}`);
172
253
  return moved;
173
254
  }
174
255
 
175
- // ===== 单次同步 =====
176
- function syncOnce(mainDb, archiveDb) {
177
- const start = Date.now();
178
- const results = {};
256
+ // ═══════════════════════════════════════════════════════════════════
257
+ // 检测(按国家)
258
+ // ═══════════════════════════════════════════════════════════════════
259
+
260
+ function countPendingByCountry() {
261
+ const rows = getDb()
262
+ .prepare(
263
+ `SELECT COALESCE(guessed_location,'') as loc, COUNT(*) as c
264
+ FROM task_queue WHERE status='pending'
265
+ GROUP BY loc`,
266
+ )
267
+ .all();
268
+ const map = {};
269
+ for (const r of rows) map[r.loc] = r.c;
270
+ // 确保所有目标国家都有值
271
+ for (const loc of LOCATIONS) {
272
+ if (!(loc in map)) map[loc] = 0;
273
+ }
274
+ return map;
275
+ }
276
+
277
+ function findNeedyCountries(byCountry, threshold) {
278
+ return LOCATIONS.filter((loc) => (byCountry[loc] || 0) < threshold);
279
+ }
280
+
281
+ // ═══════════════════════════════════════════════════════════════════
282
+ // 常规 refill(索引直查 + JS 分拣,避免 sources LIKE 大表扫描)
283
+ // ═══════════════════════════════════════════════════════════════════
284
+
285
+ function refillRegular(archiveDb, limit, locations) {
286
+ const { sql: w, args: wa } = buildWhere(locations);
287
+
288
+ // 取 limit×3 条,JS 分拣 tag > comment > other
289
+ const rows = archiveDb
290
+ .prepare(
291
+ `SELECT unique_id, sources FROM raw_jobs WHERE ${w} ORDER BY video_count DESC LIMIT ?`,
292
+ )
293
+ .all(...wa, limit * 3);
294
+ const ids = pickBySource(rows, limit);
295
+
296
+ if (!ids.length) return { moved: 0 };
297
+ const moved = moveToJobs(archiveDb, ids);
298
+ return { moved };
299
+ }
300
+
301
+ // ═══════════════════════════════════════════════════════════════════
302
+ // LLM refill — 逐国:按 scored_count 升序取 PER_COUNTRY 条,LLM 打分,通过进 jobs,不通过 +1
303
+ async function refillLLM(archiveDb, needyLocations) {
304
+ let totalMoved = 0;
305
+
306
+ log("--", `LLM开始 ${needyLocations.length}国: ${needyLocations.join(" ")}`);
179
307
 
180
- for (const country of TARGET_LOCATIONS) {
181
- // 检查主库中该国存量
182
- const countRow = mainDb
308
+ for (const loc of needyLocations) {
309
+ const { sql: w, args: wa } = buildWhere([loc], true);
310
+ const { sql: wf, args: wfa } = buildWhere([loc]);
311
+
312
+ // 1. 查询候选:取 100 条给 LLM(多于目标 50,增加通过率减少回退)
313
+ const LLM_BATCH = 100;
314
+ const tq = Date.now();
315
+ const samples = archiveDb
183
316
  .prepare(
184
- `SELECT COUNT(*) as c FROM raw_jobs
185
- WHERE UPPER(COALESCE(guessed_location, '')) = ?`,
317
+ `SELECT * FROM raw_jobs WHERE ${w} AND (llm_scored_count IS NULL OR llm_scored_count<3) ORDER BY COALESCE(llm_scored_count,0) ASC, video_count DESC LIMIT ?`,
186
318
  )
187
- .get(country);
319
+ .all(...wa, LLM_BATCH);
320
+ log(" ", `${loc} 候选${samples.length}条 ${Date.now() - tq}ms`);
321
+
322
+ if (!samples.length) {
323
+ log(" ", `${loc} 回退常规取 ${PER_COUNTRY}条...`);
324
+ const rows = archiveDb
325
+ .prepare(
326
+ `SELECT unique_id, sources FROM raw_jobs WHERE ${wf} ORDER BY video_count DESC LIMIT ?`,
327
+ )
328
+ .all(...wfa, PER_COUNTRY * 3);
329
+ const ids = pickBySource(rows, PER_COUNTRY);
330
+ const moved = moveToJobs(archiveDb, ids, loc);
331
+ totalMoved += moved;
332
+ log(" ", `${loc} → ${moved}条 回退`);
333
+ continue;
334
+ }
188
335
 
189
- const current = countRow?.c || 0;
336
+ // 2. LLM 打分
337
+ const ts = Date.now();
338
+ const scores = await scoreJobsBatch(samples, LOCATIONS);
339
+ log(" ", `${loc} 打分${scores.length}条 ${Date.now() - ts}ms`);
190
340
 
191
- if (current >= opts.threshold) {
341
+ if (
342
+ scores.every(
343
+ (s) =>
344
+ s.score === 50 && LLM_FALLBACK_KW.some((kw) => s.reason.includes(kw)),
345
+ )
346
+ ) {
347
+ log(" ", `${loc} LLM全部不可用,回退...`);
348
+ const rows = archiveDb
349
+ .prepare(
350
+ `SELECT unique_id, sources FROM raw_jobs WHERE ${wf} ORDER BY video_count DESC LIMIT ?`,
351
+ )
352
+ .all(...wfa, PER_COUNTRY * 3);
353
+ const ids = pickBySource(rows, PER_COUNTRY);
354
+ const moved = moveToJobs(archiveDb, ids, loc);
355
+ totalMoved += moved;
356
+ log(" ", `${loc} → ${moved}条 回退`);
192
357
  continue;
193
358
  }
194
359
 
195
- const need = opts.threshold - current;
196
- const limit = Math.min(need, opts.batch);
360
+ // 3. 通过 → jobs+归档删;不通过 → scored_count+1
361
+ const passIds = [],
362
+ failIds = [];
363
+ for (const s of scores) {
364
+ if (s.score >= LLM_MIN) passIds.push(s.uniqueId);
365
+ else failIds.push(s.uniqueId);
366
+ }
367
+ log(" ", `${loc} 通过${passIds.length} 淘汰${failIds.length}`);
368
+
369
+ if (failIds.length) {
370
+ archiveDb
371
+ .prepare(
372
+ `UPDATE raw_jobs SET llm_scored_count=COALESCE(llm_scored_count,0)+1 WHERE unique_id IN (${ph(failIds.length)})`,
373
+ )
374
+ .run(...failIds);
375
+ }
197
376
 
198
- const moved = moveBatch(mainDb, archiveDb, country, limit);
377
+ // 4. 写入 jobs
378
+ let moved = 0;
379
+ if (passIds.length) {
380
+ moved = moveToJobs(archiveDb, passIds, loc);
381
+ }
382
+ // 不够 → 回退补
383
+ if (moved < PER_COUNTRY) {
384
+ const gap = PER_COUNTRY - moved;
385
+ const rows = archiveDb
386
+ .prepare(
387
+ `SELECT unique_id, sources FROM raw_jobs WHERE ${wf} ORDER BY video_count DESC LIMIT ?`,
388
+ )
389
+ .all(...wfa, gap * 3);
390
+ const fbIds = pickBySource(rows, gap);
391
+ moved += moveToJobs(archiveDb, fbIds, loc);
392
+ }
393
+ totalMoved += moved;
394
+ log(" ", `${loc} → ${moved}条 LLM`);
395
+ }
396
+
397
+ log("--", `LLM完成 共写入${totalMoved}条`);
398
+ return { moved: totalMoved };
399
+ }
400
+ // ═══════════════════════════════════════════════════════════════════
401
+ // 单轮调度
402
+ let tickN = 0;
403
+
404
+ async function tick(archiveDb, opts) {
405
+ tickN++;
406
+ const t0 = Date.now();
407
+
408
+ archiveNewUsers(archiveDb);
409
+ if (opts.llm) reloadLlmConfig(); // 每轮重载,支持热切换模型
410
+
411
+ const byCountry = countPendingByCountry();
412
+ const needy = findNeedyCountries(byCountry, PER_COUNTRY);
413
+
414
+ if (!needy.length) {
415
+ log(
416
+ "--",
417
+ `轮${tickN} 全部充足 | ${LOCATIONS.map((l) => `${l}:${byCountry[l]}`).join(" ")}`,
418
+ );
419
+ return;
420
+ }
199
421
 
200
- results[country] = { before: current, moved, after: current + moved };
422
+ log(
423
+ "==",
424
+ `轮${tickN} 需补 ${needy.map((l) => `${l}(${byCountry[l]}→+${PER_COUNTRY})`).join(" ")}`,
425
+ );
201
426
 
427
+ if (opts.llm) {
428
+ const r = await refillLLM(archiveDb, needy);
429
+ const s = ((Date.now() - t0) / 1000).toFixed(1);
430
+ log(">>", `轮${tickN} 写入${r.moved}条 ${s}s`);
431
+ } else {
432
+ let total = 0;
433
+ for (const loc of needy) {
434
+ const r = refillRegular(archiveDb, PER_COUNTRY, [loc]);
435
+ total += r.moved;
436
+ log(" ", `${loc} +${r.moved}/${PER_COUNTRY}`);
437
+ }
438
+ const s = ((Date.now() - t0) / 1000).toFixed(1);
439
+ log(">>", `轮${tickN} 写入${total}条 ${s}s`);
440
+ }
441
+ }
442
+
443
+ // ═══════════════════════════════════════════════════════════════════
444
+ // 主入口
445
+ // ═══════════════════════════════════════════════════════════════════
446
+
447
+ async function main() {
448
+ const opts = parseArgs();
449
+ if (!opts.db || !opts.archive) {
202
450
  console.error(
203
- `[sync] ${country}: ${current} → ${current + moved} (搬入 ${moved} 条)`,
451
+ "用法: node scripts/sync-raw-jobs.mjs --db <主库> --archive <归档库> [--llm] [--once] [--interval 120]",
204
452
  );
453
+ process.exit(1);
205
454
  }
206
455
 
207
- const totalMoved = Object.values(results).reduce((s, r) => s + r.moved, 0);
208
- const elapsed = ((Date.now() - start) / 1000).toFixed(1);
456
+ log(
457
+ "INIT",
458
+ `LLM=${opts.llm ? "ON" : "OFF"} 每国${PER_COUNTRY}条 ${opts.once ? "单次" : "间隔" + opts.interval + "s"}`,
459
+ );
209
460
 
210
- if (totalMoved > 0) {
211
- console.error(`[sync] 本轮共搬入 ${totalMoved} 条,耗时 ${elapsed}s`);
461
+ initDb(opts.db);
462
+ loadLlmConfig();
463
+
464
+ const archiveDb = new Database(opts.archive);
465
+ archiveDb.pragma("journal_mode = WAL");
466
+ if (
467
+ !archiveDb
468
+ .prepare(
469
+ "SELECT name FROM sqlite_master WHERE type='table' AND name='raw_jobs'",
470
+ )
471
+ .get()
472
+ ) {
473
+ console.error("[sync] 归档库无 raw_jobs 表");
474
+ process.exit(1);
212
475
  }
476
+ log(
477
+ "INIT",
478
+ `归档 ${archiveDb.prepare("SELECT COUNT(*) as c FROM raw_jobs").get().c.toLocaleString()} 条`,
479
+ );
213
480
 
214
- return results;
215
- }
481
+ archiveDb.exec(
482
+ `CREATE INDEX IF NOT EXISTS idx_raw_refill ON raw_jobs(guessed_location, video_count DESC, created_at DESC) WHERE video_count>6 AND follower_count>0 AND following_count>0`,
483
+ );
484
+ // tag/回退查询用:无国家过滤,按 video_count 排序
485
+ archiveDb.exec(
486
+ `CREATE INDEX IF NOT EXISTS idx_raw_vc ON raw_jobs(video_count DESC) WHERE video_count>6 AND follower_count>0 AND following_count>0`,
487
+ );
216
488
 
217
- // ===== 主流程 =====
218
- console.error("[sync] raw_jobs 同步程序启动");
219
- console.error(`[sync] 主库: ${opts.db}`);
220
- console.error(`[sync] 归档库: ${opts.archive}`);
221
- console.error(`[sync] 最低保有量: ${opts.threshold} 条/国家`);
222
- console.error(`[sync] 检测间隔: ${opts.once ? "仅一次" : opts.interval + "s"}`);
223
- console.error("");
224
-
225
- // 打开数据库
226
- const mainDb = new Database(opts.db);
227
- mainDb.pragma("journal_mode = WAL");
228
- const archiveDb = new Database(opts.archive);
229
- archiveDb.pragma("journal_mode = WAL");
230
-
231
- // 检查主库 raw_jobs 是否存在
232
- const mainTables = mainDb
233
- .prepare(
234
- "SELECT name FROM sqlite_master WHERE type='table' AND name='raw_jobs'",
235
- )
236
- .get();
237
- if (!mainTables) {
238
- console.error("[sync] 错误: 主库中不存在 raw_jobs 表");
239
- process.exit(1);
240
- }
489
+ // LLM 打分排序索引
490
+ archiveDb.exec(
491
+ `CREATE INDEX IF NOT EXISTS idx_raw_llm_score ON raw_jobs(llm_scored_count, video_count DESC) WHERE video_count>6 AND follower_count>0 AND following_count>0`,
492
+ );
493
+ try {
494
+ archiveDb.exec(
495
+ "ALTER TABLE raw_jobs ADD COLUMN llm_scored_count INTEGER DEFAULT 0",
496
+ );
497
+ } catch {
498
+ /* exists */
499
+ }
241
500
 
242
- // 检查归档库 raw_jobs 是否存在
243
- const archiveTables = archiveDb
244
- .prepare(
245
- "SELECT name FROM sqlite_master WHERE type='table' AND name='raw_jobs'",
246
- )
247
- .get();
248
- if (!archiveTables) {
249
- console.error("[sync] 错误: 归档库中不存在 raw_jobs 表");
250
- process.exit(1);
251
- }
501
+ await tick(archiveDb, opts);
502
+ if (opts.once) {
503
+ archiveDb.close();
504
+ getDb().close();
505
+ log("DONE", "退出");
506
+ process.exit(0);
507
+ }
252
508
 
253
- const archiveTotal =
254
- archiveDb.prepare("SELECT COUNT(*) as c FROM raw_jobs").get()?.c || 0;
255
- console.error(
256
- `[sync] 归档库 raw_jobs 总量: ${archiveTotal.toLocaleString()} 条`,
257
- );
258
- console.error("");
259
-
260
- // 执行同步
261
- syncOnce(mainDb, archiveDb);
262
-
263
- if (opts.once) {
264
- mainDb.close();
265
- archiveDb.close();
266
- console.error("[sync] 完成,退出");
267
- process.exit(0);
509
+ const timer = setInterval(() => tick(archiveDb, opts), opts.interval * 1000);
510
+ const stop = () => {
511
+ clearInterval(timer);
512
+ archiveDb.close();
513
+ getDb().close();
514
+ process.exit(0);
515
+ };
516
+ process.on("SIGINT", stop);
517
+ process.on("SIGTERM", stop);
268
518
  }
269
519
 
270
- // 定时循环
271
- setInterval(() => {
272
- syncOnce(mainDb, archiveDb);
273
- }, opts.interval * 1000);
274
-
275
- // 优雅退出
276
- process.on("SIGINT", () => {
277
- console.error("\n[sync] 收到中断信号,退出...");
278
- mainDb.close();
279
- archiveDb.close();
280
- process.exit(0);
520
+ main().catch((e) => {
521
+ console.error("[sync] FATAL:", e);
522
+ process.exit(1);
281
523
  });