tt-help-cli-ycl 1.4.26 → 1.4.28
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/scripts/search-authors.mjs +21 -0
- package/scripts/sync-raw-jobs.mjs +444 -202
- package/src/cli/raw-sync.js +2 -5
- package/src/cli/search.js +160 -0
- package/src/lib/args.js +78 -5
- package/src/lib/constants.js +24 -5
- package/src/lib/search-collector.js +194 -0
- package/src/lib/target-locations.js +1 -0
- package/src/main.js +3 -0
- package/src/npm-main.js +3 -0
- package/src/watch/data-store.js +266 -577
- package/src/watch/db-columns.js +1 -0
- package/src/watch/db-crud.js +27 -3
- package/src/watch/db-raw-jobs.js +3 -3
- package/src/watch/db-schema.js +100 -0
- package/src/watch/db-stats.js +46 -23
- package/src/watch/db-tags.js +100 -1
- package/src/watch/llm-config.js +145 -1
- package/src/watch/llm-scoring.js +60 -117
- package/src/watch/public/app.js +5 -5
- package/src/watch/public/index.html +4 -5
- package/src/watch/public/style.css +16 -0
- package/src/watch/server.js +170 -59
- package/src/watch/tag-service.js +11 -67
|
@@ -1,105 +1,57 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* raw_jobs
|
|
2
|
+
* raw_jobs 同步与任务补充程序
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
4
|
+
* 职责:
|
|
5
|
+
* 1. 入库 — main.raw_jobs → archive.raw_jobs
|
|
6
|
+
* 2. 补任务 — archive.raw_jobs → main.jobs(每国 pending<50 时触发)
|
|
7
|
+
*
|
|
8
|
+
* 模式:LLM 多轮打分凑够 50 条,不可用时回退常规(tag > comment > video_count)
|
|
6
9
|
*
|
|
7
10
|
* 用法:
|
|
8
|
-
* node scripts/sync-raw-jobs.mjs --db <主库> --archive <归档库>
|
|
9
|
-
* node scripts/sync-raw-jobs.mjs --db data/result.db --archive data/result-archive.db
|
|
10
|
-
* node scripts/sync-raw-jobs.mjs --db data/result.db --archive data/result-archive.db --once
|
|
11
|
-
* node scripts/sync-raw-jobs.mjs --db data/result.db --archive data/result-archive.db --interval 60 --threshold 2000 --batch 300
|
|
11
|
+
* node scripts/sync-raw-jobs.mjs --db <主库> --archive <归档库> [--llm] [--once] [--interval 120]
|
|
12
12
|
*/
|
|
13
13
|
|
|
14
14
|
import Database from "better-sqlite3";
|
|
15
|
+
import { initDb, getDb } from "../src/watch/db-schema.js";
|
|
16
|
+
import { loadLlmConfig, reloadLlmConfig } from "../src/watch/llm-config.js";
|
|
17
|
+
import { scoreJobsBatch } from "../src/watch/llm-scoring.js";
|
|
18
|
+
import { DEFAULT_TARGET_LOCATIONS } from "../src/lib/target-locations.js";
|
|
15
19
|
|
|
16
|
-
//
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
"GR",
|
|
20
|
-
"HU",
|
|
21
|
-
"PT",
|
|
22
|
-
"ES",
|
|
23
|
-
"PL",
|
|
24
|
-
"NL",
|
|
25
|
-
"BE",
|
|
26
|
-
"DE",
|
|
27
|
-
"FR",
|
|
28
|
-
"IT",
|
|
29
|
-
"IE",
|
|
30
|
-
"AT",
|
|
31
|
-
];
|
|
32
|
-
|
|
33
|
-
// ===== 命令行参数 =====
|
|
34
|
-
function parseArgs() {
|
|
35
|
-
const args = process.argv.slice(2);
|
|
36
|
-
const opts = {
|
|
37
|
-
db: null,
|
|
38
|
-
archive: null,
|
|
39
|
-
interval: 30,
|
|
40
|
-
threshold: 1000,
|
|
41
|
-
batch: 500,
|
|
42
|
-
once: false,
|
|
43
|
-
};
|
|
44
|
-
for (let i = 0; i < args.length; i++) {
|
|
45
|
-
switch (args[i]) {
|
|
46
|
-
case "--db":
|
|
47
|
-
opts.db = args[++i];
|
|
48
|
-
break;
|
|
49
|
-
case "--archive":
|
|
50
|
-
opts.archive = args[++i];
|
|
51
|
-
break;
|
|
52
|
-
case "--interval":
|
|
53
|
-
opts.interval = parseInt(args[++i]) || 30;
|
|
54
|
-
break;
|
|
55
|
-
case "--threshold":
|
|
56
|
-
opts.threshold = parseInt(args[++i]) || 1000;
|
|
57
|
-
break;
|
|
58
|
-
case "--batch":
|
|
59
|
-
opts.batch = parseInt(args[++i]) || 500;
|
|
60
|
-
break;
|
|
61
|
-
case "--once":
|
|
62
|
-
opts.once = true;
|
|
63
|
-
break;
|
|
64
|
-
}
|
|
65
|
-
}
|
|
66
|
-
return opts;
|
|
67
|
-
}
|
|
20
|
+
// ═══════════════════════════════════════════════════════════════════
|
|
21
|
+
// 常量
|
|
22
|
+
// ═══════════════════════════════════════════════════════════════════
|
|
68
23
|
|
|
69
|
-
const
|
|
24
|
+
const LOCATIONS = DEFAULT_TARGET_LOCATIONS;
|
|
70
25
|
|
|
71
|
-
|
|
72
|
-
console.error(
|
|
73
|
-
"用法: node scripts/sync-raw-jobs.mjs --db <主库路径> --archive <归档库路径> [选项]",
|
|
74
|
-
);
|
|
75
|
-
console.error("");
|
|
76
|
-
console.error("选项:");
|
|
77
|
-
console.error(" --db <path> 主库路径(必填)");
|
|
78
|
-
console.error(" --archive <path> 归档库路径(必填)");
|
|
79
|
-
console.error(" --interval <秒> 检测间隔,默认 30");
|
|
80
|
-
console.error(" --threshold <条> 每个国家最低保有量,默认 1000");
|
|
81
|
-
console.error(" --batch <条> 每次搬移上限,默认 500");
|
|
82
|
-
console.error(" --once 只执行一次后退出");
|
|
83
|
-
process.exit(1);
|
|
84
|
-
}
|
|
26
|
+
const FILTER = "video_count>6 AND follower_count>0 AND following_count>0";
|
|
85
27
|
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
"
|
|
89
|
-
"
|
|
90
|
-
"
|
|
28
|
+
const REFILL_COLS = [
|
|
29
|
+
"unique_id",
|
|
30
|
+
"nickname",
|
|
31
|
+
"sources",
|
|
32
|
+
"pinned",
|
|
33
|
+
"tt_seller",
|
|
34
|
+
"verified",
|
|
35
|
+
"video_count",
|
|
36
|
+
"comment_count",
|
|
37
|
+
"guessed_location",
|
|
38
|
+
"location_created",
|
|
39
|
+
"confirmed_location",
|
|
40
|
+
"follower_count",
|
|
41
|
+
"following_count",
|
|
42
|
+
"heart_count",
|
|
43
|
+
"created_at",
|
|
44
|
+
"updated_at",
|
|
45
|
+
"region",
|
|
46
|
+
"signature",
|
|
47
|
+
"bio_link",
|
|
48
|
+
"sec_uid",
|
|
49
|
+
"status_code",
|
|
50
|
+
"latest_video_time",
|
|
51
|
+
"user_create_time",
|
|
91
52
|
];
|
|
92
53
|
|
|
93
|
-
const
|
|
94
|
-
ORDER BY
|
|
95
|
-
CASE WHEN sources LIKE '%tag%' THEN 0 ELSE 1 END,
|
|
96
|
-
CASE WHEN sources LIKE '%comment%' THEN 0 ELSE 1 END,
|
|
97
|
-
COALESCE(video_count, 0) DESC,
|
|
98
|
-
created_at DESC
|
|
99
|
-
`;
|
|
100
|
-
|
|
101
|
-
// raw_jobs 表完整列(与 db-schema.js 对齐)
|
|
102
|
-
const RAW_JOB_COLUMNS = [
|
|
54
|
+
const ARCHIVE_COLS = [
|
|
103
55
|
"unique_id",
|
|
104
56
|
"nickname",
|
|
105
57
|
"status",
|
|
@@ -136,146 +88,436 @@ const RAW_JOB_COLUMNS = [
|
|
|
136
88
|
"user_create_time",
|
|
137
89
|
];
|
|
138
90
|
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
91
|
+
const LLM_MIN = 60;
|
|
92
|
+
const LLM_FALLBACK_KW = [
|
|
93
|
+
"全部模型失败",
|
|
94
|
+
"全部模型不可用",
|
|
95
|
+
"解析失败",
|
|
96
|
+
"调用异常",
|
|
97
|
+
"使用默认分",
|
|
98
|
+
];
|
|
99
|
+
|
|
100
|
+
const PER_COUNTRY = 50;
|
|
101
|
+
|
|
102
|
+
function parseArgs() {
|
|
103
|
+
const a = process.argv.slice(2);
|
|
104
|
+
const o = { db: null, archive: null, interval: 120, once: false, llm: false };
|
|
105
|
+
for (let i = 0; i < a.length; i++) {
|
|
106
|
+
const v = a[i + 1];
|
|
107
|
+
switch (a[i]) {
|
|
108
|
+
case "--db":
|
|
109
|
+
o.db = v;
|
|
110
|
+
i++;
|
|
111
|
+
break;
|
|
112
|
+
case "--archive":
|
|
113
|
+
o.archive = v;
|
|
114
|
+
i++;
|
|
115
|
+
break;
|
|
116
|
+
case "--interval":
|
|
117
|
+
o.interval = parseInt(v) || 120;
|
|
118
|
+
i++;
|
|
119
|
+
break;
|
|
120
|
+
case "--llm":
|
|
121
|
+
o.llm = true;
|
|
122
|
+
break;
|
|
123
|
+
case "--once":
|
|
124
|
+
o.once = true;
|
|
125
|
+
break;
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
return o;
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
// ═══════════════════════════════════════════════════════════════════
|
|
132
|
+
// 工具
|
|
133
|
+
// ═══════════════════════════════════════════════════════════════════
|
|
134
|
+
|
|
135
|
+
const log = (tag, msg) =>
|
|
136
|
+
console.error(
|
|
137
|
+
`[sync ${new Date().toLocaleTimeString("zh-CN", { hour12: false })}] [${tag}] ${msg}`,
|
|
138
|
+
);
|
|
139
|
+
/** 按 tag > comment > other 分拣 unique_id,返回去重数组 */
|
|
140
|
+
function pickBySource(rows, limit) {
|
|
141
|
+
const ids = [
|
|
142
|
+
...rows.filter((r) => r.sources?.includes('"tag"')).map((r) => r.unique_id),
|
|
143
|
+
...rows
|
|
144
|
+
.filter(
|
|
145
|
+
(r) =>
|
|
146
|
+
r.sources?.includes('"comment"') && !r.sources?.includes('"tag"'),
|
|
147
|
+
)
|
|
148
|
+
.map((r) => r.unique_id),
|
|
149
|
+
...rows
|
|
150
|
+
.filter(
|
|
151
|
+
(r) =>
|
|
152
|
+
!r.sources?.includes('"tag"') && !r.sources?.includes('"comment"'),
|
|
153
|
+
)
|
|
154
|
+
.map((r) => r.unique_id),
|
|
155
|
+
];
|
|
156
|
+
ids.length = Math.min(ids.length, limit);
|
|
157
|
+
return ids;
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
const vals = (row, cols) => cols.map((c) => row[c] ?? null);
|
|
161
|
+
const ph = (n) => Array(n).fill("?").join(", ");
|
|
162
|
+
|
|
163
|
+
/** 构建 WHERE 子句(含国家过滤)。strict=true 时仅匹配明确国家,不含 NULL/空 */
|
|
164
|
+
function buildWhere(locations = LOCATIONS, strict = false) {
|
|
165
|
+
const locs = [
|
|
166
|
+
...new Set(
|
|
167
|
+
locations.map((l) => String(l).trim().toUpperCase()).filter(Boolean),
|
|
168
|
+
),
|
|
169
|
+
];
|
|
170
|
+
const nullClause = strict
|
|
171
|
+
? ""
|
|
172
|
+
: " OR guessed_location IS NULL OR guessed_location=''";
|
|
173
|
+
const sql =
|
|
174
|
+
locs.length === 1
|
|
175
|
+
? ` AND (guessed_location = ?${nullClause})`
|
|
176
|
+
: locs.length > 1
|
|
177
|
+
? ` AND (guessed_location IN (${ph(locs.length)})${nullClause})`
|
|
178
|
+
: "";
|
|
179
|
+
return { sql: FILTER + sql, args: locs.length ? locs : [] };
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
// ═══════════════════════════════════════════════════════════════════
|
|
183
|
+
// 共享:写入 jobs 并清理归档库
|
|
184
|
+
// ═══════════════════════════════════════════════════════════════════
|
|
185
|
+
|
|
186
|
+
function moveToJobs(archiveDb, uniqueIds, targetLocation = null) {
|
|
187
|
+
if (!uniqueIds.length) return 0;
|
|
188
|
+
const cols = REFILL_COLS.join(", ");
|
|
189
|
+
|
|
190
|
+
const rows = archiveDb
|
|
191
|
+
.prepare(
|
|
192
|
+
`SELECT ${cols} FROM raw_jobs WHERE unique_id IN (${ph(uniqueIds.length)})`,
|
|
161
193
|
)
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
194
|
+
.all(...uniqueIds);
|
|
195
|
+
if (!rows.length) return 0;
|
|
196
|
+
|
|
197
|
+
const inserted = [];
|
|
198
|
+
archiveDb.transaction(() => {
|
|
199
|
+
const insert = getDb().prepare(
|
|
200
|
+
`INSERT OR IGNORE INTO task_queue (${cols}, status) VALUES (${ph(REFILL_COLS.length)}, 'pending')`,
|
|
201
|
+
);
|
|
202
|
+
for (const row of rows) {
|
|
203
|
+
const v = vals(row, REFILL_COLS);
|
|
204
|
+
if (targetLocation && !v[REFILL_COLS.indexOf("guessed_location")]) {
|
|
205
|
+
v[REFILL_COLS.indexOf("guessed_location")] = targetLocation;
|
|
206
|
+
}
|
|
207
|
+
if (insert.run(...v).changes) inserted.push(row.unique_id);
|
|
208
|
+
}
|
|
209
|
+
if (inserted.length) {
|
|
210
|
+
archiveDb
|
|
211
|
+
.prepare(
|
|
212
|
+
`DELETE FROM raw_jobs WHERE unique_id IN (${ph(inserted.length)})`,
|
|
213
|
+
)
|
|
214
|
+
.run(...inserted);
|
|
215
|
+
}
|
|
170
216
|
})();
|
|
217
|
+
return inserted.length;
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
// ═══════════════════════════════════════════════════════════════════
|
|
221
|
+
// 入库(分批,防止大表 OOM)
|
|
222
|
+
// ═══════════════════════════════════════════════════════════════════
|
|
171
223
|
|
|
224
|
+
function archiveNewUsers(archiveDb) {
|
|
225
|
+
const mainDb = getDb();
|
|
226
|
+
const BATCH = 1000;
|
|
227
|
+
const cols = ARCHIVE_COLS.join(", ");
|
|
228
|
+
const insert = archiveDb.prepare(
|
|
229
|
+
`INSERT OR IGNORE INTO raw_jobs (${cols}) VALUES (${ph(ARCHIVE_COLS.length)})`,
|
|
230
|
+
);
|
|
231
|
+
let moved = 0;
|
|
232
|
+
|
|
233
|
+
while (true) {
|
|
234
|
+
const rows = mainDb
|
|
235
|
+
.prepare(`SELECT ${cols} FROM raw_jobs LIMIT ?`)
|
|
236
|
+
.all(BATCH);
|
|
237
|
+
if (!rows.length) break;
|
|
238
|
+
|
|
239
|
+
const ids = rows.map((r) => r.unique_id);
|
|
240
|
+
|
|
241
|
+
archiveDb.transaction(() => {
|
|
242
|
+
for (const r of rows) {
|
|
243
|
+
if (insert.run(...vals(r, ARCHIVE_COLS)).changes) moved++;
|
|
244
|
+
}
|
|
245
|
+
})();
|
|
246
|
+
|
|
247
|
+
mainDb
|
|
248
|
+
.prepare(`DELETE FROM raw_jobs WHERE unique_id IN (${ph(ids.length)})`)
|
|
249
|
+
.run(...ids);
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
if (moved) log(" ", `归档 +${moved}`);
|
|
172
253
|
return moved;
|
|
173
254
|
}
|
|
174
255
|
|
|
175
|
-
//
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
256
|
+
// ═══════════════════════════════════════════════════════════════════
|
|
257
|
+
// 检测(按国家)
|
|
258
|
+
// ═══════════════════════════════════════════════════════════════════
|
|
259
|
+
|
|
260
|
+
function countPendingByCountry() {
|
|
261
|
+
const rows = getDb()
|
|
262
|
+
.prepare(
|
|
263
|
+
`SELECT COALESCE(guessed_location,'') as loc, COUNT(*) as c
|
|
264
|
+
FROM task_queue WHERE status='pending'
|
|
265
|
+
GROUP BY loc`,
|
|
266
|
+
)
|
|
267
|
+
.all();
|
|
268
|
+
const map = {};
|
|
269
|
+
for (const r of rows) map[r.loc] = r.c;
|
|
270
|
+
// 确保所有目标国家都有值
|
|
271
|
+
for (const loc of LOCATIONS) {
|
|
272
|
+
if (!(loc in map)) map[loc] = 0;
|
|
273
|
+
}
|
|
274
|
+
return map;
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
function findNeedyCountries(byCountry, threshold) {
|
|
278
|
+
return LOCATIONS.filter((loc) => (byCountry[loc] || 0) < threshold);
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
// ═══════════════════════════════════════════════════════════════════
|
|
282
|
+
// 常规 refill(索引直查 + JS 分拣,避免 sources LIKE 大表扫描)
|
|
283
|
+
// ═══════════════════════════════════════════════════════════════════
|
|
284
|
+
|
|
285
|
+
function refillRegular(archiveDb, limit, locations) {
|
|
286
|
+
const { sql: w, args: wa } = buildWhere(locations);
|
|
287
|
+
|
|
288
|
+
// 取 limit×3 条,JS 分拣 tag > comment > other
|
|
289
|
+
const rows = archiveDb
|
|
290
|
+
.prepare(
|
|
291
|
+
`SELECT unique_id, sources FROM raw_jobs WHERE ${w} ORDER BY video_count DESC LIMIT ?`,
|
|
292
|
+
)
|
|
293
|
+
.all(...wa, limit * 3);
|
|
294
|
+
const ids = pickBySource(rows, limit);
|
|
295
|
+
|
|
296
|
+
if (!ids.length) return { moved: 0 };
|
|
297
|
+
const moved = moveToJobs(archiveDb, ids);
|
|
298
|
+
return { moved };
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
// ═══════════════════════════════════════════════════════════════════
|
|
302
|
+
// LLM refill — 逐国:按 scored_count 升序取 PER_COUNTRY 条,LLM 打分,通过进 jobs,不通过 +1
|
|
303
|
+
async function refillLLM(archiveDb, needyLocations) {
|
|
304
|
+
let totalMoved = 0;
|
|
305
|
+
|
|
306
|
+
log("--", `LLM开始 ${needyLocations.length}国: ${needyLocations.join(" ")}`);
|
|
179
307
|
|
|
180
|
-
for (const
|
|
181
|
-
|
|
182
|
-
const
|
|
308
|
+
for (const loc of needyLocations) {
|
|
309
|
+
const { sql: w, args: wa } = buildWhere([loc], true);
|
|
310
|
+
const { sql: wf, args: wfa } = buildWhere([loc]);
|
|
311
|
+
|
|
312
|
+
// 1. 查询候选:取 100 条给 LLM(多于目标 50,增加通过率减少回退)
|
|
313
|
+
const LLM_BATCH = 100;
|
|
314
|
+
const tq = Date.now();
|
|
315
|
+
const samples = archiveDb
|
|
183
316
|
.prepare(
|
|
184
|
-
`SELECT
|
|
185
|
-
WHERE UPPER(COALESCE(guessed_location, '')) = ?`,
|
|
317
|
+
`SELECT * FROM raw_jobs WHERE ${w} AND (llm_scored_count IS NULL OR llm_scored_count<3) ORDER BY COALESCE(llm_scored_count,0) ASC, video_count DESC LIMIT ?`,
|
|
186
318
|
)
|
|
187
|
-
.
|
|
319
|
+
.all(...wa, LLM_BATCH);
|
|
320
|
+
log(" ", `${loc} 候选${samples.length}条 ${Date.now() - tq}ms`);
|
|
321
|
+
|
|
322
|
+
if (!samples.length) {
|
|
323
|
+
log(" ", `${loc} 回退常规取 ${PER_COUNTRY}条...`);
|
|
324
|
+
const rows = archiveDb
|
|
325
|
+
.prepare(
|
|
326
|
+
`SELECT unique_id, sources FROM raw_jobs WHERE ${wf} ORDER BY video_count DESC LIMIT ?`,
|
|
327
|
+
)
|
|
328
|
+
.all(...wfa, PER_COUNTRY * 3);
|
|
329
|
+
const ids = pickBySource(rows, PER_COUNTRY);
|
|
330
|
+
const moved = moveToJobs(archiveDb, ids, loc);
|
|
331
|
+
totalMoved += moved;
|
|
332
|
+
log(" ", `${loc} → ${moved}条 回退`);
|
|
333
|
+
continue;
|
|
334
|
+
}
|
|
188
335
|
|
|
189
|
-
|
|
336
|
+
// 2. LLM 打分
|
|
337
|
+
const ts = Date.now();
|
|
338
|
+
const scores = await scoreJobsBatch(samples, LOCATIONS);
|
|
339
|
+
log(" ", `${loc} 打分${scores.length}条 ${Date.now() - ts}ms`);
|
|
190
340
|
|
|
191
|
-
if (
|
|
341
|
+
if (
|
|
342
|
+
scores.every(
|
|
343
|
+
(s) =>
|
|
344
|
+
s.score === 50 && LLM_FALLBACK_KW.some((kw) => s.reason.includes(kw)),
|
|
345
|
+
)
|
|
346
|
+
) {
|
|
347
|
+
log(" ", `${loc} LLM全部不可用,回退...`);
|
|
348
|
+
const rows = archiveDb
|
|
349
|
+
.prepare(
|
|
350
|
+
`SELECT unique_id, sources FROM raw_jobs WHERE ${wf} ORDER BY video_count DESC LIMIT ?`,
|
|
351
|
+
)
|
|
352
|
+
.all(...wfa, PER_COUNTRY * 3);
|
|
353
|
+
const ids = pickBySource(rows, PER_COUNTRY);
|
|
354
|
+
const moved = moveToJobs(archiveDb, ids, loc);
|
|
355
|
+
totalMoved += moved;
|
|
356
|
+
log(" ", `${loc} → ${moved}条 回退`);
|
|
192
357
|
continue;
|
|
193
358
|
}
|
|
194
359
|
|
|
195
|
-
|
|
196
|
-
const
|
|
360
|
+
// 3. 通过 → jobs+归档删;不通过 → scored_count+1
|
|
361
|
+
const passIds = [],
|
|
362
|
+
failIds = [];
|
|
363
|
+
for (const s of scores) {
|
|
364
|
+
if (s.score >= LLM_MIN) passIds.push(s.uniqueId);
|
|
365
|
+
else failIds.push(s.uniqueId);
|
|
366
|
+
}
|
|
367
|
+
log(" ", `${loc} 通过${passIds.length} 淘汰${failIds.length}`);
|
|
368
|
+
|
|
369
|
+
if (failIds.length) {
|
|
370
|
+
archiveDb
|
|
371
|
+
.prepare(
|
|
372
|
+
`UPDATE raw_jobs SET llm_scored_count=COALESCE(llm_scored_count,0)+1 WHERE unique_id IN (${ph(failIds.length)})`,
|
|
373
|
+
)
|
|
374
|
+
.run(...failIds);
|
|
375
|
+
}
|
|
197
376
|
|
|
198
|
-
|
|
377
|
+
// 4. 写入 jobs
|
|
378
|
+
let moved = 0;
|
|
379
|
+
if (passIds.length) {
|
|
380
|
+
moved = moveToJobs(archiveDb, passIds, loc);
|
|
381
|
+
}
|
|
382
|
+
// 不够 → 回退补
|
|
383
|
+
if (moved < PER_COUNTRY) {
|
|
384
|
+
const gap = PER_COUNTRY - moved;
|
|
385
|
+
const rows = archiveDb
|
|
386
|
+
.prepare(
|
|
387
|
+
`SELECT unique_id, sources FROM raw_jobs WHERE ${wf} ORDER BY video_count DESC LIMIT ?`,
|
|
388
|
+
)
|
|
389
|
+
.all(...wfa, gap * 3);
|
|
390
|
+
const fbIds = pickBySource(rows, gap);
|
|
391
|
+
moved += moveToJobs(archiveDb, fbIds, loc);
|
|
392
|
+
}
|
|
393
|
+
totalMoved += moved;
|
|
394
|
+
log(" ", `${loc} → ${moved}条 LLM`);
|
|
395
|
+
}
|
|
396
|
+
|
|
397
|
+
log("--", `LLM完成 共写入${totalMoved}条`);
|
|
398
|
+
return { moved: totalMoved };
|
|
399
|
+
}
|
|
400
|
+
// ═══════════════════════════════════════════════════════════════════
|
|
401
|
+
// 单轮调度
|
|
402
|
+
let tickN = 0;
|
|
403
|
+
|
|
404
|
+
async function tick(archiveDb, opts) {
|
|
405
|
+
tickN++;
|
|
406
|
+
const t0 = Date.now();
|
|
407
|
+
|
|
408
|
+
archiveNewUsers(archiveDb);
|
|
409
|
+
if (opts.llm) reloadLlmConfig(); // 每轮重载,支持热切换模型
|
|
410
|
+
|
|
411
|
+
const byCountry = countPendingByCountry();
|
|
412
|
+
const needy = findNeedyCountries(byCountry, PER_COUNTRY);
|
|
413
|
+
|
|
414
|
+
if (!needy.length) {
|
|
415
|
+
log(
|
|
416
|
+
"--",
|
|
417
|
+
`轮${tickN} 全部充足 | ${LOCATIONS.map((l) => `${l}:${byCountry[l]}`).join(" ")}`,
|
|
418
|
+
);
|
|
419
|
+
return;
|
|
420
|
+
}
|
|
199
421
|
|
|
200
|
-
|
|
422
|
+
log(
|
|
423
|
+
"==",
|
|
424
|
+
`轮${tickN} 需补 ${needy.map((l) => `${l}(${byCountry[l]}→+${PER_COUNTRY})`).join(" ")}`,
|
|
425
|
+
);
|
|
201
426
|
|
|
427
|
+
if (opts.llm) {
|
|
428
|
+
const r = await refillLLM(archiveDb, needy);
|
|
429
|
+
const s = ((Date.now() - t0) / 1000).toFixed(1);
|
|
430
|
+
log(">>", `轮${tickN} 写入${r.moved}条 ${s}s`);
|
|
431
|
+
} else {
|
|
432
|
+
let total = 0;
|
|
433
|
+
for (const loc of needy) {
|
|
434
|
+
const r = refillRegular(archiveDb, PER_COUNTRY, [loc]);
|
|
435
|
+
total += r.moved;
|
|
436
|
+
log(" ", `${loc} +${r.moved}/${PER_COUNTRY}`);
|
|
437
|
+
}
|
|
438
|
+
const s = ((Date.now() - t0) / 1000).toFixed(1);
|
|
439
|
+
log(">>", `轮${tickN} 写入${total}条 ${s}s`);
|
|
440
|
+
}
|
|
441
|
+
}
|
|
442
|
+
|
|
443
|
+
// ═══════════════════════════════════════════════════════════════════
|
|
444
|
+
// 主入口
|
|
445
|
+
// ═══════════════════════════════════════════════════════════════════
|
|
446
|
+
|
|
447
|
+
async function main() {
|
|
448
|
+
const opts = parseArgs();
|
|
449
|
+
if (!opts.db || !opts.archive) {
|
|
202
450
|
console.error(
|
|
203
|
-
|
|
451
|
+
"用法: node scripts/sync-raw-jobs.mjs --db <主库> --archive <归档库> [--llm] [--once] [--interval 120]",
|
|
204
452
|
);
|
|
453
|
+
process.exit(1);
|
|
205
454
|
}
|
|
206
455
|
|
|
207
|
-
|
|
208
|
-
|
|
456
|
+
log(
|
|
457
|
+
"INIT",
|
|
458
|
+
`LLM=${opts.llm ? "ON" : "OFF"} 每国${PER_COUNTRY}条 ${opts.once ? "单次" : "间隔" + opts.interval + "s"}`,
|
|
459
|
+
);
|
|
209
460
|
|
|
210
|
-
|
|
211
|
-
|
|
461
|
+
initDb(opts.db);
|
|
462
|
+
loadLlmConfig();
|
|
463
|
+
|
|
464
|
+
const archiveDb = new Database(opts.archive);
|
|
465
|
+
archiveDb.pragma("journal_mode = WAL");
|
|
466
|
+
if (
|
|
467
|
+
!archiveDb
|
|
468
|
+
.prepare(
|
|
469
|
+
"SELECT name FROM sqlite_master WHERE type='table' AND name='raw_jobs'",
|
|
470
|
+
)
|
|
471
|
+
.get()
|
|
472
|
+
) {
|
|
473
|
+
console.error("[sync] 归档库无 raw_jobs 表");
|
|
474
|
+
process.exit(1);
|
|
212
475
|
}
|
|
476
|
+
log(
|
|
477
|
+
"INIT",
|
|
478
|
+
`归档 ${archiveDb.prepare("SELECT COUNT(*) as c FROM raw_jobs").get().c.toLocaleString()} 条`,
|
|
479
|
+
);
|
|
213
480
|
|
|
214
|
-
|
|
215
|
-
|
|
481
|
+
archiveDb.exec(
|
|
482
|
+
`CREATE INDEX IF NOT EXISTS idx_raw_refill ON raw_jobs(guessed_location, video_count DESC, created_at DESC) WHERE video_count>6 AND follower_count>0 AND following_count>0`,
|
|
483
|
+
);
|
|
484
|
+
// tag/回退查询用:无国家过滤,按 video_count 排序
|
|
485
|
+
archiveDb.exec(
|
|
486
|
+
`CREATE INDEX IF NOT EXISTS idx_raw_vc ON raw_jobs(video_count DESC) WHERE video_count>6 AND follower_count>0 AND following_count>0`,
|
|
487
|
+
);
|
|
216
488
|
|
|
217
|
-
//
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
const archiveDb = new Database(opts.archive);
|
|
229
|
-
archiveDb.pragma("journal_mode = WAL");
|
|
230
|
-
|
|
231
|
-
// 检查主库 raw_jobs 是否存在
|
|
232
|
-
const mainTables = mainDb
|
|
233
|
-
.prepare(
|
|
234
|
-
"SELECT name FROM sqlite_master WHERE type='table' AND name='raw_jobs'",
|
|
235
|
-
)
|
|
236
|
-
.get();
|
|
237
|
-
if (!mainTables) {
|
|
238
|
-
console.error("[sync] 错误: 主库中不存在 raw_jobs 表");
|
|
239
|
-
process.exit(1);
|
|
240
|
-
}
|
|
489
|
+
// LLM 打分排序索引
|
|
490
|
+
archiveDb.exec(
|
|
491
|
+
`CREATE INDEX IF NOT EXISTS idx_raw_llm_score ON raw_jobs(llm_scored_count, video_count DESC) WHERE video_count>6 AND follower_count>0 AND following_count>0`,
|
|
492
|
+
);
|
|
493
|
+
try {
|
|
494
|
+
archiveDb.exec(
|
|
495
|
+
"ALTER TABLE raw_jobs ADD COLUMN llm_scored_count INTEGER DEFAULT 0",
|
|
496
|
+
);
|
|
497
|
+
} catch {
|
|
498
|
+
/* exists */
|
|
499
|
+
}
|
|
241
500
|
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
console.error("[sync] 错误: 归档库中不存在 raw_jobs 表");
|
|
250
|
-
process.exit(1);
|
|
251
|
-
}
|
|
501
|
+
await tick(archiveDb, opts);
|
|
502
|
+
if (opts.once) {
|
|
503
|
+
archiveDb.close();
|
|
504
|
+
getDb().close();
|
|
505
|
+
log("DONE", "退出");
|
|
506
|
+
process.exit(0);
|
|
507
|
+
}
|
|
252
508
|
|
|
253
|
-
const
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
);
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
if (opts.once) {
|
|
264
|
-
mainDb.close();
|
|
265
|
-
archiveDb.close();
|
|
266
|
-
console.error("[sync] 完成,退出");
|
|
267
|
-
process.exit(0);
|
|
509
|
+
const timer = setInterval(() => tick(archiveDb, opts), opts.interval * 1000);
|
|
510
|
+
const stop = () => {
|
|
511
|
+
clearInterval(timer);
|
|
512
|
+
archiveDb.close();
|
|
513
|
+
getDb().close();
|
|
514
|
+
process.exit(0);
|
|
515
|
+
};
|
|
516
|
+
process.on("SIGINT", stop);
|
|
517
|
+
process.on("SIGTERM", stop);
|
|
268
518
|
}
|
|
269
519
|
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
}, opts.interval * 1000);
|
|
274
|
-
|
|
275
|
-
// 优雅退出
|
|
276
|
-
process.on("SIGINT", () => {
|
|
277
|
-
console.error("\n[sync] 收到中断信号,退出...");
|
|
278
|
-
mainDb.close();
|
|
279
|
-
archiveDb.close();
|
|
280
|
-
process.exit(0);
|
|
520
|
+
main().catch((e) => {
|
|
521
|
+
console.error("[sync] FATAL:", e);
|
|
522
|
+
process.exit(1);
|
|
281
523
|
});
|