tt-help-cli-ycl 1.4.26 → 1.4.28
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/scripts/search-authors.mjs +21 -0
- package/scripts/sync-raw-jobs.mjs +444 -202
- package/src/cli/raw-sync.js +2 -5
- package/src/cli/search.js +160 -0
- package/src/lib/args.js +78 -5
- package/src/lib/constants.js +24 -5
- package/src/lib/search-collector.js +194 -0
- package/src/lib/target-locations.js +1 -0
- package/src/main.js +3 -0
- package/src/npm-main.js +3 -0
- package/src/watch/data-store.js +266 -577
- package/src/watch/db-columns.js +1 -0
- package/src/watch/db-crud.js +27 -3
- package/src/watch/db-raw-jobs.js +3 -3
- package/src/watch/db-schema.js +100 -0
- package/src/watch/db-stats.js +46 -23
- package/src/watch/db-tags.js +100 -1
- package/src/watch/llm-config.js +145 -1
- package/src/watch/llm-scoring.js +60 -117
- package/src/watch/public/app.js +5 -5
- package/src/watch/public/index.html +4 -5
- package/src/watch/public/style.css +16 -0
- package/src/watch/server.js +170 -59
- package/src/watch/tag-service.js +11 -67
package/src/watch/db-columns.js
CHANGED
package/src/watch/db-crud.js
CHANGED
|
@@ -158,7 +158,7 @@ export function addJobToDb(user) {
|
|
|
158
158
|
"bio_link",
|
|
159
159
|
"sec_uid",
|
|
160
160
|
];
|
|
161
|
-
db.prepare(insertIgnoreSql("
|
|
161
|
+
db.prepare(insertIgnoreSql("task_queue", columns)).run(
|
|
162
162
|
user.uniqueId,
|
|
163
163
|
user.nickname || null,
|
|
164
164
|
user.status || inferStatus(user),
|
|
@@ -312,9 +312,28 @@ export function addJob(user) {
|
|
|
312
312
|
export function getJobRow(uniqueId) {
|
|
313
313
|
const db = getDb();
|
|
314
314
|
if (!db) return null;
|
|
315
|
+
// 优先查 task_queue(热数据),回退 jobs(归档)
|
|
316
|
+
const row = db
|
|
317
|
+
.prepare("SELECT * FROM task_queue WHERE unique_id = ?")
|
|
318
|
+
.get(uniqueId);
|
|
319
|
+
if (row) return row;
|
|
315
320
|
return db.prepare("SELECT * FROM jobs WHERE unique_id = ?").get(uniqueId);
|
|
316
321
|
}
|
|
317
322
|
|
|
323
|
+
/**
|
|
324
|
+
* 判断用户在哪个表,返回 'task_queue' 或 'jobs'
|
|
325
|
+
* 用于需要更新正确表的场景(置顶、重置、移动等)
|
|
326
|
+
*/
|
|
327
|
+
export function getUserTable(uniqueId) {
|
|
328
|
+
const db = getDb();
|
|
329
|
+
if (!db) return null;
|
|
330
|
+
return db
|
|
331
|
+
.prepare("SELECT 1 FROM task_queue WHERE unique_id = ?")
|
|
332
|
+
.get(uniqueId)
|
|
333
|
+
? "task_queue"
|
|
334
|
+
: "jobs";
|
|
335
|
+
}
|
|
336
|
+
|
|
318
337
|
export function getJobBaseRow(uniqueId) {
|
|
319
338
|
const db = getDb();
|
|
320
339
|
if (!db) return null;
|
|
@@ -377,7 +396,12 @@ export function getAllVideoRows() {
|
|
|
377
396
|
|
|
378
397
|
// ===== 更新 Job 信息 =====
|
|
379
398
|
|
|
380
|
-
export function updateJobInfo(
|
|
399
|
+
export function updateJobInfo(
|
|
400
|
+
uniqueId,
|
|
401
|
+
info,
|
|
402
|
+
incrementCount = true,
|
|
403
|
+
tableName = "jobs",
|
|
404
|
+
) {
|
|
381
405
|
const db = getDb();
|
|
382
406
|
if (!db) return { error: "db not initialized" };
|
|
383
407
|
const existing = getJobRow(uniqueId);
|
|
@@ -401,7 +425,7 @@ export function updateJobInfo(uniqueId, info, incrementCount = true) {
|
|
|
401
425
|
|
|
402
426
|
const columns = Object.keys(nextValues);
|
|
403
427
|
if (columns.length > 0) {
|
|
404
|
-
const sql = `UPDATE
|
|
428
|
+
const sql = `UPDATE ${tableName} SET ${updateSetClause(columns)} WHERE unique_id = ?`;
|
|
405
429
|
db.prepare(sql).run(...columns.map((c) => nextValues[c]), uniqueId);
|
|
406
430
|
}
|
|
407
431
|
|
package/src/watch/db-raw-jobs.js
CHANGED
|
@@ -103,7 +103,7 @@ export function restoreRawJobsByCountry(country) {
|
|
|
103
103
|
|
|
104
104
|
const restoreTxn = db.transaction((targetCountry) => {
|
|
105
105
|
db.prepare(
|
|
106
|
-
`INSERT OR REPLACE INTO
|
|
106
|
+
`INSERT OR REPLACE INTO task_queue (${RESTORE_COLUMNS}) SELECT ${RESTORE_COLUMNS} FROM raw_jobs WHERE ${whereSql}`,
|
|
107
107
|
).run(targetCountry);
|
|
108
108
|
db.prepare(`DELETE FROM raw_jobs WHERE ${whereSql}`).run(targetCountry);
|
|
109
109
|
});
|
|
@@ -128,7 +128,7 @@ export function restoreRawJobById(uniqueId) {
|
|
|
128
128
|
|
|
129
129
|
const restoreTxn = db.transaction(() => {
|
|
130
130
|
db.prepare(
|
|
131
|
-
`INSERT OR REPLACE INTO
|
|
131
|
+
`INSERT OR REPLACE INTO task_queue (${RESTORE_COLUMNS}) SELECT ${RESTORE_COLUMNS} FROM raw_jobs WHERE unique_id = ?`,
|
|
132
132
|
).run(safeId);
|
|
133
133
|
db.prepare("DELETE FROM raw_jobs WHERE unique_id = ?").run(safeId);
|
|
134
134
|
});
|
|
@@ -176,7 +176,7 @@ export function restoreRawJobsByFilter({
|
|
|
176
176
|
|
|
177
177
|
const restoreTxn = db.transaction(() => {
|
|
178
178
|
db.prepare(
|
|
179
|
-
`INSERT OR REPLACE INTO
|
|
179
|
+
`INSERT OR REPLACE INTO task_queue (${RESTORE_COLUMNS}) SELECT ${RESTORE_COLUMNS} FROM raw_jobs WHERE ${whereSql}`,
|
|
180
180
|
).run(...args);
|
|
181
181
|
db.prepare(`DELETE FROM raw_jobs WHERE ${whereSql}`).run(...args);
|
|
182
182
|
});
|
package/src/watch/db-schema.js
CHANGED
|
@@ -132,6 +132,34 @@ export function initDb(filePath) {
|
|
|
132
132
|
recent_ec_videos: "TEXT",
|
|
133
133
|
});
|
|
134
134
|
|
|
135
|
+
// ===== task_queue 表(热数据:仅 pending + processing,高频读写) =====
|
|
136
|
+
// 直接复制 jobs 表结构(含所有历史迁移列),确保列完全一致
|
|
137
|
+
const tqExists = db
|
|
138
|
+
.prepare(
|
|
139
|
+
"SELECT name FROM sqlite_master WHERE type='table' AND name='task_queue'",
|
|
140
|
+
)
|
|
141
|
+
.get();
|
|
142
|
+
if (!tqExists) {
|
|
143
|
+
db.exec(`CREATE TABLE task_queue AS SELECT * FROM jobs WHERE 1=0`);
|
|
144
|
+
// 补主键(CREATE TABLE ... AS 不会复制约束)
|
|
145
|
+
db.exec(
|
|
146
|
+
`CREATE UNIQUE INDEX IF NOT EXISTS idx_tq_pk ON task_queue(unique_id)`,
|
|
147
|
+
);
|
|
148
|
+
}
|
|
149
|
+
// 后续迁移也要给 task_queue 补列
|
|
150
|
+
migrateColumns("task_queue", {
|
|
151
|
+
status_code: "INTEGER",
|
|
152
|
+
latest_video_time: "INTEGER",
|
|
153
|
+
confirmed_location: "TEXT",
|
|
154
|
+
modified_at: "INTEGER",
|
|
155
|
+
bio_link: "TEXT",
|
|
156
|
+
top_video_play_count: "INTEGER",
|
|
157
|
+
top_video_href: "TEXT",
|
|
158
|
+
top_video_is_ad: "INTEGER",
|
|
159
|
+
user_create_time: "INTEGER",
|
|
160
|
+
recent_ec_videos: "TEXT",
|
|
161
|
+
});
|
|
162
|
+
|
|
135
163
|
// ===== jobs_base 表 =====
|
|
136
164
|
db.exec(`
|
|
137
165
|
CREATE TABLE IF NOT EXISTS jobs_base (
|
|
@@ -226,6 +254,21 @@ export function initDb(filePath) {
|
|
|
226
254
|
});
|
|
227
255
|
|
|
228
256
|
// ===== videos 表 =====
|
|
257
|
+
// 首次启动时将旧 videos 重命名为 videos_old,创建全新 videos 表
|
|
258
|
+
const videosOldExists = db
|
|
259
|
+
.prepare(
|
|
260
|
+
"SELECT name FROM sqlite_master WHERE type='table' AND name='videos_old'",
|
|
261
|
+
)
|
|
262
|
+
.get();
|
|
263
|
+
const videosExists = db
|
|
264
|
+
.prepare(
|
|
265
|
+
"SELECT name FROM sqlite_master WHERE type='table' AND name='videos'",
|
|
266
|
+
)
|
|
267
|
+
.get();
|
|
268
|
+
if (!videosOldExists && videosExists) {
|
|
269
|
+
db.exec("ALTER TABLE videos RENAME TO videos_old");
|
|
270
|
+
console.log("[data-store] videos → videos_old");
|
|
271
|
+
}
|
|
229
272
|
db.exec(`
|
|
230
273
|
CREATE TABLE IF NOT EXISTS videos (
|
|
231
274
|
id TEXT PRIMARY KEY,
|
|
@@ -252,6 +295,18 @@ export function initDb(filePath) {
|
|
|
252
295
|
collect_count: "INTEGER",
|
|
253
296
|
stats_updated_at: "INTEGER",
|
|
254
297
|
create_time: "INTEGER",
|
|
298
|
+
claimed_at: "INTEGER",
|
|
299
|
+
});
|
|
300
|
+
// videos_old 同步添加列(已重命名的旧 videos 表)
|
|
301
|
+
migrateColumns("videos_old", {
|
|
302
|
+
play_count: "INTEGER",
|
|
303
|
+
digg_count: "INTEGER",
|
|
304
|
+
comment_count: "INTEGER",
|
|
305
|
+
share_count: "INTEGER",
|
|
306
|
+
collect_count: "INTEGER",
|
|
307
|
+
stats_updated_at: "INTEGER",
|
|
308
|
+
create_time: "INTEGER",
|
|
309
|
+
claimed_at: "INTEGER",
|
|
255
310
|
});
|
|
256
311
|
|
|
257
312
|
// ===== tags 表 =====
|
|
@@ -284,6 +339,9 @@ export function initDb(filePath) {
|
|
|
284
339
|
direction_id: "INTEGER",
|
|
285
340
|
refreshing: "INTEGER DEFAULT 0",
|
|
286
341
|
refreshed_at: "TEXT",
|
|
342
|
+
searching: "INTEGER DEFAULT 0",
|
|
343
|
+
searched_at: "TEXT",
|
|
344
|
+
searched_users: "INTEGER DEFAULT 0",
|
|
287
345
|
});
|
|
288
346
|
|
|
289
347
|
// ===== tag_directions 表(分层探索:通用词/分类/产品) =====
|
|
@@ -415,6 +473,7 @@ export function initDb(filePath) {
|
|
|
415
473
|
`CREATE INDEX IF NOT EXISTS idx_jobs_user_update_queue_expr ON jobs(created_at ASC, unique_id ASC) WHERE COALESCE(tt_seller, '') = '' AND COALESCE(user_update_count, 0) <= 0`,
|
|
416
474
|
`CREATE INDEX IF NOT EXISTS idx_videos_comment_queue ON videos(user_update_count, tt_seller DESC, registered_at ASC)`,
|
|
417
475
|
`CREATE INDEX IF NOT EXISTS idx_videos_comment_queue_pending ON videos(tt_seller DESC, registered_at ASC, id) WHERE user_update_count IS NULL OR user_update_count <= 0`,
|
|
476
|
+
`CREATE INDEX IF NOT EXISTS idx_videos_old_id ON videos_old(id)`,
|
|
418
477
|
`CREATE INDEX IF NOT EXISTS idx_tags_status ON tags(status)`,
|
|
419
478
|
`CREATE INDEX IF NOT EXISTS idx_tags_score ON tags(score DESC)`,
|
|
420
479
|
`CREATE INDEX IF NOT EXISTS idx_discover_log_country ON discover_log(country, round DESC)`,
|
|
@@ -426,11 +485,52 @@ export function initDb(filePath) {
|
|
|
426
485
|
`CREATE INDEX IF NOT EXISTS idx_cross_country_target ON cross_country_learnings(target_country)`,
|
|
427
486
|
`CREATE INDEX IF NOT EXISTS idx_tags_dead_reason ON tags(dead_reason)`,
|
|
428
487
|
`CREATE INDEX IF NOT EXISTS idx_jobs_recheck ON jobs(status, updated_at) WHERE status = 'done' AND (tt_seller IS NULL OR tt_seller != 1)`,
|
|
488
|
+
// 监控页面用户列表查询加速(简化排序:pinned DESC, processed_at DESC, follower_count DESC, unique_id ASC)
|
|
489
|
+
`CREATE INDEX IF NOT EXISTS idx_jobs_list_done ON jobs(pinned, processed_at, follower_count, unique_id) WHERE status = 'done'`,
|
|
490
|
+
`CREATE INDEX IF NOT EXISTS idx_jobs_list_error ON jobs(pinned, processed_at, follower_count, unique_id) WHERE status = 'error'`,
|
|
491
|
+
`CREATE INDEX IF NOT EXISTS idx_jobs_list_restricted ON jobs(pinned, processed_at, follower_count, unique_id) WHERE status = 'restricted'`,
|
|
429
492
|
];
|
|
430
493
|
for (const sql of indexes) {
|
|
431
494
|
db.exec(sql);
|
|
432
495
|
}
|
|
433
496
|
|
|
497
|
+
// ===== task_queue 索引(对应 claimNextJob UNION 查询的 4 个部分索引) =====
|
|
498
|
+
const tqIndexes = [
|
|
499
|
+
`CREATE INDEX IF NOT EXISTS idx_tq_status ON task_queue(status)`,
|
|
500
|
+
`CREATE INDEX IF NOT EXISTS idx_tq_claimed_by_status ON task_queue(claimed_by, status, claimed_at)`,
|
|
501
|
+
`CREATE INDEX IF NOT EXISTS idx_tq_status_claimed_at ON task_queue(status, claimed_at)`,
|
|
502
|
+
`CREATE INDEX IF NOT EXISTS idx_tq_claim_pending_pinned ON task_queue(created_at ASC, unique_id ASC) WHERE status = 'pending' AND COALESCE(pinned, 0) = 1`,
|
|
503
|
+
`CREATE INDEX IF NOT EXISTS idx_tq_claim_pending_seller ON task_queue(UPPER(COALESCE(guessed_location, '')), follower_count DESC, created_at ASC, unique_id ASC) WHERE status = 'pending' AND COALESCE(pinned, 0) = 0 AND tt_seller = 1 AND verified = 0`,
|
|
504
|
+
`CREATE INDEX IF NOT EXISTS idx_tq_claim_pending_follow ON task_queue(UPPER(COALESCE(guessed_location, '')), follower_count DESC, created_at ASC, unique_id ASC) WHERE status = 'pending' AND COALESCE(pinned, 0) = 0 AND (instr(COALESCE(sources, ''), '"following"') > 0 OR instr(COALESCE(sources, ''), '"follower"') > 0)`,
|
|
505
|
+
`CREATE INDEX IF NOT EXISTS idx_tq_claim_pending_other ON task_queue(UPPER(COALESCE(guessed_location, '')), follower_count DESC, created_at ASC, unique_id ASC) WHERE status = 'pending' AND COALESCE(pinned, 0) = 0`,
|
|
506
|
+
// 监控页面用户列表查询加速
|
|
507
|
+
`CREATE INDEX IF NOT EXISTS idx_tq_list_pending ON task_queue(pinned, processed_at, follower_count, unique_id) WHERE status = 'pending'`,
|
|
508
|
+
`CREATE INDEX IF NOT EXISTS idx_tq_list_processing ON task_queue(pinned, processed_at, follower_count, unique_id) WHERE status = 'processing'`,
|
|
509
|
+
];
|
|
510
|
+
for (const sql of tqIndexes) {
|
|
511
|
+
db.exec(sql);
|
|
512
|
+
}
|
|
513
|
+
|
|
514
|
+
// 首次迁移:将 jobs 中的 pending/processing 行移动到 task_queue
|
|
515
|
+
const tqCount = db.prepare("SELECT COUNT(*) as c FROM task_queue").get().c;
|
|
516
|
+
if (tqCount === 0) {
|
|
517
|
+
const moved = db
|
|
518
|
+
.prepare(
|
|
519
|
+
`
|
|
520
|
+
INSERT INTO task_queue SELECT * FROM jobs WHERE status IN ('pending', 'processing')
|
|
521
|
+
`,
|
|
522
|
+
)
|
|
523
|
+
.run().changes;
|
|
524
|
+
if (moved > 0) {
|
|
525
|
+
db.prepare(
|
|
526
|
+
`DELETE FROM jobs WHERE status IN ('pending', 'processing')`,
|
|
527
|
+
).run();
|
|
528
|
+
console.log(
|
|
529
|
+
`[data-store] 迁移 ${moved} 条 pending/processing → task_queue`,
|
|
530
|
+
);
|
|
531
|
+
}
|
|
532
|
+
}
|
|
533
|
+
|
|
434
534
|
const count = db.prepare("SELECT COUNT(*) as c FROM users").get().c;
|
|
435
535
|
console.log(`[data-store] SQLite users 表初始化完成: ${count} 条`);
|
|
436
536
|
}
|
package/src/watch/db-stats.js
CHANGED
|
@@ -15,13 +15,33 @@ export function getDashboardStatsFromDb(targetLocations = [], options = {}) {
|
|
|
15
15
|
const targetPlaceholders = targetLocations.map(() => "?").join(", ");
|
|
16
16
|
const targetParams = targetLocations.length ? targetLocations : [];
|
|
17
17
|
|
|
18
|
-
|
|
18
|
+
// Phase 2: pending/processing/source 从 task_queue,done/error/restricted 从 jobs
|
|
19
|
+
const tqAgg = db
|
|
19
20
|
.prepare(
|
|
20
|
-
`
|
|
21
|
-
|
|
22
|
-
COUNT(*) as total,
|
|
21
|
+
`SELECT
|
|
22
|
+
COUNT(*) as tqTotal,
|
|
23
23
|
SUM(CASE WHEN status = 'pending' THEN 1 ELSE 0 END) as pending,
|
|
24
24
|
SUM(CASE WHEN status = 'processing' THEN 1 ELSE 0 END) as processing,
|
|
25
|
+
SUM(CASE WHEN instr(COALESCE(sources, ''), '"video"') > 0 THEN 1 ELSE 0 END) as video,
|
|
26
|
+
SUM(CASE WHEN instr(COALESCE(sources, ''), '"comment"') > 0 THEN 1 ELSE 0 END) as comment,
|
|
27
|
+
SUM(CASE WHEN instr(COALESCE(sources, ''), '"guess"') > 0 THEN 1 ELSE 0 END) as guess,
|
|
28
|
+
SUM(CASE WHEN instr(COALESCE(sources, ''), '"following"') > 0 THEN 1 ELSE 0 END) as following,
|
|
29
|
+
SUM(CASE WHEN instr(COALESCE(sources, ''), '"follower"') > 0 THEN 1 ELSE 0 END) as follower,
|
|
30
|
+
SUM(CASE
|
|
31
|
+
WHEN instr(COALESCE(sources, ''), '"video"') = 0
|
|
32
|
+
AND instr(COALESCE(sources, ''), '"comment"') = 0
|
|
33
|
+
AND instr(COALESCE(sources, ''), '"guess"') = 0
|
|
34
|
+
AND instr(COALESCE(sources, ''), '"following"') = 0
|
|
35
|
+
AND instr(COALESCE(sources, ''), '"follower"') = 0
|
|
36
|
+
THEN 1 ELSE 0 END) as seed
|
|
37
|
+
FROM task_queue`,
|
|
38
|
+
)
|
|
39
|
+
.get();
|
|
40
|
+
|
|
41
|
+
const jobsAgg = db
|
|
42
|
+
.prepare(
|
|
43
|
+
`SELECT
|
|
44
|
+
COUNT(*) as jobsTotal,
|
|
25
45
|
SUM(CASE WHEN status = 'done' THEN 1 ELSE 0 END) as done,
|
|
26
46
|
SUM(CASE WHEN status = 'error' THEN 1 ELSE 0 END) as error,
|
|
27
47
|
SUM(CASE WHEN status = 'restricted' THEN 1 ELSE 0 END) as restricted,
|
|
@@ -30,25 +50,28 @@ export function getDashboardStatsFromDb(targetLocations = [], options = {}) {
|
|
|
30
50
|
? `AND location_created IN (${targetPlaceholders})`
|
|
31
51
|
: "AND 1 = 0"
|
|
32
52
|
} THEN 1 ELSE 0 END) as targetUsers,
|
|
33
|
-
SUM(CASE WHEN no_video = 1 THEN 1 ELSE 0 END) as noVideo
|
|
34
|
-
|
|
35
|
-
SUM(CASE WHEN status != 'done' AND instr(COALESCE(sources, ''), '"comment"') > 0 THEN 1 ELSE 0 END) as comment,
|
|
36
|
-
SUM(CASE WHEN status != 'done' AND instr(COALESCE(sources, ''), '"guess"') > 0 THEN 1 ELSE 0 END) as guess,
|
|
37
|
-
SUM(CASE WHEN status != 'done' AND instr(COALESCE(sources, ''), '"following"') > 0 THEN 1 ELSE 0 END) as following,
|
|
38
|
-
SUM(CASE WHEN status != 'done' AND instr(COALESCE(sources, ''), '"follower"') > 0 THEN 1 ELSE 0 END) as follower,
|
|
39
|
-
SUM(CASE
|
|
40
|
-
WHEN status != 'done'
|
|
41
|
-
AND instr(COALESCE(sources, ''), '"video"') = 0
|
|
42
|
-
AND instr(COALESCE(sources, ''), '"comment"') = 0
|
|
43
|
-
AND instr(COALESCE(sources, ''), '"guess"') = 0
|
|
44
|
-
AND instr(COALESCE(sources, ''), '"following"') = 0
|
|
45
|
-
AND instr(COALESCE(sources, ''), '"follower"') = 0
|
|
46
|
-
THEN 1 ELSE 0 END) as seed
|
|
47
|
-
FROM jobs
|
|
48
|
-
`,
|
|
53
|
+
SUM(CASE WHEN no_video = 1 THEN 1 ELSE 0 END) as noVideo
|
|
54
|
+
FROM jobs`,
|
|
49
55
|
)
|
|
50
56
|
.get(...targetParams);
|
|
51
57
|
|
|
58
|
+
const aggregateRow = {
|
|
59
|
+
total: tqAgg.tqTotal + jobsAgg.jobsTotal,
|
|
60
|
+
pending: tqAgg.pending,
|
|
61
|
+
processing: tqAgg.processing,
|
|
62
|
+
done: jobsAgg.done,
|
|
63
|
+
error: jobsAgg.error,
|
|
64
|
+
restricted: jobsAgg.restricted,
|
|
65
|
+
targetUsers: jobsAgg.targetUsers,
|
|
66
|
+
noVideo: jobsAgg.noVideo,
|
|
67
|
+
video: tqAgg.video,
|
|
68
|
+
comment: tqAgg.comment,
|
|
69
|
+
guess: tqAgg.guess,
|
|
70
|
+
following: tqAgg.following,
|
|
71
|
+
follower: tqAgg.follower,
|
|
72
|
+
seed: tqAgg.seed,
|
|
73
|
+
};
|
|
74
|
+
|
|
52
75
|
const userUpdateTasksRow = db
|
|
53
76
|
.prepare(
|
|
54
77
|
`SELECT COUNT(*) as userUpdateTasks FROM jobs_base WHERE COALESCE(user_update_count, 0) <= 0`,
|
|
@@ -145,7 +168,7 @@ export function getPendingByCountryFromDb() {
|
|
|
145
168
|
return db
|
|
146
169
|
.prepare(
|
|
147
170
|
`SELECT COALESCE(guessed_location, '未知') as country, COUNT(*) as count
|
|
148
|
-
FROM
|
|
171
|
+
FROM task_queue WHERE status = 'pending'
|
|
149
172
|
GROUP BY COALESCE(guessed_location, '未知')
|
|
150
173
|
ORDER BY count DESC`,
|
|
151
174
|
)
|
|
@@ -237,13 +260,13 @@ export function resetPendingByCountry(country) {
|
|
|
237
260
|
const whereSql = `status = 'pending' AND COALESCE(guessed_location, '未知') = ?`;
|
|
238
261
|
const count =
|
|
239
262
|
db
|
|
240
|
-
.prepare(`SELECT COUNT(*) as c FROM
|
|
263
|
+
.prepare(`SELECT COUNT(*) as c FROM task_queue WHERE ${whereSql}`)
|
|
241
264
|
.get(normalizedCountry)?.c || 0;
|
|
242
265
|
|
|
243
266
|
if (!count) return { reset: 0, country: normalizedCountry };
|
|
244
267
|
|
|
245
268
|
db.prepare(
|
|
246
|
-
`UPDATE
|
|
269
|
+
`UPDATE task_queue SET user_update_count = 0, updated_at = ?, claimed_by = NULL, claimed_at = NULL WHERE ${whereSql}`,
|
|
247
270
|
).run(Date.now(), normalizedCountry);
|
|
248
271
|
|
|
249
272
|
return { reset: count, country: normalizedCountry };
|
package/src/watch/db-tags.js
CHANGED
|
@@ -103,6 +103,7 @@ export function getTagsByStatus(
|
|
|
103
103
|
offset = 0,
|
|
104
104
|
country = null,
|
|
105
105
|
search = null,
|
|
106
|
+
source = null,
|
|
106
107
|
) {
|
|
107
108
|
const db = getDb();
|
|
108
109
|
if (!db) return [];
|
|
@@ -116,8 +117,12 @@ export function getTagsByStatus(
|
|
|
116
117
|
sql += " AND tag LIKE ?";
|
|
117
118
|
params.push(`%${search}%`);
|
|
118
119
|
}
|
|
120
|
+
if (source) {
|
|
121
|
+
sql += " AND source = ?";
|
|
122
|
+
params.push(source);
|
|
123
|
+
}
|
|
119
124
|
sql +=
|
|
120
|
-
" ORDER BY refreshed_at DESC NULLS LAST, score ASC, created_at ASC LIMIT ? OFFSET ?";
|
|
125
|
+
" ORDER BY CASE WHEN source = 'llm' THEN 0 ELSE 1 END, refreshed_at DESC NULLS LAST, score ASC, created_at ASC LIMIT ? OFFSET ?";
|
|
121
126
|
params.push(limit, offset);
|
|
122
127
|
const rows = db.prepare(sql).all(...params);
|
|
123
128
|
return rows.map(parseTagRow);
|
|
@@ -144,6 +149,7 @@ export function getTagStats(country = null, search = null) {
|
|
|
144
149
|
SUM(CASE WHEN status = 'productive' THEN 1 ELSE 0 END) as productive,
|
|
145
150
|
SUM(CASE WHEN status = 'dead' THEN 1 ELSE 0 END) as dead,
|
|
146
151
|
SUM(CASE WHEN status = 'new' THEN 1 ELSE 0 END) as newCount,
|
|
152
|
+
SUM(CASE WHEN status = 'new' AND source = 'llm' THEN 1 ELSE 0 END) as newLlm,
|
|
147
153
|
SUM(CASE WHEN status = 'scoring' THEN 1 ELSE 0 END) as scoring
|
|
148
154
|
FROM tags${whereClause}`;
|
|
149
155
|
const row = db.prepare(sql).get(...params);
|
|
@@ -161,6 +167,7 @@ export function getTagStats(country = null, search = null) {
|
|
|
161
167
|
productive: row.productive || 0,
|
|
162
168
|
dead: row.dead || 0,
|
|
163
169
|
new: row.newCount || 0,
|
|
170
|
+
newLlm: row.newLlm || 0,
|
|
164
171
|
scoring: row.scoring || 0,
|
|
165
172
|
countries: [...countrySet].sort(),
|
|
166
173
|
};
|
|
@@ -1649,3 +1656,95 @@ export function reportRefreshResult(tag, fields) {
|
|
|
1649
1656
|
return { ok: false, error: e.message };
|
|
1650
1657
|
}
|
|
1651
1658
|
}
|
|
1659
|
+
|
|
1660
|
+
// ===== 搜索任务(基于 productive 标签搜索作者) =====
|
|
1661
|
+
|
|
1662
|
+
/** 领取一个 productive 标签用于搜索(score>70,未被搜索过) */
|
|
1663
|
+
export function getProductiveSearchTask() {
|
|
1664
|
+
const db = getDb();
|
|
1665
|
+
if (!db) return null;
|
|
1666
|
+
|
|
1667
|
+
// 清理超时的 searching 锁(超过 30 分钟)
|
|
1668
|
+
db.prepare(
|
|
1669
|
+
`UPDATE tags SET searching = 0
|
|
1670
|
+
WHERE searching = 1 AND searched_at IS NOT NULL
|
|
1671
|
+
AND searched_at < datetime('now', '-30 minutes')`,
|
|
1672
|
+
).run();
|
|
1673
|
+
|
|
1674
|
+
const row = db
|
|
1675
|
+
.prepare(
|
|
1676
|
+
`SELECT * FROM tags
|
|
1677
|
+
WHERE status = 'productive' AND score > 70 AND searching = 0
|
|
1678
|
+
ORDER BY searched_at IS NULL DESC, searched_at ASC
|
|
1679
|
+
LIMIT 1`,
|
|
1680
|
+
)
|
|
1681
|
+
.get();
|
|
1682
|
+
if (!row) return null;
|
|
1683
|
+
const parsed = parseTagRow(row);
|
|
1684
|
+
return {
|
|
1685
|
+
tag: parsed.tag,
|
|
1686
|
+
score: parsed.score,
|
|
1687
|
+
// 取第一个国家作为搜索结果的猜测国家
|
|
1688
|
+
country: parsed.countries?.[0] || null,
|
|
1689
|
+
countries: parsed.countries,
|
|
1690
|
+
searched_at: parsed.searched_at,
|
|
1691
|
+
};
|
|
1692
|
+
}
|
|
1693
|
+
|
|
1694
|
+
/** 锁定标签用于搜索(防并发) */
|
|
1695
|
+
export function claimSearchTag(tag) {
|
|
1696
|
+
const db = getDb();
|
|
1697
|
+
if (!db) return { ok: false, error: "db not ready" };
|
|
1698
|
+
|
|
1699
|
+
const normalized = tag.replace(/^#+/, "").trim().toLowerCase();
|
|
1700
|
+
if (!normalized || normalized.length < 2) {
|
|
1701
|
+
return { ok: false, error: "invalid tag" };
|
|
1702
|
+
}
|
|
1703
|
+
|
|
1704
|
+
const result = db
|
|
1705
|
+
.prepare(
|
|
1706
|
+
`UPDATE tags SET searching = 1, searched_at = datetime('now')
|
|
1707
|
+
WHERE (tag = ? OR tag = ?) AND status = 'productive' AND score > 70 AND searching = 0`,
|
|
1708
|
+
)
|
|
1709
|
+
.run(normalized, `#${normalized}`);
|
|
1710
|
+
if (result.changes === 0) {
|
|
1711
|
+
const row = db
|
|
1712
|
+
.prepare("SELECT status, searching FROM tags WHERE tag = ? OR tag = ?")
|
|
1713
|
+
.get(normalized, `#${normalized}`);
|
|
1714
|
+
if (!row) return { ok: false, error: "tag not found" };
|
|
1715
|
+
if (row.searching) return { ok: false, error: "already being searched" };
|
|
1716
|
+
return { ok: false, error: `tag status is ${row.status}, not eligible` };
|
|
1717
|
+
}
|
|
1718
|
+
return { ok: true, tag: normalized };
|
|
1719
|
+
}
|
|
1720
|
+
|
|
1721
|
+
/** 报告搜索结果并解锁标签 */
|
|
1722
|
+
export function reportSearchResult(tag, fields) {
|
|
1723
|
+
const db = getDb();
|
|
1724
|
+
if (!db) return { ok: false, error: "db not ready" };
|
|
1725
|
+
const { searchedUsers, error } = fields;
|
|
1726
|
+
const now = new Date().toISOString();
|
|
1727
|
+
const normalized = tag.replace(/^#+/, "").trim().toLowerCase();
|
|
1728
|
+
|
|
1729
|
+
try {
|
|
1730
|
+
const result = db
|
|
1731
|
+
.prepare(
|
|
1732
|
+
`UPDATE tags SET
|
|
1733
|
+
searching = 0,
|
|
1734
|
+
searched_at = ?,
|
|
1735
|
+
searched_users = COALESCE(searched_users, 0) + COALESCE(?, 0),
|
|
1736
|
+
last_error = COALESCE(?, last_error)
|
|
1737
|
+
WHERE tag = ? OR tag = ?`,
|
|
1738
|
+
)
|
|
1739
|
+
.run(
|
|
1740
|
+
now,
|
|
1741
|
+
searchedUsers ?? null,
|
|
1742
|
+
error ?? null,
|
|
1743
|
+
normalized,
|
|
1744
|
+
`#${normalized}`,
|
|
1745
|
+
);
|
|
1746
|
+
return { ok: result.changes > 0, tag: normalized };
|
|
1747
|
+
} catch (e) {
|
|
1748
|
+
return { ok: false, error: e.message };
|
|
1749
|
+
}
|
|
1750
|
+
}
|
package/src/watch/llm-config.js
CHANGED
|
@@ -125,6 +125,16 @@ function saveToDb(key, value) {
|
|
|
125
125
|
*/
|
|
126
126
|
export function loadLlmConfig() {
|
|
127
127
|
if (_loaded) return;
|
|
128
|
+
_doLoadLlmConfig();
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
/** 强制重载配置(用于 sync 程序每轮读取最新模型配置) */
|
|
132
|
+
export function reloadLlmConfig() {
|
|
133
|
+
_loaded = false;
|
|
134
|
+
_doLoadLlmConfig();
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
function _doLoadLlmConfig() {
|
|
128
138
|
const dbConfig = loadFromDb();
|
|
129
139
|
|
|
130
140
|
_cachedUrl = dbConfig.llm_url || process.env.LLM_URL || LLM_PROVIDERS[0].url;
|
|
@@ -141,6 +151,11 @@ export function loadLlmConfig() {
|
|
|
141
151
|
"";
|
|
142
152
|
_loaded = true;
|
|
143
153
|
|
|
154
|
+
// 恢复上次 auto 模式下成功的模型(持久化到 _llm_config)
|
|
155
|
+
if (_cachedModel === "auto" && dbConfig.auto_last_model) {
|
|
156
|
+
_lastAutoSuccess = dbConfig.auto_last_model;
|
|
157
|
+
}
|
|
158
|
+
|
|
144
159
|
console.error(
|
|
145
160
|
`[llm-config] 加载完成: provider=${_cachedProvider}, model=${_cachedModel}, url=${_cachedUrl}, hasKey=${!!_cachedApiKey}`,
|
|
146
161
|
);
|
|
@@ -208,6 +223,7 @@ function clearAutoPoolCache() {
|
|
|
208
223
|
}
|
|
209
224
|
|
|
210
225
|
let _lastAutoSuccess = null; // 上次成功的模型,下次优先用它
|
|
226
|
+
let _allDeadUntil = 0; // 全部模型不可用时的冷却时间戳,避免同批次重复遍历
|
|
211
227
|
|
|
212
228
|
/** 是否处于 auto 模式 */
|
|
213
229
|
export function isAutoMode() {
|
|
@@ -215,9 +231,20 @@ export function isAutoMode() {
|
|
|
215
231
|
return _cachedModel === "auto";
|
|
216
232
|
}
|
|
217
233
|
|
|
218
|
-
/** 记录 auto
|
|
234
|
+
/** 记录 auto 模式下成功调用的模型(内存 + 持久化) */
|
|
219
235
|
export function recordAutoSuccess(model) {
|
|
220
236
|
_lastAutoSuccess = model;
|
|
237
|
+
try {
|
|
238
|
+
const db = getDb();
|
|
239
|
+
if (db) {
|
|
240
|
+
ensureTable();
|
|
241
|
+
db.prepare(
|
|
242
|
+
`INSERT INTO _llm_config (key, value) VALUES ('auto_last_model', ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value`,
|
|
243
|
+
).run(String(model));
|
|
244
|
+
}
|
|
245
|
+
} catch {
|
|
246
|
+
// 写入失败不影响主流程
|
|
247
|
+
}
|
|
221
248
|
}
|
|
222
249
|
|
|
223
250
|
/** 获取 auto 模式候选池(上次成功的排最前,失败时依次尝试) */
|
|
@@ -255,6 +282,123 @@ export function getLlmApiKey() {
|
|
|
255
282
|
return _cachedApiKey;
|
|
256
283
|
}
|
|
257
284
|
|
|
285
|
+
/**
|
|
286
|
+
* 统一的 LLM 调用入口,内部处理 auto 模式模型切换、重试、持久化。
|
|
287
|
+
* @param {{ messages: Array<{role:string,content:string}>, max_tokens?: number, temperature?: number, enable_thinking?: boolean }} opts
|
|
288
|
+
* @returns {Promise<string>} LLM 返回的文本内容
|
|
289
|
+
* @throws {Error} 全部模型均失败时抛出
|
|
290
|
+
*/
|
|
291
|
+
export async function llmFetch({
|
|
292
|
+
messages,
|
|
293
|
+
max_tokens = 1024,
|
|
294
|
+
temperature = 0.1,
|
|
295
|
+
enable_thinking = false,
|
|
296
|
+
} = {}) {
|
|
297
|
+
if (!_loaded) loadLlmConfig();
|
|
298
|
+
|
|
299
|
+
// 全部模型不可用冷却中:同批次直接跳过,不重复遍历 25 个模型
|
|
300
|
+
if (_allDeadUntil > Date.now()) {
|
|
301
|
+
throw new Error("全部模型不可用(冷却中)");
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
const url = _cachedUrl;
|
|
305
|
+
const apiKey = _cachedApiKey;
|
|
306
|
+
const models = isAutoMode() ? getAutoPool() : [null];
|
|
307
|
+
|
|
308
|
+
const { fetch } = await import("undici");
|
|
309
|
+
let lastError = "";
|
|
310
|
+
let deadCount = 0;
|
|
311
|
+
|
|
312
|
+
for (let i = 0; i < models.length; i++) {
|
|
313
|
+
if (_allDeadUntil > Date.now()) break; // 同批次其他调用已发现全挂
|
|
314
|
+
const model = models[i] !== null ? models[i] : resolveModel();
|
|
315
|
+
|
|
316
|
+
let response;
|
|
317
|
+
try {
|
|
318
|
+
response = await fetch(url, {
|
|
319
|
+
method: "POST",
|
|
320
|
+
headers: {
|
|
321
|
+
"Content-Type": "application/json",
|
|
322
|
+
Authorization: `Bearer ${apiKey}`,
|
|
323
|
+
},
|
|
324
|
+
body: JSON.stringify({
|
|
325
|
+
model,
|
|
326
|
+
messages,
|
|
327
|
+
max_tokens,
|
|
328
|
+
temperature,
|
|
329
|
+
enable_thinking,
|
|
330
|
+
}),
|
|
331
|
+
});
|
|
332
|
+
} catch (e) {
|
|
333
|
+
lastError = e.message;
|
|
334
|
+
deadCount++;
|
|
335
|
+
if (deadCount >= models.length) break;
|
|
336
|
+
console.error(`[llmFetch] ${model} 网络失败: ${e.message},尝试下一个`);
|
|
337
|
+
continue;
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
const result = await response.json().catch(() => null);
|
|
341
|
+
const content = result?.choices?.[0]?.message?.content;
|
|
342
|
+
|
|
343
|
+
if (!content) {
|
|
344
|
+
lastError = result?.error?.message || `HTTP ${response.status}`;
|
|
345
|
+
deadCount++;
|
|
346
|
+
if (deadCount >= models.length) break;
|
|
347
|
+
console.error(`[llmFetch] ${model} 失败: ${lastError},尝试下一个`);
|
|
348
|
+
continue;
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
if (i > 0) {
|
|
352
|
+
console.error(`[llmFetch] ✅ auto 切换成功: ${model}`);
|
|
353
|
+
}
|
|
354
|
+
|
|
355
|
+
if (isAutoMode()) recordAutoSuccess(model);
|
|
356
|
+
_allDeadUntil = 0; // 成功则清除冷却
|
|
357
|
+
return content;
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
// 全部失败 → auto 模式下尝试 fallback 模型
|
|
361
|
+
if (isAutoMode()) {
|
|
362
|
+
const fbModel =
|
|
363
|
+
(await loadFromDb()).fallback_model || LLM_PROVIDERS[0].models[0].id;
|
|
364
|
+
const fbUrl = (await loadFromDb()).fallback_url || LLM_PROVIDERS[0].url;
|
|
365
|
+
console.error(`[llmFetch] auto 池全挂,尝试 fallback: ${fbModel}`);
|
|
366
|
+
try {
|
|
367
|
+
const { fetch: fbFetch } = await import("undici");
|
|
368
|
+
const fbRes = await fbFetch(fbUrl, {
|
|
369
|
+
method: "POST",
|
|
370
|
+
headers: {
|
|
371
|
+
"Content-Type": "application/json",
|
|
372
|
+
Authorization: `Bearer ${apiKey}`,
|
|
373
|
+
},
|
|
374
|
+
body: JSON.stringify({
|
|
375
|
+
model: fbModel,
|
|
376
|
+
messages,
|
|
377
|
+
max_tokens,
|
|
378
|
+
temperature,
|
|
379
|
+
enable_thinking,
|
|
380
|
+
}),
|
|
381
|
+
});
|
|
382
|
+
const fbJson = await fbRes.json().catch(() => null);
|
|
383
|
+
const fbContent = fbJson?.choices?.[0]?.message?.content;
|
|
384
|
+
if (fbContent) {
|
|
385
|
+
_allDeadUntil = 0;
|
|
386
|
+
return fbContent;
|
|
387
|
+
}
|
|
388
|
+
console.error(
|
|
389
|
+
`[llmFetch] fallback 也失败: ${fbJson?.error?.message || fbRes.status}`,
|
|
390
|
+
);
|
|
391
|
+
} catch (e) {
|
|
392
|
+
console.error(`[llmFetch] fallback 异常: ${e.message}`);
|
|
393
|
+
}
|
|
394
|
+
}
|
|
395
|
+
|
|
396
|
+
// 全部失败(含 fallback),冷却 30 秒
|
|
397
|
+
_allDeadUntil = Date.now() + 30000;
|
|
398
|
+
console.error(`[llmFetch] ❌ 全部模型均失败(含 fallback),冷却 30s`);
|
|
399
|
+
throw new Error(`全部模型均失败: ${lastError}`);
|
|
400
|
+
}
|
|
401
|
+
|
|
258
402
|
/**
|
|
259
403
|
* 更新 LLM 配置(通过 API 调用)
|
|
260
404
|
* @param {{ provider?: string, model?: string, url?: string, apiKey?: string }} updates
|