tt-help-cli-ycl 1.4.16 → 1.4.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/cli/comments.js +6 -4
- package/src/cli/refresh.js +5 -0
- package/src/cli/tag.js +20 -18
- package/src/cli/watch.js +15 -4
- package/src/lib/args.js +14 -0
- package/src/lib/constants.js +13 -2
- package/src/lib/tag-fetcher.js +7 -1
- package/src/watch/data-store.js +114 -8
- package/src/watch/db-crud.js +16 -1
- package/src/watch/db-schema.js +5 -0
- package/src/watch/db-stats.js +10 -6
- package/src/watch/db-tags.js +106 -6
- package/src/watch/public/app.js +308 -10
- package/src/watch/public/index.html +6 -1
- package/src/watch/public/style.css +30 -0
- package/src/watch/server.js +240 -12
- package/src/watch/tag-service.js +317 -10
package/package.json
CHANGED
package/src/cli/comments.js
CHANGED
|
@@ -646,11 +646,13 @@ async function runAutoMode(options) {
|
|
|
646
646
|
} catch (err) {
|
|
647
647
|
console.error(` × 获取任务失败: ${err.message}`);
|
|
648
648
|
consecutiveErrors++;
|
|
649
|
-
if (consecutiveErrors >
|
|
650
|
-
console.error(
|
|
651
|
-
|
|
649
|
+
if (consecutiveErrors > 0 && consecutiveErrors % 30 === 0) {
|
|
650
|
+
console.error(
|
|
651
|
+
` ⚠ 已连续获取失败 ${consecutiveErrors} 次,继续等待服务恢复...`,
|
|
652
|
+
);
|
|
652
653
|
}
|
|
653
|
-
|
|
654
|
+
const waitMs = Math.min(consecutiveErrors * 1000, 60000);
|
|
655
|
+
await delay(waitMs, waitMs + 1000);
|
|
654
656
|
continue;
|
|
655
657
|
}
|
|
656
658
|
consecutiveErrors = 0;
|
package/src/cli/refresh.js
CHANGED
|
@@ -54,6 +54,7 @@ export async function handleRefresh(options) {
|
|
|
54
54
|
exploreMaxFollowing,
|
|
55
55
|
exploreMaxFollowers,
|
|
56
56
|
exploreLocation,
|
|
57
|
+
exploreJobLocations,
|
|
57
58
|
exploreMaxUsers,
|
|
58
59
|
serverUrl,
|
|
59
60
|
explorePort,
|
|
@@ -106,6 +107,7 @@ export async function handleRefresh(options) {
|
|
|
106
107
|
`关注/粉丝: ${exploreEnableFollow ? "启用" : "禁用"} (${exploreMaxFollowing}/${exploreMaxFollowers})`,
|
|
107
108
|
);
|
|
108
109
|
console.error(`国家筛选: ${exploreLocation}`);
|
|
110
|
+
if (exploreJobLocations) console.error(`任务国家: ${exploreJobLocations}`);
|
|
109
111
|
console.error(`空闲间隔: ${exploreInterval || 30} 秒`);
|
|
110
112
|
if (effectiveProxy) {
|
|
111
113
|
console.error(
|
|
@@ -274,6 +276,9 @@ export async function handleRefresh(options) {
|
|
|
274
276
|
if (exploreRedoMaxAge > 0) {
|
|
275
277
|
redoUrl += `&maxAge=${exploreRedoMaxAge}`;
|
|
276
278
|
}
|
|
279
|
+
if (exploreJobLocations) {
|
|
280
|
+
redoUrl += `&locations=${encodeURIComponent(exploreJobLocations)}`;
|
|
281
|
+
}
|
|
277
282
|
let job = null;
|
|
278
283
|
try {
|
|
279
284
|
job = await apiGet(redoUrl);
|
package/src/cli/tag.js
CHANGED
|
@@ -528,13 +528,13 @@ async function scoreSingleTag(
|
|
|
528
528
|
log(` ✗ 标签不存在(TikTok 提示"找不到此话题标签"),标记为 dead`);
|
|
529
529
|
result.status = "dead";
|
|
530
530
|
result.error = "tag_not_found";
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
result.status = "dead";
|
|
534
|
-
result.error = "no videos found";
|
|
531
|
+
await reportToServer(baseUrl, result);
|
|
532
|
+
return result;
|
|
535
533
|
}
|
|
536
|
-
|
|
537
|
-
|
|
534
|
+
// 非 tag_not_found 的 0 视频 → 抛异常让其走 error 重试路径,避免误标 dead
|
|
535
|
+
throw new Error(
|
|
536
|
+
"no videos returned — possible network/captcha/slow-load, will retry",
|
|
537
|
+
);
|
|
538
538
|
}
|
|
539
539
|
|
|
540
540
|
// Step 2: 补充国家信息
|
|
@@ -1064,18 +1064,20 @@ export async function handleScoreAll(parsed) {
|
|
|
1064
1064
|
|
|
1065
1065
|
if (!videos || videos.length === 0) {
|
|
1066
1066
|
const deadSec = ((Date.now() - fetchStart) / 1000).toFixed(1);
|
|
1067
|
-
|
|
1068
|
-
` ⚠️
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1067
|
+
if (tagResult.error === "tag_not_found") {
|
|
1068
|
+
log(` ⚠️ 标签不存在 (${deadSec}s),标记 dead`);
|
|
1069
|
+
result.status = "dead";
|
|
1070
|
+
result.error = "tag_not_found";
|
|
1071
|
+
await reportToServer(baseUrl, result, clientId, clientMeta);
|
|
1072
|
+
totalScored++;
|
|
1073
|
+
await randomDelay(0, 5000);
|
|
1074
|
+
await recyclePage();
|
|
1075
|
+
await maybeRecycleForMemory();
|
|
1076
|
+
continue;
|
|
1077
|
+
}
|
|
1078
|
+
// 非 tag_not_found 的 0 视频 → 抛异常让其走 error 重试路径,避免误标 dead
|
|
1079
|
+
log(` ⚠️ 无视频 (${deadSec}s),可能网络/风控/加载慢 → 抛异常重试`);
|
|
1080
|
+
throw new Error("no videos returned (network/captcha/slow-load)");
|
|
1079
1081
|
}
|
|
1080
1082
|
|
|
1081
1083
|
// enrich: 先查服务端缓存,只对未知视频发 HTTP 请求
|
package/src/cli/watch.js
CHANGED
|
@@ -5,16 +5,26 @@ import { startWatchServer, openBrowser } from "../watch/server.js";
|
|
|
5
5
|
|
|
6
6
|
export async function handleWatch(options) {
|
|
7
7
|
const dataAnchor = options.dataAnchor || options.outputFile;
|
|
8
|
-
const {
|
|
9
|
-
|
|
8
|
+
const {
|
|
9
|
+
watchPort,
|
|
10
|
+
llmRefill,
|
|
11
|
+
llmRefillMinScore,
|
|
12
|
+
llmRefillSampleSize,
|
|
13
|
+
llmDiscover,
|
|
14
|
+
} = options;
|
|
10
15
|
|
|
11
16
|
if (!dataAnchor) {
|
|
12
|
-
console.error(
|
|
17
|
+
console.error(
|
|
18
|
+
"用法: tt-help watch -o <db路径> [-p 端口] [--llm-refill] [--no-llm-discover]",
|
|
19
|
+
);
|
|
13
20
|
console.error(" tt-help watch -o data/result.db");
|
|
14
21
|
console.error(" tt-help watch -o data/result.db -p 8080");
|
|
15
22
|
console.error(
|
|
16
23
|
" tt-help watch -o data/result.db --llm-refill --llm-refill-min 60",
|
|
17
24
|
);
|
|
25
|
+
console.error(
|
|
26
|
+
" tt-help watch -o data/result.db --no-llm-discover --no-llm-refill",
|
|
27
|
+
);
|
|
18
28
|
process.exit(1);
|
|
19
29
|
}
|
|
20
30
|
|
|
@@ -37,7 +47,8 @@ export async function handleWatch(options) {
|
|
|
37
47
|
dataAnchor,
|
|
38
48
|
watchPort,
|
|
39
49
|
store,
|
|
40
|
-
storeOptions,
|
|
50
|
+
{ ...storeOptions, llmDiscover: llmDiscover ?? true },
|
|
51
|
+
options.watchPassword,
|
|
41
52
|
);
|
|
42
53
|
openBrowser(port);
|
|
43
54
|
|
package/src/lib/args.js
CHANGED
|
@@ -347,9 +347,11 @@ function parseInfoArgs(args) {
|
|
|
347
347
|
function parseWatchArgs(args) {
|
|
348
348
|
let dataAnchor = "./result.db";
|
|
349
349
|
let watchPort = 3001;
|
|
350
|
+
let watchPassword = null;
|
|
350
351
|
let llmRefill = true;
|
|
351
352
|
let llmRefillMinScore = null;
|
|
352
353
|
let llmRefillSampleSize = null;
|
|
354
|
+
let llmDiscover = true;
|
|
353
355
|
|
|
354
356
|
for (let i = 0; i < args.length; i++) {
|
|
355
357
|
const arg = args[i];
|
|
@@ -357,6 +359,8 @@ function parseWatchArgs(args) {
|
|
|
357
359
|
dataAnchor = args[++i];
|
|
358
360
|
} else if (arg === "-p") {
|
|
359
361
|
watchPort = parseInt(args[++i]) || 3001;
|
|
362
|
+
} else if (arg === "-P" || arg === "--password") {
|
|
363
|
+
watchPassword = args[++i] || null;
|
|
360
364
|
} else if (arg === "--llm-refill") {
|
|
361
365
|
llmRefill = true;
|
|
362
366
|
} else if (arg === "--no-llm-refill") {
|
|
@@ -365,6 +369,10 @@ function parseWatchArgs(args) {
|
|
|
365
369
|
llmRefillMinScore = parseInt(args[++i]) || 60;
|
|
366
370
|
} else if (arg === "--llm-refill-sample") {
|
|
367
371
|
llmRefillSampleSize = parseInt(args[++i]) || 100;
|
|
372
|
+
} else if (arg === "--llm-discover") {
|
|
373
|
+
llmDiscover = true;
|
|
374
|
+
} else if (arg === "--no-llm-discover") {
|
|
375
|
+
llmDiscover = false;
|
|
368
376
|
}
|
|
369
377
|
}
|
|
370
378
|
|
|
@@ -373,9 +381,11 @@ function parseWatchArgs(args) {
|
|
|
373
381
|
outputFile: dataAnchor,
|
|
374
382
|
dataAnchor,
|
|
375
383
|
watchPort,
|
|
384
|
+
watchPassword,
|
|
376
385
|
llmRefill,
|
|
377
386
|
llmRefillMinScore,
|
|
378
387
|
llmRefillSampleSize,
|
|
388
|
+
llmDiscover,
|
|
379
389
|
urls: [],
|
|
380
390
|
outputFormat: "json",
|
|
381
391
|
exploreCount: 0,
|
|
@@ -471,6 +481,7 @@ function parseRefreshArgs(args) {
|
|
|
471
481
|
let exploreMaxVideos = 16;
|
|
472
482
|
let exploreRedoMaxAge = 0;
|
|
473
483
|
let exploreProxy = null;
|
|
484
|
+
let exploreJobLocations = null;
|
|
474
485
|
|
|
475
486
|
for (let i = 0; i < args.length; i++) {
|
|
476
487
|
const arg = args[i];
|
|
@@ -502,6 +513,8 @@ function parseRefreshArgs(args) {
|
|
|
502
513
|
exploreProxy = args[++i];
|
|
503
514
|
} else if (arg === "--location") {
|
|
504
515
|
exploreLocation = args[++i];
|
|
516
|
+
} else if (arg === "--job-locations") {
|
|
517
|
+
exploreJobLocations = args[++i];
|
|
505
518
|
} else if (arg === "--enable-follow") {
|
|
506
519
|
exploreEnableFollow = true;
|
|
507
520
|
} else if (arg === "--disable-follow") {
|
|
@@ -519,6 +532,7 @@ function parseRefreshArgs(args) {
|
|
|
519
532
|
exploreMaxFollowing,
|
|
520
533
|
exploreMaxFollowers,
|
|
521
534
|
exploreLocation,
|
|
535
|
+
exploreJobLocations,
|
|
522
536
|
exploreMaxUsers,
|
|
523
537
|
explorePort,
|
|
524
538
|
exploreBasePort,
|
package/src/lib/constants.js
CHANGED
|
@@ -145,6 +145,7 @@ const HELP_TEXT = [
|
|
|
145
145
|
" 选项:",
|
|
146
146
|
" --server <URL> 服务端地址,默认 http://127.0.0.1:3001",
|
|
147
147
|
` --location <国家代码> 国家筛选,逗号分隔,默认 ${DEFAULT_TARGET_LOCATIONS_CSV}`,
|
|
148
|
+
" --job-locations <国家> 任务国家筛选,逗号分隔(仅筛选服务端任务),基于真实国家字段 location_created",
|
|
148
149
|
" --max-videos <数量> 每用户最大视频数,默认 16",
|
|
149
150
|
" --enable-follow 启用关注/粉丝提取(默认启用)",
|
|
150
151
|
" --disable-follow 禁用关注/粉丝提取",
|
|
@@ -176,10 +177,20 @@ const HELP_TEXT = [
|
|
|
176
177
|
" 示例: tt-help attach -p 5 -i 10",
|
|
177
178
|
" tt-help attach -c PL,DE,FR -p 5",
|
|
178
179
|
"",
|
|
179
|
-
" tt-help watch -o <db路径> [-p 端口]",
|
|
180
|
+
" tt-help watch -o <db路径> [-p 端口] [--no-llm-refill] [--no-llm-discover]",
|
|
180
181
|
" 启动监控服务;运行期主数据仅来自 SQLite",
|
|
182
|
+
" 选项:",
|
|
183
|
+
" -p, --port <端口> 服务端口,默认 3000",
|
|
184
|
+
" -P, --password <密码> Web 面板登录密码",
|
|
185
|
+
" --llm-refill 启用 LLM 自动筛选毛料任务(默认开启)",
|
|
186
|
+
" --no-llm-refill 关闭 LLM 任务打分",
|
|
187
|
+
" --llm-discover 启用 LLM 自动发现标签(默认开启)",
|
|
188
|
+
" --no-llm-discover 关闭 LLM tag 发现",
|
|
189
|
+
" --llm-refill-min <分数> 最低入围分,默认 60",
|
|
190
|
+
" --llm-refill-sample <数量> 采样量,默认 100",
|
|
181
191
|
" 示例: tt-help watch -o data/result.db",
|
|
182
192
|
" tt-help watch -o data/result.db -p 8080",
|
|
193
|
+
" tt-help watch -o data/result.db --no-llm-discover --no-llm-refill",
|
|
183
194
|
"",
|
|
184
195
|
" tt-help open <端口号>",
|
|
185
196
|
" 打开指定端口的浏览器,用于配置 TikTok 登录账户",
|
|
@@ -315,7 +326,7 @@ const HELP_TEXT = [
|
|
|
315
326
|
];
|
|
316
327
|
|
|
317
328
|
const PUBLIC_HELP_HIDDEN_HEADERS = new Set([
|
|
318
|
-
" tt-help watch -o <db路径> [-p 端口]",
|
|
329
|
+
" tt-help watch -o <db路径> [-p 端口] [--no-llm-refill] [--no-llm-discover]",
|
|
319
330
|
" videostats <db路径> -p <并发数>",
|
|
320
331
|
" db-import --db <db路径> [--users users.json] [--done done.json] [--videos videos.json]",
|
|
321
332
|
]);
|
package/src/lib/tag-fetcher.js
CHANGED
|
@@ -6,7 +6,7 @@ import { CDNBlockedError } from "./parse-ssr.mjs";
|
|
|
6
6
|
|
|
7
7
|
const TAG_URL = "https://www.tiktok.com/tag";
|
|
8
8
|
const SCROLL_INTERVAL = 3000;
|
|
9
|
-
const MAX_STALE_ROUNDS =
|
|
9
|
+
const MAX_STALE_ROUNDS = 5;
|
|
10
10
|
|
|
11
11
|
function sleep(ms) {
|
|
12
12
|
return new Promise((r) => setTimeout(r, ms));
|
|
@@ -113,6 +113,12 @@ export async function fetchTagData(tag, options = {}) {
|
|
|
113
113
|
|
|
114
114
|
await page.waitForTimeout(3000);
|
|
115
115
|
|
|
116
|
+
// 等待首屏 API 数据到达(避免网络慢时 0 视频进入滚动循环)
|
|
117
|
+
const videoWaitStart = Date.now();
|
|
118
|
+
while (seenVideoIds.size === 0 && Date.now() - videoWaitStart < 15000) {
|
|
119
|
+
await sleep(1000);
|
|
120
|
+
}
|
|
121
|
+
|
|
116
122
|
// 检测页面异常:验证码、网络错误、标签不存在
|
|
117
123
|
const pageError = await page.evaluate(() => {
|
|
118
124
|
const text = document.body?.innerText || "";
|
package/src/watch/data-store.js
CHANGED
|
@@ -100,6 +100,7 @@ import {
|
|
|
100
100
|
getProductiveRefreshTask,
|
|
101
101
|
claimRefreshTag,
|
|
102
102
|
reportRefreshResult,
|
|
103
|
+
getRefreshStats,
|
|
103
104
|
} from "./db-tags.js";
|
|
104
105
|
|
|
105
106
|
// LLM 打分
|
|
@@ -380,6 +381,9 @@ function getTargetUsersByCountryFromDb(targetLocations = [], options = {}) {
|
|
|
380
381
|
|
|
381
382
|
// 分页模式:按国家或全局分页查询用户
|
|
382
383
|
if (limit !== undefined) {
|
|
384
|
+
const filterIsTarget =
|
|
385
|
+
filterCountry && targetLocations.includes(filterCountry);
|
|
386
|
+
|
|
383
387
|
let sql = `
|
|
384
388
|
SELECT
|
|
385
389
|
unique_id,
|
|
@@ -401,13 +405,22 @@ function getTargetUsersByCountryFromDb(targetLocations = [], options = {}) {
|
|
|
401
405
|
FROM jobs
|
|
402
406
|
WHERE tt_seller = 1
|
|
403
407
|
AND verified = 0
|
|
404
|
-
AND location_created IN (${placeholders})
|
|
405
408
|
`;
|
|
406
|
-
const params = [
|
|
409
|
+
const params = [];
|
|
410
|
+
|
|
411
|
+
// 目标国家筛选:仅当 filterCountry 是目标国家或不筛选时才加 IN 限制
|
|
412
|
+
if (filterIsTarget || !filterCountry) {
|
|
413
|
+
sql += ` AND location_created IN (${placeholders})`;
|
|
414
|
+
params.push(...targetLocations);
|
|
415
|
+
}
|
|
407
416
|
|
|
408
417
|
if (filterCountry) {
|
|
409
418
|
sql += ` AND location_created = ?`;
|
|
410
419
|
params.push(filterCountry);
|
|
420
|
+
// 筛选非目标国家时,仅返回已修改的用户(与“其他国家”语义一致)
|
|
421
|
+
if (!filterIsTarget) {
|
|
422
|
+
sql += ` AND modified_at IS NOT NULL`;
|
|
423
|
+
}
|
|
411
424
|
}
|
|
412
425
|
|
|
413
426
|
if (search) {
|
|
@@ -1083,6 +1096,9 @@ export function createStore(filePath, options = {}) {
|
|
|
1083
1096
|
|
|
1084
1097
|
offsetStore.set("_tag", tagOffset);
|
|
1085
1098
|
|
|
1099
|
+
// 让出事件循环,避免长时间阻塞监控页面及其他客户端请求
|
|
1100
|
+
await new Promise(setImmediate);
|
|
1101
|
+
|
|
1086
1102
|
// ===== 第二步:一次性取 comment(全局,最多 llmCommentLimit 条,免打分)=====
|
|
1087
1103
|
let commentOffset = offsetStore.get("_comment") || 0;
|
|
1088
1104
|
const commentGlobalCount =
|
|
@@ -1134,6 +1150,9 @@ export function createStore(filePath, options = {}) {
|
|
|
1134
1150
|
|
|
1135
1151
|
offsetStore.set("_comment", commentOffset);
|
|
1136
1152
|
|
|
1153
|
+
// 让出事件循环,避免长时间阻塞监控页面及其他客户端请求
|
|
1154
|
+
await new Promise(setImmediate);
|
|
1155
|
+
|
|
1137
1156
|
// ===== 第三步:按国家取非 tag 非 comment,走 LLM 打分,直到合格数达到 llmLLMTarget =====
|
|
1138
1157
|
for (const location of normalizedLocations) {
|
|
1139
1158
|
if (allLLMQualified.length >= llmLLMTarget) break;
|
|
@@ -1182,6 +1201,9 @@ export function createStore(filePath, options = {}) {
|
|
|
1182
1201
|
|
|
1183
1202
|
if (!samples.length) break;
|
|
1184
1203
|
|
|
1204
|
+
// 让出事件循环,LLM API 调用期间允许处理其他请求
|
|
1205
|
+
await new Promise(setImmediate);
|
|
1206
|
+
|
|
1185
1207
|
const scores = await scoreJobsBatch(
|
|
1186
1208
|
samples,
|
|
1187
1209
|
DEFAULT_TARGET_LOCATIONS,
|
|
@@ -1222,6 +1244,9 @@ export function createStore(filePath, options = {}) {
|
|
|
1222
1244
|
`[data-store] LLM 调度完成: tag ${allTagQualified.length} + comment ${allCommentQualified.length} + LLM ${allLLMQualified.length} = 共 ${qualified.length} 条`,
|
|
1223
1245
|
);
|
|
1224
1246
|
|
|
1247
|
+
// 让出事件循环,避免事务写入阻塞监控页面
|
|
1248
|
+
await new Promise(setImmediate);
|
|
1249
|
+
|
|
1225
1250
|
// 移动符合条件的记录
|
|
1226
1251
|
const placeholders = qualified.map(() => "?").join(", ");
|
|
1227
1252
|
const moveTxn = getDb().transaction(() => {
|
|
@@ -2476,12 +2501,12 @@ export function createStore(filePath, options = {}) {
|
|
|
2476
2501
|
return { saved: true, pinned: user.pinned };
|
|
2477
2502
|
}
|
|
2478
2503
|
|
|
2479
|
-
function getNextRedoJob(userId, maxAgeSeconds = 43200) {
|
|
2504
|
+
function getNextRedoJob(userId, maxAgeSeconds = 43200, locations = null) {
|
|
2480
2505
|
if (getDb()) {
|
|
2481
2506
|
const now = Date.now();
|
|
2482
2507
|
const threshold = now - maxAgeSeconds * 1000;
|
|
2483
2508
|
const defaultTime = new Date("2016-01-01T00:00:00Z").getTime();
|
|
2484
|
-
const targetLocations = DEFAULT_TARGET_LOCATIONS;
|
|
2509
|
+
const targetLocations = locations || DEFAULT_TARGET_LOCATIONS;
|
|
2485
2510
|
const placeholders = targetLocations.map(() => "?").join(",");
|
|
2486
2511
|
const row = getDb()
|
|
2487
2512
|
.prepare(
|
|
@@ -2515,7 +2540,7 @@ export function createStore(filePath, options = {}) {
|
|
|
2515
2540
|
const defaultTime = new Date("2016-01-01T00:00:00Z").getTime();
|
|
2516
2541
|
|
|
2517
2542
|
// 筛选目标国家用户,按 refreshTime 升序取最远的(没有则默认 2016-01-01)
|
|
2518
|
-
const targetLocations = DEFAULT_TARGET_LOCATIONS;
|
|
2543
|
+
const targetLocations = locations || DEFAULT_TARGET_LOCATIONS;
|
|
2519
2544
|
const targetUsers = data.filter(
|
|
2520
2545
|
(u) =>
|
|
2521
2546
|
u.ttSeller &&
|
|
@@ -2699,13 +2724,15 @@ export function createStore(filePath, options = {}) {
|
|
|
2699
2724
|
|
|
2700
2725
|
if (getDb()) {
|
|
2701
2726
|
const l = Math.max(1, parseInt(limit) || 5);
|
|
2727
|
+
const fiveMinutesAgo = Date.now() - 5 * 60 * 1000;
|
|
2702
2728
|
|
|
2703
2729
|
let sql = `
|
|
2704
2730
|
SELECT *
|
|
2705
2731
|
FROM jobs_base
|
|
2706
2732
|
WHERE COALESCE(user_update_count, 0) <= 0
|
|
2733
|
+
OR (COALESCE(user_update_count, 0) > 0 AND updated_at < ?)
|
|
2707
2734
|
`;
|
|
2708
|
-
const sqlParams = [];
|
|
2735
|
+
const sqlParams = [fiveMinutesAgo];
|
|
2709
2736
|
|
|
2710
2737
|
if (hasCountryFilter) {
|
|
2711
2738
|
const placeholders = targetCountries.map(() => "?").join(", ");
|
|
@@ -2750,14 +2777,17 @@ export function createStore(filePath, options = {}) {
|
|
|
2750
2777
|
}
|
|
2751
2778
|
|
|
2752
2779
|
const l = Math.max(1, parseInt(limit) || 5);
|
|
2780
|
+
const fiveMinutesAgo = Date.now() - 5 * 60 * 1000;
|
|
2753
2781
|
const pending = data
|
|
2754
2782
|
.filter((u) => {
|
|
2755
2783
|
const updateCount = u.userUpdateCount;
|
|
2756
|
-
//
|
|
2784
|
+
// user_update_count <= 0 的记录可直接领取
|
|
2785
|
+
// user_update_count > 0 但 updatedAt 超过 5 分钟的,视为客户端失败/超时,重新回收
|
|
2757
2786
|
if (
|
|
2758
2787
|
updateCount === null ||
|
|
2759
2788
|
updateCount === undefined ||
|
|
2760
|
-
updateCount <= 0
|
|
2789
|
+
updateCount <= 0 ||
|
|
2790
|
+
(u.updatedAt && u.updatedAt < fiveMinutesAgo)
|
|
2761
2791
|
) {
|
|
2762
2792
|
if (hasCountryFilter) {
|
|
2763
2793
|
const loc = (u.guessedLocation || "").toUpperCase();
|
|
@@ -3345,6 +3375,7 @@ export function createStore(filePath, options = {}) {
|
|
|
3345
3375
|
getActiveClients,
|
|
3346
3376
|
moveSellerJobsToBase, // 将 jobs/raw_jobs 中 tt_seller=1 且 video_count=0 的记录移动到 jobs_base
|
|
3347
3377
|
moveRestrictedJobsToBase, // 将 jobs 中 status='restricted' 的记录移动到 jobs_base
|
|
3378
|
+
moveSingleJobToBase, // 将单个 job 移动到 jobs_base 并重置(目标商家页面用)
|
|
3348
3379
|
registerVideos,
|
|
3349
3380
|
getVideo,
|
|
3350
3381
|
getVideos,
|
|
@@ -3372,6 +3403,7 @@ export function createStore(filePath, options = {}) {
|
|
|
3372
3403
|
getProductiveRefreshTask,
|
|
3373
3404
|
claimRefreshTag,
|
|
3374
3405
|
reportRefreshResult,
|
|
3406
|
+
getRefreshStats,
|
|
3375
3407
|
data,
|
|
3376
3408
|
};
|
|
3377
3409
|
|
|
@@ -3539,4 +3571,78 @@ export function createStore(filePath, options = {}) {
|
|
|
3539
3571
|
markStatsDirty();
|
|
3540
3572
|
return { ok: true, moved: deleted.changes || 0 };
|
|
3541
3573
|
}
|
|
3574
|
+
|
|
3575
|
+
// 将单个 job 移动到 jobs_base(用于目标商家页面"移到base重走流程")
|
|
3576
|
+
function moveSingleJobToBase(uniqueId) {
|
|
3577
|
+
const db = getDb();
|
|
3578
|
+
if (!db) return { ok: false, error: "db not ready" };
|
|
3579
|
+
|
|
3580
|
+
const safeId = String(uniqueId).trim();
|
|
3581
|
+
if (!safeId) return { ok: false, error: "invalid uniqueId" };
|
|
3582
|
+
|
|
3583
|
+
const existing = db
|
|
3584
|
+
.prepare("SELECT * FROM jobs WHERE unique_id = ?")
|
|
3585
|
+
.get(safeId);
|
|
3586
|
+
if (!existing) return { ok: false, error: "user not found in jobs" };
|
|
3587
|
+
|
|
3588
|
+
const COLUMNS = [
|
|
3589
|
+
"unique_id",
|
|
3590
|
+
"nickname",
|
|
3591
|
+
"status",
|
|
3592
|
+
"sources",
|
|
3593
|
+
"claimed_by",
|
|
3594
|
+
"claimed_at",
|
|
3595
|
+
"error",
|
|
3596
|
+
"pinned",
|
|
3597
|
+
"no_video",
|
|
3598
|
+
"restricted",
|
|
3599
|
+
"user_update_count",
|
|
3600
|
+
"tt_seller",
|
|
3601
|
+
"verified",
|
|
3602
|
+
"video_count",
|
|
3603
|
+
"comment_count",
|
|
3604
|
+
"guessed_location",
|
|
3605
|
+
"location_created",
|
|
3606
|
+
"confirmed_location",
|
|
3607
|
+
"modified_at",
|
|
3608
|
+
"follower_count",
|
|
3609
|
+
"following_count",
|
|
3610
|
+
"heart_count",
|
|
3611
|
+
"refresh_time",
|
|
3612
|
+
"processed",
|
|
3613
|
+
"processed_at",
|
|
3614
|
+
"created_at",
|
|
3615
|
+
"updated_at",
|
|
3616
|
+
"region",
|
|
3617
|
+
"signature",
|
|
3618
|
+
"bio_link",
|
|
3619
|
+
"sec_uid",
|
|
3620
|
+
"status_code",
|
|
3621
|
+
"latest_video_time",
|
|
3622
|
+
];
|
|
3623
|
+
const cols = COLUMNS.join(",");
|
|
3624
|
+
|
|
3625
|
+
try {
|
|
3626
|
+
db.prepare(
|
|
3627
|
+
`INSERT OR REPLACE INTO jobs_base (${cols})
|
|
3628
|
+
SELECT ${cols} FROM jobs WHERE unique_id = ?`,
|
|
3629
|
+
).run(safeId);
|
|
3630
|
+
|
|
3631
|
+
// 重置 user_update_count,保留 tt_seller,状态改为 pending
|
|
3632
|
+
db.prepare(
|
|
3633
|
+
`UPDATE jobs_base SET user_update_count = 0, status = 'pending',
|
|
3634
|
+
claimed_by = NULL, claimed_at = NULL, processed = 0, processed_at = NULL,
|
|
3635
|
+
error = NULL, restricted = 0
|
|
3636
|
+
WHERE unique_id = ?`,
|
|
3637
|
+
).run(safeId);
|
|
3638
|
+
|
|
3639
|
+
db.prepare("DELETE FROM jobs WHERE unique_id = ?").run(safeId);
|
|
3640
|
+
|
|
3641
|
+
markStatsDirty();
|
|
3642
|
+
console.error(`[DB] moveSingleJobToBase: ${safeId} → jobs_base`);
|
|
3643
|
+
return { ok: true };
|
|
3644
|
+
} catch (e) {
|
|
3645
|
+
return { ok: false, error: e.message };
|
|
3646
|
+
}
|
|
3647
|
+
}
|
|
3542
3648
|
}
|
package/src/watch/db-crud.js
CHANGED
|
@@ -479,8 +479,23 @@ export function getPendingJobsUserUpdateCount() {
|
|
|
479
479
|
.get().c;
|
|
480
480
|
}
|
|
481
481
|
|
|
482
|
+
// raw_jobs 计数缓存:raw_jobs 表数据量大(千万级),COUNT(*) 全表扫描耗时长,
|
|
483
|
+
// 且该值变化频率低(仅在 batchUpdateUserInfo 移入非商家时变化),缓存 60 秒。
|
|
484
|
+
let _rawJobsCountCache = null;
|
|
485
|
+
let _rawJobsCountCacheTime = 0;
|
|
486
|
+
const RAW_JOBS_COUNT_CACHE_TTL = 60000; // 60 秒
|
|
487
|
+
|
|
482
488
|
export function getRawJobsCount() {
|
|
489
|
+
const now = Date.now();
|
|
490
|
+
if (
|
|
491
|
+
_rawJobsCountCache !== null &&
|
|
492
|
+
now - _rawJobsCountCacheTime < RAW_JOBS_COUNT_CACHE_TTL
|
|
493
|
+
) {
|
|
494
|
+
return _rawJobsCountCache;
|
|
495
|
+
}
|
|
483
496
|
const db = getDb();
|
|
484
497
|
if (!db) return 0;
|
|
485
|
-
|
|
498
|
+
_rawJobsCountCache = db.prepare("SELECT COUNT(*) as c FROM raw_jobs").get().c;
|
|
499
|
+
_rawJobsCountCacheTime = now;
|
|
500
|
+
return _rawJobsCountCache;
|
|
486
501
|
}
|
package/src/watch/db-schema.js
CHANGED
|
@@ -367,6 +367,7 @@ export function initDb(filePath) {
|
|
|
367
367
|
round INTEGER NOT NULL DEFAULT 1,
|
|
368
368
|
strategy TEXT,
|
|
369
369
|
tags_generated TEXT NOT NULL DEFAULT '[]',
|
|
370
|
+
tags_raw TEXT,
|
|
370
371
|
tags_added INTEGER DEFAULT 0,
|
|
371
372
|
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
372
373
|
productive_sample TEXT,
|
|
@@ -375,6 +376,10 @@ export function initDb(filePath) {
|
|
|
375
376
|
avg_dead_score REAL DEFAULT 0
|
|
376
377
|
)
|
|
377
378
|
`);
|
|
379
|
+
// 迁移:tags_raw 列(用于记录 LLM 原始生成的 tag,去重前)
|
|
380
|
+
try {
|
|
381
|
+
db.exec("ALTER TABLE discover_log ADD COLUMN tags_raw TEXT");
|
|
382
|
+
} catch {}
|
|
378
383
|
|
|
379
384
|
// ===== reflection_rules 表(反思闭环:LLM 自我学习规则) =====
|
|
380
385
|
db.exec(`
|
package/src/watch/db-stats.js
CHANGED
|
@@ -61,11 +61,7 @@ export function getDashboardStatsFromDb(targetLocations = []) {
|
|
|
61
61
|
COALESCE(location_created, '未知') as country,
|
|
62
62
|
COUNT(*) as count,
|
|
63
63
|
SUM(CASE
|
|
64
|
-
WHEN tt_seller = 1 AND verified = 0
|
|
65
|
-
targetLocations.length
|
|
66
|
-
? `AND location_created IN (${targetPlaceholders})`
|
|
67
|
-
: "AND 1 = 0"
|
|
68
|
-
}
|
|
64
|
+
WHEN tt_seller = 1 AND verified = 0
|
|
69
65
|
THEN 1 ELSE 0 END) as targetCount
|
|
70
66
|
FROM jobs
|
|
71
67
|
WHERE status = 'done'
|
|
@@ -73,7 +69,7 @@ export function getDashboardStatsFromDb(targetLocations = []) {
|
|
|
73
69
|
ORDER BY count DESC
|
|
74
70
|
`,
|
|
75
71
|
)
|
|
76
|
-
.all(
|
|
72
|
+
.all();
|
|
77
73
|
|
|
78
74
|
const targetCountryStats = targetLocations.length
|
|
79
75
|
? db
|
|
@@ -95,9 +91,17 @@ export function getDashboardStatsFromDb(targetLocations = []) {
|
|
|
95
91
|
|
|
96
92
|
const tagCount = db.prepare("SELECT COUNT(*) as total FROM tags").get().total;
|
|
97
93
|
|
|
94
|
+
const commentTaskCount = db
|
|
95
|
+
.prepare(
|
|
96
|
+
`SELECT COUNT(*) as total FROM videos
|
|
97
|
+
WHERE user_update_count IS NULL OR user_update_count <= 0`,
|
|
98
|
+
)
|
|
99
|
+
.get().total;
|
|
100
|
+
|
|
98
101
|
return {
|
|
99
102
|
totalUsers: aggregateRow.total,
|
|
100
103
|
tagCount,
|
|
104
|
+
commentTaskCount,
|
|
101
105
|
rawJobs: getRawJobsCount(),
|
|
102
106
|
dbTotalUsers: getUserDbCount(),
|
|
103
107
|
jobsTotal: aggregateRow.total,
|