tt-help-cli-ycl 1.4.17 → 1.4.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/cli/comments.js +6 -4
- package/src/cli/tag.js +5 -7
- package/src/cli/watch.js +15 -4
- package/src/lib/args.js +22 -7
- package/src/lib/constants.js +12 -2
- package/src/watch/data-store.js +206 -8
- package/src/watch/db-crud.js +16 -1
- package/src/watch/db-schema.js +5 -0
- package/src/watch/db-stats.js +2 -6
- package/src/watch/db-tags.js +106 -6
- package/src/watch/public/app.js +554 -11
- package/src/watch/public/index.html +4 -1
- package/src/watch/public/style.css +41 -0
- package/src/watch/server.js +273 -7
- package/src/watch/tag-service.js +317 -10
package/package.json
CHANGED
package/src/cli/comments.js
CHANGED
|
@@ -646,11 +646,13 @@ async function runAutoMode(options) {
|
|
|
646
646
|
} catch (err) {
|
|
647
647
|
console.error(` × 获取任务失败: ${err.message}`);
|
|
648
648
|
consecutiveErrors++;
|
|
649
|
-
if (consecutiveErrors >
|
|
650
|
-
console.error(
|
|
651
|
-
|
|
649
|
+
if (consecutiveErrors > 0 && consecutiveErrors % 30 === 0) {
|
|
650
|
+
console.error(
|
|
651
|
+
` ⚠ 已连续获取失败 ${consecutiveErrors} 次,继续等待服务恢复...`,
|
|
652
|
+
);
|
|
652
653
|
}
|
|
653
|
-
|
|
654
|
+
const waitMs = Math.min(consecutiveErrors * 1000, 60000);
|
|
655
|
+
await delay(waitMs, waitMs + 1000);
|
|
654
656
|
continue;
|
|
655
657
|
}
|
|
656
658
|
consecutiveErrors = 0;
|
package/src/cli/tag.js
CHANGED
|
@@ -532,7 +532,9 @@ async function scoreSingleTag(
|
|
|
532
532
|
return result;
|
|
533
533
|
}
|
|
534
534
|
// 非 tag_not_found 的 0 视频 → 抛异常让其走 error 重试路径,避免误标 dead
|
|
535
|
-
throw new Error(
|
|
535
|
+
throw new Error(
|
|
536
|
+
"no videos returned — possible network/captcha/slow-load, will retry",
|
|
537
|
+
);
|
|
536
538
|
}
|
|
537
539
|
|
|
538
540
|
// Step 2: 补充国家信息
|
|
@@ -1063,9 +1065,7 @@ export async function handleScoreAll(parsed) {
|
|
|
1063
1065
|
if (!videos || videos.length === 0) {
|
|
1064
1066
|
const deadSec = ((Date.now() - fetchStart) / 1000).toFixed(1);
|
|
1065
1067
|
if (tagResult.error === "tag_not_found") {
|
|
1066
|
-
log(
|
|
1067
|
-
` ⚠️ 标签不存在 (${deadSec}s),标记 dead`,
|
|
1068
|
-
);
|
|
1068
|
+
log(` ⚠️ 标签不存在 (${deadSec}s),标记 dead`);
|
|
1069
1069
|
result.status = "dead";
|
|
1070
1070
|
result.error = "tag_not_found";
|
|
1071
1071
|
await reportToServer(baseUrl, result, clientId, clientMeta);
|
|
@@ -1076,9 +1076,7 @@ export async function handleScoreAll(parsed) {
|
|
|
1076
1076
|
continue;
|
|
1077
1077
|
}
|
|
1078
1078
|
// 非 tag_not_found 的 0 视频 → 抛异常让其走 error 重试路径,避免误标 dead
|
|
1079
|
-
log(
|
|
1080
|
-
` ⚠️ 无视频 (${deadSec}s),可能网络/风控/加载慢 → 抛异常重试`,
|
|
1081
|
-
);
|
|
1079
|
+
log(` ⚠️ 无视频 (${deadSec}s),可能网络/风控/加载慢 → 抛异常重试`);
|
|
1082
1080
|
throw new Error("no videos returned (network/captcha/slow-load)");
|
|
1083
1081
|
}
|
|
1084
1082
|
|
package/src/cli/watch.js
CHANGED
|
@@ -5,16 +5,26 @@ import { startWatchServer, openBrowser } from "../watch/server.js";
|
|
|
5
5
|
|
|
6
6
|
export async function handleWatch(options) {
|
|
7
7
|
const dataAnchor = options.dataAnchor || options.outputFile;
|
|
8
|
-
const {
|
|
9
|
-
|
|
8
|
+
const {
|
|
9
|
+
watchPort,
|
|
10
|
+
llmRefill,
|
|
11
|
+
llmRefillMinScore,
|
|
12
|
+
llmRefillSampleSize,
|
|
13
|
+
llmDiscover,
|
|
14
|
+
} = options;
|
|
10
15
|
|
|
11
16
|
if (!dataAnchor) {
|
|
12
|
-
console.error(
|
|
17
|
+
console.error(
|
|
18
|
+
"用法: tt-help watch -o <db路径> [-p 端口] [--llm-refill] [--no-llm-discover]",
|
|
19
|
+
);
|
|
13
20
|
console.error(" tt-help watch -o data/result.db");
|
|
14
21
|
console.error(" tt-help watch -o data/result.db -p 8080");
|
|
15
22
|
console.error(
|
|
16
23
|
" tt-help watch -o data/result.db --llm-refill --llm-refill-min 60",
|
|
17
24
|
);
|
|
25
|
+
console.error(
|
|
26
|
+
" tt-help watch -o data/result.db --no-llm-discover --no-llm-refill",
|
|
27
|
+
);
|
|
18
28
|
process.exit(1);
|
|
19
29
|
}
|
|
20
30
|
|
|
@@ -37,7 +47,8 @@ export async function handleWatch(options) {
|
|
|
37
47
|
dataAnchor,
|
|
38
48
|
watchPort,
|
|
39
49
|
store,
|
|
40
|
-
storeOptions,
|
|
50
|
+
{ ...storeOptions, llmDiscover: llmDiscover ?? true },
|
|
51
|
+
options.watchPassword,
|
|
41
52
|
);
|
|
42
53
|
openBrowser(port);
|
|
43
54
|
|
package/src/lib/args.js
CHANGED
|
@@ -347,9 +347,11 @@ function parseInfoArgs(args) {
|
|
|
347
347
|
function parseWatchArgs(args) {
|
|
348
348
|
let dataAnchor = "./result.db";
|
|
349
349
|
let watchPort = 3001;
|
|
350
|
+
let watchPassword = null;
|
|
350
351
|
let llmRefill = true;
|
|
351
352
|
let llmRefillMinScore = null;
|
|
352
353
|
let llmRefillSampleSize = null;
|
|
354
|
+
let llmDiscover = true;
|
|
353
355
|
|
|
354
356
|
for (let i = 0; i < args.length; i++) {
|
|
355
357
|
const arg = args[i];
|
|
@@ -357,6 +359,8 @@ function parseWatchArgs(args) {
|
|
|
357
359
|
dataAnchor = args[++i];
|
|
358
360
|
} else if (arg === "-p") {
|
|
359
361
|
watchPort = parseInt(args[++i]) || 3001;
|
|
362
|
+
} else if (arg === "-P" || arg === "--password") {
|
|
363
|
+
watchPassword = args[++i] || null;
|
|
360
364
|
} else if (arg === "--llm-refill") {
|
|
361
365
|
llmRefill = true;
|
|
362
366
|
} else if (arg === "--no-llm-refill") {
|
|
@@ -365,6 +369,10 @@ function parseWatchArgs(args) {
|
|
|
365
369
|
llmRefillMinScore = parseInt(args[++i]) || 60;
|
|
366
370
|
} else if (arg === "--llm-refill-sample") {
|
|
367
371
|
llmRefillSampleSize = parseInt(args[++i]) || 100;
|
|
372
|
+
} else if (arg === "--llm-discover") {
|
|
373
|
+
llmDiscover = true;
|
|
374
|
+
} else if (arg === "--no-llm-discover") {
|
|
375
|
+
llmDiscover = false;
|
|
368
376
|
}
|
|
369
377
|
}
|
|
370
378
|
|
|
@@ -373,9 +381,11 @@ function parseWatchArgs(args) {
|
|
|
373
381
|
outputFile: dataAnchor,
|
|
374
382
|
dataAnchor,
|
|
375
383
|
watchPort,
|
|
384
|
+
watchPassword,
|
|
376
385
|
llmRefill,
|
|
377
386
|
llmRefillMinScore,
|
|
378
387
|
llmRefillSampleSize,
|
|
388
|
+
llmDiscover,
|
|
379
389
|
urls: [],
|
|
380
390
|
outputFormat: "json",
|
|
381
391
|
exploreCount: 0,
|
|
@@ -757,6 +767,8 @@ function parseTagArgs(args) {
|
|
|
757
767
|
let scoreTag = null;
|
|
758
768
|
let scoreCountries = null;
|
|
759
769
|
let scorePort = 9222;
|
|
770
|
+
// 延迟路由:--port 的值先暂存,等循环结束后根据已确定的 subcommand 分配
|
|
771
|
+
let rawTagPort = null;
|
|
760
772
|
|
|
761
773
|
for (let i = 0; i < args.length; i++) {
|
|
762
774
|
const arg = args[i];
|
|
@@ -798,13 +810,9 @@ function parseTagArgs(args) {
|
|
|
798
810
|
.map((s) => s.trim().toUpperCase())
|
|
799
811
|
.filter(Boolean);
|
|
800
812
|
} else if (arg === "--port") {
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
refreshPort = parseInt(args[++i]) || 8222;
|
|
805
|
-
} else {
|
|
806
|
-
scorePort = parseInt(args[++i]) || 9222;
|
|
807
|
-
}
|
|
813
|
+
// 延迟路由:暂存端口值,等循环结束后根据 subcommand 分配
|
|
814
|
+
// 避免 --port 出现在 refresh/score-all 之前时被错误分配到其他变量
|
|
815
|
+
rawTagPort = parseInt(args[++i]);
|
|
808
816
|
} else if (arg === "--proxy") {
|
|
809
817
|
scoreProxy = args[++i];
|
|
810
818
|
} else if (arg === "-p" || arg === "--prompt") {
|
|
@@ -842,6 +850,13 @@ function parseTagArgs(args) {
|
|
|
842
850
|
}
|
|
843
851
|
}
|
|
844
852
|
|
|
853
|
+
// ── 延迟路由:将 --port 值分配到正确的 subcommand 变量 ──
|
|
854
|
+
if (rawTagPort !== null && !isNaN(rawTagPort)) {
|
|
855
|
+
if (isScoreAll) scoreAllPort = rawTagPort;
|
|
856
|
+
else if (isRefresh) refreshPort = rawTagPort;
|
|
857
|
+
else scorePort = rawTagPort;
|
|
858
|
+
}
|
|
859
|
+
|
|
845
860
|
if (isDiscover) {
|
|
846
861
|
return {
|
|
847
862
|
subcommand: "tag-discover",
|
package/src/lib/constants.js
CHANGED
|
@@ -177,10 +177,20 @@ const HELP_TEXT = [
|
|
|
177
177
|
" 示例: tt-help attach -p 5 -i 10",
|
|
178
178
|
" tt-help attach -c PL,DE,FR -p 5",
|
|
179
179
|
"",
|
|
180
|
-
" tt-help watch -o <db路径> [-p 端口]",
|
|
180
|
+
" tt-help watch -o <db路径> [-p 端口] [--no-llm-refill] [--no-llm-discover]",
|
|
181
181
|
" 启动监控服务;运行期主数据仅来自 SQLite",
|
|
182
|
+
" 选项:",
|
|
183
|
+
" -p, --port <端口> 服务端口,默认 3000",
|
|
184
|
+
" -P, --password <密码> Web 面板登录密码",
|
|
185
|
+
" --llm-refill 启用 LLM 自动筛选毛料任务(默认开启)",
|
|
186
|
+
" --no-llm-refill 关闭 LLM 任务打分",
|
|
187
|
+
" --llm-discover 启用 LLM 自动发现标签(默认开启)",
|
|
188
|
+
" --no-llm-discover 关闭 LLM tag 发现",
|
|
189
|
+
" --llm-refill-min <分数> 最低入围分,默认 60",
|
|
190
|
+
" --llm-refill-sample <数量> 采样量,默认 100",
|
|
182
191
|
" 示例: tt-help watch -o data/result.db",
|
|
183
192
|
" tt-help watch -o data/result.db -p 8080",
|
|
193
|
+
" tt-help watch -o data/result.db --no-llm-discover --no-llm-refill",
|
|
184
194
|
"",
|
|
185
195
|
" tt-help open <端口号>",
|
|
186
196
|
" 打开指定端口的浏览器,用于配置 TikTok 登录账户",
|
|
@@ -316,7 +326,7 @@ const HELP_TEXT = [
|
|
|
316
326
|
];
|
|
317
327
|
|
|
318
328
|
const PUBLIC_HELP_HIDDEN_HEADERS = new Set([
|
|
319
|
-
" tt-help watch -o <db路径> [-p 端口]",
|
|
329
|
+
" tt-help watch -o <db路径> [-p 端口] [--no-llm-refill] [--no-llm-discover]",
|
|
320
330
|
" videostats <db路径> -p <并发数>",
|
|
321
331
|
" db-import --db <db路径> [--users users.json] [--done done.json] [--videos videos.json]",
|
|
322
332
|
]);
|
package/src/watch/data-store.js
CHANGED
|
@@ -100,6 +100,7 @@ import {
|
|
|
100
100
|
getProductiveRefreshTask,
|
|
101
101
|
claimRefreshTag,
|
|
102
102
|
reportRefreshResult,
|
|
103
|
+
getRefreshStats,
|
|
103
104
|
} from "./db-tags.js";
|
|
104
105
|
|
|
105
106
|
// LLM 打分
|
|
@@ -555,6 +556,64 @@ function getModifiedTargetUsersOutsideLocations(targetLocations = []) {
|
|
|
555
556
|
return { total: rows.length, countries };
|
|
556
557
|
}
|
|
557
558
|
|
|
559
|
+
/**
|
|
560
|
+
* 目标商家刷新进度统计:按国家统计最近 N 小时内刷新过的 tt_seller 用户数。
|
|
561
|
+
* 返回格式与 getRefreshStats(tags)一致,便于前端复用渲染逻辑。
|
|
562
|
+
*/
|
|
563
|
+
function getTargetRefreshStats(hours = 24) {
|
|
564
|
+
if (!getDb()) return { countries: [], total: 0, refreshed: 0 };
|
|
565
|
+
|
|
566
|
+
const targetLocations = DEFAULT_TARGET_LOCATIONS;
|
|
567
|
+
const placeholders = targetLocations.map(() => "?").join(", ");
|
|
568
|
+
|
|
569
|
+
// 按国家统计总数
|
|
570
|
+
const totalRows = getDb()
|
|
571
|
+
.prepare(
|
|
572
|
+
`
|
|
573
|
+
SELECT location_created as country, COUNT(*) as cnt
|
|
574
|
+
FROM jobs
|
|
575
|
+
WHERE tt_seller = 1
|
|
576
|
+
AND verified = 0
|
|
577
|
+
AND location_created IN (${placeholders})
|
|
578
|
+
GROUP BY location_created
|
|
579
|
+
ORDER BY cnt DESC
|
|
580
|
+
`,
|
|
581
|
+
)
|
|
582
|
+
.all(...targetLocations);
|
|
583
|
+
|
|
584
|
+
if (!totalRows.length) return { countries: [], total: 0, refreshed: 0 };
|
|
585
|
+
|
|
586
|
+
// 按国家统计最近 N 小时内刷新过的数量
|
|
587
|
+
const threshold = Date.now() - hours * 3600 * 1000;
|
|
588
|
+
const refreshedRows = getDb()
|
|
589
|
+
.prepare(
|
|
590
|
+
`
|
|
591
|
+
SELECT location_created as country, COUNT(*) as cnt
|
|
592
|
+
FROM jobs
|
|
593
|
+
WHERE tt_seller = 1
|
|
594
|
+
AND verified = 0
|
|
595
|
+
AND location_created IN (${placeholders})
|
|
596
|
+
AND refresh_time >= ?
|
|
597
|
+
GROUP BY location_created
|
|
598
|
+
`,
|
|
599
|
+
)
|
|
600
|
+
.all(...targetLocations, threshold);
|
|
601
|
+
const refreshedMap = new Map();
|
|
602
|
+
for (const r of refreshedRows) refreshedMap.set(r.country, r.cnt);
|
|
603
|
+
|
|
604
|
+
const results = [];
|
|
605
|
+
let totalAll = 0;
|
|
606
|
+
let refreshedAll = 0;
|
|
607
|
+
for (const r of totalRows) {
|
|
608
|
+
const refreshed = refreshedMap.get(r.country) || 0;
|
|
609
|
+
totalAll += r.cnt;
|
|
610
|
+
refreshedAll += refreshed;
|
|
611
|
+
results.push({ country: r.country, total: r.cnt, refreshed });
|
|
612
|
+
}
|
|
613
|
+
|
|
614
|
+
return { countries: results, total: totalAll, refreshed: refreshedAll };
|
|
615
|
+
}
|
|
616
|
+
|
|
558
617
|
export function createStore(filePath, options = {}) {
|
|
559
618
|
if (!filePath) {
|
|
560
619
|
throw new Error("createStore requires an explicit .db path");
|
|
@@ -1007,7 +1066,7 @@ export function createStore(filePath, options = {}) {
|
|
|
1007
1066
|
|
|
1008
1067
|
if (normalizedLocations && normalizedLocations.length > 0) {
|
|
1009
1068
|
conditions.push(
|
|
1010
|
-
`UPPER(COALESCE(guessed_location, '')) IN (${normalizedLocations.map(() => "?").join(", ")})`,
|
|
1069
|
+
`(UPPER(COALESCE(guessed_location, '')) IN (${normalizedLocations.map(() => "?").join(", ")}) OR COALESCE(guessed_location, '') = '')`,
|
|
1011
1070
|
);
|
|
1012
1071
|
args.push(...normalizedLocations);
|
|
1013
1072
|
}
|
|
@@ -1095,6 +1154,9 @@ export function createStore(filePath, options = {}) {
|
|
|
1095
1154
|
|
|
1096
1155
|
offsetStore.set("_tag", tagOffset);
|
|
1097
1156
|
|
|
1157
|
+
// 让出事件循环,避免长时间阻塞监控页面及其他客户端请求
|
|
1158
|
+
await new Promise(setImmediate);
|
|
1159
|
+
|
|
1098
1160
|
// ===== 第二步:一次性取 comment(全局,最多 llmCommentLimit 条,免打分)=====
|
|
1099
1161
|
let commentOffset = offsetStore.get("_comment") || 0;
|
|
1100
1162
|
const commentGlobalCount =
|
|
@@ -1146,6 +1208,9 @@ export function createStore(filePath, options = {}) {
|
|
|
1146
1208
|
|
|
1147
1209
|
offsetStore.set("_comment", commentOffset);
|
|
1148
1210
|
|
|
1211
|
+
// 让出事件循环,避免长时间阻塞监控页面及其他客户端请求
|
|
1212
|
+
await new Promise(setImmediate);
|
|
1213
|
+
|
|
1149
1214
|
// ===== 第三步:按国家取非 tag 非 comment,走 LLM 打分,直到合格数达到 llmLLMTarget =====
|
|
1150
1215
|
for (const location of normalizedLocations) {
|
|
1151
1216
|
if (allLLMQualified.length >= llmLLMTarget) break;
|
|
@@ -1157,7 +1222,7 @@ export function createStore(filePath, options = {}) {
|
|
|
1157
1222
|
const llmCount =
|
|
1158
1223
|
getDb()
|
|
1159
1224
|
.prepare(
|
|
1160
|
-
`SELECT COUNT(*) as c FROM raw_jobs WHERE ${whereSql} AND guessed_location = ? AND (sources NOT LIKE '%tag%' OR sources IS NULL) AND (sources NOT LIKE '%comment%' OR sources IS NULL)`,
|
|
1225
|
+
`SELECT COUNT(*) as c FROM raw_jobs WHERE ${whereSql} AND (guessed_location = ? OR COALESCE(guessed_location, '') = '') AND (sources NOT LIKE '%tag%' OR sources IS NULL) AND (sources NOT LIKE '%comment%' OR sources IS NULL)`,
|
|
1161
1226
|
)
|
|
1162
1227
|
.get(...locationArgs)?.c || 0;
|
|
1163
1228
|
|
|
@@ -1183,7 +1248,7 @@ export function createStore(filePath, options = {}) {
|
|
|
1183
1248
|
const samples = getDb()
|
|
1184
1249
|
.prepare(
|
|
1185
1250
|
`
|
|
1186
|
-
SELECT * FROM raw_jobs WHERE ${whereSql} AND guessed_location = ?
|
|
1251
|
+
SELECT * FROM raw_jobs WHERE ${whereSql} AND (guessed_location = ? OR COALESCE(guessed_location, '') = '')
|
|
1187
1252
|
AND (sources NOT LIKE '%tag%' OR sources IS NULL)
|
|
1188
1253
|
AND (sources NOT LIKE '%comment%' OR sources IS NULL)
|
|
1189
1254
|
ORDER BY COALESCE(video_count, 0) DESC, created_at DESC
|
|
@@ -1194,6 +1259,9 @@ export function createStore(filePath, options = {}) {
|
|
|
1194
1259
|
|
|
1195
1260
|
if (!samples.length) break;
|
|
1196
1261
|
|
|
1262
|
+
// 让出事件循环,LLM API 调用期间允许处理其他请求
|
|
1263
|
+
await new Promise(setImmediate);
|
|
1264
|
+
|
|
1197
1265
|
const scores = await scoreJobsBatch(
|
|
1198
1266
|
samples,
|
|
1199
1267
|
DEFAULT_TARGET_LOCATIONS,
|
|
@@ -1234,6 +1302,9 @@ export function createStore(filePath, options = {}) {
|
|
|
1234
1302
|
`[data-store] LLM 调度完成: tag ${allTagQualified.length} + comment ${allCommentQualified.length} + LLM ${allLLMQualified.length} = 共 ${qualified.length} 条`,
|
|
1235
1303
|
);
|
|
1236
1304
|
|
|
1305
|
+
// 让出事件循环,避免事务写入阻塞监控页面
|
|
1306
|
+
await new Promise(setImmediate);
|
|
1307
|
+
|
|
1237
1308
|
// 移动符合条件的记录
|
|
1238
1309
|
const placeholders = qualified.map(() => "?").join(", ");
|
|
1239
1310
|
const moveTxn = getDb().transaction(() => {
|
|
@@ -1348,7 +1419,7 @@ export function createStore(filePath, options = {}) {
|
|
|
1348
1419
|
const currentLimit = Math.max(0, Math.min(quota, safeLimit - moved));
|
|
1349
1420
|
if (!currentLimit) continue;
|
|
1350
1421
|
|
|
1351
|
-
const locationWhere = `${whereSql} AND UPPER(COALESCE(guessed_location, '')) =
|
|
1422
|
+
const locationWhere = `${whereSql} AND (UPPER(COALESCE(guessed_location, '')) = ? OR COALESCE(guessed_location, '') = '')`;
|
|
1352
1423
|
const locationArgs = [...args, location];
|
|
1353
1424
|
|
|
1354
1425
|
getDb()
|
|
@@ -1475,7 +1546,7 @@ export function createStore(filePath, options = {}) {
|
|
|
1475
1546
|
if (!group) return;
|
|
1476
1547
|
if (group.type === "include") {
|
|
1477
1548
|
where.push(
|
|
1478
|
-
`UPPER(COALESCE(guessed_location, '')) IN (${group.values.map(() => "?").join(", ")})`,
|
|
1549
|
+
`(UPPER(COALESCE(guessed_location, '')) IN (${group.values.map(() => "?").join(", ")}) OR COALESCE(guessed_location, '') = '')`,
|
|
1479
1550
|
);
|
|
1480
1551
|
args.push(...group.values);
|
|
1481
1552
|
return;
|
|
@@ -1546,7 +1617,7 @@ export function createStore(filePath, options = {}) {
|
|
|
1546
1617
|
}
|
|
1547
1618
|
if (normalizedLocations.length > 0) {
|
|
1548
1619
|
where.push(
|
|
1549
|
-
`UPPER(COALESCE(guessed_location, '')) IN (${normalizedLocations.map(() => "?").join(", ")})`,
|
|
1620
|
+
`(UPPER(COALESCE(guessed_location, '')) IN (${normalizedLocations.map(() => "?").join(", ")}) OR COALESCE(guessed_location, '') = '')`,
|
|
1550
1621
|
);
|
|
1551
1622
|
args.push(...normalizedLocations);
|
|
1552
1623
|
}
|
|
@@ -1944,7 +2015,7 @@ export function createStore(filePath, options = {}) {
|
|
|
1944
2015
|
if (!group) return;
|
|
1945
2016
|
if (group.type === "include") {
|
|
1946
2017
|
where.push(
|
|
1947
|
-
`UPPER(COALESCE(guessed_location, '')) IN (${group.values.map(() => "?").join(", ")})`,
|
|
2018
|
+
`(UPPER(COALESCE(guessed_location, '')) IN (${group.values.map(() => "?").join(", ")}) OR COALESCE(guessed_location, '') = '')`,
|
|
1948
2019
|
);
|
|
1949
2020
|
args.push(...group.values);
|
|
1950
2021
|
return;
|
|
@@ -2012,7 +2083,7 @@ export function createStore(filePath, options = {}) {
|
|
|
2012
2083
|
}
|
|
2013
2084
|
if (normalizedLocations.length > 0) {
|
|
2014
2085
|
where.push(
|
|
2015
|
-
`UPPER(COALESCE(guessed_location, '')) IN (${normalizedLocations.map(() => "?").join(", ")})`,
|
|
2086
|
+
`(UPPER(COALESCE(guessed_location, '')) IN (${normalizedLocations.map(() => "?").join(", ")}) OR COALESCE(guessed_location, '') = '')`,
|
|
2016
2087
|
);
|
|
2017
2088
|
args.push(...normalizedLocations);
|
|
2018
2089
|
}
|
|
@@ -3362,6 +3433,8 @@ export function createStore(filePath, options = {}) {
|
|
|
3362
3433
|
getActiveClients,
|
|
3363
3434
|
moveSellerJobsToBase, // 将 jobs/raw_jobs 中 tt_seller=1 且 video_count=0 的记录移动到 jobs_base
|
|
3364
3435
|
moveRestrictedJobsToBase, // 将 jobs 中 status='restricted' 的记录移动到 jobs_base
|
|
3436
|
+
moveNoCountryJobsToBase, // 将 jobs 中 status='done' 且无国家数据的记录移动到 jobs_base
|
|
3437
|
+
moveSingleJobToBase, // 将单个 job 移动到 jobs_base 并重置(目标商家页面用)
|
|
3365
3438
|
registerVideos,
|
|
3366
3439
|
getVideo,
|
|
3367
3440
|
getVideos,
|
|
@@ -3389,6 +3462,8 @@ export function createStore(filePath, options = {}) {
|
|
|
3389
3462
|
getProductiveRefreshTask,
|
|
3390
3463
|
claimRefreshTag,
|
|
3391
3464
|
reportRefreshResult,
|
|
3465
|
+
getRefreshStats,
|
|
3466
|
+
getTargetRefreshStats,
|
|
3392
3467
|
data,
|
|
3393
3468
|
};
|
|
3394
3469
|
|
|
@@ -3556,4 +3631,127 @@ export function createStore(filePath, options = {}) {
|
|
|
3556
3631
|
markStatsDirty();
|
|
3557
3632
|
return { ok: true, moved: deleted.changes || 0 };
|
|
3558
3633
|
}
|
|
3634
|
+
|
|
3635
|
+
// 将 jobs 中 status='done' 且无国家数据的记录移动到 jobs_base
|
|
3636
|
+
function moveNoCountryJobsToBase() {
|
|
3637
|
+
const db = getDb();
|
|
3638
|
+
if (!db) return { ok: false, error: "db not ready" };
|
|
3639
|
+
|
|
3640
|
+
const now = Date.now();
|
|
3641
|
+
const condition = `status = 'done'
|
|
3642
|
+
AND (guessed_location IS NULL OR guessed_location = '')
|
|
3643
|
+
AND (location_created IS NULL OR location_created = '')
|
|
3644
|
+
AND (confirmed_location IS NULL OR confirmed_location = '')`;
|
|
3645
|
+
|
|
3646
|
+
const countRow = db
|
|
3647
|
+
.prepare(`SELECT COUNT(*) as c FROM jobs WHERE ${condition}`)
|
|
3648
|
+
.get();
|
|
3649
|
+
if (!countRow?.c) return { ok: true, moved: 0 };
|
|
3650
|
+
|
|
3651
|
+
db.prepare(
|
|
3652
|
+
`
|
|
3653
|
+
INSERT OR REPLACE INTO jobs_base (
|
|
3654
|
+
unique_id, nickname, sources, pinned, no_video, user_update_count,
|
|
3655
|
+
tt_seller, verified, video_count, comment_count, guessed_location,
|
|
3656
|
+
location_created, follower_count, following_count, heart_count,
|
|
3657
|
+
refresh_time, region, signature, sec_uid, status_code,
|
|
3658
|
+
latest_video_time, bio_link, status, restricted, error, claimed_by,
|
|
3659
|
+
claimed_at, processed, processed_at, created_at, updated_at
|
|
3660
|
+
)
|
|
3661
|
+
SELECT
|
|
3662
|
+
unique_id, nickname, sources, pinned, no_video, 0,
|
|
3663
|
+
CASE WHEN tt_seller = 0 THEN NULL ELSE tt_seller END,
|
|
3664
|
+
verified, video_count, comment_count, guessed_location,
|
|
3665
|
+
location_created, follower_count, following_count, heart_count,
|
|
3666
|
+
refresh_time, region, signature, sec_uid, status_code,
|
|
3667
|
+
latest_video_time, bio_link,
|
|
3668
|
+
'pending', 0, NULL, NULL, NULL, 0, NULL,
|
|
3669
|
+
COALESCE(created_at, ?), ?
|
|
3670
|
+
FROM jobs WHERE ${condition}
|
|
3671
|
+
`,
|
|
3672
|
+
).run(now, now);
|
|
3673
|
+
|
|
3674
|
+
const deleted = db.prepare(`DELETE FROM jobs WHERE ${condition}`).run();
|
|
3675
|
+
|
|
3676
|
+
markStatsDirty();
|
|
3677
|
+
return {
|
|
3678
|
+
ok: true,
|
|
3679
|
+
moved: deleted.changes || 0,
|
|
3680
|
+
total: countRow.c,
|
|
3681
|
+
};
|
|
3682
|
+
}
|
|
3683
|
+
|
|
3684
|
+
// 将单个 job 移动到 jobs_base(用于目标商家页面"移到base重走流程")
|
|
3685
|
+
function moveSingleJobToBase(uniqueId) {
|
|
3686
|
+
const db = getDb();
|
|
3687
|
+
if (!db) return { ok: false, error: "db not ready" };
|
|
3688
|
+
|
|
3689
|
+
const safeId = String(uniqueId).trim();
|
|
3690
|
+
if (!safeId) return { ok: false, error: "invalid uniqueId" };
|
|
3691
|
+
|
|
3692
|
+
const existing = db
|
|
3693
|
+
.prepare("SELECT * FROM jobs WHERE unique_id = ?")
|
|
3694
|
+
.get(safeId);
|
|
3695
|
+
if (!existing) return { ok: false, error: "user not found in jobs" };
|
|
3696
|
+
|
|
3697
|
+
const COLUMNS = [
|
|
3698
|
+
"unique_id",
|
|
3699
|
+
"nickname",
|
|
3700
|
+
"status",
|
|
3701
|
+
"sources",
|
|
3702
|
+
"claimed_by",
|
|
3703
|
+
"claimed_at",
|
|
3704
|
+
"error",
|
|
3705
|
+
"pinned",
|
|
3706
|
+
"no_video",
|
|
3707
|
+
"restricted",
|
|
3708
|
+
"user_update_count",
|
|
3709
|
+
"tt_seller",
|
|
3710
|
+
"verified",
|
|
3711
|
+
"video_count",
|
|
3712
|
+
"comment_count",
|
|
3713
|
+
"guessed_location",
|
|
3714
|
+
"location_created",
|
|
3715
|
+
"confirmed_location",
|
|
3716
|
+
"modified_at",
|
|
3717
|
+
"follower_count",
|
|
3718
|
+
"following_count",
|
|
3719
|
+
"heart_count",
|
|
3720
|
+
"refresh_time",
|
|
3721
|
+
"processed",
|
|
3722
|
+
"processed_at",
|
|
3723
|
+
"created_at",
|
|
3724
|
+
"updated_at",
|
|
3725
|
+
"region",
|
|
3726
|
+
"signature",
|
|
3727
|
+
"bio_link",
|
|
3728
|
+
"sec_uid",
|
|
3729
|
+
"status_code",
|
|
3730
|
+
"latest_video_time",
|
|
3731
|
+
];
|
|
3732
|
+
const cols = COLUMNS.join(",");
|
|
3733
|
+
|
|
3734
|
+
try {
|
|
3735
|
+
db.prepare(
|
|
3736
|
+
`INSERT OR REPLACE INTO jobs_base (${cols})
|
|
3737
|
+
SELECT ${cols} FROM jobs WHERE unique_id = ?`,
|
|
3738
|
+
).run(safeId);
|
|
3739
|
+
|
|
3740
|
+
// 重置 user_update_count,保留 tt_seller,状态改为 pending
|
|
3741
|
+
db.prepare(
|
|
3742
|
+
`UPDATE jobs_base SET user_update_count = 0, status = 'pending',
|
|
3743
|
+
claimed_by = NULL, claimed_at = NULL, processed = 0, processed_at = NULL,
|
|
3744
|
+
error = NULL, restricted = 0
|
|
3745
|
+
WHERE unique_id = ?`,
|
|
3746
|
+
).run(safeId);
|
|
3747
|
+
|
|
3748
|
+
db.prepare("DELETE FROM jobs WHERE unique_id = ?").run(safeId);
|
|
3749
|
+
|
|
3750
|
+
markStatsDirty();
|
|
3751
|
+
console.error(`[DB] moveSingleJobToBase: ${safeId} → jobs_base`);
|
|
3752
|
+
return { ok: true };
|
|
3753
|
+
} catch (e) {
|
|
3754
|
+
return { ok: false, error: e.message };
|
|
3755
|
+
}
|
|
3756
|
+
}
|
|
3559
3757
|
}
|
package/src/watch/db-crud.js
CHANGED
|
@@ -479,8 +479,23 @@ export function getPendingJobsUserUpdateCount() {
|
|
|
479
479
|
.get().c;
|
|
480
480
|
}
|
|
481
481
|
|
|
482
|
+
// raw_jobs 计数缓存:raw_jobs 表数据量大(千万级),COUNT(*) 全表扫描耗时长,
|
|
483
|
+
// 且该值变化频率低(仅在 batchUpdateUserInfo 移入非商家时变化),缓存 60 秒。
|
|
484
|
+
let _rawJobsCountCache = null;
|
|
485
|
+
let _rawJobsCountCacheTime = 0;
|
|
486
|
+
const RAW_JOBS_COUNT_CACHE_TTL = 60000; // 60 秒
|
|
487
|
+
|
|
482
488
|
export function getRawJobsCount() {
|
|
489
|
+
const now = Date.now();
|
|
490
|
+
if (
|
|
491
|
+
_rawJobsCountCache !== null &&
|
|
492
|
+
now - _rawJobsCountCacheTime < RAW_JOBS_COUNT_CACHE_TTL
|
|
493
|
+
) {
|
|
494
|
+
return _rawJobsCountCache;
|
|
495
|
+
}
|
|
483
496
|
const db = getDb();
|
|
484
497
|
if (!db) return 0;
|
|
485
|
-
|
|
498
|
+
_rawJobsCountCache = db.prepare("SELECT COUNT(*) as c FROM raw_jobs").get().c;
|
|
499
|
+
_rawJobsCountCacheTime = now;
|
|
500
|
+
return _rawJobsCountCache;
|
|
486
501
|
}
|
package/src/watch/db-schema.js
CHANGED
|
@@ -367,6 +367,7 @@ export function initDb(filePath) {
|
|
|
367
367
|
round INTEGER NOT NULL DEFAULT 1,
|
|
368
368
|
strategy TEXT,
|
|
369
369
|
tags_generated TEXT NOT NULL DEFAULT '[]',
|
|
370
|
+
tags_raw TEXT,
|
|
370
371
|
tags_added INTEGER DEFAULT 0,
|
|
371
372
|
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
372
373
|
productive_sample TEXT,
|
|
@@ -375,6 +376,10 @@ export function initDb(filePath) {
|
|
|
375
376
|
avg_dead_score REAL DEFAULT 0
|
|
376
377
|
)
|
|
377
378
|
`);
|
|
379
|
+
// 迁移:tags_raw 列(用于记录 LLM 原始生成的 tag,去重前)
|
|
380
|
+
try {
|
|
381
|
+
db.exec("ALTER TABLE discover_log ADD COLUMN tags_raw TEXT");
|
|
382
|
+
} catch {}
|
|
378
383
|
|
|
379
384
|
// ===== reflection_rules 表(反思闭环:LLM 自我学习规则) =====
|
|
380
385
|
db.exec(`
|
package/src/watch/db-stats.js
CHANGED
|
@@ -61,11 +61,7 @@ export function getDashboardStatsFromDb(targetLocations = []) {
|
|
|
61
61
|
COALESCE(location_created, '未知') as country,
|
|
62
62
|
COUNT(*) as count,
|
|
63
63
|
SUM(CASE
|
|
64
|
-
WHEN tt_seller = 1 AND verified = 0
|
|
65
|
-
targetLocations.length
|
|
66
|
-
? `AND location_created IN (${targetPlaceholders})`
|
|
67
|
-
: "AND 1 = 0"
|
|
68
|
-
}
|
|
64
|
+
WHEN tt_seller = 1 AND verified = 0
|
|
69
65
|
THEN 1 ELSE 0 END) as targetCount
|
|
70
66
|
FROM jobs
|
|
71
67
|
WHERE status = 'done'
|
|
@@ -73,7 +69,7 @@ export function getDashboardStatsFromDb(targetLocations = []) {
|
|
|
73
69
|
ORDER BY count DESC
|
|
74
70
|
`,
|
|
75
71
|
)
|
|
76
|
-
.all(
|
|
72
|
+
.all();
|
|
77
73
|
|
|
78
74
|
const targetCountryStats = targetLocations.length
|
|
79
75
|
? db
|