tt-help-cli-ycl 1.4.8 → 1.4.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/cli/comments.js +95 -31
- package/src/cli/tag.js +68 -1
- package/src/lib/constants.js +1 -1
- package/src/watch/data-store.js +90 -25
package/package.json
CHANGED
package/src/cli/comments.js
CHANGED
|
@@ -5,7 +5,6 @@
|
|
|
5
5
|
// 手动模式: tt-help comments <URL> [N] [--save] 单视频评论提取
|
|
6
6
|
// ────────────────────────────────────────
|
|
7
7
|
|
|
8
|
-
import { chromium } from "playwright";
|
|
9
8
|
import os from "os";
|
|
10
9
|
import { execSync } from "child_process";
|
|
11
10
|
|
|
@@ -17,6 +16,8 @@ import {
|
|
|
17
16
|
import {
|
|
18
17
|
server as defaultServer,
|
|
19
18
|
proxy as configuredProxy,
|
|
19
|
+
userId as configuredUserId,
|
|
20
|
+
saveUserId,
|
|
20
21
|
} from "../lib/constants.js";
|
|
21
22
|
import {
|
|
22
23
|
DEFAULT_TARGET_LOCATIONS,
|
|
@@ -31,6 +32,7 @@ import {
|
|
|
31
32
|
closeCaptcha,
|
|
32
33
|
} from "../scraper/modules/captcha-handler.js";
|
|
33
34
|
import { detectModals, closeAllModals } from "../lib/modal-handler.js";
|
|
35
|
+
import { getMacOrUuid } from "../lib/mac-or-uuid.js";
|
|
34
36
|
|
|
35
37
|
// ────────────────────────────────────────
|
|
36
38
|
// Section 1: 常量 & 配置默认值
|
|
@@ -52,9 +54,9 @@ const CFG = {
|
|
|
52
54
|
/** 连续验证码后切换端口 */
|
|
53
55
|
maxConsecutiveCaptcha: 3,
|
|
54
56
|
/** 内存回收 — 触发回收的 RSS(MB) 阈值 */
|
|
55
|
-
recycleRssMb:
|
|
57
|
+
recycleRssMb: 2000,
|
|
56
58
|
/** 内存回收 — 触发回收的 Heap(MB) 阈值 */
|
|
57
|
-
recycleHeapMb:
|
|
59
|
+
recycleHeapMb: 2000,
|
|
58
60
|
/** 内存回收 — 触发回收的 Edge 进程(MB) 阈值 */
|
|
59
61
|
recycleEdgeMb: 2000,
|
|
60
62
|
/** 内存回收 — 两次回收最小间隔(ms) */
|
|
@@ -175,7 +177,43 @@ function getPage(browser) {
|
|
|
175
177
|
}
|
|
176
178
|
|
|
177
179
|
// ────────────────────────────────────────
|
|
178
|
-
// Section 4:
|
|
180
|
+
// Section 4: 页面请求拦截(独立函数,供自动/手动模式共用)
|
|
181
|
+
// ────────────────────────────────────────
|
|
182
|
+
|
|
183
|
+
const _blockedPages = new WeakSet();
|
|
184
|
+
|
|
185
|
+
async function setupPageBlocking(page) {
|
|
186
|
+
if (!page || _blockedPages.has(page)) return;
|
|
187
|
+
await page.route("**/*", (route) => {
|
|
188
|
+
const req = route.request();
|
|
189
|
+
const t = req.resourceType();
|
|
190
|
+
const url = req.url();
|
|
191
|
+
|
|
192
|
+
// 资源类型拦截
|
|
193
|
+
if (t === "image" || t === "stylesheet" || t === "media") {
|
|
194
|
+
return route.abort();
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
// 视频流拦截(TikTok 视频通过 fetch/xhr 加载 mp4 分片,
|
|
198
|
+
// resourceType 为 fetch/xhr,需按 URL 模式匹配)
|
|
199
|
+
// 真实 URL:
|
|
200
|
+
// v16-webapp-prime.us.tiktok.com/video/tos/...?mime_type=video_mp4
|
|
201
|
+
// v16m-webapp.tiktokcdn-us.com/.../video/tos/...?mime_type=video_mp4
|
|
202
|
+
if (
|
|
203
|
+
/webapp.*(tiktok\.com|tiktokcdn).*\/video\/tos\//i.test(url) ||
|
|
204
|
+
/mime_type=video_mp4/i.test(url) ||
|
|
205
|
+
/\.(mp4|m3u8|ts)(\?|$)/i.test(url)
|
|
206
|
+
) {
|
|
207
|
+
return route.abort();
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
route.continue();
|
|
211
|
+
});
|
|
212
|
+
_blockedPages.add(page);
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
// ────────────────────────────────────────
|
|
216
|
+
// Section 5: CDP 会话管理(端口池/浏览器/页面/内存回收)
|
|
179
217
|
// ────────────────────────────────────────
|
|
180
218
|
|
|
181
219
|
class CdpSessionManager {
|
|
@@ -187,7 +225,8 @@ class CdpSessionManager {
|
|
|
187
225
|
this.browser = null;
|
|
188
226
|
this.page = null;
|
|
189
227
|
this.lastRecycleAt = 0;
|
|
190
|
-
this.
|
|
228
|
+
this._lastMemCheck = 0;
|
|
229
|
+
this._shuttingDown = false;
|
|
191
230
|
}
|
|
192
231
|
|
|
193
232
|
pickNextPort() {
|
|
@@ -211,7 +250,7 @@ class CdpSessionManager {
|
|
|
211
250
|
/** 获取或恢复 page */
|
|
212
251
|
async _ensurePage() {
|
|
213
252
|
let p = getPage(this.browser);
|
|
214
|
-
if (p) await
|
|
253
|
+
if (p) await setupPageBlocking(p);
|
|
215
254
|
return p;
|
|
216
255
|
}
|
|
217
256
|
|
|
@@ -222,17 +261,6 @@ class CdpSessionManager {
|
|
|
222
261
|
return this.page;
|
|
223
262
|
}
|
|
224
263
|
|
|
225
|
-
/** 页面请求拦截:屏蔽图片/样式减少内存 */
|
|
226
|
-
async _setupBlocking(page) {
|
|
227
|
-
if (!page || this._blockedPages.has(page)) return;
|
|
228
|
-
await page.route("**/*", (route) => {
|
|
229
|
-
const t = route.request().resourceType();
|
|
230
|
-
if (t === "image" || t === "stylesheet") route.abort();
|
|
231
|
-
else route.continue();
|
|
232
|
-
});
|
|
233
|
-
this._blockedPages.add(page);
|
|
234
|
-
}
|
|
235
|
-
|
|
236
264
|
/** 轻量回收:清空当前 tab */
|
|
237
265
|
async recyclePage() {
|
|
238
266
|
if (!this.page || this.page.isClosed()) {
|
|
@@ -278,8 +306,12 @@ class CdpSessionManager {
|
|
|
278
306
|
this.lastRecycleAt = Date.now();
|
|
279
307
|
}
|
|
280
308
|
|
|
281
|
-
/** 内存检查 +
|
|
309
|
+
/** 内存检查 + 按需回收(每 30s 最多采样一次) */
|
|
282
310
|
async maybeRecycleForMemory(processedCount) {
|
|
311
|
+
const now = Date.now();
|
|
312
|
+
if (now - this._lastMemCheck < 30000) return;
|
|
313
|
+
this._lastMemCheck = now;
|
|
314
|
+
|
|
283
315
|
const mem = process.memoryUsage();
|
|
284
316
|
const rssMB = mem.rss / 1024 / 1024;
|
|
285
317
|
const heapMB = mem.heapUsed / 1024 / 1024;
|
|
@@ -334,6 +366,8 @@ class CdpSessionManager {
|
|
|
334
366
|
}
|
|
335
367
|
|
|
336
368
|
async shutdown() {
|
|
369
|
+
if (this._shuttingDown) return;
|
|
370
|
+
this._shuttingDown = true;
|
|
337
371
|
if (this.page) await this.page.close().catch(() => {});
|
|
338
372
|
await this.browser?.close().catch(() => {});
|
|
339
373
|
killEdgeProcesses(null, this.port);
|
|
@@ -341,7 +375,7 @@ class CdpSessionManager {
|
|
|
341
375
|
}
|
|
342
376
|
|
|
343
377
|
// ────────────────────────────────────────
|
|
344
|
-
// Section
|
|
378
|
+
// Section 6: 共享操作(视频页准备 & 评论提取 & 保存)
|
|
345
379
|
// ────────────────────────────────────────
|
|
346
380
|
|
|
347
381
|
async function prepareVideoPage(page, href) {
|
|
@@ -390,13 +424,12 @@ function extractUniqueAuthors(comments) {
|
|
|
390
424
|
];
|
|
391
425
|
}
|
|
392
426
|
|
|
393
|
-
async function saveAuthorsToServer(
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
});
|
|
427
|
+
async function saveAuthorsToServer(
|
|
428
|
+
apiPost,
|
|
429
|
+
serverUrl,
|
|
430
|
+
authors,
|
|
431
|
+
guessedLocation,
|
|
432
|
+
) {
|
|
400
433
|
return apiPost(`${serverUrl}/api/users`, {
|
|
401
434
|
usernames: authors.map((a) => "@" + a),
|
|
402
435
|
sources: ["comment"],
|
|
@@ -405,7 +438,7 @@ async function saveAuthorsToServer(serverUrl, authors, guessedLocation) {
|
|
|
405
438
|
}
|
|
406
439
|
|
|
407
440
|
// ────────────────────────────────────────
|
|
408
|
-
// Section
|
|
441
|
+
// Section 7: 自动模式 — 循环从服务端取任务、抓评论、提交
|
|
409
442
|
// ────────────────────────────────────────
|
|
410
443
|
|
|
411
444
|
async function runAutoMode(options) {
|
|
@@ -426,7 +459,14 @@ async function runAutoMode(options) {
|
|
|
426
459
|
|
|
427
460
|
loadEnvConfig();
|
|
428
461
|
|
|
429
|
-
|
|
462
|
+
// ── 设备标识 ──
|
|
463
|
+
let userId = configuredUserId;
|
|
464
|
+
if (!userId) {
|
|
465
|
+
userId = await getMacOrUuid();
|
|
466
|
+
saveUserId(userId);
|
|
467
|
+
}
|
|
468
|
+
|
|
469
|
+
const { apiGet, apiPost, apiPut } = createApiClient({
|
|
430
470
|
checkStatus: true,
|
|
431
471
|
maxRetries: 2,
|
|
432
472
|
backoff: 2000,
|
|
@@ -617,6 +657,16 @@ async function runAutoMode(options) {
|
|
|
617
657
|
});
|
|
618
658
|
|
|
619
659
|
if (captchaResult.detected) {
|
|
660
|
+
// 上报验证码错误
|
|
661
|
+
apiPost(`${serverUrl}/api/error-report`, {
|
|
662
|
+
userId,
|
|
663
|
+
username: "",
|
|
664
|
+
errorType: "captcha",
|
|
665
|
+
errorMessage: resultNote || "评论采集时出现验证码",
|
|
666
|
+
stage: "comments",
|
|
667
|
+
errorStack: "",
|
|
668
|
+
}).catch(() => {});
|
|
669
|
+
|
|
620
670
|
if (captchaResult.resolved) {
|
|
621
671
|
resultNote = "验证码已解决";
|
|
622
672
|
consecutiveCaptchaCount = 0;
|
|
@@ -648,6 +698,7 @@ async function runAutoMode(options) {
|
|
|
648
698
|
|
|
649
699
|
if (authors.length > 0) {
|
|
650
700
|
const saveResult = await saveAuthorsToServer(
|
|
701
|
+
apiPost,
|
|
651
702
|
serverUrl,
|
|
652
703
|
authors,
|
|
653
704
|
loc,
|
|
@@ -734,7 +785,7 @@ async function runAutoMode(options) {
|
|
|
734
785
|
}
|
|
735
786
|
|
|
736
787
|
// ────────────────────────────────────────
|
|
737
|
-
// Section
|
|
788
|
+
// Section 8: 手动模式 — 单视频评论提取 + 输出/保存
|
|
738
789
|
// ────────────────────────────────────────
|
|
739
790
|
|
|
740
791
|
async function runManualMode(url, options) {
|
|
@@ -742,17 +793,22 @@ async function runManualMode(url, options) {
|
|
|
742
793
|
maxComments = CFG.maxComments,
|
|
743
794
|
save: shouldSave,
|
|
744
795
|
serverUrl = defaultServer,
|
|
796
|
+
basePort = 9222,
|
|
745
797
|
} = options;
|
|
746
798
|
|
|
747
799
|
let browser;
|
|
748
800
|
try {
|
|
749
|
-
|
|
801
|
+
const { chromium } = await import("playwright");
|
|
802
|
+
browser = await chromium.connectOverCDP(`http://127.0.0.1:${basePort}`);
|
|
750
803
|
const page = getPage(browser);
|
|
751
804
|
if (!page) {
|
|
752
805
|
console.error("未找到可用页面");
|
|
753
806
|
process.exit(1);
|
|
754
807
|
}
|
|
755
808
|
|
|
809
|
+
// 拦截图片/视频请求
|
|
810
|
+
await setupPageBlocking(page);
|
|
811
|
+
|
|
756
812
|
console.error(`正在打开: ${url}`);
|
|
757
813
|
await prepareVideoPage(page, url);
|
|
758
814
|
|
|
@@ -788,7 +844,14 @@ async function runManualMode(url, options) {
|
|
|
788
844
|
if (guessedLocation) console.error(`猜测国家: ${guessedLocation}`);
|
|
789
845
|
|
|
790
846
|
try {
|
|
847
|
+
const { apiPost } = createApiClient({
|
|
848
|
+
checkStatus: true,
|
|
849
|
+
maxRetries: 2,
|
|
850
|
+
backoff: 2000,
|
|
851
|
+
log: true,
|
|
852
|
+
});
|
|
791
853
|
const saveResult = await saveAuthorsToServer(
|
|
854
|
+
apiPost,
|
|
792
855
|
serverUrl,
|
|
793
856
|
authors,
|
|
794
857
|
guessedLocation,
|
|
@@ -834,7 +897,7 @@ async function runManualMode(url, options) {
|
|
|
834
897
|
}
|
|
835
898
|
|
|
836
899
|
// ────────────────────────────────────────
|
|
837
|
-
// Section
|
|
900
|
+
// Section 9: 入口 — handleComments
|
|
838
901
|
// ────────────────────────────────────────
|
|
839
902
|
|
|
840
903
|
export async function handleComments(options) {
|
|
@@ -866,5 +929,6 @@ export async function handleComments(options) {
|
|
|
866
929
|
maxComments: commentsMax,
|
|
867
930
|
save: commentsSave,
|
|
868
931
|
serverUrl: commentsServer || defaultServer,
|
|
932
|
+
basePort: commentsBasePort,
|
|
869
933
|
});
|
|
870
934
|
}
|
package/src/cli/tag.js
CHANGED
|
@@ -17,7 +17,14 @@ import { discoverTags } from "../lib/tag-discover.js";
|
|
|
17
17
|
import {
|
|
18
18
|
server as cfgServer,
|
|
19
19
|
proxy as configuredProxy,
|
|
20
|
+
userId as configuredUserId,
|
|
21
|
+
saveUserId,
|
|
20
22
|
} from "../lib/constants.js";
|
|
23
|
+
import { getMacOrUuid } from "../lib/mac-or-uuid.js";
|
|
24
|
+
import {
|
|
25
|
+
waitAndGetCaptcha,
|
|
26
|
+
closeCaptcha,
|
|
27
|
+
} from "../scraper/modules/captcha-handler.js";
|
|
21
28
|
|
|
22
29
|
const ALL_COUNTRIES = DEFAULT_TARGET_LOCATIONS;
|
|
23
30
|
const DEFAULT_SERVER = cfgServer || "http://127.0.0.1:3000";
|
|
@@ -688,6 +695,13 @@ export async function handleScoreAll(parsed) {
|
|
|
688
695
|
log(`📊 启动内存: ${formatMemoryWithBrowser(cdpPort)}`);
|
|
689
696
|
log("");
|
|
690
697
|
|
|
698
|
+
// ── 设备标识 ──
|
|
699
|
+
let userId = configuredUserId;
|
|
700
|
+
if (!userId) {
|
|
701
|
+
userId = await getMacOrUuid();
|
|
702
|
+
saveUserId(userId);
|
|
703
|
+
}
|
|
704
|
+
|
|
691
705
|
// 连接 CDP 浏览器
|
|
692
706
|
const cdpOpts = { port: cdpPort };
|
|
693
707
|
if (effectiveProxy) cdpOpts.proxyServer = effectiveProxy;
|
|
@@ -731,7 +745,7 @@ export async function handleScoreAll(parsed) {
|
|
|
731
745
|
const RECYCLE_CONFIG = {
|
|
732
746
|
nodeRssMb: Math.max(
|
|
733
747
|
256,
|
|
734
|
-
parseInt(process.env.TAG_SCOREALL_RECYCLE_RSS_MB || "
|
|
748
|
+
parseInt(process.env.TAG_SCOREALL_RECYCLE_RSS_MB || "2000", 10) || 2000,
|
|
735
749
|
),
|
|
736
750
|
nodeHeapMb: Math.max(
|
|
737
751
|
128,
|
|
@@ -850,6 +864,8 @@ export async function handleScoreAll(parsed) {
|
|
|
850
864
|
|
|
851
865
|
let totalScored = 0;
|
|
852
866
|
let emptyRounds = 0; // 连续无任务的轮数
|
|
867
|
+
let consecutiveCaptcha = 0;
|
|
868
|
+
const MAX_CONSECUTIVE_CAPTCHA = 3;
|
|
853
869
|
let consecutive403 = 0; // 连续 HTTP 403 次数,达 3 次认为被封
|
|
854
870
|
const CONSECUTIVE_403_BLOCKED = 3; // 连续 3 次 403 认为账号被封
|
|
855
871
|
|
|
@@ -1098,6 +1114,57 @@ export async function handleScoreAll(parsed) {
|
|
|
1098
1114
|
}
|
|
1099
1115
|
|
|
1100
1116
|
log(` ❌ 失败: ${e.message}`);
|
|
1117
|
+
|
|
1118
|
+
// ── 验证码检测 ──
|
|
1119
|
+
const isCaptcha =
|
|
1120
|
+
e.message &&
|
|
1121
|
+
(e.message.includes("验证码") || e.message.includes("captcha"));
|
|
1122
|
+
|
|
1123
|
+
if (isCaptcha) {
|
|
1124
|
+
consecutiveCaptcha++;
|
|
1125
|
+
log(
|
|
1126
|
+
` 🔐 验证码 (${consecutiveCaptcha}/${MAX_CONSECUTIVE_CAPTCHA}),等待用户手动解决...`,
|
|
1127
|
+
);
|
|
1128
|
+
|
|
1129
|
+
// 上报验证码错误
|
|
1130
|
+
try {
|
|
1131
|
+
await fetch(`${baseUrl}/api/error-report`, {
|
|
1132
|
+
method: "POST",
|
|
1133
|
+
headers: buildClientHeaders(clientId, clientMeta, {
|
|
1134
|
+
"Content-Type": "application/json",
|
|
1135
|
+
}),
|
|
1136
|
+
body: JSON.stringify({
|
|
1137
|
+
userId,
|
|
1138
|
+
username: "",
|
|
1139
|
+
errorType: "captcha",
|
|
1140
|
+
errorMessage: e.message,
|
|
1141
|
+
stage: "tag-scoring",
|
|
1142
|
+
errorStack: "",
|
|
1143
|
+
}),
|
|
1144
|
+
});
|
|
1145
|
+
} catch {}
|
|
1146
|
+
|
|
1147
|
+
// 等待用户手动解决验证码(最多 3 分钟)
|
|
1148
|
+
try {
|
|
1149
|
+
const captchaResult = await waitAndGetCaptcha(page, {
|
|
1150
|
+
waitMs: 180000,
|
|
1151
|
+
pollInterval: 5000,
|
|
1152
|
+
log: () => {},
|
|
1153
|
+
});
|
|
1154
|
+
if (captchaResult.detected && !captchaResult.resolved) {
|
|
1155
|
+
await closeCaptcha(page).catch(() => {});
|
|
1156
|
+
}
|
|
1157
|
+
} catch {}
|
|
1158
|
+
|
|
1159
|
+
if (consecutiveCaptcha >= MAX_CONSECUTIVE_CAPTCHA) {
|
|
1160
|
+
log(` ♻️ 连续${MAX_CONSECUTIVE_CAPTCHA}次验证码 → 切换端口`);
|
|
1161
|
+
await recycleCdpSession("连续验证码", { switchPort: true });
|
|
1162
|
+
consecutiveCaptcha = 0;
|
|
1163
|
+
}
|
|
1164
|
+
} else {
|
|
1165
|
+
consecutiveCaptcha = 0;
|
|
1166
|
+
}
|
|
1167
|
+
|
|
1101
1168
|
try {
|
|
1102
1169
|
// 使用当前正在处理的 tag(tag 变量在 try 块开头已赋值,catch 中仍可访问)
|
|
1103
1170
|
const failedTag = typeof tag === "string" && tag ? tag : "unknown";
|
package/src/lib/constants.js
CHANGED
|
@@ -267,7 +267,7 @@ const HELP_TEXT = [
|
|
|
267
267
|
" 连续 3 次 HTTP 403 → 判断账号被封 → 等 2 分钟后自动切换到池中另一端口",
|
|
268
268
|
" 环境变量:",
|
|
269
269
|
" TAG_SCOREALL_PORT_POOL_SIZE=N 端口池大小(默认 10)",
|
|
270
|
-
" TAG_SCOREALL_RECYCLE_RSS_MB=N 内存回收阈值 RSS MB(默认
|
|
270
|
+
" TAG_SCOREALL_RECYCLE_RSS_MB=N 内存回收阈值 RSS MB(默认 2000)",
|
|
271
271
|
" TAG_SCOREALL_RECYCLE_HEAP_MB=N 内存回收阈值 Heap MB(默认 2000)",
|
|
272
272
|
" 示例: tt-help tag score-all",
|
|
273
273
|
" tt-help tag score-all --countries ES -s http://127.0.0.1:3001",
|
package/src/watch/data-store.js
CHANGED
|
@@ -500,6 +500,8 @@ export function createStore(filePath, options = {}) {
|
|
|
500
500
|
llmScore: false,
|
|
501
501
|
llmMinScore: 60,
|
|
502
502
|
llmSampleSize: 100,
|
|
503
|
+
llmTagRatio: 0.4,
|
|
504
|
+
llmCommentRatio: 0.3,
|
|
503
505
|
...options.refillLlm,
|
|
504
506
|
};
|
|
505
507
|
|
|
@@ -957,27 +959,30 @@ export function createStore(filePath, options = {}) {
|
|
|
957
959
|
return { moved: 0 };
|
|
958
960
|
}
|
|
959
961
|
|
|
960
|
-
// 如果启用 LLM
|
|
962
|
+
// 如果启用 LLM 打分:tag 和 comment 免打分直接入围,其余走 LLM 打分
|
|
961
963
|
if (useLlm && normalizedLocations && normalizedLocations.length > 0) {
|
|
962
964
|
const llmTotal = options.llmTotal ?? 200; // 总条数
|
|
963
|
-
const llmTagRatio = options.llmTagRatio ?? 0.
|
|
964
|
-
const
|
|
965
|
-
const
|
|
965
|
+
const llmTagRatio = options.llmTagRatio ?? 0.4; // tag 占比 40%
|
|
966
|
+
const llmCommentRatio = options.llmCommentRatio ?? 0.3; // comment 占比 30%
|
|
967
|
+
const llmTagLimit = Math.floor(llmTotal * llmTagRatio); // tag 上限
|
|
968
|
+
const llmCommentLimit = Math.floor(llmTotal * llmCommentRatio); // comment 上限
|
|
969
|
+
const llmLLMTarget = llmTotal - llmTagLimit - llmCommentLimit; // LLM 打分目标
|
|
966
970
|
const llmMinScore = options.llmMinScore ?? 60;
|
|
967
971
|
const llmSampleSize = options.llmSampleSize ?? 100;
|
|
968
972
|
const maxBatches = options.llmMaxBatches ?? 10;
|
|
969
973
|
|
|
970
974
|
console.error(
|
|
971
|
-
`[data-store] LLM
|
|
975
|
+
`[data-store] LLM 调度开始: 总目标 ${llmTotal} 条,tag ${llmTagLimit} 条,comment ${llmCommentLimit} 条(免打分),LLM 打分 ${llmLLMTarget} 条`,
|
|
972
976
|
);
|
|
973
977
|
|
|
974
978
|
// 返回 Promise,调用方需要 await
|
|
975
979
|
return (async () => {
|
|
976
980
|
const allTagQualified = [];
|
|
977
|
-
const
|
|
981
|
+
const allCommentQualified = [];
|
|
982
|
+
const allLLMQualified = [];
|
|
978
983
|
const allScores = [];
|
|
979
984
|
|
|
980
|
-
// =====
|
|
985
|
+
// ===== 第一步:一次性取 tag(全局,最多 llmTagLimit 条,免打分)=====
|
|
981
986
|
let tagOffset = offsetStore.get("_tag") || 0;
|
|
982
987
|
const tagGlobalCount =
|
|
983
988
|
getDb()
|
|
@@ -1025,43 +1030,97 @@ export function createStore(filePath, options = {}) {
|
|
|
1025
1030
|
|
|
1026
1031
|
offsetStore.set("_tag", tagOffset);
|
|
1027
1032
|
|
|
1028
|
-
// =====
|
|
1033
|
+
// ===== 第二步:一次性取 comment(全局,最多 llmCommentLimit 条,免打分)=====
|
|
1034
|
+
let commentOffset = offsetStore.get("_comment") || 0;
|
|
1035
|
+
const commentGlobalCount =
|
|
1036
|
+
getDb()
|
|
1037
|
+
.prepare(
|
|
1038
|
+
`SELECT COUNT(*) as c FROM raw_jobs WHERE ${whereSql} AND sources LIKE '%comment%' AND sources NOT LIKE '%tag%'`,
|
|
1039
|
+
)
|
|
1040
|
+
.get(...args)?.c || 0;
|
|
1041
|
+
|
|
1042
|
+
if (commentOffset >= commentGlobalCount) {
|
|
1043
|
+
commentOffset = 0;
|
|
1044
|
+
offsetStore.set("_comment", 0);
|
|
1045
|
+
}
|
|
1046
|
+
|
|
1047
|
+
console.error(
|
|
1048
|
+
`[data-store] Comment 全局共 ${commentGlobalCount} 条,从偏移量 ${commentOffset} 开始`,
|
|
1049
|
+
);
|
|
1050
|
+
|
|
1051
|
+
while (
|
|
1052
|
+
allCommentQualified.length < llmCommentLimit &&
|
|
1053
|
+
commentOffset < commentGlobalCount
|
|
1054
|
+
) {
|
|
1055
|
+
const batch = getDb()
|
|
1056
|
+
.prepare(
|
|
1057
|
+
`
|
|
1058
|
+
SELECT * FROM raw_jobs WHERE ${whereSql} AND sources LIKE '%comment%' AND sources NOT LIKE '%tag%'
|
|
1059
|
+
ORDER BY COALESCE(video_count, 0) DESC, created_at DESC
|
|
1060
|
+
LIMIT ? OFFSET ?
|
|
1061
|
+
`,
|
|
1062
|
+
)
|
|
1063
|
+
.all(
|
|
1064
|
+
...args,
|
|
1065
|
+
Math.min(
|
|
1066
|
+
llmSampleSize,
|
|
1067
|
+
llmCommentLimit - allCommentQualified.length,
|
|
1068
|
+
),
|
|
1069
|
+
commentOffset,
|
|
1070
|
+
);
|
|
1071
|
+
|
|
1072
|
+
if (!batch.length) break;
|
|
1073
|
+
|
|
1074
|
+
allCommentQualified.push(...batch.map((s) => s.unique_id));
|
|
1075
|
+
commentOffset += batch.length;
|
|
1076
|
+
|
|
1077
|
+
console.error(
|
|
1078
|
+
`[data-store] Comment 本批 ${batch.length} 条,累计 ${allCommentQualified.length}/${llmCommentLimit}`,
|
|
1079
|
+
);
|
|
1080
|
+
}
|
|
1081
|
+
|
|
1082
|
+
offsetStore.set("_comment", commentOffset);
|
|
1083
|
+
|
|
1084
|
+
// ===== 第三步:按国家取非 tag 非 comment,走 LLM 打分,直到合格数达到 llmLLMTarget =====
|
|
1029
1085
|
for (const location of normalizedLocations) {
|
|
1030
|
-
if (
|
|
1086
|
+
if (allLLMQualified.length >= llmLLMTarget) break;
|
|
1031
1087
|
|
|
1032
|
-
const
|
|
1033
|
-
let offset = offsetStore.get(
|
|
1088
|
+
const llmOffsetKey = `${location}:llm`;
|
|
1089
|
+
let offset = offsetStore.get(llmOffsetKey) || 0;
|
|
1034
1090
|
|
|
1035
1091
|
const locationArgs = [...args, location];
|
|
1036
|
-
const
|
|
1092
|
+
const llmCount =
|
|
1037
1093
|
getDb()
|
|
1038
1094
|
.prepare(
|
|
1039
|
-
`SELECT COUNT(*) as c FROM raw_jobs WHERE ${whereSql} AND guessed_location = ? AND (sources NOT LIKE '%tag%' OR sources IS NULL)`,
|
|
1095
|
+
`SELECT COUNT(*) as c FROM raw_jobs WHERE ${whereSql} AND guessed_location = ? AND (sources NOT LIKE '%tag%' OR sources IS NULL) AND (sources NOT LIKE '%comment%' OR sources IS NULL)`,
|
|
1040
1096
|
)
|
|
1041
1097
|
.get(...locationArgs)?.c || 0;
|
|
1042
1098
|
|
|
1043
|
-
if (
|
|
1044
|
-
console.error(
|
|
1099
|
+
if (llmCount === 0) {
|
|
1100
|
+
console.error(
|
|
1101
|
+
`[data-store] 国家 ${location}: 无 LLM 打分候选数据,跳过`,
|
|
1102
|
+
);
|
|
1045
1103
|
continue;
|
|
1046
1104
|
}
|
|
1047
1105
|
|
|
1048
|
-
if (offset >=
|
|
1106
|
+
if (offset >= llmCount) {
|
|
1049
1107
|
offset = 0;
|
|
1050
|
-
offsetStore.set(
|
|
1108
|
+
offsetStore.set(llmOffsetKey, 0);
|
|
1051
1109
|
}
|
|
1052
1110
|
|
|
1053
1111
|
console.error(
|
|
1054
|
-
`[data-store] 国家 ${location}:
|
|
1112
|
+
`[data-store] 国家 ${location}: LLM 候选共 ${llmCount} 条,从偏移量 ${offset} 开始`,
|
|
1055
1113
|
);
|
|
1056
1114
|
|
|
1057
1115
|
for (let batch = 0; batch < maxBatches; batch++) {
|
|
1058
|
-
if (
|
|
1116
|
+
if (allLLMQualified.length >= llmLLMTarget) break;
|
|
1059
1117
|
|
|
1060
1118
|
const samples = getDb()
|
|
1061
1119
|
.prepare(
|
|
1062
1120
|
`
|
|
1063
1121
|
SELECT * FROM raw_jobs WHERE ${whereSql} AND guessed_location = ?
|
|
1064
1122
|
AND (sources NOT LIKE '%tag%' OR sources IS NULL)
|
|
1123
|
+
AND (sources NOT LIKE '%comment%' OR sources IS NULL)
|
|
1065
1124
|
ORDER BY COALESCE(video_count, 0) DESC, created_at DESC
|
|
1066
1125
|
LIMIT ? OFFSET ?
|
|
1067
1126
|
`,
|
|
@@ -1075,24 +1134,28 @@ export function createStore(filePath, options = {}) {
|
|
|
1075
1134
|
DEFAULT_TARGET_LOCATIONS,
|
|
1076
1135
|
);
|
|
1077
1136
|
const qualified = scores.filter((s) => s.score >= llmMinScore);
|
|
1078
|
-
|
|
1137
|
+
allLLMQualified.push(...qualified.map((s) => s.uniqueId));
|
|
1079
1138
|
allScores.push(...scores);
|
|
1080
1139
|
|
|
1081
1140
|
offset += samples.length;
|
|
1082
|
-
offsetStore.set(
|
|
1141
|
+
offsetStore.set(llmOffsetKey, offset);
|
|
1083
1142
|
|
|
1084
1143
|
console.error(
|
|
1085
|
-
`[data-store] ${location} 第 ${batch + 1} 批: 采样 ${samples.length} 条,本批合格 ${qualified.length}
|
|
1144
|
+
`[data-store] ${location} 第 ${batch + 1} 批: 采样 ${samples.length} 条,本批合格 ${qualified.length} 条,LLM 累计 ${allLLMQualified.length}/${llmLLMTarget}`,
|
|
1086
1145
|
);
|
|
1087
1146
|
}
|
|
1088
1147
|
}
|
|
1089
1148
|
|
|
1090
1149
|
// ===== 最终结果 =====
|
|
1091
|
-
const qualified = [
|
|
1150
|
+
const qualified = [
|
|
1151
|
+
...allTagQualified,
|
|
1152
|
+
...allCommentQualified,
|
|
1153
|
+
...allLLMQualified,
|
|
1154
|
+
];
|
|
1092
1155
|
|
|
1093
1156
|
if (!qualified.length) {
|
|
1094
1157
|
console.error(
|
|
1095
|
-
`[data-store] LLM
|
|
1158
|
+
`[data-store] LLM 调度后无符合条件的任务(tag: ${allTagQualified.length}, comment: ${allCommentQualified.length}, LLM: ${allLLMQualified.length})`,
|
|
1096
1159
|
);
|
|
1097
1160
|
return {
|
|
1098
1161
|
moved: 0,
|
|
@@ -1103,7 +1166,7 @@ export function createStore(filePath, options = {}) {
|
|
|
1103
1166
|
}
|
|
1104
1167
|
|
|
1105
1168
|
console.error(
|
|
1106
|
-
`[data-store] LLM
|
|
1169
|
+
`[data-store] LLM 调度完成: tag ${allTagQualified.length} + comment ${allCommentQualified.length} + LLM ${allLLMQualified.length} = 共 ${qualified.length} 条`,
|
|
1107
1170
|
);
|
|
1108
1171
|
|
|
1109
1172
|
// 移动符合条件的记录
|
|
@@ -1181,6 +1244,7 @@ export function createStore(filePath, options = {}) {
|
|
|
1181
1244
|
WHERE __WHERE__
|
|
1182
1245
|
ORDER BY
|
|
1183
1246
|
CASE WHEN sources LIKE '%tag%' THEN 0 ELSE 1 END,
|
|
1247
|
+
CASE WHEN sources LIKE '%comment%' THEN 0 ELSE 1 END,
|
|
1184
1248
|
COALESCE(video_count, 0) DESC, created_at DESC
|
|
1185
1249
|
LIMIT ?
|
|
1186
1250
|
`;
|
|
@@ -1192,6 +1256,7 @@ export function createStore(filePath, options = {}) {
|
|
|
1192
1256
|
WHERE __WHERE__
|
|
1193
1257
|
ORDER BY
|
|
1194
1258
|
CASE WHEN sources LIKE '%tag%' THEN 0 ELSE 1 END,
|
|
1259
|
+
CASE WHEN sources LIKE '%comment%' THEN 0 ELSE 1 END,
|
|
1195
1260
|
COALESCE(video_count, 0) DESC, created_at DESC
|
|
1196
1261
|
LIMIT ?
|
|
1197
1262
|
)
|