tt-help-cli-ycl 1.4.8 → 1.4.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "tt-help-cli-ycl",
3
- "version": "1.4.8",
3
+ "version": "1.4.10",
4
4
  "description": "TikTok user & video data scraper - extract ttSeller, verified, locationCreated from HTML source",
5
5
  "type": "module",
6
6
  "bin": {
@@ -5,7 +5,6 @@
5
5
  // 手动模式: tt-help comments <URL> [N] [--save] 单视频评论提取
6
6
  // ────────────────────────────────────────
7
7
 
8
- import { chromium } from "playwright";
9
8
  import os from "os";
10
9
  import { execSync } from "child_process";
11
10
 
@@ -17,6 +16,8 @@ import {
17
16
  import {
18
17
  server as defaultServer,
19
18
  proxy as configuredProxy,
19
+ userId as configuredUserId,
20
+ saveUserId,
20
21
  } from "../lib/constants.js";
21
22
  import {
22
23
  DEFAULT_TARGET_LOCATIONS,
@@ -31,6 +32,7 @@ import {
31
32
  closeCaptcha,
32
33
  } from "../scraper/modules/captcha-handler.js";
33
34
  import { detectModals, closeAllModals } from "../lib/modal-handler.js";
35
+ import { getMacOrUuid } from "../lib/mac-or-uuid.js";
34
36
 
35
37
  // ────────────────────────────────────────
36
38
  // Section 1: 常量 & 配置默认值
@@ -52,9 +54,9 @@ const CFG = {
52
54
  /** 连续验证码后切换端口 */
53
55
  maxConsecutiveCaptcha: 3,
54
56
  /** 内存回收 — 触发回收的 RSS(MB) 阈值 */
55
- recycleRssMb: 900,
57
+ recycleRssMb: 2000,
56
58
  /** 内存回收 — 触发回收的 Heap(MB) 阈值 */
57
- recycleHeapMb: 512,
59
+ recycleHeapMb: 2000,
58
60
  /** 内存回收 — 触发回收的 Edge 进程(MB) 阈值 */
59
61
  recycleEdgeMb: 2000,
60
62
  /** 内存回收 — 两次回收最小间隔(ms) */
@@ -175,7 +177,43 @@ function getPage(browser) {
175
177
  }
176
178
 
177
179
  // ────────────────────────────────────────
178
- // Section 4: CDP 会话管理(端口池/浏览器/页面/内存回收)
180
+ // Section 4: 页面请求拦截(独立函数,供自动/手动模式共用)
181
+ // ────────────────────────────────────────
182
+
183
+ const _blockedPages = new WeakSet();
184
+
185
+ async function setupPageBlocking(page) {
186
+ if (!page || _blockedPages.has(page)) return;
187
+ await page.route("**/*", (route) => {
188
+ const req = route.request();
189
+ const t = req.resourceType();
190
+ const url = req.url();
191
+
192
+ // 资源类型拦截
193
+ if (t === "image" || t === "stylesheet" || t === "media") {
194
+ return route.abort();
195
+ }
196
+
197
+ // 视频流拦截(TikTok 视频通过 fetch/xhr 加载 mp4 分片,
198
+ // resourceType 为 fetch/xhr,需按 URL 模式匹配)
199
+ // 真实 URL:
200
+ // v16-webapp-prime.us.tiktok.com/video/tos/...?mime_type=video_mp4
201
+ // v16m-webapp.tiktokcdn-us.com/.../video/tos/...?mime_type=video_mp4
202
+ if (
203
+ /webapp.*(tiktok\.com|tiktokcdn).*\/video\/tos\//i.test(url) ||
204
+ /mime_type=video_mp4/i.test(url) ||
205
+ /\.(mp4|m3u8|ts)(\?|$)/i.test(url)
206
+ ) {
207
+ return route.abort();
208
+ }
209
+
210
+ route.continue();
211
+ });
212
+ _blockedPages.add(page);
213
+ }
214
+
215
+ // ────────────────────────────────────────
216
+ // Section 5: CDP 会话管理(端口池/浏览器/页面/内存回收)
179
217
  // ────────────────────────────────────────
180
218
 
181
219
  class CdpSessionManager {
@@ -187,7 +225,8 @@ class CdpSessionManager {
187
225
  this.browser = null;
188
226
  this.page = null;
189
227
  this.lastRecycleAt = 0;
190
- this._blockedPages = new WeakSet();
228
+ this._lastMemCheck = 0;
229
+ this._shuttingDown = false;
191
230
  }
192
231
 
193
232
  pickNextPort() {
@@ -211,7 +250,7 @@ class CdpSessionManager {
211
250
  /** 获取或恢复 page */
212
251
  async _ensurePage() {
213
252
  let p = getPage(this.browser);
214
- if (p) await this._setupBlocking(p);
253
+ if (p) await setupPageBlocking(p);
215
254
  return p;
216
255
  }
217
256
 
@@ -222,17 +261,6 @@ class CdpSessionManager {
222
261
  return this.page;
223
262
  }
224
263
 
225
- /** 页面请求拦截:屏蔽图片/样式减少内存 */
226
- async _setupBlocking(page) {
227
- if (!page || this._blockedPages.has(page)) return;
228
- await page.route("**/*", (route) => {
229
- const t = route.request().resourceType();
230
- if (t === "image" || t === "stylesheet") route.abort();
231
- else route.continue();
232
- });
233
- this._blockedPages.add(page);
234
- }
235
-
236
264
  /** 轻量回收:清空当前 tab */
237
265
  async recyclePage() {
238
266
  if (!this.page || this.page.isClosed()) {
@@ -278,8 +306,12 @@ class CdpSessionManager {
278
306
  this.lastRecycleAt = Date.now();
279
307
  }
280
308
 
281
- /** 内存检查 + 按需回收 */
309
+ /** 内存检查 + 按需回收(每 30s 最多采样一次) */
282
310
  async maybeRecycleForMemory(processedCount) {
311
+ const now = Date.now();
312
+ if (now - this._lastMemCheck < 30000) return;
313
+ this._lastMemCheck = now;
314
+
283
315
  const mem = process.memoryUsage();
284
316
  const rssMB = mem.rss / 1024 / 1024;
285
317
  const heapMB = mem.heapUsed / 1024 / 1024;
@@ -334,6 +366,8 @@ class CdpSessionManager {
334
366
  }
335
367
 
336
368
  async shutdown() {
369
+ if (this._shuttingDown) return;
370
+ this._shuttingDown = true;
337
371
  if (this.page) await this.page.close().catch(() => {});
338
372
  await this.browser?.close().catch(() => {});
339
373
  killEdgeProcesses(null, this.port);
@@ -341,7 +375,7 @@ class CdpSessionManager {
341
375
  }
342
376
 
343
377
  // ────────────────────────────────────────
344
- // Section 5: 共享操作(视频页准备 & 评论提取 & 保存)
378
+ // Section 6: 共享操作(视频页准备 & 评论提取 & 保存)
345
379
  // ────────────────────────────────────────
346
380
 
347
381
  async function prepareVideoPage(page, href) {
@@ -390,13 +424,12 @@ function extractUniqueAuthors(comments) {
390
424
  ];
391
425
  }
392
426
 
393
- async function saveAuthorsToServer(serverUrl, authors, guessedLocation) {
394
- const { apiPost } = createApiClient({
395
- checkStatus: true,
396
- maxRetries: 2,
397
- backoff: 2000,
398
- log: true,
399
- });
427
+ async function saveAuthorsToServer(
428
+ apiPost,
429
+ serverUrl,
430
+ authors,
431
+ guessedLocation,
432
+ ) {
400
433
  return apiPost(`${serverUrl}/api/users`, {
401
434
  usernames: authors.map((a) => "@" + a),
402
435
  sources: ["comment"],
@@ -405,7 +438,7 @@ async function saveAuthorsToServer(serverUrl, authors, guessedLocation) {
405
438
  }
406
439
 
407
440
  // ────────────────────────────────────────
408
- // Section 6: 自动模式 — 循环从服务端取任务、抓评论、提交
441
+ // Section 7: 自动模式 — 循环从服务端取任务、抓评论、提交
409
442
  // ────────────────────────────────────────
410
443
 
411
444
  async function runAutoMode(options) {
@@ -426,7 +459,14 @@ async function runAutoMode(options) {
426
459
 
427
460
  loadEnvConfig();
428
461
 
429
- const { apiGet, apiPut } = createApiClient({
462
+ // ── 设备标识 ──
463
+ let userId = configuredUserId;
464
+ if (!userId) {
465
+ userId = await getMacOrUuid();
466
+ saveUserId(userId);
467
+ }
468
+
469
+ const { apiGet, apiPost, apiPut } = createApiClient({
430
470
  checkStatus: true,
431
471
  maxRetries: 2,
432
472
  backoff: 2000,
@@ -617,6 +657,16 @@ async function runAutoMode(options) {
617
657
  });
618
658
 
619
659
  if (captchaResult.detected) {
660
+ // 上报验证码错误
661
+ apiPost(`${serverUrl}/api/error-report`, {
662
+ userId,
663
+ username: "",
664
+ errorType: "captcha",
665
+ errorMessage: resultNote || "评论采集时出现验证码",
666
+ stage: "comments",
667
+ errorStack: "",
668
+ }).catch(() => {});
669
+
620
670
  if (captchaResult.resolved) {
621
671
  resultNote = "验证码已解决";
622
672
  consecutiveCaptchaCount = 0;
@@ -648,6 +698,7 @@ async function runAutoMode(options) {
648
698
 
649
699
  if (authors.length > 0) {
650
700
  const saveResult = await saveAuthorsToServer(
701
+ apiPost,
651
702
  serverUrl,
652
703
  authors,
653
704
  loc,
@@ -734,7 +785,7 @@ async function runAutoMode(options) {
734
785
  }
735
786
 
736
787
  // ────────────────────────────────────────
737
- // Section 7: 手动模式 — 单视频评论提取 + 输出/保存
788
+ // Section 8: 手动模式 — 单视频评论提取 + 输出/保存
738
789
  // ────────────────────────────────────────
739
790
 
740
791
  async function runManualMode(url, options) {
@@ -742,17 +793,22 @@ async function runManualMode(url, options) {
742
793
  maxComments = CFG.maxComments,
743
794
  save: shouldSave,
744
795
  serverUrl = defaultServer,
796
+ basePort = 9222,
745
797
  } = options;
746
798
 
747
799
  let browser;
748
800
  try {
749
- browser = await chromium.connectOverCDP("http://127.0.0.1:9222");
801
+ const { chromium } = await import("playwright");
802
+ browser = await chromium.connectOverCDP(`http://127.0.0.1:${basePort}`);
750
803
  const page = getPage(browser);
751
804
  if (!page) {
752
805
  console.error("未找到可用页面");
753
806
  process.exit(1);
754
807
  }
755
808
 
809
+ // 拦截图片/视频请求
810
+ await setupPageBlocking(page);
811
+
756
812
  console.error(`正在打开: ${url}`);
757
813
  await prepareVideoPage(page, url);
758
814
 
@@ -788,7 +844,14 @@ async function runManualMode(url, options) {
788
844
  if (guessedLocation) console.error(`猜测国家: ${guessedLocation}`);
789
845
 
790
846
  try {
847
+ const { apiPost } = createApiClient({
848
+ checkStatus: true,
849
+ maxRetries: 2,
850
+ backoff: 2000,
851
+ log: true,
852
+ });
791
853
  const saveResult = await saveAuthorsToServer(
854
+ apiPost,
792
855
  serverUrl,
793
856
  authors,
794
857
  guessedLocation,
@@ -834,7 +897,7 @@ async function runManualMode(url, options) {
834
897
  }
835
898
 
836
899
  // ────────────────────────────────────────
837
- // Section 8: 入口 — handleComments
900
+ // Section 9: 入口 — handleComments
838
901
  // ────────────────────────────────────────
839
902
 
840
903
  export async function handleComments(options) {
@@ -866,5 +929,6 @@ export async function handleComments(options) {
866
929
  maxComments: commentsMax,
867
930
  save: commentsSave,
868
931
  serverUrl: commentsServer || defaultServer,
932
+ basePort: commentsBasePort,
869
933
  });
870
934
  }
package/src/cli/tag.js CHANGED
@@ -17,7 +17,14 @@ import { discoverTags } from "../lib/tag-discover.js";
17
17
  import {
18
18
  server as cfgServer,
19
19
  proxy as configuredProxy,
20
+ userId as configuredUserId,
21
+ saveUserId,
20
22
  } from "../lib/constants.js";
23
+ import { getMacOrUuid } from "../lib/mac-or-uuid.js";
24
+ import {
25
+ waitAndGetCaptcha,
26
+ closeCaptcha,
27
+ } from "../scraper/modules/captcha-handler.js";
21
28
 
22
29
  const ALL_COUNTRIES = DEFAULT_TARGET_LOCATIONS;
23
30
  const DEFAULT_SERVER = cfgServer || "http://127.0.0.1:3000";
@@ -688,6 +695,13 @@ export async function handleScoreAll(parsed) {
688
695
  log(`📊 启动内存: ${formatMemoryWithBrowser(cdpPort)}`);
689
696
  log("");
690
697
 
698
+ // ── 设备标识 ──
699
+ let userId = configuredUserId;
700
+ if (!userId) {
701
+ userId = await getMacOrUuid();
702
+ saveUserId(userId);
703
+ }
704
+
691
705
  // 连接 CDP 浏览器
692
706
  const cdpOpts = { port: cdpPort };
693
707
  if (effectiveProxy) cdpOpts.proxyServer = effectiveProxy;
@@ -731,7 +745,7 @@ export async function handleScoreAll(parsed) {
731
745
  const RECYCLE_CONFIG = {
732
746
  nodeRssMb: Math.max(
733
747
  256,
734
- parseInt(process.env.TAG_SCOREALL_RECYCLE_RSS_MB || "900", 10) || 900,
748
+ parseInt(process.env.TAG_SCOREALL_RECYCLE_RSS_MB || "2000", 10) || 2000,
735
749
  ),
736
750
  nodeHeapMb: Math.max(
737
751
  128,
@@ -850,6 +864,8 @@ export async function handleScoreAll(parsed) {
850
864
 
851
865
  let totalScored = 0;
852
866
  let emptyRounds = 0; // 连续无任务的轮数
867
+ let consecutiveCaptcha = 0;
868
+ const MAX_CONSECUTIVE_CAPTCHA = 3;
853
869
  let consecutive403 = 0; // 连续 HTTP 403 次数,达 3 次认为被封
854
870
  const CONSECUTIVE_403_BLOCKED = 3; // 连续 3 次 403 认为账号被封
855
871
 
@@ -1098,6 +1114,57 @@ export async function handleScoreAll(parsed) {
1098
1114
  }
1099
1115
 
1100
1116
  log(` ❌ 失败: ${e.message}`);
1117
+
1118
+ // ── 验证码检测 ──
1119
+ const isCaptcha =
1120
+ e.message &&
1121
+ (e.message.includes("验证码") || e.message.includes("captcha"));
1122
+
1123
+ if (isCaptcha) {
1124
+ consecutiveCaptcha++;
1125
+ log(
1126
+ ` 🔐 验证码 (${consecutiveCaptcha}/${MAX_CONSECUTIVE_CAPTCHA}),等待用户手动解决...`,
1127
+ );
1128
+
1129
+ // 上报验证码错误
1130
+ try {
1131
+ await fetch(`${baseUrl}/api/error-report`, {
1132
+ method: "POST",
1133
+ headers: buildClientHeaders(clientId, clientMeta, {
1134
+ "Content-Type": "application/json",
1135
+ }),
1136
+ body: JSON.stringify({
1137
+ userId,
1138
+ username: "",
1139
+ errorType: "captcha",
1140
+ errorMessage: e.message,
1141
+ stage: "tag-scoring",
1142
+ errorStack: "",
1143
+ }),
1144
+ });
1145
+ } catch {}
1146
+
1147
+ // 等待用户手动解决验证码(最多 3 分钟)
1148
+ try {
1149
+ const captchaResult = await waitAndGetCaptcha(page, {
1150
+ waitMs: 180000,
1151
+ pollInterval: 5000,
1152
+ log: () => {},
1153
+ });
1154
+ if (captchaResult.detected && !captchaResult.resolved) {
1155
+ await closeCaptcha(page).catch(() => {});
1156
+ }
1157
+ } catch {}
1158
+
1159
+ if (consecutiveCaptcha >= MAX_CONSECUTIVE_CAPTCHA) {
1160
+ log(` ♻️ 连续${MAX_CONSECUTIVE_CAPTCHA}次验证码 → 切换端口`);
1161
+ await recycleCdpSession("连续验证码", { switchPort: true });
1162
+ consecutiveCaptcha = 0;
1163
+ }
1164
+ } else {
1165
+ consecutiveCaptcha = 0;
1166
+ }
1167
+
1101
1168
  try {
1102
1169
  // 使用当前正在处理的 tag(tag 变量在 try 块开头已赋值,catch 中仍可访问)
1103
1170
  const failedTag = typeof tag === "string" && tag ? tag : "unknown";
@@ -267,7 +267,7 @@ const HELP_TEXT = [
267
267
  " 连续 3 次 HTTP 403 → 判断账号被封 → 等 2 分钟后自动切换到池中另一端口",
268
268
  " 环境变量:",
269
269
  " TAG_SCOREALL_PORT_POOL_SIZE=N 端口池大小(默认 10)",
270
- " TAG_SCOREALL_RECYCLE_RSS_MB=N 内存回收阈值 RSS MB(默认 900)",
270
+ " TAG_SCOREALL_RECYCLE_RSS_MB=N 内存回收阈值 RSS MB(默认 2000)",
271
271
  " TAG_SCOREALL_RECYCLE_HEAP_MB=N 内存回收阈值 Heap MB(默认 2000)",
272
272
  " 示例: tt-help tag score-all",
273
273
  " tt-help tag score-all --countries ES -s http://127.0.0.1:3001",
@@ -500,6 +500,8 @@ export function createStore(filePath, options = {}) {
500
500
  llmScore: false,
501
501
  llmMinScore: 60,
502
502
  llmSampleSize: 100,
503
+ llmTagRatio: 0.4,
504
+ llmCommentRatio: 0.3,
503
505
  ...options.refillLlm,
504
506
  };
505
507
 
@@ -957,27 +959,30 @@ export function createStore(filePath, options = {}) {
957
959
  return { moved: 0 };
958
960
  }
959
961
 
960
- // 如果启用 LLM 打分:先取 tag(一次性),再取非 tag 走 LLM 打分
962
+ // 如果启用 LLM 打分:tag 和 comment 免打分直接入围,其余走 LLM 打分
961
963
  if (useLlm && normalizedLocations && normalizedLocations.length > 0) {
962
964
  const llmTotal = options.llmTotal ?? 200; // 总条数
963
- const llmTagRatio = options.llmTagRatio ?? 0.6; // tag 占比 60%
964
- const llmTagLimit = Math.floor(llmTotal * llmTagRatio); // tag 上限 120
965
- const llmNonTagTarget = llmTotal - llmTagLimit; // 非 tag 目标 80
965
+ const llmTagRatio = options.llmTagRatio ?? 0.4; // tag 占比 40%
966
+ const llmCommentRatio = options.llmCommentRatio ?? 0.3; // comment 占比 30%
967
+ const llmTagLimit = Math.floor(llmTotal * llmTagRatio); // tag 上限
968
+ const llmCommentLimit = Math.floor(llmTotal * llmCommentRatio); // comment 上限
969
+ const llmLLMTarget = llmTotal - llmTagLimit - llmCommentLimit; // LLM 打分目标
966
970
  const llmMinScore = options.llmMinScore ?? 60;
967
971
  const llmSampleSize = options.llmSampleSize ?? 100;
968
972
  const maxBatches = options.llmMaxBatches ?? 10;
969
973
 
970
974
  console.error(
971
- `[data-store] LLM 打分开始: 总目标 ${llmTotal} 条,tag 最多 ${llmTagLimit} 条(一次性),非 tag 目标 ${llmNonTagTarget} 条(LLM 打分)`,
975
+ `[data-store] LLM 调度开始: 总目标 ${llmTotal} 条,tag ${llmTagLimit} 条,comment ${llmCommentLimit} 条(免打分),LLM 打分 ${llmLLMTarget} 条`,
972
976
  );
973
977
 
974
978
  // 返回 Promise,调用方需要 await
975
979
  return (async () => {
976
980
  const allTagQualified = [];
977
- const allNonTagQualified = [];
981
+ const allCommentQualified = [];
982
+ const allLLMQualified = [];
978
983
  const allScores = [];
979
984
 
980
- // ===== 第一步:一次性取所有 tag(全局,最多 llmTagLimit 条)=====
985
+ // ===== 第一步:一次性取 tag(全局,最多 llmTagLimit 条,免打分)=====
981
986
  let tagOffset = offsetStore.get("_tag") || 0;
982
987
  const tagGlobalCount =
983
988
  getDb()
@@ -1025,43 +1030,97 @@ export function createStore(filePath, options = {}) {
1025
1030
 
1026
1031
  offsetStore.set("_tag", tagOffset);
1027
1032
 
1028
- // ===== 第二步:按国家取非 tag,走 LLM 打分,直到合格数达到 llmNonTagTarget =====
1033
+ // ===== 第二步:一次性取 comment(全局,最多 llmCommentLimit 条,免打分)=====
1034
+ let commentOffset = offsetStore.get("_comment") || 0;
1035
+ const commentGlobalCount =
1036
+ getDb()
1037
+ .prepare(
1038
+ `SELECT COUNT(*) as c FROM raw_jobs WHERE ${whereSql} AND sources LIKE '%comment%' AND sources NOT LIKE '%tag%'`,
1039
+ )
1040
+ .get(...args)?.c || 0;
1041
+
1042
+ if (commentOffset >= commentGlobalCount) {
1043
+ commentOffset = 0;
1044
+ offsetStore.set("_comment", 0);
1045
+ }
1046
+
1047
+ console.error(
1048
+ `[data-store] Comment 全局共 ${commentGlobalCount} 条,从偏移量 ${commentOffset} 开始`,
1049
+ );
1050
+
1051
+ while (
1052
+ allCommentQualified.length < llmCommentLimit &&
1053
+ commentOffset < commentGlobalCount
1054
+ ) {
1055
+ const batch = getDb()
1056
+ .prepare(
1057
+ `
1058
+ SELECT * FROM raw_jobs WHERE ${whereSql} AND sources LIKE '%comment%' AND sources NOT LIKE '%tag%'
1059
+ ORDER BY COALESCE(video_count, 0) DESC, created_at DESC
1060
+ LIMIT ? OFFSET ?
1061
+ `,
1062
+ )
1063
+ .all(
1064
+ ...args,
1065
+ Math.min(
1066
+ llmSampleSize,
1067
+ llmCommentLimit - allCommentQualified.length,
1068
+ ),
1069
+ commentOffset,
1070
+ );
1071
+
1072
+ if (!batch.length) break;
1073
+
1074
+ allCommentQualified.push(...batch.map((s) => s.unique_id));
1075
+ commentOffset += batch.length;
1076
+
1077
+ console.error(
1078
+ `[data-store] Comment 本批 ${batch.length} 条,累计 ${allCommentQualified.length}/${llmCommentLimit}`,
1079
+ );
1080
+ }
1081
+
1082
+ offsetStore.set("_comment", commentOffset);
1083
+
1084
+ // ===== 第三步:按国家取非 tag 非 comment,走 LLM 打分,直到合格数达到 llmLLMTarget =====
1029
1085
  for (const location of normalizedLocations) {
1030
- if (allNonTagQualified.length >= llmNonTagTarget) break;
1086
+ if (allLLMQualified.length >= llmLLMTarget) break;
1031
1087
 
1032
- const nonTagOffsetKey = `${location}:nonTag`;
1033
- let offset = offsetStore.get(nonTagOffsetKey) || 0;
1088
+ const llmOffsetKey = `${location}:llm`;
1089
+ let offset = offsetStore.get(llmOffsetKey) || 0;
1034
1090
 
1035
1091
  const locationArgs = [...args, location];
1036
- const nonTagCount =
1092
+ const llmCount =
1037
1093
  getDb()
1038
1094
  .prepare(
1039
- `SELECT COUNT(*) as c FROM raw_jobs WHERE ${whereSql} AND guessed_location = ? AND (sources NOT LIKE '%tag%' OR sources IS NULL)`,
1095
+ `SELECT COUNT(*) as c FROM raw_jobs WHERE ${whereSql} AND guessed_location = ? AND (sources NOT LIKE '%tag%' OR sources IS NULL) AND (sources NOT LIKE '%comment%' OR sources IS NULL)`,
1040
1096
  )
1041
1097
  .get(...locationArgs)?.c || 0;
1042
1098
 
1043
- if (nonTagCount === 0) {
1044
- console.error(`[data-store] 国家 ${location}: 无非 tag 数据,跳过`);
1099
+ if (llmCount === 0) {
1100
+ console.error(
1101
+ `[data-store] 国家 ${location}: 无 LLM 打分候选数据,跳过`,
1102
+ );
1045
1103
  continue;
1046
1104
  }
1047
1105
 
1048
- if (offset >= nonTagCount) {
1106
+ if (offset >= llmCount) {
1049
1107
  offset = 0;
1050
- offsetStore.set(nonTagOffsetKey, 0);
1108
+ offsetStore.set(llmOffsetKey, 0);
1051
1109
  }
1052
1110
 
1053
1111
  console.error(
1054
- `[data-store] 国家 ${location}: 非 tag 共 ${nonTagCount} 条,从偏移量 ${offset} 开始`,
1112
+ `[data-store] 国家 ${location}: LLM 候选共 ${llmCount} 条,从偏移量 ${offset} 开始`,
1055
1113
  );
1056
1114
 
1057
1115
  for (let batch = 0; batch < maxBatches; batch++) {
1058
- if (allNonTagQualified.length >= llmNonTagTarget) break;
1116
+ if (allLLMQualified.length >= llmLLMTarget) break;
1059
1117
 
1060
1118
  const samples = getDb()
1061
1119
  .prepare(
1062
1120
  `
1063
1121
  SELECT * FROM raw_jobs WHERE ${whereSql} AND guessed_location = ?
1064
1122
  AND (sources NOT LIKE '%tag%' OR sources IS NULL)
1123
+ AND (sources NOT LIKE '%comment%' OR sources IS NULL)
1065
1124
  ORDER BY COALESCE(video_count, 0) DESC, created_at DESC
1066
1125
  LIMIT ? OFFSET ?
1067
1126
  `,
@@ -1075,24 +1134,28 @@ export function createStore(filePath, options = {}) {
1075
1134
  DEFAULT_TARGET_LOCATIONS,
1076
1135
  );
1077
1136
  const qualified = scores.filter((s) => s.score >= llmMinScore);
1078
- allNonTagQualified.push(...qualified.map((s) => s.uniqueId));
1137
+ allLLMQualified.push(...qualified.map((s) => s.uniqueId));
1079
1138
  allScores.push(...scores);
1080
1139
 
1081
1140
  offset += samples.length;
1082
- offsetStore.set(nonTagOffsetKey, offset);
1141
+ offsetStore.set(llmOffsetKey, offset);
1083
1142
 
1084
1143
  console.error(
1085
- `[data-store] ${location} 第 ${batch + 1} 批: 采样 ${samples.length} 条,本批合格 ${qualified.length} 条,非 tag 累计 ${allNonTagQualified.length}/${llmNonTagTarget}`,
1144
+ `[data-store] ${location} 第 ${batch + 1} 批: 采样 ${samples.length} 条,本批合格 ${qualified.length} 条,LLM 累计 ${allLLMQualified.length}/${llmLLMTarget}`,
1086
1145
  );
1087
1146
  }
1088
1147
  }
1089
1148
 
1090
1149
  // ===== 最终结果 =====
1091
- const qualified = [...allTagQualified, ...allNonTagQualified];
1150
+ const qualified = [
1151
+ ...allTagQualified,
1152
+ ...allCommentQualified,
1153
+ ...allLLMQualified,
1154
+ ];
1092
1155
 
1093
1156
  if (!qualified.length) {
1094
1157
  console.error(
1095
- `[data-store] LLM 打分后无符合条件的任务(tag: ${allTagQualified.length}, 非 tag: ${allNonTagQualified.length})`,
1158
+ `[data-store] LLM 调度后无符合条件的任务(tag: ${allTagQualified.length}, comment: ${allCommentQualified.length}, LLM: ${allLLMQualified.length})`,
1096
1159
  );
1097
1160
  return {
1098
1161
  moved: 0,
@@ -1103,7 +1166,7 @@ export function createStore(filePath, options = {}) {
1103
1166
  }
1104
1167
 
1105
1168
  console.error(
1106
- `[data-store] LLM 打分完成: tag ${allTagQualified.length} 条 + 非 tag ${allNonTagQualified.length} 条 = 共 ${qualified.length} 条`,
1169
+ `[data-store] LLM 调度完成: tag ${allTagQualified.length} + comment ${allCommentQualified.length} + LLM ${allLLMQualified.length} = 共 ${qualified.length} 条`,
1107
1170
  );
1108
1171
 
1109
1172
  // 移动符合条件的记录
@@ -1181,6 +1244,7 @@ export function createStore(filePath, options = {}) {
1181
1244
  WHERE __WHERE__
1182
1245
  ORDER BY
1183
1246
  CASE WHEN sources LIKE '%tag%' THEN 0 ELSE 1 END,
1247
+ CASE WHEN sources LIKE '%comment%' THEN 0 ELSE 1 END,
1184
1248
  COALESCE(video_count, 0) DESC, created_at DESC
1185
1249
  LIMIT ?
1186
1250
  `;
@@ -1192,6 +1256,7 @@ export function createStore(filePath, options = {}) {
1192
1256
  WHERE __WHERE__
1193
1257
  ORDER BY
1194
1258
  CASE WHEN sources LIKE '%tag%' THEN 0 ELSE 1 END,
1259
+ CASE WHEN sources LIKE '%comment%' THEN 0 ELSE 1 END,
1195
1260
  COALESCE(video_count, 0) DESC, created_at DESC
1196
1261
  LIMIT ?
1197
1262
  )