tt-help-cli-ycl 1.4.16 → 1.4.18

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "tt-help-cli-ycl",
3
- "version": "1.4.16",
3
+ "version": "1.4.18",
4
4
  "description": "TikTok user & video data scraper - extract ttSeller, verified, locationCreated from HTML source",
5
5
  "type": "module",
6
6
  "bin": {
@@ -646,11 +646,13 @@ async function runAutoMode(options) {
646
646
  } catch (err) {
647
647
  console.error(` × 获取任务失败: ${err.message}`);
648
648
  consecutiveErrors++;
649
- if (consecutiveErrors > 10) {
650
- console.error(" × 连续获取失败 >10 → 退出");
651
- await shutdown("too many errors");
649
+ if (consecutiveErrors > 0 && consecutiveErrors % 30 === 0) {
650
+ console.error(
651
+ ` ⚠ 已连续获取失败 ${consecutiveErrors} 次,继续等待服务恢复...`,
652
+ );
652
653
  }
653
- await delay(1000, actualInterval * 1000);
654
+ const waitMs = Math.min(consecutiveErrors * 1000, 60000);
655
+ await delay(waitMs, waitMs + 1000);
654
656
  continue;
655
657
  }
656
658
  consecutiveErrors = 0;
@@ -54,6 +54,7 @@ export async function handleRefresh(options) {
54
54
  exploreMaxFollowing,
55
55
  exploreMaxFollowers,
56
56
  exploreLocation,
57
+ exploreJobLocations,
57
58
  exploreMaxUsers,
58
59
  serverUrl,
59
60
  explorePort,
@@ -106,6 +107,7 @@ export async function handleRefresh(options) {
106
107
  `关注/粉丝: ${exploreEnableFollow ? "启用" : "禁用"} (${exploreMaxFollowing}/${exploreMaxFollowers})`,
107
108
  );
108
109
  console.error(`国家筛选: ${exploreLocation}`);
110
+ if (exploreJobLocations) console.error(`任务国家: ${exploreJobLocations}`);
109
111
  console.error(`空闲间隔: ${exploreInterval || 30} 秒`);
110
112
  if (effectiveProxy) {
111
113
  console.error(
@@ -274,6 +276,9 @@ export async function handleRefresh(options) {
274
276
  if (exploreRedoMaxAge > 0) {
275
277
  redoUrl += `&maxAge=${exploreRedoMaxAge}`;
276
278
  }
279
+ if (exploreJobLocations) {
280
+ redoUrl += `&locations=${encodeURIComponent(exploreJobLocations)}`;
281
+ }
277
282
  let job = null;
278
283
  try {
279
284
  job = await apiGet(redoUrl);
package/src/cli/tag.js CHANGED
@@ -528,13 +528,13 @@ async function scoreSingleTag(
528
528
  log(` ✗ 标签不存在(TikTok 提示"找不到此话题标签"),标记为 dead`);
529
529
  result.status = "dead";
530
530
  result.error = "tag_not_found";
531
- } else {
532
- log(` ✗ 没有视频数据,标记为 dead(可能是网络问题或页面未加载)`);
533
- result.status = "dead";
534
- result.error = "no videos found";
531
+ await reportToServer(baseUrl, result);
532
+ return result;
535
533
  }
536
- await reportToServer(baseUrl, result);
537
- return result;
534
+ // 非 tag_not_found 的 0 视频 → 抛异常让其走 error 重试路径,避免误标 dead
535
+ throw new Error(
536
+ "no videos returned — possible network/captcha/slow-load, will retry",
537
+ );
538
538
  }
539
539
 
540
540
  // Step 2: 补充国家信息
@@ -1064,18 +1064,20 @@ export async function handleScoreAll(parsed) {
1064
1064
 
1065
1065
  if (!videos || videos.length === 0) {
1066
1066
  const deadSec = ((Date.now() - fetchStart) / 1000).toFixed(1);
1067
- log(
1068
- ` ⚠️ 无视频 (${deadSec}s) mem=${formatMemoryWithBrowser(cdpPort)},标记 dead`,
1069
- );
1070
- result.status = "dead";
1071
- result.error = "no videos found";
1072
- await reportToServer(baseUrl, result, clientId, clientMeta);
1073
- totalScored++;
1074
- // 随机等待 3-7 秒,避免连续访问 TikTok 触发风控
1075
- await randomDelay(0, 5000);
1076
- await recyclePage();
1077
- await maybeRecycleForMemory();
1078
- continue;
1067
+ if (tagResult.error === "tag_not_found") {
1068
+ log(` ⚠️ 标签不存在 (${deadSec}s),标记 dead`);
1069
+ result.status = "dead";
1070
+ result.error = "tag_not_found";
1071
+ await reportToServer(baseUrl, result, clientId, clientMeta);
1072
+ totalScored++;
1073
+ await randomDelay(0, 5000);
1074
+ await recyclePage();
1075
+ await maybeRecycleForMemory();
1076
+ continue;
1077
+ }
1078
+ // 非 tag_not_found 的 0 视频 → 抛异常让其走 error 重试路径,避免误标 dead
1079
+ log(` ⚠️ 无视频 (${deadSec}s),可能网络/风控/加载慢 → 抛异常重试`);
1080
+ throw new Error("no videos returned (network/captcha/slow-load)");
1079
1081
  }
1080
1082
 
1081
1083
  // enrich: 先查服务端缓存,只对未知视频发 HTTP 请求
package/src/cli/watch.js CHANGED
@@ -5,16 +5,26 @@ import { startWatchServer, openBrowser } from "../watch/server.js";
5
5
 
6
6
  export async function handleWatch(options) {
7
7
  const dataAnchor = options.dataAnchor || options.outputFile;
8
- const { watchPort, llmRefill, llmRefillMinScore, llmRefillSampleSize } =
9
- options;
8
+ const {
9
+ watchPort,
10
+ llmRefill,
11
+ llmRefillMinScore,
12
+ llmRefillSampleSize,
13
+ llmDiscover,
14
+ } = options;
10
15
 
11
16
  if (!dataAnchor) {
12
- console.error("用法: tt-help watch -o <db路径> [-p 端口] [--llm-refill]");
17
+ console.error(
18
+ "用法: tt-help watch -o <db路径> [-p 端口] [--llm-refill] [--no-llm-discover]",
19
+ );
13
20
  console.error(" tt-help watch -o data/result.db");
14
21
  console.error(" tt-help watch -o data/result.db -p 8080");
15
22
  console.error(
16
23
  " tt-help watch -o data/result.db --llm-refill --llm-refill-min 60",
17
24
  );
25
+ console.error(
26
+ " tt-help watch -o data/result.db --no-llm-discover --no-llm-refill",
27
+ );
18
28
  process.exit(1);
19
29
  }
20
30
 
@@ -37,7 +47,8 @@ export async function handleWatch(options) {
37
47
  dataAnchor,
38
48
  watchPort,
39
49
  store,
40
- storeOptions,
50
+ { ...storeOptions, llmDiscover: llmDiscover ?? true },
51
+ options.watchPassword,
41
52
  );
42
53
  openBrowser(port);
43
54
 
package/src/lib/args.js CHANGED
@@ -347,9 +347,11 @@ function parseInfoArgs(args) {
347
347
  function parseWatchArgs(args) {
348
348
  let dataAnchor = "./result.db";
349
349
  let watchPort = 3001;
350
+ let watchPassword = null;
350
351
  let llmRefill = true;
351
352
  let llmRefillMinScore = null;
352
353
  let llmRefillSampleSize = null;
354
+ let llmDiscover = true;
353
355
 
354
356
  for (let i = 0; i < args.length; i++) {
355
357
  const arg = args[i];
@@ -357,6 +359,8 @@ function parseWatchArgs(args) {
357
359
  dataAnchor = args[++i];
358
360
  } else if (arg === "-p") {
359
361
  watchPort = parseInt(args[++i]) || 3001;
362
+ } else if (arg === "-P" || arg === "--password") {
363
+ watchPassword = args[++i] || null;
360
364
  } else if (arg === "--llm-refill") {
361
365
  llmRefill = true;
362
366
  } else if (arg === "--no-llm-refill") {
@@ -365,6 +369,10 @@ function parseWatchArgs(args) {
365
369
  llmRefillMinScore = parseInt(args[++i]) || 60;
366
370
  } else if (arg === "--llm-refill-sample") {
367
371
  llmRefillSampleSize = parseInt(args[++i]) || 100;
372
+ } else if (arg === "--llm-discover") {
373
+ llmDiscover = true;
374
+ } else if (arg === "--no-llm-discover") {
375
+ llmDiscover = false;
368
376
  }
369
377
  }
370
378
 
@@ -373,9 +381,11 @@ function parseWatchArgs(args) {
373
381
  outputFile: dataAnchor,
374
382
  dataAnchor,
375
383
  watchPort,
384
+ watchPassword,
376
385
  llmRefill,
377
386
  llmRefillMinScore,
378
387
  llmRefillSampleSize,
388
+ llmDiscover,
379
389
  urls: [],
380
390
  outputFormat: "json",
381
391
  exploreCount: 0,
@@ -471,6 +481,7 @@ function parseRefreshArgs(args) {
471
481
  let exploreMaxVideos = 16;
472
482
  let exploreRedoMaxAge = 0;
473
483
  let exploreProxy = null;
484
+ let exploreJobLocations = null;
474
485
 
475
486
  for (let i = 0; i < args.length; i++) {
476
487
  const arg = args[i];
@@ -502,6 +513,8 @@ function parseRefreshArgs(args) {
502
513
  exploreProxy = args[++i];
503
514
  } else if (arg === "--location") {
504
515
  exploreLocation = args[++i];
516
+ } else if (arg === "--job-locations") {
517
+ exploreJobLocations = args[++i];
505
518
  } else if (arg === "--enable-follow") {
506
519
  exploreEnableFollow = true;
507
520
  } else if (arg === "--disable-follow") {
@@ -519,6 +532,7 @@ function parseRefreshArgs(args) {
519
532
  exploreMaxFollowing,
520
533
  exploreMaxFollowers,
521
534
  exploreLocation,
535
+ exploreJobLocations,
522
536
  exploreMaxUsers,
523
537
  explorePort,
524
538
  exploreBasePort,
@@ -145,6 +145,7 @@ const HELP_TEXT = [
145
145
  " 选项:",
146
146
  " --server <URL> 服务端地址,默认 http://127.0.0.1:3001",
147
147
  ` --location <国家代码> 国家筛选,逗号分隔,默认 ${DEFAULT_TARGET_LOCATIONS_CSV}`,
148
+ " --job-locations <国家> 任务国家筛选,逗号分隔(仅筛选服务端任务),基于真实国家字段 location_created",
148
149
  " --max-videos <数量> 每用户最大视频数,默认 16",
149
150
  " --enable-follow 启用关注/粉丝提取(默认启用)",
150
151
  " --disable-follow 禁用关注/粉丝提取",
@@ -176,10 +177,20 @@ const HELP_TEXT = [
176
177
  " 示例: tt-help attach -p 5 -i 10",
177
178
  " tt-help attach -c PL,DE,FR -p 5",
178
179
  "",
179
- " tt-help watch -o <db路径> [-p 端口]",
180
+ " tt-help watch -o <db路径> [-p 端口] [--no-llm-refill] [--no-llm-discover]",
180
181
  " 启动监控服务;运行期主数据仅来自 SQLite",
182
+ " 选项:",
183
+ " -p, --port <端口> 服务端口,默认 3000",
184
+ " -P, --password <密码> Web 面板登录密码",
185
+ " --llm-refill 启用 LLM 自动筛选毛料任务(默认开启)",
186
+ " --no-llm-refill 关闭 LLM 任务打分",
187
+ " --llm-discover 启用 LLM 自动发现标签(默认开启)",
188
+ " --no-llm-discover 关闭 LLM tag 发现",
189
+ " --llm-refill-min <分数> 最低入围分,默认 60",
190
+ " --llm-refill-sample <数量> 采样量,默认 100",
181
191
  " 示例: tt-help watch -o data/result.db",
182
192
  " tt-help watch -o data/result.db -p 8080",
193
+ " tt-help watch -o data/result.db --no-llm-discover --no-llm-refill",
183
194
  "",
184
195
  " tt-help open <端口号>",
185
196
  " 打开指定端口的浏览器,用于配置 TikTok 登录账户",
@@ -315,7 +326,7 @@ const HELP_TEXT = [
315
326
  ];
316
327
 
317
328
  const PUBLIC_HELP_HIDDEN_HEADERS = new Set([
318
- " tt-help watch -o <db路径> [-p 端口]",
329
+ " tt-help watch -o <db路径> [-p 端口] [--no-llm-refill] [--no-llm-discover]",
319
330
  " videostats <db路径> -p <并发数>",
320
331
  " db-import --db <db路径> [--users users.json] [--done done.json] [--videos videos.json]",
321
332
  ]);
@@ -6,7 +6,7 @@ import { CDNBlockedError } from "./parse-ssr.mjs";
6
6
 
7
7
  const TAG_URL = "https://www.tiktok.com/tag";
8
8
  const SCROLL_INTERVAL = 3000;
9
- const MAX_STALE_ROUNDS = 3;
9
+ const MAX_STALE_ROUNDS = 5;
10
10
 
11
11
  function sleep(ms) {
12
12
  return new Promise((r) => setTimeout(r, ms));
@@ -113,6 +113,12 @@ export async function fetchTagData(tag, options = {}) {
113
113
 
114
114
  await page.waitForTimeout(3000);
115
115
 
116
+ // 等待首屏 API 数据到达(避免网络慢时 0 视频进入滚动循环)
117
+ const videoWaitStart = Date.now();
118
+ while (seenVideoIds.size === 0 && Date.now() - videoWaitStart < 15000) {
119
+ await sleep(1000);
120
+ }
121
+
116
122
  // 检测页面异常:验证码、网络错误、标签不存在
117
123
  const pageError = await page.evaluate(() => {
118
124
  const text = document.body?.innerText || "";
@@ -100,6 +100,7 @@ import {
100
100
  getProductiveRefreshTask,
101
101
  claimRefreshTag,
102
102
  reportRefreshResult,
103
+ getRefreshStats,
103
104
  } from "./db-tags.js";
104
105
 
105
106
  // LLM 打分
@@ -380,6 +381,9 @@ function getTargetUsersByCountryFromDb(targetLocations = [], options = {}) {
380
381
 
381
382
  // 分页模式:按国家或全局分页查询用户
382
383
  if (limit !== undefined) {
384
+ const filterIsTarget =
385
+ filterCountry && targetLocations.includes(filterCountry);
386
+
383
387
  let sql = `
384
388
  SELECT
385
389
  unique_id,
@@ -401,13 +405,22 @@ function getTargetUsersByCountryFromDb(targetLocations = [], options = {}) {
401
405
  FROM jobs
402
406
  WHERE tt_seller = 1
403
407
  AND verified = 0
404
- AND location_created IN (${placeholders})
405
408
  `;
406
- const params = [...targetLocations];
409
+ const params = [];
410
+
411
+ // 目标国家筛选:仅当 filterCountry 是目标国家或不筛选时才加 IN 限制
412
+ if (filterIsTarget || !filterCountry) {
413
+ sql += ` AND location_created IN (${placeholders})`;
414
+ params.push(...targetLocations);
415
+ }
407
416
 
408
417
  if (filterCountry) {
409
418
  sql += ` AND location_created = ?`;
410
419
  params.push(filterCountry);
420
+ // 筛选非目标国家时,仅返回已修改的用户(与“其他国家”语义一致)
421
+ if (!filterIsTarget) {
422
+ sql += ` AND modified_at IS NOT NULL`;
423
+ }
411
424
  }
412
425
 
413
426
  if (search) {
@@ -1083,6 +1096,9 @@ export function createStore(filePath, options = {}) {
1083
1096
 
1084
1097
  offsetStore.set("_tag", tagOffset);
1085
1098
 
1099
+ // 让出事件循环,避免长时间阻塞监控页面及其他客户端请求
1100
+ await new Promise(setImmediate);
1101
+
1086
1102
  // ===== 第二步:一次性取 comment(全局,最多 llmCommentLimit 条,免打分)=====
1087
1103
  let commentOffset = offsetStore.get("_comment") || 0;
1088
1104
  const commentGlobalCount =
@@ -1134,6 +1150,9 @@ export function createStore(filePath, options = {}) {
1134
1150
 
1135
1151
  offsetStore.set("_comment", commentOffset);
1136
1152
 
1153
+ // 让出事件循环,避免长时间阻塞监控页面及其他客户端请求
1154
+ await new Promise(setImmediate);
1155
+
1137
1156
  // ===== 第三步:按国家取非 tag 非 comment,走 LLM 打分,直到合格数达到 llmLLMTarget =====
1138
1157
  for (const location of normalizedLocations) {
1139
1158
  if (allLLMQualified.length >= llmLLMTarget) break;
@@ -1182,6 +1201,9 @@ export function createStore(filePath, options = {}) {
1182
1201
 
1183
1202
  if (!samples.length) break;
1184
1203
 
1204
+ // 让出事件循环,LLM API 调用期间允许处理其他请求
1205
+ await new Promise(setImmediate);
1206
+
1185
1207
  const scores = await scoreJobsBatch(
1186
1208
  samples,
1187
1209
  DEFAULT_TARGET_LOCATIONS,
@@ -1222,6 +1244,9 @@ export function createStore(filePath, options = {}) {
1222
1244
  `[data-store] LLM 调度完成: tag ${allTagQualified.length} + comment ${allCommentQualified.length} + LLM ${allLLMQualified.length} = 共 ${qualified.length} 条`,
1223
1245
  );
1224
1246
 
1247
+ // 让出事件循环,避免事务写入阻塞监控页面
1248
+ await new Promise(setImmediate);
1249
+
1225
1250
  // 移动符合条件的记录
1226
1251
  const placeholders = qualified.map(() => "?").join(", ");
1227
1252
  const moveTxn = getDb().transaction(() => {
@@ -2476,12 +2501,12 @@ export function createStore(filePath, options = {}) {
2476
2501
  return { saved: true, pinned: user.pinned };
2477
2502
  }
2478
2503
 
2479
- function getNextRedoJob(userId, maxAgeSeconds = 43200) {
2504
+ function getNextRedoJob(userId, maxAgeSeconds = 43200, locations = null) {
2480
2505
  if (getDb()) {
2481
2506
  const now = Date.now();
2482
2507
  const threshold = now - maxAgeSeconds * 1000;
2483
2508
  const defaultTime = new Date("2016-01-01T00:00:00Z").getTime();
2484
- const targetLocations = DEFAULT_TARGET_LOCATIONS;
2509
+ const targetLocations = locations || DEFAULT_TARGET_LOCATIONS;
2485
2510
  const placeholders = targetLocations.map(() => "?").join(",");
2486
2511
  const row = getDb()
2487
2512
  .prepare(
@@ -2515,7 +2540,7 @@ export function createStore(filePath, options = {}) {
2515
2540
  const defaultTime = new Date("2016-01-01T00:00:00Z").getTime();
2516
2541
 
2517
2542
  // 筛选目标国家用户,按 refreshTime 升序取最远的(没有则默认 2016-01-01)
2518
- const targetLocations = DEFAULT_TARGET_LOCATIONS;
2543
+ const targetLocations = locations || DEFAULT_TARGET_LOCATIONS;
2519
2544
  const targetUsers = data.filter(
2520
2545
  (u) =>
2521
2546
  u.ttSeller &&
@@ -2699,13 +2724,15 @@ export function createStore(filePath, options = {}) {
2699
2724
 
2700
2725
  if (getDb()) {
2701
2726
  const l = Math.max(1, parseInt(limit) || 5);
2727
+ const fiveMinutesAgo = Date.now() - 5 * 60 * 1000;
2702
2728
 
2703
2729
  let sql = `
2704
2730
  SELECT *
2705
2731
  FROM jobs_base
2706
2732
  WHERE COALESCE(user_update_count, 0) <= 0
2733
+ OR (COALESCE(user_update_count, 0) > 0 AND updated_at < ?)
2707
2734
  `;
2708
- const sqlParams = [];
2735
+ const sqlParams = [fiveMinutesAgo];
2709
2736
 
2710
2737
  if (hasCountryFilter) {
2711
2738
  const placeholders = targetCountries.map(() => "?").join(", ");
@@ -2750,14 +2777,17 @@ export function createStore(filePath, options = {}) {
2750
2777
  }
2751
2778
 
2752
2779
  const l = Math.max(1, parseInt(limit) || 5);
2780
+ const fiveMinutesAgo = Date.now() - 5 * 60 * 1000;
2753
2781
  const pending = data
2754
2782
  .filter((u) => {
2755
2783
  const updateCount = u.userUpdateCount;
2756
- // jobs_base 中所有 user_update_count <= 0 的记录都可以领取
2784
+ // user_update_count <= 0 的记录可直接领取
2785
+ // user_update_count > 0 但 updatedAt 超过 5 分钟的,视为客户端失败/超时,重新回收
2757
2786
  if (
2758
2787
  updateCount === null ||
2759
2788
  updateCount === undefined ||
2760
- updateCount <= 0
2789
+ updateCount <= 0 ||
2790
+ (u.updatedAt && u.updatedAt < fiveMinutesAgo)
2761
2791
  ) {
2762
2792
  if (hasCountryFilter) {
2763
2793
  const loc = (u.guessedLocation || "").toUpperCase();
@@ -3345,6 +3375,7 @@ export function createStore(filePath, options = {}) {
3345
3375
  getActiveClients,
3346
3376
  moveSellerJobsToBase, // 将 jobs/raw_jobs 中 tt_seller=1 且 video_count=0 的记录移动到 jobs_base
3347
3377
  moveRestrictedJobsToBase, // 将 jobs 中 status='restricted' 的记录移动到 jobs_base
3378
+ moveSingleJobToBase, // 将单个 job 移动到 jobs_base 并重置(目标商家页面用)
3348
3379
  registerVideos,
3349
3380
  getVideo,
3350
3381
  getVideos,
@@ -3372,6 +3403,7 @@ export function createStore(filePath, options = {}) {
3372
3403
  getProductiveRefreshTask,
3373
3404
  claimRefreshTag,
3374
3405
  reportRefreshResult,
3406
+ getRefreshStats,
3375
3407
  data,
3376
3408
  };
3377
3409
 
@@ -3539,4 +3571,78 @@ export function createStore(filePath, options = {}) {
3539
3571
  markStatsDirty();
3540
3572
  return { ok: true, moved: deleted.changes || 0 };
3541
3573
  }
3574
+
3575
+ // 将单个 job 移动到 jobs_base(用于目标商家页面"移到base重走流程")
3576
+ function moveSingleJobToBase(uniqueId) {
3577
+ const db = getDb();
3578
+ if (!db) return { ok: false, error: "db not ready" };
3579
+
3580
+ const safeId = String(uniqueId).trim();
3581
+ if (!safeId) return { ok: false, error: "invalid uniqueId" };
3582
+
3583
+ const existing = db
3584
+ .prepare("SELECT * FROM jobs WHERE unique_id = ?")
3585
+ .get(safeId);
3586
+ if (!existing) return { ok: false, error: "user not found in jobs" };
3587
+
3588
+ const COLUMNS = [
3589
+ "unique_id",
3590
+ "nickname",
3591
+ "status",
3592
+ "sources",
3593
+ "claimed_by",
3594
+ "claimed_at",
3595
+ "error",
3596
+ "pinned",
3597
+ "no_video",
3598
+ "restricted",
3599
+ "user_update_count",
3600
+ "tt_seller",
3601
+ "verified",
3602
+ "video_count",
3603
+ "comment_count",
3604
+ "guessed_location",
3605
+ "location_created",
3606
+ "confirmed_location",
3607
+ "modified_at",
3608
+ "follower_count",
3609
+ "following_count",
3610
+ "heart_count",
3611
+ "refresh_time",
3612
+ "processed",
3613
+ "processed_at",
3614
+ "created_at",
3615
+ "updated_at",
3616
+ "region",
3617
+ "signature",
3618
+ "bio_link",
3619
+ "sec_uid",
3620
+ "status_code",
3621
+ "latest_video_time",
3622
+ ];
3623
+ const cols = COLUMNS.join(",");
3624
+
3625
+ try {
3626
+ db.prepare(
3627
+ `INSERT OR REPLACE INTO jobs_base (${cols})
3628
+ SELECT ${cols} FROM jobs WHERE unique_id = ?`,
3629
+ ).run(safeId);
3630
+
3631
+ // 重置 user_update_count,保留 tt_seller,状态改为 pending
3632
+ db.prepare(
3633
+ `UPDATE jobs_base SET user_update_count = 0, status = 'pending',
3634
+ claimed_by = NULL, claimed_at = NULL, processed = 0, processed_at = NULL,
3635
+ error = NULL, restricted = 0
3636
+ WHERE unique_id = ?`,
3637
+ ).run(safeId);
3638
+
3639
+ db.prepare("DELETE FROM jobs WHERE unique_id = ?").run(safeId);
3640
+
3641
+ markStatsDirty();
3642
+ console.error(`[DB] moveSingleJobToBase: ${safeId} → jobs_base`);
3643
+ return { ok: true };
3644
+ } catch (e) {
3645
+ return { ok: false, error: e.message };
3646
+ }
3647
+ }
3542
3648
  }
@@ -479,8 +479,23 @@ export function getPendingJobsUserUpdateCount() {
479
479
  .get().c;
480
480
  }
481
481
 
482
+ // raw_jobs 计数缓存:raw_jobs 表数据量大(千万级),COUNT(*) 全表扫描耗时长,
483
+ // 且该值变化频率低(仅在 batchUpdateUserInfo 移入非商家时变化),缓存 60 秒。
484
+ let _rawJobsCountCache = null;
485
+ let _rawJobsCountCacheTime = 0;
486
+ const RAW_JOBS_COUNT_CACHE_TTL = 60000; // 60 秒
487
+
482
488
  export function getRawJobsCount() {
489
+ const now = Date.now();
490
+ if (
491
+ _rawJobsCountCache !== null &&
492
+ now - _rawJobsCountCacheTime < RAW_JOBS_COUNT_CACHE_TTL
493
+ ) {
494
+ return _rawJobsCountCache;
495
+ }
483
496
  const db = getDb();
484
497
  if (!db) return 0;
485
- return db.prepare("SELECT COUNT(*) as c FROM raw_jobs").get().c;
498
+ _rawJobsCountCache = db.prepare("SELECT COUNT(*) as c FROM raw_jobs").get().c;
499
+ _rawJobsCountCacheTime = now;
500
+ return _rawJobsCountCache;
486
501
  }
@@ -367,6 +367,7 @@ export function initDb(filePath) {
367
367
  round INTEGER NOT NULL DEFAULT 1,
368
368
  strategy TEXT,
369
369
  tags_generated TEXT NOT NULL DEFAULT '[]',
370
+ tags_raw TEXT,
370
371
  tags_added INTEGER DEFAULT 0,
371
372
  created_at TEXT NOT NULL DEFAULT (datetime('now')),
372
373
  productive_sample TEXT,
@@ -375,6 +376,10 @@ export function initDb(filePath) {
375
376
  avg_dead_score REAL DEFAULT 0
376
377
  )
377
378
  `);
379
+ // 迁移:tags_raw 列(用于记录 LLM 原始生成的 tag,去重前)
380
+ try {
381
+ db.exec("ALTER TABLE discover_log ADD COLUMN tags_raw TEXT");
382
+ } catch {}
378
383
 
379
384
  // ===== reflection_rules 表(反思闭环:LLM 自我学习规则) =====
380
385
  db.exec(`
@@ -61,11 +61,7 @@ export function getDashboardStatsFromDb(targetLocations = []) {
61
61
  COALESCE(location_created, '未知') as country,
62
62
  COUNT(*) as count,
63
63
  SUM(CASE
64
- WHEN tt_seller = 1 AND verified = 0 ${
65
- targetLocations.length
66
- ? `AND location_created IN (${targetPlaceholders})`
67
- : "AND 1 = 0"
68
- }
64
+ WHEN tt_seller = 1 AND verified = 0
69
65
  THEN 1 ELSE 0 END) as targetCount
70
66
  FROM jobs
71
67
  WHERE status = 'done'
@@ -73,7 +69,7 @@ export function getDashboardStatsFromDb(targetLocations = []) {
73
69
  ORDER BY count DESC
74
70
  `,
75
71
  )
76
- .all(...targetParams);
72
+ .all();
77
73
 
78
74
  const targetCountryStats = targetLocations.length
79
75
  ? db
@@ -95,9 +91,17 @@ export function getDashboardStatsFromDb(targetLocations = []) {
95
91
 
96
92
  const tagCount = db.prepare("SELECT COUNT(*) as total FROM tags").get().total;
97
93
 
94
+ const commentTaskCount = db
95
+ .prepare(
96
+ `SELECT COUNT(*) as total FROM videos
97
+ WHERE user_update_count IS NULL OR user_update_count <= 0`,
98
+ )
99
+ .get().total;
100
+
98
101
  return {
99
102
  totalUsers: aggregateRow.total,
100
103
  tagCount,
104
+ commentTaskCount,
101
105
  rawJobs: getRawJobsCount(),
102
106
  dbTotalUsers: getUserDbCount(),
103
107
  jobsTotal: aggregateRow.total,