tt-help-cli-ycl 1.4.17 → 1.4.19

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "tt-help-cli-ycl",
3
- "version": "1.4.17",
3
+ "version": "1.4.19",
4
4
  "description": "TikTok user & video data scraper - extract ttSeller, verified, locationCreated from HTML source",
5
5
  "type": "module",
6
6
  "bin": {
@@ -646,11 +646,13 @@ async function runAutoMode(options) {
646
646
  } catch (err) {
647
647
  console.error(` × 获取任务失败: ${err.message}`);
648
648
  consecutiveErrors++;
649
- if (consecutiveErrors > 10) {
650
- console.error(" × 连续获取失败 >10 → 退出");
651
- await shutdown("too many errors");
649
+ if (consecutiveErrors > 0 && consecutiveErrors % 30 === 0) {
650
+ console.error(
651
+ ` ⚠ 已连续获取失败 ${consecutiveErrors} 次,继续等待服务恢复...`,
652
+ );
652
653
  }
653
- await delay(1000, actualInterval * 1000);
654
+ const waitMs = Math.min(consecutiveErrors * 1000, 60000);
655
+ await delay(waitMs, waitMs + 1000);
654
656
  continue;
655
657
  }
656
658
  consecutiveErrors = 0;
package/src/cli/tag.js CHANGED
@@ -532,7 +532,9 @@ async function scoreSingleTag(
532
532
  return result;
533
533
  }
534
534
  // 非 tag_not_found 的 0 视频 → 抛异常让其走 error 重试路径,避免误标 dead
535
- throw new Error("no videos returned — possible network/captcha/slow-load, will retry");
535
+ throw new Error(
536
+ "no videos returned — possible network/captcha/slow-load, will retry",
537
+ );
536
538
  }
537
539
 
538
540
  // Step 2: 补充国家信息
@@ -1063,9 +1065,7 @@ export async function handleScoreAll(parsed) {
1063
1065
  if (!videos || videos.length === 0) {
1064
1066
  const deadSec = ((Date.now() - fetchStart) / 1000).toFixed(1);
1065
1067
  if (tagResult.error === "tag_not_found") {
1066
- log(
1067
- ` ⚠️ 标签不存在 (${deadSec}s),标记 dead`,
1068
- );
1068
+ log(` ⚠️ 标签不存在 (${deadSec}s),标记 dead`);
1069
1069
  result.status = "dead";
1070
1070
  result.error = "tag_not_found";
1071
1071
  await reportToServer(baseUrl, result, clientId, clientMeta);
@@ -1076,9 +1076,7 @@ export async function handleScoreAll(parsed) {
1076
1076
  continue;
1077
1077
  }
1078
1078
  // 非 tag_not_found 的 0 视频 → 抛异常让其走 error 重试路径,避免误标 dead
1079
- log(
1080
- ` ⚠️ 无视频 (${deadSec}s),可能网络/风控/加载慢 → 抛异常重试`,
1081
- );
1079
+ log(` ⚠️ 无视频 (${deadSec}s),可能网络/风控/加载慢 → 抛异常重试`);
1082
1080
  throw new Error("no videos returned (network/captcha/slow-load)");
1083
1081
  }
1084
1082
 
package/src/cli/watch.js CHANGED
@@ -5,16 +5,26 @@ import { startWatchServer, openBrowser } from "../watch/server.js";
5
5
 
6
6
  export async function handleWatch(options) {
7
7
  const dataAnchor = options.dataAnchor || options.outputFile;
8
- const { watchPort, llmRefill, llmRefillMinScore, llmRefillSampleSize } =
9
- options;
8
+ const {
9
+ watchPort,
10
+ llmRefill,
11
+ llmRefillMinScore,
12
+ llmRefillSampleSize,
13
+ llmDiscover,
14
+ } = options;
10
15
 
11
16
  if (!dataAnchor) {
12
- console.error("用法: tt-help watch -o <db路径> [-p 端口] [--llm-refill]");
17
+ console.error(
18
+ "用法: tt-help watch -o <db路径> [-p 端口] [--llm-refill] [--no-llm-discover]",
19
+ );
13
20
  console.error(" tt-help watch -o data/result.db");
14
21
  console.error(" tt-help watch -o data/result.db -p 8080");
15
22
  console.error(
16
23
  " tt-help watch -o data/result.db --llm-refill --llm-refill-min 60",
17
24
  );
25
+ console.error(
26
+ " tt-help watch -o data/result.db --no-llm-discover --no-llm-refill",
27
+ );
18
28
  process.exit(1);
19
29
  }
20
30
 
@@ -37,7 +47,8 @@ export async function handleWatch(options) {
37
47
  dataAnchor,
38
48
  watchPort,
39
49
  store,
40
- storeOptions,
50
+ { ...storeOptions, llmDiscover: llmDiscover ?? true },
51
+ options.watchPassword,
41
52
  );
42
53
  openBrowser(port);
43
54
 
package/src/lib/args.js CHANGED
@@ -347,9 +347,11 @@ function parseInfoArgs(args) {
347
347
  function parseWatchArgs(args) {
348
348
  let dataAnchor = "./result.db";
349
349
  let watchPort = 3001;
350
+ let watchPassword = null;
350
351
  let llmRefill = true;
351
352
  let llmRefillMinScore = null;
352
353
  let llmRefillSampleSize = null;
354
+ let llmDiscover = true;
353
355
 
354
356
  for (let i = 0; i < args.length; i++) {
355
357
  const arg = args[i];
@@ -357,6 +359,8 @@ function parseWatchArgs(args) {
357
359
  dataAnchor = args[++i];
358
360
  } else if (arg === "-p") {
359
361
  watchPort = parseInt(args[++i]) || 3001;
362
+ } else if (arg === "-P" || arg === "--password") {
363
+ watchPassword = args[++i] || null;
360
364
  } else if (arg === "--llm-refill") {
361
365
  llmRefill = true;
362
366
  } else if (arg === "--no-llm-refill") {
@@ -365,6 +369,10 @@ function parseWatchArgs(args) {
365
369
  llmRefillMinScore = parseInt(args[++i]) || 60;
366
370
  } else if (arg === "--llm-refill-sample") {
367
371
  llmRefillSampleSize = parseInt(args[++i]) || 100;
372
+ } else if (arg === "--llm-discover") {
373
+ llmDiscover = true;
374
+ } else if (arg === "--no-llm-discover") {
375
+ llmDiscover = false;
368
376
  }
369
377
  }
370
378
 
@@ -373,9 +381,11 @@ function parseWatchArgs(args) {
373
381
  outputFile: dataAnchor,
374
382
  dataAnchor,
375
383
  watchPort,
384
+ watchPassword,
376
385
  llmRefill,
377
386
  llmRefillMinScore,
378
387
  llmRefillSampleSize,
388
+ llmDiscover,
379
389
  urls: [],
380
390
  outputFormat: "json",
381
391
  exploreCount: 0,
@@ -757,6 +767,8 @@ function parseTagArgs(args) {
757
767
  let scoreTag = null;
758
768
  let scoreCountries = null;
759
769
  let scorePort = 9222;
770
+ // 延迟路由:--port 的值先暂存,等循环结束后根据已确定的 subcommand 分配
771
+ let rawTagPort = null;
760
772
 
761
773
  for (let i = 0; i < args.length; i++) {
762
774
  const arg = args[i];
@@ -798,13 +810,9 @@ function parseTagArgs(args) {
798
810
  .map((s) => s.trim().toUpperCase())
799
811
  .filter(Boolean);
800
812
  } else if (arg === "--port") {
801
- if (isScoreAll) {
802
- scoreAllPort = parseInt(args[++i]) || 7222;
803
- } else if (isRefresh) {
804
- refreshPort = parseInt(args[++i]) || 8222;
805
- } else {
806
- scorePort = parseInt(args[++i]) || 9222;
807
- }
813
+ // 延迟路由:暂存端口值,等循环结束后根据 subcommand 分配
814
+ // 避免 --port 出现在 refresh/score-all 之前时被错误分配到其他变量
815
+ rawTagPort = parseInt(args[++i]);
808
816
  } else if (arg === "--proxy") {
809
817
  scoreProxy = args[++i];
810
818
  } else if (arg === "-p" || arg === "--prompt") {
@@ -842,6 +850,13 @@ function parseTagArgs(args) {
842
850
  }
843
851
  }
844
852
 
853
+ // ── 延迟路由:将 --port 值分配到正确的 subcommand 变量 ──
854
+ if (rawTagPort !== null && !isNaN(rawTagPort)) {
855
+ if (isScoreAll) scoreAllPort = rawTagPort;
856
+ else if (isRefresh) refreshPort = rawTagPort;
857
+ else scorePort = rawTagPort;
858
+ }
859
+
845
860
  if (isDiscover) {
846
861
  return {
847
862
  subcommand: "tag-discover",
@@ -177,10 +177,20 @@ const HELP_TEXT = [
177
177
  " 示例: tt-help attach -p 5 -i 10",
178
178
  " tt-help attach -c PL,DE,FR -p 5",
179
179
  "",
180
- " tt-help watch -o <db路径> [-p 端口]",
180
+ " tt-help watch -o <db路径> [-p 端口] [--no-llm-refill] [--no-llm-discover]",
181
181
  " 启动监控服务;运行期主数据仅来自 SQLite",
182
+ " 选项:",
183
+ " -p, --port <端口> 服务端口,默认 3000",
184
+ " -P, --password <密码> Web 面板登录密码",
185
+ " --llm-refill 启用 LLM 自动筛选毛料任务(默认开启)",
186
+ " --no-llm-refill 关闭 LLM 任务打分",
187
+ " --llm-discover 启用 LLM 自动发现标签(默认开启)",
188
+ " --no-llm-discover 关闭 LLM tag 发现",
189
+ " --llm-refill-min <分数> 最低入围分,默认 60",
190
+ " --llm-refill-sample <数量> 采样量,默认 100",
182
191
  " 示例: tt-help watch -o data/result.db",
183
192
  " tt-help watch -o data/result.db -p 8080",
193
+ " tt-help watch -o data/result.db --no-llm-discover --no-llm-refill",
184
194
  "",
185
195
  " tt-help open <端口号>",
186
196
  " 打开指定端口的浏览器,用于配置 TikTok 登录账户",
@@ -316,7 +326,7 @@ const HELP_TEXT = [
316
326
  ];
317
327
 
318
328
  const PUBLIC_HELP_HIDDEN_HEADERS = new Set([
319
- " tt-help watch -o <db路径> [-p 端口]",
329
+ " tt-help watch -o <db路径> [-p 端口] [--no-llm-refill] [--no-llm-discover]",
320
330
  " videostats <db路径> -p <并发数>",
321
331
  " db-import --db <db路径> [--users users.json] [--done done.json] [--videos videos.json]",
322
332
  ]);
@@ -100,6 +100,7 @@ import {
100
100
  getProductiveRefreshTask,
101
101
  claimRefreshTag,
102
102
  reportRefreshResult,
103
+ getRefreshStats,
103
104
  } from "./db-tags.js";
104
105
 
105
106
  // LLM 打分
@@ -555,6 +556,64 @@ function getModifiedTargetUsersOutsideLocations(targetLocations = []) {
555
556
  return { total: rows.length, countries };
556
557
  }
557
558
 
559
+ /**
560
+ * 目标商家刷新进度统计:按国家统计最近 N 小时内刷新过的 tt_seller 用户数。
561
+ * 返回格式与 getRefreshStats(tags)一致,便于前端复用渲染逻辑。
562
+ */
563
+ function getTargetRefreshStats(hours = 24) {
564
+ if (!getDb()) return { countries: [], total: 0, refreshed: 0 };
565
+
566
+ const targetLocations = DEFAULT_TARGET_LOCATIONS;
567
+ const placeholders = targetLocations.map(() => "?").join(", ");
568
+
569
+ // 按国家统计总数
570
+ const totalRows = getDb()
571
+ .prepare(
572
+ `
573
+ SELECT location_created as country, COUNT(*) as cnt
574
+ FROM jobs
575
+ WHERE tt_seller = 1
576
+ AND verified = 0
577
+ AND location_created IN (${placeholders})
578
+ GROUP BY location_created
579
+ ORDER BY cnt DESC
580
+ `,
581
+ )
582
+ .all(...targetLocations);
583
+
584
+ if (!totalRows.length) return { countries: [], total: 0, refreshed: 0 };
585
+
586
+ // 按国家统计最近 N 小时内刷新过的数量
587
+ const threshold = Date.now() - hours * 3600 * 1000;
588
+ const refreshedRows = getDb()
589
+ .prepare(
590
+ `
591
+ SELECT location_created as country, COUNT(*) as cnt
592
+ FROM jobs
593
+ WHERE tt_seller = 1
594
+ AND verified = 0
595
+ AND location_created IN (${placeholders})
596
+ AND refresh_time >= ?
597
+ GROUP BY location_created
598
+ `,
599
+ )
600
+ .all(...targetLocations, threshold);
601
+ const refreshedMap = new Map();
602
+ for (const r of refreshedRows) refreshedMap.set(r.country, r.cnt);
603
+
604
+ const results = [];
605
+ let totalAll = 0;
606
+ let refreshedAll = 0;
607
+ for (const r of totalRows) {
608
+ const refreshed = refreshedMap.get(r.country) || 0;
609
+ totalAll += r.cnt;
610
+ refreshedAll += refreshed;
611
+ results.push({ country: r.country, total: r.cnt, refreshed });
612
+ }
613
+
614
+ return { countries: results, total: totalAll, refreshed: refreshedAll };
615
+ }
616
+
558
617
  export function createStore(filePath, options = {}) {
559
618
  if (!filePath) {
560
619
  throw new Error("createStore requires an explicit .db path");
@@ -1007,7 +1066,7 @@ export function createStore(filePath, options = {}) {
1007
1066
 
1008
1067
  if (normalizedLocations && normalizedLocations.length > 0) {
1009
1068
  conditions.push(
1010
- `UPPER(COALESCE(guessed_location, '')) IN (${normalizedLocations.map(() => "?").join(", ")})`,
1069
+ `(UPPER(COALESCE(guessed_location, '')) IN (${normalizedLocations.map(() => "?").join(", ")}) OR COALESCE(guessed_location, '') = '')`,
1011
1070
  );
1012
1071
  args.push(...normalizedLocations);
1013
1072
  }
@@ -1095,6 +1154,9 @@ export function createStore(filePath, options = {}) {
1095
1154
 
1096
1155
  offsetStore.set("_tag", tagOffset);
1097
1156
 
1157
+ // 让出事件循环,避免长时间阻塞监控页面及其他客户端请求
1158
+ await new Promise(setImmediate);
1159
+
1098
1160
  // ===== 第二步:一次性取 comment(全局,最多 llmCommentLimit 条,免打分)=====
1099
1161
  let commentOffset = offsetStore.get("_comment") || 0;
1100
1162
  const commentGlobalCount =
@@ -1146,6 +1208,9 @@ export function createStore(filePath, options = {}) {
1146
1208
 
1147
1209
  offsetStore.set("_comment", commentOffset);
1148
1210
 
1211
+ // 让出事件循环,避免长时间阻塞监控页面及其他客户端请求
1212
+ await new Promise(setImmediate);
1213
+
1149
1214
  // ===== 第三步:按国家取非 tag 非 comment,走 LLM 打分,直到合格数达到 llmLLMTarget =====
1150
1215
  for (const location of normalizedLocations) {
1151
1216
  if (allLLMQualified.length >= llmLLMTarget) break;
@@ -1157,7 +1222,7 @@ export function createStore(filePath, options = {}) {
1157
1222
  const llmCount =
1158
1223
  getDb()
1159
1224
  .prepare(
1160
- `SELECT COUNT(*) as c FROM raw_jobs WHERE ${whereSql} AND guessed_location = ? AND (sources NOT LIKE '%tag%' OR sources IS NULL) AND (sources NOT LIKE '%comment%' OR sources IS NULL)`,
1225
+ `SELECT COUNT(*) as c FROM raw_jobs WHERE ${whereSql} AND (guessed_location = ? OR COALESCE(guessed_location, '') = '') AND (sources NOT LIKE '%tag%' OR sources IS NULL) AND (sources NOT LIKE '%comment%' OR sources IS NULL)`,
1161
1226
  )
1162
1227
  .get(...locationArgs)?.c || 0;
1163
1228
 
@@ -1183,7 +1248,7 @@ export function createStore(filePath, options = {}) {
1183
1248
  const samples = getDb()
1184
1249
  .prepare(
1185
1250
  `
1186
- SELECT * FROM raw_jobs WHERE ${whereSql} AND guessed_location = ?
1251
+ SELECT * FROM raw_jobs WHERE ${whereSql} AND (guessed_location = ? OR COALESCE(guessed_location, '') = '')
1187
1252
  AND (sources NOT LIKE '%tag%' OR sources IS NULL)
1188
1253
  AND (sources NOT LIKE '%comment%' OR sources IS NULL)
1189
1254
  ORDER BY COALESCE(video_count, 0) DESC, created_at DESC
@@ -1194,6 +1259,9 @@ export function createStore(filePath, options = {}) {
1194
1259
 
1195
1260
  if (!samples.length) break;
1196
1261
 
1262
+ // 让出事件循环,LLM API 调用期间允许处理其他请求
1263
+ await new Promise(setImmediate);
1264
+
1197
1265
  const scores = await scoreJobsBatch(
1198
1266
  samples,
1199
1267
  DEFAULT_TARGET_LOCATIONS,
@@ -1234,6 +1302,9 @@ export function createStore(filePath, options = {}) {
1234
1302
  `[data-store] LLM 调度完成: tag ${allTagQualified.length} + comment ${allCommentQualified.length} + LLM ${allLLMQualified.length} = 共 ${qualified.length} 条`,
1235
1303
  );
1236
1304
 
1305
+ // 让出事件循环,避免事务写入阻塞监控页面
1306
+ await new Promise(setImmediate);
1307
+
1237
1308
  // 移动符合条件的记录
1238
1309
  const placeholders = qualified.map(() => "?").join(", ");
1239
1310
  const moveTxn = getDb().transaction(() => {
@@ -1348,7 +1419,7 @@ export function createStore(filePath, options = {}) {
1348
1419
  const currentLimit = Math.max(0, Math.min(quota, safeLimit - moved));
1349
1420
  if (!currentLimit) continue;
1350
1421
 
1351
- const locationWhere = `${whereSql} AND UPPER(COALESCE(guessed_location, '')) = ?`;
1422
+ const locationWhere = `${whereSql} AND (UPPER(COALESCE(guessed_location, '')) = ? OR COALESCE(guessed_location, '') = '')`;
1352
1423
  const locationArgs = [...args, location];
1353
1424
 
1354
1425
  getDb()
@@ -1475,7 +1546,7 @@ export function createStore(filePath, options = {}) {
1475
1546
  if (!group) return;
1476
1547
  if (group.type === "include") {
1477
1548
  where.push(
1478
- `UPPER(COALESCE(guessed_location, '')) IN (${group.values.map(() => "?").join(", ")})`,
1549
+ `(UPPER(COALESCE(guessed_location, '')) IN (${group.values.map(() => "?").join(", ")}) OR COALESCE(guessed_location, '') = '')`,
1479
1550
  );
1480
1551
  args.push(...group.values);
1481
1552
  return;
@@ -1546,7 +1617,7 @@ export function createStore(filePath, options = {}) {
1546
1617
  }
1547
1618
  if (normalizedLocations.length > 0) {
1548
1619
  where.push(
1549
- `UPPER(COALESCE(guessed_location, '')) IN (${normalizedLocations.map(() => "?").join(", ")})`,
1620
+ `(UPPER(COALESCE(guessed_location, '')) IN (${normalizedLocations.map(() => "?").join(", ")}) OR COALESCE(guessed_location, '') = '')`,
1550
1621
  );
1551
1622
  args.push(...normalizedLocations);
1552
1623
  }
@@ -1944,7 +2015,7 @@ export function createStore(filePath, options = {}) {
1944
2015
  if (!group) return;
1945
2016
  if (group.type === "include") {
1946
2017
  where.push(
1947
- `UPPER(COALESCE(guessed_location, '')) IN (${group.values.map(() => "?").join(", ")})`,
2018
+ `(UPPER(COALESCE(guessed_location, '')) IN (${group.values.map(() => "?").join(", ")}) OR COALESCE(guessed_location, '') = '')`,
1948
2019
  );
1949
2020
  args.push(...group.values);
1950
2021
  return;
@@ -2012,7 +2083,7 @@ export function createStore(filePath, options = {}) {
2012
2083
  }
2013
2084
  if (normalizedLocations.length > 0) {
2014
2085
  where.push(
2015
- `UPPER(COALESCE(guessed_location, '')) IN (${normalizedLocations.map(() => "?").join(", ")})`,
2086
+ `(UPPER(COALESCE(guessed_location, '')) IN (${normalizedLocations.map(() => "?").join(", ")}) OR COALESCE(guessed_location, '') = '')`,
2016
2087
  );
2017
2088
  args.push(...normalizedLocations);
2018
2089
  }
@@ -3362,6 +3433,8 @@ export function createStore(filePath, options = {}) {
3362
3433
  getActiveClients,
3363
3434
  moveSellerJobsToBase, // 将 jobs/raw_jobs 中 tt_seller=1 且 video_count=0 的记录移动到 jobs_base
3364
3435
  moveRestrictedJobsToBase, // 将 jobs 中 status='restricted' 的记录移动到 jobs_base
3436
+ moveNoCountryJobsToBase, // 将 jobs 中 status='done' 且无国家数据的记录移动到 jobs_base
3437
+ moveSingleJobToBase, // 将单个 job 移动到 jobs_base 并重置(目标商家页面用)
3365
3438
  registerVideos,
3366
3439
  getVideo,
3367
3440
  getVideos,
@@ -3389,6 +3462,8 @@ export function createStore(filePath, options = {}) {
3389
3462
  getProductiveRefreshTask,
3390
3463
  claimRefreshTag,
3391
3464
  reportRefreshResult,
3465
+ getRefreshStats,
3466
+ getTargetRefreshStats,
3392
3467
  data,
3393
3468
  };
3394
3469
 
@@ -3556,4 +3631,127 @@ export function createStore(filePath, options = {}) {
3556
3631
  markStatsDirty();
3557
3632
  return { ok: true, moved: deleted.changes || 0 };
3558
3633
  }
3634
+
3635
+ // 将 jobs 中 status='done' 且无国家数据的记录移动到 jobs_base
3636
+ function moveNoCountryJobsToBase() {
3637
+ const db = getDb();
3638
+ if (!db) return { ok: false, error: "db not ready" };
3639
+
3640
+ const now = Date.now();
3641
+ const condition = `status = 'done'
3642
+ AND (guessed_location IS NULL OR guessed_location = '')
3643
+ AND (location_created IS NULL OR location_created = '')
3644
+ AND (confirmed_location IS NULL OR confirmed_location = '')`;
3645
+
3646
+ const countRow = db
3647
+ .prepare(`SELECT COUNT(*) as c FROM jobs WHERE ${condition}`)
3648
+ .get();
3649
+ if (!countRow?.c) return { ok: true, moved: 0 };
3650
+
3651
+ db.prepare(
3652
+ `
3653
+ INSERT OR REPLACE INTO jobs_base (
3654
+ unique_id, nickname, sources, pinned, no_video, user_update_count,
3655
+ tt_seller, verified, video_count, comment_count, guessed_location,
3656
+ location_created, follower_count, following_count, heart_count,
3657
+ refresh_time, region, signature, sec_uid, status_code,
3658
+ latest_video_time, bio_link, status, restricted, error, claimed_by,
3659
+ claimed_at, processed, processed_at, created_at, updated_at
3660
+ )
3661
+ SELECT
3662
+ unique_id, nickname, sources, pinned, no_video, 0,
3663
+ CASE WHEN tt_seller = 0 THEN NULL ELSE tt_seller END,
3664
+ verified, video_count, comment_count, guessed_location,
3665
+ location_created, follower_count, following_count, heart_count,
3666
+ refresh_time, region, signature, sec_uid, status_code,
3667
+ latest_video_time, bio_link,
3668
+ 'pending', 0, NULL, NULL, NULL, 0, NULL,
3669
+ COALESCE(created_at, ?), ?
3670
+ FROM jobs WHERE ${condition}
3671
+ `,
3672
+ ).run(now, now);
3673
+
3674
+ const deleted = db.prepare(`DELETE FROM jobs WHERE ${condition}`).run();
3675
+
3676
+ markStatsDirty();
3677
+ return {
3678
+ ok: true,
3679
+ moved: deleted.changes || 0,
3680
+ total: countRow.c,
3681
+ };
3682
+ }
3683
+
3684
+ // 将单个 job 移动到 jobs_base(用于目标商家页面"移到base重走流程")
3685
+ function moveSingleJobToBase(uniqueId) {
3686
+ const db = getDb();
3687
+ if (!db) return { ok: false, error: "db not ready" };
3688
+
3689
+ const safeId = String(uniqueId).trim();
3690
+ if (!safeId) return { ok: false, error: "invalid uniqueId" };
3691
+
3692
+ const existing = db
3693
+ .prepare("SELECT * FROM jobs WHERE unique_id = ?")
3694
+ .get(safeId);
3695
+ if (!existing) return { ok: false, error: "user not found in jobs" };
3696
+
3697
+ const COLUMNS = [
3698
+ "unique_id",
3699
+ "nickname",
3700
+ "status",
3701
+ "sources",
3702
+ "claimed_by",
3703
+ "claimed_at",
3704
+ "error",
3705
+ "pinned",
3706
+ "no_video",
3707
+ "restricted",
3708
+ "user_update_count",
3709
+ "tt_seller",
3710
+ "verified",
3711
+ "video_count",
3712
+ "comment_count",
3713
+ "guessed_location",
3714
+ "location_created",
3715
+ "confirmed_location",
3716
+ "modified_at",
3717
+ "follower_count",
3718
+ "following_count",
3719
+ "heart_count",
3720
+ "refresh_time",
3721
+ "processed",
3722
+ "processed_at",
3723
+ "created_at",
3724
+ "updated_at",
3725
+ "region",
3726
+ "signature",
3727
+ "bio_link",
3728
+ "sec_uid",
3729
+ "status_code",
3730
+ "latest_video_time",
3731
+ ];
3732
+ const cols = COLUMNS.join(",");
3733
+
3734
+ try {
3735
+ db.prepare(
3736
+ `INSERT OR REPLACE INTO jobs_base (${cols})
3737
+ SELECT ${cols} FROM jobs WHERE unique_id = ?`,
3738
+ ).run(safeId);
3739
+
3740
+ // 重置 user_update_count,保留 tt_seller,状态改为 pending
3741
+ db.prepare(
3742
+ `UPDATE jobs_base SET user_update_count = 0, status = 'pending',
3743
+ claimed_by = NULL, claimed_at = NULL, processed = 0, processed_at = NULL,
3744
+ error = NULL, restricted = 0
3745
+ WHERE unique_id = ?`,
3746
+ ).run(safeId);
3747
+
3748
+ db.prepare("DELETE FROM jobs WHERE unique_id = ?").run(safeId);
3749
+
3750
+ markStatsDirty();
3751
+ console.error(`[DB] moveSingleJobToBase: ${safeId} → jobs_base`);
3752
+ return { ok: true };
3753
+ } catch (e) {
3754
+ return { ok: false, error: e.message };
3755
+ }
3756
+ }
3559
3757
  }
@@ -479,8 +479,23 @@ export function getPendingJobsUserUpdateCount() {
479
479
  .get().c;
480
480
  }
481
481
 
482
+ // raw_jobs 计数缓存:raw_jobs 表数据量大(千万级),COUNT(*) 全表扫描耗时长,
483
+ // 且该值变化频率低(仅在 batchUpdateUserInfo 移入非商家时变化),缓存 60 秒。
484
+ let _rawJobsCountCache = null;
485
+ let _rawJobsCountCacheTime = 0;
486
+ const RAW_JOBS_COUNT_CACHE_TTL = 60000; // 60 秒
487
+
482
488
  export function getRawJobsCount() {
489
+ const now = Date.now();
490
+ if (
491
+ _rawJobsCountCache !== null &&
492
+ now - _rawJobsCountCacheTime < RAW_JOBS_COUNT_CACHE_TTL
493
+ ) {
494
+ return _rawJobsCountCache;
495
+ }
483
496
  const db = getDb();
484
497
  if (!db) return 0;
485
- return db.prepare("SELECT COUNT(*) as c FROM raw_jobs").get().c;
498
+ _rawJobsCountCache = db.prepare("SELECT COUNT(*) as c FROM raw_jobs").get().c;
499
+ _rawJobsCountCacheTime = now;
500
+ return _rawJobsCountCache;
486
501
  }
@@ -367,6 +367,7 @@ export function initDb(filePath) {
367
367
  round INTEGER NOT NULL DEFAULT 1,
368
368
  strategy TEXT,
369
369
  tags_generated TEXT NOT NULL DEFAULT '[]',
370
+ tags_raw TEXT,
370
371
  tags_added INTEGER DEFAULT 0,
371
372
  created_at TEXT NOT NULL DEFAULT (datetime('now')),
372
373
  productive_sample TEXT,
@@ -375,6 +376,10 @@ export function initDb(filePath) {
375
376
  avg_dead_score REAL DEFAULT 0
376
377
  )
377
378
  `);
379
+ // 迁移:tags_raw 列(用于记录 LLM 原始生成的 tag,去重前)
380
+ try {
381
+ db.exec("ALTER TABLE discover_log ADD COLUMN tags_raw TEXT");
382
+ } catch {}
378
383
 
379
384
  // ===== reflection_rules 表(反思闭环:LLM 自我学习规则) =====
380
385
  db.exec(`
@@ -61,11 +61,7 @@ export function getDashboardStatsFromDb(targetLocations = []) {
61
61
  COALESCE(location_created, '未知') as country,
62
62
  COUNT(*) as count,
63
63
  SUM(CASE
64
- WHEN tt_seller = 1 AND verified = 0 ${
65
- targetLocations.length
66
- ? `AND location_created IN (${targetPlaceholders})`
67
- : "AND 1 = 0"
68
- }
64
+ WHEN tt_seller = 1 AND verified = 0
69
65
  THEN 1 ELSE 0 END) as targetCount
70
66
  FROM jobs
71
67
  WHERE status = 'done'
@@ -73,7 +69,7 @@ export function getDashboardStatsFromDb(targetLocations = []) {
73
69
  ORDER BY count DESC
74
70
  `,
75
71
  )
76
- .all(...targetParams);
72
+ .all();
77
73
 
78
74
  const targetCountryStats = targetLocations.length
79
75
  ? db