tt-help-cli-ycl 1.4.166 → 1.4.168
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/scripts/deploy-node.sh +83 -0
- package/scripts/diag-claim-branches.js +72 -0
- package/scripts/test-claim-fix.mjs +216 -0
- package/scripts/test-precise-index.mjs +114 -0
- package/scripts/test-stats-smoke.mjs +43 -0
- package/src/clash/checker.js +96 -14
- package/src/cli/attach.js +67 -19
- package/src/lib/browser/cdp.js +182 -27
- package/src/watch/data-store.js +159 -37
- package/src/watch/server.js +102 -38
package/src/watch/data-store.js
CHANGED
|
@@ -657,9 +657,23 @@ function getPreciseMerchantsFromDb(targetLocations = [], options = {}) {
|
|
|
657
657
|
searchParams.push(`%${search}%`, `%${search}%`);
|
|
658
658
|
}
|
|
659
659
|
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
660
|
+
// INDEXED BY 强制走精准商家 partial 覆盖索引(候选仅数百行)。无统计信息时
|
|
661
|
+
// 优化器会误选 idx_jobs_redo_target 全量商家回表(127 万行实测 COUNT 2.6s/次,
|
|
662
|
+
// 页面 COUNT+LIST 合计 5~6s)。索引不存在的环境(老库未重启)抛
|
|
663
|
+
// "no such index",自动回退普通查询
|
|
664
|
+
let totalRow;
|
|
665
|
+
try {
|
|
666
|
+
totalRow = getDb()
|
|
667
|
+
.prepare(
|
|
668
|
+
`SELECT COUNT(*) AS n FROM jobs INDEXED BY idx_jobs_precise_merchants WHERE ${where}${searchSql}`,
|
|
669
|
+
)
|
|
670
|
+
.get(...targetLocations, ...searchParams);
|
|
671
|
+
} catch (e) {
|
|
672
|
+
if (!/no such index/i.test(e.message)) throw e;
|
|
673
|
+
totalRow = getDb()
|
|
674
|
+
.prepare(`SELECT COUNT(*) AS n FROM jobs WHERE ${where}${searchSql}`)
|
|
675
|
+
.get(...targetLocations, ...searchParams);
|
|
676
|
+
}
|
|
663
677
|
const total = totalRow?.n || 0;
|
|
664
678
|
|
|
665
679
|
const selectSql = `
|
|
@@ -699,21 +713,47 @@ function getPreciseMerchantsFromDb(targetLocations = [], options = {}) {
|
|
|
699
713
|
ORDER BY product_sold_count DESC, unique_id ASC
|
|
700
714
|
`;
|
|
701
715
|
|
|
716
|
+
// 强制 partial 索引(同 COUNT,注释见上);"FROM jobs" 在 selectSql 中仅出现一次
|
|
717
|
+
const listSql = (indexed) =>
|
|
718
|
+
`${selectSql.replace(
|
|
719
|
+
"FROM jobs",
|
|
720
|
+
indexed ? "FROM jobs INDEXED BY idx_jobs_precise_merchants" : "FROM jobs",
|
|
721
|
+
)}`;
|
|
722
|
+
|
|
702
723
|
if (limit !== undefined) {
|
|
703
724
|
const safeLimit = Math.max(Math.min(Math.floor(limit), 1000), 1);
|
|
704
725
|
const safeOffset = Math.max(Math.floor(offset || 0), 0);
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
726
|
+
let rows;
|
|
727
|
+
try {
|
|
728
|
+
rows = getDb()
|
|
729
|
+
.prepare(`${listSql(true)} LIMIT ? OFFSET ?`)
|
|
730
|
+
.all(...targetLocations, ...searchParams, safeLimit, safeOffset);
|
|
731
|
+
} catch (e) {
|
|
732
|
+
if (!/no such index/i.test(e.message)) throw e;
|
|
733
|
+
rows = getDb()
|
|
734
|
+
.prepare(`${listSql(false)} LIMIT ? OFFSET ?`)
|
|
735
|
+
.all(...targetLocations, ...searchParams, safeLimit, safeOffset);
|
|
736
|
+
}
|
|
737
|
+
return {
|
|
738
|
+
total,
|
|
739
|
+
limit: safeLimit,
|
|
740
|
+
offset: safeOffset,
|
|
741
|
+
users: rows.map(mapJobRow),
|
|
742
|
+
};
|
|
710
743
|
}
|
|
711
744
|
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
745
|
+
let allRows;
|
|
746
|
+
try {
|
|
747
|
+
allRows = getDb()
|
|
748
|
+
.prepare(listSql(true))
|
|
749
|
+
.all(...targetLocations, ...searchParams);
|
|
750
|
+
} catch (e) {
|
|
751
|
+
if (!/no such index/i.test(e.message)) throw e;
|
|
752
|
+
allRows = getDb()
|
|
753
|
+
.prepare(listSql(false))
|
|
754
|
+
.all(...targetLocations, ...searchParams);
|
|
755
|
+
}
|
|
756
|
+
return { total, users: allRows.map(mapJobRow) };
|
|
717
757
|
}
|
|
718
758
|
|
|
719
759
|
/**
|
|
@@ -1260,6 +1300,18 @@ export function createStore(filePath, options = {}) {
|
|
|
1260
1300
|
return data;
|
|
1261
1301
|
}
|
|
1262
1302
|
|
|
1303
|
+
// 空级跳过记忆(claimNextJob 用):key = "level|登录|国家列表",value = 首次全
|
|
1304
|
+
// miss 时间。某来源级(含 pinned)全部分支 miss 后,TTL 内跳过该级,避免每次
|
|
1305
|
+
// 领任务空跑
|
|
1306
|
+
const levelEmptyCache = new Map();
|
|
1307
|
+
|
|
1308
|
+
// 任务增删/固定等使空池记忆失效的操作调用(前缀匹配,如 "pinned|")
|
|
1309
|
+
function invalidateClaimEmptyCache(prefix) {
|
|
1310
|
+
for (const key of levelEmptyCache.keys()) {
|
|
1311
|
+
if (key.startsWith(prefix)) levelEmptyCache.delete(key);
|
|
1312
|
+
}
|
|
1313
|
+
}
|
|
1314
|
+
|
|
1263
1315
|
async function claimNextJob(
|
|
1264
1316
|
userId,
|
|
1265
1317
|
expireMs = 5 * 60 * 1000,
|
|
@@ -1328,14 +1380,22 @@ export function createStore(filePath, options = {}) {
|
|
|
1328
1380
|
: "(status_code IS NULL OR status_code = 0)";
|
|
1329
1381
|
const notSellerWhere = loggedIn ? "" : "AND COALESCE(tt_seller, 0) != 1";
|
|
1330
1382
|
|
|
1331
|
-
|
|
1383
|
+
// 单国等值分支:等值条件命中 idx_tq_claim_fast 国家表达式列,点查毫秒级。
|
|
1384
|
+
// 勿与空位置 OR 合写——OR 会让国家列失效退化为 fallback 全走
|
|
1385
|
+
//(2026-10-09 生产实测 miss 单分支 2.2s)
|
|
1386
|
+
const locIn = (list) => {
|
|
1332
1387
|
if (!list || list.length === 0) return null;
|
|
1333
1388
|
const ph = list.map(() => "?").join(", ");
|
|
1334
|
-
|
|
1335
|
-
|
|
1336
|
-
:
|
|
1337
|
-
|
|
1389
|
+
return {
|
|
1390
|
+
sql: `UPPER(COALESCE(guessed_location, '')) IN (${ph})`,
|
|
1391
|
+
params: [...list],
|
|
1392
|
+
};
|
|
1338
1393
|
};
|
|
1394
|
+
// 空位置等值分支:同样走 fast 索引点查(实测 9ms)
|
|
1395
|
+
const locEmpty = () => ({
|
|
1396
|
+
sql: `UPPER(COALESCE(guessed_location, '')) = ''`,
|
|
1397
|
+
params: [],
|
|
1398
|
+
});
|
|
1339
1399
|
const locNotIn = (list) => {
|
|
1340
1400
|
if (!list || list.length === 0) return null;
|
|
1341
1401
|
const ph = list.map(() => "?").join(", ");
|
|
@@ -1366,16 +1426,30 @@ export function createStore(filePath, options = {}) {
|
|
|
1366
1426
|
branches.push({ conditions, params });
|
|
1367
1427
|
};
|
|
1368
1428
|
|
|
1369
|
-
//
|
|
1370
|
-
|
|
1371
|
-
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
// prio 1: pinned
|
|
1375
|
-
|
|
1376
|
-
|
|
1377
|
-
|
|
1378
|
-
);
|
|
1429
|
+
// 空级跳过记忆 TTL:池子空了没必要每次领任务都空跑全量分支
|
|
1430
|
+
//(2026-09-29 实测 seed 空级每次烧 1.4s CPU;2026-10-09 实测 pinned 空池
|
|
1431
|
+
// 每条分支全走 32 万行 2.1s)。TTL 过期后恢复探测一次——补进新任务即可自动恢复。
|
|
1432
|
+
const LEVEL_EMPTY_TTL = 60 * 1000;
|
|
1433
|
+
|
|
1434
|
+
// prio 0/1: pinned + hasVideo / pinned。pinned=1 无可用排序索引(列表索引
|
|
1435
|
+
// 列序不匹配 ORDER BY),池空时每条分支必全走确认 miss 且原本不在空级缓存内,
|
|
1436
|
+
// 每次领取固定烧 ~4.3s → 纳入空池记忆,TTL 内整体跳过;togglePin/resetJob 主动失效
|
|
1437
|
+
const pinnedEmptyKey = `pinned|${loggedIn ? 1 : 0}|${orderedCountries.join(",")}`;
|
|
1438
|
+
const pinnedEmptySince = levelEmptyCache.get(pinnedEmptyKey);
|
|
1439
|
+
const skipPinned =
|
|
1440
|
+
pinnedEmptySince != null && now - pinnedEmptySince < LEVEL_EMPTY_TTL;
|
|
1441
|
+
if (!skipPinned) {
|
|
1442
|
+
// prio 0: pinned + hasVideo
|
|
1443
|
+
addBranch(
|
|
1444
|
+
`status = 'pending' AND pinned = 1 AND video_count > 0 ${pinnedLocSql} AND ${statusCodeWhere} ${notSellerWhere}`,
|
|
1445
|
+
pinnedLocParams,
|
|
1446
|
+
);
|
|
1447
|
+
// prio 1: pinned (no video req)
|
|
1448
|
+
addBranch(
|
|
1449
|
+
`status = 'pending' AND pinned = 1 ${pinnedLocSql} AND ${statusCodeWhere} ${notSellerWhere}`,
|
|
1450
|
+
pinnedLocParams,
|
|
1451
|
+
);
|
|
1452
|
+
}
|
|
1379
1453
|
|
|
1380
1454
|
// prio 2-3: expired candidate(回收的超时任务优先)
|
|
1381
1455
|
if (expiredCandidate) {
|
|
@@ -1391,36 +1465,61 @@ export function createStore(filePath, options = {}) {
|
|
|
1391
1465
|
// 非 pinned 优先级:seed → seller → follow → other,每级 × 3 个 location tier × (hasVideo / noVideoReq)
|
|
1392
1466
|
const priorities = [
|
|
1393
1467
|
{
|
|
1468
|
+
name: "seed",
|
|
1394
1469
|
filter: `instr(COALESCE(sources, ''), '"seed"') > 0`,
|
|
1395
1470
|
loggedInOnly: false,
|
|
1396
1471
|
},
|
|
1397
1472
|
{
|
|
1473
|
+
name: "seller",
|
|
1398
1474
|
filter: `tt_seller = 1 AND verified = 0`,
|
|
1399
1475
|
loggedInOnly: true,
|
|
1400
1476
|
sellerAnyCountry: true,
|
|
1401
1477
|
},
|
|
1402
1478
|
{
|
|
1479
|
+
name: "follow",
|
|
1403
1480
|
filter: `(instr(COALESCE(sources, ''), '"following"') > 0 OR instr(COALESCE(sources, ''), '"follower"') > 0 OR instr(COALESCE(sources, ''), '"comment"') > 0)`,
|
|
1404
1481
|
loggedInOnly: false,
|
|
1405
1482
|
},
|
|
1406
|
-
{ filter: `1 = 1`, loggedInOnly: false },
|
|
1483
|
+
{ name: "other", filter: `1 = 1`, loggedInOnly: false },
|
|
1407
1484
|
];
|
|
1408
1485
|
|
|
1486
|
+
// 空级跳过记忆:某来源级(如 seed)全部国家分支 miss 后,记录时间;
|
|
1487
|
+
// TTL 内该级整体跳过。TTL 过期后恢复探测一次——refill 补进新任务即可自动恢复。
|
|
1488
|
+
// key 含登录态:未登录(过滤更严,无 209002/商家)miss 不代表登录端 miss,
|
|
1489
|
+
// 避免未登录轮污染登录端记忆
|
|
1490
|
+
const levelKeyOf = (name) =>
|
|
1491
|
+
`${name}|${loggedIn ? 1 : 0}|${orderedCountries.join(",")}`;
|
|
1492
|
+
for (const p of priorities) {
|
|
1493
|
+
if (p.loggedInOnly && !loggedIn) continue;
|
|
1494
|
+
const emptySince = levelEmptyCache.get(levelKeyOf(p.name));
|
|
1495
|
+
if (emptySince && now - emptySince < LEVEL_EMPTY_TTL) {
|
|
1496
|
+
p.skip = true; // 标记跳过,下面构建分支时跳过
|
|
1497
|
+
}
|
|
1498
|
+
}
|
|
1499
|
+
|
|
1409
1500
|
let basePrio = 4;
|
|
1410
1501
|
for (const p of priorities) {
|
|
1411
1502
|
if (p.loggedInOnly && !loggedIn) continue;
|
|
1412
|
-
|
|
1413
|
-
//
|
|
1414
|
-
//
|
|
1415
|
-
|
|
1416
|
-
|
|
1417
|
-
|
|
1503
|
+
if (p.skip) continue;
|
|
1504
|
+
// 逐国顺序分支:配置内国家按优先级顺序;未传子集时末尾补“配置外国家”兜底。
|
|
1505
|
+
// 空位置(loc='')拆为独立等值分支、只挂首国:原 OR 合写毁掉 fast 索引国家列
|
|
1506
|
+
// 退化为 fallback 全走(2026-10-09 实测 miss 2.2s/次),拆分后 IN 3ms / ='' 9ms。
|
|
1507
|
+
// 同层内首国等值严格优先于空位置(原为并集按粉丝竞争,属行为微调已确认可接受)
|
|
1508
|
+
const locationBranches = [];
|
|
1509
|
+
orderedCountries.forEach((loc, idx) => {
|
|
1510
|
+
locationBranches.push(locIn([loc]));
|
|
1511
|
+
if (idx === 0) locationBranches.push(locEmpty());
|
|
1512
|
+
});
|
|
1418
1513
|
if (normalizedLocations.length === 0)
|
|
1419
1514
|
locationBranches.push(locNotIn(orderedCountries));
|
|
1420
1515
|
// 商家级专用(sellerAnyCountry):国家对商家放开——末尾追加"任意国家"分支。
|
|
1421
1516
|
// 传了国家子集时对应国家的商家仍优先,领完再兜任意国家(含空位置商家);
|
|
1422
1517
|
// 未传国家时同理(配置内各国 → 配置外 → 任意)。其他来源级分支不受影响。
|
|
1423
1518
|
if (p.sellerAnyCountry) locationBranches.push(null); // null → 任意国家分支
|
|
1519
|
+
p._branchStart = branches.length; // 记录该级分支区间(用于判空)
|
|
1520
|
+
// 每个位置描述符产出 hasVideo + noVideoReq 两条分支(国家层为主序、
|
|
1521
|
+
// 是否要求视频为层内次序,与原版一致):首国等值 → 首国空位置 →
|
|
1522
|
+
// 其余配置内国家 → 配置外兜底 → 商家任意国家
|
|
1424
1523
|
let subPrio = 0;
|
|
1425
1524
|
for (const loc of locationBranches) {
|
|
1426
1525
|
// 商家任意国家分支:不限 guessed_location(含空位置的商家)
|
|
@@ -1453,12 +1552,16 @@ export function createStore(filePath, options = {}) {
|
|
|
1453
1552
|
);
|
|
1454
1553
|
subPrio++;
|
|
1455
1554
|
}
|
|
1555
|
+
p._branchEnd = branches.length;
|
|
1456
1556
|
basePrio += 2;
|
|
1457
1557
|
}
|
|
1458
1558
|
|
|
1459
|
-
//
|
|
1559
|
+
// 逐级执行:首个命中分支的最优行即领取结果(未命中继续下一优先级分支)。
|
|
1560
|
+
// 同时记录哪些来源级全部 miss → 写入 levelEmptyCache(下次领取 TTL 内直接跳过该级)
|
|
1460
1561
|
let selectedRow = null;
|
|
1461
|
-
|
|
1562
|
+
let bi = 0;
|
|
1563
|
+
for (; bi < branches.length; bi++) {
|
|
1564
|
+
const b = branches[bi];
|
|
1462
1565
|
selectedRow =
|
|
1463
1566
|
getDb()
|
|
1464
1567
|
.prepare(
|
|
@@ -1468,6 +1571,23 @@ export function createStore(filePath, options = {}) {
|
|
|
1468
1571
|
if (selectedRow) break;
|
|
1469
1572
|
}
|
|
1470
1573
|
|
|
1574
|
+
// 记录空级:分支按优先级顺序执行,命中所处级之前的所有来源级必然全 miss;
|
|
1575
|
+
// 空级写入 levelEmptyCache(TTL 内下次领取直接跳过),命中级清除记忆
|
|
1576
|
+
// bi = 命中分支下标(未命中则停在 branches.length)
|
|
1577
|
+
const hitIdx = bi;
|
|
1578
|
+
// pinned 空池记忆:仅当 pinned 分支本轮真实执行时更新(跳过轮保留原记忆)
|
|
1579
|
+
if (!skipPinned) {
|
|
1580
|
+
if (hitIdx >= 2) levelEmptyCache.set(pinnedEmptyKey, now);
|
|
1581
|
+
else levelEmptyCache.delete(pinnedEmptyKey);
|
|
1582
|
+
}
|
|
1583
|
+
for (const p of priorities) {
|
|
1584
|
+
if (p.loggedInOnly && !loggedIn) continue;
|
|
1585
|
+
if (p._branchStart == null) continue; // 被 skip 跳过的级无分支,保留原记忆
|
|
1586
|
+
const levelKey = levelKeyOf(p.name);
|
|
1587
|
+
if (hitIdx >= p._branchEnd) levelEmptyCache.set(levelKey, now);
|
|
1588
|
+
else levelEmptyCache.delete(levelKey);
|
|
1589
|
+
}
|
|
1590
|
+
|
|
1471
1591
|
if (selectedRow) {
|
|
1472
1592
|
getDb()
|
|
1473
1593
|
.prepare(
|
|
@@ -2253,6 +2373,7 @@ export function createStore(filePath, options = {}) {
|
|
|
2253
2373
|
.run(uniqueId);
|
|
2254
2374
|
}
|
|
2255
2375
|
markStatsDirty();
|
|
2376
|
+
invalidateClaimEmptyCache("pinned|");
|
|
2256
2377
|
return { saved: true };
|
|
2257
2378
|
}
|
|
2258
2379
|
|
|
@@ -2284,6 +2405,7 @@ export function createStore(filePath, options = {}) {
|
|
|
2284
2405
|
tableName,
|
|
2285
2406
|
);
|
|
2286
2407
|
if (ret.error) return { saved: false, error: ret.error };
|
|
2408
|
+
invalidateClaimEmptyCache("pinned|");
|
|
2287
2409
|
return { saved: true, pinned: nextPinned };
|
|
2288
2410
|
}
|
|
2289
2411
|
|
package/src/watch/server.js
CHANGED
|
@@ -84,44 +84,25 @@ const STAT_GROUPS = {
|
|
|
84
84
|
FROM task_queue`,
|
|
85
85
|
},
|
|
86
86
|
{
|
|
87
|
-
|
|
87
|
+
// 目标商家计数(仪表盘"目标商家"卡片 + 烟花动画的数据源):
|
|
88
|
+
// tt_seller+verified+location 三列条件完全被 idx_jobs_seller_verified_loc
|
|
89
|
+
// 覆盖,纯索引扫描无回表(127 万行实测 140ms),30s 级刷新无压力。
|
|
90
|
+
// 原随 jobsAgg 在 slow 组(5 分钟 TTL)导致烟花延迟数分钟
|
|
91
|
+
key: "targetUsersCount",
|
|
88
92
|
mode: "one",
|
|
89
93
|
params: () => getTargetLocations(),
|
|
90
|
-
sql: () => `SELECT
|
|
91
|
-
|
|
92
|
-
SUM(done) as done,
|
|
93
|
-
SUM(error) as error,
|
|
94
|
-
SUM(restricted) as restricted,
|
|
95
|
-
SUM(targetUsers) as targetUsers,
|
|
96
|
-
SUM(noVideo) as noVideo
|
|
97
|
-
FROM (
|
|
98
|
-
SELECT
|
|
99
|
-
COUNT(*) as jobsTotal,
|
|
100
|
-
SUM(CASE WHEN status = 'done' THEN 1 ELSE 0 END) as done,
|
|
101
|
-
SUM(CASE WHEN status = 'error' THEN 1 ELSE 0 END) as error,
|
|
102
|
-
SUM(CASE WHEN status = 'restricted' THEN 1 ELSE 0 END) as restricted,
|
|
103
|
-
SUM(CASE WHEN tt_seller = 1 AND verified = 0 AND ${targetInSql()} THEN 1 ELSE 0 END) as targetUsers,
|
|
104
|
-
SUM(CASE WHEN no_video = 1 THEN 1 ELSE 0 END) as noVideo
|
|
105
|
-
FROM jobs
|
|
106
|
-
UNION ALL
|
|
107
|
-
-- jobs_done 冷归档表:全部是 done 且只增不改。改用覆盖索引/部分索引计数,
|
|
108
|
-
-- 避免对 147 万行全表回表(原 SUM(CASE WHEN no_video=1) 无法匹配索引)
|
|
109
|
-
SELECT
|
|
110
|
-
c.cnt as jobsTotal,
|
|
111
|
-
c.cnt as done,
|
|
112
|
-
0 as error,
|
|
113
|
-
0 as restricted,
|
|
114
|
-
0 as targetUsers,
|
|
115
|
-
(SELECT COUNT(*) FROM jobs_done WHERE no_video = 1) as noVideo
|
|
116
|
-
FROM (SELECT COUNT(*) as cnt FROM jobs_done) c
|
|
117
|
-
)`,
|
|
94
|
+
sql: () => `SELECT COUNT(*) AS n FROM jobs
|
|
95
|
+
WHERE tt_seller = 1 AND verified = 0 AND ${targetInSql()}`,
|
|
118
96
|
},
|
|
119
97
|
{
|
|
120
98
|
// 精确商家计数:目标商家 + 7 天 EC 视频播放>1000 + 有商品 + 评论0 + 销量>10
|
|
99
|
+
// INDEXED BY 强制走 idx_jobs_precise_merchants partial 覆盖索引(候选仅数
|
|
100
|
+
// 百行,毫秒级):曾因 12.1s/次迁入 slow 组(5 分钟 TTL)导致卡片 5 分钟
|
|
101
|
+
// 才刷一次,索引修复后移回 fast 组恢复 30s 级。索引缺失时置空降级为 0
|
|
121
102
|
key: "preciseMerchants",
|
|
122
103
|
mode: "one",
|
|
123
104
|
params: () => getTargetLocations(),
|
|
124
|
-
sql: () => `SELECT COUNT(*) AS n FROM jobs
|
|
105
|
+
sql: () => `SELECT COUNT(*) AS n FROM jobs INDEXED BY idx_jobs_precise_merchants
|
|
125
106
|
WHERE tt_seller = 1 AND verified = 0 AND ${targetInSql()}
|
|
126
107
|
AND top_video_seo_url IS NOT NULL AND top_video_seo_url != ''
|
|
127
108
|
AND product_review_count = 0
|
|
@@ -132,10 +113,40 @@ const STAT_GROUPS = {
|
|
|
132
113
|
WHERE CAST(json_extract(value, '$.playCount') AS INTEGER) > 1000
|
|
133
114
|
)`,
|
|
134
115
|
},
|
|
116
|
+
{
|
|
117
|
+
// 已完成/限制/错误计数(仪表盘三个状态卡片):走部分索引覆盖计数,
|
|
118
|
+
// 索引只含对应状态的行(idx_jobs_list_done/error/restricted,
|
|
119
|
+
// WHERE status='xxx' 与查询条件完全匹配),127 万行毫秒级。
|
|
120
|
+
// 原随 jobsAgg 在 slow 组(5 分钟 TTL),2026-10-09 起错误卡片 5 分钟
|
|
121
|
+
// 才刷一次;拆出后恢复 30s 级。索引缺失查询失败时回退 slow 组 jobsAgg
|
|
122
|
+
key: "doneCount",
|
|
123
|
+
mode: "one",
|
|
124
|
+
sql: `SELECT (SELECT COUNT(*) FROM jobs INDEXED BY idx_jobs_list_done WHERE status = 'done')
|
|
125
|
+
+ (SELECT COUNT(*) FROM jobs_done) AS n`,
|
|
126
|
+
},
|
|
127
|
+
{
|
|
128
|
+
key: "restrictedCount",
|
|
129
|
+
mode: "one",
|
|
130
|
+
sql: "SELECT COUNT(*) AS n FROM jobs INDEXED BY idx_jobs_list_restricted WHERE status = 'restricted'",
|
|
131
|
+
},
|
|
132
|
+
{
|
|
133
|
+
key: "errorCount",
|
|
134
|
+
mode: "one",
|
|
135
|
+
sql: "SELECT COUNT(*) AS n FROM jobs INDEXED BY idx_jobs_list_error WHERE status = 'error'",
|
|
136
|
+
},
|
|
135
137
|
],
|
|
136
138
|
combine(results) {
|
|
137
139
|
const tq = results.tqAgg || {};
|
|
138
|
-
|
|
140
|
+
// jobs 域轻量计数 done/error/restricted 已拆出为 fast 组独立计数
|
|
141
|
+
// (部分索引覆盖计数,毫秒级,30s 级刷新);jobsAgg 仍在 slow 组提供
|
|
142
|
+
// jobsTotal(总用户)/noVideo,并作为拆出计数查询失败时的回退源。
|
|
143
|
+
// 未就绪时回退 0。
|
|
144
|
+
const slow = groupState.slow?.cache || {};
|
|
145
|
+
const jb = slow.jobsAgg || {};
|
|
146
|
+
const done = results.doneCount?.n != null ? results.doneCount.n : jb.done || 0;
|
|
147
|
+
const restricted =
|
|
148
|
+
results.restrictedCount?.n != null ? results.restrictedCount.n : jb.restricted || 0;
|
|
149
|
+
const error = results.errorCount?.n != null ? results.errorCount.n : jb.error || 0;
|
|
139
150
|
const total = (tq.tqTotal || 0) + (jb.jobsTotal || 0);
|
|
140
151
|
return {
|
|
141
152
|
totalUsers: total,
|
|
@@ -143,10 +154,10 @@ const STAT_GROUPS = {
|
|
|
143
154
|
jobsPending: tq.pending || 0,
|
|
144
155
|
pendingUsers: tq.pending || 0,
|
|
145
156
|
processingUsers: tq.processing || 0,
|
|
146
|
-
processedUsers:
|
|
147
|
-
restrictedUsers:
|
|
148
|
-
errorUsers:
|
|
149
|
-
targetUsers:
|
|
157
|
+
processedUsers: done,
|
|
158
|
+
restrictedUsers: restricted,
|
|
159
|
+
errorUsers: error,
|
|
160
|
+
targetUsers: results.targetUsersCount?.n || 0,
|
|
150
161
|
preciseMerchants: results.preciseMerchants?.n || 0,
|
|
151
162
|
sourceStats: {
|
|
152
163
|
seed: tq.seed || 0,
|
|
@@ -155,9 +166,9 @@ const STAT_GROUPS = {
|
|
|
155
166
|
guess: tq.guess || 0,
|
|
156
167
|
following: tq.following || 0,
|
|
157
168
|
follower: tq.follower || 0,
|
|
158
|
-
processed:
|
|
159
|
-
restricted:
|
|
160
|
-
error:
|
|
169
|
+
processed: done,
|
|
170
|
+
restricted: restricted,
|
|
171
|
+
error: error,
|
|
161
172
|
noVideo: jb.noVideo || 0,
|
|
162
173
|
},
|
|
163
174
|
};
|
|
@@ -222,12 +233,50 @@ const STAT_GROUPS = {
|
|
|
222
233
|
slow: {
|
|
223
234
|
ttl: SLOW_STATS_TTL,
|
|
224
235
|
// rawJobs 走 store.getRawJobsCount()(模块内部自带 60s 缓存),不进分片 SQL
|
|
236
|
+
// jobsAgg 自 fast 组迁入:jobs 表百万行级后单次 ~3.3s,只适合 5 分钟级刷新
|
|
237
|
+
// (fast 组 combine 从本组缓存读取)。targetUsers/preciseMerchants 已拆出
|
|
238
|
+
// 走覆盖索引回 fast 组(详见 fast 组注释);done/error/restricted 也已拆出
|
|
239
|
+
// 为部分索引轻量计数(详见 fast 组注释),jobsAgg 保留全量聚合作为回退源,
|
|
240
|
+
// 并继续提供 jobsTotal(总用户)/noVideo(jobs 侧无匹配索引,保持 5 分钟级)
|
|
225
241
|
sqls: [
|
|
226
242
|
{
|
|
227
243
|
key: "commentTaskCount",
|
|
228
244
|
mode: "one",
|
|
229
245
|
sql: "SELECT COUNT(*) as c FROM videos WHERE user_update_count IS NULL OR user_update_count <= 0",
|
|
230
246
|
},
|
|
247
|
+
{
|
|
248
|
+
key: "jobsAgg",
|
|
249
|
+
mode: "one",
|
|
250
|
+
params: () => getTargetLocations(),
|
|
251
|
+
sql: () => `SELECT
|
|
252
|
+
SUM(jobsTotal) as jobsTotal,
|
|
253
|
+
SUM(done) as done,
|
|
254
|
+
SUM(error) as error,
|
|
255
|
+
SUM(restricted) as restricted,
|
|
256
|
+
SUM(targetUsers) as targetUsers,
|
|
257
|
+
SUM(noVideo) as noVideo
|
|
258
|
+
FROM (
|
|
259
|
+
SELECT
|
|
260
|
+
COUNT(*) as jobsTotal,
|
|
261
|
+
SUM(CASE WHEN status = 'done' THEN 1 ELSE 0 END) as done,
|
|
262
|
+
SUM(CASE WHEN status = 'error' THEN 1 ELSE 0 END) as error,
|
|
263
|
+
SUM(CASE WHEN status = 'restricted' THEN 1 ELSE 0 END) as restricted,
|
|
264
|
+
SUM(CASE WHEN tt_seller = 1 AND verified = 0 AND ${targetInSql()} THEN 1 ELSE 0 END) as targetUsers,
|
|
265
|
+
SUM(CASE WHEN no_video = 1 THEN 1 ELSE 0 END) as noVideo
|
|
266
|
+
FROM jobs
|
|
267
|
+
UNION ALL
|
|
268
|
+
-- jobs_done 冷归档表:全部是 done 且只增不改。改用覆盖索引/部分索引计数,
|
|
269
|
+
-- 避免对 147 万行全表回表(原 SUM(CASE WHEN no_video=1) 无法匹配索引)
|
|
270
|
+
SELECT
|
|
271
|
+
c.cnt as jobsTotal,
|
|
272
|
+
c.cnt as done,
|
|
273
|
+
0 as error,
|
|
274
|
+
0 as restricted,
|
|
275
|
+
0 as targetUsers,
|
|
276
|
+
(SELECT COUNT(*) FROM jobs_done WHERE no_video = 1) as noVideo
|
|
277
|
+
FROM (SELECT COUNT(*) as cnt FROM jobs_done) c
|
|
278
|
+
)`,
|
|
279
|
+
},
|
|
231
280
|
],
|
|
232
281
|
combine(results, st) {
|
|
233
282
|
// tagCount/tagHighScore 从独立 tags.db 读取(跨库)
|
|
@@ -249,6 +298,15 @@ const STAT_GROUPS = {
|
|
|
249
298
|
return {
|
|
250
299
|
rawJobs: st.getRawJobsCount ? st.getRawJobsCount() : 0,
|
|
251
300
|
commentTaskCount: results.commentTaskCount?.c || 0,
|
|
301
|
+
// jobs 域聚合(供 fast 组 combine 读取,字段保持展开后的形态)
|
|
302
|
+
jobsAgg: {
|
|
303
|
+
jobsTotal: results.jobsAgg?.jobsTotal || 0,
|
|
304
|
+
done: results.jobsAgg?.done || 0,
|
|
305
|
+
error: results.jobsAgg?.error || 0,
|
|
306
|
+
restricted: results.jobsAgg?.restricted || 0,
|
|
307
|
+
targetUsers: results.jobsAgg?.targetUsers || 0,
|
|
308
|
+
noVideo: results.jobsAgg?.noVideo || 0,
|
|
309
|
+
},
|
|
252
310
|
tagCount,
|
|
253
311
|
tagHighScore,
|
|
254
312
|
};
|
|
@@ -595,6 +653,12 @@ export function startWatchServer(
|
|
|
595
653
|
// precheck 候选查询部分索引:预缩候选集且天然按 refresh_time 有序,
|
|
596
654
|
// 消除 TEMP B-TREE 排序(76 万行实测 ~1s → 毫秒级)
|
|
597
655
|
"CREATE INDEX IF NOT EXISTS idx_jobs_precheck ON jobs(refresh_time) WHERE tt_seller = 1 AND verified = 0 AND (pending_refresh IS NULL OR pending_refresh = 0)",
|
|
656
|
+
// 精准商家(目标商家页 + stats 计数)部分覆盖索引:WHERE 收录查询全部
|
|
657
|
+
// 便宜过滤条件,候选集从 jobs 全量商家(2026-10 已 127 万行)预缩到数百行,
|
|
658
|
+
// COUNT/LIST 由 127 万次回表(实测 COUNT 2.6s)降到毫秒级。
|
|
659
|
+
// ⚠️ WHERE 文本必须与 getPreciseMerchantsFromDb / stats preciseMerchants
|
|
660
|
+
// 查询逐词一致(SQLite 按术语文本匹配 partial index),改查询需同步改这里
|
|
661
|
+
"CREATE INDEX IF NOT EXISTS idx_jobs_precise_merchants ON jobs(location_created, product_sold_count DESC, unique_id) WHERE tt_seller = 1 AND verified = 0 AND top_video_seo_url IS NOT NULL AND top_video_seo_url != '' AND product_review_count = 0 AND product_sold_count > 10 AND recent_ec_videos IS NOT NULL AND recent_ec_videos != ''",
|
|
598
662
|
];
|
|
599
663
|
for (const sql of ensureIdx) {
|
|
600
664
|
try {
|