tt-help-cli-ycl 1.4.134 → 1.4.136
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/scripts/backup-db.mjs +73 -0
- package/scripts/check-queues.sh +37 -0
- package/scripts/migrate-jobs-done.mjs +124 -0
- package/scripts/migrate-tags-db.mjs +163 -0
- package/src/cli/tag.js +68 -9
- package/src/lib/tag-discover.js +21 -0
- package/src/watch/data-store.js +131 -77
- package/src/watch/db-crud.js +35 -10
- package/src/watch/db-schema.js +249 -100
- package/src/watch/db-stats.js +53 -23
- package/src/watch/db-tags.js +5 -2
- package/src/watch/llm-config.js +31 -1
- package/src/watch/public/app.js +162 -11
- package/src/watch/public/index.html +19 -1
- package/src/watch/server.js +338 -62
- package/src/watch/tag-selflearn.js +2 -1
- package/src/watch/watchdog-server.js +129 -1
- package/src/watchdog/agent.js +76 -0
- package/src/watchdog/process-manager.js +21 -0
- package/src/watchdog/self-log-buffer.js +25 -0
package/src/watch/server.js
CHANGED
|
@@ -7,6 +7,7 @@ import { join, dirname } from "path";
|
|
|
7
7
|
import { fileURLToPath } from "url";
|
|
8
8
|
import { spawn } from "child_process";
|
|
9
9
|
import { createStore } from "./data-store.js";
|
|
10
|
+
import { getTagDb } from "./db-schema.js";
|
|
10
11
|
import { DEFAULT_TARGET_LOCATIONS } from "../lib/target-locations.js";
|
|
11
12
|
import { tryHandleWatchdogRoute } from "./watchdog-server.js";
|
|
12
13
|
|
|
@@ -26,96 +27,345 @@ function getLocalIP() {
|
|
|
26
27
|
const __dirname = dirname(__filename);
|
|
27
28
|
const publicDir = join(__dirname, "public");
|
|
28
29
|
|
|
29
|
-
// Stats
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
//
|
|
33
|
-
//
|
|
34
|
-
|
|
30
|
+
// ===== Stats 分项缓存:快/中/慢 三档 TTL =====
|
|
31
|
+
// 原来所有统计共用一个 5 分钟缓存,快项(任务队列聚合)被慢项(千万级大表 COUNT)拖累。
|
|
32
|
+
// 现将统计拆为三组,各自独立缓存与刷新:
|
|
33
|
+
// fast (30s): task_queue + jobs 状态聚合 —— 仪表盘核心数字,接近实时
|
|
34
|
+
// medium (60s): jobs_base 计数 + 按国家分布(jobs 表 GROUP BY,较重)
|
|
35
|
+
// slow (300s): raw_jobs / videos / tags 全表 COUNT —— 最重,低频
|
|
36
|
+
// 每组过期后先返回旧数据,后台分片刷新(片间 setImmediate 让出事件循环,
|
|
37
|
+
// 避免多个大查询连续执行冻结事件循环,此前成簇超时的直接原因)。
|
|
38
|
+
// 另:任务领取/提交后主动使 fast 组失效,核心数字最快 10s(前端轮询周期)内更新。
|
|
39
|
+
const FAST_STATS_TTL = 30000; // 30 秒
|
|
40
|
+
const MEDIUM_STATS_TTL = 60000; // 60 秒
|
|
41
|
+
const SLOW_STATS_TTL = 300000; // 5 分钟
|
|
35
42
|
|
|
36
43
|
// 用户列表短缓存:同一参数组合 3 秒内复用,无论多少客户端同时查只算一次
|
|
37
44
|
const usersCache = new Map(); // key -> { result, time }
|
|
38
45
|
const USERS_CACHE_TTL = 3000; // 3 秒
|
|
39
46
|
|
|
40
|
-
//
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
47
|
+
// 目标国家占位符(jobs 表聚合需要 location_created IN (...))
|
|
48
|
+
const TARGET_PLACEHOLDERS = DEFAULT_TARGET_LOCATIONS.map(() => "?").join(", ");
|
|
49
|
+
const TARGET_IN_SQL = DEFAULT_TARGET_LOCATIONS.length
|
|
50
|
+
? `location_created IN (${TARGET_PLACEHOLDERS})`
|
|
51
|
+
: "1 = 0";
|
|
52
|
+
|
|
53
|
+
// 统计分组定义:每组包含若干 SQL(分片执行)与一个聚合函数
|
|
54
|
+
// sqls 项:{ key, sql, mode: 'one'|'all', params: 数组或返回数组的函数 }
|
|
55
|
+
const STAT_GROUPS = {
|
|
56
|
+
fast: {
|
|
57
|
+
ttl: FAST_STATS_TTL,
|
|
58
|
+
sqls: [
|
|
59
|
+
{
|
|
60
|
+
key: "tqAgg",
|
|
61
|
+
mode: "one",
|
|
62
|
+
sql: `SELECT
|
|
63
|
+
COUNT(*) as tqTotal,
|
|
64
|
+
SUM(CASE WHEN status = 'pending' THEN 1 ELSE 0 END) as pending,
|
|
65
|
+
SUM(CASE WHEN status = 'processing' THEN 1 ELSE 0 END) as processing,
|
|
66
|
+
SUM(CASE WHEN instr(COALESCE(sources, ''), '"video"') > 0 THEN 1 ELSE 0 END) as video,
|
|
67
|
+
SUM(CASE WHEN instr(COALESCE(sources, ''), '"comment"') > 0 THEN 1 ELSE 0 END) as comment,
|
|
68
|
+
SUM(CASE WHEN instr(COALESCE(sources, ''), '"guess"') > 0 THEN 1 ELSE 0 END) as guess,
|
|
69
|
+
SUM(CASE WHEN instr(COALESCE(sources, ''), '"following"') > 0 THEN 1 ELSE 0 END) as following,
|
|
70
|
+
SUM(CASE WHEN instr(COALESCE(sources, ''), '"follower"') > 0 THEN 1 ELSE 0 END) as follower,
|
|
71
|
+
SUM(CASE
|
|
72
|
+
WHEN instr(COALESCE(sources, ''), '"video"') = 0
|
|
73
|
+
AND instr(COALESCE(sources, ''), '"comment"') = 0
|
|
74
|
+
AND instr(COALESCE(sources, ''), '"guess"') = 0
|
|
75
|
+
AND instr(COALESCE(sources, ''), '"following"') = 0
|
|
76
|
+
AND instr(COALESCE(sources, ''), '"follower"') = 0
|
|
77
|
+
THEN 1 ELSE 0 END) as seed
|
|
78
|
+
FROM task_queue`,
|
|
79
|
+
},
|
|
80
|
+
{
|
|
81
|
+
key: "jobsAgg",
|
|
82
|
+
mode: "one",
|
|
83
|
+
params: DEFAULT_TARGET_LOCATIONS,
|
|
84
|
+
sql: `SELECT
|
|
85
|
+
SUM(jobsTotal) as jobsTotal,
|
|
86
|
+
SUM(done) as done,
|
|
87
|
+
SUM(error) as error,
|
|
88
|
+
SUM(restricted) as restricted,
|
|
89
|
+
SUM(targetUsers) as targetUsers,
|
|
90
|
+
SUM(noVideo) as noVideo
|
|
91
|
+
FROM (
|
|
92
|
+
SELECT
|
|
93
|
+
COUNT(*) as jobsTotal,
|
|
94
|
+
SUM(CASE WHEN status = 'done' THEN 1 ELSE 0 END) as done,
|
|
95
|
+
SUM(CASE WHEN status = 'error' THEN 1 ELSE 0 END) as error,
|
|
96
|
+
SUM(CASE WHEN status = 'restricted' THEN 1 ELSE 0 END) as restricted,
|
|
97
|
+
SUM(CASE WHEN tt_seller = 1 AND verified = 0 AND ${TARGET_IN_SQL} THEN 1 ELSE 0 END) as targetUsers,
|
|
98
|
+
SUM(CASE WHEN no_video = 1 THEN 1 ELSE 0 END) as noVideo
|
|
99
|
+
FROM jobs
|
|
100
|
+
UNION ALL
|
|
101
|
+
SELECT
|
|
102
|
+
COUNT(*) as jobsTotal,
|
|
103
|
+
COUNT(*) as done,
|
|
104
|
+
0 as error,
|
|
105
|
+
0 as restricted,
|
|
106
|
+
0 as targetUsers,
|
|
107
|
+
SUM(CASE WHEN no_video = 1 THEN 1 ELSE 0 END) as noVideo
|
|
108
|
+
FROM jobs_done
|
|
109
|
+
)`,
|
|
110
|
+
},
|
|
111
|
+
],
|
|
112
|
+
combine(results) {
|
|
113
|
+
const tq = results.tqAgg || {};
|
|
114
|
+
const jb = results.jobsAgg || {};
|
|
115
|
+
const total = (tq.tqTotal || 0) + (jb.jobsTotal || 0);
|
|
116
|
+
return {
|
|
117
|
+
totalUsers: total,
|
|
118
|
+
jobsTotal: total,
|
|
119
|
+
jobsPending: tq.pending || 0,
|
|
120
|
+
pendingUsers: tq.pending || 0,
|
|
121
|
+
processingUsers: tq.processing || 0,
|
|
122
|
+
processedUsers: jb.done || 0,
|
|
123
|
+
restrictedUsers: jb.restricted || 0,
|
|
124
|
+
errorUsers: jb.error || 0,
|
|
125
|
+
targetUsers: jb.targetUsers || 0,
|
|
126
|
+
sourceStats: {
|
|
127
|
+
seed: tq.seed || 0,
|
|
128
|
+
video: tq.video || 0,
|
|
129
|
+
comment: tq.comment || 0,
|
|
130
|
+
guess: tq.guess || 0,
|
|
131
|
+
following: tq.following || 0,
|
|
132
|
+
follower: tq.follower || 0,
|
|
133
|
+
processed: jb.done || 0,
|
|
134
|
+
restricted: jb.restricted || 0,
|
|
135
|
+
error: jb.error || 0,
|
|
136
|
+
noVideo: jb.noVideo || 0,
|
|
137
|
+
},
|
|
138
|
+
};
|
|
139
|
+
},
|
|
140
|
+
},
|
|
141
|
+
medium: {
|
|
142
|
+
ttl: MEDIUM_STATS_TTL,
|
|
143
|
+
sqls: [
|
|
144
|
+
{
|
|
145
|
+
key: "jobsBaseCount",
|
|
146
|
+
mode: "one",
|
|
147
|
+
sql: "SELECT COUNT(*) as total FROM jobs_base",
|
|
148
|
+
},
|
|
149
|
+
{
|
|
150
|
+
key: "userUpdateTasks",
|
|
151
|
+
mode: "one",
|
|
152
|
+
sql: "SELECT COUNT(*) as userUpdateTasks FROM jobs_base WHERE COALESCE(user_update_count, 0) <= 0",
|
|
153
|
+
},
|
|
154
|
+
{
|
|
155
|
+
key: "countryStats",
|
|
156
|
+
mode: "all",
|
|
157
|
+
sql: `SELECT country, SUM(cnt) as count, SUM(targetCount) as targetCount
|
|
158
|
+
FROM (
|
|
159
|
+
SELECT
|
|
160
|
+
COALESCE(location_created, '未知') as country,
|
|
161
|
+
COUNT(*) as cnt,
|
|
162
|
+
SUM(CASE WHEN tt_seller = 1 AND verified = 0 THEN 1 ELSE 0 END) as targetCount
|
|
163
|
+
FROM jobs
|
|
164
|
+
WHERE status = 'done'
|
|
165
|
+
GROUP BY COALESCE(location_created, '未知')
|
|
166
|
+
UNION ALL
|
|
167
|
+
SELECT
|
|
168
|
+
COALESCE(location_created, '未知') as country,
|
|
169
|
+
COUNT(*) as cnt,
|
|
170
|
+
0 as targetCount
|
|
171
|
+
FROM jobs_done
|
|
172
|
+
GROUP BY COALESCE(location_created, '未知')
|
|
173
|
+
)
|
|
174
|
+
GROUP BY country
|
|
175
|
+
ORDER BY count DESC`,
|
|
176
|
+
},
|
|
177
|
+
{
|
|
178
|
+
key: "targetCountryStats",
|
|
179
|
+
mode: "all",
|
|
180
|
+
params: DEFAULT_TARGET_LOCATIONS,
|
|
181
|
+
sql: `SELECT location_created as country, COUNT(*) as count
|
|
182
|
+
FROM jobs
|
|
183
|
+
WHERE tt_seller = 1 AND verified = 0 AND ${TARGET_IN_SQL}
|
|
184
|
+
GROUP BY location_created
|
|
185
|
+
ORDER BY count DESC`,
|
|
186
|
+
},
|
|
187
|
+
],
|
|
188
|
+
combine(results) {
|
|
189
|
+
return {
|
|
190
|
+
jobsBaseTotal: results.jobsBaseCount?.total || 0,
|
|
191
|
+
userUpdateTasks: results.userUpdateTasks?.userUpdateTasks || 0,
|
|
192
|
+
countryStats: results.countryStats || [],
|
|
193
|
+
targetCountryStats: results.targetCountryStats || [],
|
|
194
|
+
};
|
|
195
|
+
},
|
|
196
|
+
},
|
|
197
|
+
slow: {
|
|
198
|
+
ttl: SLOW_STATS_TTL,
|
|
199
|
+
// rawJobs 走 store.getRawJobsCount()(模块内部自带 60s 缓存),不进分片 SQL
|
|
200
|
+
sqls: [
|
|
201
|
+
{
|
|
202
|
+
key: "commentTaskCount",
|
|
203
|
+
mode: "one",
|
|
204
|
+
sql: "SELECT COUNT(*) as c FROM videos WHERE user_update_count IS NULL OR user_update_count <= 0",
|
|
205
|
+
},
|
|
206
|
+
],
|
|
207
|
+
combine(results, st) {
|
|
208
|
+
// tagCount/tagHighScore 从独立 tags.db 读取(跨库)
|
|
209
|
+
let tagCount = 0;
|
|
210
|
+
let tagHighScore = 0;
|
|
211
|
+
const tagDb = getTagDb();
|
|
212
|
+
if (tagDb) {
|
|
213
|
+
try {
|
|
214
|
+
tagCount =
|
|
215
|
+
tagDb.prepare("SELECT COUNT(*) as c FROM tags").get().c || 0;
|
|
216
|
+
tagHighScore =
|
|
217
|
+
tagDb
|
|
218
|
+
.prepare("SELECT COUNT(*) as c FROM tags WHERE score >= 70")
|
|
219
|
+
.get().c || 0;
|
|
220
|
+
} catch (e) {
|
|
221
|
+
console.error(`[stats] tag 库统计失败: ${e.message}`);
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
return {
|
|
225
|
+
rawJobs: st.getRawJobsCount ? st.getRawJobsCount() : 0,
|
|
226
|
+
commentTaskCount: results.commentTaskCount?.c || 0,
|
|
227
|
+
tagCount,
|
|
228
|
+
tagHighScore,
|
|
229
|
+
};
|
|
230
|
+
},
|
|
231
|
+
},
|
|
49
232
|
};
|
|
50
233
|
|
|
51
|
-
|
|
234
|
+
// 每组独立缓存状态
|
|
235
|
+
const groupState = {};
|
|
236
|
+
for (const name of Object.keys(STAT_GROUPS)) {
|
|
237
|
+
groupState[name] = { cache: null, cacheTime: 0, refreshing: false };
|
|
238
|
+
}
|
|
52
239
|
|
|
53
|
-
//
|
|
54
|
-
//
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
240
|
+
// 首次请求时 fast 组同步计算(task_queue + jobs 聚合,约 2s),
|
|
241
|
+
// medium/slow 先返回占位再后台补全,避免启动初期一个请求阻塞 5~30s
|
|
242
|
+
const EMPTY_GROUP = {
|
|
243
|
+
fast: {
|
|
244
|
+
totalUsers: 0,
|
|
245
|
+
jobsTotal: 0,
|
|
246
|
+
jobsPending: 0,
|
|
247
|
+
pendingUsers: 0,
|
|
248
|
+
processingUsers: 0,
|
|
249
|
+
processedUsers: 0,
|
|
250
|
+
restrictedUsers: 0,
|
|
251
|
+
errorUsers: 0,
|
|
252
|
+
targetUsers: 0,
|
|
253
|
+
sourceStats: {
|
|
254
|
+
seed: 0,
|
|
255
|
+
video: 0,
|
|
256
|
+
comment: 0,
|
|
257
|
+
guess: 0,
|
|
258
|
+
following: 0,
|
|
259
|
+
follower: 0,
|
|
260
|
+
processed: 0,
|
|
261
|
+
restricted: 0,
|
|
262
|
+
error: 0,
|
|
263
|
+
noVideo: 0,
|
|
264
|
+
},
|
|
265
|
+
},
|
|
266
|
+
medium: {
|
|
267
|
+
jobsBaseTotal: 0,
|
|
268
|
+
userUpdateTasks: 0,
|
|
269
|
+
countryStats: [],
|
|
270
|
+
targetCountryStats: [],
|
|
271
|
+
},
|
|
272
|
+
slow: { rawJobs: 0, commentTaskCount: 0, tagCount: 0, tagHighScore: 0 },
|
|
273
|
+
};
|
|
58
274
|
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
275
|
+
// 分片执行某组统计刷新:每个 SQL 单独一片,片间 setImmediate 让出事件循环
|
|
276
|
+
function refreshGroupInShards(st, name) {
|
|
277
|
+
const g = groupState[name];
|
|
278
|
+
const spec = STAT_GROUPS[name];
|
|
279
|
+
g.refreshing = true;
|
|
280
|
+
const results = {};
|
|
281
|
+
let idx = 0;
|
|
282
|
+
|
|
283
|
+
function runShard() {
|
|
284
|
+
if (idx < spec.sqls.length) {
|
|
285
|
+
const s = spec.sqls[idx++];
|
|
62
286
|
try {
|
|
63
|
-
const
|
|
64
|
-
|
|
287
|
+
const params =
|
|
288
|
+
typeof s.params === "function" ? s.params() : s.params || [];
|
|
289
|
+
const r = st.rawQuery(s.sql, params);
|
|
290
|
+
if (r?.error) {
|
|
291
|
+
console.error(`[stats] ${name}.${s.key} 查询失败: ${r.error}`);
|
|
292
|
+
results[s.key] = null;
|
|
293
|
+
} else {
|
|
294
|
+
results[s.key] =
|
|
295
|
+
s.mode === "all" ? r?.rows || [] : r?.rows?.[0] || null;
|
|
296
|
+
}
|
|
65
297
|
} catch (e) {
|
|
66
|
-
console.error(`[stats]
|
|
67
|
-
|
|
298
|
+
console.error(`[stats] ${name}.${s.key} 查询失败: ${e.message}`);
|
|
299
|
+
results[s.key] = null;
|
|
68
300
|
}
|
|
69
|
-
setImmediate(
|
|
301
|
+
setImmediate(runShard);
|
|
70
302
|
return;
|
|
71
303
|
}
|
|
72
|
-
//
|
|
304
|
+
// 该组 SQL 全部完成 → 聚合写入缓存
|
|
73
305
|
setImmediate(() => {
|
|
74
306
|
try {
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
});
|
|
78
|
-
stats.rawJobs = slowCounts.rawJobs || 0;
|
|
79
|
-
stats.commentTaskCount = slowCounts.commentTaskCount || 0;
|
|
80
|
-
stats.tagCount = slowCounts.tagCount || 0;
|
|
81
|
-
stats.tagHighScore = slowCounts.tagHighScore || 0;
|
|
82
|
-
statsCache = stats;
|
|
83
|
-
statsCacheTime = Date.now();
|
|
307
|
+
g.cache = spec.combine(results, st);
|
|
308
|
+
g.cacheTime = Date.now();
|
|
84
309
|
} catch (e) {
|
|
85
|
-
console.error(`[stats]
|
|
310
|
+
console.error(`[stats] ${name} 聚合失败: ${e.message}`);
|
|
86
311
|
} finally {
|
|
87
|
-
|
|
312
|
+
g.refreshing = false;
|
|
88
313
|
}
|
|
89
314
|
});
|
|
90
315
|
}
|
|
91
316
|
|
|
92
|
-
setImmediate(
|
|
317
|
+
setImmediate(runShard);
|
|
93
318
|
}
|
|
94
319
|
|
|
95
|
-
|
|
320
|
+
// 计算某组统计:缓存有效→直接返回;过期→先回旧数据后台刷新;
|
|
321
|
+
// 首次→fast 同步计算,medium/slow 回占位并后台补全
|
|
322
|
+
function computeGroupStats(st, name) {
|
|
323
|
+
const g = groupState[name];
|
|
96
324
|
const now = Date.now();
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
return statsCache;
|
|
325
|
+
if (g.cache && now - g.cacheTime < STAT_GROUPS[name].ttl) {
|
|
326
|
+
return g.cache;
|
|
100
327
|
}
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
refreshStatsInShards(st);
|
|
105
|
-
return statsCache;
|
|
328
|
+
if (g.cache) {
|
|
329
|
+
if (!g.refreshing) refreshGroupInShards(st, name);
|
|
330
|
+
return g.cache;
|
|
106
331
|
}
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
332
|
+
if (name === "fast") {
|
|
333
|
+
// 首次同步计算 fast 组(约 2s),仅启动后首次请求发生
|
|
334
|
+
try {
|
|
335
|
+
const results = {};
|
|
336
|
+
for (const s of STAT_GROUPS[name].sqls) {
|
|
337
|
+
const params =
|
|
338
|
+
typeof s.params === "function" ? s.params() : s.params || [];
|
|
339
|
+
const r = st.rawQuery(s.sql, params);
|
|
340
|
+
results[s.key] =
|
|
341
|
+
s.mode === "all" ? r?.rows || [] : r?.rows?.[0] || null;
|
|
342
|
+
}
|
|
343
|
+
g.cache = STAT_GROUPS[name].combine(results, st);
|
|
344
|
+
} catch (e) {
|
|
345
|
+
console.error(`[stats] fast 首次同步计算失败: ${e.message}`);
|
|
346
|
+
g.cache = EMPTY_GROUP.fast;
|
|
347
|
+
}
|
|
348
|
+
g.cacheTime = now;
|
|
349
|
+
} else {
|
|
350
|
+
g.cache = EMPTY_GROUP[name];
|
|
351
|
+
g.cacheTime = now;
|
|
117
352
|
}
|
|
118
|
-
|
|
353
|
+
if (!g.refreshing) refreshGroupInShards(st, name);
|
|
354
|
+
return g.cache;
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
// 合并三组缓存,返回完整 stats(字段结构与旧 getDashboardStatsFromDb 返回一致)
|
|
358
|
+
function computeStatsIncremental(st) {
|
|
359
|
+
return {
|
|
360
|
+
...computeGroupStats(st, "fast"),
|
|
361
|
+
...computeGroupStats(st, "medium"),
|
|
362
|
+
...computeGroupStats(st, "slow"),
|
|
363
|
+
};
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
// 快项失效:任务领取/提交后调用,下一次 /api/stats 请求即触发 fast 组后台刷新
|
|
367
|
+
function markFastStatsDirty() {
|
|
368
|
+
groupState.fast.cacheTime = 0;
|
|
119
369
|
}
|
|
120
370
|
|
|
121
371
|
function readBody(req) {
|
|
@@ -294,6 +544,30 @@ export function startWatchServer(
|
|
|
294
544
|
return new Promise((_resolve, reject) => {
|
|
295
545
|
const store = existingStore || createStore(dataAnchor, options);
|
|
296
546
|
|
|
547
|
+
// 在启动时确保关键索引存在,减少慢查询(CREATE INDEX IF NOT EXISTS 是幂等的)
|
|
548
|
+
try {
|
|
549
|
+
const ensureIdx = [
|
|
550
|
+
"CREATE INDEX IF NOT EXISTS idx_jobs_status ON jobs(status)",
|
|
551
|
+
"CREATE INDEX IF NOT EXISTS idx_jobs_loc ON jobs(location_created)",
|
|
552
|
+
"CREATE INDEX IF NOT EXISTS idx_jobs_seller_verified_loc ON jobs(tt_seller, verified, location_created)",
|
|
553
|
+
"CREATE INDEX IF NOT EXISTS idx_task_queue_status ON task_queue(status)",
|
|
554
|
+
"CREATE INDEX IF NOT EXISTS idx_jobs_done_list ON jobs_done(pinned, processed_at, follower_count, unique_id) WHERE status = 'done'",
|
|
555
|
+
"CREATE INDEX IF NOT EXISTS idx_jobs_done_loc ON jobs_done(location_created)",
|
|
556
|
+
"CREATE INDEX IF NOT EXISTS idx_videos_user_update ON videos(user_update_count)",
|
|
557
|
+
// product-fill 领任务查询的部分索引:只索引候选行,避免全表扫描 164 万行
|
|
558
|
+
"CREATE INDEX IF NOT EXISTS idx_jobs_product_fill ON jobs(claimed_at, updated_at) WHERE top_video_seo_url IS NOT NULL AND top_video_seo_url != '' AND (product_name IS NULL OR product_name = '') AND product_rating IS NULL AND product_sold_count IS NULL",
|
|
559
|
+
];
|
|
560
|
+
for (const sql of ensureIdx) {
|
|
561
|
+
try {
|
|
562
|
+
if (store && store.rawQuery) store.rawQuery(sql);
|
|
563
|
+
} catch (e) {
|
|
564
|
+
console.error(`[startup] 创建索引失败: ${e.message}`);
|
|
565
|
+
}
|
|
566
|
+
}
|
|
567
|
+
} catch (e) {
|
|
568
|
+
console.error(`[startup] 索引初始化异常: ${e.message}`);
|
|
569
|
+
}
|
|
570
|
+
|
|
297
571
|
// 初始化 LLM 动态配置(SQLite > 环境变量 > 默认值)
|
|
298
572
|
import("./llm-config.js").then((m) => m.loadLlmConfig());
|
|
299
573
|
|
|
@@ -568,6 +842,7 @@ export function startWatchServer(
|
|
|
568
842
|
clientId: userId,
|
|
569
843
|
locations: locations,
|
|
570
844
|
});
|
|
845
|
+
markFastStatsDirty();
|
|
571
846
|
sendJSON(res, 200, { hasJob: true, user: job });
|
|
572
847
|
} else {
|
|
573
848
|
logJob("CLAIM", {
|
|
@@ -711,6 +986,7 @@ export function startWatchServer(
|
|
|
711
986
|
newUsers: ret.newUsers?.length || 0,
|
|
712
987
|
});
|
|
713
988
|
if (ret.saved) {
|
|
989
|
+
markFastStatsDirty();
|
|
714
990
|
sendJSON(res, 200, ret);
|
|
715
991
|
} else {
|
|
716
992
|
sendJSON(res, 404, ret);
|
|
@@ -11,7 +11,8 @@
|
|
|
11
11
|
* 本模块不修改现有 tag-service.js,而是在其基础上叠加新能力。
|
|
12
12
|
*/
|
|
13
13
|
|
|
14
|
-
|
|
14
|
+
// 注意:tag 域数据已迁移到独立 tags.db,使用 tag 库连接(别名 getDb 保持调用点不变)
|
|
15
|
+
import { getTagDb as getDb } from "./db-schema.js";
|
|
15
16
|
import { callLLM, LLM_MODEL, getLang, COUNTRY_LANG } from "./tag-service.js";
|
|
16
17
|
|
|
17
18
|
// ====== 通用词根表(L1 探索起点) ======
|
|
@@ -10,13 +10,27 @@
|
|
|
10
10
|
* 3. 新增 POST /api/watchdog/clear-llm 端点,手动清除指定客户端的 LLM 提示
|
|
11
11
|
*/
|
|
12
12
|
|
|
13
|
-
import {
|
|
13
|
+
import {
|
|
14
|
+
readFileSync,
|
|
15
|
+
writeFileSync,
|
|
16
|
+
existsSync,
|
|
17
|
+
mkdirSync,
|
|
18
|
+
statSync,
|
|
19
|
+
} from "fs";
|
|
14
20
|
import { join } from "path";
|
|
15
21
|
import { homedir } from "os";
|
|
16
22
|
|
|
17
23
|
const homeDir = homedir() || process.env.HOME || process.env.USERPROFILE || "";
|
|
18
24
|
const WATCHDOG_CONFIG_PATH = join(homeDir, ".tt-help-watchdog.json");
|
|
19
25
|
|
|
26
|
+
// ── 按需日志拉取(后台页面点击"日志"按钮 → 心跳下发 → 客户端回传 → 覆盖写盘) ──
|
|
27
|
+
// 每个 (clientId, cmdId) 固定一个文件,永远只有最新一份,服务端内存零缓存。
|
|
28
|
+
// 目录需在 watch(SYSTEM) 与 standalone(ZKJ) 间保持一致,部署时用 WATCHDOG_LOGS_DIR 统一。
|
|
29
|
+
const LOGS_DIR =
|
|
30
|
+
process.env.WATCHDOG_LOGS_DIR || join(homeDir, ".tt-help-watchdog", "logs");
|
|
31
|
+
const LOG_REQ_TTL_MS = 5 * 60 * 1000; // 请求排队 5 分钟未取走则作废(客户端离线)
|
|
32
|
+
const logReqQueue = new Map(); // clientId -> [{cmdId, tail, requestedAt}]
|
|
33
|
+
|
|
20
34
|
// ── 内存数据结构 ──
|
|
21
35
|
const clients = new Map();
|
|
22
36
|
const CLIENT_TIMEOUT_MS = 60 * 1000;
|
|
@@ -318,6 +332,18 @@ function getAllClientCountries() {
|
|
|
318
332
|
|
|
319
333
|
// ── 工具函数 ──
|
|
320
334
|
|
|
335
|
+
/** 校验文件路径段是否安全(不含路径分隔符/Windows 保留字符/穿越串) */
|
|
336
|
+
function isSafeId(s) {
|
|
337
|
+
return (
|
|
338
|
+
typeof s === "string" &&
|
|
339
|
+
s.length > 0 &&
|
|
340
|
+
s.length <= 160 &&
|
|
341
|
+
!/[\/\\:*?"<>|]/.test(s) &&
|
|
342
|
+
!s.includes("..") &&
|
|
343
|
+
s !== "."
|
|
344
|
+
);
|
|
345
|
+
}
|
|
346
|
+
|
|
321
347
|
function readBody(req) {
|
|
322
348
|
return new Promise((resolve, reject) => {
|
|
323
349
|
let body = "";
|
|
@@ -394,6 +420,12 @@ async function handleHeartbeat(params, body, res) {
|
|
|
394
420
|
clashCmdQueue.delete(userId);
|
|
395
421
|
}
|
|
396
422
|
|
|
423
|
+
// 取出待下发的日志拉取请求(一次性,取走即清空;过期直接丢弃)
|
|
424
|
+
const pendingLogs = (logReqQueue.get(clientId) || []).filter(
|
|
425
|
+
(r) => Date.now() - r.requestedAt < LOG_REQ_TTL_MS,
|
|
426
|
+
);
|
|
427
|
+
if (logReqQueue.has(clientId)) logReqQueue.delete(clientId);
|
|
428
|
+
|
|
397
429
|
const ts = new Date().toLocaleTimeString("zh-CN", { hour12: false });
|
|
398
430
|
console.error(
|
|
399
431
|
`[watchdog ${ts}] 心跳: ${clientId} (${body.hostname || "?"}, user=${userId || "?"}) ${body.processes?.length || 0}进程, 返回${commands.length}条命令${clashCommands.length ? ", " + clashCommands.length + "条Clash命令" : ""}`,
|
|
@@ -403,6 +435,7 @@ async function handleHeartbeat(params, body, res) {
|
|
|
403
435
|
ok: true,
|
|
404
436
|
commands,
|
|
405
437
|
clashCommands: clashCommands.length > 0 ? clashCommands : undefined,
|
|
438
|
+
logRequests: pendingLogs.length > 0 ? pendingLogs : undefined,
|
|
406
439
|
clientConfig: getClientConfigStored(userId),
|
|
407
440
|
errorTrigger: getErrorTriggerStored(userId),
|
|
408
441
|
loginPorts: getLoginPortsStored(userId),
|
|
@@ -424,6 +457,88 @@ async function handleGetClients(params, body, res) {
|
|
|
424
457
|
});
|
|
425
458
|
}
|
|
426
459
|
|
|
460
|
+
// ── 按需日志拉取(点"日志"按钮 → 入队 → 心跳下发 → 客户端回传 → 覆盖写盘) ──
|
|
461
|
+
|
|
462
|
+
/** 后台页面请求某客户端日志:请求入队,等该客户端下次心跳取走 */
|
|
463
|
+
async function handleRequestLogs(params, body, res) {
|
|
464
|
+
const clientId = body.clientId ? String(body.clientId) : "";
|
|
465
|
+
const cmdId = body.cmdId ? String(body.cmdId) : "";
|
|
466
|
+
if (!isSafeId(clientId) || !isSafeId(cmdId)) {
|
|
467
|
+
sendJSON(res, 400, { ok: false, error: "clientId/cmdId 不合法" });
|
|
468
|
+
return;
|
|
469
|
+
}
|
|
470
|
+
const tail = Math.max(1, Math.min(Number(body.tail) || 500, 500));
|
|
471
|
+
// 同一 clientId 的同一 cmdId 只保留最新一条(重复点击不堆积)
|
|
472
|
+
let list = (logReqQueue.get(clientId) || []).filter(
|
|
473
|
+
(r) => r.cmdId !== cmdId && Date.now() - r.requestedAt < LOG_REQ_TTL_MS,
|
|
474
|
+
);
|
|
475
|
+
list.push({ clientId, cmdId, tail, requestedAt: Date.now() });
|
|
476
|
+
logReqQueue.set(clientId, list);
|
|
477
|
+
const ts = new Date().toLocaleTimeString("zh-CN", { hour12: false });
|
|
478
|
+
console.error(
|
|
479
|
+
`[watchdog ${ts}] 日志拉取请求: ${clientId}/${cmdId} (${tail}行)`,
|
|
480
|
+
);
|
|
481
|
+
sendJSON(res, 200, {
|
|
482
|
+
ok: true,
|
|
483
|
+
message: "已请求,等待客户端心跳回传",
|
|
484
|
+
});
|
|
485
|
+
}
|
|
486
|
+
|
|
487
|
+
/** 客户端回传日志 → 覆盖写约定文件(<LOGS_DIR>/<clientId>/<cmdId>.log) */
|
|
488
|
+
async function handleReportLogs(params, body, res) {
|
|
489
|
+
const clientId = body.clientId ? String(body.clientId) : "";
|
|
490
|
+
const cmdId = body.cmdId ? String(body.cmdId) : "";
|
|
491
|
+
if (!isSafeId(clientId) || !isSafeId(cmdId)) {
|
|
492
|
+
sendJSON(res, 400, { ok: false, error: "clientId/cmdId 不合法" });
|
|
493
|
+
return;
|
|
494
|
+
}
|
|
495
|
+
if (!Array.isArray(body.logs)) {
|
|
496
|
+
sendJSON(res, 400, { ok: false, error: "logs 必须是数组" });
|
|
497
|
+
return;
|
|
498
|
+
}
|
|
499
|
+
try {
|
|
500
|
+
const dir = join(LOGS_DIR, clientId);
|
|
501
|
+
mkdirSync(dir, { recursive: true });
|
|
502
|
+
const text = body.logs.map((l) => String(l)).join("\n") + "\n";
|
|
503
|
+
writeFileSync(join(dir, cmdId + ".log"), text, "utf-8");
|
|
504
|
+
console.error(
|
|
505
|
+
`[watchdog] 日志落盘: ${clientId}/${cmdId} (${body.logs.length}行)`,
|
|
506
|
+
);
|
|
507
|
+
sendJSON(res, 200, { ok: true, lines: body.logs.length });
|
|
508
|
+
} catch (e) {
|
|
509
|
+
console.error(`[watchdog] 日志写盘失败: ${e.message}`);
|
|
510
|
+
sendJSON(res, 500, { ok: false, error: e.message });
|
|
511
|
+
}
|
|
512
|
+
}
|
|
513
|
+
|
|
514
|
+
/** 前端弹窗轮询读取日志文件(返回内容 + mtime,无文件返回 content:null) */
|
|
515
|
+
async function handleGetLogsFile(params, body, res) {
|
|
516
|
+
const clientId = String(params.clientId || "");
|
|
517
|
+
const cmdId = String(params.cmdId || "");
|
|
518
|
+
if (!isSafeId(clientId) || !isSafeId(cmdId)) {
|
|
519
|
+
sendJSON(res, 400, { ok: false, error: "clientId/cmdId 不合法" });
|
|
520
|
+
return;
|
|
521
|
+
}
|
|
522
|
+
const file = join(LOGS_DIR, clientId, cmdId + ".log");
|
|
523
|
+
try {
|
|
524
|
+
if (!existsSync(file)) {
|
|
525
|
+
sendJSON(res, 200, { ok: false, content: null, mtimeMs: 0 });
|
|
526
|
+
return;
|
|
527
|
+
}
|
|
528
|
+
const stat = statSync(file);
|
|
529
|
+
const content = readFileSync(file, "utf-8");
|
|
530
|
+
sendJSON(res, 200, {
|
|
531
|
+
ok: true,
|
|
532
|
+
content,
|
|
533
|
+
mtimeMs: stat.mtimeMs,
|
|
534
|
+
size: content.length,
|
|
535
|
+
});
|
|
536
|
+
} catch (e) {
|
|
537
|
+
console.error(`[watchdog] 日志读取失败: ${e.message}`);
|
|
538
|
+
sendJSON(res, 500, { ok: false, error: e.message });
|
|
539
|
+
}
|
|
540
|
+
}
|
|
541
|
+
|
|
427
542
|
async function handleSetClientCommands(params, body, res) {
|
|
428
543
|
const { userId, commands } = body;
|
|
429
544
|
if (!userId) {
|
|
@@ -645,6 +760,7 @@ async function handleSetLoginPorts(params, body, res) {
|
|
|
645
760
|
export function registerWatchdogRoutes(httpServer) {
|
|
646
761
|
console.error("[watchdog] 看门狗服务端模块已加载");
|
|
647
762
|
console.error(`[watchdog] 配置文件: ${WATCHDOG_CONFIG_PATH}`);
|
|
763
|
+
console.error(`[watchdog] 日志目录: ${LOGS_DIR}`);
|
|
648
764
|
// 自动迁移:为已有命令补充唯一 ID
|
|
649
765
|
migrateCommandIds();
|
|
650
766
|
}
|
|
@@ -721,5 +837,17 @@ export async function tryHandleWatchdogRoute(
|
|
|
721
837
|
await handleSetLoginPorts(params, body, res);
|
|
722
838
|
return true;
|
|
723
839
|
}
|
|
840
|
+
if (method === "POST" && routePath === "/api/watchdog/request-logs") {
|
|
841
|
+
await handleRequestLogs(params, body, res);
|
|
842
|
+
return true;
|
|
843
|
+
}
|
|
844
|
+
if (method === "POST" && routePath === "/api/watchdog/report-logs") {
|
|
845
|
+
await handleReportLogs(params, body, res);
|
|
846
|
+
return true;
|
|
847
|
+
}
|
|
848
|
+
if (method === "GET" && routePath === "/api/watchdog/logs-file") {
|
|
849
|
+
await handleGetLogsFile(params, body, res);
|
|
850
|
+
return true;
|
|
851
|
+
}
|
|
724
852
|
return false;
|
|
725
853
|
}
|