tt-help-cli-ycl 1.4.22 → 1.4.24
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/scripts/migrate-raw-jobs.mjs +243 -0
- package/scripts/sync-raw-jobs.mjs +281 -0
- package/src/cli/raw-sync.js +41 -0
- package/src/cli/refresh.js +1 -1
- package/src/lib/args.js +50 -0
- package/src/lib/constants.js +14 -0
- package/src/main.js +3 -0
- package/src/scraper/explore-core.js +14 -4
- package/src/watch/data-store.js +256 -259
- package/src/watch/db-columns.js +2 -0
- package/src/watch/db-schema.js +2 -0
- package/src/watch/db-stats.js +18 -10
- package/src/watch/llm-config.js +66 -16
- package/src/watch/public/app.js +53 -16
- package/src/watch/server.js +122 -19
package/src/watch/db-stats.js
CHANGED
|
@@ -7,10 +7,11 @@
|
|
|
7
7
|
import { getDb } from "./db-schema.js";
|
|
8
8
|
import { getRawJobsCount, getUserDbCount } from "./db-crud.js";
|
|
9
9
|
|
|
10
|
-
export function getDashboardStatsFromDb(targetLocations = []) {
|
|
10
|
+
export function getDashboardStatsFromDb(targetLocations = [], options = {}) {
|
|
11
11
|
const db = getDb();
|
|
12
12
|
if (!db) return null;
|
|
13
13
|
|
|
14
|
+
const skipSlow = !!options.skipSlowCounts;
|
|
14
15
|
const targetPlaceholders = targetLocations.map(() => "?").join(", ");
|
|
15
16
|
const targetParams = targetLocations.length ? targetLocations : [];
|
|
16
17
|
|
|
@@ -89,21 +90,28 @@ export function getDashboardStatsFromDb(targetLocations = []) {
|
|
|
89
90
|
.prepare("SELECT COUNT(*) as total FROM jobs_base")
|
|
90
91
|
.get().total;
|
|
91
92
|
|
|
92
|
-
const tagCount =
|
|
93
|
+
const tagCount = skipSlow
|
|
94
|
+
? 0
|
|
95
|
+
: db.prepare("SELECT COUNT(*) as total FROM tags").get().total;
|
|
93
96
|
|
|
94
|
-
const commentTaskCount =
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
97
|
+
const commentTaskCount = skipSlow
|
|
98
|
+
? 0
|
|
99
|
+
: db
|
|
100
|
+
.prepare(
|
|
101
|
+
`SELECT COUNT(*) as total FROM videos
|
|
102
|
+
WHERE user_update_count IS NULL OR user_update_count <= 0`,
|
|
103
|
+
)
|
|
104
|
+
.get().total;
|
|
105
|
+
|
|
106
|
+
const rawJobsCount = skipSlow ? 0 : getRawJobsCount();
|
|
107
|
+
const dbUsersCount = skipSlow ? 0 : getUserDbCount();
|
|
100
108
|
|
|
101
109
|
return {
|
|
102
110
|
totalUsers: aggregateRow.total,
|
|
103
111
|
tagCount,
|
|
104
112
|
commentTaskCount,
|
|
105
|
-
rawJobs:
|
|
106
|
-
dbTotalUsers:
|
|
113
|
+
rawJobs: rawJobsCount,
|
|
114
|
+
dbTotalUsers: dbUsersCount,
|
|
107
115
|
jobsTotal: aggregateRow.total,
|
|
108
116
|
jobsBaseTotal: jobsBaseCount,
|
|
109
117
|
jobsPending: aggregateRow.pending,
|
package/src/watch/llm-config.js
CHANGED
|
@@ -133,7 +133,12 @@ export function loadLlmConfig() {
|
|
|
133
133
|
process.env.LLM_MODEL ||
|
|
134
134
|
LLM_PROVIDERS[0].models[0].id;
|
|
135
135
|
_cachedProvider = dbConfig.llm_provider || LLM_PROVIDERS[0].id;
|
|
136
|
-
|
|
136
|
+
// 按 provider 隔离读取 API Key:优先 llm_apikey_{provider},兼容旧 llm_apikey,最后 env
|
|
137
|
+
_cachedApiKey =
|
|
138
|
+
dbConfig[`llm_apikey_${_cachedProvider}`] ||
|
|
139
|
+
dbConfig.llm_apikey ||
|
|
140
|
+
process.env.APIKEY ||
|
|
141
|
+
"";
|
|
137
142
|
_loaded = true;
|
|
138
143
|
|
|
139
144
|
console.error(
|
|
@@ -163,15 +168,45 @@ export function getLlmUrl() {
|
|
|
163
168
|
return _cachedUrl;
|
|
164
169
|
}
|
|
165
170
|
|
|
166
|
-
// Auto
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
171
|
+
// Auto 模式的候选模型池缓存(从 DashScope provider 动态构建,排除 auto 自身)
|
|
172
|
+
let _autoPoolCache = null;
|
|
173
|
+
|
|
174
|
+
/** 从 DashScope provider 动态构建 auto 候选池(按优先级:flash/turbo > plus > 27b/35b > max/pro > 第三方) */
|
|
175
|
+
function buildAutoPool() {
|
|
176
|
+
if (_autoPoolCache) return _autoPoolCache;
|
|
177
|
+
const dashscope = LLM_PROVIDERS.find((p) => p.id === "dashscope");
|
|
178
|
+
if (!dashscope) return [];
|
|
179
|
+
const models = dashscope.models.filter((m) => m.id !== "auto");
|
|
180
|
+
|
|
181
|
+
const priority = (id) => {
|
|
182
|
+
if (/flash|turbo/i.test(id)) return 0;
|
|
183
|
+
if (/plus/i.test(id)) return 1;
|
|
184
|
+
if (/27b|35b/i.test(id)) return 2;
|
|
185
|
+
if (/max|pro/i.test(id)) return 3;
|
|
186
|
+
return 4; // deepseek, glm, kimi 等第三方
|
|
187
|
+
};
|
|
188
|
+
|
|
189
|
+
// 记录原始顺序,排序时优先按 capability tier,同 tier 保持原始顺序
|
|
190
|
+
const indexed = models.map((m, i) => ({ id: m.id, idx: i }));
|
|
191
|
+
indexed.sort((a, b) => {
|
|
192
|
+
const pa = priority(a.id);
|
|
193
|
+
const pb = priority(b.id);
|
|
194
|
+
if (pa !== pb) return pa - pb;
|
|
195
|
+
return a.idx - b.idx;
|
|
196
|
+
});
|
|
197
|
+
|
|
198
|
+
_autoPoolCache = indexed.map((m) => m.id);
|
|
199
|
+
console.error(
|
|
200
|
+
`[llm-config] auto 候选池已构建: ${_autoPoolCache.length} 个模型(${_autoPoolCache.slice(0, 5).join(", ")}...)`,
|
|
201
|
+
);
|
|
202
|
+
return _autoPoolCache;
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
/** 清除 auto 候选池缓存(provider 变更后调用) */
|
|
206
|
+
function clearAutoPoolCache() {
|
|
207
|
+
_autoPoolCache = null;
|
|
208
|
+
}
|
|
209
|
+
|
|
175
210
|
let _lastAutoSuccess = null; // 上次成功的模型,下次优先用它
|
|
176
211
|
|
|
177
212
|
/** 是否处于 auto 模式 */
|
|
@@ -187,7 +222,7 @@ export function recordAutoSuccess(model) {
|
|
|
187
222
|
|
|
188
223
|
/** 获取 auto 模式候选池(上次成功的排最前,失败时依次尝试) */
|
|
189
224
|
export function getAutoPool() {
|
|
190
|
-
const pool = [...
|
|
225
|
+
const pool = [...buildAutoPool()];
|
|
191
226
|
if (_lastAutoSuccess) {
|
|
192
227
|
const idx = pool.indexOf(_lastAutoSuccess);
|
|
193
228
|
if (idx > 0) {
|
|
@@ -202,9 +237,10 @@ export function getAutoPool() {
|
|
|
202
237
|
/** 解析当前实际使用的 model(auto 模式下轮询选一个) */
|
|
203
238
|
function resolveModel() {
|
|
204
239
|
if (_cachedModel !== "auto") return _cachedModel;
|
|
205
|
-
const
|
|
240
|
+
const pool = buildAutoPool();
|
|
241
|
+
const idx = _autoRoundIdx % pool.length;
|
|
206
242
|
_autoRoundIdx++;
|
|
207
|
-
return
|
|
243
|
+
return pool[idx];
|
|
208
244
|
}
|
|
209
245
|
|
|
210
246
|
/** 获取当前 LLM Model(内部调用,auto 模式自动轮询) */
|
|
@@ -226,17 +262,31 @@ export function getLlmApiKey() {
|
|
|
226
262
|
export function updateLlmConfig(updates = {}) {
|
|
227
263
|
if (!_loaded) loadLlmConfig();
|
|
228
264
|
|
|
229
|
-
// Provider 变更:切换 URL,Model
|
|
265
|
+
// Provider 变更:切换 URL,Model 用指定的或默认第一个,同时加载对应 provider 的 API Key
|
|
230
266
|
if (updates.provider !== undefined) {
|
|
231
267
|
const provider = findProvider(updates.provider);
|
|
268
|
+
const prevProvider = _cachedProvider;
|
|
232
269
|
_cachedProvider = provider.id;
|
|
233
270
|
_cachedUrl = provider.url;
|
|
271
|
+
clearAutoPoolCache(); // provider 变了,auto 候选池也需重建
|
|
234
272
|
// 如果同时传了 model 且在 provider 中存在,用指定的;否则用第一个
|
|
235
273
|
const specifiedModel =
|
|
236
274
|
updates.model !== undefined
|
|
237
275
|
? provider.models.find((m) => m.id === updates.model)
|
|
238
276
|
: null;
|
|
239
277
|
_cachedModel = specifiedModel ? specifiedModel.id : provider.models[0].id;
|
|
278
|
+
// 切换到新 provider:从 DB 加载该 provider 的 API Key
|
|
279
|
+
if (provider.id !== prevProvider) {
|
|
280
|
+
const dbConfig = loadFromDb();
|
|
281
|
+
_cachedApiKey =
|
|
282
|
+
dbConfig[`llm_apikey_${provider.id}`] ||
|
|
283
|
+
dbConfig.llm_apikey ||
|
|
284
|
+
process.env.APIKEY ||
|
|
285
|
+
"";
|
|
286
|
+
console.error(
|
|
287
|
+
`[llm-config] 切换到 ${provider.id},API Key: ${_cachedApiKey ? "已加载" : "未设置"}`,
|
|
288
|
+
);
|
|
289
|
+
}
|
|
240
290
|
saveToDb("llm_provider", provider.id);
|
|
241
291
|
saveToDb("llm_url", provider.url);
|
|
242
292
|
saveToDb("llm_model", _cachedModel);
|
|
@@ -252,12 +302,12 @@ export function updateLlmConfig(updates = {}) {
|
|
|
252
302
|
saveToDb("llm_url", updates.url);
|
|
253
303
|
}
|
|
254
304
|
|
|
255
|
-
// API Key
|
|
305
|
+
// API Key(按 provider 隔离存储,跳过提示字符串)
|
|
256
306
|
if (updates.apiKey !== undefined && updates.apiKey !== "") {
|
|
257
307
|
const isHint = updates.apiKey.includes("...") && updates.apiKey.length < 20;
|
|
258
308
|
if (!isHint) {
|
|
259
309
|
_cachedApiKey = updates.apiKey;
|
|
260
|
-
saveToDb(
|
|
310
|
+
saveToDb(`llm_apikey_${_cachedProvider}`, updates.apiKey);
|
|
261
311
|
}
|
|
262
312
|
}
|
|
263
313
|
|
package/src/watch/public/app.js
CHANGED
|
@@ -2289,6 +2289,10 @@ async function loadTargetUsersPage(append = false, skipLoading = false) {
|
|
|
2289
2289
|
if (currentTargetLocation)
|
|
2290
2290
|
url += `&country=${encodeURIComponent(currentTargetLocation)}`;
|
|
2291
2291
|
if (search) url += `&search=${encodeURIComponent(search)}`;
|
|
2292
|
+
if (currentTargetSort.key) {
|
|
2293
|
+
url += `&sort=${encodeURIComponent(currentTargetSort.key)}`;
|
|
2294
|
+
url += `&sortOrder=${currentTargetSort.asc ? "asc" : "desc"}`;
|
|
2295
|
+
}
|
|
2292
2296
|
|
|
2293
2297
|
const res = await fetch(url);
|
|
2294
2298
|
data = await res.json();
|
|
@@ -2329,9 +2333,9 @@ function renderTargetTable() {
|
|
|
2329
2333
|
const el = document.getElementById("targetTable");
|
|
2330
2334
|
const moreHint = document.getElementById("targetMoreHint");
|
|
2331
2335
|
|
|
2332
|
-
//
|
|
2336
|
+
// 应用本地排序(topVideoPlayCount 由服务端排序,此处跳过)
|
|
2333
2337
|
let displayUsers = currentTargetUsers;
|
|
2334
|
-
if (currentTargetSort.key) {
|
|
2338
|
+
if (currentTargetSort.key && currentTargetSort.key !== "topVideoPlayCount") {
|
|
2335
2339
|
displayUsers = [...currentTargetUsers].sort((a, b) => {
|
|
2336
2340
|
let va = a[currentTargetSort.key];
|
|
2337
2341
|
let vb = b[currentTargetSort.key];
|
|
@@ -2378,16 +2382,47 @@ function renderTargetTable() {
|
|
|
2378
2382
|
? formatTime(u.latestVideoTime * 1000)
|
|
2379
2383
|
: "-";
|
|
2380
2384
|
const refreshTime = u.refreshTime ? formatTime(u.refreshTime) : "-";
|
|
2381
|
-
|
|
2382
|
-
|
|
2383
|
-
|
|
2384
|
-
|
|
2385
|
-
|
|
2386
|
-
|
|
2387
|
-
|
|
2388
|
-
|
|
2389
|
-
|
|
2390
|
-
|
|
2385
|
+
|
|
2386
|
+
// 解析 recentEcVideos JSON,获取所有高播放 EC 视频
|
|
2387
|
+
let ecVideos = [];
|
|
2388
|
+
try {
|
|
2389
|
+
if (typeof u.recentEcVideos === "string") {
|
|
2390
|
+
ecVideos = JSON.parse(u.recentEcVideos);
|
|
2391
|
+
} else if (Array.isArray(u.recentEcVideos)) {
|
|
2392
|
+
ecVideos = u.recentEcVideos;
|
|
2393
|
+
}
|
|
2394
|
+
} catch (e) {
|
|
2395
|
+
ecVideos = [];
|
|
2396
|
+
}
|
|
2397
|
+
|
|
2398
|
+
// 最大播放量单元格:列出所有 playCount >= 1000 的视频
|
|
2399
|
+
const highPlayVideos = ecVideos.filter((v) => v.playCount >= 1000);
|
|
2400
|
+
let topPlayCountCell;
|
|
2401
|
+
if (highPlayVideos.length > 0) {
|
|
2402
|
+
const videoLinks = highPlayVideos
|
|
2403
|
+
.sort((a, b) => b.playCount - a.playCount)
|
|
2404
|
+
.map((v) => {
|
|
2405
|
+
const adTag = v.isAd ? ' <span style="color:red">(AD)</span>' : "";
|
|
2406
|
+
return `<a href="${v.href}" target="_blank" style="color:#3b82f6;text-decoration:none;display:block;line-height:1.6" title="点击查看视频">${formatNum(v.playCount)}${adTag}</a>`;
|
|
2407
|
+
})
|
|
2408
|
+
.join("");
|
|
2409
|
+
topPlayCountCell = `<td data-label="最大播放量" style="font-size:11px;color:#888">${videoLinks}</td>`;
|
|
2410
|
+
} else {
|
|
2411
|
+
topPlayCountCell = `<td data-label="最大播放量" style="font-size:11px;color:#888">-</td>`;
|
|
2412
|
+
}
|
|
2413
|
+
|
|
2414
|
+
// 最近发布时间:仅当有 playCount >= 1000 的视频时才显示
|
|
2415
|
+
let latestEcTime;
|
|
2416
|
+
if (highPlayVideos.length > 0) {
|
|
2417
|
+
const maxCreateTime = ecVideos.reduce(
|
|
2418
|
+
(max, v) => (v.createTime > max ? v.createTime : max),
|
|
2419
|
+
0,
|
|
2420
|
+
);
|
|
2421
|
+
if (maxCreateTime > 0) {
|
|
2422
|
+
latestEcTime = formatTime(maxCreateTime * 1000);
|
|
2423
|
+
}
|
|
2424
|
+
}
|
|
2425
|
+
const latestVideoCell = latestEcTime || "-";
|
|
2391
2426
|
|
|
2392
2427
|
const editIcon = ' <span style="font-size:10px;opacity:0.5">✏️</span>';
|
|
2393
2428
|
const locationCell = u.modifiedAt
|
|
@@ -2414,7 +2449,7 @@ function renderTargetTable() {
|
|
|
2414
2449
|
${locationCell}
|
|
2415
2450
|
<td data-label="确认国家" style="font-size:11px">${confirmedLocation}</td>
|
|
2416
2451
|
${topPlayCountCell}
|
|
2417
|
-
<td data-label="最近发布" style="font-size:11px;color:#888">${
|
|
2452
|
+
<td data-label="最近发布" style="font-size:11px;color:#888">${latestVideoCell}</td>
|
|
2418
2453
|
<td data-label="最近刷新" style="font-size:11px;color:#888">${refreshTime}</td>
|
|
2419
2454
|
<td data-label="操作">
|
|
2420
2455
|
<button class="btn-pending" onclick="moveToBase('${escapeJsString(u.uniqueId)}')" title="移到 jobs_base 重走流程">待确认</button>
|
|
@@ -3158,6 +3193,7 @@ function initTableSorting() {
|
|
|
3158
3193
|
}
|
|
3159
3194
|
|
|
3160
3195
|
function initTargetTableSorting() {
|
|
3196
|
+
console.log("[target-sort] 初始化排序监听...");
|
|
3161
3197
|
document.querySelectorAll("th.sortable-target").forEach((th) => {
|
|
3162
3198
|
th.addEventListener("click", () => {
|
|
3163
3199
|
const key = th.dataset.sort;
|
|
@@ -3166,7 +3202,7 @@ function initTargetTableSorting() {
|
|
|
3166
3202
|
currentTargetSort.asc = !currentTargetSort.asc;
|
|
3167
3203
|
} else {
|
|
3168
3204
|
currentTargetSort.key = key;
|
|
3169
|
-
currentTargetSort.asc =
|
|
3205
|
+
currentTargetSort.asc = false; // 默认降序(高播放在前)
|
|
3170
3206
|
}
|
|
3171
3207
|
// 更新排序指示器
|
|
3172
3208
|
document.querySelectorAll("th.sortable-target").forEach((h) => {
|
|
@@ -3177,7 +3213,8 @@ function initTargetTableSorting() {
|
|
|
3177
3213
|
th.classList.add(currentTargetSort.asc ? "sort-asc" : "sort-desc");
|
|
3178
3214
|
const icon = th.querySelector(".sort-icon");
|
|
3179
3215
|
if (icon) icon.textContent = currentTargetSort.asc ? "↑" : "↓";
|
|
3180
|
-
|
|
3216
|
+
// 重新从服务端加载(带排序参数)
|
|
3217
|
+
loadTargetUsersPage(false);
|
|
3181
3218
|
});
|
|
3182
3219
|
});
|
|
3183
3220
|
}
|
|
@@ -3319,7 +3356,7 @@ fetchClientErrors();
|
|
|
3319
3356
|
initTableSorting();
|
|
3320
3357
|
initTargetTableSorting();
|
|
3321
3358
|
|
|
3322
|
-
//
|
|
3359
|
+
// 错峰轮询(服务端已缓存,瞬时响应无压力)
|
|
3323
3360
|
setInterval(fetchStats, 10000);
|
|
3324
3361
|
setTimeout(() => setInterval(fetchUsers, 10000), 2500);
|
|
3325
3362
|
setTimeout(() => setInterval(fetchClientErrors, 10000), 5000);
|
package/src/watch/server.js
CHANGED
|
@@ -27,21 +27,87 @@ const publicDir = join(__dirname, "public");
|
|
|
27
27
|
// Stats 缓存:避免每次请求都扫描 100 万+ 条 jobs 数据
|
|
28
28
|
let statsCache = null;
|
|
29
29
|
let statsCacheTime = 0;
|
|
30
|
-
const STATS_CACHE_TTL =
|
|
30
|
+
const STATS_CACHE_TTL = 60000; // 60 秒缓存
|
|
31
|
+
|
|
32
|
+
// 慢计数缓存:每张大表独立缓存,错峰刷新,避免集中阻塞
|
|
33
|
+
const SLOW_CACHE_TTL = 600000; // 10 分钟
|
|
34
|
+
const slowCaches = {
|
|
35
|
+
rawJobs: { value: 0, time: 0, sql: "SELECT COUNT(*) as c FROM raw_jobs" },
|
|
36
|
+
commentTaskCount: {
|
|
37
|
+
value: 0,
|
|
38
|
+
time: 0,
|
|
39
|
+
sql: "SELECT COUNT(*) as c FROM videos WHERE user_update_count IS NULL OR user_update_count <= 0",
|
|
40
|
+
},
|
|
41
|
+
dbTotalUsers: { value: 0, time: 0, sql: "SELECT COUNT(*) as c FROM users" },
|
|
42
|
+
tagCount: { value: 0, time: 0, sql: "SELECT COUNT(*) as c FROM tags" },
|
|
43
|
+
};
|
|
44
|
+
|
|
45
|
+
function refreshOneSlowCount(st, key) {
|
|
46
|
+
const entry = slowCaches[key];
|
|
47
|
+
if (!entry) return;
|
|
48
|
+
try {
|
|
49
|
+
const result = st.rawQuery(entry.sql);
|
|
50
|
+
entry.value = result?.rows?.[0]?.c || 0;
|
|
51
|
+
entry.time = Date.now();
|
|
52
|
+
} catch (e) {
|
|
53
|
+
console.error(`[stats] 慢计数 ${key} 更新失败: ${e.message}`);
|
|
54
|
+
}
|
|
55
|
+
}
|
|
31
56
|
|
|
32
|
-
function
|
|
33
|
-
|
|
34
|
-
|
|
57
|
+
function getSlowCounts(st) {
|
|
58
|
+
const result = {};
|
|
59
|
+
for (const [key, entry] of Object.entries(slowCaches)) {
|
|
60
|
+
if (Date.now() - entry.time > SLOW_CACHE_TTL) {
|
|
61
|
+
refreshOneSlowCount(st, key);
|
|
62
|
+
}
|
|
63
|
+
result[key] = entry.value;
|
|
64
|
+
}
|
|
65
|
+
return result;
|
|
35
66
|
}
|
|
36
67
|
|
|
68
|
+
let statsRefreshing = false; // 防止并发刷新
|
|
69
|
+
|
|
37
70
|
function computeStatsIncremental(st) {
|
|
38
71
|
const now = Date.now();
|
|
72
|
+
// 缓存有效 → 直接返回
|
|
39
73
|
if (statsCache && now - statsCacheTime < STATS_CACHE_TTL) {
|
|
40
74
|
return statsCache;
|
|
41
75
|
}
|
|
42
|
-
|
|
76
|
+
// 缓存过期但存在 → 先返回旧数据,后台异步刷新
|
|
77
|
+
if (statsCache && !statsRefreshing) {
|
|
78
|
+
statsRefreshing = true;
|
|
79
|
+
setImmediate(async () => {
|
|
80
|
+
try {
|
|
81
|
+
const slowCounts = getSlowCounts(st);
|
|
82
|
+
const stats = st.getDashboardStats(DEFAULT_TARGET_LOCATIONS, {
|
|
83
|
+
skipSlowCounts: true,
|
|
84
|
+
});
|
|
85
|
+
stats.rawJobs = slowCounts.rawJobs;
|
|
86
|
+
stats.commentTaskCount = slowCounts.commentTaskCount;
|
|
87
|
+
stats.dbTotalUsers = slowCounts.dbTotalUsers;
|
|
88
|
+
stats.tagCount = slowCounts.tagCount;
|
|
89
|
+
statsCache = stats;
|
|
90
|
+
statsCacheTime = Date.now();
|
|
91
|
+
} catch (e) {
|
|
92
|
+
console.error(`[stats] 后台刷新失败: ${e.message}`);
|
|
93
|
+
} finally {
|
|
94
|
+
statsRefreshing = false;
|
|
95
|
+
}
|
|
96
|
+
});
|
|
97
|
+
return statsCache;
|
|
98
|
+
}
|
|
99
|
+
// 首次加载 → 同步计算
|
|
100
|
+
const slowCounts = getSlowCounts(st);
|
|
101
|
+
const stats = st.getDashboardStats(DEFAULT_TARGET_LOCATIONS, {
|
|
102
|
+
skipSlowCounts: true,
|
|
103
|
+
});
|
|
104
|
+
stats.rawJobs = slowCounts.rawJobs;
|
|
105
|
+
stats.commentTaskCount = slowCounts.commentTaskCount;
|
|
106
|
+
stats.dbTotalUsers = slowCounts.dbTotalUsers;
|
|
107
|
+
stats.tagCount = slowCounts.tagCount;
|
|
108
|
+
statsCache = stats;
|
|
43
109
|
statsCacheTime = now;
|
|
44
|
-
return
|
|
110
|
+
return stats;
|
|
45
111
|
}
|
|
46
112
|
|
|
47
113
|
function readBody(req) {
|
|
@@ -329,8 +395,24 @@ export function startWatchServer(
|
|
|
329
395
|
}
|
|
330
396
|
|
|
331
397
|
const server = http.createServer(async (req, res) => {
|
|
398
|
+
const startTime = Date.now();
|
|
332
399
|
const { path: routePath, params } = parseQuery(req.url);
|
|
333
400
|
|
|
401
|
+
// 请求耗时日志:仅慢请求打印(>1s)
|
|
402
|
+
res.on("finish", () => {
|
|
403
|
+
const elapsed = Date.now() - startTime;
|
|
404
|
+
if (elapsed <= 1000) return;
|
|
405
|
+
const method = req.method;
|
|
406
|
+
const shortPath =
|
|
407
|
+
routePath.length > 80
|
|
408
|
+
? routePath.substring(0, 80) + "..."
|
|
409
|
+
: routePath;
|
|
410
|
+
const suffix = elapsed > 5000 ? "[SLOW]" : "[WARN]";
|
|
411
|
+
console.error(
|
|
412
|
+
`${method} ${shortPath} → ${res.statusCode} (${elapsed}ms)${suffix}`,
|
|
413
|
+
);
|
|
414
|
+
});
|
|
415
|
+
|
|
334
416
|
// 客户端追踪:从 header 提取 clientId 和元数据,根据路由推断类型
|
|
335
417
|
const clientId = req.headers["x-client-id"] || "";
|
|
336
418
|
if (clientId) {
|
|
@@ -500,7 +582,7 @@ export function startWatchServer(
|
|
|
500
582
|
) {
|
|
501
583
|
try {
|
|
502
584
|
const result = store.moveNoCountryJobsToBase();
|
|
503
|
-
const ts = new Date().
|
|
585
|
+
const ts = new Date().toLocaleTimeString("zh-CN", { hour12: false });
|
|
504
586
|
if (result.ok) {
|
|
505
587
|
console.error(
|
|
506
588
|
`[JOB ${ts}] MOVE-NO-COUNTRY-TO-BASE: moved=${result.moved}, total=${result.total}`,
|
|
@@ -524,7 +606,7 @@ export function startWatchServer(
|
|
|
524
606
|
) {
|
|
525
607
|
try {
|
|
526
608
|
const result = store.moveSellerJobsToBase();
|
|
527
|
-
const ts = new Date().
|
|
609
|
+
const ts = new Date().toLocaleTimeString("zh-CN", { hour12: false });
|
|
528
610
|
if (result.ok) {
|
|
529
611
|
console.error(
|
|
530
612
|
`[JOB ${ts}] MOVE-SELLER-JOBS: jobs=${result.fromJobs}, raw_jobs=${result.fromRawJobs}, reset=${result.resetCount}, available=${result.availableInBase}`,
|
|
@@ -734,7 +816,7 @@ export function startWatchServer(
|
|
|
734
816
|
.filter(Boolean)
|
|
735
817
|
: [];
|
|
736
818
|
const tasks = store.getPendingUserUpdateTasks(limit, countries);
|
|
737
|
-
const ts = new Date().
|
|
819
|
+
const ts = new Date().toLocaleTimeString("zh-CN", { hour12: false });
|
|
738
820
|
console.error(
|
|
739
821
|
`[JOB ${ts}] USER-UPDATE-TASKS: ${tasks.length} tasks${countries.length ? ` (countries: ${countries.join(",")})` : ""}`,
|
|
740
822
|
);
|
|
@@ -749,7 +831,7 @@ export function startWatchServer(
|
|
|
749
831
|
const results = store.batchUpdateUserInfo(updates);
|
|
750
832
|
const okCount = results.filter((r) => r.ok).length;
|
|
751
833
|
const errCount = results.filter((r) => r.error).length;
|
|
752
|
-
const ts = new Date().
|
|
834
|
+
const ts = new Date().toLocaleTimeString("zh-CN", { hour12: false });
|
|
753
835
|
console.error(
|
|
754
836
|
`[JOB ${ts}] USER-INFO-BATCH: ${okCount} ok, ${errCount} error (total=${updates.length})`,
|
|
755
837
|
);
|
|
@@ -775,7 +857,7 @@ export function startWatchServer(
|
|
|
775
857
|
.filter(Boolean)
|
|
776
858
|
: [];
|
|
777
859
|
const tasks = store.getRecheckJobs(limit, countries);
|
|
778
|
-
const ts = new Date().
|
|
860
|
+
const ts = new Date().toLocaleTimeString("zh-CN", { hour12: false });
|
|
779
861
|
console.error(
|
|
780
862
|
`[JOB ${ts}] RECHECK-JOBS: ${tasks.length} tasks${countries.length ? ` (countries: ${countries.join(",")})` : ""}`,
|
|
781
863
|
);
|
|
@@ -790,7 +872,7 @@ export function startWatchServer(
|
|
|
790
872
|
const { results } = store.batchUpdateRecheckJobs(updates);
|
|
791
873
|
const okCount = results.filter((r) => r.ok).length;
|
|
792
874
|
const errCount = results.filter((r) => r.error).length;
|
|
793
|
-
const ts = new Date().
|
|
875
|
+
const ts = new Date().toLocaleTimeString("zh-CN", { hour12: false });
|
|
794
876
|
console.error(
|
|
795
877
|
`[JOB ${ts}] RECHECK-JOBS-BATCH: ${okCount} ok, ${errCount} error (total=${updates.length})`,
|
|
796
878
|
);
|
|
@@ -818,7 +900,7 @@ export function startWatchServer(
|
|
|
818
900
|
sendJSON(res, 404, { error: ret.error });
|
|
819
901
|
return;
|
|
820
902
|
}
|
|
821
|
-
const ts = new Date().
|
|
903
|
+
const ts = new Date().toLocaleTimeString("zh-CN", { hour12: false });
|
|
822
904
|
console.error(
|
|
823
905
|
`[JOB ${ts}] USER-INFO-UPDATE: ${uniqueId} (userUpdateCount=${ret.userUpdateCount})`,
|
|
824
906
|
);
|
|
@@ -849,7 +931,7 @@ export function startWatchServer(
|
|
|
849
931
|
sendJSON(res, 404, { error: ret.error });
|
|
850
932
|
return;
|
|
851
933
|
}
|
|
852
|
-
const ts = new Date().
|
|
934
|
+
const ts = new Date().toLocaleTimeString("zh-CN", { hour12: false });
|
|
853
935
|
console.error(
|
|
854
936
|
`[JOB ${ts}] USER-LOCATION: ${uniqueId} → ${body.location} (modifiedAt=${ret.modifiedAt})`,
|
|
855
937
|
);
|
|
@@ -871,7 +953,7 @@ export function startWatchServer(
|
|
|
871
953
|
sendJSON(res, 404, { error: ret.error });
|
|
872
954
|
return;
|
|
873
955
|
}
|
|
874
|
-
const ts = new Date().
|
|
956
|
+
const ts = new Date().toLocaleTimeString("zh-CN", { hour12: false });
|
|
875
957
|
console.error(`[JOB ${ts}] NON-SELLER: ${uniqueId} → ttSeller=false`);
|
|
876
958
|
sendJSON(res, 200, ret);
|
|
877
959
|
} catch (e) {
|
|
@@ -892,7 +974,7 @@ export function startWatchServer(
|
|
|
892
974
|
sendJSON(res, 404, { error: ret.error });
|
|
893
975
|
return;
|
|
894
976
|
}
|
|
895
|
-
const ts = new Date().
|
|
977
|
+
const ts = new Date().toLocaleTimeString("zh-CN", { hour12: false });
|
|
896
978
|
console.error(`[JOB ${ts}] MOVE-TO-BASE: ${uniqueId} → jobs_base`);
|
|
897
979
|
sendJSON(res, 200, ret);
|
|
898
980
|
} catch (e) {
|
|
@@ -911,7 +993,7 @@ export function startWatchServer(
|
|
|
911
993
|
.filter(Boolean)
|
|
912
994
|
: null;
|
|
913
995
|
const tasks = store.getPendingCommentTasks(limit, locations);
|
|
914
|
-
const ts = new Date().
|
|
996
|
+
const ts = new Date().toLocaleTimeString("zh-CN", { hour12: false });
|
|
915
997
|
console.error(
|
|
916
998
|
`[JOB ${ts}] COMMENT-TASKS: ${tasks.length} tasks${locations ? " (locations: " + locations.join(",") + ")" : ""}`,
|
|
917
999
|
);
|
|
@@ -930,7 +1012,7 @@ export function startWatchServer(
|
|
|
930
1012
|
sendJSON(res, 404, { error: ret.error });
|
|
931
1013
|
return;
|
|
932
1014
|
}
|
|
933
|
-
const ts = new Date().
|
|
1015
|
+
const ts = new Date().toLocaleTimeString("zh-CN", { hour12: false });
|
|
934
1016
|
console.error(
|
|
935
1017
|
`[JOB ${ts}] COMMENT-TASK-COMMIT: ${videoId} (userUpdateCount=${ret.userUpdateCount})`,
|
|
936
1018
|
);
|
|
@@ -993,6 +1075,9 @@ export function startWatchServer(
|
|
|
993
1075
|
followerCount: u.followerCount || 0,
|
|
994
1076
|
topVideoPlayCount: u.topVideoPlayCount || null,
|
|
995
1077
|
topVideoHref: u.topVideoHref || null,
|
|
1078
|
+
topVideoIsAd: u.topVideoIsAd ?? null,
|
|
1079
|
+
recentEcVideos: u.recentEcVideos || null,
|
|
1080
|
+
latestVideoTime: u.latestVideoTime || null,
|
|
996
1081
|
}));
|
|
997
1082
|
sendJSON(res, 200, { total: targets.length, users });
|
|
998
1083
|
}
|
|
@@ -1078,6 +1163,8 @@ export function startWatchServer(
|
|
|
1078
1163
|
search: params.search || undefined,
|
|
1079
1164
|
limit: parseInt(params.limit, 10),
|
|
1080
1165
|
offset: parseInt(params.offset || "0", 10),
|
|
1166
|
+
sort: params.sort || undefined,
|
|
1167
|
+
sortOrder: params.sortOrder || "desc",
|
|
1081
1168
|
},
|
|
1082
1169
|
);
|
|
1083
1170
|
// 确保每个用户对象包含 topVideoPlayCount、topVideoHref 和 topVideoIsAd
|
|
@@ -2360,6 +2447,22 @@ export function startWatchServer(
|
|
|
2360
2447
|
console.error(` 本地访问: http://127.0.0.1:${port}`);
|
|
2361
2448
|
console.error(` 局域网访问: http://${localIP}:${port}`);
|
|
2362
2449
|
|
|
2450
|
+
// 预热 stats 缓存 + 错峰后台更新慢计数(每张表独立,间隔 2.5 分钟)
|
|
2451
|
+
setTimeout(() => {
|
|
2452
|
+
for (const key of Object.keys(slowCaches))
|
|
2453
|
+
refreshOneSlowCount(store, key);
|
|
2454
|
+
computeStatsIncremental(store);
|
|
2455
|
+
}, 3000);
|
|
2456
|
+
const slowKeys = Object.keys(slowCaches);
|
|
2457
|
+
slowKeys.forEach((key, i) => {
|
|
2458
|
+
setInterval(() => refreshOneSlowCount(store, key), SLOW_CACHE_TTL);
|
|
2459
|
+
// 首次错峰:各表间隔 2.5 分钟启动
|
|
2460
|
+
setTimeout(
|
|
2461
|
+
() => refreshOneSlowCount(store, key),
|
|
2462
|
+
3000 + (i + 1) * 150000,
|
|
2463
|
+
);
|
|
2464
|
+
});
|
|
2465
|
+
|
|
2363
2466
|
// 启动时清理超时的 scoring 标签
|
|
2364
2467
|
try {
|
|
2365
2468
|
const { resetStaleScoringTags } = store;
|
|
@@ -2396,7 +2499,7 @@ export function startWatchServer(
|
|
|
2396
2499
|
setInterval(() => {
|
|
2397
2500
|
try {
|
|
2398
2501
|
const result = store.moveNoCountryJobsToBase();
|
|
2399
|
-
const ts = new Date().
|
|
2502
|
+
const ts = new Date().toLocaleTimeString("zh-CN", { hour12: false });
|
|
2400
2503
|
if (result.moved > 0) {
|
|
2401
2504
|
console.error(
|
|
2402
2505
|
`[JOB ${ts}] AUTO-MOVE-NO-COUNTRY: moved=${result.moved}, total=${result.total}`,
|