tt-help-cli-ycl 1.4.19 → 1.4.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/cli/recheck.js +381 -0
- package/src/cli/tag.js +1 -1
- package/src/lib/args.js +52 -0
- package/src/lib/constants.js +11 -0
- package/src/main.js +3 -0
- package/src/npm-main.js +3 -0
- package/src/watch/data-store.js +153 -1
- package/src/watch/db-tags.js +6 -6
- package/src/watch/public/app.js +2 -0
- package/src/watch/server.js +42 -0
package/package.json
CHANGED
|
@@ -0,0 +1,381 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* recheck — 重检 jobs 表中已完成的非商家用户
|
|
3
|
+
*
|
|
4
|
+
* 基于 attach.js 的核心循环,但任务源和提交逻辑不同:
|
|
5
|
+
* - 拉任务:GET /api/recheck-jobs(只取 status='done' 且 tt_seller!=1 且 7 天未更新的用户)
|
|
6
|
+
* - 提交结果:POST /api/recheck-jobs-batch(直接 UPDATE jobs 表,不做表迁移)
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
import { TikTokScraper } from "../lib/tiktok-scraper.mjs";
|
|
10
|
+
import { CDNBlockedError } from "../lib/parse-ssr.mjs";
|
|
11
|
+
import { proxy as configuredProxy } from "../lib/constants.js";
|
|
12
|
+
import { createApiClient } from "../lib/api-client.js";
|
|
13
|
+
import v8 from "node:v8";
|
|
14
|
+
|
|
15
|
+
const HEAP_RESTART_RATIO = 0.72;
|
|
16
|
+
const MAX_TASK_BATCHES_BEFORE_RESTART = 200;
|
|
17
|
+
|
|
18
|
+
function formatNow() {
|
|
19
|
+
return new Date().toLocaleString("zh-CN", {
|
|
20
|
+
hour12: false,
|
|
21
|
+
});
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
function recheckLog(message = "") {
|
|
25
|
+
console.error(`[${formatNow()}] ${message}`);
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
function isBrowserClosedError(err) {
|
|
29
|
+
if (!err) return false;
|
|
30
|
+
const msg = err.message || err.toString() || "";
|
|
31
|
+
return (
|
|
32
|
+
msg.includes("Target page, context or browser has been closed") ||
|
|
33
|
+
msg.includes("browser has been closed") ||
|
|
34
|
+
msg.includes("browserContext.newPage") ||
|
|
35
|
+
msg.includes("Protocol error")
|
|
36
|
+
);
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
function getHeapUsage() {
|
|
40
|
+
const heapUsed = process.memoryUsage().heapUsed;
|
|
41
|
+
const heapLimit = v8.getHeapStatistics().heap_size_limit;
|
|
42
|
+
return {
|
|
43
|
+
heapUsed,
|
|
44
|
+
heapLimit,
|
|
45
|
+
ratio: heapLimit > 0 ? heapUsed / heapLimit : 0,
|
|
46
|
+
};
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
async function recycleScraper(scraper, reason) {
|
|
50
|
+
recheckLog(`[Recheck] ${reason},正在重启浏览器实例...`);
|
|
51
|
+
await scraper.restart();
|
|
52
|
+
recheckLog("[Recheck] 浏览器实例重启完成");
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
export async function handleRecheck(options) {
|
|
56
|
+
const {
|
|
57
|
+
attachParallel,
|
|
58
|
+
attachInterval,
|
|
59
|
+
serverUrl,
|
|
60
|
+
attachCountries,
|
|
61
|
+
customProxy,
|
|
62
|
+
attachPoolSize,
|
|
63
|
+
showHelp,
|
|
64
|
+
} = options;
|
|
65
|
+
const effectiveProxy = customProxy || configuredProxy;
|
|
66
|
+
|
|
67
|
+
if (effectiveProxy) {
|
|
68
|
+
recheckLog(`[Recheck] 使用代理: ${effectiveProxy}`);
|
|
69
|
+
}
|
|
70
|
+
let shuttingDown = false;
|
|
71
|
+
let forceExitTimer = null;
|
|
72
|
+
|
|
73
|
+
if (showHelp) {
|
|
74
|
+
recheckLog(
|
|
75
|
+
"用法: tt-help recheck [-p 并行数] [-i 间隔秒数] [-s 服务端地址] [-c 国家列表]",
|
|
76
|
+
);
|
|
77
|
+
recheckLog("");
|
|
78
|
+
recheckLog("参数:");
|
|
79
|
+
recheckLog(" -p, --parallel <N> 并行抓取数(默认: 1)");
|
|
80
|
+
recheckLog(" -i, --interval <N> 无任务时轮询间隔,单位秒(默认: 10)");
|
|
81
|
+
recheckLog(
|
|
82
|
+
" -s, --server <URL> 服务端地址(默认: http://127.0.0.1:3001)",
|
|
83
|
+
);
|
|
84
|
+
recheckLog(
|
|
85
|
+
" -c, --countries <A,B,C> 目标国家列表(逗号分隔,如 PL,DE,FR),默认全部目标国家",
|
|
86
|
+
);
|
|
87
|
+
recheckLog(
|
|
88
|
+
" --proxy <代理地址> HTTP 代理地址(如 http://127.0.0.1:7890),不配置则从 ~/.tt-help.json 读取",
|
|
89
|
+
);
|
|
90
|
+
recheckLog(
|
|
91
|
+
" --size <N> 浏览器页面池大小(默认: 3,不建议超过 3)",
|
|
92
|
+
);
|
|
93
|
+
recheckLog("");
|
|
94
|
+
recheckLog("说明:");
|
|
95
|
+
recheckLog(
|
|
96
|
+
" 从 jobs 表中拉取 status='done' 且非商家(tt_seller!=1)且 7 天未更新的用户,",
|
|
97
|
+
);
|
|
98
|
+
recheckLog(
|
|
99
|
+
" 重新抓取 TikTok 用户基本信息(重点检查 tt_seller),更新回 jobs 表。",
|
|
100
|
+
);
|
|
101
|
+
recheckLog(
|
|
102
|
+
" 每 10 轮打印汇总:已跑轮数、发现 ttSeller 数量、累计检测数。",
|
|
103
|
+
);
|
|
104
|
+
recheckLog(" 一轮跑完后自动空转,7 天后可重新运行。");
|
|
105
|
+
recheckLog(" 支持多客户端并行,服务端自动分配不同用户。");
|
|
106
|
+
recheckLog("");
|
|
107
|
+
recheckLog("示例:");
|
|
108
|
+
recheckLog(" tt-help recheck");
|
|
109
|
+
recheckLog(" tt-help recheck -p 5 -i 10");
|
|
110
|
+
recheckLog(" tt-help recheck -c PL,DE,FR -p 5");
|
|
111
|
+
recheckLog(" tt-help recheck -p 5 --size 5");
|
|
112
|
+
return;
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
const countryStr =
|
|
116
|
+
attachCountries && attachCountries.length > 0
|
|
117
|
+
? `, 目标国家: ${attachCountries.join(", ")}`
|
|
118
|
+
: ", 目标国家: 全部";
|
|
119
|
+
recheckLog(
|
|
120
|
+
`[Recheck] 并行数: ${attachParallel}, 空闲间隔: ${attachInterval}秒, 服务端: ${serverUrl}${countryStr}`,
|
|
121
|
+
);
|
|
122
|
+
|
|
123
|
+
const { apiGet, apiPost } = createApiClient({ log: false });
|
|
124
|
+
|
|
125
|
+
const scraper = new TikTokScraper({
|
|
126
|
+
poolSize: attachPoolSize || 3,
|
|
127
|
+
proxyServer: effectiveProxy || null,
|
|
128
|
+
});
|
|
129
|
+
|
|
130
|
+
const shutdown = async (signal) => {
|
|
131
|
+
if (shuttingDown) return;
|
|
132
|
+
shuttingDown = true;
|
|
133
|
+
forceExitTimer = setTimeout(() => {
|
|
134
|
+
recheckLog("[Recheck] 关闭超时,强制退出");
|
|
135
|
+
process.exit(0);
|
|
136
|
+
}, 30000);
|
|
137
|
+
forceExitTimer.unref?.();
|
|
138
|
+
recheckLog(`\n[Recheck] 收到 ${signal},等待当前任务完成后退出...`);
|
|
139
|
+
};
|
|
140
|
+
|
|
141
|
+
const onSigint = () => {
|
|
142
|
+
void shutdown("SIGINT");
|
|
143
|
+
};
|
|
144
|
+
const onSigterm = () => {
|
|
145
|
+
void shutdown("SIGTERM");
|
|
146
|
+
};
|
|
147
|
+
|
|
148
|
+
process.on("SIGINT", onSigint);
|
|
149
|
+
process.on("SIGTERM", onSigterm);
|
|
150
|
+
|
|
151
|
+
try {
|
|
152
|
+
await scraper.init();
|
|
153
|
+
recheckLog("[Recheck] 浏览器初始化完成,开始循环接收任务...");
|
|
154
|
+
|
|
155
|
+
let loopCount = 0;
|
|
156
|
+
let browserRestartCount = 0;
|
|
157
|
+
let taskBatchCount = 0;
|
|
158
|
+
let totalChecked = 0;
|
|
159
|
+
let totalSellerFound = 0;
|
|
160
|
+
|
|
161
|
+
while (!shuttingDown) {
|
|
162
|
+
loopCount++;
|
|
163
|
+
|
|
164
|
+
// 检查浏览器是否存活
|
|
165
|
+
if (!scraper.isAlive()) {
|
|
166
|
+
recheckLog(
|
|
167
|
+
`[Recheck] 浏览器已关闭,正在重启 (${++browserRestartCount})...`,
|
|
168
|
+
);
|
|
169
|
+
await scraper.restart();
|
|
170
|
+
recheckLog("[Recheck] 浏览器重启完成");
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
// 收到退出信号且当前无任务正在处理,则退出
|
|
174
|
+
if (shuttingDown) {
|
|
175
|
+
recheckLog("[Recheck] 当前任务已完成,正在关闭浏览器...");
|
|
176
|
+
await scraper.close().catch(() => {});
|
|
177
|
+
if (forceExitTimer) {
|
|
178
|
+
clearTimeout(forceExitTimer);
|
|
179
|
+
forceExitTimer = null;
|
|
180
|
+
}
|
|
181
|
+
recheckLog(
|
|
182
|
+
`[Recheck] 已退出,共检测 ${totalChecked} 个用户,发现 ttSeller: ${totalSellerFound}`,
|
|
183
|
+
);
|
|
184
|
+
process.exit(0);
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
const countryParam =
|
|
188
|
+
attachCountries && attachCountries.length > 0
|
|
189
|
+
? `&countries=${encodeURIComponent(attachCountries.join(","))}`
|
|
190
|
+
: "";
|
|
191
|
+
const { total, tasks } = await apiGet(
|
|
192
|
+
`${serverUrl}/api/recheck-jobs?limit=${attachParallel}${countryParam}`,
|
|
193
|
+
);
|
|
194
|
+
|
|
195
|
+
if (!tasks || tasks.length === 0) {
|
|
196
|
+
const heap = getHeapUsage();
|
|
197
|
+
if (heap.ratio >= HEAP_RESTART_RATIO) {
|
|
198
|
+
await recycleScraper(
|
|
199
|
+
scraper,
|
|
200
|
+
`空闲时堆占用 ${(heap.ratio * 100).toFixed(1)}%`,
|
|
201
|
+
);
|
|
202
|
+
}
|
|
203
|
+
if (loopCount === 1) {
|
|
204
|
+
recheckLog(
|
|
205
|
+
`[Recheck] 当前无待重检任务(7 天内已全部检测过或无非商家用户),${attachInterval}秒后重试...`,
|
|
206
|
+
);
|
|
207
|
+
}
|
|
208
|
+
await new Promise((r) => setTimeout(r, attachInterval * 1000));
|
|
209
|
+
continue;
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
recheckLog(`\n[Recheck] 获取到 ${tasks.length} 个待重检任务...`);
|
|
213
|
+
taskBatchCount++;
|
|
214
|
+
|
|
215
|
+
if (shuttingDown) {
|
|
216
|
+
recheckLog("[Recheck] 正在处理当前批次任务,完成后退出...");
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
const results = await Promise.allSettled(
|
|
220
|
+
tasks.map(async (task) => {
|
|
221
|
+
const uniqueId = task.uniqueId;
|
|
222
|
+
const countryTag = task.guessedLocation
|
|
223
|
+
? ` [国家: ${task.guessedLocation}]`
|
|
224
|
+
: "";
|
|
225
|
+
recheckLog(` → 重检 @${uniqueId}...${countryTag}`);
|
|
226
|
+
try {
|
|
227
|
+
const info = await scraper.getUserInfo(uniqueId);
|
|
228
|
+
return { uniqueId, info, error: null };
|
|
229
|
+
} catch (err) {
|
|
230
|
+
return { uniqueId, info: null, error: err };
|
|
231
|
+
}
|
|
232
|
+
}),
|
|
233
|
+
);
|
|
234
|
+
|
|
235
|
+
let successCount = 0;
|
|
236
|
+
let failCount = 0;
|
|
237
|
+
let cdnBlockedCount = 0;
|
|
238
|
+
let needRestart = false;
|
|
239
|
+
|
|
240
|
+
const successTasks = [];
|
|
241
|
+
for (const result of results) {
|
|
242
|
+
if (result.status === "fulfilled") {
|
|
243
|
+
const { uniqueId, info, error } = result.value;
|
|
244
|
+
if (error) {
|
|
245
|
+
if (error instanceof CDNBlockedError) {
|
|
246
|
+
recheckLog(` ⚠ @${uniqueId} CDN限流 (Access Denied)`);
|
|
247
|
+
cdnBlockedCount++;
|
|
248
|
+
failCount++;
|
|
249
|
+
} else if (isBrowserClosedError(error)) {
|
|
250
|
+
needRestart = true;
|
|
251
|
+
recheckLog(` ✗ @${uniqueId} 浏览器断开: ${error.message}`);
|
|
252
|
+
failCount++;
|
|
253
|
+
} else {
|
|
254
|
+
recheckLog(
|
|
255
|
+
` ✗ @${uniqueId} 获取失败: ${error.message || "未知错误"}`,
|
|
256
|
+
);
|
|
257
|
+
failCount++;
|
|
258
|
+
}
|
|
259
|
+
} else if (info) {
|
|
260
|
+
if (info.error) {
|
|
261
|
+
// TikTok 返回了明确状态码(如 404),也提交记录
|
|
262
|
+
successTasks.push({ uniqueId, info });
|
|
263
|
+
} else {
|
|
264
|
+
successTasks.push({ uniqueId, info });
|
|
265
|
+
}
|
|
266
|
+
} else {
|
|
267
|
+
recheckLog(` ✗ @${uniqueId} 未获取到用户信息`);
|
|
268
|
+
failCount++;
|
|
269
|
+
}
|
|
270
|
+
} else {
|
|
271
|
+
recheckLog(
|
|
272
|
+
` ✗ 任务执行异常: ${result.reason?.message || "未知错误"}`,
|
|
273
|
+
);
|
|
274
|
+
failCount++;
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
// 批量提交结果
|
|
279
|
+
if (successTasks.length > 0) {
|
|
280
|
+
try {
|
|
281
|
+
const batchRet = await apiPost(
|
|
282
|
+
`${serverUrl}/api/recheck-jobs-batch`,
|
|
283
|
+
{ updates: successTasks },
|
|
284
|
+
);
|
|
285
|
+
if (batchRet && batchRet.results && Array.isArray(batchRet.results)) {
|
|
286
|
+
for (const r of batchRet.results) {
|
|
287
|
+
if (r.ok) {
|
|
288
|
+
successCount++;
|
|
289
|
+
totalChecked++;
|
|
290
|
+
const task = successTasks.find(
|
|
291
|
+
(t) => t.uniqueId === r.uniqueId,
|
|
292
|
+
);
|
|
293
|
+
const info = task?.info;
|
|
294
|
+
const isSeller = info?.ttSeller;
|
|
295
|
+
const sellerTag = isSeller ? " [ttSeller=true ✓]" : "";
|
|
296
|
+
if (isSeller) totalSellerFound++;
|
|
297
|
+
if (info && info.error) {
|
|
298
|
+
recheckLog(
|
|
299
|
+
` ⚠ @${r.uniqueId} 已跳过 (statusCode=${info.statusCode})`,
|
|
300
|
+
);
|
|
301
|
+
} else {
|
|
302
|
+
recheckLog(` ✓${sellerTag} @${r.uniqueId} 已更新`);
|
|
303
|
+
}
|
|
304
|
+
} else {
|
|
305
|
+
failCount++;
|
|
306
|
+
recheckLog(` ✗ @${r.uniqueId} 更新失败: ${r.error}`);
|
|
307
|
+
}
|
|
308
|
+
}
|
|
309
|
+
} else {
|
|
310
|
+
recheckLog(` ⚠ 后端响应格式异常,降级为批量成功`);
|
|
311
|
+
successCount = successTasks.length;
|
|
312
|
+
totalChecked += successTasks.length;
|
|
313
|
+
}
|
|
314
|
+
} catch (err) {
|
|
315
|
+
failCount += successTasks.length;
|
|
316
|
+
recheckLog(` ✗ 批量提交失败: ${err.message}`);
|
|
317
|
+
}
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
recheckLog(
|
|
321
|
+
` 本批结果: ${successCount} 成功, ${failCount} 失败 (累计: ${totalChecked})`,
|
|
322
|
+
);
|
|
323
|
+
|
|
324
|
+
// 每 10 轮打印汇总
|
|
325
|
+
if (taskBatchCount % 10 === 0) {
|
|
326
|
+
recheckLog(
|
|
327
|
+
` ── 已跑 ${taskBatchCount} 轮,发现 ttSeller: ${totalSellerFound},累计检测: ${totalChecked}`,
|
|
328
|
+
);
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
// CDN 限流比例超过 30% 时,冷却 + 重启浏览器
|
|
332
|
+
const cdnRatio = tasks.length > 0 ? cdnBlockedCount / tasks.length : 0;
|
|
333
|
+
if (cdnRatio > 0.3) {
|
|
334
|
+
const coolDownSeconds = cdnRatio > 0.8 ? 120 : 60;
|
|
335
|
+
recheckLog(
|
|
336
|
+
` [Recheck] CDN限流比例 ${(cdnRatio * 100).toFixed(0)}% (${cdnBlockedCount}/${tasks.length}),冷却 ${coolDownSeconds} 秒后重启浏览器...`,
|
|
337
|
+
);
|
|
338
|
+
await new Promise((r) => setTimeout(r, coolDownSeconds * 1000));
|
|
339
|
+
await recycleScraper(
|
|
340
|
+
scraper,
|
|
341
|
+
`CDN限流比例过高 (${cdnBlockedCount}/${tasks.length})`,
|
|
342
|
+
);
|
|
343
|
+
browserRestartCount++;
|
|
344
|
+
taskBatchCount = 0;
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
const heap = getHeapUsage();
|
|
348
|
+
if (heap.ratio >= HEAP_RESTART_RATIO) {
|
|
349
|
+
await recycleScraper(
|
|
350
|
+
scraper,
|
|
351
|
+
`本批后堆占用 ${(heap.ratio * 100).toFixed(1)}%`,
|
|
352
|
+
);
|
|
353
|
+
browserRestartCount++;
|
|
354
|
+
taskBatchCount = 0;
|
|
355
|
+
} else if (taskBatchCount >= MAX_TASK_BATCHES_BEFORE_RESTART) {
|
|
356
|
+
await recycleScraper(scraper, `已连续处理 ${taskBatchCount} 批任务`);
|
|
357
|
+
browserRestartCount++;
|
|
358
|
+
taskBatchCount = 0;
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
if (needRestart) {
|
|
362
|
+
recheckLog("[Recheck] 检测到浏览器异常,将在下一轮重启...");
|
|
363
|
+
}
|
|
364
|
+
|
|
365
|
+
await new Promise((r) => setTimeout(r, 500));
|
|
366
|
+
}
|
|
367
|
+
} catch (err) {
|
|
368
|
+
recheckLog(`[Recheck] 运行异常: ${err.message}`);
|
|
369
|
+
throw err;
|
|
370
|
+
} finally {
|
|
371
|
+
if (forceExitTimer) {
|
|
372
|
+
clearTimeout(forceExitTimer);
|
|
373
|
+
forceExitTimer = null;
|
|
374
|
+
}
|
|
375
|
+
process.removeListener("SIGINT", onSigint);
|
|
376
|
+
process.removeListener("SIGTERM", onSigterm);
|
|
377
|
+
if (!shuttingDown) {
|
|
378
|
+
await scraper.close().catch(() => {});
|
|
379
|
+
}
|
|
380
|
+
}
|
|
381
|
+
}
|
package/src/cli/tag.js
CHANGED
|
@@ -1424,7 +1424,7 @@ export async function handleTagRefresh(parsed) {
|
|
|
1424
1424
|
cdpPort +
|
|
1425
1425
|
(effectiveProxy ? " 代理: " + effectiveProxy : ""),
|
|
1426
1426
|
);
|
|
1427
|
-
log("║ 只抓取不重新打分,score>
|
|
1427
|
+
log("║ 只抓取不重新打分,score>60 的标签");
|
|
1428
1428
|
log("╚══════════════════════════════════════════╝");
|
|
1429
1429
|
log("");
|
|
1430
1430
|
|
package/src/lib/args.js
CHANGED
|
@@ -604,6 +604,54 @@ function parseAttachArgs(args) {
|
|
|
604
604
|
};
|
|
605
605
|
}
|
|
606
606
|
|
|
607
|
+
function parseRecheckArgs(args) {
|
|
608
|
+
let parallel = 1;
|
|
609
|
+
let interval = 10;
|
|
610
|
+
let serverUrl = defaultServer;
|
|
611
|
+
let countries = [];
|
|
612
|
+
let customProxy = null;
|
|
613
|
+
let poolSize = 3;
|
|
614
|
+
|
|
615
|
+
for (let i = 0; i < args.length; i++) {
|
|
616
|
+
const arg = args[i];
|
|
617
|
+
if (arg === "-p" || arg === "--parallel") {
|
|
618
|
+
parallel = parseInt(args[++i], 10) || 1;
|
|
619
|
+
} else if (arg === "-i" || arg === "--interval") {
|
|
620
|
+
interval = parseInt(args[++i], 10) || 10;
|
|
621
|
+
} else if (arg === "-s" || arg === "--server") {
|
|
622
|
+
serverUrl = args[++i];
|
|
623
|
+
} else if (arg === "-c" || arg === "--countries") {
|
|
624
|
+
countries = args[++i]
|
|
625
|
+
.split(",")
|
|
626
|
+
.map((c) => c.trim().toUpperCase())
|
|
627
|
+
.filter(Boolean);
|
|
628
|
+
} else if (arg === "--proxy") {
|
|
629
|
+
customProxy = args[++i];
|
|
630
|
+
} else if (arg === "--size") {
|
|
631
|
+
poolSize = parseInt(args[++i], 10) || 3;
|
|
632
|
+
}
|
|
633
|
+
}
|
|
634
|
+
|
|
635
|
+
return {
|
|
636
|
+
subcommand: "recheck",
|
|
637
|
+
attachParallel: parallel,
|
|
638
|
+
attachInterval: interval,
|
|
639
|
+
serverUrl,
|
|
640
|
+
attachCountries: countries,
|
|
641
|
+
customProxy,
|
|
642
|
+
attachPoolSize: poolSize,
|
|
643
|
+
urls: [],
|
|
644
|
+
outputFormat: "json",
|
|
645
|
+
exploreCount: 0,
|
|
646
|
+
showConfig: false,
|
|
647
|
+
showHelp: false,
|
|
648
|
+
configAction: null,
|
|
649
|
+
configValue: null,
|
|
650
|
+
pipeMode: false,
|
|
651
|
+
filterStr: null,
|
|
652
|
+
};
|
|
653
|
+
}
|
|
654
|
+
|
|
607
655
|
function parseOpenArgs(args) {
|
|
608
656
|
let openPort = null;
|
|
609
657
|
let openList = false;
|
|
@@ -1029,6 +1077,10 @@ export function parseArgs() {
|
|
|
1029
1077
|
return parseAttachArgs(args.slice(1));
|
|
1030
1078
|
}
|
|
1031
1079
|
|
|
1080
|
+
if (args.length > 0 && args[0] === "recheck") {
|
|
1081
|
+
return parseRecheckArgs(args.slice(1));
|
|
1082
|
+
}
|
|
1083
|
+
|
|
1032
1084
|
if (args.length > 0 && args[0] === "open") {
|
|
1033
1085
|
return parseOpenArgs(args.slice(1));
|
|
1034
1086
|
}
|
package/src/lib/constants.js
CHANGED
|
@@ -177,6 +177,17 @@ const HELP_TEXT = [
|
|
|
177
177
|
" 示例: tt-help attach -p 5 -i 10",
|
|
178
178
|
" tt-help attach -c PL,DE,FR -p 5",
|
|
179
179
|
"",
|
|
180
|
+
" tt-help recheck [-p 并行数] [-i 间隔秒数] [-s 服务端地址] [-c 国家列表]",
|
|
181
|
+
" 重检 jobs 表中 done 非商家用户(tt_seller!=1),7 天未更新才检测",
|
|
182
|
+
" 重新抓取用户基本信息(重点检查 tt_seller),更新回 jobs 表",
|
|
183
|
+
" -p, --parallel <N> 并行抓取数(默认: 1)",
|
|
184
|
+
" -i, --interval <N> 无任务时轮询间隔,单位秒(默认: 10)",
|
|
185
|
+
" -s, --server <URL> 服务端地址(默认: http://127.0.0.1:3001)",
|
|
186
|
+
" -c, --countries <A,B> 目标国家列表(逗号分隔,如 PL,DE,FR),默认全部",
|
|
187
|
+
" 示例: tt-help recheck -p 5 -i 10",
|
|
188
|
+
" tt-help recheck -c PL,DE,FR -p 5",
|
|
189
|
+
" tt-help recheck -p 5 --size 5",
|
|
190
|
+
"",
|
|
180
191
|
" tt-help watch -o <db路径> [-p 端口] [--no-llm-refill] [--no-llm-discover]",
|
|
181
192
|
" 启动监控服务;运行期主数据仅来自 SQLite",
|
|
182
193
|
" 选项:",
|
package/src/main.js
CHANGED
|
@@ -3,6 +3,7 @@ import { proxy, HELP_TEXT, getConfigText } from "./lib/constants.js";
|
|
|
3
3
|
import { handleInfo } from "./cli/info.js";
|
|
4
4
|
import { handleExplore } from "./cli/explore.js";
|
|
5
5
|
import { handleAttach } from "./cli/attach.js";
|
|
6
|
+
import { handleRecheck } from "./cli/recheck.js";
|
|
6
7
|
import { handleWatch } from "./cli/watch.js";
|
|
7
8
|
import { handleConfig, showConfig, showUsage, version } from "./cli/config.js";
|
|
8
9
|
import { handleOpen } from "./cli/open.js";
|
|
@@ -29,6 +30,8 @@ async function main() {
|
|
|
29
30
|
return handleInfo(parsed);
|
|
30
31
|
case "attach":
|
|
31
32
|
return handleAttach(parsed);
|
|
33
|
+
case "recheck":
|
|
34
|
+
return handleRecheck(parsed);
|
|
32
35
|
case "watch":
|
|
33
36
|
return handleWatch(parsed);
|
|
34
37
|
case "webserver":
|
package/src/npm-main.js
CHANGED
|
@@ -3,6 +3,7 @@ import { PUBLIC_HELP_TEXT } from "./lib/constants.js";
|
|
|
3
3
|
import { handleInfo } from "./cli/info.js";
|
|
4
4
|
import { handleExplore } from "./cli/explore.js";
|
|
5
5
|
import { handleAttach } from "./cli/attach.js";
|
|
6
|
+
import { handleRecheck } from "./cli/recheck.js";
|
|
6
7
|
import { handleConfig, showConfig, showUsage, version } from "./cli/config.js";
|
|
7
8
|
import { handleOpen } from "./cli/open.js";
|
|
8
9
|
import { handleComments } from "./cli/comments.js";
|
|
@@ -32,6 +33,8 @@ async function main() {
|
|
|
32
33
|
return handleInfo(parsed);
|
|
33
34
|
case "attach":
|
|
34
35
|
return handleAttach(parsed);
|
|
36
|
+
case "recheck":
|
|
37
|
+
return handleRecheck(parsed);
|
|
35
38
|
case "watch":
|
|
36
39
|
case "webserver":
|
|
37
40
|
case "videostats":
|
package/src/watch/data-store.js
CHANGED
|
@@ -2735,7 +2735,20 @@ export function createStore(filePath, options = {}) {
|
|
|
2735
2735
|
}
|
|
2736
2736
|
|
|
2737
2737
|
function getClientErrors() {
|
|
2738
|
-
|
|
2738
|
+
const now = Date.now();
|
|
2739
|
+
const expired = [];
|
|
2740
|
+
const result = [];
|
|
2741
|
+
for (const [userId, err] of clientErrors) {
|
|
2742
|
+
if (now - err.timestamp > 10 * 60 * 1000) {
|
|
2743
|
+
expired.push(userId);
|
|
2744
|
+
} else {
|
|
2745
|
+
result.push(err);
|
|
2746
|
+
}
|
|
2747
|
+
}
|
|
2748
|
+
for (const userId of expired) {
|
|
2749
|
+
clientErrors.delete(userId);
|
|
2750
|
+
}
|
|
2751
|
+
return result;
|
|
2739
2752
|
}
|
|
2740
2753
|
|
|
2741
2754
|
function getClientLoginStatus() {
|
|
@@ -2875,6 +2888,143 @@ export function createStore(filePath, options = {}) {
|
|
|
2875
2888
|
return pending;
|
|
2876
2889
|
}
|
|
2877
2890
|
|
|
2891
|
+
// ===== recheck-jobs:重检 jobs 表中 done 非商家用户 =====
|
|
2892
|
+
|
|
2893
|
+
function getRecheckJobs(limit, countries) {
|
|
2894
|
+
const targetCountries = countries
|
|
2895
|
+
? countries.map((c) => String(c).trim().toUpperCase())
|
|
2896
|
+
: [];
|
|
2897
|
+
const hasCountryFilter = targetCountries.length > 0;
|
|
2898
|
+
|
|
2899
|
+
if (getDb()) {
|
|
2900
|
+
const l = Math.max(1, parseInt(limit) || 5);
|
|
2901
|
+
const sevenDaysAgo = Date.now() - 7 * 24 * 60 * 60 * 1000;
|
|
2902
|
+
|
|
2903
|
+
let sql = `
|
|
2904
|
+
SELECT *
|
|
2905
|
+
FROM jobs
|
|
2906
|
+
WHERE status = 'done'
|
|
2907
|
+
AND (tt_seller IS NULL OR tt_seller != 1)
|
|
2908
|
+
AND (updated_at IS NULL OR updated_at < ?)
|
|
2909
|
+
`;
|
|
2910
|
+
const sqlParams = [sevenDaysAgo];
|
|
2911
|
+
|
|
2912
|
+
if (hasCountryFilter) {
|
|
2913
|
+
const placeholders = targetCountries.map(() => "?").join(", ");
|
|
2914
|
+
sql += ` AND UPPER(COALESCE(guessed_location, '')) IN (${placeholders})`;
|
|
2915
|
+
sqlParams.push(...targetCountries);
|
|
2916
|
+
}
|
|
2917
|
+
|
|
2918
|
+
sql += ` ORDER BY updated_at ASC LIMIT ?`;
|
|
2919
|
+
sqlParams.push(l);
|
|
2920
|
+
|
|
2921
|
+
const rows = getDb()
|
|
2922
|
+
.prepare(sql)
|
|
2923
|
+
.all(...sqlParams);
|
|
2924
|
+
if (rows.length === 0) return [];
|
|
2925
|
+
|
|
2926
|
+
const now = Date.now();
|
|
2927
|
+
const bumpStmt = getDb().prepare(
|
|
2928
|
+
`UPDATE jobs SET updated_at = ? WHERE unique_id = ?`,
|
|
2929
|
+
);
|
|
2930
|
+
const bumpTxn = getDb().transaction((items) => {
|
|
2931
|
+
for (const item of items) {
|
|
2932
|
+
bumpStmt.run(now, item.unique_id);
|
|
2933
|
+
}
|
|
2934
|
+
});
|
|
2935
|
+
bumpTxn(rows);
|
|
2936
|
+
|
|
2937
|
+
return rows.map((row) => {
|
|
2938
|
+
const mapped = mapJobRow(row);
|
|
2939
|
+
mapped.updatedAt = now;
|
|
2940
|
+
return mapped;
|
|
2941
|
+
});
|
|
2942
|
+
}
|
|
2943
|
+
|
|
2944
|
+
const l = Math.max(1, parseInt(limit) || 5);
|
|
2945
|
+
const sevenDaysAgo = Date.now() - 7 * 24 * 60 * 60 * 1000;
|
|
2946
|
+
const pending = data
|
|
2947
|
+
.filter((u) => {
|
|
2948
|
+
if (u.status !== "done") return false;
|
|
2949
|
+
if (u.ttSeller === 1) return false;
|
|
2950
|
+
if (u.updatedAt && u.updatedAt >= sevenDaysAgo) return false;
|
|
2951
|
+
if (hasCountryFilter) {
|
|
2952
|
+
const loc = (u.guessedLocation || "").toUpperCase();
|
|
2953
|
+
return targetCountries.includes(loc);
|
|
2954
|
+
}
|
|
2955
|
+
return true;
|
|
2956
|
+
})
|
|
2957
|
+
.sort((a, b) => (a.updatedAt || 0) - (b.updatedAt || 0))
|
|
2958
|
+
.slice(0, l);
|
|
2959
|
+
pending.forEach((u) => {
|
|
2960
|
+
u.updatedAt = Date.now();
|
|
2961
|
+
});
|
|
2962
|
+
save();
|
|
2963
|
+
return pending;
|
|
2964
|
+
}
|
|
2965
|
+
|
|
2966
|
+
function batchUpdateRecheckJobs(updates) {
|
|
2967
|
+
if (getDb()) {
|
|
2968
|
+
const results = [];
|
|
2969
|
+
const now = Date.now();
|
|
2970
|
+
const bumpStmt = getDb().prepare(
|
|
2971
|
+
`UPDATE jobs SET updated_at = ? WHERE unique_id = ?`,
|
|
2972
|
+
);
|
|
2973
|
+
const txn = getDb().transaction((items) => {
|
|
2974
|
+
items.forEach((item) => {
|
|
2975
|
+
const uniqueId = item.uniqueId;
|
|
2976
|
+
const info = item.info;
|
|
2977
|
+
if (info && info.error && info.statusCode !== undefined) {
|
|
2978
|
+
// 无效账号:只更新 updated_at,不污染其他字段
|
|
2979
|
+
bumpStmt.run(now, uniqueId);
|
|
2980
|
+
results.push({ uniqueId, ok: true, skipped: true });
|
|
2981
|
+
} else {
|
|
2982
|
+
// 过滤 info 中的 null/undefined/空值,避免覆盖 jobs 表已有数据
|
|
2983
|
+
const cleanInfo = {};
|
|
2984
|
+
for (const [k, v] of Object.entries(info || {})) {
|
|
2985
|
+
if (v !== null && v !== undefined && v !== "") {
|
|
2986
|
+
cleanInfo[k] = v;
|
|
2987
|
+
}
|
|
2988
|
+
}
|
|
2989
|
+
const ret = updateJobInfo(uniqueId, cleanInfo, true);
|
|
2990
|
+
results.push(
|
|
2991
|
+
ret.error
|
|
2992
|
+
? { uniqueId, error: ret.error }
|
|
2993
|
+
: { uniqueId, ok: true },
|
|
2994
|
+
);
|
|
2995
|
+
}
|
|
2996
|
+
});
|
|
2997
|
+
});
|
|
2998
|
+
txn(updates);
|
|
2999
|
+
return { results };
|
|
3000
|
+
}
|
|
3001
|
+
|
|
3002
|
+
const results = [];
|
|
3003
|
+
const now = Date.now();
|
|
3004
|
+
for (const item of updates) {
|
|
3005
|
+
const { uniqueId, info } = item;
|
|
3006
|
+
if (info && info.error && info.statusCode !== undefined) {
|
|
3007
|
+
// 无效账号:只更新 updated_at
|
|
3008
|
+
const existing = data.find((u) => u.uniqueId === uniqueId);
|
|
3009
|
+
if (existing) existing.updatedAt = now;
|
|
3010
|
+
results.push({ uniqueId, ok: true, skipped: true });
|
|
3011
|
+
} else {
|
|
3012
|
+
const cleanInfo = {};
|
|
3013
|
+
for (const [k, v] of Object.entries(info || {})) {
|
|
3014
|
+
if (v !== null && v !== undefined && v !== "") {
|
|
3015
|
+
cleanInfo[k] = v;
|
|
3016
|
+
}
|
|
3017
|
+
}
|
|
3018
|
+
const ret = updateJobInfo(uniqueId, cleanInfo, true);
|
|
3019
|
+
results.push(
|
|
3020
|
+
ret.error ? { uniqueId, error: ret.error } : { uniqueId, ok: true },
|
|
3021
|
+
);
|
|
3022
|
+
}
|
|
3023
|
+
}
|
|
3024
|
+
save();
|
|
3025
|
+
return { results };
|
|
3026
|
+
}
|
|
3027
|
+
|
|
2878
3028
|
function updateUserInfo(uniqueId, info) {
|
|
2879
3029
|
if (getDb()) {
|
|
2880
3030
|
return updateJobInfo(uniqueId, info, true);
|
|
@@ -3421,6 +3571,8 @@ export function createStore(filePath, options = {}) {
|
|
|
3421
3571
|
getNextRedoJob,
|
|
3422
3572
|
commitRedoJob,
|
|
3423
3573
|
getPendingUserUpdateTasks,
|
|
3574
|
+
getRecheckJobs,
|
|
3575
|
+
batchUpdateRecheckJobs,
|
|
3424
3576
|
updateUserInfo,
|
|
3425
3577
|
updateUserLocation,
|
|
3426
3578
|
setNonSeller,
|
package/src/watch/db-tags.js
CHANGED
|
@@ -112,7 +112,7 @@ export function getRefreshStats(hours = 24) {
|
|
|
112
112
|
|
|
113
113
|
const rows = db
|
|
114
114
|
.prepare(
|
|
115
|
-
"SELECT countries FROM tags WHERE status = 'productive' AND score >
|
|
115
|
+
"SELECT countries FROM tags WHERE status = 'productive' AND score > 60",
|
|
116
116
|
)
|
|
117
117
|
.all();
|
|
118
118
|
const countrySet = new Set();
|
|
@@ -128,10 +128,10 @@ export function getRefreshStats(hours = 24) {
|
|
|
128
128
|
const countryList = [...countrySet].sort();
|
|
129
129
|
|
|
130
130
|
const stmtTotal = db.prepare(
|
|
131
|
-
"SELECT COUNT(*) as cnt FROM tags WHERE status = 'productive' AND score >
|
|
131
|
+
"SELECT COUNT(*) as cnt FROM tags WHERE status = 'productive' AND score > 60 AND countries LIKE ?",
|
|
132
132
|
);
|
|
133
133
|
const stmtRefresh = db.prepare(
|
|
134
|
-
`SELECT COUNT(*) as cnt FROM tags WHERE status = 'productive' AND score >
|
|
134
|
+
`SELECT COUNT(*) as cnt FROM tags WHERE status = 'productive' AND score > 60 AND countries LIKE ? AND refreshed_at >= datetime('now', ?)`,
|
|
135
135
|
);
|
|
136
136
|
|
|
137
137
|
for (const c of countryList) {
|
|
@@ -1338,7 +1338,7 @@ function getLang(country) {
|
|
|
1338
1338
|
// ===== Tag 刷新 CRUD(productive 标签轮询重新抓取) =====
|
|
1339
1339
|
|
|
1340
1340
|
/**
|
|
1341
|
-
* 获取一个待刷新的 productive 标签(score >
|
|
1341
|
+
* 获取一个待刷新的 productive 标签(score > 60 且未被刷新锁定)
|
|
1342
1342
|
* 按 refreshed_at ASC 排序(最久未刷新的优先),NULL 排最前
|
|
1343
1343
|
*/
|
|
1344
1344
|
export function getProductiveRefreshTask() {
|
|
@@ -1355,7 +1355,7 @@ export function getProductiveRefreshTask() {
|
|
|
1355
1355
|
const row = db
|
|
1356
1356
|
.prepare(
|
|
1357
1357
|
`SELECT * FROM tags
|
|
1358
|
-
WHERE status = 'productive' AND score >
|
|
1358
|
+
WHERE status = 'productive' AND score > 60 AND refreshing = 0
|
|
1359
1359
|
ORDER BY refreshed_at IS NULL DESC, refreshed_at ASC
|
|
1360
1360
|
LIMIT 1`,
|
|
1361
1361
|
)
|
|
@@ -1379,7 +1379,7 @@ export function claimRefreshTag(tag) {
|
|
|
1379
1379
|
const result = db
|
|
1380
1380
|
.prepare(
|
|
1381
1381
|
`UPDATE tags SET refreshing = 1, refreshed_at = datetime('now')
|
|
1382
|
-
WHERE (tag = ? OR tag = ?) AND status = 'productive' AND score >
|
|
1382
|
+
WHERE (tag = ? OR tag = ?) AND status = 'productive' AND score > 60 AND refreshing = 0`,
|
|
1383
1383
|
)
|
|
1384
1384
|
.run(normalized, `#${normalized}`);
|
|
1385
1385
|
if (result.changes === 0) {
|
package/src/watch/public/app.js
CHANGED
|
@@ -268,6 +268,7 @@ function renderActiveClients(clients) {
|
|
|
268
268
|
"explore",
|
|
269
269
|
"refresh",
|
|
270
270
|
"attach",
|
|
271
|
+
"recheck",
|
|
271
272
|
"comments",
|
|
272
273
|
"scoring",
|
|
273
274
|
"tag-refresh",
|
|
@@ -276,6 +277,7 @@ function renderActiveClients(clients) {
|
|
|
276
277
|
explore: "Explore",
|
|
277
278
|
refresh: "Refresh",
|
|
278
279
|
attach: "Attach",
|
|
280
|
+
recheck: "Recheck",
|
|
279
281
|
comments: "Comments",
|
|
280
282
|
scoring: "Scoring",
|
|
281
283
|
"tag-refresh": "TagRefresh",
|
package/src/watch/server.js
CHANGED
|
@@ -92,6 +92,7 @@ function sendCSV(res, columns, rows) {
|
|
|
92
92
|
function inferClientType(routePath) {
|
|
93
93
|
if (routePath.startsWith("/api/redo-job")) return "refresh";
|
|
94
94
|
if (routePath.startsWith("/api/user-update-tasks")) return "attach";
|
|
95
|
+
if (routePath.startsWith("/api/recheck-jobs")) return "recheck";
|
|
95
96
|
if (routePath.startsWith("/api/comment-task")) return "comments";
|
|
96
97
|
if (routePath.startsWith("/api/tags/refresh")) return "tag-refresh";
|
|
97
98
|
if (routePath.startsWith("/api/tags")) return "scoring";
|
|
@@ -754,6 +755,47 @@ export function startWatchServer(
|
|
|
754
755
|
return;
|
|
755
756
|
}
|
|
756
757
|
|
|
758
|
+
// ── recheck-jobs 端点 ──
|
|
759
|
+
if (req.method === "GET" && routePath === "/api/recheck-jobs") {
|
|
760
|
+
const limit = params.limit;
|
|
761
|
+
const countries = params.countries
|
|
762
|
+
? params.countries
|
|
763
|
+
.split(",")
|
|
764
|
+
.map((c) => c.trim().toUpperCase())
|
|
765
|
+
.filter(Boolean)
|
|
766
|
+
: [];
|
|
767
|
+
const tasks = store.getRecheckJobs(limit, countries);
|
|
768
|
+
const ts = new Date().toISOString().slice(11, 19);
|
|
769
|
+
console.error(
|
|
770
|
+
`[JOB ${ts}] RECHECK-JOBS: ${tasks.length} tasks${countries.length ? ` (countries: ${countries.join(",")})` : ""}`,
|
|
771
|
+
);
|
|
772
|
+
sendJSON(res, 200, { total: tasks.length, tasks });
|
|
773
|
+
return;
|
|
774
|
+
}
|
|
775
|
+
|
|
776
|
+
if (req.method === "POST" && routePath === "/api/recheck-jobs-batch") {
|
|
777
|
+
try {
|
|
778
|
+
const body = await readBody(req);
|
|
779
|
+
const updates = body.updates || [];
|
|
780
|
+
const { results } = store.batchUpdateRecheckJobs(updates);
|
|
781
|
+
const okCount = results.filter((r) => r.ok).length;
|
|
782
|
+
const errCount = results.filter((r) => r.error).length;
|
|
783
|
+
const ts = new Date().toISOString().slice(11, 19);
|
|
784
|
+
console.error(
|
|
785
|
+
`[JOB ${ts}] RECHECK-JOBS-BATCH: ${okCount} ok, ${errCount} error (total=${updates.length})`,
|
|
786
|
+
);
|
|
787
|
+
sendJSON(res, 200, {
|
|
788
|
+
results,
|
|
789
|
+
total: updates.length,
|
|
790
|
+
ok: okCount,
|
|
791
|
+
error: errCount,
|
|
792
|
+
});
|
|
793
|
+
} catch (e) {
|
|
794
|
+
sendJSON(res, 400, { error: e.message });
|
|
795
|
+
}
|
|
796
|
+
return;
|
|
797
|
+
}
|
|
798
|
+
|
|
757
799
|
const userInfoCommitMatch = routePath.match(
|
|
758
800
|
/^\/api\/user-info\/([^/]+)$/,
|
|
759
801
|
);
|