tt-help-cli-ycl 1.3.83 → 1.3.84
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/cli/attach.js +3 -34
- package/src/cli/auto.js +3 -18
- package/src/cli/comments.js +13 -57
- package/src/cli/explore.js +255 -266
- package/src/cli/refresh.js +6 -21
- package/src/lib/api-client.js +101 -0
- package/src/watch/data-store.js +36 -0
- package/src/watch/public/app.js +95 -0
- package/src/watch/public/index.html +15 -0
- package/src/watch/public/style.css +107 -0
- package/src/watch/server.js +22 -0
package/package.json
CHANGED
package/src/cli/attach.js
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import { TikTokScraper } from "../lib/tiktok-scraper.mjs";
|
|
2
2
|
import { CDNBlockedError } from "../lib/parse-ssr.mjs";
|
|
3
3
|
import { proxy as configuredProxy } from "../lib/constants.js";
|
|
4
|
+
import { createApiClient } from "../lib/api-client.js";
|
|
4
5
|
import v8 from "node:v8";
|
|
5
6
|
|
|
6
|
-
const MAX_RETRY_WAIT = 5 * 60 * 1000;
|
|
7
7
|
const HEAP_RESTART_RATIO = 0.72;
|
|
8
8
|
const MAX_TASK_BATCHES_BEFORE_RESTART = 200;
|
|
9
9
|
|
|
@@ -17,39 +17,6 @@ function attachLog(message = "") {
|
|
|
17
17
|
console.error(`[${formatNow()}] ${message}`);
|
|
18
18
|
}
|
|
19
19
|
|
|
20
|
-
async function withRetry(label, fn) {
|
|
21
|
-
let backoff = 1000;
|
|
22
|
-
while (true) {
|
|
23
|
-
try {
|
|
24
|
-
return await fn();
|
|
25
|
-
} catch (err) {
|
|
26
|
-
attachLog(
|
|
27
|
-
`[连接] ${label} 失败: ${err.message},${backoff / 1000}秒后重试...`,
|
|
28
|
-
);
|
|
29
|
-
await new Promise((r) => setTimeout(r, backoff));
|
|
30
|
-
if (backoff < MAX_RETRY_WAIT) backoff *= 2;
|
|
31
|
-
}
|
|
32
|
-
}
|
|
33
|
-
}
|
|
34
|
-
|
|
35
|
-
async function apiGet(url) {
|
|
36
|
-
return withRetry(`GET ${url}`, async () => {
|
|
37
|
-
const res = await fetch(url);
|
|
38
|
-
return res.json();
|
|
39
|
-
});
|
|
40
|
-
}
|
|
41
|
-
|
|
42
|
-
async function apiPost(url, body) {
|
|
43
|
-
return withRetry(`POST ${url}`, async () => {
|
|
44
|
-
const res = await fetch(url, {
|
|
45
|
-
method: "POST",
|
|
46
|
-
headers: { "Content-Type": "application/json" },
|
|
47
|
-
body: JSON.stringify(body),
|
|
48
|
-
});
|
|
49
|
-
return res.json();
|
|
50
|
-
});
|
|
51
|
-
}
|
|
52
|
-
|
|
53
20
|
function isBrowserClosedError(err) {
|
|
54
21
|
if (!err) return false;
|
|
55
22
|
const msg = err.message || err.toString() || "";
|
|
@@ -143,6 +110,8 @@ export async function handleAttach(options) {
|
|
|
143
110
|
`[Attach] 并行数: ${attachParallel}, 空闲间隔: ${attachInterval}秒, 服务端: ${serverUrl}${countryStr}`,
|
|
144
111
|
);
|
|
145
112
|
|
|
113
|
+
const { apiGet, apiPost } = createApiClient({ log: false });
|
|
114
|
+
|
|
146
115
|
const scraper = new TikTokScraper({
|
|
147
116
|
poolSize: attachPoolSize || 3,
|
|
148
117
|
proxyServer: effectiveProxy || null,
|
package/src/cli/auto.js
CHANGED
|
@@ -6,6 +6,7 @@ import {
|
|
|
6
6
|
import { userId as configuredUserId, saveUserId } from "../lib/constants.js";
|
|
7
7
|
import { getMacOrUuid } from "../lib/mac-or-uuid.js";
|
|
8
8
|
import { ensureBrowserReady as ensureBrowserReadyCDP } from "../lib/browser/cdp.js";
|
|
9
|
+
import { createApiClient } from "../lib/api-client.js";
|
|
9
10
|
|
|
10
11
|
const MAX_RETRY_WAIT = 5 * 60 * 1000;
|
|
11
12
|
|
|
@@ -24,24 +25,6 @@ async function withRetry(label, fn) {
|
|
|
24
25
|
}
|
|
25
26
|
}
|
|
26
27
|
|
|
27
|
-
async function apiPost(url, body) {
|
|
28
|
-
return withRetry(`POST ${url}`, async () => {
|
|
29
|
-
const res = await fetch(url, {
|
|
30
|
-
method: "POST",
|
|
31
|
-
headers: { "Content-Type": "application/json" },
|
|
32
|
-
body: JSON.stringify(body),
|
|
33
|
-
});
|
|
34
|
-
return res.json();
|
|
35
|
-
});
|
|
36
|
-
}
|
|
37
|
-
|
|
38
|
-
async function apiGet(url) {
|
|
39
|
-
return withRetry(`GET ${url}`, async () => {
|
|
40
|
-
const res = await fetch(url);
|
|
41
|
-
return res.json();
|
|
42
|
-
});
|
|
43
|
-
}
|
|
44
|
-
|
|
45
28
|
export async function handleAuto(options) {
|
|
46
29
|
const {
|
|
47
30
|
autoUsernames,
|
|
@@ -87,6 +70,8 @@ export async function handleAuto(options) {
|
|
|
87
70
|
console.error(`[初始化] 未检测到本地用户编号,已生成并使用: ${userId}`);
|
|
88
71
|
}
|
|
89
72
|
|
|
73
|
+
const { apiGet, apiPost } = createApiClient();
|
|
74
|
+
|
|
90
75
|
const runOptions = {
|
|
91
76
|
collectMax: autoCollectMax,
|
|
92
77
|
scrapeDepth: autoScrapeDepth,
|
package/src/cli/comments.js
CHANGED
|
@@ -7,6 +7,7 @@ import {
|
|
|
7
7
|
isLocationInList,
|
|
8
8
|
normalizeLocation,
|
|
9
9
|
} from "../lib/target-locations.js";
|
|
10
|
+
import { createApiClient } from "../lib/api-client.js";
|
|
10
11
|
|
|
11
12
|
async function waitForPageReady(page, timeout = 30000) {
|
|
12
13
|
const startTime = Date.now();
|
|
@@ -32,64 +33,7 @@ async function safeEvaluate(page, fn) {
|
|
|
32
33
|
}
|
|
33
34
|
}
|
|
34
35
|
|
|
35
|
-
async function withRetry(label, fn, maxRetries = 3) {
|
|
36
|
-
let backoff = 2000;
|
|
37
|
-
for (let i = 0; i < maxRetries; i++) {
|
|
38
|
-
try {
|
|
39
|
-
return await fn();
|
|
40
|
-
} catch (err) {
|
|
41
|
-
if (i < maxRetries - 1) {
|
|
42
|
-
console.error(
|
|
43
|
-
` [${label}] 失败: ${err.message},${backoff / 1000}s 后重试...`,
|
|
44
|
-
);
|
|
45
|
-
await new Promise((r) => setTimeout(r, backoff));
|
|
46
|
-
backoff *= 2;
|
|
47
|
-
} else {
|
|
48
|
-
throw err;
|
|
49
|
-
}
|
|
50
|
-
}
|
|
51
|
-
}
|
|
52
|
-
}
|
|
53
|
-
|
|
54
|
-
async function apiPost(url, body) {
|
|
55
|
-
return withRetry(`POST ${url}`, async () => {
|
|
56
|
-
const res = await fetch(url, {
|
|
57
|
-
method: "POST",
|
|
58
|
-
headers: { "Content-Type": "application/json" },
|
|
59
|
-
body: JSON.stringify(body),
|
|
60
|
-
});
|
|
61
|
-
if (!res.ok) {
|
|
62
|
-
const errText = await res.text();
|
|
63
|
-
throw new Error(`HTTP ${res.status}: ${errText.substring(0, 200)}`);
|
|
64
|
-
}
|
|
65
|
-
return res.json();
|
|
66
|
-
});
|
|
67
|
-
}
|
|
68
36
|
|
|
69
|
-
async function apiPut(url) {
|
|
70
|
-
return withRetry(`PUT ${url}`, async () => {
|
|
71
|
-
const res = await fetch(url, {
|
|
72
|
-
method: "PUT",
|
|
73
|
-
headers: { "Content-Type": "application/json" },
|
|
74
|
-
});
|
|
75
|
-
if (!res.ok) {
|
|
76
|
-
const errText = await res.text();
|
|
77
|
-
throw new Error(`HTTP ${res.status}: ${errText.substring(0, 200)}`);
|
|
78
|
-
}
|
|
79
|
-
return res.json();
|
|
80
|
-
});
|
|
81
|
-
}
|
|
82
|
-
|
|
83
|
-
async function apiGet(url) {
|
|
84
|
-
return withRetry(`GET ${url}`, async () => {
|
|
85
|
-
const res = await fetch(url);
|
|
86
|
-
if (!res.ok) {
|
|
87
|
-
const errText = await res.text();
|
|
88
|
-
throw new Error(`HTTP ${res.status}: ${errText.substring(0, 200)}`);
|
|
89
|
-
}
|
|
90
|
-
return res.json();
|
|
91
|
-
});
|
|
92
|
-
}
|
|
93
37
|
|
|
94
38
|
function isBrowserClosedError(err) {
|
|
95
39
|
if (!err) return false;
|
|
@@ -110,6 +54,12 @@ function isBrowserClosedError(err) {
|
|
|
110
54
|
*/
|
|
111
55
|
async function runAutoMode(options) {
|
|
112
56
|
const { serverUrl, parallel, interval, maxComments } = options;
|
|
57
|
+
const { apiGet, apiPost, apiPut } = createApiClient({
|
|
58
|
+
checkStatus: true,
|
|
59
|
+
maxRetries: 2,
|
|
60
|
+
backoff: 2000,
|
|
61
|
+
log: true,
|
|
62
|
+
});
|
|
113
63
|
const actualParallel = Math.max(1, parallel || 1);
|
|
114
64
|
const actualInterval = interval || 10;
|
|
115
65
|
const actualMaxComments = maxComments || 200;
|
|
@@ -478,6 +428,12 @@ export async function handleComments(options) {
|
|
|
478
428
|
|
|
479
429
|
const guessedLocation = normalizeLocation(videoInfo?.locationCreated);
|
|
480
430
|
const serverUrl = commentsServer || defaultServer;
|
|
431
|
+
const { apiPost } = createApiClient({
|
|
432
|
+
checkStatus: true,
|
|
433
|
+
maxRetries: 2,
|
|
434
|
+
backoff: 2000,
|
|
435
|
+
log: true,
|
|
436
|
+
});
|
|
481
437
|
|
|
482
438
|
if (
|
|
483
439
|
guessedLocation &&
|
package/src/cli/explore.js
CHANGED
|
@@ -24,6 +24,7 @@ import {
|
|
|
24
24
|
} from "../lib/browser/cdp.js";
|
|
25
25
|
import { showResourceUsage } from "../utils/index.js";
|
|
26
26
|
import { HealthChecker } from "../lib/browser/health-checker.js";
|
|
27
|
+
import { createApiClient } from "../lib/api-client.js";
|
|
27
28
|
import path from "path";
|
|
28
29
|
import os from "os";
|
|
29
30
|
|
|
@@ -46,24 +47,6 @@ async function withRetry(label, fn) {
|
|
|
46
47
|
}
|
|
47
48
|
}
|
|
48
49
|
|
|
49
|
-
async function apiPost(url, body) {
|
|
50
|
-
return withRetry(`POST ${url}`, async () => {
|
|
51
|
-
const res = await fetch(url, {
|
|
52
|
-
method: "POST",
|
|
53
|
-
headers: { "Content-Type": "application/json" },
|
|
54
|
-
body: JSON.stringify(body),
|
|
55
|
-
});
|
|
56
|
-
return res.json();
|
|
57
|
-
});
|
|
58
|
-
}
|
|
59
|
-
|
|
60
|
-
async function apiGet(url) {
|
|
61
|
-
return withRetry(`GET ${url}`, async () => {
|
|
62
|
-
const res = await fetch(url);
|
|
63
|
-
return res.json();
|
|
64
|
-
});
|
|
65
|
-
}
|
|
66
|
-
|
|
67
50
|
export async function handleExplore(options) {
|
|
68
51
|
const {
|
|
69
52
|
exploreUsernames,
|
|
@@ -118,15 +101,6 @@ export async function handleExplore(options) {
|
|
|
118
101
|
|
|
119
102
|
setDelayConfig(explorePreset);
|
|
120
103
|
|
|
121
|
-
await apiGet(`${serverUrl}/api/stats`);
|
|
122
|
-
|
|
123
|
-
if (exploreUsernames && exploreUsernames.length > 0) {
|
|
124
|
-
const { added, skipped } = await apiPost(`${serverUrl}/api/users`, {
|
|
125
|
-
usernames: exploreUsernames,
|
|
126
|
-
});
|
|
127
|
-
console.error(`种子用户: ${added} 个新增, ${skipped} 个已存在`);
|
|
128
|
-
}
|
|
129
|
-
|
|
130
104
|
console.error(`\n国家筛选: ${exploreLocation}`);
|
|
131
105
|
if (exploreJobLocations) console.error(`任务国家: ${exploreJobLocations}`);
|
|
132
106
|
console.error(`视频采集: ${exploreMaxVideos || 1}`);
|
|
@@ -168,6 +142,17 @@ export async function handleExplore(options) {
|
|
|
168
142
|
|
|
169
143
|
console.error(`CDP 端口: ${cdpOptions.port}, 用户编号: ${userId}`);
|
|
170
144
|
console.error(`浏览器配置: ${path.basename(cdpOptions.userDataDir)}`);
|
|
145
|
+
|
|
146
|
+
const { apiGet, apiPost } = createApiClient({ meta: { port: cdpOptions.port } });
|
|
147
|
+
|
|
148
|
+
await apiGet(`${serverUrl}/api/stats`);
|
|
149
|
+
|
|
150
|
+
if (exploreUsernames && exploreUsernames.length > 0) {
|
|
151
|
+
const { added, skipped } = await apiPost(`${serverUrl}/api/users`, {
|
|
152
|
+
usernames: exploreUsernames,
|
|
153
|
+
});
|
|
154
|
+
console.error(`种子用户: ${added} 个新增, ${skipped} 个已存在`);
|
|
155
|
+
}
|
|
171
156
|
if (!explorePort) {
|
|
172
157
|
const portRange = `${currentAccount.port}-${currentAccount.port + healthChecker.accounts.length - 1}`;
|
|
173
158
|
console.error(
|
|
@@ -269,140 +254,116 @@ export async function handleExplore(options) {
|
|
|
269
254
|
? `${serverUrl}/api/job?userId=${encodeURIComponent(userId)}&locations=${encodeURIComponent(exploreJobLocations)}&loggedIn=${loggedIn}`
|
|
270
255
|
: `${serverUrl}/api/job?userId=${encodeURIComponent(userId)}&loggedIn=${loggedIn}`;
|
|
271
256
|
const job = await apiGet(jobQuery);
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
257
|
+
if (!job.hasJob) {
|
|
258
|
+
console.error(
|
|
259
|
+
`\n[Explore] 当前无任务,${exploreInterval} 秒后重试...`,
|
|
260
|
+
);
|
|
261
|
+
await new Promise((r) => setTimeout(r, exploreInterval * 1000));
|
|
262
|
+
continue;
|
|
263
|
+
}
|
|
277
264
|
|
|
278
|
-
|
|
279
|
-
|
|
265
|
+
const username = job.user.uniqueId;
|
|
266
|
+
processedCount++;
|
|
280
267
|
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
268
|
+
if (processedCount % 10 === 0) {
|
|
269
|
+
showResourceUsage();
|
|
270
|
+
}
|
|
284
271
|
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
272
|
+
const checkResult = healthChecker.check();
|
|
273
|
+
if (checkResult.shouldSwitch) {
|
|
274
|
+
await handleAccountSwitch(checkResult.reason);
|
|
275
|
+
} else if (checkResult.info && processedCount % 10 === 0) {
|
|
276
|
+
// 关注/粉丝封禁检测状态(仅登录状态)
|
|
277
|
+
let followStatus = "";
|
|
278
|
+
if (loggedIn && exploreEnableFollow) {
|
|
279
|
+
const followElapsed = Math.round(
|
|
280
|
+
(Date.now() - lastFollowSuccessTime) / 60000,
|
|
281
|
+
);
|
|
282
|
+
const followRemaining = Math.max(
|
|
283
|
+
0,
|
|
284
|
+
Math.round(FOLLOW_BLOCK_THRESHOLD / 60000) - followElapsed,
|
|
285
|
+
);
|
|
286
|
+
followStatus = ` | 关注/粉丝上次成功 ${followElapsed} 分钟前 | 封禁检测 ${followRemaining} 分钟后`;
|
|
287
|
+
}
|
|
288
|
+
console.error(`\n[健康检查] ${checkResult.info}${followStatus}`);
|
|
300
289
|
}
|
|
301
|
-
console.error(`\n[健康检查] ${checkResult.info}${followStatus}`);
|
|
302
|
-
}
|
|
303
290
|
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
291
|
+
// 切换任务前检测验证码
|
|
292
|
+
const switchCaptcha = await detectCaptcha(page);
|
|
293
|
+
if (switchCaptcha && switchCaptcha.visible) {
|
|
294
|
+
console.error(`\n[验证码] 切换任务前检测到验证码,等待3分钟...`);
|
|
295
|
+
captchaCount++;
|
|
296
|
+
console.error(` [验证码] 累计 ${captchaCount} 次`);
|
|
297
|
+
|
|
298
|
+
// 等待3分钟
|
|
299
|
+
await new Promise((r) => setTimeout(r, 3 * 60 * 1000));
|
|
300
|
+
|
|
301
|
+
// 尝试关闭
|
|
302
|
+
const closeResult = await closeCaptcha(page);
|
|
303
|
+
if (closeResult.success) {
|
|
304
|
+
console.error(` [验证码] 已关闭`);
|
|
305
|
+
} else {
|
|
306
|
+
console.error(` [验证码] 关闭失败: ${closeResult.reason}`);
|
|
307
|
+
}
|
|
310
308
|
|
|
311
|
-
|
|
312
|
-
|
|
309
|
+
// 上报
|
|
310
|
+
await withRetry("report captcha", () =>
|
|
311
|
+
apiPost(`${serverUrl}/api/error-report`, {
|
|
312
|
+
userId,
|
|
313
|
+
username: "unknown",
|
|
314
|
+
errorType: "captcha",
|
|
315
|
+
errorMessage: "切换任务前检测到验证码",
|
|
316
|
+
stage: "between-tasks",
|
|
317
|
+
errorStack: "",
|
|
318
|
+
}),
|
|
319
|
+
).catch(() => {});
|
|
313
320
|
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
console.error(` [验证码] 关闭失败: ${closeResult.reason}`);
|
|
321
|
+
// 累计2次切换账户
|
|
322
|
+
if (captchaCount >= 2) {
|
|
323
|
+
await handleAccountSwitch(`验证码累计 ${captchaCount} 次`);
|
|
324
|
+
captchaCount = 0;
|
|
325
|
+
}
|
|
320
326
|
}
|
|
321
327
|
|
|
322
|
-
//
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
}
|
|
332
|
-
).catch(() => {});
|
|
333
|
-
|
|
334
|
-
// 累计2次切换账户
|
|
335
|
-
if (captchaCount >= 2) {
|
|
336
|
-
await handleAccountSwitch(`验证码累计 ${captchaCount} 次`);
|
|
337
|
-
captchaCount = 0;
|
|
328
|
+
// 关注/粉丝封禁检测移到健康检查中(仅登录状态下)
|
|
329
|
+
if (loggedIn && exploreEnableFollow) {
|
|
330
|
+
const followElapsed = Date.now() - lastFollowSuccessTime;
|
|
331
|
+
if (followElapsed > FOLLOW_BLOCK_THRESHOLD) {
|
|
332
|
+
const minutes = Math.round(followElapsed / 60000);
|
|
333
|
+
console.error(
|
|
334
|
+
`\n[封禁检测] 关注/粉丝功能已 ${minutes} 分钟未成功获取,切换到下一个账户...`,
|
|
335
|
+
);
|
|
336
|
+
await handleAccountSwitch(`关注/粉丝功能被封`);
|
|
337
|
+
}
|
|
338
338
|
}
|
|
339
|
-
}
|
|
340
339
|
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
)
|
|
349
|
-
|
|
340
|
+
if (consecutiveNetworkErrors > 0) {
|
|
341
|
+
const waitTime =
|
|
342
|
+
consecutiveNetworkErrors <= 2
|
|
343
|
+
? 0
|
|
344
|
+
: consecutiveNetworkErrors <= 5
|
|
345
|
+
? 30000
|
|
346
|
+
: 300000;
|
|
347
|
+
if (waitTime > 0) {
|
|
348
|
+
console.error(
|
|
349
|
+
` [网络] 连续 ${consecutiveNetworkErrors} 次网络异常,等待 ${waitTime / 1000}s 后重试...`,
|
|
350
|
+
);
|
|
351
|
+
await new Promise((r) => setTimeout(r, waitTime));
|
|
352
|
+
}
|
|
350
353
|
}
|
|
351
|
-
}
|
|
352
354
|
|
|
353
|
-
|
|
354
|
-
const waitTime =
|
|
355
|
-
consecutiveNetworkErrors <= 2
|
|
356
|
-
? 0
|
|
357
|
-
: consecutiveNetworkErrors <= 5
|
|
358
|
-
? 30000
|
|
359
|
-
: 300000;
|
|
360
|
-
if (waitTime > 0) {
|
|
361
|
-
console.error(
|
|
362
|
-
` [网络] 连续 ${consecutiveNetworkErrors} 次网络异常,等待 ${waitTime / 1000}s 后重试...`,
|
|
363
|
-
);
|
|
364
|
-
await new Promise((r) => setTimeout(r, waitTime));
|
|
365
|
-
}
|
|
366
|
-
}
|
|
355
|
+
console.error(`\n[${processedCount}] 探索 @${username}...`);
|
|
367
356
|
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
const { switchMax } = getDelayConfig();
|
|
371
|
-
await delay(switchMax, switchMax * 3);
|
|
372
|
-
|
|
373
|
-
let result = await processExplore(
|
|
374
|
-
page,
|
|
375
|
-
username,
|
|
376
|
-
{
|
|
377
|
-
maxVideos: exploreMaxVideos,
|
|
378
|
-
enableFollow: exploreEnableFollow,
|
|
379
|
-
loggedIn, // 传入登录状态,避免每次调用 isLoggedIn(page)
|
|
380
|
-
maxFollowing: exploreMaxFollowing,
|
|
381
|
-
maxFollowers: exploreMaxFollowers,
|
|
382
|
-
location: exploreLocation,
|
|
383
|
-
proxyServer: cdpOptions.proxyServer || null,
|
|
384
|
-
browser,
|
|
385
|
-
},
|
|
386
|
-
console.error,
|
|
387
|
-
);
|
|
357
|
+
const { switchMax } = getDelayConfig();
|
|
358
|
+
await delay(switchMax, switchMax * 3);
|
|
388
359
|
|
|
389
|
-
|
|
390
|
-
if (result.error && isBrowserClosedError(new Error(result.error))) {
|
|
391
|
-
const newBrowser = await relaunchBrowser(
|
|
392
|
-
cdpOptions,
|
|
393
|
-
cdpOptions.port || 9222,
|
|
394
|
-
);
|
|
395
|
-
browser = newBrowser;
|
|
396
|
-
const newPage = await setupNewPage(browser);
|
|
397
|
-
Object.assign(page, newPage);
|
|
398
|
-
// 重试当前用户
|
|
399
|
-
result = await processExplore(
|
|
360
|
+
let result = await processExplore(
|
|
400
361
|
page,
|
|
401
362
|
username,
|
|
402
363
|
{
|
|
403
364
|
maxVideos: exploreMaxVideos,
|
|
404
365
|
enableFollow: exploreEnableFollow,
|
|
405
|
-
loggedIn, //
|
|
366
|
+
loggedIn, // 传入登录状态,避免每次调用 isLoggedIn(page)
|
|
406
367
|
maxFollowing: exploreMaxFollowing,
|
|
407
368
|
maxFollowers: exploreMaxFollowers,
|
|
408
369
|
location: exploreLocation,
|
|
@@ -411,36 +372,24 @@ export async function handleExplore(options) {
|
|
|
411
372
|
},
|
|
412
373
|
console.error,
|
|
413
374
|
);
|
|
414
|
-
}
|
|
415
375
|
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
continue;
|
|
427
|
-
}
|
|
428
|
-
|
|
429
|
-
if (result.error) {
|
|
430
|
-
consecutiveNetworkErrors++;
|
|
431
|
-
errorCount++;
|
|
432
|
-
|
|
433
|
-
// 临时性错误:自动重试一次
|
|
434
|
-
if (result.retryable) {
|
|
435
|
-
console.error(` [临时错误] 等待 5 秒后重试 @${username}...`);
|
|
436
|
-
await new Promise((r) => setTimeout(r, 5000));
|
|
376
|
+
// 浏览器关闭检测:processExplore 内部 catch 了异常,需要从 result.error 判断
|
|
377
|
+
if (result.error && isBrowserClosedError(new Error(result.error))) {
|
|
378
|
+
const newBrowser = await relaunchBrowser(
|
|
379
|
+
cdpOptions,
|
|
380
|
+
cdpOptions.port || 9222,
|
|
381
|
+
);
|
|
382
|
+
browser = newBrowser;
|
|
383
|
+
const newPage = await setupNewPage(browser);
|
|
384
|
+
Object.assign(page, newPage);
|
|
385
|
+
// 重试当前用户
|
|
437
386
|
result = await processExplore(
|
|
438
387
|
page,
|
|
439
388
|
username,
|
|
440
389
|
{
|
|
441
390
|
maxVideos: exploreMaxVideos,
|
|
442
391
|
enableFollow: exploreEnableFollow,
|
|
443
|
-
loggedIn,
|
|
392
|
+
loggedIn, // 传入登录状态
|
|
444
393
|
maxFollowing: exploreMaxFollowing,
|
|
445
394
|
maxFollowers: exploreMaxFollowers,
|
|
446
395
|
location: exploreLocation,
|
|
@@ -449,127 +398,167 @@ export async function handleExplore(options) {
|
|
|
449
398
|
},
|
|
450
399
|
console.error,
|
|
451
400
|
);
|
|
452
|
-
// 重试成功后继续正常流程
|
|
453
|
-
if (!result.error) {
|
|
454
|
-
consecutiveNetworkErrors = 0;
|
|
455
|
-
errorCount--;
|
|
456
|
-
}
|
|
457
401
|
}
|
|
458
402
|
|
|
459
|
-
if (result.
|
|
460
|
-
|
|
403
|
+
if (result.restricted) {
|
|
404
|
+
consecutiveNetworkErrors = 0;
|
|
461
405
|
await apiPost(`${serverUrl}/api/job/${username}`, {
|
|
462
|
-
|
|
406
|
+
restricted: true,
|
|
407
|
+
userInfo: result.userInfo || {},
|
|
463
408
|
});
|
|
464
|
-
const errorType = result.error.startsWith("被封:")
|
|
465
|
-
? "被封"
|
|
466
|
-
: consecutiveNetworkErrors > 1
|
|
467
|
-
? "network"
|
|
468
|
-
: "other";
|
|
469
|
-
await withRetry("report error", () =>
|
|
470
|
-
apiPost(`${serverUrl}/api/error-report`, {
|
|
471
|
-
userId,
|
|
472
|
-
username,
|
|
473
|
-
errorType,
|
|
474
|
-
errorMessage: result.error,
|
|
475
|
-
stage: "process",
|
|
476
|
-
errorStack: result.errorStack || "",
|
|
477
|
-
}),
|
|
478
|
-
).catch(() => {});
|
|
479
|
-
if (errorType === "被封") {
|
|
480
|
-
blockedCount++;
|
|
481
|
-
console.error(` [被封] 累计 ${blockedCount} 次`);
|
|
482
|
-
if (blockedCount >= 3) {
|
|
483
|
-
await handleAccountSwitch(`账号被封累计 ${blockedCount} 次`);
|
|
484
|
-
blockedCount = 0;
|
|
485
|
-
}
|
|
486
|
-
continue;
|
|
487
|
-
}
|
|
488
409
|
if (exploreMaxUsers > 0 && processedCount >= exploreMaxUsers) {
|
|
489
410
|
console.error(`\n已达上限 ${exploreMaxUsers} 个用户,停止处理`);
|
|
490
411
|
break;
|
|
491
412
|
}
|
|
492
413
|
continue;
|
|
493
414
|
}
|
|
494
|
-
}
|
|
495
415
|
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
416
|
+
if (result.error) {
|
|
417
|
+
consecutiveNetworkErrors++;
|
|
418
|
+
errorCount++;
|
|
419
|
+
|
|
420
|
+
// 临时性错误:自动重试一次
|
|
421
|
+
if (result.retryable) {
|
|
422
|
+
console.error(` [临时错误] 等待 5 秒后重试 @${username}...`);
|
|
423
|
+
await new Promise((r) => setTimeout(r, 5000));
|
|
424
|
+
result = await processExplore(
|
|
425
|
+
page,
|
|
426
|
+
username,
|
|
427
|
+
{
|
|
428
|
+
maxVideos: exploreMaxVideos,
|
|
429
|
+
enableFollow: exploreEnableFollow,
|
|
430
|
+
loggedIn,
|
|
431
|
+
maxFollowing: exploreMaxFollowing,
|
|
432
|
+
maxFollowers: exploreMaxFollowers,
|
|
433
|
+
location: exploreLocation,
|
|
434
|
+
proxyServer: cdpOptions.proxyServer || null,
|
|
435
|
+
browser,
|
|
436
|
+
},
|
|
437
|
+
console.error,
|
|
438
|
+
);
|
|
439
|
+
// 重试成功后继续正常流程
|
|
440
|
+
if (!result.error) {
|
|
441
|
+
consecutiveNetworkErrors = 0;
|
|
442
|
+
errorCount--;
|
|
443
|
+
}
|
|
444
|
+
}
|
|
499
445
|
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
446
|
+
if (result.error) {
|
|
447
|
+
// 上报错误(重试后仍有错误才上报)
|
|
448
|
+
await apiPost(`${serverUrl}/api/job/${username}`, {
|
|
449
|
+
error: result.error,
|
|
450
|
+
});
|
|
451
|
+
const errorType = result.error.startsWith("被封:")
|
|
452
|
+
? "被封"
|
|
453
|
+
: consecutiveNetworkErrors > 1
|
|
454
|
+
? "network"
|
|
455
|
+
: "other";
|
|
456
|
+
await withRetry("report error", () =>
|
|
457
|
+
apiPost(`${serverUrl}/api/error-report`, {
|
|
458
|
+
userId,
|
|
459
|
+
username,
|
|
460
|
+
errorType,
|
|
461
|
+
errorMessage: result.error,
|
|
462
|
+
stage: "process",
|
|
463
|
+
errorStack: result.errorStack || "",
|
|
464
|
+
}),
|
|
465
|
+
).catch(() => {});
|
|
466
|
+
if (errorType === "被封") {
|
|
467
|
+
blockedCount++;
|
|
468
|
+
console.error(` [被封] 累计 ${blockedCount} 次`);
|
|
469
|
+
if (blockedCount >= 3) {
|
|
470
|
+
await handleAccountSwitch(`账号被封累计 ${blockedCount} 次`);
|
|
471
|
+
blockedCount = 0;
|
|
472
|
+
}
|
|
473
|
+
continue;
|
|
474
|
+
}
|
|
475
|
+
if (exploreMaxUsers > 0 && processedCount >= exploreMaxUsers) {
|
|
476
|
+
console.error(`\n已达上限 ${exploreMaxUsers} 个用户,停止处理`);
|
|
477
|
+
break;
|
|
478
|
+
}
|
|
479
|
+
continue;
|
|
480
|
+
}
|
|
515
481
|
}
|
|
516
|
-
}
|
|
517
482
|
|
|
518
|
-
|
|
483
|
+
if (result.captchaDetected) {
|
|
484
|
+
captchaCount++;
|
|
485
|
+
console.error(` [验证码] 累计 ${captchaCount} 次`);
|
|
486
|
+
|
|
487
|
+
await withRetry("report captcha", () =>
|
|
488
|
+
apiPost(`${serverUrl}/api/error-report`, {
|
|
489
|
+
userId,
|
|
490
|
+
username,
|
|
491
|
+
errorType: "captcha",
|
|
492
|
+
errorMessage: result.captchaMessage || "页面出现验证码",
|
|
493
|
+
stage: result.captchaStage || "video-page",
|
|
494
|
+
errorStack: "",
|
|
495
|
+
}),
|
|
496
|
+
).catch(() => {});
|
|
519
497
|
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
if (totalFollows > 0) {
|
|
526
|
-
lastFollowSuccessTime = Date.now();
|
|
498
|
+
// 累计2次验证码,切换账户
|
|
499
|
+
if (captchaCount >= 2) {
|
|
500
|
+
await handleAccountSwitch(`验证码累计 ${captchaCount} 次`);
|
|
501
|
+
captchaCount = 0;
|
|
502
|
+
}
|
|
527
503
|
}
|
|
528
|
-
}
|
|
529
504
|
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
const payload = {
|
|
533
|
-
userInfo: result.userInfo || {},
|
|
534
|
-
discoveredFollowing: (result.discoveredFollowing || []).map((f) => ({
|
|
535
|
-
handle: Array.isArray(f) ? f[0] : f,
|
|
536
|
-
displayName: Array.isArray(f) ? f[1] : null,
|
|
537
|
-
guessedLocation,
|
|
538
|
-
})),
|
|
539
|
-
discoveredFollowers: (result.discoveredFollowers || []).map((f) => ({
|
|
540
|
-
handle: Array.isArray(f) ? f[0] : f,
|
|
541
|
-
displayName: Array.isArray(f) ? f[1] : null,
|
|
542
|
-
guessedLocation,
|
|
543
|
-
})),
|
|
544
|
-
processed: result.processed,
|
|
545
|
-
hasFollowData: result.hasFollowData,
|
|
546
|
-
keepFollow: result.keepFollow,
|
|
547
|
-
locationCreated: result.locationCreated,
|
|
548
|
-
noVideo: result.noVideo,
|
|
549
|
-
collectedVideos: result.collectedVideos,
|
|
550
|
-
};
|
|
551
|
-
await apiPost(`${serverUrl}/api/job/${username}`, payload);
|
|
552
|
-
|
|
553
|
-
// 视频登记
|
|
554
|
-
if (result.videoList && result.videoList.length > 0) {
|
|
555
|
-
const { registered, skipped } = await apiPost(
|
|
556
|
-
`${serverUrl}/api/videos`,
|
|
557
|
-
{
|
|
558
|
-
sourceUser: username,
|
|
559
|
-
videoList: result.videoList,
|
|
560
|
-
locationCreated: result.locationCreated,
|
|
561
|
-
ttSeller: result.userInfo?.ttSeller || false,
|
|
562
|
-
},
|
|
563
|
-
);
|
|
564
|
-
console.error(` 视频登记: ${registered} 条新增, ${skipped} 条已存在`);
|
|
565
|
-
}
|
|
505
|
+
consecutiveNetworkErrors = 0;
|
|
566
506
|
|
|
567
|
-
|
|
507
|
+
// 更新关注/粉丝成功时间(仅登录状态下)
|
|
508
|
+
if (result.hasFollowData && result.keepFollow) {
|
|
509
|
+
const totalFollows =
|
|
510
|
+
(result.discoveredFollowing || []).length +
|
|
511
|
+
(result.discoveredFollowers || []).length;
|
|
512
|
+
if (totalFollows > 0) {
|
|
513
|
+
lastFollowSuccessTime = Date.now();
|
|
514
|
+
}
|
|
515
|
+
}
|
|
568
516
|
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
517
|
+
const guessedLocation = result.locationCreated || null;
|
|
518
|
+
|
|
519
|
+
const payload = {
|
|
520
|
+
userInfo: result.userInfo || {},
|
|
521
|
+
discoveredFollowing: (result.discoveredFollowing || []).map((f) => ({
|
|
522
|
+
handle: Array.isArray(f) ? f[0] : f,
|
|
523
|
+
displayName: Array.isArray(f) ? f[1] : null,
|
|
524
|
+
guessedLocation,
|
|
525
|
+
})),
|
|
526
|
+
discoveredFollowers: (result.discoveredFollowers || []).map((f) => ({
|
|
527
|
+
handle: Array.isArray(f) ? f[0] : f,
|
|
528
|
+
displayName: Array.isArray(f) ? f[1] : null,
|
|
529
|
+
guessedLocation,
|
|
530
|
+
})),
|
|
531
|
+
processed: result.processed,
|
|
532
|
+
hasFollowData: result.hasFollowData,
|
|
533
|
+
keepFollow: result.keepFollow,
|
|
534
|
+
locationCreated: result.locationCreated,
|
|
535
|
+
noVideo: result.noVideo,
|
|
536
|
+
collectedVideos: result.collectedVideos,
|
|
537
|
+
};
|
|
538
|
+
await apiPost(`${serverUrl}/api/job/${username}`, payload);
|
|
539
|
+
|
|
540
|
+
// 视频登记
|
|
541
|
+
if (result.videoList && result.videoList.length > 0) {
|
|
542
|
+
const { registered, skipped } = await apiPost(
|
|
543
|
+
`${serverUrl}/api/videos`,
|
|
544
|
+
{
|
|
545
|
+
sourceUser: username,
|
|
546
|
+
videoList: result.videoList,
|
|
547
|
+
locationCreated: result.locationCreated,
|
|
548
|
+
ttSeller: result.userInfo?.ttSeller || false,
|
|
549
|
+
},
|
|
550
|
+
);
|
|
551
|
+
console.error(
|
|
552
|
+
` 视频登记: ${registered} 条新增, ${skipped} 条已存在`,
|
|
553
|
+
);
|
|
554
|
+
}
|
|
555
|
+
|
|
556
|
+
console.error(" 已提交");
|
|
557
|
+
|
|
558
|
+
if (exploreMaxUsers > 0 && processedCount >= exploreMaxUsers) {
|
|
559
|
+
console.error(`\n已达上限 ${exploreMaxUsers} 个用户,停止处理`);
|
|
560
|
+
break;
|
|
561
|
+
}
|
|
573
562
|
} catch (e) {
|
|
574
563
|
// 浏览器关闭错误:自动重建 browser + page,然后重试当前轮次
|
|
575
564
|
if (isBrowserClosedError(e)) {
|
package/src/cli/refresh.js
CHANGED
|
@@ -23,6 +23,7 @@ import {
|
|
|
23
23
|
switchAccount,
|
|
24
24
|
} from "../lib/browser/cdp.js";
|
|
25
25
|
import { HealthChecker } from "../lib/browser/health-checker.js";
|
|
26
|
+
import { createApiClient } from "../lib/api-client.js";
|
|
26
27
|
import path from "path";
|
|
27
28
|
import os from "os";
|
|
28
29
|
|
|
@@ -45,24 +46,6 @@ async function withRetry(label, fn) {
|
|
|
45
46
|
}
|
|
46
47
|
}
|
|
47
48
|
|
|
48
|
-
async function apiPost(url, body) {
|
|
49
|
-
return withRetry(`POST ${url}`, async () => {
|
|
50
|
-
const res = await fetch(url, {
|
|
51
|
-
method: "POST",
|
|
52
|
-
headers: { "Content-Type": "application/json" },
|
|
53
|
-
body: JSON.stringify(body),
|
|
54
|
-
});
|
|
55
|
-
return res.json();
|
|
56
|
-
});
|
|
57
|
-
}
|
|
58
|
-
|
|
59
|
-
async function apiGet(url) {
|
|
60
|
-
return withRetry(`GET ${url}`, async () => {
|
|
61
|
-
const res = await fetch(url);
|
|
62
|
-
return res.json();
|
|
63
|
-
});
|
|
64
|
-
}
|
|
65
|
-
|
|
66
49
|
export async function handleRefresh(options) {
|
|
67
50
|
const {
|
|
68
51
|
explorePreset,
|
|
@@ -116,9 +99,6 @@ export async function handleRefresh(options) {
|
|
|
116
99
|
|
|
117
100
|
setDelayConfig(explorePreset);
|
|
118
101
|
|
|
119
|
-
// 连接服务器验证
|
|
120
|
-
await apiGet(`${serverUrl}/api/stats`);
|
|
121
|
-
|
|
122
102
|
console.error(`\n=== Refresh 模式(基于 explore) ===`);
|
|
123
103
|
console.error(`服务器: ${serverUrl}`);
|
|
124
104
|
console.error(`视频采集: ${exploreMaxVideos || 16}`);
|
|
@@ -175,6 +155,11 @@ export async function handleRefresh(options) {
|
|
|
175
155
|
);
|
|
176
156
|
}
|
|
177
157
|
|
|
158
|
+
const { apiGet, apiPost } = createApiClient({ meta: { port: cdpOptions.port } });
|
|
159
|
+
|
|
160
|
+
// 连接服务器验证
|
|
161
|
+
await apiGet(`${serverUrl}/api/stats`);
|
|
162
|
+
|
|
178
163
|
browser = await ensureBrowserReadyCDP(cdpOptions);
|
|
179
164
|
const { processExplore } = await import("../scraper/explore-core.js");
|
|
180
165
|
const { isLoggedIn } = await import("../lib/browser/page.js");
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
import crypto from "node:crypto";
|
|
2
|
+
|
|
3
|
+
const MAX_RETRY_WAIT = 5 * 60 * 1000;
|
|
4
|
+
|
|
5
|
+
async function withRetry(label, fn, opts = {}) {
|
|
6
|
+
const { maxRetries, backoff: initialBackoff = 1000, log = true } = opts;
|
|
7
|
+
|
|
8
|
+
let backoff = initialBackoff;
|
|
9
|
+
let i = 0;
|
|
10
|
+
|
|
11
|
+
while (true) {
|
|
12
|
+
try {
|
|
13
|
+
return await fn();
|
|
14
|
+
} catch (err) {
|
|
15
|
+
if (maxRetries !== undefined && i >= maxRetries) {
|
|
16
|
+
throw err;
|
|
17
|
+
}
|
|
18
|
+
if (log) {
|
|
19
|
+
console.error(
|
|
20
|
+
`[连接] ${label} 失败: ${err.message},${backoff / 1000}秒后重试...`,
|
|
21
|
+
);
|
|
22
|
+
}
|
|
23
|
+
await new Promise((r) => setTimeout(r, backoff));
|
|
24
|
+
if (backoff < MAX_RETRY_WAIT) backoff *= 2;
|
|
25
|
+
i++;
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
function buildHeaders(baseHeaders, clientId, meta) {
|
|
31
|
+
return {
|
|
32
|
+
...baseHeaders,
|
|
33
|
+
"X-Client-Id": clientId,
|
|
34
|
+
"X-Client-Info": JSON.stringify(meta),
|
|
35
|
+
};
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export function createApiClient(opts = {}) {
|
|
39
|
+
const clientId = crypto.randomUUID();
|
|
40
|
+
const {
|
|
41
|
+
checkStatus = false,
|
|
42
|
+
maxRetries,
|
|
43
|
+
backoff = 1000,
|
|
44
|
+
log = true,
|
|
45
|
+
meta = {},
|
|
46
|
+
} = opts;
|
|
47
|
+
const retryOpts = { maxRetries, backoff, log };
|
|
48
|
+
|
|
49
|
+
async function apiGet(url) {
|
|
50
|
+
return withRetry(`GET ${url}`, async () => {
|
|
51
|
+
const res = await fetch(url, {
|
|
52
|
+
headers: buildHeaders({}, clientId, meta),
|
|
53
|
+
});
|
|
54
|
+
if (checkStatus && !res.ok) {
|
|
55
|
+
const errText = await res.text();
|
|
56
|
+
throw new Error(`HTTP ${res.status}: ${errText.substring(0, 200)}`);
|
|
57
|
+
}
|
|
58
|
+
return res.json();
|
|
59
|
+
}, retryOpts);
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
async function apiPost(url, body) {
|
|
63
|
+
return withRetry(`POST ${url}`, async () => {
|
|
64
|
+
const res = await fetch(url, {
|
|
65
|
+
method: "POST",
|
|
66
|
+
headers: buildHeaders(
|
|
67
|
+
{ "Content-Type": "application/json" },
|
|
68
|
+
clientId,
|
|
69
|
+
meta,
|
|
70
|
+
),
|
|
71
|
+
body: JSON.stringify(body),
|
|
72
|
+
});
|
|
73
|
+
if (checkStatus && !res.ok) {
|
|
74
|
+
const errText = await res.text();
|
|
75
|
+
throw new Error(`HTTP ${res.status}: ${errText.substring(0, 200)}`);
|
|
76
|
+
}
|
|
77
|
+
return res.json();
|
|
78
|
+
}, retryOpts);
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
async function apiPut(url, body) {
|
|
82
|
+
return withRetry(`PUT ${url}`, async () => {
|
|
83
|
+
const res = await fetch(url, {
|
|
84
|
+
method: "PUT",
|
|
85
|
+
headers: buildHeaders(
|
|
86
|
+
{ "Content-Type": "application/json" },
|
|
87
|
+
clientId,
|
|
88
|
+
meta,
|
|
89
|
+
),
|
|
90
|
+
body: body ? JSON.stringify(body) : undefined,
|
|
91
|
+
});
|
|
92
|
+
if (checkStatus && !res.ok) {
|
|
93
|
+
const errText = await res.text();
|
|
94
|
+
throw new Error(`HTTP ${res.status}: ${errText.substring(0, 200)}`);
|
|
95
|
+
}
|
|
96
|
+
return res.json();
|
|
97
|
+
}, retryOpts);
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
return { apiGet, apiPost, apiPut };
|
|
101
|
+
}
|
package/src/watch/data-store.js
CHANGED
|
@@ -1951,6 +1951,8 @@ export function createStore(filePath, options = {}) {
|
|
|
1951
1951
|
let clientErrors = new Map();
|
|
1952
1952
|
// 客户端登录状态:userId → boolean
|
|
1953
1953
|
let clientLoginStatus = new Map();
|
|
1954
|
+
// 活跃客户端追踪:clientId → { type, ip, port, userId, lastSeen }
|
|
1955
|
+
let activeClients = new Map();
|
|
1954
1956
|
// refill 锁:防止多个 claimNextJob 同时触发 LLM refill
|
|
1955
1957
|
let refillLock = null; // Promise | null
|
|
1956
1958
|
// LLM 采样偏移量记忆:按猜测国家记录上次查询位置,避免重复采样
|
|
@@ -3928,6 +3930,38 @@ Standards: 90-100=clear match, 70-89=likely, 50-69=possible, 20-49=low, 0-19=unl
|
|
|
3928
3930
|
return Object.fromEntries(clientLoginStatus);
|
|
3929
3931
|
}
|
|
3930
3932
|
|
|
3933
|
+
function trackClient(clientId, info) {
|
|
3934
|
+
const existing = activeClients.get(clientId);
|
|
3935
|
+
if (existing) {
|
|
3936
|
+
if (info.type) existing.type = info.type;
|
|
3937
|
+
if (info.userId) existing.userId = info.userId;
|
|
3938
|
+
if (info.ip) existing.ip = info.ip;
|
|
3939
|
+
if (info.port !== undefined) existing.port = info.port;
|
|
3940
|
+
existing.lastSeen = Date.now();
|
|
3941
|
+
} else {
|
|
3942
|
+
activeClients.set(clientId, {
|
|
3943
|
+
...info,
|
|
3944
|
+
lastSeen: Date.now(),
|
|
3945
|
+
});
|
|
3946
|
+
}
|
|
3947
|
+
}
|
|
3948
|
+
|
|
3949
|
+
function getActiveClients() {
|
|
3950
|
+
const now = Date.now();
|
|
3951
|
+
const stale = 2 * 60 * 1000;
|
|
3952
|
+
for (const [id, info] of activeClients) {
|
|
3953
|
+
if (now - info.lastSeen > stale) activeClients.delete(id);
|
|
3954
|
+
}
|
|
3955
|
+
return Array.from(activeClients.entries()).map(([clientId, info]) => ({
|
|
3956
|
+
clientId,
|
|
3957
|
+
type: info.type || "unknown",
|
|
3958
|
+
ip: info.ip || "",
|
|
3959
|
+
port: info.port || 0,
|
|
3960
|
+
userId: info.userId || "",
|
|
3961
|
+
lastSeen: info.lastSeen,
|
|
3962
|
+
}));
|
|
3963
|
+
}
|
|
3964
|
+
|
|
3931
3965
|
function getPendingUserUpdateTasks(limit, countries) {
|
|
3932
3966
|
const targetCountries = countries
|
|
3933
3967
|
? countries.map((c) => String(c).trim().toUpperCase())
|
|
@@ -4493,6 +4527,8 @@ Standards: 90-100=clear match, 70-89=likely, 50-69=possible, 20-49=low, 0-19=unl
|
|
|
4493
4527
|
deleteClientError,
|
|
4494
4528
|
getClientErrors,
|
|
4495
4529
|
getClientLoginStatus,
|
|
4530
|
+
trackClient,
|
|
4531
|
+
getActiveClients,
|
|
4496
4532
|
registerVideos,
|
|
4497
4533
|
getVideo,
|
|
4498
4534
|
getVideos,
|
package/src/watch/public/app.js
CHANGED
|
@@ -221,6 +221,7 @@ function renderStats() {
|
|
|
221
221
|
renderCountryChart(d.countryStats);
|
|
222
222
|
renderSourceChart(d.sourceStats);
|
|
223
223
|
renderTargetLocationFilter(d.targetCountryStats);
|
|
224
|
+
renderActiveClients(d.activeClients || []);
|
|
224
225
|
}
|
|
225
226
|
|
|
226
227
|
function renderTargetLocationFilter(targetCountryStats) {
|
|
@@ -237,6 +238,100 @@ function renderTargetLocationFilter(targetCountryStats) {
|
|
|
237
238
|
.join("");
|
|
238
239
|
}
|
|
239
240
|
|
|
241
|
+
function formatRelativeTime(ts) {
|
|
242
|
+
const diff = Math.max(0, Date.now() - ts);
|
|
243
|
+
if (diff < 5000) return "刚刚";
|
|
244
|
+
const s = Math.round(diff / 1000);
|
|
245
|
+
if (s < 60) return s + " 秒前";
|
|
246
|
+
const m = Math.round(s / 60);
|
|
247
|
+
if (m < 60) return m + " 分钟前";
|
|
248
|
+
return Math.round(m / 60) + " 小时前";
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
function renderActiveClients(clients) {
|
|
252
|
+
const section = document.getElementById("activeClientsSection");
|
|
253
|
+
const bar = document.getElementById("activeClientsBar");
|
|
254
|
+
const table = document.getElementById("activeClientsTable");
|
|
255
|
+
const tbody = document.getElementById("activeClientsBody");
|
|
256
|
+
if (!section || !bar) return;
|
|
257
|
+
|
|
258
|
+
const types = ["explore", "refresh", "attach", "comments"];
|
|
259
|
+
const labels = { explore: "Explore", refresh: "Refresh", attach: "Attach", comments: "Comments" };
|
|
260
|
+
const grouped = {};
|
|
261
|
+
for (const c of clients) {
|
|
262
|
+
if (!grouped[c.type]) grouped[c.type] = [];
|
|
263
|
+
grouped[c.type].push(c);
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
const counts = {};
|
|
267
|
+
for (const t of types) counts[t] = (grouped[t] || []).length;
|
|
268
|
+
const total = clients.length;
|
|
269
|
+
|
|
270
|
+
if (total === 0) {
|
|
271
|
+
section.style.display = "none";
|
|
272
|
+
return;
|
|
273
|
+
}
|
|
274
|
+
section.style.display = "";
|
|
275
|
+
|
|
276
|
+
// 清除之前的选择状态
|
|
277
|
+
let selectedType = bar.dataset.selectedType || "";
|
|
278
|
+
|
|
279
|
+
const parts = ['<span class="bar-label">活跃客户端</span>'];
|
|
280
|
+
for (const t of types) {
|
|
281
|
+
const count = counts[t] || 0;
|
|
282
|
+
const cls = count > 0 ? "active" : "inactive";
|
|
283
|
+
const dot = count > 0 ? '<span class="dot"></span>' : "";
|
|
284
|
+
const sel = t === selectedType ? " selected" : "";
|
|
285
|
+
parts.push(
|
|
286
|
+
`<span class="client-badge ${cls}${sel}" data-type="${t}" onclick="toggleClientDetail('${t}')">${dot}${labels[t]}: ${count}</span>`,
|
|
287
|
+
);
|
|
288
|
+
}
|
|
289
|
+
bar.innerHTML = parts.join("\n");
|
|
290
|
+
|
|
291
|
+
// 如果有选中类型,刷新明细
|
|
292
|
+
if (selectedType && grouped[selectedType]) {
|
|
293
|
+
showClientDetail(selectedType, grouped[selectedType]);
|
|
294
|
+
} else {
|
|
295
|
+
table.style.display = "none";
|
|
296
|
+
}
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
function toggleClientDetail(type) {
|
|
300
|
+
const bar = document.getElementById("activeClientsBar");
|
|
301
|
+
if (bar.dataset.selectedType === type) {
|
|
302
|
+
bar.dataset.selectedType = "";
|
|
303
|
+
document.getElementById("activeClientsTable").style.display = "none";
|
|
304
|
+
} else {
|
|
305
|
+
bar.dataset.selectedType = type;
|
|
306
|
+
}
|
|
307
|
+
if (currentStats) renderActiveClients(currentStats.activeClients || []);
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
function showClientDetail(type, clients) {
|
|
311
|
+
const table = document.getElementById("activeClientsTable");
|
|
312
|
+
const tbody = document.getElementById("activeClientsBody");
|
|
313
|
+
table.style.display = "";
|
|
314
|
+
tbody.innerHTML = clients
|
|
315
|
+
.map((c) => {
|
|
316
|
+
const cid = c.clientId ? c.clientId.substring(0, 8) : "-";
|
|
317
|
+
const ipPort = c.ip
|
|
318
|
+
? c.ip + (c.port ? ":" + c.port : "")
|
|
319
|
+
: "-";
|
|
320
|
+
const userId = c.userId || "-";
|
|
321
|
+
const last = formatRelativeTime(c.lastSeen);
|
|
322
|
+
return `<tr>
|
|
323
|
+
<td class="client-id">${cid}</td>
|
|
324
|
+
<td class="ip-port">${ipPort}</td>
|
|
325
|
+
<td class="ip-port">${c.port || "-"}</td>
|
|
326
|
+
<td class="user-id">${userId}</td>
|
|
327
|
+
<td class="last-seen">${last}</td>
|
|
328
|
+
</tr>`;
|
|
329
|
+
})
|
|
330
|
+
.join("");
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
window.toggleClientDetail = toggleClientDetail;
|
|
334
|
+
|
|
240
335
|
function renderCountryChart(countries) {
|
|
241
336
|
const el = document.getElementById("countryChart");
|
|
242
337
|
const filtered = countries.filter((c) => c.country !== "未知");
|
|
@@ -58,6 +58,21 @@
|
|
|
58
58
|
<div class="value target" id="statTarget">0</div>
|
|
59
59
|
</div>
|
|
60
60
|
</div>
|
|
61
|
+
<div id="activeClientsSection" class="active-clients-section" style="display:none">
|
|
62
|
+
<div class="active-clients-bar" id="activeClientsBar"></div>
|
|
63
|
+
<table class="active-clients-table" id="activeClientsTable" style="display:none">
|
|
64
|
+
<thead>
|
|
65
|
+
<tr>
|
|
66
|
+
<th>Client ID</th>
|
|
67
|
+
<th>IP</th>
|
|
68
|
+
<th>端口</th>
|
|
69
|
+
<th>用户编号</th>
|
|
70
|
+
<th>最后活跃</th>
|
|
71
|
+
</tr>
|
|
72
|
+
</thead>
|
|
73
|
+
<tbody id="activeClientsBody"></tbody>
|
|
74
|
+
</table>
|
|
75
|
+
</div>
|
|
61
76
|
<div class="client-errors-section" id="clientErrorsSection" style="display:none">
|
|
62
77
|
<div class="section-header">
|
|
63
78
|
<h3>客户端异常</h3>
|
|
@@ -118,6 +118,113 @@ body {
|
|
|
118
118
|
background: rgba(167, 139, 250, 0.15);
|
|
119
119
|
}
|
|
120
120
|
|
|
121
|
+
.active-clients-section {
|
|
122
|
+
background: #1a1a24;
|
|
123
|
+
border-radius: 8px;
|
|
124
|
+
padding: 12px 16px;
|
|
125
|
+
margin-bottom: 16px;
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
.active-clients-bar {
|
|
129
|
+
display: flex;
|
|
130
|
+
gap: 16px;
|
|
131
|
+
align-items: center;
|
|
132
|
+
flex-wrap: wrap;
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
.active-clients-bar .bar-label {
|
|
136
|
+
font-size: 13px;
|
|
137
|
+
color: #888;
|
|
138
|
+
margin-right: 4px;
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
.active-clients-bar .client-badge {
|
|
142
|
+
font-size: 13px;
|
|
143
|
+
padding: 3px 10px;
|
|
144
|
+
border-radius: 12px;
|
|
145
|
+
cursor: pointer;
|
|
146
|
+
transition: background 0.2s;
|
|
147
|
+
white-space: nowrap;
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
.active-clients-bar .client-badge .dot {
|
|
151
|
+
display: inline-block;
|
|
152
|
+
width: 8px;
|
|
153
|
+
height: 8px;
|
|
154
|
+
border-radius: 50%;
|
|
155
|
+
margin-right: 4px;
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
.active-clients-bar .client-badge.active {
|
|
159
|
+
background: #1a2e1a;
|
|
160
|
+
color: #4ade80;
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
.active-clients-bar .client-badge.active:hover {
|
|
164
|
+
background: #224422;
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
.active-clients-bar .client-badge.active .dot {
|
|
168
|
+
background: #4ade80;
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
.active-clients-bar .client-badge.inactive {
|
|
172
|
+
background: #252536;
|
|
173
|
+
color: #64748b;
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
.active-clients-bar .client-badge.selected {
|
|
177
|
+
outline: 2px solid #60a5fa;
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
.active-clients-table {
|
|
181
|
+
width: 100%;
|
|
182
|
+
border-collapse: collapse;
|
|
183
|
+
margin-top: 12px;
|
|
184
|
+
font-size: 13px;
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
.active-clients-table th {
|
|
188
|
+
text-align: left;
|
|
189
|
+
color: #888;
|
|
190
|
+
font-weight: 500;
|
|
191
|
+
padding: 6px 12px;
|
|
192
|
+
border-bottom: 1px solid #2a2a3a;
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
.active-clients-table td {
|
|
196
|
+
padding: 8px 12px;
|
|
197
|
+
border-bottom: 1px solid #1e1e2e;
|
|
198
|
+
color: #ccc;
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
.active-clients-table td.client-id {
|
|
202
|
+
font-family: monospace;
|
|
203
|
+
font-size: 12px;
|
|
204
|
+
color: #60a5fa;
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
.active-clients-table td.ip-port {
|
|
208
|
+
font-family: monospace;
|
|
209
|
+
font-size: 12px;
|
|
210
|
+
color: #facc15;
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
.active-clients-table td.user-id {
|
|
214
|
+
font-family: monospace;
|
|
215
|
+
font-size: 12px;
|
|
216
|
+
color: #c084fc;
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
.active-clients-table td.last-seen {
|
|
220
|
+
color: #94a3b8;
|
|
221
|
+
font-size: 12px;
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
.active-clients-table tr:hover td {
|
|
225
|
+
background: #1e1e2e;
|
|
226
|
+
}
|
|
227
|
+
|
|
121
228
|
.charts {
|
|
122
229
|
display: grid;
|
|
123
230
|
grid-template-columns: 1fr 1fr;
|
package/src/watch/server.js
CHANGED
|
@@ -89,6 +89,14 @@ function sendCSV(res, columns, rows) {
|
|
|
89
89
|
res.end(body);
|
|
90
90
|
}
|
|
91
91
|
|
|
92
|
+
function inferClientType(routePath) {
|
|
93
|
+
if (routePath.startsWith("/api/redo-job")) return "refresh";
|
|
94
|
+
if (routePath.startsWith("/api/user-update-tasks")) return "attach";
|
|
95
|
+
if (routePath.startsWith("/api/comment-task")) return "comments";
|
|
96
|
+
if (routePath.startsWith("/api/job") || routePath.startsWith("/api/explore-new")) return "explore";
|
|
97
|
+
return null;
|
|
98
|
+
}
|
|
99
|
+
|
|
92
100
|
export function startWatchServer(
|
|
93
101
|
dataAnchor,
|
|
94
102
|
port = 3000,
|
|
@@ -112,6 +120,19 @@ export function startWatchServer(
|
|
|
112
120
|
const server = http.createServer(async (req, res) => {
|
|
113
121
|
const { path: routePath, params } = parseQuery(req.url);
|
|
114
122
|
|
|
123
|
+
// 客户端追踪:从 header 提取 clientId 和元数据,根据路由推断类型
|
|
124
|
+
const clientId = req.headers["x-client-id"] || "";
|
|
125
|
+
if (clientId) {
|
|
126
|
+
let meta = {};
|
|
127
|
+
try {
|
|
128
|
+
meta = JSON.parse(req.headers["x-client-info"] || "{}");
|
|
129
|
+
} catch {}
|
|
130
|
+
const ip = req.socket.remoteAddress || "";
|
|
131
|
+
const userId = params.userId || "";
|
|
132
|
+
const type = inferClientType(routePath);
|
|
133
|
+
store.trackClient(clientId, { ip, port: meta.port, userId, type });
|
|
134
|
+
}
|
|
135
|
+
|
|
115
136
|
if (req.method === "POST" && routePath === "/api/users") {
|
|
116
137
|
try {
|
|
117
138
|
const { usernames, sources, guessedLocation } = await readBody(req);
|
|
@@ -382,6 +403,7 @@ export function startWatchServer(
|
|
|
382
403
|
const stats = computeStatsIncremental(store);
|
|
383
404
|
stats.targetLocations = DEFAULT_TARGET_LOCATIONS;
|
|
384
405
|
stats.clientLoginStatus = store.getClientLoginStatus();
|
|
406
|
+
stats.activeClients = store.getActiveClients();
|
|
385
407
|
stats.llmSampleOffsets = store.getLlmSampleOffsets(); // 添加偏移量状态
|
|
386
408
|
sendJSON(res, 200, stats);
|
|
387
409
|
return;
|