tt-help-cli-ycl 1.4.112 → 1.4.114
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/scripts/monitor-refresh-mini.mjs +161 -0
- package/src/clash/checker.js +18 -4
- package/src/clash/daemon.js +9 -0
- package/src/cli/product-fill.js +127 -14
- package/src/cli/refresh-mini.js +714 -0
- package/src/cli/refresh.js +28 -12
- package/src/cli/watchdog.js +5 -2
- package/src/lib/api-client.js +57 -42
- package/src/lib/api-interceptor.js +165 -49
- package/src/lib/args.js +112 -0
- package/src/lib/constants.js +22 -0
- package/src/main.js +3 -0
- package/src/npm-main.js +3 -0
- package/src/scraper/explore-core.js +24 -9
- package/src/watch/llm-config.js +14 -0
- package/src/watch/server.js +17 -0
- package/src/watchdog/process-manager.js +58 -1
- package/src/watchdog/reporter.js +1 -0
- package/src/watchdog/standalone-server.js +228 -0
|
@@ -0,0 +1,714 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* refresh-mini — 视频信息更新专用客户端
|
|
3
|
+
*
|
|
4
|
+
* 特性:
|
|
5
|
+
* - 固定 CDP 端口 4222(--port 可覆盖),不做登录检测、不做多账户轮换
|
|
6
|
+
* - 只做 mini 采集:视频列表 + 由视频推导的信息(EC视频、最新发布时间)
|
|
7
|
+
* - 代理切换采用「触发通知」模式(不自己选节点/验证):
|
|
8
|
+
* 需要切换时向 clash-daemon 发 failover 命令(fire-and-forget),
|
|
9
|
+
* 选节点、内容验证、失败回滚均由 daemon 完成
|
|
10
|
+
* 触发条件:定时轮换(--rotate-min)/ 验证码超时 / 连续错误 ≥ 3 / CDN ≥ 3
|
|
11
|
+
* 被封(0 字节空响应,api-interceptor 内部按 30/40/50s 重试 3 次)→ 仍封才触发切换
|
|
12
|
+
* - --ensure-daemon:启动时检查 daemon 是否存活,未存活则自动拉起(默认关闭)
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
import {
|
|
16
|
+
getOrCreatePage,
|
|
17
|
+
isBrowserClosedError,
|
|
18
|
+
relaunchBrowser,
|
|
19
|
+
} from "../lib/browser/page.js";
|
|
20
|
+
import {
|
|
21
|
+
delay,
|
|
22
|
+
getDelayConfig,
|
|
23
|
+
setDelayConfig,
|
|
24
|
+
} from "../scraper/modules/page-helpers.js";
|
|
25
|
+
import {
|
|
26
|
+
detectCaptcha,
|
|
27
|
+
closeCaptcha,
|
|
28
|
+
} from "../scraper/modules/captcha-handler.js";
|
|
29
|
+
import {
|
|
30
|
+
userId as configuredUserId,
|
|
31
|
+
proxy as configuredProxy,
|
|
32
|
+
saveUserId,
|
|
33
|
+
} from "../lib/constants.js";
|
|
34
|
+
import { getMacOrUuid } from "../lib/mac-or-uuid.js";
|
|
35
|
+
import {
|
|
36
|
+
ensureBrowserReady as ensureBrowserReadyCDP,
|
|
37
|
+
killEdgeProcesses,
|
|
38
|
+
} from "../lib/browser/cdp.js";
|
|
39
|
+
import { createApiClient } from "../lib/api-client.js";
|
|
40
|
+
import { requireClashOrExit } from "../clash/checker.js";
|
|
41
|
+
import { readHealth, writeCommand } from "../clash/ipc.js";
|
|
42
|
+
import { spawn } from "child_process";
|
|
43
|
+
import { fileURLToPath } from "url";
|
|
44
|
+
import path from "path";
|
|
45
|
+
import os from "os";
|
|
46
|
+
|
|
47
|
+
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
48
|
+
|
|
49
|
+
const DEFAULT_PORT = 4222;
|
|
50
|
+
const STARTUP_TIKTOK_URL = "https://www.tiktok.com/@ycl5007";
|
|
51
|
+
const PAGE_GOTO_TIMEOUT = 60000;
|
|
52
|
+
const MAX_RETRY_WAIT = 5 * 60 * 1000;
|
|
53
|
+
const ERROR_SWITCH_THRESHOLD = 3; // 连续任务错误达到该次数 → 触发切换通知
|
|
54
|
+
const SWITCH_RECEIPT_TIMEOUT_MS = 90_000; // 等待 daemon 切换回执上限(10s 轮询 + 候选逐个验证)
|
|
55
|
+
const JS_CACHE_MAX = 80; // 静态 JS 缓存条数上限(每条数百 KB,约数十 MB)
|
|
56
|
+
|
|
57
|
+
// TikTok 静态 JS 内存缓存:URL(含版本 hash,天然失效)→ Buffer
|
|
58
|
+
// 跨任务复用,第二次导航直接从本地回源,省去代理节点重复下载耗时。
|
|
59
|
+
// 注意:CDP 复用场景下 route.fetch() 不走 Edge 的代理会超时,
|
|
60
|
+
// 因此缓存来源用 page.on('response') 监听浏览器原生下载(走代理正常)
|
|
61
|
+
const jsCache = new Map();
|
|
62
|
+
|
|
63
|
+
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
64
|
+
|
|
65
|
+
async function withRetry(label, fn) {
|
|
66
|
+
let backoff = 1000;
|
|
67
|
+
while (true) {
|
|
68
|
+
try {
|
|
69
|
+
return await fn();
|
|
70
|
+
} catch (err) {
|
|
71
|
+
console.error(
|
|
72
|
+
`[连接] ${label} 失败: ${err.message},${backoff / 1000}秒后重试...`,
|
|
73
|
+
);
|
|
74
|
+
await sleep(backoff);
|
|
75
|
+
backoff = Math.min(backoff * 2, MAX_RETRY_WAIT);
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* 拉起 clash-daemon(后台分离进程,fire-and-forget)
|
|
82
|
+
* 记录 PID 供退出时清理(只清理自己拉起的)
|
|
83
|
+
*/
|
|
84
|
+
let spawnedDaemonPid = null;
|
|
85
|
+
|
|
86
|
+
function spawnClashDaemon() {
|
|
87
|
+
try {
|
|
88
|
+
const mainJs = path.join(__dirname, "..", "main.js");
|
|
89
|
+
const child = spawn(process.execPath, [mainJs, "clash-daemon"], {
|
|
90
|
+
detached: true,
|
|
91
|
+
stdio: "ignore",
|
|
92
|
+
cwd: path.dirname(mainJs),
|
|
93
|
+
});
|
|
94
|
+
child.unref();
|
|
95
|
+
spawnedDaemonPid = child.pid;
|
|
96
|
+
return true;
|
|
97
|
+
} catch (e) {
|
|
98
|
+
console.error(`[daemon] 启动失败: ${e.message}`);
|
|
99
|
+
return false;
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/** 关闭自己拉起的 daemon(daemon 原本就在跑则不碰) */
|
|
104
|
+
function stopSpawnedDaemon() {
|
|
105
|
+
if (!spawnedDaemonPid) return;
|
|
106
|
+
try {
|
|
107
|
+
process.kill(spawnedDaemonPid, "SIGTERM");
|
|
108
|
+
console.error(
|
|
109
|
+
`[daemon] 已关闭本进程拉起的 clash-daemon (pid ${spawnedDaemonPid})`,
|
|
110
|
+
);
|
|
111
|
+
} catch {
|
|
112
|
+
/* 已退出或不存在 */
|
|
113
|
+
}
|
|
114
|
+
spawnedDaemonPid = null;
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
/**
|
|
118
|
+
* 用 daemon 的 IPC 健康状态判断其是否存活(黑盒接口:clash-health.json)
|
|
119
|
+
*/
|
|
120
|
+
function isDaemonAlive() {
|
|
121
|
+
const h = readHealth();
|
|
122
|
+
return !!(h && !h.stale);
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
export async function handleRefreshMini(options) {
|
|
126
|
+
const {
|
|
127
|
+
explorePreset,
|
|
128
|
+
exploreInterval,
|
|
129
|
+
exploreLocation,
|
|
130
|
+
exploreJobLocations,
|
|
131
|
+
exploreMaxUsers,
|
|
132
|
+
explorePort,
|
|
133
|
+
exploreUserId,
|
|
134
|
+
exploreMaxVideos,
|
|
135
|
+
exploreRedoMaxAge,
|
|
136
|
+
exploreProxy,
|
|
137
|
+
serverUrl,
|
|
138
|
+
rotateMin,
|
|
139
|
+
captchaWaitMin,
|
|
140
|
+
ensureDaemon,
|
|
141
|
+
} = options;
|
|
142
|
+
|
|
143
|
+
const effectiveProxy = exploreProxy || configuredProxy;
|
|
144
|
+
let resolvedProxy = effectiveProxy;
|
|
145
|
+
if (
|
|
146
|
+
!effectiveProxy ||
|
|
147
|
+
effectiveProxy.includes("127.0.0.1") ||
|
|
148
|
+
effectiveProxy.includes("localhost")
|
|
149
|
+
) {
|
|
150
|
+
// refresh-mini 必须走本地 Clash:未配置代理时自动发现,节点轮换才有意义
|
|
151
|
+
resolvedProxy = await requireClashOrExit("refresh-mini");
|
|
152
|
+
} else {
|
|
153
|
+
console.error(
|
|
154
|
+
`[警告] 使用非本地代理 ${effectiveProxy},Clash 节点轮换将无效`,
|
|
155
|
+
);
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
let userId = exploreUserId || configuredUserId;
|
|
159
|
+
let browser = null;
|
|
160
|
+
let cdpOptions = null;
|
|
161
|
+
let shuttingDown = false;
|
|
162
|
+
|
|
163
|
+
const shutdown = async (signal) => {
|
|
164
|
+
if (shuttingDown) return;
|
|
165
|
+
shuttingDown = true;
|
|
166
|
+
console.error(`\n[refresh-mini] 收到 ${signal},正在关闭浏览器...`);
|
|
167
|
+
stopSpawnedDaemon();
|
|
168
|
+
if (cdpOptions) {
|
|
169
|
+
await killEdgeProcesses(cdpOptions.userDataDir, cdpOptions.port);
|
|
170
|
+
}
|
|
171
|
+
await browser?.close().catch(() => {});
|
|
172
|
+
console.error("[refresh-mini] 已退出");
|
|
173
|
+
process.exit(0);
|
|
174
|
+
};
|
|
175
|
+
|
|
176
|
+
const onSigint = () => void shutdown("SIGINT");
|
|
177
|
+
const onSigterm = () => void shutdown("SIGTERM");
|
|
178
|
+
process.once("SIGINT", onSigint);
|
|
179
|
+
process.once("SIGTERM", onSigterm);
|
|
180
|
+
|
|
181
|
+
try {
|
|
182
|
+
if (!userId) {
|
|
183
|
+
userId = await getMacOrUuid();
|
|
184
|
+
saveUserId(userId);
|
|
185
|
+
console.error(`[初始化] 未检测到本地用户编号,已生成并使用: ${userId}`);
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
setDelayConfig(explorePreset);
|
|
189
|
+
|
|
190
|
+
console.error(`\n=== refresh-mini 模式(视频信息更新) ===`);
|
|
191
|
+
console.error(`服务器: ${serverUrl}`);
|
|
192
|
+
console.error(`CDP 端口: ${explorePort || DEFAULT_PORT}(固定)`);
|
|
193
|
+
console.error(`采集范围: mini(仅视频及视频相关信息)`);
|
|
194
|
+
console.error(`视频采集: ${exploreMaxVideos || 16}`);
|
|
195
|
+
console.error(`国家筛选: ${exploreLocation}`);
|
|
196
|
+
if (exploreJobLocations) console.error(`任务国家: ${exploreJobLocations}`);
|
|
197
|
+
console.error(`空闲间隔: ${exploreInterval || 30} 秒`);
|
|
198
|
+
console.error(
|
|
199
|
+
`代理: ${resolvedProxy || "无"}${
|
|
200
|
+
rotateMin > 0
|
|
201
|
+
? `(每 ${rotateMin} 分钟轮换节点)`
|
|
202
|
+
: "(定时轮换已关闭)"
|
|
203
|
+
}`,
|
|
204
|
+
);
|
|
205
|
+
console.error(`验证码等待人工解决上限: ${captchaWaitMin || 10} 分钟`);
|
|
206
|
+
if (exploreMaxUsers > 0) console.error(`上限: ${exploreMaxUsers} 个用户`);
|
|
207
|
+
|
|
208
|
+
cdpOptions = {
|
|
209
|
+
port: explorePort || DEFAULT_PORT,
|
|
210
|
+
userDataDir: path.join(
|
|
211
|
+
os.homedir(),
|
|
212
|
+
"Library",
|
|
213
|
+
"Application Support",
|
|
214
|
+
`Microsoft Edge For Testing_p${explorePort || DEFAULT_PORT}`,
|
|
215
|
+
),
|
|
216
|
+
};
|
|
217
|
+
if (resolvedProxy) cdpOptions.proxyServer = resolvedProxy;
|
|
218
|
+
|
|
219
|
+
const { apiGet, apiPost } = createApiClient({
|
|
220
|
+
meta: { port: cdpOptions.port, clientType: "refresh-mini" },
|
|
221
|
+
});
|
|
222
|
+
|
|
223
|
+
// 连接服务器验证
|
|
224
|
+
await apiGet(`${serverUrl}/api/stats`);
|
|
225
|
+
|
|
226
|
+
browser = await ensureBrowserReadyCDP(cdpOptions);
|
|
227
|
+
const { processExplore } = await import("../scraper/explore-core.js");
|
|
228
|
+
|
|
229
|
+
let page = await getOrCreatePage(browser);
|
|
230
|
+
|
|
231
|
+
// 视频资源拦截:
|
|
232
|
+
// 1) 资源类型:media / image / stylesheet 直接拦
|
|
233
|
+
// 2) 视频文件后缀:.mp4/.m3u8/.ts(fetch 拉流带扩展名的情况)
|
|
234
|
+
// 3) TikTok 视频流专用 CDN:v16/v19/v77 等「v数字开头」域名,路径无扩展名
|
|
235
|
+
// (hover 预览、首页自动播放都是这类地址,fetch 类型,后缀规则拦不住)
|
|
236
|
+
// 注意不能拦 tiktokv.us / www.tiktok.com/api 等业务域名(webid/mcs 关键 API)
|
|
237
|
+
function isBlockableResource(route) {
|
|
238
|
+
const request = route.request();
|
|
239
|
+
const resourceType = request.resourceType();
|
|
240
|
+
if (
|
|
241
|
+
resourceType === "image" ||
|
|
242
|
+
resourceType === "stylesheet" ||
|
|
243
|
+
resourceType === "media"
|
|
244
|
+
) {
|
|
245
|
+
return true;
|
|
246
|
+
}
|
|
247
|
+
const url = request.url();
|
|
248
|
+
if (/\.(mp4|m3u8|ts)(\?|$)/i.test(url)) {
|
|
249
|
+
return true;
|
|
250
|
+
}
|
|
251
|
+
return /^https?:\/\/v\d+[a-z0-9-]*\.(tiktokcdn(-us)?\.(com|net)|tiktok\.com)\//i.test(
|
|
252
|
+
url,
|
|
253
|
+
);
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
async function setupNewPage(newBrowser) {
|
|
257
|
+
const newPage = await getOrCreatePage(newBrowser);
|
|
258
|
+
// 源头兜底:让 video.play() 变为空操作,无论视频流走什么地址/加载方式都无法播放
|
|
259
|
+
// (采集数据全部来自 API 拦截,不依赖视频播放,此注入无副作用)
|
|
260
|
+
await newPage.addInitScript(() => {
|
|
261
|
+
HTMLMediaElement.prototype.play = function () {
|
|
262
|
+
try {
|
|
263
|
+
this.pause();
|
|
264
|
+
} catch {}
|
|
265
|
+
return Promise.resolve();
|
|
266
|
+
};
|
|
267
|
+
});
|
|
268
|
+
// script 下载完成后缓存 body(浏览器原生下载,走代理正常;
|
|
269
|
+
// 资源已释放时 getResponseBody 会失败,忽略即可,不影响功能)
|
|
270
|
+
newPage.on("response", async (res) => {
|
|
271
|
+
if (res.request().resourceType() !== "script") return;
|
|
272
|
+
const url = res.url();
|
|
273
|
+
if (jsCache.has(url) || jsCache.size >= JS_CACHE_MAX) return;
|
|
274
|
+
try {
|
|
275
|
+
jsCache.set(url, await res.body());
|
|
276
|
+
} catch {}
|
|
277
|
+
});
|
|
278
|
+
await newPage.route("**/*", (route) => {
|
|
279
|
+
const request = route.request();
|
|
280
|
+
if (isBlockableResource(route)) {
|
|
281
|
+
route.abort();
|
|
282
|
+
return;
|
|
283
|
+
}
|
|
284
|
+
// JS 静态资源内存缓存:命中直接本地回源
|
|
285
|
+
if (request.resourceType() === "script") {
|
|
286
|
+
const cached = jsCache.get(request.url());
|
|
287
|
+
if (cached) {
|
|
288
|
+
route.fulfill({
|
|
289
|
+
status: 200,
|
|
290
|
+
contentType: "application/javascript",
|
|
291
|
+
body: cached,
|
|
292
|
+
});
|
|
293
|
+
return;
|
|
294
|
+
}
|
|
295
|
+
}
|
|
296
|
+
route.continue();
|
|
297
|
+
});
|
|
298
|
+
return newPage;
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
page = await setupNewPage(browser);
|
|
302
|
+
|
|
303
|
+
// 导航到 TikTok 页面
|
|
304
|
+
await withRetry("启动页面导航", async () => {
|
|
305
|
+
await page.goto(STARTUP_TIKTOK_URL, {
|
|
306
|
+
waitUntil: "domcontentloaded",
|
|
307
|
+
timeout: PAGE_GOTO_TIMEOUT,
|
|
308
|
+
});
|
|
309
|
+
});
|
|
310
|
+
|
|
311
|
+
// --ensure-daemon:检查 clash-daemon 是否存活,未存活则拉起(默认不检测)
|
|
312
|
+
if (ensureDaemon) {
|
|
313
|
+
if (isDaemonAlive()) {
|
|
314
|
+
const h = readHealth();
|
|
315
|
+
console.error(
|
|
316
|
+
`[daemon] 已运行(上次巡检 ${h?.ageSeconds ?? "?"}s 前)`,
|
|
317
|
+
);
|
|
318
|
+
} else {
|
|
319
|
+
console.error(`[daemon] 未检测到运行中的 clash-daemon,正在拉起...`);
|
|
320
|
+
console.error(
|
|
321
|
+
spawnClashDaemon()
|
|
322
|
+
? `[daemon] 已发送启动命令(后台分离进程)`
|
|
323
|
+
: `[daemon] 启动失败,切换通知将无人处理`,
|
|
324
|
+
);
|
|
325
|
+
}
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
let proxySwitchCount = 0;
|
|
329
|
+
let lastProxyTriggerTime = Date.now(); // 上次触发切换的时间(定时轮换节流用)
|
|
330
|
+
let consecutiveTaskErrors = 0; // 任务级连续错误计数(触发代理切换,成功或发通知后归零)
|
|
331
|
+
|
|
332
|
+
/**
|
|
333
|
+
* 触发代理切换:向 daemon 发 failover 命令,等待回执后再返回。
|
|
334
|
+
* 回执 = clash-health.json 的 lastSwitch 字段(含本次 reason),
|
|
335
|
+
* healthy 字段即 daemon 的内容验证结果(切换后是否真的可用)。
|
|
336
|
+
* 超时(daemon 未运行等)返回 null,不阻塞后续处理。
|
|
337
|
+
*/
|
|
338
|
+
async function triggerProxySwitch(reason) {
|
|
339
|
+
proxySwitchCount++;
|
|
340
|
+
lastProxyTriggerTime = Date.now();
|
|
341
|
+
consecutiveTaskErrors = 0;
|
|
342
|
+
const before = readHealth()?.lastSwitch || "";
|
|
343
|
+
console.error(
|
|
344
|
+
`[代理] 触发切换通知(${reason},累计 ${proxySwitchCount} 次),等待 daemon 回执...`,
|
|
345
|
+
);
|
|
346
|
+
writeCommand({ action: "failover", reason: `refresh-mini: ${reason}` });
|
|
347
|
+
|
|
348
|
+
const deadline = Date.now() + SWITCH_RECEIPT_TIMEOUT_MS;
|
|
349
|
+
while (Date.now() < deadline) {
|
|
350
|
+
await sleep(3000);
|
|
351
|
+
const h = readHealth();
|
|
352
|
+
if (
|
|
353
|
+
h &&
|
|
354
|
+
h.lastSwitch &&
|
|
355
|
+
h.lastSwitch !== before &&
|
|
356
|
+
h.lastSwitch.includes(reason)
|
|
357
|
+
) {
|
|
358
|
+
console.error(
|
|
359
|
+
`[代理] 切换回执: ${h.lastSwitch}${h.healthy ? "(健康)" : "(异常)"}`,
|
|
360
|
+
);
|
|
361
|
+
return !!h.healthy;
|
|
362
|
+
}
|
|
363
|
+
}
|
|
364
|
+
console.error(
|
|
365
|
+
`[代理] 等待切换回执超时(${SWITCH_RECEIPT_TIMEOUT_MS / 1000}s),daemon 可能未运行,继续处理`,
|
|
366
|
+
);
|
|
367
|
+
return null;
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
/** 等待人工在浏览器中解决验证码,超时返回 false */
|
|
371
|
+
async function waitForCaptchaResolved() {
|
|
372
|
+
const deadline = Date.now() + (captchaWaitMin || 10) * 60000;
|
|
373
|
+
console.error(
|
|
374
|
+
` [验证码] 等待人工在浏览器中解决(上限 ${captchaWaitMin || 10} 分钟)...`,
|
|
375
|
+
);
|
|
376
|
+
while (Date.now() < deadline) {
|
|
377
|
+
await sleep(15000);
|
|
378
|
+
let visible = true;
|
|
379
|
+
try {
|
|
380
|
+
const r = await detectCaptcha(page);
|
|
381
|
+
visible = !!(r && r.visible);
|
|
382
|
+
} catch {
|
|
383
|
+
visible = false;
|
|
384
|
+
}
|
|
385
|
+
if (!visible) {
|
|
386
|
+
console.error(" [验证码] 已解决,继续处理");
|
|
387
|
+
return true;
|
|
388
|
+
}
|
|
389
|
+
await closeCaptcha(page).catch(() => {});
|
|
390
|
+
}
|
|
391
|
+
console.error(" [验证码] 等待超时未解决");
|
|
392
|
+
return false;
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
let processedCount = 0;
|
|
396
|
+
let errorCount = 0;
|
|
397
|
+
let consecutiveNetworkErrors = 0; // 领取任务 API 失败计数(服务端问题,不切代理)
|
|
398
|
+
let captchaCount = 0;
|
|
399
|
+
|
|
400
|
+
const processOptions = () => ({
|
|
401
|
+
maxVideos: exploreMaxVideos || 16,
|
|
402
|
+
enableFollow: false,
|
|
403
|
+
loggedIn: false,
|
|
404
|
+
maxFollowing: 0,
|
|
405
|
+
maxFollowers: 0,
|
|
406
|
+
location: exploreLocation,
|
|
407
|
+
locationMode: "refresh",
|
|
408
|
+
browser,
|
|
409
|
+
proxyServer: cdpOptions.proxyServer || null,
|
|
410
|
+
webserverUrl: null,
|
|
411
|
+
noProduct: true,
|
|
412
|
+
mini: true,
|
|
413
|
+
});
|
|
414
|
+
|
|
415
|
+
const reportError = (
|
|
416
|
+
username,
|
|
417
|
+
errorType,
|
|
418
|
+
errorMessage,
|
|
419
|
+
stage,
|
|
420
|
+
errorStack,
|
|
421
|
+
) =>
|
|
422
|
+
withRetry("report error", () =>
|
|
423
|
+
apiPost(`${serverUrl}/api/error-report`, {
|
|
424
|
+
userId,
|
|
425
|
+
username,
|
|
426
|
+
errorType,
|
|
427
|
+
errorMessage,
|
|
428
|
+
stage,
|
|
429
|
+
errorStack: errorStack || "",
|
|
430
|
+
}),
|
|
431
|
+
).catch(() => {});
|
|
432
|
+
|
|
433
|
+
const checkMaxUsers = () => {
|
|
434
|
+
if (exploreMaxUsers > 0 && processedCount >= exploreMaxUsers) {
|
|
435
|
+
console.error(`\n已达上限 ${exploreMaxUsers} 个用户,停止处理`);
|
|
436
|
+
return true;
|
|
437
|
+
}
|
|
438
|
+
return false;
|
|
439
|
+
};
|
|
440
|
+
|
|
441
|
+
console.error(`\n开始循环刷新任务...\n`);
|
|
442
|
+
|
|
443
|
+
while (true) {
|
|
444
|
+
if (shuttingDown) break;
|
|
445
|
+
|
|
446
|
+
// 定时轮换(任务间隙,预防性换 IP,等回执后再继续)
|
|
447
|
+
if (rotateMin > 0) {
|
|
448
|
+
const elapsedMin = (Date.now() - lastProxyTriggerTime) / 60000;
|
|
449
|
+
if (elapsedMin >= rotateMin) {
|
|
450
|
+
await triggerProxySwitch(`定时轮换(满 ${rotateMin} 分钟)`);
|
|
451
|
+
}
|
|
452
|
+
}
|
|
453
|
+
|
|
454
|
+
if (processedCount % 10 === 0 && processedCount > 0) {
|
|
455
|
+
console.error(
|
|
456
|
+
`[统计] 已处理 ${processedCount}, 出错 ${errorCount}, 已触发切换 ${proxySwitchCount} 次`,
|
|
457
|
+
);
|
|
458
|
+
}
|
|
459
|
+
|
|
460
|
+
// 领取 redo 任务
|
|
461
|
+
let redoUrl = `${serverUrl}/api/redo-job?userId=${userId}`;
|
|
462
|
+
if (exploreRedoMaxAge > 0) {
|
|
463
|
+
redoUrl += `&maxAge=${exploreRedoMaxAge}`;
|
|
464
|
+
}
|
|
465
|
+
if (exploreJobLocations) {
|
|
466
|
+
redoUrl += `&locations=${encodeURIComponent(exploreJobLocations)}`;
|
|
467
|
+
}
|
|
468
|
+
let job = null;
|
|
469
|
+
try {
|
|
470
|
+
job = await apiGet(redoUrl);
|
|
471
|
+
} catch (e) {
|
|
472
|
+
consecutiveNetworkErrors++;
|
|
473
|
+
console.error(
|
|
474
|
+
` [网络] 获取任务失败 (${consecutiveNetworkErrors}): ${e.message}`,
|
|
475
|
+
);
|
|
476
|
+
await sleep(10000);
|
|
477
|
+
continue;
|
|
478
|
+
}
|
|
479
|
+
|
|
480
|
+
if (!job.hasJob) {
|
|
481
|
+
console.error(
|
|
482
|
+
`\n[空闲] 暂无 redo 任务,${exploreInterval || 30}s 后重试...`,
|
|
483
|
+
);
|
|
484
|
+
await sleep((exploreInterval || 30) * 1000);
|
|
485
|
+
consecutiveNetworkErrors = 0;
|
|
486
|
+
continue;
|
|
487
|
+
}
|
|
488
|
+
|
|
489
|
+
consecutiveNetworkErrors = 0;
|
|
490
|
+
const username = job.user.uniqueId;
|
|
491
|
+
const nickname = job.user.nickname;
|
|
492
|
+
console.error(`\n[${processedCount}] 更新 @${username} 视频信息...`);
|
|
493
|
+
|
|
494
|
+
// 切换任务前检测验证码
|
|
495
|
+
const captchaResult = await detectCaptcha(page);
|
|
496
|
+
if (captchaResult && captchaResult.visible) {
|
|
497
|
+
captchaCount++;
|
|
498
|
+
console.error(
|
|
499
|
+
`\n[验证码] 切换任务前检测到验证码(累计 ${captchaCount} 次)`,
|
|
500
|
+
);
|
|
501
|
+
await reportError(
|
|
502
|
+
"unknown",
|
|
503
|
+
"captcha",
|
|
504
|
+
"切换任务前检测到验证码",
|
|
505
|
+
"between-tasks",
|
|
506
|
+
);
|
|
507
|
+
const resolved = await waitForCaptchaResolved();
|
|
508
|
+
if (!resolved) {
|
|
509
|
+
await triggerProxySwitch(
|
|
510
|
+
`验证码等待 ${captchaWaitMin || 10} 分钟未解决`,
|
|
511
|
+
);
|
|
512
|
+
captchaCount = 0;
|
|
513
|
+
}
|
|
514
|
+
}
|
|
515
|
+
|
|
516
|
+
const { switchMax } = getDelayConfig();
|
|
517
|
+
await delay(switchMax, switchMax * 3);
|
|
518
|
+
|
|
519
|
+
let result = await processExplore(
|
|
520
|
+
page,
|
|
521
|
+
username,
|
|
522
|
+
processOptions(),
|
|
523
|
+
console.error,
|
|
524
|
+
);
|
|
525
|
+
|
|
526
|
+
// 浏览器关闭检测 → 重启浏览器并重试
|
|
527
|
+
if (result.error && isBrowserClosedError(new Error(result.error))) {
|
|
528
|
+
console.error(" [恢复] 浏览器已关闭,重启浏览器...");
|
|
529
|
+
browser = await relaunchBrowser(cdpOptions, cdpOptions.port);
|
|
530
|
+
page = await setupNewPage(browser);
|
|
531
|
+
result = await processExplore(
|
|
532
|
+
page,
|
|
533
|
+
username,
|
|
534
|
+
processOptions(),
|
|
535
|
+
console.error,
|
|
536
|
+
);
|
|
537
|
+
}
|
|
538
|
+
|
|
539
|
+
if (result.restricted) {
|
|
540
|
+
consecutiveTaskErrors = 0;
|
|
541
|
+
await apiPost(`${serverUrl}/api/redo-job/${username}`, {
|
|
542
|
+
restricted: true,
|
|
543
|
+
userInfo: result.userInfo || {},
|
|
544
|
+
});
|
|
545
|
+
if (checkMaxUsers()) break;
|
|
546
|
+
continue;
|
|
547
|
+
}
|
|
548
|
+
|
|
549
|
+
// CDN 限流:渐进退避,连续达到阈值 → 触发切换通知
|
|
550
|
+
if (result.cdnBlocked) {
|
|
551
|
+
consecutiveTaskErrors++;
|
|
552
|
+
errorCount++;
|
|
553
|
+
const waitMs = Math.min(consecutiveTaskErrors * 30000, 120000);
|
|
554
|
+
console.error(
|
|
555
|
+
` [CDN] 限流,${waitMs / 1000}s 后重试(连续 ${consecutiveTaskErrors} 次)...`,
|
|
556
|
+
);
|
|
557
|
+
await sleep(waitMs);
|
|
558
|
+
if (consecutiveTaskErrors >= ERROR_SWITCH_THRESHOLD) {
|
|
559
|
+
await triggerProxySwitch(`CDN 限流连续 ${consecutiveTaskErrors} 次`);
|
|
560
|
+
}
|
|
561
|
+
if (checkMaxUsers()) break;
|
|
562
|
+
continue;
|
|
563
|
+
}
|
|
564
|
+
|
|
565
|
+
if (result.error) {
|
|
566
|
+
const isBlocked = result.error.startsWith("被封:");
|
|
567
|
+
consecutiveTaskErrors++;
|
|
568
|
+
errorCount++;
|
|
569
|
+
|
|
570
|
+
// 临时性错误:自动重试一次
|
|
571
|
+
if (result.retryable) {
|
|
572
|
+
console.error(` [临时错误] 等待 5 秒后重试 @${username}...`);
|
|
573
|
+
await sleep(5000);
|
|
574
|
+
result = await processExplore(
|
|
575
|
+
page,
|
|
576
|
+
username,
|
|
577
|
+
processOptions(),
|
|
578
|
+
console.error,
|
|
579
|
+
);
|
|
580
|
+
if (!result.error) {
|
|
581
|
+
consecutiveTaskErrors = 0;
|
|
582
|
+
errorCount--;
|
|
583
|
+
}
|
|
584
|
+
}
|
|
585
|
+
|
|
586
|
+
if (result.error) {
|
|
587
|
+
const stillBlocked = result.error.startsWith("被封:");
|
|
588
|
+
// 被封的 30/60/90s 重试确认已在 api-interceptor 内部完成,
|
|
589
|
+
// 到这里仍被封说明重试耗尽,直接回退任务并触发切换
|
|
590
|
+
// 任务回退给服务端,之后会被重新派发
|
|
591
|
+
await apiPost(`${serverUrl}/api/redo-job/${username}`, {
|
|
592
|
+
error: result.error,
|
|
593
|
+
});
|
|
594
|
+
await reportError(
|
|
595
|
+
username,
|
|
596
|
+
stillBlocked
|
|
597
|
+
? "被封"
|
|
598
|
+
: consecutiveTaskErrors > 1
|
|
599
|
+
? "network"
|
|
600
|
+
: "other",
|
|
601
|
+
result.error,
|
|
602
|
+
"process",
|
|
603
|
+
result.errorStack || "",
|
|
604
|
+
);
|
|
605
|
+
|
|
606
|
+
const shouldSwitch =
|
|
607
|
+
stillBlocked || consecutiveTaskErrors >= ERROR_SWITCH_THRESHOLD;
|
|
608
|
+
if (shouldSwitch) {
|
|
609
|
+
await triggerProxySwitch(
|
|
610
|
+
stillBlocked
|
|
611
|
+
? "账号/IP 被封(api-interceptor 30/40/50s 重试 3 次仍封)"
|
|
612
|
+
: `连续任务错误 ${consecutiveTaskErrors} 次`,
|
|
613
|
+
);
|
|
614
|
+
}
|
|
615
|
+
if (checkMaxUsers()) break;
|
|
616
|
+
continue;
|
|
617
|
+
}
|
|
618
|
+
}
|
|
619
|
+
|
|
620
|
+
if (result.captchaDetected) {
|
|
621
|
+
captchaCount++;
|
|
622
|
+
console.error(` [验证码] 累计 ${captchaCount} 次`);
|
|
623
|
+
await reportError(
|
|
624
|
+
username,
|
|
625
|
+
"captcha",
|
|
626
|
+
result.captchaMessage || "页面出现验证码",
|
|
627
|
+
result.captchaStage || "video-page",
|
|
628
|
+
);
|
|
629
|
+
const resolved = await waitForCaptchaResolved();
|
|
630
|
+
if (!resolved || captchaCount >= ERROR_SWITCH_THRESHOLD) {
|
|
631
|
+
await triggerProxySwitch(
|
|
632
|
+
resolved
|
|
633
|
+
? `验证码累计 ${captchaCount} 次`
|
|
634
|
+
: `验证码等待 ${captchaWaitMin || 10} 分钟未解决`,
|
|
635
|
+
);
|
|
636
|
+
captchaCount = 0;
|
|
637
|
+
}
|
|
638
|
+
}
|
|
639
|
+
|
|
640
|
+
consecutiveTaskErrors = 0;
|
|
641
|
+
processedCount++;
|
|
642
|
+
|
|
643
|
+
// refresh 模式:confirmedLocation 是二次确认的国家,写入 confirmed_location 列
|
|
644
|
+
// locationCreated 保持原始值不变
|
|
645
|
+
const refreshLocation =
|
|
646
|
+
result.confirmedLocation || result.locationCreated;
|
|
647
|
+
const guessedLocation = refreshLocation || null;
|
|
648
|
+
|
|
649
|
+
// latestVideoTime 是视频推导的零成本信息,需更新服务端 latest_video_time
|
|
650
|
+
//(影响任务排序 ORDER BY latest_video_time DESC 与前端显示)
|
|
651
|
+
const refreshUserInfo = { ...(result.userInfo || {}) };
|
|
652
|
+
if (result.confirmedLocation) {
|
|
653
|
+
refreshUserInfo.confirmedLocation = result.confirmedLocation;
|
|
654
|
+
}
|
|
655
|
+
if (result.latestVideoTime) {
|
|
656
|
+
refreshUserInfo.latestVideoTime = result.latestVideoTime;
|
|
657
|
+
}
|
|
658
|
+
|
|
659
|
+
const payload = {
|
|
660
|
+
userInfo: refreshUserInfo,
|
|
661
|
+
recentEcVideos: result.recentEcVideos || null,
|
|
662
|
+
topRecentVideo: result.topRecentVideo || null,
|
|
663
|
+
productDetail: result.productDetail || null,
|
|
664
|
+
discoveredFollowing: (result.discoveredFollowing || []).map((f) => ({
|
|
665
|
+
handle: Array.isArray(f) ? f[0] : f,
|
|
666
|
+
displayName: Array.isArray(f) ? f[1] : null,
|
|
667
|
+
guessedLocation,
|
|
668
|
+
})),
|
|
669
|
+
discoveredFollowers: (result.discoveredFollowers || []).map((f) => ({
|
|
670
|
+
handle: Array.isArray(f) ? f[0] : f,
|
|
671
|
+
displayName: Array.isArray(f) ? f[1] : null,
|
|
672
|
+
guessedLocation,
|
|
673
|
+
})),
|
|
674
|
+
discoveredRecommended: (result.discoveredRecommended || []).map(
|
|
675
|
+
(f) => ({
|
|
676
|
+
handle: Array.isArray(f) ? f[0] : f,
|
|
677
|
+
displayName: Array.isArray(f) ? f[1] : null,
|
|
678
|
+
guessedLocation,
|
|
679
|
+
}),
|
|
680
|
+
),
|
|
681
|
+
processed: result.processed,
|
|
682
|
+
hasFollowData: result.hasFollowData,
|
|
683
|
+
keepFollow: result.keepFollow,
|
|
684
|
+
noVideo: result.noVideo,
|
|
685
|
+
collectedVideos: result.collectedVideos,
|
|
686
|
+
};
|
|
687
|
+
await apiPost(`${serverUrl}/api/redo-job/${username}`, payload);
|
|
688
|
+
console.error(" 已提交");
|
|
689
|
+
|
|
690
|
+
// 预防性重置:每 50 个任务刷新页面,避免 SPA 状态累积导致主线程死锁
|
|
691
|
+
//(卡死现象:日志停在"[API拦截] 获取...",刷新页面才恢复)
|
|
692
|
+
if (processedCount > 0 && processedCount % 50 === 0) {
|
|
693
|
+
console.error(" [维护] 已处理 50 个任务,刷新页面防止累积卡死...");
|
|
694
|
+
await page
|
|
695
|
+
.reload({ waitUntil: "domcontentloaded", timeout: 30000 })
|
|
696
|
+
.catch(() => {});
|
|
697
|
+
console.error(" [维护] 页面刷新完成");
|
|
698
|
+
}
|
|
699
|
+
|
|
700
|
+
if (checkMaxUsers()) break;
|
|
701
|
+
}
|
|
702
|
+
|
|
703
|
+
const stats = await apiGet(`${serverUrl}/api/stats`);
|
|
704
|
+
console.error(`\n完成: ${processedCount} 个用户处理, ${errorCount} 个出错`);
|
|
705
|
+
console.error(
|
|
706
|
+
` 总用户: ${stats.totalUsers}, 已完成: ${stats.processedUsers}, 待处理: ${stats.pendingUsers}, 错误: ${stats.errorUsers}`,
|
|
707
|
+
);
|
|
708
|
+
} finally {
|
|
709
|
+
process.removeListener("SIGINT", onSigint);
|
|
710
|
+
process.removeListener("SIGTERM", onSigterm);
|
|
711
|
+
stopSpawnedDaemon();
|
|
712
|
+
await browser?.close().catch(() => {});
|
|
713
|
+
}
|
|
714
|
+
}
|