tt-help-cli-ycl 1.4.113 → 1.4.114
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/scripts/monitor-refresh-mini.mjs +161 -0
- package/src/clash/checker.js +18 -4
- package/src/clash/daemon.js +9 -0
- package/src/cli/refresh-mini.js +714 -0
- package/src/cli/refresh.js +28 -12
- package/src/cli/watchdog.js +5 -2
- package/src/lib/api-client.js +57 -42
- package/src/lib/api-interceptor.js +165 -49
- package/src/lib/args.js +86 -0
- package/src/lib/constants.js +22 -0
- package/src/main.js +3 -0
- package/src/npm-main.js +3 -0
- package/src/scraper/explore-core.js +24 -9
- package/src/watch/llm-config.js +14 -0
- package/src/watch/server.js +17 -0
- package/src/watchdog/process-manager.js +58 -1
- package/src/watchdog/reporter.js +1 -0
- package/src/watchdog/standalone-server.js +228 -0
package/src/cli/refresh.js
CHANGED
|
@@ -71,7 +71,12 @@ export async function handleRefresh(options) {
|
|
|
71
71
|
exploreProxy,
|
|
72
72
|
exploreWebserver,
|
|
73
73
|
exploreNoProduct,
|
|
74
|
+
exploreMini,
|
|
74
75
|
} = options;
|
|
76
|
+
// --mini 模式:只采集视频列表及由视频推导的信息(EC视频、最新发布时间),
|
|
77
|
+
// 跳过用户信息、国家采样、关注/粉丝提取(最大耗时项)和商品详情抓取(由独立商品程序异步补抓),大幅提速。
|
|
78
|
+
const followEnabled = !exploreMini && exploreEnableFollow !== false;
|
|
79
|
+
const skipProduct = exploreMini || !!exploreNoProduct;
|
|
75
80
|
const effectiveProxy = exploreProxy || configuredProxy;
|
|
76
81
|
let resolvedProxy = effectiveProxy;
|
|
77
82
|
if (
|
|
@@ -122,8 +127,11 @@ export async function handleRefresh(options) {
|
|
|
122
127
|
console.error(`\n=== Refresh 模式(基于 explore) ===`);
|
|
123
128
|
console.error(`服务器: ${serverUrl}`);
|
|
124
129
|
console.error(`视频采集: ${exploreMaxVideos || 16}`);
|
|
130
|
+
if (exploreMini) {
|
|
131
|
+
console.error(`采集范围: mini(仅视频及视频相关信息)`);
|
|
132
|
+
}
|
|
125
133
|
console.error(
|
|
126
|
-
`关注/粉丝: ${
|
|
134
|
+
`关注/粉丝: ${followEnabled ? "启用" : "禁用"} (${exploreMaxFollowing}/${exploreMaxFollowers})`,
|
|
127
135
|
);
|
|
128
136
|
console.error(`国家筛选: ${exploreLocation}`);
|
|
129
137
|
if (exploreJobLocations) console.error(`任务国家: ${exploreJobLocations}`);
|
|
@@ -368,11 +376,11 @@ export async function handleRefresh(options) {
|
|
|
368
376
|
await handleAccountSwitch(checkResult.reason);
|
|
369
377
|
} else if (checkResult.info && processedCount % 10 === 0) {
|
|
370
378
|
const followElapsed =
|
|
371
|
-
loggedIn &&
|
|
379
|
+
loggedIn && followEnabled
|
|
372
380
|
? Math.round((Date.now() - lastFollowSuccessTime) / 60000)
|
|
373
381
|
: 0;
|
|
374
382
|
const followStatus =
|
|
375
|
-
loggedIn &&
|
|
383
|
+
loggedIn && followEnabled
|
|
376
384
|
? ` | 关注/粉丝上次成功 ${followElapsed} 分钟前`
|
|
377
385
|
: "";
|
|
378
386
|
console.error(`\n[健康检查] ${checkResult.info}${followStatus}`);
|
|
@@ -446,8 +454,8 @@ export async function handleRefresh(options) {
|
|
|
446
454
|
}
|
|
447
455
|
}
|
|
448
456
|
|
|
449
|
-
//
|
|
450
|
-
if (loggedIn &&
|
|
457
|
+
// 关注/粉丝封禁检测(mini 模式下不采集关注/粉丝,无需检测)
|
|
458
|
+
if (loggedIn && followEnabled) {
|
|
451
459
|
const followElapsed = Date.now() - lastFollowSuccessTime;
|
|
452
460
|
if (followElapsed > FOLLOW_BLOCK_THRESHOLD) {
|
|
453
461
|
const minutes = Math.round(followElapsed / 60000);
|
|
@@ -481,7 +489,7 @@ export async function handleRefresh(options) {
|
|
|
481
489
|
username,
|
|
482
490
|
{
|
|
483
491
|
maxVideos: exploreMaxVideos || 16,
|
|
484
|
-
enableFollow:
|
|
492
|
+
enableFollow: followEnabled,
|
|
485
493
|
loggedIn,
|
|
486
494
|
maxFollowing: exploreMaxFollowing || 100,
|
|
487
495
|
maxFollowers: exploreMaxFollowers || 100,
|
|
@@ -490,7 +498,8 @@ export async function handleRefresh(options) {
|
|
|
490
498
|
browser,
|
|
491
499
|
proxyServer: cdpOptions.proxyServer || null,
|
|
492
500
|
webserverUrl: exploreWebserver || null,
|
|
493
|
-
noProduct:
|
|
501
|
+
noProduct: skipProduct,
|
|
502
|
+
mini: !!exploreMini,
|
|
494
503
|
},
|
|
495
504
|
console.error,
|
|
496
505
|
);
|
|
@@ -509,7 +518,7 @@ export async function handleRefresh(options) {
|
|
|
509
518
|
username,
|
|
510
519
|
{
|
|
511
520
|
maxVideos: exploreMaxVideos || 16,
|
|
512
|
-
enableFollow:
|
|
521
|
+
enableFollow: followEnabled,
|
|
513
522
|
loggedIn,
|
|
514
523
|
maxFollowing: exploreMaxFollowing || 100,
|
|
515
524
|
maxFollowers: exploreMaxFollowers || 100,
|
|
@@ -518,7 +527,8 @@ export async function handleRefresh(options) {
|
|
|
518
527
|
browser,
|
|
519
528
|
proxyServer: cdpOptions.proxyServer || null,
|
|
520
529
|
webserverUrl: exploreWebserver || null,
|
|
521
|
-
noProduct:
|
|
530
|
+
noProduct: skipProduct,
|
|
531
|
+
mini: !!exploreMini,
|
|
522
532
|
},
|
|
523
533
|
console.error,
|
|
524
534
|
);
|
|
@@ -559,7 +569,7 @@ export async function handleRefresh(options) {
|
|
|
559
569
|
username,
|
|
560
570
|
{
|
|
561
571
|
maxVideos: exploreMaxVideos || 16,
|
|
562
|
-
enableFollow:
|
|
572
|
+
enableFollow: followEnabled,
|
|
563
573
|
loggedIn,
|
|
564
574
|
maxFollowing: exploreMaxFollowing || 100,
|
|
565
575
|
maxFollowers: exploreMaxFollowers || 100,
|
|
@@ -568,7 +578,8 @@ export async function handleRefresh(options) {
|
|
|
568
578
|
browser,
|
|
569
579
|
proxyServer: cdpOptions.proxyServer || null,
|
|
570
580
|
webserverUrl: exploreWebserver || null,
|
|
571
|
-
noProduct:
|
|
581
|
+
noProduct: skipProduct,
|
|
582
|
+
mini: !!exploreMini,
|
|
572
583
|
},
|
|
573
584
|
console.error,
|
|
574
585
|
);
|
|
@@ -656,11 +667,16 @@ export async function handleRefresh(options) {
|
|
|
656
667
|
result.confirmedLocation || result.locationCreated;
|
|
657
668
|
const guessedLocation = refreshLocation || null;
|
|
658
669
|
|
|
659
|
-
// 把 confirmedLocation 合并到 userInfo 中(通过 commitRedoJob 写入 DB)
|
|
670
|
+
// 把 confirmedLocation / latestVideoTime 合并到 userInfo 中(通过 commitRedoJob 写入 DB)
|
|
671
|
+
// latestVideoTime 单独合并:mini 模式下 userInfo 为 null,但 latestVideoTime
|
|
672
|
+
// 是视频推导的零成本信息,仍需更新服务端 latest_video_time(影响任务排序与前端显示)
|
|
660
673
|
const refreshUserInfo = { ...(result.userInfo || {}) };
|
|
661
674
|
if (result.confirmedLocation) {
|
|
662
675
|
refreshUserInfo.confirmedLocation = result.confirmedLocation;
|
|
663
676
|
}
|
|
677
|
+
if (result.latestVideoTime) {
|
|
678
|
+
refreshUserInfo.latestVideoTime = result.latestVideoTime;
|
|
679
|
+
}
|
|
664
680
|
|
|
665
681
|
const payload = {
|
|
666
682
|
userInfo: refreshUserInfo,
|
package/src/cli/watchdog.js
CHANGED
|
@@ -22,10 +22,13 @@ export async function handleWatchdog(options) {
|
|
|
22
22
|
watchdogNoDaemon,
|
|
23
23
|
} = options;
|
|
24
24
|
|
|
25
|
-
// 服务端地址:--server 参数 > ~/.tt-help.json server > 默认值
|
|
25
|
+
// 服务端地址:--server 参数 > ~/.tt-help.json watchdogServer > server > 默认值
|
|
26
26
|
const clientConfig = getClientConfig();
|
|
27
27
|
const serverUrl =
|
|
28
|
-
watchdogServer ||
|
|
28
|
+
watchdogServer ||
|
|
29
|
+
clientConfig.watchdogServer ||
|
|
30
|
+
clientConfig.server ||
|
|
31
|
+
"http://127.0.0.1:3001";
|
|
29
32
|
|
|
30
33
|
// 客户端 ID:优先 --client-id,其次 userId@hostname,最后 hostname+pid
|
|
31
34
|
const userId = clientConfig.userId || "";
|
package/src/lib/api-client.js
CHANGED
|
@@ -47,54 +47,69 @@ export function createApiClient(opts = {}) {
|
|
|
47
47
|
const retryOpts = { maxRetries, backoff, log };
|
|
48
48
|
|
|
49
49
|
async function apiGet(url) {
|
|
50
|
-
return withRetry(
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
50
|
+
return withRetry(
|
|
51
|
+
`GET ${url}`,
|
|
52
|
+
async () => {
|
|
53
|
+
const res = await fetch(url, {
|
|
54
|
+
headers: buildHeaders({}, clientId, meta),
|
|
55
|
+
signal: AbortSignal.timeout(30000),
|
|
56
|
+
});
|
|
57
|
+
if (checkStatus && !res.ok) {
|
|
58
|
+
const errText = await res.text();
|
|
59
|
+
throw new Error(`HTTP ${res.status}: ${errText.substring(0, 200)}`);
|
|
60
|
+
}
|
|
61
|
+
return res.json();
|
|
62
|
+
},
|
|
63
|
+
retryOpts,
|
|
64
|
+
);
|
|
60
65
|
}
|
|
61
66
|
|
|
62
67
|
async function apiPost(url, body) {
|
|
63
|
-
return withRetry(
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
68
|
+
return withRetry(
|
|
69
|
+
`POST ${url}`,
|
|
70
|
+
async () => {
|
|
71
|
+
const res = await fetch(url, {
|
|
72
|
+
method: "POST",
|
|
73
|
+
headers: buildHeaders(
|
|
74
|
+
{ "Content-Type": "application/json" },
|
|
75
|
+
clientId,
|
|
76
|
+
meta,
|
|
77
|
+
),
|
|
78
|
+
body: JSON.stringify(body),
|
|
79
|
+
signal: AbortSignal.timeout(30000),
|
|
80
|
+
});
|
|
81
|
+
if (checkStatus && !res.ok) {
|
|
82
|
+
const errText = await res.text();
|
|
83
|
+
throw new Error(`HTTP ${res.status}: ${errText.substring(0, 200)}`);
|
|
84
|
+
}
|
|
85
|
+
return res.json();
|
|
86
|
+
},
|
|
87
|
+
retryOpts,
|
|
88
|
+
);
|
|
79
89
|
}
|
|
80
90
|
|
|
81
91
|
async function apiPut(url, body) {
|
|
82
|
-
return withRetry(
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
92
|
+
return withRetry(
|
|
93
|
+
`PUT ${url}`,
|
|
94
|
+
async () => {
|
|
95
|
+
const res = await fetch(url, {
|
|
96
|
+
method: "PUT",
|
|
97
|
+
headers: buildHeaders(
|
|
98
|
+
{ "Content-Type": "application/json" },
|
|
99
|
+
clientId,
|
|
100
|
+
meta,
|
|
101
|
+
),
|
|
102
|
+
body: body ? JSON.stringify(body) : undefined,
|
|
103
|
+
signal: AbortSignal.timeout(30000),
|
|
104
|
+
});
|
|
105
|
+
if (checkStatus && !res.ok) {
|
|
106
|
+
const errText = await res.text();
|
|
107
|
+
throw new Error(`HTTP ${res.status}: ${errText.substring(0, 200)}`);
|
|
108
|
+
}
|
|
109
|
+
return res.json();
|
|
110
|
+
},
|
|
111
|
+
retryOpts,
|
|
112
|
+
);
|
|
98
113
|
}
|
|
99
114
|
|
|
100
115
|
return { apiGet, apiPost, apiPut };
|
|
@@ -142,24 +142,25 @@ async function processAPIResponse(
|
|
|
142
142
|
return videoMap;
|
|
143
143
|
}
|
|
144
144
|
|
|
145
|
-
|
|
146
|
-
|
|
145
|
+
// 默认被封重试间隔:30/40/50 秒共 3 次,仍封才抛"被封"交给上层。
|
|
146
|
+
// RM_BLOCKED_WAIT_MS 仅用于测试缩短所有等待
|
|
147
|
+
const _rmBlockedWait = Number(process.env.RM_BLOCKED_WAIT_MS);
|
|
148
|
+
const DEFAULT_BLOCKED_RETRY_WAITS =
|
|
149
|
+
_rmBlockedWait > 0
|
|
150
|
+
? [_rmBlockedWait, _rmBlockedWait, _rmBlockedWait]
|
|
151
|
+
: [30_000, 40_000, 50_000];
|
|
147
152
|
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
throw new Error(`CDP 连接异常: ${e.message}`);
|
|
154
|
-
}
|
|
155
|
-
|
|
156
|
-
let apiRequestUrl = null;
|
|
153
|
+
/**
|
|
154
|
+
* 单次尝试:注册拦截 → goto 用户主页 → 等 item_list 响应 → 解析 JSON。
|
|
155
|
+
* apiRequestUrlRef 跨次复用(URL 相同,保留第一次捕获的即可)
|
|
156
|
+
*/
|
|
157
|
+
async function attemptItemListFetch(page, username, log, apiRequestUrlRef) {
|
|
157
158
|
let sawApiRequest = false;
|
|
158
159
|
|
|
159
160
|
const requestHandler = (request) => {
|
|
160
161
|
if (request.url().includes("/api/post/item_list/")) {
|
|
161
162
|
sawApiRequest = true;
|
|
162
|
-
if (!
|
|
163
|
+
if (!apiRequestUrlRef.value) apiRequestUrlRef.value = request.url();
|
|
163
164
|
}
|
|
164
165
|
};
|
|
165
166
|
|
|
@@ -168,18 +169,28 @@ async function fetchUserVideosAPI(page, username, maxVideos, log) {
|
|
|
168
169
|
const t0 = Date.now();
|
|
169
170
|
let data = null;
|
|
170
171
|
let interceptionError = null;
|
|
172
|
+
let responsePromise = null;
|
|
173
|
+
let response = null;
|
|
174
|
+
let rawBodyBytes = -1; // -1 未知(未读取到 body);0 = 服务端空响应(封禁判定依据)
|
|
171
175
|
|
|
172
176
|
try {
|
|
177
|
+
// 响应监听必须在 goto 之前注册:domcontentloaded 返回时首个 item_list
|
|
178
|
+
// 响应可能已经到达,晚注册会错过它、被迫等下一个请求或超时
|
|
179
|
+
responsePromise = page.waitForResponse(
|
|
180
|
+
(res) => res.url().includes("/api/post/item_list/"),
|
|
181
|
+
{ timeout: 30000 },
|
|
182
|
+
);
|
|
183
|
+
// 防 unhandledRejection:goto 卡住期间该 promise 可能先超时 reject,
|
|
184
|
+
// 未被 await 的 rejection 在 Node 默认配置下会直接崩进程
|
|
185
|
+
responsePromise.catch(() => {});
|
|
186
|
+
|
|
173
187
|
await page.goto(`https://www.tiktok.com/@${username}`, {
|
|
174
188
|
waitUntil: "domcontentloaded",
|
|
175
189
|
timeout: 30000,
|
|
176
190
|
});
|
|
177
191
|
await assertPageUrl(page, `@${username}`);
|
|
178
192
|
|
|
179
|
-
|
|
180
|
-
(res) => res.url().includes("/api/post/item_list/"),
|
|
181
|
-
{ timeout: 30000 },
|
|
182
|
-
);
|
|
193
|
+
response = await responsePromise;
|
|
183
194
|
|
|
184
195
|
// 超时保护:response.json() 内部调用 CDP Network.getResponseBody,
|
|
185
196
|
// 当页面刷新/验证码导致响应资源丢失时会挂起,需独立超时控制
|
|
@@ -194,50 +205,155 @@ async function fetchUserVideosAPI(page, username, maxVideos, log) {
|
|
|
194
205
|
]);
|
|
195
206
|
} catch (e) {
|
|
196
207
|
interceptionError = e.message;
|
|
208
|
+
// JSON 解析失败(如 "Unexpected end of JSON input")时打印原始响应,
|
|
209
|
+
// 便于诊断空响应/截断响应/非 JSON 响应等异常形态
|
|
210
|
+
if (response) {
|
|
211
|
+
try {
|
|
212
|
+
// body 读取同样可能因页面异常而挂起,加独立超时保护
|
|
213
|
+
const raw = await Promise.race([
|
|
214
|
+
response.body(),
|
|
215
|
+
new Promise((_, reject) =>
|
|
216
|
+
setTimeout(() => reject(new Error("body 读取超时(15s)")), 15000),
|
|
217
|
+
),
|
|
218
|
+
]);
|
|
219
|
+
const text = raw.toString("utf-8");
|
|
220
|
+
rawBodyBytes = raw.length;
|
|
221
|
+
// content-length 用于区分两种 0 字节形态:
|
|
222
|
+
// 声明长度 > 实际字节 = 传输被截断;声明缺失/为 0 = 服务端主动空响应
|
|
223
|
+
const declared = parseInt(
|
|
224
|
+
response.headers()["content-length"] || "0",
|
|
225
|
+
10,
|
|
226
|
+
);
|
|
227
|
+
const clNote =
|
|
228
|
+
raw.length === 0
|
|
229
|
+
? declared > 0
|
|
230
|
+
? `content-length 声明 ${declared}(传输截断)`
|
|
231
|
+
: "content-length 缺失/为 0(服务端主动空响应)"
|
|
232
|
+
: `content-length ${declared || "无"}`;
|
|
233
|
+
log(
|
|
234
|
+
` [API拦截] item_list 响应异常: HTTP ${response.status()}, ${raw.length} 字节, ${clNote}, 内容: ${text.slice(0, 200).replace(/\s+/g, " ")}`,
|
|
235
|
+
);
|
|
236
|
+
} catch (e2) {
|
|
237
|
+
log(` [API拦截] item_list 响应体读取失败: ${e2.message}`);
|
|
238
|
+
}
|
|
239
|
+
}
|
|
197
240
|
} finally {
|
|
241
|
+
// goto 提前失败时丢弃未等待的响应 promise,避免 unhandledRejection
|
|
242
|
+
responsePromise?.catch(() => {});
|
|
198
243
|
page.off("request", requestHandler);
|
|
199
244
|
}
|
|
200
245
|
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
maxVideos,
|
|
209
|
-
[],
|
|
210
|
-
page,
|
|
211
|
-
apiRequestUrl,
|
|
212
|
-
log,
|
|
213
|
-
);
|
|
214
|
-
}
|
|
246
|
+
return {
|
|
247
|
+
data,
|
|
248
|
+
interceptionError,
|
|
249
|
+
rawBodyBytes,
|
|
250
|
+
elapsed: Date.now() - t0,
|
|
251
|
+
};
|
|
252
|
+
}
|
|
215
253
|
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
254
|
+
async function fetchUserVideosAPI(
|
|
255
|
+
page,
|
|
256
|
+
username,
|
|
257
|
+
maxVideos,
|
|
258
|
+
log,
|
|
259
|
+
options = {},
|
|
260
|
+
) {
|
|
261
|
+
log(` [API拦截] 获取 @${username} 视频 ...`);
|
|
220
262
|
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
263
|
+
// CDP 健康检查:带超时保护。页面主线程死锁时 evaluate 可能长时间无响应,
|
|
264
|
+
// 超时后自动刷新页面恢复(实测刷新页面即可解除死锁)
|
|
265
|
+
const evalWithTimeout = (label, ms) =>
|
|
266
|
+
Promise.race([
|
|
267
|
+
page.evaluate(() => 1),
|
|
268
|
+
new Promise((_, reject) =>
|
|
269
|
+
setTimeout(
|
|
270
|
+
() => reject(new Error(`${label} 超时(${ms / 1000}s)`)),
|
|
271
|
+
ms,
|
|
272
|
+
),
|
|
273
|
+
),
|
|
274
|
+
]);
|
|
224
275
|
|
|
225
|
-
|
|
226
|
-
|
|
276
|
+
try {
|
|
277
|
+
await evalWithTimeout("CDP 健康检查", 10000);
|
|
278
|
+
} catch (e) {
|
|
279
|
+
log(` [API拦截] CDP 连接异常: ${e.message},刷新页面恢复...`);
|
|
280
|
+
await page
|
|
281
|
+
.reload({ waitUntil: "domcontentloaded", timeout: 30000 })
|
|
282
|
+
.catch(() => {});
|
|
283
|
+
try {
|
|
284
|
+
await evalWithTimeout("刷新后重检", 10000);
|
|
285
|
+
log(` [API拦截] 页面已恢复`);
|
|
286
|
+
} catch (e2) {
|
|
287
|
+
throw new Error(`CDP 连接异常(刷新后仍失败): ${e2.message}`);
|
|
288
|
+
}
|
|
227
289
|
}
|
|
228
290
|
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
291
|
+
let apiRequestUrlRef = { value: null };
|
|
292
|
+
const { blockedRetryWaitsMs = DEFAULT_BLOCKED_RETRY_WAITS } = options;
|
|
293
|
+
const retryWaits = Array.isArray(blockedRetryWaitsMs)
|
|
294
|
+
? blockedRetryWaitsMs
|
|
295
|
+
: DEFAULT_BLOCKED_RETRY_WAITS;
|
|
296
|
+
let attempt = 0;
|
|
297
|
+
|
|
298
|
+
while (true) {
|
|
299
|
+
attempt++;
|
|
300
|
+
const { data, interceptionError, rawBodyBytes, elapsed } =
|
|
301
|
+
await attemptItemListFetch(page, username, log, apiRequestUrlRef);
|
|
302
|
+
|
|
303
|
+
if (data && Array.isArray(data.itemList) && data.itemList.length > 0) {
|
|
304
|
+
log(` [API拦截] 成功,耗时 ${elapsed}ms`);
|
|
305
|
+
return await processAPIResponse(
|
|
306
|
+
data,
|
|
307
|
+
username,
|
|
308
|
+
maxVideos,
|
|
309
|
+
[],
|
|
310
|
+
page,
|
|
311
|
+
apiRequestUrlRef.value,
|
|
312
|
+
log,
|
|
313
|
+
);
|
|
314
|
+
}
|
|
233
315
|
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
316
|
+
// 0 字节空响应 = 被封(content-length 缺失/为 0 或截断,统一按封禁处理)。
|
|
317
|
+
// 不再依赖页面"出错了/很抱歉"文案判定,静默型封禁(页面正常渲染但 API 空响应)也能识别。
|
|
318
|
+
// 默认 30/40/50 秒等待重试 3 次,仍封才抛错交给上层(explore/refresh/refresh-mini 均受益)
|
|
319
|
+
if (rawBodyBytes === 0) {
|
|
320
|
+
const retryIndex = attempt - 1;
|
|
321
|
+
if (retryIndex < retryWaits.length) {
|
|
322
|
+
const waitMs = retryWaits[retryIndex];
|
|
323
|
+
log(
|
|
324
|
+
` [API拦截] item_list 空响应(0 字节),判定为被封,${waitMs / 1000}s 后第 ${attempt}/${retryWaits.length} 次重试...`,
|
|
325
|
+
);
|
|
326
|
+
await new Promise((r) => setTimeout(r, waitMs));
|
|
327
|
+
continue;
|
|
328
|
+
}
|
|
329
|
+
log(
|
|
330
|
+
` [API拦截] item_list 空响应(0 字节),重试 ${retryWaits.length} 次仍被封`,
|
|
331
|
+
);
|
|
332
|
+
throw new Error(`被封: ${username}`);
|
|
333
|
+
}
|
|
334
|
+
|
|
335
|
+
let pageError = await detectPageError(page);
|
|
336
|
+
if (!pageError) {
|
|
337
|
+
pageError = await detectPageErrorWithWait(page, 5000);
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
if (pageError === "cdn_blocked") {
|
|
341
|
+
throw new Error(`CDN限流: ${username}`);
|
|
342
|
+
}
|
|
238
343
|
|
|
239
|
-
|
|
240
|
-
|
|
344
|
+
if (pageError) {
|
|
345
|
+
log(` [API拦截] 数据为空,识别到页面错误: ${pageError}`);
|
|
346
|
+
return new Map();
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
if (interceptionError) {
|
|
350
|
+
log(` [API拦截] 超时/拦截失败: ${interceptionError},耗时 ${elapsed}ms`);
|
|
351
|
+
throw new Error(`@${username} 页面异常(API拦截超时,未识别到错误类型)`);
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
log(` [API拦截] 数据为空,未识别到错误类型,当作无视频处理`);
|
|
355
|
+
return new Map();
|
|
356
|
+
}
|
|
241
357
|
}
|
|
242
358
|
|
|
243
359
|
export { fetchUserVideosAPI };
|
package/src/lib/args.js
CHANGED
|
@@ -515,11 +515,14 @@ function parseRefreshArgs(args) {
|
|
|
515
515
|
let exploreJobLocations = null;
|
|
516
516
|
let exploreWebserver = null;
|
|
517
517
|
let exploreNoProduct = false;
|
|
518
|
+
let exploreMini = false;
|
|
518
519
|
|
|
519
520
|
for (let i = 0; i < args.length; i++) {
|
|
520
521
|
const arg = args[i];
|
|
521
522
|
if (arg === "--server") {
|
|
522
523
|
serverUrl = args[++i];
|
|
524
|
+
} else if (arg === "--mini") {
|
|
525
|
+
exploreMini = true;
|
|
523
526
|
} else if (arg === "--preset") {
|
|
524
527
|
explorePreset = args[++i];
|
|
525
528
|
} else if (arg === "-i" || arg === "--interval") {
|
|
@@ -581,6 +584,85 @@ function parseRefreshArgs(args) {
|
|
|
581
584
|
exploreProxy,
|
|
582
585
|
exploreWebserver,
|
|
583
586
|
exploreNoProduct,
|
|
587
|
+
exploreMini,
|
|
588
|
+
serverUrl,
|
|
589
|
+
urls: [],
|
|
590
|
+
outputFormat: "json",
|
|
591
|
+
exploreCount: 0,
|
|
592
|
+
showConfig: false,
|
|
593
|
+
showHelp: false,
|
|
594
|
+
customProxy: null,
|
|
595
|
+
configAction: null,
|
|
596
|
+
configValue: null,
|
|
597
|
+
pipeMode: false,
|
|
598
|
+
filterStr: null,
|
|
599
|
+
};
|
|
600
|
+
}
|
|
601
|
+
|
|
602
|
+
function parseRefreshMiniArgs(args) {
|
|
603
|
+
let serverUrl = defaultServer;
|
|
604
|
+
let explorePreset = "normal";
|
|
605
|
+
let exploreInterval = 30;
|
|
606
|
+
let exploreLocation = DEFAULT_TARGET_LOCATIONS_CSV;
|
|
607
|
+
let exploreMaxUsers = 0;
|
|
608
|
+
let explorePort = 4222;
|
|
609
|
+
let exploreUserId = null;
|
|
610
|
+
let exploreMaxVideos = 16;
|
|
611
|
+
let exploreRedoMaxAge = 0;
|
|
612
|
+
let exploreProxy = null;
|
|
613
|
+
let exploreJobLocations = null;
|
|
614
|
+
let rotateMin = 10;
|
|
615
|
+
let captchaWaitMin = 10;
|
|
616
|
+
let ensureDaemon = false;
|
|
617
|
+
|
|
618
|
+
for (let i = 0; i < args.length; i++) {
|
|
619
|
+
const arg = args[i];
|
|
620
|
+
if (arg === "--server") {
|
|
621
|
+
serverUrl = args[++i];
|
|
622
|
+
} else if (arg === "--preset") {
|
|
623
|
+
explorePreset = args[++i];
|
|
624
|
+
} else if (arg === "-i" || arg === "--interval") {
|
|
625
|
+
exploreInterval = parseInt(args[++i], 10) || 30;
|
|
626
|
+
} else if (arg === "--port") {
|
|
627
|
+
explorePort = parseInt(args[++i]) || 4222;
|
|
628
|
+
} else if (arg === "--user-id") {
|
|
629
|
+
exploreUserId = args[++i];
|
|
630
|
+
} else if (arg === "--max-videos") {
|
|
631
|
+
exploreMaxVideos = parseInt(args[++i]) || 16;
|
|
632
|
+
} else if (arg === "--max-age") {
|
|
633
|
+
exploreRedoMaxAge = parseInt(args[++i]) || 0;
|
|
634
|
+
} else if (arg === "--proxy") {
|
|
635
|
+
exploreProxy = args[++i];
|
|
636
|
+
} else if (arg === "--location") {
|
|
637
|
+
exploreLocation = args[++i];
|
|
638
|
+
} else if (arg === "--job-locations") {
|
|
639
|
+
exploreJobLocations = args[++i];
|
|
640
|
+
} else if (arg === "--max-users") {
|
|
641
|
+
exploreMaxUsers = parseInt(args[++i]) || 0;
|
|
642
|
+
} else if (arg === "--rotate-min") {
|
|
643
|
+
rotateMin = parseInt(args[++i], 10) || 0;
|
|
644
|
+
} else if (arg === "--captcha-wait-min") {
|
|
645
|
+
captchaWaitMin = parseInt(args[++i], 10) || 10;
|
|
646
|
+
} else if (arg === "--ensure-daemon") {
|
|
647
|
+
ensureDaemon = true;
|
|
648
|
+
}
|
|
649
|
+
}
|
|
650
|
+
|
|
651
|
+
return {
|
|
652
|
+
subcommand: "refresh-mini",
|
|
653
|
+
explorePreset,
|
|
654
|
+
exploreInterval,
|
|
655
|
+
exploreLocation,
|
|
656
|
+
exploreJobLocations,
|
|
657
|
+
exploreMaxUsers,
|
|
658
|
+
explorePort,
|
|
659
|
+
exploreUserId,
|
|
660
|
+
exploreMaxVideos,
|
|
661
|
+
exploreRedoMaxAge,
|
|
662
|
+
exploreProxy,
|
|
663
|
+
rotateMin,
|
|
664
|
+
captchaWaitMin,
|
|
665
|
+
ensureDaemon,
|
|
584
666
|
serverUrl,
|
|
585
667
|
urls: [],
|
|
586
668
|
outputFormat: "json",
|
|
@@ -1472,6 +1554,10 @@ export function parseArgs() {
|
|
|
1472
1554
|
return parseRefreshArgs(args.slice(1));
|
|
1473
1555
|
}
|
|
1474
1556
|
|
|
1557
|
+
if (args.length > 0 && args[0] === "refresh-mini") {
|
|
1558
|
+
return parseRefreshMiniArgs(args.slice(1));
|
|
1559
|
+
}
|
|
1560
|
+
|
|
1475
1561
|
if (args.length > 0 && args[0] === "search") {
|
|
1476
1562
|
return parseSearchArgs(args.slice(1));
|
|
1477
1563
|
}
|
package/src/lib/constants.js
CHANGED
|
@@ -148,6 +148,7 @@ const HELP_TEXT = [
|
|
|
148
148
|
` --location <国家代码> 国家筛选,逗号分隔,默认 ${DEFAULT_TARGET_LOCATIONS_CSV}`,
|
|
149
149
|
" --job-locations <国家> 任务国家筛选,逗号分隔(仅筛选服务端任务),基于真实国家字段 location_created",
|
|
150
150
|
" --max-videos <数量> 每用户最大视频数,默认 16",
|
|
151
|
+
" --mini 精简模式:只采集视频列表及由视频推导的信息(EC视频、最新发布时间),跳过用户信息、国家采样、关注/粉丝与商品详情,大幅提速",
|
|
151
152
|
" --enable-follow 启用关注/粉丝提取(默认启用)",
|
|
152
153
|
" --disable-follow 禁用关注/粉丝提取",
|
|
153
154
|
" --max-following <数量> 最大获取关注数,默认 100",
|
|
@@ -162,6 +163,27 @@ const HELP_TEXT = [
|
|
|
162
163
|
"--proxy <地址> 浏览器代理(如 socks5://127.0.0.1:1080),未指定时使用配置文件 proxy",
|
|
163
164
|
"--webserver <URL> 集中式视频国家查询服务(如 http://127.0.0.1:3000),省去本地 Playwright 浏览器",
|
|
164
165
|
"",
|
|
166
|
+
" tt-help refresh-mini [选项]",
|
|
167
|
+
" 视频信息更新专用客户端:固定 CDP 端口 4222,只做 mini 采集(视频列表+视频推导信息)",
|
|
168
|
+
" 不检测登录、不做多账户轮换;Clash 节点自动轮换(每 10 分钟主动切换)",
|
|
169
|
+
" 验证码 → 等待人工在浏览器中解决(超时兜底切节点)",
|
|
170
|
+
" 被封 / 连续任务错误 ≥ 3 / CDN 限流连续 ≥ 3 → 切换代理节点",
|
|
171
|
+
" 选项:",
|
|
172
|
+
" --server <URL> 服务端地址,默认 http://127.0.0.1:3001",
|
|
173
|
+
` --location <国家代码> 国家筛选,逗号分隔,默认 ${DEFAULT_TARGET_LOCATIONS_CSV}`,
|
|
174
|
+
" --job-locations <国家> 任务国家筛选,逗号分隔(仅筛选服务端任务)",
|
|
175
|
+
" --max-videos <数量> 每用户最大视频数,默认 16",
|
|
176
|
+
" --max-users <数量> 最大处理用户数,默认无限制",
|
|
177
|
+
" -i, --interval <秒数> 无任务时轮询间隔,默认 30 秒",
|
|
178
|
+
" --max-age <秒数> 最小刷新间隔,默认 0(不限)",
|
|
179
|
+
" --port <端口号> 固定 CDP 端口,默认 4222",
|
|
180
|
+
" --user-id <编号> 客户端编号(设备ID),默认自动生成",
|
|
181
|
+
" --rotate-min <分钟> 代理节点轮换周期,默认 10(0=关闭定时轮换)",
|
|
182
|
+
" --captcha-wait-min <分钟> 验证码等待人工解决上限,默认 10",
|
|
183
|
+
" --ensure-daemon 启动时检查 clash-daemon 是否存活,未存活则自动拉起(默认不检测)",
|
|
184
|
+
" --preset <预设> 延迟预设 fast/normal/slow/stealth,默认 normal",
|
|
185
|
+
" --proxy <地址> 浏览器代理(默认走 Clash 本地端口,切换节点才有效)",
|
|
186
|
+
"",
|
|
165
187
|
" tt-help precheck [选项]",
|
|
166
188
|
" 对目标商家用户进行轻量预检,通过 HTTP 获取 videoCount 判断是否有新视频",
|
|
167
189
|
" 有更新 → 标记 pending_refresh=1(等待 refresh 客户端完整采集)",
|