tt-help-cli-ycl 1.4.151 → 1.4.152
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/scripts/test-api-interceptor-captcha.mjs +134 -0
- package/scripts/test-sample-all-fail.mjs +142 -0
- package/src/cli/explore.js +22 -2
- package/src/cli/refresh-mini.js +11 -0
- package/src/cli/refresh.js +11 -0
- package/src/lib/api-interceptor.js +15 -0
- package/src/lib/args.js +26 -0
- package/src/lib/constants.js +13 -2
- package/src/scraper/auto-core.js +61 -37
- package/src/scraper/explore-core.js +37 -2
- package/src/videos/core.js +5 -0
package/package.json
CHANGED
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
// 临时验证:被判重试期间弹出验证码 → fetchUserVideosAPI 立即抛 "验证码:",不再空转重试
|
|
2
|
+
// 运行:RM_BLOCKED_WAIT_MS=1 node scripts/test-api-interceptor-captcha.mjs
|
|
3
|
+
import { fetchUserVideosAPI } from "../src/lib/api-interceptor.js";
|
|
4
|
+
|
|
5
|
+
let pass = 0;
|
|
6
|
+
let fail = 0;
|
|
7
|
+
|
|
8
|
+
function check(name, ok, detail = "") {
|
|
9
|
+
if (ok) {
|
|
10
|
+
pass++;
|
|
11
|
+
console.log(`✓ ${name}${detail ? `: ${detail}` : ""}`);
|
|
12
|
+
} else {
|
|
13
|
+
fail++;
|
|
14
|
+
console.error(`✗ ${name}${detail ? `: ${detail}` : ""}`);
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
// 429 + 页面有验证码容器
|
|
19
|
+
function make429CaptchaPage(username) {
|
|
20
|
+
let gotoCount = 0;
|
|
21
|
+
const page = {
|
|
22
|
+
_handlers: {},
|
|
23
|
+
on(event, fn) {
|
|
24
|
+
(page._handlers[event] ||= []).push(fn);
|
|
25
|
+
return page;
|
|
26
|
+
},
|
|
27
|
+
off(event) {
|
|
28
|
+
delete page._handlers[event];
|
|
29
|
+
return page;
|
|
30
|
+
},
|
|
31
|
+
url: () => `https://www.tiktok.com/@${username}`,
|
|
32
|
+
goto: async () => {
|
|
33
|
+
gotoCount++;
|
|
34
|
+
},
|
|
35
|
+
// checkCdpHealth 用 () => document.readyState(fn.length===0)→ "complete"
|
|
36
|
+
// detectCaptcha 用 () => {...}(fn.length===0,通过 captchaVisible 开关控制返回值)
|
|
37
|
+
_captchaVisible: true,
|
|
38
|
+
evaluate: async (fn) => {
|
|
39
|
+
if (
|
|
40
|
+
page._captchaVisible &&
|
|
41
|
+
fn.toString().includes("captcha-verify-container")
|
|
42
|
+
) {
|
|
43
|
+
return {
|
|
44
|
+
exists: true,
|
|
45
|
+
visible: true,
|
|
46
|
+
rect: { x: 0, y: 0, w: 100, h: 100 },
|
|
47
|
+
};
|
|
48
|
+
}
|
|
49
|
+
// detectPageError(fn.length===1)→ null
|
|
50
|
+
if (fn.length === 1) return null;
|
|
51
|
+
return "complete";
|
|
52
|
+
},
|
|
53
|
+
reload: async () => {},
|
|
54
|
+
waitForFunction: async () => null,
|
|
55
|
+
waitForResponse: async (pred) => {
|
|
56
|
+
const resp = {
|
|
57
|
+
url: () => "https://www.tiktok.com/api/post/item_list/?...",
|
|
58
|
+
status: () => 429,
|
|
59
|
+
headers: () => ({ "content-length": "19" }),
|
|
60
|
+
json: async () => {
|
|
61
|
+
throw new Error("Unexpected token 'r'...");
|
|
62
|
+
},
|
|
63
|
+
body: async () => Buffer.from("ratelimit triggered"),
|
|
64
|
+
};
|
|
65
|
+
if (!pred(resp)) throw new Error("response 不匹配 item_list");
|
|
66
|
+
return resp;
|
|
67
|
+
},
|
|
68
|
+
get gotoCount() {
|
|
69
|
+
return gotoCount;
|
|
70
|
+
},
|
|
71
|
+
};
|
|
72
|
+
return page;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
// 用例1:429 + 验证码可见 → 立即抛 "验证码:",只 goto 1 次(无重试)
|
|
76
|
+
{
|
|
77
|
+
const page = make429CaptchaPage("fumolbbf08o");
|
|
78
|
+
try {
|
|
79
|
+
await fetchUserVideosAPI(page, "fumolbbf08o", 5, () => {});
|
|
80
|
+
check("429+验证码 立即转验证码", false, "未抛错");
|
|
81
|
+
} catch (e) {
|
|
82
|
+
check(
|
|
83
|
+
"429+验证码 立即转验证码",
|
|
84
|
+
e.message === "验证码: fumolbbf08o",
|
|
85
|
+
`抛错 "${e.message}"`,
|
|
86
|
+
);
|
|
87
|
+
}
|
|
88
|
+
check(
|
|
89
|
+
"429+验证码 不再重试(仅 1 次 goto)",
|
|
90
|
+
page.gotoCount === 1,
|
|
91
|
+
`goto 次数 = ${page.gotoCount}`,
|
|
92
|
+
);
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
// 用例2:429 + 验证码不可见 → 仍走正常被封重试,最终抛 "被封:"
|
|
96
|
+
{
|
|
97
|
+
const page = make429CaptchaPage("fumolbbf08o");
|
|
98
|
+
page._captchaVisible = false;
|
|
99
|
+
try {
|
|
100
|
+
await fetchUserVideosAPI(page, "fumolbbf08o", 5, () => {});
|
|
101
|
+
check("429无验证码 仍按被封重试", false, "未抛错");
|
|
102
|
+
} catch (e) {
|
|
103
|
+
check(
|
|
104
|
+
"429无验证码 仍按被封重试",
|
|
105
|
+
e.message === "被封: fumolbbf08o",
|
|
106
|
+
`抛错 "${e.message}",goto ${page.gotoCount} 次`,
|
|
107
|
+
);
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
// 用例3:429 + detectCaptcha 本身抛异常(页面上下文销毁)→ 忽略,按被封重试
|
|
112
|
+
{
|
|
113
|
+
const page = make429CaptchaPage("fumolbbf08o");
|
|
114
|
+
page.evaluate = async (fn) => {
|
|
115
|
+
if (fn.toString().includes("captcha-verify-container")) {
|
|
116
|
+
throw new Error("Execution context was destroyed");
|
|
117
|
+
}
|
|
118
|
+
if (fn.length === 1) return null;
|
|
119
|
+
return "complete";
|
|
120
|
+
};
|
|
121
|
+
try {
|
|
122
|
+
await fetchUserVideosAPI(page, "fumolbbf08o", 5, () => {});
|
|
123
|
+
check("检测异常时忽略按被封重试", false, "未抛错");
|
|
124
|
+
} catch (e) {
|
|
125
|
+
check(
|
|
126
|
+
"检测异常时忽略按被封重试",
|
|
127
|
+
e.message === "被封: fumolbbf08o",
|
|
128
|
+
`抛错 "${e.message}"`,
|
|
129
|
+
);
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
console.error(`\n结果: ${pass} 通过, ${fail} 失败`);
|
|
134
|
+
process.exit(fail > 0 ? 1 : 0);
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
// 验证:国家采样全失败(通道故障)→ 任务按 error 回流,不再静默 done
|
|
2
|
+
// 运行:node scripts/test-sample-all-fail.mjs
|
|
3
|
+
let pass = 0;
|
|
4
|
+
let fail = 0;
|
|
5
|
+
function check(name, ok, detail = "") {
|
|
6
|
+
if (ok) {
|
|
7
|
+
pass++;
|
|
8
|
+
console.log(`✓ ${name}${detail ? `: ${detail}` : ""}`);
|
|
9
|
+
} else {
|
|
10
|
+
fail++;
|
|
11
|
+
console.error(`✗ ${name}${detail ? `: ${detail}` : ""}`);
|
|
12
|
+
}
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
// [1] 全失败判定(等价于实现中 Promise.all + sampleFailCount 计数)
|
|
16
|
+
{
|
|
17
|
+
const sampleUrls = ["u1", "u2", "u3", "u4", "u5"];
|
|
18
|
+
// 模拟通道故障:每个 fetch 都抛错 → catch 内计数 +1,返回 null
|
|
19
|
+
let sampleFailCount = 0;
|
|
20
|
+
const locations = await Promise.all(
|
|
21
|
+
sampleUrls.map(async () => {
|
|
22
|
+
try {
|
|
23
|
+
throw new Error("fetch failed");
|
|
24
|
+
} catch {
|
|
25
|
+
sampleFailCount++;
|
|
26
|
+
return null;
|
|
27
|
+
}
|
|
28
|
+
}),
|
|
29
|
+
);
|
|
30
|
+
const allFailed =
|
|
31
|
+
sampleUrls.length > 0 && sampleFailCount === sampleUrls.length;
|
|
32
|
+
check(
|
|
33
|
+
"全失败判定:5/5 fetch 抛错 → allFailed=true",
|
|
34
|
+
allFailed === true && locations.every((l) => l === null),
|
|
35
|
+
);
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
// [2] 部分失败判定:2/5 失败 → allFailed=false(保持众数逻辑)
|
|
39
|
+
{
|
|
40
|
+
const sampleUrls = ["u1", "u2", "u3", "u4", "u5"];
|
|
41
|
+
let sampleFailCount = 0;
|
|
42
|
+
const locations = await Promise.all(
|
|
43
|
+
sampleUrls.map(async (u) => {
|
|
44
|
+
try {
|
|
45
|
+
if (u === "u1" || u === "u2") throw new Error("timeout");
|
|
46
|
+
return "TH";
|
|
47
|
+
} catch {
|
|
48
|
+
sampleFailCount++;
|
|
49
|
+
return null;
|
|
50
|
+
}
|
|
51
|
+
}),
|
|
52
|
+
);
|
|
53
|
+
const allFailed =
|
|
54
|
+
sampleUrls.length > 0 && sampleFailCount === sampleUrls.length;
|
|
55
|
+
check(
|
|
56
|
+
"部分失败判定:2/5 失败 → allFailed=false(走众数)",
|
|
57
|
+
allFailed === false && locations.filter(Boolean).length === 3,
|
|
58
|
+
);
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
// [3] 全成功但无数据判定:sampleFailCount=0 → 不转 error(保持 done 无国家回炉机制)
|
|
62
|
+
{
|
|
63
|
+
const sampleUrls = ["u1", "u2", "u3"];
|
|
64
|
+
let sampleFailCount = 0;
|
|
65
|
+
const locations = await Promise.all(
|
|
66
|
+
sampleUrls.map(async () => {
|
|
67
|
+
const data = { locationCreated: null }; // 成功响应但无字段
|
|
68
|
+
return data?.locationCreated || null;
|
|
69
|
+
}),
|
|
70
|
+
);
|
|
71
|
+
const allFailed =
|
|
72
|
+
sampleUrls.length > 0 && sampleFailCount === sampleUrls.length;
|
|
73
|
+
check(
|
|
74
|
+
"全成功无数据:failCount=0 → 不转 error(done 无国家,2h 回炉兜底)",
|
|
75
|
+
allFailed === false && locations.every((l) => l === null),
|
|
76
|
+
);
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
// [4] 不可达 webserver → fetch 必抛错(通道故障可稳定复现)
|
|
80
|
+
{
|
|
81
|
+
let fetchThrows = false;
|
|
82
|
+
try {
|
|
83
|
+
await fetch("http://127.0.0.1:1/api/tiktok/video", {
|
|
84
|
+
method: "POST",
|
|
85
|
+
signal: AbortSignal.timeout(2000),
|
|
86
|
+
});
|
|
87
|
+
} catch {
|
|
88
|
+
fetchThrows = true;
|
|
89
|
+
}
|
|
90
|
+
check(
|
|
91
|
+
"不可达 webserver → fetch 抛错(通道故障可复现)",
|
|
92
|
+
fetchThrows === true,
|
|
93
|
+
);
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
// [5] 真实 processExplore:page mock 让 collectVideos 走不通(API 拦截失败抛错)
|
|
97
|
+
// 无论命中哪条错误路径,都不应出现"processed=true 且无 error 且无国家"的静默 done
|
|
98
|
+
{
|
|
99
|
+
const fakePage = {
|
|
100
|
+
async evaluate() {
|
|
101
|
+
return {};
|
|
102
|
+
},
|
|
103
|
+
async goto() {},
|
|
104
|
+
on() {
|
|
105
|
+
return this;
|
|
106
|
+
},
|
|
107
|
+
off() {
|
|
108
|
+
return this;
|
|
109
|
+
},
|
|
110
|
+
url: () => "https://www.tiktok.com/@test",
|
|
111
|
+
waitForResponse() {
|
|
112
|
+
return Promise.reject(new Error("timeout"));
|
|
113
|
+
},
|
|
114
|
+
};
|
|
115
|
+
const { processExplore } = await import("../src/scraper/explore-core.js");
|
|
116
|
+
const result = await processExplore(
|
|
117
|
+
fakePage,
|
|
118
|
+
"testuser",
|
|
119
|
+
{
|
|
120
|
+
maxVideos: 1,
|
|
121
|
+
enableFollow: false,
|
|
122
|
+
loggedIn: false,
|
|
123
|
+
mini: false,
|
|
124
|
+
webserverUrl: "http://127.0.0.1:1",
|
|
125
|
+
location: "TH",
|
|
126
|
+
},
|
|
127
|
+
() => {},
|
|
128
|
+
);
|
|
129
|
+
const silentDone =
|
|
130
|
+
result.processed === true &&
|
|
131
|
+
!result.error &&
|
|
132
|
+
!result.noVideo &&
|
|
133
|
+
!result.locationCreated;
|
|
134
|
+
check(
|
|
135
|
+
"通道故障下不出现静默 done",
|
|
136
|
+
silentDone === false,
|
|
137
|
+
`error=${result.error ? `"${result.error.slice(0, 40)}"` : "无"}`,
|
|
138
|
+
);
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
console.error(`\n结果: ${pass} 通过, ${fail} 失败`);
|
|
142
|
+
process.exit(fail > 0 ? 1 : 0);
|
package/src/cli/explore.js
CHANGED
|
@@ -73,6 +73,8 @@ export async function handleExplore(options) {
|
|
|
73
73
|
exploreProxy,
|
|
74
74
|
exploreWebserver,
|
|
75
75
|
exploreNoProduct,
|
|
76
|
+
exploreSwitchDelay,
|
|
77
|
+
exploreCommentDelay,
|
|
76
78
|
} = options;
|
|
77
79
|
|
|
78
80
|
let userId = exploreUserId || configuredUserId;
|
|
@@ -111,6 +113,17 @@ export async function handleExplore(options) {
|
|
|
111
113
|
}
|
|
112
114
|
|
|
113
115
|
setDelayConfig(explorePreset);
|
|
116
|
+
// 自定义延迟:数值参数优先于预设(仅覆盖给出的项)
|
|
117
|
+
if (exploreSwitchDelay != null || exploreCommentDelay != null) {
|
|
118
|
+
const overrides = {};
|
|
119
|
+
if (exploreSwitchDelay != null) overrides.switchMax = exploreSwitchDelay;
|
|
120
|
+
if (exploreCommentDelay != null)
|
|
121
|
+
overrides.commentMax = exploreCommentDelay;
|
|
122
|
+
setDelayConfig(overrides);
|
|
123
|
+
console.error(
|
|
124
|
+
`自定义延迟: 切换 ${overrides.switchMax ?? "预设值"}ms, 评论 ${overrides.commentMax ?? "预设值"}ms`,
|
|
125
|
+
);
|
|
126
|
+
}
|
|
114
127
|
|
|
115
128
|
console.error(`\n国家筛选: ${exploreLocation}`);
|
|
116
129
|
if (exploreJobLocations) console.error(`任务国家: ${exploreJobLocations}`);
|
|
@@ -562,14 +575,21 @@ export async function handleExplore(options) {
|
|
|
562
575
|
// 可能在验证码延迟渲染前已执行,或已识别到( captchaDetected=true )但后续流程又抛了异常出现 error。
|
|
563
576
|
// 这里在把错误上报为 network/other 前对当前页面再检测一次,识别到则转走验证码分支。
|
|
564
577
|
if (result.error) {
|
|
578
|
+
// 先记住原错误类型:兜底转验证码时带上,服务端可同时看到"被判+验证码"
|
|
579
|
+
const wasBlocked = result.error.startsWith("被封:");
|
|
565
580
|
try {
|
|
566
581
|
const lateCaptcha = await detectCaptcha(page);
|
|
567
582
|
if (lateCaptcha && lateCaptcha.visible) {
|
|
568
|
-
console.error(
|
|
583
|
+
console.error(
|
|
584
|
+
` [验证码] 兜底识别到验证码 @${username}${wasBlocked ? "(被判后弹出)" : ""}`,
|
|
585
|
+
);
|
|
569
586
|
result.captchaDetected = true;
|
|
570
587
|
result.captchaStage = result.captchaStage || "process";
|
|
571
588
|
result.captchaMessage =
|
|
572
|
-
result.captchaMessage ||
|
|
589
|
+
result.captchaMessage ||
|
|
590
|
+
(wasBlocked
|
|
591
|
+
? `被判后弹出验证码(原错误: ${result.error})`
|
|
592
|
+
: "上报错误前识别到验证码");
|
|
573
593
|
result.error = null;
|
|
574
594
|
result.errorStack = "";
|
|
575
595
|
result.retryable = false;
|
package/src/cli/refresh-mini.js
CHANGED
|
@@ -191,6 +191,17 @@ export async function handleRefreshMini(options) {
|
|
|
191
191
|
}
|
|
192
192
|
|
|
193
193
|
setDelayConfig(explorePreset);
|
|
194
|
+
// 自定义延迟:数值参数优先于预设(仅覆盖给出的项)
|
|
195
|
+
if (exploreSwitchDelay != null || exploreCommentDelay != null) {
|
|
196
|
+
const overrides = {};
|
|
197
|
+
if (exploreSwitchDelay != null) overrides.switchMax = exploreSwitchDelay;
|
|
198
|
+
if (exploreCommentDelay != null)
|
|
199
|
+
overrides.commentMax = exploreCommentDelay;
|
|
200
|
+
setDelayConfig(overrides);
|
|
201
|
+
console.error(
|
|
202
|
+
`自定义延迟: 切换 ${overrides.switchMax ?? "预设值"}ms, 评论 ${overrides.commentMax ?? "预设值"}ms`,
|
|
203
|
+
);
|
|
204
|
+
}
|
|
194
205
|
|
|
195
206
|
console.error(`\n=== refresh-mini 模式(视频信息更新) ===`);
|
|
196
207
|
console.error(`服务器: ${serverUrl}`);
|
package/src/cli/refresh.js
CHANGED
|
@@ -119,6 +119,17 @@ export async function handleRefresh(options) {
|
|
|
119
119
|
}
|
|
120
120
|
|
|
121
121
|
setDelayConfig(explorePreset);
|
|
122
|
+
// 自定义延迟:数值参数优先于预设(仅覆盖给出的项)
|
|
123
|
+
if (exploreSwitchDelay != null || exploreCommentDelay != null) {
|
|
124
|
+
const overrides = {};
|
|
125
|
+
if (exploreSwitchDelay != null) overrides.switchMax = exploreSwitchDelay;
|
|
126
|
+
if (exploreCommentDelay != null)
|
|
127
|
+
overrides.commentMax = exploreCommentDelay;
|
|
128
|
+
setDelayConfig(overrides);
|
|
129
|
+
console.error(
|
|
130
|
+
`自定义延迟: 切换 ${overrides.switchMax ?? "预设值"}ms, 评论 ${overrides.commentMax ?? "预设值"}ms`,
|
|
131
|
+
);
|
|
132
|
+
}
|
|
122
133
|
|
|
123
134
|
console.error(`\n=== Refresh 模式(基于 explore) ===`);
|
|
124
135
|
console.error(`服务器: ${serverUrl}`);
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { delay } from "./delay.js";
|
|
2
2
|
import { assertPageUrl } from "./browser/page.js";
|
|
3
|
+
import { detectCaptcha } from "../scraper/modules/captcha-handler.js";
|
|
3
4
|
import {
|
|
4
5
|
detectPageError,
|
|
5
6
|
detectPageErrorWithWait,
|
|
@@ -390,6 +391,20 @@ async function fetchUserVideosAPI(
|
|
|
390
391
|
rawBodyBytes === 0
|
|
391
392
|
? "空响应(0 字节)"
|
|
392
393
|
: `限流(HTTP ${httpStatus}, ${bodyText.slice(0, 60)})`;
|
|
394
|
+
// 被封形态(尤其 429/ratelimit)常伴随页面弹出验证码:此时继续重试无意义,
|
|
395
|
+
// 人工解决验证码才是正解。检测到立即抛 "验证码:",跳过剩余 30/40/50s 空转重试
|
|
396
|
+
try {
|
|
397
|
+
const captcha = await detectCaptcha(page);
|
|
398
|
+
if (captcha && captcha.visible) {
|
|
399
|
+
log(
|
|
400
|
+
` [API拦截] item_list ${reason},页面已弹出验证码,转验证码处理(不再重试)`,
|
|
401
|
+
);
|
|
402
|
+
throw new Error(`验证码: ${username}`);
|
|
403
|
+
}
|
|
404
|
+
} catch (e) {
|
|
405
|
+
if (e.message && e.message.startsWith("验证码:")) throw e;
|
|
406
|
+
// 检测本身失败(页面导航中/上下文销毁等):忽略,继续按被封重试
|
|
407
|
+
}
|
|
393
408
|
const retryIndex = attempt - 1;
|
|
394
409
|
if (retryIndex < retryWaits.length) {
|
|
395
410
|
const waitMs = retryWaits[retryIndex];
|
package/src/lib/args.js
CHANGED
|
@@ -185,6 +185,8 @@ function parseExploreArgs(args) {
|
|
|
185
185
|
let exploreProxy = null;
|
|
186
186
|
let exploreWebserver = null;
|
|
187
187
|
let exploreNoProduct = false;
|
|
188
|
+
let exploreSwitchDelay = null;
|
|
189
|
+
let exploreCommentDelay = null;
|
|
188
190
|
|
|
189
191
|
const positional = [];
|
|
190
192
|
const PRESETS = ["fast", "normal", "slow", "stealth"];
|
|
@@ -229,6 +231,12 @@ function parseExploreArgs(args) {
|
|
|
229
231
|
exploreWebserver = args[++i];
|
|
230
232
|
} else if (arg === "--no-product") {
|
|
231
233
|
exploreNoProduct = true;
|
|
234
|
+
} else if (arg === "--preset") {
|
|
235
|
+
explorePreset = args[++i];
|
|
236
|
+
} else if (arg === "--switch-delay") {
|
|
237
|
+
exploreSwitchDelay = parseInt(args[++i], 10) || null;
|
|
238
|
+
} else if (arg === "--comment-delay") {
|
|
239
|
+
exploreCommentDelay = parseInt(args[++i], 10) || null;
|
|
232
240
|
} else {
|
|
233
241
|
positional.push(arg);
|
|
234
242
|
}
|
|
@@ -254,6 +262,8 @@ function parseExploreArgs(args) {
|
|
|
254
262
|
subcommand: "explore",
|
|
255
263
|
exploreUsernames: usernames,
|
|
256
264
|
explorePreset,
|
|
265
|
+
exploreSwitchDelay,
|
|
266
|
+
exploreCommentDelay,
|
|
257
267
|
exploreInterval,
|
|
258
268
|
exploreMaxComments,
|
|
259
269
|
exploreMaxGuess,
|
|
@@ -542,6 +552,8 @@ function parseRefreshArgs(args) {
|
|
|
542
552
|
let exploreWebserver = null;
|
|
543
553
|
let exploreNoProduct = false;
|
|
544
554
|
let exploreMini = false;
|
|
555
|
+
let exploreSwitchDelay = null;
|
|
556
|
+
let exploreCommentDelay = null;
|
|
545
557
|
|
|
546
558
|
for (let i = 0; i < args.length; i++) {
|
|
547
559
|
const arg = args[i];
|
|
@@ -551,6 +563,10 @@ function parseRefreshArgs(args) {
|
|
|
551
563
|
exploreMini = true;
|
|
552
564
|
} else if (arg === "--preset") {
|
|
553
565
|
explorePreset = args[++i];
|
|
566
|
+
} else if (arg === "--switch-delay") {
|
|
567
|
+
exploreSwitchDelay = parseInt(args[++i], 10) || null;
|
|
568
|
+
} else if (arg === "--comment-delay") {
|
|
569
|
+
exploreCommentDelay = parseInt(args[++i], 10) || null;
|
|
554
570
|
} else if (arg === "-i" || arg === "--interval") {
|
|
555
571
|
exploreInterval = parseInt(args[++i], 10) || 30;
|
|
556
572
|
} else if (arg === "--port") {
|
|
@@ -593,6 +609,8 @@ function parseRefreshArgs(args) {
|
|
|
593
609
|
return {
|
|
594
610
|
subcommand: "refresh",
|
|
595
611
|
explorePreset,
|
|
612
|
+
exploreSwitchDelay,
|
|
613
|
+
exploreCommentDelay,
|
|
596
614
|
exploreInterval,
|
|
597
615
|
exploreEnableFollow,
|
|
598
616
|
exploreMaxFollowing,
|
|
@@ -640,6 +658,8 @@ function parseRefreshMiniArgs(args) {
|
|
|
640
658
|
let rotateMin = 10;
|
|
641
659
|
let captchaWaitMin = 10;
|
|
642
660
|
let ensureDaemon = false;
|
|
661
|
+
let exploreSwitchDelay = null;
|
|
662
|
+
let exploreCommentDelay = null;
|
|
643
663
|
|
|
644
664
|
for (let i = 0; i < args.length; i++) {
|
|
645
665
|
const arg = args[i];
|
|
@@ -647,6 +667,10 @@ function parseRefreshMiniArgs(args) {
|
|
|
647
667
|
serverUrl = args[++i];
|
|
648
668
|
} else if (arg === "--preset") {
|
|
649
669
|
explorePreset = args[++i];
|
|
670
|
+
} else if (arg === "--switch-delay") {
|
|
671
|
+
exploreSwitchDelay = parseInt(args[++i], 10) || null;
|
|
672
|
+
} else if (arg === "--comment-delay") {
|
|
673
|
+
exploreCommentDelay = parseInt(args[++i], 10) || null;
|
|
650
674
|
} else if (arg === "-i" || arg === "--interval") {
|
|
651
675
|
exploreInterval = parseInt(args[++i], 10) || 30;
|
|
652
676
|
} else if (arg === "--port") {
|
|
@@ -677,6 +701,8 @@ function parseRefreshMiniArgs(args) {
|
|
|
677
701
|
return {
|
|
678
702
|
subcommand: "refresh-mini",
|
|
679
703
|
explorePreset,
|
|
704
|
+
exploreSwitchDelay,
|
|
705
|
+
exploreCommentDelay,
|
|
680
706
|
exploreInterval,
|
|
681
707
|
exploreLocation,
|
|
682
708
|
exploreJobLocations,
|
package/src/lib/constants.js
CHANGED
|
@@ -124,6 +124,9 @@ const HELP_TEXT = [
|
|
|
124
124
|
" 预设: fast, normal(默认), slow, stealth",
|
|
125
125
|
" 选项:",
|
|
126
126
|
" --server <URL> 服务端地址,默认 http://127.0.0.1:3001",
|
|
127
|
+
" --preset <预设> 延迟预设 fast/normal/slow/stealth,默认 normal",
|
|
128
|
+
" --switch-delay <毫秒> 任务切换延迟上限,覆盖预设值(实际等待随机 delay(switchMax, switchMax*3))",
|
|
129
|
+
" --comment-delay <毫秒> 评论滚动延迟上限,覆盖预设值",
|
|
127
130
|
` --location <国家代码> 国家筛选,逗号分隔,默认 ${DEFAULT_TARGET_LOCATIONS_CSV}`,
|
|
128
131
|
" --job-locations <国家> 任务国家筛选,逗号分隔(仅筛选服务端任务)",
|
|
129
132
|
" --max-comments <数量> 每视频最大评论数,默认 10",
|
|
@@ -139,8 +142,11 @@ const HELP_TEXT = [
|
|
|
139
142
|
" --base-port <端口号> 起始端口,默认 9222",
|
|
140
143
|
" --port-count <数量> 端口数量(账户数),默认 10",
|
|
141
144
|
" --user-id <编号> 客户端编号(设备ID),默认自动生成",
|
|
142
|
-
"--
|
|
143
|
-
"--
|
|
145
|
+
" --preset <预设> 延迟预设 fast/normal/slow/stealth,默认 normal",
|
|
146
|
+
" --switch-delay <毫秒> 任务切换延迟上限,覆盖预设值",
|
|
147
|
+
" --comment-delay <毫秒> 评论滚动延迟上限,覆盖预设值",
|
|
148
|
+
" --proxy <地址> 浏览器代理(如 socks5://127.0.0.1:1080),未指定时使用配置文件 proxy",
|
|
149
|
+
" --webserver <URL> 集中式视频国家查询服务(如 http://127.0.0.1:3000),省去本地 Playwright 浏览器",
|
|
144
150
|
"",
|
|
145
151
|
" tt-help refresh [选项]",
|
|
146
152
|
" 对目标商家用户进行轮回刷新,重新采集视频 + 关注 + 粉丝",
|
|
@@ -162,6 +168,9 @@ const HELP_TEXT = [
|
|
|
162
168
|
" --base-port <端口号> 起始端口,默认 9222",
|
|
163
169
|
" --port-count <数量> 端口数量(账户数),默认 10",
|
|
164
170
|
" --user-id <编号> 客户端编号(设备ID),默认自动生成",
|
|
171
|
+
" --preset <预设> 延迟预设 fast/normal/slow/stealth,默认 normal",
|
|
172
|
+
" --switch-delay <毫秒> 任务切换延迟上限,覆盖预设值",
|
|
173
|
+
" --comment-delay <毫秒> 评论滚动延迟上限,覆盖预设值",
|
|
165
174
|
"--proxy <地址> 浏览器代理(如 socks5://127.0.0.1:1080),未指定时使用配置文件 proxy",
|
|
166
175
|
"--webserver <URL> 集中式视频国家查询服务(如 http://127.0.0.1:3000),省去本地 Playwright 浏览器",
|
|
167
176
|
"",
|
|
@@ -184,6 +193,8 @@ const HELP_TEXT = [
|
|
|
184
193
|
" --captcha-wait-min <分钟> 验证码等待人工解决上限,默认 10",
|
|
185
194
|
" --ensure-daemon 启动时检查 clash-daemon 是否存活,未存活则自动拉起(默认不检测)",
|
|
186
195
|
" --preset <预设> 延迟预设 fast/normal/slow/stealth,默认 normal",
|
|
196
|
+
" --switch-delay <毫秒> 任务切换延迟上限,覆盖预设值",
|
|
197
|
+
" --comment-delay <毫秒> 评论滚动延迟上限,覆盖预设值",
|
|
187
198
|
" --proxy <地址> 浏览器代理(默认走 Clash 本地端口,切换节点才有效)",
|
|
188
199
|
"",
|
|
189
200
|
" tt-help precheck [选项]",
|
package/src/scraper/auto-core.js
CHANGED
|
@@ -9,24 +9,25 @@ import {
|
|
|
9
9
|
detectPageError,
|
|
10
10
|
isLoggedIn,
|
|
11
11
|
assertPageUrl,
|
|
12
|
-
} from
|
|
13
|
-
import { detectCaptcha } from
|
|
12
|
+
} from "./modules/page-helpers.js";
|
|
13
|
+
import { detectCaptcha } from "./modules/captcha-handler.js";
|
|
14
14
|
export { ensureBrowserReady };
|
|
15
|
-
import {
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
} from '../videos/core.js';
|
|
19
|
-
import { runScrape } from './core.js';
|
|
20
|
-
import { extractFollowAndFollowers } from './modules/follow-extractor.js';
|
|
15
|
+
import { getUserInfo, collectVideos } from "../videos/core.js";
|
|
16
|
+
import { runScrape } from "./core.js";
|
|
17
|
+
import { extractFollowAndFollowers } from "./modules/follow-extractor.js";
|
|
21
18
|
|
|
22
19
|
function mergeUserInfo(existing, incoming, source) {
|
|
23
20
|
const merged = { ...existing };
|
|
24
21
|
for (const [key, value] of Object.entries(incoming)) {
|
|
25
|
-
if (key ===
|
|
26
|
-
if (value === undefined || value === null || value ===
|
|
27
|
-
if (typeof value ===
|
|
22
|
+
if (key === "_sources") continue;
|
|
23
|
+
if (value === undefined || value === null || value === "") continue;
|
|
24
|
+
if (typeof value === "number" && typeof merged[key] === "number") {
|
|
28
25
|
merged[key] = Math.max(merged[key], value);
|
|
29
|
-
} else if (
|
|
26
|
+
} else if (
|
|
27
|
+
merged[key] === undefined ||
|
|
28
|
+
merged[key] === null ||
|
|
29
|
+
merged[key] === ""
|
|
30
|
+
) {
|
|
30
31
|
merged[key] = value;
|
|
31
32
|
}
|
|
32
33
|
}
|
|
@@ -43,7 +44,7 @@ async function processUser(page, username, options, log) {
|
|
|
43
44
|
scrapeDepth = 50,
|
|
44
45
|
maxComments = 200,
|
|
45
46
|
maxGuess = 10,
|
|
46
|
-
preset =
|
|
47
|
+
preset = "fast",
|
|
47
48
|
switchMax = null,
|
|
48
49
|
commentMax = null,
|
|
49
50
|
enableFollow = false,
|
|
@@ -65,34 +66,43 @@ async function processUser(page, username, options, log) {
|
|
|
65
66
|
|
|
66
67
|
try {
|
|
67
68
|
log(`\n[processUser] 访问 @${username}...`);
|
|
68
|
-
await retryWithBackoff(
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
69
|
+
await retryWithBackoff(
|
|
70
|
+
async () => {
|
|
71
|
+
await page.goto(`https://www.tiktok.com/@${username}`, {
|
|
72
|
+
waitUntil: "load",
|
|
73
|
+
timeout: 30000,
|
|
74
|
+
});
|
|
75
|
+
assertPageUrl(page, `@${username}`);
|
|
76
|
+
},
|
|
77
|
+
{ log },
|
|
78
|
+
);
|
|
79
|
+
await page
|
|
80
|
+
.waitForSelector('[class*="DivVideoList"]', { timeout: 10000 })
|
|
81
|
+
.catch(() => {});
|
|
75
82
|
await delay(1000, 2000);
|
|
76
83
|
|
|
77
84
|
const info = await getUserInfo(page);
|
|
78
85
|
result.userInfo = info;
|
|
79
86
|
if (!info.uniqueId) info.uniqueId = username;
|
|
80
|
-
log(` 昵称: ${info.nickname ||
|
|
87
|
+
log(` 昵称: ${info.nickname || "-"} | 粉丝: ${info.followerCount || 0}`);
|
|
81
88
|
|
|
82
89
|
if (options.enableFollow) {
|
|
83
90
|
const loggedIn = await isLoggedIn(page);
|
|
84
91
|
if (!loggedIn) {
|
|
85
|
-
log(
|
|
92
|
+
log(" [跳过] 提取关注/粉丝:未登录,请先登录 TikTok");
|
|
86
93
|
result.discoveredFollowing = [];
|
|
87
94
|
result.discoveredFollowers = [];
|
|
88
95
|
} else {
|
|
89
96
|
try {
|
|
90
|
-
log(
|
|
91
|
-
const { following, followers } = await extractFollowAndFollowers(
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
97
|
+
log(" 提取关注/粉丝列表...");
|
|
98
|
+
const { following, followers } = await extractFollowAndFollowers(
|
|
99
|
+
page,
|
|
100
|
+
{
|
|
101
|
+
maxFollowing: options.maxFollowing || 200,
|
|
102
|
+
maxFollowers: options.maxFollowers || 200,
|
|
103
|
+
log,
|
|
104
|
+
},
|
|
105
|
+
);
|
|
96
106
|
result.discoveredFollowing = following;
|
|
97
107
|
result.discoveredFollowers = followers;
|
|
98
108
|
log(` 关注: ${following.length} | 粉丝: ${followers.length}`);
|
|
@@ -108,13 +118,13 @@ async function processUser(page, username, options, log) {
|
|
|
108
118
|
if (captcha && captcha.visible) {
|
|
109
119
|
log(`[验证码] @${username} 页面出现验证码`);
|
|
110
120
|
result.captchaDetected = true;
|
|
111
|
-
result.captchaStage = result.captchaStage ||
|
|
112
|
-
result.captchaMessage = result.captchaMessage ||
|
|
121
|
+
result.captchaStage = result.captchaStage || "video-page";
|
|
122
|
+
result.captchaMessage = result.captchaMessage || "视频页出现验证码";
|
|
113
123
|
}
|
|
114
124
|
|
|
115
125
|
const videos = await collectVideos(page, username, collectMax, log);
|
|
116
126
|
const videoList = Array.from(videos.values()).slice(0, collectMax);
|
|
117
|
-
result.collectedVideos = videoList.map(v => ({
|
|
127
|
+
result.collectedVideos = videoList.map((v) => ({
|
|
118
128
|
videoId: v.id,
|
|
119
129
|
videoUrl: v.href,
|
|
120
130
|
}));
|
|
@@ -126,10 +136,12 @@ async function processUser(page, username, options, log) {
|
|
|
126
136
|
|
|
127
137
|
for (let i = 0; i < videoList.length; i++) {
|
|
128
138
|
const video = videoList[i];
|
|
129
|
-
const videoUrl = video.href.startsWith(
|
|
139
|
+
const videoUrl = video.href.startsWith("http")
|
|
130
140
|
? video.href
|
|
131
141
|
: `https://www.tiktok.com${video.href}`;
|
|
132
|
-
log(
|
|
142
|
+
log(
|
|
143
|
+
` [${i + 1}/${videoList.length}] 开始 scrape: ${videoUrl} (深度 ${scrapeDepth})`,
|
|
144
|
+
);
|
|
133
145
|
|
|
134
146
|
const scrapeResult = await runScrape({
|
|
135
147
|
videoUrl,
|
|
@@ -146,8 +158,9 @@ async function processUser(page, username, options, log) {
|
|
|
146
158
|
|
|
147
159
|
if (scrapeResult.captchaDetected) {
|
|
148
160
|
result.captchaDetected = true;
|
|
149
|
-
result.captchaStage = scrapeResult.captchaStage ||
|
|
150
|
-
result.captchaMessage =
|
|
161
|
+
result.captchaStage = scrapeResult.captchaStage || "scrape";
|
|
162
|
+
result.captchaMessage =
|
|
163
|
+
scrapeResult.captchaMessage || "scrape阶段出现验证码";
|
|
151
164
|
}
|
|
152
165
|
|
|
153
166
|
const scrapeOutput = scrapeResult.output;
|
|
@@ -181,7 +194,9 @@ async function processUser(page, username, options, log) {
|
|
|
181
194
|
result.discoveredCommentAuthors = [...allCommentAuthors];
|
|
182
195
|
result.discoveredGuessAuthors = [...allGuessAuthors];
|
|
183
196
|
|
|
184
|
-
log(
|
|
197
|
+
log(
|
|
198
|
+
` 发现: ${result.discoveredVideoAuthors.length} 个视频作者, ${result.discoveredCommentAuthors.length} 个评论作者, ${result.discoveredGuessAuthors.length} 个猜你喜欢作者`,
|
|
199
|
+
);
|
|
185
200
|
} else {
|
|
186
201
|
const pageError = await detectPageError(page);
|
|
187
202
|
result.restricted = !!pageError;
|
|
@@ -192,8 +207,17 @@ async function processUser(page, username, options, log) {
|
|
|
192
207
|
}
|
|
193
208
|
}
|
|
194
209
|
} catch (e) {
|
|
210
|
+
// 被判重试期间弹出验证码(api-interceptor 抛出 "验证码: username"):
|
|
211
|
+
// 转验证码标记,由外层 captchaDetected 计数处理
|
|
212
|
+
if (e.message.startsWith("验证码:")) {
|
|
213
|
+
result.captchaDetected = true;
|
|
214
|
+
result.captchaStage = result.captchaStage || "video-page";
|
|
215
|
+
result.captchaMessage = result.captchaMessage || "被判重试期间弹出验证码";
|
|
216
|
+
log(` @${username} 被判期间弹出验证码,转验证码处理`);
|
|
217
|
+
return result;
|
|
218
|
+
}
|
|
195
219
|
result.error = e.message;
|
|
196
|
-
result.errorStack = e.stack ||
|
|
220
|
+
result.errorStack = e.stack || "";
|
|
197
221
|
log(` [错误] ${e.message}`);
|
|
198
222
|
}
|
|
199
223
|
|
|
@@ -200,6 +200,7 @@ async function processExplore(page, username, options, log) {
|
|
|
200
200
|
` 国家采样(${sampleUrls.length}个, 已排除图文${videoArray.length - videoArray.filter((v) => !v.isPhoto).length}条): 开始查询`,
|
|
201
201
|
);
|
|
202
202
|
let locations;
|
|
203
|
+
let sampleFailCount = 0; // 查询本身失败(网络/超时/CDN)计数,与"查成功但无数据"区分
|
|
203
204
|
if (webserverUrl) {
|
|
204
205
|
// 走集中式 webserver,不再启动本地 TikTokScraper
|
|
205
206
|
locations = await Promise.all(
|
|
@@ -214,6 +215,7 @@ async function processExplore(page, username, options, log) {
|
|
|
214
215
|
const data = await res.json();
|
|
215
216
|
return data?.locationCreated || null;
|
|
216
217
|
} catch (e) {
|
|
218
|
+
sampleFailCount++;
|
|
217
219
|
log(` [webserver] 视频查询失败: ${e.message}`);
|
|
218
220
|
return null;
|
|
219
221
|
}
|
|
@@ -221,12 +223,32 @@ async function processExplore(page, username, options, log) {
|
|
|
221
223
|
);
|
|
222
224
|
} else {
|
|
223
225
|
locations = await Promise.all(
|
|
224
|
-
sampleUrls.map((url) =>
|
|
226
|
+
sampleUrls.map(async (url) => {
|
|
227
|
+
try {
|
|
228
|
+
return await extractVideoLocation(url);
|
|
229
|
+
} catch (e) {
|
|
230
|
+
sampleFailCount++;
|
|
231
|
+
log(` [采样] 视频查询失败: ${e.message}`);
|
|
232
|
+
return null;
|
|
233
|
+
}
|
|
234
|
+
}),
|
|
225
235
|
);
|
|
226
236
|
}
|
|
227
237
|
log(
|
|
228
|
-
` 国家采样(${locations.length}
|
|
238
|
+
` 国家采样(${locations.length}个${sampleFailCount > 0 ? `, 查询失败${sampleFailCount}` : ""}): [${locations.filter(Boolean).join(", ") || "无数据"}]`,
|
|
229
239
|
);
|
|
240
|
+
|
|
241
|
+
// 全部查询失败 ≠ 全部无数据:
|
|
242
|
+
// 通道故障(webserver 挂掉/超时、本地 Scraper 被封)时若按 done 无国家入库,
|
|
243
|
+
// 会陷入"无国家 → 2h 回炉 → 又全失败 → 又 done"的空转循环;
|
|
244
|
+
// 按 error 上报让任务回流重做,且面板可见
|
|
245
|
+
if (sampleUrls.length > 0 && sampleFailCount === sampleUrls.length) {
|
|
246
|
+
result.error = `国家采样全失败(${webserverUrl ? "webserver" : "本地Scraper"}通道 ${sampleFailCount}/${sampleUrls.length} 全失败)`;
|
|
247
|
+
result.errorStack = "";
|
|
248
|
+
result.processed = false;
|
|
249
|
+
log(` [错误] ${result.error},任务按错误回流`);
|
|
250
|
+
return result;
|
|
251
|
+
}
|
|
230
252
|
// 直接标准化,不去重(保留重复值用于频率统计)
|
|
231
253
|
const normalizedLocations = locations
|
|
232
254
|
.map((loc) => normalizeLocation(loc))
|
|
@@ -353,6 +375,19 @@ async function processExplore(page, username, options, log) {
|
|
|
353
375
|
log(` @${username} CDN限流`);
|
|
354
376
|
}
|
|
355
377
|
|
|
378
|
+
// 被判重试期间弹出验证码(api-interceptor 抛出 "验证码: username"):
|
|
379
|
+
// 清掉 error 转验证码分支,让外层 CLI 走 captchaDetected 上报/等待人工解决流程
|
|
380
|
+
if (e.message.startsWith("验证码:")) {
|
|
381
|
+
result.error = null;
|
|
382
|
+
result.errorStack = "";
|
|
383
|
+
result.processed = false;
|
|
384
|
+
result.captchaDetected = true;
|
|
385
|
+
result.captchaStage = result.captchaStage || "video-page";
|
|
386
|
+
result.captchaMessage = result.captchaMessage || "被判重试期间弹出验证码";
|
|
387
|
+
log(` @${username} 被判期间弹出验证码,转验证码处理`);
|
|
388
|
+
return result;
|
|
389
|
+
}
|
|
390
|
+
|
|
356
391
|
// 临时性错误:超时、执行上下文销毁、网络异常 — 标记为可重试
|
|
357
392
|
const retryablePatterns = [
|
|
358
393
|
"Timeout",
|
package/src/videos/core.js
CHANGED
|
@@ -87,6 +87,11 @@ async function collectVideos(page, username, maxVideos, log) {
|
|
|
87
87
|
if (e.message.startsWith("被封:")) {
|
|
88
88
|
throw e; // 向上抛出,由 explore-core 处理
|
|
89
89
|
}
|
|
90
|
+
// 被判重试期间弹出验证码会抛出 "验证码: username",同样原样上抛
|
|
91
|
+
// (不能被包装成 "API 拦截失败",否则上层识别不到前缀)
|
|
92
|
+
if (e.message.startsWith("验证码:")) {
|
|
93
|
+
throw e; // 向上抛出,由 explore-core 处理
|
|
94
|
+
}
|
|
90
95
|
// CDN 限流会抛出 "CDN限流..." 异常(HTTP 路径 CDNBlockedError / CDP 路径 api-interceptor)
|
|
91
96
|
if (e.message.startsWith("CDN限流")) {
|
|
92
97
|
throw e; // 向上抛出,由 explore-core 处理
|