tt-help-cli-ycl 1.3.45 → 1.3.46
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +33 -33
- package/cli.js +9 -9
- package/package.json +52 -52
- package/scripts/run-explore copy.bat +101 -101
- package/scripts/run-explore.bat +134 -134
- package/scripts/run-explore.ps1 +159 -159
- package/scripts/run-explore.sh +121 -121
- package/src/cli/attach.js +331 -313
- package/src/cli/auto.js +265 -265
- package/src/cli/comments.js +620 -620
- package/src/cli/config.js +170 -170
- package/src/cli/db-import.js +51 -51
- package/src/cli/explore.js +555 -555
- package/src/cli/info.js +10 -16
- package/src/cli/open.js +111 -111
- package/src/cli/progress.js +111 -111
- package/src/cli/refresh.js +288 -288
- package/src/cli/scrape.js +47 -47
- package/src/cli/utils.js +18 -18
- package/src/cli/videos.js +41 -41
- package/src/cli/videostats.js +196 -196
- package/src/cli/watch.js +30 -30
- package/src/cli/webserver.js +19 -0
- package/src/lib/api-interceptor.js +161 -161
- package/src/lib/args.js +809 -778
- package/src/lib/browser/anti-detect.js +23 -23
- package/src/lib/browser/cdp.js +261 -261
- package/src/lib/browser/health-checker.js +114 -114
- package/src/lib/browser/launch.js +43 -43
- package/src/lib/browser/page.js +184 -184
- package/src/lib/constants.js +297 -287
- package/src/lib/delay.js +54 -54
- package/src/lib/explore-fetch.js +118 -118
- package/src/lib/fetcher.js +45 -45
- package/src/lib/filter.js +66 -66
- package/src/lib/io.js +54 -54
- package/src/lib/output.js +80 -80
- package/src/lib/page-error-detector.js +109 -109
- package/src/lib/parse-ssr.mjs +69 -69
- package/src/lib/parser.js +47 -47
- package/src/lib/retry.js +45 -45
- package/src/lib/scrape.js +90 -89
- package/src/lib/target-locations.js +61 -61
- package/src/lib/tiktok-scraper.mjs +160 -106
- package/src/lib/url.js +52 -52
- package/src/main.js +73 -70
- package/src/npm-main.js +70 -69
- package/src/scraper/auto-core.js +203 -203
- package/src/scraper/core.js +255 -255
- package/src/scraper/explore-core.js +208 -208
- package/src/scraper/modules/captcha-handler.js +114 -114
- package/src/scraper/modules/follow-extractor.js +250 -250
- package/src/scraper/modules/guess-extractor.js +51 -51
- package/src/scraper/modules/page-helpers.js +48 -48
- package/src/scraper/refresh-core.js +213 -213
- package/src/videos/core.js +143 -143
- package/src/watch/data-store.js +2980 -2978
- package/src/watch/public/index.html +2355 -2345
- package/src/watch/server.js +727 -727
- package/src/webserver/server.mjs +174 -0
- package/scripts/test-captcha-lib.mjs +0 -68
- package/scripts/test-captcha.mjs +0 -81
- package/scripts/test-incognito-lib.mjs +0 -36
- package/scripts/test-login-state.mjs +0 -128
- package/scripts/test-safe-click.mjs +0 -45
- package/scripts/test-watch-db-smoke.mjs +0 -246
- package/src/results/user-videos-bar.lar.lar.moeta.json +0 -37
package/src/videos/core.js
CHANGED
|
@@ -1,143 +1,143 @@
|
|
|
1
|
-
import {
|
|
2
|
-
delay,
|
|
3
|
-
ensureBrowserReady,
|
|
4
|
-
ensureTikTokPage,
|
|
5
|
-
retryWithBackoff,
|
|
6
|
-
} from "../scraper/modules/page-helpers.js";
|
|
7
|
-
import { fetchUserVideosAPI } from "../lib/api-interceptor.js";
|
|
8
|
-
|
|
9
|
-
async function getUserInfo(page) {
|
|
10
|
-
// 重试包装:处理页面导航导致的执行上下文销毁
|
|
11
|
-
const evaluateWithRetry = async (fn, retries = 3) => {
|
|
12
|
-
for (let i = 0; i < retries; i++) {
|
|
13
|
-
try {
|
|
14
|
-
return await page.evaluate(fn);
|
|
15
|
-
} catch (e) {
|
|
16
|
-
if (
|
|
17
|
-
e.message.includes("Execution context was destroyed") &&
|
|
18
|
-
i < retries - 1
|
|
19
|
-
) {
|
|
20
|
-
await new Promise((r) => setTimeout(r, 500 * (i + 1)));
|
|
21
|
-
} else {
|
|
22
|
-
throw e;
|
|
23
|
-
}
|
|
24
|
-
}
|
|
25
|
-
}
|
|
26
|
-
};
|
|
27
|
-
|
|
28
|
-
return await evaluateWithRetry(() => {
|
|
29
|
-
const html = document.documentElement.outerHTML;
|
|
30
|
-
const result = {};
|
|
31
|
-
|
|
32
|
-
const m = window.location.href.match(/\/@([^/]+)/);
|
|
33
|
-
if (m) result.uniqueId = m[1];
|
|
34
|
-
|
|
35
|
-
const patterns = {
|
|
36
|
-
secUid: /"secUid":"([^"]+)"/,
|
|
37
|
-
nickname: /"nickname":"((?:[^"\\]|\\.)*)"/,
|
|
38
|
-
ttSeller: /"ttSeller":\s*(true|false)/,
|
|
39
|
-
verified: /"verified":\s*(true|false)/,
|
|
40
|
-
followerCount: /"followerCount":(\d+)/,
|
|
41
|
-
videoCount: /"videoCount":(\d+)/,
|
|
42
|
-
followingCount: /"followingCount":(\d+)/,
|
|
43
|
-
heartCount: /"heartCount":(\d+)/,
|
|
44
|
-
signature: /"signature":"((?:[^"\\]|\\.)*)"/,
|
|
45
|
-
locationCreated: /"locationCreated":"([^"]*)/,
|
|
46
|
-
region: /"region":"([^"]*)/,
|
|
47
|
-
};
|
|
48
|
-
|
|
49
|
-
const boolKeys = ["ttSeller", "verified"];
|
|
50
|
-
const numKeys = [
|
|
51
|
-
"followerCount",
|
|
52
|
-
"videoCount",
|
|
53
|
-
"followingCount",
|
|
54
|
-
"heartCount",
|
|
55
|
-
];
|
|
56
|
-
|
|
57
|
-
for (const [key, pat] of Object.entries(patterns)) {
|
|
58
|
-
const match = html.match(pat);
|
|
59
|
-
if (match) {
|
|
60
|
-
if (boolKeys.includes(key)) result[key] = match[1] === "true";
|
|
61
|
-
else if (numKeys.includes(key)) result[key] = parseInt(match[1], 10);
|
|
62
|
-
else if (key === "signature")
|
|
63
|
-
result[key] = match[1].replace(/\\n/g, "\n").replace(/\\\\/g, "\\");
|
|
64
|
-
else result[key] = match[1];
|
|
65
|
-
}
|
|
66
|
-
}
|
|
67
|
-
|
|
68
|
-
return result;
|
|
69
|
-
});
|
|
70
|
-
}
|
|
71
|
-
|
|
72
|
-
async function collectVideos(page, username, maxVideos, log) {
|
|
73
|
-
try {
|
|
74
|
-
const apiResult = await fetchUserVideosAPI(page, username, maxVideos, log);
|
|
75
|
-
if (apiResult && apiResult.size > 0) {
|
|
76
|
-
log(`收集完成: ${apiResult.size} 个视频`);
|
|
77
|
-
return apiResult;
|
|
78
|
-
}
|
|
79
|
-
// apiResult 为空 Map 表示页面有其他异常(login_required, captcha 等)
|
|
80
|
-
return new Map();
|
|
81
|
-
} catch (e) {
|
|
82
|
-
// 被封会抛出 "被封: username" 异常
|
|
83
|
-
if (e.message.startsWith("被封:")) {
|
|
84
|
-
throw e; // 向上抛出,由 explore-core 处理
|
|
85
|
-
}
|
|
86
|
-
// 其他异常也向上抛出
|
|
87
|
-
throw new Error(`API 拦截失败:@${username} ${e.message}`);
|
|
88
|
-
}
|
|
89
|
-
}
|
|
90
|
-
|
|
91
|
-
async function runGetUserVideos(options) {
|
|
92
|
-
const { username, maxVideos = 5, log = console.error } = options;
|
|
93
|
-
const url = `https://www.tiktok.com/@${username}`;
|
|
94
|
-
|
|
95
|
-
log(`用户: @${username}`);
|
|
96
|
-
log(`URL: ${url}`);
|
|
97
|
-
log(`最大视频数: ${maxVideos}\n`);
|
|
98
|
-
|
|
99
|
-
log("连接浏览器...");
|
|
100
|
-
const browser = await ensureBrowserReady();
|
|
101
|
-
|
|
102
|
-
let page;
|
|
103
|
-
try {
|
|
104
|
-
page = await ensureTikTokPage(browser, url);
|
|
105
|
-
} catch (e) {
|
|
106
|
-
await browser.close().catch(() => {});
|
|
107
|
-
throw e;
|
|
108
|
-
}
|
|
109
|
-
|
|
110
|
-
await retryWithBackoff(
|
|
111
|
-
() => page.goto(url, { waitUntil: "load", timeout: 30000 }),
|
|
112
|
-
{ log },
|
|
113
|
-
);
|
|
114
|
-
await delay(3000, 5000);
|
|
115
|
-
await page
|
|
116
|
-
.waitForSelector('[class*="DivVideoList"]', { timeout: 10000 })
|
|
117
|
-
.catch(() => {});
|
|
118
|
-
|
|
119
|
-
log("获取用户信息...");
|
|
120
|
-
const userInfo = await getUserInfo(page);
|
|
121
|
-
log("用户信息: " + JSON.stringify(userInfo, null, 2));
|
|
122
|
-
|
|
123
|
-
log("\n开始滚动收集视频...");
|
|
124
|
-
const videos = await collectVideos(page, username, maxVideos, log);
|
|
125
|
-
const allVideos = Array.from(videos.values());
|
|
126
|
-
|
|
127
|
-
log(`\n总计: ${allVideos.length} 个视频`);
|
|
128
|
-
|
|
129
|
-
const output = {
|
|
130
|
-
user: userInfo,
|
|
131
|
-
totalVideos: Math.min(allVideos.length, maxVideos),
|
|
132
|
-
videos: allVideos.slice(0, maxVideos).map((v) => ({
|
|
133
|
-
id: v.id,
|
|
134
|
-
url: v.href.startsWith("http")
|
|
135
|
-
? v.href
|
|
136
|
-
: `https://www.tiktok.com${v.href}`,
|
|
137
|
-
})),
|
|
138
|
-
};
|
|
139
|
-
|
|
140
|
-
return { output, browser };
|
|
141
|
-
}
|
|
142
|
-
|
|
143
|
-
export { getUserInfo, collectVideos, runGetUserVideos };
|
|
1
|
+
import {
|
|
2
|
+
delay,
|
|
3
|
+
ensureBrowserReady,
|
|
4
|
+
ensureTikTokPage,
|
|
5
|
+
retryWithBackoff,
|
|
6
|
+
} from "../scraper/modules/page-helpers.js";
|
|
7
|
+
import { fetchUserVideosAPI } from "../lib/api-interceptor.js";
|
|
8
|
+
|
|
9
|
+
async function getUserInfo(page) {
|
|
10
|
+
// 重试包装:处理页面导航导致的执行上下文销毁
|
|
11
|
+
const evaluateWithRetry = async (fn, retries = 3) => {
|
|
12
|
+
for (let i = 0; i < retries; i++) {
|
|
13
|
+
try {
|
|
14
|
+
return await page.evaluate(fn);
|
|
15
|
+
} catch (e) {
|
|
16
|
+
if (
|
|
17
|
+
e.message.includes("Execution context was destroyed") &&
|
|
18
|
+
i < retries - 1
|
|
19
|
+
) {
|
|
20
|
+
await new Promise((r) => setTimeout(r, 500 * (i + 1)));
|
|
21
|
+
} else {
|
|
22
|
+
throw e;
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
};
|
|
27
|
+
|
|
28
|
+
return await evaluateWithRetry(() => {
|
|
29
|
+
const html = document.documentElement.outerHTML;
|
|
30
|
+
const result = {};
|
|
31
|
+
|
|
32
|
+
const m = window.location.href.match(/\/@([^/]+)/);
|
|
33
|
+
if (m) result.uniqueId = m[1];
|
|
34
|
+
|
|
35
|
+
const patterns = {
|
|
36
|
+
secUid: /"secUid":"([^"]+)"/,
|
|
37
|
+
nickname: /"nickname":"((?:[^"\\]|\\.)*)"/,
|
|
38
|
+
ttSeller: /"ttSeller":\s*(true|false)/,
|
|
39
|
+
verified: /"verified":\s*(true|false)/,
|
|
40
|
+
followerCount: /"followerCount":(\d+)/,
|
|
41
|
+
videoCount: /"videoCount":(\d+)/,
|
|
42
|
+
followingCount: /"followingCount":(\d+)/,
|
|
43
|
+
heartCount: /"heartCount":(\d+)/,
|
|
44
|
+
signature: /"signature":"((?:[^"\\]|\\.)*)"/,
|
|
45
|
+
locationCreated: /"locationCreated":"([^"]*)/,
|
|
46
|
+
region: /"region":"([^"]*)/,
|
|
47
|
+
};
|
|
48
|
+
|
|
49
|
+
const boolKeys = ["ttSeller", "verified"];
|
|
50
|
+
const numKeys = [
|
|
51
|
+
"followerCount",
|
|
52
|
+
"videoCount",
|
|
53
|
+
"followingCount",
|
|
54
|
+
"heartCount",
|
|
55
|
+
];
|
|
56
|
+
|
|
57
|
+
for (const [key, pat] of Object.entries(patterns)) {
|
|
58
|
+
const match = html.match(pat);
|
|
59
|
+
if (match) {
|
|
60
|
+
if (boolKeys.includes(key)) result[key] = match[1] === "true";
|
|
61
|
+
else if (numKeys.includes(key)) result[key] = parseInt(match[1], 10);
|
|
62
|
+
else if (key === "signature")
|
|
63
|
+
result[key] = match[1].replace(/\\n/g, "\n").replace(/\\\\/g, "\\");
|
|
64
|
+
else result[key] = match[1];
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
return result;
|
|
69
|
+
});
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
async function collectVideos(page, username, maxVideos, log) {
|
|
73
|
+
try {
|
|
74
|
+
const apiResult = await fetchUserVideosAPI(page, username, maxVideos, log);
|
|
75
|
+
if (apiResult && apiResult.size > 0) {
|
|
76
|
+
log(`收集完成: ${apiResult.size} 个视频`);
|
|
77
|
+
return apiResult;
|
|
78
|
+
}
|
|
79
|
+
// apiResult 为空 Map 表示页面有其他异常(login_required, captcha 等)
|
|
80
|
+
return new Map();
|
|
81
|
+
} catch (e) {
|
|
82
|
+
// 被封会抛出 "被封: username" 异常
|
|
83
|
+
if (e.message.startsWith("被封:")) {
|
|
84
|
+
throw e; // 向上抛出,由 explore-core 处理
|
|
85
|
+
}
|
|
86
|
+
// 其他异常也向上抛出
|
|
87
|
+
throw new Error(`API 拦截失败:@${username} ${e.message}`);
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
async function runGetUserVideos(options) {
|
|
92
|
+
const { username, maxVideos = 5, log = console.error } = options;
|
|
93
|
+
const url = `https://www.tiktok.com/@${username}`;
|
|
94
|
+
|
|
95
|
+
log(`用户: @${username}`);
|
|
96
|
+
log(`URL: ${url}`);
|
|
97
|
+
log(`最大视频数: ${maxVideos}\n`);
|
|
98
|
+
|
|
99
|
+
log("连接浏览器...");
|
|
100
|
+
const browser = await ensureBrowserReady();
|
|
101
|
+
|
|
102
|
+
let page;
|
|
103
|
+
try {
|
|
104
|
+
page = await ensureTikTokPage(browser, url);
|
|
105
|
+
} catch (e) {
|
|
106
|
+
await browser.close().catch(() => {});
|
|
107
|
+
throw e;
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
await retryWithBackoff(
|
|
111
|
+
() => page.goto(url, { waitUntil: "load", timeout: 30000 }),
|
|
112
|
+
{ log },
|
|
113
|
+
);
|
|
114
|
+
await delay(3000, 5000);
|
|
115
|
+
await page
|
|
116
|
+
.waitForSelector('[class*="DivVideoList"]', { timeout: 10000 })
|
|
117
|
+
.catch(() => {});
|
|
118
|
+
|
|
119
|
+
log("获取用户信息...");
|
|
120
|
+
const userInfo = await getUserInfo(page);
|
|
121
|
+
log("用户信息: " + JSON.stringify(userInfo, null, 2));
|
|
122
|
+
|
|
123
|
+
log("\n开始滚动收集视频...");
|
|
124
|
+
const videos = await collectVideos(page, username, maxVideos, log);
|
|
125
|
+
const allVideos = Array.from(videos.values());
|
|
126
|
+
|
|
127
|
+
log(`\n总计: ${allVideos.length} 个视频`);
|
|
128
|
+
|
|
129
|
+
const output = {
|
|
130
|
+
user: userInfo,
|
|
131
|
+
totalVideos: Math.min(allVideos.length, maxVideos),
|
|
132
|
+
videos: allVideos.slice(0, maxVideos).map((v) => ({
|
|
133
|
+
id: v.id,
|
|
134
|
+
url: v.href.startsWith("http")
|
|
135
|
+
? v.href
|
|
136
|
+
: `https://www.tiktok.com${v.href}`,
|
|
137
|
+
})),
|
|
138
|
+
};
|
|
139
|
+
|
|
140
|
+
return { output, browser };
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
export { getUserInfo, collectVideos, runGetUserVideos };
|