tt-help-cli-ycl 1.3.21 → 1.3.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -17
- package/cli.js +9 -9
- package/package.json +47 -47
- package/scripts/run-explore copy.bat +68 -68
- package/scripts/run-explore.bat +68 -68
- package/scripts/run-explore.ps1 +81 -81
- package/scripts/run-explore.sh +73 -73
- package/scripts/test-captcha-lib.mjs +68 -0
- package/scripts/test-captcha.mjs +81 -0
- package/scripts/test-incognito-lib.mjs +36 -0
- package/scripts/test-login-state.mjs +128 -0
- package/scripts/test-safe-click.mjs +45 -0
- package/src/cli/attach.js +180 -180
- package/src/cli/auto.js +186 -186
- package/src/cli/config.js +152 -152
- package/src/cli/explore.js +253 -249
- package/src/cli/info.js +88 -88
- package/src/cli/progress.js +111 -111
- package/src/cli/refresh.js +216 -216
- package/src/cli/scrape.js +47 -47
- package/src/cli/utils.js +18 -18
- package/src/cli/videos.js +41 -41
- package/src/cli/watch.js +31 -31
- package/src/lib/api-interceptor.js +129 -129
- package/src/lib/args.js +517 -517
- package/src/lib/browser/anti-detect.js +23 -23
- package/src/lib/browser/cdp.js +194 -194
- package/src/lib/browser/launch.js +43 -43
- package/src/lib/browser/page.js +156 -156
- package/src/lib/constants.js +199 -199
- package/src/lib/delay.js +54 -54
- package/src/lib/explore-fetch.js +118 -118
- package/src/lib/fetcher.js +45 -45
- package/src/lib/filter.js +66 -66
- package/src/lib/io.js +54 -54
- package/src/lib/output.js +80 -80
- package/src/lib/page-error-detector.js +1 -0
- package/src/lib/parse-ssr.mjs +69 -69
- package/src/lib/parser.js +47 -47
- package/src/lib/retry.js +45 -45
- package/src/lib/scrape.js +89 -89
- package/src/lib/tiktok-scraper.mjs +194 -194
- package/src/lib/url.js +52 -52
- package/src/main.js +42 -42
- package/src/results/user-videos-bar.lar.lar.moeta.json +37 -0
- package/src/scraper/auto-core.js +203 -203
- package/src/scraper/core.js +211 -211
- package/src/scraper/explore-core.js +189 -170
- package/src/scraper/modules/captcha-handler.js +114 -114
- package/src/scraper/modules/comment-extractor.js +74 -74
- package/src/scraper/modules/follow-extractor.js +118 -118
- package/src/scraper/modules/guess-extractor.js +51 -51
- package/src/scraper/modules/page-helpers.js +48 -48
- package/src/scraper/refresh-core.js +179 -179
- package/src/videos/core.js +97 -97
- package/src/watch/data-store.js +828 -828
- package/src/watch/public/index.html +753 -753
- package/src/watch/server.js +705 -705
package/src/lib/output.js
CHANGED
|
@@ -1,80 +1,80 @@
|
|
|
1
|
-
export function deduplicate(results) {
|
|
2
|
-
const seen = new Set();
|
|
3
|
-
return results.filter(r => {
|
|
4
|
-
if (r.id) {
|
|
5
|
-
const key = r.id;
|
|
6
|
-
if (seen.has(key)) return false;
|
|
7
|
-
seen.add(key);
|
|
8
|
-
return true;
|
|
9
|
-
}
|
|
10
|
-
const key = r.secUid || r.uniqueId;
|
|
11
|
-
if (seen.has(key)) return false;
|
|
12
|
-
seen.add(key);
|
|
13
|
-
return true;
|
|
14
|
-
});
|
|
15
|
-
}
|
|
16
|
-
|
|
17
|
-
export function formatTable(data) {
|
|
18
|
-
if (data.length === 0) return '';
|
|
19
|
-
|
|
20
|
-
if (data.length === 1) {
|
|
21
|
-
const lines = [];
|
|
22
|
-
for (const [key, val] of Object.entries(data[0])) {
|
|
23
|
-
if (typeof val === 'string' && val.length > 80) {
|
|
24
|
-
lines.push(` ${key}: ${val.substring(0, 80)}...`);
|
|
25
|
-
} else {
|
|
26
|
-
lines.push(` ${key}: ${val}`);
|
|
27
|
-
}
|
|
28
|
-
}
|
|
29
|
-
return lines.join('\n');
|
|
30
|
-
}
|
|
31
|
-
|
|
32
|
-
const cols = [
|
|
33
|
-
{ key: 'uniqueId', label: '用户名', width: 20 },
|
|
34
|
-
{ key: 'locationCreated', label: '地区', width: 6 },
|
|
35
|
-
{ key: 'nickname', label: '昵称', width: 20 },
|
|
36
|
-
{ key: 'ttSeller', label: 'TT卖家', width: 8 },
|
|
37
|
-
{ key: 'verified', label: '已认证', width: 8 },
|
|
38
|
-
{ key: 'followerCount', label: '粉丝', width: 10 },
|
|
39
|
-
{ key: 'videoCount', label: '视频', width: 8 },
|
|
40
|
-
];
|
|
41
|
-
|
|
42
|
-
for (const row of data) {
|
|
43
|
-
for (const col of cols) {
|
|
44
|
-
const val = String(row[col.key] ?? '-');
|
|
45
|
-
col.width = Math.max(col.width, val.length, col.label.length);
|
|
46
|
-
}
|
|
47
|
-
}
|
|
48
|
-
|
|
49
|
-
const sep = (w) => '-'.repeat(w);
|
|
50
|
-
const pad = (s, w) => s.padEnd(w);
|
|
51
|
-
|
|
52
|
-
const header = cols.map(c => pad(c.label, c.width)).join(' │ ');
|
|
53
|
-
const divider = cols.map(c => sep(c.width)).join('-+-');
|
|
54
|
-
const rows = data.map(r =>
|
|
55
|
-
cols.map(c => pad(String(r[c.key] ?? '-'), c.width)).join(' │ ')
|
|
56
|
-
);
|
|
57
|
-
|
|
58
|
-
return [header, divider, ...rows].join('\n');
|
|
59
|
-
}
|
|
60
|
-
|
|
61
|
-
export function formatOutput(data, format) {
|
|
62
|
-
if (format === 'table') return formatTable(data);
|
|
63
|
-
|
|
64
|
-
if (format === 'raw') {
|
|
65
|
-
if (Array.isArray(data) && data.length > 0 && 'url' in data[0]) {
|
|
66
|
-
return data.map(d => d.url).join('\n');
|
|
67
|
-
}
|
|
68
|
-
if (Array.isArray(data) && data.length > 0 && 'uniqueId' in data[0]) {
|
|
69
|
-
return data.map(d => `https://www.tiktok.com/@${d.uniqueId}`).join('\n');
|
|
70
|
-
}
|
|
71
|
-
return JSON.stringify(data, null, 2);
|
|
72
|
-
}
|
|
73
|
-
|
|
74
|
-
// Default JSON output, but for explore results (url-only) output pure text
|
|
75
|
-
if (Array.isArray(data) && data.length > 0 && 'url' in data[0]) {
|
|
76
|
-
return data.map(d => d.url).join('\n');
|
|
77
|
-
}
|
|
78
|
-
|
|
79
|
-
return JSON.stringify(data, null, 2);
|
|
80
|
-
}
|
|
1
|
+
export function deduplicate(results) {
|
|
2
|
+
const seen = new Set();
|
|
3
|
+
return results.filter(r => {
|
|
4
|
+
if (r.id) {
|
|
5
|
+
const key = r.id;
|
|
6
|
+
if (seen.has(key)) return false;
|
|
7
|
+
seen.add(key);
|
|
8
|
+
return true;
|
|
9
|
+
}
|
|
10
|
+
const key = r.secUid || r.uniqueId;
|
|
11
|
+
if (seen.has(key)) return false;
|
|
12
|
+
seen.add(key);
|
|
13
|
+
return true;
|
|
14
|
+
});
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export function formatTable(data) {
|
|
18
|
+
if (data.length === 0) return '';
|
|
19
|
+
|
|
20
|
+
if (data.length === 1) {
|
|
21
|
+
const lines = [];
|
|
22
|
+
for (const [key, val] of Object.entries(data[0])) {
|
|
23
|
+
if (typeof val === 'string' && val.length > 80) {
|
|
24
|
+
lines.push(` ${key}: ${val.substring(0, 80)}...`);
|
|
25
|
+
} else {
|
|
26
|
+
lines.push(` ${key}: ${val}`);
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
return lines.join('\n');
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
const cols = [
|
|
33
|
+
{ key: 'uniqueId', label: '用户名', width: 20 },
|
|
34
|
+
{ key: 'locationCreated', label: '地区', width: 6 },
|
|
35
|
+
{ key: 'nickname', label: '昵称', width: 20 },
|
|
36
|
+
{ key: 'ttSeller', label: 'TT卖家', width: 8 },
|
|
37
|
+
{ key: 'verified', label: '已认证', width: 8 },
|
|
38
|
+
{ key: 'followerCount', label: '粉丝', width: 10 },
|
|
39
|
+
{ key: 'videoCount', label: '视频', width: 8 },
|
|
40
|
+
];
|
|
41
|
+
|
|
42
|
+
for (const row of data) {
|
|
43
|
+
for (const col of cols) {
|
|
44
|
+
const val = String(row[col.key] ?? '-');
|
|
45
|
+
col.width = Math.max(col.width, val.length, col.label.length);
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
const sep = (w) => '-'.repeat(w);
|
|
50
|
+
const pad = (s, w) => s.padEnd(w);
|
|
51
|
+
|
|
52
|
+
const header = cols.map(c => pad(c.label, c.width)).join(' │ ');
|
|
53
|
+
const divider = cols.map(c => sep(c.width)).join('-+-');
|
|
54
|
+
const rows = data.map(r =>
|
|
55
|
+
cols.map(c => pad(String(r[c.key] ?? '-'), c.width)).join(' │ ')
|
|
56
|
+
);
|
|
57
|
+
|
|
58
|
+
return [header, divider, ...rows].join('\n');
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
export function formatOutput(data, format) {
|
|
62
|
+
if (format === 'table') return formatTable(data);
|
|
63
|
+
|
|
64
|
+
if (format === 'raw') {
|
|
65
|
+
if (Array.isArray(data) && data.length > 0 && 'url' in data[0]) {
|
|
66
|
+
return data.map(d => d.url).join('\n');
|
|
67
|
+
}
|
|
68
|
+
if (Array.isArray(data) && data.length > 0 && 'uniqueId' in data[0]) {
|
|
69
|
+
return data.map(d => `https://www.tiktok.com/@${d.uniqueId}`).join('\n');
|
|
70
|
+
}
|
|
71
|
+
return JSON.stringify(data, null, 2);
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
// Default JSON output, but for explore results (url-only) output pure text
|
|
75
|
+
if (Array.isArray(data) && data.length > 0 && 'url' in data[0]) {
|
|
76
|
+
return data.map(d => d.url).join('\n');
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
return JSON.stringify(data, null, 2);
|
|
80
|
+
}
|
package/src/lib/parse-ssr.mjs
CHANGED
|
@@ -1,69 +1,69 @@
|
|
|
1
|
-
function parseSSR(rawHtml) {
|
|
2
|
-
if (!rawHtml.includes('__UNIVERSAL_DATA_FOR_REHYDRATION__')) return null;
|
|
3
|
-
const dataStart = rawHtml.indexOf('__UNIVERSAL_DATA_FOR_REHYDRATION__');
|
|
4
|
-
// 从该字符串后面找 <script 标签的 >,确保找到的是正确行的 >
|
|
5
|
-
const scriptStart = rawHtml.lastIndexOf('<script', dataStart);
|
|
6
|
-
const sIdx = (scriptStart >= 0 ? rawHtml.indexOf('>', scriptStart) : rawHtml.indexOf('>', dataStart)) + 1;
|
|
7
|
-
const eIdx = rawHtml.indexOf('</script>', sIdx);
|
|
8
|
-
if (sIdx < 0 || eIdx < 0) return null;
|
|
9
|
-
const jsonStr = rawHtml.substring(sIdx, eIdx);
|
|
10
|
-
return JSON.parse(jsonStr);
|
|
11
|
-
}
|
|
12
|
-
|
|
13
|
-
export function parseUserInfo(rawHtml) {
|
|
14
|
-
const data = parseSSR(rawHtml);
|
|
15
|
-
if (!data) return null;
|
|
16
|
-
const ud = data.__DEFAULT_SCOPE__['webapp.user-detail'];
|
|
17
|
-
if (!ud || !ud.userInfo) return null;
|
|
18
|
-
const u = ud.userInfo.user;
|
|
19
|
-
const s = ud.userInfo.stats;
|
|
20
|
-
return {
|
|
21
|
-
uniqueId: u.uniqueId,
|
|
22
|
-
nickname: u.nickname,
|
|
23
|
-
id: u.id,
|
|
24
|
-
verified: u.verified,
|
|
25
|
-
privateAccount: u.privateAccount,
|
|
26
|
-
language: u.language,
|
|
27
|
-
bio: u.signature || '',
|
|
28
|
-
avatar: u.avatarLarger || u.avatarMedium || u.avatarThumb || '',
|
|
29
|
-
followerCount: s.followerCount,
|
|
30
|
-
followingCount: s.followingCount,
|
|
31
|
-
heartCount: s.heartCount,
|
|
32
|
-
videoCount: s.videoCount,
|
|
33
|
-
diggCount: s.diggCount,
|
|
34
|
-
friendCount: s.friendCount,
|
|
35
|
-
secUid: u.secUid,
|
|
36
|
-
ttSeller: u.ttSeller || false,
|
|
37
|
-
locationCreated: u.locationCreated || null,
|
|
38
|
-
};
|
|
39
|
-
}
|
|
40
|
-
|
|
41
|
-
export function parseVideoInfo(rawHtml) {
|
|
42
|
-
const data = parseSSR(rawHtml);
|
|
43
|
-
if (!data) return null;
|
|
44
|
-
const vd = data.__DEFAULT_SCOPE__['webapp.video-detail'];
|
|
45
|
-
if (!vd || !vd.itemInfo || !vd.itemInfo.itemStruct) return null;
|
|
46
|
-
const item = vd.itemInfo.itemStruct;
|
|
47
|
-
const author = item.author || {};
|
|
48
|
-
const stats = item.stats || {};
|
|
49
|
-
return {
|
|
50
|
-
id: item.id,
|
|
51
|
-
desc: item.desc || '',
|
|
52
|
-
createTime: item.createTime || null,
|
|
53
|
-
locationCreated: item.locationCreated || null,
|
|
54
|
-
author: {
|
|
55
|
-
uniqueId: author.uniqueId,
|
|
56
|
-
nickname: author.nickname,
|
|
57
|
-
id: author.id,
|
|
58
|
-
verified: author.verified,
|
|
59
|
-
secUid: author.secUid,
|
|
60
|
-
},
|
|
61
|
-
stats: {
|
|
62
|
-
playCount: stats.playCount,
|
|
63
|
-
diggCount: stats.diggCount,
|
|
64
|
-
commentCount: stats.commentCount,
|
|
65
|
-
shareCount: stats.shareCount,
|
|
66
|
-
collectCount: stats.collectCount,
|
|
67
|
-
},
|
|
68
|
-
};
|
|
69
|
-
}
|
|
1
|
+
function parseSSR(rawHtml) {
|
|
2
|
+
if (!rawHtml.includes('__UNIVERSAL_DATA_FOR_REHYDRATION__')) return null;
|
|
3
|
+
const dataStart = rawHtml.indexOf('__UNIVERSAL_DATA_FOR_REHYDRATION__');
|
|
4
|
+
// 从该字符串后面找 <script 标签的 >,确保找到的是正确行的 >
|
|
5
|
+
const scriptStart = rawHtml.lastIndexOf('<script', dataStart);
|
|
6
|
+
const sIdx = (scriptStart >= 0 ? rawHtml.indexOf('>', scriptStart) : rawHtml.indexOf('>', dataStart)) + 1;
|
|
7
|
+
const eIdx = rawHtml.indexOf('</script>', sIdx);
|
|
8
|
+
if (sIdx < 0 || eIdx < 0) return null;
|
|
9
|
+
const jsonStr = rawHtml.substring(sIdx, eIdx);
|
|
10
|
+
return JSON.parse(jsonStr);
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
export function parseUserInfo(rawHtml) {
|
|
14
|
+
const data = parseSSR(rawHtml);
|
|
15
|
+
if (!data) return null;
|
|
16
|
+
const ud = data.__DEFAULT_SCOPE__['webapp.user-detail'];
|
|
17
|
+
if (!ud || !ud.userInfo) return null;
|
|
18
|
+
const u = ud.userInfo.user;
|
|
19
|
+
const s = ud.userInfo.stats;
|
|
20
|
+
return {
|
|
21
|
+
uniqueId: u.uniqueId,
|
|
22
|
+
nickname: u.nickname,
|
|
23
|
+
id: u.id,
|
|
24
|
+
verified: u.verified,
|
|
25
|
+
privateAccount: u.privateAccount,
|
|
26
|
+
language: u.language,
|
|
27
|
+
bio: u.signature || '',
|
|
28
|
+
avatar: u.avatarLarger || u.avatarMedium || u.avatarThumb || '',
|
|
29
|
+
followerCount: s.followerCount,
|
|
30
|
+
followingCount: s.followingCount,
|
|
31
|
+
heartCount: s.heartCount,
|
|
32
|
+
videoCount: s.videoCount,
|
|
33
|
+
diggCount: s.diggCount,
|
|
34
|
+
friendCount: s.friendCount,
|
|
35
|
+
secUid: u.secUid,
|
|
36
|
+
ttSeller: u.ttSeller || false,
|
|
37
|
+
locationCreated: u.locationCreated || null,
|
|
38
|
+
};
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export function parseVideoInfo(rawHtml) {
|
|
42
|
+
const data = parseSSR(rawHtml);
|
|
43
|
+
if (!data) return null;
|
|
44
|
+
const vd = data.__DEFAULT_SCOPE__['webapp.video-detail'];
|
|
45
|
+
if (!vd || !vd.itemInfo || !vd.itemInfo.itemStruct) return null;
|
|
46
|
+
const item = vd.itemInfo.itemStruct;
|
|
47
|
+
const author = item.author || {};
|
|
48
|
+
const stats = item.stats || {};
|
|
49
|
+
return {
|
|
50
|
+
id: item.id,
|
|
51
|
+
desc: item.desc || '',
|
|
52
|
+
createTime: item.createTime || null,
|
|
53
|
+
locationCreated: item.locationCreated || null,
|
|
54
|
+
author: {
|
|
55
|
+
uniqueId: author.uniqueId,
|
|
56
|
+
nickname: author.nickname,
|
|
57
|
+
id: author.id,
|
|
58
|
+
verified: author.verified,
|
|
59
|
+
secUid: author.secUid,
|
|
60
|
+
},
|
|
61
|
+
stats: {
|
|
62
|
+
playCount: stats.playCount,
|
|
63
|
+
diggCount: stats.diggCount,
|
|
64
|
+
commentCount: stats.commentCount,
|
|
65
|
+
shareCount: stats.shareCount,
|
|
66
|
+
collectCount: stats.collectCount,
|
|
67
|
+
},
|
|
68
|
+
};
|
|
69
|
+
}
|
package/src/lib/parser.js
CHANGED
|
@@ -1,47 +1,47 @@
|
|
|
1
|
-
export const USER_SECTION_SIZE = 12000;
|
|
2
|
-
|
|
3
|
-
export function extractUserSection(html) {
|
|
4
|
-
const idx = html.indexOf('"uniqueId"');
|
|
5
|
-
if (idx < 0) return null;
|
|
6
|
-
return html.substring(idx, idx + USER_SECTION_SIZE);
|
|
7
|
-
}
|
|
8
|
-
|
|
9
|
-
export function parseUserSection(section) {
|
|
10
|
-
const data = {};
|
|
11
|
-
|
|
12
|
-
for (const key of ['uniqueId', 'uid', 'secUid']) {
|
|
13
|
-
const m = section.match(new RegExp(`"${key}":"([^"]*)`));
|
|
14
|
-
if (m) data[key] = m[1];
|
|
15
|
-
}
|
|
16
|
-
|
|
17
|
-
for (const key of ['nickname', 'signature']) {
|
|
18
|
-
const m = section.match(new RegExp(`"${key}":"((?:[^"\\\\]|\\\\.)*)"`, 'g'));
|
|
19
|
-
if (m) {
|
|
20
|
-
const raw = m[0].replace(`"${key}":"`, '').replace(/"$/, '');
|
|
21
|
-
data[key] = raw.replace(/\\n/g, '\n').replace(/\\\\/g, '\\');
|
|
22
|
-
}
|
|
23
|
-
}
|
|
24
|
-
|
|
25
|
-
for (const key of ['ttSeller', 'verified']) {
|
|
26
|
-
const m = section.match(new RegExp(`"${key}":\\s*(true|false)`));
|
|
27
|
-
data[key] = m ? m[1] === 'true' : undefined;
|
|
28
|
-
}
|
|
29
|
-
|
|
30
|
-
for (const key of ['followerCount', 'followingCount', 'heartCount', 'videoCount', 'diggCount']) {
|
|
31
|
-
const m = section.match(new RegExp(`"${key}":(\\d+)`));
|
|
32
|
-
if (m) data[key] = parseInt(m[1], 10);
|
|
33
|
-
}
|
|
34
|
-
|
|
35
|
-
const mt = section.match(/"createTime":(\d+)/);
|
|
36
|
-
if (mt) data.createTime = parseInt(mt[1], 10);
|
|
37
|
-
|
|
38
|
-
const ma = section.match(/"avatarLarger":"([^"]*)/);
|
|
39
|
-
if (ma) data.avatarLarger = ma[1].replace(/\\u002F/g, '/');
|
|
40
|
-
|
|
41
|
-
return data;
|
|
42
|
-
}
|
|
43
|
-
|
|
44
|
-
export function extractLocationCreated(html) {
|
|
45
|
-
const m = html.match(/"locationCreated":"([^"]*)/);
|
|
46
|
-
return m ? m[1] : null;
|
|
47
|
-
}
|
|
1
|
+
export const USER_SECTION_SIZE = 12000;
|
|
2
|
+
|
|
3
|
+
export function extractUserSection(html) {
|
|
4
|
+
const idx = html.indexOf('"uniqueId"');
|
|
5
|
+
if (idx < 0) return null;
|
|
6
|
+
return html.substring(idx, idx + USER_SECTION_SIZE);
|
|
7
|
+
}
|
|
8
|
+
|
|
9
|
+
export function parseUserSection(section) {
|
|
10
|
+
const data = {};
|
|
11
|
+
|
|
12
|
+
for (const key of ['uniqueId', 'uid', 'secUid']) {
|
|
13
|
+
const m = section.match(new RegExp(`"${key}":"([^"]*)`));
|
|
14
|
+
if (m) data[key] = m[1];
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
for (const key of ['nickname', 'signature']) {
|
|
18
|
+
const m = section.match(new RegExp(`"${key}":"((?:[^"\\\\]|\\\\.)*)"`, 'g'));
|
|
19
|
+
if (m) {
|
|
20
|
+
const raw = m[0].replace(`"${key}":"`, '').replace(/"$/, '');
|
|
21
|
+
data[key] = raw.replace(/\\n/g, '\n').replace(/\\\\/g, '\\');
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
for (const key of ['ttSeller', 'verified']) {
|
|
26
|
+
const m = section.match(new RegExp(`"${key}":\\s*(true|false)`));
|
|
27
|
+
data[key] = m ? m[1] === 'true' : undefined;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
for (const key of ['followerCount', 'followingCount', 'heartCount', 'videoCount', 'diggCount']) {
|
|
31
|
+
const m = section.match(new RegExp(`"${key}":(\\d+)`));
|
|
32
|
+
if (m) data[key] = parseInt(m[1], 10);
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
const mt = section.match(/"createTime":(\d+)/);
|
|
36
|
+
if (mt) data.createTime = parseInt(mt[1], 10);
|
|
37
|
+
|
|
38
|
+
const ma = section.match(/"avatarLarger":"([^"]*)/);
|
|
39
|
+
if (ma) data.avatarLarger = ma[1].replace(/\\u002F/g, '/');
|
|
40
|
+
|
|
41
|
+
return data;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
export function extractLocationCreated(html) {
|
|
45
|
+
const m = html.match(/"locationCreated":"([^"]*)/);
|
|
46
|
+
return m ? m[1] : null;
|
|
47
|
+
}
|
package/src/lib/retry.js
CHANGED
|
@@ -1,45 +1,45 @@
|
|
|
1
|
-
import { delay } from './delay.js';
|
|
2
|
-
|
|
3
|
-
const RETRYABLE_PATTERNS = [
|
|
4
|
-
'interrupted',
|
|
5
|
-
'Navigation.*interrupted',
|
|
6
|
-
'net::',
|
|
7
|
-
'ECONN',
|
|
8
|
-
'ETIMEDOUT',
|
|
9
|
-
'ENOTFOUND',
|
|
10
|
-
'EAI_AGAIN',
|
|
11
|
-
'ESOCKETRESET',
|
|
12
|
-
'connection.*refused',
|
|
13
|
-
'connection.*reset',
|
|
14
|
-
'failed.*navigate',
|
|
15
|
-
'target.*closed',
|
|
16
|
-
'crash',
|
|
17
|
-
'代理错误',
|
|
18
|
-
];
|
|
19
|
-
|
|
20
|
-
export function isRetryableError(error) {
|
|
21
|
-
if (!error) return false;
|
|
22
|
-
const msg = error.message || error.toString() || '';
|
|
23
|
-
return RETRYABLE_PATTERNS.some(p => new RegExp(p, 'i').test(msg));
|
|
24
|
-
}
|
|
25
|
-
|
|
26
|
-
export async function retryWithBackoff(fn, { maxRetries = 3, baseDelay = 3000, log } = {}) {
|
|
27
|
-
let lastError;
|
|
28
|
-
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
|
29
|
-
try {
|
|
30
|
-
return await fn();
|
|
31
|
-
} catch (error) {
|
|
32
|
-
lastError = error;
|
|
33
|
-
if (attempt >= maxRetries || !isRetryableError(error)) {
|
|
34
|
-
throw error;
|
|
35
|
-
}
|
|
36
|
-
const jitter = Math.random() * 2000;
|
|
37
|
-
const waitTime = baseDelay * Math.pow(2, attempt) + jitter;
|
|
38
|
-
if (log) {
|
|
39
|
-
log(` [重试] ${attempt + 1}/${maxRetries},${Math.round(waitTime / 1000)}s 后重试...`);
|
|
40
|
-
}
|
|
41
|
-
await delay(Math.round(waitTime), Math.round(waitTime));
|
|
42
|
-
}
|
|
43
|
-
}
|
|
44
|
-
throw lastError;
|
|
45
|
-
}
|
|
1
|
+
import { delay } from './delay.js';
|
|
2
|
+
|
|
3
|
+
const RETRYABLE_PATTERNS = [
|
|
4
|
+
'interrupted',
|
|
5
|
+
'Navigation.*interrupted',
|
|
6
|
+
'net::',
|
|
7
|
+
'ECONN',
|
|
8
|
+
'ETIMEDOUT',
|
|
9
|
+
'ENOTFOUND',
|
|
10
|
+
'EAI_AGAIN',
|
|
11
|
+
'ESOCKETRESET',
|
|
12
|
+
'connection.*refused',
|
|
13
|
+
'connection.*reset',
|
|
14
|
+
'failed.*navigate',
|
|
15
|
+
'target.*closed',
|
|
16
|
+
'crash',
|
|
17
|
+
'代理错误',
|
|
18
|
+
];
|
|
19
|
+
|
|
20
|
+
export function isRetryableError(error) {
|
|
21
|
+
if (!error) return false;
|
|
22
|
+
const msg = error.message || error.toString() || '';
|
|
23
|
+
return RETRYABLE_PATTERNS.some(p => new RegExp(p, 'i').test(msg));
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
export async function retryWithBackoff(fn, { maxRetries = 3, baseDelay = 3000, log } = {}) {
|
|
27
|
+
let lastError;
|
|
28
|
+
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
|
29
|
+
try {
|
|
30
|
+
return await fn();
|
|
31
|
+
} catch (error) {
|
|
32
|
+
lastError = error;
|
|
33
|
+
if (attempt >= maxRetries || !isRetryableError(error)) {
|
|
34
|
+
throw error;
|
|
35
|
+
}
|
|
36
|
+
const jitter = Math.random() * 2000;
|
|
37
|
+
const waitTime = baseDelay * Math.pow(2, attempt) + jitter;
|
|
38
|
+
if (log) {
|
|
39
|
+
log(` [重试] ${attempt + 1}/${maxRetries},${Math.round(waitTime / 1000)}s 后重试...`);
|
|
40
|
+
}
|
|
41
|
+
await delay(Math.round(waitTime), Math.round(waitTime));
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
throw lastError;
|
|
45
|
+
}
|
package/src/lib/scrape.js
CHANGED
|
@@ -1,89 +1,89 @@
|
|
|
1
|
-
import { TikTokScraper } from './tiktok-scraper.mjs';
|
|
2
|
-
import { isProfileUrl, isVideoUrl, extractUniqueId, normalizeUsername } from './url.js';
|
|
3
|
-
|
|
4
|
-
// Lazy singleton for TikTokScraper
|
|
5
|
-
let scraperInstance = null;
|
|
6
|
-
let scraperInitPromise = null;
|
|
7
|
-
|
|
8
|
-
async function getScraper() {
|
|
9
|
-
if (scraperInstance) return scraperInstance;
|
|
10
|
-
if (scraperInitPromise) return scraperInitPromise;
|
|
11
|
-
scraperInitPromise = (async () => {
|
|
12
|
-
const scraper = new TikTokScraper();
|
|
13
|
-
await scraper.init();
|
|
14
|
-
scraperInstance = scraper;
|
|
15
|
-
scraperInitPromise = null;
|
|
16
|
-
return scraper;
|
|
17
|
-
})();
|
|
18
|
-
return scraperInitPromise;
|
|
19
|
-
}
|
|
20
|
-
|
|
21
|
-
export async function closeScraper() {
|
|
22
|
-
if (scraperInstance) {
|
|
23
|
-
await scraperInstance.close();
|
|
24
|
-
scraperInstance = null;
|
|
25
|
-
}
|
|
26
|
-
}
|
|
27
|
-
|
|
28
|
-
// Map parseUserInfo output to legacy parser.js format
|
|
29
|
-
function mapUserInfo(user) {
|
|
30
|
-
if (!user) return null;
|
|
31
|
-
return {
|
|
32
|
-
uniqueId: user.uniqueId,
|
|
33
|
-
uid: user.id,
|
|
34
|
-
secUid: user.secUid,
|
|
35
|
-
nickname: user.nickname,
|
|
36
|
-
signature: user.bio,
|
|
37
|
-
ttSeller: user.ttSeller,
|
|
38
|
-
verified: user.verified,
|
|
39
|
-
followerCount: user.followerCount,
|
|
40
|
-
followingCount: user.followingCount,
|
|
41
|
-
heartCount: user.heartCount,
|
|
42
|
-
videoCount: user.videoCount,
|
|
43
|
-
diggCount: user.diggCount,
|
|
44
|
-
avatarLarger: user.avatar,
|
|
45
|
-
locationCreated: user.locationCreated,
|
|
46
|
-
};
|
|
47
|
-
}
|
|
48
|
-
|
|
49
|
-
// Map parseVideoInfo output to legacy format
|
|
50
|
-
function mapVideoLocation(video) {
|
|
51
|
-
if (!video) return null;
|
|
52
|
-
return video.locationCreated;
|
|
53
|
-
}
|
|
54
|
-
|
|
55
|
-
export async function extractUserData(url) {
|
|
56
|
-
const scraper = await getScraper();
|
|
57
|
-
const uniqueId = extractUniqueId(url);
|
|
58
|
-
if (!uniqueId) throw new Error(`无法从URL提取用户名: ${url}`);
|
|
59
|
-
const user = await scraper.getUserInfo(normalizeUsername(uniqueId));
|
|
60
|
-
if (!user) throw new Error('无法解析用户信息');
|
|
61
|
-
return mapUserInfo(user);
|
|
62
|
-
}
|
|
63
|
-
|
|
64
|
-
export async function extractVideoLocation(videoUrl) {
|
|
65
|
-
const scraper = await getScraper();
|
|
66
|
-
const video = await scraper.getVideoInfo(videoUrl);
|
|
67
|
-
return mapVideoLocation(video);
|
|
68
|
-
}
|
|
69
|
-
|
|
70
|
-
export async function processUrl(url) {
|
|
71
|
-
if (isProfileUrl(url)) {
|
|
72
|
-
const profileData = await extractUserData(url);
|
|
73
|
-
return [profileData];
|
|
74
|
-
}
|
|
75
|
-
|
|
76
|
-
if (isVideoUrl(url)) {
|
|
77
|
-
const profileHandle = extractUniqueId(url);
|
|
78
|
-
if (!profileHandle) throw new Error(`无法从视频URL提取用户主页: ${url}`);
|
|
79
|
-
|
|
80
|
-
const [profileData, locationCreated] = await Promise.all([
|
|
81
|
-
extractUserData(url),
|
|
82
|
-
extractVideoLocation(url),
|
|
83
|
-
]);
|
|
84
|
-
|
|
85
|
-
return [{ ...profileData, locationCreated }];
|
|
86
|
-
}
|
|
87
|
-
|
|
88
|
-
return [];
|
|
89
|
-
}
|
|
1
|
+
import { TikTokScraper } from './tiktok-scraper.mjs';
|
|
2
|
+
import { isProfileUrl, isVideoUrl, extractUniqueId, normalizeUsername } from './url.js';
|
|
3
|
+
|
|
4
|
+
// Lazy singleton for TikTokScraper
|
|
5
|
+
let scraperInstance = null;
|
|
6
|
+
let scraperInitPromise = null;
|
|
7
|
+
|
|
8
|
+
async function getScraper() {
|
|
9
|
+
if (scraperInstance) return scraperInstance;
|
|
10
|
+
if (scraperInitPromise) return scraperInitPromise;
|
|
11
|
+
scraperInitPromise = (async () => {
|
|
12
|
+
const scraper = new TikTokScraper();
|
|
13
|
+
await scraper.init();
|
|
14
|
+
scraperInstance = scraper;
|
|
15
|
+
scraperInitPromise = null;
|
|
16
|
+
return scraper;
|
|
17
|
+
})();
|
|
18
|
+
return scraperInitPromise;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export async function closeScraper() {
|
|
22
|
+
if (scraperInstance) {
|
|
23
|
+
await scraperInstance.close();
|
|
24
|
+
scraperInstance = null;
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
// Map parseUserInfo output to legacy parser.js format
|
|
29
|
+
function mapUserInfo(user) {
|
|
30
|
+
if (!user) return null;
|
|
31
|
+
return {
|
|
32
|
+
uniqueId: user.uniqueId,
|
|
33
|
+
uid: user.id,
|
|
34
|
+
secUid: user.secUid,
|
|
35
|
+
nickname: user.nickname,
|
|
36
|
+
signature: user.bio,
|
|
37
|
+
ttSeller: user.ttSeller,
|
|
38
|
+
verified: user.verified,
|
|
39
|
+
followerCount: user.followerCount,
|
|
40
|
+
followingCount: user.followingCount,
|
|
41
|
+
heartCount: user.heartCount,
|
|
42
|
+
videoCount: user.videoCount,
|
|
43
|
+
diggCount: user.diggCount,
|
|
44
|
+
avatarLarger: user.avatar,
|
|
45
|
+
locationCreated: user.locationCreated,
|
|
46
|
+
};
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
// Map parseVideoInfo output to legacy format
|
|
50
|
+
function mapVideoLocation(video) {
|
|
51
|
+
if (!video) return null;
|
|
52
|
+
return video.locationCreated;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
export async function extractUserData(url) {
|
|
56
|
+
const scraper = await getScraper();
|
|
57
|
+
const uniqueId = extractUniqueId(url);
|
|
58
|
+
if (!uniqueId) throw new Error(`无法从URL提取用户名: ${url}`);
|
|
59
|
+
const user = await scraper.getUserInfo(normalizeUsername(uniqueId));
|
|
60
|
+
if (!user) throw new Error('无法解析用户信息');
|
|
61
|
+
return mapUserInfo(user);
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
export async function extractVideoLocation(videoUrl) {
|
|
65
|
+
const scraper = await getScraper();
|
|
66
|
+
const video = await scraper.getVideoInfo(videoUrl);
|
|
67
|
+
return mapVideoLocation(video);
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
export async function processUrl(url) {
|
|
71
|
+
if (isProfileUrl(url)) {
|
|
72
|
+
const profileData = await extractUserData(url);
|
|
73
|
+
return [profileData];
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
if (isVideoUrl(url)) {
|
|
77
|
+
const profileHandle = extractUniqueId(url);
|
|
78
|
+
if (!profileHandle) throw new Error(`无法从视频URL提取用户主页: ${url}`);
|
|
79
|
+
|
|
80
|
+
const [profileData, locationCreated] = await Promise.all([
|
|
81
|
+
extractUserData(url),
|
|
82
|
+
extractVideoLocation(url),
|
|
83
|
+
]);
|
|
84
|
+
|
|
85
|
+
return [{ ...profileData, locationCreated }];
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
return [];
|
|
89
|
+
}
|