tt-help-cli-ycl 1.3.34 → 1.3.35
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -17
- package/cli.js +9 -9
- package/package.json +47 -47
- package/scripts/run-explore copy.bat +101 -101
- package/scripts/run-explore.bat +132 -132
- package/scripts/run-explore.ps1 +157 -157
- package/scripts/run-explore.sh +119 -119
- package/scripts/test-captcha-lib.mjs +68 -0
- package/scripts/test-captcha.mjs +81 -0
- package/scripts/test-incognito-lib.mjs +36 -0
- package/scripts/test-login-state.mjs +128 -0
- package/scripts/test-safe-click.mjs +45 -0
- package/src/cli/attach.js +180 -180
- package/src/cli/auto.js +240 -240
- package/src/cli/config.js +152 -152
- package/src/cli/explore.js +488 -488
- package/src/cli/info.js +88 -88
- package/src/cli/open.js +111 -111
- package/src/cli/progress.js +111 -111
- package/src/cli/refresh.js +216 -216
- package/src/cli/scrape.js +47 -47
- package/src/cli/utils.js +18 -18
- package/src/cli/videos.js +41 -41
- package/src/cli/watch.js +31 -31
- package/src/lib/args.js +722 -722
- package/src/lib/browser/anti-detect.js +23 -23
- package/src/lib/browser/cdp.js +261 -261
- package/src/lib/browser/health-checker.js +114 -114
- package/src/lib/browser/launch.js +43 -43
- package/src/lib/browser/page.js +183 -183
- package/src/lib/constants.js +216 -216
- package/src/lib/delay.js +54 -54
- package/src/lib/explore-fetch.js +118 -118
- package/src/lib/fetcher.js +45 -45
- package/src/lib/filter.js +66 -66
- package/src/lib/io.js +54 -54
- package/src/lib/output.js +80 -80
- package/src/lib/page-error-detector.js +105 -105
- package/src/lib/parse-ssr.mjs +69 -69
- package/src/lib/parser.js +47 -47
- package/src/lib/retry.js +45 -45
- package/src/lib/scrape.js +89 -89
- package/src/lib/tiktok-scraper.mjs +194 -194
- package/src/lib/url.js +52 -52
- package/src/main.js +48 -48
- package/src/results/user-videos-bar.lar.lar.moeta.json +37 -0
- package/src/scraper/auto-core.js +203 -203
- package/src/scraper/core.js +211 -211
- package/src/scraper/explore-core.js +177 -167
- package/src/scraper/modules/captcha-handler.js +114 -114
- package/src/scraper/modules/follow-extractor.js +194 -194
- package/src/scraper/modules/guess-extractor.js +51 -51
- package/src/scraper/modules/page-helpers.js +48 -48
- package/src/scraper/refresh-core.js +179 -179
- package/src/videos/core.js +125 -125
- package/src/watch/data-store.js +1040 -1030
- package/src/watch/public/index.html +1458 -753
- package/src/watch/server.js +939 -933
package/src/lib/url.js
CHANGED
|
@@ -1,52 +1,52 @@
|
|
|
1
|
-
const BASE_URL = 'https://www.tiktok.com';
|
|
2
|
-
|
|
3
|
-
export function extractUniqueId(url) {
|
|
4
|
-
const m = url.match(/\/@([\w.-]+)/);
|
|
5
|
-
return m ? m[1] : null;
|
|
6
|
-
}
|
|
7
|
-
|
|
8
|
-
export function extractVideoId(url) {
|
|
9
|
-
const m = url.match(/\/video\/(\d+)/);
|
|
10
|
-
return m ? m[1] : null;
|
|
11
|
-
}
|
|
12
|
-
|
|
13
|
-
export function normalizeUsername(input) {
|
|
14
|
-
return (input || '').replace(/^@/, '');
|
|
15
|
-
}
|
|
16
|
-
|
|
17
|
-
export function toProfileUrl(handle) {
|
|
18
|
-
const clean = normalizeUsername(handle);
|
|
19
|
-
return `${BASE_URL}/@${clean}`;
|
|
20
|
-
}
|
|
21
|
-
|
|
22
|
-
export function toVideoUrl(handle, videoId) {
|
|
23
|
-
const clean = normalizeUsername(handle);
|
|
24
|
-
return `${BASE_URL}/@${clean}/video/${videoId}`;
|
|
25
|
-
}
|
|
26
|
-
|
|
27
|
-
export function ensureAbsoluteUrl(href) {
|
|
28
|
-
if (href.startsWith('http')) return href;
|
|
29
|
-
return `${BASE_URL}${href}`;
|
|
30
|
-
}
|
|
31
|
-
|
|
32
|
-
export function isProfileUrl(url) {
|
|
33
|
-
return /\/@[\w.-]+(?:$|[?#])/.test(url);
|
|
34
|
-
}
|
|
35
|
-
|
|
36
|
-
export function isVideoUrl(url) {
|
|
37
|
-
return /\/video\/\d+/.test(url);
|
|
38
|
-
}
|
|
39
|
-
|
|
40
|
-
export function extractDisplayPath(url) {
|
|
41
|
-
try {
|
|
42
|
-
const parts = new URL(url).pathname.split('/').filter(Boolean);
|
|
43
|
-
return parts.slice(-2).join('/');
|
|
44
|
-
} catch {
|
|
45
|
-
return url;
|
|
46
|
-
}
|
|
47
|
-
}
|
|
48
|
-
|
|
49
|
-
export function extractAuthorFromVideoUrl(url) {
|
|
50
|
-
const m = url.match(/@([^/]+)\/video/);
|
|
51
|
-
return m ? '@' + m[1] : null;
|
|
52
|
-
}
|
|
1
|
+
const BASE_URL = 'https://www.tiktok.com';
|
|
2
|
+
|
|
3
|
+
export function extractUniqueId(url) {
|
|
4
|
+
const m = url.match(/\/@([\w.-]+)/);
|
|
5
|
+
return m ? m[1] : null;
|
|
6
|
+
}
|
|
7
|
+
|
|
8
|
+
export function extractVideoId(url) {
|
|
9
|
+
const m = url.match(/\/video\/(\d+)/);
|
|
10
|
+
return m ? m[1] : null;
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
export function normalizeUsername(input) {
|
|
14
|
+
return (input || '').replace(/^@/, '');
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export function toProfileUrl(handle) {
|
|
18
|
+
const clean = normalizeUsername(handle);
|
|
19
|
+
return `${BASE_URL}/@${clean}`;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export function toVideoUrl(handle, videoId) {
|
|
23
|
+
const clean = normalizeUsername(handle);
|
|
24
|
+
return `${BASE_URL}/@${clean}/video/${videoId}`;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export function ensureAbsoluteUrl(href) {
|
|
28
|
+
if (href.startsWith('http')) return href;
|
|
29
|
+
return `${BASE_URL}${href}`;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
export function isProfileUrl(url) {
|
|
33
|
+
return /\/@[\w.-]+(?:$|[?#])/.test(url);
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export function isVideoUrl(url) {
|
|
37
|
+
return /\/video\/\d+/.test(url);
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
export function extractDisplayPath(url) {
|
|
41
|
+
try {
|
|
42
|
+
const parts = new URL(url).pathname.split('/').filter(Boolean);
|
|
43
|
+
return parts.slice(-2).join('/');
|
|
44
|
+
} catch {
|
|
45
|
+
return url;
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
export function extractAuthorFromVideoUrl(url) {
|
|
50
|
+
const m = url.match(/@([^/]+)\/video/);
|
|
51
|
+
return m ? '@' + m[1] : null;
|
|
52
|
+
}
|
package/src/main.js
CHANGED
|
@@ -1,48 +1,48 @@
|
|
|
1
|
-
import { parseArgs } from './lib/args.js';
|
|
2
|
-
import { proxy, HELP_TEXT, getConfigText } from './lib/constants.js';
|
|
3
|
-
import { handleInfo } from './cli/info.js';
|
|
4
|
-
import { handleExplore } from './cli/explore.js';
|
|
5
|
-
import { handleAttach } from './cli/attach.js';
|
|
6
|
-
import { handleWatch } from './cli/watch.js';
|
|
7
|
-
import { handleConfig, showConfig, showUsage, version } from './cli/config.js';
|
|
8
|
-
import { handleOpen } from './cli/open.js';
|
|
9
|
-
import { handleComments } from './cli/comments.js';
|
|
10
|
-
import { handleVideoStats } from './cli/videostats.js';
|
|
11
|
-
|
|
12
|
-
async function main() {
|
|
13
|
-
const parsed = parseArgs();
|
|
14
|
-
|
|
15
|
-
switch (parsed.subcommand) {
|
|
16
|
-
case 'explore': return handleExplore(parsed);
|
|
17
|
-
case 'info': return handleInfo(parsed);
|
|
18
|
-
case 'attach': return handleAttach(parsed);
|
|
19
|
-
case 'watch': return handleWatch(parsed);
|
|
20
|
-
case 'open': return handleOpen(parsed);
|
|
21
|
-
case 'comments': return handleComments(parsed);
|
|
22
|
-
case 'videostats': return handleVideoStats(parsed);
|
|
23
|
-
}
|
|
24
|
-
|
|
25
|
-
const { urls, outputFile, outputFormat, exploreCount, showConfig: showCfg, showHelp, showVersion, customProxy, configAction, configKey, configValue } = parsed;
|
|
26
|
-
|
|
27
|
-
if (showVersion) {
|
|
28
|
-
console.log(version);
|
|
29
|
-
process.exit(0);
|
|
30
|
-
}
|
|
31
|
-
if (showHelp) return showUsage();
|
|
32
|
-
if (configAction) return handleConfig(configAction, configKey, configValue);
|
|
33
|
-
if (showCfg) return showConfig(urls, outputFile);
|
|
34
|
-
if (urls.length === 0 && exploreCount === 0) return showUsage();
|
|
35
|
-
|
|
36
|
-
// 默认行为:URL 走 info,--explore 走 explore
|
|
37
|
-
if (exploreCount > 0) {
|
|
38
|
-
return handleExplore({ ...parsed, subcommand: 'explore' });
|
|
39
|
-
}
|
|
40
|
-
|
|
41
|
-
// 有 URL 默认走 info
|
|
42
|
-
return handleInfo(parsed);
|
|
43
|
-
}
|
|
44
|
-
|
|
45
|
-
main().catch(err => {
|
|
46
|
-
console.error(`错误: ${err.message}`);
|
|
47
|
-
process.exit(1);
|
|
48
|
-
});
|
|
1
|
+
import { parseArgs } from './lib/args.js';
|
|
2
|
+
import { proxy, HELP_TEXT, getConfigText } from './lib/constants.js';
|
|
3
|
+
import { handleInfo } from './cli/info.js';
|
|
4
|
+
import { handleExplore } from './cli/explore.js';
|
|
5
|
+
import { handleAttach } from './cli/attach.js';
|
|
6
|
+
import { handleWatch } from './cli/watch.js';
|
|
7
|
+
import { handleConfig, showConfig, showUsage, version } from './cli/config.js';
|
|
8
|
+
import { handleOpen } from './cli/open.js';
|
|
9
|
+
import { handleComments } from './cli/comments.js';
|
|
10
|
+
import { handleVideoStats } from './cli/videostats.js';
|
|
11
|
+
|
|
12
|
+
async function main() {
|
|
13
|
+
const parsed = parseArgs();
|
|
14
|
+
|
|
15
|
+
switch (parsed.subcommand) {
|
|
16
|
+
case 'explore': return handleExplore(parsed);
|
|
17
|
+
case 'info': return handleInfo(parsed);
|
|
18
|
+
case 'attach': return handleAttach(parsed);
|
|
19
|
+
case 'watch': return handleWatch(parsed);
|
|
20
|
+
case 'open': return handleOpen(parsed);
|
|
21
|
+
case 'comments': return handleComments(parsed);
|
|
22
|
+
case 'videostats': return handleVideoStats(parsed);
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
const { urls, outputFile, outputFormat, exploreCount, showConfig: showCfg, showHelp, showVersion, customProxy, configAction, configKey, configValue } = parsed;
|
|
26
|
+
|
|
27
|
+
if (showVersion) {
|
|
28
|
+
console.log(version);
|
|
29
|
+
process.exit(0);
|
|
30
|
+
}
|
|
31
|
+
if (showHelp) return showUsage();
|
|
32
|
+
if (configAction) return handleConfig(configAction, configKey, configValue);
|
|
33
|
+
if (showCfg) return showConfig(urls, outputFile);
|
|
34
|
+
if (urls.length === 0 && exploreCount === 0) return showUsage();
|
|
35
|
+
|
|
36
|
+
// 默认行为:URL 走 info,--explore 走 explore
|
|
37
|
+
if (exploreCount > 0) {
|
|
38
|
+
return handleExplore({ ...parsed, subcommand: 'explore' });
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
// 有 URL 默认走 info
|
|
42
|
+
return handleInfo(parsed);
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
main().catch(err => {
|
|
46
|
+
console.error(`错误: ${err.message}`);
|
|
47
|
+
process.exit(1);
|
|
48
|
+
});
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
{
|
|
2
|
+
"user": {
|
|
3
|
+
"uniqueId": "bar.lar.lar.moeta",
|
|
4
|
+
"secUid": "MS4wLjABAAAA3cgKTWvKfga0JAWeakAzx3zQ-aFAC8RuQvxD4HQFraKKsc_TbOIyMo3_ofVlXofV",
|
|
5
|
+
"nickname": "Bar Lar Lar Moetain",
|
|
6
|
+
"ttSeller": false,
|
|
7
|
+
"verified": false,
|
|
8
|
+
"followerCount": 24000,
|
|
9
|
+
"videoCount": 749,
|
|
10
|
+
"followingCount": 4293,
|
|
11
|
+
"heartCount": 254300,
|
|
12
|
+
"signature": ""
|
|
13
|
+
},
|
|
14
|
+
"totalVideos": 5,
|
|
15
|
+
"videos": [
|
|
16
|
+
{
|
|
17
|
+
"id": "7638231799084158228",
|
|
18
|
+
"url": "https://www.tiktok.com/@bar.lar.lar.moeta/video/7638231799084158228"
|
|
19
|
+
},
|
|
20
|
+
{
|
|
21
|
+
"id": "7638162444698914068",
|
|
22
|
+
"url": "https://www.tiktok.com/@bar.lar.lar.moeta/video/7638162444698914068"
|
|
23
|
+
},
|
|
24
|
+
{
|
|
25
|
+
"id": "7638116251767819541",
|
|
26
|
+
"url": "https://www.tiktok.com/@bar.lar.lar.moeta/video/7638116251767819541"
|
|
27
|
+
},
|
|
28
|
+
{
|
|
29
|
+
"id": "7638069637321690388",
|
|
30
|
+
"url": "https://www.tiktok.com/@bar.lar.lar.moeta/video/7638069637321690388"
|
|
31
|
+
},
|
|
32
|
+
{
|
|
33
|
+
"id": "7637927171025112341",
|
|
34
|
+
"url": "https://www.tiktok.com/@bar.lar.lar.moeta/video/7637927171025112341"
|
|
35
|
+
}
|
|
36
|
+
]
|
|
37
|
+
}
|
package/src/scraper/auto-core.js
CHANGED
|
@@ -1,203 +1,203 @@
|
|
|
1
|
-
import {
|
|
2
|
-
delay,
|
|
3
|
-
ensureBrowserReady,
|
|
4
|
-
ensureTikTokPage,
|
|
5
|
-
setDelayConfig,
|
|
6
|
-
getDelayConfig,
|
|
7
|
-
closeCommentPanel,
|
|
8
|
-
retryWithBackoff,
|
|
9
|
-
detectPageError,
|
|
10
|
-
isLoggedIn,
|
|
11
|
-
assertPageUrl,
|
|
12
|
-
} from './modules/page-helpers.js';
|
|
13
|
-
import { detectCaptcha } from './modules/captcha-handler.js';
|
|
14
|
-
export { ensureBrowserReady };
|
|
15
|
-
import {
|
|
16
|
-
getUserInfo,
|
|
17
|
-
collectVideos,
|
|
18
|
-
} from '../videos/core.js';
|
|
19
|
-
import { runScrape } from './core.js';
|
|
20
|
-
import { extractFollowAndFollowers } from './modules/follow-extractor.js';
|
|
21
|
-
|
|
22
|
-
function mergeUserInfo(existing, incoming, source) {
|
|
23
|
-
const merged = { ...existing };
|
|
24
|
-
for (const [key, value] of Object.entries(incoming)) {
|
|
25
|
-
if (key === '_sources') continue;
|
|
26
|
-
if (value === undefined || value === null || value === '') continue;
|
|
27
|
-
if (typeof value === 'number' && typeof merged[key] === 'number') {
|
|
28
|
-
merged[key] = Math.max(merged[key], value);
|
|
29
|
-
} else if (merged[key] === undefined || merged[key] === null || merged[key] === '') {
|
|
30
|
-
merged[key] = value;
|
|
31
|
-
}
|
|
32
|
-
}
|
|
33
|
-
if (source) {
|
|
34
|
-
if (!merged._sources) merged._sources = [];
|
|
35
|
-
if (!merged._sources.includes(source)) merged._sources.push(source);
|
|
36
|
-
}
|
|
37
|
-
return merged;
|
|
38
|
-
}
|
|
39
|
-
|
|
40
|
-
async function processUser(page, username, options, log) {
|
|
41
|
-
const {
|
|
42
|
-
collectMax = 1,
|
|
43
|
-
scrapeDepth = 50,
|
|
44
|
-
maxComments = 200,
|
|
45
|
-
maxGuess = 10,
|
|
46
|
-
preset = 'fast',
|
|
47
|
-
switchMax = null,
|
|
48
|
-
commentMax = null,
|
|
49
|
-
enableFollow = false,
|
|
50
|
-
maxFollowing = 200,
|
|
51
|
-
maxFollowers = 200,
|
|
52
|
-
browser = null,
|
|
53
|
-
} = options;
|
|
54
|
-
|
|
55
|
-
const result = {
|
|
56
|
-
userInfo: null,
|
|
57
|
-
collectedVideos: [],
|
|
58
|
-
discoveredVideoAuthors: [],
|
|
59
|
-
discoveredCommentAuthors: [],
|
|
60
|
-
discoveredGuessAuthors: [],
|
|
61
|
-
discoveredFollowing: [],
|
|
62
|
-
discoveredFollowers: [],
|
|
63
|
-
error: null,
|
|
64
|
-
};
|
|
65
|
-
|
|
66
|
-
try {
|
|
67
|
-
log(`\n[processUser] 访问 @${username}...`);
|
|
68
|
-
await retryWithBackoff(async () => {
|
|
69
|
-
await page.goto(`https://www.tiktok.com/@${username}`, {
|
|
70
|
-
waitUntil: 'load', timeout: 30000,
|
|
71
|
-
});
|
|
72
|
-
assertPageUrl(page, `@${username}`);
|
|
73
|
-
}, { log });
|
|
74
|
-
await page.waitForSelector('[class*="DivVideoList"]', { timeout: 10000 }).catch(() => {});
|
|
75
|
-
await delay(1000, 2000);
|
|
76
|
-
|
|
77
|
-
const info = await getUserInfo(page);
|
|
78
|
-
result.userInfo = info;
|
|
79
|
-
if (!info.uniqueId) info.uniqueId = username;
|
|
80
|
-
log(` 昵称: ${info.nickname || '-'} | 粉丝: ${info.followerCount || 0}`);
|
|
81
|
-
|
|
82
|
-
if (options.enableFollow) {
|
|
83
|
-
const loggedIn = await isLoggedIn(page);
|
|
84
|
-
if (!loggedIn) {
|
|
85
|
-
log(' [跳过] 提取关注/粉丝:未登录,请先登录 TikTok');
|
|
86
|
-
result.discoveredFollowing = [];
|
|
87
|
-
result.discoveredFollowers = [];
|
|
88
|
-
} else {
|
|
89
|
-
try {
|
|
90
|
-
log(' 提取关注/粉丝列表...');
|
|
91
|
-
const { following, followers } = await extractFollowAndFollowers(page, {
|
|
92
|
-
maxFollowing: options.maxFollowing || 200,
|
|
93
|
-
maxFollowers: options.maxFollowers || 200,
|
|
94
|
-
log,
|
|
95
|
-
});
|
|
96
|
-
result.discoveredFollowing = following;
|
|
97
|
-
result.discoveredFollowers = followers;
|
|
98
|
-
log(` 关注: ${following.length} | 粉丝: ${followers.length}`);
|
|
99
|
-
} catch (e) {
|
|
100
|
-
log(` 关注/粉丝提取失败: ${e.message}`);
|
|
101
|
-
result.discoveredFollowing = [];
|
|
102
|
-
result.discoveredFollowers = [];
|
|
103
|
-
}
|
|
104
|
-
}
|
|
105
|
-
}
|
|
106
|
-
|
|
107
|
-
const captcha = await detectCaptcha(page);
|
|
108
|
-
if (captcha && captcha.visible) {
|
|
109
|
-
log(`[验证码] @${username} 页面出现验证码`);
|
|
110
|
-
result.captchaDetected = true;
|
|
111
|
-
result.captchaStage = result.captchaStage || 'video-page';
|
|
112
|
-
result.captchaMessage = result.captchaMessage || '视频页出现验证码';
|
|
113
|
-
}
|
|
114
|
-
|
|
115
|
-
const videos = await collectVideos(page, username, collectMax, log);
|
|
116
|
-
const videoList = Array.from(videos.values()).slice(0, collectMax);
|
|
117
|
-
result.collectedVideos = videoList.map(v => ({
|
|
118
|
-
videoId: v.id,
|
|
119
|
-
videoUrl: v.href,
|
|
120
|
-
}));
|
|
121
|
-
|
|
122
|
-
if (videoList.length > 0) {
|
|
123
|
-
const allVideoAuthors = new Map();
|
|
124
|
-
const allCommentAuthors = new Set();
|
|
125
|
-
const allGuessAuthors = new Set();
|
|
126
|
-
|
|
127
|
-
for (let i = 0; i < videoList.length; i++) {
|
|
128
|
-
const video = videoList[i];
|
|
129
|
-
const videoUrl = video.href.startsWith('http')
|
|
130
|
-
? video.href
|
|
131
|
-
: `https://www.tiktok.com${video.href}`;
|
|
132
|
-
log(` [${i + 1}/${videoList.length}] 开始 scrape: ${videoUrl} (深度 ${scrapeDepth})`);
|
|
133
|
-
|
|
134
|
-
const scrapeResult = await runScrape({
|
|
135
|
-
videoUrl,
|
|
136
|
-
maxVideos: scrapeDepth,
|
|
137
|
-
maxComments,
|
|
138
|
-
maxGuess,
|
|
139
|
-
preset,
|
|
140
|
-
switchMax,
|
|
141
|
-
commentMax,
|
|
142
|
-
browser,
|
|
143
|
-
page,
|
|
144
|
-
log,
|
|
145
|
-
});
|
|
146
|
-
|
|
147
|
-
if (scrapeResult.captchaDetected) {
|
|
148
|
-
result.captchaDetected = true;
|
|
149
|
-
result.captchaStage = scrapeResult.captchaStage || 'scrape';
|
|
150
|
-
result.captchaMessage = scrapeResult.captchaMessage || 'scrape阶段出现验证码';
|
|
151
|
-
}
|
|
152
|
-
|
|
153
|
-
const scrapeOutput = scrapeResult.output;
|
|
154
|
-
|
|
155
|
-
if (scrapeOutput && scrapeOutput.videoDetails) {
|
|
156
|
-
for (const vd of scrapeOutput.videoDetails) {
|
|
157
|
-
if (!allVideoAuthors.has(vd.uniqueId)) {
|
|
158
|
-
allVideoAuthors.set(vd.uniqueId, {
|
|
159
|
-
uniqueId: vd.uniqueId,
|
|
160
|
-
nickname: vd.nickname,
|
|
161
|
-
locationCreated: vd.locationCreated,
|
|
162
|
-
});
|
|
163
|
-
}
|
|
164
|
-
}
|
|
165
|
-
}
|
|
166
|
-
|
|
167
|
-
if (scrapeOutput && scrapeOutput.commentUsers) {
|
|
168
|
-
for (const cu of scrapeOutput.commentUsers) {
|
|
169
|
-
allCommentAuthors.add(cu);
|
|
170
|
-
}
|
|
171
|
-
}
|
|
172
|
-
|
|
173
|
-
if (scrapeOutput && scrapeOutput.guessAuthors) {
|
|
174
|
-
for (const ga of scrapeOutput.guessAuthors) {
|
|
175
|
-
allGuessAuthors.add(ga);
|
|
176
|
-
}
|
|
177
|
-
}
|
|
178
|
-
}
|
|
179
|
-
|
|
180
|
-
result.discoveredVideoAuthors = [...allVideoAuthors.values()];
|
|
181
|
-
result.discoveredCommentAuthors = [...allCommentAuthors];
|
|
182
|
-
result.discoveredGuessAuthors = [...allGuessAuthors];
|
|
183
|
-
|
|
184
|
-
log(` 发现: ${result.discoveredVideoAuthors.length} 个视频作者, ${result.discoveredCommentAuthors.length} 个评论作者, ${result.discoveredGuessAuthors.length} 个猜你喜欢作者`);
|
|
185
|
-
} else {
|
|
186
|
-
const pageError = await detectPageError(page);
|
|
187
|
-
result.restricted = !!pageError;
|
|
188
|
-
if (pageError) {
|
|
189
|
-
log(` @${username} 页面受限(${pageError}),标记跳过`);
|
|
190
|
-
} else {
|
|
191
|
-
log(` @${username} 没有视频,跳过 scrape`);
|
|
192
|
-
}
|
|
193
|
-
}
|
|
194
|
-
} catch (e) {
|
|
195
|
-
result.error = e.message;
|
|
196
|
-
result.errorStack = e.stack || '';
|
|
197
|
-
log(` [错误] ${e.message}`);
|
|
198
|
-
}
|
|
199
|
-
|
|
200
|
-
return result;
|
|
201
|
-
}
|
|
202
|
-
|
|
203
|
-
export { processUser, mergeUserInfo };
|
|
1
|
+
import {
|
|
2
|
+
delay,
|
|
3
|
+
ensureBrowserReady,
|
|
4
|
+
ensureTikTokPage,
|
|
5
|
+
setDelayConfig,
|
|
6
|
+
getDelayConfig,
|
|
7
|
+
closeCommentPanel,
|
|
8
|
+
retryWithBackoff,
|
|
9
|
+
detectPageError,
|
|
10
|
+
isLoggedIn,
|
|
11
|
+
assertPageUrl,
|
|
12
|
+
} from './modules/page-helpers.js';
|
|
13
|
+
import { detectCaptcha } from './modules/captcha-handler.js';
|
|
14
|
+
export { ensureBrowserReady };
|
|
15
|
+
import {
|
|
16
|
+
getUserInfo,
|
|
17
|
+
collectVideos,
|
|
18
|
+
} from '../videos/core.js';
|
|
19
|
+
import { runScrape } from './core.js';
|
|
20
|
+
import { extractFollowAndFollowers } from './modules/follow-extractor.js';
|
|
21
|
+
|
|
22
|
+
function mergeUserInfo(existing, incoming, source) {
|
|
23
|
+
const merged = { ...existing };
|
|
24
|
+
for (const [key, value] of Object.entries(incoming)) {
|
|
25
|
+
if (key === '_sources') continue;
|
|
26
|
+
if (value === undefined || value === null || value === '') continue;
|
|
27
|
+
if (typeof value === 'number' && typeof merged[key] === 'number') {
|
|
28
|
+
merged[key] = Math.max(merged[key], value);
|
|
29
|
+
} else if (merged[key] === undefined || merged[key] === null || merged[key] === '') {
|
|
30
|
+
merged[key] = value;
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
if (source) {
|
|
34
|
+
if (!merged._sources) merged._sources = [];
|
|
35
|
+
if (!merged._sources.includes(source)) merged._sources.push(source);
|
|
36
|
+
}
|
|
37
|
+
return merged;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
async function processUser(page, username, options, log) {
|
|
41
|
+
const {
|
|
42
|
+
collectMax = 1,
|
|
43
|
+
scrapeDepth = 50,
|
|
44
|
+
maxComments = 200,
|
|
45
|
+
maxGuess = 10,
|
|
46
|
+
preset = 'fast',
|
|
47
|
+
switchMax = null,
|
|
48
|
+
commentMax = null,
|
|
49
|
+
enableFollow = false,
|
|
50
|
+
maxFollowing = 200,
|
|
51
|
+
maxFollowers = 200,
|
|
52
|
+
browser = null,
|
|
53
|
+
} = options;
|
|
54
|
+
|
|
55
|
+
const result = {
|
|
56
|
+
userInfo: null,
|
|
57
|
+
collectedVideos: [],
|
|
58
|
+
discoveredVideoAuthors: [],
|
|
59
|
+
discoveredCommentAuthors: [],
|
|
60
|
+
discoveredGuessAuthors: [],
|
|
61
|
+
discoveredFollowing: [],
|
|
62
|
+
discoveredFollowers: [],
|
|
63
|
+
error: null,
|
|
64
|
+
};
|
|
65
|
+
|
|
66
|
+
try {
|
|
67
|
+
log(`\n[processUser] 访问 @${username}...`);
|
|
68
|
+
await retryWithBackoff(async () => {
|
|
69
|
+
await page.goto(`https://www.tiktok.com/@${username}`, {
|
|
70
|
+
waitUntil: 'load', timeout: 30000,
|
|
71
|
+
});
|
|
72
|
+
assertPageUrl(page, `@${username}`);
|
|
73
|
+
}, { log });
|
|
74
|
+
await page.waitForSelector('[class*="DivVideoList"]', { timeout: 10000 }).catch(() => {});
|
|
75
|
+
await delay(1000, 2000);
|
|
76
|
+
|
|
77
|
+
const info = await getUserInfo(page);
|
|
78
|
+
result.userInfo = info;
|
|
79
|
+
if (!info.uniqueId) info.uniqueId = username;
|
|
80
|
+
log(` 昵称: ${info.nickname || '-'} | 粉丝: ${info.followerCount || 0}`);
|
|
81
|
+
|
|
82
|
+
if (options.enableFollow) {
|
|
83
|
+
const loggedIn = await isLoggedIn(page);
|
|
84
|
+
if (!loggedIn) {
|
|
85
|
+
log(' [跳过] 提取关注/粉丝:未登录,请先登录 TikTok');
|
|
86
|
+
result.discoveredFollowing = [];
|
|
87
|
+
result.discoveredFollowers = [];
|
|
88
|
+
} else {
|
|
89
|
+
try {
|
|
90
|
+
log(' 提取关注/粉丝列表...');
|
|
91
|
+
const { following, followers } = await extractFollowAndFollowers(page, {
|
|
92
|
+
maxFollowing: options.maxFollowing || 200,
|
|
93
|
+
maxFollowers: options.maxFollowers || 200,
|
|
94
|
+
log,
|
|
95
|
+
});
|
|
96
|
+
result.discoveredFollowing = following;
|
|
97
|
+
result.discoveredFollowers = followers;
|
|
98
|
+
log(` 关注: ${following.length} | 粉丝: ${followers.length}`);
|
|
99
|
+
} catch (e) {
|
|
100
|
+
log(` 关注/粉丝提取失败: ${e.message}`);
|
|
101
|
+
result.discoveredFollowing = [];
|
|
102
|
+
result.discoveredFollowers = [];
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
const captcha = await detectCaptcha(page);
|
|
108
|
+
if (captcha && captcha.visible) {
|
|
109
|
+
log(`[验证码] @${username} 页面出现验证码`);
|
|
110
|
+
result.captchaDetected = true;
|
|
111
|
+
result.captchaStage = result.captchaStage || 'video-page';
|
|
112
|
+
result.captchaMessage = result.captchaMessage || '视频页出现验证码';
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
const videos = await collectVideos(page, username, collectMax, log);
|
|
116
|
+
const videoList = Array.from(videos.values()).slice(0, collectMax);
|
|
117
|
+
result.collectedVideos = videoList.map(v => ({
|
|
118
|
+
videoId: v.id,
|
|
119
|
+
videoUrl: v.href,
|
|
120
|
+
}));
|
|
121
|
+
|
|
122
|
+
if (videoList.length > 0) {
|
|
123
|
+
const allVideoAuthors = new Map();
|
|
124
|
+
const allCommentAuthors = new Set();
|
|
125
|
+
const allGuessAuthors = new Set();
|
|
126
|
+
|
|
127
|
+
for (let i = 0; i < videoList.length; i++) {
|
|
128
|
+
const video = videoList[i];
|
|
129
|
+
const videoUrl = video.href.startsWith('http')
|
|
130
|
+
? video.href
|
|
131
|
+
: `https://www.tiktok.com${video.href}`;
|
|
132
|
+
log(` [${i + 1}/${videoList.length}] 开始 scrape: ${videoUrl} (深度 ${scrapeDepth})`);
|
|
133
|
+
|
|
134
|
+
const scrapeResult = await runScrape({
|
|
135
|
+
videoUrl,
|
|
136
|
+
maxVideos: scrapeDepth,
|
|
137
|
+
maxComments,
|
|
138
|
+
maxGuess,
|
|
139
|
+
preset,
|
|
140
|
+
switchMax,
|
|
141
|
+
commentMax,
|
|
142
|
+
browser,
|
|
143
|
+
page,
|
|
144
|
+
log,
|
|
145
|
+
});
|
|
146
|
+
|
|
147
|
+
if (scrapeResult.captchaDetected) {
|
|
148
|
+
result.captchaDetected = true;
|
|
149
|
+
result.captchaStage = scrapeResult.captchaStage || 'scrape';
|
|
150
|
+
result.captchaMessage = scrapeResult.captchaMessage || 'scrape阶段出现验证码';
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
const scrapeOutput = scrapeResult.output;
|
|
154
|
+
|
|
155
|
+
if (scrapeOutput && scrapeOutput.videoDetails) {
|
|
156
|
+
for (const vd of scrapeOutput.videoDetails) {
|
|
157
|
+
if (!allVideoAuthors.has(vd.uniqueId)) {
|
|
158
|
+
allVideoAuthors.set(vd.uniqueId, {
|
|
159
|
+
uniqueId: vd.uniqueId,
|
|
160
|
+
nickname: vd.nickname,
|
|
161
|
+
locationCreated: vd.locationCreated,
|
|
162
|
+
});
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
if (scrapeOutput && scrapeOutput.commentUsers) {
|
|
168
|
+
for (const cu of scrapeOutput.commentUsers) {
|
|
169
|
+
allCommentAuthors.add(cu);
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
if (scrapeOutput && scrapeOutput.guessAuthors) {
|
|
174
|
+
for (const ga of scrapeOutput.guessAuthors) {
|
|
175
|
+
allGuessAuthors.add(ga);
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
result.discoveredVideoAuthors = [...allVideoAuthors.values()];
|
|
181
|
+
result.discoveredCommentAuthors = [...allCommentAuthors];
|
|
182
|
+
result.discoveredGuessAuthors = [...allGuessAuthors];
|
|
183
|
+
|
|
184
|
+
log(` 发现: ${result.discoveredVideoAuthors.length} 个视频作者, ${result.discoveredCommentAuthors.length} 个评论作者, ${result.discoveredGuessAuthors.length} 个猜你喜欢作者`);
|
|
185
|
+
} else {
|
|
186
|
+
const pageError = await detectPageError(page);
|
|
187
|
+
result.restricted = !!pageError;
|
|
188
|
+
if (pageError) {
|
|
189
|
+
log(` @${username} 页面受限(${pageError}),标记跳过`);
|
|
190
|
+
} else {
|
|
191
|
+
log(` @${username} 没有视频,跳过 scrape`);
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
} catch (e) {
|
|
195
|
+
result.error = e.message;
|
|
196
|
+
result.errorStack = e.stack || '';
|
|
197
|
+
log(` [错误] ${e.message}`);
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
return result;
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
export { processUser, mergeUserInfo };
|