tt-help-cli-ycl 1.3.0 → 1.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -17
- package/cli.js +9 -9
- package/package.json +44 -44
- package/src/cli/auto.js +94 -0
- package/src/cli/explore.js +117 -0
- package/src/cli/progress.js +111 -0
- package/src/cli/scrape.js +47 -0
- package/src/cli/utils.js +18 -0
- package/src/cli/videos.js +41 -0
- package/src/cli/watch.js +28 -0
- package/src/lib/args.js +386 -397
- package/src/lib/browser/anti-detect.js +23 -0
- package/src/lib/browser/cdp.js +142 -0
- package/src/lib/browser/launch.js +43 -0
- package/src/lib/browser/page.js +80 -0
- package/src/lib/constants.js +85 -168
- package/src/lib/delay.js +54 -0
- package/src/lib/explore-fetch.js +118 -0
- package/src/lib/fetcher.js +45 -60
- package/src/lib/filter.js +66 -66
- package/src/lib/io.js +54 -76
- package/src/lib/output.js +80 -80
- package/src/lib/parser.js +47 -47
- package/src/lib/retry.js +44 -0
- package/src/lib/scrape.js +40 -39
- package/src/lib/url.js +52 -0
- package/src/main.mjs +199 -962
- package/src/results/user-videos-bar.lar.lar.moeta.json +37 -0
- package/src/scraper/auto-core.mjs +183 -0
- package/src/scraper/{core.cjs → core.mjs} +188 -214
- package/src/{explore-core.cjs → scraper/explore-core.mjs} +44 -42
- package/src/scraper/modules/captcha-handler.mjs +114 -0
- package/src/scraper/modules/comment-extractor.mjs +69 -0
- package/src/scraper/modules/follow-extractor.mjs +121 -0
- package/src/scraper/modules/{guess-extractor.cjs → guess-extractor.mjs} +51 -53
- package/src/scraper/modules/page-error-detector.mjs +70 -0
- package/src/scraper/modules/page-helpers.mjs +46 -0
- package/src/scraper/modules/scroll-collector.mjs +189 -0
- package/src/{get-user-videos-core.cjs → videos/core.mjs} +126 -143
- package/src/watch/data-store.mjs +239 -0
- package/src/watch/public/index.html +446 -271
- package/src/watch/server.mjs +257 -153
- package/src/auto-core.cjs +0 -367
- package/src/data-store.cjs +0 -69
- package/src/get-user-videos.cjs +0 -59
- package/src/lib/auto-browser.mjs +0 -13
- package/src/lib/explore.js +0 -225
- package/src/lib/get-user-videos-browser.mjs +0 -6
- package/src/lib/scrape-browser.mjs +0 -6
- package/src/scraper/index.cjs +0 -97
- package/src/scraper/modules/comment-extractor.cjs +0 -49
- package/src/scraper/modules/follow-extractor.cjs +0 -112
- package/src/scraper/modules/page-helpers.cjs +0 -422
- package/src/scraper/modules/scroll-collector.cjs +0 -173
- package/src/scraper/modules/video-scanner.cjs +0 -43
- package/src/test-auto-follow.cjs +0 -109
- package/src/test-extractors.cjs +0 -75
- package/src/test-follow.cjs +0 -41
package/src/test-auto-follow.cjs
DELETED
|
@@ -1,109 +0,0 @@
|
|
|
1
|
-
const path = require('path');
|
|
2
|
-
const fs = require('fs');
|
|
3
|
-
const { ensureBrowserReady, setDelayConfig } = require('./scraper/modules/page-helpers.cjs');
|
|
4
|
-
const { processUser } = require('./auto-core.cjs');
|
|
5
|
-
const { createStore } = require('./data-store.cjs');
|
|
6
|
-
|
|
7
|
-
async function main() {
|
|
8
|
-
const outFile = path.join(__dirname, '..', 'results', 'auto-test.json');
|
|
9
|
-
const store = createStore(outFile);
|
|
10
|
-
|
|
11
|
-
setDelayConfig('fast');
|
|
12
|
-
|
|
13
|
-
const browser = await ensureBrowserReady();
|
|
14
|
-
let page;
|
|
15
|
-
try {
|
|
16
|
-
const contexts = browser.contexts();
|
|
17
|
-
page = null;
|
|
18
|
-
for (const ctx of contexts) {
|
|
19
|
-
for (const p of ctx.pages()) {
|
|
20
|
-
if (p.url().includes('tiktok.com')) { page = p; break; }
|
|
21
|
-
}
|
|
22
|
-
if (page) break;
|
|
23
|
-
}
|
|
24
|
-
if (!page) page = await contexts[0].newPage();
|
|
25
|
-
|
|
26
|
-
console.error('========== 测试 processUser + enableFollow ==========');
|
|
27
|
-
console.error('用户: @qiqi23280\n');
|
|
28
|
-
|
|
29
|
-
const result = await processUser(page, 'qiqi23280', {
|
|
30
|
-
collectMax: 1,
|
|
31
|
-
scrapeDepth: 1,
|
|
32
|
-
maxComments: 10,
|
|
33
|
-
maxGuess: 5,
|
|
34
|
-
preset: 'fast',
|
|
35
|
-
enableFollow: true,
|
|
36
|
-
maxFollowing: 50,
|
|
37
|
-
maxFollowers: 50,
|
|
38
|
-
browser,
|
|
39
|
-
}, console.error);
|
|
40
|
-
|
|
41
|
-
console.error('\n========== 结果验证 ==========');
|
|
42
|
-
let allPassed = true;
|
|
43
|
-
|
|
44
|
-
const checks = [
|
|
45
|
-
{ label: '用户信息', ok: result.userInfo && result.userInfo.uniqueId, detail: result.userInfo?.uniqueId },
|
|
46
|
-
{ label: '关注列表', ok: Array.isArray(result.discoveredFollowing) && result.discoveredFollowing.length > 0, detail: `${result.discoveredFollowing?.length || 0} 人` },
|
|
47
|
-
{ label: '粉丝列表', ok: Array.isArray(result.discoveredFollowers) && result.discoveredFollowers.length > 0, detail: `${result.discoveredFollowers?.length || 0} 人` },
|
|
48
|
-
{ label: '关注格式', ok: result.discoveredFollowing?.every(p => Array.isArray(p) && p.length === 2 && p[0].startsWith('@')), detail: null },
|
|
49
|
-
{ label: '粉丝格式', ok: result.discoveredFollowers?.every(p => Array.isArray(p) && p.length === 2 && p[0].startsWith('@')), detail: null },
|
|
50
|
-
{ label: '无错误', ok: !result.error, detail: result.error },
|
|
51
|
-
];
|
|
52
|
-
|
|
53
|
-
for (const c of checks) {
|
|
54
|
-
const status = c.ok ? 'PASS' : 'FAIL';
|
|
55
|
-
const detailStr = c.detail !== null ? ` (${c.detail})` : '';
|
|
56
|
-
console.error(` ${status}: ${c.label}${detailStr}`);
|
|
57
|
-
if (!c.ok) allPassed = false;
|
|
58
|
-
}
|
|
59
|
-
|
|
60
|
-
// 模拟入队逻辑
|
|
61
|
-
const queue = ['qiqi23280'];
|
|
62
|
-
const followingIds = (result.discoveredFollowing || []).map(([h]) => h.replace(/^@/, ''));
|
|
63
|
-
const followerIds = (result.discoveredFollowers || []).map(([h]) => h.replace(/^@/, ''));
|
|
64
|
-
|
|
65
|
-
for (const uid of followingIds) queue.push(uid);
|
|
66
|
-
for (const uid of followerIds) queue.push(uid);
|
|
67
|
-
const uniqueQueue = [...new Set(queue)];
|
|
68
|
-
|
|
69
|
-
console.error(`\n 队列长度: ${uniqueQueue.length}(关注 ${followingIds.length} + 粉丝 ${followerIds.length} + 种子 1)`);
|
|
70
|
-
|
|
71
|
-
// 写入 store 验证
|
|
72
|
-
store.addUser({
|
|
73
|
-
uniqueId: 'qiqi23280',
|
|
74
|
-
...result.userInfo,
|
|
75
|
-
sources: ['seed'],
|
|
76
|
-
});
|
|
77
|
-
for (const [handle, name] of (result.discoveredFollowing || [])) {
|
|
78
|
-
store.addUser({ uniqueId: handle.replace(/^@/, ''), nickname: name, sources: ['following'] });
|
|
79
|
-
}
|
|
80
|
-
for (const [handle, name] of (result.discoveredFollowers || [])) {
|
|
81
|
-
store.addUser({ uniqueId: handle.replace(/^@/, ''), nickname: name, sources: ['follower'] });
|
|
82
|
-
}
|
|
83
|
-
store.save();
|
|
84
|
-
|
|
85
|
-
const allUsers = store.getAllUsers();
|
|
86
|
-
console.error(` Store 用户数: ${allUsers.length}`);
|
|
87
|
-
|
|
88
|
-
// 验证 source 标记
|
|
89
|
-
const followingUsers = allUsers.filter(u => u.sources?.includes('following'));
|
|
90
|
-
const followerUsers = allUsers.filter(u => u.sources?.includes('follower'));
|
|
91
|
-
console.error(` 关注来源: ${followingUsers.length} | 粉丝来源: ${followerUsers.length}`);
|
|
92
|
-
|
|
93
|
-
if (followingUsers.length === 0 || followerUsers.length === 0) {
|
|
94
|
-
console.error(' FAIL: 缺少 following 或 follower 来源标记');
|
|
95
|
-
allPassed = false;
|
|
96
|
-
} else {
|
|
97
|
-
console.error(' PASS: 来源标记正确');
|
|
98
|
-
}
|
|
99
|
-
|
|
100
|
-
console.error(`\n${allPassed ? 'ALL PASSED' : 'SOME FAILED'}`);
|
|
101
|
-
console.error(`数据保存到: ${outFile}`);
|
|
102
|
-
process.exit(allPassed ? 0 : 1);
|
|
103
|
-
|
|
104
|
-
} finally {
|
|
105
|
-
await browser.close().catch(() => {});
|
|
106
|
-
}
|
|
107
|
-
}
|
|
108
|
-
|
|
109
|
-
main().catch(err => { console.error('FATAL:', err.message); process.exit(1); });
|
package/src/test-extractors.cjs
DELETED
|
@@ -1,75 +0,0 @@
|
|
|
1
|
-
const { ensureBrowserReady, delay, setDelayConfig } = require('./scraper/modules/page-helpers.cjs');
|
|
2
|
-
const { extractCommentAuthors } = require('./scraper/modules/comment-extractor.cjs');
|
|
3
|
-
const { extractGuessVideos } = require('./scraper/modules/guess-extractor.cjs');
|
|
4
|
-
|
|
5
|
-
async function main() {
|
|
6
|
-
setDelayConfig('fast');
|
|
7
|
-
|
|
8
|
-
const videoUrl = process.argv[2] || 'https://www.tiktok.com/@porfirio.fructuoso/video/7615853535955111198';
|
|
9
|
-
console.error(`目标: ${videoUrl}`);
|
|
10
|
-
|
|
11
|
-
const browser = await ensureBrowserReady();
|
|
12
|
-
let page;
|
|
13
|
-
try {
|
|
14
|
-
const contexts = browser.contexts();
|
|
15
|
-
page = null;
|
|
16
|
-
for (const ctx of contexts) {
|
|
17
|
-
for (const p of ctx.pages()) {
|
|
18
|
-
if (p.url().includes('tiktok.com')) { page = p; break; }
|
|
19
|
-
}
|
|
20
|
-
if (page) break;
|
|
21
|
-
}
|
|
22
|
-
if (!page) {
|
|
23
|
-
page = await contexts[0].newPage();
|
|
24
|
-
}
|
|
25
|
-
|
|
26
|
-
await page.goto(videoUrl, { waitUntil: 'networkidle', timeout: 60000 });
|
|
27
|
-
await delay(5000, 8000);
|
|
28
|
-
|
|
29
|
-
console.error(`当前URL: ${page.url()}`);
|
|
30
|
-
|
|
31
|
-
let allPassed = true;
|
|
32
|
-
|
|
33
|
-
// ========== 评论提取 ==========
|
|
34
|
-
console.error('\n--- 评论提取 (max=30) ---');
|
|
35
|
-
const t1 = Date.now();
|
|
36
|
-
let commentUsers = [];
|
|
37
|
-
try { commentUsers = await extractCommentAuthors(page, 30); }
|
|
38
|
-
catch (e) { console.error(` 异常: ${e.message}`); }
|
|
39
|
-
console.error(` 耗时: ${((Date.now()-t1)/1000).toFixed(1)}s, 结果: ${commentUsers.length} 个`);
|
|
40
|
-
|
|
41
|
-
if (commentUsers.length > 0) {
|
|
42
|
-
const s = new Set(commentUsers);
|
|
43
|
-
const ok = s.size === commentUsers.length;
|
|
44
|
-
console.error(` ${ok ? 'PASS' : 'FAIL'}: 唯一${s.size}/总数${commentUsers.length}`);
|
|
45
|
-
if (!ok) allPassed = false;
|
|
46
|
-
}
|
|
47
|
-
|
|
48
|
-
// ========== 猜你喜欢提取 ==========
|
|
49
|
-
console.error('\n--- 猜你喜欢提取 (max=20) ---');
|
|
50
|
-
const t2 = Date.now();
|
|
51
|
-
let guessVideos = [];
|
|
52
|
-
try { guessVideos = await extractGuessVideos(page, 20); }
|
|
53
|
-
catch (e) { console.error(` 异常: ${e.message}`); }
|
|
54
|
-
console.error(` 耗时: ${((Date.now()-t2)/1000).toFixed(1)}s, 结果: ${guessVideos.length} 个`);
|
|
55
|
-
|
|
56
|
-
if (guessVideos.length > 0) {
|
|
57
|
-
const ids = guessVideos.map(v => v.videoId);
|
|
58
|
-
const s = new Set(ids);
|
|
59
|
-
const ok = s.size === ids.length;
|
|
60
|
-
console.error(` ${ok ? 'PASS' : 'FAIL'}: 唯一${s.size}/总数${ids.length}`);
|
|
61
|
-
if (!ok) allPassed = false;
|
|
62
|
-
const ok2 = guessVideos.every(v => v.author && v.videoId && v.url);
|
|
63
|
-
console.error(` ${ok2 ? 'PASS' : 'FAIL'}: 结构完整`);
|
|
64
|
-
if (!ok2) allPassed = false;
|
|
65
|
-
}
|
|
66
|
-
|
|
67
|
-
console.error(`\n${allPassed ? 'ALL PASSED' : 'SOME FAILED'}`);
|
|
68
|
-
process.exit(allPassed ? 0 : 1);
|
|
69
|
-
|
|
70
|
-
} finally {
|
|
71
|
-
await browser.close().catch(() => {});
|
|
72
|
-
}
|
|
73
|
-
}
|
|
74
|
-
|
|
75
|
-
main().catch(err => { console.error('FATAL:', err.message); process.exit(1); });
|
package/src/test-follow.cjs
DELETED
|
@@ -1,41 +0,0 @@
|
|
|
1
|
-
const path = require('path');
|
|
2
|
-
const { ensureBrowserReady, ensureTikTokPage, delay } = require('./scraper/modules/page-helpers.cjs');
|
|
3
|
-
const { extractFollowAndFollowers } = require('./scraper/modules/follow-extractor.cjs');
|
|
4
|
-
|
|
5
|
-
async function main() {
|
|
6
|
-
const url = process.argv[2] || 'https://www.tiktok.com/@qiqi23280';
|
|
7
|
-
console.error(`目标: ${url}`);
|
|
8
|
-
|
|
9
|
-
const browser = await ensureBrowserReady();
|
|
10
|
-
try {
|
|
11
|
-
const page = await ensureTikTokPage(browser, url);
|
|
12
|
-
await page.goto(url, { waitUntil: 'load', timeout: 30000 });
|
|
13
|
-
console.error('等待页面加载...');
|
|
14
|
-
await delay(3000, 5000);
|
|
15
|
-
|
|
16
|
-
console.error('开始提取关注和粉丝...\n');
|
|
17
|
-
const result = await extractFollowAndFollowers(page, {
|
|
18
|
-
log: console.error,
|
|
19
|
-
});
|
|
20
|
-
|
|
21
|
-
console.error('\n--- 提取完成 ---');
|
|
22
|
-
console.error(`关注: ${result.following.length} 人`);
|
|
23
|
-
console.error(`粉丝: ${result.followers.length} 人`);
|
|
24
|
-
|
|
25
|
-
const outDir = path.join(__dirname, '..', 'results');
|
|
26
|
-
const fs = require('fs');
|
|
27
|
-
fs.mkdirSync(outDir, { recursive: true });
|
|
28
|
-
const outPath = path.join(outDir, 'follow-result.json');
|
|
29
|
-
fs.writeFileSync(outPath, JSON.stringify(result, null, 2));
|
|
30
|
-
console.error(`已保存到 ${outPath}`);
|
|
31
|
-
|
|
32
|
-
console.log(JSON.stringify(result, null, 2));
|
|
33
|
-
} finally {
|
|
34
|
-
await browser.close().catch(() => {});
|
|
35
|
-
}
|
|
36
|
-
}
|
|
37
|
-
|
|
38
|
-
main().catch(err => {
|
|
39
|
-
console.error('错误:', err.message);
|
|
40
|
-
process.exit(1);
|
|
41
|
-
});
|