tt-help-cli-ycl 1.3.1 → 1.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -17
- package/cli.js +9 -9
- package/package.json +44 -44
- package/src/cli/auto.js +1 -1
- package/src/cli/explore.js +1 -1
- package/src/cli/progress.js +111 -111
- package/src/cli/scrape.js +47 -47
- package/src/cli/utils.js +18 -18
- package/src/cli/videos.js +41 -41
- package/src/cli/watch.js +28 -28
- package/src/lib/args.js +385 -377
- package/src/lib/browser/anti-detect.js +23 -23
- package/src/lib/browser/cdp.js +142 -142
- package/src/lib/browser/launch.js +43 -43
- package/src/lib/browser/page.js +80 -62
- package/src/lib/constants.js +84 -85
- package/src/lib/delay.js +54 -54
- package/src/lib/{explore.js → explore-fetch.js} +118 -118
- package/src/lib/fetcher.js +45 -45
- package/src/lib/filter.js +66 -66
- package/src/lib/io.js +54 -54
- package/src/lib/output.js +80 -80
- package/src/lib/parser.js +47 -47
- package/src/lib/retry.js +44 -44
- package/src/lib/scrape.js +40 -40
- package/src/lib/url.js +52 -52
- package/src/main.mjs +200 -200
- package/src/results/user-videos-bar.lar.lar.moeta.json +37 -0
- package/src/{auto-core.mjs → scraper/auto-core.mjs} +183 -174
- package/src/scraper/core.mjs +188 -182
- package/src/{explore-core.mjs → scraper/explore-core.mjs} +159 -148
- package/src/scraper/modules/captcha-handler.mjs +114 -0
- package/src/scraper/modules/comment-extractor.mjs +69 -57
- package/src/scraper/modules/follow-extractor.mjs +121 -121
- package/src/scraper/modules/guess-extractor.mjs +51 -51
- package/src/scraper/modules/page-error-detector.mjs +70 -68
- package/src/scraper/modules/page-helpers.mjs +46 -44
- package/src/scraper/modules/scroll-collector.mjs +189 -189
- package/src/{get-user-videos-core.mjs → videos/core.mjs} +126 -126
- package/src/{data-store.mjs → watch/data-store.mjs} +29 -3
- package/src/watch/public/index.html +444 -344
- package/src/watch/server.mjs +24 -1
- package/src/lib/auto-browser.mjs +0 -6
- package/src/lib/get-user-videos-browser.mjs +0 -1
- package/src/lib/scrape-browser.mjs +0 -1
- package/src/test-auto-follow.cjs +0 -109
- package/src/test-extractors.cjs +0 -75
- package/src/test-follow.cjs +0 -41
package/src/watch/server.mjs
CHANGED
|
@@ -3,7 +3,7 @@ import { readFileSync } from 'fs';
|
|
|
3
3
|
import { join, dirname } from 'path';
|
|
4
4
|
import { fileURLToPath } from 'url';
|
|
5
5
|
import { spawn } from 'child_process';
|
|
6
|
-
import { createStore } from '
|
|
6
|
+
import { createStore } from './data-store.mjs';
|
|
7
7
|
|
|
8
8
|
const __filename = fileURLToPath(import.meta.url);
|
|
9
9
|
const __dirname = dirname(__filename);
|
|
@@ -151,6 +151,26 @@ export function startWatchServer(outputFile, port = 3000) {
|
|
|
151
151
|
return;
|
|
152
152
|
}
|
|
153
153
|
|
|
154
|
+
const jobResetMatch = routePath.match(/^\/api\/job\/([^/]+)\/reset$/);
|
|
155
|
+
if (req.method === 'POST' && jobResetMatch) {
|
|
156
|
+
const uniqueId = jobResetMatch[1];
|
|
157
|
+
const ret = store.resetJob(uniqueId);
|
|
158
|
+
if (ret.saved) {
|
|
159
|
+
sendJSON(res, 200, ret);
|
|
160
|
+
} else {
|
|
161
|
+
sendJSON(res, 404, ret);
|
|
162
|
+
}
|
|
163
|
+
return;
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
const jobPinMatch = routePath.match(/^\/api\/job\/([^/]+)\/pin$/);
|
|
167
|
+
if (req.method === 'POST' && jobPinMatch) {
|
|
168
|
+
const uniqueId = jobPinMatch[1];
|
|
169
|
+
const ret = store.togglePin(uniqueId);
|
|
170
|
+
sendJSON(res, ret.saved ? 200 : 404, ret);
|
|
171
|
+
return;
|
|
172
|
+
}
|
|
173
|
+
|
|
154
174
|
if (req.method === 'GET' && routePath === '/api/stats') {
|
|
155
175
|
const stats = computeStats(store.getAllUsers());
|
|
156
176
|
sendJSON(res, 200, stats);
|
|
@@ -187,6 +207,9 @@ export function startWatchServer(outputFile, port = 3000) {
|
|
|
187
207
|
|
|
188
208
|
const statusOrder = { processing: 0, pending: 1, done: 2, error: 3, restricted: 4 };
|
|
189
209
|
filtered.sort((a, b) => {
|
|
210
|
+
// 置顶优先
|
|
211
|
+
if (a.pinned && !b.pinned) return -1;
|
|
212
|
+
if (!a.pinned && b.pinned) return 1;
|
|
190
213
|
const sa = statusOrder[a.status] ?? 9;
|
|
191
214
|
const sb = statusOrder[b.status] ?? 9;
|
|
192
215
|
if (sa !== sb) return sa - sb;
|
package/src/lib/auto-browser.mjs
DELETED
|
@@ -1,6 +0,0 @@
|
|
|
1
|
-
import { processUser } from '../auto-core.mjs';
|
|
2
|
-
import { processExplore } from '../explore-core.mjs';
|
|
3
|
-
import { ensureBrowserReady } from '../scraper/modules/page-helpers.mjs';
|
|
4
|
-
import { ensureTikTokPage, closeCommentPanel } from '../scraper/modules/page-helpers.mjs';
|
|
5
|
-
|
|
6
|
-
export { processUser, processExplore, ensureBrowserReady, ensureTikTokPage, closeCommentPanel };
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export { runGetUserVideos } from '../get-user-videos-core.mjs';
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export { runScrape } from '../scraper/core.mjs';
|
package/src/test-auto-follow.cjs
DELETED
|
@@ -1,109 +0,0 @@
|
|
|
1
|
-
const path = require('path');
|
|
2
|
-
const fs = require('fs');
|
|
3
|
-
const { ensureBrowserReady, setDelayConfig } = require('./scraper/modules/page-helpers.cjs');
|
|
4
|
-
const { processUser } = require('./auto-core.cjs');
|
|
5
|
-
const { createStore } = require('./data-store.cjs');
|
|
6
|
-
|
|
7
|
-
async function main() {
|
|
8
|
-
const outFile = path.join(__dirname, '..', 'results', 'auto-test.json');
|
|
9
|
-
const store = createStore(outFile);
|
|
10
|
-
|
|
11
|
-
setDelayConfig('fast');
|
|
12
|
-
|
|
13
|
-
const browser = await ensureBrowserReady();
|
|
14
|
-
let page;
|
|
15
|
-
try {
|
|
16
|
-
const contexts = browser.contexts();
|
|
17
|
-
page = null;
|
|
18
|
-
for (const ctx of contexts) {
|
|
19
|
-
for (const p of ctx.pages()) {
|
|
20
|
-
if (p.url().includes('tiktok.com')) { page = p; break; }
|
|
21
|
-
}
|
|
22
|
-
if (page) break;
|
|
23
|
-
}
|
|
24
|
-
if (!page) page = await contexts[0].newPage();
|
|
25
|
-
|
|
26
|
-
console.error('========== 测试 processUser + enableFollow ==========');
|
|
27
|
-
console.error('用户: @qiqi23280\n');
|
|
28
|
-
|
|
29
|
-
const result = await processUser(page, 'qiqi23280', {
|
|
30
|
-
collectMax: 1,
|
|
31
|
-
scrapeDepth: 1,
|
|
32
|
-
maxComments: 10,
|
|
33
|
-
maxGuess: 5,
|
|
34
|
-
preset: 'fast',
|
|
35
|
-
enableFollow: true,
|
|
36
|
-
maxFollowing: 50,
|
|
37
|
-
maxFollowers: 50,
|
|
38
|
-
browser,
|
|
39
|
-
}, console.error);
|
|
40
|
-
|
|
41
|
-
console.error('\n========== 结果验证 ==========');
|
|
42
|
-
let allPassed = true;
|
|
43
|
-
|
|
44
|
-
const checks = [
|
|
45
|
-
{ label: '用户信息', ok: result.userInfo && result.userInfo.uniqueId, detail: result.userInfo?.uniqueId },
|
|
46
|
-
{ label: '关注列表', ok: Array.isArray(result.discoveredFollowing) && result.discoveredFollowing.length > 0, detail: `${result.discoveredFollowing?.length || 0} 人` },
|
|
47
|
-
{ label: '粉丝列表', ok: Array.isArray(result.discoveredFollowers) && result.discoveredFollowers.length > 0, detail: `${result.discoveredFollowers?.length || 0} 人` },
|
|
48
|
-
{ label: '关注格式', ok: result.discoveredFollowing?.every(p => Array.isArray(p) && p.length === 2 && p[0].startsWith('@')), detail: null },
|
|
49
|
-
{ label: '粉丝格式', ok: result.discoveredFollowers?.every(p => Array.isArray(p) && p.length === 2 && p[0].startsWith('@')), detail: null },
|
|
50
|
-
{ label: '无错误', ok: !result.error, detail: result.error },
|
|
51
|
-
];
|
|
52
|
-
|
|
53
|
-
for (const c of checks) {
|
|
54
|
-
const status = c.ok ? 'PASS' : 'FAIL';
|
|
55
|
-
const detailStr = c.detail !== null ? ` (${c.detail})` : '';
|
|
56
|
-
console.error(` ${status}: ${c.label}${detailStr}`);
|
|
57
|
-
if (!c.ok) allPassed = false;
|
|
58
|
-
}
|
|
59
|
-
|
|
60
|
-
// 模拟入队逻辑
|
|
61
|
-
const queue = ['qiqi23280'];
|
|
62
|
-
const followingIds = (result.discoveredFollowing || []).map(([h]) => h.replace(/^@/, ''));
|
|
63
|
-
const followerIds = (result.discoveredFollowers || []).map(([h]) => h.replace(/^@/, ''));
|
|
64
|
-
|
|
65
|
-
for (const uid of followingIds) queue.push(uid);
|
|
66
|
-
for (const uid of followerIds) queue.push(uid);
|
|
67
|
-
const uniqueQueue = [...new Set(queue)];
|
|
68
|
-
|
|
69
|
-
console.error(`\n 队列长度: ${uniqueQueue.length}(关注 ${followingIds.length} + 粉丝 ${followerIds.length} + 种子 1)`);
|
|
70
|
-
|
|
71
|
-
// 写入 store 验证
|
|
72
|
-
store.addUser({
|
|
73
|
-
uniqueId: 'qiqi23280',
|
|
74
|
-
...result.userInfo,
|
|
75
|
-
sources: ['seed'],
|
|
76
|
-
});
|
|
77
|
-
for (const [handle, name] of (result.discoveredFollowing || [])) {
|
|
78
|
-
store.addUser({ uniqueId: handle.replace(/^@/, ''), nickname: name, sources: ['following'] });
|
|
79
|
-
}
|
|
80
|
-
for (const [handle, name] of (result.discoveredFollowers || [])) {
|
|
81
|
-
store.addUser({ uniqueId: handle.replace(/^@/, ''), nickname: name, sources: ['follower'] });
|
|
82
|
-
}
|
|
83
|
-
store.save();
|
|
84
|
-
|
|
85
|
-
const allUsers = store.getAllUsers();
|
|
86
|
-
console.error(` Store 用户数: ${allUsers.length}`);
|
|
87
|
-
|
|
88
|
-
// 验证 source 标记
|
|
89
|
-
const followingUsers = allUsers.filter(u => u.sources?.includes('following'));
|
|
90
|
-
const followerUsers = allUsers.filter(u => u.sources?.includes('follower'));
|
|
91
|
-
console.error(` 关注来源: ${followingUsers.length} | 粉丝来源: ${followerUsers.length}`);
|
|
92
|
-
|
|
93
|
-
if (followingUsers.length === 0 || followerUsers.length === 0) {
|
|
94
|
-
console.error(' FAIL: 缺少 following 或 follower 来源标记');
|
|
95
|
-
allPassed = false;
|
|
96
|
-
} else {
|
|
97
|
-
console.error(' PASS: 来源标记正确');
|
|
98
|
-
}
|
|
99
|
-
|
|
100
|
-
console.error(`\n${allPassed ? 'ALL PASSED' : 'SOME FAILED'}`);
|
|
101
|
-
console.error(`数据保存到: ${outFile}`);
|
|
102
|
-
process.exit(allPassed ? 0 : 1);
|
|
103
|
-
|
|
104
|
-
} finally {
|
|
105
|
-
await browser.close().catch(() => {});
|
|
106
|
-
}
|
|
107
|
-
}
|
|
108
|
-
|
|
109
|
-
main().catch(err => { console.error('FATAL:', err.message); process.exit(1); });
|
package/src/test-extractors.cjs
DELETED
|
@@ -1,75 +0,0 @@
|
|
|
1
|
-
const { ensureBrowserReady, delay, setDelayConfig } = require('./scraper/modules/page-helpers.cjs');
|
|
2
|
-
const { extractCommentAuthors } = require('./scraper/modules/comment-extractor.cjs');
|
|
3
|
-
const { extractGuessVideos } = require('./scraper/modules/guess-extractor.cjs');
|
|
4
|
-
|
|
5
|
-
async function main() {
|
|
6
|
-
setDelayConfig('fast');
|
|
7
|
-
|
|
8
|
-
const videoUrl = process.argv[2] || 'https://www.tiktok.com/@porfirio.fructuoso/video/7615853535955111198';
|
|
9
|
-
console.error(`目标: ${videoUrl}`);
|
|
10
|
-
|
|
11
|
-
const browser = await ensureBrowserReady();
|
|
12
|
-
let page;
|
|
13
|
-
try {
|
|
14
|
-
const contexts = browser.contexts();
|
|
15
|
-
page = null;
|
|
16
|
-
for (const ctx of contexts) {
|
|
17
|
-
for (const p of ctx.pages()) {
|
|
18
|
-
if (p.url().includes('tiktok.com')) { page = p; break; }
|
|
19
|
-
}
|
|
20
|
-
if (page) break;
|
|
21
|
-
}
|
|
22
|
-
if (!page) {
|
|
23
|
-
page = await contexts[0].newPage();
|
|
24
|
-
}
|
|
25
|
-
|
|
26
|
-
await page.goto(videoUrl, { waitUntil: 'networkidle', timeout: 60000 });
|
|
27
|
-
await delay(5000, 8000);
|
|
28
|
-
|
|
29
|
-
console.error(`当前URL: ${page.url()}`);
|
|
30
|
-
|
|
31
|
-
let allPassed = true;
|
|
32
|
-
|
|
33
|
-
// ========== 评论提取 ==========
|
|
34
|
-
console.error('\n--- 评论提取 (max=30) ---');
|
|
35
|
-
const t1 = Date.now();
|
|
36
|
-
let commentUsers = [];
|
|
37
|
-
try { commentUsers = await extractCommentAuthors(page, 30); }
|
|
38
|
-
catch (e) { console.error(` 异常: ${e.message}`); }
|
|
39
|
-
console.error(` 耗时: ${((Date.now()-t1)/1000).toFixed(1)}s, 结果: ${commentUsers.length} 个`);
|
|
40
|
-
|
|
41
|
-
if (commentUsers.length > 0) {
|
|
42
|
-
const s = new Set(commentUsers);
|
|
43
|
-
const ok = s.size === commentUsers.length;
|
|
44
|
-
console.error(` ${ok ? 'PASS' : 'FAIL'}: 唯一${s.size}/总数${commentUsers.length}`);
|
|
45
|
-
if (!ok) allPassed = false;
|
|
46
|
-
}
|
|
47
|
-
|
|
48
|
-
// ========== 猜你喜欢提取 ==========
|
|
49
|
-
console.error('\n--- 猜你喜欢提取 (max=20) ---');
|
|
50
|
-
const t2 = Date.now();
|
|
51
|
-
let guessVideos = [];
|
|
52
|
-
try { guessVideos = await extractGuessVideos(page, 20); }
|
|
53
|
-
catch (e) { console.error(` 异常: ${e.message}`); }
|
|
54
|
-
console.error(` 耗时: ${((Date.now()-t2)/1000).toFixed(1)}s, 结果: ${guessVideos.length} 个`);
|
|
55
|
-
|
|
56
|
-
if (guessVideos.length > 0) {
|
|
57
|
-
const ids = guessVideos.map(v => v.videoId);
|
|
58
|
-
const s = new Set(ids);
|
|
59
|
-
const ok = s.size === ids.length;
|
|
60
|
-
console.error(` ${ok ? 'PASS' : 'FAIL'}: 唯一${s.size}/总数${ids.length}`);
|
|
61
|
-
if (!ok) allPassed = false;
|
|
62
|
-
const ok2 = guessVideos.every(v => v.author && v.videoId && v.url);
|
|
63
|
-
console.error(` ${ok2 ? 'PASS' : 'FAIL'}: 结构完整`);
|
|
64
|
-
if (!ok2) allPassed = false;
|
|
65
|
-
}
|
|
66
|
-
|
|
67
|
-
console.error(`\n${allPassed ? 'ALL PASSED' : 'SOME FAILED'}`);
|
|
68
|
-
process.exit(allPassed ? 0 : 1);
|
|
69
|
-
|
|
70
|
-
} finally {
|
|
71
|
-
await browser.close().catch(() => {});
|
|
72
|
-
}
|
|
73
|
-
}
|
|
74
|
-
|
|
75
|
-
main().catch(err => { console.error('FATAL:', err.message); process.exit(1); });
|
package/src/test-follow.cjs
DELETED
|
@@ -1,41 +0,0 @@
|
|
|
1
|
-
const path = require('path');
|
|
2
|
-
const { ensureBrowserReady, ensureTikTokPage, delay } = require('./scraper/modules/page-helpers.cjs');
|
|
3
|
-
const { extractFollowAndFollowers } = require('./scraper/modules/follow-extractor.cjs');
|
|
4
|
-
|
|
5
|
-
async function main() {
|
|
6
|
-
const url = process.argv[2] || 'https://www.tiktok.com/@qiqi23280';
|
|
7
|
-
console.error(`目标: ${url}`);
|
|
8
|
-
|
|
9
|
-
const browser = await ensureBrowserReady();
|
|
10
|
-
try {
|
|
11
|
-
const page = await ensureTikTokPage(browser, url);
|
|
12
|
-
await page.goto(url, { waitUntil: 'load', timeout: 30000 });
|
|
13
|
-
console.error('等待页面加载...');
|
|
14
|
-
await delay(3000, 5000);
|
|
15
|
-
|
|
16
|
-
console.error('开始提取关注和粉丝...\n');
|
|
17
|
-
const result = await extractFollowAndFollowers(page, {
|
|
18
|
-
log: console.error,
|
|
19
|
-
});
|
|
20
|
-
|
|
21
|
-
console.error('\n--- 提取完成 ---');
|
|
22
|
-
console.error(`关注: ${result.following.length} 人`);
|
|
23
|
-
console.error(`粉丝: ${result.followers.length} 人`);
|
|
24
|
-
|
|
25
|
-
const outDir = path.join(__dirname, '..', 'results');
|
|
26
|
-
const fs = require('fs');
|
|
27
|
-
fs.mkdirSync(outDir, { recursive: true });
|
|
28
|
-
const outPath = path.join(outDir, 'follow-result.json');
|
|
29
|
-
fs.writeFileSync(outPath, JSON.stringify(result, null, 2));
|
|
30
|
-
console.error(`已保存到 ${outPath}`);
|
|
31
|
-
|
|
32
|
-
console.log(JSON.stringify(result, null, 2));
|
|
33
|
-
} finally {
|
|
34
|
-
await browser.close().catch(() => {});
|
|
35
|
-
}
|
|
36
|
-
}
|
|
37
|
-
|
|
38
|
-
main().catch(err => {
|
|
39
|
-
console.error('错误:', err.message);
|
|
40
|
-
process.exit(1);
|
|
41
|
-
});
|