tt-help-cli-ycl 1.3.24 → 1.3.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -17
- package/cli.js +9 -9
- package/package.json +47 -47
- package/scripts/run-explore copy.bat +68 -68
- package/scripts/run-explore.bat +68 -68
- package/scripts/run-explore.ps1 +81 -81
- package/scripts/run-explore.sh +73 -73
- package/scripts/test-captcha-lib.mjs +68 -0
- package/scripts/test-captcha.mjs +81 -0
- package/scripts/test-incognito-lib.mjs +36 -0
- package/scripts/test-login-state.mjs +128 -0
- package/scripts/test-safe-click.mjs +45 -0
- package/src/cli/attach.js +180 -180
- package/src/cli/auto.js +240 -186
- package/src/cli/config.js +152 -152
- package/src/cli/explore.js +439 -253
- package/src/cli/info.js +88 -88
- package/src/cli/open.js +102 -0
- package/src/cli/progress.js +111 -111
- package/src/cli/refresh.js +216 -216
- package/src/cli/scrape.js +47 -47
- package/src/cli/utils.js +18 -18
- package/src/cli/videos.js +41 -41
- package/src/cli/watch.js +31 -31
- package/src/lib/api-interceptor.js +191 -191
- package/src/lib/args.js +551 -517
- package/src/lib/browser/anti-detect.js +23 -23
- package/src/lib/browser/cdp.js +237 -194
- package/src/lib/browser/health-checker.js +93 -0
- package/src/lib/browser/launch.js +43 -43
- package/src/lib/browser/page.js +156 -156
- package/src/lib/constants.js +206 -199
- package/src/lib/delay.js +54 -54
- package/src/lib/explore-fetch.js +118 -118
- package/src/lib/fetcher.js +45 -45
- package/src/lib/filter.js +66 -66
- package/src/lib/io.js +54 -54
- package/src/lib/output.js +80 -80
- package/src/lib/page-error-detector.js +71 -71
- package/src/lib/parse-ssr.mjs +69 -69
- package/src/lib/parser.js +47 -47
- package/src/lib/retry.js +45 -45
- package/src/lib/scrape.js +89 -89
- package/src/lib/tiktok-scraper.mjs +194 -194
- package/src/lib/url.js +52 -52
- package/src/main.js +44 -42
- package/src/results/user-videos-bar.lar.lar.moeta.json +37 -0
- package/src/scraper/auto-core.js +203 -203
- package/src/scraper/core.js +211 -211
- package/src/scraper/explore-core.js +201 -189
- package/src/scraper/modules/captcha-handler.js +114 -114
- package/src/scraper/modules/comment-extractor.js +74 -74
- package/src/scraper/modules/follow-extractor.js +118 -118
- package/src/scraper/modules/guess-extractor.js +51 -51
- package/src/scraper/modules/page-helpers.js +48 -48
- package/src/scraper/refresh-core.js +179 -179
- package/src/videos/core.js +101 -101
- package/src/watch/data-store.js +828 -828
- package/src/watch/public/index.html +753 -753
- package/src/watch/server.js +705 -705
package/src/cli/info.js
CHANGED
|
@@ -1,88 +1,88 @@
|
|
|
1
|
-
import { TikTokScraper } from '../lib/tiktok-scraper.mjs';
|
|
2
|
-
import { isProfileUrl, isVideoUrl, extractUniqueId, normalizeUsername } from '../lib/url.js';
|
|
3
|
-
|
|
4
|
-
async function handleInfo(options) {
|
|
5
|
-
const { infoUrls, infoOnlyVideo } = options;
|
|
6
|
-
|
|
7
|
-
if (!infoUrls || infoUrls.length === 0) {
|
|
8
|
-
console.error('用法: tt-help info <URL> [URL2 URL3...] [--onlyvideo]');
|
|
9
|
-
console.error('');
|
|
10
|
-
console.error('参数:');
|
|
11
|
-
console.error(' <URL> TikTok 主页或视频 URL,支持多个 URL 同时查询');
|
|
12
|
-
console.error(' --onlyvideo 只返回视频信息(不返回用户信息)');
|
|
13
|
-
console.error('');
|
|
14
|
-
console.error('默认行为:');
|
|
15
|
-
console.error(' 主页 URL → 返回用户信息(bio、region、粉丝数等)');
|
|
16
|
-
console.error(' 视频 URL → 返回用户信息 + 视频信息');
|
|
17
|
-
console.error(' 视频 URL + --onlyvideo → 只返回视频信息');
|
|
18
|
-
console.error('');
|
|
19
|
-
console.error('示例:');
|
|
20
|
-
console.error(' tt-help info https://www.tiktok.com/@nike');
|
|
21
|
-
console.error(' tt-help info https://www.tiktok.com/@nike/video/7234567890');
|
|
22
|
-
console.error(' tt-help info https://www.tiktok.com/@nike https://www.tiktok.com/@apple');
|
|
23
|
-
process.exit(1);
|
|
24
|
-
}
|
|
25
|
-
|
|
26
|
-
const scraper = new TikTokScraper({ poolSize: 1 });
|
|
27
|
-
|
|
28
|
-
try {
|
|
29
|
-
await scraper.init();
|
|
30
|
-
const result = {};
|
|
31
|
-
|
|
32
|
-
for (const url of infoUrls) {
|
|
33
|
-
if (isProfileUrl(url)) {
|
|
34
|
-
const uniqueId = extractUniqueId(url);
|
|
35
|
-
const normalized = normalizeUsername(uniqueId);
|
|
36
|
-
const user = await scraper.getUserInfo(normalized);
|
|
37
|
-
if (!user) {
|
|
38
|
-
console.error(`无法获取用户 @${uniqueId} 的信息`);
|
|
39
|
-
continue;
|
|
40
|
-
}
|
|
41
|
-
result[normalized] = { user };
|
|
42
|
-
console.error(`用户: @${user.uniqueId} (${user.nickname})`);
|
|
43
|
-
} else if (isVideoUrl(url)) {
|
|
44
|
-
const uniqueId = extractUniqueId(url);
|
|
45
|
-
const normalized = normalizeUsername(uniqueId);
|
|
46
|
-
|
|
47
|
-
if (infoOnlyVideo) {
|
|
48
|
-
const video = await scraper.getVideoInfo(url);
|
|
49
|
-
if (!video) {
|
|
50
|
-
console.error(`无法获取视频信息: ${url}`);
|
|
51
|
-
continue;
|
|
52
|
-
}
|
|
53
|
-
const key = normalized + '/video/' + video.id;
|
|
54
|
-
result[key] = { video };
|
|
55
|
-
console.error(`视频: ${video.id}`);
|
|
56
|
-
} else {
|
|
57
|
-
const [user, video] = await Promise.all([
|
|
58
|
-
scraper.getUserInfo(normalized),
|
|
59
|
-
scraper.getVideoInfo(url),
|
|
60
|
-
]);
|
|
61
|
-
const entry = {};
|
|
62
|
-
if (user) {
|
|
63
|
-
entry.user = user;
|
|
64
|
-
console.error(`用户: @${user.uniqueId} (${user.nickname})`);
|
|
65
|
-
}
|
|
66
|
-
if (video) {
|
|
67
|
-
entry.video = video;
|
|
68
|
-
console.error(`视频: ${video.id}`);
|
|
69
|
-
}
|
|
70
|
-
if (!user && !video) {
|
|
71
|
-
console.error(`无法获取信息: ${url}`);
|
|
72
|
-
continue;
|
|
73
|
-
}
|
|
74
|
-
const key = normalized + '/video/' + (video ? video.id : 'unknown');
|
|
75
|
-
result[key] = entry;
|
|
76
|
-
}
|
|
77
|
-
} else {
|
|
78
|
-
console.error(`无法识别 URL: ${url}`);
|
|
79
|
-
}
|
|
80
|
-
}
|
|
81
|
-
|
|
82
|
-
console.log(JSON.stringify(result, null, 2));
|
|
83
|
-
} finally {
|
|
84
|
-
await scraper.close();
|
|
85
|
-
}
|
|
86
|
-
}
|
|
87
|
-
|
|
88
|
-
export { handleInfo };
|
|
1
|
+
import { TikTokScraper } from '../lib/tiktok-scraper.mjs';
|
|
2
|
+
import { isProfileUrl, isVideoUrl, extractUniqueId, normalizeUsername } from '../lib/url.js';
|
|
3
|
+
|
|
4
|
+
async function handleInfo(options) {
|
|
5
|
+
const { infoUrls, infoOnlyVideo } = options;
|
|
6
|
+
|
|
7
|
+
if (!infoUrls || infoUrls.length === 0) {
|
|
8
|
+
console.error('用法: tt-help info <URL> [URL2 URL3...] [--onlyvideo]');
|
|
9
|
+
console.error('');
|
|
10
|
+
console.error('参数:');
|
|
11
|
+
console.error(' <URL> TikTok 主页或视频 URL,支持多个 URL 同时查询');
|
|
12
|
+
console.error(' --onlyvideo 只返回视频信息(不返回用户信息)');
|
|
13
|
+
console.error('');
|
|
14
|
+
console.error('默认行为:');
|
|
15
|
+
console.error(' 主页 URL → 返回用户信息(bio、region、粉丝数等)');
|
|
16
|
+
console.error(' 视频 URL → 返回用户信息 + 视频信息');
|
|
17
|
+
console.error(' 视频 URL + --onlyvideo → 只返回视频信息');
|
|
18
|
+
console.error('');
|
|
19
|
+
console.error('示例:');
|
|
20
|
+
console.error(' tt-help info https://www.tiktok.com/@nike');
|
|
21
|
+
console.error(' tt-help info https://www.tiktok.com/@nike/video/7234567890');
|
|
22
|
+
console.error(' tt-help info https://www.tiktok.com/@nike https://www.tiktok.com/@apple');
|
|
23
|
+
process.exit(1);
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
const scraper = new TikTokScraper({ poolSize: 1 });
|
|
27
|
+
|
|
28
|
+
try {
|
|
29
|
+
await scraper.init();
|
|
30
|
+
const result = {};
|
|
31
|
+
|
|
32
|
+
for (const url of infoUrls) {
|
|
33
|
+
if (isProfileUrl(url)) {
|
|
34
|
+
const uniqueId = extractUniqueId(url);
|
|
35
|
+
const normalized = normalizeUsername(uniqueId);
|
|
36
|
+
const user = await scraper.getUserInfo(normalized);
|
|
37
|
+
if (!user) {
|
|
38
|
+
console.error(`无法获取用户 @${uniqueId} 的信息`);
|
|
39
|
+
continue;
|
|
40
|
+
}
|
|
41
|
+
result[normalized] = { user };
|
|
42
|
+
console.error(`用户: @${user.uniqueId} (${user.nickname})`);
|
|
43
|
+
} else if (isVideoUrl(url)) {
|
|
44
|
+
const uniqueId = extractUniqueId(url);
|
|
45
|
+
const normalized = normalizeUsername(uniqueId);
|
|
46
|
+
|
|
47
|
+
if (infoOnlyVideo) {
|
|
48
|
+
const video = await scraper.getVideoInfo(url);
|
|
49
|
+
if (!video) {
|
|
50
|
+
console.error(`无法获取视频信息: ${url}`);
|
|
51
|
+
continue;
|
|
52
|
+
}
|
|
53
|
+
const key = normalized + '/video/' + video.id;
|
|
54
|
+
result[key] = { video };
|
|
55
|
+
console.error(`视频: ${video.id}`);
|
|
56
|
+
} else {
|
|
57
|
+
const [user, video] = await Promise.all([
|
|
58
|
+
scraper.getUserInfo(normalized),
|
|
59
|
+
scraper.getVideoInfo(url),
|
|
60
|
+
]);
|
|
61
|
+
const entry = {};
|
|
62
|
+
if (user) {
|
|
63
|
+
entry.user = user;
|
|
64
|
+
console.error(`用户: @${user.uniqueId} (${user.nickname})`);
|
|
65
|
+
}
|
|
66
|
+
if (video) {
|
|
67
|
+
entry.video = video;
|
|
68
|
+
console.error(`视频: ${video.id}`);
|
|
69
|
+
}
|
|
70
|
+
if (!user && !video) {
|
|
71
|
+
console.error(`无法获取信息: ${url}`);
|
|
72
|
+
continue;
|
|
73
|
+
}
|
|
74
|
+
const key = normalized + '/video/' + (video ? video.id : 'unknown');
|
|
75
|
+
result[key] = entry;
|
|
76
|
+
}
|
|
77
|
+
} else {
|
|
78
|
+
console.error(`无法识别 URL: ${url}`);
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
console.log(JSON.stringify(result, null, 2));
|
|
83
|
+
} finally {
|
|
84
|
+
await scraper.close();
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
export { handleInfo };
|
package/src/cli/open.js
ADDED
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
import { ensureBrowserReady, killEdgeProcesses } from '../lib/browser/cdp.js';
|
|
2
|
+
import path from 'path';
|
|
3
|
+
import os from 'os';
|
|
4
|
+
|
|
5
|
+
const BASE_PORT = 9222;
|
|
6
|
+
const TOTAL_ACCOUNTS = 10;
|
|
7
|
+
|
|
8
|
+
export async function handleOpen(parsed) {
|
|
9
|
+
const { openPort, openList } = parsed;
|
|
10
|
+
|
|
11
|
+
if (openList) {
|
|
12
|
+
console.error('内置浏览器配置:');
|
|
13
|
+
for (let i = 0; i < TOTAL_ACCOUNTS; i++) {
|
|
14
|
+
const port = BASE_PORT + i;
|
|
15
|
+
const profile = `p${port}`;
|
|
16
|
+
const userDataDir = path.join(os.homedir(), 'Library', 'Application Support', `Microsoft Edge For Testing_${profile}`);
|
|
17
|
+
console.error(` open ${port} → profile: ${profile}, userDataDir: ${userDataDir}`);
|
|
18
|
+
}
|
|
19
|
+
console.error('');
|
|
20
|
+
console.error('用法: tt-help open <端口>');
|
|
21
|
+
console.error('示例: tt-help open 9222');
|
|
22
|
+
process.exit(0);
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
if (!openPort) {
|
|
26
|
+
console.error('用法: tt-help open <端口>');
|
|
27
|
+
console.error('示例: tt-help open 9222');
|
|
28
|
+
console.error('');
|
|
29
|
+
console.error('可用端口: 9222 - 9231 (共 10 个)');
|
|
30
|
+
console.error('运行 "tt-help open --list" 查看所有配置');
|
|
31
|
+
process.exit(1);
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
const port = parseInt(openPort);
|
|
35
|
+
if (isNaN(port) || port < BASE_PORT || port >= BASE_PORT + TOTAL_ACCOUNTS) {
|
|
36
|
+
console.error(`端口 ${openPort} 不在有效范围内 (9222 - 9231)`);
|
|
37
|
+
process.exit(1);
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
const profile = `p${port}`;
|
|
41
|
+
const userDataDir = path.join(os.homedir(), 'Library', 'Application Support', `Microsoft Edge For Testing_${profile}`);
|
|
42
|
+
|
|
43
|
+
console.error(`正在启动浏览器... 端口: ${port}, profile: ${profile}`);
|
|
44
|
+
console.error(`userDataDir: ${userDataDir}`);
|
|
45
|
+
console.error('');
|
|
46
|
+
console.error('启动后请在浏览器中登录 TikTok 账户');
|
|
47
|
+
console.error('登录完成后关闭浏览器即可,下次 open 会保留登录状态');
|
|
48
|
+
console.error('');
|
|
49
|
+
console.error('按 Ctrl+C 退出并关闭浏览器');
|
|
50
|
+
|
|
51
|
+
let browser = null;
|
|
52
|
+
|
|
53
|
+
try {
|
|
54
|
+
const cdpOptions = {};
|
|
55
|
+
cdpOptions.port = port;
|
|
56
|
+
cdpOptions.userDataDir = userDataDir;
|
|
57
|
+
|
|
58
|
+
browser = await ensureBrowserReady(cdpOptions);
|
|
59
|
+
|
|
60
|
+
// Ctrl+C 关闭浏览器
|
|
61
|
+
process.on('SIGINT', async () => {
|
|
62
|
+
console.error('\n正在关闭浏览器...');
|
|
63
|
+
try {
|
|
64
|
+
await killEdgeProcesses(userDataDir);
|
|
65
|
+
await browser.close();
|
|
66
|
+
} catch { /* ignore */ }
|
|
67
|
+
console.error('浏览器已关闭');
|
|
68
|
+
process.exit(0);
|
|
69
|
+
});
|
|
70
|
+
|
|
71
|
+
console.error('');
|
|
72
|
+
console.error('浏览器已启动并连接成功');
|
|
73
|
+
console.error('TikTok 页面将在浏览器中自动打开...');
|
|
74
|
+
|
|
75
|
+
// 打开 TikTok 首页
|
|
76
|
+
try {
|
|
77
|
+
const pages = await browser.pages();
|
|
78
|
+
let page = pages.find(p => !p.url().startsWith('devtools'));
|
|
79
|
+
if (!page) {
|
|
80
|
+
const context = await browser.newBrowserCDPSession();
|
|
81
|
+
await context.send('Target.createTarget', { url: '' });
|
|
82
|
+
await new Promise(r => setTimeout(r, 1000));
|
|
83
|
+
const newPages = await browser.pages();
|
|
84
|
+
page = newPages.find(p => !p.url().startsWith('devtools'));
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
if (page) {
|
|
88
|
+
await page.goto('https://www.tiktok.com', { waitUntil: 'domcontentloaded', timeout: 30000 });
|
|
89
|
+
console.error('已打开 https://www.tiktok.com');
|
|
90
|
+
}
|
|
91
|
+
} catch (e) {
|
|
92
|
+
console.error(`打开页面时出错: ${e.message}`);
|
|
93
|
+
console.error('浏览器已启动,可以手动在浏览器中打开 TikTok');
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
// 保持进程运行
|
|
97
|
+
await new Promise(() => {});
|
|
98
|
+
} catch (err) {
|
|
99
|
+
console.error(`启动失败: ${err.message}`);
|
|
100
|
+
process.exit(1);
|
|
101
|
+
}
|
|
102
|
+
}
|
package/src/cli/progress.js
CHANGED
|
@@ -1,111 +1,111 @@
|
|
|
1
|
-
import { writeFileSync } from 'fs';
|
|
2
|
-
import { formatOutput } from '../lib/output.js';
|
|
3
|
-
import { deduplicate } from '../lib/output.js';
|
|
4
|
-
import { applyFilter, formatFilterDescription } from '../lib/filter.js';
|
|
5
|
-
import { calculateConcurrency, createMultiProgressBars, renderMultiProgressBars, clearProgressBars } from '../lib/io.js';
|
|
6
|
-
import { randomDelay } from '../lib/delay.js';
|
|
7
|
-
|
|
8
|
-
export async function processUrlsWithProgress({
|
|
9
|
-
urls,
|
|
10
|
-
proxyUrl,
|
|
11
|
-
outputFile,
|
|
12
|
-
outputFormat,
|
|
13
|
-
filter,
|
|
14
|
-
processFn,
|
|
15
|
-
label = '数据',
|
|
16
|
-
}) {
|
|
17
|
-
const allResults = [];
|
|
18
|
-
const errors = [];
|
|
19
|
-
|
|
20
|
-
if (urls.length === 0) {
|
|
21
|
-
console.error('\n未获取到数据');
|
|
22
|
-
if (outputFile) writeFileSync(outputFile, '[]', 'utf-8');
|
|
23
|
-
return;
|
|
24
|
-
}
|
|
25
|
-
|
|
26
|
-
const concurrency = calculateConcurrency(urls.length);
|
|
27
|
-
const bars = createMultiProgressBars(concurrency);
|
|
28
|
-
|
|
29
|
-
const slots = Array.from({ length: concurrency }, () => []);
|
|
30
|
-
urls.forEach((url, i) => slots[i % concurrency].push(url));
|
|
31
|
-
|
|
32
|
-
bars.forEach((bar, i) => {
|
|
33
|
-
bar.total = slots[i].length;
|
|
34
|
-
bar.status = slots[i].length > 0 ? 'running' : 'done';
|
|
35
|
-
});
|
|
36
|
-
|
|
37
|
-
renderMultiProgressBars(bars);
|
|
38
|
-
|
|
39
|
-
const workers = slots.map(async (slotUrls, slotIndex) => {
|
|
40
|
-
for (const url of slotUrls) {
|
|
41
|
-
bars[slotIndex].url = url;
|
|
42
|
-
renderMultiProgressBars(bars);
|
|
43
|
-
|
|
44
|
-
await randomDelay();
|
|
45
|
-
|
|
46
|
-
try {
|
|
47
|
-
const results = await processFn(url, proxyUrl);
|
|
48
|
-
allResults.push(...results);
|
|
49
|
-
bars[slotIndex].current++;
|
|
50
|
-
bars[slotIndex].status = 'running';
|
|
51
|
-
} catch (err) {
|
|
52
|
-
errors.push({ url, message: err.message });
|
|
53
|
-
bars[slotIndex].current++;
|
|
54
|
-
bars[slotIndex].status = 'error';
|
|
55
|
-
}
|
|
56
|
-
|
|
57
|
-
renderMultiProgressBars(bars);
|
|
58
|
-
}
|
|
59
|
-
bars[slotIndex].status = bars[slotIndex].current === bars[slotIndex].total ? 'done' : 'error';
|
|
60
|
-
renderMultiProgressBars(bars);
|
|
61
|
-
});
|
|
62
|
-
|
|
63
|
-
await Promise.all(workers);
|
|
64
|
-
clearProgressBars();
|
|
65
|
-
|
|
66
|
-
const uniqueResults = deduplicate(allResults);
|
|
67
|
-
const filteredResults = applyFilter(uniqueResults, filter);
|
|
68
|
-
|
|
69
|
-
if (errors.length > 0) {
|
|
70
|
-
const firstMsg = errors[0].message;
|
|
71
|
-
const isProxyError = ['不可用', '连接被拒绝', '连接中断', '超时', '无法解析']
|
|
72
|
-
.some(kw => firstMsg.includes(kw));
|
|
73
|
-
|
|
74
|
-
if (filteredResults.length === 0) {
|
|
75
|
-
if (isProxyError) {
|
|
76
|
-
console.error(` 所有请求失败,请检查代理: ${proxyUrl}\n`);
|
|
77
|
-
} else {
|
|
78
|
-
const show = errors.slice(0, 5);
|
|
79
|
-
for (const e of show) console.error(` ✗ ${e.url}: ${e.message}\n`);
|
|
80
|
-
if (errors.length > 5) console.error(` ... 还有 ${errors.length - 5} 个失败\n`);
|
|
81
|
-
}
|
|
82
|
-
console.error('未获取到数据');
|
|
83
|
-
if (outputFile) writeFileSync(outputFile, '[]', 'utf-8');
|
|
84
|
-
return;
|
|
85
|
-
} else {
|
|
86
|
-
if (isProxyError) {
|
|
87
|
-
console.error(` ${errors.length} 个请求失败,请检查代理: ${proxyUrl}\n`);
|
|
88
|
-
} else {
|
|
89
|
-
console.error(` ${errors.length} 个失败:`);
|
|
90
|
-
const show = errors.slice(0, 5);
|
|
91
|
-
for (const e of show) console.error(` ✗ ${e.url}: ${e.message}`);
|
|
92
|
-
if (errors.length > 5) console.error(` ... 还有 ${errors.length - 5} 个`);
|
|
93
|
-
}
|
|
94
|
-
}
|
|
95
|
-
}
|
|
96
|
-
|
|
97
|
-
const output = formatOutput(filteredResults, outputFormat);
|
|
98
|
-
|
|
99
|
-
if (outputFile) {
|
|
100
|
-
writeFileSync(outputFile, output, 'utf-8');
|
|
101
|
-
console.log(`\n结果已写入: ${outputFile}`);
|
|
102
|
-
} else {
|
|
103
|
-
process.stdout.write(output + '\n');
|
|
104
|
-
}
|
|
105
|
-
|
|
106
|
-
if (filter) {
|
|
107
|
-
console.log(`\n共 ${uniqueResults.length} 个${label},过滤后 ${filteredResults.length} 个(过滤条件: ${formatFilterDescription(filter)})`);
|
|
108
|
-
} else {
|
|
109
|
-
console.log(`\n共 ${filteredResults.length} 个${label}`);
|
|
110
|
-
}
|
|
111
|
-
}
|
|
1
|
+
import { writeFileSync } from 'fs';
|
|
2
|
+
import { formatOutput } from '../lib/output.js';
|
|
3
|
+
import { deduplicate } from '../lib/output.js';
|
|
4
|
+
import { applyFilter, formatFilterDescription } from '../lib/filter.js';
|
|
5
|
+
import { calculateConcurrency, createMultiProgressBars, renderMultiProgressBars, clearProgressBars } from '../lib/io.js';
|
|
6
|
+
import { randomDelay } from '../lib/delay.js';
|
|
7
|
+
|
|
8
|
+
export async function processUrlsWithProgress({
|
|
9
|
+
urls,
|
|
10
|
+
proxyUrl,
|
|
11
|
+
outputFile,
|
|
12
|
+
outputFormat,
|
|
13
|
+
filter,
|
|
14
|
+
processFn,
|
|
15
|
+
label = '数据',
|
|
16
|
+
}) {
|
|
17
|
+
const allResults = [];
|
|
18
|
+
const errors = [];
|
|
19
|
+
|
|
20
|
+
if (urls.length === 0) {
|
|
21
|
+
console.error('\n未获取到数据');
|
|
22
|
+
if (outputFile) writeFileSync(outputFile, '[]', 'utf-8');
|
|
23
|
+
return;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
const concurrency = calculateConcurrency(urls.length);
|
|
27
|
+
const bars = createMultiProgressBars(concurrency);
|
|
28
|
+
|
|
29
|
+
const slots = Array.from({ length: concurrency }, () => []);
|
|
30
|
+
urls.forEach((url, i) => slots[i % concurrency].push(url));
|
|
31
|
+
|
|
32
|
+
bars.forEach((bar, i) => {
|
|
33
|
+
bar.total = slots[i].length;
|
|
34
|
+
bar.status = slots[i].length > 0 ? 'running' : 'done';
|
|
35
|
+
});
|
|
36
|
+
|
|
37
|
+
renderMultiProgressBars(bars);
|
|
38
|
+
|
|
39
|
+
const workers = slots.map(async (slotUrls, slotIndex) => {
|
|
40
|
+
for (const url of slotUrls) {
|
|
41
|
+
bars[slotIndex].url = url;
|
|
42
|
+
renderMultiProgressBars(bars);
|
|
43
|
+
|
|
44
|
+
await randomDelay();
|
|
45
|
+
|
|
46
|
+
try {
|
|
47
|
+
const results = await processFn(url, proxyUrl);
|
|
48
|
+
allResults.push(...results);
|
|
49
|
+
bars[slotIndex].current++;
|
|
50
|
+
bars[slotIndex].status = 'running';
|
|
51
|
+
} catch (err) {
|
|
52
|
+
errors.push({ url, message: err.message });
|
|
53
|
+
bars[slotIndex].current++;
|
|
54
|
+
bars[slotIndex].status = 'error';
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
renderMultiProgressBars(bars);
|
|
58
|
+
}
|
|
59
|
+
bars[slotIndex].status = bars[slotIndex].current === bars[slotIndex].total ? 'done' : 'error';
|
|
60
|
+
renderMultiProgressBars(bars);
|
|
61
|
+
});
|
|
62
|
+
|
|
63
|
+
await Promise.all(workers);
|
|
64
|
+
clearProgressBars();
|
|
65
|
+
|
|
66
|
+
const uniqueResults = deduplicate(allResults);
|
|
67
|
+
const filteredResults = applyFilter(uniqueResults, filter);
|
|
68
|
+
|
|
69
|
+
if (errors.length > 0) {
|
|
70
|
+
const firstMsg = errors[0].message;
|
|
71
|
+
const isProxyError = ['不可用', '连接被拒绝', '连接中断', '超时', '无法解析']
|
|
72
|
+
.some(kw => firstMsg.includes(kw));
|
|
73
|
+
|
|
74
|
+
if (filteredResults.length === 0) {
|
|
75
|
+
if (isProxyError) {
|
|
76
|
+
console.error(` 所有请求失败,请检查代理: ${proxyUrl}\n`);
|
|
77
|
+
} else {
|
|
78
|
+
const show = errors.slice(0, 5);
|
|
79
|
+
for (const e of show) console.error(` ✗ ${e.url}: ${e.message}\n`);
|
|
80
|
+
if (errors.length > 5) console.error(` ... 还有 ${errors.length - 5} 个失败\n`);
|
|
81
|
+
}
|
|
82
|
+
console.error('未获取到数据');
|
|
83
|
+
if (outputFile) writeFileSync(outputFile, '[]', 'utf-8');
|
|
84
|
+
return;
|
|
85
|
+
} else {
|
|
86
|
+
if (isProxyError) {
|
|
87
|
+
console.error(` ${errors.length} 个请求失败,请检查代理: ${proxyUrl}\n`);
|
|
88
|
+
} else {
|
|
89
|
+
console.error(` ${errors.length} 个失败:`);
|
|
90
|
+
const show = errors.slice(0, 5);
|
|
91
|
+
for (const e of show) console.error(` ✗ ${e.url}: ${e.message}`);
|
|
92
|
+
if (errors.length > 5) console.error(` ... 还有 ${errors.length - 5} 个`);
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
const output = formatOutput(filteredResults, outputFormat);
|
|
98
|
+
|
|
99
|
+
if (outputFile) {
|
|
100
|
+
writeFileSync(outputFile, output, 'utf-8');
|
|
101
|
+
console.log(`\n结果已写入: ${outputFile}`);
|
|
102
|
+
} else {
|
|
103
|
+
process.stdout.write(output + '\n');
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
if (filter) {
|
|
107
|
+
console.log(`\n共 ${uniqueResults.length} 个${label},过滤后 ${filteredResults.length} 个(过滤条件: ${formatFilterDescription(filter)})`);
|
|
108
|
+
} else {
|
|
109
|
+
console.log(`\n共 ${filteredResults.length} 个${label}`);
|
|
110
|
+
}
|
|
111
|
+
}
|