tt-help-cli-ycl 1.3.11 → 1.3.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -17
- package/cli.js +9 -9
- package/package.json +45 -46
- package/{bat → scripts}/run-explore.bat +68 -68
- package/{bat → scripts}/run-explore.ps1 +81 -81
- package/{bat → scripts}/run-explore.sh +73 -73
- package/scripts/test-captcha-lib.mjs +68 -0
- package/scripts/test-captcha.mjs +81 -0
- package/scripts/test-incognito-lib.mjs +36 -0
- package/scripts/test-login-state.mjs +128 -0
- package/scripts/test-safe-click.mjs +45 -0
- package/src/cli/auto.js +186 -157
- package/src/cli/config.js +116 -0
- package/src/cli/explore-default.js +83 -0
- package/src/cli/explore.js +227 -181
- package/src/cli/progress.js +111 -111
- package/src/cli/refresh.js +216 -0
- package/src/cli/scrape.js +47 -47
- package/src/cli/utils.js +18 -18
- package/src/cli/videos.js +41 -41
- package/src/cli/watch.js +31 -31
- package/src/lib/args.js +456 -391
- package/src/lib/browser/anti-detect.js +23 -23
- package/src/lib/browser/cdp.js +194 -142
- package/src/lib/browser/launch.js +43 -43
- package/src/lib/browser/page.js +146 -87
- package/src/lib/constants.js +119 -119
- package/src/lib/delay.js +54 -54
- package/src/lib/explore-fetch.js +118 -118
- package/src/lib/fetcher.js +45 -45
- package/src/lib/filter.js +66 -66
- package/src/lib/io.js +54 -54
- package/src/lib/output.js +80 -80
- package/src/{scraper/modules/page-error-detector.mjs → lib/page-error-detector.js} +70 -70
- package/src/lib/parser.js +47 -47
- package/src/lib/retry.js +45 -45
- package/src/lib/scrape.js +40 -40
- package/src/{scraper/modules/scroll-collector.mjs → lib/scroll-collector.js} +231 -189
- package/src/lib/url.js +52 -52
- package/src/main.js +48 -0
- package/src/results/user-videos-bar.lar.lar.moeta.json +37 -0
- package/src/scraper/{auto-core.mjs → auto-core.js} +203 -194
- package/src/scraper/{core.mjs → core.js} +211 -190
- package/src/scraper/{explore-core.mjs → explore-core.js} +180 -171
- package/src/scraper/modules/{captcha-handler.mjs → captcha-handler.js} +114 -114
- package/src/scraper/modules/{comment-extractor.mjs → comment-extractor.js} +74 -69
- package/src/scraper/modules/{follow-extractor.mjs → follow-extractor.js} +121 -121
- package/src/scraper/modules/{guess-extractor.mjs → guess-extractor.js} +51 -51
- package/src/scraper/modules/page-error-detector.js +1 -0
- package/src/scraper/modules/{page-helpers.mjs → page-helpers.js} +48 -48
- package/src/scraper/modules/scroll-collector.js +8 -0
- package/src/scraper/refresh-core.js +179 -0
- package/src/videos/{core.mjs → core.js} +126 -126
- package/src/watch/data-store.js +431 -0
- package/src/watch/public/index.html +721 -690
- package/src/watch/{server.mjs → server.js} +484 -349
- package/src/main.mjs +0 -234
- package/src/test-auto-follow.cjs +0 -109
- package/src/test-extractors.cjs +0 -75
- package/src/test-follow.cjs +0 -41
- package/src/watch/data-store.mjs +0 -274
package/src/cli/progress.js
CHANGED
|
@@ -1,111 +1,111 @@
|
|
|
1
|
-
import { writeFileSync } from 'fs';
|
|
2
|
-
import { formatOutput } from '../lib/output.js';
|
|
3
|
-
import { deduplicate } from '../lib/output.js';
|
|
4
|
-
import { applyFilter, formatFilterDescription } from '../lib/filter.js';
|
|
5
|
-
import { calculateConcurrency, createMultiProgressBars, renderMultiProgressBars, clearProgressBars } from '../lib/io.js';
|
|
6
|
-
import { randomDelay } from '../lib/delay.js';
|
|
7
|
-
|
|
8
|
-
export async function processUrlsWithProgress({
|
|
9
|
-
urls,
|
|
10
|
-
proxyUrl,
|
|
11
|
-
outputFile,
|
|
12
|
-
outputFormat,
|
|
13
|
-
filter,
|
|
14
|
-
processFn,
|
|
15
|
-
label = '数据',
|
|
16
|
-
}) {
|
|
17
|
-
const allResults = [];
|
|
18
|
-
const errors = [];
|
|
19
|
-
|
|
20
|
-
if (urls.length === 0) {
|
|
21
|
-
console.error('\n未获取到数据');
|
|
22
|
-
if (outputFile) writeFileSync(outputFile, '[]', 'utf-8');
|
|
23
|
-
return;
|
|
24
|
-
}
|
|
25
|
-
|
|
26
|
-
const concurrency = calculateConcurrency(urls.length);
|
|
27
|
-
const bars = createMultiProgressBars(concurrency);
|
|
28
|
-
|
|
29
|
-
const slots = Array.from({ length: concurrency }, () => []);
|
|
30
|
-
urls.forEach((url, i) => slots[i % concurrency].push(url));
|
|
31
|
-
|
|
32
|
-
bars.forEach((bar, i) => {
|
|
33
|
-
bar.total = slots[i].length;
|
|
34
|
-
bar.status = slots[i].length > 0 ? 'running' : 'done';
|
|
35
|
-
});
|
|
36
|
-
|
|
37
|
-
renderMultiProgressBars(bars);
|
|
38
|
-
|
|
39
|
-
const workers = slots.map(async (slotUrls, slotIndex) => {
|
|
40
|
-
for (const url of slotUrls) {
|
|
41
|
-
bars[slotIndex].url = url;
|
|
42
|
-
renderMultiProgressBars(bars);
|
|
43
|
-
|
|
44
|
-
await randomDelay();
|
|
45
|
-
|
|
46
|
-
try {
|
|
47
|
-
const results = await processFn(url, proxyUrl);
|
|
48
|
-
allResults.push(...results);
|
|
49
|
-
bars[slotIndex].current++;
|
|
50
|
-
bars[slotIndex].status = 'running';
|
|
51
|
-
} catch (err) {
|
|
52
|
-
errors.push({ url, message: err.message });
|
|
53
|
-
bars[slotIndex].current++;
|
|
54
|
-
bars[slotIndex].status = 'error';
|
|
55
|
-
}
|
|
56
|
-
|
|
57
|
-
renderMultiProgressBars(bars);
|
|
58
|
-
}
|
|
59
|
-
bars[slotIndex].status = bars[slotIndex].current === bars[slotIndex].total ? 'done' : 'error';
|
|
60
|
-
renderMultiProgressBars(bars);
|
|
61
|
-
});
|
|
62
|
-
|
|
63
|
-
await Promise.all(workers);
|
|
64
|
-
clearProgressBars();
|
|
65
|
-
|
|
66
|
-
const uniqueResults = deduplicate(allResults);
|
|
67
|
-
const filteredResults = applyFilter(uniqueResults, filter);
|
|
68
|
-
|
|
69
|
-
if (errors.length > 0) {
|
|
70
|
-
const firstMsg = errors[0].message;
|
|
71
|
-
const isProxyError = ['不可用', '连接被拒绝', '连接中断', '超时', '无法解析']
|
|
72
|
-
.some(kw => firstMsg.includes(kw));
|
|
73
|
-
|
|
74
|
-
if (filteredResults.length === 0) {
|
|
75
|
-
if (isProxyError) {
|
|
76
|
-
console.error(` 所有请求失败,请检查代理: ${proxyUrl}\n`);
|
|
77
|
-
} else {
|
|
78
|
-
const show = errors.slice(0, 5);
|
|
79
|
-
for (const e of show) console.error(` ✗ ${e.url}: ${e.message}\n`);
|
|
80
|
-
if (errors.length > 5) console.error(` ... 还有 ${errors.length - 5} 个失败\n`);
|
|
81
|
-
}
|
|
82
|
-
console.error('未获取到数据');
|
|
83
|
-
if (outputFile) writeFileSync(outputFile, '[]', 'utf-8');
|
|
84
|
-
return;
|
|
85
|
-
} else {
|
|
86
|
-
if (isProxyError) {
|
|
87
|
-
console.error(` ${errors.length} 个请求失败,请检查代理: ${proxyUrl}\n`);
|
|
88
|
-
} else {
|
|
89
|
-
console.error(` ${errors.length} 个失败:`);
|
|
90
|
-
const show = errors.slice(0, 5);
|
|
91
|
-
for (const e of show) console.error(` ✗ ${e.url}: ${e.message}`);
|
|
92
|
-
if (errors.length > 5) console.error(` ... 还有 ${errors.length - 5} 个`);
|
|
93
|
-
}
|
|
94
|
-
}
|
|
95
|
-
}
|
|
96
|
-
|
|
97
|
-
const output = formatOutput(filteredResults, outputFormat);
|
|
98
|
-
|
|
99
|
-
if (outputFile) {
|
|
100
|
-
writeFileSync(outputFile, output, 'utf-8');
|
|
101
|
-
console.log(`\n结果已写入: ${outputFile}`);
|
|
102
|
-
} else {
|
|
103
|
-
process.stdout.write(output + '\n');
|
|
104
|
-
}
|
|
105
|
-
|
|
106
|
-
if (filter) {
|
|
107
|
-
console.log(`\n共 ${uniqueResults.length} 个${label},过滤后 ${filteredResults.length} 个(过滤条件: ${formatFilterDescription(filter)})`);
|
|
108
|
-
} else {
|
|
109
|
-
console.log(`\n共 ${filteredResults.length} 个${label}`);
|
|
110
|
-
}
|
|
111
|
-
}
|
|
1
|
+
import { writeFileSync } from 'fs';
|
|
2
|
+
import { formatOutput } from '../lib/output.js';
|
|
3
|
+
import { deduplicate } from '../lib/output.js';
|
|
4
|
+
import { applyFilter, formatFilterDescription } from '../lib/filter.js';
|
|
5
|
+
import { calculateConcurrency, createMultiProgressBars, renderMultiProgressBars, clearProgressBars } from '../lib/io.js';
|
|
6
|
+
import { randomDelay } from '../lib/delay.js';
|
|
7
|
+
|
|
8
|
+
export async function processUrlsWithProgress({
|
|
9
|
+
urls,
|
|
10
|
+
proxyUrl,
|
|
11
|
+
outputFile,
|
|
12
|
+
outputFormat,
|
|
13
|
+
filter,
|
|
14
|
+
processFn,
|
|
15
|
+
label = '数据',
|
|
16
|
+
}) {
|
|
17
|
+
const allResults = [];
|
|
18
|
+
const errors = [];
|
|
19
|
+
|
|
20
|
+
if (urls.length === 0) {
|
|
21
|
+
console.error('\n未获取到数据');
|
|
22
|
+
if (outputFile) writeFileSync(outputFile, '[]', 'utf-8');
|
|
23
|
+
return;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
const concurrency = calculateConcurrency(urls.length);
|
|
27
|
+
const bars = createMultiProgressBars(concurrency);
|
|
28
|
+
|
|
29
|
+
const slots = Array.from({ length: concurrency }, () => []);
|
|
30
|
+
urls.forEach((url, i) => slots[i % concurrency].push(url));
|
|
31
|
+
|
|
32
|
+
bars.forEach((bar, i) => {
|
|
33
|
+
bar.total = slots[i].length;
|
|
34
|
+
bar.status = slots[i].length > 0 ? 'running' : 'done';
|
|
35
|
+
});
|
|
36
|
+
|
|
37
|
+
renderMultiProgressBars(bars);
|
|
38
|
+
|
|
39
|
+
const workers = slots.map(async (slotUrls, slotIndex) => {
|
|
40
|
+
for (const url of slotUrls) {
|
|
41
|
+
bars[slotIndex].url = url;
|
|
42
|
+
renderMultiProgressBars(bars);
|
|
43
|
+
|
|
44
|
+
await randomDelay();
|
|
45
|
+
|
|
46
|
+
try {
|
|
47
|
+
const results = await processFn(url, proxyUrl);
|
|
48
|
+
allResults.push(...results);
|
|
49
|
+
bars[slotIndex].current++;
|
|
50
|
+
bars[slotIndex].status = 'running';
|
|
51
|
+
} catch (err) {
|
|
52
|
+
errors.push({ url, message: err.message });
|
|
53
|
+
bars[slotIndex].current++;
|
|
54
|
+
bars[slotIndex].status = 'error';
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
renderMultiProgressBars(bars);
|
|
58
|
+
}
|
|
59
|
+
bars[slotIndex].status = bars[slotIndex].current === bars[slotIndex].total ? 'done' : 'error';
|
|
60
|
+
renderMultiProgressBars(bars);
|
|
61
|
+
});
|
|
62
|
+
|
|
63
|
+
await Promise.all(workers);
|
|
64
|
+
clearProgressBars();
|
|
65
|
+
|
|
66
|
+
const uniqueResults = deduplicate(allResults);
|
|
67
|
+
const filteredResults = applyFilter(uniqueResults, filter);
|
|
68
|
+
|
|
69
|
+
if (errors.length > 0) {
|
|
70
|
+
const firstMsg = errors[0].message;
|
|
71
|
+
const isProxyError = ['不可用', '连接被拒绝', '连接中断', '超时', '无法解析']
|
|
72
|
+
.some(kw => firstMsg.includes(kw));
|
|
73
|
+
|
|
74
|
+
if (filteredResults.length === 0) {
|
|
75
|
+
if (isProxyError) {
|
|
76
|
+
console.error(` 所有请求失败,请检查代理: ${proxyUrl}\n`);
|
|
77
|
+
} else {
|
|
78
|
+
const show = errors.slice(0, 5);
|
|
79
|
+
for (const e of show) console.error(` ✗ ${e.url}: ${e.message}\n`);
|
|
80
|
+
if (errors.length > 5) console.error(` ... 还有 ${errors.length - 5} 个失败\n`);
|
|
81
|
+
}
|
|
82
|
+
console.error('未获取到数据');
|
|
83
|
+
if (outputFile) writeFileSync(outputFile, '[]', 'utf-8');
|
|
84
|
+
return;
|
|
85
|
+
} else {
|
|
86
|
+
if (isProxyError) {
|
|
87
|
+
console.error(` ${errors.length} 个请求失败,请检查代理: ${proxyUrl}\n`);
|
|
88
|
+
} else {
|
|
89
|
+
console.error(` ${errors.length} 个失败:`);
|
|
90
|
+
const show = errors.slice(0, 5);
|
|
91
|
+
for (const e of show) console.error(` ✗ ${e.url}: ${e.message}`);
|
|
92
|
+
if (errors.length > 5) console.error(` ... 还有 ${errors.length - 5} 个`);
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
const output = formatOutput(filteredResults, outputFormat);
|
|
98
|
+
|
|
99
|
+
if (outputFile) {
|
|
100
|
+
writeFileSync(outputFile, output, 'utf-8');
|
|
101
|
+
console.log(`\n结果已写入: ${outputFile}`);
|
|
102
|
+
} else {
|
|
103
|
+
process.stdout.write(output + '\n');
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
if (filter) {
|
|
107
|
+
console.log(`\n共 ${uniqueResults.length} 个${label},过滤后 ${filteredResults.length} 个(过滤条件: ${formatFilterDescription(filter)})`);
|
|
108
|
+
} else {
|
|
109
|
+
console.log(`\n共 ${filteredResults.length} 个${label}`);
|
|
110
|
+
}
|
|
111
|
+
}
|
|
@@ -0,0 +1,216 @@
|
|
|
1
|
+
import { getOrCreatePage, isBrowserClosedError, relaunchBrowser } from '../lib/browser/page.js';
|
|
2
|
+
import { delay, setDelayConfig } from '../scraper/modules/page-helpers.js';
|
|
3
|
+
import { userId as configuredUserId, saveUserId } from '../lib/constants.js';
|
|
4
|
+
import { getMacOrUuid } from '../lib/mac-or-uuid.js';
|
|
5
|
+
import { ensureBrowserReady as ensureBrowserReadyCDP } from '../lib/browser/cdp.js';
|
|
6
|
+
import { processRefresh } from '../scraper/refresh-core.js';
|
|
7
|
+
import path from 'path';
|
|
8
|
+
import os from 'os';
|
|
9
|
+
|
|
10
|
+
async function withRetry(fn, maxRetries = 5) {
|
|
11
|
+
for (let attempt = 1; attempt <= maxRetries; attempt++) {
|
|
12
|
+
try {
|
|
13
|
+
return await fn();
|
|
14
|
+
} catch (e) {
|
|
15
|
+
if (attempt < maxRetries) {
|
|
16
|
+
const waitTime = attempt <= 2
|
|
17
|
+
? 5000 + Math.random() * 5000
|
|
18
|
+
: attempt <= 4
|
|
19
|
+
? 10000 + Math.random() * 10000
|
|
20
|
+
: 20000 + Math.random() * 10000;
|
|
21
|
+
console.error(` [网络] 请求失败 (${attempt}/${maxRetries}),${Math.round(waitTime / 1000)}s 后重试...`);
|
|
22
|
+
await delay(waitTime / 1000, waitTime / 1000);
|
|
23
|
+
} else {
|
|
24
|
+
throw e;
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
async function apiGet(url) {
|
|
31
|
+
const resp = await fetch(url);
|
|
32
|
+
if (!resp.ok) throw new Error(`HTTP ${resp.status}: ${resp.statusText}`);
|
|
33
|
+
return resp.json();
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
async function apiPost(url, body) {
|
|
37
|
+
const resp = await fetch(url, {
|
|
38
|
+
method: 'POST',
|
|
39
|
+
headers: { 'Content-Type': 'application/json' },
|
|
40
|
+
body: JSON.stringify(body),
|
|
41
|
+
});
|
|
42
|
+
if (!resp.ok) throw new Error(`HTTP ${resp.status}: ${resp.statusText}`);
|
|
43
|
+
return resp.json();
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
export async function handleRefresh(options) {
|
|
47
|
+
const {
|
|
48
|
+
explorePreset,
|
|
49
|
+
explorePort, exploreProfile, exploreUserId, serverUrl,
|
|
50
|
+
} = options;
|
|
51
|
+
|
|
52
|
+
let userId = exploreUserId || configuredUserId;
|
|
53
|
+
if (!userId) {
|
|
54
|
+
userId = await getMacOrUuid();
|
|
55
|
+
saveUserId(userId);
|
|
56
|
+
console.error(`[初始化] 未检测到本地用户编号,已生成并使用: ${userId}`);
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
setDelayConfig(explorePreset);
|
|
60
|
+
|
|
61
|
+
console.error(`\n=== Refresh 模式 ===`);
|
|
62
|
+
console.error(`服务器: ${serverUrl}`);
|
|
63
|
+
console.error(`CDP 端口: ${explorePort || 9222}, 用户编号: ${userId}`);
|
|
64
|
+
if (exploreProfile) console.error(`浏览器配置: ${exploreProfile}`);
|
|
65
|
+
console.error(`刷新: 视频 100 + 关注 100 + 粉丝 100`);
|
|
66
|
+
console.error(`新用户探索: 评论 + 猜你喜欢 + 关注/粉丝`);
|
|
67
|
+
|
|
68
|
+
const cdpOptions = {};
|
|
69
|
+
if (explorePort) cdpOptions.port = explorePort;
|
|
70
|
+
if (exploreProfile) {
|
|
71
|
+
cdpOptions.userDataDir = path.join(os.homedir(), 'Library', 'Application Support', `Microsoft Edge For Testing_${exploreProfile}`);
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
let browser = await ensureBrowserReadyCDP(cdpOptions);
|
|
75
|
+
const page = await getOrCreatePage(browser);
|
|
76
|
+
|
|
77
|
+
let processedCount = 0;
|
|
78
|
+
let errorCount = 0;
|
|
79
|
+
let consecutiveNetworkErrors = 0;
|
|
80
|
+
|
|
81
|
+
console.error(`\n开始循环刷新任务...\n`);
|
|
82
|
+
|
|
83
|
+
while (true) {
|
|
84
|
+
try {
|
|
85
|
+
const jobData = await withRetry(() =>
|
|
86
|
+
apiGet(`${serverUrl}/api/redo-job?userId=${userId}`)
|
|
87
|
+
);
|
|
88
|
+
|
|
89
|
+
if (!jobData.hasJob) {
|
|
90
|
+
console.error(`\n[空闲] 暂无 redo 任务,30s 后重试...`);
|
|
91
|
+
await delay(30000, 30000);
|
|
92
|
+
continue;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
const { uniqueId, nickname } = jobData.user;
|
|
96
|
+
consecutiveNetworkErrors = 0;
|
|
97
|
+
processedCount++;
|
|
98
|
+
|
|
99
|
+
console.error(`\n[${processedCount}] 刷新 @${uniqueId} (${nickname || '未知'})...`);
|
|
100
|
+
|
|
101
|
+
const result = await processRefresh(page, uniqueId, serverUrl, {
|
|
102
|
+
maxFollowing: 100,
|
|
103
|
+
maxFollowers: 100,
|
|
104
|
+
maxVideos: 100,
|
|
105
|
+
}, console.error);
|
|
106
|
+
|
|
107
|
+
if (result.restricted) {
|
|
108
|
+
console.error(` @${uniqueId} 页面受限,跳过`);
|
|
109
|
+
await apiPost(`${serverUrl}/api/redo-job/${uniqueId}`, {
|
|
110
|
+
restricted: true,
|
|
111
|
+
userInfo: result.userInfo || {},
|
|
112
|
+
});
|
|
113
|
+
continue;
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
if (result.error) {
|
|
117
|
+
// 浏览器关闭检测
|
|
118
|
+
if (isBrowserClosedError(new Error(result.error))) {
|
|
119
|
+
const newBrowser = await relaunchBrowser(cdpOptions, explorePort || 9222);
|
|
120
|
+
browser = newBrowser;
|
|
121
|
+
const newPage = await getOrCreatePage(browser);
|
|
122
|
+
Object.assign(page, newPage);
|
|
123
|
+
// 重试当前用户
|
|
124
|
+
const retryResult = await processRefresh(page, uniqueId, serverUrl, {
|
|
125
|
+
maxFollowing: 100,
|
|
126
|
+
maxFollowers: 100,
|
|
127
|
+
maxVideos: 100,
|
|
128
|
+
}, console.error);
|
|
129
|
+
Object.assign(result, retryResult);
|
|
130
|
+
// 继续下方逻辑,检查重试后的 result
|
|
131
|
+
} else {
|
|
132
|
+
consecutiveNetworkErrors++;
|
|
133
|
+
errorCount++;
|
|
134
|
+
console.error(` [错误] ${result.error}`);
|
|
135
|
+
|
|
136
|
+
if (consecutiveNetworkErrors >= 3) {
|
|
137
|
+
console.error(` [警告] 连续 ${consecutiveNetworkErrors} 次错误,等待 60s 后重试...`);
|
|
138
|
+
await delay(60000, 60000);
|
|
139
|
+
consecutiveNetworkErrors = 0;
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
await apiPost(`${serverUrl}/api/redo-job/${uniqueId}`, {
|
|
143
|
+
error: result.error,
|
|
144
|
+
userInfo: result.userInfo || {},
|
|
145
|
+
});
|
|
146
|
+
const errorType = consecutiveNetworkErrors > 1 ? 'network' : 'other';
|
|
147
|
+
apiPost(`${serverUrl}/api/error-report`, {
|
|
148
|
+
userId,
|
|
149
|
+
username: uniqueId,
|
|
150
|
+
errorType,
|
|
151
|
+
errorMessage: result.error,
|
|
152
|
+
stage: 'process',
|
|
153
|
+
errorStack: result.errorStack || '',
|
|
154
|
+
}).catch(() => {});
|
|
155
|
+
continue;
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
if (result.captchaDetected) {
|
|
160
|
+
await apiPost(`${serverUrl}/api/error-report`, {
|
|
161
|
+
userId,
|
|
162
|
+
username: uniqueId,
|
|
163
|
+
errorType: 'captcha',
|
|
164
|
+
errorMessage: result.captchaMessage || '页面出现验证码',
|
|
165
|
+
stage: result.captchaStage || 'video-page',
|
|
166
|
+
errorStack: '',
|
|
167
|
+
});
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
consecutiveNetworkErrors = 0;
|
|
171
|
+
|
|
172
|
+
const guessedLocation = result.locationCreated || null;
|
|
173
|
+
|
|
174
|
+
await apiPost(`${serverUrl}/api/redo-job/${uniqueId}`, {
|
|
175
|
+
userInfo: result.userInfo || {},
|
|
176
|
+
discoveredVideoAuthors: (result.discoveredVideoAuthors || []).map(item =>
|
|
177
|
+
typeof item === 'object' ? { ...item, guessedLocation } : item
|
|
178
|
+
),
|
|
179
|
+
discoveredCommentAuthors: (result.discoveredCommentAuthors || []).map(author => ({ author, guessedLocation })),
|
|
180
|
+
discoveredGuessAuthors: (result.discoveredGuessAuthors || []).map(author => ({ author, guessedLocation })),
|
|
181
|
+
discoveredFollowing: (result.discoveredFollowing || []).map(f => ({
|
|
182
|
+
handle: Array.isArray(f) ? f[0] : f,
|
|
183
|
+
displayName: Array.isArray(f) ? f[1] : null,
|
|
184
|
+
guessedLocation,
|
|
185
|
+
})),
|
|
186
|
+
discoveredFollowers: (result.discoveredFollowers || []).map(f => ({
|
|
187
|
+
handle: Array.isArray(f) ? f[0] : f,
|
|
188
|
+
displayName: Array.isArray(f) ? f[1] : null,
|
|
189
|
+
guessedLocation,
|
|
190
|
+
})),
|
|
191
|
+
newUsersAdded: result.newUsersAdded || 0,
|
|
192
|
+
collectedVideos: result.collectedVideos || 0,
|
|
193
|
+
});
|
|
194
|
+
|
|
195
|
+
console.error(` [完成] 视频: ${result.collectedVideos}, 评论作者: ${result.discoveredCommentAuthors?.length || 0}, 关注: ${result.discoveredFollowing?.length || 0}, 粉丝: ${result.discoveredFollowers?.length || 0}, 新增用户: ${result.newUsersAdded}`);
|
|
196
|
+
|
|
197
|
+
await delay(3000, 5000);
|
|
198
|
+
} catch (e) {
|
|
199
|
+
consecutiveNetworkErrors++;
|
|
200
|
+
errorCount++;
|
|
201
|
+
console.error(`\n[错误] ${e.message}`);
|
|
202
|
+
|
|
203
|
+
if (consecutiveNetworkErrors >= 3) {
|
|
204
|
+
console.error(`[警告] 连续 ${consecutiveNetworkErrors} 次网络异常,等待 60s 后重试...`);
|
|
205
|
+
await delay(60000, 60000);
|
|
206
|
+
consecutiveNetworkErrors = 0;
|
|
207
|
+
} else {
|
|
208
|
+
const waitTime = consecutiveNetworkErrors <= 2
|
|
209
|
+
? 5000 + Math.random() * 5000
|
|
210
|
+
: 10000 + Math.random() * 10000;
|
|
211
|
+
console.error(` 等待 ${Math.round(waitTime / 1000)}s 后重试...`);
|
|
212
|
+
await delay(waitTime / 1000, waitTime / 1000);
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
}
|
package/src/cli/scrape.js
CHANGED
|
@@ -1,47 +1,47 @@
|
|
|
1
|
-
import { writeFileSync } from 'fs';
|
|
2
|
-
|
|
3
|
-
export async function handleScrape(options) {
|
|
4
|
-
const { scrapeUrl, scrapePreset, scrapeMaxVideos, scrapeMaxComments, scrapeMaxGuess, scrapeSwitchDelay, scrapeCommentDelay, outputFile } = options;
|
|
5
|
-
|
|
6
|
-
if (!scrapeUrl) {
|
|
7
|
-
console.error('用法: tt-help scrape <视频URL> [preset] [最大视频数] [最大评论数] [-o 输出路径]');
|
|
8
|
-
console.error('预设: fast, normal, slow, stealth');
|
|
9
|
-
console.error('选项: -o, --output <路径> 输出到文件(默认输出到 stdout)');
|
|
10
|
-
console.error(' --switch-delay <ms> 视频切换延迟(毫秒)');
|
|
11
|
-
console.error(' --comment-delay <ms> 评论滚动延迟(毫秒)');
|
|
12
|
-
process.exit(1);
|
|
13
|
-
}
|
|
14
|
-
|
|
15
|
-
const { runScrape } = await import('../scraper/core.
|
|
16
|
-
|
|
17
|
-
let browser;
|
|
18
|
-
try {
|
|
19
|
-
const { output, browser: b } = await runScrape({
|
|
20
|
-
videoUrl: scrapeUrl,
|
|
21
|
-
maxVideos: scrapeMaxVideos,
|
|
22
|
-
maxComments: scrapeMaxComments,
|
|
23
|
-
maxGuess: scrapeMaxGuess,
|
|
24
|
-
preset: scrapePreset,
|
|
25
|
-
switchMax: scrapeSwitchDelay,
|
|
26
|
-
commentMax: scrapeCommentDelay,
|
|
27
|
-
log: console.error,
|
|
28
|
-
});
|
|
29
|
-
browser = b;
|
|
30
|
-
|
|
31
|
-
const json = JSON.stringify(output, null, 2);
|
|
32
|
-
if (outputFile) {
|
|
33
|
-
writeFileSync(outputFile, json, 'utf-8');
|
|
34
|
-
console.error(`结果已写入: ${outputFile}`);
|
|
35
|
-
} else {
|
|
36
|
-
process.stdout.write(json + '\n');
|
|
37
|
-
}
|
|
38
|
-
|
|
39
|
-
const stats = output.stats;
|
|
40
|
-
console.error(`\n共 ${stats.totalVideos} 个视频, ${stats.uniqueVideoAuthors} 个视频作者, ${stats.uniqueCommentAuthors} 个评论作者, ${stats.uniqueGuessAuthors} 个猜你喜欢作者`);
|
|
41
|
-
} catch (err) {
|
|
42
|
-
console.error(`浏览器抓取失败: ${err.message}`);
|
|
43
|
-
process.exit(1);
|
|
44
|
-
} finally {
|
|
45
|
-
if (browser) await browser.close().catch(() => {});
|
|
46
|
-
}
|
|
47
|
-
}
|
|
1
|
+
import { writeFileSync } from 'fs';
|
|
2
|
+
|
|
3
|
+
export async function handleScrape(options) {
|
|
4
|
+
const { scrapeUrl, scrapePreset, scrapeMaxVideos, scrapeMaxComments, scrapeMaxGuess, scrapeSwitchDelay, scrapeCommentDelay, outputFile } = options;
|
|
5
|
+
|
|
6
|
+
if (!scrapeUrl) {
|
|
7
|
+
console.error('用法: tt-help scrape <视频URL> [preset] [最大视频数] [最大评论数] [-o 输出路径]');
|
|
8
|
+
console.error('预设: fast, normal, slow, stealth');
|
|
9
|
+
console.error('选项: -o, --output <路径> 输出到文件(默认输出到 stdout)');
|
|
10
|
+
console.error(' --switch-delay <ms> 视频切换延迟(毫秒)');
|
|
11
|
+
console.error(' --comment-delay <ms> 评论滚动延迟(毫秒)');
|
|
12
|
+
process.exit(1);
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
const { runScrape } = await import('../scraper/core.js');
|
|
16
|
+
|
|
17
|
+
let browser;
|
|
18
|
+
try {
|
|
19
|
+
const { output, browser: b } = await runScrape({
|
|
20
|
+
videoUrl: scrapeUrl,
|
|
21
|
+
maxVideos: scrapeMaxVideos,
|
|
22
|
+
maxComments: scrapeMaxComments,
|
|
23
|
+
maxGuess: scrapeMaxGuess,
|
|
24
|
+
preset: scrapePreset,
|
|
25
|
+
switchMax: scrapeSwitchDelay,
|
|
26
|
+
commentMax: scrapeCommentDelay,
|
|
27
|
+
log: console.error,
|
|
28
|
+
});
|
|
29
|
+
browser = b;
|
|
30
|
+
|
|
31
|
+
const json = JSON.stringify(output, null, 2);
|
|
32
|
+
if (outputFile) {
|
|
33
|
+
writeFileSync(outputFile, json, 'utf-8');
|
|
34
|
+
console.error(`结果已写入: ${outputFile}`);
|
|
35
|
+
} else {
|
|
36
|
+
process.stdout.write(json + '\n');
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
const stats = output.stats;
|
|
40
|
+
console.error(`\n共 ${stats.totalVideos} 个视频, ${stats.uniqueVideoAuthors} 个视频作者, ${stats.uniqueCommentAuthors} 个评论作者, ${stats.uniqueGuessAuthors} 个猜你喜欢作者`);
|
|
41
|
+
} catch (err) {
|
|
42
|
+
console.error(`浏览器抓取失败: ${err.message}`);
|
|
43
|
+
process.exit(1);
|
|
44
|
+
} finally {
|
|
45
|
+
if (browser) await browser.close().catch(() => {});
|
|
46
|
+
}
|
|
47
|
+
}
|
package/src/cli/utils.js
CHANGED
|
@@ -1,18 +1,18 @@
|
|
|
1
|
-
import { writeFileSync } from 'fs';
|
|
2
|
-
|
|
3
|
-
export function cleanError(msg) {
|
|
4
|
-
return msg
|
|
5
|
-
.replace(/\x1b\[[0-9;]*m/g, '')
|
|
6
|
-
.replace(/\s*- navigating to.*/s, '')
|
|
7
|
-
.replace(/\s*Call log:/s, '')
|
|
8
|
-
.trim();
|
|
9
|
-
}
|
|
10
|
-
|
|
11
|
-
export function writeJson(data, outputFile) {
|
|
12
|
-
const json = JSON.stringify(data, null, 2);
|
|
13
|
-
if (outputFile) {
|
|
14
|
-
writeFileSync(outputFile, json, 'utf-8');
|
|
15
|
-
} else {
|
|
16
|
-
process.stdout.write(json + '\n');
|
|
17
|
-
}
|
|
18
|
-
}
|
|
1
|
+
import { writeFileSync } from 'fs';
|
|
2
|
+
|
|
3
|
+
export function cleanError(msg) {
|
|
4
|
+
return msg
|
|
5
|
+
.replace(/\x1b\[[0-9;]*m/g, '')
|
|
6
|
+
.replace(/\s*- navigating to.*/s, '')
|
|
7
|
+
.replace(/\s*Call log:/s, '')
|
|
8
|
+
.trim();
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
export function writeJson(data, outputFile) {
|
|
12
|
+
const json = JSON.stringify(data, null, 2);
|
|
13
|
+
if (outputFile) {
|
|
14
|
+
writeFileSync(outputFile, json, 'utf-8');
|
|
15
|
+
} else {
|
|
16
|
+
process.stdout.write(json + '\n');
|
|
17
|
+
}
|
|
18
|
+
}
|
package/src/cli/videos.js
CHANGED
|
@@ -1,41 +1,41 @@
|
|
|
1
|
-
import { writeFileSync } from 'fs';
|
|
2
|
-
|
|
3
|
-
export async function handleVideos(options) {
|
|
4
|
-
const { videosUsername, videosMax, outputFile } = options;
|
|
5
|
-
|
|
6
|
-
if (!videosUsername) {
|
|
7
|
-
console.error('用法: tt-help videos <用户名> [最大视频数] [-o 输出路径]');
|
|
8
|
-
console.error('示例: tt-help videos bar.lar.lar.moeta 1000');
|
|
9
|
-
console.error(' tt-help videos username 50 -o videos.json');
|
|
10
|
-
console.error('');
|
|
11
|
-
console.error('选项: -o, --output <路径> 输出到文件(默认输出到 stdout)');
|
|
12
|
-
process.exit(1);
|
|
13
|
-
}
|
|
14
|
-
|
|
15
|
-
const { runGetUserVideos } = await import('../videos/core.
|
|
16
|
-
|
|
17
|
-
let browser;
|
|
18
|
-
try {
|
|
19
|
-
const { output, browser: b } = await runGetUserVideos({
|
|
20
|
-
username: videosUsername,
|
|
21
|
-
maxVideos: videosMax,
|
|
22
|
-
log: console.error,
|
|
23
|
-
});
|
|
24
|
-
browser = b;
|
|
25
|
-
|
|
26
|
-
const json = JSON.stringify(output, null, 2);
|
|
27
|
-
if (outputFile) {
|
|
28
|
-
writeFileSync(outputFile, json, 'utf-8');
|
|
29
|
-
console.error(`结果已写入: ${outputFile}`);
|
|
30
|
-
} else {
|
|
31
|
-
process.stdout.write(json + '\n');
|
|
32
|
-
}
|
|
33
|
-
|
|
34
|
-
console.error(`\n共 ${output.videos.length} 个视频, 用户: @${videosUsername}`);
|
|
35
|
-
} catch (err) {
|
|
36
|
-
console.error(`获取用户视频失败: ${err.message}`);
|
|
37
|
-
process.exit(1);
|
|
38
|
-
} finally {
|
|
39
|
-
if (browser) await browser.close().catch(() => {});
|
|
40
|
-
}
|
|
41
|
-
}
|
|
1
|
+
import { writeFileSync } from 'fs';
|
|
2
|
+
|
|
3
|
+
export async function handleVideos(options) {
|
|
4
|
+
const { videosUsername, videosMax, outputFile } = options;
|
|
5
|
+
|
|
6
|
+
if (!videosUsername) {
|
|
7
|
+
console.error('用法: tt-help videos <用户名> [最大视频数] [-o 输出路径]');
|
|
8
|
+
console.error('示例: tt-help videos bar.lar.lar.moeta 1000');
|
|
9
|
+
console.error(' tt-help videos username 50 -o videos.json');
|
|
10
|
+
console.error('');
|
|
11
|
+
console.error('选项: -o, --output <路径> 输出到文件(默认输出到 stdout)');
|
|
12
|
+
process.exit(1);
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
const { runGetUserVideos } = await import('../videos/core.js');
|
|
16
|
+
|
|
17
|
+
let browser;
|
|
18
|
+
try {
|
|
19
|
+
const { output, browser: b } = await runGetUserVideos({
|
|
20
|
+
username: videosUsername,
|
|
21
|
+
maxVideos: videosMax,
|
|
22
|
+
log: console.error,
|
|
23
|
+
});
|
|
24
|
+
browser = b;
|
|
25
|
+
|
|
26
|
+
const json = JSON.stringify(output, null, 2);
|
|
27
|
+
if (outputFile) {
|
|
28
|
+
writeFileSync(outputFile, json, 'utf-8');
|
|
29
|
+
console.error(`结果已写入: ${outputFile}`);
|
|
30
|
+
} else {
|
|
31
|
+
process.stdout.write(json + '\n');
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
console.error(`\n共 ${output.videos.length} 个视频, 用户: @${videosUsername}`);
|
|
35
|
+
} catch (err) {
|
|
36
|
+
console.error(`获取用户视频失败: ${err.message}`);
|
|
37
|
+
process.exit(1);
|
|
38
|
+
} finally {
|
|
39
|
+
if (browser) await browser.close().catch(() => {});
|
|
40
|
+
}
|
|
41
|
+
}
|