tt-help-cli-ycl 1.3.34 → 1.3.36
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +33 -17
- package/cli.js +9 -9
- package/package.json +49 -47
- package/scripts/run-explore copy.bat +101 -101
- package/scripts/run-explore.bat +132 -132
- package/scripts/run-explore.ps1 +157 -157
- package/scripts/run-explore.sh +119 -119
- package/scripts/test-captcha-lib.mjs +68 -0
- package/scripts/test-captcha.mjs +81 -0
- package/scripts/test-incognito-lib.mjs +36 -0
- package/scripts/test-login-state.mjs +128 -0
- package/scripts/test-safe-click.mjs +45 -0
- package/scripts/test-watch-db-smoke.mjs +246 -0
- package/src/cli/attach.js +240 -180
- package/src/cli/auto.js +265 -240
- package/src/cli/comments.js +301 -210
- package/src/cli/config.js +170 -152
- package/src/cli/db-import.js +51 -0
- package/src/cli/explore.js +519 -488
- package/src/cli/info.js +88 -88
- package/src/cli/open.js +111 -111
- package/src/cli/progress.js +111 -111
- package/src/cli/refresh.js +288 -216
- package/src/cli/scrape.js +47 -47
- package/src/cli/utils.js +18 -18
- package/src/cli/videos.js +41 -41
- package/src/cli/videostats.js +140 -25
- package/src/cli/watch.js +30 -31
- package/src/lib/args.js +766 -722
- package/src/lib/browser/anti-detect.js +23 -23
- package/src/lib/browser/cdp.js +261 -261
- package/src/lib/browser/health-checker.js +114 -114
- package/src/lib/browser/launch.js +43 -43
- package/src/lib/browser/page.js +183 -183
- package/src/lib/constants.js +238 -216
- package/src/lib/delay.js +54 -54
- package/src/lib/explore-fetch.js +118 -118
- package/src/lib/fetcher.js +45 -45
- package/src/lib/filter.js +66 -66
- package/src/lib/io.js +54 -54
- package/src/lib/output.js +80 -80
- package/src/lib/page-error-detector.js +105 -105
- package/src/lib/parse-ssr.mjs +69 -69
- package/src/lib/parser.js +47 -47
- package/src/lib/retry.js +45 -45
- package/src/lib/scrape.js +89 -89
- package/src/lib/tiktok-scraper.mjs +232 -194
- package/src/lib/url.js +52 -52
- package/src/main.js +70 -48
- package/src/results/user-videos-bar.lar.lar.moeta.json +37 -0
- package/src/scraper/auto-core.js +203 -203
- package/src/scraper/core.js +211 -211
- package/src/scraper/explore-core.js +177 -167
- package/src/scraper/modules/captcha-handler.js +114 -114
- package/src/scraper/modules/follow-extractor.js +194 -194
- package/src/scraper/modules/guess-extractor.js +51 -51
- package/src/scraper/modules/page-helpers.js +48 -48
- package/src/scraper/refresh-core.js +179 -179
- package/src/videos/core.js +125 -125
- package/src/watch/data-store.js +2364 -1030
- package/src/watch/public/index.html +1494 -753
- package/src/watch/server.js +628 -933
package/src/lib/retry.js
CHANGED
|
@@ -1,45 +1,45 @@
|
|
|
1
|
-
import { delay } from './delay.js';
|
|
2
|
-
|
|
3
|
-
const RETRYABLE_PATTERNS = [
|
|
4
|
-
'interrupted',
|
|
5
|
-
'Navigation.*interrupted',
|
|
6
|
-
'net::',
|
|
7
|
-
'ECONN',
|
|
8
|
-
'ETIMEDOUT',
|
|
9
|
-
'ENOTFOUND',
|
|
10
|
-
'EAI_AGAIN',
|
|
11
|
-
'ESOCKETRESET',
|
|
12
|
-
'connection.*refused',
|
|
13
|
-
'connection.*reset',
|
|
14
|
-
'failed.*navigate',
|
|
15
|
-
'target.*closed',
|
|
16
|
-
'crash',
|
|
17
|
-
'代理错误',
|
|
18
|
-
];
|
|
19
|
-
|
|
20
|
-
export function isRetryableError(error) {
|
|
21
|
-
if (!error) return false;
|
|
22
|
-
const msg = error.message || error.toString() || '';
|
|
23
|
-
return RETRYABLE_PATTERNS.some(p => new RegExp(p, 'i').test(msg));
|
|
24
|
-
}
|
|
25
|
-
|
|
26
|
-
export async function retryWithBackoff(fn, { maxRetries = 3, baseDelay = 3000, log } = {}) {
|
|
27
|
-
let lastError;
|
|
28
|
-
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
|
29
|
-
try {
|
|
30
|
-
return await fn();
|
|
31
|
-
} catch (error) {
|
|
32
|
-
lastError = error;
|
|
33
|
-
if (attempt >= maxRetries || !isRetryableError(error)) {
|
|
34
|
-
throw error;
|
|
35
|
-
}
|
|
36
|
-
const jitter = Math.random() * 2000;
|
|
37
|
-
const waitTime = baseDelay * Math.pow(2, attempt) + jitter;
|
|
38
|
-
if (log) {
|
|
39
|
-
log(` [重试] ${attempt + 1}/${maxRetries},${Math.round(waitTime / 1000)}s 后重试...`);
|
|
40
|
-
}
|
|
41
|
-
await delay(Math.round(waitTime), Math.round(waitTime));
|
|
42
|
-
}
|
|
43
|
-
}
|
|
44
|
-
throw lastError;
|
|
45
|
-
}
|
|
1
|
+
import { delay } from './delay.js';
|
|
2
|
+
|
|
3
|
+
const RETRYABLE_PATTERNS = [
|
|
4
|
+
'interrupted',
|
|
5
|
+
'Navigation.*interrupted',
|
|
6
|
+
'net::',
|
|
7
|
+
'ECONN',
|
|
8
|
+
'ETIMEDOUT',
|
|
9
|
+
'ENOTFOUND',
|
|
10
|
+
'EAI_AGAIN',
|
|
11
|
+
'ESOCKETRESET',
|
|
12
|
+
'connection.*refused',
|
|
13
|
+
'connection.*reset',
|
|
14
|
+
'failed.*navigate',
|
|
15
|
+
'target.*closed',
|
|
16
|
+
'crash',
|
|
17
|
+
'代理错误',
|
|
18
|
+
];
|
|
19
|
+
|
|
20
|
+
export function isRetryableError(error) {
|
|
21
|
+
if (!error) return false;
|
|
22
|
+
const msg = error.message || error.toString() || '';
|
|
23
|
+
return RETRYABLE_PATTERNS.some(p => new RegExp(p, 'i').test(msg));
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
export async function retryWithBackoff(fn, { maxRetries = 3, baseDelay = 3000, log } = {}) {
|
|
27
|
+
let lastError;
|
|
28
|
+
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
|
29
|
+
try {
|
|
30
|
+
return await fn();
|
|
31
|
+
} catch (error) {
|
|
32
|
+
lastError = error;
|
|
33
|
+
if (attempt >= maxRetries || !isRetryableError(error)) {
|
|
34
|
+
throw error;
|
|
35
|
+
}
|
|
36
|
+
const jitter = Math.random() * 2000;
|
|
37
|
+
const waitTime = baseDelay * Math.pow(2, attempt) + jitter;
|
|
38
|
+
if (log) {
|
|
39
|
+
log(` [重试] ${attempt + 1}/${maxRetries},${Math.round(waitTime / 1000)}s 后重试...`);
|
|
40
|
+
}
|
|
41
|
+
await delay(Math.round(waitTime), Math.round(waitTime));
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
throw lastError;
|
|
45
|
+
}
|
package/src/lib/scrape.js
CHANGED
|
@@ -1,89 +1,89 @@
|
|
|
1
|
-
import { TikTokScraper } from './tiktok-scraper.mjs';
|
|
2
|
-
import { isProfileUrl, isVideoUrl, extractUniqueId, normalizeUsername } from './url.js';
|
|
3
|
-
|
|
4
|
-
// Lazy singleton for TikTokScraper
|
|
5
|
-
let scraperInstance = null;
|
|
6
|
-
let scraperInitPromise = null;
|
|
7
|
-
|
|
8
|
-
async function getScraper() {
|
|
9
|
-
if (scraperInstance) return scraperInstance;
|
|
10
|
-
if (scraperInitPromise) return scraperInitPromise;
|
|
11
|
-
scraperInitPromise = (async () => {
|
|
12
|
-
const scraper = new TikTokScraper();
|
|
13
|
-
await scraper.init();
|
|
14
|
-
scraperInstance = scraper;
|
|
15
|
-
scraperInitPromise = null;
|
|
16
|
-
return scraper;
|
|
17
|
-
})();
|
|
18
|
-
return scraperInitPromise;
|
|
19
|
-
}
|
|
20
|
-
|
|
21
|
-
export async function closeScraper() {
|
|
22
|
-
if (scraperInstance) {
|
|
23
|
-
await scraperInstance.close();
|
|
24
|
-
scraperInstance = null;
|
|
25
|
-
}
|
|
26
|
-
}
|
|
27
|
-
|
|
28
|
-
// Map parseUserInfo output to legacy parser.js format
|
|
29
|
-
function mapUserInfo(user) {
|
|
30
|
-
if (!user) return null;
|
|
31
|
-
return {
|
|
32
|
-
uniqueId: user.uniqueId,
|
|
33
|
-
uid: user.id,
|
|
34
|
-
secUid: user.secUid,
|
|
35
|
-
nickname: user.nickname,
|
|
36
|
-
signature: user.bio,
|
|
37
|
-
ttSeller: user.ttSeller,
|
|
38
|
-
verified: user.verified,
|
|
39
|
-
followerCount: user.followerCount,
|
|
40
|
-
followingCount: user.followingCount,
|
|
41
|
-
heartCount: user.heartCount,
|
|
42
|
-
videoCount: user.videoCount,
|
|
43
|
-
diggCount: user.diggCount,
|
|
44
|
-
avatarLarger: user.avatar,
|
|
45
|
-
locationCreated: user.locationCreated,
|
|
46
|
-
};
|
|
47
|
-
}
|
|
48
|
-
|
|
49
|
-
// Map parseVideoInfo output to legacy format
|
|
50
|
-
function mapVideoLocation(video) {
|
|
51
|
-
if (!video) return null;
|
|
52
|
-
return video.locationCreated;
|
|
53
|
-
}
|
|
54
|
-
|
|
55
|
-
export async function extractUserData(url) {
|
|
56
|
-
const scraper = await getScraper();
|
|
57
|
-
const uniqueId = extractUniqueId(url);
|
|
58
|
-
if (!uniqueId) throw new Error(`无法从URL提取用户名: ${url}`);
|
|
59
|
-
const user = await scraper.getUserInfo(normalizeUsername(uniqueId));
|
|
60
|
-
if (!user) throw new Error('无法解析用户信息');
|
|
61
|
-
return mapUserInfo(user);
|
|
62
|
-
}
|
|
63
|
-
|
|
64
|
-
export async function extractVideoLocation(videoUrl) {
|
|
65
|
-
const scraper = await getScraper();
|
|
66
|
-
const video = await scraper.getVideoInfo(videoUrl);
|
|
67
|
-
return mapVideoLocation(video);
|
|
68
|
-
}
|
|
69
|
-
|
|
70
|
-
export async function processUrl(url) {
|
|
71
|
-
if (isProfileUrl(url)) {
|
|
72
|
-
const profileData = await extractUserData(url);
|
|
73
|
-
return [profileData];
|
|
74
|
-
}
|
|
75
|
-
|
|
76
|
-
if (isVideoUrl(url)) {
|
|
77
|
-
const profileHandle = extractUniqueId(url);
|
|
78
|
-
if (!profileHandle) throw new Error(`无法从视频URL提取用户主页: ${url}`);
|
|
79
|
-
|
|
80
|
-
const [profileData, locationCreated] = await Promise.all([
|
|
81
|
-
extractUserData(url),
|
|
82
|
-
extractVideoLocation(url),
|
|
83
|
-
]);
|
|
84
|
-
|
|
85
|
-
return [{ ...profileData, locationCreated }];
|
|
86
|
-
}
|
|
87
|
-
|
|
88
|
-
return [];
|
|
89
|
-
}
|
|
1
|
+
import { TikTokScraper } from './tiktok-scraper.mjs';
|
|
2
|
+
import { isProfileUrl, isVideoUrl, extractUniqueId, normalizeUsername } from './url.js';
|
|
3
|
+
|
|
4
|
+
// Lazy singleton for TikTokScraper
|
|
5
|
+
let scraperInstance = null;
|
|
6
|
+
let scraperInitPromise = null;
|
|
7
|
+
|
|
8
|
+
async function getScraper() {
|
|
9
|
+
if (scraperInstance) return scraperInstance;
|
|
10
|
+
if (scraperInitPromise) return scraperInitPromise;
|
|
11
|
+
scraperInitPromise = (async () => {
|
|
12
|
+
const scraper = new TikTokScraper();
|
|
13
|
+
await scraper.init();
|
|
14
|
+
scraperInstance = scraper;
|
|
15
|
+
scraperInitPromise = null;
|
|
16
|
+
return scraper;
|
|
17
|
+
})();
|
|
18
|
+
return scraperInitPromise;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export async function closeScraper() {
|
|
22
|
+
if (scraperInstance) {
|
|
23
|
+
await scraperInstance.close();
|
|
24
|
+
scraperInstance = null;
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
// Map parseUserInfo output to legacy parser.js format
|
|
29
|
+
function mapUserInfo(user) {
|
|
30
|
+
if (!user) return null;
|
|
31
|
+
return {
|
|
32
|
+
uniqueId: user.uniqueId,
|
|
33
|
+
uid: user.id,
|
|
34
|
+
secUid: user.secUid,
|
|
35
|
+
nickname: user.nickname,
|
|
36
|
+
signature: user.bio,
|
|
37
|
+
ttSeller: user.ttSeller,
|
|
38
|
+
verified: user.verified,
|
|
39
|
+
followerCount: user.followerCount,
|
|
40
|
+
followingCount: user.followingCount,
|
|
41
|
+
heartCount: user.heartCount,
|
|
42
|
+
videoCount: user.videoCount,
|
|
43
|
+
diggCount: user.diggCount,
|
|
44
|
+
avatarLarger: user.avatar,
|
|
45
|
+
locationCreated: user.locationCreated,
|
|
46
|
+
};
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
// Map parseVideoInfo output to legacy format
|
|
50
|
+
function mapVideoLocation(video) {
|
|
51
|
+
if (!video) return null;
|
|
52
|
+
return video.locationCreated;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
export async function extractUserData(url) {
|
|
56
|
+
const scraper = await getScraper();
|
|
57
|
+
const uniqueId = extractUniqueId(url);
|
|
58
|
+
if (!uniqueId) throw new Error(`无法从URL提取用户名: ${url}`);
|
|
59
|
+
const user = await scraper.getUserInfo(normalizeUsername(uniqueId));
|
|
60
|
+
if (!user) throw new Error('无法解析用户信息');
|
|
61
|
+
return mapUserInfo(user);
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
export async function extractVideoLocation(videoUrl) {
|
|
65
|
+
const scraper = await getScraper();
|
|
66
|
+
const video = await scraper.getVideoInfo(videoUrl);
|
|
67
|
+
return mapVideoLocation(video);
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
export async function processUrl(url) {
|
|
71
|
+
if (isProfileUrl(url)) {
|
|
72
|
+
const profileData = await extractUserData(url);
|
|
73
|
+
return [profileData];
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
if (isVideoUrl(url)) {
|
|
77
|
+
const profileHandle = extractUniqueId(url);
|
|
78
|
+
if (!profileHandle) throw new Error(`无法从视频URL提取用户主页: ${url}`);
|
|
79
|
+
|
|
80
|
+
const [profileData, locationCreated] = await Promise.all([
|
|
81
|
+
extractUserData(url),
|
|
82
|
+
extractVideoLocation(url),
|
|
83
|
+
]);
|
|
84
|
+
|
|
85
|
+
return [{ ...profileData, locationCreated }];
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
return [];
|
|
89
|
+
}
|