tt-help-cli-ycl 1.3.12 → 1.3.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -17
- package/cli.js +9 -9
- package/package.json +45 -45
- package/scripts/run-explore.bat +68 -68
- package/scripts/run-explore.ps1 +81 -81
- package/scripts/run-explore.sh +73 -73
- package/scripts/test-captcha-lib.mjs +68 -0
- package/scripts/test-captcha.mjs +81 -0
- package/scripts/test-incognito-lib.mjs +36 -0
- package/scripts/test-login-state.mjs +128 -0
- package/scripts/test-safe-click.mjs +45 -0
- package/src/cli/auto.js +186 -157
- package/src/cli/explore.js +227 -193
- package/src/cli/progress.js +111 -111
- package/src/cli/refresh.js +216 -0
- package/src/cli/scrape.js +47 -47
- package/src/cli/utils.js +18 -18
- package/src/cli/videos.js +41 -41
- package/src/cli/watch.js +31 -31
- package/src/lib/args.js +456 -402
- package/src/lib/browser/anti-detect.js +23 -23
- package/src/lib/browser/cdp.js +52 -10
- package/src/lib/browser/launch.js +43 -43
- package/src/lib/browser/page.js +146 -87
- package/src/lib/constants.js +119 -115
- package/src/lib/delay.js +54 -54
- package/src/lib/explore-fetch.js +118 -118
- package/src/lib/fetcher.js +45 -45
- package/src/lib/filter.js +66 -66
- package/src/lib/io.js +54 -54
- package/src/lib/output.js +80 -80
- package/src/lib/parser.js +47 -47
- package/src/lib/retry.js +45 -45
- package/src/lib/scrape.js +40 -40
- package/src/lib/url.js +52 -52
- package/src/main.js +2 -0
- package/src/results/user-videos-bar.lar.lar.moeta.json +37 -0
- package/src/scraper/auto-core.js +203 -194
- package/src/scraper/core.js +211 -190
- package/src/scraper/explore-core.js +180 -171
- package/src/scraper/modules/captcha-handler.js +114 -114
- package/src/scraper/modules/comment-extractor.js +74 -69
- package/src/scraper/modules/follow-extractor.js +121 -121
- package/src/scraper/modules/guess-extractor.js +51 -51
- package/src/scraper/modules/page-helpers.js +48 -48
- package/src/scraper/refresh-core.js +179 -0
- package/src/videos/core.js +126 -126
- package/src/watch/data-store.js +431 -302
- package/src/watch/public/index.html +721 -701
- package/src/watch/server.js +483 -359
package/src/lib/retry.js
CHANGED
|
@@ -1,45 +1,45 @@
|
|
|
1
|
-
import { delay } from './delay.js';
|
|
2
|
-
|
|
3
|
-
const RETRYABLE_PATTERNS = [
|
|
4
|
-
'interrupted',
|
|
5
|
-
'Navigation.*interrupted',
|
|
6
|
-
'net::',
|
|
7
|
-
'ECONN',
|
|
8
|
-
'ETIMEDOUT',
|
|
9
|
-
'ENOTFOUND',
|
|
10
|
-
'EAI_AGAIN',
|
|
11
|
-
'ESOCKETRESET',
|
|
12
|
-
'connection.*refused',
|
|
13
|
-
'connection.*reset',
|
|
14
|
-
'failed.*navigate',
|
|
15
|
-
'target.*closed',
|
|
16
|
-
'crash',
|
|
17
|
-
'代理错误',
|
|
18
|
-
];
|
|
19
|
-
|
|
20
|
-
export function isRetryableError(error) {
|
|
21
|
-
if (!error) return false;
|
|
22
|
-
const msg = error.message || error.toString() || '';
|
|
23
|
-
return RETRYABLE_PATTERNS.some(p => new RegExp(p, 'i').test(msg));
|
|
24
|
-
}
|
|
25
|
-
|
|
26
|
-
export async function retryWithBackoff(fn, { maxRetries = 3, baseDelay = 3000, log } = {}) {
|
|
27
|
-
let lastError;
|
|
28
|
-
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
|
29
|
-
try {
|
|
30
|
-
return await fn();
|
|
31
|
-
} catch (error) {
|
|
32
|
-
lastError = error;
|
|
33
|
-
if (attempt >= maxRetries || !isRetryableError(error)) {
|
|
34
|
-
throw error;
|
|
35
|
-
}
|
|
36
|
-
const jitter = Math.random() * 2000;
|
|
37
|
-
const waitTime = baseDelay * Math.pow(2, attempt) + jitter;
|
|
38
|
-
if (log) {
|
|
39
|
-
log(` [重试] ${attempt + 1}/${maxRetries},${Math.round(waitTime / 1000)}s 后重试...`);
|
|
40
|
-
}
|
|
41
|
-
await delay(Math.round(waitTime), Math.round(waitTime));
|
|
42
|
-
}
|
|
43
|
-
}
|
|
44
|
-
throw lastError;
|
|
45
|
-
}
|
|
1
|
+
import { delay } from './delay.js';
|
|
2
|
+
|
|
3
|
+
const RETRYABLE_PATTERNS = [
|
|
4
|
+
'interrupted',
|
|
5
|
+
'Navigation.*interrupted',
|
|
6
|
+
'net::',
|
|
7
|
+
'ECONN',
|
|
8
|
+
'ETIMEDOUT',
|
|
9
|
+
'ENOTFOUND',
|
|
10
|
+
'EAI_AGAIN',
|
|
11
|
+
'ESOCKETRESET',
|
|
12
|
+
'connection.*refused',
|
|
13
|
+
'connection.*reset',
|
|
14
|
+
'failed.*navigate',
|
|
15
|
+
'target.*closed',
|
|
16
|
+
'crash',
|
|
17
|
+
'代理错误',
|
|
18
|
+
];
|
|
19
|
+
|
|
20
|
+
export function isRetryableError(error) {
|
|
21
|
+
if (!error) return false;
|
|
22
|
+
const msg = error.message || error.toString() || '';
|
|
23
|
+
return RETRYABLE_PATTERNS.some(p => new RegExp(p, 'i').test(msg));
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
export async function retryWithBackoff(fn, { maxRetries = 3, baseDelay = 3000, log } = {}) {
|
|
27
|
+
let lastError;
|
|
28
|
+
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
|
29
|
+
try {
|
|
30
|
+
return await fn();
|
|
31
|
+
} catch (error) {
|
|
32
|
+
lastError = error;
|
|
33
|
+
if (attempt >= maxRetries || !isRetryableError(error)) {
|
|
34
|
+
throw error;
|
|
35
|
+
}
|
|
36
|
+
const jitter = Math.random() * 2000;
|
|
37
|
+
const waitTime = baseDelay * Math.pow(2, attempt) + jitter;
|
|
38
|
+
if (log) {
|
|
39
|
+
log(` [重试] ${attempt + 1}/${maxRetries},${Math.round(waitTime / 1000)}s 后重试...`);
|
|
40
|
+
}
|
|
41
|
+
await delay(Math.round(waitTime), Math.round(waitTime));
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
throw lastError;
|
|
45
|
+
}
|
package/src/lib/scrape.js
CHANGED
|
@@ -1,40 +1,40 @@
|
|
|
1
|
-
import { extractUserSection, parseUserSection, extractLocationCreated } from './parser.js';
|
|
2
|
-
import { fetchHtml, isProfileUrl } from './fetcher.js';
|
|
3
|
-
import { toProfileUrl, isVideoUrl, extractUniqueId } from './url.js';
|
|
4
|
-
|
|
5
|
-
export async function extractUserData(profileUrl, proxyUrl) {
|
|
6
|
-
const profileHtml = await fetchHtml(profileUrl, proxyUrl);
|
|
7
|
-
const section = extractUserSection(profileHtml);
|
|
8
|
-
if (!section) throw new Error('无法解析用户信息');
|
|
9
|
-
const data = parseUserSection(section);
|
|
10
|
-
data.locationCreated = extractLocationCreated(profileHtml);
|
|
11
|
-
return data;
|
|
12
|
-
}
|
|
13
|
-
|
|
14
|
-
export async function extractVideoLocation(videoUrl, proxyUrl) {
|
|
15
|
-
const videoHtml = await fetchHtml(videoUrl, proxyUrl);
|
|
16
|
-
return extractLocationCreated(videoHtml);
|
|
17
|
-
}
|
|
18
|
-
|
|
19
|
-
export async function processUrl(url, proxyUrl) {
|
|
20
|
-
if (isProfileUrl(url)) {
|
|
21
|
-
const profileUrl = toProfileUrl(url);
|
|
22
|
-
const profileData = await extractUserData(profileUrl, proxyUrl);
|
|
23
|
-
return [profileData];
|
|
24
|
-
}
|
|
25
|
-
|
|
26
|
-
if (isVideoUrl(url)) {
|
|
27
|
-
const profileHandle = extractUniqueId(url);
|
|
28
|
-
if (!profileHandle) throw new Error(`无法从视频URL提取用户主页: ${url}`);
|
|
29
|
-
|
|
30
|
-
const profileUrl = toProfileUrl(profileHandle);
|
|
31
|
-
const [profileData, locationCreated] = await Promise.all([
|
|
32
|
-
extractUserData(profileUrl, proxyUrl),
|
|
33
|
-
extractVideoLocation(url, proxyUrl),
|
|
34
|
-
]);
|
|
35
|
-
|
|
36
|
-
return [{ ...profileData, locationCreated }];
|
|
37
|
-
}
|
|
38
|
-
|
|
39
|
-
return [];
|
|
40
|
-
}
|
|
1
|
+
import { extractUserSection, parseUserSection, extractLocationCreated } from './parser.js';
|
|
2
|
+
import { fetchHtml, isProfileUrl } from './fetcher.js';
|
|
3
|
+
import { toProfileUrl, isVideoUrl, extractUniqueId } from './url.js';
|
|
4
|
+
|
|
5
|
+
export async function extractUserData(profileUrl, proxyUrl) {
|
|
6
|
+
const profileHtml = await fetchHtml(profileUrl, proxyUrl);
|
|
7
|
+
const section = extractUserSection(profileHtml);
|
|
8
|
+
if (!section) throw new Error('无法解析用户信息');
|
|
9
|
+
const data = parseUserSection(section);
|
|
10
|
+
data.locationCreated = extractLocationCreated(profileHtml);
|
|
11
|
+
return data;
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
export async function extractVideoLocation(videoUrl, proxyUrl) {
|
|
15
|
+
const videoHtml = await fetchHtml(videoUrl, proxyUrl);
|
|
16
|
+
return extractLocationCreated(videoHtml);
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
export async function processUrl(url, proxyUrl) {
|
|
20
|
+
if (isProfileUrl(url)) {
|
|
21
|
+
const profileUrl = toProfileUrl(url);
|
|
22
|
+
const profileData = await extractUserData(profileUrl, proxyUrl);
|
|
23
|
+
return [profileData];
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
if (isVideoUrl(url)) {
|
|
27
|
+
const profileHandle = extractUniqueId(url);
|
|
28
|
+
if (!profileHandle) throw new Error(`无法从视频URL提取用户主页: ${url}`);
|
|
29
|
+
|
|
30
|
+
const profileUrl = toProfileUrl(profileHandle);
|
|
31
|
+
const [profileData, locationCreated] = await Promise.all([
|
|
32
|
+
extractUserData(profileUrl, proxyUrl),
|
|
33
|
+
extractVideoLocation(url, proxyUrl),
|
|
34
|
+
]);
|
|
35
|
+
|
|
36
|
+
return [{ ...profileData, locationCreated }];
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
return [];
|
|
40
|
+
}
|
package/src/lib/url.js
CHANGED
|
@@ -1,52 +1,52 @@
|
|
|
1
|
-
const BASE_URL = 'https://www.tiktok.com';
|
|
2
|
-
|
|
3
|
-
export function extractUniqueId(url) {
|
|
4
|
-
const m = url.match(/\/@([^/]+)/);
|
|
5
|
-
return m ? m[1] : null;
|
|
6
|
-
}
|
|
7
|
-
|
|
8
|
-
export function extractVideoId(url) {
|
|
9
|
-
const m = url.match(/\/video\/(\d+)/);
|
|
10
|
-
return m ? m[1] : null;
|
|
11
|
-
}
|
|
12
|
-
|
|
13
|
-
export function normalizeUsername(input) {
|
|
14
|
-
return (input || '').replace(/^@/, '');
|
|
15
|
-
}
|
|
16
|
-
|
|
17
|
-
export function toProfileUrl(handle) {
|
|
18
|
-
const clean = normalizeUsername(handle);
|
|
19
|
-
return `${BASE_URL}/@${clean}`;
|
|
20
|
-
}
|
|
21
|
-
|
|
22
|
-
export function toVideoUrl(handle, videoId) {
|
|
23
|
-
const clean = normalizeUsername(handle);
|
|
24
|
-
return `${BASE_URL}/@${clean}/video/${videoId}`;
|
|
25
|
-
}
|
|
26
|
-
|
|
27
|
-
export function ensureAbsoluteUrl(href) {
|
|
28
|
-
if (href.startsWith('http')) return href;
|
|
29
|
-
return `${BASE_URL}${href}`;
|
|
30
|
-
}
|
|
31
|
-
|
|
32
|
-
export function isProfileUrl(url) {
|
|
33
|
-
return /\/@[\w-]+(?:$|[?#])/.test(url);
|
|
34
|
-
}
|
|
35
|
-
|
|
36
|
-
export function isVideoUrl(url) {
|
|
37
|
-
return /\/video\/\d+/.test(url);
|
|
38
|
-
}
|
|
39
|
-
|
|
40
|
-
export function extractDisplayPath(url) {
|
|
41
|
-
try {
|
|
42
|
-
const parts = new URL(url).pathname.split('/').filter(Boolean);
|
|
43
|
-
return parts.slice(-2).join('/');
|
|
44
|
-
} catch {
|
|
45
|
-
return url;
|
|
46
|
-
}
|
|
47
|
-
}
|
|
48
|
-
|
|
49
|
-
export function extractAuthorFromVideoUrl(url) {
|
|
50
|
-
const m = url.match(/@([^/]+)\/video/);
|
|
51
|
-
return m ? '@' + m[1] : null;
|
|
52
|
-
}
|
|
1
|
+
const BASE_URL = 'https://www.tiktok.com';
|
|
2
|
+
|
|
3
|
+
export function extractUniqueId(url) {
|
|
4
|
+
const m = url.match(/\/@([^/]+)/);
|
|
5
|
+
return m ? m[1] : null;
|
|
6
|
+
}
|
|
7
|
+
|
|
8
|
+
export function extractVideoId(url) {
|
|
9
|
+
const m = url.match(/\/video\/(\d+)/);
|
|
10
|
+
return m ? m[1] : null;
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
export function normalizeUsername(input) {
|
|
14
|
+
return (input || '').replace(/^@/, '');
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export function toProfileUrl(handle) {
|
|
18
|
+
const clean = normalizeUsername(handle);
|
|
19
|
+
return `${BASE_URL}/@${clean}`;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export function toVideoUrl(handle, videoId) {
|
|
23
|
+
const clean = normalizeUsername(handle);
|
|
24
|
+
return `${BASE_URL}/@${clean}/video/${videoId}`;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export function ensureAbsoluteUrl(href) {
|
|
28
|
+
if (href.startsWith('http')) return href;
|
|
29
|
+
return `${BASE_URL}${href}`;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
export function isProfileUrl(url) {
|
|
33
|
+
return /\/@[\w-]+(?:$|[?#])/.test(url);
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export function isVideoUrl(url) {
|
|
37
|
+
return /\/video\/\d+/.test(url);
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
export function extractDisplayPath(url) {
|
|
41
|
+
try {
|
|
42
|
+
const parts = new URL(url).pathname.split('/').filter(Boolean);
|
|
43
|
+
return parts.slice(-2).join('/');
|
|
44
|
+
} catch {
|
|
45
|
+
return url;
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
export function extractAuthorFromVideoUrl(url) {
|
|
50
|
+
const m = url.match(/@([^/]+)\/video/);
|
|
51
|
+
return m ? '@' + m[1] : null;
|
|
52
|
+
}
|
package/src/main.js
CHANGED
|
@@ -5,6 +5,7 @@ import { handleScrape } from './cli/scrape.js';
|
|
|
5
5
|
import { handleVideos } from './cli/videos.js';
|
|
6
6
|
import { handleAuto } from './cli/auto.js';
|
|
7
7
|
import { handleExplore } from './cli/explore.js';
|
|
8
|
+
import { handleRefresh } from './cli/refresh.js';
|
|
8
9
|
import { handleWatch } from './cli/watch.js';
|
|
9
10
|
import { handleConfig, showConfig, showUsage, version } from './cli/config.js';
|
|
10
11
|
import { runExploreDefault, runScrapeDefault } from './cli/explore-default.js';
|
|
@@ -17,6 +18,7 @@ async function main() {
|
|
|
17
18
|
case 'videos': return handleVideos(parsed);
|
|
18
19
|
case 'auto': return handleAuto(parsed);
|
|
19
20
|
case 'explore':return handleExplore(parsed);
|
|
21
|
+
case 'refresh':return handleRefresh(parsed);
|
|
20
22
|
case 'watch': return handleWatch(parsed);
|
|
21
23
|
}
|
|
22
24
|
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
{
|
|
2
|
+
"user": {
|
|
3
|
+
"uniqueId": "bar.lar.lar.moeta",
|
|
4
|
+
"secUid": "MS4wLjABAAAA3cgKTWvKfga0JAWeakAzx3zQ-aFAC8RuQvxD4HQFraKKsc_TbOIyMo3_ofVlXofV",
|
|
5
|
+
"nickname": "Bar Lar Lar Moetain",
|
|
6
|
+
"ttSeller": false,
|
|
7
|
+
"verified": false,
|
|
8
|
+
"followerCount": 24000,
|
|
9
|
+
"videoCount": 749,
|
|
10
|
+
"followingCount": 4293,
|
|
11
|
+
"heartCount": 254300,
|
|
12
|
+
"signature": ""
|
|
13
|
+
},
|
|
14
|
+
"totalVideos": 5,
|
|
15
|
+
"videos": [
|
|
16
|
+
{
|
|
17
|
+
"id": "7638231799084158228",
|
|
18
|
+
"url": "https://www.tiktok.com/@bar.lar.lar.moeta/video/7638231799084158228"
|
|
19
|
+
},
|
|
20
|
+
{
|
|
21
|
+
"id": "7638162444698914068",
|
|
22
|
+
"url": "https://www.tiktok.com/@bar.lar.lar.moeta/video/7638162444698914068"
|
|
23
|
+
},
|
|
24
|
+
{
|
|
25
|
+
"id": "7638116251767819541",
|
|
26
|
+
"url": "https://www.tiktok.com/@bar.lar.lar.moeta/video/7638116251767819541"
|
|
27
|
+
},
|
|
28
|
+
{
|
|
29
|
+
"id": "7638069637321690388",
|
|
30
|
+
"url": "https://www.tiktok.com/@bar.lar.lar.moeta/video/7638069637321690388"
|
|
31
|
+
},
|
|
32
|
+
{
|
|
33
|
+
"id": "7637927171025112341",
|
|
34
|
+
"url": "https://www.tiktok.com/@bar.lar.lar.moeta/video/7637927171025112341"
|
|
35
|
+
}
|
|
36
|
+
]
|
|
37
|
+
}
|