tt-help-cli-ycl 1.3.48 → 1.3.50

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. package/README.md +33 -33
  2. package/cli.js +9 -9
  3. package/package.json +52 -52
  4. package/scripts/run-explore copy.bat +101 -101
  5. package/scripts/run-explore.bat +134 -134
  6. package/scripts/run-explore.ps1 +159 -159
  7. package/scripts/run-explore.sh +121 -121
  8. package/scripts/test-captcha-lib.mjs +68 -0
  9. package/scripts/test-captcha.mjs +81 -0
  10. package/scripts/test-incognito-lib.mjs +36 -0
  11. package/scripts/test-login-state.mjs +128 -0
  12. package/scripts/test-safe-click.mjs +45 -0
  13. package/scripts/test-watch-db-smoke.mjs +246 -0
  14. package/src/cli/attach.js +331 -331
  15. package/src/cli/auto.js +265 -265
  16. package/src/cli/comments.js +620 -620
  17. package/src/cli/config.js +170 -170
  18. package/src/cli/db-import.js +51 -51
  19. package/src/cli/explore.js +555 -555
  20. package/src/cli/open.js +109 -111
  21. package/src/cli/progress.js +111 -111
  22. package/src/cli/refresh.js +288 -288
  23. package/src/cli/scrape.js +47 -47
  24. package/src/cli/utils.js +18 -18
  25. package/src/cli/videos.js +41 -41
  26. package/src/cli/videostats.js +196 -196
  27. package/src/cli/watch.js +30 -30
  28. package/src/lib/api-interceptor.js +161 -161
  29. package/src/lib/args.js +809 -809
  30. package/src/lib/browser/anti-detect.js +23 -23
  31. package/src/lib/browser/cdp.js +261 -261
  32. package/src/lib/browser/health-checker.js +114 -114
  33. package/src/lib/browser/launch.js +43 -43
  34. package/src/lib/browser/page.js +184 -184
  35. package/src/lib/constants.js +297 -297
  36. package/src/lib/delay.js +54 -54
  37. package/src/lib/explore-fetch.js +118 -118
  38. package/src/lib/fetcher.js +45 -45
  39. package/src/lib/filter.js +66 -66
  40. package/src/lib/io.js +54 -54
  41. package/src/lib/output.js +80 -80
  42. package/src/lib/page-error-detector.js +109 -109
  43. package/src/lib/parse-ssr.mjs +69 -69
  44. package/src/lib/parser.js +47 -47
  45. package/src/lib/retry.js +45 -45
  46. package/src/lib/scrape.js +90 -90
  47. package/src/lib/target-locations.js +61 -61
  48. package/src/lib/tiktok-scraper.mjs +98 -61
  49. package/src/lib/url.js +52 -52
  50. package/src/main.js +73 -73
  51. package/src/npm-main.js +70 -70
  52. package/src/results/user-videos-bar.lar.lar.moeta.json +37 -0
  53. package/src/scraper/auto-core.js +203 -203
  54. package/src/scraper/core.js +255 -255
  55. package/src/scraper/explore-core.js +208 -208
  56. package/src/scraper/modules/captcha-handler.js +114 -114
  57. package/src/scraper/modules/follow-extractor.js +250 -250
  58. package/src/scraper/modules/guess-extractor.js +51 -51
  59. package/src/scraper/modules/page-helpers.js +48 -48
  60. package/src/scraper/refresh-core.js +213 -213
  61. package/src/videos/core.js +143 -143
  62. package/src/watch/data-store.js +2980 -2980
  63. package/src/watch/public/index.html +2355 -2355
  64. package/src/watch/server.js +727 -727
package/src/cli/utils.js CHANGED
@@ -1,18 +1,18 @@
1
- import { writeFileSync } from 'fs';
2
-
3
- export function cleanError(msg) {
4
- return msg
5
- .replace(/\x1b\[[0-9;]*m/g, '')
6
- .replace(/\s*- navigating to.*/s, '')
7
- .replace(/\s*Call log:/s, '')
8
- .trim();
9
- }
10
-
11
- export function writeJson(data, outputFile) {
12
- const json = JSON.stringify(data, null, 2);
13
- if (outputFile) {
14
- writeFileSync(outputFile, json, 'utf-8');
15
- } else {
16
- process.stdout.write(json + '\n');
17
- }
18
- }
1
+ import { writeFileSync } from 'fs';
2
+
3
+ export function cleanError(msg) {
4
+ return msg
5
+ .replace(/\x1b\[[0-9;]*m/g, '')
6
+ .replace(/\s*- navigating to.*/s, '')
7
+ .replace(/\s*Call log:/s, '')
8
+ .trim();
9
+ }
10
+
11
+ export function writeJson(data, outputFile) {
12
+ const json = JSON.stringify(data, null, 2);
13
+ if (outputFile) {
14
+ writeFileSync(outputFile, json, 'utf-8');
15
+ } else {
16
+ process.stdout.write(json + '\n');
17
+ }
18
+ }
package/src/cli/videos.js CHANGED
@@ -1,41 +1,41 @@
1
- import { writeFileSync } from 'fs';
2
-
3
- export async function handleVideos(options) {
4
- const { videosUsername, videosMax, outputFile } = options;
5
-
6
- if (!videosUsername) {
7
- console.error('用法: tt-help videos <用户名> [最大视频数] [-o 输出路径]');
8
- console.error('示例: tt-help videos bar.lar.lar.moeta 1000');
9
- console.error(' tt-help videos username 50 -o videos.json');
10
- console.error('');
11
- console.error('选项: -o, --output <路径> 输出到文件(默认输出到 stdout)');
12
- process.exit(1);
13
- }
14
-
15
- const { runGetUserVideos } = await import('../videos/core.js');
16
-
17
- let browser;
18
- try {
19
- const { output, browser: b } = await runGetUserVideos({
20
- username: videosUsername,
21
- maxVideos: videosMax,
22
- log: console.error,
23
- });
24
- browser = b;
25
-
26
- const json = JSON.stringify(output, null, 2);
27
- if (outputFile) {
28
- writeFileSync(outputFile, json, 'utf-8');
29
- console.error(`结果已写入: ${outputFile}`);
30
- } else {
31
- process.stdout.write(json + '\n');
32
- }
33
-
34
- console.error(`\n共 ${output.videos.length} 个视频, 用户: @${videosUsername}`);
35
- } catch (err) {
36
- console.error(`获取用户视频失败: ${err.message}`);
37
- process.exit(1);
38
- } finally {
39
- if (browser) await browser.close().catch(() => {});
40
- }
41
- }
1
+ import { writeFileSync } from 'fs';
2
+
3
+ export async function handleVideos(options) {
4
+ const { videosUsername, videosMax, outputFile } = options;
5
+
6
+ if (!videosUsername) {
7
+ console.error('用法: tt-help videos <用户名> [最大视频数] [-o 输出路径]');
8
+ console.error('示例: tt-help videos bar.lar.lar.moeta 1000');
9
+ console.error(' tt-help videos username 50 -o videos.json');
10
+ console.error('');
11
+ console.error('选项: -o, --output <路径> 输出到文件(默认输出到 stdout)');
12
+ process.exit(1);
13
+ }
14
+
15
+ const { runGetUserVideos } = await import('../videos/core.js');
16
+
17
+ let browser;
18
+ try {
19
+ const { output, browser: b } = await runGetUserVideos({
20
+ username: videosUsername,
21
+ maxVideos: videosMax,
22
+ log: console.error,
23
+ });
24
+ browser = b;
25
+
26
+ const json = JSON.stringify(output, null, 2);
27
+ if (outputFile) {
28
+ writeFileSync(outputFile, json, 'utf-8');
29
+ console.error(`结果已写入: ${outputFile}`);
30
+ } else {
31
+ process.stdout.write(json + '\n');
32
+ }
33
+
34
+ console.error(`\n共 ${output.videos.length} 个视频, 用户: @${videosUsername}`);
35
+ } catch (err) {
36
+ console.error(`获取用户视频失败: ${err.message}`);
37
+ process.exit(1);
38
+ } finally {
39
+ if (browser) await browser.close().catch(() => {});
40
+ }
41
+ }
@@ -1,196 +1,196 @@
1
- import path from "path";
2
- import Database from "better-sqlite3";
3
- import { TikTokScraper } from "../lib/tiktok-scraper.mjs";
4
-
5
- function resolveVideoSource(inputPath) {
6
- if (!inputPath) return null;
7
-
8
- const resolved = path.resolve(inputPath);
9
- const ext = path.extname(resolved).toLowerCase();
10
-
11
- if (ext === ".db") {
12
- return { kind: "db", dbPath: resolved, label: resolved };
13
- }
14
-
15
- return null;
16
- }
17
-
18
- function ensureVideoStatsColumns(db) {
19
- const existing = new Set(
20
- db
21
- .prepare("PRAGMA table_info(videos)")
22
- .all()
23
- .map((column) => column.name),
24
- );
25
- const required = {
26
- play_count: "INTEGER",
27
- digg_count: "INTEGER",
28
- comment_count: "INTEGER",
29
- share_count: "INTEGER",
30
- collect_count: "INTEGER",
31
- stats_updated_at: "INTEGER",
32
- };
33
- for (const [column, type] of Object.entries(required)) {
34
- if (!existing.has(column)) {
35
- db.exec(`ALTER TABLE videos ADD COLUMN ${column} ${type}`);
36
- }
37
- }
38
- }
39
-
40
- function loadVideosFromDb(dbPath) {
41
- const db = new Database(dbPath);
42
- ensureVideoStatsColumns(db);
43
- const rows = db
44
- .prepare("SELECT * FROM videos ORDER BY registered_at ASC")
45
- .all();
46
- const videos = rows.map((row) => ({
47
- id: row.id,
48
- href: row.href,
49
- authorUniqueId: row.author_unique_id,
50
- locationCreated: row.location_created,
51
- ttSeller: !!row.tt_seller,
52
- registeredAt: row.registered_at,
53
- userUpdateCount: row.user_update_count,
54
- stats:
55
- row.play_count === null &&
56
- row.digg_count === null &&
57
- row.comment_count === null &&
58
- row.share_count === null &&
59
- row.collect_count === null
60
- ? undefined
61
- : {
62
- playCount: row.play_count,
63
- diggCount: row.digg_count,
64
- commentCount: row.comment_count,
65
- shareCount: row.share_count,
66
- collectCount: row.collect_count,
67
- },
68
- }));
69
- return { db, videos };
70
- }
71
-
72
- function writeVideosToDb(db, videos) {
73
- const stmt = db.prepare(`
74
- UPDATE videos
75
- SET play_count = ?,
76
- digg_count = ?,
77
- comment_count = ?,
78
- share_count = ?,
79
- collect_count = ?,
80
- stats_updated_at = ?
81
- WHERE id = ?
82
- `);
83
- const now = Date.now();
84
- const txn = db.transaction((items) => {
85
- for (const video of items) {
86
- if (!video.stats) continue;
87
- stmt.run(
88
- video.stats.playCount ?? null,
89
- video.stats.diggCount ?? null,
90
- video.stats.commentCount ?? null,
91
- video.stats.shareCount ?? null,
92
- video.stats.collectCount ?? null,
93
- now,
94
- video.id,
95
- );
96
- }
97
- });
98
- txn(videos);
99
- }
100
-
101
- export async function handleVideoStats(options) {
102
- const statsSource = options.statsSource || options.statsFile;
103
- const { statsParallel } = options;
104
-
105
- if (!statsSource) {
106
- console.error("用法: tt-help videostats <db路径> -p <并发数>");
107
- console.error(" tt-help videostats data/result.db -p 3");
108
- console.error("");
109
- console.error("选项: -p, --parallel <N> 并发数(默认: 3)");
110
- process.exit(1);
111
- }
112
-
113
- const source = resolveVideoSource(statsSource);
114
- if (!source) {
115
- console.error("[videostats] 仅支持 .db 路径");
116
- process.exit(1);
117
- }
118
-
119
- let db = null;
120
- let videos = [];
121
- ({ db, videos } = loadVideosFromDb(source.dbPath));
122
-
123
- if (!Array.isArray(videos) || videos.length === 0) {
124
- console.error("[videostats] 视频源为空或格式错误");
125
- db?.close();
126
- process.exit(1);
127
- }
128
-
129
- const poolSize = Math.min(statsParallel, videos.length);
130
- console.error(
131
- `[videostats] 读取 ${source.label}: ${videos.length} 个视频,并发: ${poolSize}`,
132
- );
133
-
134
- // 初始化 scraper
135
- const scraper = new TikTokScraper({ poolSize });
136
- await scraper.init();
137
- console.error("[videostats] 浏览器初始化完成,开始刷新...");
138
-
139
- let success = 0,
140
- failed = 0,
141
- skipped = 0;
142
-
143
- try {
144
- for (let i = 0; i < videos.length; i += poolSize) {
145
- const batch = videos.slice(i, i + poolSize);
146
-
147
- const results = await Promise.allSettled(
148
- batch.map(async (v) => {
149
- if (!v.href) return { video: v, error: "no href" };
150
- try {
151
- const info = await scraper.getVideoInfo(v.href);
152
- if (!info) return { video: v, error: "no info" };
153
- return { video: v, info };
154
- } catch (err) {
155
- return { video: v, error: err.message };
156
- }
157
- }),
158
- );
159
-
160
- for (const result of results) {
161
- if (result.status === "fulfilled") {
162
- const { video, info, error } = result.value;
163
- if (info && info.stats) {
164
- video.stats = info.stats;
165
- success++;
166
- } else {
167
- failed++;
168
- const id = video.id || video.href || "?";
169
- console.error(
170
- ` [失败] ${id.substring(0, 30)} - ${error || "获取失败"}`,
171
- );
172
- }
173
- } else {
174
- failed++;
175
- }
176
- }
177
-
178
- const processed = Math.min(i + poolSize, videos.length);
179
- console.error(
180
- `[videostats] 进度: ${processed}/${videos.length} (成功: ${success}, 失败: ${failed})`,
181
- );
182
- }
183
-
184
- writeVideosToDb(db, videos);
185
- console.error(``);
186
- console.error(`[videostats] 完成: 成功 ${success}, 失败 ${failed}`);
187
- console.error(`[videostats] 已更新 ${source.dbPath}`);
188
- } catch (err) {
189
- console.error(`[videostats] 异常: ${err.message}`);
190
- db?.close();
191
- process.exit(1);
192
- } finally {
193
- db?.close();
194
- await scraper.close();
195
- }
196
- }
1
+ import path from "path";
2
+ import Database from "better-sqlite3";
3
+ import { TikTokScraper } from "../lib/tiktok-scraper.mjs";
4
+
5
+ function resolveVideoSource(inputPath) {
6
+ if (!inputPath) return null;
7
+
8
+ const resolved = path.resolve(inputPath);
9
+ const ext = path.extname(resolved).toLowerCase();
10
+
11
+ if (ext === ".db") {
12
+ return { kind: "db", dbPath: resolved, label: resolved };
13
+ }
14
+
15
+ return null;
16
+ }
17
+
18
+ function ensureVideoStatsColumns(db) {
19
+ const existing = new Set(
20
+ db
21
+ .prepare("PRAGMA table_info(videos)")
22
+ .all()
23
+ .map((column) => column.name),
24
+ );
25
+ const required = {
26
+ play_count: "INTEGER",
27
+ digg_count: "INTEGER",
28
+ comment_count: "INTEGER",
29
+ share_count: "INTEGER",
30
+ collect_count: "INTEGER",
31
+ stats_updated_at: "INTEGER",
32
+ };
33
+ for (const [column, type] of Object.entries(required)) {
34
+ if (!existing.has(column)) {
35
+ db.exec(`ALTER TABLE videos ADD COLUMN ${column} ${type}`);
36
+ }
37
+ }
38
+ }
39
+
40
+ function loadVideosFromDb(dbPath) {
41
+ const db = new Database(dbPath);
42
+ ensureVideoStatsColumns(db);
43
+ const rows = db
44
+ .prepare("SELECT * FROM videos ORDER BY registered_at ASC")
45
+ .all();
46
+ const videos = rows.map((row) => ({
47
+ id: row.id,
48
+ href: row.href,
49
+ authorUniqueId: row.author_unique_id,
50
+ locationCreated: row.location_created,
51
+ ttSeller: !!row.tt_seller,
52
+ registeredAt: row.registered_at,
53
+ userUpdateCount: row.user_update_count,
54
+ stats:
55
+ row.play_count === null &&
56
+ row.digg_count === null &&
57
+ row.comment_count === null &&
58
+ row.share_count === null &&
59
+ row.collect_count === null
60
+ ? undefined
61
+ : {
62
+ playCount: row.play_count,
63
+ diggCount: row.digg_count,
64
+ commentCount: row.comment_count,
65
+ shareCount: row.share_count,
66
+ collectCount: row.collect_count,
67
+ },
68
+ }));
69
+ return { db, videos };
70
+ }
71
+
72
+ function writeVideosToDb(db, videos) {
73
+ const stmt = db.prepare(`
74
+ UPDATE videos
75
+ SET play_count = ?,
76
+ digg_count = ?,
77
+ comment_count = ?,
78
+ share_count = ?,
79
+ collect_count = ?,
80
+ stats_updated_at = ?
81
+ WHERE id = ?
82
+ `);
83
+ const now = Date.now();
84
+ const txn = db.transaction((items) => {
85
+ for (const video of items) {
86
+ if (!video.stats) continue;
87
+ stmt.run(
88
+ video.stats.playCount ?? null,
89
+ video.stats.diggCount ?? null,
90
+ video.stats.commentCount ?? null,
91
+ video.stats.shareCount ?? null,
92
+ video.stats.collectCount ?? null,
93
+ now,
94
+ video.id,
95
+ );
96
+ }
97
+ });
98
+ txn(videos);
99
+ }
100
+
101
+ export async function handleVideoStats(options) {
102
+ const statsSource = options.statsSource || options.statsFile;
103
+ const { statsParallel } = options;
104
+
105
+ if (!statsSource) {
106
+ console.error("用法: tt-help videostats <db路径> -p <并发数>");
107
+ console.error(" tt-help videostats data/result.db -p 3");
108
+ console.error("");
109
+ console.error("选项: -p, --parallel <N> 并发数(默认: 3)");
110
+ process.exit(1);
111
+ }
112
+
113
+ const source = resolveVideoSource(statsSource);
114
+ if (!source) {
115
+ console.error("[videostats] 仅支持 .db 路径");
116
+ process.exit(1);
117
+ }
118
+
119
+ let db = null;
120
+ let videos = [];
121
+ ({ db, videos } = loadVideosFromDb(source.dbPath));
122
+
123
+ if (!Array.isArray(videos) || videos.length === 0) {
124
+ console.error("[videostats] 视频源为空或格式错误");
125
+ db?.close();
126
+ process.exit(1);
127
+ }
128
+
129
+ const poolSize = Math.min(statsParallel, videos.length);
130
+ console.error(
131
+ `[videostats] 读取 ${source.label}: ${videos.length} 个视频,并发: ${poolSize}`,
132
+ );
133
+
134
+ // 初始化 scraper
135
+ const scraper = new TikTokScraper({ poolSize });
136
+ await scraper.init();
137
+ console.error("[videostats] 浏览器初始化完成,开始刷新...");
138
+
139
+ let success = 0,
140
+ failed = 0,
141
+ skipped = 0;
142
+
143
+ try {
144
+ for (let i = 0; i < videos.length; i += poolSize) {
145
+ const batch = videos.slice(i, i + poolSize);
146
+
147
+ const results = await Promise.allSettled(
148
+ batch.map(async (v) => {
149
+ if (!v.href) return { video: v, error: "no href" };
150
+ try {
151
+ const info = await scraper.getVideoInfo(v.href);
152
+ if (!info) return { video: v, error: "no info" };
153
+ return { video: v, info };
154
+ } catch (err) {
155
+ return { video: v, error: err.message };
156
+ }
157
+ }),
158
+ );
159
+
160
+ for (const result of results) {
161
+ if (result.status === "fulfilled") {
162
+ const { video, info, error } = result.value;
163
+ if (info && info.stats) {
164
+ video.stats = info.stats;
165
+ success++;
166
+ } else {
167
+ failed++;
168
+ const id = video.id || video.href || "?";
169
+ console.error(
170
+ ` [失败] ${id.substring(0, 30)} - ${error || "获取失败"}`,
171
+ );
172
+ }
173
+ } else {
174
+ failed++;
175
+ }
176
+ }
177
+
178
+ const processed = Math.min(i + poolSize, videos.length);
179
+ console.error(
180
+ `[videostats] 进度: ${processed}/${videos.length} (成功: ${success}, 失败: ${failed})`,
181
+ );
182
+ }
183
+
184
+ writeVideosToDb(db, videos);
185
+ console.error(``);
186
+ console.error(`[videostats] 完成: 成功 ${success}, 失败 ${failed}`);
187
+ console.error(`[videostats] 已更新 ${source.dbPath}`);
188
+ } catch (err) {
189
+ console.error(`[videostats] 异常: ${err.message}`);
190
+ db?.close();
191
+ process.exit(1);
192
+ } finally {
193
+ db?.close();
194
+ await scraper.close();
195
+ }
196
+ }
package/src/cli/watch.js CHANGED
@@ -1,30 +1,30 @@
1
- import { mkdirSync } from "fs";
2
- import path from "path";
3
- import { createStore } from "../watch/data-store.js";
4
- import { startWatchServer, openBrowser } from "../watch/server.js";
5
-
6
- export async function handleWatch(options) {
7
- const dataAnchor = options.dataAnchor || options.outputFile;
8
- const { watchPort } = options;
9
-
10
- if (!dataAnchor) {
11
- console.error("用法: tt-help watch -o <db路径> [-p 端口]");
12
- console.error(" tt-help watch -o data/result.db");
13
- console.error(" tt-help watch -o data/result.db -p 8080");
14
- process.exit(1);
15
- }
16
-
17
- mkdirSync(path.dirname(path.resolve(dataAnchor)), { recursive: true });
18
-
19
- const store = createStore(dataAnchor);
20
- const { server, port } = await startWatchServer(dataAnchor, watchPort, store);
21
- openBrowser(port);
22
-
23
- process.once("SIGINT", () => {
24
- store.stopBackup();
25
- server.close();
26
- process.exit(0);
27
- });
28
-
29
- console.error("按 Ctrl+C 停止监控服务");
30
- }
1
+ import { mkdirSync } from "fs";
2
+ import path from "path";
3
+ import { createStore } from "../watch/data-store.js";
4
+ import { startWatchServer, openBrowser } from "../watch/server.js";
5
+
6
+ export async function handleWatch(options) {
7
+ const dataAnchor = options.dataAnchor || options.outputFile;
8
+ const { watchPort } = options;
9
+
10
+ if (!dataAnchor) {
11
+ console.error("用法: tt-help watch -o <db路径> [-p 端口]");
12
+ console.error(" tt-help watch -o data/result.db");
13
+ console.error(" tt-help watch -o data/result.db -p 8080");
14
+ process.exit(1);
15
+ }
16
+
17
+ mkdirSync(path.dirname(path.resolve(dataAnchor)), { recursive: true });
18
+
19
+ const store = createStore(dataAnchor);
20
+ const { server, port } = await startWatchServer(dataAnchor, watchPort, store);
21
+ openBrowser(port);
22
+
23
+ process.once("SIGINT", () => {
24
+ store.stopBackup();
25
+ server.close();
26
+ process.exit(0);
27
+ });
28
+
29
+ console.error("按 Ctrl+C 停止监控服务");
30
+ }