tt-help-cli-ycl 1.3.44 → 1.3.46

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. package/README.md +33 -33
  2. package/cli.js +9 -9
  3. package/package.json +52 -52
  4. package/scripts/run-explore copy.bat +101 -101
  5. package/scripts/run-explore.bat +134 -134
  6. package/scripts/run-explore.ps1 +159 -159
  7. package/scripts/run-explore.sh +121 -121
  8. package/src/cli/attach.js +331 -301
  9. package/src/cli/auto.js +265 -265
  10. package/src/cli/comments.js +620 -620
  11. package/src/cli/config.js +170 -170
  12. package/src/cli/db-import.js +51 -51
  13. package/src/cli/explore.js +555 -555
  14. package/src/cli/info.js +10 -16
  15. package/src/cli/open.js +111 -111
  16. package/src/cli/progress.js +111 -111
  17. package/src/cli/refresh.js +288 -288
  18. package/src/cli/scrape.js +47 -47
  19. package/src/cli/utils.js +18 -18
  20. package/src/cli/videos.js +41 -41
  21. package/src/cli/videostats.js +196 -196
  22. package/src/cli/watch.js +30 -30
  23. package/src/cli/webserver.js +19 -0
  24. package/src/lib/api-interceptor.js +161 -161
  25. package/src/lib/args.js +809 -771
  26. package/src/lib/browser/anti-detect.js +23 -23
  27. package/src/lib/browser/cdp.js +261 -261
  28. package/src/lib/browser/health-checker.js +114 -114
  29. package/src/lib/browser/launch.js +43 -43
  30. package/src/lib/browser/page.js +184 -184
  31. package/src/lib/constants.js +297 -285
  32. package/src/lib/delay.js +54 -54
  33. package/src/lib/explore-fetch.js +118 -118
  34. package/src/lib/fetcher.js +45 -45
  35. package/src/lib/filter.js +66 -66
  36. package/src/lib/io.js +54 -54
  37. package/src/lib/output.js +80 -80
  38. package/src/lib/page-error-detector.js +109 -109
  39. package/src/lib/parse-ssr.mjs +69 -69
  40. package/src/lib/parser.js +47 -47
  41. package/src/lib/retry.js +45 -45
  42. package/src/lib/scrape.js +90 -89
  43. package/src/lib/target-locations.js +61 -61
  44. package/src/lib/tiktok-scraper.mjs +160 -106
  45. package/src/lib/url.js +52 -52
  46. package/src/main.js +73 -70
  47. package/src/npm-main.js +70 -69
  48. package/src/scraper/auto-core.js +203 -203
  49. package/src/scraper/core.js +255 -255
  50. package/src/scraper/explore-core.js +208 -208
  51. package/src/scraper/modules/captcha-handler.js +114 -114
  52. package/src/scraper/modules/follow-extractor.js +250 -250
  53. package/src/scraper/modules/guess-extractor.js +51 -51
  54. package/src/scraper/modules/page-helpers.js +48 -48
  55. package/src/scraper/refresh-core.js +213 -213
  56. package/src/videos/core.js +143 -143
  57. package/src/watch/data-store.js +2980 -2846
  58. package/src/watch/public/index.html +2355 -2285
  59. package/src/watch/server.js +727 -711
  60. package/src/webserver/server.mjs +174 -0
  61. package/scripts/test-captcha-lib.mjs +0 -68
  62. package/scripts/test-captcha.mjs +0 -81
  63. package/scripts/test-incognito-lib.mjs +0 -36
  64. package/scripts/test-login-state.mjs +0 -128
  65. package/scripts/test-safe-click.mjs +0 -45
  66. package/scripts/test-watch-db-smoke.mjs +0 -246
  67. package/src/results/user-videos-bar.lar.lar.moeta.json +0 -37
package/src/cli/utils.js CHANGED
@@ -1,18 +1,18 @@
1
- import { writeFileSync } from 'fs';
2
-
3
- export function cleanError(msg) {
4
- return msg
5
- .replace(/\x1b\[[0-9;]*m/g, '')
6
- .replace(/\s*- navigating to.*/s, '')
7
- .replace(/\s*Call log:/s, '')
8
- .trim();
9
- }
10
-
11
- export function writeJson(data, outputFile) {
12
- const json = JSON.stringify(data, null, 2);
13
- if (outputFile) {
14
- writeFileSync(outputFile, json, 'utf-8');
15
- } else {
16
- process.stdout.write(json + '\n');
17
- }
18
- }
1
+ import { writeFileSync } from 'fs';
2
+
3
+ export function cleanError(msg) {
4
+ return msg
5
+ .replace(/\x1b\[[0-9;]*m/g, '')
6
+ .replace(/\s*- navigating to.*/s, '')
7
+ .replace(/\s*Call log:/s, '')
8
+ .trim();
9
+ }
10
+
11
+ export function writeJson(data, outputFile) {
12
+ const json = JSON.stringify(data, null, 2);
13
+ if (outputFile) {
14
+ writeFileSync(outputFile, json, 'utf-8');
15
+ } else {
16
+ process.stdout.write(json + '\n');
17
+ }
18
+ }
package/src/cli/videos.js CHANGED
@@ -1,41 +1,41 @@
1
- import { writeFileSync } from 'fs';
2
-
3
- export async function handleVideos(options) {
4
- const { videosUsername, videosMax, outputFile } = options;
5
-
6
- if (!videosUsername) {
7
- console.error('用法: tt-help videos <用户名> [最大视频数] [-o 输出路径]');
8
- console.error('示例: tt-help videos bar.lar.lar.moeta 1000');
9
- console.error(' tt-help videos username 50 -o videos.json');
10
- console.error('');
11
- console.error('选项: -o, --output <路径> 输出到文件(默认输出到 stdout)');
12
- process.exit(1);
13
- }
14
-
15
- const { runGetUserVideos } = await import('../videos/core.js');
16
-
17
- let browser;
18
- try {
19
- const { output, browser: b } = await runGetUserVideos({
20
- username: videosUsername,
21
- maxVideos: videosMax,
22
- log: console.error,
23
- });
24
- browser = b;
25
-
26
- const json = JSON.stringify(output, null, 2);
27
- if (outputFile) {
28
- writeFileSync(outputFile, json, 'utf-8');
29
- console.error(`结果已写入: ${outputFile}`);
30
- } else {
31
- process.stdout.write(json + '\n');
32
- }
33
-
34
- console.error(`\n共 ${output.videos.length} 个视频, 用户: @${videosUsername}`);
35
- } catch (err) {
36
- console.error(`获取用户视频失败: ${err.message}`);
37
- process.exit(1);
38
- } finally {
39
- if (browser) await browser.close().catch(() => {});
40
- }
41
- }
1
+ import { writeFileSync } from 'fs';
2
+
3
+ export async function handleVideos(options) {
4
+ const { videosUsername, videosMax, outputFile } = options;
5
+
6
+ if (!videosUsername) {
7
+ console.error('用法: tt-help videos <用户名> [最大视频数] [-o 输出路径]');
8
+ console.error('示例: tt-help videos bar.lar.lar.moeta 1000');
9
+ console.error(' tt-help videos username 50 -o videos.json');
10
+ console.error('');
11
+ console.error('选项: -o, --output <路径> 输出到文件(默认输出到 stdout)');
12
+ process.exit(1);
13
+ }
14
+
15
+ const { runGetUserVideos } = await import('../videos/core.js');
16
+
17
+ let browser;
18
+ try {
19
+ const { output, browser: b } = await runGetUserVideos({
20
+ username: videosUsername,
21
+ maxVideos: videosMax,
22
+ log: console.error,
23
+ });
24
+ browser = b;
25
+
26
+ const json = JSON.stringify(output, null, 2);
27
+ if (outputFile) {
28
+ writeFileSync(outputFile, json, 'utf-8');
29
+ console.error(`结果已写入: ${outputFile}`);
30
+ } else {
31
+ process.stdout.write(json + '\n');
32
+ }
33
+
34
+ console.error(`\n共 ${output.videos.length} 个视频, 用户: @${videosUsername}`);
35
+ } catch (err) {
36
+ console.error(`获取用户视频失败: ${err.message}`);
37
+ process.exit(1);
38
+ } finally {
39
+ if (browser) await browser.close().catch(() => {});
40
+ }
41
+ }
@@ -1,196 +1,196 @@
1
- import path from "path";
2
- import Database from "better-sqlite3";
3
- import { TikTokScraper } from "../lib/tiktok-scraper.mjs";
4
-
5
- function resolveVideoSource(inputPath) {
6
- if (!inputPath) return null;
7
-
8
- const resolved = path.resolve(inputPath);
9
- const ext = path.extname(resolved).toLowerCase();
10
-
11
- if (ext === ".db") {
12
- return { kind: "db", dbPath: resolved, label: resolved };
13
- }
14
-
15
- return null;
16
- }
17
-
18
- function ensureVideoStatsColumns(db) {
19
- const existing = new Set(
20
- db
21
- .prepare("PRAGMA table_info(videos)")
22
- .all()
23
- .map((column) => column.name),
24
- );
25
- const required = {
26
- play_count: "INTEGER",
27
- digg_count: "INTEGER",
28
- comment_count: "INTEGER",
29
- share_count: "INTEGER",
30
- collect_count: "INTEGER",
31
- stats_updated_at: "INTEGER",
32
- };
33
- for (const [column, type] of Object.entries(required)) {
34
- if (!existing.has(column)) {
35
- db.exec(`ALTER TABLE videos ADD COLUMN ${column} ${type}`);
36
- }
37
- }
38
- }
39
-
40
- function loadVideosFromDb(dbPath) {
41
- const db = new Database(dbPath);
42
- ensureVideoStatsColumns(db);
43
- const rows = db
44
- .prepare("SELECT * FROM videos ORDER BY registered_at ASC")
45
- .all();
46
- const videos = rows.map((row) => ({
47
- id: row.id,
48
- href: row.href,
49
- authorUniqueId: row.author_unique_id,
50
- locationCreated: row.location_created,
51
- ttSeller: !!row.tt_seller,
52
- registeredAt: row.registered_at,
53
- userUpdateCount: row.user_update_count,
54
- stats:
55
- row.play_count === null &&
56
- row.digg_count === null &&
57
- row.comment_count === null &&
58
- row.share_count === null &&
59
- row.collect_count === null
60
- ? undefined
61
- : {
62
- playCount: row.play_count,
63
- diggCount: row.digg_count,
64
- commentCount: row.comment_count,
65
- shareCount: row.share_count,
66
- collectCount: row.collect_count,
67
- },
68
- }));
69
- return { db, videos };
70
- }
71
-
72
- function writeVideosToDb(db, videos) {
73
- const stmt = db.prepare(`
74
- UPDATE videos
75
- SET play_count = ?,
76
- digg_count = ?,
77
- comment_count = ?,
78
- share_count = ?,
79
- collect_count = ?,
80
- stats_updated_at = ?
81
- WHERE id = ?
82
- `);
83
- const now = Date.now();
84
- const txn = db.transaction((items) => {
85
- for (const video of items) {
86
- if (!video.stats) continue;
87
- stmt.run(
88
- video.stats.playCount ?? null,
89
- video.stats.diggCount ?? null,
90
- video.stats.commentCount ?? null,
91
- video.stats.shareCount ?? null,
92
- video.stats.collectCount ?? null,
93
- now,
94
- video.id,
95
- );
96
- }
97
- });
98
- txn(videos);
99
- }
100
-
101
- export async function handleVideoStats(options) {
102
- const statsSource = options.statsSource || options.statsFile;
103
- const { statsParallel } = options;
104
-
105
- if (!statsSource) {
106
- console.error("用法: tt-help videostats <db路径> -p <并发数>");
107
- console.error(" tt-help videostats data/result.db -p 3");
108
- console.error("");
109
- console.error("选项: -p, --parallel <N> 并发数(默认: 3)");
110
- process.exit(1);
111
- }
112
-
113
- const source = resolveVideoSource(statsSource);
114
- if (!source) {
115
- console.error("[videostats] 仅支持 .db 路径");
116
- process.exit(1);
117
- }
118
-
119
- let db = null;
120
- let videos = [];
121
- ({ db, videos } = loadVideosFromDb(source.dbPath));
122
-
123
- if (!Array.isArray(videos) || videos.length === 0) {
124
- console.error("[videostats] 视频源为空或格式错误");
125
- db?.close();
126
- process.exit(1);
127
- }
128
-
129
- const poolSize = Math.min(statsParallel, videos.length);
130
- console.error(
131
- `[videostats] 读取 ${source.label}: ${videos.length} 个视频,并发: ${poolSize}`,
132
- );
133
-
134
- // 初始化 scraper
135
- const scraper = new TikTokScraper({ poolSize });
136
- await scraper.init();
137
- console.error("[videostats] 浏览器初始化完成,开始刷新...");
138
-
139
- let success = 0,
140
- failed = 0,
141
- skipped = 0;
142
-
143
- try {
144
- for (let i = 0; i < videos.length; i += poolSize) {
145
- const batch = videos.slice(i, i + poolSize);
146
-
147
- const results = await Promise.allSettled(
148
- batch.map(async (v) => {
149
- if (!v.href) return { video: v, error: "no href" };
150
- try {
151
- const info = await scraper.getVideoInfo(v.href);
152
- if (!info) return { video: v, error: "no info" };
153
- return { video: v, info };
154
- } catch (err) {
155
- return { video: v, error: err.message };
156
- }
157
- }),
158
- );
159
-
160
- for (const result of results) {
161
- if (result.status === "fulfilled") {
162
- const { video, info, error } = result.value;
163
- if (info && info.stats) {
164
- video.stats = info.stats;
165
- success++;
166
- } else {
167
- failed++;
168
- const id = video.id || video.href || "?";
169
- console.error(
170
- ` [失败] ${id.substring(0, 30)} - ${error || "获取失败"}`,
171
- );
172
- }
173
- } else {
174
- failed++;
175
- }
176
- }
177
-
178
- const processed = Math.min(i + poolSize, videos.length);
179
- console.error(
180
- `[videostats] 进度: ${processed}/${videos.length} (成功: ${success}, 失败: ${failed})`,
181
- );
182
- }
183
-
184
- writeVideosToDb(db, videos);
185
- console.error(``);
186
- console.error(`[videostats] 完成: 成功 ${success}, 失败 ${failed}`);
187
- console.error(`[videostats] 已更新 ${source.dbPath}`);
188
- } catch (err) {
189
- console.error(`[videostats] 异常: ${err.message}`);
190
- db?.close();
191
- process.exit(1);
192
- } finally {
193
- db?.close();
194
- await scraper.close();
195
- }
196
- }
1
+ import path from "path";
2
+ import Database from "better-sqlite3";
3
+ import { TikTokScraper } from "../lib/tiktok-scraper.mjs";
4
+
5
+ function resolveVideoSource(inputPath) {
6
+ if (!inputPath) return null;
7
+
8
+ const resolved = path.resolve(inputPath);
9
+ const ext = path.extname(resolved).toLowerCase();
10
+
11
+ if (ext === ".db") {
12
+ return { kind: "db", dbPath: resolved, label: resolved };
13
+ }
14
+
15
+ return null;
16
+ }
17
+
18
+ function ensureVideoStatsColumns(db) {
19
+ const existing = new Set(
20
+ db
21
+ .prepare("PRAGMA table_info(videos)")
22
+ .all()
23
+ .map((column) => column.name),
24
+ );
25
+ const required = {
26
+ play_count: "INTEGER",
27
+ digg_count: "INTEGER",
28
+ comment_count: "INTEGER",
29
+ share_count: "INTEGER",
30
+ collect_count: "INTEGER",
31
+ stats_updated_at: "INTEGER",
32
+ };
33
+ for (const [column, type] of Object.entries(required)) {
34
+ if (!existing.has(column)) {
35
+ db.exec(`ALTER TABLE videos ADD COLUMN ${column} ${type}`);
36
+ }
37
+ }
38
+ }
39
+
40
+ function loadVideosFromDb(dbPath) {
41
+ const db = new Database(dbPath);
42
+ ensureVideoStatsColumns(db);
43
+ const rows = db
44
+ .prepare("SELECT * FROM videos ORDER BY registered_at ASC")
45
+ .all();
46
+ const videos = rows.map((row) => ({
47
+ id: row.id,
48
+ href: row.href,
49
+ authorUniqueId: row.author_unique_id,
50
+ locationCreated: row.location_created,
51
+ ttSeller: !!row.tt_seller,
52
+ registeredAt: row.registered_at,
53
+ userUpdateCount: row.user_update_count,
54
+ stats:
55
+ row.play_count === null &&
56
+ row.digg_count === null &&
57
+ row.comment_count === null &&
58
+ row.share_count === null &&
59
+ row.collect_count === null
60
+ ? undefined
61
+ : {
62
+ playCount: row.play_count,
63
+ diggCount: row.digg_count,
64
+ commentCount: row.comment_count,
65
+ shareCount: row.share_count,
66
+ collectCount: row.collect_count,
67
+ },
68
+ }));
69
+ return { db, videos };
70
+ }
71
+
72
+ function writeVideosToDb(db, videos) {
73
+ const stmt = db.prepare(`
74
+ UPDATE videos
75
+ SET play_count = ?,
76
+ digg_count = ?,
77
+ comment_count = ?,
78
+ share_count = ?,
79
+ collect_count = ?,
80
+ stats_updated_at = ?
81
+ WHERE id = ?
82
+ `);
83
+ const now = Date.now();
84
+ const txn = db.transaction((items) => {
85
+ for (const video of items) {
86
+ if (!video.stats) continue;
87
+ stmt.run(
88
+ video.stats.playCount ?? null,
89
+ video.stats.diggCount ?? null,
90
+ video.stats.commentCount ?? null,
91
+ video.stats.shareCount ?? null,
92
+ video.stats.collectCount ?? null,
93
+ now,
94
+ video.id,
95
+ );
96
+ }
97
+ });
98
+ txn(videos);
99
+ }
100
+
101
+ export async function handleVideoStats(options) {
102
+ const statsSource = options.statsSource || options.statsFile;
103
+ const { statsParallel } = options;
104
+
105
+ if (!statsSource) {
106
+ console.error("用法: tt-help videostats <db路径> -p <并发数>");
107
+ console.error(" tt-help videostats data/result.db -p 3");
108
+ console.error("");
109
+ console.error("选项: -p, --parallel <N> 并发数(默认: 3)");
110
+ process.exit(1);
111
+ }
112
+
113
+ const source = resolveVideoSource(statsSource);
114
+ if (!source) {
115
+ console.error("[videostats] 仅支持 .db 路径");
116
+ process.exit(1);
117
+ }
118
+
119
+ let db = null;
120
+ let videos = [];
121
+ ({ db, videos } = loadVideosFromDb(source.dbPath));
122
+
123
+ if (!Array.isArray(videos) || videos.length === 0) {
124
+ console.error("[videostats] 视频源为空或格式错误");
125
+ db?.close();
126
+ process.exit(1);
127
+ }
128
+
129
+ const poolSize = Math.min(statsParallel, videos.length);
130
+ console.error(
131
+ `[videostats] 读取 ${source.label}: ${videos.length} 个视频,并发: ${poolSize}`,
132
+ );
133
+
134
+ // 初始化 scraper
135
+ const scraper = new TikTokScraper({ poolSize });
136
+ await scraper.init();
137
+ console.error("[videostats] 浏览器初始化完成,开始刷新...");
138
+
139
+ let success = 0,
140
+ failed = 0,
141
+ skipped = 0;
142
+
143
+ try {
144
+ for (let i = 0; i < videos.length; i += poolSize) {
145
+ const batch = videos.slice(i, i + poolSize);
146
+
147
+ const results = await Promise.allSettled(
148
+ batch.map(async (v) => {
149
+ if (!v.href) return { video: v, error: "no href" };
150
+ try {
151
+ const info = await scraper.getVideoInfo(v.href);
152
+ if (!info) return { video: v, error: "no info" };
153
+ return { video: v, info };
154
+ } catch (err) {
155
+ return { video: v, error: err.message };
156
+ }
157
+ }),
158
+ );
159
+
160
+ for (const result of results) {
161
+ if (result.status === "fulfilled") {
162
+ const { video, info, error } = result.value;
163
+ if (info && info.stats) {
164
+ video.stats = info.stats;
165
+ success++;
166
+ } else {
167
+ failed++;
168
+ const id = video.id || video.href || "?";
169
+ console.error(
170
+ ` [失败] ${id.substring(0, 30)} - ${error || "获取失败"}`,
171
+ );
172
+ }
173
+ } else {
174
+ failed++;
175
+ }
176
+ }
177
+
178
+ const processed = Math.min(i + poolSize, videos.length);
179
+ console.error(
180
+ `[videostats] 进度: ${processed}/${videos.length} (成功: ${success}, 失败: ${failed})`,
181
+ );
182
+ }
183
+
184
+ writeVideosToDb(db, videos);
185
+ console.error(``);
186
+ console.error(`[videostats] 完成: 成功 ${success}, 失败 ${failed}`);
187
+ console.error(`[videostats] 已更新 ${source.dbPath}`);
188
+ } catch (err) {
189
+ console.error(`[videostats] 异常: ${err.message}`);
190
+ db?.close();
191
+ process.exit(1);
192
+ } finally {
193
+ db?.close();
194
+ await scraper.close();
195
+ }
196
+ }
package/src/cli/watch.js CHANGED
@@ -1,30 +1,30 @@
1
- import { mkdirSync } from "fs";
2
- import path from "path";
3
- import { createStore } from "../watch/data-store.js";
4
- import { startWatchServer, openBrowser } from "../watch/server.js";
5
-
6
- export async function handleWatch(options) {
7
- const dataAnchor = options.dataAnchor || options.outputFile;
8
- const { watchPort } = options;
9
-
10
- if (!dataAnchor) {
11
- console.error("用法: tt-help watch -o <db路径> [-p 端口]");
12
- console.error(" tt-help watch -o data/result.db");
13
- console.error(" tt-help watch -o data/result.db -p 8080");
14
- process.exit(1);
15
- }
16
-
17
- mkdirSync(path.dirname(path.resolve(dataAnchor)), { recursive: true });
18
-
19
- const store = createStore(dataAnchor);
20
- const { server, port } = await startWatchServer(dataAnchor, watchPort, store);
21
- openBrowser(port);
22
-
23
- process.once("SIGINT", () => {
24
- store.stopBackup();
25
- server.close();
26
- process.exit(0);
27
- });
28
-
29
- console.error("按 Ctrl+C 停止监控服务");
30
- }
1
+ import { mkdirSync } from "fs";
2
+ import path from "path";
3
+ import { createStore } from "../watch/data-store.js";
4
+ import { startWatchServer, openBrowser } from "../watch/server.js";
5
+
6
+ export async function handleWatch(options) {
7
+ const dataAnchor = options.dataAnchor || options.outputFile;
8
+ const { watchPort } = options;
9
+
10
+ if (!dataAnchor) {
11
+ console.error("用法: tt-help watch -o <db路径> [-p 端口]");
12
+ console.error(" tt-help watch -o data/result.db");
13
+ console.error(" tt-help watch -o data/result.db -p 8080");
14
+ process.exit(1);
15
+ }
16
+
17
+ mkdirSync(path.dirname(path.resolve(dataAnchor)), { recursive: true });
18
+
19
+ const store = createStore(dataAnchor);
20
+ const { server, port } = await startWatchServer(dataAnchor, watchPort, store);
21
+ openBrowser(port);
22
+
23
+ process.once("SIGINT", () => {
24
+ store.stopBackup();
25
+ server.close();
26
+ process.exit(0);
27
+ });
28
+
29
+ console.error("按 Ctrl+C 停止监控服务");
30
+ }
@@ -0,0 +1,19 @@
1
+ import { startWebserver, closeScraper } from "../webserver/server.mjs";
2
+
3
+ export async function handleWebserver(options) {
4
+ const { webserverPort } = options;
5
+
6
+ const { server, port } = await startWebserver(webserverPort);
7
+
8
+ process.once("SIGINT", async () => {
9
+ await closeScraper();
10
+ server.close();
11
+ process.exit(0);
12
+ });
13
+
14
+ process.once("SIGTERM", async () => {
15
+ await closeScraper();
16
+ server.close();
17
+ process.exit(0);
18
+ });
19
+ }