tt-help-cli-ycl 1.3.35 → 1.3.37
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -1
- package/cli.js +3 -3
- package/package.json +7 -2
- package/scripts/test-watch-db-smoke.mjs +246 -0
- package/src/cli/attach.js +149 -89
- package/src/cli/auto.js +180 -155
- package/src/cli/comments.js +301 -210
- package/src/cli/config.js +62 -44
- package/src/cli/db-import.js +51 -0
- package/src/cli/explore.js +373 -342
- package/src/cli/refresh.js +223 -151
- package/src/cli/videostats.js +140 -25
- package/src/cli/watch.js +15 -16
- package/src/lib/args.js +50 -6
- package/src/lib/constants.js +159 -92
- package/src/lib/tiktok-scraper.mjs +59 -21
- package/src/main.js +42 -20
- package/src/npm-main.js +69 -0
- package/src/watch/data-store.js +1560 -236
- package/src/watch/public/index.html +51 -15
- package/src/watch/server.js +63 -374
package/src/cli/watch.js
CHANGED
|
@@ -1,31 +1,30 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import
|
|
3
|
-
import {
|
|
1
|
+
import { mkdirSync } from "fs";
|
|
2
|
+
import path from "path";
|
|
3
|
+
import { createStore } from "../watch/data-store.js";
|
|
4
|
+
import { startWatchServer, openBrowser } from "../watch/server.js";
|
|
4
5
|
|
|
5
6
|
export async function handleWatch(options) {
|
|
6
|
-
const
|
|
7
|
+
const dataAnchor = options.dataAnchor || options.outputFile;
|
|
8
|
+
const { watchPort } = options;
|
|
7
9
|
|
|
8
|
-
if (!
|
|
9
|
-
console.error(
|
|
10
|
-
console.error(
|
|
11
|
-
console.error(
|
|
10
|
+
if (!dataAnchor) {
|
|
11
|
+
console.error("用法: tt-help watch -o <db路径> [-p 端口]");
|
|
12
|
+
console.error(" tt-help watch -o data/result.db");
|
|
13
|
+
console.error(" tt-help watch -o data/result.db -p 8080");
|
|
12
14
|
process.exit(1);
|
|
13
15
|
}
|
|
14
16
|
|
|
15
|
-
|
|
16
|
-
console.error(`文件不存�? ${outputFile}`);
|
|
17
|
-
process.exit(1);
|
|
18
|
-
}
|
|
17
|
+
mkdirSync(path.dirname(path.resolve(dataAnchor)), { recursive: true });
|
|
19
18
|
|
|
20
|
-
const store = createStore(
|
|
21
|
-
const { server, port } = await startWatchServer(
|
|
19
|
+
const store = createStore(dataAnchor);
|
|
20
|
+
const { server, port } = await startWatchServer(dataAnchor, watchPort, store);
|
|
22
21
|
openBrowser(port);
|
|
23
22
|
|
|
24
|
-
process.once(
|
|
23
|
+
process.once("SIGINT", () => {
|
|
25
24
|
store.stopBackup();
|
|
26
25
|
server.close();
|
|
27
26
|
process.exit(0);
|
|
28
27
|
});
|
|
29
28
|
|
|
30
|
-
console.error(
|
|
29
|
+
console.error("按 Ctrl+C 停止监控服务");
|
|
31
30
|
}
|
package/src/lib/args.js
CHANGED
|
@@ -336,13 +336,13 @@ function parseInfoArgs(args) {
|
|
|
336
336
|
}
|
|
337
337
|
|
|
338
338
|
function parseWatchArgs(args) {
|
|
339
|
-
let
|
|
339
|
+
let dataAnchor = "./result.db";
|
|
340
340
|
let watchPort = 3001;
|
|
341
341
|
|
|
342
342
|
for (let i = 0; i < args.length; i++) {
|
|
343
343
|
const arg = args[i];
|
|
344
344
|
if (arg === "-o" || arg === "--output") {
|
|
345
|
-
|
|
345
|
+
dataAnchor = args[++i];
|
|
346
346
|
} else if (arg === "-p") {
|
|
347
347
|
watchPort = parseInt(args[++i]) || 3001;
|
|
348
348
|
}
|
|
@@ -350,7 +350,8 @@ function parseWatchArgs(args) {
|
|
|
350
350
|
|
|
351
351
|
return {
|
|
352
352
|
subcommand: "watch",
|
|
353
|
-
outputFile,
|
|
353
|
+
outputFile: dataAnchor,
|
|
354
|
+
dataAnchor,
|
|
354
355
|
watchPort,
|
|
355
356
|
urls: [],
|
|
356
357
|
outputFormat: "json",
|
|
@@ -365,6 +366,44 @@ function parseWatchArgs(args) {
|
|
|
365
366
|
};
|
|
366
367
|
}
|
|
367
368
|
|
|
369
|
+
function parseDbImportArgs(args) {
|
|
370
|
+
let dbPath = "./result.db";
|
|
371
|
+
let usersFilePath = null;
|
|
372
|
+
let doneFilePath = null;
|
|
373
|
+
let videosFilePath = null;
|
|
374
|
+
|
|
375
|
+
for (let i = 0; i < args.length; i++) {
|
|
376
|
+
const arg = args[i];
|
|
377
|
+
if (arg === "--db") {
|
|
378
|
+
dbPath = args[++i] || dbPath;
|
|
379
|
+
} else if (arg === "--users") {
|
|
380
|
+
usersFilePath = args[++i] || null;
|
|
381
|
+
} else if (arg === "--done") {
|
|
382
|
+
doneFilePath = args[++i] || null;
|
|
383
|
+
} else if (arg === "--videos") {
|
|
384
|
+
videosFilePath = args[++i] || null;
|
|
385
|
+
}
|
|
386
|
+
}
|
|
387
|
+
|
|
388
|
+
return {
|
|
389
|
+
subcommand: "db-import",
|
|
390
|
+
dbPath,
|
|
391
|
+
usersFilePath,
|
|
392
|
+
doneFilePath,
|
|
393
|
+
videosFilePath,
|
|
394
|
+
urls: [],
|
|
395
|
+
outputFormat: "json",
|
|
396
|
+
exploreCount: 0,
|
|
397
|
+
showConfig: false,
|
|
398
|
+
showHelp: false,
|
|
399
|
+
customProxy: null,
|
|
400
|
+
configAction: null,
|
|
401
|
+
configValue: null,
|
|
402
|
+
pipeMode: false,
|
|
403
|
+
filterStr: null,
|
|
404
|
+
};
|
|
405
|
+
}
|
|
406
|
+
|
|
368
407
|
function parseRefreshArgs(args) {
|
|
369
408
|
let serverUrl = defaultServer;
|
|
370
409
|
let explorePreset = "normal";
|
|
@@ -480,7 +519,7 @@ function parseOpenArgs(args) {
|
|
|
480
519
|
}
|
|
481
520
|
|
|
482
521
|
function parseVideoStatsArgs(args) {
|
|
483
|
-
let
|
|
522
|
+
let statsSource = null;
|
|
484
523
|
let statsParallel = 3;
|
|
485
524
|
|
|
486
525
|
for (let i = 0; i < args.length; i++) {
|
|
@@ -488,13 +527,14 @@ function parseVideoStatsArgs(args) {
|
|
|
488
527
|
if (arg === "-p" || arg === "--parallel") {
|
|
489
528
|
statsParallel = parseInt(args[++i]) || 3;
|
|
490
529
|
} else if (!arg.startsWith("-")) {
|
|
491
|
-
|
|
530
|
+
statsSource = args[i] || null;
|
|
492
531
|
}
|
|
493
532
|
}
|
|
494
533
|
|
|
495
534
|
return {
|
|
496
535
|
subcommand: "videostats",
|
|
497
|
-
statsFile,
|
|
536
|
+
statsFile: statsSource,
|
|
537
|
+
statsSource,
|
|
498
538
|
statsParallel,
|
|
499
539
|
commentsUrl: null,
|
|
500
540
|
commentsMax: 20,
|
|
@@ -634,6 +674,10 @@ export function parseArgs() {
|
|
|
634
674
|
return parseVideoStatsArgs(args.slice(1));
|
|
635
675
|
}
|
|
636
676
|
|
|
677
|
+
if (args.length > 0 && args[0] === "db-import") {
|
|
678
|
+
return parseDbImportArgs(args.slice(1));
|
|
679
|
+
}
|
|
680
|
+
|
|
637
681
|
const urls = [];
|
|
638
682
|
let inputFile = null;
|
|
639
683
|
let outputFile = null;
|
package/src/lib/constants.js
CHANGED
|
@@ -1,17 +1,17 @@
|
|
|
1
|
-
import { join, dirname } from
|
|
2
|
-
import { readFileSync, writeFileSync, existsSync } from
|
|
3
|
-
import { fileURLToPath } from
|
|
1
|
+
import { join, dirname } from "path";
|
|
2
|
+
import { readFileSync, writeFileSync, existsSync } from "fs";
|
|
3
|
+
import { fileURLToPath } from "url";
|
|
4
4
|
|
|
5
5
|
const __filename = fileURLToPath(import.meta.url);
|
|
6
6
|
const __dirname = dirname(__filename);
|
|
7
|
-
const homeDir = process.env.HOME || process.env.USERPROFILE ||
|
|
8
|
-
const configPath = join(homeDir,
|
|
7
|
+
const homeDir = process.env.HOME || process.env.USERPROFILE || "";
|
|
8
|
+
const configPath = join(homeDir, ".tt-help.json");
|
|
9
9
|
|
|
10
|
-
const DEFAULT_PROXY =
|
|
11
|
-
const DEFAULT_OUTPUT =
|
|
10
|
+
const DEFAULT_PROXY = "http://127.0.0.1:7897";
|
|
11
|
+
const DEFAULT_OUTPUT = "tiktok_data.json";
|
|
12
12
|
|
|
13
13
|
let proxy = DEFAULT_PROXY;
|
|
14
|
-
let server =
|
|
14
|
+
let server = "http://127.0.0.1:3001";
|
|
15
15
|
let configFile = null;
|
|
16
16
|
let browser = null;
|
|
17
17
|
let userId = null;
|
|
@@ -22,7 +22,7 @@ let maxComments = 10;
|
|
|
22
22
|
|
|
23
23
|
try {
|
|
24
24
|
if (existsSync(configPath)) {
|
|
25
|
-
const cfg = JSON.parse(readFileSync(configPath,
|
|
25
|
+
const cfg = JSON.parse(readFileSync(configPath, "utf-8"));
|
|
26
26
|
if (cfg.proxy) {
|
|
27
27
|
proxy = cfg.proxy;
|
|
28
28
|
}
|
|
@@ -54,143 +54,209 @@ try {
|
|
|
54
54
|
}
|
|
55
55
|
|
|
56
56
|
function saveBrowser(path) {
|
|
57
|
-
const cfg = existsSync(configPath)
|
|
57
|
+
const cfg = existsSync(configPath)
|
|
58
|
+
? JSON.parse(readFileSync(configPath, "utf-8"))
|
|
59
|
+
: {};
|
|
58
60
|
cfg.browser = path;
|
|
59
|
-
writeFileSync(configPath, JSON.stringify(cfg, null, 2),
|
|
61
|
+
writeFileSync(configPath, JSON.stringify(cfg, null, 2), "utf-8");
|
|
60
62
|
browser = path;
|
|
61
63
|
configFile = configPath;
|
|
62
64
|
}
|
|
63
65
|
|
|
64
66
|
function saveUserId(id) {
|
|
65
|
-
const cfg = existsSync(configPath)
|
|
67
|
+
const cfg = existsSync(configPath)
|
|
68
|
+
? JSON.parse(readFileSync(configPath, "utf-8"))
|
|
69
|
+
: {};
|
|
66
70
|
cfg.userId = id;
|
|
67
|
-
writeFileSync(configPath, JSON.stringify(cfg, null, 2),
|
|
71
|
+
writeFileSync(configPath, JSON.stringify(cfg, null, 2), "utf-8");
|
|
68
72
|
userId = id;
|
|
69
73
|
configFile = configPath;
|
|
70
74
|
}
|
|
71
75
|
|
|
72
76
|
function saveMaxFollowing(val) {
|
|
73
|
-
const cfg = existsSync(configPath)
|
|
77
|
+
const cfg = existsSync(configPath)
|
|
78
|
+
? JSON.parse(readFileSync(configPath, "utf-8"))
|
|
79
|
+
: {};
|
|
74
80
|
cfg.maxFollowing = parseInt(val) || 5;
|
|
75
|
-
writeFileSync(configPath, JSON.stringify(cfg, null, 2),
|
|
81
|
+
writeFileSync(configPath, JSON.stringify(cfg, null, 2), "utf-8");
|
|
76
82
|
maxFollowing = cfg.maxFollowing;
|
|
77
83
|
configFile = configPath;
|
|
78
84
|
}
|
|
79
85
|
|
|
80
86
|
function saveMaxFollowers(val) {
|
|
81
|
-
const cfg = existsSync(configPath)
|
|
87
|
+
const cfg = existsSync(configPath)
|
|
88
|
+
? JSON.parse(readFileSync(configPath, "utf-8"))
|
|
89
|
+
: {};
|
|
82
90
|
cfg.maxFollowers = parseInt(val) || 5;
|
|
83
|
-
writeFileSync(configPath, JSON.stringify(cfg, null, 2),
|
|
91
|
+
writeFileSync(configPath, JSON.stringify(cfg, null, 2), "utf-8");
|
|
84
92
|
maxFollowers = cfg.maxFollowers;
|
|
85
93
|
configFile = configPath;
|
|
86
94
|
}
|
|
87
95
|
|
|
88
96
|
function saveMaxVideos(val) {
|
|
89
|
-
const cfg = existsSync(configPath)
|
|
97
|
+
const cfg = existsSync(configPath)
|
|
98
|
+
? JSON.parse(readFileSync(configPath, "utf-8"))
|
|
99
|
+
: {};
|
|
90
100
|
cfg.maxVideos = parseInt(val) || 16;
|
|
91
|
-
writeFileSync(configPath, JSON.stringify(cfg, null, 2),
|
|
101
|
+
writeFileSync(configPath, JSON.stringify(cfg, null, 2), "utf-8");
|
|
92
102
|
maxVideos = cfg.maxVideos;
|
|
93
103
|
configFile = configPath;
|
|
94
104
|
}
|
|
95
105
|
|
|
96
106
|
function saveMaxComments(val) {
|
|
97
|
-
const cfg = existsSync(configPath)
|
|
107
|
+
const cfg = existsSync(configPath)
|
|
108
|
+
? JSON.parse(readFileSync(configPath, "utf-8"))
|
|
109
|
+
: {};
|
|
98
110
|
cfg.maxComments = parseInt(val) || 10;
|
|
99
|
-
writeFileSync(configPath, JSON.stringify(cfg, null, 2),
|
|
111
|
+
writeFileSync(configPath, JSON.stringify(cfg, null, 2), "utf-8");
|
|
100
112
|
maxComments = cfg.maxComments;
|
|
101
113
|
configFile = configPath;
|
|
102
114
|
}
|
|
103
115
|
|
|
104
116
|
const HELP_TEXT = [
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
117
|
+
"用法:",
|
|
118
|
+
"",
|
|
119
|
+
" tt-help explore <用户名> [preset] [选项]",
|
|
120
|
+
" 支持多个用户名: tt-help explore @user1 @user2 --server http://127.0.0.1:3001",
|
|
121
|
+
" 预设: fast, normal(默认), slow, stealth",
|
|
122
|
+
" 选项:",
|
|
123
|
+
" --server <URL> 服务端地址,默认 http://127.0.0.1:3001",
|
|
124
|
+
" --location <国家代码> 国家筛选,逗号分隔,默认 PL,NL,BE,DE,FR,IT,ES,IE,AT",
|
|
125
|
+
" --job-locations <国家> 任务国家筛选,逗号分隔(仅筛选服务端任务)",
|
|
126
|
+
" --max-comments <数量> 每视频最大评论数,默认 10",
|
|
127
|
+
" --max-guess <数量> 每视频最大猜你喜欢数,默认 0",
|
|
128
|
+
" --max-videos <数量> 每用户最大视频数,默认 16",
|
|
129
|
+
" --enable-follow 启用关注/粉丝提取(默认启用)",
|
|
130
|
+
" --disable-follow 禁用关注/粉丝提取",
|
|
131
|
+
" --max-following <数量> 最大获取关注数,默认 50(同时设置粉丝数)",
|
|
132
|
+
" --max-followers <数量> 最大获取粉丝数,默认 50",
|
|
133
|
+
" --max-users <数量> 最大处理用户数,默认无限制",
|
|
134
|
+
" --port <端口号> 固定 CDP 端口(调试用,关闭自动轮换)",
|
|
135
|
+
" --base-port <端口号> 起始端口,默认 9222",
|
|
136
|
+
" --port-count <数量> 端口数量(账户数),默认 10",
|
|
137
|
+
" --user-id <编号> 客户端编号(设备ID),默认自动生成",
|
|
138
|
+
"",
|
|
139
|
+
" tt-help info <URL> [URL2 ...] [--onlyvideo]",
|
|
140
|
+
" 获取用户/视频信息,支持多个 URL",
|
|
141
|
+
" 主页 URL → 返回用户信息",
|
|
142
|
+
" 视频 URL → 返回用户信息 + 视频信息",
|
|
143
|
+
" 视频 URL + --onlyvideo → 只返回视频信息",
|
|
144
|
+
" 示例: tt-help info https://www.tiktok.com/@nike",
|
|
145
|
+
" tt-help info https://www.tiktok.com/@nike/video/7234567890 --onlyvideo",
|
|
146
|
+
"",
|
|
147
|
+
" tt-help attach [-p 并行数] [-i 间隔秒数] [-s 服务端地址]",
|
|
148
|
+
" 后台轮询服务端任务接口,自动抓取 TikTok 用户信息",
|
|
149
|
+
" -p, --parallel <N> 并行抓取数(默认: 1)",
|
|
150
|
+
" -i, --interval <N> 无任务时轮询间隔,单位秒(默认: 10)",
|
|
151
|
+
" -s, --server <URL> 服务端地址(默认: http://127.0.0.1:3001)",
|
|
152
|
+
" 示例: tt-help attach -p 5 -i 10",
|
|
153
|
+
"",
|
|
154
|
+
" tt-help watch -o <db路径> [-p 端口]",
|
|
155
|
+
" 启动监控服务;运行期主数据仅来自 SQLite",
|
|
156
|
+
" 示例: tt-help watch -o data/result.db",
|
|
157
|
+
" tt-help watch -o data/result.db -p 8080",
|
|
158
|
+
"",
|
|
159
|
+
" tt-help open <端口号>",
|
|
160
|
+
" 打开指定端口的浏览器,用于配置 TikTok 登录账户",
|
|
161
|
+
" 端口范围: 9222 - 9231(对应 explore 的 10 个内置账户)",
|
|
162
|
+
" --list 列出所有端口和配置",
|
|
163
|
+
" 示例: tt-help open 9222",
|
|
164
|
+
" tt-help open --list",
|
|
165
|
+
"",
|
|
166
|
+
" videostats <db路径> -p <并发数>",
|
|
167
|
+
" 批量刷新视频统计数据(playCount, diggCount, commentCount, shareCount, collectCount)",
|
|
168
|
+
" 直接读取并更新 SQLite videos 表",
|
|
169
|
+
" -p, --parallel <N> 并发数(默认: 3)",
|
|
170
|
+
" 示例: tt-help videostats data/result.db -p 3",
|
|
171
|
+
"",
|
|
172
|
+
" db-import --db <db路径> [--users users.json] [--done done.json] [--videos videos.json]",
|
|
173
|
+
" 将 legacy JSON 显式导入 SQLite;运行期不再自动读取 result*.json",
|
|
174
|
+
" 示例: tt-help db-import --db data/result.db --users data/result.json --done data/result-done.json --videos data/result-videos.json",
|
|
175
|
+
"",
|
|
176
|
+
" config [show|set|reset]",
|
|
177
|
+
" config 查看当前配置",
|
|
178
|
+
" config set <key> <value> 设置配置(key: proxy, server, browser, userId, maxFollowing, maxFollowers, maxVideos, maxComments)",
|
|
179
|
+
" config reset 重置所有配置为默认",
|
|
180
|
+
"",
|
|
181
|
+
" 全局选项:",
|
|
182
|
+
" -h, --help 显示帮助",
|
|
183
|
+
" --version 显示版本号",
|
|
184
|
+
"",
|
|
185
|
+
" 示例: tt-help info https://www.tiktok.com/@nike https://www.tiktok.com/@adidas",
|
|
186
|
+
" tt-help explore qiqi23280 fast --location ES --max-comments 50",
|
|
187
|
+
" tt-help config set server http://127.0.0.1:3001",
|
|
188
|
+
" tt-help attach -p 5 -i 10",
|
|
189
|
+
" tt-help watch -o data/result.db",
|
|
190
|
+
" tt-help videostats data/result.db -p 3",
|
|
169
191
|
];
|
|
170
192
|
|
|
193
|
+
const PUBLIC_HELP_HIDDEN_HEADERS = new Set([
|
|
194
|
+
" tt-help watch -o <db路径> [-p 端口]",
|
|
195
|
+
" videostats <db路径> -p <并发数>",
|
|
196
|
+
" db-import --db <db路径> [--users users.json] [--done done.json] [--videos videos.json]",
|
|
197
|
+
]);
|
|
198
|
+
|
|
199
|
+
const PUBLIC_HELP_HIDDEN_LINES = new Set([
|
|
200
|
+
" tt-help watch -o data/result.db",
|
|
201
|
+
" tt-help videostats data/result.db -p 3",
|
|
202
|
+
]);
|
|
203
|
+
|
|
204
|
+
function removeHelpSections(lines, hiddenHeaders, hiddenLines = new Set()) {
|
|
205
|
+
const result = [];
|
|
206
|
+
let skipping = false;
|
|
207
|
+
|
|
208
|
+
for (const line of lines) {
|
|
209
|
+
if (hiddenHeaders.has(line)) {
|
|
210
|
+
skipping = true;
|
|
211
|
+
continue;
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
if (skipping) {
|
|
215
|
+
if (line === "") {
|
|
216
|
+
skipping = false;
|
|
217
|
+
}
|
|
218
|
+
continue;
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
if (hiddenLines.has(line)) {
|
|
222
|
+
continue;
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
result.push(line);
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
return result;
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
const PUBLIC_HELP_TEXT = removeHelpSections(
|
|
232
|
+
HELP_TEXT,
|
|
233
|
+
PUBLIC_HELP_HIDDEN_HEADERS,
|
|
234
|
+
PUBLIC_HELP_HIDDEN_LINES,
|
|
235
|
+
);
|
|
236
|
+
|
|
171
237
|
function getConfigText() {
|
|
172
238
|
let currentUserId = userId;
|
|
173
239
|
if (!currentUserId && existsSync(configPath)) {
|
|
174
240
|
try {
|
|
175
|
-
const cfg = JSON.parse(readFileSync(configPath,
|
|
241
|
+
const cfg = JSON.parse(readFileSync(configPath, "utf-8"));
|
|
176
242
|
if (cfg.userId) currentUserId = cfg.userId;
|
|
177
243
|
} catch {}
|
|
178
244
|
}
|
|
179
245
|
return [
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
246
|
+
"tt-help v1.0.1",
|
|
247
|
+
"",
|
|
248
|
+
"配置:",
|
|
183
249
|
` 代理: ${proxy}`,
|
|
184
250
|
` 服务端: ${server}`,
|
|
185
|
-
` 浏览器: ${browser ||
|
|
186
|
-
` 用户号: ${currentUserId ||
|
|
251
|
+
` 浏览器: ${browser || "未配置(将自动探测或回退)"}`,
|
|
252
|
+
` 用户号: ${currentUserId || "未设置(首次运行 auto 自动创建)"}`,
|
|
187
253
|
` 商家关注采集数: ${maxFollowing}`,
|
|
188
254
|
` 粉丝采集数: ${maxFollowers}`,
|
|
189
255
|
` 视频采集数: ${maxVideos}`,
|
|
190
256
|
` 评论采集数: ${maxComments}`,
|
|
191
257
|
` 输出格式: json`,
|
|
192
258
|
` 默认输出: ${DEFAULT_OUTPUT}`,
|
|
193
|
-
` 配置文件: ${configFile ||
|
|
259
|
+
` 配置文件: ${configFile || "无(使用默认值)"}`,
|
|
194
260
|
];
|
|
195
261
|
}
|
|
196
262
|
|
|
@@ -200,6 +266,7 @@ export {
|
|
|
200
266
|
configPath,
|
|
201
267
|
DEFAULT_PROXY,
|
|
202
268
|
HELP_TEXT,
|
|
269
|
+
PUBLIC_HELP_TEXT,
|
|
203
270
|
browser,
|
|
204
271
|
userId,
|
|
205
272
|
maxFollowing,
|
|
@@ -1,13 +1,14 @@
|
|
|
1
|
-
import { chromium } from
|
|
2
|
-
import { detectBrowser } from
|
|
3
|
-
import { parseUserInfo, parseVideoInfo } from
|
|
1
|
+
import { chromium } from "playwright";
|
|
2
|
+
import { detectBrowser } from "./browser/launch.js";
|
|
3
|
+
import { parseUserInfo, parseVideoInfo } from "./parse-ssr.mjs";
|
|
4
4
|
|
|
5
5
|
const DEFAULT_POOL_SIZE = 3;
|
|
6
6
|
const DEFAULT_WAF_TTL = 120000;
|
|
7
|
-
const DEFAULT_WARM_URL =
|
|
7
|
+
const DEFAULT_WARM_URL = "https://www.tiktok.com/@nike";
|
|
8
|
+
const BROWSER_CLOSE_TIMEOUT = 5000;
|
|
8
9
|
|
|
9
10
|
function delay(ms) {
|
|
10
|
-
return new Promise(r => setTimeout(r, ms));
|
|
11
|
+
return new Promise((r) => setTimeout(r, ms));
|
|
11
12
|
}
|
|
12
13
|
|
|
13
14
|
class PageSlot {
|
|
@@ -45,7 +46,11 @@ class PromiseQueue {
|
|
|
45
46
|
}
|
|
46
47
|
|
|
47
48
|
export class TikTokScraper {
|
|
48
|
-
constructor({
|
|
49
|
+
constructor({
|
|
50
|
+
poolSize = DEFAULT_POOL_SIZE,
|
|
51
|
+
wafTtl = DEFAULT_WAF_TTL,
|
|
52
|
+
warmUrl = DEFAULT_WARM_URL,
|
|
53
|
+
} = {}) {
|
|
49
54
|
this.poolSize = poolSize;
|
|
50
55
|
this.wafTtl = wafTtl;
|
|
51
56
|
this.warmUrl = warmUrl;
|
|
@@ -60,12 +65,21 @@ export class TikTokScraper {
|
|
|
60
65
|
async init() {
|
|
61
66
|
const executablePath = detectBrowser();
|
|
62
67
|
if (!executablePath) {
|
|
63
|
-
throw new Error(
|
|
68
|
+
throw new Error(
|
|
69
|
+
"未找到本地浏览器(Chrome/Edge),请先安装浏览器或执行 npx playwright install",
|
|
70
|
+
);
|
|
64
71
|
}
|
|
65
72
|
this.browser = await chromium.launch({
|
|
66
73
|
headless: true,
|
|
67
74
|
executablePath,
|
|
68
|
-
|
|
75
|
+
handleSIGINT: false,
|
|
76
|
+
handleSIGTERM: false,
|
|
77
|
+
handleSIGHUP: false,
|
|
78
|
+
args: [
|
|
79
|
+
"--no-sandbox",
|
|
80
|
+
"--disable-setuid-sandbox",
|
|
81
|
+
"--disable-dev-shm-usage",
|
|
82
|
+
],
|
|
69
83
|
});
|
|
70
84
|
this.context = await this.browser.newContext();
|
|
71
85
|
for (let i = 0; i < this.poolSize; i++) {
|
|
@@ -76,7 +90,24 @@ export class TikTokScraper {
|
|
|
76
90
|
|
|
77
91
|
async close() {
|
|
78
92
|
if (this.browser) {
|
|
79
|
-
|
|
93
|
+
const browser = this.browser;
|
|
94
|
+
let closeTimedOut = false;
|
|
95
|
+
const closePromise = browser.close().catch((error) => {
|
|
96
|
+
console.error(
|
|
97
|
+
`[TikTokScraper] browser.close() failed: ${error.message}`,
|
|
98
|
+
);
|
|
99
|
+
});
|
|
100
|
+
await Promise.race([
|
|
101
|
+
closePromise,
|
|
102
|
+
delay(BROWSER_CLOSE_TIMEOUT).then(() => {
|
|
103
|
+
closeTimedOut = true;
|
|
104
|
+
}),
|
|
105
|
+
]);
|
|
106
|
+
if (closeTimedOut) {
|
|
107
|
+
console.error(
|
|
108
|
+
`[TikTokScraper] browser.close() 超时 ${BROWSER_CLOSE_TIMEOUT}ms,跳过等待并继续退出`,
|
|
109
|
+
);
|
|
110
|
+
}
|
|
80
111
|
this.browser = null;
|
|
81
112
|
this.context = null;
|
|
82
113
|
this.slots = [];
|
|
@@ -101,7 +132,10 @@ export class TikTokScraper {
|
|
|
101
132
|
this.warmPromise = (async () => {
|
|
102
133
|
const page = await this.context.newPage();
|
|
103
134
|
try {
|
|
104
|
-
await page.goto(this.warmUrl, {
|
|
135
|
+
await page.goto(this.warmUrl, {
|
|
136
|
+
waitUntil: "domcontentloaded",
|
|
137
|
+
timeout: 15000,
|
|
138
|
+
});
|
|
105
139
|
await delay(1500);
|
|
106
140
|
this.lastWarmTime = Date.now();
|
|
107
141
|
} catch (e) {
|
|
@@ -135,17 +169,17 @@ export class TikTokScraper {
|
|
|
135
169
|
async _fetchViewSource(url, slot) {
|
|
136
170
|
const page = await this._ensurePage(slot);
|
|
137
171
|
|
|
138
|
-
await page.goto(
|
|
139
|
-
waitUntil:
|
|
172
|
+
await page.goto("view-source:" + url, {
|
|
173
|
+
waitUntil: "domcontentloaded",
|
|
140
174
|
timeout: 15000,
|
|
141
175
|
});
|
|
142
176
|
|
|
143
177
|
return await page.evaluate(() => {
|
|
144
|
-
const rows = document.querySelectorAll(
|
|
145
|
-
let content =
|
|
146
|
-
rows.forEach(r => {
|
|
147
|
-
const lc = r.querySelector(
|
|
148
|
-
if (lc) content += lc.textContent +
|
|
178
|
+
const rows = document.querySelectorAll("tr");
|
|
179
|
+
let content = "";
|
|
180
|
+
rows.forEach((r) => {
|
|
181
|
+
const lc = r.querySelector(".line-content");
|
|
182
|
+
if (lc) content += lc.textContent + "\n";
|
|
149
183
|
});
|
|
150
184
|
return content;
|
|
151
185
|
});
|
|
@@ -156,14 +190,16 @@ export class TikTokScraper {
|
|
|
156
190
|
return slot.lock.run(async () => {
|
|
157
191
|
let rawHtml = await this._fetchViewSource(
|
|
158
192
|
`https://www.tiktok.com/@${uniqueId}`,
|
|
159
|
-
slot
|
|
193
|
+
slot,
|
|
160
194
|
);
|
|
161
195
|
let result = parseUserInfo(rawHtml);
|
|
162
196
|
if (!result) {
|
|
163
|
-
try {
|
|
197
|
+
try {
|
|
198
|
+
await this.warmWaf();
|
|
199
|
+
} catch {}
|
|
164
200
|
rawHtml = await this._fetchViewSource(
|
|
165
201
|
`https://www.tiktok.com/@${uniqueId}`,
|
|
166
|
-
slot
|
|
202
|
+
slot,
|
|
167
203
|
);
|
|
168
204
|
result = parseUserInfo(rawHtml);
|
|
169
205
|
}
|
|
@@ -177,7 +213,9 @@ export class TikTokScraper {
|
|
|
177
213
|
let rawHtml = await this._fetchViewSource(videoUrl, slot);
|
|
178
214
|
let result = parseVideoInfo(rawHtml);
|
|
179
215
|
if (!result) {
|
|
180
|
-
try {
|
|
216
|
+
try {
|
|
217
|
+
await this.warmWaf();
|
|
218
|
+
} catch {}
|
|
181
219
|
rawHtml = await this._fetchViewSource(videoUrl, slot);
|
|
182
220
|
result = parseVideoInfo(rawHtml);
|
|
183
221
|
}
|