tt-help-cli-ycl 1.3.35 → 1.3.37
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -1
- package/cli.js +3 -3
- package/package.json +7 -2
- package/scripts/test-watch-db-smoke.mjs +246 -0
- package/src/cli/attach.js +149 -89
- package/src/cli/auto.js +180 -155
- package/src/cli/comments.js +301 -210
- package/src/cli/config.js +62 -44
- package/src/cli/db-import.js +51 -0
- package/src/cli/explore.js +373 -342
- package/src/cli/refresh.js +223 -151
- package/src/cli/videostats.js +140 -25
- package/src/cli/watch.js +15 -16
- package/src/lib/args.js +50 -6
- package/src/lib/constants.js +159 -92
- package/src/lib/tiktok-scraper.mjs +59 -21
- package/src/main.js +42 -20
- package/src/npm-main.js +69 -0
- package/src/watch/data-store.js +1560 -236
- package/src/watch/public/index.html +51 -15
- package/src/watch/server.js +63 -374
package/src/cli/auto.js
CHANGED
|
@@ -57,184 +57,209 @@ export async function handleAuto(options) {
|
|
|
57
57
|
autoMaxFollowing,
|
|
58
58
|
autoMaxFollowers,
|
|
59
59
|
} = options;
|
|
60
|
+
let browser = null;
|
|
61
|
+
let shuttingDown = false;
|
|
60
62
|
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
63
|
+
const shutdown = async (signal) => {
|
|
64
|
+
if (shuttingDown) return;
|
|
65
|
+
shuttingDown = true;
|
|
66
|
+
console.error(`\n[Auto] 收到 ${signal},正在关闭浏览器...`);
|
|
67
|
+
await browser?.close().catch(() => {});
|
|
68
|
+
console.error("[Auto] 已退出");
|
|
69
|
+
process.exit(0);
|
|
70
|
+
};
|
|
67
71
|
|
|
68
|
-
const
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
preset: autoPreset,
|
|
74
|
-
switchMax: autoSwitchDelay,
|
|
75
|
-
commentMax: autoCommentDelay,
|
|
76
|
-
enableFollow: autoEnableFollow,
|
|
77
|
-
maxFollowing: autoMaxFollowing,
|
|
78
|
-
maxFollowers: autoMaxFollowers,
|
|
79
|
-
userId,
|
|
72
|
+
const onSigint = () => {
|
|
73
|
+
void shutdown("SIGINT");
|
|
74
|
+
};
|
|
75
|
+
const onSigterm = () => {
|
|
76
|
+
void shutdown("SIGTERM");
|
|
80
77
|
};
|
|
81
78
|
|
|
82
|
-
|
|
79
|
+
process.once("SIGINT", onSigint);
|
|
80
|
+
process.once("SIGTERM", onSigterm);
|
|
83
81
|
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
82
|
+
try {
|
|
83
|
+
let userId = configuredUserId;
|
|
84
|
+
if (!userId) {
|
|
85
|
+
userId = await getMacOrUuid();
|
|
86
|
+
saveUserId(userId);
|
|
87
|
+
console.error(`[初始化] 未检测到本地用户编号,已生成并使用: ${userId}`);
|
|
88
|
+
}
|
|
90
89
|
|
|
91
|
-
|
|
90
|
+
const runOptions = {
|
|
91
|
+
collectMax: autoCollectMax,
|
|
92
|
+
scrapeDepth: autoScrapeDepth,
|
|
93
|
+
maxComments: autoMaxComments,
|
|
94
|
+
maxGuess: autoMaxGuess,
|
|
95
|
+
preset: autoPreset,
|
|
96
|
+
switchMax: autoSwitchDelay,
|
|
97
|
+
commentMax: autoCommentDelay,
|
|
98
|
+
enableFollow: autoEnableFollow,
|
|
99
|
+
maxFollowing: autoMaxFollowing,
|
|
100
|
+
maxFollowers: autoMaxFollowers,
|
|
101
|
+
userId,
|
|
102
|
+
};
|
|
92
103
|
|
|
93
|
-
|
|
94
|
-
await import("../scraper/auto-core.js");
|
|
95
|
-
let browser = await ensureBrowserReady();
|
|
104
|
+
await apiGet(`${serverUrl}/api/stats`);
|
|
96
105
|
|
|
97
|
-
|
|
106
|
+
if (autoUsernames.length > 0) {
|
|
107
|
+
const { added, skipped } = await apiPost(`${serverUrl}/api/users`, {
|
|
108
|
+
usernames: autoUsernames,
|
|
109
|
+
});
|
|
110
|
+
console.error(`种子用户: ${added} 个新增, ${skipped} 个已存在`);
|
|
111
|
+
}
|
|
98
112
|
|
|
99
|
-
|
|
100
|
-
let errorCount = 0;
|
|
101
|
-
let consecutiveNetworkErrors = 0;
|
|
102
|
-
let captchaCount = 0; // 验证码累计计数
|
|
113
|
+
console.error(`服务器: ${serverUrl}(断开会自动重连)`);
|
|
103
114
|
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
);
|
|
108
|
-
if (!job.hasJob) break;
|
|
109
|
-
|
|
110
|
-
const username = job.user.uniqueId;
|
|
111
|
-
processedCount++;
|
|
112
|
-
|
|
113
|
-
if (consecutiveNetworkErrors > 0) {
|
|
114
|
-
const waitTime =
|
|
115
|
-
consecutiveNetworkErrors <= 2
|
|
116
|
-
? 0
|
|
117
|
-
: consecutiveNetworkErrors <= 5
|
|
118
|
-
? 30000
|
|
119
|
-
: 300000;
|
|
120
|
-
if (waitTime > 0) {
|
|
121
|
-
console.error(
|
|
122
|
-
` [网络] 连续 ${consecutiveNetworkErrors} 次网络异常,等待 ${waitTime / 1000}s 后重试...`,
|
|
123
|
-
);
|
|
124
|
-
await new Promise((r) => setTimeout(r, waitTime));
|
|
125
|
-
}
|
|
126
|
-
}
|
|
115
|
+
const { ensureBrowserReady, processUser } =
|
|
116
|
+
await import("../scraper/auto-core.js");
|
|
117
|
+
browser = await ensureBrowserReady();
|
|
127
118
|
|
|
128
|
-
|
|
119
|
+
const page = await getOrCreatePage(browser);
|
|
129
120
|
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
console.error,
|
|
135
|
-
);
|
|
121
|
+
let processedCount = 0;
|
|
122
|
+
let errorCount = 0;
|
|
123
|
+
let consecutiveNetworkErrors = 0;
|
|
124
|
+
let captchaCount = 0; // 验证码累计计数
|
|
136
125
|
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
126
|
+
while (!shuttingDown) {
|
|
127
|
+
const job = await apiGet(
|
|
128
|
+
`${serverUrl}/api/job?userId=${encodeURIComponent(userId)}`,
|
|
129
|
+
);
|
|
130
|
+
if (!job.hasJob) break;
|
|
131
|
+
|
|
132
|
+
const username = job.user.uniqueId;
|
|
133
|
+
processedCount++;
|
|
134
|
+
|
|
135
|
+
if (consecutiveNetworkErrors > 0) {
|
|
136
|
+
const waitTime =
|
|
137
|
+
consecutiveNetworkErrors <= 2
|
|
138
|
+
? 0
|
|
139
|
+
: consecutiveNetworkErrors <= 5
|
|
140
|
+
? 30000
|
|
141
|
+
: 300000;
|
|
142
|
+
if (waitTime > 0) {
|
|
143
|
+
console.error(
|
|
144
|
+
` [网络] 连续 ${consecutiveNetworkErrors} 次网络异常,等待 ${waitTime / 1000}s 后重试...`,
|
|
145
|
+
);
|
|
146
|
+
await new Promise((r) => setTimeout(r, waitTime));
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
console.error(`\n[${processedCount}] 处理 @${username}...`);
|
|
142
151
|
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
page,
|
|
153
|
-
username,
|
|
154
|
-
{ ...runOptions, browser },
|
|
155
|
-
console.error,
|
|
156
|
-
);
|
|
157
|
-
Object.assign(result, retryResult);
|
|
158
|
-
// 继续下方逻辑
|
|
159
|
-
} else {
|
|
160
|
-
consecutiveNetworkErrors++;
|
|
161
|
-
errorCount++;
|
|
152
|
+
const result = await processUser(
|
|
153
|
+
page,
|
|
154
|
+
username,
|
|
155
|
+
{ ...runOptions, browser },
|
|
156
|
+
console.error,
|
|
157
|
+
);
|
|
158
|
+
|
|
159
|
+
if (result.restricted) {
|
|
160
|
+
consecutiveNetworkErrors = 0;
|
|
162
161
|
await apiPost(`${serverUrl}/api/job/${username}`, result);
|
|
163
|
-
|
|
164
|
-
|
|
162
|
+
continue;
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
if (result.error) {
|
|
166
|
+
// 浏览器关闭检测
|
|
167
|
+
if (isBrowserClosedError(new Error(result.error))) {
|
|
168
|
+
const newBrowser = await relaunchBrowser({}, 9222);
|
|
169
|
+
browser = newBrowser;
|
|
170
|
+
const newPage = await getOrCreatePage(browser);
|
|
171
|
+
Object.assign(page, newPage);
|
|
172
|
+
// 重试当前用户
|
|
173
|
+
const retryResult = await processUser(
|
|
174
|
+
page,
|
|
175
|
+
username,
|
|
176
|
+
{ ...runOptions, browser },
|
|
177
|
+
console.error,
|
|
178
|
+
);
|
|
179
|
+
Object.assign(result, retryResult);
|
|
180
|
+
// 继续下方逻辑
|
|
181
|
+
} else {
|
|
182
|
+
consecutiveNetworkErrors++;
|
|
183
|
+
errorCount++;
|
|
184
|
+
await apiPost(`${serverUrl}/api/job/${username}`, result);
|
|
185
|
+
const errorType = consecutiveNetworkErrors > 1 ? "network" : "other";
|
|
186
|
+
await withRetry("report error", () =>
|
|
187
|
+
apiPost(`${serverUrl}/api/error-report`, {
|
|
188
|
+
userId,
|
|
189
|
+
username,
|
|
190
|
+
errorType,
|
|
191
|
+
errorMessage: result.error,
|
|
192
|
+
stage: "process",
|
|
193
|
+
errorStack: result.errorStack || "",
|
|
194
|
+
}),
|
|
195
|
+
).catch(() => {});
|
|
196
|
+
continue;
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
if (result.captchaDetected) {
|
|
201
|
+
captchaCount++;
|
|
202
|
+
console.error(` [验证码] 累计 ${captchaCount} 次`);
|
|
203
|
+
|
|
204
|
+
await withRetry("report captcha", () =>
|
|
165
205
|
apiPost(`${serverUrl}/api/error-report`, {
|
|
166
206
|
userId,
|
|
167
207
|
username,
|
|
168
|
-
errorType,
|
|
169
|
-
errorMessage: result.
|
|
170
|
-
stage: "
|
|
171
|
-
errorStack:
|
|
208
|
+
errorType: "captcha",
|
|
209
|
+
errorMessage: result.captchaMessage || "页面出现验证码",
|
|
210
|
+
stage: result.captchaStage || "video-page",
|
|
211
|
+
errorStack: "",
|
|
172
212
|
}),
|
|
173
213
|
).catch(() => {});
|
|
174
|
-
continue;
|
|
175
|
-
}
|
|
176
|
-
}
|
|
177
214
|
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
username,
|
|
186
|
-
errorType: "captcha",
|
|
187
|
-
errorMessage: result.captchaMessage || "页面出现验证码",
|
|
188
|
-
stage: result.captchaStage || "video-page",
|
|
189
|
-
errorStack: "",
|
|
190
|
-
}),
|
|
191
|
-
).catch(() => {});
|
|
192
|
-
|
|
193
|
-
// 累计2次验证码,标记异常(auto模式暂不支持账户切换)
|
|
194
|
-
if (captchaCount >= 2) {
|
|
195
|
-
console.error(
|
|
196
|
-
` [警告] 验证码累计 ${captchaCount} 次,建议检查账户状态`,
|
|
197
|
-
);
|
|
198
|
-
captchaCount = 0;
|
|
215
|
+
// 累计2次验证码,标记异常(auto模式暂不支持账户切换)
|
|
216
|
+
if (captchaCount >= 2) {
|
|
217
|
+
console.error(
|
|
218
|
+
` [警告] 验证码累计 ${captchaCount} 次,建议检查账户状态`,
|
|
219
|
+
);
|
|
220
|
+
captchaCount = 0;
|
|
221
|
+
}
|
|
199
222
|
}
|
|
200
|
-
}
|
|
201
223
|
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
const payload = {
|
|
207
|
-
userInfo: result.userInfo || {},
|
|
208
|
-
discoveredVideoAuthors: (result.discoveredVideoAuthors || []).map(
|
|
209
|
-
(item) =>
|
|
210
|
-
typeof item === "object" ? { ...item, guessedLocation } : item,
|
|
211
|
-
),
|
|
212
|
-
discoveredCommentAuthors: (result.discoveredCommentAuthors || []).map(
|
|
213
|
-
(author) => ({ author, guessedLocation }),
|
|
214
|
-
),
|
|
215
|
-
discoveredGuessAuthors: (result.discoveredGuessAuthors || []).map(
|
|
216
|
-
(author) => ({ author, guessedLocation }),
|
|
217
|
-
),
|
|
218
|
-
discoveredFollowing: (result.discoveredFollowing || []).map((f) => ({
|
|
219
|
-
handle: Array.isArray(f) ? f[0] : f,
|
|
220
|
-
displayName: Array.isArray(f) ? f[1] : null,
|
|
221
|
-
guessedLocation,
|
|
222
|
-
})),
|
|
223
|
-
discoveredFollowers: (result.discoveredFollowers || []).map((f) => ({
|
|
224
|
-
handle: Array.isArray(f) ? f[0] : f,
|
|
225
|
-
displayName: Array.isArray(f) ? f[1] : null,
|
|
226
|
-
guessedLocation,
|
|
227
|
-
})),
|
|
228
|
-
};
|
|
229
|
-
await apiPost(`${serverUrl}/api/job/${username}`, payload);
|
|
230
|
-
console.error(" 已提交");
|
|
231
|
-
}
|
|
224
|
+
consecutiveNetworkErrors = 0;
|
|
225
|
+
|
|
226
|
+
const guessedLocation = result.locationCreated || null;
|
|
232
227
|
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
228
|
+
const payload = {
|
|
229
|
+
userInfo: result.userInfo || {},
|
|
230
|
+
discoveredVideoAuthors: (result.discoveredVideoAuthors || []).map(
|
|
231
|
+
(item) =>
|
|
232
|
+
typeof item === "object" ? { ...item, guessedLocation } : item,
|
|
233
|
+
),
|
|
234
|
+
discoveredCommentAuthors: (result.discoveredCommentAuthors || []).map(
|
|
235
|
+
(author) => ({ author, guessedLocation }),
|
|
236
|
+
),
|
|
237
|
+
discoveredGuessAuthors: (result.discoveredGuessAuthors || []).map(
|
|
238
|
+
(author) => ({ author, guessedLocation }),
|
|
239
|
+
),
|
|
240
|
+
discoveredFollowing: (result.discoveredFollowing || []).map((f) => ({
|
|
241
|
+
handle: Array.isArray(f) ? f[0] : f,
|
|
242
|
+
displayName: Array.isArray(f) ? f[1] : null,
|
|
243
|
+
guessedLocation,
|
|
244
|
+
})),
|
|
245
|
+
discoveredFollowers: (result.discoveredFollowers || []).map((f) => ({
|
|
246
|
+
handle: Array.isArray(f) ? f[0] : f,
|
|
247
|
+
displayName: Array.isArray(f) ? f[1] : null,
|
|
248
|
+
guessedLocation,
|
|
249
|
+
})),
|
|
250
|
+
};
|
|
251
|
+
await apiPost(`${serverUrl}/api/job/${username}`, payload);
|
|
252
|
+
console.error(" 已提交");
|
|
253
|
+
}
|
|
238
254
|
|
|
239
|
-
|
|
255
|
+
const stats = await apiGet(`${serverUrl}/api/stats`);
|
|
256
|
+
console.error(`\n完成: ${processedCount} 个用户处理, ${errorCount} 个出错`);
|
|
257
|
+
console.error(
|
|
258
|
+
` 总用户: ${stats.totalUsers}, 已完成: ${stats.processedUsers}, 待处理: ${stats.pendingUsers}, 错误: ${stats.errorUsers}`,
|
|
259
|
+
);
|
|
260
|
+
} finally {
|
|
261
|
+
process.removeListener("SIGINT", onSigint);
|
|
262
|
+
process.removeListener("SIGTERM", onSigterm);
|
|
263
|
+
await browser?.close().catch(() => {});
|
|
264
|
+
}
|
|
240
265
|
}
|