tt-help-cli-ycl 1.3.62 → 1.3.63
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/cli/explore.js +5 -0
- package/src/cli/refresh.js +437 -163
- package/src/lib/args.js +54 -8
- package/src/lib/browser/cdp.js +11 -5
- package/src/lib/constants.js +31 -9
- package/src/main.js +3 -0
- package/src/npm-main.js +3 -0
- package/src/watch/data-store.js +17 -7
- package/src/watch/public/app.js +5 -0
- package/src/watch/public/index.html +1 -0
- package/src/watch/server.js +3 -1
- package/src/scraper/refresh-core.js +0 -213
package/package.json
CHANGED
package/src/cli/explore.js
CHANGED
|
@@ -78,6 +78,7 @@ export async function handleExplore(options) {
|
|
|
78
78
|
explorePortCount,
|
|
79
79
|
exploreUserId,
|
|
80
80
|
exploreMaxVideos,
|
|
81
|
+
exploreProxy,
|
|
81
82
|
} = options;
|
|
82
83
|
|
|
83
84
|
let userId = exploreUserId || configuredUserId;
|
|
@@ -150,6 +151,10 @@ export async function handleExplore(options) {
|
|
|
150
151
|
cdpOptions.userDataDir = currentAccount.userDataDir;
|
|
151
152
|
}
|
|
152
153
|
|
|
154
|
+
if (exploreProxy) {
|
|
155
|
+
cdpOptions.proxyServer = exploreProxy;
|
|
156
|
+
}
|
|
157
|
+
|
|
153
158
|
console.error(`CDP 端口: ${cdpOptions.port}, 用户编号: ${userId}`);
|
|
154
159
|
console.error(`浏览器配置: ${path.basename(cdpOptions.userDataDir)}`);
|
|
155
160
|
if (!explorePort) {
|
package/src/cli/refresh.js
CHANGED
|
@@ -3,61 +3,82 @@ import {
|
|
|
3
3
|
isBrowserClosedError,
|
|
4
4
|
relaunchBrowser,
|
|
5
5
|
} from "../lib/browser/page.js";
|
|
6
|
-
import {
|
|
6
|
+
import {
|
|
7
|
+
delay,
|
|
8
|
+
getDelayConfig,
|
|
9
|
+
setDelayConfig,
|
|
10
|
+
} from "../scraper/modules/page-helpers.js";
|
|
11
|
+
import {
|
|
12
|
+
detectCaptcha,
|
|
13
|
+
closeCaptcha,
|
|
14
|
+
} from "../scraper/modules/captcha-handler.js";
|
|
7
15
|
import { userId as configuredUserId, saveUserId } from "../lib/constants.js";
|
|
8
16
|
import { getMacOrUuid } from "../lib/mac-or-uuid.js";
|
|
9
|
-
import {
|
|
10
|
-
|
|
17
|
+
import {
|
|
18
|
+
ensureBrowserReady as ensureBrowserReadyCDP,
|
|
19
|
+
switchAccount,
|
|
20
|
+
} from "../lib/browser/cdp.js";
|
|
21
|
+
import { HealthChecker } from "../lib/browser/health-checker.js";
|
|
11
22
|
import path from "path";
|
|
12
23
|
import os from "os";
|
|
13
24
|
|
|
14
|
-
|
|
15
|
-
|
|
25
|
+
const MAX_RETRY_WAIT = 5 * 60 * 1000;
|
|
26
|
+
const STARTUP_TIKTOK_URL = "https://www.tiktok.com/@ycl5007";
|
|
27
|
+
|
|
28
|
+
async function withRetry(label, fn) {
|
|
29
|
+
let backoff = 1000;
|
|
30
|
+
while (true) {
|
|
16
31
|
try {
|
|
17
32
|
return await fn();
|
|
18
|
-
} catch (
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
? 10000 + Math.random() * 10000
|
|
25
|
-
: 20000 + Math.random() * 10000;
|
|
26
|
-
console.error(
|
|
27
|
-
` [网络] 请求失败 (${attempt}/${maxRetries}),${Math.round(waitTime / 1000)}s 后重试...`,
|
|
28
|
-
);
|
|
29
|
-
await delay(waitTime / 1000, waitTime / 1000);
|
|
30
|
-
} else {
|
|
31
|
-
throw e;
|
|
32
|
-
}
|
|
33
|
+
} catch (err) {
|
|
34
|
+
console.error(
|
|
35
|
+
`[连接] ${label} 失败: ${err.message},${backoff / 1000}秒后重试...`,
|
|
36
|
+
);
|
|
37
|
+
await new Promise((r) => setTimeout(r, backoff));
|
|
38
|
+
if (backoff < MAX_RETRY_WAIT) backoff *= 2;
|
|
33
39
|
}
|
|
34
40
|
}
|
|
35
41
|
}
|
|
36
42
|
|
|
37
|
-
async function
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
43
|
+
async function apiPost(url, body) {
|
|
44
|
+
return withRetry(`POST ${url}`, async () => {
|
|
45
|
+
const res = await fetch(url, {
|
|
46
|
+
method: "POST",
|
|
47
|
+
headers: { "Content-Type": "application/json" },
|
|
48
|
+
body: JSON.stringify(body),
|
|
49
|
+
});
|
|
50
|
+
return res.json();
|
|
51
|
+
});
|
|
41
52
|
}
|
|
42
53
|
|
|
43
|
-
async function
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
body: JSON.stringify(body),
|
|
54
|
+
async function apiGet(url) {
|
|
55
|
+
return withRetry(`GET ${url}`, async () => {
|
|
56
|
+
const res = await fetch(url);
|
|
57
|
+
return res.json();
|
|
48
58
|
});
|
|
49
|
-
if (!resp.ok) throw new Error(`HTTP ${resp.status}: ${resp.statusText}`);
|
|
50
|
-
return resp.json();
|
|
51
59
|
}
|
|
52
60
|
|
|
53
61
|
export async function handleRefresh(options) {
|
|
54
62
|
const {
|
|
55
63
|
explorePreset,
|
|
64
|
+
exploreInterval,
|
|
65
|
+
exploreEnableFollow,
|
|
66
|
+
exploreMaxFollowing,
|
|
67
|
+
exploreMaxFollowers,
|
|
68
|
+
exploreLocation,
|
|
69
|
+
exploreMaxUsers,
|
|
70
|
+
serverUrl,
|
|
56
71
|
explorePort,
|
|
57
|
-
|
|
72
|
+
exploreBasePort,
|
|
73
|
+
explorePortCount,
|
|
58
74
|
exploreUserId,
|
|
59
|
-
|
|
75
|
+
exploreMaxVideos,
|
|
76
|
+
exploreProfile,
|
|
77
|
+
exploreRedoMaxAge,
|
|
78
|
+
exploreProxy,
|
|
60
79
|
} = options;
|
|
80
|
+
|
|
81
|
+
let userId = exploreUserId || configuredUserId;
|
|
61
82
|
let browser = null;
|
|
62
83
|
let shuttingDown = false;
|
|
63
84
|
|
|
@@ -81,7 +102,6 @@ export async function handleRefresh(options) {
|
|
|
81
102
|
process.once("SIGTERM", onSigterm);
|
|
82
103
|
|
|
83
104
|
try {
|
|
84
|
-
let userId = exploreUserId || configuredUserId;
|
|
85
105
|
if (!userId) {
|
|
86
106
|
userId = await getMacOrUuid();
|
|
87
107
|
saveUserId(userId);
|
|
@@ -90,196 +110,450 @@ export async function handleRefresh(options) {
|
|
|
90
110
|
|
|
91
111
|
setDelayConfig(explorePreset);
|
|
92
112
|
|
|
93
|
-
|
|
113
|
+
// 连接服务器验证
|
|
114
|
+
await apiGet(`${serverUrl}/api/stats`);
|
|
115
|
+
|
|
116
|
+
console.error(`\n=== Refresh 模式(基于 explore) ===`);
|
|
94
117
|
console.error(`服务器: ${serverUrl}`);
|
|
95
|
-
console.error(
|
|
96
|
-
|
|
97
|
-
console.error(
|
|
98
|
-
console.error(
|
|
118
|
+
console.error(`视频采集: ${exploreMaxVideos || 16}`);
|
|
119
|
+
console.error(`关注/粉丝: ${exploreEnableFollow ? "启用" : "禁用"} (${exploreMaxFollowing}/${exploreMaxFollowers})`);
|
|
120
|
+
console.error(`国家筛选: ${exploreLocation}`);
|
|
121
|
+
console.error(`空闲间隔: ${exploreInterval || 30} 秒`);
|
|
122
|
+
if (exploreMaxUsers > 0) console.error(`上限: ${exploreMaxUsers} 个用户`);
|
|
123
|
+
|
|
124
|
+
const healthChecker = new HealthChecker({
|
|
125
|
+
basePort: exploreBasePort,
|
|
126
|
+
totalAccounts: explorePortCount,
|
|
127
|
+
});
|
|
128
|
+
let currentAccount = healthChecker.getCurrentAccount();
|
|
99
129
|
|
|
100
130
|
const cdpOptions = {};
|
|
101
|
-
if (explorePort)
|
|
102
|
-
|
|
131
|
+
if (explorePort) {
|
|
132
|
+
cdpOptions.port = explorePort;
|
|
133
|
+
cdpOptions.userDataDir = path.join(
|
|
134
|
+
os.homedir(),
|
|
135
|
+
"Library",
|
|
136
|
+
"Application Support",
|
|
137
|
+
`Microsoft Edge For Testing_p${explorePort}`,
|
|
138
|
+
);
|
|
139
|
+
} else if (exploreProfile) {
|
|
103
140
|
cdpOptions.userDataDir = path.join(
|
|
104
141
|
os.homedir(),
|
|
105
142
|
"Library",
|
|
106
143
|
"Application Support",
|
|
107
144
|
`Microsoft Edge For Testing_${exploreProfile}`,
|
|
108
145
|
);
|
|
146
|
+
cdpOptions.port = currentAccount.port;
|
|
147
|
+
} else {
|
|
148
|
+
cdpOptions.port = currentAccount.port;
|
|
149
|
+
cdpOptions.userDataDir = currentAccount.userDataDir;
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
if (exploreProxy) {
|
|
153
|
+
cdpOptions.proxyServer = exploreProxy;
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
console.error(`CDP 端口: ${cdpOptions.port}, 用户编号: ${userId}`);
|
|
157
|
+
console.error(`浏览器配置: ${path.basename(cdpOptions.userDataDir)}`);
|
|
158
|
+
if (!explorePort && !exploreProfile) {
|
|
159
|
+
const portRange = `${currentAccount.port}-${currentAccount.port + healthChecker.accounts.length - 1}`;
|
|
160
|
+
console.error(
|
|
161
|
+
`端口轮换范围: ${portRange}(共 ${healthChecker.accounts.length} 个)`,
|
|
162
|
+
);
|
|
109
163
|
}
|
|
110
164
|
|
|
111
165
|
browser = await ensureBrowserReadyCDP(cdpOptions);
|
|
166
|
+
const { processExplore } = await import("../scraper/explore-core.js");
|
|
167
|
+
const { isLoggedIn } = await import("../lib/browser/page.js");
|
|
168
|
+
|
|
112
169
|
const page = await getOrCreatePage(browser);
|
|
113
170
|
|
|
171
|
+
// 导航到 TikTok 页面
|
|
172
|
+
await page.goto(STARTUP_TIKTOK_URL, {
|
|
173
|
+
waitUntil: "domcontentloaded",
|
|
174
|
+
});
|
|
175
|
+
|
|
176
|
+
// 检测登录状态
|
|
177
|
+
let loggedIn = await isLoggedIn(page);
|
|
178
|
+
console.error(`登录状态: ${loggedIn ? "已登录" : "未登录"}`);
|
|
179
|
+
|
|
180
|
+
// 全局拦截图片资源
|
|
181
|
+
await page.route("**/*", (route) => {
|
|
182
|
+
const resourceType = route.request().resourceType();
|
|
183
|
+
if (resourceType === "image" || resourceType === "stylesheet") {
|
|
184
|
+
route.abort();
|
|
185
|
+
} else {
|
|
186
|
+
route.continue();
|
|
187
|
+
}
|
|
188
|
+
});
|
|
189
|
+
|
|
190
|
+
// 账户切换后,重新初始化 page
|
|
191
|
+
async function setupNewPage(newBrowser) {
|
|
192
|
+
const newPage = await getOrCreatePage(newBrowser);
|
|
193
|
+
await newPage.route("**/*", (route) => {
|
|
194
|
+
const resourceType = route.request().resourceType();
|
|
195
|
+
if (resourceType === "image" || resourceType === "stylesheet") {
|
|
196
|
+
route.abort();
|
|
197
|
+
} else {
|
|
198
|
+
route.continue();
|
|
199
|
+
}
|
|
200
|
+
});
|
|
201
|
+
return newPage;
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
async function handleAccountSwitch(reason) {
|
|
205
|
+
console.error(`\n[健康检查] ${reason},切换到下一个账户...`);
|
|
206
|
+
const oldAccount = currentAccount;
|
|
207
|
+
const nextAccount = healthChecker.getNextAccount();
|
|
208
|
+
currentAccount = nextAccount;
|
|
209
|
+
const newBrowser = await switchAccount(
|
|
210
|
+
{ port: oldAccount.port, userDataDir: oldAccount.userDataDir },
|
|
211
|
+
{ port: nextAccount.port, userDataDir: nextAccount.userDataDir },
|
|
212
|
+
);
|
|
213
|
+
browser = newBrowser;
|
|
214
|
+
const newPage = await setupNewPage(browser);
|
|
215
|
+
Object.assign(page, newPage);
|
|
216
|
+
Object.assign(cdpOptions, {
|
|
217
|
+
port: nextAccount.port,
|
|
218
|
+
userDataDir: nextAccount.userDataDir,
|
|
219
|
+
});
|
|
220
|
+
console.error(`[健康检查] 已切换到端口 ${nextAccount.port}`);
|
|
221
|
+
await page.goto(STARTUP_TIKTOK_URL, {
|
|
222
|
+
waitUntil: "domcontentloaded",
|
|
223
|
+
});
|
|
224
|
+
loggedIn = await isLoggedIn(page);
|
|
225
|
+
console.error(
|
|
226
|
+
`[健康检查] 新账户登录状态: ${loggedIn ? "已登录" : "未登录"}`,
|
|
227
|
+
);
|
|
228
|
+
lastFollowSuccessTime = Date.now();
|
|
229
|
+
captchaCount = 0;
|
|
230
|
+
blockedCount = 0;
|
|
231
|
+
consecutiveNetworkErrors = 0;
|
|
232
|
+
}
|
|
233
|
+
|
|
114
234
|
let processedCount = 0;
|
|
115
235
|
let errorCount = 0;
|
|
116
236
|
let consecutiveNetworkErrors = 0;
|
|
237
|
+
let captchaCount = 0;
|
|
238
|
+
let blockedCount = 0;
|
|
239
|
+
let lastFollowSuccessTime = Date.now();
|
|
240
|
+
const FOLLOW_BLOCK_THRESHOLD = 10 * 60 * 1000;
|
|
117
241
|
|
|
118
242
|
console.error(`\n开始循环刷新任务...\n`);
|
|
119
243
|
|
|
120
|
-
while (
|
|
244
|
+
while (true) {
|
|
245
|
+
if (shuttingDown) break;
|
|
246
|
+
|
|
247
|
+
// 健康检查
|
|
248
|
+
const checkResult = healthChecker.check();
|
|
249
|
+
if (checkResult.shouldSwitch) {
|
|
250
|
+
await handleAccountSwitch(checkResult.reason);
|
|
251
|
+
} else if (checkResult.info && processedCount % 10 === 0) {
|
|
252
|
+
const followElapsed = loggedIn && exploreEnableFollow
|
|
253
|
+
? Math.round((Date.now() - lastFollowSuccessTime) / 60000)
|
|
254
|
+
: 0;
|
|
255
|
+
const followStatus =
|
|
256
|
+
loggedIn && exploreEnableFollow
|
|
257
|
+
? ` | 关注/粉丝上次成功 ${followElapsed} 分钟前`
|
|
258
|
+
: "";
|
|
259
|
+
console.error(`\n[健康检查] ${checkResult.info}${followStatus}`);
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
// 领取 redo 任务
|
|
263
|
+
let redoUrl = `${serverUrl}/api/redo-job?userId=${userId}`;
|
|
264
|
+
if (exploreRedoMaxAge > 0) {
|
|
265
|
+
redoUrl += `&maxAge=${exploreRedoMaxAge}`;
|
|
266
|
+
}
|
|
267
|
+
let job = null;
|
|
121
268
|
try {
|
|
122
|
-
|
|
123
|
-
|
|
269
|
+
job = await apiGet(redoUrl);
|
|
270
|
+
} catch (e) {
|
|
271
|
+
consecutiveNetworkErrors++;
|
|
272
|
+
console.error(
|
|
273
|
+
` [网络] 获取任务失败 (${consecutiveNetworkErrors}): ${e.message}`
|
|
124
274
|
);
|
|
275
|
+
await new Promise((r) => setTimeout(r, 10000));
|
|
276
|
+
continue;
|
|
277
|
+
}
|
|
125
278
|
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
279
|
+
if (!job.hasJob) {
|
|
280
|
+
console.error(`\n[空闲] 暂无 redo 任务,${exploreInterval || 30}s 后重试...`);
|
|
281
|
+
await new Promise((r) => setTimeout(r, (exploreInterval || 30) * 1000));
|
|
282
|
+
consecutiveNetworkErrors = 0;
|
|
283
|
+
continue;
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
consecutiveNetworkErrors = 0;
|
|
287
|
+
const username = job.user.uniqueId;
|
|
288
|
+
const nickname = job.user.nickname;
|
|
289
|
+
console.error(`\n[${processedCount}] 刷新 @${username}...`);
|
|
290
|
+
|
|
291
|
+
// 切换任务前检测验证码
|
|
292
|
+
const captchaResult = await detectCaptcha(page);
|
|
293
|
+
if (captchaResult && captchaResult.visible) {
|
|
294
|
+
console.error(`\n[验证码] 切换任务前检测到验证码,等待3分钟...`);
|
|
295
|
+
captchaCount++;
|
|
296
|
+
console.error(` [验证码] 累计 ${captchaCount} 次`);
|
|
297
|
+
|
|
298
|
+
await new Promise((r) => setTimeout(r, 180000));
|
|
299
|
+
|
|
300
|
+
const closeResult = await closeCaptcha(page);
|
|
301
|
+
if (closeResult.success) {
|
|
302
|
+
console.error(" [验证码] 已关闭验证码");
|
|
303
|
+
await new Promise((r) => setTimeout(r, 10000));
|
|
304
|
+
} else {
|
|
305
|
+
console.error(" [验证码] 无法关闭验证码,继续处理");
|
|
130
306
|
}
|
|
131
307
|
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
308
|
+
await withRetry("report captcha", () =>
|
|
309
|
+
apiPost(`${serverUrl}/api/error-report`, {
|
|
310
|
+
userId,
|
|
311
|
+
username: "unknown",
|
|
312
|
+
errorType: "captcha",
|
|
313
|
+
errorMessage: "切换任务前检测到验证码",
|
|
314
|
+
stage: "between-tasks",
|
|
315
|
+
errorStack: "",
|
|
316
|
+
}),
|
|
317
|
+
).catch(() => {});
|
|
135
318
|
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
319
|
+
if (captchaCount >= 2) {
|
|
320
|
+
await handleAccountSwitch(`验证码累计 ${captchaCount} 次`);
|
|
321
|
+
captchaCount = 0;
|
|
322
|
+
}
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
// 关注/粉丝封禁检测
|
|
326
|
+
if (loggedIn && exploreEnableFollow) {
|
|
327
|
+
const followElapsed = Date.now() - lastFollowSuccessTime;
|
|
328
|
+
if (followElapsed > FOLLOW_BLOCK_THRESHOLD) {
|
|
329
|
+
const minutes = Math.round(followElapsed / 60000);
|
|
330
|
+
console.error(
|
|
331
|
+
`\n[封禁检测] 关注/粉丝功能已 ${minutes} 分钟未成功获取,切换到下一个账户...`,
|
|
332
|
+
);
|
|
333
|
+
await handleAccountSwitch(`关注/粉丝功能被封`);
|
|
334
|
+
}
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
if (consecutiveNetworkErrors > 0) {
|
|
338
|
+
const waitTime =
|
|
339
|
+
consecutiveNetworkErrors <= 2
|
|
340
|
+
? 0
|
|
341
|
+
: consecutiveNetworkErrors <= 5
|
|
342
|
+
? 30000
|
|
343
|
+
: 300000;
|
|
344
|
+
if (waitTime > 0) {
|
|
345
|
+
console.error(
|
|
346
|
+
` [网络] 连续 ${consecutiveNetworkErrors} 次网络异常,等待 ${waitTime / 1000}s 后重试...`,
|
|
347
|
+
);
|
|
348
|
+
await new Promise((r) => setTimeout(r, waitTime));
|
|
349
|
+
}
|
|
350
|
+
}
|
|
139
351
|
|
|
140
|
-
|
|
352
|
+
const { switchMax } = getDelayConfig();
|
|
353
|
+
await delay(switchMax, switchMax * 3);
|
|
354
|
+
|
|
355
|
+
let result = await processExplore(
|
|
356
|
+
page,
|
|
357
|
+
username,
|
|
358
|
+
{
|
|
359
|
+
maxVideos: exploreMaxVideos || 16,
|
|
360
|
+
enableFollow: exploreEnableFollow !== false,
|
|
361
|
+
loggedIn,
|
|
362
|
+
maxFollowing: exploreMaxFollowing || 100,
|
|
363
|
+
maxFollowers: exploreMaxFollowers || 100,
|
|
364
|
+
location: exploreLocation,
|
|
365
|
+
browser,
|
|
366
|
+
},
|
|
367
|
+
console.error,
|
|
368
|
+
);
|
|
369
|
+
|
|
370
|
+
// 浏览器关闭检测
|
|
371
|
+
if (result.error && isBrowserClosedError(new Error(result.error))) {
|
|
372
|
+
const newBrowser = await relaunchBrowser(
|
|
373
|
+
cdpOptions,
|
|
374
|
+
cdpOptions.port || 9222,
|
|
375
|
+
);
|
|
376
|
+
browser = newBrowser;
|
|
377
|
+
const newPage = await setupNewPage(browser);
|
|
378
|
+
Object.assign(page, newPage);
|
|
379
|
+
result = await processExplore(
|
|
141
380
|
page,
|
|
142
|
-
|
|
143
|
-
serverUrl,
|
|
381
|
+
username,
|
|
144
382
|
{
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
383
|
+
maxVideos: exploreMaxVideos || 16,
|
|
384
|
+
enableFollow: exploreEnableFollow !== false,
|
|
385
|
+
loggedIn,
|
|
386
|
+
maxFollowing: exploreMaxFollowing || 100,
|
|
387
|
+
maxFollowers: exploreMaxFollowers || 100,
|
|
388
|
+
location: exploreLocation,
|
|
389
|
+
browser,
|
|
148
390
|
},
|
|
149
391
|
console.error,
|
|
150
392
|
);
|
|
393
|
+
}
|
|
151
394
|
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
395
|
+
if (result.restricted) {
|
|
396
|
+
consecutiveNetworkErrors = 0;
|
|
397
|
+
await apiPost(`${serverUrl}/api/redo-job/${username}`, {
|
|
398
|
+
restricted: true,
|
|
399
|
+
userInfo: result.userInfo || {},
|
|
400
|
+
});
|
|
401
|
+
if (exploreMaxUsers > 0 && processedCount >= exploreMaxUsers) {
|
|
402
|
+
console.error(`\n已达上限 ${exploreMaxUsers} 个用户,停止处理`);
|
|
403
|
+
break;
|
|
159
404
|
}
|
|
405
|
+
continue;
|
|
406
|
+
}
|
|
160
407
|
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
console.error(` [错误] ${result.error}`);
|
|
189
|
-
|
|
190
|
-
if (consecutiveNetworkErrors >= 3) {
|
|
191
|
-
console.error(
|
|
192
|
-
` [警告] 连续 ${consecutiveNetworkErrors} 次错误,等待 60s 后重试...`,
|
|
193
|
-
);
|
|
194
|
-
await delay(60000, 60000);
|
|
195
|
-
consecutiveNetworkErrors = 0;
|
|
196
|
-
}
|
|
408
|
+
if (result.error) {
|
|
409
|
+
consecutiveNetworkErrors++;
|
|
410
|
+
errorCount++;
|
|
411
|
+
|
|
412
|
+
// 临时性错误:自动重试一次
|
|
413
|
+
if (result.retryable) {
|
|
414
|
+
console.error(` [临时错误] 等待 5 秒后重试 @${username}...`);
|
|
415
|
+
await new Promise((r) => setTimeout(r, 5000));
|
|
416
|
+
result = await processExplore(
|
|
417
|
+
page,
|
|
418
|
+
username,
|
|
419
|
+
{
|
|
420
|
+
maxVideos: exploreMaxVideos || 16,
|
|
421
|
+
enableFollow: exploreEnableFollow !== false,
|
|
422
|
+
loggedIn,
|
|
423
|
+
maxFollowing: exploreMaxFollowing || 100,
|
|
424
|
+
maxFollowers: exploreMaxFollowers || 100,
|
|
425
|
+
location: exploreLocation,
|
|
426
|
+
browser,
|
|
427
|
+
},
|
|
428
|
+
console.error,
|
|
429
|
+
);
|
|
430
|
+
if (!result.error) {
|
|
431
|
+
consecutiveNetworkErrors = 0;
|
|
432
|
+
errorCount--;
|
|
433
|
+
}
|
|
434
|
+
}
|
|
197
435
|
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
436
|
+
if (result.error) {
|
|
437
|
+
await apiPost(`${serverUrl}/api/redo-job/${username}`, {
|
|
438
|
+
error: result.error,
|
|
439
|
+
});
|
|
440
|
+
const errorType = result.error.startsWith("被封:")
|
|
441
|
+
? "被封"
|
|
442
|
+
: consecutiveNetworkErrors > 1
|
|
443
|
+
? "network"
|
|
444
|
+
: "other";
|
|
445
|
+
await withRetry("report error", () =>
|
|
204
446
|
apiPost(`${serverUrl}/api/error-report`, {
|
|
205
447
|
userId,
|
|
206
|
-
username
|
|
448
|
+
username,
|
|
207
449
|
errorType,
|
|
208
450
|
errorMessage: result.error,
|
|
209
451
|
stage: "process",
|
|
210
452
|
errorStack: result.errorStack || "",
|
|
211
|
-
})
|
|
453
|
+
}),
|
|
454
|
+
).catch(() => {});
|
|
455
|
+
if (errorType === "被封") {
|
|
456
|
+
blockedCount++;
|
|
457
|
+
console.error(` [被封] 累计 ${blockedCount} 次`);
|
|
458
|
+
if (blockedCount >= 3) {
|
|
459
|
+
await handleAccountSwitch(`账号被封累计 ${blockedCount} 次`);
|
|
460
|
+
blockedCount = 0;
|
|
461
|
+
}
|
|
212
462
|
continue;
|
|
213
463
|
}
|
|
464
|
+
if (exploreMaxUsers > 0 && processedCount >= exploreMaxUsers) {
|
|
465
|
+
console.error(`\n已达上限 ${exploreMaxUsers} 个用户,停止处理`);
|
|
466
|
+
break;
|
|
467
|
+
}
|
|
468
|
+
continue;
|
|
214
469
|
}
|
|
470
|
+
}
|
|
471
|
+
|
|
472
|
+
if (result.captchaDetected) {
|
|
473
|
+
captchaCount++;
|
|
474
|
+
console.error(` [验证码] 累计 ${captchaCount} 次`);
|
|
215
475
|
|
|
216
|
-
|
|
217
|
-
|
|
476
|
+
await withRetry("report captcha", () =>
|
|
477
|
+
apiPost(`${serverUrl}/api/error-report`, {
|
|
218
478
|
userId,
|
|
219
|
-
username
|
|
479
|
+
username,
|
|
220
480
|
errorType: "captcha",
|
|
221
481
|
errorMessage: result.captchaMessage || "页面出现验证码",
|
|
222
482
|
stage: result.captchaStage || "video-page",
|
|
223
483
|
errorStack: "",
|
|
224
|
-
})
|
|
484
|
+
}),
|
|
485
|
+
).catch(() => {});
|
|
486
|
+
|
|
487
|
+
if (captchaCount >= 2) {
|
|
488
|
+
await handleAccountSwitch(`验证码累计 ${captchaCount} 次`);
|
|
489
|
+
captchaCount = 0;
|
|
225
490
|
}
|
|
491
|
+
}
|
|
226
492
|
|
|
227
|
-
|
|
493
|
+
consecutiveNetworkErrors = 0;
|
|
494
|
+
|
|
495
|
+
// 更新关注/粉丝成功时间
|
|
496
|
+
if (result.hasFollowData && result.keepFollow) {
|
|
497
|
+
const totalFollows =
|
|
498
|
+
(result.discoveredFollowing || []).length +
|
|
499
|
+
(result.discoveredFollowers || []).length;
|
|
500
|
+
if (totalFollows > 0) {
|
|
501
|
+
lastFollowSuccessTime = Date.now();
|
|
502
|
+
}
|
|
503
|
+
}
|
|
228
504
|
|
|
229
|
-
|
|
505
|
+
processedCount++;
|
|
230
506
|
|
|
231
|
-
|
|
232
|
-
userInfo: result.userInfo || {},
|
|
233
|
-
discoveredVideoAuthors: (result.discoveredVideoAuthors || []).map(
|
|
234
|
-
(item) =>
|
|
235
|
-
typeof item === "object" ? { ...item, guessedLocation } : item,
|
|
236
|
-
),
|
|
237
|
-
discoveredCommentAuthors: (result.discoveredCommentAuthors || []).map(
|
|
238
|
-
(author) => ({ author, guessedLocation }),
|
|
239
|
-
),
|
|
240
|
-
discoveredGuessAuthors: (result.discoveredGuessAuthors || []).map(
|
|
241
|
-
(author) => ({ author, guessedLocation }),
|
|
242
|
-
),
|
|
243
|
-
discoveredFollowing: (result.discoveredFollowing || []).map((f) => ({
|
|
244
|
-
handle: Array.isArray(f) ? f[0] : f,
|
|
245
|
-
displayName: Array.isArray(f) ? f[1] : null,
|
|
246
|
-
guessedLocation,
|
|
247
|
-
})),
|
|
248
|
-
discoveredFollowers: (result.discoveredFollowers || []).map((f) => ({
|
|
249
|
-
handle: Array.isArray(f) ? f[0] : f,
|
|
250
|
-
displayName: Array.isArray(f) ? f[1] : null,
|
|
251
|
-
guessedLocation,
|
|
252
|
-
})),
|
|
253
|
-
newUsersAdded: result.newUsersAdded || 0,
|
|
254
|
-
collectedVideos: result.collectedVideos || 0,
|
|
255
|
-
});
|
|
507
|
+
const guessedLocation = result.locationCreated || null;
|
|
256
508
|
|
|
257
|
-
|
|
258
|
-
|
|
509
|
+
const payload = {
|
|
510
|
+
userInfo: result.userInfo || {},
|
|
511
|
+
discoveredFollowing: (result.discoveredFollowing || []).map((f) => ({
|
|
512
|
+
handle: Array.isArray(f) ? f[0] : f,
|
|
513
|
+
displayName: Array.isArray(f) ? f[1] : null,
|
|
514
|
+
guessedLocation,
|
|
515
|
+
})),
|
|
516
|
+
discoveredFollowers: (result.discoveredFollowers || []).map((f) => ({
|
|
517
|
+
handle: Array.isArray(f) ? f[0] : f,
|
|
518
|
+
displayName: Array.isArray(f) ? f[1] : null,
|
|
519
|
+
guessedLocation,
|
|
520
|
+
})),
|
|
521
|
+
processed: result.processed,
|
|
522
|
+
hasFollowData: result.hasFollowData,
|
|
523
|
+
keepFollow: result.keepFollow,
|
|
524
|
+
locationCreated: result.locationCreated,
|
|
525
|
+
noVideo: result.noVideo,
|
|
526
|
+
collectedVideos: result.collectedVideos,
|
|
527
|
+
};
|
|
528
|
+
await apiPost(`${serverUrl}/api/redo-job/${username}`, payload);
|
|
529
|
+
|
|
530
|
+
// 视频登记
|
|
531
|
+
if (result.videoList && result.videoList.length > 0) {
|
|
532
|
+
const { registered, skipped } = await apiPost(
|
|
533
|
+
`${serverUrl}/api/videos`,
|
|
534
|
+
{
|
|
535
|
+
sourceUser: username,
|
|
536
|
+
videoList: result.videoList,
|
|
537
|
+
locationCreated: result.locationCreated,
|
|
538
|
+
ttSeller: result.userInfo?.ttSeller || false,
|
|
539
|
+
},
|
|
259
540
|
);
|
|
541
|
+
console.error(` 视频登记: ${registered} 条新增, ${skipped} 条已存在`);
|
|
542
|
+
}
|
|
260
543
|
|
|
261
|
-
|
|
262
|
-
} catch (e) {
|
|
263
|
-
consecutiveNetworkErrors++;
|
|
264
|
-
errorCount++;
|
|
265
|
-
console.error(`\n[错误] ${e.message}`);
|
|
544
|
+
console.error(" 已提交");
|
|
266
545
|
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
);
|
|
271
|
-
await delay(60000, 60000);
|
|
272
|
-
consecutiveNetworkErrors = 0;
|
|
273
|
-
} else {
|
|
274
|
-
const waitTime =
|
|
275
|
-
consecutiveNetworkErrors <= 2
|
|
276
|
-
? 5000 + Math.random() * 5000
|
|
277
|
-
: 10000 + Math.random() * 10000;
|
|
278
|
-
console.error(` 等待 ${Math.round(waitTime / 1000)}s 后重试...`);
|
|
279
|
-
await delay(waitTime / 1000, waitTime / 1000);
|
|
280
|
-
}
|
|
546
|
+
if (exploreMaxUsers > 0 && processedCount >= exploreMaxUsers) {
|
|
547
|
+
console.error(`\n已达上限 ${exploreMaxUsers} 个用户,停止处理`);
|
|
548
|
+
break;
|
|
281
549
|
}
|
|
282
550
|
}
|
|
551
|
+
|
|
552
|
+
const stats = await apiGet(`${serverUrl}/api/stats`);
|
|
553
|
+
console.error(`\n完成: ${processedCount} 个用户处理, ${errorCount} 个出错`);
|
|
554
|
+
console.error(
|
|
555
|
+
` 总用户: ${stats.totalUsers}, 已完成: ${stats.processedUsers}, 待处理: ${stats.pendingUsers}, 错误: ${stats.errorUsers}`,
|
|
556
|
+
);
|
|
283
557
|
} finally {
|
|
284
558
|
process.removeListener("SIGINT", onSigint);
|
|
285
559
|
process.removeListener("SIGTERM", onSigterm);
|
package/src/lib/args.js
CHANGED
|
@@ -180,6 +180,7 @@ function parseExploreArgs(args) {
|
|
|
180
180
|
let explorePortCount = null;
|
|
181
181
|
let exploreUserId = null;
|
|
182
182
|
let exploreMaxVideos = 16;
|
|
183
|
+
let exploreProxy = null;
|
|
183
184
|
|
|
184
185
|
const positional = [];
|
|
185
186
|
const PRESETS = ["fast", "normal", "slow", "stealth"];
|
|
@@ -218,6 +219,8 @@ function parseExploreArgs(args) {
|
|
|
218
219
|
exploreUserId = args[++i];
|
|
219
220
|
} else if (arg === "--max-videos") {
|
|
220
221
|
exploreMaxVideos = parseInt(args[++i]) || 16;
|
|
222
|
+
} else if (arg === "--proxy") {
|
|
223
|
+
exploreProxy = args[++i];
|
|
221
224
|
} else {
|
|
222
225
|
positional.push(arg);
|
|
223
226
|
}
|
|
@@ -258,6 +261,7 @@ function parseExploreArgs(args) {
|
|
|
258
261
|
explorePortCount,
|
|
259
262
|
exploreUserId,
|
|
260
263
|
exploreMaxVideos,
|
|
264
|
+
exploreProxy,
|
|
261
265
|
urls: [],
|
|
262
266
|
outputFormat: "json",
|
|
263
267
|
exploreCount: 0,
|
|
@@ -439,39 +443,77 @@ function parseDbImportArgs(args) {
|
|
|
439
443
|
function parseRefreshArgs(args) {
|
|
440
444
|
let serverUrl = defaultServer;
|
|
441
445
|
let explorePreset = "normal";
|
|
442
|
-
let
|
|
443
|
-
let
|
|
446
|
+
let exploreInterval = 30;
|
|
447
|
+
let exploreEnableFollow = true;
|
|
448
|
+
let exploreMaxFollowing = 100;
|
|
449
|
+
let exploreMaxFollowers = 100;
|
|
450
|
+
let exploreLocation = DEFAULT_TARGET_LOCATIONS_CSV;
|
|
451
|
+
let exploreMaxUsers = 0;
|
|
444
452
|
let explorePort = null;
|
|
453
|
+
let exploreBasePort = null;
|
|
454
|
+
let explorePortCount = null;
|
|
445
455
|
let exploreProfile = null;
|
|
446
456
|
let exploreUserId = null;
|
|
457
|
+
let exploreMaxVideos = 16;
|
|
458
|
+
let exploreRedoMaxAge = 0;
|
|
459
|
+
let exploreProxy = null;
|
|
447
460
|
|
|
448
461
|
for (let i = 0; i < args.length; i++) {
|
|
449
462
|
const arg = args[i];
|
|
450
463
|
if (arg === "--server") {
|
|
451
464
|
serverUrl = args[++i];
|
|
452
|
-
} else if (arg === "--comments") {
|
|
453
|
-
exploreMaxComments = parseInt(args[++i]) || 10;
|
|
454
|
-
} else if (arg === "--guess") {
|
|
455
|
-
exploreMaxGuess = parseInt(args[++i]) || 0;
|
|
456
465
|
} else if (arg === "--preset") {
|
|
457
466
|
explorePreset = args[++i];
|
|
467
|
+
} else if (arg === "-i" || arg === "--interval") {
|
|
468
|
+
exploreInterval = parseInt(args[++i], 10) || 30;
|
|
458
469
|
} else if (arg === "--port") {
|
|
459
470
|
explorePort = parseInt(args[++i]) || 9222;
|
|
471
|
+
} else if (arg === "--base-port") {
|
|
472
|
+
exploreBasePort = parseInt(args[++i]) || 9222;
|
|
473
|
+
} else if (arg === "--port-count") {
|
|
474
|
+
explorePortCount = parseInt(args[++i]) || 10;
|
|
460
475
|
} else if (arg === "--profile") {
|
|
461
476
|
exploreProfile = args[++i];
|
|
462
477
|
} else if (arg === "--user-id") {
|
|
463
478
|
exploreUserId = args[++i];
|
|
479
|
+
} else if (arg === "--max-videos") {
|
|
480
|
+
exploreMaxVideos = parseInt(args[++i]) || 16;
|
|
481
|
+
} else if (arg === "--max-following") {
|
|
482
|
+
exploreMaxFollowing = parseInt(args[++i]) || 100;
|
|
483
|
+
} else if (arg === "--max-followers") {
|
|
484
|
+
exploreMaxFollowers = parseInt(args[++i]) || 100;
|
|
485
|
+
} else if (arg === "--max-age") {
|
|
486
|
+
exploreRedoMaxAge = parseInt(args[++i]) || 43200;
|
|
487
|
+
} else if (arg === "--proxy") {
|
|
488
|
+
exploreProxy = args[++i];
|
|
489
|
+
} else if (arg === "--location") {
|
|
490
|
+
exploreLocation = args[++i];
|
|
491
|
+
} else if (arg === "--enable-follow") {
|
|
492
|
+
exploreEnableFollow = true;
|
|
493
|
+
} else if (arg === "--disable-follow") {
|
|
494
|
+
exploreEnableFollow = false;
|
|
495
|
+
} else if (arg === "--max-users") {
|
|
496
|
+
exploreMaxUsers = parseInt(args[++i]) || 0;
|
|
464
497
|
}
|
|
465
498
|
}
|
|
466
499
|
|
|
467
500
|
return {
|
|
468
501
|
subcommand: "refresh",
|
|
469
502
|
explorePreset,
|
|
470
|
-
|
|
471
|
-
|
|
503
|
+
exploreInterval,
|
|
504
|
+
exploreEnableFollow,
|
|
505
|
+
exploreMaxFollowing,
|
|
506
|
+
exploreMaxFollowers,
|
|
507
|
+
exploreLocation,
|
|
508
|
+
exploreMaxUsers,
|
|
472
509
|
explorePort,
|
|
510
|
+
exploreBasePort,
|
|
511
|
+
explorePortCount,
|
|
473
512
|
exploreProfile,
|
|
474
513
|
exploreUserId,
|
|
514
|
+
exploreMaxVideos,
|
|
515
|
+
exploreRedoMaxAge,
|
|
516
|
+
exploreProxy,
|
|
475
517
|
serverUrl,
|
|
476
518
|
urls: [],
|
|
477
519
|
outputFormat: "json",
|
|
@@ -721,6 +763,10 @@ export function parseArgs() {
|
|
|
721
763
|
return parseDbImportArgs(args.slice(1));
|
|
722
764
|
}
|
|
723
765
|
|
|
766
|
+
if (args.length > 0 && args[0] === "refresh") {
|
|
767
|
+
return parseRefreshArgs(args.slice(1));
|
|
768
|
+
}
|
|
769
|
+
|
|
724
770
|
const urls = [];
|
|
725
771
|
let inputFile = null;
|
|
726
772
|
let outputFile = null;
|
package/src/lib/browser/cdp.js
CHANGED
|
@@ -143,7 +143,7 @@ function killEdgeProcesses(targetDir) {
|
|
|
143
143
|
});
|
|
144
144
|
}
|
|
145
145
|
|
|
146
|
-
function launchEdgeWithCDP(port, userDataDir) {
|
|
146
|
+
function launchEdgeWithCDP(port, userDataDir, proxyServer) {
|
|
147
147
|
return new Promise((resolve, reject) => {
|
|
148
148
|
const platform = os.platform();
|
|
149
149
|
const edgePath = getEdgePath();
|
|
@@ -162,14 +162,19 @@ function launchEdgeWithCDP(port, userDataDir) {
|
|
|
162
162
|
"--disable-breakpad",
|
|
163
163
|
"--disable-background-networking",
|
|
164
164
|
"--disable-sync",
|
|
165
|
-
]
|
|
165
|
+
];
|
|
166
|
+
if (proxyServer) {
|
|
167
|
+
extraArgs.push(`--proxy-server="${proxyServer}"`);
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
const argsStr = extraArgs.join(" ");
|
|
166
171
|
|
|
167
172
|
if (platform === "darwin") {
|
|
168
|
-
command = `open -a ${edgePath} --new --args ${
|
|
173
|
+
command = `open -a ${edgePath} --new --args ${argsStr}`;
|
|
169
174
|
} else if (platform === "win32") {
|
|
170
|
-
command = `start msedge ${
|
|
175
|
+
command = `start msedge ${argsStr}`;
|
|
171
176
|
} else {
|
|
172
|
-
command = `msedge ${
|
|
177
|
+
command = `msedge ${argsStr} &`;
|
|
173
178
|
}
|
|
174
179
|
|
|
175
180
|
exec(command, (err) => {
|
|
@@ -194,6 +199,7 @@ export { killEdgeProcesses };
|
|
|
194
199
|
export async function ensureBrowserReady(options = {}) {
|
|
195
200
|
const port = options.port || DEFAULT_CDP_PORT;
|
|
196
201
|
const userDataDir = options.userDataDir || DEFAULT_USER_DATA_DIR;
|
|
202
|
+
const proxyServer = options.proxyServer || null;
|
|
197
203
|
const isCustom = port !== DEFAULT_CDP_PORT || !!options.userDataDir;
|
|
198
204
|
|
|
199
205
|
const isReady = await checkCDPPort(port);
|
package/src/lib/constants.js
CHANGED
|
@@ -131,12 +131,33 @@ const HELP_TEXT = [
|
|
|
131
131
|
" --disable-follow 禁用关注/粉丝提取",
|
|
132
132
|
" --max-following <数量> 最大获取关注数,默认 50",
|
|
133
133
|
" --max-followers <数量> 最大获取粉丝数,默认 50",
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
134
|
+
" --max-users <数量> 最大处理用户数,默认无限制",
|
|
135
|
+
" -i, --interval <秒数> 无任务时轮询间隔,默认 10 秒",
|
|
136
|
+
" --port <端口号> 固定 CDP 端口(调试用,关闭自动轮换)",
|
|
137
|
+
" --base-port <端口号> 起始端口,默认 9222",
|
|
138
|
+
" --port-count <数量> 端口数量(账户数),默认 10",
|
|
139
|
+
" --user-id <编号> 客户端编号(设备ID),默认自动生成",
|
|
140
|
+
" --proxy <地址> 浏览器代理(如 socks5://127.0.0.1:1080)",
|
|
141
|
+
"",
|
|
142
|
+
" tt-help refresh [选项]",
|
|
143
|
+
" 对目标商家用户进行轮回刷新,重新采集视频 + 关注 + 粉丝",
|
|
144
|
+
" 筛选条件: tt_seller=1, verified=0, 目标国家",
|
|
145
|
+
" 选项:",
|
|
146
|
+
" --server <URL> 服务端地址,默认 http://127.0.0.1:3001",
|
|
147
|
+
` --location <国家代码> 国家筛选,逗号分隔,默认 ${DEFAULT_TARGET_LOCATIONS_CSV}`,
|
|
148
|
+
" --max-videos <数量> 每用户最大视频数,默认 16",
|
|
149
|
+
" --enable-follow 启用关注/粉丝提取(默认启用)",
|
|
150
|
+
" --disable-follow 禁用关注/粉丝提取",
|
|
151
|
+
" --max-following <数量> 最大获取关注数,默认 100",
|
|
152
|
+
" --max-followers <数量> 最大获取粉丝数,默认 100",
|
|
153
|
+
" --max-users <数量> 最大处理用户数,默认无限制",
|
|
154
|
+
" -i, --interval <秒数> 无任务时轮询间隔,默认 30 秒",
|
|
155
|
+
" --max-age <秒数> 最小刷新间隔,默认 43200(12小时)",
|
|
156
|
+
" --port <端口号> 固定 CDP 端口(调试用,关闭自动轮换)",
|
|
157
|
+
" --base-port <端口号> 起始端口,默认 9222",
|
|
158
|
+
" --port-count <数量> 端口数量(账户数),默认 10",
|
|
159
|
+
" --user-id <编号> 客户端编号(设备ID),默认自动生成",
|
|
160
|
+
" --proxy <地址> 浏览器代理(如 socks5://127.0.0.1:1080)",
|
|
140
161
|
"",
|
|
141
162
|
" tt-help info <URL> [URL2 ...] [--onlyvideo]",
|
|
142
163
|
" 获取用户/视频信息,支持多个 URL",
|
|
@@ -196,9 +217,10 @@ const HELP_TEXT = [
|
|
|
196
217
|
" -h, --help 显示帮助",
|
|
197
218
|
" --version 显示版本号",
|
|
198
219
|
"",
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
220
|
+
" 示例: tt-help info https://www.tiktok.com/@nike https://www.tiktok.com/@adidas",
|
|
221
|
+
" tt-help explore qiqi23280 fast --location ES --max-comments 50",
|
|
222
|
+
" tt-help refresh --server http://127.0.0.1:3001 --port 9222",
|
|
223
|
+
" tt-help config set server http://127.0.0.1:3001",
|
|
202
224
|
" tt-help attach -p 5 -i 10",
|
|
203
225
|
" tt-help watch -o data/result.db",
|
|
204
226
|
" tt-help videostats data/result.db -p 3",
|
package/src/main.js
CHANGED
|
@@ -10,6 +10,7 @@ import { handleComments } from "./cli/comments.js";
|
|
|
10
10
|
import { handleVideoStats } from "./cli/videostats.js";
|
|
11
11
|
import { handleDbImport } from "./cli/db-import.js";
|
|
12
12
|
import { handleWebserver } from "./cli/webserver.js";
|
|
13
|
+
import { handleRefresh } from "./cli/refresh.js";
|
|
13
14
|
|
|
14
15
|
async function main() {
|
|
15
16
|
const parsed = parseArgs();
|
|
@@ -33,6 +34,8 @@ async function main() {
|
|
|
33
34
|
return handleVideoStats(parsed);
|
|
34
35
|
case "db-import":
|
|
35
36
|
return handleDbImport(parsed);
|
|
37
|
+
case "refresh":
|
|
38
|
+
return handleRefresh(parsed);
|
|
36
39
|
}
|
|
37
40
|
|
|
38
41
|
const {
|
package/src/npm-main.js
CHANGED
|
@@ -6,6 +6,7 @@ import { handleAttach } from "./cli/attach.js";
|
|
|
6
6
|
import { handleConfig, showConfig, showUsage, version } from "./cli/config.js";
|
|
7
7
|
import { handleOpen } from "./cli/open.js";
|
|
8
8
|
import { handleComments } from "./cli/comments.js";
|
|
9
|
+
import { handleRefresh } from "./cli/refresh.js";
|
|
9
10
|
|
|
10
11
|
function exitUnsupportedCommand(command) {
|
|
11
12
|
console.error(
|
|
@@ -33,6 +34,8 @@ async function main() {
|
|
|
33
34
|
return handleOpen(parsed);
|
|
34
35
|
case "comments":
|
|
35
36
|
return handleComments(parsed);
|
|
37
|
+
case "refresh":
|
|
38
|
+
return handleRefresh(parsed);
|
|
36
39
|
}
|
|
37
40
|
|
|
38
41
|
const {
|
package/src/watch/data-store.js
CHANGED
|
@@ -2786,9 +2786,10 @@ export function createStore(filePath) {
|
|
|
2786
2786
|
return { saved: true, pinned: user.pinned };
|
|
2787
2787
|
}
|
|
2788
2788
|
|
|
2789
|
-
function getNextRedoJob(userId) {
|
|
2789
|
+
function getNextRedoJob(userId, maxAgeSeconds = 43200) {
|
|
2790
2790
|
if (db) {
|
|
2791
2791
|
const now = Date.now();
|
|
2792
|
+
const threshold = now - maxAgeSeconds * 1000;
|
|
2792
2793
|
const defaultTime = new Date("2016-01-01T00:00:00Z").getTime();
|
|
2793
2794
|
const targetLocations = [
|
|
2794
2795
|
"CZ",
|
|
@@ -2813,11 +2814,12 @@ export function createStore(filePath) {
|
|
|
2813
2814
|
WHERE tt_seller = 1
|
|
2814
2815
|
AND verified = 0
|
|
2815
2816
|
AND location_created IN (${placeholders})
|
|
2817
|
+
AND COALESCE(refresh_time, ?) < ?
|
|
2816
2818
|
ORDER BY COALESCE(refresh_time, ?) ASC
|
|
2817
2819
|
LIMIT 1
|
|
2818
2820
|
`,
|
|
2819
2821
|
)
|
|
2820
|
-
.get(...targetLocations, defaultTime);
|
|
2822
|
+
.get(...targetLocations, defaultTime, threshold, defaultTime);
|
|
2821
2823
|
if (!row) return null;
|
|
2822
2824
|
db.prepare(
|
|
2823
2825
|
"UPDATE jobs SET refresh_time = ?, updated_at = ? WHERE unique_id = ?",
|
|
@@ -2830,6 +2832,7 @@ export function createStore(filePath) {
|
|
|
2830
2832
|
}
|
|
2831
2833
|
|
|
2832
2834
|
const now = Date.now();
|
|
2835
|
+
const threshold = now - maxAgeSeconds * 1000;
|
|
2833
2836
|
const defaultTime = new Date("2016-01-01T00:00:00Z").getTime();
|
|
2834
2837
|
|
|
2835
2838
|
// 筛选目标国家用户,按 refreshTime 升序取最远的(没有则默认 2016-01-01)
|
|
@@ -2855,13 +2858,19 @@ export function createStore(filePath) {
|
|
|
2855
2858
|
);
|
|
2856
2859
|
if (targetUsers.length === 0) return null;
|
|
2857
2860
|
|
|
2858
|
-
targetUsers.
|
|
2861
|
+
const recentEnough = targetUsers.filter((u) => {
|
|
2862
|
+
const rt = u.refreshTime || defaultTime;
|
|
2863
|
+
return rt < threshold;
|
|
2864
|
+
});
|
|
2865
|
+
if (recentEnough.length === 0) return null;
|
|
2866
|
+
|
|
2867
|
+
recentEnough.sort((a, b) => {
|
|
2859
2868
|
const ta = a.refreshTime || defaultTime;
|
|
2860
2869
|
const tb = b.refreshTime || defaultTime;
|
|
2861
2870
|
return ta - tb;
|
|
2862
2871
|
});
|
|
2863
2872
|
|
|
2864
|
-
const next =
|
|
2873
|
+
const next = recentEnough[0];
|
|
2865
2874
|
next.refreshTime = now;
|
|
2866
2875
|
save();
|
|
2867
2876
|
return {
|
|
@@ -2889,9 +2898,10 @@ export function createStore(filePath) {
|
|
|
2889
2898
|
}
|
|
2890
2899
|
}
|
|
2891
2900
|
}
|
|
2901
|
+
const newUsers = processDiscoveredUsers(result);
|
|
2892
2902
|
const ret = updateJobInfo(uniqueId, user, false);
|
|
2893
2903
|
if (ret.error) return { saved: false, error: ret.error };
|
|
2894
|
-
return { saved: true };
|
|
2904
|
+
return { saved: true, newUsers };
|
|
2895
2905
|
}
|
|
2896
2906
|
|
|
2897
2907
|
const user = getUser(uniqueId);
|
|
@@ -2908,8 +2918,8 @@ export function createStore(filePath) {
|
|
|
2908
2918
|
}
|
|
2909
2919
|
}
|
|
2910
2920
|
}
|
|
2911
|
-
|
|
2912
|
-
return { saved: true };
|
|
2921
|
+
const newUsers = processDiscoveredUsers(result);
|
|
2922
|
+
return { saved: true, newUsers };
|
|
2913
2923
|
}
|
|
2914
2924
|
|
|
2915
2925
|
function reportClientError(
|
package/src/watch/public/app.js
CHANGED
|
@@ -368,6 +368,10 @@ function renderTable(users) {
|
|
|
368
368
|
}
|
|
369
369
|
const claimTime = u.claimedAt ? formatTime(u.claimedAt) : "-";
|
|
370
370
|
const procTime = u.processedAt ? formatTime(u.processedAt) : "-";
|
|
371
|
+
const refreshTime =
|
|
372
|
+
u.ttSeller && !u.verified && u.refreshTime
|
|
373
|
+
? formatTime(u.refreshTime)
|
|
374
|
+
: "-";
|
|
371
375
|
const statusCodeDisplay =
|
|
372
376
|
u.statusCode != null && u.statusCode !== 0
|
|
373
377
|
? `<span class="tag error" style="font-size:10px">${u.statusCode}</span>`
|
|
@@ -378,6 +382,7 @@ function renderTable(users) {
|
|
|
378
382
|
<td data-label="粉丝">${fans}</td>
|
|
379
383
|
<td data-label="视频">${videos}</td>
|
|
380
384
|
<td data-label="国家">${loc}</td>
|
|
385
|
+
<td data-label="最近刷新" style="font-size:11px;color:#888">${refreshTime}</td>
|
|
381
386
|
<td data-label="最近发布" style="font-size:11px;color:#888">${latestVideo}</td>
|
|
382
387
|
<td data-label="猜测国家">${guessedLoc}</td>
|
|
383
388
|
<td data-label="来源">${sources || "-"}</td>
|
|
@@ -125,6 +125,7 @@
|
|
|
125
125
|
<th>粉丝</th>
|
|
126
126
|
<th>视频</th>
|
|
127
127
|
<th>国家</th>
|
|
128
|
+
<th class="sortable" data-sort="refreshTime">最近刷新 <span class="sort-icon">↕</span></th>
|
|
128
129
|
<th class="sortable" data-sort="latestVideoTime">最近发布 <span class="sort-icon">↕</span></th>
|
|
129
130
|
<th>猜测国家</th>
|
|
130
131
|
<th>来源</th>
|
package/src/watch/server.js
CHANGED
|
@@ -356,7 +356,8 @@ export function startWatchServer(dataAnchor, port = 3000, existingStore) {
|
|
|
356
356
|
|
|
357
357
|
if (req.method === "GET" && routePath === "/api/redo-job") {
|
|
358
358
|
const userId = params.userId || "";
|
|
359
|
-
const
|
|
359
|
+
const maxAge = parseInt(params.maxAge) || 43200;
|
|
360
|
+
const job = store.getNextRedoJob(userId, maxAge);
|
|
360
361
|
if (job) {
|
|
361
362
|
logJob("REDO-CLAIM", { user: job.uniqueId, clientId: userId });
|
|
362
363
|
sendJSON(res, 200, { hasJob: true, user: job });
|
|
@@ -683,6 +684,7 @@ export function startWatchServer(dataAnchor, port = 3000, existingStore) {
|
|
|
683
684
|
followerCount: u.followerCount,
|
|
684
685
|
locationCreated: u.locationCreated,
|
|
685
686
|
latestVideoTime: u.latestVideoTime,
|
|
687
|
+
refreshTime: u.refreshTime,
|
|
686
688
|
guessedLocation: u.guessedLocation,
|
|
687
689
|
pinned: u.pinned,
|
|
688
690
|
processedAt: u.processedAt,
|
|
@@ -1,213 +0,0 @@
|
|
|
1
|
-
import {
|
|
2
|
-
delay,
|
|
3
|
-
retryWithBackoff,
|
|
4
|
-
detectPageError,
|
|
5
|
-
assertPageUrl,
|
|
6
|
-
} from "./modules/page-helpers.js";
|
|
7
|
-
import { detectCaptcha } from "./modules/captcha-handler.js";
|
|
8
|
-
import { getUserInfo, collectVideos } from "../videos/core.js";
|
|
9
|
-
import { extractFollowAndFollowers } from "./modules/follow-extractor.js";
|
|
10
|
-
import { processExplore } from "./explore-core.js";
|
|
11
|
-
import { DEFAULT_TARGET_LOCATIONS_CSV } from "../lib/target-locations.js";
|
|
12
|
-
|
|
13
|
-
export async function processRefresh(page, username, serverUrl, options, log) {
|
|
14
|
-
const { maxFollowing = 100, maxFollowers = 100, maxVideos = 100 } = options;
|
|
15
|
-
|
|
16
|
-
const result = {
|
|
17
|
-
userInfo: null,
|
|
18
|
-
discoveredVideoAuthors: [],
|
|
19
|
-
discoveredFollowing: [],
|
|
20
|
-
discoveredFollowers: [],
|
|
21
|
-
newUsersAdded: 0,
|
|
22
|
-
collectedVideos: 0,
|
|
23
|
-
error: null,
|
|
24
|
-
};
|
|
25
|
-
|
|
26
|
-
try {
|
|
27
|
-
log(` 访问 @${username} 主页...`);
|
|
28
|
-
const homeUrl = `https://www.tiktok.com/@${username}`;
|
|
29
|
-
await retryWithBackoff(
|
|
30
|
-
async () => {
|
|
31
|
-
await page.goto(homeUrl, {
|
|
32
|
-
waitUntil: "domcontentloaded",
|
|
33
|
-
timeout: 30000,
|
|
34
|
-
});
|
|
35
|
-
assertPageUrl(page, `@${username}`);
|
|
36
|
-
},
|
|
37
|
-
{ log },
|
|
38
|
-
);
|
|
39
|
-
await page
|
|
40
|
-
.waitForSelector('[class*="DivVideoList"]', { timeout: 10000 })
|
|
41
|
-
.catch(() => {});
|
|
42
|
-
await delay(1000, 2000);
|
|
43
|
-
|
|
44
|
-
log(" 获取用户信息...");
|
|
45
|
-
const info = await getUserInfo(page);
|
|
46
|
-
if (info) {
|
|
47
|
-
result.userInfo = info;
|
|
48
|
-
log(
|
|
49
|
-
` 用户: ${info.nickname || username} | 粉丝: ${info.followerCount || "-"} | 视频: ${info.videoCount || "-"}`,
|
|
50
|
-
);
|
|
51
|
-
}
|
|
52
|
-
|
|
53
|
-
const captcha = await detectCaptcha(page);
|
|
54
|
-
if (captcha && captcha.visible) {
|
|
55
|
-
log(`[验证码] @${username} 页面出现验证码`);
|
|
56
|
-
result.captchaDetected = true;
|
|
57
|
-
result.captchaStage = result.captchaStage || "video-page";
|
|
58
|
-
result.captchaMessage = result.captchaMessage || "视频页出现验证码";
|
|
59
|
-
}
|
|
60
|
-
|
|
61
|
-
// 采集视频
|
|
62
|
-
log(` 采集视频 (最多 ${maxVideos} 个)...`);
|
|
63
|
-
const videoList = await collectVideos(page, username, maxVideos, log);
|
|
64
|
-
const videoArray = videoList ? [...videoList.values()] : [];
|
|
65
|
-
result.collectedVideos = videoArray.length;
|
|
66
|
-
result.discoveredVideoAuthors = videoArray.map((v) => v.author);
|
|
67
|
-
|
|
68
|
-
if (videoArray.length <= 0) {
|
|
69
|
-
result.noVideo = true;
|
|
70
|
-
const pageError = await detectPageError(page);
|
|
71
|
-
if (pageError) {
|
|
72
|
-
result.restricted = true;
|
|
73
|
-
log(` @${username} 页面受限(${pageError}),标记跳过`);
|
|
74
|
-
}
|
|
75
|
-
return result;
|
|
76
|
-
}
|
|
77
|
-
|
|
78
|
-
// 采集关注和粉丝
|
|
79
|
-
log(` 采集关注 (最多 ${maxFollowing}) + 粉丝 (最多 ${maxFollowers})...`);
|
|
80
|
-
try {
|
|
81
|
-
const followResult = await extractFollowAndFollowers(page, {
|
|
82
|
-
maxFollowing,
|
|
83
|
-
maxFollowers,
|
|
84
|
-
});
|
|
85
|
-
result.discoveredFollowing = followResult.following || [];
|
|
86
|
-
result.discoveredFollowers = followResult.followers || [];
|
|
87
|
-
log(
|
|
88
|
-
` 关注: ${result.discoveredFollowing.length}, 粉丝: ${result.discoveredFollowers.length}`,
|
|
89
|
-
);
|
|
90
|
-
} catch (e) {
|
|
91
|
-
log(` [关注/粉丝采集失败] ${e.message}`);
|
|
92
|
-
result.discoveredFollowing = [];
|
|
93
|
-
result.discoveredFollowers = [];
|
|
94
|
-
}
|
|
95
|
-
|
|
96
|
-
// 处理新发现的用户(关注 + 粉丝),循环执行完整 explore
|
|
97
|
-
// follow-extractor 返回 [handle, displayName] 数组
|
|
98
|
-
const allDiscovered = [
|
|
99
|
-
...result.discoveredFollowing.map((h) => ({
|
|
100
|
-
handle: Array.isArray(h) ? h[0] : h,
|
|
101
|
-
source: "refresh-following",
|
|
102
|
-
})),
|
|
103
|
-
...result.discoveredFollowers.map((h) => ({
|
|
104
|
-
handle: Array.isArray(h) ? h[0] : h,
|
|
105
|
-
source: "refresh-follower",
|
|
106
|
-
})),
|
|
107
|
-
];
|
|
108
|
-
|
|
109
|
-
for (const { handle, source } of allDiscovered) {
|
|
110
|
-
const uniqueId = handle.replace("@", "");
|
|
111
|
-
|
|
112
|
-
// 检查用户是否已存在
|
|
113
|
-
const existsResp = await fetch(
|
|
114
|
-
`${serverUrl}/api/user-exists/${encodeURIComponent(uniqueId)}`,
|
|
115
|
-
);
|
|
116
|
-
const existsData = await existsResp.json();
|
|
117
|
-
|
|
118
|
-
if (existsData.exists) {
|
|
119
|
-
continue;
|
|
120
|
-
}
|
|
121
|
-
|
|
122
|
-
log(` [新用户] @${uniqueId} 不存在,开始探索 (来源: ${source})...`);
|
|
123
|
-
await delay(1000, 2000);
|
|
124
|
-
|
|
125
|
-
// 对新用户做完整 explore(与 explore 命令逻辑一致)
|
|
126
|
-
const exploreResult = await processExplore(
|
|
127
|
-
page,
|
|
128
|
-
uniqueId,
|
|
129
|
-
{
|
|
130
|
-
maxComments: 10,
|
|
131
|
-
maxGuess: 0,
|
|
132
|
-
enableFollow: true,
|
|
133
|
-
maxFollowing: 5,
|
|
134
|
-
maxFollowers: 5,
|
|
135
|
-
location: DEFAULT_TARGET_LOCATIONS_CSV,
|
|
136
|
-
},
|
|
137
|
-
log,
|
|
138
|
-
);
|
|
139
|
-
|
|
140
|
-
// 提交 explore 结果到服务端(和 explore 命令的 commitJob 一致)
|
|
141
|
-
if (exploreResult.userInfo) {
|
|
142
|
-
const guessedLocation = exploreResult.locationCreated || null;
|
|
143
|
-
|
|
144
|
-
const payload = {
|
|
145
|
-
userInfo: exploreResult.userInfo || {},
|
|
146
|
-
discoveredVideoAuthors: (
|
|
147
|
-
exploreResult.discoveredVideoAuthors || []
|
|
148
|
-
).map((item) =>
|
|
149
|
-
typeof item === "object" ? { ...item, guessedLocation } : item,
|
|
150
|
-
),
|
|
151
|
-
discoveredCommentAuthors: (
|
|
152
|
-
exploreResult.discoveredCommentAuthors || []
|
|
153
|
-
).map((author) => ({ author, guessedLocation })),
|
|
154
|
-
discoveredGuessAuthors: (
|
|
155
|
-
exploreResult.discoveredGuessAuthors || []
|
|
156
|
-
).map((author) => ({ author, guessedLocation })),
|
|
157
|
-
discoveredFollowing: (exploreResult.discoveredFollowing || []).map(
|
|
158
|
-
(f) => ({
|
|
159
|
-
handle: Array.isArray(f) ? f[0] : f,
|
|
160
|
-
displayName: Array.isArray(f) ? f[1] : null,
|
|
161
|
-
guessedLocation,
|
|
162
|
-
}),
|
|
163
|
-
),
|
|
164
|
-
discoveredFollowers: (exploreResult.discoveredFollowers || []).map(
|
|
165
|
-
(f) => ({
|
|
166
|
-
handle: Array.isArray(f) ? f[0] : f,
|
|
167
|
-
displayName: Array.isArray(f) ? f[1] : null,
|
|
168
|
-
guessedLocation,
|
|
169
|
-
}),
|
|
170
|
-
),
|
|
171
|
-
processed: exploreResult.processed,
|
|
172
|
-
hasFollowData: exploreResult.hasFollowData,
|
|
173
|
-
keepFollow: exploreResult.keepFollow,
|
|
174
|
-
locationCreated: exploreResult.locationCreated,
|
|
175
|
-
noVideo: exploreResult.noVideo,
|
|
176
|
-
restricted: exploreResult.restricted,
|
|
177
|
-
error: exploreResult.error,
|
|
178
|
-
};
|
|
179
|
-
|
|
180
|
-
const addResp = await fetch(
|
|
181
|
-
`${serverUrl}/api/explore-new/${uniqueId}`,
|
|
182
|
-
{
|
|
183
|
-
method: "POST",
|
|
184
|
-
headers: { "Content-Type": "application/json" },
|
|
185
|
-
body: JSON.stringify(payload),
|
|
186
|
-
},
|
|
187
|
-
);
|
|
188
|
-
const addResult = await addResp.json();
|
|
189
|
-
|
|
190
|
-
if (!addResult.saved) {
|
|
191
|
-
log(` [跳过] @${uniqueId} 提交失败`);
|
|
192
|
-
continue;
|
|
193
|
-
}
|
|
194
|
-
|
|
195
|
-
result.newUsersAdded++;
|
|
196
|
-
if (exploreResult.captchaDetected) {
|
|
197
|
-
result.captchaDetected = true;
|
|
198
|
-
}
|
|
199
|
-
log(
|
|
200
|
-
` [已提交] @${uniqueId} ${addResult.created ? "(新用户)" : "(已存在)"} | 发现: ${addResult.newUsers?.length || 0} 个`,
|
|
201
|
-
);
|
|
202
|
-
}
|
|
203
|
-
|
|
204
|
-
await delay(2000, 4000);
|
|
205
|
-
}
|
|
206
|
-
} catch (e) {
|
|
207
|
-
log(` [错误] ${e.message}`);
|
|
208
|
-
result.error = e.message;
|
|
209
|
-
result.errorStack = e.stack || "";
|
|
210
|
-
}
|
|
211
|
-
|
|
212
|
-
return result;
|
|
213
|
-
}
|