tt-help-cli-ycl 1.3.62 → 1.3.63

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "tt-help-cli-ycl",
3
- "version": "1.3.62",
3
+ "version": "1.3.63",
4
4
  "description": "TikTok user & video data scraper - extract ttSeller, verified, locationCreated from HTML source",
5
5
  "type": "module",
6
6
  "bin": {
@@ -78,6 +78,7 @@ export async function handleExplore(options) {
78
78
  explorePortCount,
79
79
  exploreUserId,
80
80
  exploreMaxVideos,
81
+ exploreProxy,
81
82
  } = options;
82
83
 
83
84
  let userId = exploreUserId || configuredUserId;
@@ -150,6 +151,10 @@ export async function handleExplore(options) {
150
151
  cdpOptions.userDataDir = currentAccount.userDataDir;
151
152
  }
152
153
 
154
+ if (exploreProxy) {
155
+ cdpOptions.proxyServer = exploreProxy;
156
+ }
157
+
153
158
  console.error(`CDP 端口: ${cdpOptions.port}, 用户编号: ${userId}`);
154
159
  console.error(`浏览器配置: ${path.basename(cdpOptions.userDataDir)}`);
155
160
  if (!explorePort) {
@@ -3,61 +3,82 @@ import {
3
3
  isBrowserClosedError,
4
4
  relaunchBrowser,
5
5
  } from "../lib/browser/page.js";
6
- import { delay, setDelayConfig } from "../scraper/modules/page-helpers.js";
6
+ import {
7
+ delay,
8
+ getDelayConfig,
9
+ setDelayConfig,
10
+ } from "../scraper/modules/page-helpers.js";
11
+ import {
12
+ detectCaptcha,
13
+ closeCaptcha,
14
+ } from "../scraper/modules/captcha-handler.js";
7
15
  import { userId as configuredUserId, saveUserId } from "../lib/constants.js";
8
16
  import { getMacOrUuid } from "../lib/mac-or-uuid.js";
9
- import { ensureBrowserReady as ensureBrowserReadyCDP } from "../lib/browser/cdp.js";
10
- import { processRefresh } from "../scraper/refresh-core.js";
17
+ import {
18
+ ensureBrowserReady as ensureBrowserReadyCDP,
19
+ switchAccount,
20
+ } from "../lib/browser/cdp.js";
21
+ import { HealthChecker } from "../lib/browser/health-checker.js";
11
22
  import path from "path";
12
23
  import os from "os";
13
24
 
14
- async function withRetry(fn, maxRetries = 5) {
15
- for (let attempt = 1; attempt <= maxRetries; attempt++) {
25
+ const MAX_RETRY_WAIT = 5 * 60 * 1000;
26
+ const STARTUP_TIKTOK_URL = "https://www.tiktok.com/@ycl5007";
27
+
28
+ async function withRetry(label, fn) {
29
+ let backoff = 1000;
30
+ while (true) {
16
31
  try {
17
32
  return await fn();
18
- } catch (e) {
19
- if (attempt < maxRetries) {
20
- const waitTime =
21
- attempt <= 2
22
- ? 5000 + Math.random() * 5000
23
- : attempt <= 4
24
- ? 10000 + Math.random() * 10000
25
- : 20000 + Math.random() * 10000;
26
- console.error(
27
- ` [网络] 请求失败 (${attempt}/${maxRetries}),${Math.round(waitTime / 1000)}s 后重试...`,
28
- );
29
- await delay(waitTime / 1000, waitTime / 1000);
30
- } else {
31
- throw e;
32
- }
33
+ } catch (err) {
34
+ console.error(
35
+ `[连接] ${label} 失败: ${err.message},${backoff / 1000}秒后重试...`,
36
+ );
37
+ await new Promise((r) => setTimeout(r, backoff));
38
+ if (backoff < MAX_RETRY_WAIT) backoff *= 2;
33
39
  }
34
40
  }
35
41
  }
36
42
 
37
- async function apiGet(url) {
38
- const resp = await fetch(url);
39
- if (!resp.ok) throw new Error(`HTTP ${resp.status}: ${resp.statusText}`);
40
- return resp.json();
43
+ async function apiPost(url, body) {
44
+ return withRetry(`POST ${url}`, async () => {
45
+ const res = await fetch(url, {
46
+ method: "POST",
47
+ headers: { "Content-Type": "application/json" },
48
+ body: JSON.stringify(body),
49
+ });
50
+ return res.json();
51
+ });
41
52
  }
42
53
 
43
- async function apiPost(url, body) {
44
- const resp = await fetch(url, {
45
- method: "POST",
46
- headers: { "Content-Type": "application/json" },
47
- body: JSON.stringify(body),
54
+ async function apiGet(url) {
55
+ return withRetry(`GET ${url}`, async () => {
56
+ const res = await fetch(url);
57
+ return res.json();
48
58
  });
49
- if (!resp.ok) throw new Error(`HTTP ${resp.status}: ${resp.statusText}`);
50
- return resp.json();
51
59
  }
52
60
 
53
61
  export async function handleRefresh(options) {
54
62
  const {
55
63
  explorePreset,
64
+ exploreInterval,
65
+ exploreEnableFollow,
66
+ exploreMaxFollowing,
67
+ exploreMaxFollowers,
68
+ exploreLocation,
69
+ exploreMaxUsers,
70
+ serverUrl,
56
71
  explorePort,
57
- exploreProfile,
72
+ exploreBasePort,
73
+ explorePortCount,
58
74
  exploreUserId,
59
- serverUrl,
75
+ exploreMaxVideos,
76
+ exploreProfile,
77
+ exploreRedoMaxAge,
78
+ exploreProxy,
60
79
  } = options;
80
+
81
+ let userId = exploreUserId || configuredUserId;
61
82
  let browser = null;
62
83
  let shuttingDown = false;
63
84
 
@@ -81,7 +102,6 @@ export async function handleRefresh(options) {
81
102
  process.once("SIGTERM", onSigterm);
82
103
 
83
104
  try {
84
- let userId = exploreUserId || configuredUserId;
85
105
  if (!userId) {
86
106
  userId = await getMacOrUuid();
87
107
  saveUserId(userId);
@@ -90,196 +110,450 @@ export async function handleRefresh(options) {
90
110
 
91
111
  setDelayConfig(explorePreset);
92
112
 
93
- console.error(`\n=== Refresh 模式 ===`);
113
+ // 连接服务器验证
114
+ await apiGet(`${serverUrl}/api/stats`);
115
+
116
+ console.error(`\n=== Refresh 模式(基于 explore) ===`);
94
117
  console.error(`服务器: ${serverUrl}`);
95
- console.error(`CDP 端口: ${explorePort || 9222}, 用户编号: ${userId}`);
96
- if (exploreProfile) console.error(`浏览器配置: ${exploreProfile}`);
97
- console.error(`刷新: 视频 100 + 关注 100 + 粉丝 100`);
98
- console.error(`新用户探索: 评论 + 猜你喜欢 + 关注/粉丝`);
118
+ console.error(`视频采集: ${exploreMaxVideos || 16}`);
119
+ console.error(`关注/粉丝: ${exploreEnableFollow ? "启用" : "禁用"} (${exploreMaxFollowing}/${exploreMaxFollowers})`);
120
+ console.error(`国家筛选: ${exploreLocation}`);
121
+ console.error(`空闲间隔: ${exploreInterval || 30} 秒`);
122
+ if (exploreMaxUsers > 0) console.error(`上限: ${exploreMaxUsers} 个用户`);
123
+
124
+ const healthChecker = new HealthChecker({
125
+ basePort: exploreBasePort,
126
+ totalAccounts: explorePortCount,
127
+ });
128
+ let currentAccount = healthChecker.getCurrentAccount();
99
129
 
100
130
  const cdpOptions = {};
101
- if (explorePort) cdpOptions.port = explorePort;
102
- if (exploreProfile) {
131
+ if (explorePort) {
132
+ cdpOptions.port = explorePort;
133
+ cdpOptions.userDataDir = path.join(
134
+ os.homedir(),
135
+ "Library",
136
+ "Application Support",
137
+ `Microsoft Edge For Testing_p${explorePort}`,
138
+ );
139
+ } else if (exploreProfile) {
103
140
  cdpOptions.userDataDir = path.join(
104
141
  os.homedir(),
105
142
  "Library",
106
143
  "Application Support",
107
144
  `Microsoft Edge For Testing_${exploreProfile}`,
108
145
  );
146
+ cdpOptions.port = currentAccount.port;
147
+ } else {
148
+ cdpOptions.port = currentAccount.port;
149
+ cdpOptions.userDataDir = currentAccount.userDataDir;
150
+ }
151
+
152
+ if (exploreProxy) {
153
+ cdpOptions.proxyServer = exploreProxy;
154
+ }
155
+
156
+ console.error(`CDP 端口: ${cdpOptions.port}, 用户编号: ${userId}`);
157
+ console.error(`浏览器配置: ${path.basename(cdpOptions.userDataDir)}`);
158
+ if (!explorePort && !exploreProfile) {
159
+ const portRange = `${currentAccount.port}-${currentAccount.port + healthChecker.accounts.length - 1}`;
160
+ console.error(
161
+ `端口轮换范围: ${portRange}(共 ${healthChecker.accounts.length} 个)`,
162
+ );
109
163
  }
110
164
 
111
165
  browser = await ensureBrowserReadyCDP(cdpOptions);
166
+ const { processExplore } = await import("../scraper/explore-core.js");
167
+ const { isLoggedIn } = await import("../lib/browser/page.js");
168
+
112
169
  const page = await getOrCreatePage(browser);
113
170
 
171
+ // 导航到 TikTok 页面
172
+ await page.goto(STARTUP_TIKTOK_URL, {
173
+ waitUntil: "domcontentloaded",
174
+ });
175
+
176
+ // 检测登录状态
177
+ let loggedIn = await isLoggedIn(page);
178
+ console.error(`登录状态: ${loggedIn ? "已登录" : "未登录"}`);
179
+
180
+ // 全局拦截图片资源
181
+ await page.route("**/*", (route) => {
182
+ const resourceType = route.request().resourceType();
183
+ if (resourceType === "image" || resourceType === "stylesheet") {
184
+ route.abort();
185
+ } else {
186
+ route.continue();
187
+ }
188
+ });
189
+
190
+ // 账户切换后,重新初始化 page
191
+ async function setupNewPage(newBrowser) {
192
+ const newPage = await getOrCreatePage(newBrowser);
193
+ await newPage.route("**/*", (route) => {
194
+ const resourceType = route.request().resourceType();
195
+ if (resourceType === "image" || resourceType === "stylesheet") {
196
+ route.abort();
197
+ } else {
198
+ route.continue();
199
+ }
200
+ });
201
+ return newPage;
202
+ }
203
+
204
+ async function handleAccountSwitch(reason) {
205
+ console.error(`\n[健康检查] ${reason},切换到下一个账户...`);
206
+ const oldAccount = currentAccount;
207
+ const nextAccount = healthChecker.getNextAccount();
208
+ currentAccount = nextAccount;
209
+ const newBrowser = await switchAccount(
210
+ { port: oldAccount.port, userDataDir: oldAccount.userDataDir },
211
+ { port: nextAccount.port, userDataDir: nextAccount.userDataDir },
212
+ );
213
+ browser = newBrowser;
214
+ const newPage = await setupNewPage(browser);
215
+ Object.assign(page, newPage);
216
+ Object.assign(cdpOptions, {
217
+ port: nextAccount.port,
218
+ userDataDir: nextAccount.userDataDir,
219
+ });
220
+ console.error(`[健康检查] 已切换到端口 ${nextAccount.port}`);
221
+ await page.goto(STARTUP_TIKTOK_URL, {
222
+ waitUntil: "domcontentloaded",
223
+ });
224
+ loggedIn = await isLoggedIn(page);
225
+ console.error(
226
+ `[健康检查] 新账户登录状态: ${loggedIn ? "已登录" : "未登录"}`,
227
+ );
228
+ lastFollowSuccessTime = Date.now();
229
+ captchaCount = 0;
230
+ blockedCount = 0;
231
+ consecutiveNetworkErrors = 0;
232
+ }
233
+
114
234
  let processedCount = 0;
115
235
  let errorCount = 0;
116
236
  let consecutiveNetworkErrors = 0;
237
+ let captchaCount = 0;
238
+ let blockedCount = 0;
239
+ let lastFollowSuccessTime = Date.now();
240
+ const FOLLOW_BLOCK_THRESHOLD = 10 * 60 * 1000;
117
241
 
118
242
  console.error(`\n开始循环刷新任务...\n`);
119
243
 
120
- while (!shuttingDown) {
244
+ while (true) {
245
+ if (shuttingDown) break;
246
+
247
+ // 健康检查
248
+ const checkResult = healthChecker.check();
249
+ if (checkResult.shouldSwitch) {
250
+ await handleAccountSwitch(checkResult.reason);
251
+ } else if (checkResult.info && processedCount % 10 === 0) {
252
+ const followElapsed = loggedIn && exploreEnableFollow
253
+ ? Math.round((Date.now() - lastFollowSuccessTime) / 60000)
254
+ : 0;
255
+ const followStatus =
256
+ loggedIn && exploreEnableFollow
257
+ ? ` | 关注/粉丝上次成功 ${followElapsed} 分钟前`
258
+ : "";
259
+ console.error(`\n[健康检查] ${checkResult.info}${followStatus}`);
260
+ }
261
+
262
+ // 领取 redo 任务
263
+ let redoUrl = `${serverUrl}/api/redo-job?userId=${userId}`;
264
+ if (exploreRedoMaxAge > 0) {
265
+ redoUrl += `&maxAge=${exploreRedoMaxAge}`;
266
+ }
267
+ let job = null;
121
268
  try {
122
- const jobData = await withRetry(() =>
123
- apiGet(`${serverUrl}/api/redo-job?userId=${userId}`),
269
+ job = await apiGet(redoUrl);
270
+ } catch (e) {
271
+ consecutiveNetworkErrors++;
272
+ console.error(
273
+ ` [网络] 获取任务失败 (${consecutiveNetworkErrors}): ${e.message}`
124
274
  );
275
+ await new Promise((r) => setTimeout(r, 10000));
276
+ continue;
277
+ }
125
278
 
126
- if (!jobData.hasJob) {
127
- console.error(`\n[空闲] 暂无 redo 任务,30s 后重试...`);
128
- await delay(30000, 30000);
129
- continue;
279
+ if (!job.hasJob) {
280
+ console.error(`\n[空闲] 暂无 redo 任务,${exploreInterval || 30}s 后重试...`);
281
+ await new Promise((r) => setTimeout(r, (exploreInterval || 30) * 1000));
282
+ consecutiveNetworkErrors = 0;
283
+ continue;
284
+ }
285
+
286
+ consecutiveNetworkErrors = 0;
287
+ const username = job.user.uniqueId;
288
+ const nickname = job.user.nickname;
289
+ console.error(`\n[${processedCount}] 刷新 @${username}...`);
290
+
291
+ // 切换任务前检测验证码
292
+ const captchaResult = await detectCaptcha(page);
293
+ if (captchaResult && captchaResult.visible) {
294
+ console.error(`\n[验证码] 切换任务前检测到验证码,等待3分钟...`);
295
+ captchaCount++;
296
+ console.error(` [验证码] 累计 ${captchaCount} 次`);
297
+
298
+ await new Promise((r) => setTimeout(r, 180000));
299
+
300
+ const closeResult = await closeCaptcha(page);
301
+ if (closeResult.success) {
302
+ console.error(" [验证码] 已关闭验证码");
303
+ await new Promise((r) => setTimeout(r, 10000));
304
+ } else {
305
+ console.error(" [验证码] 无法关闭验证码,继续处理");
130
306
  }
131
307
 
132
- const { uniqueId, nickname } = jobData.user;
133
- consecutiveNetworkErrors = 0;
134
- processedCount++;
308
+ await withRetry("report captcha", () =>
309
+ apiPost(`${serverUrl}/api/error-report`, {
310
+ userId,
311
+ username: "unknown",
312
+ errorType: "captcha",
313
+ errorMessage: "切换任务前检测到验证码",
314
+ stage: "between-tasks",
315
+ errorStack: "",
316
+ }),
317
+ ).catch(() => {});
135
318
 
136
- console.error(
137
- `\n[${processedCount}] 刷新 @${uniqueId} (${nickname || "未知"})...`,
138
- );
319
+ if (captchaCount >= 2) {
320
+ await handleAccountSwitch(`验证码累计 ${captchaCount} 次`);
321
+ captchaCount = 0;
322
+ }
323
+ }
324
+
325
+ // 关注/粉丝封禁检测
326
+ if (loggedIn && exploreEnableFollow) {
327
+ const followElapsed = Date.now() - lastFollowSuccessTime;
328
+ if (followElapsed > FOLLOW_BLOCK_THRESHOLD) {
329
+ const minutes = Math.round(followElapsed / 60000);
330
+ console.error(
331
+ `\n[封禁检测] 关注/粉丝功能已 ${minutes} 分钟未成功获取,切换到下一个账户...`,
332
+ );
333
+ await handleAccountSwitch(`关注/粉丝功能被封`);
334
+ }
335
+ }
336
+
337
+ if (consecutiveNetworkErrors > 0) {
338
+ const waitTime =
339
+ consecutiveNetworkErrors <= 2
340
+ ? 0
341
+ : consecutiveNetworkErrors <= 5
342
+ ? 30000
343
+ : 300000;
344
+ if (waitTime > 0) {
345
+ console.error(
346
+ ` [网络] 连续 ${consecutiveNetworkErrors} 次网络异常,等待 ${waitTime / 1000}s 后重试...`,
347
+ );
348
+ await new Promise((r) => setTimeout(r, waitTime));
349
+ }
350
+ }
139
351
 
140
- const result = await processRefresh(
352
+ const { switchMax } = getDelayConfig();
353
+ await delay(switchMax, switchMax * 3);
354
+
355
+ let result = await processExplore(
356
+ page,
357
+ username,
358
+ {
359
+ maxVideos: exploreMaxVideos || 16,
360
+ enableFollow: exploreEnableFollow !== false,
361
+ loggedIn,
362
+ maxFollowing: exploreMaxFollowing || 100,
363
+ maxFollowers: exploreMaxFollowers || 100,
364
+ location: exploreLocation,
365
+ browser,
366
+ },
367
+ console.error,
368
+ );
369
+
370
+ // 浏览器关闭检测
371
+ if (result.error && isBrowserClosedError(new Error(result.error))) {
372
+ const newBrowser = await relaunchBrowser(
373
+ cdpOptions,
374
+ cdpOptions.port || 9222,
375
+ );
376
+ browser = newBrowser;
377
+ const newPage = await setupNewPage(browser);
378
+ Object.assign(page, newPage);
379
+ result = await processExplore(
141
380
  page,
142
- uniqueId,
143
- serverUrl,
381
+ username,
144
382
  {
145
- maxFollowing: 100,
146
- maxFollowers: 100,
147
- maxVideos: 100,
383
+ maxVideos: exploreMaxVideos || 16,
384
+ enableFollow: exploreEnableFollow !== false,
385
+ loggedIn,
386
+ maxFollowing: exploreMaxFollowing || 100,
387
+ maxFollowers: exploreMaxFollowers || 100,
388
+ location: exploreLocation,
389
+ browser,
148
390
  },
149
391
  console.error,
150
392
  );
393
+ }
151
394
 
152
- if (result.restricted) {
153
- console.error(` @${uniqueId} 页面受限,跳过`);
154
- await apiPost(`${serverUrl}/api/redo-job/${uniqueId}`, {
155
- restricted: true,
156
- userInfo: result.userInfo || {},
157
- });
158
- continue;
395
+ if (result.restricted) {
396
+ consecutiveNetworkErrors = 0;
397
+ await apiPost(`${serverUrl}/api/redo-job/${username}`, {
398
+ restricted: true,
399
+ userInfo: result.userInfo || {},
400
+ });
401
+ if (exploreMaxUsers > 0 && processedCount >= exploreMaxUsers) {
402
+ console.error(`\n已达上限 ${exploreMaxUsers} 个用户,停止处理`);
403
+ break;
159
404
  }
405
+ continue;
406
+ }
160
407
 
161
- if (result.error) {
162
- // 浏览器关闭检测
163
- if (isBrowserClosedError(new Error(result.error))) {
164
- const newBrowser = await relaunchBrowser(
165
- cdpOptions,
166
- explorePort || 9222,
167
- );
168
- browser = newBrowser;
169
- const newPage = await getOrCreatePage(browser);
170
- Object.assign(page, newPage);
171
- // 重试当前用户
172
- const retryResult = await processRefresh(
173
- page,
174
- uniqueId,
175
- serverUrl,
176
- {
177
- maxFollowing: 100,
178
- maxFollowers: 100,
179
- maxVideos: 100,
180
- },
181
- console.error,
182
- );
183
- Object.assign(result, retryResult);
184
- // 继续下方逻辑,检查重试后的 result
185
- } else {
186
- consecutiveNetworkErrors++;
187
- errorCount++;
188
- console.error(` [错误] ${result.error}`);
189
-
190
- if (consecutiveNetworkErrors >= 3) {
191
- console.error(
192
- ` [警告] 连续 ${consecutiveNetworkErrors} 次错误,等待 60s 后重试...`,
193
- );
194
- await delay(60000, 60000);
195
- consecutiveNetworkErrors = 0;
196
- }
408
+ if (result.error) {
409
+ consecutiveNetworkErrors++;
410
+ errorCount++;
411
+
412
+ // 临时性错误:自动重试一次
413
+ if (result.retryable) {
414
+ console.error(` [临时错误] 等待 5 秒后重试 @${username}...`);
415
+ await new Promise((r) => setTimeout(r, 5000));
416
+ result = await processExplore(
417
+ page,
418
+ username,
419
+ {
420
+ maxVideos: exploreMaxVideos || 16,
421
+ enableFollow: exploreEnableFollow !== false,
422
+ loggedIn,
423
+ maxFollowing: exploreMaxFollowing || 100,
424
+ maxFollowers: exploreMaxFollowers || 100,
425
+ location: exploreLocation,
426
+ browser,
427
+ },
428
+ console.error,
429
+ );
430
+ if (!result.error) {
431
+ consecutiveNetworkErrors = 0;
432
+ errorCount--;
433
+ }
434
+ }
197
435
 
198
- await apiPost(`${serverUrl}/api/redo-job/${uniqueId}`, {
199
- error: result.error,
200
- userInfo: result.userInfo || {},
201
- });
202
- const errorType =
203
- consecutiveNetworkErrors > 1 ? "network" : "other";
436
+ if (result.error) {
437
+ await apiPost(`${serverUrl}/api/redo-job/${username}`, {
438
+ error: result.error,
439
+ });
440
+ const errorType = result.error.startsWith("被封:")
441
+ ? "被封"
442
+ : consecutiveNetworkErrors > 1
443
+ ? "network"
444
+ : "other";
445
+ await withRetry("report error", () =>
204
446
  apiPost(`${serverUrl}/api/error-report`, {
205
447
  userId,
206
- username: uniqueId,
448
+ username,
207
449
  errorType,
208
450
  errorMessage: result.error,
209
451
  stage: "process",
210
452
  errorStack: result.errorStack || "",
211
- }).catch(() => {});
453
+ }),
454
+ ).catch(() => {});
455
+ if (errorType === "被封") {
456
+ blockedCount++;
457
+ console.error(` [被封] 累计 ${blockedCount} 次`);
458
+ if (blockedCount >= 3) {
459
+ await handleAccountSwitch(`账号被封累计 ${blockedCount} 次`);
460
+ blockedCount = 0;
461
+ }
212
462
  continue;
213
463
  }
464
+ if (exploreMaxUsers > 0 && processedCount >= exploreMaxUsers) {
465
+ console.error(`\n已达上限 ${exploreMaxUsers} 个用户,停止处理`);
466
+ break;
467
+ }
468
+ continue;
214
469
  }
470
+ }
471
+
472
+ if (result.captchaDetected) {
473
+ captchaCount++;
474
+ console.error(` [验证码] 累计 ${captchaCount} 次`);
215
475
 
216
- if (result.captchaDetected) {
217
- await apiPost(`${serverUrl}/api/error-report`, {
476
+ await withRetry("report captcha", () =>
477
+ apiPost(`${serverUrl}/api/error-report`, {
218
478
  userId,
219
- username: uniqueId,
479
+ username,
220
480
  errorType: "captcha",
221
481
  errorMessage: result.captchaMessage || "页面出现验证码",
222
482
  stage: result.captchaStage || "video-page",
223
483
  errorStack: "",
224
- });
484
+ }),
485
+ ).catch(() => {});
486
+
487
+ if (captchaCount >= 2) {
488
+ await handleAccountSwitch(`验证码累计 ${captchaCount} 次`);
489
+ captchaCount = 0;
225
490
  }
491
+ }
226
492
 
227
- consecutiveNetworkErrors = 0;
493
+ consecutiveNetworkErrors = 0;
494
+
495
+ // 更新关注/粉丝成功时间
496
+ if (result.hasFollowData && result.keepFollow) {
497
+ const totalFollows =
498
+ (result.discoveredFollowing || []).length +
499
+ (result.discoveredFollowers || []).length;
500
+ if (totalFollows > 0) {
501
+ lastFollowSuccessTime = Date.now();
502
+ }
503
+ }
228
504
 
229
- const guessedLocation = result.locationCreated || null;
505
+ processedCount++;
230
506
 
231
- await apiPost(`${serverUrl}/api/redo-job/${uniqueId}`, {
232
- userInfo: result.userInfo || {},
233
- discoveredVideoAuthors: (result.discoveredVideoAuthors || []).map(
234
- (item) =>
235
- typeof item === "object" ? { ...item, guessedLocation } : item,
236
- ),
237
- discoveredCommentAuthors: (result.discoveredCommentAuthors || []).map(
238
- (author) => ({ author, guessedLocation }),
239
- ),
240
- discoveredGuessAuthors: (result.discoveredGuessAuthors || []).map(
241
- (author) => ({ author, guessedLocation }),
242
- ),
243
- discoveredFollowing: (result.discoveredFollowing || []).map((f) => ({
244
- handle: Array.isArray(f) ? f[0] : f,
245
- displayName: Array.isArray(f) ? f[1] : null,
246
- guessedLocation,
247
- })),
248
- discoveredFollowers: (result.discoveredFollowers || []).map((f) => ({
249
- handle: Array.isArray(f) ? f[0] : f,
250
- displayName: Array.isArray(f) ? f[1] : null,
251
- guessedLocation,
252
- })),
253
- newUsersAdded: result.newUsersAdded || 0,
254
- collectedVideos: result.collectedVideos || 0,
255
- });
507
+ const guessedLocation = result.locationCreated || null;
256
508
 
257
- console.error(
258
- ` [完成] 视频: ${result.collectedVideos}, 评论作者: ${result.discoveredCommentAuthors?.length || 0}, 关注: ${result.discoveredFollowing?.length || 0}, 粉丝: ${result.discoveredFollowers?.length || 0}, 新增用户: ${result.newUsersAdded}`,
509
+ const payload = {
510
+ userInfo: result.userInfo || {},
511
+ discoveredFollowing: (result.discoveredFollowing || []).map((f) => ({
512
+ handle: Array.isArray(f) ? f[0] : f,
513
+ displayName: Array.isArray(f) ? f[1] : null,
514
+ guessedLocation,
515
+ })),
516
+ discoveredFollowers: (result.discoveredFollowers || []).map((f) => ({
517
+ handle: Array.isArray(f) ? f[0] : f,
518
+ displayName: Array.isArray(f) ? f[1] : null,
519
+ guessedLocation,
520
+ })),
521
+ processed: result.processed,
522
+ hasFollowData: result.hasFollowData,
523
+ keepFollow: result.keepFollow,
524
+ locationCreated: result.locationCreated,
525
+ noVideo: result.noVideo,
526
+ collectedVideos: result.collectedVideos,
527
+ };
528
+ await apiPost(`${serverUrl}/api/redo-job/${username}`, payload);
529
+
530
+ // 视频登记
531
+ if (result.videoList && result.videoList.length > 0) {
532
+ const { registered, skipped } = await apiPost(
533
+ `${serverUrl}/api/videos`,
534
+ {
535
+ sourceUser: username,
536
+ videoList: result.videoList,
537
+ locationCreated: result.locationCreated,
538
+ ttSeller: result.userInfo?.ttSeller || false,
539
+ },
259
540
  );
541
+ console.error(` 视频登记: ${registered} 条新增, ${skipped} 条已存在`);
542
+ }
260
543
 
261
- await delay(3000, 5000);
262
- } catch (e) {
263
- consecutiveNetworkErrors++;
264
- errorCount++;
265
- console.error(`\n[错误] ${e.message}`);
544
+ console.error(" 已提交");
266
545
 
267
- if (consecutiveNetworkErrors >= 3) {
268
- console.error(
269
- `[警告] 连续 ${consecutiveNetworkErrors} 次网络异常,等待 60s 后重试...`,
270
- );
271
- await delay(60000, 60000);
272
- consecutiveNetworkErrors = 0;
273
- } else {
274
- const waitTime =
275
- consecutiveNetworkErrors <= 2
276
- ? 5000 + Math.random() * 5000
277
- : 10000 + Math.random() * 10000;
278
- console.error(` 等待 ${Math.round(waitTime / 1000)}s 后重试...`);
279
- await delay(waitTime / 1000, waitTime / 1000);
280
- }
546
+ if (exploreMaxUsers > 0 && processedCount >= exploreMaxUsers) {
547
+ console.error(`\n已达上限 ${exploreMaxUsers} 个用户,停止处理`);
548
+ break;
281
549
  }
282
550
  }
551
+
552
+ const stats = await apiGet(`${serverUrl}/api/stats`);
553
+ console.error(`\n完成: ${processedCount} 个用户处理, ${errorCount} 个出错`);
554
+ console.error(
555
+ ` 总用户: ${stats.totalUsers}, 已完成: ${stats.processedUsers}, 待处理: ${stats.pendingUsers}, 错误: ${stats.errorUsers}`,
556
+ );
283
557
  } finally {
284
558
  process.removeListener("SIGINT", onSigint);
285
559
  process.removeListener("SIGTERM", onSigterm);
package/src/lib/args.js CHANGED
@@ -180,6 +180,7 @@ function parseExploreArgs(args) {
180
180
  let explorePortCount = null;
181
181
  let exploreUserId = null;
182
182
  let exploreMaxVideos = 16;
183
+ let exploreProxy = null;
183
184
 
184
185
  const positional = [];
185
186
  const PRESETS = ["fast", "normal", "slow", "stealth"];
@@ -218,6 +219,8 @@ function parseExploreArgs(args) {
218
219
  exploreUserId = args[++i];
219
220
  } else if (arg === "--max-videos") {
220
221
  exploreMaxVideos = parseInt(args[++i]) || 16;
222
+ } else if (arg === "--proxy") {
223
+ exploreProxy = args[++i];
221
224
  } else {
222
225
  positional.push(arg);
223
226
  }
@@ -258,6 +261,7 @@ function parseExploreArgs(args) {
258
261
  explorePortCount,
259
262
  exploreUserId,
260
263
  exploreMaxVideos,
264
+ exploreProxy,
261
265
  urls: [],
262
266
  outputFormat: "json",
263
267
  exploreCount: 0,
@@ -439,39 +443,77 @@ function parseDbImportArgs(args) {
439
443
  function parseRefreshArgs(args) {
440
444
  let serverUrl = defaultServer;
441
445
  let explorePreset = "normal";
442
- let exploreMaxComments = 10;
443
- let exploreMaxGuess = 0;
446
+ let exploreInterval = 30;
447
+ let exploreEnableFollow = true;
448
+ let exploreMaxFollowing = 100;
449
+ let exploreMaxFollowers = 100;
450
+ let exploreLocation = DEFAULT_TARGET_LOCATIONS_CSV;
451
+ let exploreMaxUsers = 0;
444
452
  let explorePort = null;
453
+ let exploreBasePort = null;
454
+ let explorePortCount = null;
445
455
  let exploreProfile = null;
446
456
  let exploreUserId = null;
457
+ let exploreMaxVideos = 16;
458
+ let exploreRedoMaxAge = 0;
459
+ let exploreProxy = null;
447
460
 
448
461
  for (let i = 0; i < args.length; i++) {
449
462
  const arg = args[i];
450
463
  if (arg === "--server") {
451
464
  serverUrl = args[++i];
452
- } else if (arg === "--comments") {
453
- exploreMaxComments = parseInt(args[++i]) || 10;
454
- } else if (arg === "--guess") {
455
- exploreMaxGuess = parseInt(args[++i]) || 0;
456
465
  } else if (arg === "--preset") {
457
466
  explorePreset = args[++i];
467
+ } else if (arg === "-i" || arg === "--interval") {
468
+ exploreInterval = parseInt(args[++i], 10) || 30;
458
469
  } else if (arg === "--port") {
459
470
  explorePort = parseInt(args[++i]) || 9222;
471
+ } else if (arg === "--base-port") {
472
+ exploreBasePort = parseInt(args[++i]) || 9222;
473
+ } else if (arg === "--port-count") {
474
+ explorePortCount = parseInt(args[++i]) || 10;
460
475
  } else if (arg === "--profile") {
461
476
  exploreProfile = args[++i];
462
477
  } else if (arg === "--user-id") {
463
478
  exploreUserId = args[++i];
479
+ } else if (arg === "--max-videos") {
480
+ exploreMaxVideos = parseInt(args[++i]) || 16;
481
+ } else if (arg === "--max-following") {
482
+ exploreMaxFollowing = parseInt(args[++i]) || 100;
483
+ } else if (arg === "--max-followers") {
484
+ exploreMaxFollowers = parseInt(args[++i]) || 100;
485
+ } else if (arg === "--max-age") {
486
+ exploreRedoMaxAge = parseInt(args[++i]) || 43200;
487
+ } else if (arg === "--proxy") {
488
+ exploreProxy = args[++i];
489
+ } else if (arg === "--location") {
490
+ exploreLocation = args[++i];
491
+ } else if (arg === "--enable-follow") {
492
+ exploreEnableFollow = true;
493
+ } else if (arg === "--disable-follow") {
494
+ exploreEnableFollow = false;
495
+ } else if (arg === "--max-users") {
496
+ exploreMaxUsers = parseInt(args[++i]) || 0;
464
497
  }
465
498
  }
466
499
 
467
500
  return {
468
501
  subcommand: "refresh",
469
502
  explorePreset,
470
- exploreMaxComments,
471
- exploreMaxGuess,
503
+ exploreInterval,
504
+ exploreEnableFollow,
505
+ exploreMaxFollowing,
506
+ exploreMaxFollowers,
507
+ exploreLocation,
508
+ exploreMaxUsers,
472
509
  explorePort,
510
+ exploreBasePort,
511
+ explorePortCount,
473
512
  exploreProfile,
474
513
  exploreUserId,
514
+ exploreMaxVideos,
515
+ exploreRedoMaxAge,
516
+ exploreProxy,
475
517
  serverUrl,
476
518
  urls: [],
477
519
  outputFormat: "json",
@@ -721,6 +763,10 @@ export function parseArgs() {
721
763
  return parseDbImportArgs(args.slice(1));
722
764
  }
723
765
 
766
+ if (args.length > 0 && args[0] === "refresh") {
767
+ return parseRefreshArgs(args.slice(1));
768
+ }
769
+
724
770
  const urls = [];
725
771
  let inputFile = null;
726
772
  let outputFile = null;
@@ -143,7 +143,7 @@ function killEdgeProcesses(targetDir) {
143
143
  });
144
144
  }
145
145
 
146
- function launchEdgeWithCDP(port, userDataDir) {
146
+ function launchEdgeWithCDP(port, userDataDir, proxyServer) {
147
147
  return new Promise((resolve, reject) => {
148
148
  const platform = os.platform();
149
149
  const edgePath = getEdgePath();
@@ -162,14 +162,19 @@ function launchEdgeWithCDP(port, userDataDir) {
162
162
  "--disable-breakpad",
163
163
  "--disable-background-networking",
164
164
  "--disable-sync",
165
- ].join(" ");
165
+ ];
166
+ if (proxyServer) {
167
+ extraArgs.push(`--proxy-server="${proxyServer}"`);
168
+ }
169
+
170
+ const argsStr = extraArgs.join(" ");
166
171
 
167
172
  if (platform === "darwin") {
168
- command = `open -a ${edgePath} --new --args ${extraArgs}`;
173
+ command = `open -a ${edgePath} --new --args ${argsStr}`;
169
174
  } else if (platform === "win32") {
170
- command = `start msedge ${extraArgs}`;
175
+ command = `start msedge ${argsStr}`;
171
176
  } else {
172
- command = `msedge ${extraArgs} &`;
177
+ command = `msedge ${argsStr} &`;
173
178
  }
174
179
 
175
180
  exec(command, (err) => {
@@ -194,6 +199,7 @@ export { killEdgeProcesses };
194
199
  export async function ensureBrowserReady(options = {}) {
195
200
  const port = options.port || DEFAULT_CDP_PORT;
196
201
  const userDataDir = options.userDataDir || DEFAULT_USER_DATA_DIR;
202
+ const proxyServer = options.proxyServer || null;
197
203
  const isCustom = port !== DEFAULT_CDP_PORT || !!options.userDataDir;
198
204
 
199
205
  const isReady = await checkCDPPort(port);
@@ -131,12 +131,33 @@ const HELP_TEXT = [
131
131
  " --disable-follow 禁用关注/粉丝提取",
132
132
  " --max-following <数量> 最大获取关注数,默认 50",
133
133
  " --max-followers <数量> 最大获取粉丝数,默认 50",
134
- " --max-users <数量> 最大处理用户数,默认无限制",
135
- " -i, --interval <秒数> 无任务时轮询间隔,默认 10 秒",
136
- " --port <端口号> 固定 CDP 端口(调试用,关闭自动轮换)",
137
- " --base-port <端口号> 起始端口,默认 9222",
138
- " --port-count <数量> 端口数量(账户数),默认 10",
139
- " --user-id <编号> 客户端编号(设备ID),默认自动生成",
134
+ " --max-users <数量> 最大处理用户数,默认无限制",
135
+ " -i, --interval <秒数> 无任务时轮询间隔,默认 10 秒",
136
+ " --port <端口号> 固定 CDP 端口(调试用,关闭自动轮换)",
137
+ " --base-port <端口号> 起始端口,默认 9222",
138
+ " --port-count <数量> 端口数量(账户数),默认 10",
139
+ " --user-id <编号> 客户端编号(设备ID),默认自动生成",
140
+ " --proxy <地址> 浏览器代理(如 socks5://127.0.0.1:1080)",
141
+ "",
142
+ " tt-help refresh [选项]",
143
+ " 对目标商家用户进行轮回刷新,重新采集视频 + 关注 + 粉丝",
144
+ " 筛选条件: tt_seller=1, verified=0, 目标国家",
145
+ " 选项:",
146
+ " --server <URL> 服务端地址,默认 http://127.0.0.1:3001",
147
+ ` --location <国家代码> 国家筛选,逗号分隔,默认 ${DEFAULT_TARGET_LOCATIONS_CSV}`,
148
+ " --max-videos <数量> 每用户最大视频数,默认 16",
149
+ " --enable-follow 启用关注/粉丝提取(默认启用)",
150
+ " --disable-follow 禁用关注/粉丝提取",
151
+ " --max-following <数量> 最大获取关注数,默认 100",
152
+ " --max-followers <数量> 最大获取粉丝数,默认 100",
153
+ " --max-users <数量> 最大处理用户数,默认无限制",
154
+ " -i, --interval <秒数> 无任务时轮询间隔,默认 30 秒",
155
+ " --max-age <秒数> 最小刷新间隔,默认 43200(12小时)",
156
+ " --port <端口号> 固定 CDP 端口(调试用,关闭自动轮换)",
157
+ " --base-port <端口号> 起始端口,默认 9222",
158
+ " --port-count <数量> 端口数量(账户数),默认 10",
159
+ " --user-id <编号> 客户端编号(设备ID),默认自动生成",
160
+ " --proxy <地址> 浏览器代理(如 socks5://127.0.0.1:1080)",
140
161
  "",
141
162
  " tt-help info <URL> [URL2 ...] [--onlyvideo]",
142
163
  " 获取用户/视频信息,支持多个 URL",
@@ -196,9 +217,10 @@ const HELP_TEXT = [
196
217
  " -h, --help 显示帮助",
197
218
  " --version 显示版本号",
198
219
  "",
199
- " 示例: tt-help info https://www.tiktok.com/@nike https://www.tiktok.com/@adidas",
200
- " tt-help explore qiqi23280 fast --location ES --max-comments 50",
201
- " tt-help config set server http://127.0.0.1:3001",
220
+ " 示例: tt-help info https://www.tiktok.com/@nike https://www.tiktok.com/@adidas",
221
+ " tt-help explore qiqi23280 fast --location ES --max-comments 50",
222
+ " tt-help refresh --server http://127.0.0.1:3001 --port 9222",
223
+ " tt-help config set server http://127.0.0.1:3001",
202
224
  " tt-help attach -p 5 -i 10",
203
225
  " tt-help watch -o data/result.db",
204
226
  " tt-help videostats data/result.db -p 3",
package/src/main.js CHANGED
@@ -10,6 +10,7 @@ import { handleComments } from "./cli/comments.js";
10
10
  import { handleVideoStats } from "./cli/videostats.js";
11
11
  import { handleDbImport } from "./cli/db-import.js";
12
12
  import { handleWebserver } from "./cli/webserver.js";
13
+ import { handleRefresh } from "./cli/refresh.js";
13
14
 
14
15
  async function main() {
15
16
  const parsed = parseArgs();
@@ -33,6 +34,8 @@ async function main() {
33
34
  return handleVideoStats(parsed);
34
35
  case "db-import":
35
36
  return handleDbImport(parsed);
37
+ case "refresh":
38
+ return handleRefresh(parsed);
36
39
  }
37
40
 
38
41
  const {
package/src/npm-main.js CHANGED
@@ -6,6 +6,7 @@ import { handleAttach } from "./cli/attach.js";
6
6
  import { handleConfig, showConfig, showUsage, version } from "./cli/config.js";
7
7
  import { handleOpen } from "./cli/open.js";
8
8
  import { handleComments } from "./cli/comments.js";
9
+ import { handleRefresh } from "./cli/refresh.js";
9
10
 
10
11
  function exitUnsupportedCommand(command) {
11
12
  console.error(
@@ -33,6 +34,8 @@ async function main() {
33
34
  return handleOpen(parsed);
34
35
  case "comments":
35
36
  return handleComments(parsed);
37
+ case "refresh":
38
+ return handleRefresh(parsed);
36
39
  }
37
40
 
38
41
  const {
@@ -2786,9 +2786,10 @@ export function createStore(filePath) {
2786
2786
  return { saved: true, pinned: user.pinned };
2787
2787
  }
2788
2788
 
2789
- function getNextRedoJob(userId) {
2789
+ function getNextRedoJob(userId, maxAgeSeconds = 43200) {
2790
2790
  if (db) {
2791
2791
  const now = Date.now();
2792
+ const threshold = now - maxAgeSeconds * 1000;
2792
2793
  const defaultTime = new Date("2016-01-01T00:00:00Z").getTime();
2793
2794
  const targetLocations = [
2794
2795
  "CZ",
@@ -2813,11 +2814,12 @@ export function createStore(filePath) {
2813
2814
  WHERE tt_seller = 1
2814
2815
  AND verified = 0
2815
2816
  AND location_created IN (${placeholders})
2817
+ AND COALESCE(refresh_time, ?) < ?
2816
2818
  ORDER BY COALESCE(refresh_time, ?) ASC
2817
2819
  LIMIT 1
2818
2820
  `,
2819
2821
  )
2820
- .get(...targetLocations, defaultTime);
2822
+ .get(...targetLocations, defaultTime, threshold, defaultTime);
2821
2823
  if (!row) return null;
2822
2824
  db.prepare(
2823
2825
  "UPDATE jobs SET refresh_time = ?, updated_at = ? WHERE unique_id = ?",
@@ -2830,6 +2832,7 @@ export function createStore(filePath) {
2830
2832
  }
2831
2833
 
2832
2834
  const now = Date.now();
2835
+ const threshold = now - maxAgeSeconds * 1000;
2833
2836
  const defaultTime = new Date("2016-01-01T00:00:00Z").getTime();
2834
2837
 
2835
2838
  // 筛选目标国家用户,按 refreshTime 升序取最远的(没有则默认 2016-01-01)
@@ -2855,13 +2858,19 @@ export function createStore(filePath) {
2855
2858
  );
2856
2859
  if (targetUsers.length === 0) return null;
2857
2860
 
2858
- targetUsers.sort((a, b) => {
2861
+ const recentEnough = targetUsers.filter((u) => {
2862
+ const rt = u.refreshTime || defaultTime;
2863
+ return rt < threshold;
2864
+ });
2865
+ if (recentEnough.length === 0) return null;
2866
+
2867
+ recentEnough.sort((a, b) => {
2859
2868
  const ta = a.refreshTime || defaultTime;
2860
2869
  const tb = b.refreshTime || defaultTime;
2861
2870
  return ta - tb;
2862
2871
  });
2863
2872
 
2864
- const next = targetUsers[0];
2873
+ const next = recentEnough[0];
2865
2874
  next.refreshTime = now;
2866
2875
  save();
2867
2876
  return {
@@ -2889,9 +2898,10 @@ export function createStore(filePath) {
2889
2898
  }
2890
2899
  }
2891
2900
  }
2901
+ const newUsers = processDiscoveredUsers(result);
2892
2902
  const ret = updateJobInfo(uniqueId, user, false);
2893
2903
  if (ret.error) return { saved: false, error: ret.error };
2894
- return { saved: true };
2904
+ return { saved: true, newUsers };
2895
2905
  }
2896
2906
 
2897
2907
  const user = getUser(uniqueId);
@@ -2908,8 +2918,8 @@ export function createStore(filePath) {
2908
2918
  }
2909
2919
  }
2910
2920
  }
2911
-
2912
- return { saved: true };
2921
+ const newUsers = processDiscoveredUsers(result);
2922
+ return { saved: true, newUsers };
2913
2923
  }
2914
2924
 
2915
2925
  function reportClientError(
@@ -368,6 +368,10 @@ function renderTable(users) {
368
368
  }
369
369
  const claimTime = u.claimedAt ? formatTime(u.claimedAt) : "-";
370
370
  const procTime = u.processedAt ? formatTime(u.processedAt) : "-";
371
+ const refreshTime =
372
+ u.ttSeller && !u.verified && u.refreshTime
373
+ ? formatTime(u.refreshTime)
374
+ : "-";
371
375
  const statusCodeDisplay =
372
376
  u.statusCode != null && u.statusCode !== 0
373
377
  ? `<span class="tag error" style="font-size:10px">${u.statusCode}</span>`
@@ -378,6 +382,7 @@ function renderTable(users) {
378
382
  <td data-label="粉丝">${fans}</td>
379
383
  <td data-label="视频">${videos}</td>
380
384
  <td data-label="国家">${loc}</td>
385
+ <td data-label="最近刷新" style="font-size:11px;color:#888">${refreshTime}</td>
381
386
  <td data-label="最近发布" style="font-size:11px;color:#888">${latestVideo}</td>
382
387
  <td data-label="猜测国家">${guessedLoc}</td>
383
388
  <td data-label="来源">${sources || "-"}</td>
@@ -125,6 +125,7 @@
125
125
  <th>粉丝</th>
126
126
  <th>视频</th>
127
127
  <th>国家</th>
128
+ <th class="sortable" data-sort="refreshTime">最近刷新 <span class="sort-icon">↕</span></th>
128
129
  <th class="sortable" data-sort="latestVideoTime">最近发布 <span class="sort-icon">↕</span></th>
129
130
  <th>猜测国家</th>
130
131
  <th>来源</th>
@@ -356,7 +356,8 @@ export function startWatchServer(dataAnchor, port = 3000, existingStore) {
356
356
 
357
357
  if (req.method === "GET" && routePath === "/api/redo-job") {
358
358
  const userId = params.userId || "";
359
- const job = store.getNextRedoJob(userId);
359
+ const maxAge = parseInt(params.maxAge) || 43200;
360
+ const job = store.getNextRedoJob(userId, maxAge);
360
361
  if (job) {
361
362
  logJob("REDO-CLAIM", { user: job.uniqueId, clientId: userId });
362
363
  sendJSON(res, 200, { hasJob: true, user: job });
@@ -683,6 +684,7 @@ export function startWatchServer(dataAnchor, port = 3000, existingStore) {
683
684
  followerCount: u.followerCount,
684
685
  locationCreated: u.locationCreated,
685
686
  latestVideoTime: u.latestVideoTime,
687
+ refreshTime: u.refreshTime,
686
688
  guessedLocation: u.guessedLocation,
687
689
  pinned: u.pinned,
688
690
  processedAt: u.processedAt,
@@ -1,213 +0,0 @@
1
- import {
2
- delay,
3
- retryWithBackoff,
4
- detectPageError,
5
- assertPageUrl,
6
- } from "./modules/page-helpers.js";
7
- import { detectCaptcha } from "./modules/captcha-handler.js";
8
- import { getUserInfo, collectVideos } from "../videos/core.js";
9
- import { extractFollowAndFollowers } from "./modules/follow-extractor.js";
10
- import { processExplore } from "./explore-core.js";
11
- import { DEFAULT_TARGET_LOCATIONS_CSV } from "../lib/target-locations.js";
12
-
13
- export async function processRefresh(page, username, serverUrl, options, log) {
14
- const { maxFollowing = 100, maxFollowers = 100, maxVideos = 100 } = options;
15
-
16
- const result = {
17
- userInfo: null,
18
- discoveredVideoAuthors: [],
19
- discoveredFollowing: [],
20
- discoveredFollowers: [],
21
- newUsersAdded: 0,
22
- collectedVideos: 0,
23
- error: null,
24
- };
25
-
26
- try {
27
- log(` 访问 @${username} 主页...`);
28
- const homeUrl = `https://www.tiktok.com/@${username}`;
29
- await retryWithBackoff(
30
- async () => {
31
- await page.goto(homeUrl, {
32
- waitUntil: "domcontentloaded",
33
- timeout: 30000,
34
- });
35
- assertPageUrl(page, `@${username}`);
36
- },
37
- { log },
38
- );
39
- await page
40
- .waitForSelector('[class*="DivVideoList"]', { timeout: 10000 })
41
- .catch(() => {});
42
- await delay(1000, 2000);
43
-
44
- log(" 获取用户信息...");
45
- const info = await getUserInfo(page);
46
- if (info) {
47
- result.userInfo = info;
48
- log(
49
- ` 用户: ${info.nickname || username} | 粉丝: ${info.followerCount || "-"} | 视频: ${info.videoCount || "-"}`,
50
- );
51
- }
52
-
53
- const captcha = await detectCaptcha(page);
54
- if (captcha && captcha.visible) {
55
- log(`[验证码] @${username} 页面出现验证码`);
56
- result.captchaDetected = true;
57
- result.captchaStage = result.captchaStage || "video-page";
58
- result.captchaMessage = result.captchaMessage || "视频页出现验证码";
59
- }
60
-
61
- // 采集视频
62
- log(` 采集视频 (最多 ${maxVideos} 个)...`);
63
- const videoList = await collectVideos(page, username, maxVideos, log);
64
- const videoArray = videoList ? [...videoList.values()] : [];
65
- result.collectedVideos = videoArray.length;
66
- result.discoveredVideoAuthors = videoArray.map((v) => v.author);
67
-
68
- if (videoArray.length <= 0) {
69
- result.noVideo = true;
70
- const pageError = await detectPageError(page);
71
- if (pageError) {
72
- result.restricted = true;
73
- log(` @${username} 页面受限(${pageError}),标记跳过`);
74
- }
75
- return result;
76
- }
77
-
78
- // 采集关注和粉丝
79
- log(` 采集关注 (最多 ${maxFollowing}) + 粉丝 (最多 ${maxFollowers})...`);
80
- try {
81
- const followResult = await extractFollowAndFollowers(page, {
82
- maxFollowing,
83
- maxFollowers,
84
- });
85
- result.discoveredFollowing = followResult.following || [];
86
- result.discoveredFollowers = followResult.followers || [];
87
- log(
88
- ` 关注: ${result.discoveredFollowing.length}, 粉丝: ${result.discoveredFollowers.length}`,
89
- );
90
- } catch (e) {
91
- log(` [关注/粉丝采集失败] ${e.message}`);
92
- result.discoveredFollowing = [];
93
- result.discoveredFollowers = [];
94
- }
95
-
96
- // 处理新发现的用户(关注 + 粉丝),循环执行完整 explore
97
- // follow-extractor 返回 [handle, displayName] 数组
98
- const allDiscovered = [
99
- ...result.discoveredFollowing.map((h) => ({
100
- handle: Array.isArray(h) ? h[0] : h,
101
- source: "refresh-following",
102
- })),
103
- ...result.discoveredFollowers.map((h) => ({
104
- handle: Array.isArray(h) ? h[0] : h,
105
- source: "refresh-follower",
106
- })),
107
- ];
108
-
109
- for (const { handle, source } of allDiscovered) {
110
- const uniqueId = handle.replace("@", "");
111
-
112
- // 检查用户是否已存在
113
- const existsResp = await fetch(
114
- `${serverUrl}/api/user-exists/${encodeURIComponent(uniqueId)}`,
115
- );
116
- const existsData = await existsResp.json();
117
-
118
- if (existsData.exists) {
119
- continue;
120
- }
121
-
122
- log(` [新用户] @${uniqueId} 不存在,开始探索 (来源: ${source})...`);
123
- await delay(1000, 2000);
124
-
125
- // 对新用户做完整 explore(与 explore 命令逻辑一致)
126
- const exploreResult = await processExplore(
127
- page,
128
- uniqueId,
129
- {
130
- maxComments: 10,
131
- maxGuess: 0,
132
- enableFollow: true,
133
- maxFollowing: 5,
134
- maxFollowers: 5,
135
- location: DEFAULT_TARGET_LOCATIONS_CSV,
136
- },
137
- log,
138
- );
139
-
140
- // 提交 explore 结果到服务端(和 explore 命令的 commitJob 一致)
141
- if (exploreResult.userInfo) {
142
- const guessedLocation = exploreResult.locationCreated || null;
143
-
144
- const payload = {
145
- userInfo: exploreResult.userInfo || {},
146
- discoveredVideoAuthors: (
147
- exploreResult.discoveredVideoAuthors || []
148
- ).map((item) =>
149
- typeof item === "object" ? { ...item, guessedLocation } : item,
150
- ),
151
- discoveredCommentAuthors: (
152
- exploreResult.discoveredCommentAuthors || []
153
- ).map((author) => ({ author, guessedLocation })),
154
- discoveredGuessAuthors: (
155
- exploreResult.discoveredGuessAuthors || []
156
- ).map((author) => ({ author, guessedLocation })),
157
- discoveredFollowing: (exploreResult.discoveredFollowing || []).map(
158
- (f) => ({
159
- handle: Array.isArray(f) ? f[0] : f,
160
- displayName: Array.isArray(f) ? f[1] : null,
161
- guessedLocation,
162
- }),
163
- ),
164
- discoveredFollowers: (exploreResult.discoveredFollowers || []).map(
165
- (f) => ({
166
- handle: Array.isArray(f) ? f[0] : f,
167
- displayName: Array.isArray(f) ? f[1] : null,
168
- guessedLocation,
169
- }),
170
- ),
171
- processed: exploreResult.processed,
172
- hasFollowData: exploreResult.hasFollowData,
173
- keepFollow: exploreResult.keepFollow,
174
- locationCreated: exploreResult.locationCreated,
175
- noVideo: exploreResult.noVideo,
176
- restricted: exploreResult.restricted,
177
- error: exploreResult.error,
178
- };
179
-
180
- const addResp = await fetch(
181
- `${serverUrl}/api/explore-new/${uniqueId}`,
182
- {
183
- method: "POST",
184
- headers: { "Content-Type": "application/json" },
185
- body: JSON.stringify(payload),
186
- },
187
- );
188
- const addResult = await addResp.json();
189
-
190
- if (!addResult.saved) {
191
- log(` [跳过] @${uniqueId} 提交失败`);
192
- continue;
193
- }
194
-
195
- result.newUsersAdded++;
196
- if (exploreResult.captchaDetected) {
197
- result.captchaDetected = true;
198
- }
199
- log(
200
- ` [已提交] @${uniqueId} ${addResult.created ? "(新用户)" : "(已存在)"} | 发现: ${addResult.newUsers?.length || 0} 个`,
201
- );
202
- }
203
-
204
- await delay(2000, 4000);
205
- }
206
- } catch (e) {
207
- log(` [错误] ${e.message}`);
208
- result.error = e.message;
209
- result.errorStack = e.stack || "";
210
- }
211
-
212
- return result;
213
- }