tt-help-cli-ycl 1.4.28 → 1.4.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/cli/comments.js +22 -0
- package/src/cli/search.js +521 -98
- package/src/lib/args.js +9 -5
- package/src/lib/constants.js +3 -2
- package/src/lib/search-collector.js +22 -13
package/package.json
CHANGED
package/src/cli/comments.js
CHANGED
|
@@ -732,6 +732,28 @@ async function runAutoMode(options) {
|
|
|
732
732
|
await session.ensurePage();
|
|
733
733
|
if (!session.page) throw new Error("未找到可用页面");
|
|
734
734
|
|
|
735
|
+
// ── 登录弹窗检测(任务前)──
|
|
736
|
+
try {
|
|
737
|
+
const modals = await detectModals(session.page);
|
|
738
|
+
const loginModal = modals.find(
|
|
739
|
+
(m) =>
|
|
740
|
+
m.type === "login-modal" || m.type === "modal-container",
|
|
741
|
+
);
|
|
742
|
+
if (loginModal) {
|
|
743
|
+
console.error(` 🔔 检测到登录弹窗,正在关闭...`);
|
|
744
|
+
const { closed, failed } = await closeAllModals(
|
|
745
|
+
session.page,
|
|
746
|
+
{ waitMs: 300 },
|
|
747
|
+
);
|
|
748
|
+
console.error(
|
|
749
|
+
` 🔔 弹窗关闭: closed=${JSON.stringify(closed)} failed=${JSON.stringify(failed)}`,
|
|
750
|
+
);
|
|
751
|
+
await new Promise((r) => setTimeout(r, 2000));
|
|
752
|
+
}
|
|
753
|
+
} catch (e) {
|
|
754
|
+
// 忽略弹窗检测异常
|
|
755
|
+
}
|
|
756
|
+
|
|
735
757
|
// 导航到视频页
|
|
736
758
|
await prepareVideoPage(session.page, href);
|
|
737
759
|
|
package/src/cli/search.js
CHANGED
|
@@ -6,25 +6,74 @@
|
|
|
6
6
|
* 2. POST /api/tags/search-claim → 锁定标签
|
|
7
7
|
* 3. collectSearchAuthors() → 并行搜索 5 个 tab
|
|
8
8
|
* 4. POST /api/tags/search-result → 提交结果
|
|
9
|
+
*
|
|
10
|
+
* 监控检测流程(对齐 explore):
|
|
11
|
+
* - 验证码检测(切换任务前 + 搜索完成后)
|
|
12
|
+
* - 浏览器关闭恢复(isBrowserClosedError + relaunchBrowser)
|
|
13
|
+
* - 网络错误退避(consecutiveNetworkErrors 累进等待)
|
|
14
|
+
* - 临时错误重试(Timeout/ExecutionContext)
|
|
15
|
+
* - 错误上报(POST /api/error-report)
|
|
16
|
+
*
|
|
17
|
+
* 健康检测流程(对齐 explore):
|
|
18
|
+
* - HealthChecker 监控内存(>95%)+ 运行时间(>48分钟)
|
|
19
|
+
* - switchAccount 自动切换账户
|
|
9
20
|
*/
|
|
10
21
|
|
|
11
22
|
import { randomUUID } from "crypto";
|
|
12
|
-
import { ensureBrowserReady } from "../lib/browser/cdp.js";
|
|
13
|
-
import {
|
|
23
|
+
import { ensureBrowserReady as ensureBrowserReadyCDP } from "../lib/browser/cdp.js";
|
|
24
|
+
import { switchAccount, killEdgeProcesses } from "../lib/browser/cdp.js";
|
|
25
|
+
import {
|
|
26
|
+
getOrCreatePage,
|
|
27
|
+
safeCheckLogin,
|
|
28
|
+
isBrowserClosedError,
|
|
29
|
+
relaunchBrowser,
|
|
30
|
+
} from "../lib/browser/page.js";
|
|
14
31
|
import { collectSearchAuthors } from "../lib/search-collector.js";
|
|
32
|
+
import {
|
|
33
|
+
delay,
|
|
34
|
+
getDelayConfig,
|
|
35
|
+
setDelayConfig,
|
|
36
|
+
} from "../scraper/modules/page-helpers.js";
|
|
37
|
+
import {
|
|
38
|
+
detectCaptcha,
|
|
39
|
+
closeCaptcha,
|
|
40
|
+
} from "../scraper/modules/captcha-handler.js";
|
|
41
|
+
import { createApiClient } from "../lib/api-client.js";
|
|
42
|
+
import { HealthChecker } from "../lib/browser/health-checker.js";
|
|
15
43
|
import {
|
|
16
44
|
server as cfgServer,
|
|
17
45
|
proxy as configuredProxy,
|
|
18
46
|
} from "../lib/constants.js";
|
|
19
47
|
|
|
20
48
|
const DEFAULT_SERVER = cfgServer || "http://127.0.0.1:3000";
|
|
49
|
+
const STARTUP_TIKTOK_URL = "https://www.tiktok.com/@ycl5007";
|
|
50
|
+
const PAGE_GOTO_TIMEOUT = 60000;
|
|
51
|
+
const MAX_RETRY_WAIT = 5 * 60 * 1000;
|
|
52
|
+
|
|
53
|
+
async function withRetry(label, fn) {
|
|
54
|
+
let backoff = 1000;
|
|
55
|
+
while (true) {
|
|
56
|
+
try {
|
|
57
|
+
return await fn();
|
|
58
|
+
} catch (err) {
|
|
59
|
+
console.error(
|
|
60
|
+
`[连接] ${label} 失败: ${err.message},${backoff / 1000}秒后重试...`,
|
|
61
|
+
);
|
|
62
|
+
await new Promise((r) => setTimeout(r, backoff));
|
|
63
|
+
if (backoff < MAX_RETRY_WAIT) backoff *= 2;
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
}
|
|
21
67
|
|
|
22
68
|
export async function handleSearch(parsed) {
|
|
23
69
|
const { search: searchOpts } = parsed;
|
|
24
|
-
const {
|
|
70
|
+
const {
|
|
71
|
+
serverUrl,
|
|
72
|
+
basePort = 3333,
|
|
73
|
+
portCount = 10,
|
|
74
|
+
proxy: cliProxy,
|
|
75
|
+
} = searchOpts || {};
|
|
25
76
|
|
|
26
|
-
const baseUrl = serverUrl || DEFAULT_SERVER;
|
|
27
|
-
const cdpPort = port || 3333;
|
|
28
77
|
const effectiveProxy = cliProxy || configuredProxy;
|
|
29
78
|
|
|
30
79
|
const log = (...args) => process.stderr.write(args.join(" ") + "\n");
|
|
@@ -32,129 +81,503 @@ export async function handleSearch(parsed) {
|
|
|
32
81
|
log("");
|
|
33
82
|
log("╔════════════════════════════════════╗");
|
|
34
83
|
log("║ TikTok 搜索作者采集(标签驱动)");
|
|
35
|
-
log("║ 服务端: " +
|
|
84
|
+
log("║ 服务端: " + (serverUrl || DEFAULT_SERVER));
|
|
36
85
|
log(
|
|
37
|
-
"║
|
|
38
|
-
|
|
39
|
-
|
|
86
|
+
"║ 端口: " +
|
|
87
|
+
basePort +
|
|
88
|
+
"-" +
|
|
89
|
+
(basePort + portCount - 1) +
|
|
90
|
+
"(共 " +
|
|
91
|
+
portCount +
|
|
92
|
+
" 个)",
|
|
40
93
|
);
|
|
41
|
-
log("║
|
|
94
|
+
log("║ 代理: " + (effectiveProxy || "无"));
|
|
42
95
|
log("╚════════════════════════════════════╝");
|
|
43
96
|
log("");
|
|
44
97
|
|
|
45
|
-
//
|
|
46
|
-
const
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
log("导航到 TikTok...");
|
|
53
|
-
try {
|
|
54
|
-
await page.goto("https://www.tiktok.com/@ycl5007", {
|
|
55
|
-
waitUntil: "domcontentloaded",
|
|
56
|
-
timeout: 30000,
|
|
57
|
-
});
|
|
58
|
-
} catch (e) {
|
|
59
|
-
log(`⚠️ 导航失败: ${e.message}`);
|
|
60
|
-
}
|
|
98
|
+
// ── 初始化 ──
|
|
99
|
+
const baseUrl = serverUrl || DEFAULT_SERVER;
|
|
100
|
+
const healthChecker = new HealthChecker({
|
|
101
|
+
basePort,
|
|
102
|
+
totalAccounts: portCount,
|
|
103
|
+
});
|
|
104
|
+
let currentAccount = healthChecker.getCurrentAccount();
|
|
61
105
|
|
|
62
|
-
const
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
106
|
+
const cdpOptions = {};
|
|
107
|
+
cdpOptions.port = currentAccount.port;
|
|
108
|
+
cdpOptions.userDataDir = currentAccount.userDataDir;
|
|
109
|
+
if (effectiveProxy) {
|
|
110
|
+
cdpOptions.proxyServer = effectiveProxy;
|
|
67
111
|
}
|
|
68
|
-
log("✅ 已登录 TikTok");
|
|
69
|
-
|
|
70
|
-
const clientId = randomUUID();
|
|
71
|
-
const clientMeta = { type: "tag-search", port: cdpPort };
|
|
72
|
-
const headers = (extra = {}) => ({
|
|
73
|
-
"X-Client-Id": clientId,
|
|
74
|
-
"X-Client-Info": JSON.stringify(clientMeta),
|
|
75
|
-
...extra,
|
|
76
|
-
});
|
|
77
|
-
log(`客户端 ID: ${clientId.substring(0, 8)}...`);
|
|
78
|
-
log("");
|
|
79
112
|
|
|
80
|
-
let
|
|
81
|
-
let
|
|
113
|
+
let browser = null;
|
|
114
|
+
let shuttingDown = false;
|
|
82
115
|
|
|
83
116
|
const cleanup = () => {
|
|
117
|
+
if (shuttingDown) return;
|
|
118
|
+
shuttingDown = true;
|
|
84
119
|
log("\n正在清理...");
|
|
85
|
-
browser
|
|
120
|
+
browser?.close().catch(() => {});
|
|
121
|
+
killEdgeProcesses(cdpOptions.userDataDir, cdpOptions.port).catch(() => {});
|
|
86
122
|
process.exit(0);
|
|
87
123
|
};
|
|
88
124
|
process.on("SIGINT", cleanup);
|
|
125
|
+
process.on("SIGTERM", cleanup);
|
|
89
126
|
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
127
|
+
const { apiGet, apiPost } = createApiClient({
|
|
128
|
+
meta: { port: cdpOptions.port, type: "tag-search" },
|
|
129
|
+
});
|
|
130
|
+
|
|
131
|
+
setDelayConfig("normal");
|
|
132
|
+
|
|
133
|
+
try {
|
|
134
|
+
// ── 启动:逐个尝试账户,直到找到一个已登录的 ──
|
|
135
|
+
let page = null;
|
|
136
|
+
let loggedIn = false;
|
|
137
|
+
let triedAccounts = 0;
|
|
138
|
+
const maxAccounts = portCount;
|
|
139
|
+
|
|
140
|
+
while (!loggedIn && triedAccounts < maxAccounts) {
|
|
141
|
+
if (triedAccounts > 0) {
|
|
142
|
+
// 切换到下一个账户
|
|
143
|
+
const prevAccount = currentAccount;
|
|
144
|
+
currentAccount = healthChecker.getNextAccount();
|
|
145
|
+
log(
|
|
146
|
+
`\n[启动] 尝试下一个账户: 端口 ${prevAccount.port} → ${currentAccount.port}`,
|
|
147
|
+
);
|
|
148
|
+
Object.assign(cdpOptions, {
|
|
149
|
+
port: currentAccount.port,
|
|
150
|
+
userDataDir: currentAccount.userDataDir,
|
|
151
|
+
...(effectiveProxy ? { proxyServer: effectiveProxy } : {}),
|
|
152
|
+
});
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
triedAccounts++;
|
|
156
|
+
log(
|
|
157
|
+
`\n[启动] 尝试连接端口 ${cdpOptions.port} (${triedAccounts}/${maxAccounts})...`,
|
|
158
|
+
);
|
|
159
|
+
|
|
160
|
+
try {
|
|
161
|
+
// 连接 CDP 浏览器
|
|
162
|
+
browser = await ensureBrowserReadyCDP(cdpOptions);
|
|
163
|
+
page = await getOrCreatePage(browser);
|
|
164
|
+
|
|
165
|
+
// 导航到 TikTok 并检测登录
|
|
166
|
+
await withRetry("启动页面导航", async () => {
|
|
167
|
+
await page.goto(STARTUP_TIKTOK_URL, {
|
|
168
|
+
waitUntil: "domcontentloaded",
|
|
169
|
+
timeout: PAGE_GOTO_TIMEOUT,
|
|
170
|
+
});
|
|
171
|
+
});
|
|
172
|
+
|
|
173
|
+
loggedIn = await safeCheckLogin(page);
|
|
174
|
+
if (!loggedIn) {
|
|
175
|
+
log(` ⚠️ 端口 ${cdpOptions.port} 未登录 TikTok`);
|
|
176
|
+
await browser.close().catch(() => {});
|
|
177
|
+
browser = null;
|
|
105
178
|
}
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
179
|
+
} catch (e) {
|
|
180
|
+
log(` ❌ 端口 ${cdpOptions.port} 连接失败: ${e.message}`);
|
|
181
|
+
await browser?.close().catch(() => {});
|
|
182
|
+
browser = null;
|
|
109
183
|
}
|
|
110
|
-
|
|
184
|
+
}
|
|
111
185
|
|
|
112
|
-
|
|
113
|
-
const country = task.country;
|
|
186
|
+
if (!loggedIn || !browser || !page) {
|
|
114
187
|
log(
|
|
115
|
-
|
|
188
|
+
`❌ 所有 ${triedAccounts} 个账户均未登录或连接失败,请至少在一个端口的浏览器中登录 TikTok 后重试`,
|
|
116
189
|
);
|
|
190
|
+
process.exit(1);
|
|
191
|
+
}
|
|
192
|
+
log(`✅ 端口 ${cdpOptions.port} 已登录 TikTok`);
|
|
193
|
+
|
|
194
|
+
// 提取当前登录用户名(通过 TikTok 内部 API)
|
|
195
|
+
async function extractMyUsername(p) {
|
|
196
|
+
try {
|
|
197
|
+
return await p.evaluate(async () => {
|
|
198
|
+
try {
|
|
199
|
+
const res = await fetch("https://www.tiktok.com/passport/web/account/info/");
|
|
200
|
+
const data = await res.json();
|
|
201
|
+
return data?.data?.unique_id || null;
|
|
202
|
+
} catch {}
|
|
203
|
+
return null;
|
|
204
|
+
});
|
|
205
|
+
} catch {
|
|
206
|
+
return null;
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
let myUsername = await extractMyUsername(page);
|
|
211
|
+
if (myUsername) {
|
|
212
|
+
log(` 当前用户: @${myUsername}`);
|
|
213
|
+
} else {
|
|
214
|
+
log(` ⚠️ 无法提取当前用户名,将不会排除自身`);
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
// 全局拦截图片/样式资源,减少内存和带宽
|
|
218
|
+
await page.route("**/*", (route) => {
|
|
219
|
+
const resourceType = route.request().resourceType();
|
|
220
|
+
if (resourceType === "image" || resourceType === "stylesheet") {
|
|
221
|
+
route.abort();
|
|
222
|
+
} else {
|
|
223
|
+
route.continue();
|
|
224
|
+
}
|
|
225
|
+
});
|
|
226
|
+
|
|
227
|
+
// ── 账户切换后重建 page ──
|
|
228
|
+
async function setupNewPage(newBrowser) {
|
|
229
|
+
const newPage = await getOrCreatePage(newBrowser);
|
|
230
|
+
await newPage.route("**/*", (route) => {
|
|
231
|
+
const resourceType = route.request().resourceType();
|
|
232
|
+
if (resourceType === "image" || resourceType === "stylesheet") {
|
|
233
|
+
route.abort();
|
|
234
|
+
} else {
|
|
235
|
+
route.continue();
|
|
236
|
+
}
|
|
237
|
+
});
|
|
238
|
+
return newPage;
|
|
239
|
+
}
|
|
117
240
|
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
241
|
+
// ── 账户切换处理 ──
|
|
242
|
+
async function handleAccountSwitch(reason) {
|
|
243
|
+
log(`\n[健康检查] ${reason},切换到下一个账户...`);
|
|
244
|
+
const oldAccount = currentAccount;
|
|
245
|
+
const nextAccount = healthChecker.getNextAccount();
|
|
246
|
+
currentAccount = nextAccount;
|
|
247
|
+
// 先关闭旧浏览器连接,再切换
|
|
248
|
+
await browser.close().catch(() => {});
|
|
249
|
+
const newBrowser = await switchAccount(
|
|
250
|
+
{ port: oldAccount.port, userDataDir: oldAccount.userDataDir },
|
|
251
|
+
{ port: nextAccount.port, userDataDir: nextAccount.userDataDir },
|
|
252
|
+
effectiveProxy,
|
|
253
|
+
);
|
|
254
|
+
browser = newBrowser;
|
|
255
|
+
const newPage = await setupNewPage(browser);
|
|
256
|
+
page = newPage;
|
|
257
|
+
Object.assign(cdpOptions, {
|
|
258
|
+
port: nextAccount.port,
|
|
259
|
+
userDataDir: nextAccount.userDataDir,
|
|
260
|
+
...(effectiveProxy ? { proxyServer: effectiveProxy } : {}),
|
|
123
261
|
});
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
262
|
+
log(
|
|
263
|
+
`[健康检查] 已切换到端口 ${nextAccount.port}${effectiveProxy ? ", 代理: " + effectiveProxy : ""}`,
|
|
264
|
+
);
|
|
265
|
+
// 重新导航并检测登录
|
|
266
|
+
await withRetry("账户切换后页面导航", async () => {
|
|
267
|
+
await page.goto(STARTUP_TIKTOK_URL, {
|
|
268
|
+
waitUntil: "domcontentloaded",
|
|
269
|
+
timeout: PAGE_GOTO_TIMEOUT,
|
|
270
|
+
});
|
|
271
|
+
});
|
|
272
|
+
loggedIn = await safeCheckLogin(page);
|
|
273
|
+
log(`[健康检查] 新账户登录状态: ${loggedIn ? "已登录" : "未登录"}`);
|
|
274
|
+
// 重新提取用户名
|
|
275
|
+
myUsername = await extractMyUsername(page);
|
|
276
|
+
if (myUsername) {
|
|
277
|
+
log(`[健康检查] 新账户用户名: @${myUsername}`);
|
|
128
278
|
}
|
|
279
|
+
captchaCount = 0;
|
|
280
|
+
staleRounds = 0;
|
|
281
|
+
consecutiveNetworkErrors = 0;
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
const clientId = randomUUID();
|
|
285
|
+
const headers = (extra = {}) => ({
|
|
286
|
+
"X-Client-Id": clientId,
|
|
287
|
+
"X-Client-Info": JSON.stringify({
|
|
288
|
+
type: "tag-search",
|
|
289
|
+
port: cdpOptions.port,
|
|
290
|
+
}),
|
|
291
|
+
...extra,
|
|
292
|
+
});
|
|
293
|
+
log(`客户端 ID: ${clientId.substring(0, 8)}...`);
|
|
294
|
+
log("");
|
|
129
295
|
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
296
|
+
let totalSearched = 0;
|
|
297
|
+
let emptyRounds = 0;
|
|
298
|
+
let staleRounds = 0;
|
|
299
|
+
let captchaCount = 0;
|
|
300
|
+
let consecutiveNetworkErrors = 0;
|
|
135
301
|
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
302
|
+
// ── 主循环 ──
|
|
303
|
+
while (!shuttingDown) {
|
|
304
|
+
try {
|
|
305
|
+
// ── 健康检查 ──
|
|
306
|
+
const checkResult = healthChecker.check();
|
|
307
|
+
if (checkResult.shouldSwitch) {
|
|
308
|
+
await handleAccountSwitch(checkResult.reason);
|
|
309
|
+
} else if (checkResult.info && totalSearched % 10 === 0) {
|
|
310
|
+
log(`\n[健康检查] ${checkResult.info}`);
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
// ── 验证码检测(切换任务前) ──
|
|
314
|
+
const preCaptcha = await detectCaptcha(page);
|
|
315
|
+
if (preCaptcha && preCaptcha.visible) {
|
|
316
|
+
log(`\n[验证码] 切换任务前检测到验证码,等待3分钟...`);
|
|
317
|
+
captchaCount++;
|
|
318
|
+
log(` [验证码] 累计 ${captchaCount} 次`);
|
|
319
|
+
|
|
320
|
+
await new Promise((r) => setTimeout(r, 3 * 60 * 1000));
|
|
321
|
+
|
|
322
|
+
const closeResult = await closeCaptcha(page);
|
|
323
|
+
if (closeResult.success) {
|
|
324
|
+
log(` [验证码] 已关闭`);
|
|
325
|
+
} else {
|
|
326
|
+
log(` [验证码] 关闭失败: ${closeResult.reason}`);
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
// 上报
|
|
330
|
+
await withRetry("report captcha", () =>
|
|
331
|
+
apiPost(`${baseUrl}/api/error-report`, {
|
|
332
|
+
userId: clientId,
|
|
333
|
+
username: "unknown",
|
|
334
|
+
errorType: "captcha",
|
|
335
|
+
errorMessage: "切换任务前检测到验证码",
|
|
336
|
+
stage: "between-tasks",
|
|
337
|
+
errorStack: "",
|
|
338
|
+
}),
|
|
339
|
+
).catch(() => {});
|
|
340
|
+
|
|
341
|
+
// 累计2次切换账户
|
|
342
|
+
if (captchaCount >= 2) {
|
|
343
|
+
await handleAccountSwitch(`验证码累计 ${captchaCount} 次`);
|
|
344
|
+
captchaCount = 0;
|
|
345
|
+
}
|
|
346
|
+
}
|
|
347
|
+
|
|
348
|
+
// ── 网络错误退避 ──
|
|
349
|
+
if (consecutiveNetworkErrors > 0) {
|
|
350
|
+
const waitTime =
|
|
351
|
+
consecutiveNetworkErrors <= 2
|
|
352
|
+
? 0
|
|
353
|
+
: consecutiveNetworkErrors <= 5
|
|
354
|
+
? 30000
|
|
355
|
+
: 300000;
|
|
356
|
+
if (waitTime > 0) {
|
|
357
|
+
log(
|
|
358
|
+
` [网络] 连续 ${consecutiveNetworkErrors} 次网络异常,等待 ${waitTime / 1000}s 后重试...`,
|
|
359
|
+
);
|
|
360
|
+
await new Promise((r) => setTimeout(r, waitTime));
|
|
361
|
+
}
|
|
362
|
+
}
|
|
363
|
+
|
|
364
|
+
// ── 任务间延迟 ──
|
|
365
|
+
const { switchMax } = getDelayConfig();
|
|
366
|
+
await delay(switchMax, switchMax * 3);
|
|
367
|
+
|
|
368
|
+
// ── 1. 获取搜索任务 ──
|
|
369
|
+
const taskRes = await fetch(`${baseUrl}/api/tags/search-task`, {
|
|
370
|
+
headers: headers(),
|
|
142
371
|
});
|
|
143
|
-
const
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
372
|
+
const task = await taskRes.json();
|
|
373
|
+
|
|
374
|
+
if (!task.tag) {
|
|
375
|
+
emptyRounds++;
|
|
376
|
+
if (emptyRounds >= 5) {
|
|
377
|
+
log(`⏳ 连续 ${emptyRounds} 轮无标签可搜索(30 秒后重试)...`);
|
|
378
|
+
await new Promise((r) => setTimeout(r, 30000));
|
|
379
|
+
emptyRounds = 0;
|
|
380
|
+
await handleAccountSwitch("连续5轮无标签可搜索");
|
|
381
|
+
continue;
|
|
382
|
+
}
|
|
383
|
+
log(`⏳ 暂无标签可搜索(连续 ${emptyRounds} 轮),10 秒后重试...`);
|
|
384
|
+
await new Promise((r) => setTimeout(r, 10000));
|
|
385
|
+
continue;
|
|
386
|
+
}
|
|
387
|
+
emptyRounds = 0;
|
|
388
|
+
|
|
389
|
+
const tag = task.tag;
|
|
390
|
+
const country = task.country;
|
|
391
|
+
log(
|
|
392
|
+
`[${totalSearched + 1}] 搜索: "${tag}" (score: ${task.score}, 国家: ${country})`,
|
|
393
|
+
);
|
|
394
|
+
|
|
395
|
+
// ── 2. 锁定标签 ──
|
|
396
|
+
const claimRes = await fetch(`${baseUrl}/api/tags/search-claim`, {
|
|
147
397
|
method: "POST",
|
|
148
398
|
headers: headers({ "Content-Type": "application/json" }),
|
|
149
|
-
body: JSON.stringify({ tag
|
|
399
|
+
body: JSON.stringify({ tag }),
|
|
150
400
|
});
|
|
151
|
-
|
|
152
|
-
|
|
401
|
+
const claimData = await claimRes.json();
|
|
402
|
+
if (!claimData.ok) {
|
|
403
|
+
log(` ⏭️ ${claimData.error},跳过`);
|
|
404
|
+
continue;
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
// ── 3. 并行搜索 ──
|
|
408
|
+
const startTime = Date.now();
|
|
409
|
+
let ids = [];
|
|
410
|
+
let searchError = null;
|
|
411
|
+
try {
|
|
412
|
+
ids = await collectSearchAuthors(browser, tag, {
|
|
413
|
+
log,
|
|
414
|
+
exclude: myUsername ? [myUsername] : [],
|
|
415
|
+
});
|
|
416
|
+
} catch (e) {
|
|
417
|
+
searchError = e;
|
|
418
|
+
}
|
|
419
|
+
const elapsed = ((Date.now() - startTime) / 1000).toFixed(1);
|
|
420
|
+
|
|
421
|
+
// ── 浏览器关闭恢复 ──
|
|
422
|
+
if (searchError && isBrowserClosedError(searchError)) {
|
|
423
|
+
log(
|
|
424
|
+
`\n[浏览器] 检测到浏览器关闭 (${searchError.message}),正在重建...`,
|
|
425
|
+
);
|
|
426
|
+
const newBrowser = await relaunchBrowser(cdpOptions, cdpOptions.port);
|
|
427
|
+
browser = newBrowser;
|
|
428
|
+
page = await setupNewPage(browser);
|
|
429
|
+
await new Promise((r) => setTimeout(r, 3000));
|
|
430
|
+
log(`[浏览器] 已重建,重试当前搜索...`);
|
|
431
|
+
|
|
432
|
+
// 重试搜索
|
|
433
|
+
try {
|
|
434
|
+
ids = await collectSearchAuthors(browser, tag, {
|
|
435
|
+
log,
|
|
436
|
+
exclude: myUsername ? [myUsername] : [],
|
|
437
|
+
});
|
|
438
|
+
searchError = null;
|
|
439
|
+
} catch (e2) {
|
|
440
|
+
searchError = e2;
|
|
441
|
+
}
|
|
442
|
+
}
|
|
443
|
+
|
|
444
|
+
// ── 临时错误重试 ──
|
|
445
|
+
if (searchError) {
|
|
446
|
+
const retryablePatterns = [
|
|
447
|
+
"Timeout",
|
|
448
|
+
"Execution context was destroyed",
|
|
449
|
+
"ERR_CONNECTION",
|
|
450
|
+
"net::",
|
|
451
|
+
];
|
|
452
|
+
const isRetryable = retryablePatterns.some((p) =>
|
|
453
|
+
searchError.message.includes(p),
|
|
454
|
+
);
|
|
455
|
+
if (isRetryable) {
|
|
456
|
+
log(` [临时错误] 等待 5 秒后重试...`);
|
|
457
|
+
await new Promise((r) => setTimeout(r, 5000));
|
|
458
|
+
try {
|
|
459
|
+
ids = await collectSearchAuthors(browser, tag, {
|
|
460
|
+
log,
|
|
461
|
+
exclude: myUsername ? [myUsername] : [],
|
|
462
|
+
});
|
|
463
|
+
searchError = null;
|
|
464
|
+
} catch (e2) {
|
|
465
|
+
searchError = e2;
|
|
466
|
+
}
|
|
467
|
+
}
|
|
468
|
+
}
|
|
469
|
+
|
|
470
|
+
// ── 错误上报 ──
|
|
471
|
+
if (searchError) {
|
|
472
|
+
consecutiveNetworkErrors++;
|
|
473
|
+
log(` ❌ 搜索异常: ${searchError.message}`);
|
|
474
|
+
|
|
475
|
+
const errorType = consecutiveNetworkErrors > 1 ? "network" : "other";
|
|
476
|
+
await withRetry("report error", () =>
|
|
477
|
+
apiPost(`${baseUrl}/api/error-report`, {
|
|
478
|
+
userId: clientId,
|
|
479
|
+
username: tag,
|
|
480
|
+
errorType,
|
|
481
|
+
errorMessage: searchError.message,
|
|
482
|
+
stage: "search",
|
|
483
|
+
errorStack: searchError.stack || "",
|
|
484
|
+
}),
|
|
485
|
+
).catch(() => {});
|
|
486
|
+
|
|
487
|
+
// 仍提交空结果,释放标签锁定
|
|
488
|
+
await fetch(`${baseUrl}/api/tags/search-result`, {
|
|
489
|
+
method: "POST",
|
|
490
|
+
headers: headers({ "Content-Type": "application/json" }),
|
|
491
|
+
body: JSON.stringify({ tag, authors: [], country }),
|
|
492
|
+
}).catch(() => {});
|
|
493
|
+
log(` 提交: 0 个作者(搜索异常)`);
|
|
494
|
+
totalSearched++;
|
|
495
|
+
continue;
|
|
496
|
+
}
|
|
153
497
|
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
498
|
+
log(` 搜索完成: ${ids.length} 个作者 (${elapsed}s)`);
|
|
499
|
+
|
|
500
|
+
// ── 搜索完成后验证码检测 ──
|
|
501
|
+
const postCaptcha = await detectCaptcha(page);
|
|
502
|
+
if (postCaptcha && postCaptcha.visible) {
|
|
503
|
+
captchaCount++;
|
|
504
|
+
log(` [验证码] 搜索后检测到验证码,累计 ${captchaCount} 次`);
|
|
505
|
+
|
|
506
|
+
await withRetry("report captcha", () =>
|
|
507
|
+
apiPost(`${baseUrl}/api/error-report`, {
|
|
508
|
+
userId: clientId,
|
|
509
|
+
username: tag,
|
|
510
|
+
errorType: "captcha",
|
|
511
|
+
errorMessage: "搜索后页面出现验证码",
|
|
512
|
+
stage: "after-search",
|
|
513
|
+
errorStack: "",
|
|
514
|
+
}),
|
|
515
|
+
).catch(() => {});
|
|
516
|
+
|
|
517
|
+
if (captchaCount >= 2) {
|
|
518
|
+
await handleAccountSwitch(`验证码累计 ${captchaCount} 次`);
|
|
519
|
+
captchaCount = 0;
|
|
520
|
+
}
|
|
521
|
+
}
|
|
522
|
+
|
|
523
|
+
consecutiveNetworkErrors = 0;
|
|
524
|
+
|
|
525
|
+
// ── 4. 提交结果 ──
|
|
526
|
+
let pushed = 0;
|
|
527
|
+
if (ids.length > 0) {
|
|
528
|
+
const resultRes = await fetch(`${baseUrl}/api/tags/search-result`, {
|
|
529
|
+
method: "POST",
|
|
530
|
+
headers: headers({ "Content-Type": "application/json" }),
|
|
531
|
+
body: JSON.stringify({ tag, authors: ids, country }),
|
|
532
|
+
});
|
|
533
|
+
const result = await resultRes.json();
|
|
534
|
+
pushed = result.pushed || 0;
|
|
535
|
+
log(` 提交: 新增 ${pushed} 个, 跳过 ${result.skipped} 个`);
|
|
536
|
+
} else {
|
|
537
|
+
await fetch(`${baseUrl}/api/tags/search-result`, {
|
|
538
|
+
method: "POST",
|
|
539
|
+
headers: headers({ "Content-Type": "application/json" }),
|
|
540
|
+
body: JSON.stringify({ tag, authors: [], country }),
|
|
541
|
+
});
|
|
542
|
+
log(` 提交: 0 个作者`);
|
|
543
|
+
}
|
|
544
|
+
|
|
545
|
+
// 无新作者计数 & 切换
|
|
546
|
+
if (pushed > 0) {
|
|
547
|
+
staleRounds = 0;
|
|
548
|
+
} else {
|
|
549
|
+
staleRounds++;
|
|
550
|
+
log(` [无新增] 连续 ${staleRounds} 轮`);
|
|
551
|
+
if (staleRounds >= 5) {
|
|
552
|
+
staleRounds = 0;
|
|
553
|
+
await handleAccountSwitch("连续5轮无新作者");
|
|
554
|
+
continue;
|
|
555
|
+
}
|
|
556
|
+
}
|
|
557
|
+
|
|
558
|
+
totalSearched++;
|
|
559
|
+
} catch (e) {
|
|
560
|
+
// 浏览器关闭错误:自动重建
|
|
561
|
+
if (isBrowserClosedError(e)) {
|
|
562
|
+
log(`\n[浏览器] 检测到浏览器关闭 (${e.message}),正在重建...`);
|
|
563
|
+
const newBrowser = await relaunchBrowser(cdpOptions, cdpOptions.port);
|
|
564
|
+
browser = newBrowser;
|
|
565
|
+
page = await setupNewPage(browser);
|
|
566
|
+
await new Promise((r) => setTimeout(r, 3000));
|
|
567
|
+
log(`[浏览器] 已重建,继续处理...`);
|
|
568
|
+
continue;
|
|
569
|
+
}
|
|
570
|
+
// 其他未预期错误
|
|
571
|
+
log(`❌ 异常: ${e.message}`);
|
|
572
|
+
log(e.stack || "");
|
|
573
|
+
await new Promise((r) => setTimeout(r, 5000));
|
|
574
|
+
}
|
|
158
575
|
}
|
|
576
|
+
} finally {
|
|
577
|
+
process.removeListener("SIGINT", cleanup);
|
|
578
|
+
process.removeListener("SIGTERM", cleanup);
|
|
579
|
+
await browser?.close().catch(() => {});
|
|
580
|
+
killEdgeProcesses(cdpOptions.userDataDir, cdpOptions.port).catch(() => {});
|
|
581
|
+
log("\n搜索采集已停止");
|
|
159
582
|
}
|
|
160
583
|
}
|
package/src/lib/args.js
CHANGED
|
@@ -558,15 +558,18 @@ function parseRefreshArgs(args) {
|
|
|
558
558
|
|
|
559
559
|
function parseSearchArgs(args) {
|
|
560
560
|
let serverUrl = null;
|
|
561
|
-
let
|
|
561
|
+
let basePort = 3333;
|
|
562
|
+
let portCount = 10;
|
|
562
563
|
let proxy = null;
|
|
563
564
|
|
|
564
565
|
for (let i = 0; i < args.length; i++) {
|
|
565
566
|
const arg = args[i];
|
|
566
567
|
if (arg === "-s" || arg === "--server") {
|
|
567
568
|
serverUrl = args[++i];
|
|
568
|
-
} else if (arg === "--port") {
|
|
569
|
-
|
|
569
|
+
} else if (arg === "--base-port") {
|
|
570
|
+
basePort = parseInt(args[++i]) || 3333;
|
|
571
|
+
} else if (arg === "--port-count") {
|
|
572
|
+
portCount = parseInt(args[++i]) || 10;
|
|
570
573
|
} else if (arg === "--proxy") {
|
|
571
574
|
proxy = args[++i];
|
|
572
575
|
}
|
|
@@ -574,7 +577,7 @@ function parseSearchArgs(args) {
|
|
|
574
577
|
|
|
575
578
|
return {
|
|
576
579
|
subcommand: "search",
|
|
577
|
-
search: { serverUrl,
|
|
580
|
+
search: { serverUrl, basePort, portCount, proxy },
|
|
578
581
|
urls: [],
|
|
579
582
|
outputFormat: "json",
|
|
580
583
|
exploreCount: 0,
|
|
@@ -1092,7 +1095,8 @@ function parseTagArgs(args) {
|
|
|
1092
1095
|
subcommand: "search",
|
|
1093
1096
|
search: {
|
|
1094
1097
|
serverUrl,
|
|
1095
|
-
|
|
1098
|
+
basePort: searchPort,
|
|
1099
|
+
portCount: 10,
|
|
1096
1100
|
proxy: scoreProxy,
|
|
1097
1101
|
},
|
|
1098
1102
|
urls: [],
|
package/src/lib/constants.js
CHANGED
|
@@ -319,7 +319,8 @@ const HELP_TEXT = [
|
|
|
319
319
|
" 5 个搜索 Tab 并行采集(综合/视频/用户/直播/照片),去重后提交到 job_base",
|
|
320
320
|
" 选项:",
|
|
321
321
|
" -s, --server <URL> 服务端地址(默认 http://127.0.0.1:3000)",
|
|
322
|
-
" --port
|
|
322
|
+
" --base-port <端口号> 起始 CDP 端口,默认 3333",
|
|
323
|
+
" --port-count <数量> 端口数量(账户数),默认 10",
|
|
323
324
|
" --proxy <地址> 浏览器代理",
|
|
324
325
|
" 行为:",
|
|
325
326
|
" - 从 tags 表取 score>70 的 productive 标签",
|
|
@@ -328,7 +329,7 @@ const HELP_TEXT = [
|
|
|
328
329
|
" - 猜测国家取标签 countries 数组的第一个",
|
|
329
330
|
" 示例: tt-help search",
|
|
330
331
|
" tt-help search -s http://127.0.0.1:3001",
|
|
331
|
-
" tt-help search --port 3333 -s http://127.0.0.1:3001",
|
|
332
|
+
" tt-help search --base-port 3333 -s http://127.0.0.1:3001",
|
|
332
333
|
" tt-help search --proxy http://127.0.0.1:7890",
|
|
333
334
|
"",
|
|
334
335
|
" raw-sync --db <主库路径> --archive <归档库路径> [选项]",
|
|
@@ -18,7 +18,7 @@ const DEFAULT_TABS = ["综合", "视频", "用户", "直播", "照片"];
|
|
|
18
18
|
|
|
19
19
|
const SCROLL = {
|
|
20
20
|
maxRounds: 60,
|
|
21
|
-
staleThreshold:
|
|
21
|
+
staleThreshold: 3,
|
|
22
22
|
delayMin: 800,
|
|
23
23
|
delayMax: 1500,
|
|
24
24
|
waitAfterScroll: 4000,
|
|
@@ -30,15 +30,21 @@ const randDelay = () =>
|
|
|
30
30
|
SCROLL.delayMin,
|
|
31
31
|
);
|
|
32
32
|
|
|
33
|
-
async function extractAuthors(page) {
|
|
34
|
-
return page.evaluate(
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
const
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
33
|
+
async function extractAuthors(page, excludeSet) {
|
|
34
|
+
return page.evaluate(
|
|
35
|
+
(exclude) => {
|
|
36
|
+
const ids = new Set();
|
|
37
|
+
const skip = new Set(exclude || []);
|
|
38
|
+
for (const a of document.querySelectorAll('a[href*="/@"]')) {
|
|
39
|
+
const m = (a.getAttribute("href") || "").match(/\/@([^/?]+)/);
|
|
40
|
+
if (m && m[1] !== "undefined" && m[1].length <= 30 && !skip.has(m[1])) {
|
|
41
|
+
ids.add(m[1]);
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
return [...ids];
|
|
45
|
+
},
|
|
46
|
+
[...excludeSet],
|
|
47
|
+
);
|
|
42
48
|
}
|
|
43
49
|
|
|
44
50
|
async function scrollPage(page) {
|
|
@@ -52,7 +58,7 @@ async function scrollPage(page) {
|
|
|
52
58
|
});
|
|
53
59
|
}
|
|
54
60
|
|
|
55
|
-
async function processOneTab(page, tabName, keyword, allIds, log) {
|
|
61
|
+
async function processOneTab(page, tabName, keyword, allIds, excludeSet, log) {
|
|
56
62
|
const urlFn = TAB_URLS[tabName];
|
|
57
63
|
if (!urlFn) {
|
|
58
64
|
log(`[${tabName}] 不支持`);
|
|
@@ -89,7 +95,7 @@ async function processOneTab(page, tabName, keyword, allIds, log) {
|
|
|
89
95
|
await delay(SCROLL.waitAfterScroll);
|
|
90
96
|
}
|
|
91
97
|
|
|
92
|
-
const ids = await extractAuthors(page);
|
|
98
|
+
const ids = await extractAuthors(page, excludeSet);
|
|
93
99
|
let added = 0;
|
|
94
100
|
for (const id of ids) {
|
|
95
101
|
if (!tabSet.has(id)) {
|
|
@@ -138,8 +144,11 @@ async function processOneTab(page, tabName, keyword, allIds, log) {
|
|
|
138
144
|
export async function collectSearchAuthors(browser, keyword, opts = {}) {
|
|
139
145
|
const tabs = opts.tabs || DEFAULT_TABS;
|
|
140
146
|
const log = opts.log || ((...args) => console.error(...args));
|
|
147
|
+
const excludeSet = new Set(opts.exclude || []);
|
|
141
148
|
|
|
142
149
|
log(`[collect] "${keyword}" Tab: [${tabs.join(", ")}]`);
|
|
150
|
+
if (excludeSet.size > 0)
|
|
151
|
+
log(`[collect] 排除: [${[...excludeSet].join(", ")}]`);
|
|
143
152
|
|
|
144
153
|
const context = browser.contexts()[0];
|
|
145
154
|
const existingPage = context.pages()[0];
|
|
@@ -152,7 +161,7 @@ export async function collectSearchAuthors(browser, keyword, opts = {}) {
|
|
|
152
161
|
try {
|
|
153
162
|
page = i === 0 ? existingPage : await context.newPage();
|
|
154
163
|
if (i > 0) newPages.push(page);
|
|
155
|
-
await processOneTab(page, tabName, keyword, allIds, log);
|
|
164
|
+
await processOneTab(page, tabName, keyword, allIds, excludeSet, log);
|
|
156
165
|
} catch (e) {
|
|
157
166
|
log(`[${tabName}] 异常: ${e.message}`);
|
|
158
167
|
} finally {
|