tt-help-cli-ycl 1.4.145 → 1.4.146
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/scripts/test-product-fill-cdp.mjs +168 -0
- package/src/cli/comments.js +265 -388
- package/src/cli/product-fill.js +255 -128
- package/src/cli/tag.js +43 -9
- package/src/lib/args.js +15 -5
- package/src/lib/constants.js +8 -5
- package/src/lib/shop-scraper.mjs +69 -6
- package/src/watch/data-store.js +41 -6
- package/src/watch/public/app.js +58 -0
- package/src/watch/public/index.html +46 -1
- package/src/watch/server.js +4 -1
- package/scripts/test-product-fill.mjs +0 -306
package/src/cli/product-fill.js
CHANGED
|
@@ -1,34 +1,43 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* product-fill 命令 — 循环到服务端领取"缺商品详情"任务,补全商品信息
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
4
|
+
* 使用前提:--profiles 国家/端口对 配置端口与国家绑定(如 US/9223,JP/1222)。
|
|
5
|
+
* TikTok 区域判定 = 浏览器 profile 区域 + 代理出口,商品国家必须与两者
|
|
6
|
+
* 匹配才能抓到数据(如 9223=US profile、1222=JP profile)。
|
|
7
|
+
*
|
|
8
|
+
* 流程:
|
|
9
|
+
* 1. 每轮领取前探测代理出口国家,与配置不匹配则跳过本轮
|
|
10
|
+
* 2. GET /api/product-tasks?country=XX 领取该国家任务
|
|
11
|
+
* 3. CDP 直抓商品详情(评分/销量/评论),验证码弹浏览器人工解决
|
|
8
12
|
* 4. 成功 → POST /api/product-fill/:uniqueId 回写 jobs 表
|
|
13
|
+
* 5. 下架壳页/连续失败/无效 URL → 回写 rating=-1 哨兵,永久移出候选
|
|
9
14
|
*
|
|
10
|
-
* 用法: tt-help product-fill [
|
|
15
|
+
* 用法: tt-help product-fill --profiles <XX/port>[,<XX/port>...] [-s 服务端] [-i 间隔秒] [-l 每批上限] [--proxy URL]
|
|
11
16
|
*/
|
|
12
|
-
import {
|
|
13
|
-
import { fileURLToPath } from "node:url";
|
|
17
|
+
import { ProxyAgent, request } from "undici";
|
|
14
18
|
import { getMacOrUuid } from "../lib/mac-or-uuid.js";
|
|
15
19
|
import { createApiClient } from "../lib/api-client.js";
|
|
16
20
|
import { userId as configuredUserId, saveUserId } from "../lib/constants.js";
|
|
21
|
+
import { ensureBrowserReady } from "../lib/browser/cdp.js";
|
|
22
|
+
import {
|
|
23
|
+
getOrCreatePage,
|
|
24
|
+
isBrowserClosedError,
|
|
25
|
+
relaunchBrowser,
|
|
26
|
+
} from "../lib/browser/page.js";
|
|
27
|
+
import {
|
|
28
|
+
getProductInfoCDP,
|
|
29
|
+
isEmptyProductDetail,
|
|
30
|
+
} from "../lib/shop-scraper.mjs";
|
|
31
|
+
import {
|
|
32
|
+
detectCaptcha,
|
|
33
|
+
waitAndGetCaptcha,
|
|
34
|
+
} from "../scraper/modules/captcha-handler.js";
|
|
17
35
|
|
|
18
36
|
const CAPTCHA_BACKOFF_MS = 30_000; // 验证码后冷却时长
|
|
19
|
-
const
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
try {
|
|
24
|
-
const res = await fetch(`${url}/health`, {
|
|
25
|
-
signal: AbortSignal.timeout(timeoutMs),
|
|
26
|
-
});
|
|
27
|
-
return res.ok;
|
|
28
|
-
} catch {
|
|
29
|
-
return false;
|
|
30
|
-
}
|
|
31
|
-
}
|
|
37
|
+
const CDP_CAPTCHA_WAIT_MS = 180_000; // CDP 模式人工解验证码最长等待
|
|
38
|
+
const FAIL_MARK_LIMIT = 3; // 连续失败达到该次数后哨兵放弃
|
|
39
|
+
const SENTINEL_RATING = -1; // 哨兵:商品不可售/已下架(正常评分 0-5,-1 不会冲突)
|
|
40
|
+
const DEFAULT_PROXY = "http://127.0.0.1:7897"; // 未传 --proxy 时按本机 Clash 默认端口探测
|
|
32
41
|
|
|
33
42
|
function formatNow() {
|
|
34
43
|
return new Date().toLocaleString("zh-CN", { hour12: false });
|
|
@@ -38,26 +47,28 @@ function log(message = "") {
|
|
|
38
47
|
console.error(`[${formatNow()}] ${message}`);
|
|
39
48
|
}
|
|
40
49
|
|
|
41
|
-
/**
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
50
|
+
/**
|
|
51
|
+
* 通过代理探测出口 IP 国家码(ip-api.com 免费接口)。
|
|
52
|
+
* 探测失败(超时/代理未开/接口异常)返回 null。
|
|
53
|
+
*/
|
|
54
|
+
export async function detectProxyCountry(proxyUrl, timeoutMs = 8000) {
|
|
55
|
+
const dispatcher = proxyUrl ? new ProxyAgent(proxyUrl) : null;
|
|
56
|
+
try {
|
|
57
|
+
const { statusCode, body } = await request(
|
|
58
|
+
"http://ip-api.com/json?fields=countryCode",
|
|
59
|
+
{
|
|
60
|
+
method: "GET",
|
|
61
|
+
...(dispatcher ? { dispatcher } : {}),
|
|
62
|
+
headersTimeout: timeoutMs,
|
|
63
|
+
bodyTimeout: timeoutMs,
|
|
64
|
+
},
|
|
53
65
|
);
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
66
|
+
if (statusCode !== 200) return null;
|
|
67
|
+
const data = JSON.parse(await body.text());
|
|
68
|
+
return data?.countryCode || null;
|
|
69
|
+
} catch {
|
|
70
|
+
return null;
|
|
59
71
|
}
|
|
60
|
-
return res.json();
|
|
61
72
|
}
|
|
62
73
|
|
|
63
74
|
export async function handleProductFill(options) {
|
|
@@ -65,48 +76,55 @@ export async function handleProductFill(options) {
|
|
|
65
76
|
productFillLimit,
|
|
66
77
|
productFillInterval,
|
|
67
78
|
serverUrl,
|
|
68
|
-
productFillWebserver,
|
|
69
79
|
productFillProxy,
|
|
80
|
+
productFillProfiles,
|
|
70
81
|
showHelp,
|
|
71
82
|
} = options;
|
|
72
|
-
// 不指定 -w 时默认使用本地 webserver(http://127.0.0.1:3000),
|
|
73
|
-
// 且本地无响应时自动拉起(适合单独机器:验证码时弹浏览器人工处理)
|
|
74
|
-
const spawnLocalWebserver = !productFillWebserver;
|
|
75
|
-
const webserverUrl = productFillWebserver || "http://127.0.0.1:3000";
|
|
76
|
-
// 自动拉起 webserver 时透传的代理(默认本机 clash 7897)
|
|
77
|
-
const localWebserverProxy = productFillProxy || "http://127.0.0.1:7897";
|
|
78
|
-
let localWebserverChild = null;
|
|
79
83
|
|
|
80
84
|
if (showHelp) {
|
|
81
85
|
log(
|
|
82
|
-
"用法: tt-help product-fill [
|
|
86
|
+
"用法: tt-help product-fill --profiles <国家/端口>[,<国家/端口>...] [-s 服务端] [-i 间隔秒] [-l 每批上限] [--proxy URL]",
|
|
83
87
|
);
|
|
84
88
|
log("");
|
|
85
89
|
log("参数:");
|
|
86
|
-
log(" -s, --server <URL> 服务端地址(默认: http://127.0.0.1:3001)");
|
|
87
90
|
log(
|
|
88
|
-
"
|
|
91
|
+
" --profiles <XX/port> 必填。国家/端口对,逗号分隔可传多个(如 US/9223,JP/1222);",
|
|
92
|
+
);
|
|
93
|
+
log(
|
|
94
|
+
" 每轮探测代理出口,在配置中匹配对应端口自动领取该国家任务",
|
|
89
95
|
);
|
|
96
|
+
log(" -s, --server <URL> 服务端地址(默认: http://127.0.0.1:3001)");
|
|
90
97
|
log(
|
|
91
|
-
" --proxy <URL>
|
|
98
|
+
" --proxy <URL> 浏览器启动代理及出口探测(默认: http://127.0.0.1:7897)",
|
|
92
99
|
);
|
|
93
100
|
log(" -i, --interval <N> 无任务时轮询间隔,单位秒(默认: 10)");
|
|
94
101
|
log(" -l, --limit <N> 每批领取任务数上限(默认: 5)");
|
|
95
102
|
log("");
|
|
96
103
|
log("说明:");
|
|
97
|
-
log(" 循环向服务端 /api/product-tasks
|
|
98
|
-
log("
|
|
99
|
-
log("
|
|
100
|
-
log("
|
|
101
|
-
log(
|
|
102
|
-
" 不传 -w 时本地无 webserver 会自动拉起(验证码时弹浏览器,人工解决后继续)。",
|
|
103
|
-
);
|
|
104
|
+
log(" 循环向服务端 /api/product-tasks?country=XX 领取缺商品详情任务;");
|
|
105
|
+
log(" 代理出口不在配置范围 → 本轮不领任务,下轮重新探测;");
|
|
106
|
+
log(" 商品下架壳页立即标记不可售(rating=-1),连续失败 3 次同样标记;");
|
|
107
|
+
log(" 验证码在浏览器中弹出时等待人工解决(最长 3 分钟)后自动重试。");
|
|
104
108
|
log("");
|
|
105
109
|
log("示例:");
|
|
110
|
+
log(" tt-help product-fill --profiles US/9223 -s http://127.0.0.1:3001");
|
|
106
111
|
log(
|
|
107
|
-
" tt-help product-fill -s http://
|
|
112
|
+
" tt-help product-fill --profiles US/9223,JP/1222 -s http://117.71.53.99:17301 -i 5",
|
|
113
|
+
);
|
|
114
|
+
return;
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
// ── 参数校验:至少一个有效的国家/端口对;同端口重复保留最后一个 ──
|
|
118
|
+
const finalProfiles = [];
|
|
119
|
+
for (const p of productFillProfiles || []) {
|
|
120
|
+
const idx = finalProfiles.findIndex((x) => x.port === p.port);
|
|
121
|
+
if (idx >= 0) finalProfiles[idx] = p;
|
|
122
|
+
else finalProfiles.push(p);
|
|
123
|
+
}
|
|
124
|
+
if (finalProfiles.length === 0) {
|
|
125
|
+
log(
|
|
126
|
+
"[ProductFill] 错误: 请传 --profiles 国家/端口对(如 --profiles US/9223,JP/1222)",
|
|
108
127
|
);
|
|
109
|
-
log(" tt-help product-fill -w http://117.71.53.99:17300 -i 5 -l 10");
|
|
110
128
|
return;
|
|
111
129
|
}
|
|
112
130
|
|
|
@@ -118,7 +136,7 @@ export async function handleProductFill(options) {
|
|
|
118
136
|
log(`[ProductFill] 未检测到本地用户编号,已生成并使用: ${userId}`);
|
|
119
137
|
}
|
|
120
138
|
log(
|
|
121
|
-
`[ProductFill] 用户编号: ${userId} | 服务端: ${serverUrl} |
|
|
139
|
+
`[ProductFill] 用户编号: ${userId} | 服务端: ${serverUrl} | 端口配置: ${finalProfiles.map((p) => `${p.port}=${p.country}`).join(", ")}`,
|
|
122
140
|
);
|
|
123
141
|
log(
|
|
124
142
|
`[ProductFill] 每批上限: ${productFillLimit} | 空闲间隔: ${productFillInterval}秒`,
|
|
@@ -128,17 +146,10 @@ export async function handleProductFill(options) {
|
|
|
128
146
|
|
|
129
147
|
let shuttingDown = false;
|
|
130
148
|
let forceExitTimer = null;
|
|
131
|
-
/** 强制退出路径清理自动拉起的 webserver(防止孤儿进程) */
|
|
132
|
-
const killLocalWebserver = () => {
|
|
133
|
-
if (localWebserverChild && !localWebserverChild.killed) {
|
|
134
|
-
localWebserverChild.kill("SIGKILL");
|
|
135
|
-
}
|
|
136
|
-
};
|
|
137
149
|
const shutdown = () => {
|
|
138
150
|
if (shuttingDown) {
|
|
139
151
|
// 第二次信号:立即强制退出(网络重试等场景可能无法优雅退出)
|
|
140
152
|
log(`\n[ProductFill] 再次收到信号,强制退出...`);
|
|
141
|
-
killLocalWebserver();
|
|
142
153
|
process.exit(1);
|
|
143
154
|
return;
|
|
144
155
|
}
|
|
@@ -149,7 +160,6 @@ export async function handleProductFill(options) {
|
|
|
149
160
|
// 兜底:卡在网络重试/等待中时 10 秒后强制退出
|
|
150
161
|
forceExitTimer = setTimeout(() => {
|
|
151
162
|
log("[ProductFill] 等待超时,强制退出");
|
|
152
|
-
killLocalWebserver();
|
|
153
163
|
process.exit(0);
|
|
154
164
|
}, 10_000);
|
|
155
165
|
forceExitTimer.unref?.();
|
|
@@ -157,57 +167,43 @@ export async function handleProductFill(options) {
|
|
|
157
167
|
process.on("SIGINT", () => shutdown());
|
|
158
168
|
process.on("SIGTERM", () => shutdown());
|
|
159
169
|
|
|
160
|
-
// ──
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
);
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
)
|
|
191
|
-
|
|
192
|
-
if (!shuttingDown && code !== null && code !== 0) {
|
|
193
|
-
log(`[ProductFill] ⚠ 本地 webserver 已退出 (code=${code})`);
|
|
194
|
-
}
|
|
195
|
-
});
|
|
196
|
-
// 等待就绪(最多 60 秒)
|
|
197
|
-
const deadline = Date.now() + WEBSERVER_READY_TIMEOUT_MS;
|
|
198
|
-
while (!shuttingDown && Date.now() < deadline) {
|
|
199
|
-
if (await probeWebserverHealth(webserverUrl, 1000)) break;
|
|
200
|
-
await new Promise((r) => setTimeout(r, 1000));
|
|
201
|
-
}
|
|
202
|
-
if (await probeWebserverHealth(webserverUrl, 1000)) {
|
|
203
|
-
log(`[ProductFill] webserver 已就绪: ${webserverUrl}`);
|
|
170
|
+
// ── 浏览器连接管理:按出口国家自动切换端口(只换 CDP 连接,Edge 实例不杀) ──
|
|
171
|
+
let browser = null;
|
|
172
|
+
let page = null;
|
|
173
|
+
let currentPort = null;
|
|
174
|
+
const connectProfile = async (profile) => {
|
|
175
|
+
if (currentPort === profile.port && browser) return;
|
|
176
|
+
if (browser) {
|
|
177
|
+
await browser.close().catch(() => {});
|
|
178
|
+
browser = null;
|
|
179
|
+
page = null;
|
|
180
|
+
}
|
|
181
|
+
const cdpOptions = { port: profile.port };
|
|
182
|
+
if (productFillProxy) cdpOptions.proxyServer = productFillProxy;
|
|
183
|
+
log(
|
|
184
|
+
`[ProductFill] 切换浏览器端口 ${currentPort ?? "-"} → ${profile.port}(country=${profile.country}${productFillProxy ? `,代理 ${productFillProxy}` : ""})`,
|
|
185
|
+
);
|
|
186
|
+
browser = await ensureBrowserReady(cdpOptions);
|
|
187
|
+
page = await getOrCreatePage(browser);
|
|
188
|
+
currentPort = profile.port;
|
|
189
|
+
log(`[ProductFill] CDP 浏览器已就绪(当前页: ${page.url()})`);
|
|
190
|
+
};
|
|
191
|
+
|
|
192
|
+
// ── 代理出口国家探测 ──
|
|
193
|
+
const effectiveProxyUrl = productFillProxy || DEFAULT_PROXY;
|
|
194
|
+
let proxyCountry = null;
|
|
195
|
+
const refreshProxyCountry = async () => {
|
|
196
|
+
const prev = proxyCountry;
|
|
197
|
+
proxyCountry = await detectProxyCountry(effectiveProxyUrl);
|
|
198
|
+
// 只在国家变化时打印,避免每轮探测刷屏
|
|
199
|
+
if (proxyCountry !== prev) {
|
|
200
|
+
if (proxyCountry) {
|
|
201
|
+
log(`[ProductFill] 代理出口国家: ${proxyCountry}`);
|
|
204
202
|
} else {
|
|
205
|
-
log(
|
|
206
|
-
`[ProductFill] 警告: webserver 启动超时,继续运行(抓取失败将自动重试)`,
|
|
207
|
-
);
|
|
203
|
+
log(`[ProductFill] ⚠ 无法探测代理出口国家`);
|
|
208
204
|
}
|
|
209
205
|
}
|
|
210
|
-
}
|
|
206
|
+
};
|
|
211
207
|
|
|
212
208
|
/** 可被退出信号中断的等待(500ms 粒度检查 shuttingDown) */
|
|
213
209
|
const interruptibleSleep = (ms) =>
|
|
@@ -226,14 +222,124 @@ export async function handleProductFill(options) {
|
|
|
226
222
|
let totalSuccess = 0;
|
|
227
223
|
let totalFailed = 0;
|
|
228
224
|
let totalSkipped = 0;
|
|
225
|
+
let totalMarked = 0; // 标记不可售/放弃数
|
|
229
226
|
let captchaBackoffUntil = 0; // 验证码后冷却到的时间戳
|
|
227
|
+
let lastMismatchLogAt = 0; // 出口不一致日志节流
|
|
228
|
+
/** 连续失败计数:uniqueId → 次数,达到 FAIL_MARK_LIMIT 后哨兵放弃 */
|
|
229
|
+
const failCount = new Map();
|
|
230
|
+
|
|
231
|
+
/** 哨兵回写:product_rating = -1,任务永久移出候选(以后不再领取) */
|
|
232
|
+
const markUnavailable = async (uniqueId, reason) => {
|
|
233
|
+
try {
|
|
234
|
+
const post = await apiPost(
|
|
235
|
+
`${serverUrl}/api/product-fill/${encodeURIComponent(uniqueId)}`,
|
|
236
|
+
{ productDetail: { rating: SENTINEL_RATING } },
|
|
237
|
+
);
|
|
238
|
+
if (post && post.ok) {
|
|
239
|
+
totalMarked++;
|
|
240
|
+
failCount.delete(uniqueId);
|
|
241
|
+
log(` ⊘ @${uniqueId} 已标记不可售(${reason})`);
|
|
242
|
+
return true;
|
|
243
|
+
}
|
|
244
|
+
log(` ✗ @${uniqueId} 标记失败: ${post?.error || "未知"}`);
|
|
245
|
+
} catch (e) {
|
|
246
|
+
log(` ✗ @${uniqueId} 标记异常: ${e.message || "未知"}`);
|
|
247
|
+
}
|
|
248
|
+
return false;
|
|
249
|
+
};
|
|
250
|
+
|
|
251
|
+
/** 记录抓取失败次数,达到上限后哨兵放弃(防止死任务无限轮转) */
|
|
252
|
+
const noteFail = async (uniqueId) => {
|
|
253
|
+
const n = (failCount.get(uniqueId) || 0) + 1;
|
|
254
|
+
if (n < FAIL_MARK_LIMIT) {
|
|
255
|
+
failCount.set(uniqueId, n);
|
|
256
|
+
return;
|
|
257
|
+
}
|
|
258
|
+
failCount.delete(uniqueId);
|
|
259
|
+
await markUnavailable(uniqueId, `连续失败 ${n} 次`);
|
|
260
|
+
};
|
|
261
|
+
|
|
262
|
+
/** CDP 直抓:验证码在已登录浏览器中弹出时,等待人工解决后自动重试 */
|
|
263
|
+
const fetchViaCDP = async (seoUrl) => {
|
|
264
|
+
try {
|
|
265
|
+
return await getProductInfoCDP(page, seoUrl);
|
|
266
|
+
} catch (e) {
|
|
267
|
+
if (!e.message?.includes("Blocked by TikTok")) throw e;
|
|
268
|
+
// 验证码:优先检测可见容器并等待人工解决(浏览器就在本机屏幕上)
|
|
269
|
+
const cap = await detectCaptcha(page);
|
|
270
|
+
if (cap?.exists) {
|
|
271
|
+
log(" ⏳ 检测到验证码,请在已登录的浏览器窗口中手动解决...");
|
|
272
|
+
const r = await waitAndGetCaptcha(page, {
|
|
273
|
+
waitMs: CDP_CAPTCHA_WAIT_MS,
|
|
274
|
+
pollInterval: 5000,
|
|
275
|
+
log,
|
|
276
|
+
});
|
|
277
|
+
if (r.resolved) {
|
|
278
|
+
log(" ✓ 验证码已解决,重新抓取...");
|
|
279
|
+
return getProductInfoCDP(page, seoUrl);
|
|
280
|
+
}
|
|
281
|
+
}
|
|
282
|
+
const err = new Error(
|
|
283
|
+
cap?.exists
|
|
284
|
+
? "验证码等待人工解决超时"
|
|
285
|
+
: "验证码拦截(无可见验证码容器)",
|
|
286
|
+
);
|
|
287
|
+
err.captcha = true;
|
|
288
|
+
throw err;
|
|
289
|
+
}
|
|
290
|
+
};
|
|
291
|
+
|
|
292
|
+
/** 抓取商品详情,带浏览器断连恢复与空数据防御 */
|
|
293
|
+
const getTaskDetail = async (seoUrl) => {
|
|
294
|
+
let detail;
|
|
295
|
+
try {
|
|
296
|
+
detail = await fetchViaCDP(seoUrl);
|
|
297
|
+
} catch (e) {
|
|
298
|
+
if (!isBrowserClosedError(e)) throw e;
|
|
299
|
+
log(" ↻ 浏览器断连,重启浏览器后重试...");
|
|
300
|
+
try {
|
|
301
|
+
const cdpOptions = { port: currentPort };
|
|
302
|
+
if (productFillProxy) cdpOptions.proxyServer = productFillProxy;
|
|
303
|
+
browser = await relaunchBrowser(cdpOptions, currentPort);
|
|
304
|
+
page = await getOrCreatePage(browser);
|
|
305
|
+
} catch (relaunchErr) {
|
|
306
|
+
relaunchErr.retryable = true;
|
|
307
|
+
throw relaunchErr;
|
|
308
|
+
}
|
|
309
|
+
detail = await fetchViaCDP(seoUrl);
|
|
310
|
+
}
|
|
311
|
+
// 空数据防御:回写字段全空,不回写(防止假成功死循环)
|
|
312
|
+
if (isEmptyProductDetail(detail)) {
|
|
313
|
+
const err = new Error("商品详情数据为空(页面被拦截或结构变化)");
|
|
314
|
+
err.retryable = true;
|
|
315
|
+
err.empty = true;
|
|
316
|
+
throw err;
|
|
317
|
+
}
|
|
318
|
+
return detail;
|
|
319
|
+
};
|
|
230
320
|
|
|
231
321
|
try {
|
|
232
322
|
while (!shuttingDown) {
|
|
233
323
|
loopCount++;
|
|
234
324
|
if (shuttingDown) break;
|
|
235
325
|
|
|
236
|
-
//
|
|
326
|
+
// 每轮领取前探测出口:在配置中找匹配端口,找不到则不领任务,
|
|
327
|
+
// 下轮重新探测(代理切换生效有窗口期,等生效即可)
|
|
328
|
+
await refreshProxyCountry();
|
|
329
|
+
const profile = finalProfiles.find((p) => p.country === proxyCountry);
|
|
330
|
+
if (!profile) {
|
|
331
|
+
if (loopCount === 1 || Date.now() - lastMismatchLogAt > 60_000) {
|
|
332
|
+
log(
|
|
333
|
+
`[ProductFill] ⏳ 出口 ${proxyCountry || "未知"} 不在配置范围(${finalProfiles.map((p) => p.country).join("/")}),本轮跳过(等待代理切换)`,
|
|
334
|
+
);
|
|
335
|
+
lastMismatchLogAt = Date.now();
|
|
336
|
+
}
|
|
337
|
+
await interruptibleSleep(productFillInterval * 1000);
|
|
338
|
+
continue;
|
|
339
|
+
}
|
|
340
|
+
await connectProfile(profile);
|
|
341
|
+
|
|
342
|
+
// 验证码冷却期间不领任务
|
|
237
343
|
const now = Date.now();
|
|
238
344
|
if (now < captchaBackoffUntil) {
|
|
239
345
|
const waitMs = captchaBackoffUntil - now;
|
|
@@ -244,8 +350,14 @@ export async function handleProductFill(options) {
|
|
|
244
350
|
continue;
|
|
245
351
|
}
|
|
246
352
|
|
|
353
|
+
// 领任务:服务端按国家过滤,只发该国家任务
|
|
354
|
+
const taskQuery = new URLSearchParams({
|
|
355
|
+
limit: String(productFillLimit),
|
|
356
|
+
userId,
|
|
357
|
+
country: profile.country,
|
|
358
|
+
});
|
|
247
359
|
const { tasks = [] } = await apiGet(
|
|
248
|
-
`${serverUrl}/api/product-tasks
|
|
360
|
+
`${serverUrl}/api/product-tasks?${taskQuery.toString()}`,
|
|
249
361
|
);
|
|
250
362
|
|
|
251
363
|
if (!tasks || tasks.length === 0) {
|
|
@@ -263,19 +375,22 @@ export async function handleProductFill(options) {
|
|
|
263
375
|
let batchSuccess = 0;
|
|
264
376
|
let batchFailed = 0;
|
|
265
377
|
let batchSkipped = 0;
|
|
378
|
+
let batchMarked = 0;
|
|
266
379
|
|
|
267
380
|
for (const task of tasks) {
|
|
268
381
|
const uniqueId = task.uniqueId;
|
|
269
382
|
const seoUrl = task.seoUrl;
|
|
270
|
-
if (!seoUrl) {
|
|
271
|
-
|
|
383
|
+
if (!seoUrl || !String(seoUrl).startsWith("http")) {
|
|
384
|
+
// 残缺 URL(如仅 ?source=anchor):永远无效,直接哨兵放弃,
|
|
385
|
+
// 避免进入 page.goto 触发 Protocol error 被误判为浏览器断连
|
|
272
386
|
batchSkipped++;
|
|
273
387
|
totalSkipped++;
|
|
388
|
+
if (await markUnavailable(uniqueId, "seoUrl 无效")) batchMarked++;
|
|
274
389
|
continue;
|
|
275
390
|
}
|
|
276
391
|
log(` → 补商品 @${uniqueId} ...`);
|
|
277
392
|
try {
|
|
278
|
-
const detail = await
|
|
393
|
+
const detail = await getTaskDetail(seoUrl);
|
|
279
394
|
const post = await apiPost(
|
|
280
395
|
`${serverUrl}/api/product-fill/${encodeURIComponent(uniqueId)}`,
|
|
281
396
|
{ productDetail: detail },
|
|
@@ -283,6 +398,7 @@ export async function handleProductFill(options) {
|
|
|
283
398
|
if (post && post.ok) {
|
|
284
399
|
batchSuccess++;
|
|
285
400
|
totalSuccess++;
|
|
401
|
+
failCount.delete(uniqueId);
|
|
286
402
|
log(
|
|
287
403
|
` ✓ @${uniqueId} 商品已补全 (评分${detail.rating ?? "-"} | 销量${detail.soldCount ?? "-"} | 评论${detail.reviewCount ?? "-"})`,
|
|
288
404
|
);
|
|
@@ -290,6 +406,7 @@ export async function handleProductFill(options) {
|
|
|
290
406
|
batchFailed++;
|
|
291
407
|
totalFailed++;
|
|
292
408
|
log(` ✗ @${uniqueId} 回传失败: ${post?.error || "未知"}`);
|
|
409
|
+
await noteFail(uniqueId);
|
|
293
410
|
}
|
|
294
411
|
} catch (e) {
|
|
295
412
|
if (e.captcha) {
|
|
@@ -299,8 +416,21 @@ export async function handleProductFill(options) {
|
|
|
299
416
|
log(
|
|
300
417
|
` ⏳ @${uniqueId} 验证码拦截,冷却 ${CAPTCHA_BACKOFF_MS / 1000} 秒`,
|
|
301
418
|
);
|
|
419
|
+
} else if (e.delisted) {
|
|
420
|
+
// 壳页/下架:seller_id 为空且无销售数据 → 立即哨兵放弃
|
|
421
|
+
batchSkipped++;
|
|
422
|
+
totalSkipped++;
|
|
423
|
+
if (await markUnavailable(uniqueId, "商品下架/不可售")) {
|
|
424
|
+
batchMarked++;
|
|
425
|
+
}
|
|
426
|
+
} else if (e.empty) {
|
|
427
|
+
// 解析全空(回写字段全无):计入熔断,连续 N 次后哨兵放弃
|
|
428
|
+
batchFailed++;
|
|
429
|
+
totalFailed++;
|
|
430
|
+
log(` ✗ @${uniqueId} 商品详情数据为空: ${e.message}`);
|
|
431
|
+
await noteFail(uniqueId);
|
|
302
432
|
} else if (e.retryable) {
|
|
303
|
-
//
|
|
433
|
+
// 超时等可重试错误:未回写,下轮自然重试,不触发全局冷却
|
|
304
434
|
batchSkipped++;
|
|
305
435
|
totalSkipped++;
|
|
306
436
|
log(` ⏳ @${uniqueId} 临时失败(可重试): ${e.message}`);
|
|
@@ -308,12 +438,13 @@ export async function handleProductFill(options) {
|
|
|
308
438
|
batchFailed++;
|
|
309
439
|
totalFailed++;
|
|
310
440
|
log(` ✗ @${uniqueId} 商品详情失败: ${e.message || "未知错误"}`);
|
|
441
|
+
await noteFail(uniqueId);
|
|
311
442
|
}
|
|
312
443
|
}
|
|
313
444
|
}
|
|
314
445
|
|
|
315
446
|
log(
|
|
316
|
-
` 本批: 成功 ${batchSuccess} | 失败 ${batchFailed} | 跳过 ${batchSkipped}(累计: 成功 ${totalSuccess} | 失败 ${totalFailed} | 跳过 ${totalSkipped})\n`,
|
|
447
|
+
` 本批: 成功 ${batchSuccess} | 失败 ${batchFailed} | 跳过 ${batchSkipped} | 标记 ${batchMarked}(累计: 成功 ${totalSuccess} | 失败 ${totalFailed} | 跳过 ${totalSkipped} | 标记 ${totalMarked})\n`,
|
|
317
448
|
);
|
|
318
449
|
await interruptibleSleep(300);
|
|
319
450
|
}
|
|
@@ -322,12 +453,8 @@ export async function handleProductFill(options) {
|
|
|
322
453
|
throw err;
|
|
323
454
|
} finally {
|
|
324
455
|
if (forceExitTimer) clearTimeout(forceExitTimer);
|
|
325
|
-
if (localWebserverChild && !localWebserverChild.killed) {
|
|
326
|
-
log("[ProductFill] 关闭自动拉起的本地 webserver...");
|
|
327
|
-
localWebserverChild.kill("SIGTERM");
|
|
328
|
-
}
|
|
329
456
|
log(
|
|
330
|
-
`[ProductFill] 已退出。总计: 成功 ${totalSuccess} | 失败 ${totalFailed} | 跳过 ${totalSkipped}`,
|
|
457
|
+
`[ProductFill] 已退出。总计: 成功 ${totalSuccess} | 失败 ${totalFailed} | 跳过 ${totalSkipped} | 标记不可售 ${totalMarked}`,
|
|
331
458
|
);
|
|
332
459
|
}
|
|
333
460
|
}
|
package/src/cli/tag.js
CHANGED
|
@@ -9,7 +9,7 @@ import {
|
|
|
9
9
|
import { reportClientErrorAndFailover } from "../lib/client-error-report.js";
|
|
10
10
|
import { fetchTagData, enrichVideosWithLocation } from "../lib/tag-fetcher.js";
|
|
11
11
|
import { killEdgeProcesses, ensureBrowserReady } from "../lib/browser/cdp.js";
|
|
12
|
-
import { getOrCreatePage
|
|
12
|
+
import { getOrCreatePage } from "../lib/browser/page.js";
|
|
13
13
|
import { TikTokScraper } from "../lib/tiktok-scraper.mjs";
|
|
14
14
|
import { isLocationInList } from "../lib/target-locations.js";
|
|
15
15
|
import { getTargetLocations } from "../lib/app-config.js";
|
|
@@ -34,11 +34,17 @@ function resolveTargetCountries(countries) {
|
|
|
34
34
|
}
|
|
35
35
|
|
|
36
36
|
/**
|
|
37
|
-
*
|
|
37
|
+
* 判断浏览器是否真的已断开(用户手动关窗口 / CDP 断开)。
|
|
38
38
|
* 用于让采集循环(score-all / refresh)在浏览器被关闭后自动退出,避免无限重试。
|
|
39
|
+
*
|
|
40
|
+
* 注意:page.js 的 isBrowserClosedError 会把 net::ERR_CONNECTION_* 等网络层错误
|
|
41
|
+
* 也判为“浏览器关闭”。实际上这类错误是代理掂断/连接重置,浏览器进程还活着,
|
|
42
|
+
* 属于网络错误而不是窗口关闭。此处只认两种确凿信号:
|
|
43
|
+
* 1. browser.isConnected() === false(CDP 连接已断)
|
|
44
|
+
* 2. 明确的关闭消息(Target closed / Browser has been closed)
|
|
45
|
+
* net::ERR_* 一律放行,交给错误计数处理,避免误退出循环。
|
|
39
46
|
*/
|
|
40
47
|
function isBrowserDisconnected(browser, e) {
|
|
41
|
-
if (e && isBrowserClosedError(e)) return true;
|
|
42
48
|
try {
|
|
43
49
|
if (
|
|
44
50
|
browser &&
|
|
@@ -48,7 +54,16 @@ function isBrowserDisconnected(browser, e) {
|
|
|
48
54
|
return true;
|
|
49
55
|
}
|
|
50
56
|
} catch {}
|
|
51
|
-
return false;
|
|
57
|
+
if (!e) return false;
|
|
58
|
+
const msg = e.message || "";
|
|
59
|
+
// 网络层错误(net::ERR_CONNECTION_CLOSED/RESET/REFUSED...)不代表浏览器关闭
|
|
60
|
+
if (msg.includes("net::ERR_")) return false;
|
|
61
|
+
// 明确的 CDP 关闭信号
|
|
62
|
+
return (
|
|
63
|
+
msg.includes("Target closed") ||
|
|
64
|
+
msg.includes("Browser has been closed") ||
|
|
65
|
+
msg.includes("Target page, context or browser has been closed")
|
|
66
|
+
);
|
|
52
67
|
}
|
|
53
68
|
|
|
54
69
|
// 构建带客户端追踪 header 的 fetch 封装
|
|
@@ -851,10 +866,10 @@ export async function handleScoreAll(parsed) {
|
|
|
851
866
|
// ── 错误计数(纯观察,不做任何自愈) ──
|
|
852
867
|
// 阈值语义:连续超过 N 次触发动作(即第 N+1 次)
|
|
853
868
|
// 403 连续 >2 次 → 上报服务端 + 强制 fallback(daemon 换节点),清零后继续跑
|
|
854
|
-
// 网络错误 连续 >10 次 →
|
|
869
|
+
// 网络错误 连续 >10 次 → 触发 daemon fallback 换节点(本地 IPC,不依赖服务端可达)+ 退出
|
|
855
870
|
// 验证码 连续 >3 次(人工一直未解决)→ 打印退出
|
|
856
871
|
// 其他错误 连续 >10 次 → 打印退出
|
|
857
|
-
// 清零规则:成功处理一个 tag → 全部清零;fallback 触发 →
|
|
872
|
+
// 清零规则:成功处理一个 tag → 全部清零;fallback 触发 → 对应计数清零;
|
|
858
873
|
// 人工点掉验证码 → 验证码计数清零
|
|
859
874
|
const errorCounts = { http403: 0, network: 0, captcha: 0, other: 0 };
|
|
860
875
|
const ERROR_LIMITS = { http403: 2, network: 10, captcha: 3, other: 10 };
|
|
@@ -1207,16 +1222,35 @@ export async function handleScoreAll(parsed) {
|
|
|
1207
1222
|
(e.message &&
|
|
1208
1223
|
(e.message.includes("ECONNREFUSED") ||
|
|
1209
1224
|
e.message.includes("fetch failed") ||
|
|
1210
|
-
e.message.includes("network")
|
|
1225
|
+
e.message.includes("network") ||
|
|
1226
|
+
e.message.includes("net::ERR_")));
|
|
1211
1227
|
if (isNetworkError) {
|
|
1212
1228
|
const over = bumpErrorCount("network");
|
|
1213
1229
|
log(
|
|
1214
|
-
` ⚠️
|
|
1230
|
+
` ⚠️ 网络错误 (${e.message}),15 秒后重试(连续网络错误 ${errorCounts.network} 次)...`,
|
|
1215
1231
|
);
|
|
1216
1232
|
if (over) {
|
|
1217
1233
|
log(
|
|
1218
|
-
` ❌ 连续网络错误超过 ${ERROR_LIMITS.network} 次 →
|
|
1234
|
+
` ❌ 连续网络错误超过 ${ERROR_LIMITS.network} 次 → 触发 daemon fallback 换节点后退出`,
|
|
1219
1235
|
);
|
|
1236
|
+
// 网络错误多为代理节点问题:fallback 走本地 IPC 触发 daemon 换节点
|
|
1237
|
+
// (不依赖服务端可达),上报失败会被内部忽略;随后退出由看门狗拉起
|
|
1238
|
+
await reportClientErrorAndFailover({
|
|
1239
|
+
post: (url, body) =>
|
|
1240
|
+
fetch(url, {
|
|
1241
|
+
method: "POST",
|
|
1242
|
+
headers: buildClientHeaders(clientId, clientMeta, {
|
|
1243
|
+
"Content-Type": "application/json",
|
|
1244
|
+
}),
|
|
1245
|
+
body: JSON.stringify(body),
|
|
1246
|
+
}),
|
|
1247
|
+
baseUrl,
|
|
1248
|
+
userId,
|
|
1249
|
+
errorType: "network_failure",
|
|
1250
|
+
message: `连续 ${ERROR_LIMITS.network + 1} 次网络错误 (port=${cdpPort})`,
|
|
1251
|
+
stage: "tag-scoring",
|
|
1252
|
+
log,
|
|
1253
|
+
});
|
|
1220
1254
|
await cleanup();
|
|
1221
1255
|
return;
|
|
1222
1256
|
}
|