tt-help-cli-ycl 1.4.111 → 1.4.113
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/scripts/test-product-fill-db.mjs +64 -4
- package/src/cli/product-fill.js +158 -30
- package/src/lib/args.js +26 -0
- package/src/watch/data-store.js +47 -6
- package/src/watch/db-columns.js +1 -0
- package/src/watch/public/app.js +2 -0
- package/src/watch/server.js +8 -2
package/package.json
CHANGED
|
@@ -37,6 +37,21 @@ db.prepare(
|
|
|
37
37
|
`INSERT INTO jobs (unique_id, status, tt_seller, top_video_seo_url, product_name, updated_at)
|
|
38
38
|
VALUES ('sellerC', 'done', 1, NULL, NULL, ?)`,
|
|
39
39
|
).run(now);
|
|
40
|
+
// D: 有 seoUrl、name 空但 rating 有值 → 已抓到数据,不应被领取(防死循环)
|
|
41
|
+
db.prepare(
|
|
42
|
+
`INSERT INTO jobs (unique_id, status, tt_seller, top_video_seo_url, product_name, product_rating, updated_at)
|
|
43
|
+
VALUES ('sellerD', 'done', 1, 'https://www.tiktok.com/shop/pdp/d1', NULL, 4.5, ?)`,
|
|
44
|
+
).run(now);
|
|
45
|
+
// E: 有 seoUrl、name 空但 sold_count 有值 → 已抓到数据,不应被领取(防死循环)
|
|
46
|
+
db.prepare(
|
|
47
|
+
`INSERT INTO jobs (unique_id, status, tt_seller, top_video_seo_url, product_name, product_sold_count, updated_at)
|
|
48
|
+
VALUES ('sellerE', 'done', 1, 'https://www.tiktok.com/shop/pdp/e1', '', 120, ?)`,
|
|
49
|
+
).run(now);
|
|
50
|
+
// F: 有 seoUrl、无商品 → 与 A 一起验证 claim 防重复领取
|
|
51
|
+
db.prepare(
|
|
52
|
+
`INSERT INTO jobs (unique_id, status, tt_seller, top_video_seo_url, product_name, updated_at)
|
|
53
|
+
VALUES ('sellerF', 'done', 1, 'https://www.tiktok.com/shop/pdp/f1', NULL, ?)`,
|
|
54
|
+
).run(now + 1);
|
|
40
55
|
|
|
41
56
|
let passed = 0;
|
|
42
57
|
function check(name, fn) {
|
|
@@ -54,16 +69,61 @@ console.log("===== product-fill 服务端数据层测试 =====");
|
|
|
54
69
|
|
|
55
70
|
console.log("\n[1] getPendingProductTasks:只领缺商品的");
|
|
56
71
|
const tasks = store.getPendingProductTasks(5);
|
|
57
|
-
check("
|
|
72
|
+
check("只返回缺商品的(sellerA/sellerF,不含 B/C/D/E)", () => {
|
|
58
73
|
const ids = tasks.map((t) => t.uniqueId);
|
|
59
|
-
assert(ids.length ===
|
|
74
|
+
assert(ids.length === 2, `实际: ${ids.join(",")}`);
|
|
75
|
+
assert(
|
|
76
|
+
ids.includes("sellerA") && ids.includes("sellerF"),
|
|
77
|
+
`实际: ${ids.join(",")}`,
|
|
78
|
+
);
|
|
60
79
|
});
|
|
61
80
|
check("任务带 seoUrl / productId / productTitle", () => {
|
|
62
|
-
|
|
63
|
-
assert(
|
|
81
|
+
const a = tasks.find((t) => t.uniqueId === "sellerA");
|
|
82
|
+
assert(a.seoUrl === "https://www.tiktok.com/shop/pdp/a1");
|
|
83
|
+
assert("productId" in a && "productTitle" in a);
|
|
64
84
|
});
|
|
65
85
|
check("limit=1 只取 1 条", () => {
|
|
86
|
+
db.prepare(
|
|
87
|
+
"UPDATE jobs SET claimed_at = NULL, claimed_by = NULL WHERE unique_id IN ('sellerA','sellerF')",
|
|
88
|
+
).run();
|
|
66
89
|
assert(store.getPendingProductTasks(1).length === 1);
|
|
90
|
+
db.prepare(
|
|
91
|
+
"UPDATE jobs SET claimed_at = NULL, claimed_by = NULL WHERE unique_id IN ('sellerA','sellerF')",
|
|
92
|
+
).run();
|
|
93
|
+
});
|
|
94
|
+
check("领取即锁定:写入 claimed_by/claimed_at", () => {
|
|
95
|
+
const claimed = store.getPendingProductTasks(5, "testerA");
|
|
96
|
+
assert(claimed.length === 2, `实际: ${claimed.length}`);
|
|
97
|
+
const row = db
|
|
98
|
+
.prepare(
|
|
99
|
+
"SELECT claimed_by, claimed_at FROM jobs WHERE unique_id = 'sellerA'",
|
|
100
|
+
)
|
|
101
|
+
.get();
|
|
102
|
+
assert(row.claimed_by === "testerA", JSON.stringify(row));
|
|
103
|
+
assert(row.claimed_at !== null && row.claimed_at !== undefined);
|
|
104
|
+
});
|
|
105
|
+
check("锁定期间不重复领取(多客户端防重复)", () => {
|
|
106
|
+
const again = store.getPendingProductTasks(5, "testerB");
|
|
107
|
+
assert(
|
|
108
|
+
again.length === 0,
|
|
109
|
+
`实际领到: ${again.map((t) => t.uniqueId).join(",")}`,
|
|
110
|
+
);
|
|
111
|
+
});
|
|
112
|
+
check("TTL 过期后自动回收", () => {
|
|
113
|
+
db.prepare("UPDATE jobs SET claimed_at = ? WHERE unique_id = 'sellerF'").run(
|
|
114
|
+
Date.now() - 6 * 60 * 1000,
|
|
115
|
+
);
|
|
116
|
+
const recycled = store.getPendingProductTasks(5, "testerC");
|
|
117
|
+
assert(
|
|
118
|
+
recycled.some((t) => t.uniqueId === "sellerF"),
|
|
119
|
+
`实际: ${recycled.map((t) => t.uniqueId).join(",")}`,
|
|
120
|
+
);
|
|
121
|
+
});
|
|
122
|
+
check("name 空但 rating 有值 → 不进候选", () => {
|
|
123
|
+
assert(!tasks.some((t) => t.uniqueId === "sellerD"));
|
|
124
|
+
});
|
|
125
|
+
check("name 空但 soldCount 有值 → 不进候选", () => {
|
|
126
|
+
assert(!tasks.some((t) => t.uniqueId === "sellerE"));
|
|
67
127
|
});
|
|
68
128
|
|
|
69
129
|
console.log("\n[2] updateProductInfo:回写商品详情");
|
package/src/cli/product-fill.js
CHANGED
|
@@ -9,11 +9,26 @@
|
|
|
9
9
|
*
|
|
10
10
|
* 用法: tt-help product-fill [-s 服务端] [-w webserver] [-i 间隔秒] [-l 每批上限]
|
|
11
11
|
*/
|
|
12
|
+
import { spawn } from "node:child_process";
|
|
13
|
+
import { fileURLToPath } from "node:url";
|
|
12
14
|
import { getMacOrUuid } from "../lib/mac-or-uuid.js";
|
|
13
15
|
import { createApiClient } from "../lib/api-client.js";
|
|
14
16
|
import { userId as configuredUserId, saveUserId } from "../lib/constants.js";
|
|
15
17
|
|
|
16
18
|
const CAPTCHA_BACKOFF_MS = 30_000; // 验证码后冷却时长
|
|
19
|
+
const WEBSERVER_READY_TIMEOUT_MS = 60_000; // 等待自动拉起的 webserver 就绪上限
|
|
20
|
+
|
|
21
|
+
/** 探测 webserver /health 是否可用 */
|
|
22
|
+
async function probeWebserverHealth(url, timeoutMs = 1500) {
|
|
23
|
+
try {
|
|
24
|
+
const res = await fetch(`${url}/health`, {
|
|
25
|
+
signal: AbortSignal.timeout(timeoutMs),
|
|
26
|
+
});
|
|
27
|
+
return res.ok;
|
|
28
|
+
} catch {
|
|
29
|
+
return false;
|
|
30
|
+
}
|
|
31
|
+
}
|
|
17
32
|
|
|
18
33
|
function formatNow() {
|
|
19
34
|
return new Date().toLocaleString("zh-CN", { hour12: false });
|
|
@@ -31,16 +46,15 @@ async function fetchProductDetail(webserverUrl, seoUrl) {
|
|
|
31
46
|
body: JSON.stringify({ url: seoUrl }),
|
|
32
47
|
signal: AbortSignal.timeout(30_000),
|
|
33
48
|
});
|
|
34
|
-
if (res.
|
|
49
|
+
if (!res.ok) {
|
|
50
|
+
const body = await res.json().catch(() => ({}));
|
|
35
51
|
const err = new Error(
|
|
36
|
-
`
|
|
52
|
+
`${body.error || `HTTP ${res.status}`}${res.status === 503 ? " (captcha)" : ""}`,
|
|
37
53
|
);
|
|
38
|
-
err.retryable = true;
|
|
39
|
-
throw err;
|
|
40
|
-
}
|
|
41
|
-
if (!res.ok) {
|
|
42
|
-
const err = new Error(`HTTP ${res.status}`);
|
|
43
54
|
err.status = res.status;
|
|
55
|
+
// webserver 约定:503=验证码;504/502 响应体带 retryable=true 表示可重试
|
|
56
|
+
err.captcha = res.status === 503;
|
|
57
|
+
err.retryable = !!body.retryable || res.status === 503;
|
|
44
58
|
throw err;
|
|
45
59
|
}
|
|
46
60
|
return res.json();
|
|
@@ -52,9 +66,16 @@ export async function handleProductFill(options) {
|
|
|
52
66
|
productFillInterval,
|
|
53
67
|
serverUrl,
|
|
54
68
|
productFillWebserver,
|
|
69
|
+
productFillProxy,
|
|
55
70
|
showHelp,
|
|
56
71
|
} = options;
|
|
57
|
-
|
|
72
|
+
// 不指定 -w 时默认使用本地 webserver(http://127.0.0.1:3000),
|
|
73
|
+
// 且本地无响应时自动拉起(适合单独机器:验证码时弹浏览器人工处理)
|
|
74
|
+
const spawnLocalWebserver = !productFillWebserver;
|
|
75
|
+
const webserverUrl = productFillWebserver || "http://127.0.0.1:3000";
|
|
76
|
+
// 自动拉起 webserver 时透传的代理(默认本机 clash 7897)
|
|
77
|
+
const localWebserverProxy = productFillProxy || "http://127.0.0.1:7897";
|
|
78
|
+
let localWebserverChild = null;
|
|
58
79
|
|
|
59
80
|
if (showHelp) {
|
|
60
81
|
log(
|
|
@@ -64,7 +85,10 @@ export async function handleProductFill(options) {
|
|
|
64
85
|
log("参数:");
|
|
65
86
|
log(" -s, --server <URL> 服务端地址(默认: http://127.0.0.1:3001)");
|
|
66
87
|
log(
|
|
67
|
-
" -w, --webserver <URL> 商品详情 webserver
|
|
88
|
+
" -w, --webserver <URL> 商品详情 webserver 地址(默认: http://127.0.0.1:3000,本地未运行会自动拉起)",
|
|
89
|
+
);
|
|
90
|
+
log(
|
|
91
|
+
" --proxy <URL> 自动拉起 webserver 时使用的代理(默认: http://127.0.0.1:7897)",
|
|
68
92
|
);
|
|
69
93
|
log(" -i, --interval <N> 无任务时轮询间隔,单位秒(默认: 10)");
|
|
70
94
|
log(" -l, --limit <N> 每批领取任务数上限(默认: 5)");
|
|
@@ -74,6 +98,9 @@ export async function handleProductFill(options) {
|
|
|
74
98
|
log(" 对每个任务调用 webserver /api/tiktok/product 抓取评分/销量/评论;");
|
|
75
99
|
log(" 成功后回传 /api/product-fill/:uniqueId 写入 jobs 表。");
|
|
76
100
|
log(" 商品详情验证码(503)自动冷却 30 秒后重试,其他失败计数后继续。");
|
|
101
|
+
log(
|
|
102
|
+
" 不传 -w 时本地无 webserver 会自动拉起(验证码时弹浏览器,人工解决后继续)。",
|
|
103
|
+
);
|
|
77
104
|
log("");
|
|
78
105
|
log("示例:");
|
|
79
106
|
log(
|
|
@@ -83,13 +110,6 @@ export async function handleProductFill(options) {
|
|
|
83
110
|
return;
|
|
84
111
|
}
|
|
85
112
|
|
|
86
|
-
if (!webserverUrl) {
|
|
87
|
-
log(
|
|
88
|
-
"错误: 缺少 webserver 地址,请用 -w/--webserver 指定(如 http://127.0.0.1:3000)",
|
|
89
|
-
);
|
|
90
|
-
return;
|
|
91
|
-
}
|
|
92
|
-
|
|
93
113
|
// ── 设备标识 ──
|
|
94
114
|
let userId = configuredUserId;
|
|
95
115
|
if (!userId) {
|
|
@@ -107,14 +127,101 @@ export async function handleProductFill(options) {
|
|
|
107
127
|
const { apiGet, apiPost } = createApiClient({ log: false });
|
|
108
128
|
|
|
109
129
|
let shuttingDown = false;
|
|
130
|
+
let forceExitTimer = null;
|
|
131
|
+
/** 强制退出路径清理自动拉起的 webserver(防止孤儿进程) */
|
|
132
|
+
const killLocalWebserver = () => {
|
|
133
|
+
if (localWebserverChild && !localWebserverChild.killed) {
|
|
134
|
+
localWebserverChild.kill("SIGKILL");
|
|
135
|
+
}
|
|
136
|
+
};
|
|
110
137
|
const shutdown = () => {
|
|
111
|
-
if (shuttingDown)
|
|
138
|
+
if (shuttingDown) {
|
|
139
|
+
// 第二次信号:立即强制退出(网络重试等场景可能无法优雅退出)
|
|
140
|
+
log(`\n[ProductFill] 再次收到信号,强制退出...`);
|
|
141
|
+
killLocalWebserver();
|
|
142
|
+
process.exit(1);
|
|
143
|
+
return;
|
|
144
|
+
}
|
|
112
145
|
shuttingDown = true;
|
|
113
|
-
log(
|
|
146
|
+
log(
|
|
147
|
+
`\n[ProductFill] 收到退出信号,完成当前批次后退出(再次按 Ctrl+C 强制退出)...`,
|
|
148
|
+
);
|
|
149
|
+
// 兜底:卡在网络重试/等待中时 10 秒后强制退出
|
|
150
|
+
forceExitTimer = setTimeout(() => {
|
|
151
|
+
log("[ProductFill] 等待超时,强制退出");
|
|
152
|
+
killLocalWebserver();
|
|
153
|
+
process.exit(0);
|
|
154
|
+
}, 10_000);
|
|
155
|
+
forceExitTimer.unref?.();
|
|
114
156
|
};
|
|
115
157
|
process.on("SIGINT", () => shutdown());
|
|
116
158
|
process.on("SIGTERM", () => shutdown());
|
|
117
159
|
|
|
160
|
+
// ── 确保 webserver 可用:未指定 -w 且本地无响应时自动拉起 ──
|
|
161
|
+
if (await probeWebserverHealth(webserverUrl)) {
|
|
162
|
+
log(`[ProductFill] webserver 已就绪: ${webserverUrl}`);
|
|
163
|
+
} else {
|
|
164
|
+
if (!spawnLocalWebserver) {
|
|
165
|
+
log(
|
|
166
|
+
`[ProductFill] 警告: webserver ${webserverUrl} 无响应,任务将按失败重试`,
|
|
167
|
+
);
|
|
168
|
+
} else {
|
|
169
|
+
const portMatch = webserverUrl.match(/:(\d+)\/?$/);
|
|
170
|
+
const port = portMatch ? parseInt(portMatch[1], 10) : 3000;
|
|
171
|
+
// cli.js 在包根(源码与 npm 包结构一致),webserver 命令两者都支持
|
|
172
|
+
const cliJs = fileURLToPath(new URL("../../cli.js", import.meta.url));
|
|
173
|
+
log(
|
|
174
|
+
`[ProductFill] 本地 webserver 未运行,自动拉起(端口 ${port},代理 ${localWebserverProxy})...`,
|
|
175
|
+
);
|
|
176
|
+
localWebserverChild = spawn(
|
|
177
|
+
process.execPath,
|
|
178
|
+
[
|
|
179
|
+
cliJs,
|
|
180
|
+
"webserver",
|
|
181
|
+
"-p",
|
|
182
|
+
String(port),
|
|
183
|
+
"--proxy",
|
|
184
|
+
localWebserverProxy,
|
|
185
|
+
],
|
|
186
|
+
{
|
|
187
|
+
stdio: ["ignore", "ignore", "inherit"], // webserver 日志(含验证码提示)透传到本终端
|
|
188
|
+
windowsHide: true,
|
|
189
|
+
},
|
|
190
|
+
);
|
|
191
|
+
localWebserverChild.on("exit", (code) => {
|
|
192
|
+
if (!shuttingDown && code !== null && code !== 0) {
|
|
193
|
+
log(`[ProductFill] ⚠ 本地 webserver 已退出 (code=${code})`);
|
|
194
|
+
}
|
|
195
|
+
});
|
|
196
|
+
// 等待就绪(最多 60 秒)
|
|
197
|
+
const deadline = Date.now() + WEBSERVER_READY_TIMEOUT_MS;
|
|
198
|
+
while (!shuttingDown && Date.now() < deadline) {
|
|
199
|
+
if (await probeWebserverHealth(webserverUrl, 1000)) break;
|
|
200
|
+
await new Promise((r) => setTimeout(r, 1000));
|
|
201
|
+
}
|
|
202
|
+
if (await probeWebserverHealth(webserverUrl, 1000)) {
|
|
203
|
+
log(`[ProductFill] webserver 已就绪: ${webserverUrl}`);
|
|
204
|
+
} else {
|
|
205
|
+
log(
|
|
206
|
+
`[ProductFill] 警告: webserver 启动超时,继续运行(抓取失败将自动重试)`,
|
|
207
|
+
);
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
/** 可被退出信号中断的等待(500ms 粒度检查 shuttingDown) */
|
|
213
|
+
const interruptibleSleep = (ms) =>
|
|
214
|
+
new Promise((resolve) => {
|
|
215
|
+
const start = Date.now();
|
|
216
|
+
const tick = () => {
|
|
217
|
+
if (shuttingDown) return resolve();
|
|
218
|
+
const left = ms - (Date.now() - start);
|
|
219
|
+
if (left <= 0) return resolve();
|
|
220
|
+
setTimeout(tick, Math.min(500, left));
|
|
221
|
+
};
|
|
222
|
+
tick();
|
|
223
|
+
});
|
|
224
|
+
|
|
118
225
|
let loopCount = 0;
|
|
119
226
|
let totalSuccess = 0;
|
|
120
227
|
let totalFailed = 0;
|
|
@@ -126,6 +233,17 @@ export async function handleProductFill(options) {
|
|
|
126
233
|
loopCount++;
|
|
127
234
|
if (shuttingDown) break;
|
|
128
235
|
|
|
236
|
+
// 验证码冷却期间不领任务(避免反复触发 webserver 限流)
|
|
237
|
+
const now = Date.now();
|
|
238
|
+
if (now < captchaBackoffUntil) {
|
|
239
|
+
const waitMs = captchaBackoffUntil - now;
|
|
240
|
+
log(
|
|
241
|
+
`[ProductFill] ⏳ 验证码冷却中(剩余 ${Math.ceil(waitMs / 1000)}s),暂不领取任务...`,
|
|
242
|
+
);
|
|
243
|
+
await interruptibleSleep(waitMs);
|
|
244
|
+
continue;
|
|
245
|
+
}
|
|
246
|
+
|
|
129
247
|
const { tasks = [] } = await apiGet(
|
|
130
248
|
`${serverUrl}/api/product-tasks?limit=${productFillLimit}&userId=${encodeURIComponent(userId)}`,
|
|
131
249
|
);
|
|
@@ -136,26 +254,22 @@ export async function handleProductFill(options) {
|
|
|
136
254
|
`[ProductFill] 当前无缺商品任务,${productFillInterval}秒后重试...`,
|
|
137
255
|
);
|
|
138
256
|
}
|
|
139
|
-
await
|
|
257
|
+
await interruptibleSleep(productFillInterval * 1000);
|
|
140
258
|
continue;
|
|
141
259
|
}
|
|
142
260
|
|
|
143
261
|
log(`\n[ProductFill] 获取到 ${tasks.length} 个补商品任务...`);
|
|
144
262
|
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
const waitSec = Math.ceil((captchaBackoffUntil - now) / 1000);
|
|
149
|
-
log(` ⏳ 验证码冷却中(剩余 ${waitSec}s),本批跳过`);
|
|
150
|
-
await new Promise((r) => setTimeout(r, captchaBackoffUntil - now));
|
|
151
|
-
continue;
|
|
152
|
-
}
|
|
263
|
+
let batchSuccess = 0;
|
|
264
|
+
let batchFailed = 0;
|
|
265
|
+
let batchSkipped = 0;
|
|
153
266
|
|
|
154
267
|
for (const task of tasks) {
|
|
155
268
|
const uniqueId = task.uniqueId;
|
|
156
269
|
const seoUrl = task.seoUrl;
|
|
157
270
|
if (!seoUrl) {
|
|
158
271
|
log(` ✗ @${uniqueId} 无 seoUrl,跳过`);
|
|
272
|
+
batchSkipped++;
|
|
159
273
|
totalSkipped++;
|
|
160
274
|
continue;
|
|
161
275
|
}
|
|
@@ -167,22 +281,31 @@ export async function handleProductFill(options) {
|
|
|
167
281
|
{ productDetail: detail },
|
|
168
282
|
);
|
|
169
283
|
if (post && post.ok) {
|
|
284
|
+
batchSuccess++;
|
|
170
285
|
totalSuccess++;
|
|
171
286
|
log(
|
|
172
287
|
` ✓ @${uniqueId} 商品已补全 (评分${detail.rating ?? "-"} | 销量${detail.soldCount ?? "-"} | 评论${detail.reviewCount ?? "-"})`,
|
|
173
288
|
);
|
|
174
289
|
} else {
|
|
290
|
+
batchFailed++;
|
|
175
291
|
totalFailed++;
|
|
176
292
|
log(` ✗ @${uniqueId} 回传失败: ${post?.error || "未知"}`);
|
|
177
293
|
}
|
|
178
294
|
} catch (e) {
|
|
179
|
-
if (e.
|
|
295
|
+
if (e.captcha) {
|
|
296
|
+
batchSkipped++;
|
|
180
297
|
totalSkipped++;
|
|
181
298
|
captchaBackoffUntil = Date.now() + CAPTCHA_BACKOFF_MS;
|
|
182
299
|
log(
|
|
183
300
|
` ⏳ @${uniqueId} 验证码拦截,冷却 ${CAPTCHA_BACKOFF_MS / 1000} 秒`,
|
|
184
301
|
);
|
|
302
|
+
} else if (e.retryable) {
|
|
303
|
+
// 504 超时等可重试错误:未回写,下轮自然重试,不触发全局冷却
|
|
304
|
+
batchSkipped++;
|
|
305
|
+
totalSkipped++;
|
|
306
|
+
log(` ⏳ @${uniqueId} 临时失败(可重试): ${e.message}`);
|
|
185
307
|
} else {
|
|
308
|
+
batchFailed++;
|
|
186
309
|
totalFailed++;
|
|
187
310
|
log(` ✗ @${uniqueId} 商品详情失败: ${e.message || "未知错误"}`);
|
|
188
311
|
}
|
|
@@ -190,14 +313,19 @@ export async function handleProductFill(options) {
|
|
|
190
313
|
}
|
|
191
314
|
|
|
192
315
|
log(
|
|
193
|
-
`
|
|
316
|
+
` 本批: 成功 ${batchSuccess} | 失败 ${batchFailed} | 跳过 ${batchSkipped}(累计: 成功 ${totalSuccess} | 失败 ${totalFailed} | 跳过 ${totalSkipped})\n`,
|
|
194
317
|
);
|
|
195
|
-
await
|
|
318
|
+
await interruptibleSleep(300);
|
|
196
319
|
}
|
|
197
320
|
} catch (err) {
|
|
198
321
|
log(`[ProductFill] 运行异常: ${err.message}`);
|
|
199
322
|
throw err;
|
|
200
323
|
} finally {
|
|
324
|
+
if (forceExitTimer) clearTimeout(forceExitTimer);
|
|
325
|
+
if (localWebserverChild && !localWebserverChild.killed) {
|
|
326
|
+
log("[ProductFill] 关闭自动拉起的本地 webserver...");
|
|
327
|
+
localWebserverChild.kill("SIGTERM");
|
|
328
|
+
}
|
|
201
329
|
log(
|
|
202
330
|
`[ProductFill] 已退出。总计: 成功 ${totalSuccess} | 失败 ${totalFailed} | 跳过 ${totalSkipped}`,
|
|
203
331
|
);
|
package/src/lib/args.js
CHANGED
|
@@ -766,6 +766,7 @@ function parseProductFillArgs(args) {
|
|
|
766
766
|
let interval = 10;
|
|
767
767
|
let serverUrl = defaultServer;
|
|
768
768
|
let webserverUrl = null;
|
|
769
|
+
let proxyUrl = null;
|
|
769
770
|
|
|
770
771
|
for (let i = 0; i < args.length; i++) {
|
|
771
772
|
const arg = args[i];
|
|
@@ -773,6 +774,8 @@ function parseProductFillArgs(args) {
|
|
|
773
774
|
serverUrl = args[++i];
|
|
774
775
|
} else if (arg === "-w" || arg === "--webserver") {
|
|
775
776
|
webserverUrl = args[++i];
|
|
777
|
+
} else if (arg === "--proxy") {
|
|
778
|
+
proxyUrl = args[++i] || null;
|
|
776
779
|
} else if (arg === "-i" || arg === "--interval") {
|
|
777
780
|
interval = parseInt(args[++i], 10) || 10;
|
|
778
781
|
} else if (arg === "-l" || arg === "--limit") {
|
|
@@ -786,6 +789,7 @@ function parseProductFillArgs(args) {
|
|
|
786
789
|
productFillInterval: interval,
|
|
787
790
|
serverUrl,
|
|
788
791
|
productFillWebserver: webserverUrl,
|
|
792
|
+
productFillProxy: proxyUrl,
|
|
789
793
|
urls: [],
|
|
790
794
|
outputFormat: "json",
|
|
791
795
|
exploreCount: 0,
|
|
@@ -1368,6 +1372,28 @@ export function parseArgs() {
|
|
|
1368
1372
|
const args = process.argv.slice(2);
|
|
1369
1373
|
|
|
1370
1374
|
if (args.includes("-h") || args.includes("--help")) {
|
|
1375
|
+
// 子命令级帮助:product-fill -h 显示专属帮助,而非全局 HELP_TEXT
|
|
1376
|
+
if (args[0] === "product-fill") {
|
|
1377
|
+
return {
|
|
1378
|
+
subcommand: "product-fill",
|
|
1379
|
+
productFillLimit: 5,
|
|
1380
|
+
productFillInterval: 10,
|
|
1381
|
+
serverUrl: defaultServer,
|
|
1382
|
+
productFillWebserver: null,
|
|
1383
|
+
showHelp: true,
|
|
1384
|
+
showVersion: false,
|
|
1385
|
+
urls: [],
|
|
1386
|
+
outputFile: null,
|
|
1387
|
+
outputFormat: "json",
|
|
1388
|
+
exploreCount: 0,
|
|
1389
|
+
showConfig: false,
|
|
1390
|
+
customProxy: null,
|
|
1391
|
+
configAction: null,
|
|
1392
|
+
configValue: null,
|
|
1393
|
+
pipeMode: false,
|
|
1394
|
+
filterStr: null,
|
|
1395
|
+
};
|
|
1396
|
+
}
|
|
1371
1397
|
return {
|
|
1372
1398
|
showHelp: true,
|
|
1373
1399
|
showVersion: false,
|
package/src/watch/data-store.js
CHANGED
|
@@ -2029,7 +2029,14 @@ export function createStore(filePath, options = {}) {
|
|
|
2029
2029
|
const existing = getJob(uniqueId);
|
|
2030
2030
|
if (existing) {
|
|
2031
2031
|
updateUserFromResult(existing, result);
|
|
2032
|
-
|
|
2032
|
+
// 用户在哪个表就更新哪个表(task_queue 热表 / jobs 归档),
|
|
2033
|
+
// 避免固定写 jobs 导致的假成功(UPDATE 0 行仍返回 ok)
|
|
2034
|
+
const persistRet = updateJobInfo(
|
|
2035
|
+
uniqueId,
|
|
2036
|
+
existing,
|
|
2037
|
+
false,
|
|
2038
|
+
getUserTable(uniqueId) || "jobs",
|
|
2039
|
+
);
|
|
2033
2040
|
if (persistRet.error) {
|
|
2034
2041
|
return { saved: false, error: persistRet.error };
|
|
2035
2042
|
}
|
|
@@ -2049,6 +2056,13 @@ export function createStore(filePath, options = {}) {
|
|
|
2049
2056
|
};
|
|
2050
2057
|
updateUserFromResult(userObj, result);
|
|
2051
2058
|
addJob(userObj);
|
|
2059
|
+
// addJobToDb 的 INSERT 列不含商品列(top_video_* / product_*),
|
|
2060
|
+
// 新用户行需显式补写 task_queue,否则探索发现的商品线索丢失,
|
|
2061
|
+
// 且与 product-fill 领取/回写的位置(jobs 归档表)断链
|
|
2062
|
+
const persistRet = updateJobInfo(uniqueId, userObj, false, "task_queue");
|
|
2063
|
+
if (persistRet.error) {
|
|
2064
|
+
return { saved: false, error: persistRet.error };
|
|
2065
|
+
}
|
|
2052
2066
|
const newUsers = processDiscoveredUsers(result);
|
|
2053
2067
|
return { saved: true, created: true, status: userObj.status, newUsers };
|
|
2054
2068
|
}
|
|
@@ -2601,14 +2615,28 @@ export function createStore(filePath, options = {}) {
|
|
|
2601
2615
|
// ===== product-fill:给缺商品详情的商家补商品信息(jobs 表) =====
|
|
2602
2616
|
|
|
2603
2617
|
/**
|
|
2604
|
-
* 领取缺商品详情任务:jobs 表中有 top_video_seo_url
|
|
2605
|
-
*
|
|
2618
|
+
* 领取缺商品详情任务:jobs 表中有 top_video_seo_url 但未补过商品信息
|
|
2619
|
+
* 完成判定为「名称/评分/销量三者都空」:商品页可能解析不出 name,
|
|
2620
|
+
* 若只以 product_name 为空作候选条件,这类任务会永远留在候选池重复抓取
|
|
2621
|
+
*
|
|
2622
|
+
* 多客户端防重复领取(claim 机制):
|
|
2623
|
+
* - 领取时写入 claimed_at/claimed_by,锁定期间不再返回同一任务
|
|
2624
|
+
* - 回写成功(product_* 有值)后自然移出候选;回写失败等 TTL 过期重新可领
|
|
2625
|
+
* - 客户端崩溃/断连 → claimed_at 超过 TTL 自动回收(与 claimNextJob 的 5 分钟一致)
|
|
2606
2626
|
* @param {number|string} limit
|
|
2627
|
+
* @param {string|null} claimBy 领取者标识(userId),仅作标记/日志
|
|
2628
|
+
* @param {number} claimTtlMs 锁定超时(默认 5 分钟)
|
|
2607
2629
|
* @returns {{uniqueId:string, nickname:string|null, seoUrl:string, productId:string|null, productTitle:string|null}[]}
|
|
2608
2630
|
*/
|
|
2609
|
-
function getPendingProductTasks(
|
|
2631
|
+
function getPendingProductTasks(
|
|
2632
|
+
limit,
|
|
2633
|
+
claimBy = null,
|
|
2634
|
+
claimTtlMs = 5 * 60 * 1000,
|
|
2635
|
+
) {
|
|
2610
2636
|
if (getDb()) {
|
|
2611
2637
|
const l = Math.max(1, parseInt(limit) || 5);
|
|
2638
|
+
const now = Date.now();
|
|
2639
|
+
const claimExpireBefore = now - claimTtlMs;
|
|
2612
2640
|
const rows = getDb()
|
|
2613
2641
|
.prepare(
|
|
2614
2642
|
`
|
|
@@ -2617,11 +2645,24 @@ export function createStore(filePath, options = {}) {
|
|
|
2617
2645
|
FROM jobs
|
|
2618
2646
|
WHERE top_video_seo_url IS NOT NULL AND top_video_seo_url != ''
|
|
2619
2647
|
AND (product_name IS NULL OR product_name = '')
|
|
2620
|
-
|
|
2648
|
+
AND product_rating IS NULL
|
|
2649
|
+
AND product_sold_count IS NULL
|
|
2650
|
+
AND (claimed_at IS NULL OR claimed_at < ?)
|
|
2651
|
+
ORDER BY COALESCE(claimed_at, 0) ASC, updated_at DESC
|
|
2621
2652
|
LIMIT ?
|
|
2622
2653
|
`,
|
|
2623
2654
|
)
|
|
2624
|
-
.all(l);
|
|
2655
|
+
.all(claimExpireBefore, l);
|
|
2656
|
+
if (rows.length > 0) {
|
|
2657
|
+
// 领取即锁定:写入 claimed_at/claimed_by,避免多客户端重复抓取同一商品
|
|
2658
|
+
const ids = rows.map((r) => r.unique_id);
|
|
2659
|
+
const ph = ids.map(() => "?").join(", ");
|
|
2660
|
+
getDb()
|
|
2661
|
+
.prepare(
|
|
2662
|
+
`UPDATE jobs SET claimed_at = ?, claimed_by = ? WHERE unique_id IN (${ph})`,
|
|
2663
|
+
)
|
|
2664
|
+
.run(now, claimBy || null, ...ids);
|
|
2665
|
+
}
|
|
2625
2666
|
return rows.map((row) => ({
|
|
2626
2667
|
uniqueId: row.unique_id,
|
|
2627
2668
|
nickname: row.nickname || null,
|
package/src/watch/db-columns.js
CHANGED
package/src/watch/public/app.js
CHANGED
|
@@ -296,6 +296,7 @@ function renderActiveClients(clients) {
|
|
|
296
296
|
"attach",
|
|
297
297
|
"recheck",
|
|
298
298
|
"comments",
|
|
299
|
+
"product-fill",
|
|
299
300
|
"scoring",
|
|
300
301
|
"tag-refresh",
|
|
301
302
|
"tag-search",
|
|
@@ -307,6 +308,7 @@ function renderActiveClients(clients) {
|
|
|
307
308
|
attach: "Attach",
|
|
308
309
|
recheck: "Recheck",
|
|
309
310
|
comments: "Comments",
|
|
311
|
+
"product-fill": "ProductFill",
|
|
310
312
|
scoring: "Scoring",
|
|
311
313
|
"tag-refresh": "TagRefresh",
|
|
312
314
|
"tag-search": "Search",
|
package/src/watch/server.js
CHANGED
|
@@ -957,9 +957,15 @@ export function startWatchServer(
|
|
|
957
957
|
// ── product-fill 端点:补商品信息(缺商品详情的商家) ──
|
|
958
958
|
if (req.method === "GET" && routePath === "/api/product-tasks") {
|
|
959
959
|
const limit = params.limit;
|
|
960
|
-
const
|
|
960
|
+
const clientId = params.userId || "?";
|
|
961
|
+
const tasks = store.getPendingProductTasks(
|
|
962
|
+
limit,
|
|
963
|
+
params.userId || null,
|
|
964
|
+
);
|
|
961
965
|
const ts = new Date().toLocaleTimeString("zh-CN", { hour12: false });
|
|
962
|
-
console.error(
|
|
966
|
+
console.error(
|
|
967
|
+
`[JOB ${ts}] PRODUCT-TASKS: ${tasks.length} tasks (client=${clientId})`,
|
|
968
|
+
);
|
|
963
969
|
sendJSON(res, 200, { total: tasks.length, tasks });
|
|
964
970
|
return;
|
|
965
971
|
}
|