tt-help-cli-ycl 1.4.98 → 1.4.99
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/scripts/test-product-fill-db.mjs +113 -0
- package/scripts/test-product-fill-http.mjs +117 -0
- package/src/cli/product-fill.js +205 -0
- package/src/lib/args.js +42 -0
- package/src/lib/constants.js +9 -0
- package/src/main.js +3 -0
- package/src/npm-main.js +3 -0
- package/src/watch/data-store.js +65 -0
- package/src/watch/db-schema.js +15 -1
- package/src/watch/server.js +36 -0
- package/src/watchdog/upgrader.js +31 -4
package/package.json
CHANGED
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* product-fill 服务端数据层集成测试
|
|
3
|
+
*
|
|
4
|
+
* 用临时 SQLite 库验证:
|
|
5
|
+
* - getPendingProductTasks:只领取"有 seoUrl 且缺商品详情"的商家,已补全/无 seoUrl 的不领
|
|
6
|
+
* - updateProductInfo:回写 product_* 字段到 jobs 表,成功后移出候选
|
|
7
|
+
*
|
|
8
|
+
* 运行: TMPDIR=$(mktemp -d) node scripts/test-product-fill-db.mjs
|
|
9
|
+
*/
|
|
10
|
+
import { createStore } from "../src/watch/data-store.js";
|
|
11
|
+
import { getDb } from "../src/watch/db-schema.js";
|
|
12
|
+
import { join } from "node:path";
|
|
13
|
+
|
|
14
|
+
const tmp = process.env.TMPDIR || "/tmp";
|
|
15
|
+
const dbPath = join(tmp, `tt-test-product-${Date.now()}.db`);
|
|
16
|
+
|
|
17
|
+
const store = createStore(dbPath);
|
|
18
|
+
const db = getDb();
|
|
19
|
+
if (!db) {
|
|
20
|
+
console.error("❌ db 未初始化");
|
|
21
|
+
process.exit(1);
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
const now = Date.now();
|
|
25
|
+
// A: 有 seoUrl、无商品 → 应被领取
|
|
26
|
+
db.prepare(
|
|
27
|
+
`INSERT INTO jobs (unique_id, status, tt_seller, top_video_seo_url, product_name, updated_at)
|
|
28
|
+
VALUES ('sellerA', 'done', 1, 'https://www.tiktok.com/shop/pdp/a1', NULL, ?)`,
|
|
29
|
+
).run(now);
|
|
30
|
+
// B: 有 seoUrl、已有商品 → 不应被领取
|
|
31
|
+
db.prepare(
|
|
32
|
+
`INSERT INTO jobs (unique_id, status, tt_seller, top_video_seo_url, product_name, updated_at)
|
|
33
|
+
VALUES ('sellerB', 'done', 1, 'https://www.tiktok.com/shop/pdp/b1', '已有商品', ?)`,
|
|
34
|
+
).run(now);
|
|
35
|
+
// C: 无 seoUrl → 不应被领取
|
|
36
|
+
db.prepare(
|
|
37
|
+
`INSERT INTO jobs (unique_id, status, tt_seller, top_video_seo_url, product_name, updated_at)
|
|
38
|
+
VALUES ('sellerC', 'done', 1, NULL, NULL, ?)`,
|
|
39
|
+
).run(now);
|
|
40
|
+
|
|
41
|
+
let passed = 0;
|
|
42
|
+
function check(name, fn) {
|
|
43
|
+
try {
|
|
44
|
+
fn();
|
|
45
|
+
passed++;
|
|
46
|
+
console.log(` ✅ ${name}`);
|
|
47
|
+
} catch (e) {
|
|
48
|
+
console.error(` ❌ ${name}\n ${e.message}`);
|
|
49
|
+
process.exitCode = 1;
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
console.log("===== product-fill 服务端数据层测试 =====");
|
|
54
|
+
|
|
55
|
+
console.log("\n[1] getPendingProductTasks:只领缺商品的");
|
|
56
|
+
const tasks = store.getPendingProductTasks(5);
|
|
57
|
+
check("返回 1 条(仅 sellerA)", () => {
|
|
58
|
+
const ids = tasks.map((t) => t.uniqueId);
|
|
59
|
+
assert(ids.length === 1 && ids[0] === "sellerA", `实际: ${ids.join(",")}`);
|
|
60
|
+
});
|
|
61
|
+
check("任务带 seoUrl / productId / productTitle", () => {
|
|
62
|
+
assert(tasks[0].seoUrl === "https://www.tiktok.com/shop/pdp/a1");
|
|
63
|
+
assert("productId" in tasks[0] && "productTitle" in tasks[0]);
|
|
64
|
+
});
|
|
65
|
+
check("limit=1 只取 1 条", () => {
|
|
66
|
+
assert(store.getPendingProductTasks(1).length === 1);
|
|
67
|
+
});
|
|
68
|
+
|
|
69
|
+
console.log("\n[2] updateProductInfo:回写商品详情");
|
|
70
|
+
const ret = store.updateProductInfo("sellerA", {
|
|
71
|
+
rating: 4.7,
|
|
72
|
+
reviewCount: 3303,
|
|
73
|
+
soldCount: 296,
|
|
74
|
+
productName: "测试商品A",
|
|
75
|
+
});
|
|
76
|
+
check("返回 ok", () => assert(ret.ok === true, JSON.stringify(ret)));
|
|
77
|
+
const rowA = db
|
|
78
|
+
.prepare("SELECT * FROM jobs WHERE unique_id = ?")
|
|
79
|
+
.get("sellerA");
|
|
80
|
+
check("jobs 表已写入 product_* 字段", () => {
|
|
81
|
+
assert(rowA.product_rating === 4.7);
|
|
82
|
+
assert(rowA.product_review_count === 3303);
|
|
83
|
+
assert(rowA.product_sold_count === 296);
|
|
84
|
+
assert(rowA.product_name === "测试商品A");
|
|
85
|
+
});
|
|
86
|
+
check("未增加 user_update_count", () => {
|
|
87
|
+
assert((rowA.user_update_count || 0) === 0);
|
|
88
|
+
});
|
|
89
|
+
|
|
90
|
+
console.log("\n[3] 补全后移出候选");
|
|
91
|
+
const tasksAfter = store.getPendingProductTasks(5);
|
|
92
|
+
check("sellerA 不再出现在候选", () => {
|
|
93
|
+
assert(!tasksAfter.some((t) => t.uniqueId === "sellerA"));
|
|
94
|
+
});
|
|
95
|
+
|
|
96
|
+
console.log("\n[4] 异常输入");
|
|
97
|
+
check("缺 productDetail → error", () => {
|
|
98
|
+
const r = store.updateProductInfo("sellerB", null);
|
|
99
|
+
assert(r.ok !== true && r.error);
|
|
100
|
+
});
|
|
101
|
+
check("不存在的用户 → error", () => {
|
|
102
|
+
const r = store.updateProductInfo("ghostUser", { rating: 1 });
|
|
103
|
+
assert(r.ok !== true && r.error);
|
|
104
|
+
});
|
|
105
|
+
|
|
106
|
+
store.stopBackup?.();
|
|
107
|
+
console.log(
|
|
108
|
+
`\n结果: ${passed} 通过${process.exitCode ? ", 有失败" : ", 0 失败"}`,
|
|
109
|
+
);
|
|
110
|
+
|
|
111
|
+
function assert(cond, msg = "断言失败") {
|
|
112
|
+
if (!cond) throw new Error(msg);
|
|
113
|
+
}
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* product-fill HTTP 端到端测试
|
|
3
|
+
*
|
|
4
|
+
* 起真实 watch server(临时库),验证两个端点:
|
|
5
|
+
* - GET /api/product-tasks?limit= → 领取缺商品任务
|
|
6
|
+
* - POST /api/product-fill/:uniqueId → 回写商品详情
|
|
7
|
+
*
|
|
8
|
+
* 运行: TMPDIR=$(mktemp -d) node scripts/test-product-fill-http.mjs
|
|
9
|
+
*/
|
|
10
|
+
import { startWatchServer } from "../src/watch/server.js";
|
|
11
|
+
import { createStore } from "../src/watch/data-store.js";
|
|
12
|
+
import { getDb } from "../src/watch/db-schema.js";
|
|
13
|
+
import { join } from "node:path";
|
|
14
|
+
|
|
15
|
+
const tmp = process.env.TMPDIR || "/tmp";
|
|
16
|
+
const dbPath = join(tmp, `tt-test-http-${Date.now()}.db`);
|
|
17
|
+
|
|
18
|
+
const store = createStore(dbPath);
|
|
19
|
+
const db = getDb();
|
|
20
|
+
const now = Date.now();
|
|
21
|
+
db.prepare(
|
|
22
|
+
`INSERT INTO jobs (unique_id, status, tt_seller, top_video_seo_url, product_name, updated_at)
|
|
23
|
+
VALUES ('sellerA', 'done', 1, 'https://www.tiktok.com/shop/pdp/a1', NULL, ?)`,
|
|
24
|
+
).run(now);
|
|
25
|
+
db.prepare(
|
|
26
|
+
`INSERT INTO jobs (unique_id, status, tt_seller, top_video_seo_url, product_name, updated_at)
|
|
27
|
+
VALUES ('sellerB', 'done', 1, 'https://www.tiktok.com/shop/pdp/b1', '已有', ?)`,
|
|
28
|
+
).run(now);
|
|
29
|
+
|
|
30
|
+
let passed = 0;
|
|
31
|
+
function check(name, fn) {
|
|
32
|
+
try {
|
|
33
|
+
fn();
|
|
34
|
+
passed++;
|
|
35
|
+
console.log(` ✅ ${name}`);
|
|
36
|
+
} catch (e) {
|
|
37
|
+
console.error(` ❌ ${name}\n ${e.message}`);
|
|
38
|
+
process.exitCode = 1;
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
function assert(cond, msg = "断言失败") {
|
|
42
|
+
if (!cond) throw new Error(msg);
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
const { server } = await startWatchServer(dbPath, 0, store, {
|
|
46
|
+
llmDiscover: false,
|
|
47
|
+
});
|
|
48
|
+
const port = server.address().port;
|
|
49
|
+
const base = `http://127.0.0.1:${port}`;
|
|
50
|
+
|
|
51
|
+
console.log("===== product-fill HTTP 端到端测试 =====");
|
|
52
|
+
try {
|
|
53
|
+
console.log("\n[1] GET /api/product-tasks");
|
|
54
|
+
const res = await fetch(`${base}/api/product-tasks?limit=5`);
|
|
55
|
+
const data = await res.json();
|
|
56
|
+
check("返回 200 + total/tasks", () => {
|
|
57
|
+
assert(
|
|
58
|
+
res.ok && typeof data.total === "number" && Array.isArray(data.tasks),
|
|
59
|
+
);
|
|
60
|
+
});
|
|
61
|
+
check("只返回缺商品的 sellerA", () => {
|
|
62
|
+
const ids = data.tasks.map((t) => t.uniqueId);
|
|
63
|
+
assert(ids.length === 1 && ids[0] === "sellerA", `实际: ${ids.join(",")}`);
|
|
64
|
+
assert(data.tasks[0].seoUrl === "https://www.tiktok.com/shop/pdp/a1");
|
|
65
|
+
});
|
|
66
|
+
|
|
67
|
+
console.log("\n[2] POST /api/product-fill/:uniqueId");
|
|
68
|
+
const fill = await fetch(`${base}/api/product-fill/sellerA`, {
|
|
69
|
+
method: "POST",
|
|
70
|
+
headers: { "Content-Type": "application/json" },
|
|
71
|
+
body: JSON.stringify({
|
|
72
|
+
productDetail: {
|
|
73
|
+
rating: 4.7,
|
|
74
|
+
reviewCount: 3303,
|
|
75
|
+
soldCount: 296,
|
|
76
|
+
productName: "HTTP测试商品",
|
|
77
|
+
},
|
|
78
|
+
}),
|
|
79
|
+
});
|
|
80
|
+
const fillData = await fill.json();
|
|
81
|
+
check("回传 ok", () =>
|
|
82
|
+
assert(fillData.ok === true, JSON.stringify(fillData)),
|
|
83
|
+
);
|
|
84
|
+
|
|
85
|
+
console.log("\n[3] 补全后任务消失");
|
|
86
|
+
const res2 = await fetch(`${base}/api/product-tasks?limit=5`);
|
|
87
|
+
const data2 = await res2.json();
|
|
88
|
+
check("sellerA 不再出现", () =>
|
|
89
|
+
assert(!data2.tasks.some((t) => t.uniqueId === "sellerA")),
|
|
90
|
+
);
|
|
91
|
+
|
|
92
|
+
console.log("\n[4] 无效回传(缺 body)");
|
|
93
|
+
const bad = await fetch(`${base}/api/product-fill/sellerB`, {
|
|
94
|
+
method: "POST",
|
|
95
|
+
headers: { "Content-Type": "application/json" },
|
|
96
|
+
body: JSON.stringify({}),
|
|
97
|
+
});
|
|
98
|
+
const badData = await bad.json();
|
|
99
|
+
check("返回 error", () =>
|
|
100
|
+
assert(bad.status === 400 && badData.error, JSON.stringify(badData)),
|
|
101
|
+
);
|
|
102
|
+
} finally {
|
|
103
|
+
// 强制关闭连接 + 超时兜底,避免 keep-alive/定时器拖住进程
|
|
104
|
+
try {
|
|
105
|
+
server.closeAllConnections?.();
|
|
106
|
+
} catch {}
|
|
107
|
+
await Promise.race([
|
|
108
|
+
new Promise((r) => server.close(r)),
|
|
109
|
+
new Promise((r) => setTimeout(r, 1000)),
|
|
110
|
+
]);
|
|
111
|
+
store.stopBackup?.();
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
console.log(
|
|
115
|
+
`\n结果: ${passed} 通过${process.exitCode ? ", 有失败" : ", 0 失败"}`,
|
|
116
|
+
);
|
|
117
|
+
process.exit(process.exitCode || 0);
|
|
@@ -0,0 +1,205 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* product-fill 命令 — 循环到服务端领取"缺商品详情"任务,补全商品信息
|
|
3
|
+
*
|
|
4
|
+
* 与 attach 类似的多用户循环客户端:
|
|
5
|
+
* 1. 通过 webserver(集中式)抓取商品详情(PDP 评分/销量/评论)
|
|
6
|
+
* 2. 循环 GET /api/product-tasks 领取任务
|
|
7
|
+
* 3. 对每个任务 POST webserver /api/tiktok/product 抓取商品详情
|
|
8
|
+
* 4. 成功 → POST /api/product-fill/:uniqueId 回写 jobs 表
|
|
9
|
+
*
|
|
10
|
+
* 用法: tt-help product-fill [-s 服务端] [-w webserver] [-i 间隔秒] [-l 每批上限]
|
|
11
|
+
*/
|
|
12
|
+
import { getMacOrUuid } from "../lib/mac-or-uuid.js";
|
|
13
|
+
import { createApiClient } from "../lib/api-client.js";
|
|
14
|
+
import { userId as configuredUserId, saveUserId } from "../lib/constants.js";
|
|
15
|
+
|
|
16
|
+
const CAPTCHA_BACKOFF_MS = 30_000; // 验证码后冷却时长
|
|
17
|
+
|
|
18
|
+
function formatNow() {
|
|
19
|
+
return new Date().toLocaleString("zh-CN", { hour12: false });
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
function log(message = "") {
|
|
23
|
+
console.error(`[${formatNow()}] ${message}`);
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/** 抓取商品详情(webserver /api/tiktok/product);503 验证码 → retryable 错误 */
|
|
27
|
+
async function fetchProductDetail(webserverUrl, seoUrl) {
|
|
28
|
+
const res = await fetch(`${webserverUrl}/api/tiktok/product`, {
|
|
29
|
+
method: "POST",
|
|
30
|
+
headers: { "Content-Type": "application/json" },
|
|
31
|
+
body: JSON.stringify({ url: seoUrl }),
|
|
32
|
+
signal: AbortSignal.timeout(30_000),
|
|
33
|
+
});
|
|
34
|
+
if (res.status === 503) {
|
|
35
|
+
const err = new Error(
|
|
36
|
+
`captcha: ${(await res.json().catch(() => ({}))).error || "Blocked"}`,
|
|
37
|
+
);
|
|
38
|
+
err.retryable = true;
|
|
39
|
+
throw err;
|
|
40
|
+
}
|
|
41
|
+
if (!res.ok) {
|
|
42
|
+
const err = new Error(`HTTP ${res.status}`);
|
|
43
|
+
err.status = res.status;
|
|
44
|
+
throw err;
|
|
45
|
+
}
|
|
46
|
+
return res.json();
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
export async function handleProductFill(options) {
|
|
50
|
+
const {
|
|
51
|
+
productFillLimit,
|
|
52
|
+
productFillInterval,
|
|
53
|
+
serverUrl,
|
|
54
|
+
productFillWebserver,
|
|
55
|
+
showHelp,
|
|
56
|
+
} = options;
|
|
57
|
+
const webserverUrl = productFillWebserver || null;
|
|
58
|
+
|
|
59
|
+
if (showHelp) {
|
|
60
|
+
log(
|
|
61
|
+
"用法: tt-help product-fill [-s 服务端] [-w webserver地址] [-i 间隔秒] [-l 每批上限]",
|
|
62
|
+
);
|
|
63
|
+
log("");
|
|
64
|
+
log("参数:");
|
|
65
|
+
log(" -s, --server <URL> 服务端地址(默认: http://127.0.0.1:3001)");
|
|
66
|
+
log(
|
|
67
|
+
" -w, --webserver <URL> 商品详情 webserver 地址(必填,如 http://127.0.0.1:3000)",
|
|
68
|
+
);
|
|
69
|
+
log(" -i, --interval <N> 无任务时轮询间隔,单位秒(默认: 10)");
|
|
70
|
+
log(" -l, --limit <N> 每批领取任务数上限(默认: 5)");
|
|
71
|
+
log("");
|
|
72
|
+
log("说明:");
|
|
73
|
+
log(" 循环向服务端 /api/product-tasks 领取缺商品详情的任务;");
|
|
74
|
+
log(" 对每个任务调用 webserver /api/tiktok/product 抓取评分/销量/评论;");
|
|
75
|
+
log(" 成功后回传 /api/product-fill/:uniqueId 写入 jobs 表。");
|
|
76
|
+
log(" 商品详情验证码(503)自动冷却 30 秒后重试,其他失败计数后继续。");
|
|
77
|
+
log("");
|
|
78
|
+
log("示例:");
|
|
79
|
+
log(
|
|
80
|
+
" tt-help product-fill -s http://127.0.0.1:3001 -w http://127.0.0.1:3000",
|
|
81
|
+
);
|
|
82
|
+
log(" tt-help product-fill -w http://117.71.53.99:17300 -i 5 -l 10");
|
|
83
|
+
return;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
if (!webserverUrl) {
|
|
87
|
+
log(
|
|
88
|
+
"错误: 缺少 webserver 地址,请用 -w/--webserver 指定(如 http://127.0.0.1:3000)",
|
|
89
|
+
);
|
|
90
|
+
return;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
// ── 设备标识 ──
|
|
94
|
+
let userId = configuredUserId;
|
|
95
|
+
if (!userId) {
|
|
96
|
+
userId = await getMacOrUuid();
|
|
97
|
+
saveUserId(userId);
|
|
98
|
+
log(`[ProductFill] 未检测到本地用户编号,已生成并使用: ${userId}`);
|
|
99
|
+
}
|
|
100
|
+
log(
|
|
101
|
+
`[ProductFill] 用户编号: ${userId} | 服务端: ${serverUrl} | webserver: ${webserverUrl}`,
|
|
102
|
+
);
|
|
103
|
+
log(
|
|
104
|
+
`[ProductFill] 每批上限: ${productFillLimit} | 空闲间隔: ${productFillInterval}秒`,
|
|
105
|
+
);
|
|
106
|
+
|
|
107
|
+
const { apiGet, apiPost } = createApiClient({ log: false });
|
|
108
|
+
|
|
109
|
+
let shuttingDown = false;
|
|
110
|
+
const shutdown = () => {
|
|
111
|
+
if (shuttingDown) return;
|
|
112
|
+
shuttingDown = true;
|
|
113
|
+
log(`\n[ProductFill] 收到退出信号,完成当前批次后退出...`);
|
|
114
|
+
};
|
|
115
|
+
process.on("SIGINT", () => shutdown());
|
|
116
|
+
process.on("SIGTERM", () => shutdown());
|
|
117
|
+
|
|
118
|
+
let loopCount = 0;
|
|
119
|
+
let totalSuccess = 0;
|
|
120
|
+
let totalFailed = 0;
|
|
121
|
+
let totalSkipped = 0;
|
|
122
|
+
let captchaBackoffUntil = 0; // 验证码后冷却到的时间戳
|
|
123
|
+
|
|
124
|
+
try {
|
|
125
|
+
while (!shuttingDown) {
|
|
126
|
+
loopCount++;
|
|
127
|
+
if (shuttingDown) break;
|
|
128
|
+
|
|
129
|
+
const { tasks = [] } = await apiGet(
|
|
130
|
+
`${serverUrl}/api/product-tasks?limit=${productFillLimit}&userId=${encodeURIComponent(userId)}`,
|
|
131
|
+
);
|
|
132
|
+
|
|
133
|
+
if (!tasks || tasks.length === 0) {
|
|
134
|
+
if (loopCount === 1) {
|
|
135
|
+
log(
|
|
136
|
+
`[ProductFill] 当前无缺商品任务,${productFillInterval}秒后重试...`,
|
|
137
|
+
);
|
|
138
|
+
}
|
|
139
|
+
await new Promise((r) => setTimeout(r, productFillInterval * 1000));
|
|
140
|
+
continue;
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
log(`\n[ProductFill] 获取到 ${tasks.length} 个补商品任务...`);
|
|
144
|
+
|
|
145
|
+
// 验证码冷却期间整批跳过(避免反复触发 webserver 限流)
|
|
146
|
+
const now = Date.now();
|
|
147
|
+
if (now < captchaBackoffUntil) {
|
|
148
|
+
const waitSec = Math.ceil((captchaBackoffUntil - now) / 1000);
|
|
149
|
+
log(` ⏳ 验证码冷却中(剩余 ${waitSec}s),本批跳过`);
|
|
150
|
+
await new Promise((r) => setTimeout(r, captchaBackoffUntil - now));
|
|
151
|
+
continue;
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
for (const task of tasks) {
|
|
155
|
+
const uniqueId = task.uniqueId;
|
|
156
|
+
const seoUrl = task.seoUrl;
|
|
157
|
+
if (!seoUrl) {
|
|
158
|
+
log(` ✗ @${uniqueId} 无 seoUrl,跳过`);
|
|
159
|
+
totalSkipped++;
|
|
160
|
+
continue;
|
|
161
|
+
}
|
|
162
|
+
log(` → 补商品 @${uniqueId} ...`);
|
|
163
|
+
try {
|
|
164
|
+
const detail = await fetchProductDetail(webserverUrl, seoUrl);
|
|
165
|
+
const post = await apiPost(
|
|
166
|
+
`${serverUrl}/api/product-fill/${encodeURIComponent(uniqueId)}`,
|
|
167
|
+
{ productDetail: detail },
|
|
168
|
+
);
|
|
169
|
+
if (post && post.ok) {
|
|
170
|
+
totalSuccess++;
|
|
171
|
+
log(
|
|
172
|
+
` ✓ @${uniqueId} 商品已补全 (评分${detail.rating ?? "-"} | 销量${detail.soldCount ?? "-"} | 评论${detail.reviewCount ?? "-"})`,
|
|
173
|
+
);
|
|
174
|
+
} else {
|
|
175
|
+
totalFailed++;
|
|
176
|
+
log(` ✗ @${uniqueId} 回传失败: ${post?.error || "未知"}`);
|
|
177
|
+
}
|
|
178
|
+
} catch (e) {
|
|
179
|
+
if (e.retryable) {
|
|
180
|
+
totalSkipped++;
|
|
181
|
+
captchaBackoffUntil = Date.now() + CAPTCHA_BACKOFF_MS;
|
|
182
|
+
log(
|
|
183
|
+
` ⏳ @${uniqueId} 验证码拦截,冷却 ${CAPTCHA_BACKOFF_MS / 1000} 秒`,
|
|
184
|
+
);
|
|
185
|
+
} else {
|
|
186
|
+
totalFailed++;
|
|
187
|
+
log(` ✗ @${uniqueId} 商品详情失败: ${e.message || "未知错误"}`);
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
log(
|
|
193
|
+
` 本批结果: 成功 ${totalSuccess} | 失败 ${totalFailed} | 跳过 ${totalSkipped}\n`,
|
|
194
|
+
);
|
|
195
|
+
await new Promise((r) => setTimeout(r, 300));
|
|
196
|
+
}
|
|
197
|
+
} catch (err) {
|
|
198
|
+
log(`[ProductFill] 运行异常: ${err.message}`);
|
|
199
|
+
throw err;
|
|
200
|
+
} finally {
|
|
201
|
+
log(
|
|
202
|
+
`[ProductFill] 已退出。总计: 成功 ${totalSuccess} | 失败 ${totalFailed} | 跳过 ${totalSkipped}`,
|
|
203
|
+
);
|
|
204
|
+
}
|
|
205
|
+
}
|
package/src/lib/args.js
CHANGED
|
@@ -761,6 +761,44 @@ function parseAttachArgs(args) {
|
|
|
761
761
|
};
|
|
762
762
|
}
|
|
763
763
|
|
|
764
|
+
function parseProductFillArgs(args) {
|
|
765
|
+
let limit = 5;
|
|
766
|
+
let interval = 10;
|
|
767
|
+
let serverUrl = defaultServer;
|
|
768
|
+
let webserverUrl = null;
|
|
769
|
+
|
|
770
|
+
for (let i = 0; i < args.length; i++) {
|
|
771
|
+
const arg = args[i];
|
|
772
|
+
if (arg === "-s" || arg === "--server") {
|
|
773
|
+
serverUrl = args[++i];
|
|
774
|
+
} else if (arg === "-w" || arg === "--webserver") {
|
|
775
|
+
webserverUrl = args[++i];
|
|
776
|
+
} else if (arg === "-i" || arg === "--interval") {
|
|
777
|
+
interval = parseInt(args[++i], 10) || 10;
|
|
778
|
+
} else if (arg === "-l" || arg === "--limit") {
|
|
779
|
+
limit = parseInt(args[++i], 10) || 5;
|
|
780
|
+
}
|
|
781
|
+
}
|
|
782
|
+
|
|
783
|
+
return {
|
|
784
|
+
subcommand: "product-fill",
|
|
785
|
+
productFillLimit: limit,
|
|
786
|
+
productFillInterval: interval,
|
|
787
|
+
serverUrl,
|
|
788
|
+
productFillWebserver: webserverUrl,
|
|
789
|
+
urls: [],
|
|
790
|
+
outputFormat: "json",
|
|
791
|
+
exploreCount: 0,
|
|
792
|
+
showConfig: false,
|
|
793
|
+
showHelp: false,
|
|
794
|
+
customProxy: null,
|
|
795
|
+
configAction: null,
|
|
796
|
+
configValue: null,
|
|
797
|
+
pipeMode: false,
|
|
798
|
+
filterStr: null,
|
|
799
|
+
};
|
|
800
|
+
}
|
|
801
|
+
|
|
764
802
|
function parseRecheckArgs(args) {
|
|
765
803
|
let parallel = 1;
|
|
766
804
|
let interval = 10;
|
|
@@ -1384,6 +1422,10 @@ export function parseArgs() {
|
|
|
1384
1422
|
return parseAttachArgs(args.slice(1));
|
|
1385
1423
|
}
|
|
1386
1424
|
|
|
1425
|
+
if (args.length > 0 && args[0] === "product-fill") {
|
|
1426
|
+
return parseProductFillArgs(args.slice(1));
|
|
1427
|
+
}
|
|
1428
|
+
|
|
1387
1429
|
if (args.length > 0 && args[0] === "recheck") {
|
|
1388
1430
|
return parseRecheckArgs(args.slice(1));
|
|
1389
1431
|
}
|
package/src/lib/constants.js
CHANGED
|
@@ -198,6 +198,15 @@ const HELP_TEXT = [
|
|
|
198
198
|
" 示例: tt-help attach -p 5 -i 10",
|
|
199
199
|
" tt-help attach -c PL,DE,FR -p 5",
|
|
200
200
|
"",
|
|
201
|
+
" tt-help product-fill [-s 服务端地址] [-w webserver地址] [-i 间隔秒数] [-l 每批上限]",
|
|
202
|
+
" 循环领取缺商品详情任务,通过 webserver 抓取评分/销量/评论并回写 jobs 表",
|
|
203
|
+
" -s, --server <URL> 服务端地址(默认: http://127.0.0.1:3001)",
|
|
204
|
+
" -w, --webserver <URL> 商品详情 webserver 地址(必填)",
|
|
205
|
+
" -i, --interval <N> 无任务时轮询间隔,单位秒(默认: 10)",
|
|
206
|
+
" -l, --limit <N> 每批领取任务数上限(默认: 5)",
|
|
207
|
+
" 示例: tt-help product-fill -w http://127.0.0.1:3000",
|
|
208
|
+
" tt-help product-fill -s http://127.0.0.1:3001 -w http://117.71.53.99:17300 -i 5 -l 10",
|
|
209
|
+
"",
|
|
201
210
|
" tt-help recheck [-p 并行数] [-i 间隔秒数] [-s 服务端地址] [-c 国家列表]",
|
|
202
211
|
" 重检 jobs 表中 done 非商家用户(tt_seller!=1),7 天未更新才检测",
|
|
203
212
|
" 重新抓取用户基本信息(重点检查 tt_seller),更新回 jobs 表",
|
package/src/main.js
CHANGED
|
@@ -3,6 +3,7 @@ import { proxy, HELP_TEXT, getConfigText } from "./lib/constants.js";
|
|
|
3
3
|
import { handleInfo } from "./cli/info.js";
|
|
4
4
|
import { handleExplore } from "./cli/explore.js";
|
|
5
5
|
import { handleAttach } from "./cli/attach.js";
|
|
6
|
+
import { handleProductFill } from "./cli/product-fill.js";
|
|
6
7
|
import { handleRecheck } from "./cli/recheck.js";
|
|
7
8
|
import { handleWatch } from "./cli/watch.js";
|
|
8
9
|
import { handleConfig, showConfig, showUsage, version } from "./cli/config.js";
|
|
@@ -35,6 +36,8 @@ async function main() {
|
|
|
35
36
|
return handleInfo(parsed);
|
|
36
37
|
case "attach":
|
|
37
38
|
return handleAttach(parsed);
|
|
39
|
+
case "product-fill":
|
|
40
|
+
return handleProductFill(parsed);
|
|
38
41
|
case "recheck":
|
|
39
42
|
return handleRecheck(parsed);
|
|
40
43
|
case "watch":
|
package/src/npm-main.js
CHANGED
|
@@ -3,6 +3,7 @@ import { PUBLIC_HELP_TEXT } from "./lib/constants.js";
|
|
|
3
3
|
import { handleInfo } from "./cli/info.js";
|
|
4
4
|
import { handleExplore } from "./cli/explore.js";
|
|
5
5
|
import { handleAttach } from "./cli/attach.js";
|
|
6
|
+
import { handleProductFill } from "./cli/product-fill.js";
|
|
6
7
|
import { handleRecheck } from "./cli/recheck.js";
|
|
7
8
|
import { handleConfig, showConfig, showUsage, version } from "./cli/config.js";
|
|
8
9
|
import { handleOpen } from "./cli/open.js";
|
|
@@ -38,6 +39,8 @@ async function main() {
|
|
|
38
39
|
return handleInfo(parsed);
|
|
39
40
|
case "attach":
|
|
40
41
|
return handleAttach(parsed);
|
|
42
|
+
case "product-fill":
|
|
43
|
+
return handleProductFill(parsed);
|
|
41
44
|
case "recheck":
|
|
42
45
|
return handleRecheck(parsed);
|
|
43
46
|
case "webserver":
|
package/src/watch/data-store.js
CHANGED
|
@@ -2598,6 +2598,69 @@ export function createStore(filePath, options = {}) {
|
|
|
2598
2598
|
return pending;
|
|
2599
2599
|
}
|
|
2600
2600
|
|
|
2601
|
+
// ===== product-fill:给缺商品详情的商家补商品信息(jobs 表) =====
|
|
2602
|
+
|
|
2603
|
+
/**
|
|
2604
|
+
* 领取缺商品详情任务:jobs 表中有 top_video_seo_url 但未补过商品信息(product_name 为空)
|
|
2605
|
+
* 补全是幂等的(同一 seoUrl 补出相同详情),成功回写后自动移出候选,无需额外锁定
|
|
2606
|
+
* @param {number|string} limit
|
|
2607
|
+
* @returns {{uniqueId:string, nickname:string|null, seoUrl:string, productId:string|null, productTitle:string|null}[]}
|
|
2608
|
+
*/
|
|
2609
|
+
function getPendingProductTasks(limit) {
|
|
2610
|
+
if (getDb()) {
|
|
2611
|
+
const l = Math.max(1, parseInt(limit) || 5);
|
|
2612
|
+
const rows = getDb()
|
|
2613
|
+
.prepare(
|
|
2614
|
+
`
|
|
2615
|
+
SELECT unique_id, nickname, top_video_seo_url, top_video_product_id,
|
|
2616
|
+
top_video_product_title
|
|
2617
|
+
FROM jobs
|
|
2618
|
+
WHERE top_video_seo_url IS NOT NULL AND top_video_seo_url != ''
|
|
2619
|
+
AND (product_name IS NULL OR product_name = '')
|
|
2620
|
+
ORDER BY updated_at DESC
|
|
2621
|
+
LIMIT ?
|
|
2622
|
+
`,
|
|
2623
|
+
)
|
|
2624
|
+
.all(l);
|
|
2625
|
+
return rows.map((row) => ({
|
|
2626
|
+
uniqueId: row.unique_id,
|
|
2627
|
+
nickname: row.nickname || null,
|
|
2628
|
+
seoUrl: row.top_video_seo_url,
|
|
2629
|
+
productId: row.top_video_product_id || null,
|
|
2630
|
+
productTitle: row.top_video_product_title || null,
|
|
2631
|
+
}));
|
|
2632
|
+
}
|
|
2633
|
+
// 内存模式无商品列,返回空
|
|
2634
|
+
return [];
|
|
2635
|
+
}
|
|
2636
|
+
|
|
2637
|
+
/**
|
|
2638
|
+
* 回写商品详情到 jobs 表(product_* 列)
|
|
2639
|
+
* @param {string} uniqueId
|
|
2640
|
+
* @param {{rating?:number, reviewCount?:number, soldCount?:number, productName?:string}} productDetail
|
|
2641
|
+
*/
|
|
2642
|
+
function updateProductInfo(uniqueId, productDetail) {
|
|
2643
|
+
if (getDb()) {
|
|
2644
|
+
if (!productDetail || typeof productDetail !== "object") {
|
|
2645
|
+
return { ok: false, error: "productDetail 缺失" };
|
|
2646
|
+
}
|
|
2647
|
+
const ret = updateJobInfo(
|
|
2648
|
+
uniqueId,
|
|
2649
|
+
{
|
|
2650
|
+
productRating: productDetail.rating ?? null,
|
|
2651
|
+
productReviewCount: productDetail.reviewCount ?? null,
|
|
2652
|
+
productSoldCount: productDetail.soldCount ?? null,
|
|
2653
|
+
productName: productDetail.productName || null,
|
|
2654
|
+
},
|
|
2655
|
+
false, // 商品补全与用户刷新计数无关,不增加 user_update_count
|
|
2656
|
+
"jobs",
|
|
2657
|
+
);
|
|
2658
|
+
if (ret.error) return ret;
|
|
2659
|
+
return { ok: true };
|
|
2660
|
+
}
|
|
2661
|
+
return { ok: false, error: "db not initialized" };
|
|
2662
|
+
}
|
|
2663
|
+
|
|
2601
2664
|
// ===== recheck-jobs:重检 jobs 表中 done 非商家用户 =====
|
|
2602
2665
|
|
|
2603
2666
|
function getRecheckJobs(limit, countries) {
|
|
@@ -3383,6 +3446,8 @@ export function createStore(filePath, options = {}) {
|
|
|
3383
3446
|
getPrecheckCandidates,
|
|
3384
3447
|
markPrecheckResult,
|
|
3385
3448
|
getPendingUserUpdateTasks,
|
|
3449
|
+
getPendingProductTasks,
|
|
3450
|
+
updateProductInfo,
|
|
3386
3451
|
getRecheckJobs,
|
|
3387
3452
|
batchUpdateRecheckJobs,
|
|
3388
3453
|
updateUserInfo,
|
package/src/watch/db-schema.js
CHANGED
|
@@ -49,6 +49,11 @@ export function resetDbConnection() {
|
|
|
49
49
|
* @param {Object} requiredColumns - { columnName: columnType }
|
|
50
50
|
*/
|
|
51
51
|
function migrateColumns(tableName, requiredColumns) {
|
|
52
|
+
// 表不存在则跳过(videos_old 等表仅在历史 rename 后才存在,全新库不建)
|
|
53
|
+
const tableExists = db
|
|
54
|
+
.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name = ?")
|
|
55
|
+
.get(tableName);
|
|
56
|
+
if (!tableExists) return;
|
|
52
57
|
const existing = new Set(
|
|
53
58
|
db
|
|
54
59
|
.prepare(`PRAGMA table_info(${tableName})`)
|
|
@@ -493,7 +498,6 @@ export function initDb(filePath) {
|
|
|
493
498
|
`CREATE INDEX IF NOT EXISTS idx_jobs_user_update_queue_expr ON jobs(created_at ASC, unique_id ASC) WHERE COALESCE(tt_seller, '') = '' AND COALESCE(user_update_count, 0) <= 0`,
|
|
494
499
|
`CREATE INDEX IF NOT EXISTS idx_videos_comment_queue ON videos(user_update_count, tt_seller DESC, registered_at ASC)`,
|
|
495
500
|
`CREATE INDEX IF NOT EXISTS idx_videos_comment_queue_pending ON videos(tt_seller DESC, registered_at ASC, id) WHERE user_update_count IS NULL OR user_update_count <= 0`,
|
|
496
|
-
`CREATE INDEX IF NOT EXISTS idx_videos_old_id ON videos_old(id)`,
|
|
497
501
|
`CREATE INDEX IF NOT EXISTS idx_tags_status ON tags(status)`,
|
|
498
502
|
`CREATE INDEX IF NOT EXISTS idx_tags_score ON tags(score DESC)`,
|
|
499
503
|
`CREATE INDEX IF NOT EXISTS idx_discover_log_country ON discover_log(country, round DESC)`,
|
|
@@ -513,6 +517,16 @@ export function initDb(filePath) {
|
|
|
513
517
|
for (const sql of indexes) {
|
|
514
518
|
db.exec(sql);
|
|
515
519
|
}
|
|
520
|
+
// videos_old 索引(该表仅历史 rename 后存在,全新库跳过)
|
|
521
|
+
if (
|
|
522
|
+
db
|
|
523
|
+
.prepare(
|
|
524
|
+
"SELECT 1 FROM sqlite_master WHERE type='table' AND name='videos_old'",
|
|
525
|
+
)
|
|
526
|
+
.get()
|
|
527
|
+
) {
|
|
528
|
+
db.exec(`CREATE INDEX IF NOT EXISTS idx_videos_old_id ON videos_old(id)`);
|
|
529
|
+
}
|
|
516
530
|
|
|
517
531
|
// ===== task_queue 索引(对应 claimNextJob UNION 查询的 4 个部分索引) =====
|
|
518
532
|
const tqIndexes = [
|
package/src/watch/server.js
CHANGED
|
@@ -155,6 +155,8 @@ function inferClientType(routePath) {
|
|
|
155
155
|
if (routePath.startsWith("/api/precheck")) return "precheck";
|
|
156
156
|
if (routePath.startsWith("/api/redo-job")) return "refresh";
|
|
157
157
|
if (routePath.startsWith("/api/user-update-tasks")) return "attach";
|
|
158
|
+
if (routePath.startsWith("/api/product-tasks")) return "product-fill";
|
|
159
|
+
if (routePath.startsWith("/api/product-fill")) return "product-fill";
|
|
158
160
|
if (routePath.startsWith("/api/recheck-jobs")) return "recheck";
|
|
159
161
|
if (routePath.startsWith("/api/comment-task")) return "comments";
|
|
160
162
|
if (routePath.startsWith("/api/tags/refresh")) return "tag-refresh";
|
|
@@ -952,6 +954,40 @@ export function startWatchServer(
|
|
|
952
954
|
return;
|
|
953
955
|
}
|
|
954
956
|
|
|
957
|
+
// ── product-fill 端点:补商品信息(缺商品详情的商家) ──
|
|
958
|
+
if (req.method === "GET" && routePath === "/api/product-tasks") {
|
|
959
|
+
const limit = params.limit;
|
|
960
|
+
const tasks = store.getPendingProductTasks(limit);
|
|
961
|
+
const ts = new Date().toLocaleTimeString("zh-CN", { hour12: false });
|
|
962
|
+
console.error(`[JOB ${ts}] PRODUCT-TASKS: ${tasks.length} tasks`);
|
|
963
|
+
sendJSON(res, 200, { total: tasks.length, tasks });
|
|
964
|
+
return;
|
|
965
|
+
}
|
|
966
|
+
|
|
967
|
+
const productFillMatch = routePath.match(
|
|
968
|
+
/^\/api\/product-fill\/([^/]+)$/,
|
|
969
|
+
);
|
|
970
|
+
if (req.method === "POST" && productFillMatch) {
|
|
971
|
+
const uniqueId = decodeURIComponent(productFillMatch[1]);
|
|
972
|
+
try {
|
|
973
|
+
const body = await readBody(req);
|
|
974
|
+
const ret = store.updateProductInfo(uniqueId, body.productDetail);
|
|
975
|
+
const ts = new Date().toLocaleTimeString("zh-CN", { hour12: false });
|
|
976
|
+
if (ret.ok) {
|
|
977
|
+
console.error(`[JOB ${ts}] PRODUCT-FILL: ${uniqueId} ok`);
|
|
978
|
+
sendJSON(res, 200, { ok: true });
|
|
979
|
+
} else {
|
|
980
|
+
console.error(
|
|
981
|
+
`[JOB ${ts}] PRODUCT-FILL: ${uniqueId} 失败: ${ret.error}`,
|
|
982
|
+
);
|
|
983
|
+
sendJSON(res, 400, { error: ret.error });
|
|
984
|
+
}
|
|
985
|
+
} catch (e) {
|
|
986
|
+
sendJSON(res, 400, { error: e.message });
|
|
987
|
+
}
|
|
988
|
+
return;
|
|
989
|
+
}
|
|
990
|
+
|
|
955
991
|
// ── recheck-jobs 端点 ──
|
|
956
992
|
if (req.method === "GET" && routePath === "/api/recheck-jobs") {
|
|
957
993
|
const limit = params.limit;
|
package/src/watchdog/upgrader.js
CHANGED
|
@@ -197,9 +197,17 @@ function openLogFile(prefix, label, what) {
|
|
|
197
197
|
}
|
|
198
198
|
|
|
199
199
|
/**
|
|
200
|
-
* spawn 子进程(升级器 / 新版 watchdog 共用)
|
|
201
|
-
*
|
|
200
|
+
* spawn 子进程(升级器 U / 新版 watchdog 共用)
|
|
201
|
+
*
|
|
202
|
+
* interactive(前端):stdio inherit,继承 cmd /k 前端窗口。
|
|
203
|
+
* 用于 cmd /k 启动的 watchdog 升级链路——旧 watchdog 被杀后 cmd /k 窗口保持,
|
|
204
|
+
* U 与新版继承该窗口,运维可远程(RDP)在前端窗口看日志。
|
|
205
|
+
* 前提:watchdog 由 cmd /k 启动(保证窗口在杀旧后不关闭)。
|
|
206
|
+
*
|
|
207
|
+
* 非交互:detached 后台 + 独立日志(旧 watchdog 非交互启动时的兜底,保证升级不丢)。
|
|
208
|
+
*
|
|
202
209
|
* @param {string[]} args 子进程完整参数(node 脚本路径 + 参数)
|
|
210
|
+
* @param {{interactive?: boolean, logFd?: number}} opts
|
|
203
211
|
* @returns {import("child_process").ChildProcess}
|
|
204
212
|
*/
|
|
205
213
|
function spawnChild(args, { interactive = false, logFd } = {}) {
|
|
@@ -300,7 +308,9 @@ export async function runUpgradeMode(argv) {
|
|
|
300
308
|
await new Promise((r) => setTimeout(r, 1000)); // 给强杀一点生效时间
|
|
301
309
|
}
|
|
302
310
|
|
|
303
|
-
// 4) 旧进程已退,启新版 watchdog
|
|
311
|
+
// 4) 旧进程已退,启新版 watchdog
|
|
312
|
+
// interactive:继承 cmd /k 前端窗口(运维远程看日志);
|
|
313
|
+
// 旧 watchdog 非交互启动时自动退化为 detached 后台 + 独立日志(保证不丢)
|
|
304
314
|
console.error(`[upgrade-proc] 启动新版 watchdog ...`);
|
|
305
315
|
const cliPath = join(__dirname, "..", "..", "cli.js");
|
|
306
316
|
const args = buildWatchdogArgs({
|
|
@@ -314,13 +324,28 @@ export async function runUpgradeMode(argv) {
|
|
|
314
324
|
"upgrade-proc",
|
|
315
325
|
"新版 watchdog 日志",
|
|
316
326
|
);
|
|
327
|
+
let child;
|
|
317
328
|
try {
|
|
318
|
-
|
|
329
|
+
child = spawnChild([cliPath, ...args], { interactive, logFd });
|
|
319
330
|
console.error(`[upgrade-proc] 新版 watchdog 已启动 (pid=${child.pid})`);
|
|
320
331
|
} catch (e) {
|
|
321
332
|
console.error(`[upgrade-proc] 新版 watchdog 启动失败: ${e.message}`);
|
|
322
333
|
process.exit(1);
|
|
323
334
|
}
|
|
335
|
+
|
|
336
|
+
// 5) 存活校验:3s 后确认新版进程存活;未存活则明确报错(可看日志定位),不静默
|
|
337
|
+
await new Promise((r) => setTimeout(r, 3000));
|
|
338
|
+
try {
|
|
339
|
+
process.kill(child.pid, 0);
|
|
340
|
+
console.error(
|
|
341
|
+
`[upgrade-proc] 新版 watchdog 存活确认 (pid=${child.pid}),升级完成`,
|
|
342
|
+
);
|
|
343
|
+
} catch {
|
|
344
|
+
console.error(
|
|
345
|
+
`[upgrade-proc] 新版 watchdog (pid=${child.pid}) 启动后未存活,请查看日志排障`,
|
|
346
|
+
);
|
|
347
|
+
process.exit(1);
|
|
348
|
+
}
|
|
324
349
|
process.exit(0);
|
|
325
350
|
}
|
|
326
351
|
|
|
@@ -348,6 +373,8 @@ export function spawnUpgrader({
|
|
|
348
373
|
|
|
349
374
|
const logFd = openLogFile("tt-help-upgrader", "watchdog", "升级程序日志");
|
|
350
375
|
try {
|
|
376
|
+
// U 按旧 watchdog 的 interactive 透传:cmd /k 前端窗口启动时 U 继承前端窗口(运维可看升级日志),
|
|
377
|
+
// 旧 watchdog 非交互启动时 U 自动 detached 后台 + 独立日志
|
|
351
378
|
return spawnChild([upgraderPath, ...args], { interactive, logFd });
|
|
352
379
|
} catch (e) {
|
|
353
380
|
console.error(`[watchdog] 升级程序启动失败: ${e.message}`);
|