tt-help-cli-ycl 1.4.150 → 1.4.152
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/scripts/test-api-interceptor-captcha.mjs +134 -0
- package/scripts/test-photo-post.mjs +156 -0
- package/scripts/test-precheck-perf.mjs +270 -0
- package/scripts/test-product-fill-cdp.mjs +44 -0
- package/scripts/test-product-fill-db.mjs +43 -0
- package/scripts/test-sample-all-fail.mjs +142 -0
- package/scripts/test-seller-any-country.mjs +167 -0
- package/src/cli/explore.js +22 -2
- package/src/cli/product-fill.js +13 -1
- package/src/cli/refresh-mini.js +11 -0
- package/src/cli/refresh.js +11 -0
- package/src/lib/api-interceptor.js +25 -1
- package/src/lib/args.js +26 -0
- package/src/lib/constants.js +13 -2
- package/src/lib/shop-scraper.mjs +79 -0
- package/src/scraper/auto-core.js +61 -37
- package/src/scraper/explore-core.js +45 -4
- package/src/videos/core.js +5 -0
- package/src/watch/data-store.js +67 -20
- package/src/watch/db-columns.js +1 -0
- package/src/watch/db-schema.js +3 -0
- package/src/watch/db-tags.js +15 -5
- package/src/watch/public/app.js +89 -17
- package/src/watch/public/index.html +60 -55
- package/src/watch/public/style.css +80 -4
- package/src/watch/server.js +14 -0
package/src/lib/shop-scraper.mjs
CHANGED
|
@@ -97,6 +97,81 @@ function extractJsonByKey(html, keyPattern) {
|
|
|
97
97
|
return null;
|
|
98
98
|
}
|
|
99
99
|
|
|
100
|
+
/**
|
|
101
|
+
* Extract a JSON array by its key name (counterpart of extractJsonByKey for `[`).
|
|
102
|
+
* 用于 PDP SSR HTML 中的 "skus":[...] 段。
|
|
103
|
+
*/
|
|
104
|
+
function extractJsonArrayByKey(html, keyPattern) {
|
|
105
|
+
const match = html.match(keyPattern);
|
|
106
|
+
if (!match) return null;
|
|
107
|
+
|
|
108
|
+
const bracketIdx = html.indexOf("[", match.index);
|
|
109
|
+
if (bracketIdx === -1) return null;
|
|
110
|
+
|
|
111
|
+
let depth = 0;
|
|
112
|
+
let inString = false;
|
|
113
|
+
let escaped = false;
|
|
114
|
+
|
|
115
|
+
for (let i = bracketIdx; i < html.length; i++) {
|
|
116
|
+
const ch = html[i];
|
|
117
|
+
if (escaped) {
|
|
118
|
+
escaped = false;
|
|
119
|
+
continue;
|
|
120
|
+
}
|
|
121
|
+
if (ch === "\\") {
|
|
122
|
+
escaped = true;
|
|
123
|
+
continue;
|
|
124
|
+
}
|
|
125
|
+
if (ch === '"') {
|
|
126
|
+
inString = !inString;
|
|
127
|
+
continue;
|
|
128
|
+
}
|
|
129
|
+
if (inString) continue;
|
|
130
|
+
if (ch === "[") depth++;
|
|
131
|
+
else if (ch === "]") {
|
|
132
|
+
depth--;
|
|
133
|
+
if (depth === 0) {
|
|
134
|
+
try {
|
|
135
|
+
return JSON.parse(html.substring(bracketIdx, i + 1));
|
|
136
|
+
} catch {
|
|
137
|
+
return null;
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
return null;
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
/**
|
|
146
|
+
* 从 PDP SSR HTML 的 "skus":[...] 提取多 SKU 概要(原有商品级字段不动)。
|
|
147
|
+
* sku_id 为 19 位数字,必须按字符串保留(同 product_id 的精度坑)。
|
|
148
|
+
* @returns {Array<{skuId, specs, stock, stockStatus, warehouseId}>|null}
|
|
149
|
+
* 无 skus 段或解析失败返回 null(与现有字段缺失口径一致)
|
|
150
|
+
*/
|
|
151
|
+
function extractSkus(html) {
|
|
152
|
+
const skus = extractJsonArrayByKey(html, /"skus"\s*:/);
|
|
153
|
+
if (!Array.isArray(skus) || skus.length === 0) return null;
|
|
154
|
+
const out = [];
|
|
155
|
+
for (const s of skus) {
|
|
156
|
+
try {
|
|
157
|
+
out.push({
|
|
158
|
+
skuId: String(s.sku_id ?? ""),
|
|
159
|
+
specs: (s.property_pairs || [])
|
|
160
|
+
.map((p) => `${p.sku_property_name}: ${p.sku_property_value_name}`)
|
|
161
|
+
.join(" | "),
|
|
162
|
+
stock: s.sku_quantity?.available_quantity ?? null,
|
|
163
|
+
stockStatus: s.sku_stock_status ?? null,
|
|
164
|
+
warehouseId: s.sku_quantity?.warehouse_id
|
|
165
|
+
? String(s.sku_quantity.warehouse_id)
|
|
166
|
+
: null,
|
|
167
|
+
});
|
|
168
|
+
} catch {
|
|
169
|
+
// 单条损坏跳过,不影响其余 SKU
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
return out.length > 0 ? out : null;
|
|
173
|
+
}
|
|
174
|
+
|
|
100
175
|
/**
|
|
101
176
|
* Open a headed browser for manual captcha solving, extract cookies
|
|
102
177
|
* so undici can reuse them for a while.
|
|
@@ -278,6 +353,7 @@ export function parseProductHtml(html) {
|
|
|
278
353
|
ratingBreakdown: null,
|
|
279
354
|
productId: null,
|
|
280
355
|
sellerId: null,
|
|
356
|
+
skus: null, // 多 SKU 概要(无 skus 段或解析失败为 null,不影响原有字段)
|
|
281
357
|
};
|
|
282
358
|
|
|
283
359
|
// sold_count from product_model (string value, not shop-level integer)
|
|
@@ -308,6 +384,9 @@ export function parseProductHtml(html) {
|
|
|
308
384
|
result.ratingBreakdown = ratings.rating_result ?? null;
|
|
309
385
|
}
|
|
310
386
|
|
|
387
|
+
// skus(多规格概要:规格名/库存/仓库,sku_id 字符串保精度)
|
|
388
|
+
result.skus = extractSkus(html);
|
|
389
|
+
|
|
311
390
|
// 壳页识别:有 product_id 骨架但 seller_id 为空且无任何销售/名称数据
|
|
312
391
|
// → 商品已下架/店铺异常(200 空壳页),抛 delisted 供调用方标记放弃
|
|
313
392
|
const hasSaleData =
|
package/src/scraper/auto-core.js
CHANGED
|
@@ -9,24 +9,25 @@ import {
|
|
|
9
9
|
detectPageError,
|
|
10
10
|
isLoggedIn,
|
|
11
11
|
assertPageUrl,
|
|
12
|
-
} from
|
|
13
|
-
import { detectCaptcha } from
|
|
12
|
+
} from "./modules/page-helpers.js";
|
|
13
|
+
import { detectCaptcha } from "./modules/captcha-handler.js";
|
|
14
14
|
export { ensureBrowserReady };
|
|
15
|
-
import {
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
} from '../videos/core.js';
|
|
19
|
-
import { runScrape } from './core.js';
|
|
20
|
-
import { extractFollowAndFollowers } from './modules/follow-extractor.js';
|
|
15
|
+
import { getUserInfo, collectVideos } from "../videos/core.js";
|
|
16
|
+
import { runScrape } from "./core.js";
|
|
17
|
+
import { extractFollowAndFollowers } from "./modules/follow-extractor.js";
|
|
21
18
|
|
|
22
19
|
function mergeUserInfo(existing, incoming, source) {
|
|
23
20
|
const merged = { ...existing };
|
|
24
21
|
for (const [key, value] of Object.entries(incoming)) {
|
|
25
|
-
if (key ===
|
|
26
|
-
if (value === undefined || value === null || value ===
|
|
27
|
-
if (typeof value ===
|
|
22
|
+
if (key === "_sources") continue;
|
|
23
|
+
if (value === undefined || value === null || value === "") continue;
|
|
24
|
+
if (typeof value === "number" && typeof merged[key] === "number") {
|
|
28
25
|
merged[key] = Math.max(merged[key], value);
|
|
29
|
-
} else if (
|
|
26
|
+
} else if (
|
|
27
|
+
merged[key] === undefined ||
|
|
28
|
+
merged[key] === null ||
|
|
29
|
+
merged[key] === ""
|
|
30
|
+
) {
|
|
30
31
|
merged[key] = value;
|
|
31
32
|
}
|
|
32
33
|
}
|
|
@@ -43,7 +44,7 @@ async function processUser(page, username, options, log) {
|
|
|
43
44
|
scrapeDepth = 50,
|
|
44
45
|
maxComments = 200,
|
|
45
46
|
maxGuess = 10,
|
|
46
|
-
preset =
|
|
47
|
+
preset = "fast",
|
|
47
48
|
switchMax = null,
|
|
48
49
|
commentMax = null,
|
|
49
50
|
enableFollow = false,
|
|
@@ -65,34 +66,43 @@ async function processUser(page, username, options, log) {
|
|
|
65
66
|
|
|
66
67
|
try {
|
|
67
68
|
log(`\n[processUser] 访问 @${username}...`);
|
|
68
|
-
await retryWithBackoff(
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
69
|
+
await retryWithBackoff(
|
|
70
|
+
async () => {
|
|
71
|
+
await page.goto(`https://www.tiktok.com/@${username}`, {
|
|
72
|
+
waitUntil: "load",
|
|
73
|
+
timeout: 30000,
|
|
74
|
+
});
|
|
75
|
+
assertPageUrl(page, `@${username}`);
|
|
76
|
+
},
|
|
77
|
+
{ log },
|
|
78
|
+
);
|
|
79
|
+
await page
|
|
80
|
+
.waitForSelector('[class*="DivVideoList"]', { timeout: 10000 })
|
|
81
|
+
.catch(() => {});
|
|
75
82
|
await delay(1000, 2000);
|
|
76
83
|
|
|
77
84
|
const info = await getUserInfo(page);
|
|
78
85
|
result.userInfo = info;
|
|
79
86
|
if (!info.uniqueId) info.uniqueId = username;
|
|
80
|
-
log(` 昵称: ${info.nickname ||
|
|
87
|
+
log(` 昵称: ${info.nickname || "-"} | 粉丝: ${info.followerCount || 0}`);
|
|
81
88
|
|
|
82
89
|
if (options.enableFollow) {
|
|
83
90
|
const loggedIn = await isLoggedIn(page);
|
|
84
91
|
if (!loggedIn) {
|
|
85
|
-
log(
|
|
92
|
+
log(" [跳过] 提取关注/粉丝:未登录,请先登录 TikTok");
|
|
86
93
|
result.discoveredFollowing = [];
|
|
87
94
|
result.discoveredFollowers = [];
|
|
88
95
|
} else {
|
|
89
96
|
try {
|
|
90
|
-
log(
|
|
91
|
-
const { following, followers } = await extractFollowAndFollowers(
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
97
|
+
log(" 提取关注/粉丝列表...");
|
|
98
|
+
const { following, followers } = await extractFollowAndFollowers(
|
|
99
|
+
page,
|
|
100
|
+
{
|
|
101
|
+
maxFollowing: options.maxFollowing || 200,
|
|
102
|
+
maxFollowers: options.maxFollowers || 200,
|
|
103
|
+
log,
|
|
104
|
+
},
|
|
105
|
+
);
|
|
96
106
|
result.discoveredFollowing = following;
|
|
97
107
|
result.discoveredFollowers = followers;
|
|
98
108
|
log(` 关注: ${following.length} | 粉丝: ${followers.length}`);
|
|
@@ -108,13 +118,13 @@ async function processUser(page, username, options, log) {
|
|
|
108
118
|
if (captcha && captcha.visible) {
|
|
109
119
|
log(`[验证码] @${username} 页面出现验证码`);
|
|
110
120
|
result.captchaDetected = true;
|
|
111
|
-
result.captchaStage = result.captchaStage ||
|
|
112
|
-
result.captchaMessage = result.captchaMessage ||
|
|
121
|
+
result.captchaStage = result.captchaStage || "video-page";
|
|
122
|
+
result.captchaMessage = result.captchaMessage || "视频页出现验证码";
|
|
113
123
|
}
|
|
114
124
|
|
|
115
125
|
const videos = await collectVideos(page, username, collectMax, log);
|
|
116
126
|
const videoList = Array.from(videos.values()).slice(0, collectMax);
|
|
117
|
-
result.collectedVideos = videoList.map(v => ({
|
|
127
|
+
result.collectedVideos = videoList.map((v) => ({
|
|
118
128
|
videoId: v.id,
|
|
119
129
|
videoUrl: v.href,
|
|
120
130
|
}));
|
|
@@ -126,10 +136,12 @@ async function processUser(page, username, options, log) {
|
|
|
126
136
|
|
|
127
137
|
for (let i = 0; i < videoList.length; i++) {
|
|
128
138
|
const video = videoList[i];
|
|
129
|
-
const videoUrl = video.href.startsWith(
|
|
139
|
+
const videoUrl = video.href.startsWith("http")
|
|
130
140
|
? video.href
|
|
131
141
|
: `https://www.tiktok.com${video.href}`;
|
|
132
|
-
log(
|
|
142
|
+
log(
|
|
143
|
+
` [${i + 1}/${videoList.length}] 开始 scrape: ${videoUrl} (深度 ${scrapeDepth})`,
|
|
144
|
+
);
|
|
133
145
|
|
|
134
146
|
const scrapeResult = await runScrape({
|
|
135
147
|
videoUrl,
|
|
@@ -146,8 +158,9 @@ async function processUser(page, username, options, log) {
|
|
|
146
158
|
|
|
147
159
|
if (scrapeResult.captchaDetected) {
|
|
148
160
|
result.captchaDetected = true;
|
|
149
|
-
result.captchaStage = scrapeResult.captchaStage ||
|
|
150
|
-
result.captchaMessage =
|
|
161
|
+
result.captchaStage = scrapeResult.captchaStage || "scrape";
|
|
162
|
+
result.captchaMessage =
|
|
163
|
+
scrapeResult.captchaMessage || "scrape阶段出现验证码";
|
|
151
164
|
}
|
|
152
165
|
|
|
153
166
|
const scrapeOutput = scrapeResult.output;
|
|
@@ -181,7 +194,9 @@ async function processUser(page, username, options, log) {
|
|
|
181
194
|
result.discoveredCommentAuthors = [...allCommentAuthors];
|
|
182
195
|
result.discoveredGuessAuthors = [...allGuessAuthors];
|
|
183
196
|
|
|
184
|
-
log(
|
|
197
|
+
log(
|
|
198
|
+
` 发现: ${result.discoveredVideoAuthors.length} 个视频作者, ${result.discoveredCommentAuthors.length} 个评论作者, ${result.discoveredGuessAuthors.length} 个猜你喜欢作者`,
|
|
199
|
+
);
|
|
185
200
|
} else {
|
|
186
201
|
const pageError = await detectPageError(page);
|
|
187
202
|
result.restricted = !!pageError;
|
|
@@ -192,8 +207,17 @@ async function processUser(page, username, options, log) {
|
|
|
192
207
|
}
|
|
193
208
|
}
|
|
194
209
|
} catch (e) {
|
|
210
|
+
// 被判重试期间弹出验证码(api-interceptor 抛出 "验证码: username"):
|
|
211
|
+
// 转验证码标记,由外层 captchaDetected 计数处理
|
|
212
|
+
if (e.message.startsWith("验证码:")) {
|
|
213
|
+
result.captchaDetected = true;
|
|
214
|
+
result.captchaStage = result.captchaStage || "video-page";
|
|
215
|
+
result.captchaMessage = result.captchaMessage || "被判重试期间弹出验证码";
|
|
216
|
+
log(` @${username} 被判期间弹出验证码,转验证码处理`);
|
|
217
|
+
return result;
|
|
218
|
+
}
|
|
195
219
|
result.error = e.message;
|
|
196
|
-
result.errorStack = e.stack ||
|
|
220
|
+
result.errorStack = e.stack || "";
|
|
197
221
|
log(` [错误] ${e.message}`);
|
|
198
222
|
}
|
|
199
223
|
|
|
@@ -181,19 +181,26 @@ async function processExplore(page, username, options, log) {
|
|
|
181
181
|
return result;
|
|
182
182
|
}
|
|
183
183
|
|
|
184
|
-
//
|
|
184
|
+
// 国家采样判断(只用真视频:图文帖 /photo/ 链接查不到 locationCreated,
|
|
185
|
+
// 会浪费采样位并污染众数统计;图文 EC 帖仍在上方 EC 筛选中正常保留)
|
|
185
186
|
// explore 模式:采样 5 个,优先命中目标国家,不匹配则回退众数 → 写 locationCreated
|
|
186
187
|
// refresh 模式:采样 7 个,纯众数逻辑 → 写 confirmedLocation(二次确认)
|
|
187
188
|
const SAMPLE_SIZE = locationMode === "refresh" ? 7 : 5;
|
|
188
189
|
let locationCreated = null;
|
|
189
190
|
let confirmedLocation = null;
|
|
190
191
|
let locationDecision = null;
|
|
191
|
-
const sampleVideos = videoArray
|
|
192
|
+
const sampleVideos = videoArray
|
|
193
|
+
.filter((v) => !v.isPhoto)
|
|
194
|
+
.slice(0, SAMPLE_SIZE);
|
|
192
195
|
if (sampleVideos.length > 0) {
|
|
193
196
|
const sampleUrls = sampleVideos.map((v) =>
|
|
194
197
|
v.href.startsWith("http") ? v.href : `https://www.tiktok.com${v.href}`,
|
|
195
198
|
);
|
|
199
|
+
log(
|
|
200
|
+
` 国家采样(${sampleUrls.length}个, 已排除图文${videoArray.length - videoArray.filter((v) => !v.isPhoto).length}条): 开始查询`,
|
|
201
|
+
);
|
|
196
202
|
let locations;
|
|
203
|
+
let sampleFailCount = 0; // 查询本身失败(网络/超时/CDN)计数,与"查成功但无数据"区分
|
|
197
204
|
if (webserverUrl) {
|
|
198
205
|
// 走集中式 webserver,不再启动本地 TikTokScraper
|
|
199
206
|
locations = await Promise.all(
|
|
@@ -208,6 +215,7 @@ async function processExplore(page, username, options, log) {
|
|
|
208
215
|
const data = await res.json();
|
|
209
216
|
return data?.locationCreated || null;
|
|
210
217
|
} catch (e) {
|
|
218
|
+
sampleFailCount++;
|
|
211
219
|
log(` [webserver] 视频查询失败: ${e.message}`);
|
|
212
220
|
return null;
|
|
213
221
|
}
|
|
@@ -215,12 +223,32 @@ async function processExplore(page, username, options, log) {
|
|
|
215
223
|
);
|
|
216
224
|
} else {
|
|
217
225
|
locations = await Promise.all(
|
|
218
|
-
sampleUrls.map((url) =>
|
|
226
|
+
sampleUrls.map(async (url) => {
|
|
227
|
+
try {
|
|
228
|
+
return await extractVideoLocation(url);
|
|
229
|
+
} catch (e) {
|
|
230
|
+
sampleFailCount++;
|
|
231
|
+
log(` [采样] 视频查询失败: ${e.message}`);
|
|
232
|
+
return null;
|
|
233
|
+
}
|
|
234
|
+
}),
|
|
219
235
|
);
|
|
220
236
|
}
|
|
221
237
|
log(
|
|
222
|
-
` 国家采样(${locations.length}
|
|
238
|
+
` 国家采样(${locations.length}个${sampleFailCount > 0 ? `, 查询失败${sampleFailCount}` : ""}): [${locations.filter(Boolean).join(", ") || "无数据"}]`,
|
|
223
239
|
);
|
|
240
|
+
|
|
241
|
+
// 全部查询失败 ≠ 全部无数据:
|
|
242
|
+
// 通道故障(webserver 挂掉/超时、本地 Scraper 被封)时若按 done 无国家入库,
|
|
243
|
+
// 会陷入"无国家 → 2h 回炉 → 又全失败 → 又 done"的空转循环;
|
|
244
|
+
// 按 error 上报让任务回流重做,且面板可见
|
|
245
|
+
if (sampleUrls.length > 0 && sampleFailCount === sampleUrls.length) {
|
|
246
|
+
result.error = `国家采样全失败(${webserverUrl ? "webserver" : "本地Scraper"}通道 ${sampleFailCount}/${sampleUrls.length} 全失败)`;
|
|
247
|
+
result.errorStack = "";
|
|
248
|
+
result.processed = false;
|
|
249
|
+
log(` [错误] ${result.error},任务按错误回流`);
|
|
250
|
+
return result;
|
|
251
|
+
}
|
|
224
252
|
// 直接标准化,不去重(保留重复值用于频率统计)
|
|
225
253
|
const normalizedLocations = locations
|
|
226
254
|
.map((loc) => normalizeLocation(loc))
|
|
@@ -347,6 +375,19 @@ async function processExplore(page, username, options, log) {
|
|
|
347
375
|
log(` @${username} CDN限流`);
|
|
348
376
|
}
|
|
349
377
|
|
|
378
|
+
// 被判重试期间弹出验证码(api-interceptor 抛出 "验证码: username"):
|
|
379
|
+
// 清掉 error 转验证码分支,让外层 CLI 走 captchaDetected 上报/等待人工解决流程
|
|
380
|
+
if (e.message.startsWith("验证码:")) {
|
|
381
|
+
result.error = null;
|
|
382
|
+
result.errorStack = "";
|
|
383
|
+
result.processed = false;
|
|
384
|
+
result.captchaDetected = true;
|
|
385
|
+
result.captchaStage = result.captchaStage || "video-page";
|
|
386
|
+
result.captchaMessage = result.captchaMessage || "被判重试期间弹出验证码";
|
|
387
|
+
log(` @${username} 被判期间弹出验证码,转验证码处理`);
|
|
388
|
+
return result;
|
|
389
|
+
}
|
|
390
|
+
|
|
350
391
|
// 临时性错误:超时、执行上下文销毁、网络异常 — 标记为可重试
|
|
351
392
|
const retryablePatterns = [
|
|
352
393
|
"Timeout",
|
package/src/videos/core.js
CHANGED
|
@@ -87,6 +87,11 @@ async function collectVideos(page, username, maxVideos, log) {
|
|
|
87
87
|
if (e.message.startsWith("被封:")) {
|
|
88
88
|
throw e; // 向上抛出,由 explore-core 处理
|
|
89
89
|
}
|
|
90
|
+
// 被判重试期间弹出验证码会抛出 "验证码: username",同样原样上抛
|
|
91
|
+
// (不能被包装成 "API 拦截失败",否则上层识别不到前缀)
|
|
92
|
+
if (e.message.startsWith("验证码:")) {
|
|
93
|
+
throw e; // 向上抛出,由 explore-core 处理
|
|
94
|
+
}
|
|
90
95
|
// CDN 限流会抛出 "CDN限流..." 异常(HTTP 路径 CDNBlockedError / CDP 路径 api-interceptor)
|
|
91
96
|
if (e.message.startsWith("CDN限流")) {
|
|
92
97
|
throw e; // 向上抛出,由 explore-core 处理
|
package/src/watch/data-store.js
CHANGED
|
@@ -683,7 +683,13 @@ function getPreciseMerchantsFromDb(targetLocations = [], options = {}) {
|
|
|
683
683
|
product_review_count,
|
|
684
684
|
product_sold_count,
|
|
685
685
|
product_name,
|
|
686
|
+
product_skus,
|
|
686
687
|
recent_ec_videos,
|
|
688
|
+
(CASE WHEN json_valid(product_skus) THEN
|
|
689
|
+
(SELECT COUNT(*) FROM json_each(product_skus)) ELSE 0 END) AS skuCount,
|
|
690
|
+
(CASE WHEN json_valid(product_skus) THEN
|
|
691
|
+
(SELECT COALESCE(SUM(CAST(json_extract(value, '$.stock') AS INTEGER)), 0)
|
|
692
|
+
FROM json_each(product_skus)) ELSE 0 END) AS skuTotalStock,
|
|
687
693
|
(SELECT COUNT(*) FROM json_each(recent_ec_videos)
|
|
688
694
|
WHERE CAST(json_extract(value, '$.playCount') AS INTEGER) > 1000) AS ecOver1k,
|
|
689
695
|
(SELECT MAX(CAST(json_extract(value, '$.playCount') AS INTEGER))
|
|
@@ -1388,7 +1394,11 @@ export function createStore(filePath, options = {}) {
|
|
|
1388
1394
|
filter: `instr(COALESCE(sources, ''), '"seed"') > 0`,
|
|
1389
1395
|
loggedInOnly: false,
|
|
1390
1396
|
},
|
|
1391
|
-
{
|
|
1397
|
+
{
|
|
1398
|
+
filter: `tt_seller = 1 AND verified = 0`,
|
|
1399
|
+
loggedInOnly: true,
|
|
1400
|
+
sellerAnyCountry: true,
|
|
1401
|
+
},
|
|
1392
1402
|
{
|
|
1393
1403
|
filter: `(instr(COALESCE(sources, ''), '"following"') > 0 OR instr(COALESCE(sources, ''), '"follower"') > 0 OR instr(COALESCE(sources, ''), '"comment"') > 0)`,
|
|
1394
1404
|
loggedInOnly: false,
|
|
@@ -1407,8 +1417,28 @@ export function createStore(filePath, options = {}) {
|
|
|
1407
1417
|
);
|
|
1408
1418
|
if (normalizedLocations.length === 0)
|
|
1409
1419
|
locationBranches.push(locNotIn(orderedCountries));
|
|
1420
|
+
// 商家级专用(sellerAnyCountry):国家对商家放开——末尾追加"任意国家"分支。
|
|
1421
|
+
// 传了国家子集时对应国家的商家仍优先,领完再兜任意国家(含空位置商家);
|
|
1422
|
+
// 未传国家时同理(配置内各国 → 配置外 → 任意)。其他来源级分支不受影响。
|
|
1423
|
+
if (p.sellerAnyCountry) locationBranches.push(null); // null → 任意国家分支
|
|
1410
1424
|
let subPrio = 0;
|
|
1411
1425
|
for (const loc of locationBranches) {
|
|
1426
|
+
// 商家任意国家分支:不限 guessed_location(含空位置的商家)
|
|
1427
|
+
if (loc === null) {
|
|
1428
|
+
// hasVideo
|
|
1429
|
+
addBranch(
|
|
1430
|
+
`status = 'pending' AND pinned = 0 AND ${p.filter} AND video_count > 0 AND ${statusCodeWhere} ${notSellerWhere}`,
|
|
1431
|
+
[],
|
|
1432
|
+
);
|
|
1433
|
+
subPrio++;
|
|
1434
|
+
// noVideoReq
|
|
1435
|
+
addBranch(
|
|
1436
|
+
`status = 'pending' AND pinned = 0 AND ${p.filter} AND ${statusCodeWhere} ${notSellerWhere}`,
|
|
1437
|
+
[],
|
|
1438
|
+
);
|
|
1439
|
+
subPrio++;
|
|
1440
|
+
continue;
|
|
1441
|
+
}
|
|
1412
1442
|
if (!loc) continue;
|
|
1413
1443
|
// hasVideo
|
|
1414
1444
|
addBranch(
|
|
@@ -2452,28 +2482,40 @@ export function createStore(filePath, options = {}) {
|
|
|
2452
2482
|
const placeholders = targetLocations.map(() => "?").join(",");
|
|
2453
2483
|
|
|
2454
2484
|
// SELECT 候选用户
|
|
2455
|
-
|
|
2456
|
-
|
|
2457
|
-
|
|
2458
|
-
|
|
2459
|
-
|
|
2485
|
+
// 性能三件套(76 万行实测 1~3s → ~80ms):
|
|
2486
|
+
// 1. WHERE/ORDER BY 不用 COALESCE(refresh_time, ?)(表达式挡索引);
|
|
2487
|
+
// ASC 排序 NULL 天然最前,与 COALESCE 2016 默认值语义等价
|
|
2488
|
+
// 2. INDEXED BY 强制走 idx_jobs_precheck 部分索引(优化器无统计时会
|
|
2489
|
+
// 误选 idx_jobs_seller_verified_loc 走 TEMP B-TREE 全排序)
|
|
2490
|
+
// 3. ORDER BY jobs.refresh_time 限定表名——裸列名会被 SELECT 输出别名
|
|
2491
|
+
// refresh_time(COALESCE 表达式)遮蔽,又退回表达式排序
|
|
2492
|
+
// 索引不存在的环境(老库未重启)抛 "no such index",自动回退普通查询。
|
|
2493
|
+
const baseWhere = `
|
|
2460
2494
|
WHERE tt_seller = 1
|
|
2461
2495
|
AND verified = 0
|
|
2462
2496
|
AND (pending_refresh IS NULL OR pending_refresh = 0)
|
|
2463
2497
|
AND location_created IN (${placeholders})
|
|
2464
|
-
AND
|
|
2465
|
-
|
|
2466
|
-
|
|
2467
|
-
|
|
2468
|
-
|
|
2469
|
-
|
|
2470
|
-
|
|
2471
|
-
|
|
2472
|
-
|
|
2473
|
-
|
|
2474
|
-
|
|
2475
|
-
|
|
2476
|
-
|
|
2498
|
+
AND (refresh_time IS NULL OR refresh_time < ?)`;
|
|
2499
|
+
const orderLimit = `
|
|
2500
|
+
ORDER BY jobs.refresh_time ASC
|
|
2501
|
+
LIMIT ?`;
|
|
2502
|
+
const selectSql = `SELECT unique_id, nickname, video_count, COALESCE(refresh_time, ?) AS refresh_time
|
|
2503
|
+
FROM jobs`;
|
|
2504
|
+
const args = [defaultTime, ...targetLocations, threshold, limit];
|
|
2505
|
+
|
|
2506
|
+
let selected;
|
|
2507
|
+
try {
|
|
2508
|
+
selected = getDb()
|
|
2509
|
+
.prepare(
|
|
2510
|
+
`${selectSql} INDEXED BY idx_jobs_precheck${baseWhere}${orderLimit}`,
|
|
2511
|
+
)
|
|
2512
|
+
.all(...args);
|
|
2513
|
+
} catch (e) {
|
|
2514
|
+
if (!/no such index/i.test(e.message)) throw e;
|
|
2515
|
+
selected = getDb()
|
|
2516
|
+
.prepare(`${selectSql}${baseWhere}${orderLimit}`)
|
|
2517
|
+
.all(...args);
|
|
2518
|
+
}
|
|
2477
2519
|
|
|
2478
2520
|
// 立即更新 refresh_time 防重复领取(非事务,小概率并发重复可接受)
|
|
2479
2521
|
if (selected.length > 0) {
|
|
@@ -2799,8 +2841,9 @@ export function createStore(filePath, options = {}) {
|
|
|
2799
2841
|
/**
|
|
2800
2842
|
* 回写商品详情到 jobs 表(product_* 列)
|
|
2801
2843
|
* @param {string} uniqueId
|
|
2802
|
-
* @param {{rating?:number, reviewCount?:number, soldCount?:number, productName?:string}} productDetail
|
|
2844
|
+
* @param {{rating?:number, reviewCount?:number, soldCount?:number, productName?:string, skus?:Array}} productDetail
|
|
2803
2845
|
* rating = -1 为哨兵值:商品不可售/已下架,写入后任务移出候选(正常评分 0-5)
|
|
2846
|
+
* skus = 多 SKU 概要数组(skuId/specs/stock/stockStatus/warehouseId),存 product_skus JSON
|
|
2804
2847
|
*/
|
|
2805
2848
|
function updateProductInfo(uniqueId, productDetail) {
|
|
2806
2849
|
if (getDb()) {
|
|
@@ -2814,6 +2857,10 @@ export function createStore(filePath, options = {}) {
|
|
|
2814
2857
|
productReviewCount: productDetail.reviewCount ?? null,
|
|
2815
2858
|
productSoldCount: productDetail.soldCount ?? null,
|
|
2816
2859
|
productName: productDetail.productName || null,
|
|
2860
|
+
// 多 SKU 概要:数组经 normalizeJobValue 自动序列化为 JSON(无 skus 存 null)
|
|
2861
|
+
productSkus: Array.isArray(productDetail.skus)
|
|
2862
|
+
? productDetail.skus
|
|
2863
|
+
: null,
|
|
2817
2864
|
},
|
|
2818
2865
|
false, // 商品补全与用户刷新计数无关,不增加 user_update_count
|
|
2819
2866
|
"jobs",
|
package/src/watch/db-columns.js
CHANGED
package/src/watch/db-schema.js
CHANGED
|
@@ -173,6 +173,7 @@ export function initDb(filePath) {
|
|
|
173
173
|
product_review_count: "INTEGER",
|
|
174
174
|
product_sold_count: "INTEGER",
|
|
175
175
|
product_name: "TEXT",
|
|
176
|
+
product_skus: "TEXT",
|
|
176
177
|
user_create_time: "INTEGER",
|
|
177
178
|
recent_ec_videos: "TEXT",
|
|
178
179
|
});
|
|
@@ -209,6 +210,7 @@ export function initDb(filePath) {
|
|
|
209
210
|
product_review_count: "INTEGER",
|
|
210
211
|
product_sold_count: "INTEGER",
|
|
211
212
|
product_name: "TEXT",
|
|
213
|
+
product_skus: "TEXT",
|
|
212
214
|
user_create_time: "INTEGER",
|
|
213
215
|
recent_ec_videos: "TEXT",
|
|
214
216
|
});
|
|
@@ -246,6 +248,7 @@ export function initDb(filePath) {
|
|
|
246
248
|
product_review_count: "INTEGER",
|
|
247
249
|
product_sold_count: "INTEGER",
|
|
248
250
|
product_name: "TEXT",
|
|
251
|
+
product_skus: "TEXT",
|
|
249
252
|
user_create_time: "INTEGER",
|
|
250
253
|
recent_ec_videos: "TEXT",
|
|
251
254
|
});
|
package/src/watch/db-tags.js
CHANGED
|
@@ -375,15 +375,25 @@ export function getAllTags(
|
|
|
375
375
|
}
|
|
376
376
|
|
|
377
377
|
export function rawQuery(sql, params = []) {
|
|
378
|
-
// 通用查询器:默认查主库(stats/db-query 复用),勿切到 tag
|
|
378
|
+
// 通用查询器:默认查主库(stats/db-query 复用),勿切到 tag 库。
|
|
379
|
+
// 注意:原实现用 prepare().all() 跑一切语句,DDL(CREATE INDEX 等)会抛
|
|
380
|
+
// "This statement does not return data" 且被 catch 吞成 {error} 返回——
|
|
381
|
+
// server.js 启动清单 ensureIdx 曾因此静默失效(生产库从未建过启动索引)。
|
|
382
|
+
// 现按语句类型分流:DDL/写语句走 exec,查询走 all;失败时抛错让调用方感知。
|
|
379
383
|
const db = getMainDb();
|
|
380
|
-
if (!db)
|
|
381
|
-
|
|
384
|
+
if (!db) throw new Error("db not ready");
|
|
385
|
+
const trimmed = sql.trim().toUpperCase();
|
|
386
|
+
const isQuery =
|
|
387
|
+
trimmed.startsWith("SELECT") ||
|
|
388
|
+
trimmed.startsWith("PRAGMA") ||
|
|
389
|
+
trimmed.startsWith("EXPLAIN") ||
|
|
390
|
+
trimmed.startsWith("WITH");
|
|
391
|
+
if (isQuery) {
|
|
382
392
|
const rows = db.prepare(sql).all(...params);
|
|
383
393
|
return { rows };
|
|
384
|
-
} catch (e) {
|
|
385
|
-
return { error: e.message };
|
|
386
394
|
}
|
|
395
|
+
db.exec(sql);
|
|
396
|
+
return { rows: [] };
|
|
387
397
|
}
|
|
388
398
|
|
|
389
399
|
export function normalizeTags() {
|