tt-help-cli-ycl 1.3.79 → 1.3.81
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/cli/attach.js +30 -5
- package/src/cli/explore.js +11 -4
- package/src/cli/refresh.js +11 -4
- package/src/cli/test-real-attach.js +0 -0
- package/src/lib/api-interceptor.js +14 -3
- package/src/lib/browser/cdp.js +2 -1
- package/src/lib/page-error-detector.js +31 -14
- package/src/lib/parse-ssr.mjs +35 -0
- package/src/lib/scroll-collector.js +1 -1
- package/src/lib/tiktok-scraper.mjs +18 -0
- package/src/videos/core.js +6 -2
- package/src/watch/data-store.js +118 -31
package/package.json
CHANGED
package/src/cli/attach.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { TikTokScraper } from "../lib/tiktok-scraper.mjs";
|
|
2
|
+
import { CDNBlockedError } from "../lib/parse-ssr.mjs";
|
|
2
3
|
import { proxy as configuredProxy } from "../lib/constants.js";
|
|
3
4
|
import v8 from "node:v8";
|
|
4
5
|
|
|
@@ -249,6 +250,7 @@ export async function handleAttach(options) {
|
|
|
249
250
|
|
|
250
251
|
let successCount = 0;
|
|
251
252
|
let failCount = 0;
|
|
253
|
+
let cdnBlockedCount = 0;
|
|
252
254
|
let needRestart = false;
|
|
253
255
|
|
|
254
256
|
// 收集抓取成功的任务,记录抓取失败的
|
|
@@ -257,13 +259,20 @@ export async function handleAttach(options) {
|
|
|
257
259
|
if (result.status === "fulfilled") {
|
|
258
260
|
const { uniqueId, info, error } = result.value;
|
|
259
261
|
if (error) {
|
|
260
|
-
if (
|
|
262
|
+
if (error instanceof CDNBlockedError) {
|
|
263
|
+
attachLog(` ⚠ @${uniqueId} CDN限流 (Access Denied)`);
|
|
264
|
+
cdnBlockedCount++;
|
|
265
|
+
failCount++;
|
|
266
|
+
} else if (isBrowserClosedError(error)) {
|
|
261
267
|
needRestart = true;
|
|
268
|
+
attachLog(` ✗ @${uniqueId} 浏览器断开: ${error.message}`);
|
|
269
|
+
failCount++;
|
|
270
|
+
} else {
|
|
271
|
+
attachLog(
|
|
272
|
+
` ✗ @${uniqueId} 获取失败: ${error.message || "未知错误"}`,
|
|
273
|
+
);
|
|
274
|
+
failCount++;
|
|
262
275
|
}
|
|
263
|
-
attachLog(
|
|
264
|
-
` ✗ @${uniqueId} 获取失败: ${error.message || "未知错误"}`,
|
|
265
|
-
);
|
|
266
|
-
failCount++;
|
|
267
276
|
} else if (info) {
|
|
268
277
|
// info 可能是 { error: true, statusCode: xxx } 表示 TikTok 给了明确响应
|
|
269
278
|
if (info.error) {
|
|
@@ -327,6 +336,22 @@ export async function handleAttach(options) {
|
|
|
327
336
|
|
|
328
337
|
attachLog(` 本批结果: ${successCount} 成功, ${failCount} 失败\n`);
|
|
329
338
|
|
|
339
|
+
// CDN 限流比例超过 30% 时,冷却 + 重启浏览器
|
|
340
|
+
const cdnRatio = cdnBlockedCount / tasks.length;
|
|
341
|
+
if (cdnRatio > 0.3) {
|
|
342
|
+
const coolDownSeconds = cdnRatio > 0.8 ? 120 : 60;
|
|
343
|
+
attachLog(
|
|
344
|
+
` [Attach] CDN限流比例 ${(cdnRatio * 100).toFixed(0)}% (${cdnBlockedCount}/${tasks.length}),冷却 ${coolDownSeconds} 秒后重启浏览器...`,
|
|
345
|
+
);
|
|
346
|
+
await new Promise((r) => setTimeout(r, coolDownSeconds * 1000));
|
|
347
|
+
await recycleScraper(
|
|
348
|
+
scraper,
|
|
349
|
+
`CDN限流比例过高 (${cdnBlockedCount}/${tasks.length})`,
|
|
350
|
+
);
|
|
351
|
+
browserRestartCount++;
|
|
352
|
+
taskBatchCount = 0;
|
|
353
|
+
}
|
|
354
|
+
|
|
330
355
|
const heap = getHeapUsage();
|
|
331
356
|
if (heap.ratio >= HEAP_RESTART_RATIO) {
|
|
332
357
|
await recycleScraper(
|
package/src/cli/explore.js
CHANGED
|
@@ -29,6 +29,7 @@ import os from "os";
|
|
|
29
29
|
|
|
30
30
|
const MAX_RETRY_WAIT = 5 * 60 * 1000;
|
|
31
31
|
const STARTUP_TIKTOK_URL = "https://www.tiktok.com/@ycl5007";
|
|
32
|
+
const PAGE_GOTO_TIMEOUT = 60000; // 页面导航超时 60 秒(账户切换后需要更长时间)
|
|
32
33
|
|
|
33
34
|
async function withRetry(label, fn) {
|
|
34
35
|
let backoff = 1000;
|
|
@@ -181,8 +182,11 @@ export async function handleExplore(options) {
|
|
|
181
182
|
const page = await getOrCreatePage(browser);
|
|
182
183
|
|
|
183
184
|
// 先导航到 TikTok 页面,再检测登录状态
|
|
184
|
-
await
|
|
185
|
-
|
|
185
|
+
await withRetry("启动页面导航", async () => {
|
|
186
|
+
await page.goto(STARTUP_TIKTOK_URL, {
|
|
187
|
+
waitUntil: "domcontentloaded",
|
|
188
|
+
timeout: PAGE_GOTO_TIMEOUT,
|
|
189
|
+
});
|
|
186
190
|
});
|
|
187
191
|
|
|
188
192
|
// 检测登录状态(启动时只检测一次)
|
|
@@ -235,8 +239,11 @@ export async function handleExplore(options) {
|
|
|
235
239
|
`[健康检查] 已切换到端口 ${nextAccount.port}${effectiveProxy ? ", 代理: " + effectiveProxy : ""}`,
|
|
236
240
|
);
|
|
237
241
|
// 切换账户后先导航到 TikTok 页面,再重新检测登录状态
|
|
238
|
-
await
|
|
239
|
-
|
|
242
|
+
await withRetry("账户切换后页面导航", async () => {
|
|
243
|
+
await page.goto(STARTUP_TIKTOK_URL, {
|
|
244
|
+
waitUntil: "domcontentloaded",
|
|
245
|
+
timeout: PAGE_GOTO_TIMEOUT,
|
|
246
|
+
});
|
|
240
247
|
});
|
|
241
248
|
loggedIn = await safeCheckLogin(page);
|
|
242
249
|
console.error(
|
package/src/cli/refresh.js
CHANGED
|
@@ -28,6 +28,7 @@ import os from "os";
|
|
|
28
28
|
|
|
29
29
|
const MAX_RETRY_WAIT = 5 * 60 * 1000;
|
|
30
30
|
const STARTUP_TIKTOK_URL = "https://www.tiktok.com/@ycl5007";
|
|
31
|
+
const PAGE_GOTO_TIMEOUT = 60000; // 页面导航超时 60 秒(账户切换后需要更长时间)
|
|
31
32
|
|
|
32
33
|
async function withRetry(label, fn) {
|
|
33
34
|
let backoff = 1000;
|
|
@@ -181,8 +182,11 @@ export async function handleRefresh(options) {
|
|
|
181
182
|
const page = await getOrCreatePage(browser);
|
|
182
183
|
|
|
183
184
|
// 导航到 TikTok 页面
|
|
184
|
-
await
|
|
185
|
-
|
|
185
|
+
await withRetry("启动页面导航", async () => {
|
|
186
|
+
await page.goto(STARTUP_TIKTOK_URL, {
|
|
187
|
+
waitUntil: "domcontentloaded",
|
|
188
|
+
timeout: PAGE_GOTO_TIMEOUT,
|
|
189
|
+
});
|
|
186
190
|
});
|
|
187
191
|
|
|
188
192
|
// 检测登录状态
|
|
@@ -233,8 +237,11 @@ export async function handleRefresh(options) {
|
|
|
233
237
|
cdpOptions.proxyServer = effectiveProxy;
|
|
234
238
|
}
|
|
235
239
|
console.error(`[健康检查] 已切换到端口 ${nextAccount.port}`);
|
|
236
|
-
await
|
|
237
|
-
|
|
240
|
+
await withRetry("账户切换后页面导航", async () => {
|
|
241
|
+
await page.goto(STARTUP_TIKTOK_URL, {
|
|
242
|
+
waitUntil: "domcontentloaded",
|
|
243
|
+
timeout: PAGE_GOTO_TIMEOUT,
|
|
244
|
+
});
|
|
238
245
|
});
|
|
239
246
|
loggedIn = await isLoggedIn(page);
|
|
240
247
|
console.error(
|
|
File without changes
|
|
@@ -32,7 +32,7 @@ async function processAPIResponse(
|
|
|
32
32
|
|
|
33
33
|
try {
|
|
34
34
|
const pageData = await (() => {
|
|
35
|
-
//
|
|
35
|
+
// 重试包装:处理页面导航导致的执行上下文销毁和 CDP 断连
|
|
36
36
|
const tryEval = async (retries = 3) => {
|
|
37
37
|
for (let i = 0; i < retries; i++) {
|
|
38
38
|
try {
|
|
@@ -42,8 +42,11 @@ async function processAPIResponse(
|
|
|
42
42
|
}, newUrl);
|
|
43
43
|
} catch (e) {
|
|
44
44
|
if (
|
|
45
|
-
e.message
|
|
46
|
-
|
|
45
|
+
e.message &&
|
|
46
|
+
(e.message.includes("Execution context was destroyed") ||
|
|
47
|
+
e.message.includes("Target closed") ||
|
|
48
|
+
e.message.includes("Connection closed") ||
|
|
49
|
+
e.message.includes("Protocol error"))
|
|
47
50
|
) {
|
|
48
51
|
await delay(500 * (i + 1), 500 * (i + 1));
|
|
49
52
|
} else {
|
|
@@ -89,6 +92,14 @@ async function processAPIResponse(
|
|
|
89
92
|
async function fetchUserVideosAPI(page, username, maxVideos, log) {
|
|
90
93
|
log(` [API拦截] 获取 @${username} 视频 ...`);
|
|
91
94
|
|
|
95
|
+
// CDP 健康检查:确保 page 可用
|
|
96
|
+
try {
|
|
97
|
+
await page.evaluate(() => 1);
|
|
98
|
+
} catch (e) {
|
|
99
|
+
log(` [API拦截] CDP 连接异常: ${e.message}`);
|
|
100
|
+
throw new Error(`CDP 连接异常: ${e.message}`);
|
|
101
|
+
}
|
|
102
|
+
|
|
92
103
|
let apiRequestUrl = null;
|
|
93
104
|
let sawApiRequest = false;
|
|
94
105
|
|
package/src/lib/browser/cdp.js
CHANGED
|
@@ -277,7 +277,8 @@ export async function switchAccount(oldAccount, newAccount, proxyServer) {
|
|
|
277
277
|
|
|
278
278
|
const browser = await ensureBrowserReady(newCdpOptions);
|
|
279
279
|
|
|
280
|
-
|
|
280
|
+
// 等待浏览器完全稳定(Windows 下 Edge 启动后需要更长时间)
|
|
281
|
+
await new Promise((r) => setTimeout(r, 15000));
|
|
281
282
|
|
|
282
283
|
return browser;
|
|
283
284
|
}
|
|
@@ -54,23 +54,40 @@ const PATTERNS = {
|
|
|
54
54
|
service_error: ["出错了", "很抱歉"],
|
|
55
55
|
};
|
|
56
56
|
|
|
57
|
-
export async function detectPageError(page) {
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
57
|
+
export async function detectPageError(page, timeout = 10000) {
|
|
58
|
+
try {
|
|
59
|
+
return await page.evaluate(
|
|
60
|
+
(patterns) => {
|
|
61
|
+
const body = document.body;
|
|
62
|
+
if (!body) return null;
|
|
63
|
+
const bodyText = body.innerText;
|
|
64
|
+
const lower = bodyText.toLowerCase();
|
|
63
65
|
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
66
|
+
for (const [type, phrases] of Object.entries(patterns)) {
|
|
67
|
+
for (const phrase of phrases) {
|
|
68
|
+
if (lower.includes(phrase.toLowerCase())) {
|
|
69
|
+
return type;
|
|
70
|
+
}
|
|
71
|
+
}
|
|
68
72
|
}
|
|
69
|
-
}
|
|
70
|
-
}
|
|
71
73
|
|
|
72
|
-
|
|
73
|
-
|
|
74
|
+
return null;
|
|
75
|
+
},
|
|
76
|
+
PATTERNS,
|
|
77
|
+
);
|
|
78
|
+
} catch (e) {
|
|
79
|
+
// CDP 断连或超时:返回 null 而非永久挂起
|
|
80
|
+
if (
|
|
81
|
+
e.message &&
|
|
82
|
+
(e.message.includes("Timeout") ||
|
|
83
|
+
e.message.includes("Target closed") ||
|
|
84
|
+
e.message.includes("Connection closed") ||
|
|
85
|
+
e.message.includes("Protocol error"))
|
|
86
|
+
) {
|
|
87
|
+
return null;
|
|
88
|
+
}
|
|
89
|
+
throw e;
|
|
90
|
+
}
|
|
74
91
|
}
|
|
75
92
|
|
|
76
93
|
/**
|
package/src/lib/parse-ssr.mjs
CHANGED
|
@@ -1,3 +1,25 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* CDN 限流错误(Akamai Access Denied)
|
|
3
|
+
*/
|
|
4
|
+
export class CDNBlockedError extends Error {
|
|
5
|
+
constructor(message = "CDN限流 (Access Denied)", reference) {
|
|
6
|
+
super(message);
|
|
7
|
+
this.name = "CDNBlockedError";
|
|
8
|
+
this.reference = reference;
|
|
9
|
+
}
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* 检测 HTML 是否为 CDN Access Denied
|
|
14
|
+
* 返回 { isBlocked: true, reference: "xxx" } 或 null
|
|
15
|
+
*/
|
|
16
|
+
export function detectAccessDenied(rawHtml) {
|
|
17
|
+
if (!rawHtml || typeof rawHtml !== "string") return null;
|
|
18
|
+
if (!rawHtml.includes("Access Denied")) return null;
|
|
19
|
+
const refMatch = rawHtml.match(/Reference\s*#\s*([\w.]+)/);
|
|
20
|
+
return { isBlocked: true, reference: refMatch ? refMatch[1] : null };
|
|
21
|
+
}
|
|
22
|
+
|
|
1
23
|
/**
|
|
2
24
|
* 判断失败是否可重试
|
|
3
25
|
* - 有 statusCode(无论值是多少):TikTok 给了明确响应,不可重试
|
|
@@ -5,6 +27,10 @@
|
|
|
5
27
|
*/
|
|
6
28
|
export function isRetryableFailure(rawHtml) {
|
|
7
29
|
if (!rawHtml || typeof rawHtml !== "string") return false;
|
|
30
|
+
// Access Denied = CDN 限流 = 可重试
|
|
31
|
+
if (detectAccessDenied(rawHtml)) {
|
|
32
|
+
return true;
|
|
33
|
+
}
|
|
8
34
|
// 没有 SSR 标记 = 空壳 HTML = 可重试
|
|
9
35
|
if (!rawHtml.includes("__UNIVERSAL_DATA_FOR_REHYDRATION__")) {
|
|
10
36
|
return true;
|
|
@@ -49,6 +75,15 @@ function parseSSR(rawHtml) {
|
|
|
49
75
|
}
|
|
50
76
|
|
|
51
77
|
export function parseUserInfo(rawHtml) {
|
|
78
|
+
// 先检查 CDN 限流
|
|
79
|
+
const denied = detectAccessDenied(rawHtml);
|
|
80
|
+
if (denied) {
|
|
81
|
+
throw new CDNBlockedError(
|
|
82
|
+
`CDN限流 (Access Denied, ref:${denied.reference || "N/A"})`,
|
|
83
|
+
denied.reference,
|
|
84
|
+
);
|
|
85
|
+
}
|
|
86
|
+
|
|
52
87
|
const data = parseSSR(rawHtml);
|
|
53
88
|
if (!data) return null;
|
|
54
89
|
const scopeKeys = data.__DEFAULT_SCOPE__
|
|
@@ -30,7 +30,7 @@ async function doCollect(
|
|
|
30
30
|
}
|
|
31
31
|
|
|
32
32
|
const fn = eval("(" + fnStr + ")");
|
|
33
|
-
return fn(el, args);
|
|
33
|
+
return args !== undefined ? fn(el, args) : fn(el);
|
|
34
34
|
},
|
|
35
35
|
{ fn: fnStr, containerSelector: container, findScrollableFlag: findScrollable, args: extraArgs },
|
|
36
36
|
);
|
|
@@ -4,6 +4,8 @@ import {
|
|
|
4
4
|
parseUserInfo,
|
|
5
5
|
parseVideoInfo,
|
|
6
6
|
isRetryableFailure,
|
|
7
|
+
CDNBlockedError,
|
|
8
|
+
detectAccessDenied,
|
|
7
9
|
} from "./parse-ssr.mjs";
|
|
8
10
|
|
|
9
11
|
const DEFAULT_POOL_SIZE = 3;
|
|
@@ -225,6 +227,14 @@ export class TikTokScraper {
|
|
|
225
227
|
`https://www.tiktok.com/@${uniqueId}`,
|
|
226
228
|
slot,
|
|
227
229
|
);
|
|
230
|
+
// CDN 限流立即抛出,不重试
|
|
231
|
+
if (detectAccessDenied(rawHtml)) {
|
|
232
|
+
const denied = detectAccessDenied(rawHtml);
|
|
233
|
+
throw new CDNBlockedError(
|
|
234
|
+
`CDN限流 (Access Denied, ref:${denied.reference || "N/A"})`,
|
|
235
|
+
denied.reference,
|
|
236
|
+
);
|
|
237
|
+
}
|
|
228
238
|
let result = parseUserInfo(rawHtml);
|
|
229
239
|
for (let attempt = 1; !result && attempt <= maxRetries; attempt++) {
|
|
230
240
|
// 检查是否值得重试:用户异常/不存在则跳过重试
|
|
@@ -239,6 +249,14 @@ export class TikTokScraper {
|
|
|
239
249
|
`https://www.tiktok.com/@${uniqueId}`,
|
|
240
250
|
slot,
|
|
241
251
|
);
|
|
252
|
+
// 重试中也检查 CDN 限流
|
|
253
|
+
if (detectAccessDenied(rawHtml)) {
|
|
254
|
+
const denied = detectAccessDenied(rawHtml);
|
|
255
|
+
throw new CDNBlockedError(
|
|
256
|
+
`CDN限流 (Access Denied, ref:${denied.reference || "N/A"})`,
|
|
257
|
+
denied.reference,
|
|
258
|
+
);
|
|
259
|
+
}
|
|
242
260
|
result = parseUserInfo(rawHtml);
|
|
243
261
|
}
|
|
244
262
|
return result || null;
|
package/src/videos/core.js
CHANGED
|
@@ -7,14 +7,18 @@ import {
|
|
|
7
7
|
import { fetchUserVideosAPI } from "../lib/api-interceptor.js";
|
|
8
8
|
|
|
9
9
|
async function getUserInfo(page) {
|
|
10
|
-
//
|
|
10
|
+
// 重试包装:处理页面导航导致的执行上下文销毁和 CDP 断连
|
|
11
11
|
const evaluateWithRetry = async (fn, retries = 3) => {
|
|
12
12
|
for (let i = 0; i < retries; i++) {
|
|
13
13
|
try {
|
|
14
14
|
return await page.evaluate(fn);
|
|
15
15
|
} catch (e) {
|
|
16
16
|
if (
|
|
17
|
-
e.message
|
|
17
|
+
e.message &&
|
|
18
|
+
(e.message.includes("Execution context was destroyed") ||
|
|
19
|
+
e.message.includes("Target closed") ||
|
|
20
|
+
e.message.includes("Connection closed") ||
|
|
21
|
+
e.message.includes("Protocol error")) &&
|
|
18
22
|
i < retries - 1
|
|
19
23
|
) {
|
|
20
24
|
await new Promise((r) => setTimeout(r, 500 * (i + 1)));
|
package/src/watch/data-store.js
CHANGED
|
@@ -650,13 +650,24 @@ function getDashboardStatsFromDb(targetLocations = []) {
|
|
|
650
650
|
AND instr(COALESCE(sources, ''), '"guess"') = 0
|
|
651
651
|
AND instr(COALESCE(sources, ''), '"following"') = 0
|
|
652
652
|
AND instr(COALESCE(sources, ''), '"follower"') = 0
|
|
653
|
-
THEN 1 ELSE 0 END) as seed
|
|
654
|
-
SUM(CASE WHEN COALESCE(tt_seller, '') = '' AND COALESCE(user_update_count, 0) <= 0 THEN 1 ELSE 0 END) as userUpdateTasks
|
|
653
|
+
THEN 1 ELSE 0 END) as seed
|
|
655
654
|
FROM jobs
|
|
656
655
|
`,
|
|
657
656
|
)
|
|
658
657
|
.get(...targetParams);
|
|
659
658
|
|
|
659
|
+
// userUpdateTasks 单独从 jobs_base 统计
|
|
660
|
+
const userUpdateTasksRow = db
|
|
661
|
+
.prepare(
|
|
662
|
+
`
|
|
663
|
+
SELECT COUNT(*) as userUpdateTasks
|
|
664
|
+
FROM jobs_base
|
|
665
|
+
WHERE COALESCE(tt_seller, '') = ''
|
|
666
|
+
AND COALESCE(user_update_count, 0) <= 0
|
|
667
|
+
`,
|
|
668
|
+
)
|
|
669
|
+
.get();
|
|
670
|
+
|
|
660
671
|
// countryStats 和 targetCountryStats 需要 GROUP BY,保留为独立查询
|
|
661
672
|
const countryStats = db
|
|
662
673
|
.prepare(
|
|
@@ -712,7 +723,7 @@ function getDashboardStatsFromDb(targetLocations = []) {
|
|
|
712
723
|
restrictedUsers: aggregateRow.restricted,
|
|
713
724
|
errorUsers: aggregateRow.error,
|
|
714
725
|
targetUsers: aggregateRow.targetUsers,
|
|
715
|
-
userUpdateTasks:
|
|
726
|
+
userUpdateTasks: userUpdateTasksRow.userUpdateTasks,
|
|
716
727
|
targetCountryStats,
|
|
717
728
|
countryStats,
|
|
718
729
|
sourceStats: {
|
|
@@ -761,7 +772,7 @@ function getUserUpdateByCountryFromDb() {
|
|
|
761
772
|
SELECT
|
|
762
773
|
COALESCE(guessed_location, '未知') as country,
|
|
763
774
|
COUNT(*) as count
|
|
764
|
-
FROM
|
|
775
|
+
FROM jobs_base
|
|
765
776
|
WHERE COALESCE(tt_seller, '') = ''
|
|
766
777
|
AND COALESCE(user_update_count, 0) <= 0
|
|
767
778
|
GROUP BY COALESCE(guessed_location, '未知')
|
|
@@ -782,7 +793,7 @@ function getAttachStuckByCountryFromDb() {
|
|
|
782
793
|
SELECT
|
|
783
794
|
COALESCE(guessed_location, '未知') as country,
|
|
784
795
|
COUNT(*) as count
|
|
785
|
-
FROM
|
|
796
|
+
FROM jobs_base
|
|
786
797
|
WHERE COALESCE(tt_seller, '') = ''
|
|
787
798
|
AND COALESCE(user_update_count, 0) = 1
|
|
788
799
|
GROUP BY COALESCE(guessed_location, '未知')
|
|
@@ -816,7 +827,7 @@ function restoreAttachStuckByCountry(country) {
|
|
|
816
827
|
.prepare(
|
|
817
828
|
`
|
|
818
829
|
SELECT COUNT(*) as c
|
|
819
|
-
FROM
|
|
830
|
+
FROM jobs_base
|
|
820
831
|
WHERE ${whereSql}
|
|
821
832
|
`,
|
|
822
833
|
)
|
|
@@ -828,7 +839,7 @@ function restoreAttachStuckByCountry(country) {
|
|
|
828
839
|
|
|
829
840
|
db.prepare(
|
|
830
841
|
`
|
|
831
|
-
UPDATE
|
|
842
|
+
UPDATE jobs_base
|
|
832
843
|
SET user_update_count = 0,
|
|
833
844
|
updated_at = ?,
|
|
834
845
|
claimed_by = NULL,
|
|
@@ -943,7 +954,7 @@ function moveJobsToRawByCountry(scope, country) {
|
|
|
943
954
|
.prepare(
|
|
944
955
|
`
|
|
945
956
|
SELECT COUNT(*) as c
|
|
946
|
-
FROM
|
|
957
|
+
FROM jobs_base
|
|
947
958
|
WHERE ${whereSql}
|
|
948
959
|
`,
|
|
949
960
|
)
|
|
@@ -1017,14 +1028,14 @@ function moveJobsToRawByCountry(scope, country) {
|
|
|
1017
1028
|
signature,
|
|
1018
1029
|
sec_uid,
|
|
1019
1030
|
latest_video_time
|
|
1020
|
-
FROM
|
|
1031
|
+
FROM jobs_base
|
|
1021
1032
|
WHERE ${whereSql}
|
|
1022
1033
|
`,
|
|
1023
1034
|
).run(targetCountry);
|
|
1024
1035
|
|
|
1025
1036
|
db.prepare(
|
|
1026
1037
|
`
|
|
1027
|
-
DELETE FROM
|
|
1038
|
+
DELETE FROM jobs_base
|
|
1028
1039
|
WHERE ${whereSql}
|
|
1029
1040
|
`,
|
|
1030
1041
|
).run(targetCountry);
|
|
@@ -1718,6 +1729,13 @@ function getJobRow(uniqueId) {
|
|
|
1718
1729
|
return db.prepare("SELECT * FROM jobs WHERE unique_id = ?").get(uniqueId);
|
|
1719
1730
|
}
|
|
1720
1731
|
|
|
1732
|
+
function getJobBaseRow(uniqueId) {
|
|
1733
|
+
if (!db) return null;
|
|
1734
|
+
return db
|
|
1735
|
+
.prepare("SELECT * FROM jobs_base WHERE unique_id = ?")
|
|
1736
|
+
.get(uniqueId);
|
|
1737
|
+
}
|
|
1738
|
+
|
|
1721
1739
|
function getJob(uniqueId) {
|
|
1722
1740
|
return mapJobRow(getJobRow(uniqueId));
|
|
1723
1741
|
}
|
|
@@ -1795,6 +1813,43 @@ function inferStatus(u) {
|
|
|
1795
1813
|
return "pending";
|
|
1796
1814
|
}
|
|
1797
1815
|
|
|
1816
|
+
function updateJobBaseInfo(uniqueId, info, incrementCount = true) {
|
|
1817
|
+
if (!db) return { error: "db not initialized" };
|
|
1818
|
+
const existing = getJobBaseRow(uniqueId);
|
|
1819
|
+
if (!existing) return { error: "user not found" };
|
|
1820
|
+
|
|
1821
|
+
const nextValues = {};
|
|
1822
|
+
for (const [key, value] of Object.entries(info || {})) {
|
|
1823
|
+
if (key === "uniqueId" || key === "unique_id") continue;
|
|
1824
|
+
if (value === undefined || value === "") continue;
|
|
1825
|
+
let column = camelToSnake(key);
|
|
1826
|
+
// 字段别名:bio → signature
|
|
1827
|
+
if (column === "bio") column = "signature";
|
|
1828
|
+
if (!writableJobColumns.has(column)) continue;
|
|
1829
|
+
nextValues[column] = normalizeJobValue(column, value);
|
|
1830
|
+
}
|
|
1831
|
+
|
|
1832
|
+
nextValues.updated_at = Date.now();
|
|
1833
|
+
if (incrementCount) {
|
|
1834
|
+
nextValues.user_update_count = (existing.user_update_count || 0) + 1;
|
|
1835
|
+
}
|
|
1836
|
+
|
|
1837
|
+
const columns = Object.keys(nextValues);
|
|
1838
|
+
if (columns.length > 0) {
|
|
1839
|
+
const sql = `UPDATE jobs_base SET ${columns.map((column) => `${column} = ?`).join(", ")} WHERE unique_id = ?`;
|
|
1840
|
+
db.prepare(sql).run(
|
|
1841
|
+
...columns.map((column) => nextValues[column]),
|
|
1842
|
+
uniqueId,
|
|
1843
|
+
);
|
|
1844
|
+
}
|
|
1845
|
+
|
|
1846
|
+
return {
|
|
1847
|
+
ok: true,
|
|
1848
|
+
userUpdateCount:
|
|
1849
|
+
nextValues.user_update_count ?? existing.user_update_count ?? 0,
|
|
1850
|
+
};
|
|
1851
|
+
}
|
|
1852
|
+
|
|
1798
1853
|
function addJobBaseToDb(user) {
|
|
1799
1854
|
if (!db) return;
|
|
1800
1855
|
const now = Date.now();
|
|
@@ -3375,7 +3430,7 @@ export function createStore(filePath) {
|
|
|
3375
3430
|
|
|
3376
3431
|
let sql = `
|
|
3377
3432
|
SELECT *
|
|
3378
|
-
FROM
|
|
3433
|
+
FROM jobs_base
|
|
3379
3434
|
WHERE COALESCE(tt_seller, '') = ''
|
|
3380
3435
|
AND COALESCE(user_update_count, 0) <= 0
|
|
3381
3436
|
`;
|
|
@@ -3395,7 +3450,7 @@ export function createStore(filePath) {
|
|
|
3395
3450
|
const now = Date.now();
|
|
3396
3451
|
const bumpStmt = db.prepare(
|
|
3397
3452
|
`
|
|
3398
|
-
UPDATE
|
|
3453
|
+
UPDATE jobs_base
|
|
3399
3454
|
SET user_update_count = COALESCE(user_update_count, 0) + 1,
|
|
3400
3455
|
updated_at = ?
|
|
3401
3456
|
WHERE unique_id = ?
|
|
@@ -3526,7 +3581,8 @@ export function createStore(filePath) {
|
|
|
3526
3581
|
function batchUpdateUserInfo(updates) {
|
|
3527
3582
|
if (db) {
|
|
3528
3583
|
const results = [];
|
|
3529
|
-
const
|
|
3584
|
+
const rawMoveList = [];
|
|
3585
|
+
const sellerMoveList = [];
|
|
3530
3586
|
|
|
3531
3587
|
const txn = db.transaction((items) => {
|
|
3532
3588
|
items.forEach((item) => {
|
|
@@ -3536,13 +3592,13 @@ export function createStore(filePath) {
|
|
|
3536
3592
|
let updateResult;
|
|
3537
3593
|
if (info && info.error && info.statusCode !== undefined) {
|
|
3538
3594
|
// 只更新 status_code,不更新其他字段
|
|
3539
|
-
updateResult =
|
|
3595
|
+
updateResult = updateJobBaseInfo(
|
|
3540
3596
|
uniqueId,
|
|
3541
3597
|
{ statusCode: info.statusCode },
|
|
3542
3598
|
true,
|
|
3543
3599
|
);
|
|
3544
3600
|
} else {
|
|
3545
|
-
updateResult =
|
|
3601
|
+
updateResult = updateJobBaseInfo(uniqueId, info, true);
|
|
3546
3602
|
}
|
|
3547
3603
|
|
|
3548
3604
|
if (updateResult.error) {
|
|
@@ -3550,34 +3606,66 @@ export function createStore(filePath) {
|
|
|
3550
3606
|
return;
|
|
3551
3607
|
}
|
|
3552
3608
|
|
|
3553
|
-
// 检查 tt_seller
|
|
3554
|
-
const row =
|
|
3609
|
+
// 检查 tt_seller:商家移到 jobs,非商家移到 raw_jobs
|
|
3610
|
+
const row = getJobBaseRow(uniqueId);
|
|
3555
3611
|
const ttSeller = row ? row.tt_seller : null;
|
|
3556
3612
|
if (ttSeller) {
|
|
3557
|
-
//
|
|
3613
|
+
// 商家:标记移动到 jobs
|
|
3558
3614
|
results.push({
|
|
3559
3615
|
uniqueId,
|
|
3560
3616
|
ok: true,
|
|
3561
3617
|
userUpdateCount: updateResult.userUpdateCount,
|
|
3618
|
+
_movedToJobs: true,
|
|
3562
3619
|
});
|
|
3620
|
+
sellerMoveList.push(uniqueId);
|
|
3563
3621
|
} else {
|
|
3564
|
-
//
|
|
3622
|
+
// 非商家:标记移动到 raw_jobs
|
|
3565
3623
|
results.push({
|
|
3566
3624
|
uniqueId,
|
|
3567
3625
|
ok: true,
|
|
3568
3626
|
userUpdateCount: updateResult.userUpdateCount,
|
|
3569
3627
|
_movedToRaw: true,
|
|
3570
3628
|
});
|
|
3571
|
-
|
|
3629
|
+
rawMoveList.push(uniqueId);
|
|
3572
3630
|
}
|
|
3573
3631
|
});
|
|
3574
3632
|
});
|
|
3575
3633
|
txn(updates);
|
|
3576
3634
|
|
|
3577
|
-
//
|
|
3578
|
-
if (
|
|
3579
|
-
const placeholders =
|
|
3580
|
-
|
|
3635
|
+
// 批量移动商家用户到 jobs
|
|
3636
|
+
if (sellerMoveList.length > 0) {
|
|
3637
|
+
const placeholders = sellerMoveList.map(() => "?").join(",");
|
|
3638
|
+
db.prepare(
|
|
3639
|
+
`
|
|
3640
|
+
INSERT OR REPLACE INTO jobs (
|
|
3641
|
+
unique_id, nickname, status, sources, claimed_by, claimed_at,
|
|
3642
|
+
error, pinned, no_video, restricted, user_update_count,
|
|
3643
|
+
tt_seller, verified, video_count, comment_count,
|
|
3644
|
+
guessed_location, location_created, confirmed_location, modified_at,
|
|
3645
|
+
follower_count, following_count, heart_count, refresh_time,
|
|
3646
|
+
processed, processed_at, created_at, updated_at,
|
|
3647
|
+
region, signature, bio_link, sec_uid, status_code, latest_video_time
|
|
3648
|
+
)
|
|
3649
|
+
SELECT
|
|
3650
|
+
unique_id, nickname, status, sources, claimed_by, claimed_at,
|
|
3651
|
+
error, pinned, no_video, restricted, user_update_count,
|
|
3652
|
+
tt_seller, verified, video_count, comment_count,
|
|
3653
|
+
guessed_location, location_created, confirmed_location, modified_at,
|
|
3654
|
+
follower_count, following_count, heart_count, refresh_time,
|
|
3655
|
+
processed, processed_at, created_at, updated_at,
|
|
3656
|
+
region, signature, bio_link, sec_uid, status_code, latest_video_time
|
|
3657
|
+
FROM jobs_base WHERE unique_id IN (${placeholders})
|
|
3658
|
+
`,
|
|
3659
|
+
).run(...sellerMoveList);
|
|
3660
|
+
|
|
3661
|
+
db.prepare(
|
|
3662
|
+
`DELETE FROM jobs_base WHERE unique_id IN (${placeholders})`,
|
|
3663
|
+
).run(...sellerMoveList);
|
|
3664
|
+
}
|
|
3665
|
+
|
|
3666
|
+
// 批量移动非商家用户到 raw_jobs
|
|
3667
|
+
if (rawMoveList.length > 0) {
|
|
3668
|
+
const placeholders = rawMoveList.map(() => "?").join(",");
|
|
3581
3669
|
db.prepare(
|
|
3582
3670
|
`
|
|
3583
3671
|
INSERT OR REPLACE INTO raw_jobs (
|
|
@@ -3597,19 +3685,18 @@ export function createStore(filePath) {
|
|
|
3597
3685
|
follower_count, following_count, heart_count, refresh_time,
|
|
3598
3686
|
processed, processed_at, created_at, updated_at,
|
|
3599
3687
|
region, signature, bio_link, sec_uid, status_code, latest_video_time
|
|
3600
|
-
FROM
|
|
3688
|
+
FROM jobs_base WHERE unique_id IN (${placeholders})
|
|
3601
3689
|
`,
|
|
3602
|
-
).run(...
|
|
3690
|
+
).run(...rawMoveList);
|
|
3603
3691
|
|
|
3604
|
-
|
|
3605
|
-
|
|
3606
|
-
|
|
3607
|
-
);
|
|
3692
|
+
db.prepare(
|
|
3693
|
+
`DELETE FROM jobs_base WHERE unique_id IN (${placeholders})`,
|
|
3694
|
+
).run(...rawMoveList);
|
|
3608
3695
|
}
|
|
3609
3696
|
|
|
3610
3697
|
// 清理内部标记
|
|
3611
3698
|
return results.map((r) => {
|
|
3612
|
-
const { _movedToRaw, ...rest } = r;
|
|
3699
|
+
const { _movedToRaw, _movedToJobs, ...rest } = r;
|
|
3613
3700
|
return rest;
|
|
3614
3701
|
});
|
|
3615
3702
|
}
|