tt-help-cli-ycl 1.4.142 → 1.4.144
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/scripts/sync-raw-jobs.mjs +13 -7
- package/scripts/test-api-interceptor-429.mjs +122 -0
- package/src/cli/comments.js +5 -5
- package/src/cli/precheck.js +2 -2
- package/src/cli/refresh-mini.js +1 -1
- package/src/cli/tag.js +6 -24
- package/src/lib/api-interceptor.js +26 -8
- package/src/lib/app-config.js +119 -0
- package/src/lib/args.js +3 -1
- package/src/lib/constants.js +3 -1
- package/src/lib/target-locations.js +1 -0
- package/src/scraper/core.js +2 -2
- package/src/scraper/explore-core.js +2 -2
- package/src/watch/data-store.js +6 -8
- package/src/watch/public/app.js +180 -26
- package/src/watch/public/index.html +21 -1
- package/src/watch/public/style.css +16 -0
- package/src/watch/server.js +84 -40
package/package.json
CHANGED
|
@@ -15,13 +15,16 @@ import Database from "better-sqlite3";
|
|
|
15
15
|
import { initDb, getDb } from "../src/watch/db-schema.js";
|
|
16
16
|
import { loadLlmConfig, reloadLlmConfig } from "../src/watch/llm-config.js";
|
|
17
17
|
import { scoreJobsBatch } from "../src/watch/llm-scoring.js";
|
|
18
|
-
import {
|
|
18
|
+
import { getTargetLocations, reloadConfig } from "../src/lib/app-config.js";
|
|
19
19
|
|
|
20
20
|
// ═══════════════════════════════════════════════════════════════════
|
|
21
21
|
// 常量
|
|
22
22
|
// ═══════════════════════════════════════════════════════════════════
|
|
23
23
|
|
|
24
|
-
|
|
24
|
+
// 目标国家:每轮 tick 前 reloadConfig() 重读配置文件,捕捉服务端(设置页)修改
|
|
25
|
+
function currentLocations() {
|
|
26
|
+
return getTargetLocations();
|
|
27
|
+
}
|
|
25
28
|
|
|
26
29
|
const FILTER = "video_count>6 AND follower_count>0 AND following_count>0";
|
|
27
30
|
|
|
@@ -161,7 +164,7 @@ const vals = (row, cols) => cols.map((c) => row[c] ?? null);
|
|
|
161
164
|
const ph = (n) => Array(n).fill("?").join(", ");
|
|
162
165
|
|
|
163
166
|
/** 构建 WHERE 子句(含国家过滤)。strict=true 时仅匹配明确国家,不含 NULL/空 */
|
|
164
|
-
function buildWhere(locations =
|
|
167
|
+
function buildWhere(locations = currentLocations(), strict = false) {
|
|
165
168
|
const locs = [
|
|
166
169
|
...new Set(
|
|
167
170
|
locations.map((l) => String(l).trim().toUpperCase()).filter(Boolean),
|
|
@@ -268,14 +271,14 @@ function countPendingByCountry() {
|
|
|
268
271
|
const map = {};
|
|
269
272
|
for (const r of rows) map[r.loc] = r.c;
|
|
270
273
|
// 确保所有目标国家都有值
|
|
271
|
-
for (const loc of
|
|
274
|
+
for (const loc of currentLocations()) {
|
|
272
275
|
if (!(loc in map)) map[loc] = 0;
|
|
273
276
|
}
|
|
274
277
|
return map;
|
|
275
278
|
}
|
|
276
279
|
|
|
277
280
|
function findNeedyCountries(byCountry, threshold) {
|
|
278
|
-
return
|
|
281
|
+
return currentLocations().filter((loc) => (byCountry[loc] || 0) < threshold);
|
|
279
282
|
}
|
|
280
283
|
|
|
281
284
|
// ═══════════════════════════════════════════════════════════════════
|
|
@@ -335,7 +338,7 @@ async function refillLLM(archiveDb, needyLocations) {
|
|
|
335
338
|
|
|
336
339
|
// 2. LLM 打分
|
|
337
340
|
const ts = Date.now();
|
|
338
|
-
const scores = await scoreJobsBatch(samples,
|
|
341
|
+
const scores = await scoreJobsBatch(samples, currentLocations());
|
|
339
342
|
log(" ", `${loc} 打分${scores.length}条 ${Date.now() - ts}ms`);
|
|
340
343
|
|
|
341
344
|
if (
|
|
@@ -406,6 +409,7 @@ async function tick(archiveDb, opts) {
|
|
|
406
409
|
const t0 = Date.now();
|
|
407
410
|
|
|
408
411
|
archiveNewUsers(archiveDb);
|
|
412
|
+
reloadConfig(); // 每轮重读目标国家配置,支持服务端(设置页)修改
|
|
409
413
|
if (opts.llm) reloadLlmConfig(); // 每轮重载,支持热切换模型
|
|
410
414
|
|
|
411
415
|
const byCountry = countPendingByCountry();
|
|
@@ -414,7 +418,9 @@ async function tick(archiveDb, opts) {
|
|
|
414
418
|
if (!needy.length) {
|
|
415
419
|
log(
|
|
416
420
|
"--",
|
|
417
|
-
`轮${tickN} 全部充足 | ${
|
|
421
|
+
`轮${tickN} 全部充足 | ${currentLocations()
|
|
422
|
+
.map((l) => `${l}:${byCountry[l]}`)
|
|
423
|
+
.join(" ")}`,
|
|
418
424
|
);
|
|
419
425
|
return;
|
|
420
426
|
}
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
// 临时验证:fetchUserVideosAPI 对 HTTP 429 / ratelimit triggered 判定为"被封"
|
|
2
|
+
// 运行:RM_BLOCKED_WAIT_MS=1 node scripts/test-api-interceptor-429.mjs
|
|
3
|
+
import { fetchUserVideosAPI } from "../src/lib/api-interceptor.js";
|
|
4
|
+
|
|
5
|
+
// 构造一个可被 waitForResponse 返回的 429 响应
|
|
6
|
+
function make429Response() {
|
|
7
|
+
return {
|
|
8
|
+
url: () => "https://www.tiktok.com/api/post/item_list/?...",
|
|
9
|
+
status: () => 429,
|
|
10
|
+
headers: () => ({ "content-length": "19" }),
|
|
11
|
+
json: async () => {
|
|
12
|
+
throw new Error("Unexpected token 'r'...");
|
|
13
|
+
},
|
|
14
|
+
body: async () => Buffer.from("ratelimit triggered"),
|
|
15
|
+
};
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
// 构造一个正常响应(itemList 非空)
|
|
19
|
+
function make200Response(username) {
|
|
20
|
+
const itemList = [
|
|
21
|
+
{ id: "111", stats: { playCount: 1 }, isECVideo: 0, isAd: 0 },
|
|
22
|
+
];
|
|
23
|
+
return {
|
|
24
|
+
url: () => "https://www.tiktok.com/api/post/item_list/?...",
|
|
25
|
+
status: () => 200,
|
|
26
|
+
headers: () => ({}),
|
|
27
|
+
json: async () => ({ itemList, cursor: 0, hasMore: false }),
|
|
28
|
+
body: async () => Buffer.from(JSON.stringify({ itemList })),
|
|
29
|
+
username,
|
|
30
|
+
};
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
// 构造 mock page:封禁场景时 waitForResponse 返回 429 响应
|
|
34
|
+
function makePage(username, respFactory) {
|
|
35
|
+
const page = {
|
|
36
|
+
_handlers: {},
|
|
37
|
+
on(event, fn) {
|
|
38
|
+
(page._handlers[event] ||= []).push(fn);
|
|
39
|
+
return page;
|
|
40
|
+
},
|
|
41
|
+
off(event) {
|
|
42
|
+
delete page._handlers[event];
|
|
43
|
+
return page;
|
|
44
|
+
},
|
|
45
|
+
url: () => `https://www.tiktok.com/@${username}`,
|
|
46
|
+
goto: async () => {},
|
|
47
|
+
// checkCdpHealth 用 () => document.readyState(fn.length===0)→ 返回 "complete";
|
|
48
|
+
// detectPageError 用 (patterns)=>...(fn.length===1)→ 返回 null(页面无错误文案)
|
|
49
|
+
evaluate: async (fn) => (fn.length === 1 ? null : "complete"),
|
|
50
|
+
reload: async () => {},
|
|
51
|
+
waitForFunction: async () => null,
|
|
52
|
+
waitForResponse: async (pred, opts) => {
|
|
53
|
+
const resp = respFactory();
|
|
54
|
+
if (!pred(resp)) throw new Error("response 不匹配 item_list");
|
|
55
|
+
return resp;
|
|
56
|
+
},
|
|
57
|
+
};
|
|
58
|
+
return page;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
let pass = 0;
|
|
62
|
+
let fail = 0;
|
|
63
|
+
|
|
64
|
+
async function expectReject(name, fn, expectedMsg) {
|
|
65
|
+
try {
|
|
66
|
+
await fn();
|
|
67
|
+
fail++;
|
|
68
|
+
console.error(`✗ ${name}: 未抛错(期望 "${expectedMsg}")`);
|
|
69
|
+
} catch (e) {
|
|
70
|
+
if (e.message === expectedMsg || e.message.includes(expectedMsg)) {
|
|
71
|
+
pass++;
|
|
72
|
+
console.log(`✓ ${name}: 抛错 "${e.message}"`);
|
|
73
|
+
} else {
|
|
74
|
+
fail++;
|
|
75
|
+
console.error(
|
|
76
|
+
`✗ ${name}: 抛错 "${e.message}"(期望含 "${expectedMsg}")`,
|
|
77
|
+
);
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
async function expectResolve(name, fn) {
|
|
83
|
+
try {
|
|
84
|
+
const r = await fn();
|
|
85
|
+
if (r instanceof Map && r.size === 1) {
|
|
86
|
+
pass++;
|
|
87
|
+
console.log(`✓ ${name}: 正常返回 ${r.size} 个视频`);
|
|
88
|
+
} else {
|
|
89
|
+
fail++;
|
|
90
|
+
console.error(`✗ ${name}: 返回异常 ${JSON.stringify(r)}`);
|
|
91
|
+
}
|
|
92
|
+
} catch (e) {
|
|
93
|
+
fail++;
|
|
94
|
+
console.error(`✗ ${name}: 抛错 ${e.message}`);
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
// 用例1:429 + ratelimit triggered → 判定为"被封"
|
|
99
|
+
await expectReject(
|
|
100
|
+
"HTTP 429 ratelimit triggered 判为被封",
|
|
101
|
+
() =>
|
|
102
|
+
fetchUserVideosAPI(
|
|
103
|
+
makePage("fumolbbf08o", make429Response),
|
|
104
|
+
"fumolbbf08o",
|
|
105
|
+
5,
|
|
106
|
+
() => {},
|
|
107
|
+
),
|
|
108
|
+
"被封: fumolbbf08o",
|
|
109
|
+
);
|
|
110
|
+
|
|
111
|
+
// 用例2:正常 200 itemList → 不误判,正常返回视频
|
|
112
|
+
await expectResolve("HTTP 200 正常 itemList 不被误伤", () =>
|
|
113
|
+
fetchUserVideosAPI(
|
|
114
|
+
makePage("fumolbbf08o", make200Response),
|
|
115
|
+
"fumolbbf08o",
|
|
116
|
+
5,
|
|
117
|
+
() => {},
|
|
118
|
+
),
|
|
119
|
+
);
|
|
120
|
+
|
|
121
|
+
console.log(`\n结果: ${pass} 通过, ${fail} 失败`);
|
|
122
|
+
process.exit(fail > 0 ? 1 : 0);
|
package/src/cli/comments.js
CHANGED
|
@@ -23,10 +23,10 @@ import {
|
|
|
23
23
|
saveUserId,
|
|
24
24
|
} from "../lib/constants.js";
|
|
25
25
|
import {
|
|
26
|
-
DEFAULT_TARGET_LOCATIONS,
|
|
27
26
|
isLocationInList,
|
|
28
27
|
normalizeLocation,
|
|
29
28
|
} from "../lib/target-locations.js";
|
|
29
|
+
import { getTargetLocations } from "../lib/app-config.js";
|
|
30
30
|
import { createApiClient } from "../lib/api-client.js";
|
|
31
31
|
import { ensureBrowserReady, killEdgeProcesses } from "../lib/browser/cdp.js";
|
|
32
32
|
import { delay } from "../lib/delay.js";
|
|
@@ -562,7 +562,7 @@ async function runAutoMode(options) {
|
|
|
562
562
|
console.error(`\n╔══════════════════════════════════════════╗`);
|
|
563
563
|
console.error(`║ 自动评论采集模式(CDP 连接已登录浏览器)`);
|
|
564
564
|
console.error(
|
|
565
|
-
`║ 目标国家: ${jobLocations ||
|
|
565
|
+
`║ 目标国家: ${jobLocations || getTargetLocations().slice(0, 6).join(",") + "..."}`,
|
|
566
566
|
);
|
|
567
567
|
if (jobLocations) console.error(`║ 任务过滤: ${jobLocations}`);
|
|
568
568
|
console.error(`║ 服务端: ${serverUrl}`);
|
|
@@ -636,7 +636,7 @@ async function runAutoMode(options) {
|
|
|
636
636
|
const taskNum = totalTasks() + 1;
|
|
637
637
|
|
|
638
638
|
// 非目标国家 → 跳过
|
|
639
|
-
if (loc && !isLocationInList(loc,
|
|
639
|
+
if (loc && !isLocationInList(loc, getTargetLocations())) {
|
|
640
640
|
await apiPut(`${serverUrl}/api/comment-task/${id}`);
|
|
641
641
|
skippedCount++;
|
|
642
642
|
console.error(
|
|
@@ -960,10 +960,10 @@ async function runManualMode(url, options) {
|
|
|
960
960
|
// ── --save: 去重保存到服务端 ──
|
|
961
961
|
if (
|
|
962
962
|
guessedLocation &&
|
|
963
|
-
!isLocationInList(guessedLocation,
|
|
963
|
+
!isLocationInList(guessedLocation, getTargetLocations())
|
|
964
964
|
) {
|
|
965
965
|
console.error(`\n猜测国家: ${guessedLocation},非目标国家,跳过保存`);
|
|
966
|
-
console.error(`目标国家: ${
|
|
966
|
+
console.error(`目标国家: ${getTargetLocations().join(", ")}`);
|
|
967
967
|
} else {
|
|
968
968
|
console.error(`\n正在提交 ${authors.length} 个评论作者到服务端...`);
|
|
969
969
|
console.error(`服务端: ${serverUrl}`);
|
package/src/cli/precheck.js
CHANGED
|
@@ -8,7 +8,7 @@ import {
|
|
|
8
8
|
} from "../lib/constants.js";
|
|
9
9
|
import { getMacOrUuid } from "../lib/mac-or-uuid.js";
|
|
10
10
|
import { createApiClient } from "../lib/api-client.js";
|
|
11
|
-
import {
|
|
11
|
+
import { getTargetLocations } from "../lib/app-config.js";
|
|
12
12
|
import v8 from "node:v8";
|
|
13
13
|
|
|
14
14
|
const HEAP_RESTART_RATIO = 0.72;
|
|
@@ -104,7 +104,7 @@ export async function handlePrecheck(options) {
|
|
|
104
104
|
const locations =
|
|
105
105
|
precheckCountries && precheckCountries.length > 0
|
|
106
106
|
? precheckCountries
|
|
107
|
-
:
|
|
107
|
+
: getTargetLocations();
|
|
108
108
|
|
|
109
109
|
precheckLog(`[Precheck] 服务端: ${serverUrl}`);
|
|
110
110
|
precheckLog(
|
package/src/cli/refresh-mini.js
CHANGED
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
* 需要切换时向 clash-daemon 发 failover 命令(fire-and-forget),
|
|
9
9
|
* 选节点、内容验证、失败回滚均由 daemon 完成
|
|
10
10
|
* 触发条件:定时轮换(--rotate-min)/ 验证码超时 / 连续错误 ≥ 3 / CDN ≥ 3
|
|
11
|
-
* 被封(0
|
|
11
|
+
* 被封(0 字节空响应 / HTTP 429 限流,api-interceptor 内部按 30/40/50s 重试 3 次)→ 仍封才触发切换
|
|
12
12
|
* - --ensure-daemon:启动时检查 daemon 是否存活,未存活则自动拉起(默认关闭)
|
|
13
13
|
*/
|
|
14
14
|
|
package/src/cli/tag.js
CHANGED
|
@@ -15,10 +15,8 @@ import { fetchTagData, enrichVideosWithLocation } from "../lib/tag-fetcher.js";
|
|
|
15
15
|
import { killEdgeProcesses, ensureBrowserReady } from "../lib/browser/cdp.js";
|
|
16
16
|
import { getOrCreatePage, isBrowserClosedError } from "../lib/browser/page.js";
|
|
17
17
|
import { TikTokScraper } from "../lib/tiktok-scraper.mjs";
|
|
18
|
-
import {
|
|
19
|
-
|
|
20
|
-
isLocationInList,
|
|
21
|
-
} from "../lib/target-locations.js";
|
|
18
|
+
import { isLocationInList } from "../lib/target-locations.js";
|
|
19
|
+
import { getTargetLocations } from "../lib/app-config.js";
|
|
22
20
|
import { delay as randomDelay } from "../lib/delay.js";
|
|
23
21
|
import { discoverTags } from "../lib/tag-discover.js";
|
|
24
22
|
import {
|
|
@@ -33,26 +31,10 @@ import {
|
|
|
33
31
|
closeCaptcha,
|
|
34
32
|
} from "../scraper/modules/captcha-handler.js";
|
|
35
33
|
|
|
36
|
-
const ALL_COUNTRIES = DEFAULT_TARGET_LOCATIONS;
|
|
37
34
|
const DEFAULT_SERVER = cfgServer || "http://127.0.0.1:3000";
|
|
38
|
-
const DEFAULT_SCORE_COUNTRIES = [
|
|
39
|
-
"ES",
|
|
40
|
-
"FR",
|
|
41
|
-
"DE",
|
|
42
|
-
"PT",
|
|
43
|
-
"IT",
|
|
44
|
-
"NL",
|
|
45
|
-
"BE",
|
|
46
|
-
"AT",
|
|
47
|
-
"IE",
|
|
48
|
-
"PL",
|
|
49
|
-
"CZ",
|
|
50
|
-
"GR",
|
|
51
|
-
"HU",
|
|
52
|
-
];
|
|
53
35
|
|
|
54
36
|
function resolveTargetCountries(countries) {
|
|
55
|
-
return countries ||
|
|
37
|
+
return countries || getTargetLocations();
|
|
56
38
|
}
|
|
57
39
|
|
|
58
40
|
/**
|
|
@@ -465,7 +447,7 @@ export async function handleDiscover(parsed) {
|
|
|
465
447
|
countries.length === 1 &&
|
|
466
448
|
countries[0].toUpperCase() === "ALL"
|
|
467
449
|
) {
|
|
468
|
-
countries =
|
|
450
|
+
countries = getTargetLocations();
|
|
469
451
|
}
|
|
470
452
|
|
|
471
453
|
if (!countries || countries.length === 0) {
|
|
@@ -1649,7 +1631,7 @@ export async function handleTag(parsed) {
|
|
|
1649
1631
|
.split(",")
|
|
1650
1632
|
.map((s) => s.trim().toUpperCase())
|
|
1651
1633
|
.filter(Boolean)
|
|
1652
|
-
:
|
|
1634
|
+
: getTargetLocations();
|
|
1653
1635
|
|
|
1654
1636
|
const autoEnrich = enrich || !!discover;
|
|
1655
1637
|
|
|
@@ -2129,7 +2111,7 @@ export async function handleTagRefresh(parsed) {
|
|
|
2129
2111
|
}
|
|
2130
2112
|
|
|
2131
2113
|
// 过滤目标国家的视频和作者
|
|
2132
|
-
const targetLocations =
|
|
2114
|
+
const targetLocations = getTargetLocations();
|
|
2133
2115
|
const filteredVideos = videos.filter((v) =>
|
|
2134
2116
|
isLocationInList(v.locationCreated, targetLocations),
|
|
2135
2117
|
);
|
|
@@ -172,6 +172,8 @@ async function attemptItemListFetch(page, username, log, apiRequestUrlRef) {
|
|
|
172
172
|
let responsePromise = null;
|
|
173
173
|
let response = null;
|
|
174
174
|
let rawBodyBytes = -1; // -1 未知(未读取到 body);0 = 服务端空响应(封禁判定依据)
|
|
175
|
+
let httpStatus = null; // 响应 HTTP 状态码
|
|
176
|
+
let bodyText = ""; // 响应体文本(识别 ratelimit triggered 等限流文案)
|
|
175
177
|
|
|
176
178
|
try {
|
|
177
179
|
// 响应监听必须在 goto 之前注册:domcontentloaded 返回时首个 item_list
|
|
@@ -208,6 +210,7 @@ async function attemptItemListFetch(page, username, log, apiRequestUrlRef) {
|
|
|
208
210
|
// JSON 解析失败(如 "Unexpected end of JSON input")时打印原始响应,
|
|
209
211
|
// 便于诊断空响应/截断响应/非 JSON 响应等异常形态
|
|
210
212
|
if (response) {
|
|
213
|
+
httpStatus = response.status();
|
|
211
214
|
try {
|
|
212
215
|
// body 读取同样可能因页面异常而挂起,加独立超时保护
|
|
213
216
|
const raw = await Promise.race([
|
|
@@ -218,6 +221,7 @@ async function attemptItemListFetch(page, username, log, apiRequestUrlRef) {
|
|
|
218
221
|
]);
|
|
219
222
|
const text = raw.toString("utf-8");
|
|
220
223
|
rawBodyBytes = raw.length;
|
|
224
|
+
bodyText = text;
|
|
221
225
|
// content-length 用于区分两种 0 字节形态:
|
|
222
226
|
// 声明长度 > 实际字节 = 传输被截断;声明缺失/为 0 = 服务端主动空响应
|
|
223
227
|
const declared = parseInt(
|
|
@@ -247,6 +251,8 @@ async function attemptItemListFetch(page, username, log, apiRequestUrlRef) {
|
|
|
247
251
|
data,
|
|
248
252
|
interceptionError,
|
|
249
253
|
rawBodyBytes,
|
|
254
|
+
httpStatus,
|
|
255
|
+
bodyText,
|
|
250
256
|
elapsed: Date.now() - t0,
|
|
251
257
|
};
|
|
252
258
|
}
|
|
@@ -343,8 +349,14 @@ async function fetchUserVideosAPI(
|
|
|
343
349
|
|
|
344
350
|
while (true) {
|
|
345
351
|
attempt++;
|
|
346
|
-
const {
|
|
347
|
-
|
|
352
|
+
const {
|
|
353
|
+
data,
|
|
354
|
+
interceptionError,
|
|
355
|
+
rawBodyBytes,
|
|
356
|
+
httpStatus,
|
|
357
|
+
bodyText,
|
|
358
|
+
elapsed,
|
|
359
|
+
} = await attemptItemListFetch(page, username, log, apiRequestUrlRef);
|
|
348
360
|
|
|
349
361
|
if (data && Array.isArray(data.itemList) && data.itemList.length > 0) {
|
|
350
362
|
log(` [API拦截] 成功,耗时 ${elapsed}ms`);
|
|
@@ -359,21 +371,27 @@ async function fetchUserVideosAPI(
|
|
|
359
371
|
);
|
|
360
372
|
}
|
|
361
373
|
|
|
362
|
-
//
|
|
363
|
-
//
|
|
364
|
-
//
|
|
365
|
-
|
|
374
|
+
// 判定为被封的两种形态,统一按封禁处理(默认 30/40/50 秒重试 3 次,仍封才抛错交给上层):
|
|
375
|
+
// 1. 0 字节空响应(content-length 缺失/为 0 或截断)—— 静默型封禁(页面正常渲染但 API 空响应)
|
|
376
|
+
// 2. HTTP 429 或 body 含 ratelimit —— 服务端主动限流(ratelimit triggered)
|
|
377
|
+
// 两种都不再依赖页面"出错了/很抱歉"文案判定,explore/refresh/refresh-mini 均受益
|
|
378
|
+
const rateLimited = httpStatus === 429 || /ratelimit/i.test(bodyText || "");
|
|
379
|
+
if (rawBodyBytes === 0 || rateLimited) {
|
|
380
|
+
const reason =
|
|
381
|
+
rawBodyBytes === 0
|
|
382
|
+
? "空响应(0 字节)"
|
|
383
|
+
: `限流(HTTP ${httpStatus}, ${bodyText.slice(0, 60)})`;
|
|
366
384
|
const retryIndex = attempt - 1;
|
|
367
385
|
if (retryIndex < retryWaits.length) {
|
|
368
386
|
const waitMs = retryWaits[retryIndex];
|
|
369
387
|
log(
|
|
370
|
-
` [API拦截] item_list
|
|
388
|
+
` [API拦截] item_list ${reason},判定为被封,${waitMs / 1000}s 后第 ${attempt}/${retryWaits.length} 次重试...`,
|
|
371
389
|
);
|
|
372
390
|
await new Promise((r) => setTimeout(r, waitMs));
|
|
373
391
|
continue;
|
|
374
392
|
}
|
|
375
393
|
log(
|
|
376
|
-
` [API拦截] item_list
|
|
394
|
+
` [API拦截] item_list ${reason},重试 ${retryWaits.length} 次仍被封`,
|
|
377
395
|
);
|
|
378
396
|
throw new Error(`被封: ${username}`);
|
|
379
397
|
}
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 通用应用配置模块(key-value,落盘到 ~/.tt-help-config.json)
|
|
3
|
+
*
|
|
4
|
+
* 设计为通用命名:任意配置项以 `key` 形式存放,后续可加入其他配置。
|
|
5
|
+
* 例如当前使用 `targetLocations`,未来可扩展 `xxx` 等。
|
|
6
|
+
*
|
|
7
|
+
* 读取规则:文件存在则用文件内容,否则回退内置默认值。
|
|
8
|
+
* 写入规则:整份 JSON 落盘(原子性不强,但配置变更频率低,可接受)。
|
|
9
|
+
*/
|
|
10
|
+
import { existsSync, readFileSync, writeFileSync } from "fs";
|
|
11
|
+
import { homedir } from "os";
|
|
12
|
+
import { join } from "path";
|
|
13
|
+
import { DEFAULT_TARGET_LOCATIONS } from "./target-locations.js";
|
|
14
|
+
|
|
15
|
+
// 通用配置文件路径(与 ~/.tt-help.json 隔离,避免与看门狗 clientConfig 写入冲突)
|
|
16
|
+
const CONFIG_FILE = join(homedir(), ".tt-help-config.json");
|
|
17
|
+
|
|
18
|
+
let _cache = null;
|
|
19
|
+
|
|
20
|
+
export function getConfigFilePath() {
|
|
21
|
+
return CONFIG_FILE;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
function readConfig() {
|
|
25
|
+
if (_cache) return _cache;
|
|
26
|
+
try {
|
|
27
|
+
if (existsSync(CONFIG_FILE)) {
|
|
28
|
+
const parsed = JSON.parse(readFileSync(CONFIG_FILE, "utf-8"));
|
|
29
|
+
_cache = parsed && typeof parsed === "object" ? parsed : {};
|
|
30
|
+
} else {
|
|
31
|
+
_cache = {};
|
|
32
|
+
}
|
|
33
|
+
} catch {
|
|
34
|
+
_cache = {};
|
|
35
|
+
}
|
|
36
|
+
return _cache;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* 通用读取:getConfig() 返回整份配置;getConfig(key) 返回指定项。
|
|
41
|
+
* @param {string|null} key
|
|
42
|
+
* @param {*} fallback
|
|
43
|
+
*/
|
|
44
|
+
export function getConfig(key = null, fallback = null) {
|
|
45
|
+
const cfg = readConfig();
|
|
46
|
+
if (key == null) return { ...cfg };
|
|
47
|
+
return cfg[key] !== undefined ? cfg[key] : fallback;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* 通用写入:设置指定 key 并落盘,返回更新后的整份配置。
|
|
52
|
+
* @param {string} key
|
|
53
|
+
* @param {*} value
|
|
54
|
+
*/
|
|
55
|
+
export function setConfig(key, value) {
|
|
56
|
+
const cfg = readConfig();
|
|
57
|
+
cfg[key] = value;
|
|
58
|
+
_cache = cfg;
|
|
59
|
+
try {
|
|
60
|
+
writeFileSync(CONFIG_FILE, JSON.stringify(cfg, null, 2), "utf-8");
|
|
61
|
+
} catch {
|
|
62
|
+
// 写失败不抛出,保证调用方不至于崩溃;仅保留内存值
|
|
63
|
+
}
|
|
64
|
+
return { ...cfg };
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* 通用删除:移除指定 key 并落盘,返回更新后的整份配置。
|
|
69
|
+
* @param {string} key
|
|
70
|
+
*/
|
|
71
|
+
export function removeConfig(key) {
|
|
72
|
+
const cfg = readConfig();
|
|
73
|
+
if (key in cfg) {
|
|
74
|
+
delete cfg[key];
|
|
75
|
+
_cache = cfg;
|
|
76
|
+
try {
|
|
77
|
+
writeFileSync(CONFIG_FILE, JSON.stringify(cfg, null, 2), "utf-8");
|
|
78
|
+
} catch {
|
|
79
|
+
// 写失败忽略,仅保留内存值
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
return { ...cfg };
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
function normalizeLocations(locations) {
|
|
86
|
+
if (!Array.isArray(locations)) return [];
|
|
87
|
+
return [
|
|
88
|
+
...new Set(
|
|
89
|
+
locations.map((l) => String(l).trim().toUpperCase()).filter(Boolean),
|
|
90
|
+
),
|
|
91
|
+
];
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/** 读取当前目标国家,未配置则回退 DEFAULT_TARGET_LOCATIONS */
|
|
95
|
+
export function getTargetLocations() {
|
|
96
|
+
const raw = getConfig("targetLocations", []);
|
|
97
|
+
const normalized = normalizeLocations(raw);
|
|
98
|
+
return normalized.length > 0 ? normalized : [...DEFAULT_TARGET_LOCATIONS];
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/**
|
|
102
|
+
* 保存目标国家(自动去重、大写)。
|
|
103
|
+
* 传入空数组/非法值 = 移除配置项,回退到内置默认列表。
|
|
104
|
+
* 始终返回"生效后的列表"(getTargetLocations()),保证响应与读取一致。
|
|
105
|
+
*/
|
|
106
|
+
export function setTargetLocations(locations) {
|
|
107
|
+
const normalized = normalizeLocations(locations);
|
|
108
|
+
if (normalized.length === 0) {
|
|
109
|
+
removeConfig("targetLocations");
|
|
110
|
+
} else {
|
|
111
|
+
setConfig("targetLocations", normalized);
|
|
112
|
+
}
|
|
113
|
+
return getTargetLocations();
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
/** 清除内存缓存,强制下次读盘(配置被外部修改时调用) */
|
|
117
|
+
export function reloadConfig() {
|
|
118
|
+
_cache = null;
|
|
119
|
+
}
|
package/src/lib/args.js
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import { readFileSync } from "fs";
|
|
2
2
|
import { server as defaultServer } from "./constants.js";
|
|
3
|
-
import {
|
|
3
|
+
import { getTargetLocations } from "./app-config.js";
|
|
4
|
+
|
|
5
|
+
const DEFAULT_TARGET_LOCATIONS_CSV = getTargetLocations().join(",");
|
|
4
6
|
|
|
5
7
|
const PRESETS = ["fast", "normal", "slow", "stealth"];
|
|
6
8
|
|
package/src/lib/constants.js
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
import { join, dirname } from "path";
|
|
2
2
|
import { readFileSync, writeFileSync, existsSync } from "fs";
|
|
3
3
|
import { fileURLToPath } from "url";
|
|
4
|
-
import {
|
|
4
|
+
import { getTargetLocations } from "./app-config.js";
|
|
5
|
+
|
|
6
|
+
const DEFAULT_TARGET_LOCATIONS_CSV = getTargetLocations().join(",");
|
|
5
7
|
|
|
6
8
|
const __filename = fileURLToPath(import.meta.url);
|
|
7
9
|
const __dirname = dirname(__filename);
|
package/src/scraper/core.js
CHANGED
|
@@ -11,17 +11,17 @@ import {
|
|
|
11
11
|
import { extractCommentAuthors } from "./modules/comment-extractor.js";
|
|
12
12
|
import { extractGuessVideos } from "./modules/guess-extractor.js";
|
|
13
13
|
import {
|
|
14
|
-
DEFAULT_TARGET_LOCATIONS_CSV,
|
|
15
14
|
isLocationInList,
|
|
16
15
|
normalizeLocationList,
|
|
17
16
|
} from "../lib/target-locations.js";
|
|
17
|
+
import { getTargetLocations } from "../lib/app-config.js";
|
|
18
18
|
|
|
19
19
|
async function scrapeSingleVideo(
|
|
20
20
|
page,
|
|
21
21
|
maxComments,
|
|
22
22
|
maxGuess,
|
|
23
23
|
log,
|
|
24
|
-
location =
|
|
24
|
+
location = getTargetLocations().join(","),
|
|
25
25
|
) {
|
|
26
26
|
const config = getDelayConfig();
|
|
27
27
|
|
|
@@ -5,12 +5,12 @@ import { getUserInfo, collectVideos } from "../videos/core.js";
|
|
|
5
5
|
import { extractFollowAndFollowers } from "./modules/follow-extractor.js";
|
|
6
6
|
import { extractVideoLocation, setScraperProxy } from "../lib/scrape.js";
|
|
7
7
|
import {
|
|
8
|
-
DEFAULT_TARGET_LOCATIONS_CSV,
|
|
9
8
|
findBestMatchingLocation,
|
|
10
9
|
isLocationInList,
|
|
11
10
|
normalizeLocation,
|
|
12
11
|
normalizeLocationList,
|
|
13
12
|
} from "../lib/target-locations.js";
|
|
13
|
+
import { getTargetLocations } from "../lib/app-config.js";
|
|
14
14
|
import {
|
|
15
15
|
maxFollowing as globalMaxFollowing,
|
|
16
16
|
maxFollowers as globalMaxFollowers,
|
|
@@ -23,7 +23,7 @@ async function processExplore(page, username, options, log) {
|
|
|
23
23
|
loggedIn = false, // 由外部传入登录状态,避免每次调用 isLoggedIn(page)
|
|
24
24
|
maxFollowing = 50,
|
|
25
25
|
maxFollowers = 50,
|
|
26
|
-
location =
|
|
26
|
+
location = getTargetLocations().join(","),
|
|
27
27
|
locationMode = "explore", // "explore" | "refresh"
|
|
28
28
|
proxyServer = null,
|
|
29
29
|
webserverUrl = null,
|
package/src/watch/data-store.js
CHANGED
|
@@ -19,10 +19,8 @@
|
|
|
19
19
|
import fs from "fs";
|
|
20
20
|
import path from "path";
|
|
21
21
|
import Database from "better-sqlite3";
|
|
22
|
-
import {
|
|
23
|
-
|
|
24
|
-
DEFAULT_TARGET_LOCATIONS,
|
|
25
|
-
} from "../lib/target-locations.js";
|
|
22
|
+
import { isLocationInList } from "../lib/target-locations.js";
|
|
23
|
+
import { getTargetLocations } from "../lib/app-config.js";
|
|
26
24
|
|
|
27
25
|
// Schema 与连接管理
|
|
28
26
|
import {
|
|
@@ -665,7 +663,7 @@ function getModifiedTargetUsersOutsideLocations(targetLocations = []) {
|
|
|
665
663
|
function getTargetRefreshStats(hours = 24) {
|
|
666
664
|
if (!getDb()) return { countries: [], total: 0, refreshed: 0 };
|
|
667
665
|
|
|
668
|
-
const targetLocations =
|
|
666
|
+
const targetLocations = getTargetLocations();
|
|
669
667
|
const placeholders = targetLocations.map(() => "?").join(", ");
|
|
670
668
|
|
|
671
669
|
// 按国家统计总数
|
|
@@ -2226,7 +2224,7 @@ export function createStore(filePath, options = {}) {
|
|
|
2226
2224
|
const now = Date.now();
|
|
2227
2225
|
const threshold = now - maxAgeSeconds * 1000;
|
|
2228
2226
|
const defaultTime = new Date("2016-01-01T00:00:00Z").getTime();
|
|
2229
|
-
const targetLocations = locations ||
|
|
2227
|
+
const targetLocations = locations || getTargetLocations();
|
|
2230
2228
|
const placeholders = targetLocations.map(() => "?").join(",");
|
|
2231
2229
|
const row = getDb()
|
|
2232
2230
|
.prepare(
|
|
@@ -2261,7 +2259,7 @@ export function createStore(filePath, options = {}) {
|
|
|
2261
2259
|
const defaultTime = new Date("2016-01-01T00:00:00Z").getTime();
|
|
2262
2260
|
|
|
2263
2261
|
// 筛选目标国家用户,按 refreshTime 升序取最远的(没有则默认 2016-01-01)
|
|
2264
|
-
const targetLocations = locations ||
|
|
2262
|
+
const targetLocations = locations || getTargetLocations();
|
|
2265
2263
|
const targetUsers = data.filter(
|
|
2266
2264
|
(u) =>
|
|
2267
2265
|
u.ttSeller &&
|
|
@@ -2405,7 +2403,7 @@ export function createStore(filePath, options = {}) {
|
|
|
2405
2403
|
const now = Date.now();
|
|
2406
2404
|
const threshold = now - intervalSeconds * 1000;
|
|
2407
2405
|
const defaultTime = new Date("2016-01-01T00:00:00Z").getTime();
|
|
2408
|
-
const targetLocations = locations ||
|
|
2406
|
+
const targetLocations = locations || getTargetLocations();
|
|
2409
2407
|
const placeholders = targetLocations.map(() => "?").join(",");
|
|
2410
2408
|
|
|
2411
2409
|
// SELECT 候选用户
|
package/src/watch/public/app.js
CHANGED
|
@@ -683,7 +683,7 @@ function closeAddModal() {
|
|
|
683
683
|
|
|
684
684
|
// ========== 国家修改模态框 ==========
|
|
685
685
|
|
|
686
|
-
const
|
|
686
|
+
const DEFAULT_LOCATION_OPTIONS = [
|
|
687
687
|
"AT",
|
|
688
688
|
"BE",
|
|
689
689
|
"CZ",
|
|
@@ -697,19 +697,35 @@ const TARGET_LOCATIONS = [
|
|
|
697
697
|
"NL",
|
|
698
698
|
"PL",
|
|
699
699
|
"PT",
|
|
700
|
+
"TH",
|
|
701
|
+
"US",
|
|
700
702
|
];
|
|
703
|
+
let locationOptions = [...DEFAULT_LOCATION_OPTIONS];
|
|
701
704
|
|
|
702
|
-
function openLocationModal(uniqueId, currentLocation) {
|
|
705
|
+
async function openLocationModal(uniqueId, currentLocation) {
|
|
703
706
|
let overlay = document.getElementById("locationModalOverlay");
|
|
704
707
|
if (overlay) return;
|
|
708
|
+
// 优先使用服务端配置的目标国家作为候选,失败则回退默认
|
|
709
|
+
try {
|
|
710
|
+
const res = await fetch("/api/settings/target-locations");
|
|
711
|
+
const data = await res.json();
|
|
712
|
+
const locs = (data.locations || []).filter(Boolean);
|
|
713
|
+
if (locs.length) {
|
|
714
|
+
locationOptions = [...new Set([...locs, ...DEFAULT_LOCATION_OPTIONS])];
|
|
715
|
+
}
|
|
716
|
+
} catch {
|
|
717
|
+
locationOptions = [...DEFAULT_LOCATION_OPTIONS];
|
|
718
|
+
}
|
|
705
719
|
overlay = document.createElement("div");
|
|
706
720
|
overlay.id = "locationModalOverlay";
|
|
707
721
|
overlay.className = "modal-overlay";
|
|
708
722
|
const safeId = escapeJsString(uniqueId);
|
|
709
|
-
const options =
|
|
710
|
-
(
|
|
711
|
-
|
|
712
|
-
|
|
723
|
+
const options = locationOptions
|
|
724
|
+
.map(
|
|
725
|
+
(loc) =>
|
|
726
|
+
`<button class="loc-option ${loc === currentLocation ? "active" : ""}" onclick="selectLocation('${safeId}','${loc}')">${loc}</button>`,
|
|
727
|
+
)
|
|
728
|
+
.join("");
|
|
713
729
|
overlay.innerHTML = `
|
|
714
730
|
<div class="modal" style="max-width:420px">
|
|
715
731
|
<h3>修改用户国家</h3>
|
|
@@ -1392,6 +1408,157 @@ function showSettingsPage() {
|
|
|
1392
1408
|
hideAllPages();
|
|
1393
1409
|
document.getElementById("settingsPage").classList.add("active");
|
|
1394
1410
|
fetchLlmConfig();
|
|
1411
|
+
loadTargetLocations();
|
|
1412
|
+
}
|
|
1413
|
+
|
|
1414
|
+
// ========== 目标国家配置(通用 app-config,落盘 ~/.tt-help-config.json) ==========
|
|
1415
|
+
|
|
1416
|
+
const AVAILABLE_COUNTRIES = [
|
|
1417
|
+
"AT",
|
|
1418
|
+
"BE",
|
|
1419
|
+
"CZ",
|
|
1420
|
+
"DE",
|
|
1421
|
+
"ES",
|
|
1422
|
+
"FR",
|
|
1423
|
+
"GR",
|
|
1424
|
+
"HU",
|
|
1425
|
+
"IE",
|
|
1426
|
+
"IT",
|
|
1427
|
+
"NL",
|
|
1428
|
+
"PL",
|
|
1429
|
+
"PT",
|
|
1430
|
+
"TH",
|
|
1431
|
+
"US",
|
|
1432
|
+
"GB",
|
|
1433
|
+
"SE",
|
|
1434
|
+
"NO",
|
|
1435
|
+
"FI",
|
|
1436
|
+
"DK",
|
|
1437
|
+
"RO",
|
|
1438
|
+
"BG",
|
|
1439
|
+
"CH",
|
|
1440
|
+
"SK",
|
|
1441
|
+
"SI",
|
|
1442
|
+
"HR",
|
|
1443
|
+
"RS",
|
|
1444
|
+
"UA",
|
|
1445
|
+
"TR",
|
|
1446
|
+
"MX",
|
|
1447
|
+
"BR",
|
|
1448
|
+
"AR",
|
|
1449
|
+
"CL",
|
|
1450
|
+
"CO",
|
|
1451
|
+
];
|
|
1452
|
+
|
|
1453
|
+
function normalizeCountryInput(str) {
|
|
1454
|
+
return [
|
|
1455
|
+
...new Set(
|
|
1456
|
+
String(str || "")
|
|
1457
|
+
.split(",")
|
|
1458
|
+
.map((s) => s.trim().toUpperCase())
|
|
1459
|
+
.filter(Boolean),
|
|
1460
|
+
),
|
|
1461
|
+
];
|
|
1462
|
+
}
|
|
1463
|
+
|
|
1464
|
+
async function loadTargetLocations() {
|
|
1465
|
+
const input = document.getElementById("targetLocationsInput");
|
|
1466
|
+
const status = document.getElementById("targetLocationsStatus");
|
|
1467
|
+
if (!input) return;
|
|
1468
|
+
try {
|
|
1469
|
+
const res = await fetch("/api/settings/target-locations");
|
|
1470
|
+
const data = await res.json();
|
|
1471
|
+
const locations = data.locations || [];
|
|
1472
|
+
input.value = locations.join(",");
|
|
1473
|
+
renderTargetCountryGrid(locations);
|
|
1474
|
+
if (status) {
|
|
1475
|
+
status.textContent = `当前 ${locations.length} 个国家: ${locations.join(", ")}`;
|
|
1476
|
+
status.className = "settings-status ok";
|
|
1477
|
+
}
|
|
1478
|
+
} catch (e) {
|
|
1479
|
+
console.error("loadTargetLocations error:", e);
|
|
1480
|
+
if (status) {
|
|
1481
|
+
status.textContent = "读取目标国家失败";
|
|
1482
|
+
status.className = "settings-status warn";
|
|
1483
|
+
}
|
|
1484
|
+
}
|
|
1485
|
+
}
|
|
1486
|
+
|
|
1487
|
+
function renderTargetCountryGrid(selected) {
|
|
1488
|
+
const grid = document.getElementById("targetCountryGrid");
|
|
1489
|
+
if (!grid) return;
|
|
1490
|
+
const selectedSet = new Set(selected || []);
|
|
1491
|
+
grid.innerHTML = AVAILABLE_COUNTRIES.map(
|
|
1492
|
+
(loc) =>
|
|
1493
|
+
`<button type="button" class="loc-option ${selectedSet.has(loc) ? "active" : ""}" onclick="toggleTargetCountry('${loc}')">${loc}</button>`,
|
|
1494
|
+
).join("");
|
|
1495
|
+
}
|
|
1496
|
+
|
|
1497
|
+
function toggleTargetCountry(loc) {
|
|
1498
|
+
const input = document.getElementById("targetLocationsInput");
|
|
1499
|
+
if (!input) return;
|
|
1500
|
+
let current = normalizeCountryInput(input.value);
|
|
1501
|
+
if (current.includes(loc)) {
|
|
1502
|
+
current = current.filter((l) => l !== loc);
|
|
1503
|
+
} else {
|
|
1504
|
+
current.push(loc);
|
|
1505
|
+
}
|
|
1506
|
+
input.value = current.join(",");
|
|
1507
|
+
renderTargetCountryGrid(current);
|
|
1508
|
+
}
|
|
1509
|
+
|
|
1510
|
+
async function saveTargetLocations() {
|
|
1511
|
+
const input = document.getElementById("targetLocationsInput");
|
|
1512
|
+
const status = document.getElementById("targetLocationsStatus");
|
|
1513
|
+
if (!input) return;
|
|
1514
|
+
const locations = normalizeCountryInput(input.value);
|
|
1515
|
+
status.textContent = "保存中...";
|
|
1516
|
+
status.className = "settings-status loading";
|
|
1517
|
+
try {
|
|
1518
|
+
const res = await fetch("/api/settings/target-locations", {
|
|
1519
|
+
method: "POST",
|
|
1520
|
+
headers: { "Content-Type": "application/json" },
|
|
1521
|
+
body: JSON.stringify({ locations }),
|
|
1522
|
+
});
|
|
1523
|
+
const data = await res.json();
|
|
1524
|
+
input.value = (data.locations || []).join(",");
|
|
1525
|
+
renderTargetCountryGrid(data.locations || []);
|
|
1526
|
+
status.textContent = `已保存 ${(data.locations || []).length} 个国家`;
|
|
1527
|
+
status.className = "settings-status ok";
|
|
1528
|
+
} catch (e) {
|
|
1529
|
+
console.error("saveTargetLocations error:", e);
|
|
1530
|
+
status.textContent = "保存失败";
|
|
1531
|
+
status.className = "settings-status warn";
|
|
1532
|
+
}
|
|
1533
|
+
}
|
|
1534
|
+
|
|
1535
|
+
async function resetTargetLocations() {
|
|
1536
|
+
const status = document.getElementById("targetLocationsStatus");
|
|
1537
|
+
if (status) {
|
|
1538
|
+
status.textContent = "保存中...";
|
|
1539
|
+
status.className = "settings-status loading";
|
|
1540
|
+
}
|
|
1541
|
+
try {
|
|
1542
|
+
const res = await fetch("/api/settings/target-locations", {
|
|
1543
|
+
method: "POST",
|
|
1544
|
+
headers: { "Content-Type": "application/json" },
|
|
1545
|
+
body: JSON.stringify({ locations: [] }),
|
|
1546
|
+
});
|
|
1547
|
+
const data = await res.json();
|
|
1548
|
+
const input = document.getElementById("targetLocationsInput");
|
|
1549
|
+
if (input) input.value = (data.locations || []).join(",");
|
|
1550
|
+
renderTargetCountryGrid(data.locations || []);
|
|
1551
|
+
if (status) {
|
|
1552
|
+
status.textContent = `已恢复默认 ${(data.locations || []).length} 个国家`;
|
|
1553
|
+
status.className = "settings-status ok";
|
|
1554
|
+
}
|
|
1555
|
+
} catch (e) {
|
|
1556
|
+
console.error("resetTargetLocations error:", e);
|
|
1557
|
+
if (status) {
|
|
1558
|
+
status.textContent = "恢复默认失败";
|
|
1559
|
+
status.className = "settings-status warn";
|
|
1560
|
+
}
|
|
1561
|
+
}
|
|
1395
1562
|
}
|
|
1396
1563
|
|
|
1397
1564
|
let _llmProvidersCache = null;
|
|
@@ -1596,27 +1763,14 @@ function openAddTagModal() {
|
|
|
1596
1763
|
if (e.target === overlay) closeAddTagModal();
|
|
1597
1764
|
});
|
|
1598
1765
|
|
|
1599
|
-
//
|
|
1766
|
+
// 渲染国家复选框(使用动态目标国家列表,含 TH/US,失败回退默认)
|
|
1600
1767
|
const countryGrid = document.getElementById("addTagCountryCheckboxes");
|
|
1601
|
-
|
|
1602
|
-
|
|
1603
|
-
|
|
1604
|
-
|
|
1605
|
-
|
|
1606
|
-
"
|
|
1607
|
-
"FR",
|
|
1608
|
-
"GR",
|
|
1609
|
-
"HU",
|
|
1610
|
-
"IE",
|
|
1611
|
-
"IT",
|
|
1612
|
-
"NL",
|
|
1613
|
-
"PL",
|
|
1614
|
-
"PT",
|
|
1615
|
-
];
|
|
1616
|
-
countryGrid.innerHTML = TARGET_LOCATIONS.map(
|
|
1617
|
-
(c) =>
|
|
1618
|
-
`<label class="checkbox-label"><input type="checkbox" value="${c}" checked> ${c}</label>`,
|
|
1619
|
-
).join("");
|
|
1768
|
+
countryGrid.innerHTML = locationOptions
|
|
1769
|
+
.map(
|
|
1770
|
+
(c) =>
|
|
1771
|
+
`<label class="checkbox-label"><input type="checkbox" value="${c}" checked> ${c}</label>`,
|
|
1772
|
+
)
|
|
1773
|
+
.join("");
|
|
1620
1774
|
|
|
1621
1775
|
const ta = document.getElementById("addTagModalInput");
|
|
1622
1776
|
ta.focus();
|
|
@@ -20,7 +20,7 @@
|
|
|
20
20
|
<input type="checkbox" id="fireworkToggle" checked onchange="toggleFireworks()">
|
|
21
21
|
<span>🎆</span>
|
|
22
22
|
</label>
|
|
23
|
-
<a class="script-link" onclick="navigateToSettings()" style="cursor:pointer">⚙️
|
|
23
|
+
<a class="script-link" onclick="navigateToSettings()" style="cursor:pointer">⚙️ 配置</a>
|
|
24
24
|
<a class="script-link" onclick="navigateToWatchdog()" style="cursor:pointer">🐕 看门狗</a>
|
|
25
25
|
<span class="status" id="lastUpdate">--</span>
|
|
26
26
|
</div>
|
|
@@ -488,6 +488,26 @@
|
|
|
488
488
|
</div>
|
|
489
489
|
</div>
|
|
490
490
|
</div>
|
|
491
|
+
<div class="settings-card">
|
|
492
|
+
<h3>🎯 目标国家</h3>
|
|
493
|
+
<p class="settings-hint">设置采集的目标国家列表,保存后写入通用配置文件(~/.tt-help-config.json)。当前仅保存配置,后续接入采集流程后生效。</p>
|
|
494
|
+
<div class="settings-form">
|
|
495
|
+
<div class="settings-field">
|
|
496
|
+
<label>目标国家(英文国家代码,逗号分隔,如 CZ,GR,TH)</label>
|
|
497
|
+
<input type="text" id="targetLocationsInput" placeholder="CZ,GR,HU,PT,ES,PL,NL,BE,DE,FR,IT,IE,AT,TH,US">
|
|
498
|
+
<span class="settings-note">可手动输入,或点击下方国家快速选择。</span>
|
|
499
|
+
</div>
|
|
500
|
+
<div class="settings-field">
|
|
501
|
+
<label>快速选择</label>
|
|
502
|
+
<div class="loc-grid" id="targetCountryGrid"></div>
|
|
503
|
+
</div>
|
|
504
|
+
<div class="settings-status" id="targetLocationsStatus"></div>
|
|
505
|
+
<div class="btn-row">
|
|
506
|
+
<button class="btn-submit" onclick="saveTargetLocations()">保存目标国家</button>
|
|
507
|
+
<button class="btn-cancel" onclick="resetTargetLocations()">恢复默认</button>
|
|
508
|
+
</div>
|
|
509
|
+
</div>
|
|
510
|
+
</div>
|
|
491
511
|
</div>
|
|
492
512
|
<div id="watchdogPage">
|
|
493
513
|
<div class="stats" style="margin-bottom:16px">
|
|
@@ -1834,6 +1834,22 @@ td.user-id:hover {
|
|
|
1834
1834
|
background: #e61944;
|
|
1835
1835
|
}
|
|
1836
1836
|
|
|
1837
|
+
.settings-form .btn-cancel {
|
|
1838
|
+
padding: 10px 20px;
|
|
1839
|
+
border: 1px solid #333;
|
|
1840
|
+
border-radius: 8px;
|
|
1841
|
+
background: #2a2a3a;
|
|
1842
|
+
color: #ccc;
|
|
1843
|
+
font-size: 14px;
|
|
1844
|
+
font-weight: 600;
|
|
1845
|
+
cursor: pointer;
|
|
1846
|
+
transition: background 0.2s;
|
|
1847
|
+
}
|
|
1848
|
+
|
|
1849
|
+
.settings-form .btn-cancel:hover {
|
|
1850
|
+
background: #333;
|
|
1851
|
+
}
|
|
1852
|
+
|
|
1837
1853
|
/* ── 看门狗页面 ── */
|
|
1838
1854
|
|
|
1839
1855
|
#watchdogPage {
|
package/src/watch/server.js
CHANGED
|
@@ -8,7 +8,7 @@ import { fileURLToPath } from "url";
|
|
|
8
8
|
import { spawn } from "child_process";
|
|
9
9
|
import { createStore } from "./data-store.js";
|
|
10
10
|
import { getTagDb } from "./db-schema.js";
|
|
11
|
-
import {
|
|
11
|
+
import { getTargetLocations } from "../lib/app-config.js";
|
|
12
12
|
import { tryHandleWatchdogRoute } from "./watchdog-server.js";
|
|
13
13
|
|
|
14
14
|
const __filename = fileURLToPath(import.meta.url);
|
|
@@ -44,11 +44,17 @@ const SLOW_STATS_TTL = 300000; // 5 分钟
|
|
|
44
44
|
const usersCache = new Map(); // key -> { result, time }
|
|
45
45
|
const USERS_CACHE_TTL = 3000; // 3 秒
|
|
46
46
|
|
|
47
|
-
// 目标国家占位符(jobs 表聚合需要 location_created IN (...)
|
|
48
|
-
|
|
49
|
-
const
|
|
50
|
-
|
|
51
|
-
|
|
47
|
+
// 目标国家占位符(jobs 表聚合需要 location_created IN (...),运行时按 getTargetLocations() 动态生成)
|
|
48
|
+
function targetPlaceholders() {
|
|
49
|
+
const locs = getTargetLocations();
|
|
50
|
+
return locs.map(() => "?").join(", ");
|
|
51
|
+
}
|
|
52
|
+
function targetInSql() {
|
|
53
|
+
const locs = getTargetLocations();
|
|
54
|
+
return locs.length
|
|
55
|
+
? `location_created IN (${targetPlaceholders()})`
|
|
56
|
+
: "1 = 0";
|
|
57
|
+
}
|
|
52
58
|
|
|
53
59
|
// 统计分组定义:每组包含若干 SQL(分片执行)与一个聚合函数
|
|
54
60
|
// sqls 项:{ key, sql, mode: 'one'|'all', params: 数组或返回数组的函数 }
|
|
@@ -80,8 +86,8 @@ const STAT_GROUPS = {
|
|
|
80
86
|
{
|
|
81
87
|
key: "jobsAgg",
|
|
82
88
|
mode: "one",
|
|
83
|
-
params:
|
|
84
|
-
sql: `SELECT
|
|
89
|
+
params: () => getTargetLocations(),
|
|
90
|
+
sql: () => `SELECT
|
|
85
91
|
SUM(jobsTotal) as jobsTotal,
|
|
86
92
|
SUM(done) as done,
|
|
87
93
|
SUM(error) as error,
|
|
@@ -94,7 +100,7 @@ const STAT_GROUPS = {
|
|
|
94
100
|
SUM(CASE WHEN status = 'done' THEN 1 ELSE 0 END) as done,
|
|
95
101
|
SUM(CASE WHEN status = 'error' THEN 1 ELSE 0 END) as error,
|
|
96
102
|
SUM(CASE WHEN status = 'restricted' THEN 1 ELSE 0 END) as restricted,
|
|
97
|
-
SUM(CASE WHEN tt_seller = 1 AND verified = 0 AND ${
|
|
103
|
+
SUM(CASE WHEN tt_seller = 1 AND verified = 0 AND ${targetInSql()} THEN 1 ELSE 0 END) as targetUsers,
|
|
98
104
|
SUM(CASE WHEN no_video = 1 THEN 1 ELSE 0 END) as noVideo
|
|
99
105
|
FROM jobs
|
|
100
106
|
UNION ALL
|
|
@@ -177,10 +183,10 @@ const STAT_GROUPS = {
|
|
|
177
183
|
{
|
|
178
184
|
key: "targetCountryStats",
|
|
179
185
|
mode: "all",
|
|
180
|
-
params:
|
|
181
|
-
sql: `SELECT location_created as country, COUNT(*) as count
|
|
186
|
+
params: () => getTargetLocations(),
|
|
187
|
+
sql: () => `SELECT location_created as country, COUNT(*) as count
|
|
182
188
|
FROM jobs
|
|
183
|
-
WHERE tt_seller = 1 AND verified = 0 AND ${
|
|
189
|
+
WHERE tt_seller = 1 AND verified = 0 AND ${targetInSql()}
|
|
184
190
|
GROUP BY location_created
|
|
185
191
|
ORDER BY count DESC`,
|
|
186
192
|
},
|
|
@@ -284,9 +290,10 @@ function refreshGroupInShards(st, name) {
|
|
|
284
290
|
if (idx < spec.sqls.length) {
|
|
285
291
|
const s = spec.sqls[idx++];
|
|
286
292
|
try {
|
|
293
|
+
const sql = typeof s.sql === "function" ? s.sql() : s.sql;
|
|
287
294
|
const params =
|
|
288
295
|
typeof s.params === "function" ? s.params() : s.params || [];
|
|
289
|
-
const r = st.rawQuery(
|
|
296
|
+
const r = st.rawQuery(sql, params);
|
|
290
297
|
if (r?.error) {
|
|
291
298
|
console.error(`[stats] ${name}.${s.key} 查询失败: ${r.error}`);
|
|
292
299
|
results[s.key] = null;
|
|
@@ -334,9 +341,10 @@ function computeGroupStats(st, name) {
|
|
|
334
341
|
try {
|
|
335
342
|
const results = {};
|
|
336
343
|
for (const s of STAT_GROUPS[name].sqls) {
|
|
344
|
+
const sql = typeof s.sql === "function" ? s.sql() : s.sql;
|
|
337
345
|
const params =
|
|
338
346
|
typeof s.params === "function" ? s.params() : s.params || [];
|
|
339
|
-
const r = st.rawQuery(
|
|
347
|
+
const r = st.rawQuery(sql, params);
|
|
340
348
|
results[s.key] =
|
|
341
349
|
s.mode === "all" ? r?.rows || [] : r?.rows?.[0] || null;
|
|
342
350
|
}
|
|
@@ -641,7 +649,7 @@ export function startWatchServer(
|
|
|
641
649
|
// 按国家维度判断:只为 new 标签不足的国家发现
|
|
642
650
|
const countriesToDiscover = [];
|
|
643
651
|
const { getTagsByStatus } = await import("./db-tags.js");
|
|
644
|
-
for (const country of
|
|
652
|
+
for (const country of getTargetLocations()) {
|
|
645
653
|
const newTags = getTagsByStatus("new", 100, 0, country, null, "llm");
|
|
646
654
|
if (newTags.length < DISCOVER_PER_COUNTRY_THRESHOLD) {
|
|
647
655
|
countriesToDiscover.push(country);
|
|
@@ -1137,7 +1145,7 @@ export function startWatchServer(
|
|
|
1137
1145
|
|
|
1138
1146
|
if (req.method === "GET" && routePath === "/api/stats") {
|
|
1139
1147
|
const stats = computeStatsIncremental(store);
|
|
1140
|
-
stats.targetLocations =
|
|
1148
|
+
stats.targetLocations = getTargetLocations();
|
|
1141
1149
|
stats.clientLoginStatus = store.getClientLoginStatus();
|
|
1142
1150
|
stats.activeClients = store.getActiveClients();
|
|
1143
1151
|
sendJSON(res, 200, stats);
|
|
@@ -1494,7 +1502,7 @@ export function startWatchServer(
|
|
|
1494
1502
|
}
|
|
1495
1503
|
|
|
1496
1504
|
if (req.method === "GET" && routePath === "/api/target-users") {
|
|
1497
|
-
const targetResult = store.getTargetUsers(
|
|
1505
|
+
const targetResult = store.getTargetUsers(getTargetLocations());
|
|
1498
1506
|
const targets = targetResult.users;
|
|
1499
1507
|
if (req.headers["accept"]?.includes("text/csv")) {
|
|
1500
1508
|
const columns = [
|
|
@@ -1549,19 +1557,16 @@ export function startWatchServer(
|
|
|
1549
1557
|
) {
|
|
1550
1558
|
// 摘要模式:只返回各国统计数
|
|
1551
1559
|
if (params.summary === "1") {
|
|
1552
|
-
const result = store.getTargetUsersByCountry(
|
|
1553
|
-
|
|
1554
|
-
|
|
1555
|
-
);
|
|
1560
|
+
const result = store.getTargetUsersByCountry(getTargetLocations(), {
|
|
1561
|
+
summaryOnly: true,
|
|
1562
|
+
});
|
|
1556
1563
|
sendJSON(res, 200, result);
|
|
1557
1564
|
return;
|
|
1558
1565
|
}
|
|
1559
1566
|
|
|
1560
1567
|
// CSV 导出:全量数据
|
|
1561
1568
|
if (req.headers["accept"]?.includes("text/csv")) {
|
|
1562
|
-
const result = store.getTargetUsersByCountry(
|
|
1563
|
-
DEFAULT_TARGET_LOCATIONS,
|
|
1564
|
-
);
|
|
1569
|
+
const result = store.getTargetUsersByCountry(getTargetLocations());
|
|
1565
1570
|
const columns = [
|
|
1566
1571
|
"uniqueId",
|
|
1567
1572
|
"nickname",
|
|
@@ -1615,17 +1620,14 @@ export function startWatchServer(
|
|
|
1615
1620
|
|
|
1616
1621
|
// 分页模式:按国家+搜索+分页查询用户
|
|
1617
1622
|
if (params.limit) {
|
|
1618
|
-
const result = store.getTargetUsersByCountry(
|
|
1619
|
-
|
|
1620
|
-
|
|
1621
|
-
|
|
1622
|
-
|
|
1623
|
-
|
|
1624
|
-
|
|
1625
|
-
|
|
1626
|
-
sortOrder: params.sortOrder || "desc",
|
|
1627
|
-
},
|
|
1628
|
-
);
|
|
1623
|
+
const result = store.getTargetUsersByCountry(getTargetLocations(), {
|
|
1624
|
+
country: params.country || undefined,
|
|
1625
|
+
search: params.search || undefined,
|
|
1626
|
+
limit: parseInt(params.limit, 10),
|
|
1627
|
+
offset: parseInt(params.offset || "0", 10),
|
|
1628
|
+
sort: params.sort || undefined,
|
|
1629
|
+
sortOrder: params.sortOrder || "desc",
|
|
1630
|
+
});
|
|
1629
1631
|
// 确保每个用户对象包含 topVideoPlayCount、topVideoHref 和 topVideoIsAd
|
|
1630
1632
|
if (result.users && Array.isArray(result.users)) {
|
|
1631
1633
|
result.users = result.users.map((u) => ({
|
|
@@ -1640,7 +1642,7 @@ export function startWatchServer(
|
|
|
1640
1642
|
}
|
|
1641
1643
|
|
|
1642
1644
|
// 默认:全量(兼容旧调用)
|
|
1643
|
-
const result = store.getTargetUsersByCountry(
|
|
1645
|
+
const result = store.getTargetUsersByCountry(getTargetLocations());
|
|
1644
1646
|
// 确保每个用户对象包含 topVideoPlayCount、topVideoHref 和 topVideoIsAd
|
|
1645
1647
|
if (result.countries && Array.isArray(result.countries)) {
|
|
1646
1648
|
for (const country of result.countries) {
|
|
@@ -1662,9 +1664,8 @@ export function startWatchServer(
|
|
|
1662
1664
|
req.method === "GET" &&
|
|
1663
1665
|
routePath === "/api/target-users-other-countries"
|
|
1664
1666
|
) {
|
|
1665
|
-
const result =
|
|
1666
|
-
|
|
1667
|
-
);
|
|
1667
|
+
const result =
|
|
1668
|
+
store.getModifiedTargetUsersOutsideLocations(getTargetLocations());
|
|
1668
1669
|
sendJSON(res, 200, result);
|
|
1669
1670
|
return;
|
|
1670
1671
|
}
|
|
@@ -1842,6 +1843,49 @@ export function startWatchServer(
|
|
|
1842
1843
|
return;
|
|
1843
1844
|
}
|
|
1844
1845
|
|
|
1846
|
+
// ====== 目标国家配置(通用 app-config 落盘) ======
|
|
1847
|
+
|
|
1848
|
+
// GET /api/settings/target-locations — 读取当前目标国家
|
|
1849
|
+
if (
|
|
1850
|
+
req.method === "GET" &&
|
|
1851
|
+
routePath === "/api/settings/target-locations"
|
|
1852
|
+
) {
|
|
1853
|
+
try {
|
|
1854
|
+
const { getTargetLocations } = await import("../lib/app-config.js");
|
|
1855
|
+
sendJSON(res, 200, { locations: getTargetLocations() });
|
|
1856
|
+
} catch (e) {
|
|
1857
|
+
sendJSON(res, 500, { error: e.message });
|
|
1858
|
+
}
|
|
1859
|
+
return;
|
|
1860
|
+
}
|
|
1861
|
+
|
|
1862
|
+
// POST /api/settings/target-locations — 保存目标国家(body: { locations: ["CZ","GR",...] })
|
|
1863
|
+
if (
|
|
1864
|
+
req.method === "POST" &&
|
|
1865
|
+
routePath === "/api/settings/target-locations"
|
|
1866
|
+
) {
|
|
1867
|
+
try {
|
|
1868
|
+
const body = await readBody(req);
|
|
1869
|
+
const { setTargetLocations } = await import("../lib/app-config.js");
|
|
1870
|
+
const saved = setTargetLocations(body.locations);
|
|
1871
|
+
sendJSON(res, 200, { locations: saved });
|
|
1872
|
+
} catch (e) {
|
|
1873
|
+
sendJSON(res, 400, { error: e.message });
|
|
1874
|
+
}
|
|
1875
|
+
return;
|
|
1876
|
+
}
|
|
1877
|
+
|
|
1878
|
+
// GET /api/settings/config — 读取整份通用配置(便于后续扩展其他配置)
|
|
1879
|
+
if (req.method === "GET" && routePath === "/api/settings/config") {
|
|
1880
|
+
try {
|
|
1881
|
+
const { getConfig } = await import("../lib/app-config.js");
|
|
1882
|
+
sendJSON(res, 200, { config: getConfig() });
|
|
1883
|
+
} catch (e) {
|
|
1884
|
+
sendJSON(res, 500, { error: e.message });
|
|
1885
|
+
}
|
|
1886
|
+
return;
|
|
1887
|
+
}
|
|
1888
|
+
|
|
1845
1889
|
// GET /api/llm-test — 直接测试 LLM API 连通性
|
|
1846
1890
|
if (req.method === "GET" && routePath === "/api/llm-test") {
|
|
1847
1891
|
try {
|
|
@@ -1939,7 +1983,7 @@ export function startWatchServer(
|
|
|
1939
1983
|
targetLocation: params.targetLocation,
|
|
1940
1984
|
limit: params.limit,
|
|
1941
1985
|
offset: params.offset,
|
|
1942
|
-
targetLocations:
|
|
1986
|
+
targetLocations: getTargetLocations(),
|
|
1943
1987
|
});
|
|
1944
1988
|
usersCache.set(cacheKey, { result, time: Date.now() });
|
|
1945
1989
|
}
|