tt-help-cli-ycl 1.4.27 → 1.4.28
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/scripts/search-authors.mjs +21 -0
- package/scripts/sync-raw-jobs.mjs +450 -181
- package/src/cli/raw-sync.js +2 -5
- package/src/cli/search.js +160 -0
- package/src/lib/args.js +81 -8
- package/src/lib/constants.js +24 -5
- package/src/lib/search-collector.js +194 -0
- package/src/lib/target-locations.js +1 -0
- package/src/main.js +3 -0
- package/src/npm-main.js +3 -0
- package/src/watch/data-store.js +266 -577
- package/src/watch/db-columns.js +1 -0
- package/src/watch/db-crud.js +27 -3
- package/src/watch/db-raw-jobs.js +3 -3
- package/src/watch/db-schema.js +100 -0
- package/src/watch/db-stats.js +46 -23
- package/src/watch/db-tags.js +100 -1
- package/src/watch/llm-config.js +145 -1
- package/src/watch/llm-scoring.js +60 -117
- package/src/watch/public/app.js +5 -5
- package/src/watch/public/index.html +4 -5
- package/src/watch/public/style.css +16 -0
- package/src/watch/server.js +170 -59
- package/src/watch/tag-service.js +11 -67
package/src/watch/llm-scoring.js
CHANGED
|
@@ -14,8 +14,6 @@ import { getDb } from "./db-schema.js";
|
|
|
14
14
|
* @returns {Promise<{ uniqueId: string, score: number, reason: string }>}
|
|
15
15
|
*/
|
|
16
16
|
export async function scoreJobLocation(job, targetLocations) {
|
|
17
|
-
const { fetch: undiciFetch } = await import("undici");
|
|
18
|
-
|
|
19
17
|
const prompt = `
|
|
20
18
|
你是一个 TikTok 用户数据分析助手。请根据以下用户信息,判断该用户是否来自以下**任意一个**目标国家。
|
|
21
19
|
|
|
@@ -42,149 +40,94 @@ Standards: 90-100=clear match, 70-89=likely, 50-69=possible, 20-49=low, 0-19=unl
|
|
|
42
40
|
`;
|
|
43
41
|
|
|
44
42
|
try {
|
|
45
|
-
const {
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
messages: [{ role: "user", content: prompt }],
|
|
69
|
-
max_tokens: 512,
|
|
70
|
-
temperature: 0.1,
|
|
71
|
-
enable_thinking: false,
|
|
72
|
-
}),
|
|
73
|
-
});
|
|
74
|
-
|
|
75
|
-
const result = await response.json().catch(() => null);
|
|
76
|
-
const content = result?.choices?.[0]?.message?.content || "";
|
|
77
|
-
|
|
78
|
-
if (!content) {
|
|
79
|
-
const errMsg = result?.error?.message || `HTTP ${response.status}`;
|
|
80
|
-
lastError = errMsg;
|
|
81
|
-
console.error(
|
|
82
|
-
`[scoreJobLocation] ${llmModel} 失败: ${errMsg},尝试下一个`,
|
|
83
|
-
);
|
|
84
|
-
continue;
|
|
85
|
-
}
|
|
86
|
-
|
|
87
|
-
if (i > 0) {
|
|
88
|
-
console.error(`[scoreJobLocation] ✅ auto 切换成功: ${llmModel}`);
|
|
89
|
-
}
|
|
90
|
-
|
|
91
|
-
let parsed = null;
|
|
92
|
-
|
|
93
|
-
// 尝试 1: 直接解析
|
|
94
|
-
try {
|
|
95
|
-
parsed = JSON.parse(content);
|
|
96
|
-
} catch {
|
|
97
|
-
// 尝试 2: 提取 {} 包裹的内容
|
|
98
|
-
const match = content.match(/\{[\s\S]*\}/);
|
|
99
|
-
if (match) {
|
|
43
|
+
const { llmFetch } = await import("./llm-config.js");
|
|
44
|
+
const content = await llmFetch({
|
|
45
|
+
messages: [{ role: "user", content: prompt }],
|
|
46
|
+
max_tokens: 512,
|
|
47
|
+
temperature: 0.1,
|
|
48
|
+
enable_thinking: false,
|
|
49
|
+
});
|
|
50
|
+
|
|
51
|
+
// ── 解析 JSON ──
|
|
52
|
+
let parsed = null;
|
|
53
|
+
|
|
54
|
+
try {
|
|
55
|
+
parsed = JSON.parse(content);
|
|
56
|
+
} catch {
|
|
57
|
+
const match = content.match(/\{[\s\S]*\}/);
|
|
58
|
+
if (match) {
|
|
59
|
+
try {
|
|
60
|
+
parsed = JSON.parse(match[0]);
|
|
61
|
+
} catch {
|
|
62
|
+
const cleaned = match[0]
|
|
63
|
+
.replace(/"/g, '"')
|
|
64
|
+
.replace(/\s+/g, " ")
|
|
65
|
+
.trim();
|
|
100
66
|
try {
|
|
101
|
-
parsed = JSON.parse(
|
|
67
|
+
parsed = JSON.parse(cleaned);
|
|
102
68
|
} catch {
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
.
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
const valueStart = afterKey.indexOf('"', colonPos + 1) + 1;
|
|
120
|
-
const rawValue = afterKey.substring(valueStart);
|
|
121
|
-
const lastBrace = content.lastIndexOf("}");
|
|
122
|
-
const reasonEnd = lastBrace - reasonKeyPos - valueStart;
|
|
123
|
-
if (reasonEnd > 0) {
|
|
124
|
-
reason = rawValue.substring(0, reasonEnd).trim();
|
|
125
|
-
if (reason.startsWith('"')) reason = reason.substring(1);
|
|
126
|
-
if (reason.endsWith('"'))
|
|
127
|
-
reason = reason.substring(0, reason.length - 1);
|
|
128
|
-
}
|
|
69
|
+
const scoreMatch = content.match(/"?score"?\s*:\s*(\d+)/i);
|
|
70
|
+
if (scoreMatch) {
|
|
71
|
+
let reason = "解析降级";
|
|
72
|
+
const reasonKeyPos = content.search(/"?reason"?\s*:\s*"/i);
|
|
73
|
+
if (reasonKeyPos !== -1) {
|
|
74
|
+
const afterKey = content.substring(reasonKeyPos);
|
|
75
|
+
const colonPos = afterKey.indexOf(":");
|
|
76
|
+
const valueStart = afterKey.indexOf('"', colonPos + 1) + 1;
|
|
77
|
+
const rawValue = afterKey.substring(valueStart);
|
|
78
|
+
const lastBrace = content.lastIndexOf("}");
|
|
79
|
+
const reasonEnd = lastBrace - reasonKeyPos - valueStart;
|
|
80
|
+
if (reasonEnd > 0) {
|
|
81
|
+
reason = rawValue.substring(0, reasonEnd).trim();
|
|
82
|
+
if (reason.startsWith('"')) reason = reason.substring(1);
|
|
83
|
+
if (reason.endsWith('"'))
|
|
84
|
+
reason = reason.substring(0, reason.length - 1);
|
|
129
85
|
}
|
|
130
|
-
parsed = { score: parseInt(scoreMatch[1]) || 50, reason };
|
|
131
86
|
}
|
|
87
|
+
parsed = { score: parseInt(scoreMatch[1]) || 50, reason };
|
|
132
88
|
}
|
|
133
89
|
}
|
|
134
90
|
}
|
|
135
|
-
|
|
136
|
-
// 尝试 5: 更宽松的正则提取
|
|
137
|
-
if (!parsed) {
|
|
138
|
-
const scoreMatch = content.match(/"score"\s*:\s*(\d+)/);
|
|
139
|
-
const reasonMatch = content.match(/"reason"\s*:\s*"([^"]*)"/);
|
|
140
|
-
if (scoreMatch) {
|
|
141
|
-
parsed = {
|
|
142
|
-
score: parseInt(scoreMatch[1]) || 50,
|
|
143
|
-
reason: reasonMatch ? reasonMatch[1] : "解析降级 - 宽松模式",
|
|
144
|
-
};
|
|
145
|
-
}
|
|
146
|
-
}
|
|
147
91
|
}
|
|
148
92
|
|
|
149
|
-
if (parsed
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
93
|
+
if (!parsed) {
|
|
94
|
+
const scoreMatch = content.match(/"score"\s*:\s*(\d+)/);
|
|
95
|
+
const reasonMatch = content.match(/"reason"\s*:\s*"([^"]*)"/);
|
|
96
|
+
if (scoreMatch) {
|
|
97
|
+
parsed = {
|
|
98
|
+
score: parseInt(scoreMatch[1]) || 50,
|
|
99
|
+
reason: reasonMatch ? reasonMatch[1] : "解析降级 - 宽松模式",
|
|
100
|
+
};
|
|
153
101
|
}
|
|
154
|
-
return {
|
|
155
|
-
uniqueId: job.unique_id,
|
|
156
|
-
score: Math.max(0, Math.min(100, parsed.score)),
|
|
157
|
-
reason: parsed.reason || "",
|
|
158
|
-
};
|
|
159
102
|
}
|
|
103
|
+
}
|
|
160
104
|
|
|
161
|
-
|
|
162
|
-
`[scoreJobLocation] JSON 解析失败 (${job.unique_id}): ${content.substring(0, 100)}`,
|
|
163
|
-
);
|
|
105
|
+
if (parsed && typeof parsed.score === "number") {
|
|
164
106
|
return {
|
|
165
107
|
uniqueId: job.unique_id,
|
|
166
|
-
score:
|
|
167
|
-
reason:
|
|
108
|
+
score: Math.max(0, Math.min(100, parsed.score)),
|
|
109
|
+
reason: parsed.reason || "",
|
|
168
110
|
};
|
|
169
|
-
}
|
|
111
|
+
}
|
|
170
112
|
|
|
171
|
-
// 全部模型均失败
|
|
172
113
|
console.error(
|
|
173
|
-
`[scoreJobLocation]
|
|
114
|
+
`[scoreJobLocation] JSON 解析失败 (${job.unique_id}): ${content.substring(0, 100)}`,
|
|
174
115
|
);
|
|
175
116
|
return {
|
|
176
117
|
uniqueId: job.unique_id,
|
|
177
118
|
score: 50,
|
|
178
|
-
reason:
|
|
119
|
+
reason: "LLM 响应解析失败,使用默认分",
|
|
179
120
|
};
|
|
180
121
|
} catch (e) {
|
|
181
122
|
console.error(
|
|
182
|
-
`[scoreJobLocation] LLM
|
|
123
|
+
`[scoreJobLocation] LLM 调用失败 (${job.unique_id}): ${e.message}`,
|
|
183
124
|
);
|
|
184
125
|
return {
|
|
185
126
|
uniqueId: job.unique_id,
|
|
186
127
|
score: 50,
|
|
187
|
-
reason:
|
|
128
|
+
reason: e.message.includes("全部")
|
|
129
|
+
? e.message
|
|
130
|
+
: `LLM 调用异常: ${e.message}`,
|
|
188
131
|
};
|
|
189
132
|
}
|
|
190
133
|
}
|
package/src/watch/public/app.js
CHANGED
|
@@ -144,7 +144,8 @@ function flashEl(id, value, options) {
|
|
|
144
144
|
const el = document.getElementById(id);
|
|
145
145
|
if (!el) return;
|
|
146
146
|
const prev = prevStatValues[id];
|
|
147
|
-
|
|
147
|
+
const suffix = options?.suffix || "";
|
|
148
|
+
el.textContent = formatStatNum(value, options) + suffix;
|
|
148
149
|
if (prev !== undefined && prev !== value) {
|
|
149
150
|
el.classList.remove("flash-change");
|
|
150
151
|
void el.offsetWidth;
|
|
@@ -160,9 +161,6 @@ function renderStats() {
|
|
|
160
161
|
flashEl("statProcessing", d.processingUsers || 0);
|
|
161
162
|
flashEl("statDone", d.processedUsers);
|
|
162
163
|
flashEl("statPending", d.pendingUsers);
|
|
163
|
-
const statJobsBase = document.getElementById("statJobsBase");
|
|
164
|
-
if (statJobsBase)
|
|
165
|
-
statJobsBase.textContent = `jobs_base: ${formatStatNum(d.jobsBaseTotal || 0)}`;
|
|
166
164
|
flashEl("statError", d.errorUsers);
|
|
167
165
|
flashEl("statTarget", d.targetUsers, { full: true });
|
|
168
166
|
// 检测目标商家新增,放烟花(仅首页)
|
|
@@ -175,7 +173,7 @@ function renderStats() {
|
|
|
175
173
|
flashEl("statUserUpdateTasks", d.userUpdateTasks || 0, { full: true });
|
|
176
174
|
flashEl("statRawJobs", d.rawJobs || 0);
|
|
177
175
|
flashEl("statCommentTask", d.commentTaskCount || 0, { full: true });
|
|
178
|
-
flashEl("statTags", d.
|
|
176
|
+
flashEl("statTags", d.tagHighScore || 0);
|
|
179
177
|
// 同步子页面 stats
|
|
180
178
|
const pendingTotal = document.getElementById("pendingStatTotal");
|
|
181
179
|
if (pendingTotal) pendingTotal.textContent = formatStatNum(d.totalUsers);
|
|
@@ -272,6 +270,7 @@ function renderActiveClients(clients) {
|
|
|
272
270
|
"comments",
|
|
273
271
|
"scoring",
|
|
274
272
|
"tag-refresh",
|
|
273
|
+
"tag-search",
|
|
275
274
|
];
|
|
276
275
|
const labels = {
|
|
277
276
|
explore: "Explore",
|
|
@@ -281,6 +280,7 @@ function renderActiveClients(clients) {
|
|
|
281
280
|
comments: "Comments",
|
|
282
281
|
scoring: "Scoring",
|
|
283
282
|
"tag-refresh": "TagRefresh",
|
|
283
|
+
"tag-search": "Search",
|
|
284
284
|
};
|
|
285
285
|
const grouped = {};
|
|
286
286
|
for (const c of clients) {
|
|
@@ -37,7 +37,6 @@
|
|
|
37
37
|
<div class="stat-card clickable pending-card" id="statPendingCard" onclick="navigateToPending()">
|
|
38
38
|
<div class="label">待处理</div>
|
|
39
39
|
<div class="value pending" id="statPending">0</div>
|
|
40
|
-
<div class="value-sub" id="statJobsBase">jobs_base: 0</div>
|
|
41
40
|
</div>
|
|
42
41
|
<div class="stat-card">
|
|
43
42
|
<div class="label">错误</div>
|
|
@@ -45,11 +44,11 @@
|
|
|
45
44
|
</div>
|
|
46
45
|
<div class="stat-card clickable pending-card" id="statUserUpdateCard" onclick="navigateToUserUpdate()">
|
|
47
46
|
<div class="label">待补资料</div>
|
|
48
|
-
<div class="value
|
|
47
|
+
<div class="value warn" id="statUserUpdateTasks">0</div>
|
|
49
48
|
</div>
|
|
50
49
|
<div class="stat-card clickable" id="statRawCard" onclick="navigateToRaw()">
|
|
51
50
|
<div class="label">毛料库</div>
|
|
52
|
-
<div class="value
|
|
51
|
+
<div class="value slate" id="statRawJobs">0</div>
|
|
53
52
|
</div>
|
|
54
53
|
<div class="stat-card clickable" id="statTargetCard">
|
|
55
54
|
<div class="label">目标商家</div>
|
|
@@ -57,11 +56,11 @@
|
|
|
57
56
|
</div>
|
|
58
57
|
<div class="stat-card clickable pending-card" id="statCommentTaskCard">
|
|
59
58
|
<div class="label">待评论任务</div>
|
|
60
|
-
<div class="value
|
|
59
|
+
<div class="value rose" id="statCommentTask">0</div>
|
|
61
60
|
</div>
|
|
62
61
|
<div class="stat-card clickable pending-card" id="statTagsCard" onclick="navigateToTags()" style="cursor:pointer">
|
|
63
62
|
<div class="label">关键词</div>
|
|
64
|
-
<div class="value
|
|
63
|
+
<div class="value teal" id="statTags">0</div>
|
|
65
64
|
</div>
|
|
66
65
|
</div>
|
|
67
66
|
<div id="activeClientsSection" class="active-clients-section" style="display:none">
|
|
@@ -95,6 +95,22 @@ body {
|
|
|
95
95
|
color: #a78bfa;
|
|
96
96
|
}
|
|
97
97
|
|
|
98
|
+
.stat-card .value.warn {
|
|
99
|
+
color: #fb923c;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
.stat-card .value.slate {
|
|
103
|
+
color: #94a3b8;
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
.stat-card .value.teal {
|
|
107
|
+
color: #2dd4bf;
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
.stat-card .value.rose {
|
|
111
|
+
color: #f472b6;
|
|
112
|
+
}
|
|
113
|
+
|
|
98
114
|
.stat-card .value-sub {
|
|
99
115
|
font-size: 12px;
|
|
100
116
|
font-weight: 400;
|
package/src/watch/server.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import http from "http";
|
|
2
2
|
import os from "os";
|
|
3
|
+
import Database from "better-sqlite3";
|
|
3
4
|
|
|
4
5
|
import { readFileSync, existsSync } from "fs";
|
|
5
6
|
import { join, dirname } from "path";
|
|
@@ -27,40 +28,32 @@ const publicDir = join(__dirname, "public");
|
|
|
27
28
|
// Stats 缓存:避免每次请求都扫描 100 万+ 条 jobs 数据
|
|
28
29
|
let statsCache = null;
|
|
29
30
|
let statsCacheTime = 0;
|
|
30
|
-
const STATS_CACHE_TTL =
|
|
31
|
-
|
|
32
|
-
//
|
|
33
|
-
const
|
|
34
|
-
const
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
dbTotalUsers:
|
|
42
|
-
tagCount:
|
|
31
|
+
const STATS_CACHE_TTL = 15000; // 15 秒缓存
|
|
32
|
+
|
|
33
|
+
// 用户列表短缓存:同一参数组合 3 秒内复用,无论多少客户端同时查只算一次
|
|
34
|
+
const usersCache = new Map(); // key -> { result, time }
|
|
35
|
+
const USERS_CACHE_TTL = 3000; // 3 秒
|
|
36
|
+
|
|
37
|
+
// 慢计数实时查询(数据库已分拆,直接查无压力)
|
|
38
|
+
const SLOW_COUNT_SQLS = {
|
|
39
|
+
rawJobs: "SELECT COUNT(*) as c FROM raw_jobs",
|
|
40
|
+
commentTaskCount:
|
|
41
|
+
"SELECT COUNT(*) as c FROM videos WHERE user_update_count IS NULL OR user_update_count <= 0",
|
|
42
|
+
dbTotalUsers: "SELECT COUNT(*) as c FROM users",
|
|
43
|
+
tagCount: "SELECT COUNT(*) as c FROM tags",
|
|
44
|
+
tagHighScore: "SELECT COUNT(*) as c FROM tags WHERE score >= 70",
|
|
43
45
|
};
|
|
44
46
|
|
|
45
|
-
function
|
|
46
|
-
const entry = slowCaches[key];
|
|
47
|
-
if (!entry) return;
|
|
48
|
-
try {
|
|
49
|
-
const result = st.rawQuery(entry.sql);
|
|
50
|
-
entry.value = result?.rows?.[0]?.c || 0;
|
|
51
|
-
entry.time = Date.now();
|
|
52
|
-
} catch (e) {
|
|
53
|
-
console.error(`[stats] 慢计数 ${key} 更新失败: ${e.message}`);
|
|
54
|
-
}
|
|
55
|
-
}
|
|
56
|
-
|
|
57
|
-
function getSlowCounts(st) {
|
|
47
|
+
function computeSlowCounts(st) {
|
|
58
48
|
const result = {};
|
|
59
|
-
for (const [key,
|
|
60
|
-
|
|
61
|
-
|
|
49
|
+
for (const [key, sql] of Object.entries(SLOW_COUNT_SQLS)) {
|
|
50
|
+
try {
|
|
51
|
+
const r = st.rawQuery(sql);
|
|
52
|
+
result[key] = r?.rows?.[0]?.c || 0;
|
|
53
|
+
} catch (e) {
|
|
54
|
+
console.error(`[stats] 慢计数 ${key} 查询失败: ${e.message}`);
|
|
55
|
+
result[key] = 0;
|
|
62
56
|
}
|
|
63
|
-
result[key] = entry.value;
|
|
64
57
|
}
|
|
65
58
|
return result;
|
|
66
59
|
}
|
|
@@ -78,7 +71,7 @@ function computeStatsIncremental(st) {
|
|
|
78
71
|
statsRefreshing = true;
|
|
79
72
|
setImmediate(async () => {
|
|
80
73
|
try {
|
|
81
|
-
const slowCounts =
|
|
74
|
+
const slowCounts = computeSlowCounts(st);
|
|
82
75
|
const stats = st.getDashboardStats(DEFAULT_TARGET_LOCATIONS, {
|
|
83
76
|
skipSlowCounts: true,
|
|
84
77
|
});
|
|
@@ -86,6 +79,7 @@ function computeStatsIncremental(st) {
|
|
|
86
79
|
stats.commentTaskCount = slowCounts.commentTaskCount;
|
|
87
80
|
stats.dbTotalUsers = slowCounts.dbTotalUsers;
|
|
88
81
|
stats.tagCount = slowCounts.tagCount;
|
|
82
|
+
stats.tagHighScore = slowCounts.tagHighScore;
|
|
89
83
|
statsCache = stats;
|
|
90
84
|
statsCacheTime = Date.now();
|
|
91
85
|
} catch (e) {
|
|
@@ -97,7 +91,7 @@ function computeStatsIncremental(st) {
|
|
|
97
91
|
return statsCache;
|
|
98
92
|
}
|
|
99
93
|
// 首次加载 → 同步计算
|
|
100
|
-
const slowCounts =
|
|
94
|
+
const slowCounts = computeSlowCounts(st);
|
|
101
95
|
const stats = st.getDashboardStats(DEFAULT_TARGET_LOCATIONS, {
|
|
102
96
|
skipSlowCounts: true,
|
|
103
97
|
});
|
|
@@ -105,6 +99,7 @@ function computeStatsIncremental(st) {
|
|
|
105
99
|
stats.commentTaskCount = slowCounts.commentTaskCount;
|
|
106
100
|
stats.dbTotalUsers = slowCounts.dbTotalUsers;
|
|
107
101
|
stats.tagCount = slowCounts.tagCount;
|
|
102
|
+
stats.tagHighScore = slowCounts.tagHighScore;
|
|
108
103
|
statsCache = stats;
|
|
109
104
|
statsCacheTime = now;
|
|
110
105
|
return stats;
|
|
@@ -161,6 +156,7 @@ function inferClientType(routePath) {
|
|
|
161
156
|
if (routePath.startsWith("/api/recheck-jobs")) return "recheck";
|
|
162
157
|
if (routePath.startsWith("/api/comment-task")) return "comments";
|
|
163
158
|
if (routePath.startsWith("/api/tags/refresh")) return "tag-refresh";
|
|
159
|
+
if (routePath.startsWith("/api/tags/search")) return "tag-search";
|
|
164
160
|
if (routePath.startsWith("/api/tags")) return "scoring";
|
|
165
161
|
if (
|
|
166
162
|
routePath.startsWith("/api/job") ||
|
|
@@ -356,7 +352,7 @@ export function startWatchServer(
|
|
|
356
352
|
const countriesToDiscover = [];
|
|
357
353
|
const { getTagsByStatus } = await import("./db-tags.js");
|
|
358
354
|
for (const country of DEFAULT_TARGET_LOCATIONS) {
|
|
359
|
-
const newTags = getTagsByStatus("new", 100, 0, country);
|
|
355
|
+
const newTags = getTagsByStatus("new", 100, 0, country, null, "llm");
|
|
360
356
|
if (newTags.length < DISCOVER_PER_COUNTRY_THRESHOLD) {
|
|
361
357
|
countriesToDiscover.push(country);
|
|
362
358
|
}
|
|
@@ -626,10 +622,22 @@ export function startWatchServer(
|
|
|
626
622
|
if (req.method === "GET" && routePath === "/api/db-query") {
|
|
627
623
|
const sql = params.sql || "SELECT * FROM jobs LIMIT 10";
|
|
628
624
|
const limit = Math.min(parseInt(params.limit) || 100, 1000);
|
|
625
|
+
const dbPath = params.db || "";
|
|
629
626
|
// 安全限制:自动加 LIMIT
|
|
630
627
|
const safeSql = sql.replace(/LIMIT\s+\d+/gi, "") + ` LIMIT ${limit}`;
|
|
631
628
|
try {
|
|
632
|
-
|
|
629
|
+
let result;
|
|
630
|
+
if (dbPath) {
|
|
631
|
+
const extDb = new Database(dbPath);
|
|
632
|
+
extDb.pragma("journal_mode = WAL");
|
|
633
|
+
try {
|
|
634
|
+
result = { rows: extDb.prepare(safeSql).all() };
|
|
635
|
+
} finally {
|
|
636
|
+
extDb.close();
|
|
637
|
+
}
|
|
638
|
+
} else {
|
|
639
|
+
result = store.rawQuery(safeSql);
|
|
640
|
+
}
|
|
633
641
|
sendJSON(res, 200, result);
|
|
634
642
|
} catch (e) {
|
|
635
643
|
sendJSON(res, 400, { error: e.message });
|
|
@@ -760,6 +768,35 @@ export function startWatchServer(
|
|
|
760
768
|
return;
|
|
761
769
|
}
|
|
762
770
|
|
|
771
|
+
// 从 videos_old 恢复视频到 videos(通用 where 条件)
|
|
772
|
+
// ?where=tt_seller=1&limit=500
|
|
773
|
+
// ?where=tt_seller=1 AND registered_at>=1234567890&limit=500
|
|
774
|
+
if (req.method === "POST" && routePath === "/api/videos/restore") {
|
|
775
|
+
const limit = Math.max(
|
|
776
|
+
1,
|
|
777
|
+
Math.min(10000, parseInt(params.limit) || 500),
|
|
778
|
+
);
|
|
779
|
+
const where = decodeURIComponent(params.where || "");
|
|
780
|
+
const ret = store.restoreVideosFromOld({ limit, where });
|
|
781
|
+
sendJSON(res, 200, ret);
|
|
782
|
+
return;
|
|
783
|
+
}
|
|
784
|
+
|
|
785
|
+
// 将 videos 中符合条件的视频归档到 videos_old(通用 where 条件)
|
|
786
|
+
// ?where=tt_seller=0&limit=500
|
|
787
|
+
// ?where=user_update_count>1&limit=500
|
|
788
|
+
// ?where=tt_seller=0 OR user_update_count>1&limit=500
|
|
789
|
+
if (req.method === "POST" && routePath === "/api/videos/archive") {
|
|
790
|
+
const limit = Math.max(
|
|
791
|
+
1,
|
|
792
|
+
Math.min(10000, parseInt(params.limit) || 500),
|
|
793
|
+
);
|
|
794
|
+
const where = decodeURIComponent(params.where || "");
|
|
795
|
+
const ret = store.archiveVideosToOld({ limit, where });
|
|
796
|
+
sendJSON(res, 200, ret);
|
|
797
|
+
return;
|
|
798
|
+
}
|
|
799
|
+
|
|
763
800
|
const jobPinMatch = routePath.match(/^\/api\/job\/([^/]+)\/pin$/);
|
|
764
801
|
if (req.method === "POST" && jobPinMatch) {
|
|
765
802
|
const uniqueId = jobPinMatch[1];
|
|
@@ -773,7 +810,6 @@ export function startWatchServer(
|
|
|
773
810
|
stats.targetLocations = DEFAULT_TARGET_LOCATIONS;
|
|
774
811
|
stats.clientLoginStatus = store.getClientLoginStatus();
|
|
775
812
|
stats.activeClients = store.getActiveClients();
|
|
776
|
-
stats.llmSampleOffsets = store.getLlmSampleOffsets(); // 添加偏移量状态
|
|
777
813
|
sendJSON(res, 200, stats);
|
|
778
814
|
return;
|
|
779
815
|
}
|
|
@@ -1465,16 +1501,24 @@ export function startWatchServer(
|
|
|
1465
1501
|
}
|
|
1466
1502
|
|
|
1467
1503
|
if (req.method === "GET" && routePath === "/api/users") {
|
|
1468
|
-
const
|
|
1469
|
-
|
|
1470
|
-
|
|
1471
|
-
|
|
1472
|
-
|
|
1473
|
-
|
|
1474
|
-
|
|
1475
|
-
|
|
1476
|
-
|
|
1477
|
-
|
|
1504
|
+
const cacheKey = `users|${params.status || ""}|${params.search || ""}|${params.location || ""}|${params.target || ""}|${params.targetLocation || ""}|${params.limit || ""}|${params.offset || ""}`;
|
|
1505
|
+
const cached = usersCache.get(cacheKey);
|
|
1506
|
+
let result;
|
|
1507
|
+
if (cached && Date.now() - cached.time < USERS_CACHE_TTL) {
|
|
1508
|
+
result = cached.result;
|
|
1509
|
+
} else {
|
|
1510
|
+
result = store.getUsersPage({
|
|
1511
|
+
status: params.status,
|
|
1512
|
+
search: params.search,
|
|
1513
|
+
location: params.location,
|
|
1514
|
+
target: params.target,
|
|
1515
|
+
targetLocation: params.targetLocation,
|
|
1516
|
+
limit: params.limit,
|
|
1517
|
+
offset: params.offset,
|
|
1518
|
+
targetLocations: DEFAULT_TARGET_LOCATIONS,
|
|
1519
|
+
});
|
|
1520
|
+
usersCache.set(cacheKey, { result, time: Date.now() });
|
|
1521
|
+
}
|
|
1478
1522
|
|
|
1479
1523
|
if (params.view === "light") {
|
|
1480
1524
|
result.users = result.users.map((u) => ({
|
|
@@ -1674,20 +1718,22 @@ export function startWatchServer(
|
|
|
1674
1718
|
// 获取总数(带搜索过滤,按 status 返回对应计数)
|
|
1675
1719
|
const stats = store.getTagStats(upperCountry, searchTerm);
|
|
1676
1720
|
let total;
|
|
1721
|
+
let newLlmTotal = 0;
|
|
1677
1722
|
if (stats) {
|
|
1678
1723
|
if (status === "new") total = stats.new;
|
|
1679
1724
|
else if (status === "productive") total = stats.productive;
|
|
1680
1725
|
else if (status === "dead") total = stats.dead;
|
|
1681
1726
|
else if (status === "scoring") total = stats.scoring;
|
|
1682
1727
|
else total = stats.total;
|
|
1728
|
+
newLlmTotal = stats.newLlm || 0;
|
|
1683
1729
|
} else {
|
|
1684
1730
|
total = tags.length;
|
|
1685
1731
|
}
|
|
1686
1732
|
|
|
1687
|
-
// 按需发现:客户端取 new
|
|
1733
|
+
// 按需发现:客户端取 new 标签时,如果 LLM 来源的 new 池子低于阈值,立即触发发现
|
|
1688
1734
|
if (
|
|
1689
1735
|
status === "new" &&
|
|
1690
|
-
|
|
1736
|
+
newLlmTotal < DISCOVER_LOW_THRESHOLD &&
|
|
1691
1737
|
!discoverRunning
|
|
1692
1738
|
) {
|
|
1693
1739
|
runAutoDiscover("low-pool"); // 异步触发,不阻塞响应
|
|
@@ -1785,6 +1831,82 @@ export function startWatchServer(
|
|
|
1785
1831
|
return;
|
|
1786
1832
|
}
|
|
1787
1833
|
|
|
1834
|
+
// GET /api/tags/search-task — 领取一个 productive 标签用于搜索(score>70,未被搜索过)
|
|
1835
|
+
if (req.method === "GET" && routePath === "/api/tags/search-task") {
|
|
1836
|
+
const task = store.getProductiveSearchTask();
|
|
1837
|
+
if (!task) {
|
|
1838
|
+
sendJSON(res, 200, {
|
|
1839
|
+
tag: null,
|
|
1840
|
+
message: "no productive tags available for search",
|
|
1841
|
+
});
|
|
1842
|
+
return;
|
|
1843
|
+
}
|
|
1844
|
+
sendJSON(res, 200, task);
|
|
1845
|
+
return;
|
|
1846
|
+
}
|
|
1847
|
+
|
|
1848
|
+
// POST /api/tags/search-claim { tag } — 锁定标签用于搜索(防并发)
|
|
1849
|
+
if (req.method === "POST" && routePath === "/api/tags/search-claim") {
|
|
1850
|
+
try {
|
|
1851
|
+
const body = await readBody(req);
|
|
1852
|
+
const tag = (body.tag || "").trim().toLowerCase();
|
|
1853
|
+
if (!tag) {
|
|
1854
|
+
sendJSON(res, 400, { error: "tag 不能为空" });
|
|
1855
|
+
return;
|
|
1856
|
+
}
|
|
1857
|
+
const result = store.claimSearchTag(tag);
|
|
1858
|
+
sendJSON(res, result.ok ? 200 : 409, result);
|
|
1859
|
+
} catch (e) {
|
|
1860
|
+
sendJSON(res, 500, { error: e.message });
|
|
1861
|
+
}
|
|
1862
|
+
return;
|
|
1863
|
+
}
|
|
1864
|
+
|
|
1865
|
+
// POST /api/tags/search-result { tag, authors[], country } — 提交搜索结果 + 解锁
|
|
1866
|
+
if (req.method === "POST" && routePath === "/api/tags/search-result") {
|
|
1867
|
+
try {
|
|
1868
|
+
const body = await readBody(req);
|
|
1869
|
+
const tag = (body.tag || "").trim().toLowerCase();
|
|
1870
|
+
const authors = body.authors || [];
|
|
1871
|
+
const country = body.country || null;
|
|
1872
|
+
|
|
1873
|
+
if (!tag) {
|
|
1874
|
+
sendJSON(res, 400, { error: "tag 不能为空" });
|
|
1875
|
+
return;
|
|
1876
|
+
}
|
|
1877
|
+
|
|
1878
|
+
// 提交作者到 job_base(来源=search,国家=tag的国家)
|
|
1879
|
+
let pushed = 0;
|
|
1880
|
+
let skipped = 0;
|
|
1881
|
+
if (authors.length > 0) {
|
|
1882
|
+
const users = authors.map((uniqueId) => ({
|
|
1883
|
+
uniqueId,
|
|
1884
|
+
sources: ["search"],
|
|
1885
|
+
locationCreated: country,
|
|
1886
|
+
}));
|
|
1887
|
+
const pushResult = store.addRawUsers(users);
|
|
1888
|
+
pushed = pushResult.added || 0;
|
|
1889
|
+
skipped = pushResult.skipped || 0;
|
|
1890
|
+
}
|
|
1891
|
+
|
|
1892
|
+
// 报告搜索结果并解锁标签
|
|
1893
|
+
const result = store.reportSearchResult(tag, {
|
|
1894
|
+
searchedUsers: pushed,
|
|
1895
|
+
error: pushed === 0 && authors.length > 0 ? "all duplicates" : null,
|
|
1896
|
+
});
|
|
1897
|
+
|
|
1898
|
+
sendJSON(res, 200, {
|
|
1899
|
+
ok: true,
|
|
1900
|
+
pushed,
|
|
1901
|
+
skipped,
|
|
1902
|
+
total: authors.length,
|
|
1903
|
+
});
|
|
1904
|
+
} catch (e) {
|
|
1905
|
+
sendJSON(res, 500, { error: e.message });
|
|
1906
|
+
}
|
|
1907
|
+
return;
|
|
1908
|
+
}
|
|
1909
|
+
|
|
1788
1910
|
// GET /api/tags/recent-generated?country=ES&lookback=10 — 最近N轮生成的标签(去重)
|
|
1789
1911
|
if (req.method === "GET" && routePath === "/api/tags/recent-generated") {
|
|
1790
1912
|
try {
|
|
@@ -2447,21 +2569,10 @@ export function startWatchServer(
|
|
|
2447
2569
|
console.error(` 本地访问: http://127.0.0.1:${port}`);
|
|
2448
2570
|
console.error(` 局域网访问: http://${localIP}:${port}`);
|
|
2449
2571
|
|
|
2450
|
-
// 预热 stats 缓存
|
|
2572
|
+
// 预热 stats 缓存
|
|
2451
2573
|
setTimeout(() => {
|
|
2452
|
-
for (const key of Object.keys(slowCaches))
|
|
2453
|
-
refreshOneSlowCount(store, key);
|
|
2454
2574
|
computeStatsIncremental(store);
|
|
2455
2575
|
}, 3000);
|
|
2456
|
-
const slowKeys = Object.keys(slowCaches);
|
|
2457
|
-
slowKeys.forEach((key, i) => {
|
|
2458
|
-
setInterval(() => refreshOneSlowCount(store, key), SLOW_CACHE_TTL);
|
|
2459
|
-
// 首次错峰:各表间隔 2.5 分钟启动
|
|
2460
|
-
setTimeout(
|
|
2461
|
-
() => refreshOneSlowCount(store, key),
|
|
2462
|
-
3000 + (i + 1) * 150000,
|
|
2463
|
-
);
|
|
2464
|
-
});
|
|
2465
2576
|
|
|
2466
2577
|
// 启动时清理超时的 scoring 标签
|
|
2467
2578
|
try {
|