tt-help-cli-ycl 1.3.43 → 1.3.45
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/cli/attach.js +16 -4
- package/src/cli/comments.js +14 -22
- package/src/lib/args.js +9 -1
- package/src/lib/browser/page.js +8 -7
- package/src/lib/constants.js +5 -2
- package/src/lib/target-locations.js +61 -0
- package/src/scraper/core.js +8 -9
- package/src/scraper/explore-core.js +32 -14
- package/src/scraper/modules/follow-extractor.js +35 -6
- package/src/scraper/refresh-core.js +2 -1
- package/src/watch/data-store.js +149 -17
- package/src/watch/public/index.html +61 -1
- package/src/watch/server.js +24 -23
package/package.json
CHANGED
package/src/cli/attach.js
CHANGED
|
@@ -76,12 +76,12 @@ async function recycleScraper(scraper, reason) {
|
|
|
76
76
|
}
|
|
77
77
|
|
|
78
78
|
export async function handleAttach(options) {
|
|
79
|
-
const { attachParallel, attachInterval, serverUrl, showHelp } = options;
|
|
79
|
+
const { attachParallel, attachInterval, serverUrl, attachCountries, showHelp } = options;
|
|
80
80
|
let shuttingDown = false;
|
|
81
81
|
let forceExitTimer = null;
|
|
82
82
|
|
|
83
83
|
if (showHelp) {
|
|
84
|
-
attachLog("用法: tt-help attach [-p 并行数] [-i 间隔秒数] [-s 服务端地址]");
|
|
84
|
+
attachLog("用法: tt-help attach [-p 并行数] [-i 间隔秒数] [-s 服务端地址] [-c 国家列表]");
|
|
85
85
|
attachLog("");
|
|
86
86
|
attachLog("参数:");
|
|
87
87
|
attachLog(" -p, --parallel <N> 并行抓取数(默认: 1)");
|
|
@@ -89,6 +89,9 @@ export async function handleAttach(options) {
|
|
|
89
89
|
attachLog(
|
|
90
90
|
" -s, --server <URL> 服务端地址(默认: http://127.0.0.1:3001)",
|
|
91
91
|
);
|
|
92
|
+
attachLog(
|
|
93
|
+
" -c, --countries <A,B,C> 猜测国家列表(逗号分隔,如 PL,DE,FR),服务端优先返回这些国家的任务",
|
|
94
|
+
);
|
|
92
95
|
attachLog("");
|
|
93
96
|
attachLog("说明:");
|
|
94
97
|
attachLog(
|
|
@@ -101,11 +104,16 @@ export async function handleAttach(options) {
|
|
|
101
104
|
attachLog(" tt-help attach");
|
|
102
105
|
attachLog(" tt-help attach -p 5 -i 10");
|
|
103
106
|
attachLog(" tt-help attach -p 3 -i 5 -s http://127.0.0.1:3001");
|
|
107
|
+
attachLog(" tt-help attach -c PL,DE,FR -p 5");
|
|
104
108
|
return;
|
|
105
109
|
}
|
|
106
110
|
|
|
111
|
+
const countryStr =
|
|
112
|
+
attachCountries && attachCountries.length > 0
|
|
113
|
+
? `, 猜测国家: ${attachCountries.join(", ")}`
|
|
114
|
+
: "";
|
|
107
115
|
attachLog(
|
|
108
|
-
`[Attach] 并行数: ${attachParallel}, 空闲间隔: ${attachInterval}秒, 服务端: ${serverUrl}`,
|
|
116
|
+
`[Attach] 并行数: ${attachParallel}, 空闲间隔: ${attachInterval}秒, 服务端: ${serverUrl}${countryStr}`,
|
|
109
117
|
);
|
|
110
118
|
|
|
111
119
|
const scraper = new TikTokScraper();
|
|
@@ -162,8 +170,12 @@ export async function handleAttach(options) {
|
|
|
162
170
|
process.exit(0);
|
|
163
171
|
}
|
|
164
172
|
|
|
173
|
+
const countryParam =
|
|
174
|
+
attachCountries && attachCountries.length > 0
|
|
175
|
+
? `&countries=${encodeURIComponent(attachCountries.join(","))}`
|
|
176
|
+
: "";
|
|
165
177
|
const { total, tasks } = await apiGet(
|
|
166
|
-
`${serverUrl}/api/user-update-tasks?limit=${attachParallel}`,
|
|
178
|
+
`${serverUrl}/api/user-update-tasks?limit=${attachParallel}${countryParam}`,
|
|
167
179
|
);
|
|
168
180
|
|
|
169
181
|
if (!tasks || tasks.length === 0) {
|
package/src/cli/comments.js
CHANGED
|
@@ -2,22 +2,11 @@ import { chromium } from "playwright";
|
|
|
2
2
|
import { fetchUserCommentsAPI } from "../lib/api-interceptor-comment.js";
|
|
3
3
|
import { closeCommentPanel } from "../lib/browser/page.js";
|
|
4
4
|
import { server as defaultServer } from "../lib/constants.js";
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
"PT",
|
|
11
|
-
"ES",
|
|
12
|
-
"PL",
|
|
13
|
-
"NL",
|
|
14
|
-
"BE",
|
|
15
|
-
"DE",
|
|
16
|
-
"FR",
|
|
17
|
-
"IT",
|
|
18
|
-
"IE",
|
|
19
|
-
"AT",
|
|
20
|
-
];
|
|
5
|
+
import {
|
|
6
|
+
DEFAULT_TARGET_LOCATIONS,
|
|
7
|
+
isLocationInList,
|
|
8
|
+
normalizeLocation,
|
|
9
|
+
} from "../lib/target-locations.js";
|
|
21
10
|
|
|
22
11
|
async function waitForPageReady(page, timeout = 30000) {
|
|
23
12
|
const startTime = Date.now();
|
|
@@ -130,7 +119,7 @@ async function runAutoMode(options) {
|
|
|
130
119
|
`\n[Comments Auto] 并行: ${actualParallel}, 间隔: ${actualInterval}s, 评论数: ${actualMaxComments}`,
|
|
131
120
|
);
|
|
132
121
|
console.error(`服务器: ${serverUrl}`);
|
|
133
|
-
console.error(`目标国家: ${
|
|
122
|
+
console.error(`目标国家: ${DEFAULT_TARGET_LOCATIONS.join(", ")}`);
|
|
134
123
|
console.error(`开始循环接收任务...\n`);
|
|
135
124
|
|
|
136
125
|
let browser;
|
|
@@ -228,10 +217,10 @@ async function runAutoMode(options) {
|
|
|
228
217
|
for (const task of tasks) {
|
|
229
218
|
try {
|
|
230
219
|
const { id, href, locationCreated, ttSeller } = task;
|
|
231
|
-
const loc = (locationCreated
|
|
220
|
+
const loc = normalizeLocation(locationCreated);
|
|
232
221
|
|
|
233
222
|
// 检查目标国家
|
|
234
|
-
if (loc && !
|
|
223
|
+
if (loc && !isLocationInList(loc, DEFAULT_TARGET_LOCATIONS)) {
|
|
235
224
|
// 非目标国家,直接 commit 跳过
|
|
236
225
|
await apiPut(`${serverUrl}/api/comment-task/${id}`);
|
|
237
226
|
skippedCount++;
|
|
@@ -487,12 +476,15 @@ export async function handleComments(options) {
|
|
|
487
476
|
),
|
|
488
477
|
];
|
|
489
478
|
|
|
490
|
-
const guessedLocation = (videoInfo?.locationCreated
|
|
479
|
+
const guessedLocation = normalizeLocation(videoInfo?.locationCreated);
|
|
491
480
|
const serverUrl = commentsServer || defaultServer;
|
|
492
481
|
|
|
493
|
-
if (
|
|
482
|
+
if (
|
|
483
|
+
guessedLocation &&
|
|
484
|
+
!isLocationInList(guessedLocation, DEFAULT_TARGET_LOCATIONS)
|
|
485
|
+
) {
|
|
494
486
|
console.error(`\n猜测国家: ${guessedLocation},非目标国家,跳过保存`);
|
|
495
|
-
console.error(`目标国家: ${
|
|
487
|
+
console.error(`目标国家: ${DEFAULT_TARGET_LOCATIONS.join(", ")}`);
|
|
496
488
|
} else {
|
|
497
489
|
console.error(`\n正在提交 ${authors.length} 个评论作者到服务端...`);
|
|
498
490
|
console.error(`服务端: ${serverUrl}`);
|
package/src/lib/args.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { readFileSync } from "fs";
|
|
2
2
|
import { server as defaultServer } from "./constants.js";
|
|
3
|
+
import { DEFAULT_TARGET_LOCATIONS_CSV } from "./target-locations.js";
|
|
3
4
|
|
|
4
5
|
const PRESETS = ["fast", "normal", "slow", "stealth"];
|
|
5
6
|
|
|
@@ -171,7 +172,7 @@ function parseExploreArgs(args) {
|
|
|
171
172
|
let exploreEnableFollow = true;
|
|
172
173
|
let exploreMaxFollowing = 50;
|
|
173
174
|
let exploreMaxFollowers = 50;
|
|
174
|
-
let exploreLocation =
|
|
175
|
+
let exploreLocation = DEFAULT_TARGET_LOCATIONS_CSV;
|
|
175
176
|
let exploreJobLocations = null;
|
|
176
177
|
let exploreMaxUsers = 0;
|
|
177
178
|
let explorePort = null;
|
|
@@ -462,6 +463,7 @@ function parseAttachArgs(args) {
|
|
|
462
463
|
let parallel = 1;
|
|
463
464
|
let interval = 10;
|
|
464
465
|
let serverUrl = defaultServer;
|
|
466
|
+
let countries = [];
|
|
465
467
|
|
|
466
468
|
for (let i = 0; i < args.length; i++) {
|
|
467
469
|
const arg = args[i];
|
|
@@ -471,6 +473,11 @@ function parseAttachArgs(args) {
|
|
|
471
473
|
interval = parseInt(args[++i], 10) || 10;
|
|
472
474
|
} else if (arg === "-s" || arg === "--server") {
|
|
473
475
|
serverUrl = args[++i];
|
|
476
|
+
} else if (arg === "-c" || arg === "--countries") {
|
|
477
|
+
countries = args[++i]
|
|
478
|
+
.split(",")
|
|
479
|
+
.map((c) => c.trim().toUpperCase())
|
|
480
|
+
.filter(Boolean);
|
|
474
481
|
}
|
|
475
482
|
}
|
|
476
483
|
|
|
@@ -479,6 +486,7 @@ function parseAttachArgs(args) {
|
|
|
479
486
|
attachParallel: parallel,
|
|
480
487
|
attachInterval: interval,
|
|
481
488
|
serverUrl,
|
|
489
|
+
attachCountries: countries,
|
|
482
490
|
urls: [],
|
|
483
491
|
outputFormat: "json",
|
|
484
492
|
exploreCount: 0,
|
package/src/lib/browser/page.js
CHANGED
|
@@ -67,19 +67,20 @@ export async function withBrowserRecovery(fn, browser, page, cdpOptions, port) {
|
|
|
67
67
|
}
|
|
68
68
|
|
|
69
69
|
/**
|
|
70
|
-
*
|
|
71
|
-
*
|
|
72
|
-
* 未登录:不存在 sessionid Cookie
|
|
70
|
+
* 稳定判断登录状态:先检查 sessionid Cookie,再用页面 DOM 做验真。
|
|
71
|
+
* 仅有 sessionid 不足以说明会话仍然有效,因此需要二次确认。
|
|
73
72
|
*/
|
|
74
73
|
export async function isLoggedIn(page) {
|
|
75
74
|
const cookies = await page.context().cookies("https://www.tiktok.com");
|
|
76
|
-
|
|
75
|
+
const hasSessionId = cookies.some((c) => c.name === "sessionid");
|
|
76
|
+
if (!hasSessionId) return false;
|
|
77
|
+
|
|
78
|
+
return await isLoggedInByDom(page);
|
|
77
79
|
}
|
|
78
80
|
|
|
79
81
|
/**
|
|
80
|
-
* 通过 DOM
|
|
81
|
-
*
|
|
82
|
-
* @deprecated 推荐使用 isLoggedIn(基于 Cookie)
|
|
82
|
+
* 通过 DOM 元素判断登录状态(验真方案)
|
|
83
|
+
* 依赖页面渲染完成,因此不单独作为主判断,而是和 Cookie 组合使用。
|
|
83
84
|
*/
|
|
84
85
|
export async function isLoggedInByDom(page) {
|
|
85
86
|
// 先等客户端渲染完成:登录态元素或登录按钮,哪个先出现就停止等待
|
package/src/lib/constants.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { join, dirname } from "path";
|
|
2
2
|
import { readFileSync, writeFileSync, existsSync } from "fs";
|
|
3
3
|
import { fileURLToPath } from "url";
|
|
4
|
+
import { DEFAULT_TARGET_LOCATIONS_CSV } from "./target-locations.js";
|
|
4
5
|
|
|
5
6
|
const __filename = fileURLToPath(import.meta.url);
|
|
6
7
|
const __dirname = dirname(__filename);
|
|
@@ -121,7 +122,7 @@ const HELP_TEXT = [
|
|
|
121
122
|
" 预设: fast, normal(默认), slow, stealth",
|
|
122
123
|
" 选项:",
|
|
123
124
|
" --server <URL> 服务端地址,默认 http://127.0.0.1:3001",
|
|
124
|
-
|
|
125
|
+
` --location <国家代码> 国家筛选,逗号分隔,默认 ${DEFAULT_TARGET_LOCATIONS_CSV}`,
|
|
125
126
|
" --job-locations <国家> 任务国家筛选,逗号分隔(仅筛选服务端任务)",
|
|
126
127
|
" --max-comments <数量> 每视频最大评论数,默认 10",
|
|
127
128
|
" --max-guess <数量> 每视频最大猜你喜欢数,默认 0",
|
|
@@ -145,12 +146,14 @@ const HELP_TEXT = [
|
|
|
145
146
|
" 示例: tt-help info https://www.tiktok.com/@nike",
|
|
146
147
|
" tt-help info https://www.tiktok.com/@nike/video/7234567890 --onlyvideo",
|
|
147
148
|
"",
|
|
148
|
-
" tt-help attach [-p 并行数] [-i 间隔秒数] [-s 服务端地址]",
|
|
149
|
+
" tt-help attach [-p 并行数] [-i 间隔秒数] [-s 服务端地址] [-c 国家列表]",
|
|
149
150
|
" 后台轮询服务端任务接口,自动抓取 TikTok 用户信息",
|
|
150
151
|
" -p, --parallel <N> 并行抓取数(默认: 1)",
|
|
151
152
|
" -i, --interval <N> 无任务时轮询间隔,单位秒(默认: 10)",
|
|
152
153
|
" -s, --server <URL> 服务端地址(默认: http://127.0.0.1:3001)",
|
|
154
|
+
" -c, --countries <A,B> 猜测国家列表(逗号分隔,如 PL,DE,FR)",
|
|
153
155
|
" 示例: tt-help attach -p 5 -i 10",
|
|
156
|
+
" tt-help attach -c PL,DE,FR -p 5",
|
|
154
157
|
"",
|
|
155
158
|
" tt-help watch -o <db路径> [-p 端口]",
|
|
156
159
|
" 启动监控服务;运行期主数据仅来自 SQLite",
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
const DEFAULT_TARGET_LOCATIONS = [
|
|
2
|
+
"CZ",
|
|
3
|
+
"GR",
|
|
4
|
+
"HU",
|
|
5
|
+
"PT",
|
|
6
|
+
"ES",
|
|
7
|
+
"PL",
|
|
8
|
+
"NL",
|
|
9
|
+
"BE",
|
|
10
|
+
"DE",
|
|
11
|
+
"FR",
|
|
12
|
+
"IT",
|
|
13
|
+
"IE",
|
|
14
|
+
"AT",
|
|
15
|
+
];
|
|
16
|
+
|
|
17
|
+
const DEFAULT_TARGET_LOCATIONS_CSV = DEFAULT_TARGET_LOCATIONS.join(",");
|
|
18
|
+
|
|
19
|
+
function normalizeLocation(location) {
|
|
20
|
+
if (location == null) return null;
|
|
21
|
+
const normalized = String(location).trim().toUpperCase();
|
|
22
|
+
return normalized || null;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
function normalizeLocationList(locations, fallback = DEFAULT_TARGET_LOCATIONS) {
|
|
26
|
+
const source = Array.isArray(locations)
|
|
27
|
+
? locations
|
|
28
|
+
: String(locations == null ? "" : locations).split(",");
|
|
29
|
+
const normalized = source
|
|
30
|
+
.map((location) => normalizeLocation(location))
|
|
31
|
+
.filter(Boolean);
|
|
32
|
+
return normalized.length > 0 ? [...new Set(normalized)] : [...fallback];
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
function isLocationInList(location, locations) {
|
|
36
|
+
const normalizedLocation = normalizeLocation(location);
|
|
37
|
+
if (!normalizedLocation) return false;
|
|
38
|
+
return normalizeLocationList(locations).includes(normalizedLocation);
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
function findFirstMatchingLocation(
|
|
42
|
+
locations,
|
|
43
|
+
targetLocations = DEFAULT_TARGET_LOCATIONS,
|
|
44
|
+
) {
|
|
45
|
+
const normalizedTargetLocations = normalizeLocationList(targetLocations);
|
|
46
|
+
const normalizedLocations = normalizeLocationList(locations, []);
|
|
47
|
+
return (
|
|
48
|
+
normalizedLocations.find((location) =>
|
|
49
|
+
normalizedTargetLocations.includes(location),
|
|
50
|
+
) || null
|
|
51
|
+
);
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
export {
|
|
55
|
+
DEFAULT_TARGET_LOCATIONS,
|
|
56
|
+
DEFAULT_TARGET_LOCATIONS_CSV,
|
|
57
|
+
findFirstMatchingLocation,
|
|
58
|
+
isLocationInList,
|
|
59
|
+
normalizeLocation,
|
|
60
|
+
normalizeLocationList,
|
|
61
|
+
};
|
package/src/scraper/core.js
CHANGED
|
@@ -10,13 +10,18 @@ import {
|
|
|
10
10
|
} from "./modules/page-helpers.js";
|
|
11
11
|
import { extractCommentAuthors } from "./modules/comment-extractor.js";
|
|
12
12
|
import { extractGuessVideos } from "./modules/guess-extractor.js";
|
|
13
|
+
import {
|
|
14
|
+
DEFAULT_TARGET_LOCATIONS_CSV,
|
|
15
|
+
isLocationInList,
|
|
16
|
+
normalizeLocationList,
|
|
17
|
+
} from "../lib/target-locations.js";
|
|
13
18
|
|
|
14
19
|
async function scrapeSingleVideo(
|
|
15
20
|
page,
|
|
16
21
|
maxComments,
|
|
17
22
|
maxGuess,
|
|
18
23
|
log,
|
|
19
|
-
location =
|
|
24
|
+
location = DEFAULT_TARGET_LOCATIONS_CSV,
|
|
20
25
|
) {
|
|
21
26
|
const config = getDelayConfig();
|
|
22
27
|
|
|
@@ -52,14 +57,8 @@ async function scrapeSingleVideo(
|
|
|
52
57
|
let captchaStage = "";
|
|
53
58
|
let captchaMessage = "";
|
|
54
59
|
|
|
55
|
-
const locationList = (location
|
|
56
|
-
|
|
57
|
-
.map((s) => s.trim().toUpperCase());
|
|
58
|
-
if (
|
|
59
|
-
locationList.includes(
|
|
60
|
-
userData.locationCreated?.toUpperCase?.() || userData.locationCreated,
|
|
61
|
-
)
|
|
62
|
-
) {
|
|
60
|
+
const locationList = normalizeLocationList(location);
|
|
61
|
+
if (isLocationInList(userData.locationCreated, locationList)) {
|
|
63
62
|
if (maxGuess > 0) {
|
|
64
63
|
guessVideos = await extractGuessVideos(page, maxGuess);
|
|
65
64
|
}
|
|
@@ -4,6 +4,12 @@ export { ensureBrowserReady };
|
|
|
4
4
|
import { getUserInfo, collectVideos } from "../videos/core.js";
|
|
5
5
|
import { extractFollowAndFollowers } from "./modules/follow-extractor.js";
|
|
6
6
|
import { extractVideoLocation } from "../lib/scrape.js";
|
|
7
|
+
import {
|
|
8
|
+
DEFAULT_TARGET_LOCATIONS_CSV,
|
|
9
|
+
findFirstMatchingLocation,
|
|
10
|
+
isLocationInList,
|
|
11
|
+
normalizeLocationList,
|
|
12
|
+
} from "../lib/target-locations.js";
|
|
7
13
|
import {
|
|
8
14
|
maxFollowing as globalMaxFollowing,
|
|
9
15
|
maxFollowers as globalMaxFollowers,
|
|
@@ -16,7 +22,7 @@ async function processExplore(page, username, options, log) {
|
|
|
16
22
|
loggedIn = false, // 由外部传入登录状态,避免每次调用 isLoggedIn(page)
|
|
17
23
|
maxFollowing = 50,
|
|
18
24
|
maxFollowers = 50,
|
|
19
|
-
location =
|
|
25
|
+
location = DEFAULT_TARGET_LOCATIONS_CSV,
|
|
20
26
|
} = options;
|
|
21
27
|
|
|
22
28
|
const result = {
|
|
@@ -35,6 +41,8 @@ async function processExplore(page, username, options, log) {
|
|
|
35
41
|
error: null,
|
|
36
42
|
};
|
|
37
43
|
|
|
44
|
+
const locationList = normalizeLocationList(location);
|
|
45
|
+
|
|
38
46
|
try {
|
|
39
47
|
log(` 访问 @${username} 主页...`);
|
|
40
48
|
const videoList = await collectVideos(page, username, maxVideos, log);
|
|
@@ -66,9 +74,11 @@ async function processExplore(page, username, options, log) {
|
|
|
66
74
|
return result;
|
|
67
75
|
}
|
|
68
76
|
|
|
69
|
-
// 从最多 5 个视频并发获取 locationCreated
|
|
77
|
+
// 从最多 5 个视频并发获取 locationCreated。
|
|
78
|
+
// 新规则:采样列表里只要有任一国家命中目标列表,就直接采用该国家;否则回退到众数。
|
|
70
79
|
const SAMPLE_SIZE = 5;
|
|
71
80
|
let locationCreated = null;
|
|
81
|
+
let locationDecision = null;
|
|
72
82
|
const sampleVideos = videoArray.slice(0, SAMPLE_SIZE);
|
|
73
83
|
if (sampleVideos.length > 0) {
|
|
74
84
|
const sampleUrls = sampleVideos.map((v) =>
|
|
@@ -80,26 +90,34 @@ async function processExplore(page, username, options, log) {
|
|
|
80
90
|
log(
|
|
81
91
|
` 国家采样(${locations.length}个): [${locations.filter(Boolean).join(", ") || "无数据"}]`,
|
|
82
92
|
);
|
|
93
|
+
const normalizedLocations = normalizeLocationList(locations, []);
|
|
94
|
+
const matchedTargetLocation = findFirstMatchingLocation(
|
|
95
|
+
normalizedLocations,
|
|
96
|
+
locationList,
|
|
97
|
+
);
|
|
83
98
|
const freq = {};
|
|
84
|
-
for (const
|
|
85
|
-
|
|
86
|
-
const key = loc.toUpperCase();
|
|
87
|
-
freq[key] = (freq[key] || 0) + 1;
|
|
88
|
-
}
|
|
99
|
+
for (const key of normalizedLocations) {
|
|
100
|
+
freq[key] = (freq[key] || 0) + 1;
|
|
89
101
|
}
|
|
90
102
|
const entries = Object.entries(freq).sort((a, b) => b[1] - a[1]);
|
|
91
|
-
|
|
103
|
+
if (matchedTargetLocation) {
|
|
104
|
+
locationCreated = matchedTargetLocation;
|
|
105
|
+
locationDecision = "命中目标国家";
|
|
106
|
+
} else if (entries.length > 0) {
|
|
107
|
+
locationCreated = entries[0][0];
|
|
108
|
+
locationDecision = "回退众数";
|
|
109
|
+
}
|
|
92
110
|
}
|
|
93
111
|
|
|
94
112
|
result.locationCreated = locationCreated || null;
|
|
95
|
-
log(
|
|
113
|
+
log(
|
|
114
|
+
` 国家: ${result.locationCreated || "未知"}${locationDecision ? ` (${locationDecision})` : ""}`,
|
|
115
|
+
);
|
|
96
116
|
|
|
97
117
|
// 国家筛选
|
|
98
|
-
const
|
|
99
|
-
.
|
|
100
|
-
|
|
101
|
-
const isTargetLocation = locationList.includes(
|
|
102
|
-
result.locationCreated?.toUpperCase?.() || result.locationCreated,
|
|
118
|
+
const isTargetLocation = isLocationInList(
|
|
119
|
+
result.locationCreated,
|
|
120
|
+
locationList,
|
|
103
121
|
);
|
|
104
122
|
|
|
105
123
|
if (isTargetLocation) {
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { delay, getDelayConfig } from "./page-helpers.js";
|
|
2
2
|
import { scrollAndCollect } from "./scroll-collector.js";
|
|
3
|
+
import { extractUniqueId, toProfileUrl } from "../../lib/url.js";
|
|
3
4
|
|
|
4
5
|
const FILTER_WORDS = ["主页", "已关注", "粉丝", "推荐"];
|
|
5
6
|
|
|
@@ -48,7 +49,14 @@ async function waitForListContent(page, minChildren = 1, timeout = 15000) {
|
|
|
48
49
|
.catch(() => {});
|
|
49
50
|
}
|
|
50
51
|
|
|
51
|
-
|
|
52
|
+
function getProfileUrlForRetry(page) {
|
|
53
|
+
const currentUrl = page.url();
|
|
54
|
+
const uniqueId = extractUniqueId(currentUrl);
|
|
55
|
+
if (uniqueId) return toProfileUrl(uniqueId);
|
|
56
|
+
return currentUrl;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
async function openFollowModal(page, log = () => {}) {
|
|
52
60
|
const tryOpen = async () =>
|
|
53
61
|
page.evaluate((selectors) => {
|
|
54
62
|
const clickTarget = (node) => {
|
|
@@ -83,7 +91,16 @@ async function openFollowModal(page) {
|
|
|
83
91
|
for (let attempt = 1; attempt <= 3; attempt++) {
|
|
84
92
|
await waitForFollowTrigger(page, attempt === 1 ? 15000 : 8000);
|
|
85
93
|
opened = await tryOpen();
|
|
86
|
-
if (opened)
|
|
94
|
+
if (opened) {
|
|
95
|
+
if (attempt >= 2) {
|
|
96
|
+
log(` [关注弹窗] 第 ${attempt} 次已点击关注入口: ${opened}`);
|
|
97
|
+
}
|
|
98
|
+
break;
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
if (attempt >= 2) {
|
|
102
|
+
log(` [关注弹窗] 第 ${attempt} 次仍未找到可点击的关注入口`);
|
|
103
|
+
}
|
|
87
104
|
|
|
88
105
|
await page
|
|
89
106
|
.evaluate(() => {
|
|
@@ -94,6 +111,7 @@ async function openFollowModal(page) {
|
|
|
94
111
|
}
|
|
95
112
|
|
|
96
113
|
if (!opened) {
|
|
114
|
+
log(" [关注弹窗] 重试后仍未找到关注入口");
|
|
97
115
|
throw new Error(
|
|
98
116
|
"未找到关注入口元素,请确认当前页面为用户主页或页面结构已变化",
|
|
99
117
|
);
|
|
@@ -107,20 +125,31 @@ async function openFollowModal(page) {
|
|
|
107
125
|
timeout: 30000,
|
|
108
126
|
});
|
|
109
127
|
containerReady = true;
|
|
128
|
+
if (attempt >= 2) {
|
|
129
|
+
log(` [关注弹窗] 第 ${attempt} 次等待后,关注列表容器已出现`);
|
|
130
|
+
}
|
|
110
131
|
break;
|
|
111
132
|
} catch (e) {
|
|
112
133
|
if (attempt === 1) {
|
|
113
|
-
//
|
|
114
|
-
|
|
134
|
+
// 第一次超时,重新 goto 用户主页后重试
|
|
135
|
+
const retryUrl = getProfileUrlForRetry(page);
|
|
136
|
+
log(
|
|
137
|
+
` [关注弹窗] 第 1 次等待列表超时,准备 goto 主页重试: ${retryUrl} | ${e.message}`,
|
|
138
|
+
);
|
|
139
|
+
await page.goto(retryUrl, { waitUntil: "domcontentloaded" });
|
|
115
140
|
await delay(2000, 3000);
|
|
141
|
+
log(" [关注弹窗] 主页跳转完成,重新点击关注入口");
|
|
116
142
|
// 重新点击 follow 入口
|
|
117
143
|
opened = await tryOpen();
|
|
118
144
|
if (!opened) {
|
|
145
|
+
log(" [关注弹窗] goto 主页后仍未找到关注入口");
|
|
119
146
|
throw new Error(
|
|
120
|
-
"
|
|
147
|
+
"goto 主页后仍未找到关注入口元素,请确认当前页面为用户主页",
|
|
121
148
|
);
|
|
122
149
|
}
|
|
150
|
+
log(` [关注弹窗] goto 主页后已重新点击关注入口: ${opened}`);
|
|
123
151
|
} else {
|
|
152
|
+
log(` [关注弹窗] 第 2 次等待列表仍超时,结束本次提取: ${e.message}`);
|
|
124
153
|
throw e;
|
|
125
154
|
}
|
|
126
155
|
}
|
|
@@ -200,7 +229,7 @@ async function extractUsersFromModal(page, maxUsers) {
|
|
|
200
229
|
async function extractFollowAndFollowers(page, options = {}) {
|
|
201
230
|
const { maxFollowing = 999, maxFollowers = 999, log = () => {} } = options;
|
|
202
231
|
|
|
203
|
-
await openFollowModal(page);
|
|
232
|
+
await openFollowModal(page, log);
|
|
204
233
|
|
|
205
234
|
const following = await extractUsersFromModal(page, maxFollowing);
|
|
206
235
|
log(` 已关注: ${following.length}`);
|
|
@@ -8,6 +8,7 @@ import { detectCaptcha } from "./modules/captcha-handler.js";
|
|
|
8
8
|
import { getUserInfo, collectVideos } from "../videos/core.js";
|
|
9
9
|
import { extractFollowAndFollowers } from "./modules/follow-extractor.js";
|
|
10
10
|
import { processExplore } from "./explore-core.js";
|
|
11
|
+
import { DEFAULT_TARGET_LOCATIONS_CSV } from "../lib/target-locations.js";
|
|
11
12
|
|
|
12
13
|
export async function processRefresh(page, username, serverUrl, options, log) {
|
|
13
14
|
const { maxFollowing = 100, maxFollowers = 100, maxVideos = 100 } = options;
|
|
@@ -131,7 +132,7 @@ export async function processRefresh(page, username, serverUrl, options, log) {
|
|
|
131
132
|
enableFollow: true,
|
|
132
133
|
maxFollowing: 5,
|
|
133
134
|
maxFollowers: 5,
|
|
134
|
-
location:
|
|
135
|
+
location: DEFAULT_TARGET_LOCATIONS_CSV,
|
|
135
136
|
},
|
|
136
137
|
log,
|
|
137
138
|
);
|
package/src/watch/data-store.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import fs from "fs";
|
|
2
2
|
import path from "path";
|
|
3
3
|
import Database from "better-sqlite3";
|
|
4
|
+
import { isLocationInList } from "../lib/target-locations.js";
|
|
4
5
|
|
|
5
6
|
// SQLite 用户表(用于判重)
|
|
6
7
|
let db = null;
|
|
@@ -989,6 +990,117 @@ function restoreRawJobsByCountry(country) {
|
|
|
989
990
|
return { restored: count, country: normalizedCountry };
|
|
990
991
|
}
|
|
991
992
|
|
|
993
|
+
function restoreRawJobById(uniqueId) {
|
|
994
|
+
if (!db) {
|
|
995
|
+
return { restored: 0, uniqueId, error: "db not ready" };
|
|
996
|
+
}
|
|
997
|
+
|
|
998
|
+
const safeId = String(uniqueId).trim();
|
|
999
|
+
if (!safeId) {
|
|
1000
|
+
return { restored: 0, uniqueId: safeId, error: "uniqueId is required" };
|
|
1001
|
+
}
|
|
1002
|
+
|
|
1003
|
+
const exists =
|
|
1004
|
+
db
|
|
1005
|
+
.prepare("SELECT COUNT(*) as c FROM raw_jobs WHERE unique_id = ?")
|
|
1006
|
+
.get(safeId)?.c || 0;
|
|
1007
|
+
|
|
1008
|
+
if (!exists) {
|
|
1009
|
+
return { restored: 0, uniqueId: safeId };
|
|
1010
|
+
}
|
|
1011
|
+
|
|
1012
|
+
const restoreTxn = db.transaction(() => {
|
|
1013
|
+
db.prepare(
|
|
1014
|
+
`
|
|
1015
|
+
INSERT OR REPLACE INTO jobs (
|
|
1016
|
+
unique_id, nickname, status, sources, claimed_by, claimed_at, error,
|
|
1017
|
+
pinned, no_video, restricted, user_update_count, tt_seller, verified,
|
|
1018
|
+
video_count, comment_count, guessed_location, location_created,
|
|
1019
|
+
follower_count, following_count, heart_count, refresh_time,
|
|
1020
|
+
processed, processed_at, created_at, updated_at, region, signature, sec_uid
|
|
1021
|
+
)
|
|
1022
|
+
SELECT
|
|
1023
|
+
unique_id, nickname, status, sources, claimed_by, claimed_at, error,
|
|
1024
|
+
pinned, no_video, restricted, user_update_count, tt_seller, verified,
|
|
1025
|
+
video_count, comment_count, guessed_location, location_created,
|
|
1026
|
+
follower_count, following_count, heart_count, refresh_time,
|
|
1027
|
+
processed, processed_at, created_at, updated_at, region, signature, sec_uid
|
|
1028
|
+
FROM raw_jobs WHERE unique_id = ?
|
|
1029
|
+
`,
|
|
1030
|
+
).run(safeId);
|
|
1031
|
+
|
|
1032
|
+
db.prepare("DELETE FROM raw_jobs WHERE unique_id = ?").run(safeId);
|
|
1033
|
+
});
|
|
1034
|
+
|
|
1035
|
+
restoreTxn();
|
|
1036
|
+
return { restored: 1, uniqueId: safeId };
|
|
1037
|
+
}
|
|
1038
|
+
|
|
1039
|
+
function restoreRawJobsByFilter({ search, location }) {
|
|
1040
|
+
if (!db) {
|
|
1041
|
+
return { restored: 0, error: "db not ready" };
|
|
1042
|
+
}
|
|
1043
|
+
|
|
1044
|
+
const where = [];
|
|
1045
|
+
const args = [];
|
|
1046
|
+
|
|
1047
|
+
if (search) {
|
|
1048
|
+
where.push(
|
|
1049
|
+
"(LOWER(unique_id) LIKE ? OR LOWER(COALESCE(nickname, '')) LIKE ?)",
|
|
1050
|
+
);
|
|
1051
|
+
const likeVal = `%${search.toLowerCase()}%`;
|
|
1052
|
+
args.push(likeVal, likeVal);
|
|
1053
|
+
}
|
|
1054
|
+
|
|
1055
|
+
if (location) {
|
|
1056
|
+
where.push("COALESCE(guessed_location, '未知') = ?");
|
|
1057
|
+
args.push(location);
|
|
1058
|
+
}
|
|
1059
|
+
|
|
1060
|
+
if (where.length === 0) {
|
|
1061
|
+
return { restored: 0, error: "at least one filter is required" };
|
|
1062
|
+
}
|
|
1063
|
+
|
|
1064
|
+
const whereSql = where.join(" AND ");
|
|
1065
|
+
|
|
1066
|
+
const count =
|
|
1067
|
+
db
|
|
1068
|
+
.prepare(
|
|
1069
|
+
`SELECT COUNT(*) as c FROM raw_jobs WHERE ${whereSql}`,
|
|
1070
|
+
)
|
|
1071
|
+
.get(...args)?.c || 0;
|
|
1072
|
+
|
|
1073
|
+
if (!count) {
|
|
1074
|
+
return { restored: 0 };
|
|
1075
|
+
}
|
|
1076
|
+
|
|
1077
|
+
const restoreTxn = db.transaction(() => {
|
|
1078
|
+
db.prepare(
|
|
1079
|
+
`
|
|
1080
|
+
INSERT OR REPLACE INTO jobs (
|
|
1081
|
+
unique_id, nickname, status, sources, claimed_by, claimed_at, error,
|
|
1082
|
+
pinned, no_video, restricted, user_update_count, tt_seller, verified,
|
|
1083
|
+
video_count, comment_count, guessed_location, location_created,
|
|
1084
|
+
follower_count, following_count, heart_count, refresh_time,
|
|
1085
|
+
processed, processed_at, created_at, updated_at, region, signature, sec_uid
|
|
1086
|
+
)
|
|
1087
|
+
SELECT
|
|
1088
|
+
unique_id, nickname, status, sources, claimed_by, claimed_at, error,
|
|
1089
|
+
pinned, no_video, restricted, user_update_count, tt_seller, verified,
|
|
1090
|
+
video_count, comment_count, guessed_location, location_created,
|
|
1091
|
+
follower_count, following_count, heart_count, refresh_time,
|
|
1092
|
+
processed, processed_at, created_at, updated_at, region, signature, sec_uid
|
|
1093
|
+
FROM raw_jobs WHERE ${whereSql}
|
|
1094
|
+
`,
|
|
1095
|
+
).run(...args);
|
|
1096
|
+
|
|
1097
|
+
db.prepare(`DELETE FROM raw_jobs WHERE ${whereSql}`).run(...args);
|
|
1098
|
+
});
|
|
1099
|
+
|
|
1100
|
+
restoreTxn();
|
|
1101
|
+
return { restored: count };
|
|
1102
|
+
}
|
|
1103
|
+
|
|
992
1104
|
function getRawJobsPageFromDb({ search, location, limit, offset }) {
|
|
993
1105
|
if (!db) return null;
|
|
994
1106
|
|
|
@@ -1911,8 +2023,7 @@ export function createStore(filePath) {
|
|
|
1911
2023
|
// 国家过滤:如果指定了 locations,只保留 guessedLocation 在列表中的用户
|
|
1912
2024
|
function locationFilter(u) {
|
|
1913
2025
|
if (!locations || locations.length === 0) return true;
|
|
1914
|
-
|
|
1915
|
-
return locations.includes(loc);
|
|
2026
|
+
return isLocationInList(u.guessedLocation, locations);
|
|
1916
2027
|
}
|
|
1917
2028
|
|
|
1918
2029
|
// 从候选列表中按优先级取第一个:pinned > 超时回收 > seed > ttSeller(仅登录) > follow > other
|
|
@@ -2474,21 +2585,33 @@ export function createStore(filePath) {
|
|
|
2474
2585
|
return Array.from(clientErrors.values());
|
|
2475
2586
|
}
|
|
2476
2587
|
|
|
2477
|
-
function getPendingUserUpdateTasks(limit) {
|
|
2588
|
+
function getPendingUserUpdateTasks(limit, countries) {
|
|
2589
|
+
const targetCountries = countries
|
|
2590
|
+
? countries.map((c) => String(c).trim().toUpperCase())
|
|
2591
|
+
: [];
|
|
2592
|
+
const hasCountryFilter = targetCountries.length > 0;
|
|
2593
|
+
|
|
2478
2594
|
if (db) {
|
|
2479
2595
|
const l = Math.max(1, parseInt(limit) || 5);
|
|
2480
|
-
|
|
2481
|
-
|
|
2482
|
-
|
|
2483
|
-
|
|
2484
|
-
|
|
2485
|
-
|
|
2486
|
-
|
|
2487
|
-
|
|
2488
|
-
|
|
2489
|
-
|
|
2490
|
-
)
|
|
2491
|
-
|
|
2596
|
+
|
|
2597
|
+
let sql = `
|
|
2598
|
+
SELECT *
|
|
2599
|
+
FROM jobs
|
|
2600
|
+
WHERE COALESCE(tt_seller, '') = ''
|
|
2601
|
+
AND COALESCE(user_update_count, 0) <= 0
|
|
2602
|
+
`;
|
|
2603
|
+
const sqlParams = [];
|
|
2604
|
+
|
|
2605
|
+
if (hasCountryFilter) {
|
|
2606
|
+
const placeholders = targetCountries.map(() => "?").join(", ");
|
|
2607
|
+
sql += ` AND UPPER(COALESCE(guessed_location, '')) IN (${placeholders})`;
|
|
2608
|
+
sqlParams.push(...targetCountries);
|
|
2609
|
+
}
|
|
2610
|
+
|
|
2611
|
+
sql += ` ORDER BY created_at ASC, unique_id ASC LIMIT ?`;
|
|
2612
|
+
sqlParams.push(l);
|
|
2613
|
+
|
|
2614
|
+
const rows = db.prepare(sql).all(...sqlParams);
|
|
2492
2615
|
if (rows.length === 0) return [];
|
|
2493
2616
|
const now = Date.now();
|
|
2494
2617
|
const bumpStmt = db.prepare(
|
|
@@ -2520,9 +2643,16 @@ export function createStore(filePath) {
|
|
|
2520
2643
|
const ttSellerEmpty =
|
|
2521
2644
|
u.ttSeller === null || u.ttSeller === undefined || u.ttSeller === "";
|
|
2522
2645
|
if (!ttSellerEmpty) return false;
|
|
2523
|
-
|
|
2646
|
+
if (
|
|
2524
2647
|
updateCount === null || updateCount === undefined || updateCount <= 0
|
|
2525
|
-
)
|
|
2648
|
+
) {
|
|
2649
|
+
if (hasCountryFilter) {
|
|
2650
|
+
const loc = (u.guessedLocation || "").toUpperCase();
|
|
2651
|
+
return targetCountries.includes(loc);
|
|
2652
|
+
}
|
|
2653
|
+
return true;
|
|
2654
|
+
}
|
|
2655
|
+
return false;
|
|
2526
2656
|
})
|
|
2527
2657
|
.slice(0, l);
|
|
2528
2658
|
// 接受任务时 userUpdateCount + 1
|
|
@@ -2814,6 +2944,8 @@ export function createStore(filePath) {
|
|
|
2814
2944
|
moveJobsToRawByCountry,
|
|
2815
2945
|
restoreAttachStuckByCountry,
|
|
2816
2946
|
restoreRawJobsByCountry,
|
|
2947
|
+
restoreRawJobById,
|
|
2948
|
+
restoreRawJobsByFilter,
|
|
2817
2949
|
getUsersPage: getUsersPageFromDb,
|
|
2818
2950
|
getRawJobsPage: getRawJobsPageFromDb,
|
|
2819
2951
|
getTargetUsers: getTargetUsersFromDb,
|
|
@@ -1265,6 +1265,7 @@
|
|
|
1265
1265
|
<option value="">全部国家</option>
|
|
1266
1266
|
</select>
|
|
1267
1267
|
<button onclick="clearRawFilters()">清空筛选</button>
|
|
1268
|
+
<button onclick="restoreFilteredRawJobs()" style="background:#22c55e;color:#fff;border:none;padding:6px 12px;border-radius:6px;cursor:pointer;font-size:12px;">恢复当前筛选到 jobs</button>
|
|
1268
1269
|
</div>
|
|
1269
1270
|
<div class="table-scroll">
|
|
1270
1271
|
<table>
|
|
@@ -1279,6 +1280,7 @@
|
|
|
1279
1280
|
<th>来源</th>
|
|
1280
1281
|
<th>状态</th>
|
|
1281
1282
|
<th>创建时间</th>
|
|
1283
|
+
<th>操作</th>
|
|
1282
1284
|
</tr>
|
|
1283
1285
|
</thead>
|
|
1284
1286
|
<tbody id="rawTable"></tbody>
|
|
@@ -2181,7 +2183,7 @@
|
|
|
2181
2183
|
function renderRawJobsTable(users) {
|
|
2182
2184
|
const el = document.getElementById('rawTable');
|
|
2183
2185
|
if (!users.length) {
|
|
2184
|
-
el.innerHTML = '<tr><td colspan="
|
|
2186
|
+
el.innerHTML = '<tr><td colspan="10" style="color:#666;text-align:center;padding:24px">暂无毛料库任务</td></tr>';
|
|
2185
2187
|
return;
|
|
2186
2188
|
}
|
|
2187
2189
|
el.innerHTML = users.map(u => {
|
|
@@ -2203,6 +2205,7 @@
|
|
|
2203
2205
|
<td data-label="来源">${sources || '-'}</td>
|
|
2204
2206
|
<td data-label="状态">${statusTag}</td>
|
|
2205
2207
|
<td data-label="创建时间" style="font-size:11px;color:#888">${created}</td>
|
|
2208
|
+
<td data-label="操作" style="text-align:center"><button onclick="restoreRawJob('${u.uniqueId}')" style="background:#22c55e;color:#fff;border:none;padding:4px 10px;border-radius:4px;cursor:pointer;font-size:11px;">恢复</button></td>
|
|
2206
2209
|
</tr>`;
|
|
2207
2210
|
}).join('');
|
|
2208
2211
|
}
|
|
@@ -2237,6 +2240,63 @@
|
|
|
2237
2240
|
fetchRawJobs();
|
|
2238
2241
|
}
|
|
2239
2242
|
|
|
2243
|
+
async function restoreRawJob(uniqueId) {
|
|
2244
|
+
if (!window.confirm(`确认将 @${uniqueId} 从毛料库恢复到 jobs 队列吗?`)) {
|
|
2245
|
+
return;
|
|
2246
|
+
}
|
|
2247
|
+
try {
|
|
2248
|
+
const res = await fetch('/api/raw-jobs/restore', {
|
|
2249
|
+
method: 'POST',
|
|
2250
|
+
headers: { 'Content-Type': 'application/json' },
|
|
2251
|
+
body: JSON.stringify({ uniqueId })
|
|
2252
|
+
});
|
|
2253
|
+
const data = await res.json();
|
|
2254
|
+
if (!res.ok || data.error) {
|
|
2255
|
+
showToast(data.error || '恢复失败', true);
|
|
2256
|
+
return;
|
|
2257
|
+
}
|
|
2258
|
+
showToast(`@${uniqueId} 已恢复到队列`);
|
|
2259
|
+
await fetchStats();
|
|
2260
|
+
await fetchRawByCountry();
|
|
2261
|
+
await fetchRawJobs();
|
|
2262
|
+
} catch (e) {
|
|
2263
|
+
showToast('恢复失败: ' + e.message, true);
|
|
2264
|
+
}
|
|
2265
|
+
}
|
|
2266
|
+
|
|
2267
|
+
async function restoreFilteredRawJobs() {
|
|
2268
|
+
const search = document.getElementById('rawSearchInput').value.trim();
|
|
2269
|
+
const location = currentRawLocation;
|
|
2270
|
+
let desc = '当前筛选条件';
|
|
2271
|
+
if (search && location) desc = `搜索="${search}" + 国家=${location}`;
|
|
2272
|
+
else if (search) desc = `搜索="${search}"`;
|
|
2273
|
+
else if (location) desc = `国家=${location}`;
|
|
2274
|
+
if (!window.confirm(`确认将毛料库中符合【${desc}】的任务恢复到 jobs 队列吗?`)) {
|
|
2275
|
+
return;
|
|
2276
|
+
}
|
|
2277
|
+
try {
|
|
2278
|
+
const body = {};
|
|
2279
|
+
if (search) body.search = search;
|
|
2280
|
+
if (location) body.location = location;
|
|
2281
|
+
const res = await fetch('/api/raw-jobs/restore', {
|
|
2282
|
+
method: 'POST',
|
|
2283
|
+
headers: { 'Content-Type': 'application/json' },
|
|
2284
|
+
body: JSON.stringify(body)
|
|
2285
|
+
});
|
|
2286
|
+
const data = await res.json();
|
|
2287
|
+
if (!res.ok || data.error) {
|
|
2288
|
+
showToast(data.error || '恢复失败', true);
|
|
2289
|
+
return;
|
|
2290
|
+
}
|
|
2291
|
+
showToast(`已恢复 ${data.restored} 条任务到队列`);
|
|
2292
|
+
await fetchStats();
|
|
2293
|
+
await fetchRawByCountry();
|
|
2294
|
+
await fetchRawJobs();
|
|
2295
|
+
} catch (e) {
|
|
2296
|
+
showToast('恢复失败: ' + e.message, true);
|
|
2297
|
+
}
|
|
2298
|
+
}
|
|
2299
|
+
|
|
2240
2300
|
async function restoreRawJobsByCountry(country, count) {
|
|
2241
2301
|
const countText = count != null ? `将恢复 ${formatStatNum(count)} 条。` : '';
|
|
2242
2302
|
if (!window.confirm(`确认将 ${country} 从毛料库恢复到 jobs 队列吗?${countText}`)) {
|
package/src/watch/server.js
CHANGED
|
@@ -6,22 +6,7 @@ import { join, dirname } from "path";
|
|
|
6
6
|
import { fileURLToPath } from "url";
|
|
7
7
|
import { spawn } from "child_process";
|
|
8
8
|
import { createStore } from "./data-store.js";
|
|
9
|
-
|
|
10
|
-
const TARGET_LOCATIONS = [
|
|
11
|
-
"CZ",
|
|
12
|
-
"GR",
|
|
13
|
-
"HU",
|
|
14
|
-
"PT",
|
|
15
|
-
"ES",
|
|
16
|
-
"PL",
|
|
17
|
-
"NL",
|
|
18
|
-
"BE",
|
|
19
|
-
"DE",
|
|
20
|
-
"FR",
|
|
21
|
-
"IT",
|
|
22
|
-
"IE",
|
|
23
|
-
"AT",
|
|
24
|
-
];
|
|
9
|
+
import { DEFAULT_TARGET_LOCATIONS } from "../lib/target-locations.js";
|
|
25
10
|
|
|
26
11
|
const __filename = fileURLToPath(import.meta.url);
|
|
27
12
|
|
|
@@ -40,7 +25,7 @@ const __dirname = dirname(__filename);
|
|
|
40
25
|
const publicDir = join(__dirname, "public");
|
|
41
26
|
|
|
42
27
|
function computeStatsIncremental(st) {
|
|
43
|
-
return st.getDashboardStats(
|
|
28
|
+
return st.getDashboardStats(DEFAULT_TARGET_LOCATIONS);
|
|
44
29
|
}
|
|
45
30
|
|
|
46
31
|
function readBody(req) {
|
|
@@ -328,7 +313,7 @@ export function startWatchServer(dataAnchor, port = 3000, existingStore) {
|
|
|
328
313
|
|
|
329
314
|
if (req.method === "GET" && routePath === "/api/stats") {
|
|
330
315
|
const stats = computeStatsIncremental(store);
|
|
331
|
-
stats.targetLocations =
|
|
316
|
+
stats.targetLocations = DEFAULT_TARGET_LOCATIONS;
|
|
332
317
|
sendJSON(res, 200, stats);
|
|
333
318
|
return;
|
|
334
319
|
}
|
|
@@ -348,9 +333,12 @@ export function startWatchServer(dataAnchor, port = 3000, existingStore) {
|
|
|
348
333
|
|
|
349
334
|
if (req.method === "GET" && routePath === "/api/user-update-tasks") {
|
|
350
335
|
const limit = params.limit;
|
|
351
|
-
const
|
|
336
|
+
const countries = params.countries
|
|
337
|
+
? params.countries.split(",").map((c) => c.trim().toUpperCase()).filter(Boolean)
|
|
338
|
+
: [];
|
|
339
|
+
const tasks = store.getPendingUserUpdateTasks(limit, countries);
|
|
352
340
|
const ts = new Date().toISOString().slice(11, 19);
|
|
353
|
-
console.error(`[JOB ${ts}] USER-UPDATE-TASKS: ${tasks.length} tasks`);
|
|
341
|
+
console.error(`[JOB ${ts}] USER-UPDATE-TASKS: ${tasks.length} tasks${countries.length ? ` (countries: ${countries.join(",")})` : ""}`);
|
|
354
342
|
sendJSON(res, 200, { total: tasks.length, tasks });
|
|
355
343
|
return;
|
|
356
344
|
}
|
|
@@ -459,7 +447,7 @@ export function startWatchServer(dataAnchor, port = 3000, existingStore) {
|
|
|
459
447
|
}
|
|
460
448
|
|
|
461
449
|
if (req.method === "GET" && routePath === "/api/target-users") {
|
|
462
|
-
const targetResult = store.getTargetUsers(
|
|
450
|
+
const targetResult = store.getTargetUsers(DEFAULT_TARGET_LOCATIONS);
|
|
463
451
|
const targets = targetResult.users;
|
|
464
452
|
if (req.headers["accept"]?.includes("text/csv")) {
|
|
465
453
|
const columns = [
|
|
@@ -555,7 +543,20 @@ export function startWatchServer(dataAnchor, port = 3000, existingStore) {
|
|
|
555
543
|
if (req.method === "POST" && routePath === "/api/raw-jobs/restore") {
|
|
556
544
|
try {
|
|
557
545
|
const body = await readBody(req);
|
|
558
|
-
|
|
546
|
+
let result;
|
|
547
|
+
if (body.uniqueId) {
|
|
548
|
+
result = store.restoreRawJobById(body.uniqueId);
|
|
549
|
+
} else if (body.search || body.location) {
|
|
550
|
+
result = store.restoreRawJobsByFilter({
|
|
551
|
+
search: body.search || "",
|
|
552
|
+
location: body.location || "",
|
|
553
|
+
});
|
|
554
|
+
} else if (body.country) {
|
|
555
|
+
result = store.restoreRawJobsByCountry(body.country);
|
|
556
|
+
} else {
|
|
557
|
+
sendJSON(res, 400, { error: "missing filter: uniqueId, country, or search/location" });
|
|
558
|
+
return;
|
|
559
|
+
}
|
|
559
560
|
if (result.error) {
|
|
560
561
|
sendJSON(res, 400, result);
|
|
561
562
|
return;
|
|
@@ -623,7 +624,7 @@ export function startWatchServer(dataAnchor, port = 3000, existingStore) {
|
|
|
623
624
|
targetLocation: params.targetLocation,
|
|
624
625
|
limit: params.limit,
|
|
625
626
|
offset: params.offset,
|
|
626
|
-
targetLocations:
|
|
627
|
+
targetLocations: DEFAULT_TARGET_LOCATIONS,
|
|
627
628
|
});
|
|
628
629
|
|
|
629
630
|
if (params.view === "light") {
|