tt-help-cli-ycl 1.3.37 → 1.3.39
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/cli/comments.js +15 -1
- package/src/lib/api-interceptor.js +23 -4
- package/src/lib/args.js +1 -1
- package/src/lib/constants.js +1 -1
- package/src/scraper/core.js +94 -49
- package/src/scraper/explore-core.js +9 -5
- package/src/scraper/refresh-core.js +90 -57
- package/src/videos/core.js +19 -1
- package/src/watch/data-store.js +28 -2
- package/src/watch/public/index.html +15 -2
- package/src/watch/server.js +16 -1
package/package.json
CHANGED
package/src/cli/comments.js
CHANGED
|
@@ -3,7 +3,21 @@ import { fetchUserCommentsAPI } from "../lib/api-interceptor-comment.js";
|
|
|
3
3
|
import { closeCommentPanel } from "../lib/browser/page.js";
|
|
4
4
|
import { server as defaultServer } from "../lib/constants.js";
|
|
5
5
|
|
|
6
|
-
const TARGET_LOCATIONS = [
|
|
6
|
+
const TARGET_LOCATIONS = [
|
|
7
|
+
"CZ",
|
|
8
|
+
"GR",
|
|
9
|
+
"HU",
|
|
10
|
+
"PT",
|
|
11
|
+
"ES",
|
|
12
|
+
"PL",
|
|
13
|
+
"NL",
|
|
14
|
+
"BE",
|
|
15
|
+
"DE",
|
|
16
|
+
"FR",
|
|
17
|
+
"IT",
|
|
18
|
+
"IE",
|
|
19
|
+
"AT",
|
|
20
|
+
];
|
|
7
21
|
|
|
8
22
|
async function waitForPageReady(page, timeout = 30000) {
|
|
9
23
|
const startTime = Date.now();
|
|
@@ -31,10 +31,29 @@ async function processAPIResponse(
|
|
|
31
31
|
const newUrl = reqUrl.replace(/cursor=\d+/, `cursor=${cursor}`);
|
|
32
32
|
|
|
33
33
|
try {
|
|
34
|
-
const pageData = await
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
34
|
+
const pageData = await (() => {
|
|
35
|
+
// 重试包装:处理页面导航导致的执行上下文销毁
|
|
36
|
+
const tryEval = async (retries = 3) => {
|
|
37
|
+
for (let i = 0; i < retries; i++) {
|
|
38
|
+
try {
|
|
39
|
+
return await page.evaluate(async (u) => {
|
|
40
|
+
const res = await fetch(u);
|
|
41
|
+
return await res.json();
|
|
42
|
+
}, newUrl);
|
|
43
|
+
} catch (e) {
|
|
44
|
+
if (
|
|
45
|
+
e.message.includes('Execution context was destroyed') &&
|
|
46
|
+
i < retries - 1
|
|
47
|
+
) {
|
|
48
|
+
await delay(500 * (i + 1), 500 * (i + 1));
|
|
49
|
+
} else {
|
|
50
|
+
throw e;
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
};
|
|
55
|
+
return tryEval();
|
|
56
|
+
})();
|
|
38
57
|
|
|
39
58
|
if (pageData && pageData.itemList) {
|
|
40
59
|
for (const item of pageData.itemList) {
|
package/src/lib/args.js
CHANGED
|
@@ -170,7 +170,7 @@ function parseExploreArgs(args) {
|
|
|
170
170
|
let exploreEnableFollow = true;
|
|
171
171
|
let exploreMaxFollowing = 50;
|
|
172
172
|
let exploreMaxFollowers = 50;
|
|
173
|
-
let exploreLocation = "PL,NL,BE,DE,FR,IT,ES,IE,AT";
|
|
173
|
+
let exploreLocation = "CZ,GR,HU,PT,PL,NL,BE,DE,FR,IT,ES,IE,AT";
|
|
174
174
|
let exploreJobLocations = null;
|
|
175
175
|
let exploreMaxUsers = 0;
|
|
176
176
|
let explorePort = null;
|
package/src/lib/constants.js
CHANGED
|
@@ -121,7 +121,7 @@ const HELP_TEXT = [
|
|
|
121
121
|
" 预设: fast, normal(默认), slow, stealth",
|
|
122
122
|
" 选项:",
|
|
123
123
|
" --server <URL> 服务端地址,默认 http://127.0.0.1:3001",
|
|
124
|
-
" --location <国家代码> 国家筛选,逗号分隔,默认 PL,NL,BE,DE,FR,IT,ES,IE,AT",
|
|
124
|
+
" --location <国家代码> 国家筛选,逗号分隔,默认 CZ,GR,HU,PT,PL,NL,BE,DE,FR,IT,ES,IE,AT",
|
|
125
125
|
" --job-locations <国家> 任务国家筛选,逗号分隔(仅筛选服务端任务)",
|
|
126
126
|
" --max-comments <数量> 每视频最大评论数,默认 10",
|
|
127
127
|
" --max-guess <数量> 每视频最大猜你喜欢数,默认 0",
|
package/src/scraper/core.js
CHANGED
|
@@ -7,14 +7,22 @@ import {
|
|
|
7
7
|
getDelayConfig,
|
|
8
8
|
retryWithBackoff,
|
|
9
9
|
assertPageUrl,
|
|
10
|
-
} from
|
|
11
|
-
import { extractCommentAuthors } from
|
|
12
|
-
import { extractGuessVideos } from
|
|
10
|
+
} from "./modules/page-helpers.js";
|
|
11
|
+
import { extractCommentAuthors } from "./modules/comment-extractor.js";
|
|
12
|
+
import { extractGuessVideos } from "./modules/guess-extractor.js";
|
|
13
13
|
|
|
14
|
-
async function scrapeSingleVideo(
|
|
14
|
+
async function scrapeSingleVideo(
|
|
15
|
+
page,
|
|
16
|
+
maxComments,
|
|
17
|
+
maxGuess,
|
|
18
|
+
log,
|
|
19
|
+
location = "CZ,GR,HU,PT,PL,NL,BE,DE,FR,IT,ES,IE",
|
|
20
|
+
) {
|
|
15
21
|
const config = getDelayConfig();
|
|
16
22
|
|
|
17
|
-
await page
|
|
23
|
+
await page
|
|
24
|
+
.waitForSelector('[class*="VideoMeta"]', { timeout: 10000 })
|
|
25
|
+
.catch(() => {});
|
|
18
26
|
await delay(Math.round(config.commentMax * 0.3), config.commentMax);
|
|
19
27
|
|
|
20
28
|
const userData = await page.evaluate(() => {
|
|
@@ -23,8 +31,8 @@ async function scrapeSingleVideo(page, maxComments, maxGuess, log, location = 'P
|
|
|
23
31
|
if (m) result.uniqueId = m[1];
|
|
24
32
|
const authorEls = document.querySelectorAll('[class*="Author"]');
|
|
25
33
|
for (const el of authorEls) {
|
|
26
|
-
const text = (el.textContent ||
|
|
27
|
-
if (text && !text.includes(
|
|
34
|
+
const text = (el.textContent || "").trim();
|
|
35
|
+
if (text && !text.includes("TikTok") && !text.includes("Share")) {
|
|
28
36
|
result.nickname = text;
|
|
29
37
|
break;
|
|
30
38
|
}
|
|
@@ -35,34 +43,40 @@ async function scrapeSingleVideo(page, maxComments, maxGuess, log, location = 'P
|
|
|
35
43
|
return result;
|
|
36
44
|
});
|
|
37
45
|
|
|
38
|
-
const videoAuthor = userData.uniqueId ?
|
|
39
|
-
if (!videoAuthor) throw new Error(
|
|
46
|
+
const videoAuthor = userData.uniqueId ? "@" + userData.uniqueId : null;
|
|
47
|
+
if (!videoAuthor) throw new Error("无法获取视频作者");
|
|
40
48
|
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
49
|
+
let guessVideos = [];
|
|
50
|
+
let commentUsers = [];
|
|
51
|
+
let captchaDetected = false;
|
|
52
|
+
let captchaStage = "";
|
|
53
|
+
let captchaMessage = "";
|
|
46
54
|
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
55
|
+
const locationList = (location || "ES")
|
|
56
|
+
.split(",")
|
|
57
|
+
.map((s) => s.trim().toUpperCase());
|
|
58
|
+
if (
|
|
59
|
+
locationList.includes(
|
|
60
|
+
userData.locationCreated?.toUpperCase?.() || userData.locationCreated,
|
|
61
|
+
)
|
|
62
|
+
) {
|
|
63
|
+
if (maxGuess > 0) {
|
|
64
|
+
guessVideos = await extractGuessVideos(page, maxGuess);
|
|
65
|
+
}
|
|
66
|
+
if (maxComments > 0) {
|
|
67
|
+
const commentResult = await extractCommentAuthors(page, maxComments);
|
|
68
|
+
commentUsers = commentResult.authors || [];
|
|
69
|
+
if (commentResult.captchaDetected) {
|
|
70
|
+
captchaDetected = true;
|
|
71
|
+
captchaStage = "comment";
|
|
72
|
+
captchaMessage = "评论阶段出现验证码";
|
|
64
73
|
}
|
|
65
74
|
}
|
|
75
|
+
await closeCommentPanel(page);
|
|
76
|
+
if (maxGuess > 0 || maxComments > 0) {
|
|
77
|
+
await delay(Math.round(config.commentMax * 0.3), config.commentMax);
|
|
78
|
+
}
|
|
79
|
+
}
|
|
66
80
|
|
|
67
81
|
return {
|
|
68
82
|
videoAuthor,
|
|
@@ -79,16 +93,25 @@ async function scrapeSingleVideo(page, maxComments, maxGuess, log, location = 'P
|
|
|
79
93
|
|
|
80
94
|
async function runScrape(options) {
|
|
81
95
|
const {
|
|
82
|
-
videoUrl,
|
|
83
|
-
|
|
96
|
+
videoUrl,
|
|
97
|
+
maxVideos = 20,
|
|
98
|
+
maxComments = 999,
|
|
99
|
+
maxGuess = 10,
|
|
100
|
+
preset = null,
|
|
101
|
+
switchMax = null,
|
|
102
|
+
commentMax = null,
|
|
84
103
|
log = console.error,
|
|
85
|
-
browser: externalBrowser = null,
|
|
104
|
+
browser: externalBrowser = null,
|
|
105
|
+
page: externalPage = null,
|
|
86
106
|
} = options;
|
|
87
107
|
|
|
88
108
|
if (preset) {
|
|
89
109
|
setDelayConfig(preset);
|
|
90
110
|
} else if (switchMax || commentMax) {
|
|
91
|
-
setDelayConfig({
|
|
111
|
+
setDelayConfig({
|
|
112
|
+
switchMax: switchMax || 5000,
|
|
113
|
+
commentMax: commentMax || 3000,
|
|
114
|
+
});
|
|
92
115
|
}
|
|
93
116
|
|
|
94
117
|
const config = getDelayConfig();
|
|
@@ -97,7 +120,9 @@ async function runScrape(options) {
|
|
|
97
120
|
|
|
98
121
|
if (!isExternal) {
|
|
99
122
|
log(`视频地址: ${videoUrl}`);
|
|
100
|
-
log(
|
|
123
|
+
log(
|
|
124
|
+
`视频数: ${maxVideos}, 评论数: ${maxComments}, 猜你喜欢: ${maxGuess}, 切换延迟: ${config.switchMax}ms, 评论延迟: ${config.commentMax}ms`,
|
|
125
|
+
);
|
|
101
126
|
}
|
|
102
127
|
|
|
103
128
|
if (isExternal) {
|
|
@@ -113,16 +138,19 @@ async function runScrape(options) {
|
|
|
113
138
|
}
|
|
114
139
|
}
|
|
115
140
|
|
|
116
|
-
await retryWithBackoff(
|
|
117
|
-
|
|
141
|
+
await retryWithBackoff(
|
|
142
|
+
() => page.goto(videoUrl, { waitUntil: "load", timeout: 30000 }),
|
|
143
|
+
{ log },
|
|
144
|
+
);
|
|
145
|
+
assertPageUrl(page, videoUrl.split("/video/")[0]);
|
|
118
146
|
await delay(Math.round(config.switchMax * 0.5), config.switchMax);
|
|
119
147
|
await closeCommentPanel(page);
|
|
120
148
|
await delay(Math.round(config.commentMax * 0.5), config.commentMax);
|
|
121
149
|
|
|
122
150
|
const allResults = [];
|
|
123
151
|
let anyCaptchaDetected = false;
|
|
124
|
-
let anyCaptchaStage =
|
|
125
|
-
let anyCaptchaMessage =
|
|
152
|
+
let anyCaptchaStage = "";
|
|
153
|
+
let anyCaptchaMessage = "";
|
|
126
154
|
const videoAuthors = new Set();
|
|
127
155
|
const commentUsers = new Set();
|
|
128
156
|
const allCommentAuthorsList = [];
|
|
@@ -139,7 +167,9 @@ async function runScrape(options) {
|
|
|
139
167
|
log(`[${i + 1}/${maxVideos}] 跳过: ${e.message}`);
|
|
140
168
|
if (i < maxVideos - 1) {
|
|
141
169
|
await page.evaluate(() => {
|
|
142
|
-
const container = document.querySelector(
|
|
170
|
+
const container = document.querySelector(
|
|
171
|
+
'[class*="ColumnListContainer"]',
|
|
172
|
+
);
|
|
143
173
|
if (container) container.scrollTop += 700;
|
|
144
174
|
else window.scrollBy(0, 700);
|
|
145
175
|
});
|
|
@@ -151,31 +181,39 @@ async function runScrape(options) {
|
|
|
151
181
|
allResults.push(result);
|
|
152
182
|
if (result.captchaDetected) {
|
|
153
183
|
anyCaptchaDetected = true;
|
|
154
|
-
anyCaptchaStage = result.captchaStage ||
|
|
155
|
-
anyCaptchaMessage = result.captchaMessage ||
|
|
184
|
+
anyCaptchaStage = result.captchaStage || "";
|
|
185
|
+
anyCaptchaMessage = result.captchaMessage || "";
|
|
156
186
|
}
|
|
157
187
|
videoAuthors.add(result.videoAuthor);
|
|
158
|
-
result.commentUsers.forEach(u => commentUsers.add(u));
|
|
188
|
+
result.commentUsers.forEach((u) => commentUsers.add(u));
|
|
159
189
|
allCommentAuthorsList.push(...result.commentUsers);
|
|
160
190
|
if (result.guessVideos) {
|
|
161
191
|
allGuessVideos.push(...result.guessVideos);
|
|
162
|
-
result.guessVideos.forEach(v => {
|
|
192
|
+
result.guessVideos.forEach((v) => {
|
|
193
|
+
if (v.author) allGuessAuthors.add(v.author);
|
|
194
|
+
});
|
|
163
195
|
}
|
|
164
196
|
|
|
165
197
|
if ((i + 1) % 5 === 0 || i === 0) {
|
|
166
|
-
log(
|
|
198
|
+
log(
|
|
199
|
+
`[${i + 1}/${maxVideos}] ${result.videoAuthor} | 昵称: ${result.nickname || "-"} | 评论用户: ${result.commentUsers.length} | 猜你喜欢: ${result.guessVideos ? result.guessVideos.length : 0}`,
|
|
200
|
+
);
|
|
167
201
|
}
|
|
168
202
|
|
|
169
203
|
if (i < maxVideos - 1) {
|
|
170
204
|
await page.evaluate(() => {
|
|
171
|
-
const container = document.querySelector(
|
|
205
|
+
const container = document.querySelector(
|
|
206
|
+
'[class*="ColumnListContainer"]',
|
|
207
|
+
);
|
|
172
208
|
if (container) container.scrollTop += 700;
|
|
173
209
|
});
|
|
174
210
|
await delay(2000, config.switchMax);
|
|
175
211
|
}
|
|
176
212
|
}
|
|
177
213
|
|
|
178
|
-
log(
|
|
214
|
+
log(
|
|
215
|
+
`\n结果: 视频作者 ${videoAuthors.size} | 评论用户 ${commentUsers.size} | 总评论 ${allCommentAuthorsList.length} | 猜你喜欢作者 ${allGuessAuthors.size} | 总猜中视频 ${allGuessVideos.length}`,
|
|
216
|
+
);
|
|
179
217
|
|
|
180
218
|
const videoDetails = {};
|
|
181
219
|
for (const r of allResults) {
|
|
@@ -205,7 +243,14 @@ async function runScrape(options) {
|
|
|
205
243
|
},
|
|
206
244
|
};
|
|
207
245
|
|
|
208
|
-
return {
|
|
246
|
+
return {
|
|
247
|
+
output,
|
|
248
|
+
browser,
|
|
249
|
+
isExternal,
|
|
250
|
+
captchaDetected: anyCaptchaDetected,
|
|
251
|
+
captchaStage: anyCaptchaStage,
|
|
252
|
+
captchaMessage: anyCaptchaMessage,
|
|
253
|
+
};
|
|
209
254
|
}
|
|
210
255
|
|
|
211
256
|
export { scrapeSingleVideo, runScrape };
|
|
@@ -16,7 +16,7 @@ async function processExplore(page, username, options, log) {
|
|
|
16
16
|
loggedIn = false, // 由外部传入登录状态,避免每次调用 isLoggedIn(page)
|
|
17
17
|
maxFollowing = 50,
|
|
18
18
|
maxFollowers = 50,
|
|
19
|
-
location = "PL,NL,BE,DE,FR,IT,ES,IE",
|
|
19
|
+
location = "CZ,GR,HU,PT,PL,NL,BE,DE,FR,IT,ES,IE",
|
|
20
20
|
} = options;
|
|
21
21
|
|
|
22
22
|
const result = {
|
|
@@ -71,11 +71,15 @@ async function processExplore(page, username, options, log) {
|
|
|
71
71
|
let locationCreated = null;
|
|
72
72
|
const sampleVideos = videoArray.slice(0, SAMPLE_SIZE);
|
|
73
73
|
if (sampleVideos.length > 0) {
|
|
74
|
-
const sampleUrls = sampleVideos.map(v =>
|
|
75
|
-
v.href.startsWith("http") ? v.href : `https://www.tiktok.com${v.href}
|
|
74
|
+
const sampleUrls = sampleVideos.map((v) =>
|
|
75
|
+
v.href.startsWith("http") ? v.href : `https://www.tiktok.com${v.href}`,
|
|
76
|
+
);
|
|
77
|
+
const locations = await Promise.all(
|
|
78
|
+
sampleUrls.map((url) => extractVideoLocation(url)),
|
|
79
|
+
);
|
|
80
|
+
log(
|
|
81
|
+
` 国家采样(${locations.length}个): [${locations.filter(Boolean).join(", ") || "无数据"}]`,
|
|
76
82
|
);
|
|
77
|
-
const locations = await Promise.all(sampleUrls.map(url => extractVideoLocation(url)));
|
|
78
|
-
log(` 国家采样(${locations.length}个): [${locations.filter(Boolean).join(", ") || "无数据"}]`);
|
|
79
83
|
const freq = {};
|
|
80
84
|
for (const loc of locations) {
|
|
81
85
|
if (loc) {
|
|
@@ -3,21 +3,14 @@ import {
|
|
|
3
3
|
retryWithBackoff,
|
|
4
4
|
detectPageError,
|
|
5
5
|
assertPageUrl,
|
|
6
|
-
} from
|
|
7
|
-
import { detectCaptcha } from
|
|
8
|
-
import {
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
} from '../videos/core.js';
|
|
12
|
-
import { extractFollowAndFollowers } from './modules/follow-extractor.js';
|
|
13
|
-
import { processExplore } from './explore-core.js';
|
|
6
|
+
} from "./modules/page-helpers.js";
|
|
7
|
+
import { detectCaptcha } from "./modules/captcha-handler.js";
|
|
8
|
+
import { getUserInfo, collectVideos } from "../videos/core.js";
|
|
9
|
+
import { extractFollowAndFollowers } from "./modules/follow-extractor.js";
|
|
10
|
+
import { processExplore } from "./explore-core.js";
|
|
14
11
|
|
|
15
12
|
export async function processRefresh(page, username, serverUrl, options, log) {
|
|
16
|
-
const {
|
|
17
|
-
maxFollowing = 100,
|
|
18
|
-
maxFollowers = 100,
|
|
19
|
-
maxVideos = 100,
|
|
20
|
-
} = options;
|
|
13
|
+
const { maxFollowing = 100, maxFollowers = 100, maxVideos = 100 } = options;
|
|
21
14
|
|
|
22
15
|
const result = {
|
|
23
16
|
userInfo: null,
|
|
@@ -32,26 +25,36 @@ export async function processRefresh(page, username, serverUrl, options, log) {
|
|
|
32
25
|
try {
|
|
33
26
|
log(` 访问 @${username} 主页...`);
|
|
34
27
|
const homeUrl = `https://www.tiktok.com/@${username}`;
|
|
35
|
-
await retryWithBackoff(
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
28
|
+
await retryWithBackoff(
|
|
29
|
+
async () => {
|
|
30
|
+
await page.goto(homeUrl, {
|
|
31
|
+
waitUntil: "domcontentloaded",
|
|
32
|
+
timeout: 30000,
|
|
33
|
+
});
|
|
34
|
+
assertPageUrl(page, `@${username}`);
|
|
35
|
+
},
|
|
36
|
+
{ log },
|
|
37
|
+
);
|
|
38
|
+
await page
|
|
39
|
+
.waitForSelector('[class*="DivVideoList"]', { timeout: 10000 })
|
|
40
|
+
.catch(() => {});
|
|
40
41
|
await delay(1000, 2000);
|
|
41
42
|
|
|
42
|
-
log(
|
|
43
|
+
log(" 获取用户信息...");
|
|
43
44
|
const info = await getUserInfo(page);
|
|
44
45
|
if (info) {
|
|
45
46
|
result.userInfo = info;
|
|
46
|
-
log(
|
|
47
|
+
log(
|
|
48
|
+
` 用户: ${info.nickname || username} | 粉丝: ${info.followerCount || "-"} | 视频: ${info.videoCount || "-"}`,
|
|
49
|
+
);
|
|
47
50
|
}
|
|
48
51
|
|
|
49
52
|
const captcha = await detectCaptcha(page);
|
|
50
53
|
if (captcha && captcha.visible) {
|
|
51
54
|
log(`[验证码] @${username} 页面出现验证码`);
|
|
52
55
|
result.captchaDetected = true;
|
|
53
|
-
result.captchaStage = result.captchaStage ||
|
|
54
|
-
result.captchaMessage = result.captchaMessage ||
|
|
56
|
+
result.captchaStage = result.captchaStage || "video-page";
|
|
57
|
+
result.captchaMessage = result.captchaMessage || "视频页出现验证码";
|
|
55
58
|
}
|
|
56
59
|
|
|
57
60
|
// 采集视频
|
|
@@ -59,7 +62,7 @@ export async function processRefresh(page, username, serverUrl, options, log) {
|
|
|
59
62
|
const videoList = await collectVideos(page, username, maxVideos, log);
|
|
60
63
|
const videoArray = videoList ? [...videoList.values()] : [];
|
|
61
64
|
result.collectedVideos = videoArray.length;
|
|
62
|
-
result.discoveredVideoAuthors = videoArray.map(v => v.author);
|
|
65
|
+
result.discoveredVideoAuthors = videoArray.map((v) => v.author);
|
|
63
66
|
|
|
64
67
|
if (videoArray.length <= 0) {
|
|
65
68
|
result.noVideo = true;
|
|
@@ -80,7 +83,9 @@ export async function processRefresh(page, username, serverUrl, options, log) {
|
|
|
80
83
|
});
|
|
81
84
|
result.discoveredFollowing = followResult.following || [];
|
|
82
85
|
result.discoveredFollowers = followResult.followers || [];
|
|
83
|
-
log(
|
|
86
|
+
log(
|
|
87
|
+
` 关注: ${result.discoveredFollowing.length}, 粉丝: ${result.discoveredFollowers.length}`,
|
|
88
|
+
);
|
|
84
89
|
} catch (e) {
|
|
85
90
|
log(` [关注/粉丝采集失败] ${e.message}`);
|
|
86
91
|
result.discoveredFollowing = [];
|
|
@@ -90,15 +95,23 @@ export async function processRefresh(page, username, serverUrl, options, log) {
|
|
|
90
95
|
// 处理新发现的用户(关注 + 粉丝),循环执行完整 explore
|
|
91
96
|
// follow-extractor 返回 [handle, displayName] 数组
|
|
92
97
|
const allDiscovered = [
|
|
93
|
-
...result.discoveredFollowing.map(h => ({
|
|
94
|
-
|
|
98
|
+
...result.discoveredFollowing.map((h) => ({
|
|
99
|
+
handle: Array.isArray(h) ? h[0] : h,
|
|
100
|
+
source: "refresh-following",
|
|
101
|
+
})),
|
|
102
|
+
...result.discoveredFollowers.map((h) => ({
|
|
103
|
+
handle: Array.isArray(h) ? h[0] : h,
|
|
104
|
+
source: "refresh-follower",
|
|
105
|
+
})),
|
|
95
106
|
];
|
|
96
107
|
|
|
97
108
|
for (const { handle, source } of allDiscovered) {
|
|
98
|
-
const uniqueId = handle.replace(
|
|
109
|
+
const uniqueId = handle.replace("@", "");
|
|
99
110
|
|
|
100
111
|
// 检查用户是否已存在
|
|
101
|
-
const existsResp = await fetch(
|
|
112
|
+
const existsResp = await fetch(
|
|
113
|
+
`${serverUrl}/api/user-exists/${encodeURIComponent(uniqueId)}`,
|
|
114
|
+
);
|
|
102
115
|
const existsData = await existsResp.json();
|
|
103
116
|
|
|
104
117
|
if (existsData.exists) {
|
|
@@ -109,14 +122,19 @@ export async function processRefresh(page, username, serverUrl, options, log) {
|
|
|
109
122
|
await delay(1000, 2000);
|
|
110
123
|
|
|
111
124
|
// 对新用户做完整 explore(与 explore 命令逻辑一致)
|
|
112
|
-
const exploreResult = await processExplore(
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
125
|
+
const exploreResult = await processExplore(
|
|
126
|
+
page,
|
|
127
|
+
uniqueId,
|
|
128
|
+
{
|
|
129
|
+
maxComments: 10,
|
|
130
|
+
maxGuess: 0,
|
|
131
|
+
enableFollow: true,
|
|
132
|
+
maxFollowing: 5,
|
|
133
|
+
maxFollowers: 5,
|
|
134
|
+
location: "CZ,GR,HU,PT,PL,NL,BE,DE,FR,IT,ES,IE",
|
|
135
|
+
},
|
|
136
|
+
log,
|
|
137
|
+
);
|
|
120
138
|
|
|
121
139
|
// 提交 explore 结果到服务端(和 explore 命令的 commitJob 一致)
|
|
122
140
|
if (exploreResult.userInfo) {
|
|
@@ -124,21 +142,31 @@ export async function processRefresh(page, username, serverUrl, options, log) {
|
|
|
124
142
|
|
|
125
143
|
const payload = {
|
|
126
144
|
userInfo: exploreResult.userInfo || {},
|
|
127
|
-
discoveredVideoAuthors: (
|
|
128
|
-
|
|
145
|
+
discoveredVideoAuthors: (
|
|
146
|
+
exploreResult.discoveredVideoAuthors || []
|
|
147
|
+
).map((item) =>
|
|
148
|
+
typeof item === "object" ? { ...item, guessedLocation } : item,
|
|
149
|
+
),
|
|
150
|
+
discoveredCommentAuthors: (
|
|
151
|
+
exploreResult.discoveredCommentAuthors || []
|
|
152
|
+
).map((author) => ({ author, guessedLocation })),
|
|
153
|
+
discoveredGuessAuthors: (
|
|
154
|
+
exploreResult.discoveredGuessAuthors || []
|
|
155
|
+
).map((author) => ({ author, guessedLocation })),
|
|
156
|
+
discoveredFollowing: (exploreResult.discoveredFollowing || []).map(
|
|
157
|
+
(f) => ({
|
|
158
|
+
handle: Array.isArray(f) ? f[0] : f,
|
|
159
|
+
displayName: Array.isArray(f) ? f[1] : null,
|
|
160
|
+
guessedLocation,
|
|
161
|
+
}),
|
|
162
|
+
),
|
|
163
|
+
discoveredFollowers: (exploreResult.discoveredFollowers || []).map(
|
|
164
|
+
(f) => ({
|
|
165
|
+
handle: Array.isArray(f) ? f[0] : f,
|
|
166
|
+
displayName: Array.isArray(f) ? f[1] : null,
|
|
167
|
+
guessedLocation,
|
|
168
|
+
}),
|
|
129
169
|
),
|
|
130
|
-
discoveredCommentAuthors: (exploreResult.discoveredCommentAuthors || []).map(author => ({ author, guessedLocation })),
|
|
131
|
-
discoveredGuessAuthors: (exploreResult.discoveredGuessAuthors || []).map(author => ({ author, guessedLocation })),
|
|
132
|
-
discoveredFollowing: (exploreResult.discoveredFollowing || []).map(f => ({
|
|
133
|
-
handle: Array.isArray(f) ? f[0] : f,
|
|
134
|
-
displayName: Array.isArray(f) ? f[1] : null,
|
|
135
|
-
guessedLocation,
|
|
136
|
-
})),
|
|
137
|
-
discoveredFollowers: (exploreResult.discoveredFollowers || []).map(f => ({
|
|
138
|
-
handle: Array.isArray(f) ? f[0] : f,
|
|
139
|
-
displayName: Array.isArray(f) ? f[1] : null,
|
|
140
|
-
guessedLocation,
|
|
141
|
-
})),
|
|
142
170
|
processed: exploreResult.processed,
|
|
143
171
|
hasFollowData: exploreResult.hasFollowData,
|
|
144
172
|
keepFollow: exploreResult.keepFollow,
|
|
@@ -148,11 +176,14 @@ export async function processRefresh(page, username, serverUrl, options, log) {
|
|
|
148
176
|
error: exploreResult.error,
|
|
149
177
|
};
|
|
150
178
|
|
|
151
|
-
const addResp = await fetch(
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
179
|
+
const addResp = await fetch(
|
|
180
|
+
`${serverUrl}/api/explore-new/${uniqueId}`,
|
|
181
|
+
{
|
|
182
|
+
method: "POST",
|
|
183
|
+
headers: { "Content-Type": "application/json" },
|
|
184
|
+
body: JSON.stringify(payload),
|
|
185
|
+
},
|
|
186
|
+
);
|
|
156
187
|
const addResult = await addResp.json();
|
|
157
188
|
|
|
158
189
|
if (!addResult.saved) {
|
|
@@ -164,7 +195,9 @@ export async function processRefresh(page, username, serverUrl, options, log) {
|
|
|
164
195
|
if (exploreResult.captchaDetected) {
|
|
165
196
|
result.captchaDetected = true;
|
|
166
197
|
}
|
|
167
|
-
log(
|
|
198
|
+
log(
|
|
199
|
+
` [已提交] @${uniqueId} ${addResult.created ? "(新用户)" : "(已存在)"} | 发现: ${addResult.newUsers?.length || 0} 个`,
|
|
200
|
+
);
|
|
168
201
|
}
|
|
169
202
|
|
|
170
203
|
await delay(2000, 4000);
|
|
@@ -172,7 +205,7 @@ export async function processRefresh(page, username, serverUrl, options, log) {
|
|
|
172
205
|
} catch (e) {
|
|
173
206
|
log(` [错误] ${e.message}`);
|
|
174
207
|
result.error = e.message;
|
|
175
|
-
result.errorStack = e.stack ||
|
|
208
|
+
result.errorStack = e.stack || "";
|
|
176
209
|
}
|
|
177
210
|
|
|
178
211
|
return result;
|
package/src/videos/core.js
CHANGED
|
@@ -7,7 +7,25 @@ import {
|
|
|
7
7
|
import { fetchUserVideosAPI } from "../lib/api-interceptor.js";
|
|
8
8
|
|
|
9
9
|
async function getUserInfo(page) {
|
|
10
|
-
|
|
10
|
+
// 重试包装:处理页面导航导致的执行上下文销毁
|
|
11
|
+
const evaluateWithRetry = async (fn, retries = 3) => {
|
|
12
|
+
for (let i = 0; i < retries; i++) {
|
|
13
|
+
try {
|
|
14
|
+
return await page.evaluate(fn);
|
|
15
|
+
} catch (e) {
|
|
16
|
+
if (
|
|
17
|
+
e.message.includes('Execution context was destroyed') &&
|
|
18
|
+
i < retries - 1
|
|
19
|
+
) {
|
|
20
|
+
await new Promise((r) => setTimeout(r, 500 * (i + 1)));
|
|
21
|
+
} else {
|
|
22
|
+
throw e;
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
};
|
|
27
|
+
|
|
28
|
+
return await evaluateWithRetry(() => {
|
|
11
29
|
const html = document.documentElement.outerHTML;
|
|
12
30
|
const result = {};
|
|
13
31
|
|
package/src/watch/data-store.js
CHANGED
|
@@ -1854,7 +1854,20 @@ export function createStore(filePath) {
|
|
|
1854
1854
|
if (db) {
|
|
1855
1855
|
const now = Date.now();
|
|
1856
1856
|
const defaultTime = new Date("2016-01-01T00:00:00Z").getTime();
|
|
1857
|
-
const targetLocations = [
|
|
1857
|
+
const targetLocations = [
|
|
1858
|
+
"CZ",
|
|
1859
|
+
"GR",
|
|
1860
|
+
"HU",
|
|
1861
|
+
"PT",
|
|
1862
|
+
"ES",
|
|
1863
|
+
"PL",
|
|
1864
|
+
"NL",
|
|
1865
|
+
"BE",
|
|
1866
|
+
"DE",
|
|
1867
|
+
"FR",
|
|
1868
|
+
"IT",
|
|
1869
|
+
"IE",
|
|
1870
|
+
];
|
|
1858
1871
|
const placeholders = targetLocations.map(() => "?").join(",");
|
|
1859
1872
|
const row = db
|
|
1860
1873
|
.prepare(
|
|
@@ -1884,7 +1897,20 @@ export function createStore(filePath) {
|
|
|
1884
1897
|
const defaultTime = new Date("2016-01-01T00:00:00Z").getTime();
|
|
1885
1898
|
|
|
1886
1899
|
// 筛选目标国家用户,按 refreshTime 升序取最远的(没有则默认 2016-01-01)
|
|
1887
|
-
const targetLocations = [
|
|
1900
|
+
const targetLocations = [
|
|
1901
|
+
"CZ",
|
|
1902
|
+
"GR",
|
|
1903
|
+
"HU",
|
|
1904
|
+
"PT",
|
|
1905
|
+
"ES",
|
|
1906
|
+
"PL",
|
|
1907
|
+
"NL",
|
|
1908
|
+
"BE",
|
|
1909
|
+
"DE",
|
|
1910
|
+
"FR",
|
|
1911
|
+
"IT",
|
|
1912
|
+
"IE",
|
|
1913
|
+
];
|
|
1888
1914
|
const targetUsers = data.filter(
|
|
1889
1915
|
(u) =>
|
|
1890
1916
|
u.ttSeller &&
|
|
@@ -187,6 +187,17 @@
|
|
|
187
187
|
flex-shrink: 0;
|
|
188
188
|
}
|
|
189
189
|
|
|
190
|
+
.bar-row.is-target {
|
|
191
|
+
background: rgba(167, 139, 250, 0.08);
|
|
192
|
+
border-radius: 4px;
|
|
193
|
+
padding: 2px 0;
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
.bar-row.is-target .name {
|
|
197
|
+
color: #a78bfa;
|
|
198
|
+
font-weight: 600;
|
|
199
|
+
}
|
|
200
|
+
|
|
190
201
|
.source-row {
|
|
191
202
|
display: flex;
|
|
192
203
|
align-items: center;
|
|
@@ -1094,14 +1105,16 @@
|
|
|
1094
1105
|
if (!filtered.length) { el.innerHTML = '<span style="color:#666;font-size:12px">\u6682\u65e0\u6570\u636e</span>'; return; }
|
|
1095
1106
|
const max = filtered[0].count;
|
|
1096
1107
|
const top = filtered.slice(0, 15);
|
|
1108
|
+
const targetLocations = currentStats?.targetLocations || [];
|
|
1097
1109
|
el.innerHTML = top.map((c, i) => {
|
|
1110
|
+
const isTarget = targetLocations.includes(c.country);
|
|
1098
1111
|
const targetBadge = c.targetCount > 0
|
|
1099
1112
|
? `<span class="target-badge">🎯 ${c.targetCount}</span>`
|
|
1100
1113
|
: `<span class="target-badge" style="visibility:hidden"> </span>`;
|
|
1101
1114
|
return `
|
|
1102
|
-
<div class="bar-row">
|
|
1115
|
+
<div class="bar-row${isTarget ? ' is-target' : ''}">
|
|
1103
1116
|
<span class="name">${c.country}</span>
|
|
1104
|
-
<div class="bar-bg"><div class="bar-fill" style="width:${(c.count / max * 100)}%;background:${COLORS[i % COLORS.length]}">${c.count}</div></div>
|
|
1117
|
+
<div class="bar-bg"><div class="bar-fill" style="width:${(c.count / max * 100)}%;background:${isTarget ? '#a78bfa' : COLORS[i % COLORS.length]}">${c.count}</div></div>
|
|
1105
1118
|
${targetBadge}
|
|
1106
1119
|
<span class="count">${(currentStats ? (c.count / currentStats.totalUsers * 100).toFixed(1) : 0)}%</span>
|
|
1107
1120
|
</div>
|
package/src/watch/server.js
CHANGED
|
@@ -7,7 +7,21 @@ import { fileURLToPath } from "url";
|
|
|
7
7
|
import { spawn } from "child_process";
|
|
8
8
|
import { createStore } from "./data-store.js";
|
|
9
9
|
|
|
10
|
-
const TARGET_LOCATIONS = [
|
|
10
|
+
const TARGET_LOCATIONS = [
|
|
11
|
+
"CZ",
|
|
12
|
+
"GR",
|
|
13
|
+
"HU",
|
|
14
|
+
"PT",
|
|
15
|
+
"ES",
|
|
16
|
+
"PL",
|
|
17
|
+
"NL",
|
|
18
|
+
"BE",
|
|
19
|
+
"DE",
|
|
20
|
+
"FR",
|
|
21
|
+
"IT",
|
|
22
|
+
"IE",
|
|
23
|
+
"AT",
|
|
24
|
+
];
|
|
11
25
|
|
|
12
26
|
const __filename = fileURLToPath(import.meta.url);
|
|
13
27
|
|
|
@@ -314,6 +328,7 @@ export function startWatchServer(dataAnchor, port = 3000, existingStore) {
|
|
|
314
328
|
|
|
315
329
|
if (req.method === "GET" && routePath === "/api/stats") {
|
|
316
330
|
const stats = computeStatsIncremental(store);
|
|
331
|
+
stats.targetLocations = TARGET_LOCATIONS;
|
|
317
332
|
sendJSON(res, 200, stats);
|
|
318
333
|
return;
|
|
319
334
|
}
|