tt-help-cli-ycl 1.3.32 → 1.3.33
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/scripts/run-explore copy.bat +35 -2
- package/scripts/run-explore.bat +40 -10
- package/scripts/run-explore.ps1 +51 -5
- package/scripts/run-explore.sh +20 -5
- package/src/cli/explore.js +6 -3
- package/src/lib/api-interceptor.js +110 -37
- package/src/lib/args.js +4 -0
- package/src/scraper/modules/follow-extractor.js +82 -6
- package/src/watch/data-store.js +340 -165
- package/src/watch/server.js +400 -201
package/src/watch/data-store.js
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
|
-
import fs from
|
|
2
|
-
import { promises as fsPromises } from
|
|
3
|
-
import path from
|
|
1
|
+
import fs from "fs";
|
|
2
|
+
import { promises as fsPromises } from "fs";
|
|
3
|
+
import path from "path";
|
|
4
4
|
|
|
5
5
|
function inferStatus(u) {
|
|
6
|
-
if (u.restricted) return
|
|
7
|
-
if (u.error) return
|
|
8
|
-
if (u.processed) return
|
|
9
|
-
return
|
|
6
|
+
if (u.restricted) return "restricted";
|
|
7
|
+
if (u.error) return "error";
|
|
8
|
+
if (u.processed) return "done";
|
|
9
|
+
return "pending";
|
|
10
10
|
}
|
|
11
11
|
|
|
12
12
|
export function createStore(filePath) {
|
|
@@ -20,10 +20,10 @@ export function createStore(filePath) {
|
|
|
20
20
|
let doneArchivePath = null;
|
|
21
21
|
let doneUidIndex = new Map();
|
|
22
22
|
if (filePath) {
|
|
23
|
-
doneArchivePath = path.resolve(filePath).replace(/\.json$/,
|
|
23
|
+
doneArchivePath = path.resolve(filePath).replace(/\.json$/, "-done.json");
|
|
24
24
|
if (fs.existsSync(doneArchivePath)) {
|
|
25
25
|
try {
|
|
26
|
-
const content = fs.readFileSync(doneArchivePath,
|
|
26
|
+
const content = fs.readFileSync(doneArchivePath, "utf-8");
|
|
27
27
|
const parsed = JSON.parse(content);
|
|
28
28
|
if (Array.isArray(parsed)) doneArchive = parsed;
|
|
29
29
|
} catch (e) {
|
|
@@ -43,7 +43,13 @@ export function createStore(filePath) {
|
|
|
43
43
|
|
|
44
44
|
function computeStatsInternal() {
|
|
45
45
|
const total = data.length;
|
|
46
|
-
const statusCounts = {
|
|
46
|
+
const statusCounts = {
|
|
47
|
+
pending: 0,
|
|
48
|
+
processing: 0,
|
|
49
|
+
done: 0,
|
|
50
|
+
error: 0,
|
|
51
|
+
restricted: 0,
|
|
52
|
+
};
|
|
47
53
|
for (const u of data) {
|
|
48
54
|
statusCounts[u.status] = (statusCounts[u.status] || 0) + 1;
|
|
49
55
|
}
|
|
@@ -63,36 +69,45 @@ export function createStore(filePath) {
|
|
|
63
69
|
let statusGroups = null;
|
|
64
70
|
let groupsDirty = true;
|
|
65
71
|
|
|
66
|
-
const tier1LocSet = new Set([
|
|
67
|
-
const tier2LocSet = new Set([
|
|
72
|
+
const tier1LocSet = new Set(["PL", "NL", "BE"]);
|
|
73
|
+
const tier2LocSet = new Set(["DE", "FR", "IT", "IE", "ES"]);
|
|
68
74
|
function locationTier(u) {
|
|
69
|
-
const loc = (u.guessedLocation ||
|
|
75
|
+
const loc = (u.guessedLocation || "").toUpperCase();
|
|
70
76
|
if (tier1LocSet.has(loc)) return 0;
|
|
71
77
|
if (tier2LocSet.has(loc)) return 1;
|
|
72
78
|
return 2;
|
|
73
79
|
}
|
|
74
80
|
|
|
75
81
|
function sortGroup(key, arr) {
|
|
76
|
-
if (key ===
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
82
|
+
if (key === "done")
|
|
83
|
+
arr.sort((a, b) => (b.processedAt || 0) - (a.processedAt || 0));
|
|
84
|
+
else if (key === "pending")
|
|
85
|
+
arr.sort((a, b) => {
|
|
86
|
+
const aSeller = a.ttSeller === true && a.verified === false ? 0 : 1;
|
|
87
|
+
const bSeller = b.ttSeller === true && b.verified === false ? 0 : 1;
|
|
88
|
+
if (aSeller !== bSeller) return aSeller - bSeller;
|
|
89
|
+
const la = locationTier(a),
|
|
90
|
+
lb = locationTier(b);
|
|
91
|
+
if (la !== lb) return la - lb;
|
|
92
|
+
return (b.followerCount || 0) - (a.followerCount || 0);
|
|
93
|
+
});
|
|
85
94
|
else arr.sort((a, b) => (b.followerCount || 0) - (a.followerCount || 0));
|
|
86
95
|
// 置顶冒泡到组首
|
|
87
|
-
const pinned = arr.filter(u => u.pinned);
|
|
88
|
-
const unpinned = arr.filter(u => !u.pinned);
|
|
96
|
+
const pinned = arr.filter((u) => u.pinned);
|
|
97
|
+
const unpinned = arr.filter((u) => !u.pinned);
|
|
89
98
|
return pinned.concat(unpinned);
|
|
90
99
|
}
|
|
91
100
|
|
|
92
101
|
function rebuildStatusGroups() {
|
|
93
|
-
statusGroups = {
|
|
102
|
+
statusGroups = {
|
|
103
|
+
pending: [],
|
|
104
|
+
processing: [],
|
|
105
|
+
done: [],
|
|
106
|
+
error: [],
|
|
107
|
+
restricted: [],
|
|
108
|
+
};
|
|
94
109
|
for (const u of data) {
|
|
95
|
-
const key = u.status ||
|
|
110
|
+
const key = u.status || "pending";
|
|
96
111
|
if (statusGroups[key]) statusGroups[key].push(u);
|
|
97
112
|
else statusGroups[key] = [u];
|
|
98
113
|
}
|
|
@@ -121,10 +136,10 @@ export function createStore(filePath) {
|
|
|
121
136
|
let videoFilePath = null;
|
|
122
137
|
if (filePath) {
|
|
123
138
|
const resolved = path.resolve(filePath);
|
|
124
|
-
videoFilePath = resolved.replace(/\.json$/,
|
|
139
|
+
videoFilePath = resolved.replace(/\.json$/, "-videos.json");
|
|
125
140
|
if (fs.existsSync(videoFilePath)) {
|
|
126
141
|
try {
|
|
127
|
-
const content = fs.readFileSync(videoFilePath,
|
|
142
|
+
const content = fs.readFileSync(videoFilePath, "utf-8");
|
|
128
143
|
const parsed = JSON.parse(content);
|
|
129
144
|
if (Array.isArray(parsed)) {
|
|
130
145
|
videos = parsed;
|
|
@@ -139,12 +154,12 @@ export function createStore(filePath) {
|
|
|
139
154
|
|
|
140
155
|
if (filePath) {
|
|
141
156
|
const resolved = path.resolve(filePath);
|
|
142
|
-
const backupDir = path.join(path.dirname(resolved),
|
|
157
|
+
const backupDir = path.join(path.dirname(resolved), ".backup");
|
|
143
158
|
const maxBackups = 3;
|
|
144
159
|
|
|
145
160
|
if (fs.existsSync(resolved)) {
|
|
146
161
|
try {
|
|
147
|
-
const content = fs.readFileSync(resolved,
|
|
162
|
+
const content = fs.readFileSync(resolved, "utf-8");
|
|
148
163
|
data = JSON.parse(content);
|
|
149
164
|
if (!Array.isArray(data)) {
|
|
150
165
|
data = [];
|
|
@@ -157,16 +172,18 @@ export function createStore(filePath) {
|
|
|
157
172
|
|
|
158
173
|
function runBackup() {
|
|
159
174
|
if (!fs.existsSync(resolved)) return;
|
|
160
|
-
if (!fs.existsSync(backupDir))
|
|
175
|
+
if (!fs.existsSync(backupDir))
|
|
176
|
+
fs.mkdirSync(backupDir, { recursive: true });
|
|
161
177
|
const now = new Date();
|
|
162
|
-
const timestamp = now.toISOString().replace(/[:.]/g,
|
|
178
|
+
const timestamp = now.toISOString().replace(/[:.]/g, "-").slice(0, 13);
|
|
163
179
|
const backupFile = path.join(backupDir, `data-${timestamp}.json`);
|
|
164
180
|
try {
|
|
165
181
|
fs.copyFileSync(resolved, backupFile);
|
|
166
|
-
const files = fs
|
|
167
|
-
.
|
|
182
|
+
const files = fs
|
|
183
|
+
.readdirSync(backupDir)
|
|
184
|
+
.filter((f) => f.startsWith("data-") && f.endsWith(".json"))
|
|
168
185
|
.sort()
|
|
169
|
-
.map(f => path.join(backupDir, f));
|
|
186
|
+
.map((f) => path.join(backupDir, f));
|
|
170
187
|
while (files.length > maxBackups) {
|
|
171
188
|
fs.unlinkSync(files.shift());
|
|
172
189
|
}
|
|
@@ -220,7 +237,7 @@ export function createStore(filePath) {
|
|
|
220
237
|
const active = [];
|
|
221
238
|
const toArchive = [];
|
|
222
239
|
for (const u of data) {
|
|
223
|
-
if (u.status ===
|
|
240
|
+
if (u.status === "done") {
|
|
224
241
|
toArchive.push(u);
|
|
225
242
|
} else {
|
|
226
243
|
active.push(u);
|
|
@@ -239,18 +256,20 @@ export function createStore(filePath) {
|
|
|
239
256
|
}
|
|
240
257
|
|
|
241
258
|
const json = JSON.stringify(data);
|
|
242
|
-
const tmpPath = resolved +
|
|
243
|
-
fsPromises
|
|
259
|
+
const tmpPath = resolved + ".tmp";
|
|
260
|
+
fsPromises
|
|
261
|
+
.writeFile(tmpPath, json, "utf-8")
|
|
244
262
|
.then(() => fsPromises.rename(tmpPath, resolved))
|
|
245
263
|
.then(() => {
|
|
246
264
|
if (doneArchivePath && doneArchive.length > 0) {
|
|
247
265
|
const doneJson = JSON.stringify(doneArchive);
|
|
248
|
-
const doneTmp = doneArchivePath +
|
|
249
|
-
return fsPromises
|
|
266
|
+
const doneTmp = doneArchivePath + ".tmp";
|
|
267
|
+
return fsPromises
|
|
268
|
+
.writeFile(doneTmp, doneJson, "utf-8")
|
|
250
269
|
.then(() => fsPromises.rename(doneTmp, doneArchivePath));
|
|
251
270
|
}
|
|
252
271
|
})
|
|
253
|
-
.catch(err => {
|
|
272
|
+
.catch((err) => {
|
|
254
273
|
console.error(`[data-store] save 写入失败: ${err.message}`);
|
|
255
274
|
});
|
|
256
275
|
}
|
|
@@ -260,13 +279,16 @@ export function createStore(filePath) {
|
|
|
260
279
|
}
|
|
261
280
|
|
|
262
281
|
function flushSave() {
|
|
263
|
-
return new Promise(resolve => {
|
|
282
|
+
return new Promise((resolve) => {
|
|
264
283
|
if (saveTimer) {
|
|
265
284
|
clearTimeout(saveTimer);
|
|
266
285
|
saveTimer = null;
|
|
267
286
|
savePending = false;
|
|
268
287
|
}
|
|
269
|
-
if (!filePath) {
|
|
288
|
+
if (!filePath) {
|
|
289
|
+
resolve();
|
|
290
|
+
return;
|
|
291
|
+
}
|
|
270
292
|
const resolved = path.resolve(filePath);
|
|
271
293
|
|
|
272
294
|
// 将 done 用户归档到独立文件
|
|
@@ -274,7 +296,7 @@ export function createStore(filePath) {
|
|
|
274
296
|
const active = [];
|
|
275
297
|
const toArchive = [];
|
|
276
298
|
for (const u of data) {
|
|
277
|
-
if (u.status ===
|
|
299
|
+
if (u.status === "done") {
|
|
278
300
|
toArchive.push(u);
|
|
279
301
|
} else {
|
|
280
302
|
active.push(u);
|
|
@@ -292,19 +314,21 @@ export function createStore(filePath) {
|
|
|
292
314
|
}
|
|
293
315
|
|
|
294
316
|
const json = JSON.stringify(data);
|
|
295
|
-
const tmpPath = resolved +
|
|
296
|
-
fsPromises
|
|
317
|
+
const tmpPath = resolved + ".tmp";
|
|
318
|
+
fsPromises
|
|
319
|
+
.writeFile(tmpPath, json, "utf-8")
|
|
297
320
|
.then(() => fsPromises.rename(tmpPath, resolved))
|
|
298
321
|
.then(() => {
|
|
299
322
|
if (doneArchivePath && doneArchive.length > 0) {
|
|
300
323
|
const doneJson = JSON.stringify(doneArchive);
|
|
301
|
-
const doneTmp = doneArchivePath +
|
|
302
|
-
return fsPromises
|
|
324
|
+
const doneTmp = doneArchivePath + ".tmp";
|
|
325
|
+
return fsPromises
|
|
326
|
+
.writeFile(doneTmp, doneJson, "utf-8")
|
|
303
327
|
.then(() => fsPromises.rename(doneTmp, doneArchivePath));
|
|
304
328
|
}
|
|
305
329
|
resolve();
|
|
306
330
|
})
|
|
307
|
-
.catch(err => {
|
|
331
|
+
.catch((err) => {
|
|
308
332
|
console.error(`[data-store] flushSave 写入失败: ${err.message}`);
|
|
309
333
|
resolve();
|
|
310
334
|
});
|
|
@@ -319,10 +343,9 @@ export function createStore(filePath) {
|
|
|
319
343
|
videosSaveTimer = setTimeout(() => {
|
|
320
344
|
videosSaveTimer = null;
|
|
321
345
|
const json = JSON.stringify(videos, null, 2);
|
|
322
|
-
fsPromises.writeFile(videoFilePath, json,
|
|
323
|
-
.
|
|
324
|
-
|
|
325
|
-
});
|
|
346
|
+
fsPromises.writeFile(videoFilePath, json, "utf-8").catch((err) => {
|
|
347
|
+
console.error(`[data-store] saveVideos 写入失败: ${err.message}`);
|
|
348
|
+
});
|
|
326
349
|
}, 2000);
|
|
327
350
|
videosSaveTimer.unref();
|
|
328
351
|
}
|
|
@@ -354,8 +377,8 @@ export function createStore(filePath) {
|
|
|
354
377
|
const existing = getUser(user.uniqueId);
|
|
355
378
|
if (existing) {
|
|
356
379
|
for (const key of Object.keys(user)) {
|
|
357
|
-
if (key ===
|
|
358
|
-
if (user[key] !== undefined && user[key] !== null && user[key] !==
|
|
380
|
+
if (key === "uniqueId" || key === "sources") continue;
|
|
381
|
+
if (user[key] !== undefined && user[key] !== null && user[key] !== "") {
|
|
359
382
|
existing[key] = user[key];
|
|
360
383
|
}
|
|
361
384
|
}
|
|
@@ -375,11 +398,13 @@ export function createStore(filePath) {
|
|
|
375
398
|
}
|
|
376
399
|
|
|
377
400
|
function getPendingUsers() {
|
|
378
|
-
return data.filter(u => u.status ===
|
|
401
|
+
return data.filter((u) => u.status === "pending");
|
|
379
402
|
}
|
|
380
403
|
|
|
381
404
|
function getProcessedUsers() {
|
|
382
|
-
return doneArchive.length > 0
|
|
405
|
+
return doneArchive.length > 0
|
|
406
|
+
? data.filter((u) => u.status === "done").concat(doneArchive)
|
|
407
|
+
: data.filter((u) => u.status === "done");
|
|
383
408
|
}
|
|
384
409
|
|
|
385
410
|
function getAllUsers() {
|
|
@@ -389,38 +414,57 @@ export function createStore(filePath) {
|
|
|
389
414
|
return data;
|
|
390
415
|
}
|
|
391
416
|
|
|
392
|
-
function claimNextJob(userId, expireMs = 5 * 60 * 1000) {
|
|
417
|
+
function claimNextJob(userId, expireMs = 5 * 60 * 1000, locations = null) {
|
|
393
418
|
const now = Date.now();
|
|
394
419
|
|
|
395
420
|
// 0. 该客户端有未过期的任务,续期返回
|
|
396
|
-
const ongoing = data.find(
|
|
397
|
-
|
|
421
|
+
const ongoing = data.find(
|
|
422
|
+
(u) =>
|
|
423
|
+
u.status === "processing" &&
|
|
424
|
+
u.claimedBy === userId &&
|
|
425
|
+
u.claimedAt &&
|
|
426
|
+
now - u.claimedAt < expireMs,
|
|
398
427
|
);
|
|
399
428
|
if (ongoing) {
|
|
400
429
|
ongoing.claimedAt = now;
|
|
401
|
-
return {
|
|
430
|
+
return {
|
|
431
|
+
uniqueId: ongoing.uniqueId,
|
|
432
|
+
nickname: ongoing.nickname,
|
|
433
|
+
claimedAt: ongoing.claimedAt,
|
|
434
|
+
claimedBy: userId,
|
|
435
|
+
};
|
|
402
436
|
}
|
|
403
437
|
|
|
404
438
|
// 按猜测国家梯队排序
|
|
405
|
-
const tier1 = new Set([
|
|
406
|
-
const tier2 = new Set([
|
|
439
|
+
const tier1 = new Set(["PL", "NL", "BE"]);
|
|
440
|
+
const tier2 = new Set(["DE", "FR", "IT", "IE", "ES"]);
|
|
407
441
|
function locationTier(u) {
|
|
408
|
-
const loc = (u.guessedLocation ||
|
|
442
|
+
const loc = (u.guessedLocation || "").toUpperCase();
|
|
409
443
|
if (tier1.has(loc)) return 0;
|
|
410
444
|
if (tier2.has(loc)) return 1;
|
|
411
445
|
return 2;
|
|
412
446
|
}
|
|
413
447
|
|
|
448
|
+
// 国家过滤:如果指定了 locations,只保留 guessedLocation 在列表中的用户
|
|
449
|
+
function locationFilter(u) {
|
|
450
|
+
if (!locations || locations.length === 0) return true;
|
|
451
|
+
const loc = (u.guessedLocation || "").toUpperCase();
|
|
452
|
+
return locations.includes(loc);
|
|
453
|
+
}
|
|
454
|
+
|
|
414
455
|
// 从候选列表中按优先级取第一个:pinned > 超时回收 > seed > ttSeller > follow > other
|
|
415
456
|
function pickCandidate(candidates) {
|
|
416
|
-
let next = candidates.find(u => u.pinned);
|
|
457
|
+
let next = candidates.find((u) => u.pinned);
|
|
417
458
|
|
|
418
459
|
if (!next) {
|
|
419
|
-
const expired = data.find(
|
|
420
|
-
|
|
460
|
+
const expired = data.find(
|
|
461
|
+
(u) =>
|
|
462
|
+
u.status === "processing" &&
|
|
463
|
+
u.claimedAt &&
|
|
464
|
+
now - u.claimedAt > expireMs,
|
|
421
465
|
);
|
|
422
466
|
if (expired) {
|
|
423
|
-
expired.status =
|
|
467
|
+
expired.status = "pending";
|
|
424
468
|
markStatsDirty();
|
|
425
469
|
delete expired.claimedAt;
|
|
426
470
|
next = expired;
|
|
@@ -428,19 +472,27 @@ export function createStore(filePath) {
|
|
|
428
472
|
}
|
|
429
473
|
|
|
430
474
|
if (!next) {
|
|
431
|
-
const seed = candidates.filter(
|
|
475
|
+
const seed = candidates.filter(
|
|
476
|
+
(u) => u.sources && u.sources.includes("seed"),
|
|
477
|
+
);
|
|
432
478
|
seed.sort((a, b) => locationTier(a) - locationTier(b));
|
|
433
479
|
next = seed[0] || null;
|
|
434
480
|
}
|
|
435
481
|
|
|
436
482
|
if (!next) {
|
|
437
|
-
const ttSeller = candidates.filter(
|
|
483
|
+
const ttSeller = candidates.filter(
|
|
484
|
+
(u) => u.ttSeller === true && u.verified === false,
|
|
485
|
+
);
|
|
438
486
|
ttSeller.sort((a, b) => locationTier(a) - locationTier(b));
|
|
439
487
|
next = ttSeller[0] || null;
|
|
440
488
|
}
|
|
441
489
|
|
|
442
490
|
if (!next) {
|
|
443
|
-
const follow = candidates.filter(
|
|
491
|
+
const follow = candidates.filter(
|
|
492
|
+
(u) =>
|
|
493
|
+
u.sources &&
|
|
494
|
+
(u.sources.includes("following") || u.sources.includes("follower")),
|
|
495
|
+
);
|
|
444
496
|
follow.sort((a, b) => locationTier(a) - locationTier(b));
|
|
445
497
|
next = follow[0] || null;
|
|
446
498
|
}
|
|
@@ -454,16 +506,25 @@ export function createStore(filePath) {
|
|
|
454
506
|
}
|
|
455
507
|
|
|
456
508
|
// 先在有视频的 pending 用户中找;找不到再用全部 pending 用户兜底
|
|
457
|
-
|
|
458
|
-
|
|
509
|
+
let pending = data.filter((u) => u.status === "pending");
|
|
510
|
+
// 应用国家过滤
|
|
511
|
+
if (locations && locations.length > 0) {
|
|
512
|
+
pending = pending.filter(locationFilter);
|
|
513
|
+
}
|
|
514
|
+
let hasVideo = pending.filter((u) => u.videoCount > 0);
|
|
459
515
|
const next = pickCandidate(hasVideo) || pickCandidate(pending);
|
|
460
516
|
|
|
461
517
|
if (next) {
|
|
462
|
-
next.status =
|
|
518
|
+
next.status = "processing";
|
|
463
519
|
markStatsDirty();
|
|
464
520
|
next.claimedAt = now;
|
|
465
521
|
next.claimedBy = userId;
|
|
466
|
-
return {
|
|
522
|
+
return {
|
|
523
|
+
uniqueId: next.uniqueId,
|
|
524
|
+
nickname: next.nickname,
|
|
525
|
+
claimedAt: next.claimedAt,
|
|
526
|
+
claimedBy: userId,
|
|
527
|
+
};
|
|
467
528
|
}
|
|
468
529
|
return null;
|
|
469
530
|
}
|
|
@@ -471,32 +532,71 @@ export function createStore(filePath) {
|
|
|
471
532
|
function processDiscoveredUsers(result) {
|
|
472
533
|
const guessedLocation = result.guessedLocation || null;
|
|
473
534
|
const discovered = [
|
|
474
|
-
...(result.discoveredVideoAuthors || []).map(v => ({
|
|
475
|
-
uniqueId:
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
535
|
+
...(result.discoveredVideoAuthors || []).map((v) => ({
|
|
536
|
+
uniqueId:
|
|
537
|
+
typeof v === "string"
|
|
538
|
+
? v.replace(/^@/, "")
|
|
539
|
+
: v.uniqueId?.replace(/^@/, "") || "",
|
|
540
|
+
nickname: typeof v === "string" ? null : v.nickname || null,
|
|
541
|
+
locationCreated:
|
|
542
|
+
typeof v === "string" ? null : v.locationCreated || null,
|
|
543
|
+
guessedLocation:
|
|
544
|
+
typeof v === "string"
|
|
545
|
+
? guessedLocation
|
|
546
|
+
: v.guessedLocation || guessedLocation,
|
|
547
|
+
sources: ["video"],
|
|
480
548
|
})),
|
|
481
|
-
...(result.discoveredCommentAuthors || []).map(c => {
|
|
482
|
-
if (typeof c ===
|
|
483
|
-
|
|
549
|
+
...(result.discoveredCommentAuthors || []).map((c) => {
|
|
550
|
+
if (typeof c === "string")
|
|
551
|
+
return {
|
|
552
|
+
uniqueId: c.replace(/^@/, ""),
|
|
553
|
+
sources: ["comment"],
|
|
554
|
+
guessedLocation,
|
|
555
|
+
};
|
|
556
|
+
return {
|
|
557
|
+
uniqueId: (c.author || c.uniqueId || "").replace(/^@/, ""),
|
|
558
|
+
nickname: c.nickname || null,
|
|
559
|
+
sources: ["comment"],
|
|
560
|
+
guessedLocation: c.guessedLocation || guessedLocation,
|
|
561
|
+
};
|
|
484
562
|
}),
|
|
485
|
-
...(result.discoveredGuessAuthors || []).map(g => {
|
|
486
|
-
if (typeof g ===
|
|
487
|
-
|
|
563
|
+
...(result.discoveredGuessAuthors || []).map((g) => {
|
|
564
|
+
if (typeof g === "string")
|
|
565
|
+
return {
|
|
566
|
+
uniqueId: g.replace(/^@/, ""),
|
|
567
|
+
sources: ["guess"],
|
|
568
|
+
guessedLocation,
|
|
569
|
+
};
|
|
570
|
+
return {
|
|
571
|
+
uniqueId: (g.author || g.uniqueId || "").replace(/^@/, ""),
|
|
572
|
+
nickname: g.nickname || null,
|
|
573
|
+
sources: ["guess"],
|
|
574
|
+
guessedLocation: g.guessedLocation || guessedLocation,
|
|
575
|
+
};
|
|
488
576
|
}),
|
|
489
|
-
...(result.discoveredFollowing || []).map(f => {
|
|
490
|
-
const handle = Array.isArray(f) ? f[0] :
|
|
491
|
-
const name = Array.isArray(f) ? f[1] :
|
|
492
|
-
return {
|
|
577
|
+
...(result.discoveredFollowing || []).map((f) => {
|
|
578
|
+
const handle = Array.isArray(f) ? f[0] : f.handle || "";
|
|
579
|
+
const name = Array.isArray(f) ? f[1] : f.displayName || null;
|
|
580
|
+
return {
|
|
581
|
+
uniqueId: handle.replace(/^@/, ""),
|
|
582
|
+
nickname: name,
|
|
583
|
+
sources: ["following"],
|
|
584
|
+
guessedLocation:
|
|
585
|
+
(typeof f === "object" && f.guessedLocation) || guessedLocation,
|
|
586
|
+
};
|
|
493
587
|
}),
|
|
494
|
-
...(result.discoveredFollowers || []).map(f => {
|
|
495
|
-
const handle = Array.isArray(f) ? f[0] :
|
|
496
|
-
const name = Array.isArray(f) ? f[1] :
|
|
497
|
-
return {
|
|
588
|
+
...(result.discoveredFollowers || []).map((f) => {
|
|
589
|
+
const handle = Array.isArray(f) ? f[0] : f.handle || "";
|
|
590
|
+
const name = Array.isArray(f) ? f[1] : f.displayName || null;
|
|
591
|
+
return {
|
|
592
|
+
uniqueId: handle.replace(/^@/, ""),
|
|
593
|
+
nickname: name,
|
|
594
|
+
sources: ["follower"],
|
|
595
|
+
guessedLocation:
|
|
596
|
+
(typeof f === "object" && f.guessedLocation) || guessedLocation,
|
|
597
|
+
};
|
|
498
598
|
}),
|
|
499
|
-
].filter(u => u.uniqueId);
|
|
599
|
+
].filter((u) => u.uniqueId);
|
|
500
600
|
|
|
501
601
|
// 先对 discovered 内部去重,再用 uidIndex 批量判断
|
|
502
602
|
const seen = new Set();
|
|
@@ -521,25 +621,29 @@ export function createStore(filePath) {
|
|
|
521
621
|
function updateUserFromResult(user, result) {
|
|
522
622
|
const oldStatus = user.status;
|
|
523
623
|
if (result.restricted) {
|
|
524
|
-
user.status =
|
|
624
|
+
user.status = "restricted";
|
|
525
625
|
if (result.userInfo) {
|
|
526
626
|
const info = result.userInfo;
|
|
527
627
|
for (const key of Object.keys(info)) {
|
|
528
|
-
if (key ===
|
|
529
|
-
if (
|
|
628
|
+
if (key === "uniqueId" || key === "sources") continue;
|
|
629
|
+
if (
|
|
630
|
+
info[key] !== undefined &&
|
|
631
|
+
info[key] !== null &&
|
|
632
|
+
info[key] !== ""
|
|
633
|
+
) {
|
|
530
634
|
user[key] = info[key];
|
|
531
635
|
}
|
|
532
636
|
}
|
|
533
637
|
}
|
|
534
638
|
user.processed = true;
|
|
535
639
|
user.processedAt = Date.now();
|
|
536
|
-
user.sources = [...new Set([...(user.sources || []),
|
|
640
|
+
user.sources = [...new Set([...(user.sources || []), "restricted"])];
|
|
537
641
|
} else if (result.error) {
|
|
538
|
-
user.status =
|
|
642
|
+
user.status = "error";
|
|
539
643
|
user.error = result.error;
|
|
540
|
-
user.sources = [...new Set([...(user.sources || []),
|
|
644
|
+
user.sources = [...new Set([...(user.sources || []), "error"])];
|
|
541
645
|
} else {
|
|
542
|
-
user.status =
|
|
646
|
+
user.status = "done";
|
|
543
647
|
user.processed = true;
|
|
544
648
|
user.processedAt = Date.now();
|
|
545
649
|
user.noVideo = result.noVideo || false;
|
|
@@ -549,8 +653,12 @@ export function createStore(filePath) {
|
|
|
549
653
|
if (result.userInfo) {
|
|
550
654
|
const info = result.userInfo;
|
|
551
655
|
for (const key of Object.keys(info)) {
|
|
552
|
-
if (key ===
|
|
553
|
-
if (
|
|
656
|
+
if (key === "uniqueId" || key === "sources") continue;
|
|
657
|
+
if (
|
|
658
|
+
info[key] !== undefined &&
|
|
659
|
+
info[key] !== null &&
|
|
660
|
+
info[key] !== ""
|
|
661
|
+
) {
|
|
554
662
|
user[key] = info[key];
|
|
555
663
|
}
|
|
556
664
|
}
|
|
@@ -559,31 +667,46 @@ export function createStore(filePath) {
|
|
|
559
667
|
user.followerCount = result.userInfo?.followerCount ?? user.followerCount;
|
|
560
668
|
user.videoCount = result.userInfo?.videoCount ?? user.videoCount;
|
|
561
669
|
user.nickname = result.userInfo?.nickname || user.nickname;
|
|
562
|
-
user.locationCreated =
|
|
670
|
+
user.locationCreated =
|
|
671
|
+
result.userInfo?.locationCreated || user.locationCreated;
|
|
563
672
|
user.ttSeller = result.userInfo?.ttSeller ?? user.ttSeller;
|
|
564
673
|
user.verified = result.userInfo?.verified ?? user.verified;
|
|
565
674
|
user.region = result.userInfo?.region || user.region;
|
|
566
675
|
user.signature = result.userInfo?.signature ?? user.signature;
|
|
567
|
-
user.followingCount =
|
|
676
|
+
user.followingCount =
|
|
677
|
+
result.userInfo?.followingCount ?? user.followingCount;
|
|
568
678
|
user.heartCount = result.userInfo?.heartCount ?? user.heartCount;
|
|
569
679
|
if (result.userInfo?.secUid) user.secUid = result.userInfo.secUid;
|
|
570
|
-
const extraFields = [
|
|
571
|
-
|
|
572
|
-
|
|
680
|
+
const extraFields = [
|
|
681
|
+
"restricted",
|
|
682
|
+
"error",
|
|
683
|
+
"userInfo",
|
|
684
|
+
"discoveredVideoAuthors",
|
|
685
|
+
"discoveredCommentAuthors",
|
|
686
|
+
"discoveredGuessAuthors",
|
|
687
|
+
"discoveredFollowing",
|
|
688
|
+
"discoveredFollowers",
|
|
689
|
+
"uniqueId",
|
|
690
|
+
"sources",
|
|
691
|
+
];
|
|
573
692
|
for (const key of Object.keys(result)) {
|
|
574
693
|
if (extraFields.includes(key)) continue;
|
|
575
|
-
if (
|
|
694
|
+
if (
|
|
695
|
+
result[key] !== undefined &&
|
|
696
|
+
result[key] !== null &&
|
|
697
|
+
result[key] !== ""
|
|
698
|
+
) {
|
|
576
699
|
user[key] = result[key];
|
|
577
700
|
}
|
|
578
701
|
}
|
|
579
|
-
user.sources = [...new Set([...(user.sources || []),
|
|
702
|
+
user.sources = [...new Set([...(user.sources || []), "processed"])];
|
|
580
703
|
}
|
|
581
704
|
if (user.status !== oldStatus) markStatsDirty();
|
|
582
705
|
}
|
|
583
706
|
|
|
584
707
|
function commitJob(uniqueId, result) {
|
|
585
708
|
const user = getUser(uniqueId);
|
|
586
|
-
if (!user) return { saved: false, error:
|
|
709
|
+
if (!user) return { saved: false, error: "user not found" };
|
|
587
710
|
|
|
588
711
|
updateUserFromResult(user, result);
|
|
589
712
|
delete user.claimedAt;
|
|
@@ -605,7 +728,7 @@ export function createStore(filePath) {
|
|
|
605
728
|
const userObj = {
|
|
606
729
|
uniqueId,
|
|
607
730
|
...(result.userInfo || {}),
|
|
608
|
-
sources: [
|
|
731
|
+
sources: ["refresh-explore"],
|
|
609
732
|
};
|
|
610
733
|
updateUserFromResult(userObj, result);
|
|
611
734
|
addUser(userObj, true);
|
|
@@ -617,8 +740,8 @@ export function createStore(filePath) {
|
|
|
617
740
|
|
|
618
741
|
function resetJob(uniqueId) {
|
|
619
742
|
const user = getUser(uniqueId);
|
|
620
|
-
if (!user) return { saved: false, error:
|
|
621
|
-
user.status =
|
|
743
|
+
if (!user) return { saved: false, error: "user not found" };
|
|
744
|
+
user.status = "pending";
|
|
622
745
|
markStatsDirty();
|
|
623
746
|
delete user.claimedAt;
|
|
624
747
|
delete user.processedAt;
|
|
@@ -632,7 +755,7 @@ export function createStore(filePath) {
|
|
|
632
755
|
|
|
633
756
|
function togglePin(uniqueId) {
|
|
634
757
|
const user = getUser(uniqueId);
|
|
635
|
-
if (!user) return { saved: false, error:
|
|
758
|
+
if (!user) return { saved: false, error: "user not found" };
|
|
636
759
|
user.pinned = !user.pinned;
|
|
637
760
|
save();
|
|
638
761
|
return { saved: true, pinned: user.pinned };
|
|
@@ -640,11 +763,16 @@ export function createStore(filePath) {
|
|
|
640
763
|
|
|
641
764
|
function getNextRedoJob(userId) {
|
|
642
765
|
const now = Date.now();
|
|
643
|
-
const defaultTime = new Date(
|
|
766
|
+
const defaultTime = new Date("2016-01-01T00:00:00Z").getTime();
|
|
644
767
|
|
|
645
768
|
// 筛选目标国家用户,按 refreshTime 升序取最远的(没有则默认 2016-01-01)
|
|
646
|
-
const targetLocations = [
|
|
647
|
-
const targetUsers = data.filter(
|
|
769
|
+
const targetLocations = ["ES", "PL", "NL", "BE", "DE", "FR", "IT", "IE"];
|
|
770
|
+
const targetUsers = data.filter(
|
|
771
|
+
(u) =>
|
|
772
|
+
u.ttSeller &&
|
|
773
|
+
u.verified === false &&
|
|
774
|
+
targetLocations.includes(u.locationCreated),
|
|
775
|
+
);
|
|
648
776
|
if (targetUsers.length === 0) return null;
|
|
649
777
|
|
|
650
778
|
targetUsers.sort((a, b) => {
|
|
@@ -655,20 +783,24 @@ export function createStore(filePath) {
|
|
|
655
783
|
|
|
656
784
|
const next = targetUsers[0];
|
|
657
785
|
next.refreshTime = now;
|
|
658
|
-
return {
|
|
786
|
+
return {
|
|
787
|
+
uniqueId: next.uniqueId,
|
|
788
|
+
nickname: next.nickname,
|
|
789
|
+
refreshTime: next.refreshTime,
|
|
790
|
+
};
|
|
659
791
|
}
|
|
660
792
|
|
|
661
793
|
function commitRedoJob(uniqueId, result) {
|
|
662
794
|
const user = getUser(uniqueId);
|
|
663
|
-
if (!user) return { saved: false, error:
|
|
795
|
+
if (!user) return { saved: false, error: "user not found" };
|
|
664
796
|
|
|
665
797
|
user.refreshTime = Date.now();
|
|
666
798
|
|
|
667
799
|
if (result.userInfo) {
|
|
668
800
|
const info = result.userInfo;
|
|
669
801
|
for (const key of Object.keys(info)) {
|
|
670
|
-
if (key ===
|
|
671
|
-
if (info[key] !== undefined && info[key] !== null && info[key] !==
|
|
802
|
+
if (key === "uniqueId" || key === "sources") continue;
|
|
803
|
+
if (info[key] !== undefined && info[key] !== null && info[key] !== "") {
|
|
672
804
|
user[key] = info[key];
|
|
673
805
|
}
|
|
674
806
|
}
|
|
@@ -677,20 +809,28 @@ export function createStore(filePath) {
|
|
|
677
809
|
return { saved: true };
|
|
678
810
|
}
|
|
679
811
|
|
|
680
|
-
function reportClientError(
|
|
812
|
+
function reportClientError(
|
|
813
|
+
userId,
|
|
814
|
+
errorType,
|
|
815
|
+
errorMessage,
|
|
816
|
+
username,
|
|
817
|
+
stage,
|
|
818
|
+
errorStack,
|
|
819
|
+
) {
|
|
681
820
|
const existing = clientErrors.get(userId);
|
|
682
821
|
if (existing) {
|
|
683
822
|
existing.timestamp = Date.now();
|
|
684
|
-
if (errorType ===
|
|
823
|
+
if (errorType === "captcha") {
|
|
685
824
|
existing.captchaCount = (existing.captchaCount || 0) + 1;
|
|
686
|
-
if (!existing.captchaStage) existing.captchaStage = stage ||
|
|
687
|
-
if (!existing.captchaMessage)
|
|
688
|
-
|
|
825
|
+
if (!existing.captchaStage) existing.captchaStage = stage || "";
|
|
826
|
+
if (!existing.captchaMessage)
|
|
827
|
+
existing.captchaMessage = errorMessage || "";
|
|
828
|
+
if (!existing.captchaStack) existing.captchaStack = errorStack || "";
|
|
689
829
|
} else {
|
|
690
830
|
existing.errorType = errorType;
|
|
691
|
-
existing.errorMessage = errorMessage ||
|
|
692
|
-
existing.errorStack = errorStack ||
|
|
693
|
-
existing.stage = stage ||
|
|
831
|
+
existing.errorMessage = errorMessage || "";
|
|
832
|
+
existing.errorStack = errorStack || "";
|
|
833
|
+
existing.stage = stage || "";
|
|
694
834
|
existing.reportCount = (existing.reportCount || 1) + 1;
|
|
695
835
|
}
|
|
696
836
|
if (username) existing.username = username;
|
|
@@ -698,16 +838,16 @@ export function createStore(filePath) {
|
|
|
698
838
|
clientErrors.set(userId, {
|
|
699
839
|
userId,
|
|
700
840
|
errorType,
|
|
701
|
-
errorMessage: errorMessage ||
|
|
702
|
-
errorStack: errorStack ||
|
|
841
|
+
errorMessage: errorMessage || "",
|
|
842
|
+
errorStack: errorStack || "",
|
|
703
843
|
username,
|
|
704
|
-
stage: stage ||
|
|
844
|
+
stage: stage || "",
|
|
705
845
|
timestamp: Date.now(),
|
|
706
846
|
reportCount: 1,
|
|
707
|
-
captchaCount: errorType ===
|
|
708
|
-
captchaStage: errorType ===
|
|
709
|
-
captchaMessage: errorType ===
|
|
710
|
-
captchaStack: errorType ===
|
|
847
|
+
captchaCount: errorType === "captcha" ? 1 : 0,
|
|
848
|
+
captchaStage: errorType === "captcha" ? stage || "" : "",
|
|
849
|
+
captchaMessage: errorType === "captcha" ? errorMessage || "" : "",
|
|
850
|
+
captchaStack: errorType === "captcha" ? errorStack || "" : "",
|
|
711
851
|
});
|
|
712
852
|
}
|
|
713
853
|
}
|
|
@@ -722,14 +862,19 @@ export function createStore(filePath) {
|
|
|
722
862
|
|
|
723
863
|
function getPendingUserUpdateTasks(limit) {
|
|
724
864
|
const l = Math.max(1, parseInt(limit) || 5);
|
|
725
|
-
const pending = data
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
865
|
+
const pending = data
|
|
866
|
+
.filter((u) => {
|
|
867
|
+
const updateCount = u.userUpdateCount;
|
|
868
|
+
const ttSellerEmpty =
|
|
869
|
+
u.ttSeller === null || u.ttSeller === undefined || u.ttSeller === "";
|
|
870
|
+
if (!ttSellerEmpty) return false;
|
|
871
|
+
return (
|
|
872
|
+
updateCount === null || updateCount === undefined || updateCount <= 0
|
|
873
|
+
);
|
|
874
|
+
})
|
|
875
|
+
.slice(0, l);
|
|
731
876
|
// 接受任务时 userUpdateCount + 1
|
|
732
|
-
pending.forEach(u => {
|
|
877
|
+
pending.forEach((u) => {
|
|
733
878
|
u.userUpdateCount = (u.userUpdateCount || 0) + 1;
|
|
734
879
|
u.updatedAt = Date.now();
|
|
735
880
|
});
|
|
@@ -739,10 +884,10 @@ export function createStore(filePath) {
|
|
|
739
884
|
|
|
740
885
|
function updateUserInfo(uniqueId, info) {
|
|
741
886
|
const user = getUser(uniqueId);
|
|
742
|
-
if (!user) return { error:
|
|
887
|
+
if (!user) return { error: "user not found" };
|
|
743
888
|
for (const key of Object.keys(info)) {
|
|
744
|
-
if (key ===
|
|
745
|
-
if (info[key] !== undefined && info[key] !== null && info[key] !==
|
|
889
|
+
if (key === "uniqueId" || key === "sources") continue;
|
|
890
|
+
if (info[key] !== undefined && info[key] !== null && info[key] !== "") {
|
|
746
891
|
user[key] = info[key];
|
|
747
892
|
}
|
|
748
893
|
}
|
|
@@ -757,18 +902,26 @@ export function createStore(filePath) {
|
|
|
757
902
|
for (const item of updates) {
|
|
758
903
|
const user = getUser(item.uniqueId);
|
|
759
904
|
if (!user) {
|
|
760
|
-
results.push({ uniqueId: item.uniqueId, error:
|
|
905
|
+
results.push({ uniqueId: item.uniqueId, error: "user not found" });
|
|
761
906
|
continue;
|
|
762
907
|
}
|
|
763
908
|
for (const key of Object.keys(item.info)) {
|
|
764
|
-
if (key ===
|
|
765
|
-
if (
|
|
909
|
+
if (key === "uniqueId" || key === "sources") continue;
|
|
910
|
+
if (
|
|
911
|
+
item.info[key] !== undefined &&
|
|
912
|
+
item.info[key] !== null &&
|
|
913
|
+
item.info[key] !== ""
|
|
914
|
+
) {
|
|
766
915
|
user[key] = item.info[key];
|
|
767
916
|
}
|
|
768
917
|
}
|
|
769
918
|
user.userUpdateCount = (user.userUpdateCount || 0) + 1;
|
|
770
919
|
user.updatedAt = Date.now();
|
|
771
|
-
results.push({
|
|
920
|
+
results.push({
|
|
921
|
+
uniqueId: item.uniqueId,
|
|
922
|
+
ok: true,
|
|
923
|
+
userUpdateCount: user.userUpdateCount,
|
|
924
|
+
});
|
|
772
925
|
}
|
|
773
926
|
save();
|
|
774
927
|
return results;
|
|
@@ -780,7 +933,7 @@ export function createStore(filePath) {
|
|
|
780
933
|
return { registered: 0, skipped: 0 };
|
|
781
934
|
}
|
|
782
935
|
|
|
783
|
-
const existingIds = new Set(videos.map(v => v.id));
|
|
936
|
+
const existingIds = new Set(videos.map((v) => v.id));
|
|
784
937
|
let registered = 0;
|
|
785
938
|
let skipped = 0;
|
|
786
939
|
|
|
@@ -815,7 +968,7 @@ export function createStore(filePath) {
|
|
|
815
968
|
|
|
816
969
|
function getPendingCommentTasks(limit) {
|
|
817
970
|
// 筛选待处理视频(userUpdateCount <= 0 或 null/undefined)
|
|
818
|
-
const pending = videos.filter(v => (v.userUpdateCount || 0) <= 0);
|
|
971
|
+
const pending = videos.filter((v) => (v.userUpdateCount || 0) <= 0);
|
|
819
972
|
// ttSeller=true 优先
|
|
820
973
|
pending.sort((a, b) => {
|
|
821
974
|
if (a.ttSeller && !b.ttSeller) return -1;
|
|
@@ -833,22 +986,44 @@ export function createStore(filePath) {
|
|
|
833
986
|
}
|
|
834
987
|
|
|
835
988
|
function commitCommentTask(videoId) {
|
|
836
|
-
const video = videos.find(v => v.id === videoId);
|
|
837
|
-
if (!video) return { ok: false, error:
|
|
989
|
+
const video = videos.find((v) => v.id === videoId);
|
|
990
|
+
if (!video) return { ok: false, error: "video not found" };
|
|
838
991
|
video.userUpdateCount = (video.userUpdateCount || 0) + 1;
|
|
839
992
|
saveVideos();
|
|
840
993
|
return { ok: true, userUpdateCount: video.userUpdateCount };
|
|
841
994
|
}
|
|
842
995
|
|
|
843
996
|
return {
|
|
844
|
-
save,
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
997
|
+
save,
|
|
998
|
+
flushSave,
|
|
999
|
+
getUser,
|
|
1000
|
+
hasUser,
|
|
1001
|
+
userExists,
|
|
1002
|
+
addUser,
|
|
1003
|
+
getPendingUsers,
|
|
1004
|
+
getProcessedUsers,
|
|
1005
|
+
getAllUsers,
|
|
1006
|
+
getStats,
|
|
1007
|
+
getStatusGroups,
|
|
1008
|
+
markGroupsDirty,
|
|
1009
|
+
claimNextJob,
|
|
1010
|
+
commitJob,
|
|
1011
|
+
commitNewExplore,
|
|
1012
|
+
resetJob,
|
|
1013
|
+
togglePin,
|
|
1014
|
+
getNextRedoJob,
|
|
1015
|
+
commitRedoJob,
|
|
1016
|
+
getPendingUserUpdateTasks,
|
|
1017
|
+
updateUserInfo,
|
|
1018
|
+
batchUpdateUserInfo,
|
|
1019
|
+
reportClientError,
|
|
1020
|
+
deleteClientError,
|
|
1021
|
+
getClientErrors,
|
|
1022
|
+
registerVideos,
|
|
1023
|
+
getVideos,
|
|
1024
|
+
getVideoCount,
|
|
1025
|
+
getPendingCommentTasks,
|
|
1026
|
+
commitCommentTask,
|
|
852
1027
|
stopBackup,
|
|
853
1028
|
data,
|
|
854
1029
|
};
|