tt-help-cli-ycl 1.3.65 → 1.3.73
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/cli/attach.js +8 -4
- package/src/cli/refresh.js +16 -4
- package/src/lib/parse-ssr.mjs +1 -0
- package/src/scraper/explore-core.js +36 -19
- package/src/watch/data-store.js +390 -155
- package/src/watch/public/app.js +325 -62
- package/src/watch/public/index.html +10 -6
- package/src/watch/public/style.css +88 -1
- package/src/watch/server.js +73 -3
package/src/watch/data-store.js
CHANGED
|
@@ -104,6 +104,8 @@ function initUserDb(filePath) {
|
|
|
104
104
|
comment_count INTEGER DEFAULT 0,
|
|
105
105
|
guessed_location TEXT,
|
|
106
106
|
location_created TEXT,
|
|
107
|
+
confirmed_location TEXT,
|
|
108
|
+
modified_at INTEGER,
|
|
107
109
|
follower_count INTEGER DEFAULT 0,
|
|
108
110
|
following_count INTEGER DEFAULT 0,
|
|
109
111
|
heart_count INTEGER DEFAULT 0,
|
|
@@ -132,6 +134,15 @@ function initUserDb(filePath) {
|
|
|
132
134
|
if (!existingJobColumns.has("latest_video_time")) {
|
|
133
135
|
db.exec(`ALTER TABLE jobs ADD COLUMN latest_video_time INTEGER`);
|
|
134
136
|
}
|
|
137
|
+
if (!existingJobColumns.has("confirmed_location")) {
|
|
138
|
+
db.exec(`ALTER TABLE jobs ADD COLUMN confirmed_location TEXT`);
|
|
139
|
+
}
|
|
140
|
+
if (!existingJobColumns.has("modified_at")) {
|
|
141
|
+
db.exec(`ALTER TABLE jobs ADD COLUMN modified_at INTEGER`);
|
|
142
|
+
}
|
|
143
|
+
if (!existingJobColumns.has("bio_link")) {
|
|
144
|
+
db.exec(`ALTER TABLE jobs ADD COLUMN bio_link TEXT`);
|
|
145
|
+
}
|
|
135
146
|
db.exec(`
|
|
136
147
|
CREATE TABLE IF NOT EXISTS raw_jobs (
|
|
137
148
|
unique_id TEXT PRIMARY KEY,
|
|
@@ -151,6 +162,8 @@ function initUserDb(filePath) {
|
|
|
151
162
|
comment_count INTEGER DEFAULT 0,
|
|
152
163
|
guessed_location TEXT,
|
|
153
164
|
location_created TEXT,
|
|
165
|
+
confirmed_location TEXT,
|
|
166
|
+
modified_at INTEGER,
|
|
154
167
|
follower_count INTEGER DEFAULT 0,
|
|
155
168
|
following_count INTEGER DEFAULT 0,
|
|
156
169
|
heart_count INTEGER DEFAULT 0,
|
|
@@ -180,6 +193,15 @@ function initUserDb(filePath) {
|
|
|
180
193
|
if (!existingRawJobColumns.has("latest_video_time")) {
|
|
181
194
|
db.exec(`ALTER TABLE raw_jobs ADD COLUMN latest_video_time INTEGER`);
|
|
182
195
|
}
|
|
196
|
+
if (!existingRawJobColumns.has("confirmed_location")) {
|
|
197
|
+
db.exec(`ALTER TABLE raw_jobs ADD COLUMN confirmed_location TEXT`);
|
|
198
|
+
}
|
|
199
|
+
if (!existingRawJobColumns.has("modified_at")) {
|
|
200
|
+
db.exec(`ALTER TABLE raw_jobs ADD COLUMN modified_at INTEGER`);
|
|
201
|
+
}
|
|
202
|
+
if (!existingRawJobColumns.has("bio_link")) {
|
|
203
|
+
db.exec(`ALTER TABLE raw_jobs ADD COLUMN bio_link TEXT`);
|
|
204
|
+
}
|
|
183
205
|
db.exec(`
|
|
184
206
|
CREATE TABLE IF NOT EXISTS videos (
|
|
185
207
|
id TEXT PRIMARY KEY,
|
|
@@ -445,9 +467,10 @@ function addJobToDb(user) {
|
|
|
445
467
|
updated_at,
|
|
446
468
|
region,
|
|
447
469
|
signature,
|
|
470
|
+
bio_link,
|
|
448
471
|
sec_uid
|
|
449
472
|
)
|
|
450
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
473
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
451
474
|
`,
|
|
452
475
|
).run(
|
|
453
476
|
user.uniqueId,
|
|
@@ -491,6 +514,7 @@ function addJobToDb(user) {
|
|
|
491
514
|
user.updatedAt || now,
|
|
492
515
|
user.region || null,
|
|
493
516
|
user.signature || null,
|
|
517
|
+
user.bioLink?.link || user.bioLink?.url || user.bioLink || null,
|
|
494
518
|
user.secUid || null,
|
|
495
519
|
);
|
|
496
520
|
}
|
|
@@ -534,44 +558,46 @@ function getRawJobsCount() {
|
|
|
534
558
|
function getDashboardStatsFromDb(targetLocations = []) {
|
|
535
559
|
if (!db) return null;
|
|
536
560
|
|
|
537
|
-
const
|
|
538
|
-
const
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
error: 0,
|
|
543
|
-
restricted: 0,
|
|
544
|
-
};
|
|
545
|
-
const rows = db
|
|
561
|
+
const targetPlaceholders = targetLocations.map(() => "?").join(", ");
|
|
562
|
+
const targetParams = targetLocations.length ? targetLocations : [];
|
|
563
|
+
|
|
564
|
+
// 合并所有 jobs 表的聚合统计为单次扫描
|
|
565
|
+
const aggregateRow = db
|
|
546
566
|
.prepare(
|
|
547
567
|
`
|
|
548
|
-
SELECT
|
|
568
|
+
SELECT
|
|
569
|
+
COUNT(*) as total,
|
|
570
|
+
SUM(CASE WHEN status = 'pending' THEN 1 ELSE 0 END) as pending,
|
|
571
|
+
SUM(CASE WHEN status = 'processing' THEN 1 ELSE 0 END) as processing,
|
|
572
|
+
SUM(CASE WHEN status = 'done' THEN 1 ELSE 0 END) as done,
|
|
573
|
+
SUM(CASE WHEN status = 'error' THEN 1 ELSE 0 END) as error,
|
|
574
|
+
SUM(CASE WHEN status = 'restricted' THEN 1 ELSE 0 END) as restricted,
|
|
575
|
+
SUM(CASE WHEN tt_seller = 1 AND verified = 0 ${
|
|
576
|
+
targetLocations.length
|
|
577
|
+
? `AND location_created IN (${targetPlaceholders})`
|
|
578
|
+
: "AND 1 = 0"
|
|
579
|
+
} THEN 1 ELSE 0 END) as targetUsers,
|
|
580
|
+
SUM(CASE WHEN no_video = 1 THEN 1 ELSE 0 END) as noVideo,
|
|
581
|
+
SUM(CASE WHEN status != 'done' AND instr(COALESCE(sources, ''), '"video"') > 0 THEN 1 ELSE 0 END) as video,
|
|
582
|
+
SUM(CASE WHEN status != 'done' AND instr(COALESCE(sources, ''), '"comment"') > 0 THEN 1 ELSE 0 END) as comment,
|
|
583
|
+
SUM(CASE WHEN status != 'done' AND instr(COALESCE(sources, ''), '"guess"') > 0 THEN 1 ELSE 0 END) as guess,
|
|
584
|
+
SUM(CASE WHEN status != 'done' AND instr(COALESCE(sources, ''), '"following"') > 0 THEN 1 ELSE 0 END) as following,
|
|
585
|
+
SUM(CASE WHEN status != 'done' AND instr(COALESCE(sources, ''), '"follower"') > 0 THEN 1 ELSE 0 END) as follower,
|
|
586
|
+
SUM(CASE
|
|
587
|
+
WHEN status != 'done'
|
|
588
|
+
AND instr(COALESCE(sources, ''), '"video"') = 0
|
|
589
|
+
AND instr(COALESCE(sources, ''), '"comment"') = 0
|
|
590
|
+
AND instr(COALESCE(sources, ''), '"guess"') = 0
|
|
591
|
+
AND instr(COALESCE(sources, ''), '"following"') = 0
|
|
592
|
+
AND instr(COALESCE(sources, ''), '"follower"') = 0
|
|
593
|
+
THEN 1 ELSE 0 END) as seed,
|
|
594
|
+
SUM(CASE WHEN COALESCE(tt_seller, '') = '' AND COALESCE(user_update_count, 0) <= 0 THEN 1 ELSE 0 END) as userUpdateTasks
|
|
549
595
|
FROM jobs
|
|
550
|
-
GROUP BY status
|
|
551
596
|
`,
|
|
552
597
|
)
|
|
553
|
-
.
|
|
554
|
-
for (const row of rows) {
|
|
555
|
-
if (row.status && statusCounts[row.status] !== undefined) {
|
|
556
|
-
statusCounts[row.status] = row.count;
|
|
557
|
-
}
|
|
558
|
-
}
|
|
559
|
-
|
|
560
|
-
const targetPlaceholders = targetLocations.map(() => "?").join(", ");
|
|
561
|
-
const targetUsers = targetLocations.length
|
|
562
|
-
? db
|
|
563
|
-
.prepare(
|
|
564
|
-
`
|
|
565
|
-
SELECT COUNT(*) as c
|
|
566
|
-
FROM jobs
|
|
567
|
-
WHERE tt_seller = 1
|
|
568
|
-
AND verified = 0
|
|
569
|
-
AND location_created IN (${targetPlaceholders})
|
|
570
|
-
`,
|
|
571
|
-
)
|
|
572
|
-
.get(...targetLocations).c
|
|
573
|
-
: 0;
|
|
598
|
+
.get(...targetParams);
|
|
574
599
|
|
|
600
|
+
// countryStats 和 targetCountryStats 需要 GROUP BY,保留为独立查询
|
|
575
601
|
const countryStats = db
|
|
576
602
|
.prepare(
|
|
577
603
|
`
|
|
@@ -591,7 +617,7 @@ function getDashboardStatsFromDb(targetLocations = []) {
|
|
|
591
617
|
ORDER BY count DESC
|
|
592
618
|
`,
|
|
593
619
|
)
|
|
594
|
-
.all(...
|
|
620
|
+
.all(...targetParams);
|
|
595
621
|
|
|
596
622
|
const targetCountryStats = targetLocations.length
|
|
597
623
|
? db
|
|
@@ -609,58 +635,32 @@ function getDashboardStatsFromDb(targetLocations = []) {
|
|
|
609
635
|
.all(...targetLocations)
|
|
610
636
|
: [];
|
|
611
637
|
|
|
612
|
-
const sourceRow = db
|
|
613
|
-
.prepare(
|
|
614
|
-
`
|
|
615
|
-
SELECT
|
|
616
|
-
SUM(CASE WHEN status = 'done' THEN 1 ELSE 0 END) as processed,
|
|
617
|
-
SUM(CASE WHEN status = 'restricted' THEN 1 ELSE 0 END) as restricted,
|
|
618
|
-
SUM(CASE WHEN status = 'error' THEN 1 ELSE 0 END) as error,
|
|
619
|
-
SUM(CASE WHEN no_video = 1 THEN 1 ELSE 0 END) as noVideo,
|
|
620
|
-
SUM(CASE WHEN status != 'done' AND instr(COALESCE(sources, ''), '"video"') > 0 THEN 1 ELSE 0 END) as video,
|
|
621
|
-
SUM(CASE WHEN status != 'done' AND instr(COALESCE(sources, ''), '"comment"') > 0 THEN 1 ELSE 0 END) as comment,
|
|
622
|
-
SUM(CASE WHEN status != 'done' AND instr(COALESCE(sources, ''), '"guess"') > 0 THEN 1 ELSE 0 END) as guess,
|
|
623
|
-
SUM(CASE WHEN status != 'done' AND instr(COALESCE(sources, ''), '"following"') > 0 THEN 1 ELSE 0 END) as following,
|
|
624
|
-
SUM(CASE WHEN status != 'done' AND instr(COALESCE(sources, ''), '"follower"') > 0 THEN 1 ELSE 0 END) as follower,
|
|
625
|
-
SUM(CASE
|
|
626
|
-
WHEN status != 'done'
|
|
627
|
-
AND instr(COALESCE(sources, ''), '"video"') = 0
|
|
628
|
-
AND instr(COALESCE(sources, ''), '"comment"') = 0
|
|
629
|
-
AND instr(COALESCE(sources, ''), '"guess"') = 0
|
|
630
|
-
AND instr(COALESCE(sources, ''), '"following"') = 0
|
|
631
|
-
AND instr(COALESCE(sources, ''), '"follower"') = 0
|
|
632
|
-
THEN 1 ELSE 0 END) as seed
|
|
633
|
-
FROM jobs
|
|
634
|
-
`,
|
|
635
|
-
)
|
|
636
|
-
.get();
|
|
637
|
-
|
|
638
638
|
return {
|
|
639
|
-
totalUsers: total,
|
|
639
|
+
totalUsers: aggregateRow.total,
|
|
640
640
|
rawJobs: getRawJobsCount(),
|
|
641
641
|
dbTotalUsers: getUserDbCount(),
|
|
642
|
-
jobsTotal: total,
|
|
643
|
-
jobsPending:
|
|
644
|
-
processedUsers:
|
|
645
|
-
pendingUsers:
|
|
646
|
-
processingUsers:
|
|
647
|
-
restrictedUsers:
|
|
648
|
-
errorUsers:
|
|
649
|
-
targetUsers,
|
|
650
|
-
userUpdateTasks:
|
|
642
|
+
jobsTotal: aggregateRow.total,
|
|
643
|
+
jobsPending: aggregateRow.pending,
|
|
644
|
+
processedUsers: aggregateRow.done,
|
|
645
|
+
pendingUsers: aggregateRow.pending,
|
|
646
|
+
processingUsers: aggregateRow.processing,
|
|
647
|
+
restrictedUsers: aggregateRow.restricted,
|
|
648
|
+
errorUsers: aggregateRow.error,
|
|
649
|
+
targetUsers: aggregateRow.targetUsers,
|
|
650
|
+
userUpdateTasks: aggregateRow.userUpdateTasks,
|
|
651
651
|
targetCountryStats,
|
|
652
652
|
countryStats,
|
|
653
653
|
sourceStats: {
|
|
654
|
-
seed:
|
|
655
|
-
video:
|
|
656
|
-
comment:
|
|
657
|
-
guess:
|
|
658
|
-
following:
|
|
659
|
-
follower:
|
|
660
|
-
processed:
|
|
661
|
-
restricted:
|
|
662
|
-
error:
|
|
663
|
-
noVideo:
|
|
654
|
+
seed: aggregateRow.seed || 0,
|
|
655
|
+
video: aggregateRow.video || 0,
|
|
656
|
+
comment: aggregateRow.comment || 0,
|
|
657
|
+
guess: aggregateRow.guess || 0,
|
|
658
|
+
following: aggregateRow.following || 0,
|
|
659
|
+
follower: aggregateRow.follower || 0,
|
|
660
|
+
processed: aggregateRow.done,
|
|
661
|
+
restricted: aggregateRow.restricted,
|
|
662
|
+
error: aggregateRow.error,
|
|
663
|
+
noVideo: aggregateRow.noVideo || 0,
|
|
664
664
|
},
|
|
665
665
|
};
|
|
666
666
|
}
|
|
@@ -857,7 +857,7 @@ function moveJobsToRawByCountry(scope, country) {
|
|
|
857
857
|
|
|
858
858
|
let scopeWhere = "";
|
|
859
859
|
if (normalizedScope === "pending") {
|
|
860
|
-
scopeWhere = `status = 'pending'`;
|
|
860
|
+
scopeWhere = `status = 'pending' AND COALESCE(user_update_count, 0) >= 2`;
|
|
861
861
|
} else if (normalizedScope === "userUpdate") {
|
|
862
862
|
scopeWhere = `COALESCE(tt_seller, '') = '' AND COALESCE(user_update_count, 0) <= 0`;
|
|
863
863
|
} else {
|
|
@@ -1105,14 +1105,14 @@ function restoreRawJobById(uniqueId) {
|
|
|
1105
1105
|
pinned, no_video, restricted, user_update_count, tt_seller, verified,
|
|
1106
1106
|
video_count, comment_count, guessed_location, location_created,
|
|
1107
1107
|
follower_count, following_count, heart_count, refresh_time,
|
|
1108
|
-
processed, processed_at, created_at, updated_at, region, signature, sec_uid
|
|
1108
|
+
processed, processed_at, created_at, updated_at, region, signature, bio_link, sec_uid
|
|
1109
1109
|
)
|
|
1110
1110
|
SELECT
|
|
1111
1111
|
unique_id, nickname, status, sources, claimed_by, claimed_at, error,
|
|
1112
1112
|
pinned, no_video, restricted, user_update_count, tt_seller, verified,
|
|
1113
1113
|
video_count, comment_count, guessed_location, location_created,
|
|
1114
1114
|
follower_count, following_count, heart_count, refresh_time,
|
|
1115
|
-
processed, processed_at, created_at, updated_at, region, signature, sec_uid
|
|
1115
|
+
processed, processed_at, created_at, updated_at, region, signature, bio_link, sec_uid
|
|
1116
1116
|
FROM raw_jobs WHERE unique_id = ?
|
|
1117
1117
|
`,
|
|
1118
1118
|
).run(safeId);
|
|
@@ -1176,14 +1176,14 @@ function restoreRawJobsByFilter({ search, location, hasVideo, hasFollower }) {
|
|
|
1176
1176
|
pinned, no_video, restricted, user_update_count, tt_seller, verified,
|
|
1177
1177
|
video_count, comment_count, guessed_location, location_created,
|
|
1178
1178
|
follower_count, following_count, heart_count, refresh_time,
|
|
1179
|
-
processed, processed_at, created_at, updated_at, region, signature, sec_uid
|
|
1179
|
+
processed, processed_at, created_at, updated_at, region, signature, bio_link, sec_uid
|
|
1180
1180
|
)
|
|
1181
1181
|
SELECT
|
|
1182
1182
|
unique_id, nickname, status, sources, claimed_by, claimed_at, error,
|
|
1183
1183
|
pinned, no_video, restricted, user_update_count, tt_seller, verified,
|
|
1184
1184
|
video_count, comment_count, guessed_location, location_created,
|
|
1185
1185
|
follower_count, following_count, heart_count, refresh_time,
|
|
1186
|
-
processed, processed_at, created_at, updated_at, region, signature, sec_uid
|
|
1186
|
+
processed, processed_at, created_at, updated_at, region, signature, bio_link, sec_uid
|
|
1187
1187
|
FROM raw_jobs WHERE ${whereSql}
|
|
1188
1188
|
`,
|
|
1189
1189
|
).run(...args);
|
|
@@ -1313,46 +1313,59 @@ function getUsersPageFromDb({
|
|
|
1313
1313
|
}
|
|
1314
1314
|
|
|
1315
1315
|
const whereSql = where.length ? `WHERE ${where.join(" AND ")}` : "";
|
|
1316
|
-
const total = db
|
|
1317
|
-
.prepare(`SELECT COUNT(*) as c FROM jobs ${whereSql}`)
|
|
1318
|
-
.get(...args).c;
|
|
1319
1316
|
|
|
1317
|
+
// COUNT 缓存:134 万条数据全表扫描慢,5 秒内返回缓存值
|
|
1318
|
+
const cacheKey = whereSql + "|" + args.join(",");
|
|
1319
|
+
if (!getUsersPageFromDb._countCache)
|
|
1320
|
+
getUsersPageFromDb._countCache = new Map();
|
|
1321
|
+
const cachedCount = getUsersPageFromDb._countCache.get(cacheKey);
|
|
1322
|
+
let total;
|
|
1323
|
+
if (cachedCount && Date.now() - cachedCount.time < 5000) {
|
|
1324
|
+
total = cachedCount.c;
|
|
1325
|
+
} else {
|
|
1326
|
+
total = db
|
|
1327
|
+
.prepare(`SELECT COUNT(*) as c FROM jobs ${whereSql}`)
|
|
1328
|
+
.get(...args).c;
|
|
1329
|
+
getUsersPageFromDb._countCache.set(cacheKey, {
|
|
1330
|
+
c: total,
|
|
1331
|
+
time: Date.now(),
|
|
1332
|
+
});
|
|
1333
|
+
}
|
|
1334
|
+
|
|
1335
|
+
// 只查询前端需要的列,避免 SELECT * 带来的大字段传输和 mapJobRow 开销
|
|
1320
1336
|
const rows = db
|
|
1321
1337
|
.prepare(
|
|
1322
1338
|
`
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
|
|
1335
|
-
|
|
1336
|
-
|
|
1337
|
-
|
|
1338
|
-
|
|
1339
|
-
|
|
1340
|
-
|
|
1341
|
-
|
|
1342
|
-
|
|
1343
|
-
|
|
1344
|
-
|
|
1345
|
-
|
|
1346
|
-
|
|
1347
|
-
|
|
1348
|
-
|
|
1349
|
-
|
|
1350
|
-
|
|
1351
|
-
unique_id ASC
|
|
1352
|
-
LIMIT ? OFFSET ?
|
|
1353
|
-
`,
|
|
1339
|
+
SELECT
|
|
1340
|
+
unique_id, nickname, sec_uid, status, sources,
|
|
1341
|
+
tt_seller, verified, follower_count, following_count,
|
|
1342
|
+
location_created, latest_video_time, refresh_time,
|
|
1343
|
+
guessed_location, pinned, processed_at, video_count,
|
|
1344
|
+
no_video, claimed_by, claimed_at, created_at, updated_at
|
|
1345
|
+
FROM jobs
|
|
1346
|
+
${whereSql}
|
|
1347
|
+
ORDER BY
|
|
1348
|
+
pinned DESC,
|
|
1349
|
+
CASE
|
|
1350
|
+
WHEN ? = 'done' THEN COALESCE(processed_at, 0) * -1
|
|
1351
|
+
WHEN ? = '1' THEN COALESCE(refresh_time, 0) * -1
|
|
1352
|
+
ELSE 0
|
|
1353
|
+
END ASC,
|
|
1354
|
+
CASE status
|
|
1355
|
+
WHEN 'processing' THEN 0
|
|
1356
|
+
WHEN 'pending' THEN 1
|
|
1357
|
+
WHEN 'done' THEN 2
|
|
1358
|
+
WHEN 'error' THEN 3
|
|
1359
|
+
WHEN 'restricted' THEN 4
|
|
1360
|
+
ELSE 9
|
|
1361
|
+
END ASC,
|
|
1362
|
+
COALESCE(follower_count, 0) DESC,
|
|
1363
|
+
COALESCE(processed_at, 0) DESC,
|
|
1364
|
+
unique_id ASC
|
|
1365
|
+
LIMIT ? OFFSET ?
|
|
1366
|
+
`,
|
|
1354
1367
|
)
|
|
1355
|
-
.all(...args, safeLimit, safeOffset)
|
|
1368
|
+
.all(...args, status || "", target, safeLimit, safeOffset)
|
|
1356
1369
|
.map(mapJobRow);
|
|
1357
1370
|
|
|
1358
1371
|
return {
|
|
@@ -1371,18 +1384,12 @@ function getTargetUsersFromDb(targetLocations = []) {
|
|
|
1371
1384
|
const rows = db
|
|
1372
1385
|
.prepare(
|
|
1373
1386
|
`
|
|
1374
|
-
|
|
1375
|
-
|
|
1376
|
-
|
|
1377
|
-
|
|
1378
|
-
|
|
1379
|
-
|
|
1380
|
-
location_created,
|
|
1381
|
-
latest_video_time,
|
|
1382
|
-
status,
|
|
1383
|
-
sources
|
|
1384
|
-
FROM jobs
|
|
1385
|
-
WHERE tt_seller = 1
|
|
1387
|
+
SELECT
|
|
1388
|
+
unique_id, nickname, sec_uid, status, sources,
|
|
1389
|
+
tt_seller, verified, follower_count, following_count,
|
|
1390
|
+
location_created, latest_video_time, refresh_time,
|
|
1391
|
+
guessed_location, pinned, processed_at, video_count,
|
|
1392
|
+
no_video, claimed_by, claimed_at, created_at, updated_at
|
|
1386
1393
|
AND verified = 0
|
|
1387
1394
|
AND location_created IN (${placeholders})
|
|
1388
1395
|
ORDER BY COALESCE(follower_count, 0) DESC, unique_id ASC
|
|
@@ -1397,13 +1404,109 @@ function getTargetUsersFromDb(targetLocations = []) {
|
|
|
1397
1404
|
};
|
|
1398
1405
|
}
|
|
1399
1406
|
|
|
1400
|
-
function getTargetUsersByCountryFromDb(targetLocations = []) {
|
|
1407
|
+
function getTargetUsersByCountryFromDb(targetLocations = [], options = {}) {
|
|
1401
1408
|
if (!db) return null;
|
|
1402
1409
|
if (!targetLocations.length) {
|
|
1403
1410
|
return { countries: [] };
|
|
1404
1411
|
}
|
|
1405
1412
|
|
|
1413
|
+
const {
|
|
1414
|
+
summaryOnly = false,
|
|
1415
|
+
country: filterCountry,
|
|
1416
|
+
search,
|
|
1417
|
+
limit,
|
|
1418
|
+
offset,
|
|
1419
|
+
} = options;
|
|
1406
1420
|
const placeholders = targetLocations.map(() => "?").join(", ");
|
|
1421
|
+
const baseParams = [...targetLocations];
|
|
1422
|
+
|
|
1423
|
+
// 摘要模式:只返回各国统计数,不返回用户数据
|
|
1424
|
+
if (summaryOnly) {
|
|
1425
|
+
const statsRows = db
|
|
1426
|
+
.prepare(
|
|
1427
|
+
`
|
|
1428
|
+
SELECT location_created as country, COUNT(*) as count
|
|
1429
|
+
FROM jobs
|
|
1430
|
+
WHERE tt_seller = 1
|
|
1431
|
+
AND verified = 0
|
|
1432
|
+
AND location_created IN (${placeholders})
|
|
1433
|
+
GROUP BY location_created
|
|
1434
|
+
ORDER BY count DESC
|
|
1435
|
+
`,
|
|
1436
|
+
)
|
|
1437
|
+
.all(...targetLocations);
|
|
1438
|
+
|
|
1439
|
+
const countries = statsRows.map((r) => ({
|
|
1440
|
+
country: r.country,
|
|
1441
|
+
count: r.count,
|
|
1442
|
+
users: undefined,
|
|
1443
|
+
}));
|
|
1444
|
+
return {
|
|
1445
|
+
total: statsRows.reduce((s, r) => s + r.count, 0),
|
|
1446
|
+
countries,
|
|
1447
|
+
};
|
|
1448
|
+
}
|
|
1449
|
+
|
|
1450
|
+
// 分页模式:按国家或全局分页查询用户
|
|
1451
|
+
if (limit !== undefined) {
|
|
1452
|
+
let sql = `
|
|
1453
|
+
SELECT
|
|
1454
|
+
unique_id,
|
|
1455
|
+
nickname,
|
|
1456
|
+
follower_count,
|
|
1457
|
+
video_count,
|
|
1458
|
+
tt_seller,
|
|
1459
|
+
verified,
|
|
1460
|
+
location_created,
|
|
1461
|
+
confirmed_location,
|
|
1462
|
+
modified_at,
|
|
1463
|
+
latest_video_time,
|
|
1464
|
+
refresh_time,
|
|
1465
|
+
status,
|
|
1466
|
+
sources
|
|
1467
|
+
FROM jobs
|
|
1468
|
+
WHERE tt_seller = 1
|
|
1469
|
+
AND verified = 0
|
|
1470
|
+
AND location_created IN (${placeholders})
|
|
1471
|
+
`;
|
|
1472
|
+
const params = [...targetLocations];
|
|
1473
|
+
|
|
1474
|
+
if (filterCountry) {
|
|
1475
|
+
sql += ` AND location_created = ?`;
|
|
1476
|
+
params.push(filterCountry);
|
|
1477
|
+
}
|
|
1478
|
+
|
|
1479
|
+
if (search) {
|
|
1480
|
+
sql += ` AND (unique_id LIKE ? OR nickname LIKE ?)`;
|
|
1481
|
+
const likeSearch = `%${search}%`;
|
|
1482
|
+
params.push(likeSearch, likeSearch);
|
|
1483
|
+
}
|
|
1484
|
+
|
|
1485
|
+
sql += ` ORDER BY location_created ASC, COALESCE(latest_video_time, 0) DESC`;
|
|
1486
|
+
|
|
1487
|
+
const countSql = sql.replace(
|
|
1488
|
+
/SELECT[^FROM]*FROM/,
|
|
1489
|
+
"SELECT COUNT(*) as cnt FROM",
|
|
1490
|
+
);
|
|
1491
|
+
const total = db.prepare(countSql).get(...params)?.cnt || 0;
|
|
1492
|
+
|
|
1493
|
+
sql += ` LIMIT ? OFFSET ?`;
|
|
1494
|
+
const safeLimit = Math.min(Math.floor(limit), 10000);
|
|
1495
|
+
const safeOffset = Math.max(Math.floor(offset), 0);
|
|
1496
|
+
|
|
1497
|
+
const rows = db
|
|
1498
|
+
.prepare(sql)
|
|
1499
|
+
.all(...params, safeLimit, safeOffset)
|
|
1500
|
+
.map(mapJobRow);
|
|
1501
|
+
|
|
1502
|
+
return {
|
|
1503
|
+
total,
|
|
1504
|
+
limit: safeLimit,
|
|
1505
|
+
offset: safeOffset,
|
|
1506
|
+
users: rows,
|
|
1507
|
+
};
|
|
1508
|
+
}
|
|
1509
|
+
|
|
1407
1510
|
const rows = db
|
|
1408
1511
|
.prepare(
|
|
1409
1512
|
`
|
|
@@ -1415,6 +1518,8 @@ function getTargetUsersByCountryFromDb(targetLocations = []) {
|
|
|
1415
1518
|
tt_seller,
|
|
1416
1519
|
verified,
|
|
1417
1520
|
location_created,
|
|
1521
|
+
confirmed_location,
|
|
1522
|
+
modified_at,
|
|
1418
1523
|
latest_video_time,
|
|
1419
1524
|
refresh_time,
|
|
1420
1525
|
status,
|
|
@@ -1490,6 +1595,8 @@ const writableJobColumns = new Set([
|
|
|
1490
1595
|
"comment_count",
|
|
1491
1596
|
"guessed_location",
|
|
1492
1597
|
"location_created",
|
|
1598
|
+
"confirmed_location",
|
|
1599
|
+
"modified_at",
|
|
1493
1600
|
"follower_count",
|
|
1494
1601
|
"following_count",
|
|
1495
1602
|
"heart_count",
|
|
@@ -1499,6 +1606,7 @@ const writableJobColumns = new Set([
|
|
|
1499
1606
|
"updated_at",
|
|
1500
1607
|
"region",
|
|
1501
1608
|
"signature",
|
|
1609
|
+
"bio_link",
|
|
1502
1610
|
"sec_uid",
|
|
1503
1611
|
"status_code",
|
|
1504
1612
|
"latest_video_time",
|
|
@@ -1587,7 +1695,9 @@ function updateJobInfo(uniqueId, info, incrementCount = true) {
|
|
|
1587
1695
|
for (const [key, value] of Object.entries(info || {})) {
|
|
1588
1696
|
if (key === "uniqueId" || key === "unique_id") continue;
|
|
1589
1697
|
if (value === undefined || value === "") continue;
|
|
1590
|
-
|
|
1698
|
+
let column = camelToSnake(key);
|
|
1699
|
+
// 字段别名:bio → signature
|
|
1700
|
+
if (column === "bio") column = "signature";
|
|
1591
1701
|
if (!writableJobColumns.has(column)) continue;
|
|
1592
1702
|
nextValues[column] = normalizeJobValue(column, value);
|
|
1593
1703
|
}
|
|
@@ -2754,7 +2864,9 @@ export function createStore(filePath) {
|
|
|
2754
2864
|
user.ttSeller = result.userInfo?.ttSeller ?? user.ttSeller;
|
|
2755
2865
|
user.verified = result.userInfo?.verified ?? user.verified;
|
|
2756
2866
|
user.region = result.userInfo?.region || user.region;
|
|
2757
|
-
user.signature =
|
|
2867
|
+
user.signature =
|
|
2868
|
+
result.userInfo?.signature ?? result.userInfo?.bio ?? user.signature;
|
|
2869
|
+
user.bioLink = result.userInfo?.bioLink ?? user.bioLink;
|
|
2758
2870
|
user.followingCount =
|
|
2759
2871
|
result.userInfo?.followingCount ?? user.followingCount;
|
|
2760
2872
|
user.heartCount = result.userInfo?.heartCount ?? user.heartCount;
|
|
@@ -3201,39 +3313,161 @@ export function createStore(filePath) {
|
|
|
3201
3313
|
return { ok: true, userUpdateCount: user.userUpdateCount };
|
|
3202
3314
|
}
|
|
3203
3315
|
|
|
3316
|
+
function updateUserLocation(uniqueId, location) {
|
|
3317
|
+
if (db) {
|
|
3318
|
+
const existing = db
|
|
3319
|
+
.prepare("SELECT * FROM jobs WHERE unique_id = ?")
|
|
3320
|
+
.get(uniqueId);
|
|
3321
|
+
if (!existing) return { error: "user not found" };
|
|
3322
|
+
const now = Date.now();
|
|
3323
|
+
db.prepare(
|
|
3324
|
+
"UPDATE jobs SET location_created = ?, modified_at = ?, updated_at = ? WHERE unique_id = ?",
|
|
3325
|
+
).run(location, now, now, uniqueId);
|
|
3326
|
+
return { ok: true, location, modifiedAt: now };
|
|
3327
|
+
}
|
|
3328
|
+
|
|
3329
|
+
const user = getUser(uniqueId);
|
|
3330
|
+
if (!user) return { error: "user not found" };
|
|
3331
|
+
user.locationCreated = location;
|
|
3332
|
+
user.modifiedAt = Date.now();
|
|
3333
|
+
user.updatedAt = Date.now();
|
|
3334
|
+
user.userUpdateCount = (user.userUpdateCount || 0) + 1;
|
|
3335
|
+
save();
|
|
3336
|
+
return { ok: true, location, modifiedAt: user.modifiedAt };
|
|
3337
|
+
}
|
|
3338
|
+
|
|
3339
|
+
// 将单个 job 移动到 raw_jobs 表(完整字段复制 + 删除原记录)
|
|
3340
|
+
function moveJobToRaw(uniqueId) {
|
|
3341
|
+
if (!db) return false;
|
|
3342
|
+
const safeId = String(uniqueId).trim();
|
|
3343
|
+
if (!safeId) return false;
|
|
3344
|
+
|
|
3345
|
+
const moveSingleTxn = db.transaction(() => {
|
|
3346
|
+
db.prepare(
|
|
3347
|
+
`
|
|
3348
|
+
INSERT OR REPLACE INTO raw_jobs (
|
|
3349
|
+
unique_id, nickname, status, sources, claimed_by, claimed_at,
|
|
3350
|
+
error, pinned, no_video, restricted, user_update_count,
|
|
3351
|
+
tt_seller, verified, video_count, comment_count,
|
|
3352
|
+
guessed_location, location_created, confirmed_location, modified_at,
|
|
3353
|
+
follower_count, following_count, heart_count, refresh_time,
|
|
3354
|
+
processed, processed_at, created_at, updated_at,
|
|
3355
|
+
region, signature, bio_link, sec_uid, status_code, latest_video_time
|
|
3356
|
+
)
|
|
3357
|
+
SELECT
|
|
3358
|
+
unique_id, nickname, status, sources, claimed_by, claimed_at,
|
|
3359
|
+
error, pinned, no_video, restricted, user_update_count,
|
|
3360
|
+
tt_seller, verified, video_count, comment_count,
|
|
3361
|
+
guessed_location, location_created, confirmed_location, modified_at,
|
|
3362
|
+
follower_count, following_count, heart_count, refresh_time,
|
|
3363
|
+
processed, processed_at, created_at, updated_at,
|
|
3364
|
+
region, signature, bio_link, sec_uid, status_code, latest_video_time
|
|
3365
|
+
FROM jobs WHERE unique_id = ?
|
|
3366
|
+
`,
|
|
3367
|
+
).run(safeId);
|
|
3368
|
+
|
|
3369
|
+
db.prepare("DELETE FROM jobs WHERE unique_id = ?").run(safeId);
|
|
3370
|
+
});
|
|
3371
|
+
moveSingleTxn();
|
|
3372
|
+
return true;
|
|
3373
|
+
}
|
|
3374
|
+
|
|
3204
3375
|
function batchUpdateUserInfo(updates) {
|
|
3205
3376
|
if (db) {
|
|
3206
|
-
const
|
|
3207
|
-
|
|
3377
|
+
const results = [];
|
|
3378
|
+
const moveList = [];
|
|
3379
|
+
|
|
3380
|
+
const txn = db.transaction((items) => {
|
|
3381
|
+
items.forEach((item) => {
|
|
3382
|
+
const uniqueId = item.uniqueId;
|
|
3208
3383
|
// 处理 { error: true, statusCode: xxx } 的情况
|
|
3209
3384
|
const info = item.info;
|
|
3385
|
+
let updateResult;
|
|
3210
3386
|
if (info && info.error && info.statusCode !== undefined) {
|
|
3211
3387
|
// 只更新 status_code,不更新其他字段
|
|
3212
|
-
|
|
3213
|
-
|
|
3388
|
+
updateResult = updateJobInfo(
|
|
3389
|
+
uniqueId,
|
|
3214
3390
|
{ statusCode: info.statusCode },
|
|
3215
3391
|
true,
|
|
3216
3392
|
);
|
|
3393
|
+
} else {
|
|
3394
|
+
updateResult = updateJobInfo(uniqueId, info, true);
|
|
3217
3395
|
}
|
|
3218
|
-
|
|
3219
|
-
|
|
3220
|
-
|
|
3221
|
-
|
|
3222
|
-
|
|
3223
|
-
|
|
3224
|
-
|
|
3225
|
-
|
|
3396
|
+
|
|
3397
|
+
if (updateResult.error) {
|
|
3398
|
+
results.push({ uniqueId, error: updateResult.error });
|
|
3399
|
+
return;
|
|
3400
|
+
}
|
|
3401
|
+
|
|
3402
|
+
// 检查 tt_seller:非商家则标记为需要移动到毛料表
|
|
3403
|
+
const row = getJobRow(uniqueId);
|
|
3404
|
+
const ttSeller = row ? row.tt_seller : null;
|
|
3405
|
+
if (ttSeller) {
|
|
3406
|
+
// 商家:保持当前逻辑
|
|
3407
|
+
results.push({
|
|
3408
|
+
uniqueId,
|
|
3226
3409
|
ok: true,
|
|
3227
|
-
userUpdateCount:
|
|
3228
|
-
}
|
|
3229
|
-
|
|
3410
|
+
userUpdateCount: updateResult.userUpdateCount,
|
|
3411
|
+
});
|
|
3412
|
+
} else {
|
|
3413
|
+
// 非商家:标记移动
|
|
3414
|
+
results.push({
|
|
3415
|
+
uniqueId,
|
|
3416
|
+
ok: true,
|
|
3417
|
+
userUpdateCount: updateResult.userUpdateCount,
|
|
3418
|
+
_movedToRaw: true,
|
|
3419
|
+
});
|
|
3420
|
+
moveList.push(uniqueId);
|
|
3421
|
+
}
|
|
3422
|
+
});
|
|
3423
|
+
});
|
|
3424
|
+
txn(updates);
|
|
3425
|
+
|
|
3426
|
+
// 在事务外执行移动操作(避免嵌套 transaction 问题)
|
|
3427
|
+
if (moveList.length > 0) {
|
|
3428
|
+
const moveTxn = db.transaction((ids) => {
|
|
3429
|
+
ids.forEach((uid) => {
|
|
3430
|
+
db.prepare(
|
|
3431
|
+
`
|
|
3432
|
+
INSERT OR REPLACE INTO raw_jobs (
|
|
3433
|
+
unique_id, nickname, status, sources, claimed_by, claimed_at,
|
|
3434
|
+
error, pinned, no_video, restricted, user_update_count,
|
|
3435
|
+
tt_seller, verified, video_count, comment_count,
|
|
3436
|
+
guessed_location, location_created, confirmed_location, modified_at,
|
|
3437
|
+
follower_count, following_count, heart_count, refresh_time,
|
|
3438
|
+
processed, processed_at, created_at, updated_at,
|
|
3439
|
+
region, signature, bio_link, sec_uid, status_code, latest_video_time
|
|
3440
|
+
)
|
|
3441
|
+
SELECT
|
|
3442
|
+
unique_id, nickname, status, sources, claimed_by, claimed_at,
|
|
3443
|
+
error, pinned, no_video, restricted, user_update_count,
|
|
3444
|
+
tt_seller, verified, video_count, comment_count,
|
|
3445
|
+
guessed_location, location_created, confirmed_location, modified_at,
|
|
3446
|
+
follower_count, following_count, heart_count, refresh_time,
|
|
3447
|
+
processed, processed_at, created_at, updated_at,
|
|
3448
|
+
region, signature, bio_link, sec_uid, status_code, latest_video_time
|
|
3449
|
+
FROM jobs WHERE unique_id = ?
|
|
3450
|
+
`,
|
|
3451
|
+
).run(uid);
|
|
3452
|
+
|
|
3453
|
+
db.prepare("DELETE FROM jobs WHERE unique_id = ?").run(uid);
|
|
3454
|
+
});
|
|
3455
|
+
});
|
|
3456
|
+
moveTxn(moveList);
|
|
3457
|
+
}
|
|
3458
|
+
|
|
3459
|
+
// 清理内部标记
|
|
3460
|
+
return results.map((r) => {
|
|
3461
|
+
const { _movedToRaw, ...rest } = r;
|
|
3462
|
+
return rest;
|
|
3463
|
+
});
|
|
3230
3464
|
}
|
|
3231
3465
|
|
|
3232
|
-
const
|
|
3466
|
+
const memResults = [];
|
|
3233
3467
|
for (const item of updates) {
|
|
3234
3468
|
const user = getUser(item.uniqueId);
|
|
3235
3469
|
if (!user) {
|
|
3236
|
-
|
|
3470
|
+
memResults.push({ uniqueId: item.uniqueId, error: "user not found" });
|
|
3237
3471
|
continue;
|
|
3238
3472
|
}
|
|
3239
3473
|
const info = item.info;
|
|
@@ -3255,14 +3489,14 @@ export function createStore(filePath) {
|
|
|
3255
3489
|
}
|
|
3256
3490
|
user.userUpdateCount = (user.userUpdateCount || 0) + 1;
|
|
3257
3491
|
user.updatedAt = Date.now();
|
|
3258
|
-
|
|
3492
|
+
memResults.push({
|
|
3259
3493
|
uniqueId: item.uniqueId,
|
|
3260
3494
|
ok: true,
|
|
3261
3495
|
userUpdateCount: user.userUpdateCount,
|
|
3262
3496
|
});
|
|
3263
3497
|
}
|
|
3264
3498
|
save();
|
|
3265
|
-
return
|
|
3499
|
+
return memResults;
|
|
3266
3500
|
}
|
|
3267
3501
|
|
|
3268
3502
|
// 视频登记
|
|
@@ -3350,7 +3584,7 @@ export function createStore(filePath) {
|
|
|
3350
3584
|
}
|
|
3351
3585
|
|
|
3352
3586
|
function getVideosPage(limit, offset) {
|
|
3353
|
-
const safeLimit = Math.max(1, Math.min(
|
|
3587
|
+
const safeLimit = Math.max(1, Math.min(100, parseInt(limit) || 50));
|
|
3354
3588
|
const safeOffset = Math.max(0, parseInt(offset) || 0);
|
|
3355
3589
|
|
|
3356
3590
|
if (db) {
|
|
@@ -3503,6 +3737,7 @@ export function createStore(filePath) {
|
|
|
3503
3737
|
commitRedoJob,
|
|
3504
3738
|
getPendingUserUpdateTasks,
|
|
3505
3739
|
updateUserInfo,
|
|
3740
|
+
updateUserLocation,
|
|
3506
3741
|
batchUpdateUserInfo,
|
|
3507
3742
|
reportClientError,
|
|
3508
3743
|
deleteClientError,
|