tt-help-cli-ycl 1.3.83 → 1.3.85
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/scripts/test-refill-order.mjs +218 -0
- package/src/cli/attach.js +3 -34
- package/src/cli/auto.js +3 -18
- package/src/cli/comments.js +13 -57
- package/src/cli/explore.js +255 -266
- package/src/cli/refresh.js +6 -21
- package/src/cli/tag.js +712 -0
- package/src/lib/api-client.js +101 -0
- package/src/lib/args.js +182 -6
- package/src/lib/constants.js +43 -0
- package/src/lib/parse-ssr.mjs +1 -0
- package/src/lib/tag-discover.js +124 -0
- package/src/lib/tag-fetcher.js +296 -0
- package/src/lib/target-locations.js +18 -0
- package/src/main.js +14 -0
- package/src/npm-main.js +3 -0
- package/src/scraper/explore-core.js +6 -6
- package/src/watch/data-store.js +304 -49
- package/src/watch/public/app.js +95 -0
- package/src/watch/public/index.html +15 -0
- package/src/watch/public/style.css +107 -0
- package/src/watch/server.js +185 -0
- package/src/watch/tag-service.js +334 -0
package/src/watch/data-store.js
CHANGED
|
@@ -152,6 +152,9 @@ function initUserDb(filePath) {
|
|
|
152
152
|
if (!existingJobColumns.has("top_video_href")) {
|
|
153
153
|
db.exec(`ALTER TABLE jobs ADD COLUMN top_video_href TEXT`);
|
|
154
154
|
}
|
|
155
|
+
if (!existingJobColumns.has("user_create_time")) {
|
|
156
|
+
db.exec(`ALTER TABLE jobs ADD COLUMN user_create_time INTEGER`);
|
|
157
|
+
}
|
|
155
158
|
db.exec(`
|
|
156
159
|
CREATE TABLE IF NOT EXISTS jobs_base (
|
|
157
160
|
unique_id TEXT PRIMARY KEY,
|
|
@@ -212,6 +215,9 @@ function initUserDb(filePath) {
|
|
|
212
215
|
if (!existingJobBaseColumns.has("bio_link")) {
|
|
213
216
|
db.exec(`ALTER TABLE jobs_base ADD COLUMN bio_link TEXT`);
|
|
214
217
|
}
|
|
218
|
+
if (!existingJobBaseColumns.has("user_create_time")) {
|
|
219
|
+
db.exec(`ALTER TABLE jobs_base ADD COLUMN user_create_time INTEGER`);
|
|
220
|
+
}
|
|
215
221
|
db.exec(`
|
|
216
222
|
CREATE TABLE IF NOT EXISTS raw_jobs (
|
|
217
223
|
unique_id TEXT PRIMARY KEY,
|
|
@@ -271,6 +277,9 @@ function initUserDb(filePath) {
|
|
|
271
277
|
if (!existingRawJobColumns.has("bio_link")) {
|
|
272
278
|
db.exec(`ALTER TABLE raw_jobs ADD COLUMN bio_link TEXT`);
|
|
273
279
|
}
|
|
280
|
+
if (!existingRawJobColumns.has("user_create_time")) {
|
|
281
|
+
db.exec(`ALTER TABLE raw_jobs ADD COLUMN user_create_time INTEGER`);
|
|
282
|
+
}
|
|
274
283
|
db.exec(`
|
|
275
284
|
CREATE TABLE IF NOT EXISTS videos (
|
|
276
285
|
id TEXT PRIMARY KEY,
|
|
@@ -384,6 +393,30 @@ function initUserDb(filePath) {
|
|
|
384
393
|
db.exec(`ALTER TABLE videos ADD COLUMN create_time INTEGER`);
|
|
385
394
|
}
|
|
386
395
|
|
|
396
|
+
// tags 表:标签发现与打分系统
|
|
397
|
+
db.exec(`
|
|
398
|
+
CREATE TABLE IF NOT EXISTS tags (
|
|
399
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
400
|
+
tag TEXT NOT NULL UNIQUE,
|
|
401
|
+
status TEXT NOT NULL DEFAULT 'new',
|
|
402
|
+
score REAL NOT NULL DEFAULT 0,
|
|
403
|
+
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
404
|
+
scored_at TEXT,
|
|
405
|
+
score_count INTEGER NOT NULL DEFAULT 0,
|
|
406
|
+
countries TEXT NOT NULL DEFAULT '[]',
|
|
407
|
+
matched_countries TEXT DEFAULT '[]',
|
|
408
|
+
total_posts INTEGER DEFAULT 0,
|
|
409
|
+
author_count INTEGER DEFAULT 0,
|
|
410
|
+
matched_authors INTEGER DEFAULT 0,
|
|
411
|
+
pushed_users INTEGER DEFAULT 0,
|
|
412
|
+
source TEXT NOT NULL DEFAULT 'llm',
|
|
413
|
+
user_prompt TEXT,
|
|
414
|
+
last_error TEXT
|
|
415
|
+
)
|
|
416
|
+
`);
|
|
417
|
+
db.exec(`CREATE INDEX IF NOT EXISTS idx_tags_status ON tags(status)`);
|
|
418
|
+
db.exec(`CREATE INDEX IF NOT EXISTS idx_tags_score ON tags(score DESC)`);
|
|
419
|
+
|
|
387
420
|
const count = db.prepare("SELECT COUNT(*) as c FROM users").get().c;
|
|
388
421
|
console.log(`[data-store] SQLite users 表初始化完成: ${count} 条`);
|
|
389
422
|
}
|
|
@@ -956,7 +989,7 @@ function moveJobsToRawByCountry(scope, country) {
|
|
|
956
989
|
guessed_location, location_created, follower_count,
|
|
957
990
|
following_count, heart_count, refresh_time, processed,
|
|
958
991
|
processed_at, created_at, updated_at, region, signature,
|
|
959
|
-
sec_uid, latest_video_time
|
|
992
|
+
sec_uid, latest_video_time, user_create_time
|
|
960
993
|
`;
|
|
961
994
|
} else if (normalizedScope === "userUpdate") {
|
|
962
995
|
sourceTable = "jobs_base";
|
|
@@ -968,7 +1001,7 @@ function moveJobsToRawByCountry(scope, country) {
|
|
|
968
1001
|
guessed_location, location_created, follower_count,
|
|
969
1002
|
following_count, heart_count, refresh_time, processed,
|
|
970
1003
|
processed_at, created_at, updated_at, region, signature,
|
|
971
|
-
sec_uid, latest_video_time
|
|
1004
|
+
sec_uid, latest_video_time, user_create_time
|
|
972
1005
|
`;
|
|
973
1006
|
} else {
|
|
974
1007
|
return {
|
|
@@ -1307,6 +1340,158 @@ function getRawJobsPageFromDb({
|
|
|
1307
1340
|
};
|
|
1308
1341
|
}
|
|
1309
1342
|
|
|
1343
|
+
// ====== Tag 发现与打分 CRUD ======
|
|
1344
|
+
|
|
1345
|
+
function insertTag(tag, countries, source = "llm") {
|
|
1346
|
+
if (!db) return { inserted: false, error: "db not ready" };
|
|
1347
|
+
try {
|
|
1348
|
+
const result = db
|
|
1349
|
+
.prepare(
|
|
1350
|
+
`
|
|
1351
|
+
INSERT OR IGNORE INTO tags (tag, countries, source)
|
|
1352
|
+
VALUES (?, ?, ?)
|
|
1353
|
+
`,
|
|
1354
|
+
)
|
|
1355
|
+
.run(tag, JSON.stringify(countries), source);
|
|
1356
|
+
return { inserted: result.changes > 0, tag };
|
|
1357
|
+
} catch (e) {
|
|
1358
|
+
return { inserted: false, error: e.message };
|
|
1359
|
+
}
|
|
1360
|
+
}
|
|
1361
|
+
|
|
1362
|
+
function getTagsByStatus(status, limit = 100) {
|
|
1363
|
+
if (!db) return [];
|
|
1364
|
+
const rows = db
|
|
1365
|
+
.prepare(
|
|
1366
|
+
`
|
|
1367
|
+
SELECT * FROM tags WHERE status = ? ORDER BY score ASC, created_at ASC LIMIT ?
|
|
1368
|
+
`,
|
|
1369
|
+
)
|
|
1370
|
+
.all(status, limit);
|
|
1371
|
+
return rows.map((r) => ({
|
|
1372
|
+
...r,
|
|
1373
|
+
countries: JSON.parse(r.countries || "[]"),
|
|
1374
|
+
matched_countries: JSON.parse(r.matched_countries || "[]"),
|
|
1375
|
+
}));
|
|
1376
|
+
}
|
|
1377
|
+
|
|
1378
|
+
function getTagsByCountry(country, minScore = 0) {
|
|
1379
|
+
if (!db) return [];
|
|
1380
|
+
const rows = db
|
|
1381
|
+
.prepare(
|
|
1382
|
+
`
|
|
1383
|
+
SELECT * FROM tags WHERE status != 'dead'
|
|
1384
|
+
ORDER BY score DESC
|
|
1385
|
+
`,
|
|
1386
|
+
)
|
|
1387
|
+
.all();
|
|
1388
|
+
// Filter in JS since countries is JSON
|
|
1389
|
+
return rows
|
|
1390
|
+
.map((r) => ({
|
|
1391
|
+
...r,
|
|
1392
|
+
countries: JSON.parse(r.countries || "[]"),
|
|
1393
|
+
matched_countries: JSON.parse(r.matched_countries || "[]"),
|
|
1394
|
+
}))
|
|
1395
|
+
.filter((r) => r.countries.includes(country) && r.score >= minScore);
|
|
1396
|
+
}
|
|
1397
|
+
|
|
1398
|
+
function getDeadTags(country) {
|
|
1399
|
+
if (!db) return [];
|
|
1400
|
+
const rows = db
|
|
1401
|
+
.prepare(
|
|
1402
|
+
`
|
|
1403
|
+
SELECT * FROM tags WHERE status = 'dead' ORDER BY score ASC
|
|
1404
|
+
`,
|
|
1405
|
+
)
|
|
1406
|
+
.all();
|
|
1407
|
+
return rows
|
|
1408
|
+
.map((r) => ({
|
|
1409
|
+
...r,
|
|
1410
|
+
countries: JSON.parse(r.countries || "[]"),
|
|
1411
|
+
matched_countries: JSON.parse(r.matched_countries || "[]"),
|
|
1412
|
+
}))
|
|
1413
|
+
.filter((r) => r.countries.includes(country));
|
|
1414
|
+
}
|
|
1415
|
+
|
|
1416
|
+
function claimTag(tag) {
|
|
1417
|
+
if (!db) return { ok: false, error: "db not ready" };
|
|
1418
|
+
const row = db.prepare("SELECT status FROM tags WHERE tag = ?").get(tag);
|
|
1419
|
+
if (!row) return { ok: false, error: "tag not found" };
|
|
1420
|
+
if (row.status !== "new")
|
|
1421
|
+
return { ok: false, error: `tag status is ${row.status}, not new` };
|
|
1422
|
+
db.prepare("UPDATE tags SET status = 'scoring' WHERE tag = ?").run(tag);
|
|
1423
|
+
return { ok: true, tag, previousStatus: row.status };
|
|
1424
|
+
}
|
|
1425
|
+
|
|
1426
|
+
function reportTagScore(tag, fields) {
|
|
1427
|
+
if (!db) return { ok: false, error: "db not ready" };
|
|
1428
|
+
const {
|
|
1429
|
+
score,
|
|
1430
|
+
status,
|
|
1431
|
+
totalPosts,
|
|
1432
|
+
authorCount,
|
|
1433
|
+
matchedAuthors,
|
|
1434
|
+
matchedCountries,
|
|
1435
|
+
pushedUsers,
|
|
1436
|
+
error,
|
|
1437
|
+
} = fields;
|
|
1438
|
+
const matchedCountriesJson = matchedCountries
|
|
1439
|
+
? JSON.stringify(matchedCountries)
|
|
1440
|
+
: null;
|
|
1441
|
+
const now = new Date().toISOString();
|
|
1442
|
+
|
|
1443
|
+
try {
|
|
1444
|
+
const result = db
|
|
1445
|
+
.prepare(
|
|
1446
|
+
`
|
|
1447
|
+
UPDATE tags SET
|
|
1448
|
+
score = COALESCE(?, score),
|
|
1449
|
+
status = COALESCE(?, status),
|
|
1450
|
+
total_posts = COALESCE(?, total_posts),
|
|
1451
|
+
author_count = COALESCE(?, author_count),
|
|
1452
|
+
matched_authors = COALESCE(?, matched_authors),
|
|
1453
|
+
matched_countries = COALESCE(?, matched_countries),
|
|
1454
|
+
pushed_users = COALESCE(?, pushed_users),
|
|
1455
|
+
last_error = COALESCE(?, last_error),
|
|
1456
|
+
scored_at = ?,
|
|
1457
|
+
score_count = score_count + 1
|
|
1458
|
+
WHERE tag = ?
|
|
1459
|
+
`,
|
|
1460
|
+
)
|
|
1461
|
+
.run(
|
|
1462
|
+
score ?? null,
|
|
1463
|
+
status ?? null,
|
|
1464
|
+
totalPosts ?? null,
|
|
1465
|
+
authorCount ?? null,
|
|
1466
|
+
matchedAuthors ?? null,
|
|
1467
|
+
matchedCountriesJson,
|
|
1468
|
+
pushedUsers ?? null,
|
|
1469
|
+
error ?? null,
|
|
1470
|
+
now,
|
|
1471
|
+
tag,
|
|
1472
|
+
);
|
|
1473
|
+
return { ok: result.changes > 0, tag };
|
|
1474
|
+
} catch (e) {
|
|
1475
|
+
return { ok: false, error: e.message };
|
|
1476
|
+
}
|
|
1477
|
+
}
|
|
1478
|
+
|
|
1479
|
+
function getAllTags(limit = 200) {
|
|
1480
|
+
if (!db) return [];
|
|
1481
|
+
const rows = db
|
|
1482
|
+
.prepare(
|
|
1483
|
+
`
|
|
1484
|
+
SELECT * FROM tags ORDER BY score DESC, created_at DESC LIMIT ?
|
|
1485
|
+
`,
|
|
1486
|
+
)
|
|
1487
|
+
.all(limit);
|
|
1488
|
+
return rows.map((r) => ({
|
|
1489
|
+
...r,
|
|
1490
|
+
countries: JSON.parse(r.countries || "[]"),
|
|
1491
|
+
matched_countries: JSON.parse(r.matched_countries || "[]"),
|
|
1492
|
+
}));
|
|
1493
|
+
}
|
|
1494
|
+
|
|
1310
1495
|
// 调试接口:直接执行 SQL 查询,返回原始数据
|
|
1311
1496
|
function rawQuery(sql, params = []) {
|
|
1312
1497
|
if (!db) return { error: "db not ready" };
|
|
@@ -1668,6 +1853,7 @@ const writableJobColumns = new Set([
|
|
|
1668
1853
|
"latest_video_time",
|
|
1669
1854
|
"top_video_play_count",
|
|
1670
1855
|
"top_video_href",
|
|
1856
|
+
"user_create_time",
|
|
1671
1857
|
]);
|
|
1672
1858
|
|
|
1673
1859
|
function normalizeJobValue(column, value) {
|
|
@@ -1761,8 +1947,9 @@ function updateJobInfo(uniqueId, info, incrementCount = true) {
|
|
|
1761
1947
|
if (key === "uniqueId" || key === "unique_id") continue;
|
|
1762
1948
|
if (value === undefined || value === "") continue;
|
|
1763
1949
|
let column = camelToSnake(key);
|
|
1764
|
-
// 字段别名:bio → signature
|
|
1950
|
+
// 字段别名:bio → signature, createTime → user_create_time
|
|
1765
1951
|
if (column === "bio") column = "signature";
|
|
1952
|
+
if (column === "create_time") column = "user_create_time";
|
|
1766
1953
|
if (!writableJobColumns.has(column)) continue;
|
|
1767
1954
|
nextValues[column] = normalizeJobValue(column, value);
|
|
1768
1955
|
}
|
|
@@ -1805,8 +1992,9 @@ function updateJobBaseInfo(uniqueId, info, incrementCount = true) {
|
|
|
1805
1992
|
if (key === "uniqueId" || key === "unique_id") continue;
|
|
1806
1993
|
if (value === undefined || value === "") continue;
|
|
1807
1994
|
let column = camelToSnake(key);
|
|
1808
|
-
// 字段别名:bio → signature
|
|
1995
|
+
// 字段别名:bio → signature, createTime → user_create_time
|
|
1809
1996
|
if (column === "bio") column = "signature";
|
|
1997
|
+
if (column === "create_time") column = "user_create_time";
|
|
1810
1998
|
if (!writableJobColumns.has(column)) continue;
|
|
1811
1999
|
nextValues[column] = normalizeJobValue(column, value);
|
|
1812
2000
|
}
|
|
@@ -1951,6 +2139,8 @@ export function createStore(filePath, options = {}) {
|
|
|
1951
2139
|
let clientErrors = new Map();
|
|
1952
2140
|
// 客户端登录状态:userId → boolean
|
|
1953
2141
|
let clientLoginStatus = new Map();
|
|
2142
|
+
// 活跃客户端追踪:clientId → { type, ip, port, userId, lastSeen }
|
|
2143
|
+
let activeClients = new Map();
|
|
1954
2144
|
// refill 锁:防止多个 claimNextJob 同时触发 LLM refill
|
|
1955
2145
|
let refillLock = null; // Promise | null
|
|
1956
2146
|
// LLM 采样偏移量记忆:按猜测国家记录上次查询位置,避免重复采样
|
|
@@ -2192,6 +2382,43 @@ export function createStore(filePath, options = {}) {
|
|
|
2192
2382
|
}
|
|
2193
2383
|
}
|
|
2194
2384
|
|
|
2385
|
+
function addRawUsers(users) {
|
|
2386
|
+
if (!Array.isArray(users)) return { added: 0, skipped: 0 };
|
|
2387
|
+
const now = Date.now();
|
|
2388
|
+
let added = 0;
|
|
2389
|
+
let skipped = 0;
|
|
2390
|
+
|
|
2391
|
+
for (const u of users) {
|
|
2392
|
+
const uniqueId = (u.uniqueId || "").replace(/^@/, "").trim();
|
|
2393
|
+
if (!uniqueId) continue;
|
|
2394
|
+
if (hasUser(uniqueId)) {
|
|
2395
|
+
skipped++;
|
|
2396
|
+
continue;
|
|
2397
|
+
}
|
|
2398
|
+
const userObj = {
|
|
2399
|
+
uniqueId,
|
|
2400
|
+
status: "pending",
|
|
2401
|
+
sources: Array.isArray(u.sources)
|
|
2402
|
+
? u.sources
|
|
2403
|
+
: u.sources
|
|
2404
|
+
? [u.sources]
|
|
2405
|
+
: ["tag"],
|
|
2406
|
+
guessedLocation: u.guessedLocation || u.locationCreated || null,
|
|
2407
|
+
locationCreated: u.locationCreated || null,
|
|
2408
|
+
createdAt: now,
|
|
2409
|
+
updatedAt: now,
|
|
2410
|
+
};
|
|
2411
|
+
const writeTxn = db.transaction((job) => {
|
|
2412
|
+
addUserToDb(job);
|
|
2413
|
+
addJobBaseToDb(job);
|
|
2414
|
+
});
|
|
2415
|
+
writeTxn(userObj);
|
|
2416
|
+
added++;
|
|
2417
|
+
}
|
|
2418
|
+
|
|
2419
|
+
return { added, skipped };
|
|
2420
|
+
}
|
|
2421
|
+
|
|
2195
2422
|
function getPendingUsers() {
|
|
2196
2423
|
if (db) {
|
|
2197
2424
|
return getAllJobs().filter((u) => u.status === "pending");
|
|
@@ -2408,7 +2635,7 @@ Standards: 90-100=clear match, 70-89=likely, 50-69=possible, 20-49=low, 0-19=unl
|
|
|
2408
2635
|
|
|
2409
2636
|
// 构建 WHERE 条件
|
|
2410
2637
|
const conditions = [
|
|
2411
|
-
"COALESCE(video_count, 0) >
|
|
2638
|
+
"COALESCE(video_count, 0) > 6",
|
|
2412
2639
|
"COALESCE(follower_count, 0) > 0",
|
|
2413
2640
|
"COALESCE(following_count, 0) > 0",
|
|
2414
2641
|
];
|
|
@@ -2494,7 +2721,7 @@ Standards: 90-100=clear match, 70-89=likely, 50-69=possible, 20-49=low, 0-19=unl
|
|
|
2494
2721
|
.prepare(
|
|
2495
2722
|
`
|
|
2496
2723
|
SELECT * FROM raw_jobs WHERE ${whereSql} AND guessed_location = ?
|
|
2497
|
-
ORDER BY created_at DESC
|
|
2724
|
+
ORDER BY COALESCE(video_count, 0) DESC, created_at DESC
|
|
2498
2725
|
LIMIT ? OFFSET ?
|
|
2499
2726
|
`,
|
|
2500
2727
|
)
|
|
@@ -2608,7 +2835,7 @@ Standards: 90-100=clear match, 70-89=likely, 50-69=possible, 20-49=low, 0-19=unl
|
|
|
2608
2835
|
guessed_location, location_created, confirmed_location,
|
|
2609
2836
|
follower_count, following_count, heart_count,
|
|
2610
2837
|
created_at, updated_at, region, signature, bio_link, sec_uid,
|
|
2611
|
-
status_code, latest_video_time
|
|
2838
|
+
status_code, latest_video_time, user_create_time
|
|
2612
2839
|
)
|
|
2613
2840
|
SELECT
|
|
2614
2841
|
unique_id, nickname, 'pending', sources, pinned,
|
|
@@ -2616,10 +2843,10 @@ Standards: 90-100=clear match, 70-89=likely, 50-69=possible, 20-49=low, 0-19=unl
|
|
|
2616
2843
|
guessed_location, location_created, confirmed_location,
|
|
2617
2844
|
follower_count, following_count, heart_count,
|
|
2618
2845
|
created_at, updated_at, region, signature, bio_link, sec_uid,
|
|
2619
|
-
status_code, latest_video_time
|
|
2846
|
+
status_code, latest_video_time, user_create_time
|
|
2620
2847
|
FROM raw_jobs
|
|
2621
2848
|
WHERE ${whereSql}
|
|
2622
|
-
ORDER BY created_at DESC
|
|
2849
|
+
ORDER BY COALESCE(video_count, 0) DESC, created_at DESC
|
|
2623
2850
|
LIMIT ?
|
|
2624
2851
|
`,
|
|
2625
2852
|
).run(...args, safeLimit);
|
|
@@ -2631,7 +2858,7 @@ Standards: 90-100=clear match, 70-89=likely, 50-69=possible, 20-49=low, 0-19=unl
|
|
|
2631
2858
|
WHERE unique_id IN (
|
|
2632
2859
|
SELECT unique_id FROM raw_jobs
|
|
2633
2860
|
WHERE ${whereSql}
|
|
2634
|
-
ORDER BY created_at DESC
|
|
2861
|
+
ORDER BY COALESCE(video_count, 0) DESC, created_at DESC
|
|
2635
2862
|
LIMIT ?
|
|
2636
2863
|
)
|
|
2637
2864
|
`,
|
|
@@ -3732,20 +3959,7 @@ Standards: 90-100=clear match, 70-89=likely, 50-69=possible, 20-49=low, 0-19=unl
|
|
|
3732
3959
|
const now = Date.now();
|
|
3733
3960
|
const threshold = now - maxAgeSeconds * 1000;
|
|
3734
3961
|
const defaultTime = new Date("2016-01-01T00:00:00Z").getTime();
|
|
3735
|
-
const targetLocations =
|
|
3736
|
-
"CZ",
|
|
3737
|
-
"GR",
|
|
3738
|
-
"HU",
|
|
3739
|
-
"PT",
|
|
3740
|
-
"ES",
|
|
3741
|
-
"PL",
|
|
3742
|
-
"NL",
|
|
3743
|
-
"BE",
|
|
3744
|
-
"DE",
|
|
3745
|
-
"FR",
|
|
3746
|
-
"IT",
|
|
3747
|
-
"IE",
|
|
3748
|
-
];
|
|
3962
|
+
const targetLocations = DEFAULT_TARGET_LOCATIONS;
|
|
3749
3963
|
const placeholders = targetLocations.map(() => "?").join(",");
|
|
3750
3964
|
const row = db
|
|
3751
3965
|
.prepare(
|
|
@@ -3756,7 +3970,7 @@ Standards: 90-100=clear match, 70-89=likely, 50-69=possible, 20-49=low, 0-19=unl
|
|
|
3756
3970
|
AND verified = 0
|
|
3757
3971
|
AND location_created IN (${placeholders})
|
|
3758
3972
|
AND COALESCE(refresh_time, ?) < ?
|
|
3759
|
-
ORDER BY COALESCE(refresh_time, ?) ASC
|
|
3973
|
+
ORDER BY COALESCE(pinned, 0) DESC, COALESCE(refresh_time, ?) ASC
|
|
3760
3974
|
LIMIT 1
|
|
3761
3975
|
`,
|
|
3762
3976
|
)
|
|
@@ -3777,20 +3991,7 @@ Standards: 90-100=clear match, 70-89=likely, 50-69=possible, 20-49=low, 0-19=unl
|
|
|
3777
3991
|
const defaultTime = new Date("2016-01-01T00:00:00Z").getTime();
|
|
3778
3992
|
|
|
3779
3993
|
// 筛选目标国家用户,按 refreshTime 升序取最远的(没有则默认 2016-01-01)
|
|
3780
|
-
const targetLocations =
|
|
3781
|
-
"CZ",
|
|
3782
|
-
"GR",
|
|
3783
|
-
"HU",
|
|
3784
|
-
"PT",
|
|
3785
|
-
"ES",
|
|
3786
|
-
"PL",
|
|
3787
|
-
"NL",
|
|
3788
|
-
"BE",
|
|
3789
|
-
"DE",
|
|
3790
|
-
"FR",
|
|
3791
|
-
"IT",
|
|
3792
|
-
"IE",
|
|
3793
|
-
];
|
|
3994
|
+
const targetLocations = DEFAULT_TARGET_LOCATIONS;
|
|
3794
3995
|
const targetUsers = data.filter(
|
|
3795
3996
|
(u) =>
|
|
3796
3997
|
u.ttSeller &&
|
|
@@ -3806,6 +4007,10 @@ Standards: 90-100=clear match, 70-89=likely, 50-69=possible, 20-49=low, 0-19=unl
|
|
|
3806
4007
|
if (recentEnough.length === 0) return null;
|
|
3807
4008
|
|
|
3808
4009
|
recentEnough.sort((a, b) => {
|
|
4010
|
+
// pinned 优先,其次按 refreshTime 升序
|
|
4011
|
+
if ((a.pinned ? 1 : 0) !== (b.pinned ? 1 : 0)) {
|
|
4012
|
+
return (b.pinned ? 1 : 0) - (a.pinned ? 1 : 0);
|
|
4013
|
+
}
|
|
3809
4014
|
const ta = a.refreshTime || defaultTime;
|
|
3810
4015
|
const tb = b.refreshTime || defaultTime;
|
|
3811
4016
|
return ta - tb;
|
|
@@ -3928,6 +4133,38 @@ Standards: 90-100=clear match, 70-89=likely, 50-69=possible, 20-49=low, 0-19=unl
|
|
|
3928
4133
|
return Object.fromEntries(clientLoginStatus);
|
|
3929
4134
|
}
|
|
3930
4135
|
|
|
4136
|
+
function trackClient(clientId, info) {
|
|
4137
|
+
const existing = activeClients.get(clientId);
|
|
4138
|
+
if (existing) {
|
|
4139
|
+
if (info.type) existing.type = info.type;
|
|
4140
|
+
if (info.userId) existing.userId = info.userId;
|
|
4141
|
+
if (info.ip) existing.ip = info.ip;
|
|
4142
|
+
if (info.port !== undefined) existing.port = info.port;
|
|
4143
|
+
existing.lastSeen = Date.now();
|
|
4144
|
+
} else {
|
|
4145
|
+
activeClients.set(clientId, {
|
|
4146
|
+
...info,
|
|
4147
|
+
lastSeen: Date.now(),
|
|
4148
|
+
});
|
|
4149
|
+
}
|
|
4150
|
+
}
|
|
4151
|
+
|
|
4152
|
+
function getActiveClients() {
|
|
4153
|
+
const now = Date.now();
|
|
4154
|
+
const stale = 2 * 60 * 1000;
|
|
4155
|
+
for (const [id, info] of activeClients) {
|
|
4156
|
+
if (now - info.lastSeen > stale) activeClients.delete(id);
|
|
4157
|
+
}
|
|
4158
|
+
return Array.from(activeClients.entries()).map(([clientId, info]) => ({
|
|
4159
|
+
clientId,
|
|
4160
|
+
type: info.type || "unknown",
|
|
4161
|
+
ip: info.ip || "",
|
|
4162
|
+
port: info.port || 0,
|
|
4163
|
+
userId: info.userId || "",
|
|
4164
|
+
lastSeen: info.lastSeen,
|
|
4165
|
+
}));
|
|
4166
|
+
}
|
|
4167
|
+
|
|
3931
4168
|
function getPendingUserUpdateTasks(limit, countries) {
|
|
3932
4169
|
const targetCountries = countries
|
|
3933
4170
|
? countries.map((c) => String(c).trim().toUpperCase())
|
|
@@ -4067,7 +4304,8 @@ Standards: 90-100=clear match, 70-89=likely, 50-69=possible, 20-49=low, 0-19=unl
|
|
|
4067
4304
|
guessed_location, location_created, confirmed_location, modified_at,
|
|
4068
4305
|
follower_count, following_count, heart_count, refresh_time,
|
|
4069
4306
|
processed, processed_at, created_at, updated_at,
|
|
4070
|
-
region, signature, bio_link, sec_uid, status_code, latest_video_time
|
|
4307
|
+
region, signature, bio_link, sec_uid, status_code, latest_video_time,
|
|
4308
|
+
user_create_time
|
|
4071
4309
|
)
|
|
4072
4310
|
SELECT
|
|
4073
4311
|
unique_id, nickname, status, sources, claimed_by, claimed_at,
|
|
@@ -4076,7 +4314,8 @@ Standards: 90-100=clear match, 70-89=likely, 50-69=possible, 20-49=low, 0-19=unl
|
|
|
4076
4314
|
guessed_location, location_created, confirmed_location, modified_at,
|
|
4077
4315
|
follower_count, following_count, heart_count, refresh_time,
|
|
4078
4316
|
processed, processed_at, created_at, updated_at,
|
|
4079
|
-
region, signature, bio_link, sec_uid, status_code, latest_video_time
|
|
4317
|
+
region, signature, bio_link, sec_uid, status_code, latest_video_time,
|
|
4318
|
+
user_create_time
|
|
4080
4319
|
FROM jobs WHERE unique_id = ?
|
|
4081
4320
|
`,
|
|
4082
4321
|
).run(safeId);
|
|
@@ -4115,11 +4354,12 @@ Standards: 90-100=clear match, 70-89=likely, 50-69=possible, 20-49=low, 0-19=unl
|
|
|
4115
4354
|
return;
|
|
4116
4355
|
}
|
|
4117
4356
|
|
|
4118
|
-
// 检查 tt_seller
|
|
4357
|
+
// 检查 tt_seller:商家且视频数>0移到 jobs,否则移到 raw_jobs
|
|
4119
4358
|
const row = getJobBaseRow(uniqueId);
|
|
4120
4359
|
const ttSeller = row ? row.tt_seller : null;
|
|
4121
|
-
|
|
4122
|
-
|
|
4360
|
+
const videoCount = row ? row.video_count || 0 : 0;
|
|
4361
|
+
if (ttSeller && videoCount > 0) {
|
|
4362
|
+
// 商家且有视频:标记移动到 jobs
|
|
4123
4363
|
results.push({
|
|
4124
4364
|
uniqueId,
|
|
4125
4365
|
ok: true,
|
|
@@ -4128,7 +4368,7 @@ Standards: 90-100=clear match, 70-89=likely, 50-69=possible, 20-49=low, 0-19=unl
|
|
|
4128
4368
|
});
|
|
4129
4369
|
sellerMoveList.push(uniqueId);
|
|
4130
4370
|
} else {
|
|
4131
|
-
//
|
|
4371
|
+
// 非商家或无视频:标记移动到 raw_jobs
|
|
4132
4372
|
results.push({
|
|
4133
4373
|
uniqueId,
|
|
4134
4374
|
ok: true,
|
|
@@ -4153,7 +4393,8 @@ Standards: 90-100=clear match, 70-89=likely, 50-69=possible, 20-49=low, 0-19=unl
|
|
|
4153
4393
|
guessed_location, location_created, confirmed_location, modified_at,
|
|
4154
4394
|
follower_count, following_count, heart_count, refresh_time,
|
|
4155
4395
|
processed, processed_at, created_at, updated_at,
|
|
4156
|
-
region, signature, bio_link, sec_uid, status_code, latest_video_time
|
|
4396
|
+
region, signature, bio_link, sec_uid, status_code, latest_video_time,
|
|
4397
|
+
user_create_time
|
|
4157
4398
|
)
|
|
4158
4399
|
SELECT
|
|
4159
4400
|
unique_id, nickname, status, sources, claimed_by, claimed_at,
|
|
@@ -4162,7 +4403,8 @@ Standards: 90-100=clear match, 70-89=likely, 50-69=possible, 20-49=low, 0-19=unl
|
|
|
4162
4403
|
guessed_location, location_created, confirmed_location, modified_at,
|
|
4163
4404
|
follower_count, following_count, heart_count, refresh_time,
|
|
4164
4405
|
processed, processed_at, created_at, updated_at,
|
|
4165
|
-
region, signature, bio_link, sec_uid, status_code, latest_video_time
|
|
4406
|
+
region, signature, bio_link, sec_uid, status_code, latest_video_time,
|
|
4407
|
+
user_create_time
|
|
4166
4408
|
FROM jobs_base WHERE unique_id IN (${placeholders})
|
|
4167
4409
|
`,
|
|
4168
4410
|
).run(...sellerMoveList);
|
|
@@ -4184,7 +4426,8 @@ Standards: 90-100=clear match, 70-89=likely, 50-69=possible, 20-49=low, 0-19=unl
|
|
|
4184
4426
|
guessed_location, location_created, confirmed_location, modified_at,
|
|
4185
4427
|
follower_count, following_count, heart_count, refresh_time,
|
|
4186
4428
|
processed, processed_at, created_at, updated_at,
|
|
4187
|
-
region, signature, bio_link, sec_uid, status_code, latest_video_time
|
|
4429
|
+
region, signature, bio_link, sec_uid, status_code, latest_video_time,
|
|
4430
|
+
user_create_time
|
|
4188
4431
|
)
|
|
4189
4432
|
SELECT
|
|
4190
4433
|
unique_id, nickname, status, sources, claimed_by, claimed_at,
|
|
@@ -4193,7 +4436,8 @@ Standards: 90-100=clear match, 70-89=likely, 50-69=possible, 20-49=low, 0-19=unl
|
|
|
4193
4436
|
guessed_location, location_created, confirmed_location, modified_at,
|
|
4194
4437
|
follower_count, following_count, heart_count, refresh_time,
|
|
4195
4438
|
processed, processed_at, created_at, updated_at,
|
|
4196
|
-
region, signature, bio_link, sec_uid, status_code, latest_video_time
|
|
4439
|
+
region, signature, bio_link, sec_uid, status_code, latest_video_time,
|
|
4440
|
+
user_create_time
|
|
4197
4441
|
FROM jobs_base WHERE unique_id IN (${placeholders})
|
|
4198
4442
|
`,
|
|
4199
4443
|
).run(...rawMoveList);
|
|
@@ -4449,6 +4693,7 @@ Standards: 90-100=clear match, 70-89=likely, 50-69=possible, 20-49=low, 0-19=unl
|
|
|
4449
4693
|
hasUser,
|
|
4450
4694
|
userExists,
|
|
4451
4695
|
addUser,
|
|
4696
|
+
addRawUsers,
|
|
4452
4697
|
getPendingUsers,
|
|
4453
4698
|
getProcessedUsers,
|
|
4454
4699
|
getAllUsers,
|
|
@@ -4493,6 +4738,8 @@ Standards: 90-100=clear match, 70-89=likely, 50-69=possible, 20-49=low, 0-19=unl
|
|
|
4493
4738
|
deleteClientError,
|
|
4494
4739
|
getClientErrors,
|
|
4495
4740
|
getClientLoginStatus,
|
|
4741
|
+
trackClient,
|
|
4742
|
+
getActiveClients,
|
|
4496
4743
|
registerVideos,
|
|
4497
4744
|
getVideo,
|
|
4498
4745
|
getVideos,
|
|
@@ -4504,6 +4751,14 @@ Standards: 90-100=clear match, 70-89=likely, 50-69=possible, 20-49=low, 0-19=unl
|
|
|
4504
4751
|
stopBackup,
|
|
4505
4752
|
rawQuery,
|
|
4506
4753
|
getLlmSampleOffsets, // 获取 LLM 采样偏移量状态
|
|
4754
|
+
// Tag 发现与打分
|
|
4755
|
+
insertTag,
|
|
4756
|
+
getTagsByStatus,
|
|
4757
|
+
getTagsByCountry,
|
|
4758
|
+
getDeadTags,
|
|
4759
|
+
claimTag,
|
|
4760
|
+
reportTagScore,
|
|
4761
|
+
getAllTags,
|
|
4507
4762
|
data,
|
|
4508
4763
|
};
|
|
4509
4764
|
|
package/src/watch/public/app.js
CHANGED
|
@@ -221,6 +221,7 @@ function renderStats() {
|
|
|
221
221
|
renderCountryChart(d.countryStats);
|
|
222
222
|
renderSourceChart(d.sourceStats);
|
|
223
223
|
renderTargetLocationFilter(d.targetCountryStats);
|
|
224
|
+
renderActiveClients(d.activeClients || []);
|
|
224
225
|
}
|
|
225
226
|
|
|
226
227
|
function renderTargetLocationFilter(targetCountryStats) {
|
|
@@ -237,6 +238,100 @@ function renderTargetLocationFilter(targetCountryStats) {
|
|
|
237
238
|
.join("");
|
|
238
239
|
}
|
|
239
240
|
|
|
241
|
+
function formatRelativeTime(ts) {
|
|
242
|
+
const diff = Math.max(0, Date.now() - ts);
|
|
243
|
+
if (diff < 5000) return "刚刚";
|
|
244
|
+
const s = Math.round(diff / 1000);
|
|
245
|
+
if (s < 60) return s + " 秒前";
|
|
246
|
+
const m = Math.round(s / 60);
|
|
247
|
+
if (m < 60) return m + " 分钟前";
|
|
248
|
+
return Math.round(m / 60) + " 小时前";
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
function renderActiveClients(clients) {
|
|
252
|
+
const section = document.getElementById("activeClientsSection");
|
|
253
|
+
const bar = document.getElementById("activeClientsBar");
|
|
254
|
+
const table = document.getElementById("activeClientsTable");
|
|
255
|
+
const tbody = document.getElementById("activeClientsBody");
|
|
256
|
+
if (!section || !bar) return;
|
|
257
|
+
|
|
258
|
+
const types = ["explore", "refresh", "attach", "comments"];
|
|
259
|
+
const labels = { explore: "Explore", refresh: "Refresh", attach: "Attach", comments: "Comments" };
|
|
260
|
+
const grouped = {};
|
|
261
|
+
for (const c of clients) {
|
|
262
|
+
if (!grouped[c.type]) grouped[c.type] = [];
|
|
263
|
+
grouped[c.type].push(c);
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
const counts = {};
|
|
267
|
+
for (const t of types) counts[t] = (grouped[t] || []).length;
|
|
268
|
+
const total = clients.length;
|
|
269
|
+
|
|
270
|
+
if (total === 0) {
|
|
271
|
+
section.style.display = "none";
|
|
272
|
+
return;
|
|
273
|
+
}
|
|
274
|
+
section.style.display = "";
|
|
275
|
+
|
|
276
|
+
// 清除之前的选择状态
|
|
277
|
+
let selectedType = bar.dataset.selectedType || "";
|
|
278
|
+
|
|
279
|
+
const parts = ['<span class="bar-label">活跃客户端</span>'];
|
|
280
|
+
for (const t of types) {
|
|
281
|
+
const count = counts[t] || 0;
|
|
282
|
+
const cls = count > 0 ? "active" : "inactive";
|
|
283
|
+
const dot = count > 0 ? '<span class="dot"></span>' : "";
|
|
284
|
+
const sel = t === selectedType ? " selected" : "";
|
|
285
|
+
parts.push(
|
|
286
|
+
`<span class="client-badge ${cls}${sel}" data-type="${t}" onclick="toggleClientDetail('${t}')">${dot}${labels[t]}: ${count}</span>`,
|
|
287
|
+
);
|
|
288
|
+
}
|
|
289
|
+
bar.innerHTML = parts.join("\n");
|
|
290
|
+
|
|
291
|
+
// 如果有选中类型,刷新明细
|
|
292
|
+
if (selectedType && grouped[selectedType]) {
|
|
293
|
+
showClientDetail(selectedType, grouped[selectedType]);
|
|
294
|
+
} else {
|
|
295
|
+
table.style.display = "none";
|
|
296
|
+
}
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
function toggleClientDetail(type) {
|
|
300
|
+
const bar = document.getElementById("activeClientsBar");
|
|
301
|
+
if (bar.dataset.selectedType === type) {
|
|
302
|
+
bar.dataset.selectedType = "";
|
|
303
|
+
document.getElementById("activeClientsTable").style.display = "none";
|
|
304
|
+
} else {
|
|
305
|
+
bar.dataset.selectedType = type;
|
|
306
|
+
}
|
|
307
|
+
if (currentStats) renderActiveClients(currentStats.activeClients || []);
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
function showClientDetail(type, clients) {
|
|
311
|
+
const table = document.getElementById("activeClientsTable");
|
|
312
|
+
const tbody = document.getElementById("activeClientsBody");
|
|
313
|
+
table.style.display = "";
|
|
314
|
+
tbody.innerHTML = clients
|
|
315
|
+
.map((c) => {
|
|
316
|
+
const cid = c.clientId ? c.clientId.substring(0, 8) : "-";
|
|
317
|
+
const ipPort = c.ip
|
|
318
|
+
? c.ip + (c.port ? ":" + c.port : "")
|
|
319
|
+
: "-";
|
|
320
|
+
const userId = c.userId || "-";
|
|
321
|
+
const last = formatRelativeTime(c.lastSeen);
|
|
322
|
+
return `<tr>
|
|
323
|
+
<td class="client-id">${cid}</td>
|
|
324
|
+
<td class="ip-port">${ipPort}</td>
|
|
325
|
+
<td class="ip-port">${c.port || "-"}</td>
|
|
326
|
+
<td class="user-id">${userId}</td>
|
|
327
|
+
<td class="last-seen">${last}</td>
|
|
328
|
+
</tr>`;
|
|
329
|
+
})
|
|
330
|
+
.join("");
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
window.toggleClientDetail = toggleClientDetail;
|
|
334
|
+
|
|
240
335
|
function renderCountryChart(countries) {
|
|
241
336
|
const el = document.getElementById("countryChart");
|
|
242
337
|
const filtered = countries.filter((c) => c.country !== "未知");
|