tt-help-cli-ycl 1.4.22 → 1.4.24
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/scripts/migrate-raw-jobs.mjs +243 -0
- package/scripts/sync-raw-jobs.mjs +281 -0
- package/src/cli/raw-sync.js +41 -0
- package/src/cli/refresh.js +1 -1
- package/src/lib/args.js +50 -0
- package/src/lib/constants.js +14 -0
- package/src/main.js +3 -0
- package/src/scraper/explore-core.js +14 -4
- package/src/watch/data-store.js +256 -259
- package/src/watch/db-columns.js +2 -0
- package/src/watch/db-schema.js +2 -0
- package/src/watch/db-stats.js +18 -10
- package/src/watch/llm-config.js +66 -16
- package/src/watch/public/app.js +53 -16
- package/src/watch/server.js +122 -19
package/package.json
CHANGED
|
@@ -0,0 +1,243 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* raw_jobs 分库迁移脚本
|
|
3
|
+
*
|
|
4
|
+
* 用途: 将主库中 raw_jobs 表数据全部迁至归档库,主库仅保留空表结构。
|
|
5
|
+
* 用法: node scripts/migrate-raw-jobs.mjs [--db <主库路径>] [--archive <归档库路径>]
|
|
6
|
+
* 示例:
|
|
7
|
+
* node scripts/migrate-raw-jobs.mjs
|
|
8
|
+
* node scripts/migrate-raw-jobs.mjs --db D:\tiktok\data\result.db
|
|
9
|
+
* node scripts/migrate-raw-jobs.mjs --db data/result.db --archive data/archive.db
|
|
10
|
+
* 前提: 主程序(watch server)已停止
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
import Database from "better-sqlite3";
|
|
14
|
+
import { copyFileSync, existsSync, statSync } from "fs";
|
|
15
|
+
import { resolve, dirname, extname } from "path";
|
|
16
|
+
import { fileURLToPath } from "url";
|
|
17
|
+
|
|
18
|
+
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
19
|
+
|
|
20
|
+
// ===== 命令行参数解析 =====
|
|
21
|
+
function parseArgs() {
|
|
22
|
+
const args = process.argv.slice(2);
|
|
23
|
+
const opts = {
|
|
24
|
+
mainDb: null,
|
|
25
|
+
archiveDb: null,
|
|
26
|
+
};
|
|
27
|
+
for (let i = 0; i < args.length; i++) {
|
|
28
|
+
if (args[i] === "--db" && i + 1 < args.length) {
|
|
29
|
+
opts.mainDb = resolve(args[++i]);
|
|
30
|
+
} else if (args[i] === "--archive" && i + 1 < args.length) {
|
|
31
|
+
opts.archiveDb = resolve(args[++i]);
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
return opts;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
const cliArgs = parseArgs();
|
|
38
|
+
|
|
39
|
+
// ===== 配置 =====
|
|
40
|
+
const MAIN_DB = cliArgs.mainDb || resolve(__dirname, "..", "data", "result.db");
|
|
41
|
+
const ARCHIVE_DB =
|
|
42
|
+
cliArgs.archiveDb ||
|
|
43
|
+
(() => {
|
|
44
|
+
const ext = extname(MAIN_DB);
|
|
45
|
+
const base = MAIN_DB.slice(0, -ext.length);
|
|
46
|
+
return `${base}-archive${ext}`;
|
|
47
|
+
})();
|
|
48
|
+
|
|
49
|
+
// ===== 工具函数 =====
|
|
50
|
+
function formatSize(bytes) {
|
|
51
|
+
if (bytes < 1024) return `${bytes} B`;
|
|
52
|
+
if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`;
|
|
53
|
+
if (bytes < 1024 * 1024 * 1024)
|
|
54
|
+
return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
|
|
55
|
+
return `${(bytes / (1024 * 1024 * 1024)).toFixed(2)} GB`;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
// ===== 前置检查 =====
|
|
59
|
+
console.error("=== raw_jobs 分库迁移 ===\n");
|
|
60
|
+
|
|
61
|
+
if (!existsSync(MAIN_DB)) {
|
|
62
|
+
console.error(`错误: 主库不存在: ${MAIN_DB}`);
|
|
63
|
+
process.exit(1);
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
// 检查 WAL 文件,提示用户确保程序已停止
|
|
67
|
+
const walPath = MAIN_DB + "-wal";
|
|
68
|
+
const shmPath = MAIN_DB + "-shm";
|
|
69
|
+
const hasWal = existsSync(walPath);
|
|
70
|
+
const hasShm = existsSync(shmPath);
|
|
71
|
+
|
|
72
|
+
if (hasWal || hasShm) {
|
|
73
|
+
console.error("警告: 检测到 WAL 文件,请先确保主程序已停止!");
|
|
74
|
+
console.error(` ${walPath}: ${hasWal ? "存在" : "不存在"}`);
|
|
75
|
+
console.error(` ${shmPath}: ${hasShm ? "存在" : "不存在"}`);
|
|
76
|
+
console.error("");
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
// ===== 步骤 0: 备份主库 =====
|
|
80
|
+
console.error("[0/6] 备份主库...");
|
|
81
|
+
|
|
82
|
+
const BACKUP_DB = MAIN_DB.replace(/\.db$/, `-bak-${Date.now()}.db`);
|
|
83
|
+
copyFileSync(MAIN_DB, BACKUP_DB);
|
|
84
|
+
console.error(` 备份已保存: ${BACKUP_DB}`);
|
|
85
|
+
|
|
86
|
+
// ===== 步骤 1: 检查主库状态 =====
|
|
87
|
+
console.error("\n[1/6] 检查主库...");
|
|
88
|
+
|
|
89
|
+
const main = new Database(MAIN_DB);
|
|
90
|
+
main.pragma("journal_mode = WAL");
|
|
91
|
+
|
|
92
|
+
// 先做一次 CHECKPOINT,把 WAL 合并回主文件
|
|
93
|
+
main.pragma("wal_checkpoint(TRUNCATE)");
|
|
94
|
+
|
|
95
|
+
const rawCount = main.prepare("SELECT COUNT(*) as c FROM raw_jobs").get().c;
|
|
96
|
+
const mainSize = statSync(MAIN_DB).size;
|
|
97
|
+
|
|
98
|
+
console.error(` 主库大小: ${formatSize(mainSize)}`);
|
|
99
|
+
console.error(` raw_jobs 记录数: ${rawCount.toLocaleString()}`);
|
|
100
|
+
|
|
101
|
+
if (rawCount === 0) {
|
|
102
|
+
console.error(" raw_jobs 已为空,无需迁移");
|
|
103
|
+
main.close();
|
|
104
|
+
process.exit(0);
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
// ===== 步骤 2: 提取 raw_jobs 完整 DDL =====
|
|
108
|
+
console.error("\n[2/6] 提取 raw_jobs 表结构...");
|
|
109
|
+
|
|
110
|
+
const createTableSql = main
|
|
111
|
+
.prepare(
|
|
112
|
+
"SELECT sql FROM sqlite_master WHERE type='table' AND name='raw_jobs'",
|
|
113
|
+
)
|
|
114
|
+
.get()?.sql;
|
|
115
|
+
|
|
116
|
+
if (!createTableSql) {
|
|
117
|
+
console.error(" 错误: 未找到 raw_jobs 建表语句");
|
|
118
|
+
main.close();
|
|
119
|
+
process.exit(1);
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
// 提取所有 raw_jobs 相关的索引
|
|
123
|
+
const indexes = main
|
|
124
|
+
.prepare(
|
|
125
|
+
"SELECT sql FROM sqlite_master WHERE type='index' AND tbl_name='raw_jobs' AND sql IS NOT NULL",
|
|
126
|
+
)
|
|
127
|
+
.all()
|
|
128
|
+
.map((r) => r.sql);
|
|
129
|
+
|
|
130
|
+
console.error(` 建表语句: ${createTableSql.substring(0, 60)}...`);
|
|
131
|
+
console.error(` 索引数量: ${indexes.length}`);
|
|
132
|
+
|
|
133
|
+
main.close();
|
|
134
|
+
|
|
135
|
+
// ===== 步骤 3: 拷贝主库 → 归档库 =====
|
|
136
|
+
console.error("\n[3/6] 拷贝主库 → 归档库...");
|
|
137
|
+
|
|
138
|
+
if (existsSync(ARCHIVE_DB)) {
|
|
139
|
+
console.error(` 归档库已存在,将覆盖: ${ARCHIVE_DB}`);
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
const startCopy = Date.now();
|
|
143
|
+
copyFileSync(MAIN_DB, ARCHIVE_DB);
|
|
144
|
+
const copyElapsed = ((Date.now() - startCopy) / 1000).toFixed(1);
|
|
145
|
+
const archiveSize = statSync(ARCHIVE_DB).size;
|
|
146
|
+
console.error(` 拷贝完成: ${formatSize(archiveSize)},耗时 ${copyElapsed}s`);
|
|
147
|
+
|
|
148
|
+
// ===== 步骤 4: 归档库清理 =====
|
|
149
|
+
console.error("\n[4/6] 归档库清理(仅保留 raw_jobs)...");
|
|
150
|
+
|
|
151
|
+
const archive = new Database(ARCHIVE_DB);
|
|
152
|
+
archive.pragma("journal_mode = WAL");
|
|
153
|
+
|
|
154
|
+
// 删除 raw_jobs 以外的所有表(排除 SQLite 系统表)
|
|
155
|
+
const SKIP_TABLES = new Set(["raw_jobs", "sqlite_sequence"]);
|
|
156
|
+
const tables = archive
|
|
157
|
+
.prepare("SELECT name FROM sqlite_master WHERE type='table'")
|
|
158
|
+
.all()
|
|
159
|
+
.filter((t) => !SKIP_TABLES.has(t.name) && !t.name.startsWith("sqlite_"));
|
|
160
|
+
|
|
161
|
+
for (const t of tables) {
|
|
162
|
+
archive.exec(`DROP TABLE IF EXISTS "${t.name}"`);
|
|
163
|
+
console.error(` 删除表: ${t.name}`);
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
// 删除 raw_jobs 以外的所有索引(排除 SQLite 系统索引)
|
|
167
|
+
const otherIndexes = archive
|
|
168
|
+
.prepare("SELECT name FROM sqlite_master WHERE type='index'")
|
|
169
|
+
.all()
|
|
170
|
+
.filter(
|
|
171
|
+
(idx) => idx.tbl_name !== "raw_jobs" && !idx.name.startsWith("sqlite_"),
|
|
172
|
+
);
|
|
173
|
+
for (const idx of otherIndexes) {
|
|
174
|
+
archive.exec(`DROP INDEX IF EXISTS "${idx.name}"`);
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
// 回收空间
|
|
178
|
+
archive.exec("VACUUM");
|
|
179
|
+
const archiveFinalSize = statSync(ARCHIVE_DB).size;
|
|
180
|
+
archive.close();
|
|
181
|
+
|
|
182
|
+
console.error(` 归档库最终大小: ${formatSize(archiveFinalSize)}`);
|
|
183
|
+
|
|
184
|
+
// ===== 步骤 5: 主库清空 raw_jobs =====
|
|
185
|
+
console.error("\n[5/6] 主库清空 raw_jobs...");
|
|
186
|
+
|
|
187
|
+
const main2 = new Database(MAIN_DB);
|
|
188
|
+
main2.pragma("journal_mode = WAL");
|
|
189
|
+
main2.pragma("wal_checkpoint(TRUNCATE)");
|
|
190
|
+
|
|
191
|
+
// DROP + 重建(使用原始建表语句)
|
|
192
|
+
main2.exec("DROP TABLE IF EXISTS raw_jobs");
|
|
193
|
+
main2.exec(createTableSql);
|
|
194
|
+
|
|
195
|
+
// 重建索引
|
|
196
|
+
for (const idxSql of indexes) {
|
|
197
|
+
main2.exec(idxSql);
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
main2.pragma("wal_checkpoint(TRUNCATE)");
|
|
201
|
+
main2.close();
|
|
202
|
+
|
|
203
|
+
console.error(" raw_jobs 已重建(空表)");
|
|
204
|
+
|
|
205
|
+
// ===== 步骤 6: 主库 VACUUM,回收空间 =====
|
|
206
|
+
console.error("\n[6/6] 主库空间回收 (VACUUM)...");
|
|
207
|
+
|
|
208
|
+
const beforeVacuum = statSync(MAIN_DB).size;
|
|
209
|
+
const startVacuum = Date.now();
|
|
210
|
+
|
|
211
|
+
const main3 = new Database(MAIN_DB);
|
|
212
|
+
main3.pragma("journal_mode = WAL");
|
|
213
|
+
main3.exec("VACUUM");
|
|
214
|
+
main3.close();
|
|
215
|
+
|
|
216
|
+
const afterVacuum = statSync(MAIN_DB).size;
|
|
217
|
+
const vacuumElapsed = ((Date.now() - startVacuum) / 1000).toFixed(1);
|
|
218
|
+
console.error(
|
|
219
|
+
` 主库: ${formatSize(beforeVacuum)} → ${formatSize(afterVacuum)},耗时 ${vacuumElapsed}s`,
|
|
220
|
+
);
|
|
221
|
+
|
|
222
|
+
// 验证
|
|
223
|
+
const verifyDb = new Database(MAIN_DB);
|
|
224
|
+
const verifyCount = verifyDb
|
|
225
|
+
.prepare("SELECT COUNT(*) as c FROM raw_jobs")
|
|
226
|
+
.get().c;
|
|
227
|
+
verifyDb.close();
|
|
228
|
+
|
|
229
|
+
// ===== 结果 =====
|
|
230
|
+
console.error("\n=== 迁移完成 ===");
|
|
231
|
+
console.error(`备份: ${BACKUP_DB}`);
|
|
232
|
+
console.error(`主库: ${MAIN_DB}`);
|
|
233
|
+
console.error(` 大小: ${formatSize(afterVacuum)}`);
|
|
234
|
+
console.error(` raw_jobs: ${verifyCount} 条`);
|
|
235
|
+
console.error("");
|
|
236
|
+
console.error(`归档库: ${ARCHIVE_DB}`);
|
|
237
|
+
console.error(` 大小: ${formatSize(archiveFinalSize)}`);
|
|
238
|
+
console.error(` raw_jobs: ${rawCount.toLocaleString()} 条`);
|
|
239
|
+
console.error("");
|
|
240
|
+
console.error("下一步:");
|
|
241
|
+
console.error(" 1. 启动主程序: tt-help watch -o data/result.db");
|
|
242
|
+
console.error(" 2. raw_jobs 空表,refill 不再卡顿");
|
|
243
|
+
console.error(" 3. 后续需编写同步程序,从归档库补充数据到主库 raw_jobs");
|
|
@@ -0,0 +1,281 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* raw_jobs 同步程序
|
|
3
|
+
*
|
|
4
|
+
* 定时检测主库中各目标国家的 raw_jobs 数量,不足时从归档库搬移补充。
|
|
5
|
+
* 搬移逻辑复用 refillJobsFromRaw 非 LLM 模式的排序规则。
|
|
6
|
+
*
|
|
7
|
+
* 用法:
|
|
8
|
+
* node scripts/sync-raw-jobs.mjs --db <主库> --archive <归档库>
|
|
9
|
+
* node scripts/sync-raw-jobs.mjs --db data/result.db --archive data/result-archive.db
|
|
10
|
+
* node scripts/sync-raw-jobs.mjs --db data/result.db --archive data/result-archive.db --once
|
|
11
|
+
* node scripts/sync-raw-jobs.mjs --db data/result.db --archive data/result-archive.db --interval 60 --threshold 2000 --batch 300
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
import Database from "better-sqlite3";
|
|
15
|
+
|
|
16
|
+
// ===== 目标国家 =====
|
|
17
|
+
const TARGET_LOCATIONS = [
|
|
18
|
+
"CZ",
|
|
19
|
+
"GR",
|
|
20
|
+
"HU",
|
|
21
|
+
"PT",
|
|
22
|
+
"ES",
|
|
23
|
+
"PL",
|
|
24
|
+
"NL",
|
|
25
|
+
"BE",
|
|
26
|
+
"DE",
|
|
27
|
+
"FR",
|
|
28
|
+
"IT",
|
|
29
|
+
"IE",
|
|
30
|
+
"AT",
|
|
31
|
+
];
|
|
32
|
+
|
|
33
|
+
// ===== 命令行参数 =====
|
|
34
|
+
function parseArgs() {
|
|
35
|
+
const args = process.argv.slice(2);
|
|
36
|
+
const opts = {
|
|
37
|
+
db: null,
|
|
38
|
+
archive: null,
|
|
39
|
+
interval: 30,
|
|
40
|
+
threshold: 1000,
|
|
41
|
+
batch: 500,
|
|
42
|
+
once: false,
|
|
43
|
+
};
|
|
44
|
+
for (let i = 0; i < args.length; i++) {
|
|
45
|
+
switch (args[i]) {
|
|
46
|
+
case "--db":
|
|
47
|
+
opts.db = args[++i];
|
|
48
|
+
break;
|
|
49
|
+
case "--archive":
|
|
50
|
+
opts.archive = args[++i];
|
|
51
|
+
break;
|
|
52
|
+
case "--interval":
|
|
53
|
+
opts.interval = parseInt(args[++i]) || 30;
|
|
54
|
+
break;
|
|
55
|
+
case "--threshold":
|
|
56
|
+
opts.threshold = parseInt(args[++i]) || 1000;
|
|
57
|
+
break;
|
|
58
|
+
case "--batch":
|
|
59
|
+
opts.batch = parseInt(args[++i]) || 500;
|
|
60
|
+
break;
|
|
61
|
+
case "--once":
|
|
62
|
+
opts.once = true;
|
|
63
|
+
break;
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
return opts;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
const opts = parseArgs();
|
|
70
|
+
|
|
71
|
+
if (!opts.db || !opts.archive) {
|
|
72
|
+
console.error(
|
|
73
|
+
"用法: node scripts/sync-raw-jobs.mjs --db <主库路径> --archive <归档库路径> [选项]",
|
|
74
|
+
);
|
|
75
|
+
console.error("");
|
|
76
|
+
console.error("选项:");
|
|
77
|
+
console.error(" --db <path> 主库路径(必填)");
|
|
78
|
+
console.error(" --archive <path> 归档库路径(必填)");
|
|
79
|
+
console.error(" --interval <秒> 检测间隔,默认 30");
|
|
80
|
+
console.error(" --threshold <条> 每个国家最低保有量,默认 1000");
|
|
81
|
+
console.error(" --batch <条> 每次搬移上限,默认 500");
|
|
82
|
+
console.error(" --once 只执行一次后退出");
|
|
83
|
+
process.exit(1);
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
// ===== 过滤与排序 =====
|
|
87
|
+
const FILTER_CONDITIONS = [
|
|
88
|
+
"COALESCE(video_count, 0) > 6",
|
|
89
|
+
"COALESCE(follower_count, 0) > 0",
|
|
90
|
+
"COALESCE(following_count, 0) > 0",
|
|
91
|
+
];
|
|
92
|
+
|
|
93
|
+
const ORDER_CLAUSE = `
|
|
94
|
+
ORDER BY
|
|
95
|
+
CASE WHEN sources LIKE '%tag%' THEN 0 ELSE 1 END,
|
|
96
|
+
CASE WHEN sources LIKE '%comment%' THEN 0 ELSE 1 END,
|
|
97
|
+
COALESCE(video_count, 0) DESC,
|
|
98
|
+
created_at DESC
|
|
99
|
+
`;
|
|
100
|
+
|
|
101
|
+
// raw_jobs 表完整列(与 db-schema.js 对齐)
|
|
102
|
+
const RAW_JOB_COLUMNS = [
|
|
103
|
+
"unique_id",
|
|
104
|
+
"nickname",
|
|
105
|
+
"status",
|
|
106
|
+
"sources",
|
|
107
|
+
"claimed_by",
|
|
108
|
+
"claimed_at",
|
|
109
|
+
"error",
|
|
110
|
+
"pinned",
|
|
111
|
+
"no_video",
|
|
112
|
+
"restricted",
|
|
113
|
+
"user_update_count",
|
|
114
|
+
"tt_seller",
|
|
115
|
+
"verified",
|
|
116
|
+
"video_count",
|
|
117
|
+
"comment_count",
|
|
118
|
+
"guessed_location",
|
|
119
|
+
"location_created",
|
|
120
|
+
"confirmed_location",
|
|
121
|
+
"modified_at",
|
|
122
|
+
"follower_count",
|
|
123
|
+
"following_count",
|
|
124
|
+
"heart_count",
|
|
125
|
+
"refresh_time",
|
|
126
|
+
"processed",
|
|
127
|
+
"processed_at",
|
|
128
|
+
"created_at",
|
|
129
|
+
"updated_at",
|
|
130
|
+
"region",
|
|
131
|
+
"signature",
|
|
132
|
+
"bio_link",
|
|
133
|
+
"sec_uid",
|
|
134
|
+
"status_code",
|
|
135
|
+
"latest_video_time",
|
|
136
|
+
"user_create_time",
|
|
137
|
+
];
|
|
138
|
+
|
|
139
|
+
// ===== 核心:从归档库搬移一批到主库 =====
|
|
140
|
+
function moveBatch(mainDb, archiveDb, country, limit) {
|
|
141
|
+
const whereSql = [
|
|
142
|
+
...FILTER_CONDITIONS,
|
|
143
|
+
`(UPPER(COALESCE(guessed_location, '')) = ? OR COALESCE(guessed_location, '') = '')`,
|
|
144
|
+
].join(" AND ");
|
|
145
|
+
|
|
146
|
+
const cols = RAW_JOB_COLUMNS.join(", ");
|
|
147
|
+
|
|
148
|
+
// 1. INSERT OR IGNORE 到主库
|
|
149
|
+
const insertSql = `
|
|
150
|
+
INSERT OR IGNORE INTO raw_jobs (${cols})
|
|
151
|
+
SELECT ${cols}
|
|
152
|
+
FROM (SELECT * FROM raw_jobs WHERE ${whereSql} ${ORDER_CLAUSE} LIMIT ?)
|
|
153
|
+
`;
|
|
154
|
+
|
|
155
|
+
// 2. DELETE 归档库中已搬走的数据(匹配同一批次的 unique_id)
|
|
156
|
+
const deleteSql = `
|
|
157
|
+
DELETE FROM raw_jobs
|
|
158
|
+
WHERE unique_id IN (
|
|
159
|
+
SELECT unique_id FROM raw_jobs
|
|
160
|
+
WHERE ${whereSql} ${ORDER_CLAUSE} LIMIT ?
|
|
161
|
+
)
|
|
162
|
+
`;
|
|
163
|
+
|
|
164
|
+
const moved = archiveDb.transaction(() => {
|
|
165
|
+
// 先删后插?不——先插后删,防止 INSERT OR IGNORE 冲突导致删了没插
|
|
166
|
+
// INSERT OR IGNORE: 如果主库已有该 unique_id,跳过但不报错
|
|
167
|
+
mainDb.prepare(insertSql).run(country, limit);
|
|
168
|
+
const del = archiveDb.prepare(deleteSql).run(country, limit);
|
|
169
|
+
return del.changes;
|
|
170
|
+
})();
|
|
171
|
+
|
|
172
|
+
return moved;
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
// ===== 单次同步 =====
|
|
176
|
+
function syncOnce(mainDb, archiveDb) {
|
|
177
|
+
const start = Date.now();
|
|
178
|
+
const results = {};
|
|
179
|
+
|
|
180
|
+
for (const country of TARGET_LOCATIONS) {
|
|
181
|
+
// 检查主库中该国存量
|
|
182
|
+
const countRow = mainDb
|
|
183
|
+
.prepare(
|
|
184
|
+
`SELECT COUNT(*) as c FROM raw_jobs
|
|
185
|
+
WHERE UPPER(COALESCE(guessed_location, '')) = ?`,
|
|
186
|
+
)
|
|
187
|
+
.get(country);
|
|
188
|
+
|
|
189
|
+
const current = countRow?.c || 0;
|
|
190
|
+
|
|
191
|
+
if (current >= opts.threshold) {
|
|
192
|
+
continue;
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
const need = opts.threshold - current;
|
|
196
|
+
const limit = Math.min(need, opts.batch);
|
|
197
|
+
|
|
198
|
+
const moved = moveBatch(mainDb, archiveDb, country, limit);
|
|
199
|
+
|
|
200
|
+
results[country] = { before: current, moved, after: current + moved };
|
|
201
|
+
|
|
202
|
+
console.error(
|
|
203
|
+
`[sync] ${country}: ${current} → ${current + moved} (搬入 ${moved} 条)`,
|
|
204
|
+
);
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
const totalMoved = Object.values(results).reduce((s, r) => s + r.moved, 0);
|
|
208
|
+
const elapsed = ((Date.now() - start) / 1000).toFixed(1);
|
|
209
|
+
|
|
210
|
+
if (totalMoved > 0) {
|
|
211
|
+
console.error(`[sync] 本轮共搬入 ${totalMoved} 条,耗时 ${elapsed}s`);
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
return results;
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
// ===== 主流程 =====
|
|
218
|
+
console.error("[sync] raw_jobs 同步程序启动");
|
|
219
|
+
console.error(`[sync] 主库: ${opts.db}`);
|
|
220
|
+
console.error(`[sync] 归档库: ${opts.archive}`);
|
|
221
|
+
console.error(`[sync] 最低保有量: ${opts.threshold} 条/国家`);
|
|
222
|
+
console.error(`[sync] 检测间隔: ${opts.once ? "仅一次" : opts.interval + "s"}`);
|
|
223
|
+
console.error("");
|
|
224
|
+
|
|
225
|
+
// 打开数据库
|
|
226
|
+
const mainDb = new Database(opts.db);
|
|
227
|
+
mainDb.pragma("journal_mode = WAL");
|
|
228
|
+
const archiveDb = new Database(opts.archive);
|
|
229
|
+
archiveDb.pragma("journal_mode = WAL");
|
|
230
|
+
|
|
231
|
+
// 检查主库 raw_jobs 是否存在
|
|
232
|
+
const mainTables = mainDb
|
|
233
|
+
.prepare(
|
|
234
|
+
"SELECT name FROM sqlite_master WHERE type='table' AND name='raw_jobs'",
|
|
235
|
+
)
|
|
236
|
+
.get();
|
|
237
|
+
if (!mainTables) {
|
|
238
|
+
console.error("[sync] 错误: 主库中不存在 raw_jobs 表");
|
|
239
|
+
process.exit(1);
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
// 检查归档库 raw_jobs 是否存在
|
|
243
|
+
const archiveTables = archiveDb
|
|
244
|
+
.prepare(
|
|
245
|
+
"SELECT name FROM sqlite_master WHERE type='table' AND name='raw_jobs'",
|
|
246
|
+
)
|
|
247
|
+
.get();
|
|
248
|
+
if (!archiveTables) {
|
|
249
|
+
console.error("[sync] 错误: 归档库中不存在 raw_jobs 表");
|
|
250
|
+
process.exit(1);
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
const archiveTotal =
|
|
254
|
+
archiveDb.prepare("SELECT COUNT(*) as c FROM raw_jobs").get()?.c || 0;
|
|
255
|
+
console.error(
|
|
256
|
+
`[sync] 归档库 raw_jobs 总量: ${archiveTotal.toLocaleString()} 条`,
|
|
257
|
+
);
|
|
258
|
+
console.error("");
|
|
259
|
+
|
|
260
|
+
// 执行同步
|
|
261
|
+
syncOnce(mainDb, archiveDb);
|
|
262
|
+
|
|
263
|
+
if (opts.once) {
|
|
264
|
+
mainDb.close();
|
|
265
|
+
archiveDb.close();
|
|
266
|
+
console.error("[sync] 完成,退出");
|
|
267
|
+
process.exit(0);
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
// 定时循环
|
|
271
|
+
setInterval(() => {
|
|
272
|
+
syncOnce(mainDb, archiveDb);
|
|
273
|
+
}, opts.interval * 1000);
|
|
274
|
+
|
|
275
|
+
// 优雅退出
|
|
276
|
+
process.on("SIGINT", () => {
|
|
277
|
+
console.error("\n[sync] 收到中断信号,退出...");
|
|
278
|
+
mainDb.close();
|
|
279
|
+
archiveDb.close();
|
|
280
|
+
process.exit(0);
|
|
281
|
+
});
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* raw-sync CLI 入口
|
|
3
|
+
*
|
|
4
|
+
* 启动 raw_jobs 同步程序(独立子进程)。
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
import { spawn } from "child_process";
|
|
8
|
+
import { resolve, dirname } from "path";
|
|
9
|
+
import { fileURLToPath } from "url";
|
|
10
|
+
|
|
11
|
+
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
12
|
+
const SCRIPT = resolve(__dirname, "..", "..", "scripts", "sync-raw-jobs.mjs");
|
|
13
|
+
|
|
14
|
+
export function handleRawSync(parsed) {
|
|
15
|
+
const args = ["--db", parsed.rawSyncDb, "--archive", parsed.rawSyncArchive];
|
|
16
|
+
|
|
17
|
+
if (parsed.rawSyncInterval != null)
|
|
18
|
+
args.push("--interval", String(parsed.rawSyncInterval));
|
|
19
|
+
if (parsed.rawSyncThreshold != null)
|
|
20
|
+
args.push("--threshold", String(parsed.rawSyncThreshold));
|
|
21
|
+
if (parsed.rawSyncBatch != null)
|
|
22
|
+
args.push("--batch", String(parsed.rawSyncBatch));
|
|
23
|
+
if (parsed.rawSyncOnce) args.push("--once");
|
|
24
|
+
|
|
25
|
+
console.error(`[raw-sync] 启动同步程序: node ${SCRIPT} ${args.join(" ")}`);
|
|
26
|
+
|
|
27
|
+
const child = spawn("node", [SCRIPT, ...args], {
|
|
28
|
+
stdio: "inherit",
|
|
29
|
+
});
|
|
30
|
+
|
|
31
|
+
child.on("exit", (code) => {
|
|
32
|
+
if (code !== 0) {
|
|
33
|
+
console.error(`[raw-sync] 同步程序退出,退出码: ${code}`);
|
|
34
|
+
}
|
|
35
|
+
process.exit(code);
|
|
36
|
+
});
|
|
37
|
+
|
|
38
|
+
// 转发信号给子进程
|
|
39
|
+
process.on("SIGINT", () => child.kill("SIGINT"));
|
|
40
|
+
process.on("SIGTERM", () => child.kill("SIGTERM"));
|
|
41
|
+
}
|
package/src/cli/refresh.js
CHANGED
|
@@ -541,7 +541,7 @@ export async function handleRefresh(options) {
|
|
|
541
541
|
|
|
542
542
|
const payload = {
|
|
543
543
|
userInfo: refreshUserInfo,
|
|
544
|
-
|
|
544
|
+
recentEcVideos: result.recentEcVideos || null,
|
|
545
545
|
discoveredFollowing: (result.discoveredFollowing || []).map((f) => ({
|
|
546
546
|
handle: Array.isArray(f) ? f[0] : f,
|
|
547
547
|
displayName: Array.isArray(f) ? f[1] : null,
|
package/src/lib/args.js
CHANGED
|
@@ -686,6 +686,52 @@ function parseOpenArgs(args) {
|
|
|
686
686
|
};
|
|
687
687
|
}
|
|
688
688
|
|
|
689
|
+
function parseRawSyncArgs(args) {
|
|
690
|
+
let rawSyncDb = null;
|
|
691
|
+
let rawSyncArchive = null;
|
|
692
|
+
let rawSyncInterval = null;
|
|
693
|
+
let rawSyncThreshold = null;
|
|
694
|
+
let rawSyncBatch = null;
|
|
695
|
+
let rawSyncOnce = false;
|
|
696
|
+
|
|
697
|
+
for (let i = 0; i < args.length; i++) {
|
|
698
|
+
const arg = args[i];
|
|
699
|
+
if (arg === "--db" && i + 1 < args.length) {
|
|
700
|
+
rawSyncDb = args[++i];
|
|
701
|
+
} else if (arg === "--archive" && i + 1 < args.length) {
|
|
702
|
+
rawSyncArchive = args[++i];
|
|
703
|
+
} else if (arg === "--interval" && i + 1 < args.length) {
|
|
704
|
+
rawSyncInterval = parseInt(args[++i]) || null;
|
|
705
|
+
} else if (arg === "--threshold" && i + 1 < args.length) {
|
|
706
|
+
rawSyncThreshold = parseInt(args[++i]) || null;
|
|
707
|
+
} else if (arg === "--batch" && i + 1 < args.length) {
|
|
708
|
+
rawSyncBatch = parseInt(args[++i]) || null;
|
|
709
|
+
} else if (arg === "--once") {
|
|
710
|
+
rawSyncOnce = true;
|
|
711
|
+
}
|
|
712
|
+
}
|
|
713
|
+
|
|
714
|
+
return {
|
|
715
|
+
subcommand: "raw-sync",
|
|
716
|
+
rawSyncDb,
|
|
717
|
+
rawSyncArchive,
|
|
718
|
+
rawSyncInterval,
|
|
719
|
+
rawSyncThreshold,
|
|
720
|
+
rawSyncBatch,
|
|
721
|
+
rawSyncOnce,
|
|
722
|
+
urls: [],
|
|
723
|
+
outputFormat: "json",
|
|
724
|
+
exploreCount: 0,
|
|
725
|
+
showConfig: false,
|
|
726
|
+
showHelp: false,
|
|
727
|
+
customProxy: null,
|
|
728
|
+
configAction: null,
|
|
729
|
+
configValue: null,
|
|
730
|
+
pipeMode: false,
|
|
731
|
+
filterStr: null,
|
|
732
|
+
};
|
|
733
|
+
}
|
|
734
|
+
|
|
689
735
|
function parseVideoStatsArgs(args) {
|
|
690
736
|
let statsSource = null;
|
|
691
737
|
let statsParallel = 3;
|
|
@@ -1097,6 +1143,10 @@ export function parseArgs() {
|
|
|
1097
1143
|
return parseRefreshArgs(args.slice(1));
|
|
1098
1144
|
}
|
|
1099
1145
|
|
|
1146
|
+
if (args.length > 0 && args[0] === "raw-sync") {
|
|
1147
|
+
return parseRawSyncArgs(args.slice(1));
|
|
1148
|
+
}
|
|
1149
|
+
|
|
1100
1150
|
const urls = [];
|
|
1101
1151
|
let inputFile = null;
|
|
1102
1152
|
let outputFile = null;
|
package/src/lib/constants.js
CHANGED
|
@@ -314,6 +314,19 @@ const HELP_TEXT = [
|
|
|
314
314
|
" tt-help tag refresh -s http://127.0.0.1:3001",
|
|
315
315
|
" tt-help tag refresh --port 8222 -s http://127.0.0.1:3001",
|
|
316
316
|
"",
|
|
317
|
+
" raw-sync --db <主库路径> --archive <归档库路径> [选项]",
|
|
318
|
+
" 从归档库向主库 raw_jobs 同步数据,按国家保持最低保有量",
|
|
319
|
+
" 选项:",
|
|
320
|
+
" --db <路径> 主库路径(必填)",
|
|
321
|
+
" --archive <路径> 归档库路径(必填)",
|
|
322
|
+
" --interval <秒> 检测间隔,默认 30",
|
|
323
|
+
" --threshold <条> 每个国家最低保有量,默认 1000",
|
|
324
|
+
" --batch <条> 每次搬移上限,默认 500",
|
|
325
|
+
" --once 只执行一次后退出",
|
|
326
|
+
" 示例: tt-help raw-sync --db data/result.db --archive data/result-archive.db",
|
|
327
|
+
" tt-help raw-sync --db data/result.db --archive data/result-archive.db --once",
|
|
328
|
+
" tt-help raw-sync --db data/result.db --archive data/result-archive.db --interval 60 --threshold 2000",
|
|
329
|
+
"",
|
|
317
330
|
" config [show|set|unset|reset]",
|
|
318
331
|
" config 查看当前配置",
|
|
319
332
|
" config set <key> <value> 设置配置(key: proxy, server, browser, userId, maxFollowing, maxFollowers, maxVideos, maxComments)",
|
|
@@ -331,6 +344,7 @@ const HELP_TEXT = [
|
|
|
331
344
|
" tt-help attach -p 5 -i 10",
|
|
332
345
|
" tt-help watch -o data/result.db",
|
|
333
346
|
" tt-help videostats data/result.db -p 3",
|
|
347
|
+
" tt-help raw-sync --db data/result.db --archive data/result-archive.db",
|
|
334
348
|
" tt-help tag discover ES FR --count 5",
|
|
335
349
|
" tt-help tag score ventas --countries ES",
|
|
336
350
|
" tt-help tag score-all --countries ES,FR -s http://127.0.0.1:3001",
|
package/src/main.js
CHANGED
|
@@ -19,6 +19,7 @@ import {
|
|
|
19
19
|
handleScoreAll,
|
|
20
20
|
handleTagRefresh,
|
|
21
21
|
} from "./cli/tag.js";
|
|
22
|
+
import { handleRawSync } from "./cli/raw-sync.js";
|
|
22
23
|
|
|
23
24
|
async function main() {
|
|
24
25
|
const parsed = parseArgs();
|
|
@@ -48,6 +49,8 @@ async function main() {
|
|
|
48
49
|
return handleRefresh(parsed);
|
|
49
50
|
case "tag":
|
|
50
51
|
return handleTag(parsed);
|
|
52
|
+
case "raw-sync":
|
|
53
|
+
return handleRawSync(parsed);
|
|
51
54
|
case "tag-discover":
|
|
52
55
|
return handleDiscover(parsed);
|
|
53
56
|
case "tag-score":
|