rssany 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. package/.env.example +52 -0
  2. package/README.md +196 -0
  3. package/bin/rssany.js +6 -0
  4. package/config.examples.json +11 -0
  5. package/dist/index.js +4037 -0
  6. package/dist/index.js.map +1 -0
  7. package/package.json +98 -0
  8. package/plugins/sources/email.rssany.js +96 -0
  9. package/plugins/sources/rss.rssany.js +83 -0
  10. package/plugins/templates/site.rssany.js +26 -0
  11. package/scripts/reset.mjs +136 -0
  12. package/sources.example.json +562 -0
  13. package/statics/401.html +56 -0
  14. package/statics/404.html +12 -0
  15. package/statics/README.md +7 -0
  16. package/statics/image.png +0 -0
  17. package/webui/build/200.html +51 -0
  18. package/webui/build/_app/env.js +1 -0
  19. package/webui/build/_app/immutable/assets/0.BUAXpTm6.css +1 -0
  20. package/webui/build/_app/immutable/assets/10.I1OuCLrU.css +1 -0
  21. package/webui/build/_app/immutable/assets/11.CrO9xaki.css +1 -0
  22. package/webui/build/_app/immutable/assets/12.BEi6fInA.css +1 -0
  23. package/webui/build/_app/immutable/assets/14.Ctlgn1LZ.css +1 -0
  24. package/webui/build/_app/immutable/assets/2.eJ80XOGm.css +1 -0
  25. package/webui/build/_app/immutable/assets/4.B8-jYAVj.css +1 -0
  26. package/webui/build/_app/immutable/assets/5.ClehBQ0g.css +1 -0
  27. package/webui/build/_app/immutable/assets/6.Drn-0DON.css +1 -0
  28. package/webui/build/_app/immutable/assets/7.ms2diq_q.css +1 -0
  29. package/webui/build/_app/immutable/assets/8.DKymkjjs.css +1 -0
  30. package/webui/build/_app/immutable/assets/9.BZheTlzZ.css +1 -0
  31. package/webui/build/_app/immutable/assets/SourcesList.BhtYlRsQ.css +1 -0
  32. package/webui/build/_app/immutable/chunks/BUApaBEI.js +1 -0
  33. package/webui/build/_app/immutable/chunks/BUngiKFg.js +1 -0
  34. package/webui/build/_app/immutable/chunks/Bfc47y5P.js +1 -0
  35. package/webui/build/_app/immutable/chunks/Bt0fzibd.js +1 -0
  36. package/webui/build/_app/immutable/chunks/BxHqDcpw.js +1 -0
  37. package/webui/build/_app/immutable/chunks/ByQRbEUX.js +1 -0
  38. package/webui/build/_app/immutable/chunks/C12mHcUp.js +6 -0
  39. package/webui/build/_app/immutable/chunks/C1kQ4pHy.js +1 -0
  40. package/webui/build/_app/immutable/chunks/C74gbb4Q.js +1 -0
  41. package/webui/build/_app/immutable/chunks/CAtemnMo.js +1 -0
  42. package/webui/build/_app/immutable/chunks/CBY2biv-.js +1 -0
  43. package/webui/build/_app/immutable/chunks/CVjCNJia.js +1 -0
  44. package/webui/build/_app/immutable/chunks/Cg3zih_x.js +1 -0
  45. package/webui/build/_app/immutable/chunks/CjQQ9_Q2.js +2 -0
  46. package/webui/build/_app/immutable/chunks/CkS2JMkE.js +1 -0
  47. package/webui/build/_app/immutable/chunks/CtHRh_pJ.js +1 -0
  48. package/webui/build/_app/immutable/chunks/D-6mYMI1.js +1 -0
  49. package/webui/build/_app/immutable/chunks/D1Gs8-g3.js +1 -0
  50. package/webui/build/_app/immutable/chunks/D9dRVKgL.js +1 -0
  51. package/webui/build/_app/immutable/chunks/DCEY1XiC.js +1 -0
  52. package/webui/build/_app/immutable/chunks/DI-t-G_K.js +2 -0
  53. package/webui/build/_app/immutable/chunks/DTUxjyWL.js +1 -0
  54. package/webui/build/_app/immutable/chunks/DWJZOHke.js +1 -0
  55. package/webui/build/_app/immutable/chunks/Dgs6d7X5.js +1 -0
  56. package/webui/build/_app/immutable/chunks/DjpPK99f.js +71 -0
  57. package/webui/build/_app/immutable/chunks/DjzVVxpy.js +1 -0
  58. package/webui/build/_app/immutable/chunks/LQVMBmDN.js +1 -0
  59. package/webui/build/_app/immutable/chunks/Qw0Qgx6J.js +1 -0
  60. package/webui/build/_app/immutable/chunks/V2-VOe88.js +1 -0
  61. package/webui/build/_app/immutable/chunks/bohabpgg.js +1 -0
  62. package/webui/build/_app/immutable/chunks/c-YfbAB_.js +8 -0
  63. package/webui/build/_app/immutable/chunks/hp4PFHFv.js +1 -0
  64. package/webui/build/_app/immutable/chunks/tpTQfoNn.js +1 -0
  65. package/webui/build/_app/immutable/entry/app.4I2fqDIL.js +2 -0
  66. package/webui/build/_app/immutable/entry/start.CrgdT2Qb.js +1 -0
  67. package/webui/build/_app/immutable/nodes/0.gA9sQtoM.js +11 -0
  68. package/webui/build/_app/immutable/nodes/1.Bybh7btp.js +1 -0
  69. package/webui/build/_app/immutable/nodes/10.DEkJCZ6X.js +1 -0
  70. package/webui/build/_app/immutable/nodes/11.CDNNJqlQ.js +1 -0
  71. package/webui/build/_app/immutable/nodes/12.D9g8GCjm.js +24 -0
  72. package/webui/build/_app/immutable/nodes/13.DRpZV72T.js +1 -0
  73. package/webui/build/_app/immutable/nodes/14.DVeJW6bd.js +1 -0
  74. package/webui/build/_app/immutable/nodes/15.BtYZF6FM.js +1 -0
  75. package/webui/build/_app/immutable/nodes/16.Ba_qJjp6.js +1 -0
  76. package/webui/build/_app/immutable/nodes/2.DIZ4IPNm.js +1 -0
  77. package/webui/build/_app/immutable/nodes/3.BFSNf0FK.js +1 -0
  78. package/webui/build/_app/immutable/nodes/4.BSsIjejE.js +2 -0
  79. package/webui/build/_app/immutable/nodes/5.COxRT9Oe.js +1 -0
  80. package/webui/build/_app/immutable/nodes/6.CBgQ4YzB.js +1 -0
  81. package/webui/build/_app/immutable/nodes/7.BbzWOL0V.js +6 -0
  82. package/webui/build/_app/immutable/nodes/8.C8120200.js +1 -0
  83. package/webui/build/_app/immutable/nodes/9.BH_BGQQ4.js +1 -0
  84. package/webui/build/_app/version.json +1 -0
package/dist/index.js ADDED
@@ -0,0 +1,4037 @@
1
+ import "dotenv/config";
2
+ import { existsSync, unlinkSync, openSync, writeSync, closeSync, readFileSync, watch } from "node:fs";
3
+ import { platform, homedir, networkInterfaces } from "node:os";
4
+ import { serve } from "@hono/node-server";
5
+ import { Hono } from "hono";
6
+ import { cors } from "hono/cors";
7
+ import { exec } from "node:child_process";
8
+ import { join, dirname, basename, resolve, sep, relative } from "node:path";
9
+ import { promisify } from "node:util";
10
+ import puppeteerCore from "puppeteer-core";
11
+ import { parse, NodeType } from "node-html-parser";
12
+ import Database from "better-sqlite3";
13
+ import { mkdir, copyFile, access, rename, readFile, writeFile, readdir } from "node:fs/promises";
14
+ import { fileURLToPath, pathToFileURL } from "node:url";
15
+ import { createHash, randomUUID } from "node:crypto";
16
+ import { JSDOM } from "jsdom";
17
+ import { Readability } from "@mozilla/readability";
18
+ import OpenAI from "openai";
19
+ import { EventEmitter } from "node:events";
20
+ import { CronExpressionParser } from "cron-parser";
21
+ import { validate, schedule as schedule$1 } from "node-cron";
22
+ import { streamSSE } from "hono/streaming";
23
+ import { serveStatic } from "@hono/node-server/serve-static";
24
+ const TAGS_TO_REMOVE = [
25
+ "script",
26
+ "style",
27
+ "svg",
28
+ "symbol",
29
+ "link",
30
+ "meta",
31
+ "input",
32
+ "embed",
33
+ "button",
34
+ "select",
35
+ "textarea",
36
+ "nav",
37
+ "iframe",
38
+ "noscript",
39
+ "template",
40
+ "object",
41
+ "canvas"
42
+ ];
43
+ const BASE64_IMG_PATTERN = /^data:image\/[^;"'\s]+;base64,/i;
44
+ function collectCommentNodes(node, out) {
45
+ if (node.nodeType === NodeType.COMMENT_NODE) {
46
+ out.push(node);
47
+ return;
48
+ }
49
+ if ("childNodes" in node && Array.isArray(node.childNodes)) {
50
+ for (const child of node.childNodes) {
51
+ collectCommentNodes(child, out);
52
+ }
53
+ }
54
+ }
55
+ function stripRssIrrelevantAttributes(root) {
56
+ const toProcess = [root];
57
+ const all = root.querySelectorAll("*");
58
+ for (const el of all) {
59
+ if (el.nodeType === NodeType.ELEMENT_NODE) toProcess.push(el);
60
+ }
61
+ for (const elem of toProcess) {
62
+ elem.removeAttribute("class");
63
+ elem.removeAttribute("style");
64
+ const src = elem.getAttribute("src");
65
+ if (src && BASE64_IMG_PATTERN.test(src)) {
66
+ elem.removeAttribute("src");
67
+ }
68
+ }
69
+ }
70
+ function applyPurify(html, purify) {
71
+ if (purify === false) return html;
72
+ const root = parse(html, { comment: true });
73
+ for (const tag of TAGS_TO_REMOVE) {
74
+ const list = root.querySelectorAll(tag);
75
+ for (const el of list) {
76
+ el.remove();
77
+ }
78
+ }
79
+ const commentNodes = [];
80
+ collectCommentNodes(root, commentNodes);
81
+ for (const node of commentNodes) {
82
+ node.remove();
83
+ }
84
+ stripRssIrrelevantAttributes(root);
85
+ return root.toString();
86
+ }
87
+ function findChromeExecutable() {
88
+ const platformName = platform();
89
+ const paths = [];
90
+ const envChrome = process.env.CHROME_PATH || process.env.CHROMIUM_PATH;
91
+ if (envChrome) paths.push(envChrome);
92
+ if (platformName === "darwin") {
93
+ paths.push(
94
+ "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
95
+ "/Applications/Chromium.app/Contents/MacOS/Chromium",
96
+ "/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge"
97
+ );
98
+ } else if (platformName === "linux") {
99
+ paths.push(
100
+ "/usr/bin/google-chrome",
101
+ "/usr/bin/google-chrome-stable",
102
+ "/usr/bin/chromium",
103
+ "/usr/bin/chromium-browser",
104
+ "/snap/bin/chromium"
105
+ );
106
+ } else if (platformName === "win32") {
107
+ const programFiles = process.env["ProgramFiles"] || "C:\\Program Files";
108
+ const programFilesX86 = process.env["ProgramFiles(x86)"] || "C:\\Program Files (x86)";
109
+ paths.push(
110
+ join(programFiles, "Google", "Chrome", "Application", "chrome.exe"),
111
+ join(programFilesX86, "Google", "Chrome", "Application", "chrome.exe"),
112
+ join(programFiles, "Microsoft", "Edge", "Application", "msedge.exe"),
113
+ join(programFilesX86, "Microsoft", "Edge", "Application", "msedge.exe")
114
+ );
115
+ }
116
+ for (const p of paths) {
117
+ try {
118
+ if (existsSync(p)) return p;
119
+ } catch {
120
+ }
121
+ }
122
+ return null;
123
+ }
124
+ function getEffectiveItemFields(item, lng) {
125
+ const raw = lng && lng !== "" ? item.translations?.[lng] : void 0;
126
+ const t = raw && typeof raw === "object" ? raw : void 0;
127
+ return {
128
+ title: (t?.title != null && t.title !== "" ? t.title : item.title) ?? "",
129
+ summary: (t?.summary != null && t.summary !== "" ? t.summary : item.summary) ?? "",
130
+ content: (t?.content != null && t.content !== "" ? t.content : item.content) ?? ""
131
+ };
132
+ }
133
+ function normalizeAuthor(author) {
134
+ if (author == null) return void 0;
135
+ if (Array.isArray(author)) return author.filter((s2) => typeof s2 === "string" && s2.trim()).map((s2) => s2.trim());
136
+ const s = String(author).trim();
137
+ return s ? [s] : void 0;
138
+ }
139
+ const PIPELINE_DROP_EXTRA_KEY = "_rssanyPipelineDrop";
140
+ function markPipelineDrop(item) {
141
+ item.extra = { ...item.extra, [PIPELINE_DROP_EXTRA_KEY]: true };
142
+ return item;
143
+ }
144
+ function isPipelineDroppedItem(item) {
145
+ return item.extra?.[PIPELINE_DROP_EXTRA_KEY] === true;
146
+ }
147
+ const __dir = dirname(fileURLToPath(import.meta.url));
148
+ const base = basename(__dir);
149
+ const PACKAGE_ROOT = base === "app" || base === "dist" ? join(__dir, "..") : __dir;
150
+ const envUserDir = process.env.RSSANY_USER_DIR?.trim();
151
+ const USER_DIR = envUserDir && envUserDir.length > 0 ? envUserDir : join(homedir(), ".rssany");
152
+ const DATA_DIR = join(USER_DIR, "data");
153
+ const CACHE_DIR = process.env.CACHE_DIR ?? join(USER_DIR, "cache");
154
+ join(USER_DIR, "sites.json");
155
+ const SOURCES_CONFIG_PATH = join(USER_DIR, "sources.json");
156
+ const TAGS_CONFIG_PATH = join(USER_DIR, "tags.json");
157
+ const CONFIG_PATH = join(USER_DIR, "config.json");
158
+ const LEGACY_SUBSCRIPTIONS_PATH = join(USER_DIR, "subscriptions.json");
159
+ const BUILTIN_PLUGINS_DIR = join(PACKAGE_ROOT, "plugins");
160
+ const USER_PLUGINS_DIR = join(USER_DIR, "plugins");
161
+ const BUILTIN_SOURCES_DIR = join(BUILTIN_PLUGINS_DIR, "sources");
162
+ const USER_SOURCES_DIR = join(USER_PLUGINS_DIR, "sources");
163
+ const BUILTIN_ENRICH_DIR = join(BUILTIN_PLUGINS_DIR, "enrich");
164
+ const USER_ENRICH_DIR = join(USER_PLUGINS_DIR, "enrich");
165
+ async function pathExists(p) {
166
+ try {
167
+ await access(p);
168
+ return true;
169
+ } catch {
170
+ return false;
171
+ }
172
+ }
173
+ async function migrateFile(from, to) {
174
+ if (!await pathExists(from)) return;
175
+ if (await pathExists(to)) return;
176
+ try {
177
+ await rename(from, to);
178
+ logger.info("config", "配置已迁移", { from, to });
179
+ } catch (err) {
180
+ logger.warn("config", "配置迁移失败", { from, to, err: err instanceof Error ? err.message : String(err) });
181
+ }
182
+ }
183
+ const EXAMPLE_SOURCES = join(PACKAGE_ROOT, "sources.example.json");
184
+ const EXAMPLE_CONFIG = join(PACKAGE_ROOT, "config.examples.json");
185
+ async function seedExampleConfigsIfMissing() {
186
+ if (!await pathExists(SOURCES_CONFIG_PATH) && await pathExists(EXAMPLE_SOURCES)) {
187
+ try {
188
+ await copyFile(EXAMPLE_SOURCES, SOURCES_CONFIG_PATH);
189
+ logger.info("config", "已写入默认信源示例", { path: SOURCES_CONFIG_PATH });
190
+ } catch (err) {
191
+ logger.warn("config", "写入 sources 示例失败", {
192
+ err: err instanceof Error ? err.message : String(err)
193
+ });
194
+ }
195
+ }
196
+ if (!await pathExists(CONFIG_PATH) && await pathExists(EXAMPLE_CONFIG)) {
197
+ try {
198
+ await copyFile(EXAMPLE_CONFIG, CONFIG_PATH);
199
+ logger.info("config", "已写入默认配置示例", { path: CONFIG_PATH });
200
+ } catch (err) {
201
+ logger.warn("config", "写入 config 示例失败", {
202
+ err: err instanceof Error ? err.message : String(err)
203
+ });
204
+ }
205
+ }
206
+ }
207
+ async function initUserDir() {
208
+ await mkdir(USER_DIR, { recursive: true });
209
+ await mkdir(DATA_DIR, { recursive: true });
210
+ await mkdir(CACHE_DIR, { recursive: true });
211
+ await mkdir(USER_PLUGINS_DIR, { recursive: true });
212
+ await mkdir(USER_SOURCES_DIR, { recursive: true });
213
+ await mkdir(USER_ENRICH_DIR, { recursive: true });
214
+ await seedExampleConfigsIfMissing();
215
+ if (!await pathExists(SOURCES_CONFIG_PATH) && await pathExists(LEGACY_SUBSCRIPTIONS_PATH)) {
216
+ await migrateFile(LEGACY_SUBSCRIPTIONS_PATH, SOURCES_CONFIG_PATH);
217
+ }
218
+ }
219
+ const MAIN_DB_JOURNAL = (process.env.RSSANY_DB_JOURNAL ?? "wal").toLowerCase() === "delete" ? "DELETE" : "WAL";
220
+ let _db = null;
221
+ let _dbInit = null;
222
+ let _writeLock = Promise.resolve();
223
+ const MAIN_DB_LOCK_PATH = join(DATA_DIR, "rssany.db.lock");
224
+ function logCorruptDiagnostic(operation, err) {
225
+ const code = err?.code;
226
+ const msg = err instanceof Error ? err.message : String(err);
227
+ const lines = [
228
+ "[rssany db] ??????????????",
229
+ ` ?????: ${operation}`,
230
+ ` ????: ${code ?? "unknown"} - ${msg}`,
231
+ " ????????:",
232
+ " 1. ??????/????????????????????????? tsx --watch ?????????????????????????????????????????????",
233
+ " 2. ???????????????????WAL ????? checkpoint",
234
+ " 3. ????/?????/?????????????????????????",
235
+ " ??:",
236
+ " - ??????????????? --watch???????????????????????????????",
237
+ " - ???????????? RSSANY_DB_JOURNAL=delete ?????? DELETE ???????????????????",
238
+ " - ?????? .rssany/data/rssany.db ????? -wal???-shm???rssany.db.lock ??????"
239
+ ];
240
+ process.stderr.write(lines.join("\n") + "\n");
241
+ }
242
+ function acquireDbLock(dbDir) {
243
+ const lockPath = join(dbDir, "rssany.db.lock");
244
+ const pid = process.pid;
245
+ const tryCreate = () => {
246
+ try {
247
+ const fd = openSync(lockPath, "wx");
248
+ writeSync(fd, String(pid), 0, "utf8");
249
+ closeSync(fd);
250
+ return;
251
+ } catch (e) {
252
+ const code = e?.code;
253
+ if (code !== "EEXIST") throw e;
254
+ }
255
+ if (!existsSync(lockPath)) {
256
+ tryCreate();
257
+ return;
258
+ }
259
+ let oldPid = null;
260
+ try {
261
+ const buf = readFileSync(lockPath, "utf8");
262
+ const n = parseInt(buf.trim(), 10);
263
+ if (!Number.isNaN(n)) oldPid = n;
264
+ } catch {
265
+ }
266
+ if (oldPid !== null && oldPid !== pid) {
267
+ const stillRunning = (() => {
268
+ try {
269
+ process.kill(oldPid, 0);
270
+ return true;
271
+ } catch {
272
+ return false;
273
+ }
274
+ })();
275
+ if (stillRunning) {
276
+ throw new Error(
277
+ `???????????????????????????PID ${oldPid}??????????? tsx --watch ?????????????????????????????????? ${lockPath} ??????????`
278
+ );
279
+ }
280
+ }
281
+ try {
282
+ unlinkSync(lockPath);
283
+ } catch {
284
+ }
285
+ tryCreate();
286
+ };
287
+ tryCreate();
288
+ }
289
+ function releaseDbLock() {
290
+ if (!existsSync(MAIN_DB_LOCK_PATH)) return;
291
+ try {
292
+ unlinkSync(MAIN_DB_LOCK_PATH);
293
+ } catch {
294
+ }
295
+ }
296
+ function withWriteLock(fn) {
297
+ const prev = _writeLock;
298
+ let resolveOut;
299
+ let rejectOut;
300
+ const out = new Promise((res, rej) => {
301
+ resolveOut = res;
302
+ rejectOut = rej;
303
+ });
304
+ _writeLock = prev.then(() => fn()).then(
305
+ (v) => {
306
+ resolveOut(v);
307
+ },
308
+ (e) => {
309
+ if (isCorruptError(e)) {
310
+ logCorruptDiagnostic("???????????? updateItemContent/upsertItems??", e);
311
+ }
312
+ rejectOut(e);
313
+ throw e;
314
+ }
315
+ );
316
+ return out;
317
+ }
318
+ const DATE_ONLY_TITLE_RE = /^(?:jan|feb|mar|apr|may|jun|jul|aug|sep|sept|oct|nov|dec)\b[\s\d,??./-]*(?:st|nd|rd|th)?[\s\d,??./-]*$/i;
319
+ function normalizeText(text) {
320
+ return (text ?? "").replace(/\s+/g, " ").trim();
321
+ }
322
+ function isDateOnlyTitle(title) {
323
+ const normalized = normalizeText(title);
324
+ if (!normalized) return false;
325
+ return DATE_ONLY_TITLE_RE.test(normalized);
326
+ }
327
+ function toMs(input) {
328
+ if (!input) return null;
329
+ const ms = Date.parse(input);
330
+ return Number.isNaN(ms) ? null : ms;
331
+ }
332
+ function parseAuthorFromDb(raw) {
333
+ if (!raw?.trim()) return void 0;
334
+ try {
335
+ const p = JSON.parse(raw);
336
+ if (Array.isArray(p)) return p.filter((s) => typeof s === "string").map((s) => String(s).trim()).filter(Boolean);
337
+ return [String(p).trim()];
338
+ } catch {
339
+ return [raw.trim()];
340
+ }
341
+ }
342
+ function toDbItem(row) {
343
+ const author = parseAuthorFromDb(row.author) ?? null;
344
+ const parseJsonArr = (v) => {
345
+ try {
346
+ return v ? JSON.parse(v) : null;
347
+ } catch {
348
+ return null;
349
+ }
350
+ };
351
+ const tags = parseJsonArr(row.tags);
352
+ let translations = null;
353
+ try {
354
+ if (row.translations && typeof row.translations === "string") {
355
+ const p = JSON.parse(row.translations);
356
+ if (p && typeof p === "object") translations = p;
357
+ }
358
+ } catch {
359
+ }
360
+ return { ...row, author, tags, translations };
361
+ }
362
+ function mapRowsToDbItems(rows) {
363
+ return rows.map(toDbItem);
364
+ }
365
+ function isCorruptError(err) {
366
+ const code = err?.code;
367
+ const msg = err instanceof Error ? err.message : String(err);
368
+ return code === "SQLITE_CORRUPT" || code === "SQLITE_CORRUPT_VTAB" || msg.includes("database disk image is malformed");
369
+ }
370
+ async function getDb() {
371
+ if (_db) return _db;
372
+ if (_dbInit) return _dbInit;
373
+ const dbPath = join(DATA_DIR, "rssany.db");
374
+ _dbInit = (async () => {
375
+ await mkdir(DATA_DIR, { recursive: true });
376
+ acquireDbLock(DATA_DIR);
377
+ let db = null;
378
+ try {
379
+ db = new Database(dbPath);
380
+ db.pragma(`journal_mode = ${MAIN_DB_JOURNAL}`);
381
+ db.pragma("synchronous = NORMAL");
382
+ initSchema(db);
383
+ _db = db;
384
+ return db;
385
+ } catch (err) {
386
+ _dbInit = null;
387
+ releaseDbLock();
388
+ if (db) {
389
+ try {
390
+ db.close();
391
+ } catch {
392
+ }
393
+ db = null;
394
+ }
395
+ if (isCorruptError(err)) {
396
+ logCorruptDiagnostic("?????/??????????? (getDb)", err);
397
+ }
398
+ throw err;
399
+ }
400
+ })();
401
+ return _dbInit;
402
+ }
403
+ async function runIntegrityCheck() {
404
+ const db = await getDb();
405
+ try {
406
+ const row = db.prepare("PRAGMA integrity_check").get();
407
+ return row?.integrity_check ?? "unknown";
408
+ } catch (err) {
409
+ const msg = err instanceof Error ? err.message : String(err);
410
+ return `integrity_check ???????: ${msg}`;
411
+ }
412
+ }
413
+ const LOGS_DB_PATH = join(DATA_DIR, "logs.db");
414
+ let _logsDb = null;
415
+ let _logsDbInit = null;
416
+ function initLogsSchema(db) {
417
+ db.exec(`
418
+ CREATE TABLE IF NOT EXISTS logs (
419
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
420
+ level TEXT NOT NULL,
421
+ category TEXT NOT NULL,
422
+ message TEXT NOT NULL,
423
+ payload TEXT,
424
+ source_url TEXT,
425
+ created_at TEXT NOT NULL
426
+ );
427
+ CREATE INDEX IF NOT EXISTS idx_logs_level_created ON logs(level, created_at);
428
+ CREATE INDEX IF NOT EXISTS idx_logs_source_created ON logs(source_url, created_at);
429
+ `);
430
+ }
431
+ async function getLogsDb() {
432
+ if (_logsDb) return _logsDb;
433
+ if (_logsDbInit) return _logsDbInit;
434
+ _logsDbInit = (async () => {
435
+ await mkdir(DATA_DIR, { recursive: true });
436
+ const db = new Database(LOGS_DB_PATH);
437
+ db.pragma("journal_mode = WAL");
438
+ db.pragma("synchronous = NORMAL");
439
+ initLogsSchema(db);
440
+ _logsDb = db;
441
+ return db;
442
+ })();
443
+ return _logsDbInit;
444
+ }
445
+ function initSchema(db) {
446
+ db.exec(`
447
+ CREATE TABLE IF NOT EXISTS items (
448
+ id TEXT PRIMARY KEY,
449
+ url TEXT UNIQUE NOT NULL,
450
+ source_url TEXT NOT NULL,
451
+ title TEXT,
452
+ author TEXT,
453
+ summary TEXT,
454
+ content TEXT,
455
+ image_url TEXT,
456
+ tags TEXT,
457
+ translations TEXT,
458
+ pub_date TEXT,
459
+ fetched_at TEXT NOT NULL,
460
+ pushed_at TEXT
461
+ );
462
+ CREATE INDEX IF NOT EXISTS idx_items_source ON items(source_url);
463
+ CREATE INDEX IF NOT EXISTS idx_items_fetched ON items(fetched_at);
464
+ CREATE INDEX IF NOT EXISTS idx_items_pushed ON items(pushed_at);
465
+ `);
466
+ db.exec(`
467
+ CREATE VIEW IF NOT EXISTS items_fts_src AS
468
+ SELECT rowid, title, summary, content,
469
+ json_extract(translations, '$."zh-CN".title') AS title_zh,
470
+ json_extract(translations, '$."zh-CN".summary') AS summary_zh,
471
+ json_extract(translations, '$."zh-CN".content') AS content_zh
472
+ FROM items;
473
+ CREATE VIRTUAL TABLE IF NOT EXISTS items_fts USING fts5(
474
+ title, summary, content, title_zh, summary_zh, content_zh,
475
+ content='items_fts_src',
476
+ content_rowid='rowid'
477
+ );
478
+ CREATE TRIGGER IF NOT EXISTS items_fts_after_insert AFTER INSERT ON items
479
+ BEGIN
480
+ INSERT INTO items_fts(rowid, title, summary, content, title_zh, summary_zh, content_zh)
481
+ VALUES (
482
+ NEW.rowid, NEW.title, NEW.summary, NEW.content,
483
+ json_extract(NEW.translations, '$."zh-CN".title'),
484
+ json_extract(NEW.translations, '$."zh-CN".summary'),
485
+ json_extract(NEW.translations, '$."zh-CN".content')
486
+ );
487
+ END;
488
+ CREATE TRIGGER IF NOT EXISTS items_fts_after_update AFTER UPDATE ON items
489
+ BEGIN
490
+ INSERT INTO items_fts(items_fts, rowid, title, summary, content, title_zh, summary_zh, content_zh)
491
+ VALUES (
492
+ 'delete', OLD.rowid, OLD.title, OLD.summary, OLD.content,
493
+ json_extract(OLD.translations, '$."zh-CN".title'),
494
+ json_extract(OLD.translations, '$."zh-CN".summary'),
495
+ json_extract(OLD.translations, '$."zh-CN".content')
496
+ );
497
+ INSERT INTO items_fts(rowid, title, summary, content, title_zh, summary_zh, content_zh)
498
+ VALUES (
499
+ NEW.rowid, NEW.title, NEW.summary, NEW.content,
500
+ json_extract(NEW.translations, '$."zh-CN".title'),
501
+ json_extract(NEW.translations, '$."zh-CN".summary'),
502
+ json_extract(NEW.translations, '$."zh-CN".content')
503
+ );
504
+ END;
505
+ CREATE TRIGGER IF NOT EXISTS items_fts_after_delete AFTER DELETE ON items
506
+ BEGIN
507
+ INSERT INTO items_fts(items_fts, rowid, title, summary, content, title_zh, summary_zh, content_zh)
508
+ VALUES (
509
+ 'delete', OLD.rowid, OLD.title, OLD.summary, OLD.content,
510
+ json_extract(OLD.translations, '$."zh-CN".title'),
511
+ json_extract(OLD.translations, '$."zh-CN".summary'),
512
+ json_extract(OLD.translations, '$."zh-CN".content')
513
+ );
514
+ END;
515
+ `);
516
+ try {
517
+ const info = db.prepare("PRAGMA table_info(items)").all();
518
+ if (info && !info.some((c) => c.name === "image_url")) {
519
+ db.exec("ALTER TABLE items ADD COLUMN image_url TEXT");
520
+ }
521
+ } catch {
522
+ }
523
+ }
524
+ async function upsertItems(items, sourceUrlOverride) {
525
+ if (items.length === 0) return { newCount: 0, newIds: /* @__PURE__ */ new Set() };
526
+ const sourceUrl = items[0].sourceRef;
527
+ if (!sourceUrl) {
528
+ throw new Error("upsertItems: ???????? item ???? sourceRef????????? sourceUrlOverride");
529
+ }
530
+ return withWriteLock(async () => {
531
+ const db = await getDb();
532
+ const stmt = db.prepare(`
533
+ INSERT OR IGNORE INTO items (id, url, source_url, title, author, summary, image_url, tags, pub_date, fetched_at)
534
+ VALUES (@id, @url, @sourceUrl, @title, @author, @summary, @imageUrl, @tags, @pubDate, @fetchedAt)
535
+ `);
536
+ const selectExistingStmt = db.prepare(`
537
+ SELECT id, title, author, summary, image_url, pub_date, fetched_at
538
+ FROM items
539
+ WHERE id = @id
540
+ `);
541
+ const repairExistingStmt = db.prepare(`
542
+ UPDATE items
543
+ SET title = @title,
544
+ author = @author,
545
+ summary = @summary,
546
+ image_url = @imageUrl,
547
+ pub_date = @pubDate,
548
+ fetched_at = @fetchedAt
549
+ WHERE id = @id
550
+ `);
551
+ const now2 = (/* @__PURE__ */ new Date()).toISOString();
552
+ let newCount = 0;
553
+ const newIds = /* @__PURE__ */ new Set();
554
+ const run = db.transaction((rows) => {
555
+ for (const item of rows) {
556
+ const nextTitle = normalizeText(item.title) || null;
557
+ const nextSummary = normalizeText(item.summary) || null;
558
+ const nextAuthorArr = normalizeAuthor(item.author);
559
+ const nextAuthor = nextAuthorArr?.length ? JSON.stringify(nextAuthorArr) : null;
560
+ const nextPubDate = item.pubDate instanceof Date ? item.pubDate.toISOString() : item.pubDate ?? null;
561
+ const nextTags = item.tags?.length ? JSON.stringify(item.tags) : null;
562
+ const nextImageUrl = typeof item.imageUrl === "string" && item.imageUrl.trim() ? item.imageUrl.trim() : null;
563
+ const info = stmt.run({
564
+ id: item.guid,
565
+ url: item.link,
566
+ sourceUrl,
567
+ title: nextTitle,
568
+ author: nextAuthor,
569
+ summary: nextSummary,
570
+ imageUrl: nextImageUrl,
571
+ tags: nextTags,
572
+ pubDate: nextPubDate,
573
+ fetchedAt: now2
574
+ });
575
+ newCount += info.changes;
576
+ if (info.changes > 0) newIds.add(item.guid);
577
+ if (info.changes > 0) continue;
578
+ const existing = selectExistingStmt.get({ id: item.guid });
579
+ if (!existing) continue;
580
+ const shouldRepairTitle = !!nextTitle && !isDateOnlyTitle(nextTitle) && (isDateOnlyTitle(existing.title) || !normalizeText(existing.title));
581
+ const shouldRepairSummary = !!nextSummary && normalizeText(existing.summary).length < nextSummary.length;
582
+ const shouldRepairImageUrl = !!nextImageUrl && !existing.image_url?.trim();
583
+ const existingAuthorArr = parseAuthorFromDb(existing.author);
584
+ const shouldRepairAuthor = !!nextAuthorArr?.length && !existingAuthorArr?.length;
585
+ const existingPubDateMs = toMs(existing.pub_date);
586
+ const existingFetchedAtMs = toMs(existing.fetched_at);
587
+ const nextPubDateMs = toMs(nextPubDate);
588
+ const existingPubDateLooksFallback = existingPubDateMs != null && existingFetchedAtMs != null && Math.abs(existingPubDateMs - existingFetchedAtMs) <= 5 * 60 * 1e3;
589
+ const shouldRepairPubDate = nextPubDateMs != null && (existingPubDateMs == null || existingPubDateLooksFallback && nextPubDateMs < existingPubDateMs - 24 * 60 * 60 * 1e3);
590
+ if (!(shouldRepairTitle || shouldRepairSummary || shouldRepairImageUrl || shouldRepairAuthor || shouldRepairPubDate)) {
591
+ continue;
592
+ }
593
+ repairExistingStmt.run({
594
+ id: item.guid,
595
+ title: shouldRepairTitle ? nextTitle : existing.title,
596
+ author: shouldRepairAuthor ? nextAuthor : existing.author ?? null,
597
+ summary: shouldRepairSummary ? nextSummary : existing.summary,
598
+ imageUrl: shouldRepairImageUrl ? nextImageUrl : existing.image_url ?? null,
599
+ pubDate: shouldRepairPubDate ? nextPubDate : existing.pub_date,
600
+ fetchedAt: now2
601
+ });
602
+ }
603
+ });
604
+ run(items);
605
+ return { newCount, newIds };
606
+ });
607
+ }
608
+ async function updateItemContent(item) {
609
+ return withWriteLock(async () => {
610
+ const db = await getDb();
611
+ db.prepare(`
612
+ UPDATE items
613
+ SET content = COALESCE(content, @content),
614
+ image_url = COALESCE(@imageUrl, image_url),
615
+ author = COALESCE(@author, author),
616
+ pub_date = COALESCE(@pubDate, pub_date),
617
+ tags = @tags,
618
+ translations = COALESCE(@translations, translations)
619
+ WHERE id = @id
620
+ `).run({
621
+ id: item.guid,
622
+ content: item.content ?? null,
623
+ imageUrl: typeof item.imageUrl === "string" && item.imageUrl.trim() ? item.imageUrl.trim() : null,
624
+ author: (() => {
625
+ const arr = normalizeAuthor(item.author);
626
+ return arr?.length ? JSON.stringify(arr) : null;
627
+ })(),
628
+ pubDate: item.pubDate instanceof Date ? item.pubDate.toISOString() : item.pubDate ?? null,
629
+ tags: item.tags?.length ? JSON.stringify(item.tags) : null,
630
+ translations: item.translations && Object.keys(item.translations).length > 0 ? JSON.stringify(item.translations) : null
631
+ });
632
+ });
633
+ }
634
+ async function queryFeedItems(sourceUrls, limit, offset, opts) {
635
+ if (sourceUrls.length === 0) return { items: [], hasMore: false };
636
+ const db = await getDb();
637
+ const placeholders = sourceUrls.map((_, i) => `@u${i}`).join(", ");
638
+ const conditions = [`source_url IN (${placeholders})`];
639
+ const params = { lim: limit + 1, off: offset };
640
+ sourceUrls.forEach((url, i) => {
641
+ params[`u${i}`] = url;
642
+ });
643
+ if (opts?.since) {
644
+ conditions.push("COALESCE(pub_date, fetched_at) >= @since");
645
+ params.since = opts.since.length === 10 ? `${opts.since}T00:00:00.000Z` : opts.since;
646
+ }
647
+ if (opts?.until) {
648
+ conditions.push("COALESCE(pub_date, fetched_at) < @until");
649
+ if (opts.until.length === 10) {
650
+ const d = /* @__PURE__ */ new Date(opts.until + "T12:00:00Z");
651
+ d.setUTCDate(d.getUTCDate() + 1);
652
+ params.until = d.toISOString();
653
+ } else {
654
+ params.until = opts.until;
655
+ }
656
+ }
657
+ const where = conditions.length ? `WHERE ${conditions.join(" AND ")}` : "";
658
+ const rows = db.prepare(`
659
+ SELECT * FROM items
660
+ ${where}
661
+ ORDER BY COALESCE(pub_date, fetched_at) DESC
662
+ LIMIT @lim OFFSET @off
663
+ `).all(params);
664
+ const hasMore = rows.length > limit;
665
+ const items = mapRowsToDbItems(hasMore ? rows.slice(0, limit) : rows);
666
+ return { items, hasMore };
667
+ }
668
+ async function queryItems(opts) {
669
+ const db = await getDb();
670
+ const { sourceUrl, sourceUrls, author, q, tags: tagsFilter, limit = 20, offset = 0, since, until } = opts;
671
+ const conditions = [];
672
+ const params = { limit, offset };
673
+ if (sourceUrl) {
674
+ conditions.push("i.source_url = @sourceUrl");
675
+ params.sourceUrl = sourceUrl;
676
+ } else if (sourceUrls && sourceUrls.length > 0) {
677
+ const placeholders = sourceUrls.map((_, i) => `@src${i}`).join(", ");
678
+ conditions.push(`i.source_url IN (${placeholders})`);
679
+ sourceUrls.forEach((s, i) => params[`src${i}`] = s);
680
+ }
681
+ if (author && author.trim().length >= 2) {
682
+ conditions.push("instr(i.author, @author) > 0");
683
+ params.author = author.trim();
684
+ }
685
+ if (q) {
686
+ conditions.push("i.rowid IN (SELECT rowid FROM items_fts WHERE items_fts MATCH @q)");
687
+ params.q = q;
688
+ }
689
+ if (tagsFilter && tagsFilter.length > 0) {
690
+ const trimmed = tagsFilter.filter((t) => typeof t === "string" && t.trim()).map((t) => t.trim());
691
+ if (trimmed.length > 0) {
692
+ const tagConds = trimmed.map((_, i) => `LOWER(TRIM(json_each.value)) = LOWER(@tag${i})`).join(" OR ");
693
+ conditions.push(
694
+ `i.tags IS NOT NULL AND EXISTS (SELECT 1 FROM json_each(i.tags) WHERE ${tagConds})`
695
+ );
696
+ trimmed.forEach((t, i) => {
697
+ params[`tag${i}`] = t;
698
+ });
699
+ }
700
+ }
701
+ if (since) {
702
+ conditions.push("COALESCE(i.pub_date, i.fetched_at) >= @since");
703
+ params.since = since.toISOString();
704
+ }
705
+ if (until) {
706
+ conditions.push("COALESCE(i.pub_date, i.fetched_at) < @until");
707
+ params.until = until.toISOString();
708
+ }
709
+ const where = conditions.length ? `WHERE ${conditions.join(" AND ")}` : "";
710
+ const rows = db.prepare(`
711
+ SELECT i.id, i.url, i.source_url, i.title, i.author, i.summary, i.content, i.tags, i.translations, i.pub_date, i.fetched_at, i.pushed_at
712
+ FROM items i ${where}
713
+ ORDER BY COALESCE(i.pub_date, i.fetched_at) DESC
714
+ LIMIT @limit OFFSET @offset
715
+ `).all(params);
716
+ const { count } = db.prepare(`SELECT COUNT(*) as count FROM items i ${where}`).get(params);
717
+ return { items: mapRowsToDbItems(rows), total: count };
718
+ }
719
+ async function removeTagFromAllItems(tag) {
720
+ const trimmed = String(tag ?? "").trim();
721
+ if (!trimmed) return 0;
722
+ const targetLower = trimmed.toLowerCase();
723
+ return withWriteLock(async () => {
724
+ const db = await getDb();
725
+ const rows = db.prepare("SELECT id, tags FROM items WHERE tags IS NOT NULL AND tags != ''").all();
726
+ const updateStmt = db.prepare("UPDATE items SET tags = @tags WHERE id = @id");
727
+ let count = 0;
728
+ const run = db.transaction(() => {
729
+ for (const row of rows) {
730
+ let itemTags;
731
+ try {
732
+ itemTags = JSON.parse(row.tags);
733
+ } catch {
734
+ continue;
735
+ }
736
+ const filtered = itemTags.filter((t) => String(t).trim().toLowerCase() !== targetLower);
737
+ if (filtered.length === itemTags.length) continue;
738
+ const nextTags = filtered.length > 0 ? JSON.stringify(filtered) : null;
739
+ updateStmt.run({ id: row.id, tags: nextTags });
740
+ count += 1;
741
+ }
742
+ });
743
+ run();
744
+ return count;
745
+ });
746
+ }
747
+ async function markPushed(ids) {
748
+ if (ids.length === 0) return;
749
+ return withWriteLock(async () => {
750
+ const db = await getDb();
751
+ const now2 = (/* @__PURE__ */ new Date()).toISOString();
752
+ const stmt = db.prepare("UPDATE items SET pushed_at = @now WHERE id = @id");
753
+ const run = db.transaction((list) => {
754
+ for (const id of list) stmt.run({ now: now2, id });
755
+ });
756
+ run(ids);
757
+ });
758
+ }
759
+ async function deleteItem(id) {
760
+ if (!id?.trim()) return false;
761
+ return withWriteLock(async () => {
762
+ const db = await getDb();
763
+ const run = db.transaction(() => {
764
+ const row = db.prepare("SELECT rowid FROM items WHERE id = @id").get({ id: id.trim() });
765
+ if (!row) return 0;
766
+ db.prepare("DELETE FROM items_fts WHERE rowid = @rowid").run({ rowid: row.rowid });
767
+ const info = db.prepare("DELETE FROM items WHERE id = @id").run({ id: id.trim() });
768
+ return info.changes;
769
+ });
770
+ return run() > 0;
771
+ });
772
+ }
773
+ async function deleteItemsBySourceUrl(sourceUrl) {
774
+ if (!sourceUrl?.trim()) return 0;
775
+ return withWriteLock(async () => {
776
+ const db = await getDb();
777
+ const info = db.prepare("DELETE FROM items WHERE source_url = @sourceUrl").run({ sourceUrl: sourceUrl.trim() });
778
+ return info.changes;
779
+ });
780
+ }
781
+ async function getPendingPushItems(limit = 100) {
782
+ const db = await getDb();
783
+ const rows = db.prepare(`
784
+ SELECT * FROM items
785
+ WHERE pushed_at IS NULL AND content IS NOT NULL
786
+ ORDER BY fetched_at ASC
787
+ LIMIT @limit
788
+ `).all({ limit });
789
+ return mapRowsToDbItems(rows);
790
+ }
791
+ async function getSourceStats() {
792
+ const db = await getDb();
793
+ return db.prepare(
794
+ "SELECT source_url, COUNT(*) as count, MAX(COALESCE(pub_date, fetched_at)) as latest_at FROM items GROUP BY source_url ORDER BY count DESC"
795
+ ).all();
796
+ }
797
+ async function insertLog(entry) {
798
+ const db = await getLogsDb();
799
+ db.prepare(`
800
+ INSERT INTO logs (level, category, message, payload, source_url, created_at)
801
+ VALUES (@level, @category, @message, @payload, NULL, @created_at)
802
+ `).run({
803
+ level: entry.level,
804
+ category: entry.category,
805
+ message: entry.message,
806
+ payload: entry.payload != null ? JSON.stringify(entry.payload) : null,
807
+ created_at: entry.created_at
808
+ });
809
+ }
810
+ async function queryLogs(opts) {
811
+ const db = await getLogsDb();
812
+ const { level, category, limit = 50, offset = 0, since } = opts;
813
+ const conditions = [];
814
+ const params = { limit, offset };
815
+ if (level) {
816
+ conditions.push("level = @level");
817
+ params.level = level;
818
+ }
819
+ if (category) {
820
+ conditions.push("INSTR(LOWER(category), LOWER(@categoryPattern)) > 0");
821
+ params.categoryPattern = category;
822
+ }
823
+ if (since) {
824
+ conditions.push("created_at >= @since");
825
+ params.since = since.toISOString();
826
+ }
827
+ const where = conditions.length ? `WHERE ${conditions.join(" AND ")}` : "";
828
+ const rows = db.prepare(`
829
+ SELECT id, level, category, message, payload, created_at
830
+ FROM logs ${where}
831
+ ORDER BY created_at DESC
832
+ LIMIT @limit OFFSET @offset
833
+ `).all(params);
834
+ const { count } = db.prepare(`SELECT COUNT(*) as count FROM logs ${where}`).get(params);
835
+ return { items: rows, total: count };
836
+ }
837
+ async function clearAllLogs() {
838
+ const db = await getLogsDb();
839
+ const r = db.prepare("DELETE FROM logs").run();
840
+ return r.changes;
841
+ }
842
+ async function getSystemTags() {
843
+ try {
844
+ const raw = await readFile(TAGS_CONFIG_PATH, "utf-8");
845
+ const parsed = JSON.parse(raw);
846
+ if (!Array.isArray(parsed?.tags)) return [];
847
+ return parsed.tags.filter((t) => typeof t === "string" && t.trim().length > 0).map((t) => t.trim());
848
+ } catch {
849
+ return [];
850
+ }
851
+ }
852
+ async function saveSystemTagsToFile(tags) {
853
+ const list = tags.filter((t) => typeof t === "string" && t.trim()).map((t) => t.trim());
854
+ await writeFile(TAGS_CONFIG_PATH, JSON.stringify({ tags: list }, null, 2), "utf-8");
855
+ }
856
+ async function getSystemTagStats() {
857
+ const systemTags = await getSystemTags();
858
+ if (systemTags.length === 0) return [];
859
+ const db = await getDb();
860
+ const rows = db.prepare("SELECT tags, pub_date, fetched_at FROM items WHERE tags IS NOT NULL AND tags != ''").all();
861
+ const now2 = Date.now();
862
+ const tagMap = /* @__PURE__ */ new Map();
863
+ for (const name of systemTags) {
864
+ tagMap.set(name.toLowerCase(), { count: 0, hotness: 0 });
865
+ }
866
+ for (const row of rows) {
867
+ let itemTags;
868
+ try {
869
+ itemTags = JSON.parse(row.tags);
870
+ } catch {
871
+ continue;
872
+ }
873
+ const pubMs = row.pub_date ? Date.parse(row.pub_date) : null;
874
+ const fetchedMs = Date.parse(row.fetched_at);
875
+ const factor = recencyFactor(pubMs, fetchedMs, now2);
876
+ for (const t of itemTags) {
877
+ const key = String(t).trim().toLowerCase();
878
+ const entry = tagMap.get(key);
879
+ if (entry) {
880
+ entry.count += 1;
881
+ entry.hotness += factor;
882
+ }
883
+ }
884
+ }
885
+ return systemTags.map((name) => {
886
+ const entry = tagMap.get(name.toLowerCase()) ?? { count: 0, hotness: 0 };
887
+ return {
888
+ name,
889
+ count: entry.count,
890
+ hotness: Math.round(entry.hotness * 100) / 100
891
+ };
892
+ });
893
+ }
894
+ function recencyFactor(pubDateMs, fetchedAtMs, nowMs) {
895
+ const ref = pubDateMs ?? fetchedAtMs;
896
+ const daysAgo = (nowMs - ref) / (24 * 60 * 60 * 1e3);
897
+ return 1 / (1 + Math.max(0, daysAgo) / 7);
898
+ }
899
+ async function getSuggestedTags() {
900
+ const systemTags = await getSystemTags();
901
+ const systemLower = new Set(systemTags.map((t) => t.toLowerCase().trim()));
902
+ const db = await getDb();
903
+ const rows = db.prepare("SELECT tags, pub_date, fetched_at FROM items WHERE tags IS NOT NULL AND tags != ''").all();
904
+ const tagMap = /* @__PURE__ */ new Map();
905
+ const now2 = Date.now();
906
+ for (const row of rows) {
907
+ let tags;
908
+ try {
909
+ tags = JSON.parse(row.tags);
910
+ } catch {
911
+ continue;
912
+ }
913
+ const pubMs = row.pub_date ? Date.parse(row.pub_date) : null;
914
+ const fetchedMs = Date.parse(row.fetched_at);
915
+ const factor = recencyFactor(pubMs, fetchedMs, now2);
916
+ for (const t of tags) {
917
+ const trimmed = String(t).trim();
918
+ if (!trimmed) continue;
919
+ const key = trimmed.toLowerCase();
920
+ if (systemLower.has(key)) continue;
921
+ const existing = tagMap.get(key);
922
+ if (existing) {
923
+ existing.count += 1;
924
+ existing.hotness += factor;
925
+ } else {
926
+ tagMap.set(key, { name: trimmed, count: 1, hotness: factor });
927
+ }
928
+ }
929
+ }
930
+ return Array.from(tagMap.values()).filter((s) => s.hotness > 20).sort((a, b) => b.hotness - a.hotness).slice(0, 5).map((s) => ({ name: s.name, count: s.count, hotness: Math.round(s.hotness * 100) / 100 }));
931
+ }
932
+ function getLogToDb() {
933
+ const v = process.env.LOG_TO_DB;
934
+ if (v === "0" || v === "false") return false;
935
+ if (v === "1" || v === "true") return true;
936
+ return true;
937
+ }
938
+ function now() {
939
+ return (/* @__PURE__ */ new Date()).toISOString();
940
+ }
941
+ function writeDb(entry) {
942
+ insertLog(entry).catch((err) => {
943
+ process.stderr.write(`[logger] 写入日志表失败: ${err instanceof Error ? err.message : String(err)}
944
+ `);
945
+ });
946
+ }
947
+ function emit(level, category, message, meta) {
948
+ const payload = meta && Object.keys(meta).length > 0 ? { ...meta } : void 0;
949
+ const entry = {
950
+ level,
951
+ category,
952
+ message,
953
+ payload: payload && Object.keys(payload).length > 0 ? payload : void 0,
954
+ created_at: now()
955
+ };
956
+ if (getLogToDb()) {
957
+ writeDb(entry);
958
+ }
959
+ }
960
+ const logger = {
961
+ error(category, message, meta) {
962
+ emit("error", category, message, meta);
963
+ },
964
+ warn(category, message, meta) {
965
+ emit("warn", category, message, meta);
966
+ },
967
+ info(category, message, meta) {
968
+ emit("info", category, message, meta);
969
+ },
970
+ debug(category, message, meta) {
971
+ emit("debug", category, message, meta);
972
+ }
973
+ };
974
+ const execAsync = promisify(exec);
975
+ function resolveProxy(config) {
976
+ return config?.proxy ?? process.env.HTTP_PROXY ?? process.env.HTTPS_PROXY;
977
+ }
978
+ function parseProxy(proxy) {
979
+ const u = new URL(proxy);
980
+ const serverUrl = u.port ? `${u.protocol}//${u.hostname}:${u.port}` : `${u.protocol}//${u.hostname}`;
981
+ const username = u.username || void 0;
982
+ const password = u.password || void 0;
983
+ return { serverUrl, username, password };
984
+ }
985
+ function launchArgs(config) {
986
+ const base2 = [
987
+ "--disable-blink-features=AutomationControlled",
988
+ "--no-sandbox",
989
+ "--disable-setuid-sandbox",
990
+ "--disable-dev-shm-usage",
991
+ "--disable-web-security",
992
+ "--disable-features=IsolateOrigins,site-per-process",
993
+ "--disable-site-isolation-trials",
994
+ "--disable-infobars"
995
+ ];
996
+ const height = config?.headless !== false ? 5e3 : 960;
997
+ base2.push(`--window-size=1366,${height}`);
998
+ const proxy = resolveProxy(config);
999
+ if (proxy) {
1000
+ const { serverUrl } = parseProxy(proxy);
1001
+ base2.push(`--proxy-server=${serverUrl}`);
1002
+ }
1003
+ return base2;
1004
+ }
1005
+ function getUserDataDir(cacheDir) {
1006
+ if (!cacheDir) return void 0;
1007
+ return join(cacheDir, "browser_data", "main");
1008
+ }
1009
+ function isAlreadyRunningError(e) {
1010
+ const msg = e instanceof Error ? e.message : String(e);
1011
+ return /already running/i.test(msg) && /userDataDir|user-data-dir|user data dir/i.test(msg);
1012
+ }
1013
+ async function killStaleChromeProcesses(absUserDataDir) {
1014
+ const plat = platform();
1015
+ if (plat !== "darwin" && plat !== "linux") {
1016
+ return;
1017
+ }
1018
+ try {
1019
+ const psCmd = plat === "darwin" ? `ps -eww -o pid= -o args= 2>/dev/null` : `ps -eo pid,args --no-headers 2>/dev/null`;
1020
+ const { stdout } = await execAsync(psCmd, { maxBuffer: 4 * 1024 * 1024 });
1021
+ const pids = /* @__PURE__ */ new Set();
1022
+ const lineRegex = /^\s*(\d+)\s+/;
1023
+ for (const line of stdout.split("\n")) {
1024
+ if (!line.includes(absUserDataDir)) continue;
1025
+ const m = line.match(lineRegex);
1026
+ if (m) pids.add(parseInt(m[1], 10));
1027
+ }
1028
+ if (pids.size === 0) return;
1029
+ logger.info("scraper", "发现占用 browser_data 的 Chrome 进程,正在结束", { pids: [...pids], userDataDir: absUserDataDir });
1030
+ for (const pid of pids) {
1031
+ try {
1032
+ process.kill(pid, "SIGTERM");
1033
+ } catch {
1034
+ }
1035
+ }
1036
+ await new Promise((r) => setTimeout(r, 800));
1037
+ for (const pid of pids) {
1038
+ try {
1039
+ process.kill(pid, "SIGKILL");
1040
+ } catch {
1041
+ }
1042
+ }
1043
+ await new Promise((r) => setTimeout(r, 300));
1044
+ } catch (err) {
1045
+ logger.warn("scraper", "结束残留 Chrome 进程时出错", { err: err instanceof Error ? err.message : String(err) });
1046
+ }
1047
+ }
1048
+ async function stealthPage(page) {
1049
+ await page.evaluateOnNewDocument(() => {
1050
+ Object.defineProperty(navigator, "webdriver", { get: () => false });
1051
+ Object.defineProperty(navigator, "plugins", { get: () => [1, 2, 3, 4, 5] });
1052
+ Object.defineProperty(navigator, "languages", { get: () => ["zh-CN", "zh", "en"] });
1053
+ const originalQuery = window.navigator.permissions.query;
1054
+ window.navigator.permissions.query = (parameters) => parameters.name === "notifications" ? Promise.resolve({ state: Notification.permission }) : originalQuery(parameters);
1055
+ window.chrome = { runtime: {} };
1056
+ Object.defineProperty(Notification, "permission", { get: () => "default" });
1057
+ const nav = navigator;
1058
+ if (nav.getBattery) {
1059
+ nav.getBattery = () => Promise.resolve({ charging: true, chargingTime: 0, dischargingTime: Infinity, level: 1 });
1060
+ }
1061
+ });
1062
+ await page.setExtraHTTPHeaders({
1063
+ "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8",
1064
+ "Accept-Encoding": "gzip, deflate, br",
1065
+ "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
1066
+ "Cache-Control": "max-age=0",
1067
+ "Sec-Ch-Ua": '"Google Chrome";v="131", "Chromium";v="131", "Not_A Brand";v="24"',
1068
+ "Sec-Ch-Ua-Mobile": "?0",
1069
+ "Sec-Ch-Ua-Platform": '"macOS"',
1070
+ "Sec-Fetch-Dest": "document",
1071
+ "Sec-Fetch-Mode": "navigate",
1072
+ "Sec-Fetch-Site": "none",
1073
+ "Sec-Fetch-User": "?1",
1074
+ "Upgrade-Insecure-Requests": "1"
1075
+ });
1076
+ }
1077
+ function headersToRecord(headers) {
1078
+ const out = {};
1079
+ for (const [k, v] of Object.entries(headers)) {
1080
+ out[k.toLowerCase()] = String(v);
1081
+ }
1082
+ return out;
1083
+ }
1084
+ async function setupPage(page, headless = true) {
1085
+ const realUserAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36";
1086
+ await page.setUserAgent(realUserAgent);
1087
+ await page.setViewport({ width: 1366, height: headless ? 5e3 : 960 });
1088
+ await stealthPage(page);
1089
+ }
1090
+ let _browser = null;
1091
+ let _browserHeadless = true;
1092
+ let _launchPromise = null;
1093
+ function isFrameDetachedError(e) {
1094
+ const msg = e instanceof Error ? e.message : String(e);
1095
+ return /detached|Navigating frame was detached|Session closed/i.test(msg);
1096
+ }
1097
+ async function isBrowserAlive() {
1098
+ if (!_browser) return false;
1099
+ try {
1100
+ await _browser.version();
1101
+ return true;
1102
+ } catch {
1103
+ _browser = null;
1104
+ return false;
1105
+ }
1106
+ }
1107
+ async function getOrCreateBrowser(config) {
1108
+ const wantHeadless = config.headless !== false;
1109
+ if (await isBrowserAlive()) {
1110
+ if (_browserHeadless === wantHeadless) {
1111
+ return _browser;
1112
+ }
1113
+ logger.info("scraper", "浏览器切换模式", { from: _browserHeadless ? "无头" : "有头", to: wantHeadless ? "无头" : "有头" });
1114
+ await _browser.close().catch(() => {
1115
+ });
1116
+ _browser = null;
1117
+ _launchPromise = null;
1118
+ }
1119
+ if (!_launchPromise) {
1120
+ _launchPromise = (async () => {
1121
+ const executablePath = config.chromeExecutablePath ?? process.env.CHROME_PATH ?? findChromeExecutable();
1122
+ if (!executablePath) {
1123
+ throw new Error("未找到 Chrome 可执行文件,请安装 Google Chrome 或设置 CHROME_PATH 环境变量");
1124
+ }
1125
+ const userDataDir = getUserDataDir(config.cacheDir);
1126
+ const maxRetries = 2;
1127
+ let lastErr;
1128
+ for (let attempt = 0; attempt <= maxRetries; attempt++) {
1129
+ try {
1130
+ if (attempt === 0 && userDataDir) {
1131
+ const absUserDataDir = resolve(userDataDir);
1132
+ await killStaleChromeProcesses(absUserDataDir);
1133
+ }
1134
+ if (attempt > 0) {
1135
+ const waitMs = attempt * 2e3;
1136
+ logger.info("scraper", "userDataDir 曾被占用,等待后重试", { waitMs, attempt });
1137
+ await new Promise((r) => setTimeout(r, waitMs));
1138
+ }
1139
+ logger.info("scraper", "启动 Chrome", { headless: wantHeadless, executablePath });
1140
+ const browser = await puppeteerCore.launch({
1141
+ headless: wantHeadless,
1142
+ args: launchArgs({ proxy: config.proxy, headless: wantHeadless }),
1143
+ userDataDir,
1144
+ executablePath,
1145
+ ignoreDefaultArgs: ["--enable-automation"]
1146
+ });
1147
+ browser.on("disconnected", () => {
1148
+ _browser = null;
1149
+ _launchPromise = null;
1150
+ });
1151
+ _browser = browser;
1152
+ _browserHeadless = wantHeadless;
1153
+ return browser;
1154
+ } catch (e) {
1155
+ lastErr = e;
1156
+ if (attempt < maxRetries && isAlreadyRunningError(e)) {
1157
+ continue;
1158
+ }
1159
+ if (isAlreadyRunningError(e)) {
1160
+ const dir = userDataDir ?? "browser_data/main";
1161
+ throw new Error(
1162
+ `Chrome 的 profile 目录已被占用(${dir})。通常是因为上次未正常退出或同时运行了多个本服务实例。请关闭占用该目录的 Chrome 进程后重试,或设置环境变量 CACHE_DIR 使用不同缓存目录。`
1163
+ );
1164
+ }
1165
+ throw e;
1166
+ }
1167
+ }
1168
+ throw lastErr;
1169
+ })().catch((e) => {
1170
+ _launchPromise = null;
1171
+ throw e;
1172
+ });
1173
+ }
1174
+ return _launchPromise;
1175
+ }
1176
+ process.once("exit", () => {
1177
+ _browser?.close().catch(() => {
1178
+ });
1179
+ });
1180
+ process.once("SIGINT", async () => {
1181
+ await _browser?.close().catch(() => {
1182
+ });
1183
+ process.exit(0);
1184
+ });
1185
+ process.once("SIGTERM", async () => {
1186
+ await _browser?.close().catch(() => {
1187
+ });
1188
+ process.exit(0);
1189
+ });
1190
+ async function preCheckAuth(authFlow, cacheDir) {
1191
+ const { checkAuth, loginUrl, domain } = authFlow;
1192
+ if (domain == null || !cacheDir) return true;
1193
+ const browser = await getOrCreateBrowser({ headless: true, cacheDir });
1194
+ const page = await browser.newPage();
1195
+ try {
1196
+ await setupPage(page, true);
1197
+ await page.goto(loginUrl, { waitUntil: "domcontentloaded", timeout: 6e4 });
1198
+ await new Promise((resolve2) => setTimeout(resolve2, 3e3));
1199
+ return await checkAuth(page, page.url());
1200
+ } finally {
1201
+ await page.close().catch(() => {
1202
+ });
1203
+ }
1204
+ }
1205
+ async function ensureAuth(authFlow, cacheDir) {
1206
+ const { checkAuth, loginUrl, loginTimeoutMs = 60 * 1e3, pollIntervalMs = 2e3 } = authFlow;
1207
+ const browser = await getOrCreateBrowser({ headless: false, cacheDir });
1208
+ const page = await browser.newPage();
1209
+ try {
1210
+ await setupPage(page, false);
1211
+ await page.goto(loginUrl, { waitUntil: "domcontentloaded", timeout: 6e4 });
1212
+ await new Promise((resolve2) => setTimeout(resolve2, 3e3));
1213
+ const authenticated = await checkAuth(page, page.url());
1214
+ if (authenticated) return;
1215
+ const startTime = Date.now();
1216
+ while (Date.now() - startTime < loginTimeoutMs) {
1217
+ await new Promise((resolve2) => setTimeout(resolve2, pollIntervalMs));
1218
+ const authenticated2 = await checkAuth(page, page.url());
1219
+ if (authenticated2) return;
1220
+ }
1221
+ throw new Error(`登录超时(${loginTimeoutMs}ms)`);
1222
+ } finally {
1223
+ await page.close().catch(() => {
1224
+ });
1225
+ }
1226
+ }
1227
+ async function fetchHtml(url, config = {}) {
1228
+ const { timeoutMs, headers, cookies, cacheDir, checkAuth, authFlow, purify, headless, waitAfterLoadMs, waitForSelector, waitForSelectorTimeoutMs } = config;
1229
+ const isHeadless = headless !== false;
1230
+ const browser = await getOrCreateBrowser({
1231
+ headless: isHeadless,
1232
+ cacheDir,
1233
+ proxy: resolveProxy(config),
1234
+ chromeExecutablePath: config.chromeExecutablePath
1235
+ });
1236
+ const navigationTimeout = timeoutMs ?? 6e4;
1237
+ const maxAttempts = 2;
1238
+ let lastError;
1239
+ for (let attempt = 0; attempt < maxAttempts; attempt++) {
1240
+ const page = await browser.newPage();
1241
+ const isRetry = attempt === 1;
1242
+ const waitUntil = isRetry ? "domcontentloaded" : "load";
1243
+ const extraWaitMs = isRetry ? Math.min(500, Math.max(0, waitAfterLoadMs ?? 2e3)) : Math.max(0, waitAfterLoadMs ?? 2e3);
1244
+ try {
1245
+ if (config.browserContext) {
1246
+ await config.browserContext(page.browserContext());
1247
+ }
1248
+ await setupPage(page, isHeadless);
1249
+ const extraHeaders = { "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8", ...headers ?? {} };
1250
+ if (cookies != null && cookies !== "") {
1251
+ extraHeaders.cookie = cookies;
1252
+ }
1253
+ await page.setExtraHTTPHeaders(extraHeaders);
1254
+ const proxy = resolveProxy(config);
1255
+ if (proxy) {
1256
+ const { username, password } = parseProxy(proxy);
1257
+ if (username !== void 0 || password !== void 0) {
1258
+ await page.authenticate({ username: username ?? "", password: password ?? "" });
1259
+ }
1260
+ }
1261
+ if (timeoutMs != null) {
1262
+ await page.setDefaultNavigationTimeout(timeoutMs);
1263
+ }
1264
+ const response = await page.goto(url, { waitUntil, timeout: navigationTimeout });
1265
+ if (extraWaitMs > 0) {
1266
+ await new Promise((resolve2) => setTimeout(resolve2, extraWaitMs));
1267
+ }
1268
+ if (waitForSelector != null && waitForSelector !== "" && !isRetry) {
1269
+ const selectorTimeout = waitForSelectorTimeoutMs ?? 2e4;
1270
+ await page.waitForSelector(waitForSelector, { timeout: selectorTimeout });
1271
+ }
1272
+ if (checkAuth != null || authFlow != null) {
1273
+ const authCheck = checkAuth ?? authFlow?.checkAuth;
1274
+ if (authCheck != null) {
1275
+ const ok = await authCheck(page, url);
1276
+ if (!ok) {
1277
+ throw new Error("checkAuth failed: 未通过认证检查,请先调用 ensureAuth 进行预处理登录");
1278
+ }
1279
+ }
1280
+ }
1281
+ const rawBody = await page.content();
1282
+ const finalUrl = response?.url() ?? page.url() ?? String(url);
1283
+ const status = response?.status() ?? 0;
1284
+ const statusText = response?.statusText() ?? "";
1285
+ const rawHeaders = response?.headers() ?? {};
1286
+ const normalizedHeaders = headersToRecord(rawHeaders);
1287
+ const body = applyPurify(rawBody, purify);
1288
+ await page.close().catch(() => {
1289
+ });
1290
+ return { finalUrl, status, statusText, headers: normalizedHeaders, body };
1291
+ } catch (e) {
1292
+ lastError = e;
1293
+ await page.close().catch(() => {
1294
+ });
1295
+ if (isRetry || !isFrameDetachedError(e)) {
1296
+ throw e;
1297
+ }
1298
+ logger.warn("scraper", "fetchHtml 因 frame 分离重试", { url, attempt: attempt + 1, err: e instanceof Error ? e.message : String(e) });
1299
+ await new Promise((r) => setTimeout(r, 800));
1300
+ }
1301
+ }
1302
+ throw lastError;
1303
+ }
1304
+ const VALID_INTERVALS = ["1min", "5min", "10min", "30min", "1h", "6h", "12h", "1day", "3day", "7day"];
1305
+ function cronToRefreshInterval(cronExpr) {
1306
+ const parts = cronExpr.trim().split(/\s+/);
1307
+ const is6Field = parts.length >= 6;
1308
+ const minute = parts[is6Field ? 1 : 0];
1309
+ const hour = parts[is6Field ? 2 : 1];
1310
+ const day = parts[is6Field ? 3 : 2];
1311
+ const weekday = parts[is6Field ? 5 : 4];
1312
+ const minMatch = minute.match(/^\*\/(\d+)$/);
1313
+ if (minute === "*" || minMatch && parseInt(minMatch[1], 10) <= 1) return "1min";
1314
+ if (minMatch) {
1315
+ const n = parseInt(minMatch[1], 10);
1316
+ if (n <= 5) return "5min";
1317
+ if (n <= 10) return "10min";
1318
+ if (n <= 30) return "30min";
1319
+ }
1320
+ if (minute === "0" || minute === "00") {
1321
+ const hourMatch = hour.match(/^\*\/(\d+)$/);
1322
+ if (hour === "*" || hourMatch && parseInt(hourMatch[1], 10) <= 1) return "1h";
1323
+ if (hourMatch) {
1324
+ const n = parseInt(hourMatch[1], 10);
1325
+ if (n <= 2) return "1h";
1326
+ if (n <= 6) return "6h";
1327
+ if (n <= 12) return "12h";
1328
+ }
1329
+ if (day === "*" || day === "?") return "1day";
1330
+ if (weekday !== "*" && weekday !== "?") return "7day";
1331
+ }
1332
+ return "1h";
1333
+ }
1334
+ function refreshIntervalToCron(interval) {
1335
+ const map = {
1336
+ "1min": "* * * * *",
1337
+ "5min": "*/5 * * * *",
1338
+ "10min": "*/10 * * * *",
1339
+ "30min": "*/30 * * * *",
1340
+ "1h": "0 * * * *",
1341
+ "6h": "0 */6 * * *",
1342
+ "12h": "0 0,12 * * *",
1343
+ "1day": "0 0 * * *",
1344
+ "3day": "0 0 */3 * *",
1345
+ "7day": "0 0 * * 0"
1346
+ };
1347
+ return map[interval];
1348
+ }
1349
+ function urlHash(url) {
1350
+ return createHash("sha256").update(url).digest("hex");
1351
+ }
1352
+ function timeBucket(strategy, now2) {
1353
+ const y = now2.getUTCFullYear();
1354
+ const mo = String(now2.getUTCMonth() + 1).padStart(2, "0");
1355
+ const d = String(now2.getUTCDate()).padStart(2, "0");
1356
+ const h = now2.getUTCHours();
1357
+ const hs = String(h).padStart(2, "0");
1358
+ const min = now2.getUTCMinutes();
1359
+ if (strategy === "1min") return `${y}-${mo}-${d}T${hs}:${String(min).padStart(2, "0")}`;
1360
+ if (strategy === "5min") return `${y}-${mo}-${d}T${hs}:${String(Math.floor(min / 5) * 5).padStart(2, "0")}`;
1361
+ if (strategy === "10min") return `${y}-${mo}-${d}T${hs}:${String(Math.floor(min / 10) * 10).padStart(2, "0")}`;
1362
+ if (strategy === "30min") return `${y}-${mo}-${d}T${hs}:${min < 30 ? "00" : "30"}`;
1363
+ if (strategy === "1h") return `${y}-${mo}-${d}T${hs}`;
1364
+ if (strategy === "6h") return `${y}-${mo}-${d}T${String(Math.floor(h / 6) * 6).padStart(2, "0")}`;
1365
+ if (strategy === "12h") return `${y}-${mo}-${d}T${h < 12 ? "00" : "12"}`;
1366
+ if (strategy === "1day") return `${y}-${mo}-${d}`;
1367
+ if (strategy === "3day") {
1368
+ const epochDay = Math.floor(now2.getTime() / 864e5);
1369
+ return `d${Math.floor(epochDay / 3) * 3}`;
1370
+ }
1371
+ if (strategy === "7day") {
1372
+ const adjusted = new Date(now2);
1373
+ adjusted.setUTCHours(0, 0, 0, 0);
1374
+ adjusted.setUTCDate(adjusted.getUTCDate() + 3 - (adjusted.getUTCDay() + 6) % 7);
1375
+ const week1 = new Date(Date.UTC(adjusted.getUTCFullYear(), 0, 4));
1376
+ const isoYear = adjusted.getUTCFullYear();
1377
+ const isoWeek = 1 + Math.round(((adjusted.getTime() - week1.getTime()) / 864e5 - 3 + (week1.getUTCDay() + 6) % 7) / 7);
1378
+ return `${isoYear}-W${String(isoWeek).padStart(2, "0")}`;
1379
+ }
1380
+ return "";
1381
+ }
1382
+ function cacheKey(url, strategy = "forever", now2 = /* @__PURE__ */ new Date()) {
1383
+ const hash = urlHash(url);
1384
+ if (strategy === "forever") return hash;
1385
+ return `${timeBucket(strategy, now2)}-${hash}`;
1386
+ }
1387
+ function cacheKeyFromCron(url, cron, now2 = /* @__PURE__ */ new Date()) {
1388
+ return cacheKey(url, cronToRefreshInterval(cron), now2);
1389
+ }
1390
+ const EXTRACTED_SUBDIR = "extracted";
1391
+ async function readCachedExtracted(cacheDir, key) {
1392
+ const filePath = join(cacheDir, EXTRACTED_SUBDIR, `${key}.json`);
1393
+ try {
1394
+ const raw = await readFile(filePath, "utf-8");
1395
+ const parsed = JSON.parse(raw);
1396
+ return {
1397
+ author: parsed.author,
1398
+ title: parsed.title,
1399
+ summary: parsed.summary,
1400
+ content: parsed.content ?? parsed.contentMarkdown ?? parsed.contentHtml,
1401
+ pubDate: parsed.pubDate
1402
+ };
1403
+ } catch {
1404
+ return null;
1405
+ }
1406
+ }
1407
+ async function writeCachedExtracted(cacheDir, key, result) {
1408
+ const dir = join(cacheDir, EXTRACTED_SUBDIR);
1409
+ await mkdir(dir, { recursive: true });
1410
+ const filePath = join(dir, `${key}.json`);
1411
+ const cache = { ...result, cachedAt: (/* @__PURE__ */ new Date()).toISOString() };
1412
+ await writeFile(filePath, JSON.stringify(cache, null, 2), "utf-8");
1413
+ }
1414
+ function extractedCacheKey(url, config) {
1415
+ if (config.cacheKey != null && config.cacheKey !== "") return config.cacheKey;
1416
+ if (url) return cacheKey(url, "forever");
1417
+ return "";
1418
+ }
1419
+ function extractWithReadability(html, url) {
1420
+ const dom = new JSDOM(html, { url });
1421
+ const reader = new Readability(dom.window.document);
1422
+ const article = reader.parse();
1423
+ if (!article) return {};
1424
+ const summary = article.excerpt && article.excerpt.length > 200 ? article.excerpt.slice(0, 200) + "…" : article.excerpt;
1425
+ return {
1426
+ author: article.byline || void 0,
1427
+ title: article.title || void 0,
1428
+ summary: summary || void 0,
1429
+ content: article.content || void 0
1430
+ };
1431
+ }
1432
+ async function extractHtml(html, config = {}) {
1433
+ const { url = "", mode, customExtractor, cacheDir, useCache = true } = config;
1434
+ const key = extractedCacheKey(url, config);
1435
+ if (useCache !== false && cacheDir != null && cacheDir !== "" && key) {
1436
+ const cached = await readCachedExtracted(cacheDir, key);
1437
+ if (cached != null) return cached;
1438
+ }
1439
+ if (customExtractor != null) {
1440
+ const result = await Promise.resolve(customExtractor(html, url));
1441
+ if (cacheDir != null && cacheDir !== "" && key) {
1442
+ await writeCachedExtracted(cacheDir, key, result);
1443
+ }
1444
+ return result;
1445
+ }
1446
+ if (mode === "readability") {
1447
+ const result = extractWithReadability(html, url);
1448
+ if (cacheDir != null && cacheDir !== "" && key) {
1449
+ await writeCachedExtracted(cacheDir, key, result);
1450
+ }
1451
+ return result;
1452
+ }
1453
+ return {};
1454
+ }
1455
+ async function extractFromLink(link, extractorConfig = {}, fetchConfig = {}) {
1456
+ const { cacheDir } = extractorConfig;
1457
+ const fetchOpts = {
1458
+ cacheDir: fetchConfig.cacheDir ?? cacheDir,
1459
+ useCache: false,
1460
+ timeoutMs: fetchConfig.timeoutMs ?? 15e3,
1461
+ ...fetchConfig
1462
+ };
1463
+ const res = await fetchHtml(link, fetchOpts);
1464
+ if (res.status !== 200 && res.status !== 304) {
1465
+ throw new Error(`fetch 失败: ${res.status} ${res.statusText} for ${link}`);
1466
+ }
1467
+ const finalUrl = res.finalUrl ?? link;
1468
+ return extractHtml(res.body, {
1469
+ ...extractorConfig,
1470
+ url: finalUrl,
1471
+ cacheKey: extractorConfig.cacheKey ?? (cacheDir ? cacheKey(link, "forever") : void 0)
1472
+ });
1473
+ }
1474
+ function getLLMConfig() {
1475
+ return {
1476
+ apiKey: process.env.OPENAI_API_KEY,
1477
+ baseUrl: process.env.OPENAI_BASE_URL || "https://api.openai.com/v1",
1478
+ model: process.env.OPENAI_MODEL || "gpt-4o-mini"
1479
+ };
1480
+ }
1481
+ function mergeConfig(override) {
1482
+ const env = getLLMConfig();
1483
+ const apiKey = override?.apiKey ?? env.apiKey;
1484
+ const baseUrl = override?.apiUrl ?? override?.baseUrl ?? env.baseUrl;
1485
+ const model = override?.model ?? env.model;
1486
+ if (!apiKey) throw new Error("LLM API Key 未配置,请设置 OPENAI_API_KEY 或传入 apiKey");
1487
+ return { apiKey, baseUrl, model };
1488
+ }
1489
+ async function chatJson(prompt, config, options) {
1490
+ const { apiKey, baseUrl, model } = mergeConfig(config);
1491
+ const openai = new OpenAI({ apiKey, baseURL: baseUrl });
1492
+ const completion = await openai.chat.completions.create({
1493
+ model,
1494
+ messages: [{ role: "user", content: prompt }],
1495
+ max_tokens: options?.maxTokens ?? 8192,
1496
+ response_format: { type: "json_object" }
1497
+ });
1498
+ const content = completion.choices[0]?.message?.content;
1499
+ if (!content) throw new Error("LLM 返回空内容");
1500
+ return JSON.parse(content);
1501
+ }
1502
+ async function chatText(prompt, config, options) {
1503
+ const { apiKey, baseUrl, model } = mergeConfig(config);
1504
+ const openai = new OpenAI({ apiKey, baseURL: baseUrl });
1505
+ const completion = await openai.chat.completions.create({
1506
+ model,
1507
+ messages: [{ role: "user", content: prompt }],
1508
+ max_tokens: options?.maxTokens ?? 8192
1509
+ });
1510
+ const content = completion.choices[0]?.message?.content;
1511
+ if (!content) throw new Error("LLM 返回空内容");
1512
+ return content;
1513
+ }
1514
+ function generateGuid(link) {
1515
+ return createHash("sha256").update(link).digest("hex");
1516
+ }
1517
+ async function parseWithLLM(html, url, config) {
1518
+ const prompt = config.prompt ?? `你是一个专业的 HTML 内容提取助手。请仔细分析以下 HTML 页面,提取所有可点击的内容条目(如文章、笔记、帖子、视频等)。
1519
+
1520
+ 要求:
1521
+ 1. 返回 JSON 对象,格式为 {"items": [...]}
1522
+ 2. 每个条目必须包含以下字段:
1523
+ - title: 标题(必填)
1524
+ - link: 完整链接(必填)。**必须从 HTML 的 href 原样提取**:若为绝对路径(以 / 开头)则直接使用;若为相对路径则补全为完整 URL。当前页: ${url}
1525
+ - description: 摘要或描述(200字内,必填)
1526
+ - author: 作者(可选)
1527
+ - published: 发布日期 ISO 字符串(可选)
1528
+ 3. 如果页面是列表页,提取所有列表项
1529
+ 4. 如果页面是详情页,将整个页面作为一个条目提取
1530
+ 5. 如果页面是用户主页/个人资料页:
1531
+ - 优先尝试提取该用户发布的内容列表
1532
+ - 如果无法提取列表,至少提取用户基本信息作为一个条目(title 为用户名称或页面标题,description 为用户简介或页面描述,link 为当前 URL)
1533
+ 6. 如果页面是单页应用(SPA),尝试从以下位置提取数据:
1534
+ - <title> 标签中的页面标题
1535
+ - <meta name="description"> 或 <meta property="og:description"> 中的描述
1536
+ - <meta property="og:title"> 中的标题
1537
+ - <link rel="canonical" href="..."> 中的规范链接(优先使用)
1538
+ - <script> 标签中的 JSON 数据(搜索包含 "title", "description", "user", "author", "items", "list" 等关键词的 JSON)
1539
+ - 任何包含 href 属性的 <a> 标签,**link 必须使用 href 的原始值**(以 / 开头的路径表示从站根算起)
1540
+ - 任何包含文本内容的元素
1541
+ 7. **重要**:即使页面看起来是空的 SPA,也必须至少返回一个条目:
1542
+ - 从 <title> 提取标题(如果没有则使用 URL)
1543
+ - 从 <meta> 标签提取描述(如果没有则使用 "页面内容通过 JavaScript 动态加载")
1544
+ - link 使用当前 URL: ${url}
1545
+ - 如果页面 URL 包含 "user" 或 "profile",可以推断这是一个用户主页,title 可以是 "用户主页",description 可以是页面 URL
1546
+
1547
+ **强制要求**:绝对不能返回空数组 {"items": []},至少要返回一个包含页面基本信息的条目。
1548
+
1549
+ HTML:
1550
+ ${html}`;
1551
+ const parsed = await chatJson(
1552
+ prompt,
1553
+ { apiKey: config.apiKey, apiUrl: config.apiUrl, model: config.model },
1554
+ {}
1555
+ );
1556
+ let entries = [];
1557
+ if (parsed.items && Array.isArray(parsed.items)) {
1558
+ entries = parsed.items;
1559
+ } else if (parsed.entries && Array.isArray(parsed.entries)) {
1560
+ entries = parsed.entries;
1561
+ } else if (Array.isArray(parsed)) {
1562
+ entries = parsed;
1563
+ } else if (parsed && typeof parsed === "object") {
1564
+ entries = [parsed];
1565
+ }
1566
+ return entries.map((e) => {
1567
+ const raw = e.link || url;
1568
+ if (raw.startsWith("http")) return { ...e, link: raw };
1569
+ const path = raw.startsWith("/") ? raw : `/${raw}`;
1570
+ try {
1571
+ const base2 = new URL(url);
1572
+ return { ...e, link: new URL(path, base2.origin).href };
1573
+ } catch {
1574
+ return { ...e, link: new URL(raw, url).href };
1575
+ }
1576
+ });
1577
+ }
1578
+ function toFeedItem(entry, includeContent) {
1579
+ return {
1580
+ guid: entry.guid ?? generateGuid(entry.link),
1581
+ title: entry.title,
1582
+ link: entry.link,
1583
+ pubDate: entry.published ? new Date(entry.published) : /* @__PURE__ */ new Date(),
1584
+ author: entry.author ? [entry.author] : void 0,
1585
+ summary: entry.description,
1586
+ content: includeContent ? entry.content : void 0,
1587
+ imageUrl: entry.imageUrl
1588
+ };
1589
+ }
1590
+ async function parseHtml(html, config = {}) {
1591
+ const { url = "", mode, customParser, llmConfig, includeContent = false, purify } = config;
1592
+ let entries = [];
1593
+ const actualMode = mode ?? (llmConfig != null ? "llm" : customParser != null ? "custom" : "llm");
1594
+ if (actualMode === "llm") {
1595
+ if (llmConfig == null && !getLLMConfig().apiKey) {
1596
+ throw new Error('mode 为 "llm" 时必须提供 llmConfig 或设置 OPENAI_API_KEY 环境变量');
1597
+ }
1598
+ const htmlForLLM = applyPurify(html, purify !== false);
1599
+ entries = await parseWithLLM(htmlForLLM, url, llmConfig ?? {});
1600
+ } else if (actualMode === "custom") {
1601
+ if (customParser == null) {
1602
+ throw new Error('mode 为 "custom" 时必须提供 customParser');
1603
+ }
1604
+ entries = await customParser(html, url);
1605
+ if (!Array.isArray(entries)) {
1606
+ entries = [entries];
1607
+ }
1608
+ } else {
1609
+ throw new Error(`不支持的解析模式: ${actualMode}`);
1610
+ }
1611
+ const items = entries.map((e) => toFeedItem(e, includeContent));
1612
+ return {
1613
+ items,
1614
+ url,
1615
+ mode: actualMode
1616
+ };
1617
+ }
1618
+ function patternToRegex(pattern) {
1619
+ if (pattern instanceof RegExp) return pattern;
1620
+ const pathOnly = pattern.split("?")[0];
1621
+ const pl = "<<<__PL__>>>";
1622
+ const s = pathOnly.replace(/\{[^}]*\}/g, pl).replace(/[.*+?^${}()|[\]\\]/g, (c) => "\\" + c).replace(new RegExp(pl.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"), "g"), "[^/]+");
1623
+ return new RegExp("^" + s + "(\\?.*)?$");
1624
+ }
1625
+ function toAuthFlow(site) {
1626
+ if (!site.checkAuth || !site.loginUrl) return void 0;
1627
+ return {
1628
+ checkAuth: site.checkAuth,
1629
+ loginUrl: site.loginUrl,
1630
+ domain: site.domain ?? void 0,
1631
+ loginTimeoutMs: site.loginTimeoutMs ?? void 0,
1632
+ pollIntervalMs: site.pollIntervalMs ?? void 0
1633
+ };
1634
+ }
1635
+ function matchesListUrl(site, url) {
1636
+ try {
1637
+ return patternToRegex(site.listUrlPattern).test(url);
1638
+ } catch {
1639
+ return false;
1640
+ }
1641
+ }
1642
+ function computeSpecificity(site, url) {
1643
+ if (!matchesListUrl(site, url)) return -1;
1644
+ const p = site.listUrlPattern;
1645
+ if (typeof p === "string") {
1646
+ const pathOnly = p.split("?")[0];
1647
+ return 1e3 + pathOnly.split("/").filter(Boolean).length;
1648
+ }
1649
+ return p.source.length;
1650
+ }
1651
+ function getSiteByUrl(url, sites) {
1652
+ const matched = sites.map((s) => ({ site: s, score: computeSpecificity(s, url) })).filter((x) => x.score >= 0).sort((a, b) => b.score - a.score);
1653
+ return matched[0]?.site;
1654
+ }
1655
+ class AuthRequiredError extends Error {
1656
+ constructor(message = "需要登录") {
1657
+ super(message);
1658
+ this.name = "AuthRequiredError";
1659
+ }
1660
+ }
1661
+ class NotFoundError extends Error {
1662
+ constructor(message = "未找到") {
1663
+ super(message);
1664
+ this.name = "NotFoundError";
1665
+ }
1666
+ }
1667
+ const PLUGIN_EXTENSIONS = [".rssany.js", ".rssany.ts"];
1668
+ function isValidSite(obj) {
1669
+ if (obj == null || typeof obj !== "object") return false;
1670
+ const s = obj;
1671
+ return typeof s.id === "string" && (typeof s.listUrlPattern === "string" || s.listUrlPattern instanceof RegExp) && typeof s.fetchItems === "function";
1672
+ }
1673
+ function isValidSource(obj) {
1674
+ if (obj == null || typeof obj !== "object") return false;
1675
+ const s = obj;
1676
+ return typeof s.id === "string" && (typeof s.pattern === "string" || s.pattern instanceof RegExp) && typeof s.fetchItems === "function" && s.listUrlPattern === void 0;
1677
+ }
1678
+ function isValidEnrichPlugin(obj) {
1679
+ if (obj == null || typeof obj !== "object") return false;
1680
+ const p = obj;
1681
+ return typeof p.id === "string" && typeof p.match === "function" && typeof p.enrichItem === "function";
1682
+ }
1683
+ async function loadSourcePluginsFromDir(dir, label) {
1684
+ const siteEntries = [];
1685
+ const sources = [];
1686
+ let entries;
1687
+ try {
1688
+ const raw = await readdir(dir, { withFileTypes: true, encoding: "utf-8" });
1689
+ entries = raw;
1690
+ } catch {
1691
+ return { siteEntries, sources };
1692
+ }
1693
+ for (const e of entries) {
1694
+ const name = String(e.name);
1695
+ if (!e.isFile()) continue;
1696
+ if (!PLUGIN_EXTENSIONS.some((ext) => name.endsWith(ext))) continue;
1697
+ const filePath = join(dir, name);
1698
+ try {
1699
+ const mod = await import(pathToFileURL(filePath).href);
1700
+ const plugin = mod.default ?? mod;
1701
+ if (isValidSite(plugin)) {
1702
+ siteEntries.push({ site: plugin, filePath });
1703
+ } else if (isValidSource(plugin)) {
1704
+ sources.push({ source: plugin, filePath });
1705
+ } else {
1706
+ logger.warn("plugin", "插件未实现 Site 或 Source 接口,已跳过", { label, name });
1707
+ }
1708
+ } catch (err) {
1709
+ logger.warn("plugin", "插件加载失败", { label, name, err: err instanceof Error ? err.message : String(err) });
1710
+ }
1711
+ }
1712
+ return { siteEntries, sources };
1713
+ }
1714
+ async function loadPluginsFromDir(dir, label, validator) {
1715
+ const result = [];
1716
+ let entries;
1717
+ try {
1718
+ const raw = await readdir(dir, { withFileTypes: true, encoding: "utf-8" });
1719
+ entries = raw;
1720
+ } catch {
1721
+ return result;
1722
+ }
1723
+ for (const e of entries) {
1724
+ const name = String(e.name);
1725
+ if (!e.isFile()) continue;
1726
+ if (!PLUGIN_EXTENSIONS.some((ext) => name.endsWith(ext))) continue;
1727
+ const filePath = join(dir, name);
1728
+ try {
1729
+ const mod = await import(pathToFileURL(filePath).href);
1730
+ const plugin = mod.default ?? mod;
1731
+ if (validator(plugin)) {
1732
+ result.push(plugin);
1733
+ } else {
1734
+ logger.warn("plugin", "插件接口不匹配,已跳过", { label, name });
1735
+ }
1736
+ } catch (err) {
1737
+ logger.warn("plugin", "插件加载失败", { label, name, err: err instanceof Error ? err.message : String(err) });
1738
+ }
1739
+ }
1740
+ return result;
1741
+ }
1742
+ async function loadFromSourcesOrRoot() {
1743
+ const [builtinFromSources, userFromSources] = await Promise.all([
1744
+ loadSourcePluginsFromDir(BUILTIN_SOURCES_DIR, "builtin:sources"),
1745
+ loadSourcePluginsFromDir(USER_SOURCES_DIR, "user:sources")
1746
+ ]);
1747
+ const hasAny = builtinFromSources.siteEntries.length + builtinFromSources.sources.length + userFromSources.siteEntries.length + userFromSources.sources.length > 0;
1748
+ if (hasAny) return { builtin: builtinFromSources, user: userFromSources };
1749
+ const [builtinRoot, userRoot] = await Promise.all([
1750
+ loadSourcePluginsFromDir(BUILTIN_PLUGINS_DIR, "builtin"),
1751
+ loadSourcePluginsFromDir(USER_PLUGINS_DIR, "user")
1752
+ ]);
1753
+ return { builtin: builtinRoot, user: userRoot };
1754
+ }
1755
+ const pluginSitePaths = /* @__PURE__ */ new Map();
1756
+ function mergeSourcePluginPaths(siteIds, pathMap, builtinSources, userSources) {
1757
+ for (const { source, filePath } of builtinSources) {
1758
+ if (siteIds.has(source.id)) {
1759
+ logger.warn("plugin", "Source 插件 id 与 Site 插件冲突,已忽略 Source 路径", { sourceId: source.id });
1760
+ continue;
1761
+ }
1762
+ pathMap.set(source.id, filePath);
1763
+ }
1764
+ for (const { source, filePath } of userSources) {
1765
+ if (siteIds.has(source.id)) {
1766
+ logger.warn("plugin", "Source 插件 id 与 Site 插件冲突,已忽略 Source 路径", { sourceId: source.id });
1767
+ continue;
1768
+ }
1769
+ if (pathMap.has(source.id)) logger.info("plugin", "用户 Source 插件覆盖同名内置", { sourceId: source.id });
1770
+ pathMap.set(source.id, filePath);
1771
+ }
1772
+ }
1773
+ function getPluginFilePath(id) {
1774
+ return pluginSitePaths.get(id);
1775
+ }
1776
+ async function loadSiteAndSourcePlugins() {
1777
+ const { builtin, user } = await loadFromSourcesOrRoot();
1778
+ const siteMap = /* @__PURE__ */ new Map();
1779
+ const pathMap = /* @__PURE__ */ new Map();
1780
+ for (const { site: s, filePath } of builtin.siteEntries) {
1781
+ siteMap.set(s.id, s);
1782
+ pathMap.set(s.id, filePath);
1783
+ }
1784
+ for (const { site: s, filePath } of user.siteEntries) {
1785
+ if (siteMap.has(s.id)) logger.info("plugin", "用户插件覆盖同名内置", { pluginId: s.id });
1786
+ siteMap.set(s.id, s);
1787
+ pathMap.set(s.id, filePath);
1788
+ }
1789
+ mergeSourcePluginPaths(new Set(siteMap.keys()), pathMap, builtin.sources, user.sources);
1790
+ const sourceMap = /* @__PURE__ */ new Map();
1791
+ for (const { source } of builtin.sources) sourceMap.set(source.id, source);
1792
+ for (const { source } of user.sources) {
1793
+ if (sourceMap.has(source.id)) logger.info("plugin", "用户 Source 插件覆盖同名内置", { sourceId: source.id });
1794
+ sourceMap.set(source.id, source);
1795
+ }
1796
+ pluginSitePaths.clear();
1797
+ pathMap.forEach((path, id) => pluginSitePaths.set(id, path));
1798
+ return { sites: Array.from(siteMap.values()), sources: Array.from(sourceMap.values()) };
1799
+ }
1800
+ let registeredEnrichPlugins = [];
1801
+ async function loadEnrichPlugins() {
1802
+ const [builtin, user] = await Promise.all([
1803
+ loadPluginsFromDir(BUILTIN_ENRICH_DIR, "builtin:enrich", isValidEnrichPlugin),
1804
+ loadPluginsFromDir(USER_ENRICH_DIR, "user:enrich", isValidEnrichPlugin)
1805
+ ]);
1806
+ const merged = /* @__PURE__ */ new Map();
1807
+ for (const p of builtin) merged.set(p.id, p);
1808
+ for (const p of user) {
1809
+ if (merged.has(p.id)) logger.info("plugin", "用户 Enrich 插件覆盖同名内置", { pluginId: p.id });
1810
+ merged.set(p.id, p);
1811
+ }
1812
+ const list = Array.from(merged.values());
1813
+ list.sort((a, b) => (a.priority ?? 100) - (b.priority ?? 100));
1814
+ registeredEnrichPlugins = list;
1815
+ return list;
1816
+ }
1817
+ function getMatchedEnrichPlugin(item, ctx) {
1818
+ return registeredEnrichPlugins.find((p) => p.match(item, ctx));
1819
+ }
1820
+ function buildEnrichContext(ctx) {
1821
+ return {
1822
+ cacheDir: ctx.cacheDir,
1823
+ headless: ctx.headless,
1824
+ proxy: ctx.proxy,
1825
+ async fetchHtml(url, opts) {
1826
+ const res = await fetchHtml(url, {
1827
+ cacheDir: ctx.cacheDir,
1828
+ useCache: false,
1829
+ authFlow: void 0,
1830
+ headless: ctx.headless,
1831
+ proxy: ctx.proxy,
1832
+ waitAfterLoadMs: opts?.waitMs,
1833
+ purify: opts?.purify
1834
+ });
1835
+ return { html: res.body, finalUrl: res.finalUrl ?? url, status: res.status };
1836
+ },
1837
+ async extractItem(item, opts) {
1838
+ const res = await fetchHtml(item.link, {
1839
+ cacheDir: ctx.cacheDir,
1840
+ useCache: false,
1841
+ authFlow: void 0,
1842
+ headless: ctx.headless,
1843
+ proxy: ctx.proxy
1844
+ });
1845
+ if (res.status !== 200 && res.status !== 304) {
1846
+ throw new Error(`默认正文提取失败: HTTP ${res.status} ${res.statusText} for ${item.link}`);
1847
+ }
1848
+ const extracted = await extractHtml(res.body, {
1849
+ url: res.finalUrl ?? item.link,
1850
+ cacheDir: ctx.cacheDir ?? void 0,
1851
+ mode: "readability",
1852
+ useCache: true,
1853
+ cacheKey: opts?.cacheKey
1854
+ });
1855
+ const pubDate = extracted.pubDate != null ? typeof extracted.pubDate === "string" ? new Date(extracted.pubDate) : extracted.pubDate : item.pubDate;
1856
+ return {
1857
+ ...item,
1858
+ author: normalizeAuthor(extracted.author ?? item.author),
1859
+ title: extracted.title ?? item.title,
1860
+ summary: extracted.summary ?? item.summary,
1861
+ content: extracted.content ?? item.content,
1862
+ pubDate
1863
+ };
1864
+ }
1865
+ };
1866
+ }
1867
+ function buildSiteContext(site, ctx) {
1868
+ const proxy = ctx.proxy ?? site.proxy;
1869
+ const authFlow = toAuthFlow(site);
1870
+ return {
1871
+ cacheDir: ctx.cacheDir,
1872
+ headless: ctx.headless,
1873
+ proxy,
1874
+ async fetchHtml(url, opts) {
1875
+ const res = await fetchHtml(url, {
1876
+ cacheDir: ctx.cacheDir,
1877
+ useCache: false,
1878
+ authFlow,
1879
+ headless: ctx.headless,
1880
+ proxy,
1881
+ waitAfterLoadMs: opts?.waitMs,
1882
+ purify: opts?.purify,
1883
+ waitForSelector: opts?.waitForSelector,
1884
+ waitForSelectorTimeoutMs: opts?.waitForSelectorTimeoutMs
1885
+ });
1886
+ return { html: res.body, finalUrl: res.finalUrl ?? url, status: res.status };
1887
+ },
1888
+ async extractItem(item, opts) {
1889
+ const res = await fetchHtml(item.link, {
1890
+ cacheDir: ctx.cacheDir,
1891
+ useCache: false,
1892
+ authFlow,
1893
+ headless: ctx.headless,
1894
+ proxy
1895
+ });
1896
+ if (res.status !== 200 && res.status !== 304) {
1897
+ throw new Error(`默认正文提取失败: HTTP ${res.status} ${res.statusText} for ${item.link}`);
1898
+ }
1899
+ const extracted = await extractHtml(res.body, {
1900
+ url: res.finalUrl ?? item.link,
1901
+ cacheDir: ctx.cacheDir ?? void 0,
1902
+ mode: "readability",
1903
+ useCache: true,
1904
+ cacheKey: opts?.cacheKey
1905
+ });
1906
+ const pubDate = extracted.pubDate != null ? typeof extracted.pubDate === "string" ? new Date(extracted.pubDate) : extracted.pubDate : item.pubDate;
1907
+ return {
1908
+ ...item,
1909
+ author: normalizeAuthor(extracted.author ?? item.author),
1910
+ title: extracted.title ?? item.title,
1911
+ summary: extracted.summary ?? item.summary,
1912
+ content: extracted.content ?? item.content,
1913
+ pubDate
1914
+ };
1915
+ }
1916
+ };
1917
+ }
1918
+ function createWebSource(site) {
1919
+ const authFlow = toAuthFlow(site);
1920
+ return {
1921
+ id: site.id,
1922
+ pattern: site.listUrlPattern,
1923
+ priority: 50,
1924
+ refreshInterval: site.refreshInterval ?? void 0,
1925
+ proxy: site.proxy ?? void 0,
1926
+ preCheck: authFlow ? async (ctx) => {
1927
+ if (!ctx.cacheDir) return;
1928
+ const passed = await preCheckAuth(authFlow, ctx.cacheDir);
1929
+ if (!passed) throw new AuthRequiredError(`站点 ${site.id} 需要登录,请先执行 ensureAuth`);
1930
+ } : void 0,
1931
+ async fetchItems(sourceId, ctx) {
1932
+ return site.fetchItems(sourceId, buildSiteContext(site, ctx));
1933
+ },
1934
+ enrichItem: site.enrichItem ? async (item, ctx) => {
1935
+ return site.enrichItem(item, buildSiteContext(site, ctx));
1936
+ } : void 0
1937
+ };
1938
+ }
1939
+ const genericWebSource = {
1940
+ id: "generic",
1941
+ pattern: /^https?:\/\//,
1942
+ priority: 200,
1943
+ async fetchItems(sourceId, ctx) {
1944
+ const res = await fetchHtml(sourceId, {
1945
+ cacheDir: ctx.cacheDir,
1946
+ useCache: true,
1947
+ headless: ctx.headless,
1948
+ proxy: ctx.proxy
1949
+ });
1950
+ if (res.status !== 200 && res.status !== 304) {
1951
+ throw new Error(`抓取失败: HTTP ${res.status} ${res.statusText}`);
1952
+ }
1953
+ const parsed = await parseHtml(res.body, {
1954
+ url: res.finalUrl ?? sourceId
1955
+ });
1956
+ return parsed.items;
1957
+ }
1958
+ };
1959
+ const loadedSites = [];
1960
+ function setLoadedSites(sites) {
1961
+ loadedSites.length = 0;
1962
+ loadedSites.push(...sites);
1963
+ }
1964
+ function getWebSite(id) {
1965
+ return loadedSites.find((s) => s.id === id);
1966
+ }
1967
+ function getPluginSites() {
1968
+ return loadedSites.filter((s) => s.id !== "generic");
1969
+ }
1970
+ function getBestSite(url) {
1971
+ return getSiteByUrl(url, loadedSites);
1972
+ }
1973
+ const registeredSources = [];
1974
+ function sourcePatternToRegex(pattern) {
1975
+ if (pattern instanceof RegExp) return pattern;
1976
+ const pathOnly = pattern.split("?")[0];
1977
+ const pl = "<<<__PL__>>>";
1978
+ const escaped = pathOnly.replace(/\{[^}]*\}/g, pl).replace(/[.*+?^${}()|[\]\\]/g, (c) => "\\" + c).replace(new RegExp(pl.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"), "g"), "[^/]+");
1979
+ return new RegExp("^" + escaped + "(\\?.*)?$");
1980
+ }
1981
+ function getSource(sourceId) {
1982
+ for (const source of registeredSources) {
1983
+ const matches = source.match ? source.match(sourceId) : (() => {
1984
+ try {
1985
+ return sourcePatternToRegex(source.pattern).test(sourceId);
1986
+ } catch {
1987
+ return false;
1988
+ }
1989
+ })();
1990
+ if (matches) return source;
1991
+ }
1992
+ return genericWebSource;
1993
+ }
1994
+ async function initSources() {
1995
+ const [siteResult] = await Promise.all([
1996
+ loadSiteAndSourcePlugins(),
1997
+ loadEnrichPlugins()
1998
+ ]);
1999
+ const { sites, sources: sourcePlugins } = siteResult;
2000
+ setLoadedSites(sites);
2001
+ registeredSources.length = 0;
2002
+ const webSources = sites.map((s) => createWebSource(s));
2003
+ const all = [
2004
+ ...sourcePlugins,
2005
+ ...webSources,
2006
+ genericWebSource
2007
+ ];
2008
+ all.sort((a, b) => (a.priority ?? 100) - (b.priority ?? 100));
2009
+ registeredSources.push(...all);
2010
+ logger.info("scheduler", "信源已注册", {
2011
+ total: registeredSources.length,
2012
+ siteCount: sites.length,
2013
+ sourcePluginCount: sourcePlugins.length
2014
+ });
2015
+ }
2016
+ function resolveRef(src) {
2017
+ return src.ref ?? src.url ?? "";
2018
+ }
2019
+ async function loadSourcesFile() {
2020
+ try {
2021
+ const raw = await readFile(SOURCES_CONFIG_PATH, "utf-8");
2022
+ const parsed = JSON.parse(raw);
2023
+ if (!parsed || typeof parsed !== "object") return [];
2024
+ if (Array.isArray(parsed.sources)) {
2025
+ return parsed.sources.filter((s) => resolveRef(s));
2026
+ }
2027
+ const entries = Object.values(parsed);
2028
+ const list = [];
2029
+ for (const entry of entries) {
2030
+ if (entry && Array.isArray(entry.sources)) {
2031
+ for (const s of entry.sources) {
2032
+ if (resolveRef(s)) list.push(s);
2033
+ }
2034
+ }
2035
+ }
2036
+ return list;
2037
+ } catch {
2038
+ return [];
2039
+ }
2040
+ }
2041
+ async function getAllSources() {
2042
+ return loadSourcesFile();
2043
+ }
2044
+ async function getAllSubscriptionRefs() {
2045
+ const list = await loadSourcesFile();
2046
+ const seen = /* @__PURE__ */ new Set();
2047
+ const out = [];
2048
+ for (const s of list) {
2049
+ const r = resolveRef(s);
2050
+ if (r && !seen.has(r)) {
2051
+ seen.add(r);
2052
+ out.push(r);
2053
+ }
2054
+ }
2055
+ return out;
2056
+ }
2057
+ async function saveSourcesFile(sources) {
2058
+ await writeFile(
2059
+ SOURCES_CONFIG_PATH,
2060
+ JSON.stringify({ sources }, null, 2) + "\n",
2061
+ "utf-8"
2062
+ );
2063
+ }
2064
+ async function getSourcesRaw() {
2065
+ try {
2066
+ const raw = await readFile(SOURCES_CONFIG_PATH, "utf-8");
2067
+ const parsed = JSON.parse(raw);
2068
+ if (!parsed || typeof parsed !== "object") return JSON.stringify({ sources: [] }, null, 2);
2069
+ if (Array.isArray(parsed.sources)) {
2070
+ return raw;
2071
+ }
2072
+ const list = await loadSourcesFile();
2073
+ return JSON.stringify({ sources: list }, null, 2);
2074
+ } catch {
2075
+ return JSON.stringify({ sources: [] }, null, 2);
2076
+ }
2077
+ }
2078
+ const MAX_BODY_CHARS = 4e3;
2079
+ const SYSTEM$2 = `你是信息质量评估助手。根据标题、摘要与正文片段,判断该条是否值得保留在订阅信息流中。
2080
+ - 保留(keep: true):有实质信息、观点、技术内容、新闻价值,或对读者有参考意义。
2081
+ - 过滤(keep: false):明显垃圾广告、纯日期/站务占位、无意义一句话、乱码或空白、纯导航无内容、重复无信息量的模板句。
2082
+ - 只输出 JSON,格式:{"keep": true 或 false, "reason": "一句话中文理由"}`;
2083
+ function combinedForJudge(item) {
2084
+ const title = (item.title ?? "").trim();
2085
+ const summary = (item.summary ?? "").trim();
2086
+ let body = (item.content ?? "").trim();
2087
+ if (body.length > MAX_BODY_CHARS) body = body.slice(0, MAX_BODY_CHARS) + "\n\n[... 正文已截断 ...]";
2088
+ const parts = [];
2089
+ if (title) parts.push(`标题:
2090
+ ${title}`);
2091
+ if (summary) parts.push(`摘要:
2092
+ ${summary}`);
2093
+ if (body) parts.push(`正文:
2094
+ ${body}`);
2095
+ return parts.join("\n\n---\n\n");
2096
+ }
2097
+ function qualityFilterMatch(_item, ctx) {
2098
+ return !!ctx.llm;
2099
+ }
2100
+ async function runQualityFilter(item, ctx) {
2101
+ if (!ctx.llm) return item;
2102
+ const text = combinedForJudge(item);
2103
+ if (!text.trim()) return item;
2104
+ const prompt = `${SYSTEM$2}
2105
+
2106
+ 请评估以下条目:
2107
+
2108
+ ${text}`;
2109
+ let res;
2110
+ try {
2111
+ res = await ctx.llm.chatJson(prompt, void 0, { maxTokens: 256, debugLabel: "qualityFilter" });
2112
+ } catch {
2113
+ return item;
2114
+ }
2115
+ const isKeep = res.keep === true || res.keep === "true";
2116
+ const isDrop = res.keep === false || res.keep === "false";
2117
+ if (isKeep) return item;
2118
+ if (!isDrop) return item;
2119
+ const reason = typeof res.reason === "string" ? res.reason.trim() : "";
2120
+ logger.info("pipeline", "质量过滤移除条目", { link: item.link, reason: reason || void 0 });
2121
+ return markPipelineDrop(item);
2122
+ }
2123
+ const SYSTEM$1 = `你是一个信息分类助手。根据文章标题和摘要,从候选标签库中选出最匹配的标签,最多 5 个。
2124
+ - 只能使用候选标签库中已有的标签,不要输出库中不存在的标签。
2125
+ - 如果没有合适的标签,输出空数组 {"tags": []},不要硬选不相关的标签。
2126
+ - 只输出 JSON,格式:{"tags": ["tag1", "tag2", ...]}`;
2127
+ function taggerMatch(item, ctx) {
2128
+ return !item.tags?.length && !!ctx.llm;
2129
+ }
2130
+ async function runTagger(item, ctx) {
2131
+ if (!ctx.db) return item;
2132
+ const systemTags = await ctx.db.getSystemTags();
2133
+ if (!systemTags.length || !ctx.llm) return item;
2134
+ const candidateList = `候选标签库(只能从中选取):
2135
+ ${systemTags.join(", ")}
2136
+
2137
+ `;
2138
+ const prompt = `${SYSTEM$1}
2139
+
2140
+ ${candidateList}文章标题:${item.title ?? ""}
2141
+ 文章摘要:${item.summary ?? item.content?.slice(0, 300) ?? "(无摘要)"}`;
2142
+ let suggested;
2143
+ try {
2144
+ const res = await ctx.llm.chatJson(prompt, void 0, { maxTokens: 256, debugLabel: "tagger" });
2145
+ suggested = Array.isArray(res.tags) ? res.tags.filter((t) => typeof t === "string") : [];
2146
+ } catch {
2147
+ return item;
2148
+ }
2149
+ if (!suggested.length) return item;
2150
+ const set = new Set(systemTags.map((t) => t.toLowerCase()));
2151
+ const confirmed = suggested.filter((t) => set.has(t.toLowerCase()));
2152
+ if (confirmed.length) {
2153
+ item.tags = [.../* @__PURE__ */ new Set([...item.tags ?? [], ...confirmed])];
2154
+ }
2155
+ return item;
2156
+ }
2157
+ const ZH_CN = "zh-CN";
2158
+ const MAX_CONTENT_CHARS = 6e3;
2159
+ const DETECT_CONTENT_PREFIX = 2e3;
2160
+ const SYSTEM = `你是一个专业翻译助手。将用户提供的英文(或其他语言)内容翻译为简体中文。
2161
+ - 保持专业、准确、流畅。
2162
+ - 若原文已是中文,则保持原样或轻微润色。
2163
+ - 只输出 JSON,格式:{"title": "译文标题", "summary": "译文摘要", "content": "译文正文"}
2164
+ - 若某字段为空或用户未提供,对应输出空字符串 ""。`;
2165
+ function combinedTextForDetection(item) {
2166
+ const title = (item.title ?? "").trim();
2167
+ const summary = (item.summary ?? item.content?.slice(0, 500) ?? "").trim();
2168
+ const content = (item.content ?? "").trim().slice(0, DETECT_CONTENT_PREFIX);
2169
+ return `${title}
2170
+ ${summary}
2171
+ ${content}`;
2172
+ }
2173
+ function isLikelyChineseContent(text) {
2174
+ const t = text.trim();
2175
+ if (t.length < 8) return false;
2176
+ if (/[\u3040-\u309f\u30a0-\u30ff]/.test(t)) return false;
2177
+ if (/[\uac00-\ud7af]/.test(t)) return false;
2178
+ const cjk = (t.match(/[\u4e00-\u9fff\u3400-\u4dbf]/g) ?? []).length;
2179
+ const latin = (t.match(/[A-Za-z]/g) ?? []).length;
2180
+ const letterish = cjk + latin;
2181
+ if (letterish < 12) return false;
2182
+ return cjk / letterish >= 0.55;
2183
+ }
2184
+ function translatorMatch(item, ctx) {
2185
+ const hasZh = item.translations?.[ZH_CN];
2186
+ if (hasZh || !ctx.llm) return false;
2187
+ if (isLikelyChineseContent(combinedTextForDetection(item))) return false;
2188
+ return true;
2189
+ }
2190
+ async function runTranslator(item, ctx) {
2191
+ if (!ctx.llm) return item;
2192
+ const title = (item.title ?? "").trim();
2193
+ const summary = (item.summary ?? item.content?.slice(0, 500) ?? "").trim();
2194
+ const content = (item.content ?? "").trim();
2195
+ const contentTruncated = content.length > MAX_CONTENT_CHARS ? content.slice(0, MAX_CONTENT_CHARS) + "\n\n[... 内容已截断 ...]" : content;
2196
+ if (!title && !summary && !content) return item;
2197
+ const parts = [];
2198
+ if (title) parts.push(`标题:
2199
+ ${title}`);
2200
+ if (summary) parts.push(`摘要:
2201
+ ${summary}`);
2202
+ if (contentTruncated) parts.push(`正文:
2203
+ ${contentTruncated}`);
2204
+ const prompt = `${SYSTEM}
2205
+
2206
+ 请翻译以下内容:
2207
+
2208
+ ${parts.join("\n\n---\n\n")}`;
2209
+ let res;
2210
+ try {
2211
+ res = await ctx.llm.chatJson(prompt, void 0, {
2212
+ maxTokens: Math.min(8192, Math.ceil((title.length + summary.length + contentTruncated.length) * 1.5)),
2213
+ debugLabel: "translator"
2214
+ });
2215
+ } catch {
2216
+ return item;
2217
+ }
2218
+ const tTitle = typeof res.title === "string" ? res.title.trim() : "";
2219
+ const tSummary = typeof res.summary === "string" ? res.summary.trim() : "";
2220
+ const tContent = typeof res.content === "string" ? res.content.trim() : "";
2221
+ if (!tTitle && !tSummary && !tContent) return item;
2222
+ item.translations = item.translations ?? {};
2223
+ item.translations[ZH_CN] = {
2224
+ title: tTitle || void 0,
2225
+ summary: tSummary || void 0,
2226
+ content: tContent || void 0
2227
+ };
2228
+ return item;
2229
+ }
2230
+ const DEFAULT_PIPELINE_STEPS = [
2231
+ { id: "qualityFilter", enabled: false },
2232
+ { id: "tagger", enabled: false },
2233
+ { id: "translator", enabled: false }
2234
+ ];
2235
+ const PIPELINE_STEP_IDS = ["qualityFilter", "tagger", "translator"];
2236
+ function parseSteps$1(rawSteps) {
2237
+ const steps = [];
2238
+ const seen = /* @__PURE__ */ new Set();
2239
+ for (const s of rawSteps) {
2240
+ if (s && typeof s === "object" && typeof s.id === "string") {
2241
+ const obj = s;
2242
+ const id = obj.id.trim();
2243
+ if (!id || seen.has(id)) continue;
2244
+ seen.add(id);
2245
+ const enabled = obj.enabled;
2246
+ steps.push({
2247
+ id,
2248
+ enabled: enabled !== false && enabled !== 0
2249
+ });
2250
+ }
2251
+ }
2252
+ return steps;
2253
+ }
2254
+ function mergeWithDefaultSteps(userSteps) {
2255
+ const map = new Map(userSteps.map((s) => [s.id, s]));
2256
+ return DEFAULT_PIPELINE_STEPS.map((def) => {
2257
+ const u = map.get(def.id);
2258
+ return { id: def.id, enabled: u ? u.enabled : def.enabled };
2259
+ });
2260
+ }
2261
+ async function loadPipelineConfig() {
2262
+ try {
2263
+ const raw = await readFile(CONFIG_PATH, "utf-8");
2264
+ const parsed = JSON.parse(raw);
2265
+ const rawSteps = Array.isArray(parsed?.pipeline?.steps) ? parsed.pipeline.steps : [];
2266
+ const steps = mergeWithDefaultSteps(parseSteps$1(rawSteps));
2267
+ if (steps.length > 0) return { steps };
2268
+ } catch {
2269
+ }
2270
+ return { steps: [...DEFAULT_PIPELINE_STEPS] };
2271
+ }
2272
+ async function savePipelineConfig(config) {
2273
+ let root = {};
2274
+ try {
2275
+ const raw = await readFile(CONFIG_PATH, "utf-8");
2276
+ root = JSON.parse(raw);
2277
+ } catch {
2278
+ }
2279
+ root.pipeline = { steps: config.steps };
2280
+ await writeFile(CONFIG_PATH, JSON.stringify(root, null, 2), "utf-8");
2281
+ }
2282
+ const STEP_REGISTRY = {
2283
+ qualityFilter: { match: qualityFilterMatch, run: runQualityFilter },
2284
+ tagger: { match: taggerMatch, run: runTagger },
2285
+ translator: { match: translatorMatch, run: runTranslator }
2286
+ };
2287
+ async function getResolvedSteps() {
2288
+ const config = await loadPipelineConfig();
2289
+ const out = [];
2290
+ for (const { id, enabled } of config.steps) {
2291
+ if (!enabled) continue;
2292
+ const step = STEP_REGISTRY[id];
2293
+ if (!step) {
2294
+ logger.debug("pipeline", "未知步骤已跳过", { id });
2295
+ continue;
2296
+ }
2297
+ out.push({ id, ...step });
2298
+ }
2299
+ return out;
2300
+ }
2301
+ async function runPipeline(item, ctx) {
2302
+ const steps = await getResolvedSteps();
2303
+ let current = item;
2304
+ for (const step of steps) {
2305
+ if (!step.match(current, ctx)) continue;
2306
+ try {
2307
+ current = await step.run(current, ctx);
2308
+ } catch (err) {
2309
+ logger.warn("pipeline", "步骤执行失败", {
2310
+ stepId: step.id,
2311
+ item_url: item.link,
2312
+ err: err instanceof Error ? err.message : String(err)
2313
+ });
2314
+ }
2315
+ }
2316
+ return current;
2317
+ }
2318
+ function escapeXml(s) {
2319
+ return s.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;").replace(/'/g, "&apos;");
2320
+ }
2321
+ function isoToRfc822(iso) {
2322
+ const d = new Date(iso);
2323
+ if (Number.isNaN(d.getTime())) return "";
2324
+ return d.toUTCString();
2325
+ }
2326
+ function buildItem(entry) {
2327
+ const title = escapeXml(entry.title ?? "");
2328
+ const link = escapeXml(entry.link ?? "");
2329
+ const descRaw = entry.description ?? "";
2330
+ const desc = descRaw.includes("<") || descRaw.includes(">") ? `<![CDATA[${descRaw.replace(/\]\]>/g, "]]]]><![CDATA[>")}]]>` : escapeXml(descRaw);
2331
+ const pubDate = entry.published ? escapeXml(isoToRfc822(entry.published)) : "";
2332
+ const guid = entry.guid ?? entry.link ?? "";
2333
+ const guidEscaped = escapeXml(guid);
2334
+ let buf = ` <item>
2335
+ <title>${title}</title>
2336
+ <link>${link}</link>
2337
+ <description>${desc}</description>
2338
+ `;
2339
+ if (pubDate) buf += ` <pubDate>${pubDate}</pubDate>
2340
+ `;
2341
+ if (entry.imageUrl?.trim()) {
2342
+ const encUrl = escapeXml(entry.imageUrl.trim());
2343
+ const encType = escapeXml(entry.imageType?.trim() || "image/jpeg");
2344
+ buf += ` <enclosure url="${encUrl}" length="0" type="${encType}"/>
2345
+ `;
2346
+ }
2347
+ buf += ` <guid isPermaLink="true">${guidEscaped}</guid>
2348
+ `;
2349
+ buf += ` </item>
2350
+ `;
2351
+ return buf;
2352
+ }
2353
+ function buildRssXml(channel, entries) {
2354
+ const title = escapeXml(channel.title);
2355
+ const link = escapeXml(channel.link);
2356
+ const desc = escapeXml(channel.description ?? "");
2357
+ const lang = escapeXml(channel.language ?? "zh-CN");
2358
+ const now2 = (/* @__PURE__ */ new Date()).toUTCString();
2359
+ const items = entries.map(buildItem).join("");
2360
+ return `<?xml version="1.0" encoding="UTF-8"?>
2361
+ <rss version="2.0">
2362
+ <channel>
2363
+ <title>${title}</title>
2364
+ <link>${link}</link>
2365
+ <description>${desc}</description>
2366
+ <language>${lang}</language>
2367
+ <lastBuildDate>${now2}</lastBuildDate>
2368
+
2369
+ ${items} </channel>
2370
+ </rss>`;
2371
+ }
2372
+ const eventBus = new EventEmitter();
2373
+ eventBus.setMaxListeners(200);
2374
+ function emitFeedUpdated(payload) {
2375
+ eventBus.emit("feed:updated", payload);
2376
+ }
2377
+ function onFeedUpdated(fn) {
2378
+ eventBus.on("feed:updated", fn);
2379
+ return () => eventBus.off("feed:updated", fn);
2380
+ }
2381
+ const DEFAULTS = {
2382
+ concurrency: 2,
2383
+ maxRetries: 2
2384
+ };
2385
+ async function loadEnrichConfig() {
2386
+ let fileEnrich = {};
2387
+ try {
2388
+ const raw = await readFile(join(USER_DIR, "config.json"), "utf-8");
2389
+ const parsed = JSON.parse(raw);
2390
+ if (parsed.enrich && typeof parsed.enrich === "object") {
2391
+ fileEnrich = parsed.enrich;
2392
+ }
2393
+ } catch {
2394
+ }
2395
+ return {
2396
+ concurrency: Number(fileEnrich["concurrency"] ?? process.env.ENRICH_CONCURRENCY ?? DEFAULTS.concurrency),
2397
+ maxRetries: Number(fileEnrich["maxRetries"] ?? process.env.ENRICH_MAX_RETRIES ?? DEFAULTS.maxRetries)
2398
+ };
2399
+ }
2400
+ const validateCron = validate;
2401
+ const tasks$1 = /* @__PURE__ */ new Map();
2402
+ const groups = /* @__PURE__ */ new Map();
2403
+ const DEFAULT_RETRY_DELAY_MS = 5e3;
2404
+ const DEFAULT_GROUP_CONCURRENCY = 10;
2405
+ async function runWithRetry(task, options) {
2406
+ const retries = options.retries ?? 0;
2407
+ const retryDelayMs = options.retryDelayMs ?? DEFAULT_RETRY_DELAY_MS;
2408
+ let lastErr;
2409
+ for (let attempt = 0; attempt <= retries; attempt++) {
2410
+ try {
2411
+ await task();
2412
+ return;
2413
+ } catch (err) {
2414
+ lastErr = err;
2415
+ if (attempt < retries) {
2416
+ await new Promise((r) => setTimeout(r, retryDelayMs));
2417
+ }
2418
+ }
2419
+ }
2420
+ throw lastErr;
2421
+ }
2422
+ async function runWithRetryAndResult(task, options) {
2423
+ const retries = options.retries ?? 0;
2424
+ const retryDelayMs = options.retryDelayMs ?? DEFAULT_RETRY_DELAY_MS;
2425
+ let lastErr;
2426
+ for (let attempt = 0; attempt <= retries; attempt++) {
2427
+ try {
2428
+ return await task();
2429
+ } catch (err) {
2430
+ lastErr = err;
2431
+ if (attempt < retries) {
2432
+ await new Promise((r) => setTimeout(r, retryDelayMs));
2433
+ }
2434
+ }
2435
+ }
2436
+ throw lastErr;
2437
+ }
2438
+ function ensureGroup(group, concurrency) {
2439
+ if (!groups.has(group)) {
2440
+ groups.set(group, { config: { concurrency }, running: 0, queue: [], completedCount: 0 });
2441
+ } else {
2442
+ const g = groups.get(group);
2443
+ g.config.concurrency = concurrency;
2444
+ if (g.completedCount === void 0) g.completedCount = 0;
2445
+ }
2446
+ }
2447
+ function enqueueAndProcess(group, id, task, options, resolve2, priority, resolveValue, rejectValue) {
2448
+ const g = groups.get(group);
2449
+ if (!g) return;
2450
+ g.queue = g.queue.filter((it) => it.id !== id);
2451
+ const item = { id, task, options, resolve: resolve2, resolveValue, rejectValue };
2452
+ if (priority) {
2453
+ g.queue.unshift(item);
2454
+ } else {
2455
+ g.queue.push(item);
2456
+ }
2457
+ processGroupQueue(group);
2458
+ }
2459
+ function processGroupQueue(group) {
2460
+ const g = groups.get(group);
2461
+ if (!g || g.running >= g.config.concurrency || g.queue.length === 0) return;
2462
+ const item = g.queue.shift();
2463
+ g.running += 1;
2464
+ const done = () => {
2465
+ g.running -= 1;
2466
+ g.completedCount = (g.completedCount ?? 0) + 1;
2467
+ processGroupQueue(group);
2468
+ };
2469
+ if (item.resolveValue != null || item.rejectValue != null) {
2470
+ runWithRetryAndResult(item.task, item.options).then((result) => {
2471
+ item.resolveValue?.(result);
2472
+ }).catch((err) => {
2473
+ item.rejectValue?.(err);
2474
+ }).finally(done);
2475
+ } else {
2476
+ runWithRetry(item.task, item.options).catch(() => {
2477
+ }).finally(() => {
2478
+ item.resolve?.();
2479
+ done();
2480
+ });
2481
+ }
2482
+ }
2483
+ function schedule(group, id, task, options = {}) {
2484
+ const cronExpr = options.cron?.trim();
2485
+ ensureGroup(group, options.concurrency ?? groups.get(group)?.config.concurrency ?? DEFAULT_GROUP_CONCURRENCY);
2486
+ if (cronExpr && validate(cronExpr)) {
2487
+ unschedule(id);
2488
+ const optsWithGroup = { ...options, group };
2489
+ const job = schedule$1(cronExpr, () => {
2490
+ const reg = tasks$1.get(id);
2491
+ if (reg) reg.lastRunTime = Date.now();
2492
+ enqueueAndProcess(group, id, task, optsWithGroup);
2493
+ });
2494
+ tasks$1.set(id, {
2495
+ id,
2496
+ cronExpr,
2497
+ task,
2498
+ options: optsWithGroup,
2499
+ stop: () => job.stop(),
2500
+ lastRunTime: 0
2501
+ });
2502
+ if (options.runNow) {
2503
+ runNow(id, true).catch(() => {
2504
+ });
2505
+ }
2506
+ return true;
2507
+ }
2508
+ return new Promise((resolve2, reject) => {
2509
+ enqueueAndProcess(
2510
+ group,
2511
+ id,
2512
+ task,
2513
+ { ...options, group },
2514
+ void 0,
2515
+ options.priority ?? false,
2516
+ resolve2,
2517
+ reject
2518
+ );
2519
+ });
2520
+ }
2521
+ function unschedule(id) {
2522
+ const reg = tasks$1.get(id);
2523
+ if (reg) {
2524
+ reg.stop();
2525
+ tasks$1.delete(id);
2526
+ }
2527
+ }
2528
+ function unscheduleGroup(group) {
2529
+ const ids = [...tasks$1.entries()].filter(([, reg]) => reg.options.group === group).map(([id]) => id);
2530
+ for (const id of ids) unschedule(id);
2531
+ }
2532
+ function runNow(id, priority = false) {
2533
+ const reg = tasks$1.get(id);
2534
+ if (!reg) return Promise.resolve();
2535
+ reg.lastRunTime = Date.now();
2536
+ const group = reg.options.group;
2537
+ if (group) {
2538
+ return new Promise((resolve2) => {
2539
+ enqueueAndProcess(group, id, reg.task, reg.options, resolve2, priority);
2540
+ });
2541
+ }
2542
+ return runWithRetry(reg.task, reg.options);
2543
+ }
2544
+ function getNextRunForTask(reg) {
2545
+ try {
2546
+ const base2 = reg.lastRunTime > 0 ? new Date(reg.lastRunTime) : /* @__PURE__ */ new Date();
2547
+ const expr = CronExpressionParser.parse(reg.cronExpr, { currentDate: base2 });
2548
+ let next = expr.next().getTime();
2549
+ if (next <= Date.now()) {
2550
+ const retry = CronExpressionParser.parse(reg.cronExpr, { currentDate: /* @__PURE__ */ new Date() });
2551
+ next = retry.next().getTime();
2552
+ }
2553
+ return next;
2554
+ } catch {
2555
+ return -1;
2556
+ }
2557
+ }
2558
+ function getGroupStats() {
2559
+ const result = {};
2560
+ for (const [name, g] of groups) {
2561
+ const groupTasks = [...tasks$1.values()].filter((t) => t.options.group === name);
2562
+ const scheduledCount = groupTasks.length;
2563
+ let nextRunTime;
2564
+ if (g.running > 0) {
2565
+ nextRunTime = 0;
2566
+ } else if (scheduledCount === 0) {
2567
+ nextRunTime = -1;
2568
+ } else {
2569
+ const times = groupTasks.map(getNextRunForTask).filter((t) => t > 0);
2570
+ nextRunTime = times.length > 0 ? Math.min(...times) : -1;
2571
+ }
2572
+ result[name] = {
2573
+ running: g.running,
2574
+ queued: g.queue.length,
2575
+ concurrency: g.config.concurrency,
2576
+ scheduledCount,
2577
+ completedCount: g.completedCount ?? 0,
2578
+ nextRunTime
2579
+ };
2580
+ }
2581
+ return result;
2582
+ }
2583
+ const ENRICH_GROUP = "enrich";
2584
+ const MAX_STORED_TASKS = 200;
2585
+ const RETRY_DELAY_MS = 3e3;
2586
+ class EnrichQueue {
2587
+ tasks = /* @__PURE__ */ new Map();
2588
+ taskItems = /* @__PURE__ */ new Map();
2589
+ taskCallbacks = /* @__PURE__ */ new Map();
2590
+ configLoaded = false;
2591
+ async ensureConfig() {
2592
+ if (this.configLoaded) return { concurrency: 2, maxRetries: 2 };
2593
+ const config = await loadEnrichConfig();
2594
+ this.configLoaded = true;
2595
+ logger.info("scraper", "配置加载完成", { concurrency: config.concurrency, maxRetries: config.maxRetries });
2596
+ return config;
2597
+ }
2598
+ evictIfNeeded() {
2599
+ if (this.tasks.size <= MAX_STORED_TASKS) return;
2600
+ const ids = [...this.tasks.keys()];
2601
+ for (const id of ids) {
2602
+ if (this.tasks.get(id)?.status === "done") {
2603
+ this.removeTask(id);
2604
+ if (this.tasks.size <= MAX_STORED_TASKS) return;
2605
+ }
2606
+ }
2607
+ if (this.tasks.size > MAX_STORED_TASKS) this.removeTask(ids[0]);
2608
+ }
2609
+ removeTask(id) {
2610
+ this.tasks.delete(id);
2611
+ this.taskItems.delete(id);
2612
+ this.taskCallbacks.delete(id);
2613
+ }
2614
+ checkTaskComplete(taskId) {
2615
+ const task = this.tasks.get(taskId);
2616
+ const items = this.taskItems.get(taskId);
2617
+ const callbacks = this.taskCallbacks.get(taskId);
2618
+ if (!task || !items) return;
2619
+ const allSettled = task.itemResults.every((r) => r.status === "done" || r.status === "failed");
2620
+ if (!allSettled) return;
2621
+ task.status = "done";
2622
+ task.completedAt = (/* @__PURE__ */ new Date()).toISOString();
2623
+ logger.info("scraper", "任务完成", {
2624
+ source_url: task.sourceUrl,
2625
+ taskId,
2626
+ done: task.progress.done,
2627
+ failed: task.progress.failed
2628
+ });
2629
+ Promise.resolve(callbacks?.onAllDone?.(items)).catch((err) => {
2630
+ logger.warn("scraper", "onAllDone 回调异常", { taskId, err: err instanceof Error ? err.message : String(err) });
2631
+ });
2632
+ }
2633
+ async submit(items, enrichFn, ctx, opts) {
2634
+ const config = await this.ensureConfig();
2635
+ const id = randomUUID();
2636
+ const itemResults = items.map((_, i) => ({
2637
+ index: i,
2638
+ status: "pending",
2639
+ retries: 0
2640
+ }));
2641
+ const task = {
2642
+ id,
2643
+ sourceUrl: opts.sourceUrl,
2644
+ status: items.length === 0 ? "done" : "pending",
2645
+ progress: { total: items.length, done: 0, failed: 0 },
2646
+ itemResults,
2647
+ createdAt: (/* @__PURE__ */ new Date()).toISOString(),
2648
+ completedAt: items.length === 0 ? (/* @__PURE__ */ new Date()).toISOString() : void 0
2649
+ };
2650
+ const itemsCopy = [...items];
2651
+ this.tasks.set(id, task);
2652
+ this.taskItems.set(id, itemsCopy);
2653
+ this.taskCallbacks.set(id, opts);
2654
+ this.evictIfNeeded();
2655
+ for (let i = 0; i < items.length; i++) {
2656
+ const itemIndex = i;
2657
+ const workId = `${id}-${i}`;
2658
+ const taskFn = async () => {
2659
+ const t = this.tasks.get(id);
2660
+ const its = this.taskItems.get(id);
2661
+ const cbs = this.taskCallbacks.get(id);
2662
+ if (!t || !its || !cbs) return;
2663
+ const itemResult = t.itemResults[itemIndex];
2664
+ if (!itemResult) return;
2665
+ itemResult.status = "running";
2666
+ if (t.status === "pending") t.status = "running";
2667
+ for (let r = 0; r <= config.maxRetries; r++) {
2668
+ try {
2669
+ const enriched = await enrichFn(its[itemIndex], ctx);
2670
+ its[itemIndex] = enriched;
2671
+ itemResult.item = enriched;
2672
+ itemResult.status = "done";
2673
+ t.progress.done++;
2674
+ await Promise.resolve(cbs.onItemDone?.(enriched, itemIndex));
2675
+ this.checkTaskComplete(id);
2676
+ return;
2677
+ } catch (err) {
2678
+ const msg = err instanceof Error ? err.message : String(err);
2679
+ if (r < config.maxRetries) {
2680
+ logger.warn("scraper", "提取失败,重试中", {
2681
+ source_url: t.sourceUrl,
2682
+ item_url: its[itemIndex]?.link,
2683
+ retries: r + 1,
2684
+ maxRetries: config.maxRetries,
2685
+ err: msg
2686
+ });
2687
+ await new Promise((resolve2) => setTimeout(resolve2, RETRY_DELAY_MS));
2688
+ } else {
2689
+ itemResult.status = "failed";
2690
+ itemResult.error = msg;
2691
+ t.progress.failed++;
2692
+ logger.warn("scraper", "提取最终失败", {
2693
+ source_url: t.sourceUrl,
2694
+ item_url: its[itemIndex]?.link,
2695
+ err: msg
2696
+ });
2697
+ const failedItem = { ...its[itemIndex], enrichFailed: true };
2698
+ its[itemIndex] = failedItem;
2699
+ await Promise.resolve(cbs.onItemDone?.(failedItem, itemIndex));
2700
+ this.checkTaskComplete(id);
2701
+ }
2702
+ }
2703
+ }
2704
+ };
2705
+ schedule(ENRICH_GROUP, workId, taskFn, { concurrency: config.concurrency }).catch(() => {
2706
+ });
2707
+ }
2708
+ return id;
2709
+ }
2710
+ getTask(id) {
2711
+ return this.tasks.get(id);
2712
+ }
2713
+ getTaskItems(id) {
2714
+ return this.taskItems.get(id);
2715
+ }
2716
+ }
2717
+ const enrichQueue = new EnrichQueue();
2718
+ async function getDeliverUrl() {
2719
+ try {
2720
+ const raw = await readFile(CONFIG_PATH, "utf-8");
2721
+ const j = JSON.parse(raw);
2722
+ const u = j?.deliver?.url;
2723
+ return typeof u === "string" ? u.trim() : "";
2724
+ } catch {
2725
+ return "";
2726
+ }
2727
+ }
2728
+ async function saveDeliverUrl(url) {
2729
+ let root = {};
2730
+ try {
2731
+ const raw = await readFile(CONFIG_PATH, "utf-8");
2732
+ root = JSON.parse(raw);
2733
+ } catch {
2734
+ }
2735
+ root.deliver = { url: url.trim() };
2736
+ await writeFile(CONFIG_PATH, JSON.stringify(root, null, 2) + "\n", "utf-8");
2737
+ }
2738
+ function feedItemsToPayload(items) {
2739
+ return items.map((i) => ({
2740
+ guid: i.guid,
2741
+ title: i.title,
2742
+ link: i.link,
2743
+ pubDate: i.pubDate instanceof Date ? i.pubDate.toISOString() : (/* @__PURE__ */ new Date()).toISOString(),
2744
+ author: i.author,
2745
+ summary: i.summary,
2746
+ content: i.content,
2747
+ tags: i.tags,
2748
+ sourceRef: i.sourceRef,
2749
+ translations: i.translations
2750
+ }));
2751
+ }
2752
+ async function postDeliverItems(url, sourceRef, items) {
2753
+ if (!url.trim() || items.length === 0) return;
2754
+ const body = JSON.stringify({ sourceRef, items: feedItemsToPayload(items) });
2755
+ const res = await fetch(url.trim(), {
2756
+ method: "POST",
2757
+ headers: { "Content-Type": "application/json" },
2758
+ body,
2759
+ signal: AbortSignal.timeout(12e4)
2760
+ });
2761
+ if (!res.ok) {
2762
+ const text = await res.text().catch(() => "");
2763
+ throw new Error(`HTTP ${res.status}${text ? `: ${text.slice(0, 200)}` : ""}`);
2764
+ }
2765
+ }
2766
+ async function postDeliverItemsSafe(url, sourceRef, items) {
2767
+ try {
2768
+ await postDeliverItems(url, sourceRef, items);
2769
+ } catch (err) {
2770
+ logger.warn("deliver", "投递失败", {
2771
+ sourceRef,
2772
+ count: items.length,
2773
+ err: err instanceof Error ? err.message : String(err)
2774
+ });
2775
+ }
2776
+ }
2777
+ function buildChannelFromItems(listUrl, items, lng) {
2778
+ const channel = {
2779
+ title: items[0]?.author?.length ? `${items[0].author[0]} 的订阅` : "RSS 订阅",
2780
+ link: listUrl,
2781
+ description: `来自 ${listUrl} 的订阅`
2782
+ };
2783
+ if (lng) channel.language = lng;
2784
+ return channel;
2785
+ }
2786
+ function toRssEntry(item, lng) {
2787
+ const eff = getEffectiveItemFields(item, lng);
2788
+ const hasContent = eff.content != null && eff.content !== "";
2789
+ const desc = hasContent ? eff.content : eff.summary;
2790
+ return {
2791
+ title: eff.title,
2792
+ link: item.link,
2793
+ description: desc,
2794
+ guid: item.guid,
2795
+ published: item.pubDate?.toISOString?.() ?? void 0,
2796
+ imageUrl: item.imageUrl
2797
+ };
2798
+ }
2799
+ const generatingKeys = /* @__PURE__ */ new Map();
2800
+ const pipelineCtx = {
2801
+ llm: { chatJson, chatText },
2802
+ db: { getSystemTags }
2803
+ };
2804
+ async function runPipelineOnItem(item, ctx) {
2805
+ return runPipeline(item, { ...pipelineCtx, ...ctx });
2806
+ }
2807
+ function buildEnrichFn(source, listUrl, ctx) {
2808
+ const enrichCtx = buildEnrichContext(ctx);
2809
+ enrichCtx.sourceUrl = listUrl;
2810
+ return async (item) => {
2811
+ let result = item;
2812
+ if (source.enrichItem) {
2813
+ result = await source.enrichItem(item, ctx);
2814
+ }
2815
+ const plugin = getMatchedEnrichPlugin(result, { sourceUrl: listUrl });
2816
+ if (plugin) {
2817
+ result = await plugin.enrichItem(result, enrichCtx);
2818
+ }
2819
+ return result;
2820
+ };
2821
+ }
2822
+ async function generateAndCache(listUrl, key, config) {
2823
+ const { cacheDir = "cache", includeContent = true, headless } = config;
2824
+ const source = getSource(listUrl);
2825
+ const ctx = { cacheDir, headless, proxy: config.proxy ?? source.proxy };
2826
+ let items;
2827
+ try {
2828
+ items = await source.fetchItems(listUrl, ctx);
2829
+ } catch (err) {
2830
+ generatingKeys.delete(key);
2831
+ const message = err instanceof Error ? err.message : String(err);
2832
+ logger.error("scraper", "抓取失败", { source_url: listUrl, err: message });
2833
+ throw err;
2834
+ }
2835
+ items.forEach((i) => {
2836
+ i.sourceRef = listUrl;
2837
+ i.author = normalizeAuthor(i.author);
2838
+ });
2839
+ generatingKeys.delete(key);
2840
+ logger.info("scraper", "抓取成功", { source_url: listUrl, count: items.length });
2841
+ const deliverUrl = await getDeliverUrl();
2842
+ let newCount = 0;
2843
+ let newIds = /* @__PURE__ */ new Set();
2844
+ const upsertResult = await upsertItems(items).catch((err) => {
2845
+ logger.warn("db", "upsertItems 失败", { source_url: listUrl, err: err instanceof Error ? err.message : String(err) });
2846
+ return { newCount: 0, newIds: /* @__PURE__ */ new Set() };
2847
+ });
2848
+ newCount = upsertResult.newCount;
2849
+ newIds = upsertResult.newIds;
2850
+ let pipelineDroppedNew = 0;
2851
+ const shouldRunPipelineRow = (guid) => newIds.has(guid);
2852
+ const hasEnrich = source.enrichItem != null || items.some((i) => getMatchedEnrichPlugin(i, { sourceUrl: listUrl }));
2853
+ if (!includeContent || items.length === 0 || !hasEnrich) {
2854
+ for (let i = 0; i < items.length; i++) {
2855
+ if (!shouldRunPipelineRow(items[i].guid)) continue;
2856
+ const processed = await runPipelineOnItem(items[i], { sourceUrl: listUrl, isEnriched: false });
2857
+ items[i] = processed;
2858
+ if (isPipelineDroppedItem(processed)) {
2859
+ await deleteItem(processed.guid).catch(
2860
+ (err) => logger.warn("db", "质量过滤后删除条目失败", { source_url: listUrl, err: err instanceof Error ? err.message : String(err) })
2861
+ );
2862
+ pipelineDroppedNew++;
2863
+ } else {
2864
+ updateItemContent(processed).catch(
2865
+ (err) => logger.warn("db", "updateItemContent 失败", { source_url: listUrl, err: err instanceof Error ? err.message : String(err) })
2866
+ );
2867
+ }
2868
+ }
2869
+ if (newCount > 0) {
2870
+ emitFeedUpdated({ sourceUrl: listUrl, newCount: newCount - pipelineDroppedNew });
2871
+ }
2872
+ const out = items.filter((i) => !isPipelineDroppedItem(i));
2873
+ if (deliverUrl && out.length > 0) {
2874
+ await postDeliverItemsSafe(deliverUrl, listUrl, out);
2875
+ }
2876
+ return { items: out };
2877
+ }
2878
+ const enrichFn = (item, _ctx) => buildEnrichFn(source, listUrl, ctx)(item);
2879
+ await enrichQueue.submit(
2880
+ items,
2881
+ enrichFn,
2882
+ ctx,
2883
+ {
2884
+ sourceUrl: listUrl,
2885
+ onItemDone: async (enrichedItem, index) => {
2886
+ enrichedItem.sourceRef = listUrl;
2887
+ const processed = shouldRunPipelineRow(enrichedItem.guid) ? await runPipelineOnItem(enrichedItem, { sourceUrl: listUrl, isEnriched: true }) : enrichedItem;
2888
+ items[index] = processed;
2889
+ if (isPipelineDroppedItem(processed)) {
2890
+ await deleteItem(processed.guid).catch(
2891
+ (err) => logger.warn("db", "质量过滤后删除条目失败", { source_url: listUrl, err: err instanceof Error ? err.message : String(err) })
2892
+ );
2893
+ pipelineDroppedNew++;
2894
+ } else {
2895
+ updateItemContent(processed).catch(
2896
+ (err) => logger.warn("db", "updateItemContent 失败", { source_url: listUrl, err: err instanceof Error ? err.message : String(err) })
2897
+ );
2898
+ }
2899
+ },
2900
+ onAllDone: async () => {
2901
+ for (let i = items.length - 1; i >= 0; i--) {
2902
+ if (isPipelineDroppedItem(items[i])) items.splice(i, 1);
2903
+ }
2904
+ if (newCount > 0) {
2905
+ emitFeedUpdated({ sourceUrl: listUrl, newCount: newCount - pipelineDroppedNew });
2906
+ }
2907
+ if (deliverUrl && items.length > 0) {
2908
+ await postDeliverItemsSafe(deliverUrl, listUrl, items);
2909
+ }
2910
+ }
2911
+ }
2912
+ );
2913
+ return { items };
2914
+ }
2915
+ async function getItems(listUrl, config = {}) {
2916
+ const source = getSource(listUrl);
2917
+ const key = config.cron ? cacheKeyFromCron(listUrl, config.cron) : cacheKey(listUrl, config.refreshInterval ?? source.refreshInterval ?? "1day");
2918
+ if (source.preCheck != null) {
2919
+ try {
2920
+ await source.preCheck({ cacheDir: config.cacheDir ?? "cache", headless: config.headless, proxy: config.proxy ?? source.proxy });
2921
+ } catch (err) {
2922
+ if (err instanceof AuthRequiredError) throw err;
2923
+ throw err;
2924
+ }
2925
+ }
2926
+ let task = config.force ? void 0 : generatingKeys.get(key);
2927
+ if (!task) {
2928
+ task = generateAndCache(listUrl, key, config);
2929
+ if (!config.force) generatingKeys.set(key, task);
2930
+ }
2931
+ const { items } = await task;
2932
+ return { items, fromCache: false };
2933
+ }
2934
+ function feedItemsToRssXml(items, listUrl, lng, opts) {
2935
+ const channel = buildChannelFromItems(listUrl, items, lng);
2936
+ if (opts?.channelTitle) channel.title = opts.channelTitle;
2937
+ if (opts?.channelDesc) channel.description = opts.channelDesc;
2938
+ return buildRssXml(channel, items.map((it) => toRssEntry(it, lng)));
2939
+ }
2940
+ const DEFAULT_REFRESH = "1day";
2941
+ const SOURCES_CONCURRENCY = 5;
2942
+ function createPullTask(ref, cacheDir, cronExpr) {
2943
+ return async () => {
2944
+ try {
2945
+ await getItems(ref, {
2946
+ cacheDir,
2947
+ cron: cronExpr
2948
+ });
2949
+ } catch (err) {
2950
+ throw err;
2951
+ }
2952
+ };
2953
+ }
2954
+ const SOURCES_GROUP = "sources";
2955
+ async function rescheduleSources(cacheDir, runNow2) {
2956
+ unscheduleGroup(SOURCES_GROUP);
2957
+ let sources;
2958
+ try {
2959
+ sources = await getAllSources();
2960
+ } catch {
2961
+ sources = [];
2962
+ }
2963
+ for (const src of sources) {
2964
+ const ref = resolveRef(src);
2965
+ if (!ref) continue;
2966
+ const cronExpr = src.cron ? src.cron : refreshIntervalToCron(src.refresh ?? DEFAULT_REFRESH);
2967
+ if (!validateCron(cronExpr)) continue;
2968
+ schedule(SOURCES_GROUP, ref, createPullTask(ref, cacheDir, cronExpr), {
2969
+ cron: cronExpr,
2970
+ retries: 2,
2971
+ retryDelayMs: 5e3,
2972
+ concurrency: SOURCES_CONCURRENCY,
2973
+ runNow: runNow2
2974
+ });
2975
+ }
2976
+ }
2977
+ async function initScheduler(cacheDir) {
2978
+ await rescheduleSources(cacheDir, true);
2979
+ let debounceTimer = null;
2980
+ try {
2981
+ const watcher = watch(SOURCES_CONFIG_PATH, () => {
2982
+ if (debounceTimer) clearTimeout(debounceTimer);
2983
+ debounceTimer = setTimeout(() => {
2984
+ rescheduleSources(cacheDir, false).catch(() => {
2985
+ });
2986
+ }, 500);
2987
+ });
2988
+ watcher.on("error", () => {
2989
+ });
2990
+ } catch {
2991
+ }
2992
+ }
2993
+ function requireAdmin() {
2994
+ return async (_c, next) => {
2995
+ return next();
2996
+ };
2997
+ }
2998
+ const PORT$1 = Number(process.env.PORT) || 18473;
2999
+ function registerServerRoutes(app) {
3000
+ app.get("/api/server-info", requireAdmin(), (c) => {
3001
+ const lanIp = Object.values(networkInterfaces()).flat().find((iface) => iface?.family === "IPv4" && !iface.internal)?.address;
3002
+ const lanUrl = lanIp ? `http://${lanIp}:${PORT$1}` : null;
3003
+ return c.json({ port: PORT$1, lanUrl });
3004
+ });
3005
+ }
3006
+ function registerRssApiRoutes(app) {
3007
+ app.get("/api/rss", async (c) => {
3008
+ const url = c.req.query("url");
3009
+ if (!url) return c.json({ error: "url 参数缺失" }, 400);
3010
+ const headlessParam = c.req.query("headless");
3011
+ const headless = headlessParam === "false" || headlessParam === "0" ? false : void 0;
3012
+ const lng = c.req.query("lng") ?? void 0;
3013
+ const limit = Math.min(Number(c.req.query("limit") ?? 20), 200);
3014
+ const offset = Number(c.req.query("offset") ?? 0);
3015
+ try {
3016
+ const httpId = "http-" + createHash("sha256").update(url).digest("hex").slice(0, 16);
3017
+ const { items: allItems, fromCache } = await schedule(
3018
+ SOURCES_GROUP,
3019
+ httpId,
3020
+ () => getItems(url, { cacheDir: CACHE_DIR, headless, lng })
3021
+ );
3022
+ const total = allItems.length;
3023
+ const pageItems = allItems.slice(offset, offset + limit);
3024
+ return c.json({
3025
+ fromCache,
3026
+ total,
3027
+ hasMore: offset + pageItems.length < total,
3028
+ items: pageItems.map((item) => {
3029
+ const { title, summary } = lng ? getEffectiveItemFields(item, lng) : { title: item.title, summary: item.summary ?? "" };
3030
+ return {
3031
+ guid: item.guid,
3032
+ title,
3033
+ link: item.link,
3034
+ summary,
3035
+ author: item.author,
3036
+ pubDate: item.pubDate instanceof Date ? item.pubDate.toISOString() : item.pubDate
3037
+ };
3038
+ })
3039
+ });
3040
+ } catch (err) {
3041
+ if (err instanceof AuthRequiredError) return c.json({ error: "需要登录", code: "AUTH_REQUIRED" }, 401);
3042
+ if (err instanceof NotFoundError) return c.json({ error: err.message, code: "NOT_FOUND" }, 404);
3043
+ return c.json({ error: err instanceof Error ? err.message : String(err) }, 500);
3044
+ }
3045
+ });
3046
+ }
3047
+ function registerEnrichRoutes(app) {
3048
+ app.get("/api/enrich/:taskId", (c) => {
3049
+ const taskId = c.req.param("taskId");
3050
+ const task = enrichQueue.getTask(taskId);
3051
+ if (!task) return c.json({ error: "任务不存在或已过期" }, 404);
3052
+ return c.json(task);
3053
+ });
3054
+ }
3055
+ function registerSchedulerRoutes(app) {
3056
+ app.get("/api/scheduler/stats", requireAdmin(), (c) => {
3057
+ const stats = getGroupStats();
3058
+ return c.json(stats);
3059
+ });
3060
+ }
3061
+ const USER_SITE_TEMPLATE = join(BUILTIN_PLUGINS_DIR, "templates", "site.rssany.js");
3062
+ const SITE_TEMPLATE_FALLBACK = `/**
3063
+ * Site 插件模板(由管理页添加,位于 .rssany/plugins/sources/)
3064
+ */
3065
+ export default {
3066
+ id: "__PLUGIN_ID__",
3067
+ listUrlPattern: "https://example.com/{segment}",
3068
+ refreshInterval: "1day",
3069
+
3070
+ async fetchItems(sourceId, ctx) {
3071
+ const { html, finalUrl } = await ctx.fetchHtml(sourceId, {
3072
+ waitMs: 2000,
3073
+ purify: true,
3074
+ });
3075
+ void html;
3076
+ void finalUrl;
3077
+ return [];
3078
+ },
3079
+ };
3080
+ `;
3081
+ function isValidNewPluginId(id) {
3082
+ return /^[a-zA-Z][a-zA-Z0-9_-]{0,63}$/.test(id) && id !== "generic" && id !== "new";
3083
+ }
3084
+ async function fileExists(p) {
3085
+ try {
3086
+ await access(p);
3087
+ return true;
3088
+ } catch {
3089
+ return false;
3090
+ }
3091
+ }
3092
+ function isAllowedPluginPath(absPath) {
3093
+ const f = resolve(absPath);
3094
+ for (const root of [BUILTIN_PLUGINS_DIR, USER_PLUGINS_DIR]) {
3095
+ const r = resolve(root);
3096
+ if (f === r || f.startsWith(r + sep)) return true;
3097
+ }
3098
+ return false;
3099
+ }
3100
+ function registerPluginsRoutes(app) {
3101
+ app.post("/api/plugins", requireAdmin(), async (c) => {
3102
+ let body;
3103
+ try {
3104
+ body = await c.req.json();
3105
+ } catch {
3106
+ return c.json({ error: "无效 JSON" }, 400);
3107
+ }
3108
+ const id = typeof body.id === "string" ? body.id.trim() : "";
3109
+ if (!id) return c.json({ error: "缺少 id" }, 400);
3110
+ if (!isValidNewPluginId(id)) {
3111
+ return c.json({ error: "id 须为字母开头,仅含字母数字、下划线、连字符;不能为 generic 或 new" }, 400);
3112
+ }
3113
+ await mkdir(USER_PLUGINS_DIR, { recursive: true });
3114
+ await mkdir(USER_SOURCES_DIR, { recursive: true });
3115
+ const outPath = join(USER_SOURCES_DIR, `${id}.rssany.ts`);
3116
+ if (await fileExists(outPath)) return c.json({ error: "该 id 已存在同名文件" }, 409);
3117
+ let tpl = SITE_TEMPLATE_FALLBACK;
3118
+ try {
3119
+ tpl = await readFile(USER_SITE_TEMPLATE, "utf-8");
3120
+ } catch {
3121
+ }
3122
+ const content = tpl.replace(/__PLUGIN_ID__/g, id);
3123
+ if (!isAllowedPluginPath(outPath)) return c.json({ error: "路径不允许" }, 403);
3124
+ try {
3125
+ await writeFile(outPath, content, "utf-8");
3126
+ await initSources();
3127
+ return c.json({ ok: true, filePath: outPath, id });
3128
+ } catch (e) {
3129
+ return c.json({ error: e instanceof Error ? e.message : String(e) }, 500);
3130
+ }
3131
+ });
3132
+ app.get("/api/plugins", requireAdmin(), (c) => {
3133
+ const sites = getPluginSites().map((s) => ({
3134
+ kind: "site",
3135
+ id: s.id,
3136
+ listUrlPattern: typeof s.listUrlPattern === "string" ? s.listUrlPattern : String(s.listUrlPattern),
3137
+ hasEnrich: !!s.enrichItem,
3138
+ hasAuth: !!(s.checkAuth && s.loginUrl)
3139
+ }));
3140
+ const siteIds = new Set(sites.map((p) => p.id));
3141
+ const sources = registeredSources.filter((src) => src.id !== "generic" && !siteIds.has(src.id)).map((src) => ({
3142
+ kind: "source",
3143
+ id: src.id,
3144
+ listUrlPattern: typeof src.pattern === "string" ? src.pattern : String(src.pattern),
3145
+ hasEnrich: !!src.enrichItem,
3146
+ hasAuth: false
3147
+ }));
3148
+ return c.json([...sites, ...sources]);
3149
+ });
3150
+ app.get("/api/plugins/:id", requireAdmin(), async (c) => {
3151
+ const id = decodeURIComponent(c.req.param("id") ?? "").trim();
3152
+ if (!id) return c.json({ error: "缺少 id" }, 400);
3153
+ const filePath = getPluginFilePath(id);
3154
+ if (!filePath) return c.json({ error: "未找到该插件或无可编辑文件" }, 404);
3155
+ if (!isAllowedPluginPath(filePath)) return c.json({ error: "路径不允许" }, 403);
3156
+ try {
3157
+ const content = await readFile(filePath, "utf-8");
3158
+ return c.json({ id, filePath, content });
3159
+ } catch (e) {
3160
+ return c.json({ error: e instanceof Error ? e.message : String(e) }, 500);
3161
+ }
3162
+ });
3163
+ app.put("/api/plugins/:id", requireAdmin(), async (c) => {
3164
+ const id = decodeURIComponent(c.req.param("id") ?? "").trim();
3165
+ if (!id) return c.json({ error: "缺少 id" }, 400);
3166
+ let body;
3167
+ try {
3168
+ body = await c.req.json();
3169
+ } catch {
3170
+ return c.json({ error: "无效 JSON" }, 400);
3171
+ }
3172
+ if (typeof body.content !== "string") return c.json({ error: "需要 content 字符串" }, 400);
3173
+ const filePath = getPluginFilePath(id);
3174
+ if (!filePath) return c.json({ error: "未找到该插件" }, 404);
3175
+ if (!isAllowedPluginPath(filePath)) return c.json({ error: "路径不允许" }, 403);
3176
+ try {
3177
+ await writeFile(filePath, body.content, "utf-8");
3178
+ await initSources();
3179
+ return c.json({ ok: true });
3180
+ } catch (e) {
3181
+ return c.json({ error: e instanceof Error ? e.message : String(e) }, 500);
3182
+ }
3183
+ });
3184
+ }
3185
+ function parseSteps(rawSteps) {
3186
+ return rawSteps.filter((s) => s && typeof s === "object" && typeof s.id === "string").map((s) => {
3187
+ const obj = s;
3188
+ const e = obj.enabled;
3189
+ return {
3190
+ id: String(obj.id).trim(),
3191
+ enabled: e !== false && e !== 0
3192
+ };
3193
+ }).filter((s) => s.id.length > 0);
3194
+ }
3195
+ function dedupeSteps(steps) {
3196
+ const seen = /* @__PURE__ */ new Set();
3197
+ const out = [];
3198
+ for (const s of steps) {
3199
+ if (seen.has(s.id)) continue;
3200
+ seen.add(s.id);
3201
+ out.push(s);
3202
+ }
3203
+ return out;
3204
+ }
3205
+ function registerPipelineRoutes(app) {
3206
+ app.get("/api/pipeline", requireAdmin(), async (c) => {
3207
+ const config = await loadPipelineConfig();
3208
+ return c.json({
3209
+ steps: config.steps,
3210
+ availableIds: [...PIPELINE_STEP_IDS],
3211
+ defaults: DEFAULT_PIPELINE_STEPS
3212
+ });
3213
+ });
3214
+ app.put("/api/pipeline", requireAdmin(), async (c) => {
3215
+ try {
3216
+ const body = await c.req.json();
3217
+ const rawSteps = Array.isArray(body?.steps) ? body.steps : [];
3218
+ const steps = dedupeSteps(parseSteps(rawSteps));
3219
+ await savePipelineConfig({ steps });
3220
+ return c.json({ ok: true, steps });
3221
+ } catch (err) {
3222
+ return c.json({ ok: false, message: err instanceof Error ? err.message : String(err) }, 400);
3223
+ }
3224
+ });
3225
+ }
3226
+ function registerFeedRoutes(app) {
3227
+ app.get("/api/feed", async (c) => {
3228
+ const limit = Math.min(Number(c.req.query("limit") ?? 50), 200);
3229
+ const offset = Number(c.req.query("offset") ?? 0);
3230
+ const refFilter = c.req.query("ref") ?? c.req.query("source") ?? void 0;
3231
+ const lng = c.req.query("lng") ?? void 0;
3232
+ const since = c.req.query("since");
3233
+ const until = c.req.query("until");
3234
+ const sources = await getAllSources();
3235
+ const refSet = new Set(sources.map((s) => resolveRef(s)).filter(Boolean));
3236
+ let sourceRefs;
3237
+ if (refFilter) {
3238
+ sourceRefs = refSet.has(refFilter) ? [refFilter] : [];
3239
+ } else {
3240
+ sourceRefs = await getAllSubscriptionRefs();
3241
+ }
3242
+ const labelByRef = new Map(sources.map((s) => [resolveRef(s), s.label ?? resolveRef(s)]));
3243
+ const sourcesMeta = sources.map((s) => ({
3244
+ ref: resolveRef(s),
3245
+ label: s.label ?? resolveRef(s)
3246
+ }));
3247
+ const dateOpts = since || until ? { since: since ?? void 0, until: until ?? void 0 } : void 0;
3248
+ const { items: dbItems, hasMore } = await queryFeedItems(sourceRefs, limit, offset, dateOpts);
3249
+ const items = dbItems.map((item) => {
3250
+ const refKey = item.source_url ?? "";
3251
+ const base2 = {
3252
+ ...item,
3253
+ sub_id: refKey,
3254
+ sub_title: labelByRef.get(refKey) ?? ""
3255
+ };
3256
+ if (!lng) return base2;
3257
+ const view = {
3258
+ title: item.title ?? "",
3259
+ summary: item.summary ?? "",
3260
+ content: item.content ?? "",
3261
+ translations: item.translations
3262
+ };
3263
+ const eff = getEffectiveItemFields(view, lng);
3264
+ return { ...base2, title: eff.title, summary: eff.summary, content: eff.content };
3265
+ });
3266
+ return c.json({ sources: sourcesMeta, items, hasMore });
3267
+ });
3268
+ app.get("/api/events", (c) => {
3269
+ return streamSSE(c, async (stream) => {
3270
+ await stream.writeSSE({ data: JSON.stringify({ type: "connected" }) });
3271
+ const off = onFeedUpdated((e) => {
3272
+ stream.writeSSE({ data: JSON.stringify({ type: "feed:updated", sourceUrl: e.sourceUrl, newCount: e.newCount }) }).catch(() => {
3273
+ });
3274
+ });
3275
+ const heartbeat = setInterval(() => {
3276
+ stream.writeSSE({ data: "", event: "ping" }).catch(() => {
3277
+ });
3278
+ }, 25e3);
3279
+ stream.onAbort(() => {
3280
+ off();
3281
+ clearInterval(heartbeat);
3282
+ });
3283
+ await new Promise((resolve2) => stream.onAbort(resolve2));
3284
+ });
3285
+ });
3286
+ }
3287
+ function parseSubscribedFlag$1(v) {
3288
+ return v === "1" || v === "true" || v === "yes";
3289
+ }
3290
+ function registerItemsRoutes(app) {
3291
+ app.get("/api/items/pending-push", async (c) => {
3292
+ const limit = Math.min(Number(c.req.query("limit") ?? 100), 500);
3293
+ const items = await getPendingPushItems(limit);
3294
+ return c.json({ items, count: items.length });
3295
+ });
3296
+ app.post("/api/items/mark-pushed", async (c) => {
3297
+ try {
3298
+ const { ids } = await c.req.json();
3299
+ if (!Array.isArray(ids) || ids.length === 0) return c.json({ ok: false, message: "ids 不能为空" }, 400);
3300
+ await markPushed(ids);
3301
+ return c.json({ ok: true, count: ids.length });
3302
+ } catch (err) {
3303
+ return c.json({ ok: false, message: err instanceof Error ? err.message : String(err) }, 400);
3304
+ }
3305
+ });
3306
+ app.delete("/api/items/:id", async (c) => {
3307
+ const id = decodeURIComponent(c.req.param("id") ?? "").trim();
3308
+ if (!id) return c.json({ ok: false, message: "id 不能为空" }, 400);
3309
+ const deleted = await deleteItem(id);
3310
+ if (!deleted) return c.json({ ok: false, message: "条目不存在或已删除" }, 404);
3311
+ return c.json({ ok: true });
3312
+ });
3313
+ app.delete("/api/items/by-source", requireAdmin(), async (c) => {
3314
+ const sourceUrl = (c.req.query("source_url") ?? "").trim();
3315
+ if (!sourceUrl) return c.json({ ok: false, message: "source_url 不能为空" }, 400);
3316
+ const deleted = await deleteItemsBySourceUrl(sourceUrl);
3317
+ return c.json({ ok: true, deleted });
3318
+ });
3319
+ app.get("/api/items", async (c) => {
3320
+ const ref = c.req.query("ref") ?? c.req.query("source") ?? void 0;
3321
+ const subscribed = parseSubscribedFlag$1(c.req.query("subscribed"));
3322
+ const author = c.req.query("author") ?? void 0;
3323
+ const q = c.req.query("q") ?? void 0;
3324
+ const tagsParam = c.req.query("tags") ?? void 0;
3325
+ const tags = tagsParam ? tagsParam.split(",").map((t) => t.trim()).filter(Boolean) : void 0;
3326
+ const daysParam = c.req.query("days");
3327
+ const sinceParam = c.req.query("since") ?? void 0;
3328
+ const untilParam = c.req.query("until") ?? void 0;
3329
+ let since;
3330
+ let until;
3331
+ const daysNum = daysParam !== void 0 && daysParam !== "" ? Number(daysParam) : NaN;
3332
+ if (Number.isFinite(daysNum) && daysNum > 0) {
3333
+ const n = Math.max(1, Math.min(365, Math.floor(daysNum)));
3334
+ const now2 = /* @__PURE__ */ new Date();
3335
+ const todayStart = new Date(now2.getFullYear(), now2.getMonth(), now2.getDate());
3336
+ const todayEnd = new Date(todayStart);
3337
+ todayEnd.setDate(todayEnd.getDate() + 1);
3338
+ since = new Date(todayStart);
3339
+ since.setDate(since.getDate() - (n - 1));
3340
+ until = todayEnd;
3341
+ } else {
3342
+ since = sinceParam ? new Date(sinceParam) : void 0;
3343
+ if (untilParam) {
3344
+ if (untilParam.length === 10) {
3345
+ const d = /* @__PURE__ */ new Date(untilParam + "T12:00:00Z");
3346
+ d.setUTCDate(d.getUTCDate() + 1);
3347
+ until = d;
3348
+ } else {
3349
+ until = new Date(untilParam);
3350
+ }
3351
+ }
3352
+ }
3353
+ const limit = Math.min(Number(c.req.query("limit") ?? 200), 500);
3354
+ const offset = Number(c.req.query("offset") ?? 0);
3355
+ const lng = c.req.query("lng") ?? void 0;
3356
+ let sourceUrls;
3357
+ let effectiveSourceUrl;
3358
+ if (ref) {
3359
+ effectiveSourceUrl = ref;
3360
+ sourceUrls = void 0;
3361
+ } else if (subscribed) {
3362
+ const refs = await getAllSubscriptionRefs();
3363
+ sourceUrls = refs.length > 0 ? refs : [];
3364
+ }
3365
+ if (!effectiveSourceUrl && sourceUrls?.length === 0) {
3366
+ return c.json({ items: [], total: 0, hasMore: false });
3367
+ }
3368
+ const result = await queryItems({
3369
+ sourceUrl: effectiveSourceUrl ?? (sourceUrls ? void 0 : ref),
3370
+ sourceUrls,
3371
+ author,
3372
+ q,
3373
+ tags,
3374
+ since,
3375
+ until,
3376
+ limit,
3377
+ offset
3378
+ });
3379
+ const items = lng && result.items.length > 0 ? result.items.map((it) => {
3380
+ const view = {
3381
+ title: it.title ?? "",
3382
+ summary: it.summary ?? "",
3383
+ content: it.content ?? "",
3384
+ translations: it.translations
3385
+ };
3386
+ const eff = getEffectiveItemFields(view, lng);
3387
+ return { ...it, title: eff.title, summary: eff.summary, content: eff.content };
3388
+ }) : result.items;
3389
+ const hasMore = offset + items.length < result.total;
3390
+ return c.json({ items, total: result.total, hasMore });
3391
+ });
3392
+ }
3393
+ function registerLogsRoutes(app) {
3394
+ app.delete("/api/logs", requireAdmin(), async (c) => {
3395
+ const deleted = await clearAllLogs();
3396
+ return c.json({ ok: true, deleted });
3397
+ });
3398
+ app.get("/api/logs", requireAdmin(), async (c) => {
3399
+ const levelParam = c.req.query("level");
3400
+ const level = levelParam === "error" || levelParam === "warn" || levelParam === "info" || levelParam === "debug" ? levelParam : void 0;
3401
+ const categoryRaw = c.req.query("category");
3402
+ const category = typeof categoryRaw === "string" && categoryRaw.trim() ? categoryRaw.trim() : void 0;
3403
+ const limit = Math.min(Number(c.req.query("limit") ?? 100), 200);
3404
+ const offset = Number(c.req.query("offset") ?? 0);
3405
+ const sinceParam = c.req.query("since");
3406
+ const since = sinceParam ? new Date(sinceParam) : void 0;
3407
+ const result = await queryLogs({ level, category, limit, offset, since });
3408
+ return c.json(result);
3409
+ });
3410
+ }
3411
+ function registerAdminApiRoutes(app) {
3412
+ app.get("/api/admin/verify", requireAdmin(), async (c) => {
3413
+ return c.json({ ok: true });
3414
+ });
3415
+ app.get("/api/admin/integrity-check", requireAdmin(), async (c) => {
3416
+ try {
3417
+ const result = await runIntegrityCheck();
3418
+ const ok = result === "ok";
3419
+ return c.json({ ok, result });
3420
+ } catch (err) {
3421
+ const message = err instanceof Error ? err.message : String(err);
3422
+ return c.json({ ok: false, result: `error: ${message}` }, 500);
3423
+ }
3424
+ });
3425
+ }
3426
+ function registerSourcesRoutes(app) {
3427
+ app.get("/api/sources/stats", requireAdmin(), async (c) => {
3428
+ const stats = await getSourceStats();
3429
+ return c.json(stats);
3430
+ });
3431
+ app.post("/api/sources/plugin-match", requireAdmin(), async (c) => {
3432
+ try {
3433
+ const body = await c.req.json();
3434
+ const refs = Array.isArray(body?.refs) ? body.refs : [];
3435
+ const pluginIds = new Set(getPluginSites().map((s) => s.id));
3436
+ const result = {};
3437
+ for (const ref of refs) {
3438
+ const source = getSource(ref);
3439
+ result[ref] = pluginIds.has(source.id) ? source.id : null;
3440
+ }
3441
+ return c.json(result);
3442
+ } catch {
3443
+ return c.json({});
3444
+ }
3445
+ });
3446
+ app.get("/api/sources/raw", requireAdmin(), async (c) => {
3447
+ try {
3448
+ const raw = await getSourcesRaw();
3449
+ return c.text(raw, 200, { "Content-Type": "application/json; charset=utf-8" });
3450
+ } catch {
3451
+ return c.text(JSON.stringify({ sources: [] }, null, 2), 200, { "Content-Type": "application/json; charset=utf-8" });
3452
+ }
3453
+ });
3454
+ app.put("/api/sources/raw", requireAdmin(), async (c) => {
3455
+ try {
3456
+ const body = await c.req.json();
3457
+ const list = Array.isArray(body?.sources) ? body.sources : [];
3458
+ const sources = list.filter((s) => s != null && typeof s === "object" && typeof s.ref === "string").map((s) => {
3459
+ const t = s.type;
3460
+ const type = t === "web" || t === "rss" || t === "email" ? t : void 0;
3461
+ const r = s.refresh;
3462
+ const refresh = r && VALID_INTERVALS.includes(r) ? r : void 0;
3463
+ const w = s.weight;
3464
+ const weight = typeof w === "number" ? w : void 0;
3465
+ return {
3466
+ ref: String(s.ref),
3467
+ type,
3468
+ label: s.label,
3469
+ description: s.description,
3470
+ refresh,
3471
+ proxy: s.proxy,
3472
+ weight
3473
+ };
3474
+ });
3475
+ await saveSourcesFile(sources);
3476
+ return c.json({ ok: true });
3477
+ } catch (err) {
3478
+ return c.json({ ok: false, message: err instanceof Error ? err.message : String(err) }, 400);
3479
+ }
3480
+ });
3481
+ }
3482
+ function registerTopicsRoutes(app) {
3483
+ app.get("/api/tags", async (c) => {
3484
+ const [tags, stats, suggested] = await Promise.all([
3485
+ getSystemTags(),
3486
+ getSystemTagStats(),
3487
+ getSuggestedTags()
3488
+ ]);
3489
+ return c.json({
3490
+ tags,
3491
+ stats: stats.map((s) => ({ name: s.name, count: s.count, hotness: s.hotness })),
3492
+ suggestedTags: suggested.map((s) => ({ name: s.name, count: s.count, hotness: s.hotness }))
3493
+ });
3494
+ });
3495
+ app.put("/api/tags", requireAdmin(), async (c) => {
3496
+ try {
3497
+ const body = await c.req.json();
3498
+ const list = Array.isArray(body?.tags) ? body.tags : [];
3499
+ await saveSystemTagsToFile(list);
3500
+ const [tags, stats, suggested] = await Promise.all([
3501
+ getSystemTags(),
3502
+ getSystemTagStats(),
3503
+ getSuggestedTags()
3504
+ ]);
3505
+ return c.json({
3506
+ ok: true,
3507
+ tags,
3508
+ stats: stats.map((s) => ({ name: s.name, count: s.count, hotness: s.hotness })),
3509
+ suggestedTags: suggested.map((s) => ({ name: s.name, count: s.count, hotness: s.hotness }))
3510
+ });
3511
+ } catch (err) {
3512
+ return c.json({ ok: false, message: err instanceof Error ? err.message : String(err) }, 400);
3513
+ }
3514
+ });
3515
+ app.post("/api/tags/remove-from-items", requireAdmin(), async (c) => {
3516
+ try {
3517
+ const body = await c.req.json();
3518
+ const tag = typeof body?.tag === "string" ? body.tag.trim() : "";
3519
+ if (!tag) return c.json({ ok: false, message: "tag 参数缺失" }, 400);
3520
+ const count = await removeTagFromAllItems(tag);
3521
+ const [tags, stats, suggested] = await Promise.all([
3522
+ getSystemTags(),
3523
+ getSystemTagStats(),
3524
+ getSuggestedTags()
3525
+ ]);
3526
+ return c.json({
3527
+ ok: true,
3528
+ removedCount: count,
3529
+ tags,
3530
+ stats: stats.map((s) => ({ name: s.name, count: s.count, hotness: s.hotness })),
3531
+ suggestedTags: suggested.map((s) => ({ name: s.name, count: s.count, hotness: s.hotness }))
3532
+ });
3533
+ } catch (err) {
3534
+ return c.json({ ok: false, message: err instanceof Error ? err.message : String(err) }, 400);
3535
+ }
3536
+ });
3537
+ }
3538
+ function registerDeliverRoutes(app) {
3539
+ app.get("/api/deliver", requireAdmin(), async (c) => {
3540
+ const url = await getDeliverUrl();
3541
+ return c.json({ url });
3542
+ });
3543
+ app.put("/api/deliver", requireAdmin(), async (c) => {
3544
+ try {
3545
+ const body = await c.req.json();
3546
+ const url = typeof body?.url === "string" ? body.url.trim() : "";
3547
+ await saveDeliverUrl(url);
3548
+ return c.json({ ok: true, url });
3549
+ } catch (err) {
3550
+ return c.json({ ok: false, message: err instanceof Error ? err.message : String(err) }, 400);
3551
+ }
3552
+ });
3553
+ app.post("/api/deliver/test", requireAdmin(), async (c) => {
3554
+ try {
3555
+ const body = await c.req.json();
3556
+ const url = typeof body?.url === "string" ? body.url.trim() : "";
3557
+ if (!url) return c.json({ ok: false, message: "url 不能为空" }, 400);
3558
+ const sample = {
3559
+ guid: "deliver-test-" + Date.now(),
3560
+ title: "投递连通性测试",
3561
+ link: "https://example.com/rssany-deliver-test",
3562
+ pubDate: (/* @__PURE__ */ new Date()).toISOString(),
3563
+ summary: "若下游收到此条,说明投递 URL 可用。"
3564
+ };
3565
+ await postDeliverItems(url, "rssany-deliver-test", [
3566
+ {
3567
+ guid: sample.guid,
3568
+ title: sample.title,
3569
+ link: sample.link,
3570
+ pubDate: new Date(sample.pubDate),
3571
+ summary: sample.summary,
3572
+ sourceRef: "rssany-deliver-test"
3573
+ }
3574
+ ]);
3575
+ return c.json({ ok: true });
3576
+ } catch (err) {
3577
+ return c.json({ ok: false, message: err instanceof Error ? err.message : String(err) }, 400);
3578
+ }
3579
+ });
3580
+ }
3581
+ const tasks = /* @__PURE__ */ new Map();
3582
+ let idCounter = 0;
3583
+ function nextId() {
3584
+ idCounter += 1;
3585
+ return `t_${Date.now().toString(36)}_${idCounter}`;
3586
+ }
3587
+ function createTask() {
3588
+ const id = nextId();
3589
+ const now2 = Date.now();
3590
+ tasks.set(id, { id, status: "pending", createdAt: now2, updatedAt: now2 });
3591
+ return id;
3592
+ }
3593
+ function getTask(id) {
3594
+ return tasks.get(id) ?? null;
3595
+ }
3596
+ function setTaskRunning(id) {
3597
+ const t = tasks.get(id);
3598
+ if (t) {
3599
+ t.status = "running";
3600
+ t.updatedAt = Date.now();
3601
+ }
3602
+ }
3603
+ function setTaskDone(id, result) {
3604
+ const t = tasks.get(id);
3605
+ if (t) {
3606
+ t.status = "done";
3607
+ t.result = result;
3608
+ t.updatedAt = Date.now();
3609
+ }
3610
+ }
3611
+ function setTaskError(id, error) {
3612
+ const t = tasks.get(id);
3613
+ if (t) {
3614
+ t.status = "error";
3615
+ t.error = error;
3616
+ t.updatedAt = Date.now();
3617
+ }
3618
+ }
3619
+ function registerTasksRoutes(app) {
3620
+ app.get("/api/tasks/:id", (c) => {
3621
+ const id = c.req.param("id") ?? "";
3622
+ const task = getTask(id);
3623
+ if (!task) return c.json({ error: "任务不存在" }, 404);
3624
+ return c.json(task);
3625
+ });
3626
+ app.post("/api/tasks", requireAdmin(), async (c) => {
3627
+ try {
3628
+ const body = await c.req.json().catch(() => ({}));
3629
+ const type = body.type ?? "";
3630
+ if (type === "source-pull") {
3631
+ const ref = typeof body.ref === "string" ? body.ref.trim() : "";
3632
+ if (!ref) return c.json({ error: "ref 不能为空" }, 400);
3633
+ const taskId = createTask();
3634
+ schedule(SOURCES_GROUP, taskId, async () => {
3635
+ setTaskRunning(taskId);
3636
+ try {
3637
+ await getItems(ref, { cacheDir: CACHE_DIR, force: true });
3638
+ setTaskDone(taskId, { ok: true });
3639
+ } catch (err) {
3640
+ const msg = err instanceof Error ? err.message : String(err);
3641
+ setTaskError(taskId, msg);
3642
+ throw err;
3643
+ }
3644
+ }, { priority: true }).catch(() => {
3645
+ });
3646
+ return c.json({ taskId });
3647
+ }
3648
+ return c.json({ error: `未知任务类型: ${type}` }, 400);
3649
+ } catch (err) {
3650
+ return c.json({ error: err instanceof Error ? err.message : String(err) }, 500);
3651
+ }
3652
+ });
3653
+ }
3654
+ function registerApiRoutes(app) {
3655
+ registerServerRoutes(app);
3656
+ registerRssApiRoutes(app);
3657
+ registerEnrichRoutes(app);
3658
+ registerSchedulerRoutes(app);
3659
+ registerPluginsRoutes(app);
3660
+ registerPipelineRoutes(app);
3661
+ registerFeedRoutes(app);
3662
+ registerItemsRoutes(app);
3663
+ registerLogsRoutes(app);
3664
+ registerAdminApiRoutes(app);
3665
+ registerSourcesRoutes(app);
3666
+ registerTopicsRoutes(app);
3667
+ registerDeliverRoutes(app);
3668
+ registerTasksRoutes(app);
3669
+ }
3670
+ function registerAuthRoutes(app) {
3671
+ app.get("/auth/check", async (c) => {
3672
+ const siteIdParam = c.req.query("siteId");
3673
+ if (!siteIdParam) {
3674
+ return c.json({ ok: false, message: "请提供 siteId" }, 400);
3675
+ }
3676
+ const site = getWebSite(siteIdParam);
3677
+ if (!site) return c.json({ ok: false, message: "无此站点" }, 404);
3678
+ const authFlow = toAuthFlow(site);
3679
+ if (!authFlow) return c.json({ ok: false, message: "该站点无需登录" }, 400);
3680
+ try {
3681
+ const authenticated = await preCheckAuth(authFlow, CACHE_DIR);
3682
+ return c.json({ ok: true, authenticated });
3683
+ } catch (err) {
3684
+ const msg = err instanceof Error ? err.message : String(err);
3685
+ return c.json({ ok: false, message: `检查失败: ${msg}` }, 500);
3686
+ }
3687
+ });
3688
+ app.post("/auth/open", async (c) => {
3689
+ const siteIdParam = c.req.query("siteId");
3690
+ if (!siteIdParam) {
3691
+ return c.json({ ok: false, message: "请提供 siteId" }, 400);
3692
+ }
3693
+ const site = getWebSite(siteIdParam);
3694
+ if (!site) return c.json({ ok: false, message: "无此站点" }, 404);
3695
+ const authFlow = toAuthFlow(site);
3696
+ if (!authFlow) return c.json({ ok: false, message: "该站点无需登录" }, 400);
3697
+ const { loginUrl } = authFlow;
3698
+ getOrCreateBrowser({ headless: false, cacheDir: CACHE_DIR }).then(async (browser) => {
3699
+ const page = await browser.newPage();
3700
+ const realUserAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36";
3701
+ await page.setUserAgent(realUserAgent);
3702
+ await page.setViewport({ width: 1366, height: 960 });
3703
+ await page.goto(loginUrl, { waitUntil: "domcontentloaded", timeout: 6e4 });
3704
+ }).catch(() => {
3705
+ });
3706
+ return c.json({ ok: true, message: "已打开登录页面" });
3707
+ });
3708
+ app.post("/auth/ensure", async (c) => {
3709
+ const urlParam = c.req.query("url");
3710
+ const siteIdParam = c.req.query("siteId");
3711
+ let site;
3712
+ if (urlParam) {
3713
+ const decoded = decodeURIComponent(urlParam);
3714
+ site = getBestSite(decoded);
3715
+ if (!site) return c.json({ ok: false, message: "无匹配站点" }, 404);
3716
+ } else if (siteIdParam) {
3717
+ site = getWebSite(siteIdParam);
3718
+ if (!site) return c.json({ ok: false, message: "无此站点" }, 404);
3719
+ } else {
3720
+ return c.json({ ok: false, message: "请提供 url 或 siteId" }, 400);
3721
+ }
3722
+ const authFlow = toAuthFlow(site);
3723
+ if (!authFlow) return c.json({ ok: false, message: "该站点无需登录" }, 400);
3724
+ ensureAuth(authFlow, CACHE_DIR).then(() => {
3725
+ }).catch(() => {
3726
+ });
3727
+ return c.json({ ok: true, message: "已打开登录窗口,请在弹出的浏览器中完成登录,完成后刷新订阅页面即可。" });
3728
+ });
3729
+ }
3730
+ const STATICS_DIR = join(PACKAGE_ROOT, "statics");
3731
+ function parseUrlFromPath(path, prefix) {
3732
+ const raw = path.slice(prefix.length) || "";
3733
+ const decoded = decodeURIComponent(raw.startsWith("/") ? raw.slice(1) : raw);
3734
+ if (!decoded) return null;
3735
+ return decoded.startsWith("http") ? decoded : `https://${decoded}`;
3736
+ }
3737
+ async function readStaticHtml(name, fallback) {
3738
+ try {
3739
+ return await readFile(join(STATICS_DIR, `${name}.html`), "utf-8");
3740
+ } catch {
3741
+ return fallback;
3742
+ }
3743
+ }
3744
+ function escapeHtml(s) {
3745
+ return s.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;").replace(/'/g, "&#39;");
3746
+ }
3747
+ function registerAdminRoutes(app) {
3748
+ async function render401(listUrl) {
3749
+ const raw = await readStaticHtml("401", '<!DOCTYPE html><html><head><meta charset="utf-8"><title>401</title></head><body><h1>401 需要登录</h1></body></html>');
3750
+ return raw.replace(/\{\{listUrl\}\}/g, escapeHtml(listUrl));
3751
+ }
3752
+ app.get("/admin/parse/*", requireAdmin(), async (c) => {
3753
+ const url = parseUrlFromPath(c.req.path, "/admin/parse");
3754
+ if (!url) return c.text("无效 URL,格式: /admin/parse/https://... 或 /admin/parse/example.com/...", 400);
3755
+ try {
3756
+ const headlessParam = c.req.query("headless");
3757
+ const headless = headlessParam === "false" || headlessParam === "0" ? false : void 0;
3758
+ const source = getSource(url);
3759
+ const ctx = { cacheDir: CACHE_DIR, headless, proxy: source.proxy };
3760
+ const items = await source.fetchItems(url, ctx);
3761
+ const mode = source.id === "generic" ? "generic" : "plugin";
3762
+ return c.json({ items, url, mode, pluginId: source.id });
3763
+ } catch (err) {
3764
+ if (err instanceof AuthRequiredError) {
3765
+ const html = await render401(url);
3766
+ return c.html(html, 401);
3767
+ }
3768
+ const msg = err instanceof Error ? err.message : String(err);
3769
+ return c.text(`解析失败: ${msg}`, 500);
3770
+ }
3771
+ });
3772
+ app.get("/admin/extractor/*", requireAdmin(), async (c) => {
3773
+ const url = parseUrlFromPath(c.req.path, "/admin/extractor");
3774
+ if (!url) return c.text("无效 URL,格式: /admin/extractor/https://... 或 /admin/extractor/example.com/...", 400);
3775
+ try {
3776
+ const headlessParam = c.req.query("headless");
3777
+ const headless = headlessParam === "false" || headlessParam === "0" ? false : void 0;
3778
+ const site = getBestSite(url);
3779
+ if (site?.enrichItem) {
3780
+ const siteCtx = buildSiteContext(site, { cacheDir: CACHE_DIR, headless });
3781
+ const stub = { guid: url, title: "", link: url, pubDate: /* @__PURE__ */ new Date() };
3782
+ const enriched = await site.enrichItem(stub, siteCtx);
3783
+ return c.json({
3784
+ title: enriched.title ?? null,
3785
+ author: enriched.author ?? null,
3786
+ pubDate: enriched.pubDate instanceof Date ? enriched.pubDate.toISOString() : enriched.pubDate ?? null,
3787
+ content: enriched.content ?? null,
3788
+ _extractor: site.id
3789
+ });
3790
+ }
3791
+ const proxy = site?.proxy;
3792
+ const result = await extractFromLink(url, {}, { timeoutMs: 6e4, headless, proxy });
3793
+ return c.json({
3794
+ title: result.title ?? null,
3795
+ author: result.author ?? null,
3796
+ pubDate: result.pubDate ?? null,
3797
+ content: result.content ?? null,
3798
+ _extractor: "readability"
3799
+ });
3800
+ } catch (err) {
3801
+ const msg = err instanceof Error ? err.message : String(err);
3802
+ return c.text(`提取失败: ${msg}`, 500);
3803
+ }
3804
+ });
3805
+ }
3806
+ function parseSubscribedFlag(v) {
3807
+ return v === "1" || v === "true" || v === "yes";
3808
+ }
3809
+ function registerRssRoutes(app) {
3810
+ async function render401(listUrl) {
3811
+ const raw = await readStaticHtml("401", '<!DOCTYPE html><html><head><meta charset="utf-8"><title>401</title></head><body><h1>401 需要登录</h1></body></html>');
3812
+ return raw.replace(/\{\{listUrl\}\}/g, escapeHtml(listUrl));
3813
+ }
3814
+ app.get("/rss", async (c) => {
3815
+ const search = c.req.query("search") ?? c.req.query("q") ?? void 0;
3816
+ const ref = c.req.query("ref") ?? c.req.query("source") ?? c.req.query("sourceUrl") ?? void 0;
3817
+ const subscribed = parseSubscribedFlag(c.req.query("subscribed"));
3818
+ const author = c.req.query("author") ?? void 0;
3819
+ const tagsParam = c.req.query("tags") ?? void 0;
3820
+ const tags = tagsParam ? tagsParam.split(",").map((t) => t.trim()).filter(Boolean) : void 0;
3821
+ const lng = c.req.query("lng") ?? void 0;
3822
+ const limit = Math.min(Number(c.req.query("limit") ?? 50), 200);
3823
+ const offset = Number(c.req.query("offset") ?? 0);
3824
+ const title = c.req.query("title") ?? void 0;
3825
+ const daysParam = c.req.query("days");
3826
+ const sinceParam = c.req.query("since") ?? void 0;
3827
+ const untilParam = c.req.query("until") ?? void 0;
3828
+ let since;
3829
+ let until;
3830
+ if (daysParam) {
3831
+ const n = Math.max(1, Math.min(365, Number(daysParam) || 1));
3832
+ const now2 = /* @__PURE__ */ new Date();
3833
+ const todayStart = new Date(now2.getFullYear(), now2.getMonth(), now2.getDate());
3834
+ const todayEnd = new Date(todayStart);
3835
+ todayEnd.setDate(todayEnd.getDate() + 1);
3836
+ since = new Date(todayStart);
3837
+ since.setDate(since.getDate() - (n - 1));
3838
+ until = todayEnd;
3839
+ } else {
3840
+ since = sinceParam ? new Date(sinceParam) : void 0;
3841
+ if (untilParam) {
3842
+ if (untilParam.length === 10) {
3843
+ const d = /* @__PURE__ */ new Date(untilParam + "T12:00:00Z");
3844
+ d.setUTCDate(d.getUTCDate() + 1);
3845
+ until = d;
3846
+ } else {
3847
+ until = new Date(untilParam);
3848
+ }
3849
+ }
3850
+ }
3851
+ let sourceUrls;
3852
+ if (ref) {
3853
+ sourceUrls = void 0;
3854
+ } else if (subscribed) {
3855
+ sourceUrls = await getAllSubscriptionRefs();
3856
+ }
3857
+ if (sourceUrls?.length === 0) {
3858
+ const xml2 = feedItemsToRssXml([], new URL(c.req.url).href, lng, {
3859
+ channelTitle: title ?? "RSS 订阅",
3860
+ channelDesc: "无匹配条目"
3861
+ });
3862
+ return c.body(xml2, 200, { "Content-Type": "application/rss+xml; charset=utf-8" });
3863
+ }
3864
+ const result = await queryItems({
3865
+ sourceUrl: sourceUrls ? void 0 : ref,
3866
+ sourceUrls,
3867
+ author,
3868
+ q: search,
3869
+ tags,
3870
+ since,
3871
+ until,
3872
+ limit,
3873
+ offset
3874
+ });
3875
+ const feedItems = result.items.map((dbItem) => ({
3876
+ guid: dbItem.id,
3877
+ title: dbItem.title ?? "",
3878
+ link: dbItem.url,
3879
+ pubDate: dbItem.pub_date ? new Date(dbItem.pub_date) : /* @__PURE__ */ new Date(),
3880
+ author: dbItem.author ?? void 0,
3881
+ summary: dbItem.summary ?? void 0,
3882
+ content: dbItem.content ?? void 0,
3883
+ imageUrl: dbItem.image_url ?? void 0,
3884
+ tags: dbItem.tags ?? void 0,
3885
+ sourceRef: dbItem.source_url,
3886
+ translations: dbItem.translations ?? void 0
3887
+ }));
3888
+ const rssUrl = new URL(c.req.url);
3889
+ const channelTitle = title ?? "RSS 订阅";
3890
+ const xml = feedItemsToRssXml(feedItems, rssUrl.href, lng, {
3891
+ channelTitle,
3892
+ channelDesc: `来自 ${rssUrl.href} 的订阅`
3893
+ });
3894
+ return c.body(xml, 200, {
3895
+ "Content-Type": "application/rss+xml; charset=utf-8"
3896
+ });
3897
+ });
3898
+ app.get("/rss/*", async (c) => {
3899
+ const url = parseUrlFromPath(c.req.path, "/rss");
3900
+ if (!url) return c.text("无效 URL,格式: /rss/https://... 或 /rss/www.xiaohongshu.com/...", 400);
3901
+ try {
3902
+ const headlessParam = c.req.query("headless");
3903
+ const headless = headlessParam === "false" || headlessParam === "0" ? false : void 0;
3904
+ const lng = c.req.query("lng") ?? void 0;
3905
+ const httpId = "rss-" + createHash("sha256").update(url).digest("hex").slice(0, 16);
3906
+ const { items } = await schedule(
3907
+ SOURCES_GROUP,
3908
+ httpId,
3909
+ () => getItems(url, { cacheDir: CACHE_DIR, headless, lng })
3910
+ );
3911
+ const xml = feedItemsToRssXml(items, url, lng);
3912
+ return c.body(xml, 200, {
3913
+ "Content-Type": "application/rss+xml; charset=utf-8"
3914
+ });
3915
+ } catch (err) {
3916
+ if (err instanceof AuthRequiredError) {
3917
+ const html = await render401(url);
3918
+ return c.html(html, 401);
3919
+ }
3920
+ if (err instanceof NotFoundError) {
3921
+ const html = await readStaticHtml("404", '<!DOCTYPE html><html><head><meta charset="utf-8"><title>404</title></head><body><h1>404 未找到</h1></body></html>');
3922
+ return c.html(html, 404);
3923
+ }
3924
+ const msg = err instanceof Error ? err.message : String(err);
3925
+ return c.text(`生成 RSS 失败: ${msg}`, 500);
3926
+ }
3927
+ });
3928
+ }
3929
+ function getWebUiBuildDir() {
3930
+ const w = process.env.WEBUI_BUILD_DIR?.trim();
3931
+ if (w) {
3932
+ if (w.startsWith("/") || /^[A-Za-z]:[\\/]/.test(w)) return w;
3933
+ return join(process.cwd(), w);
3934
+ }
3935
+ return join(PACKAGE_ROOT, "webui/build");
3936
+ }
3937
+ function isBackendOnlyPath(pathname) {
3938
+ if (pathname.startsWith("/api")) return true;
3939
+ if (pathname.startsWith("/rss")) return true;
3940
+ if (pathname.startsWith("/auth")) return true;
3941
+ if (pathname.startsWith("/admin/parse") || pathname.startsWith("/admin/extractor")) return true;
3942
+ return false;
3943
+ }
3944
+ function looksLikeStaticAsset(pathname) {
3945
+ return /\.[a-zA-Z0-9]{1,12}$/.test(pathname);
3946
+ }
3947
+ function registerWebUiRoutes(app) {
3948
+ const absRoot = getWebUiBuildDir();
3949
+ if (!existsSync(absRoot)) {
3950
+ console.warn(
3951
+ "未找到 WebUI 构建目录,跳过根路径静态托管:",
3952
+ absRoot,
3953
+ "(构建前端:pnpm run webui:build)"
3954
+ );
3955
+ return;
3956
+ }
3957
+ const relRoot = relative(process.cwd(), absRoot).replace(/\\/g, "/");
3958
+ const staticRoot = relRoot === "" || relRoot === "." ? "." : relRoot.startsWith(".") || relRoot.startsWith("/") || /^[A-Za-z]:/.test(relRoot) ? relRoot : `./${relRoot}`;
3959
+ const staticMw = serveStatic({
3960
+ root: staticRoot,
3961
+ index: "200.html"
3962
+ });
3963
+ app.use("*", async (c, next) => {
3964
+ if (isBackendOnlyPath(c.req.path)) return next();
3965
+ return staticMw(c, next);
3966
+ });
3967
+ const spaFallback = async (c) => {
3968
+ const p = c.req.path;
3969
+ if (isBackendOnlyPath(p)) return c.notFound();
3970
+ if (looksLikeStaticAsset(p)) return c.notFound();
3971
+ try {
3972
+ const html = await readFile(join(absRoot, "200.html"), "utf-8");
3973
+ return c.html(html);
3974
+ } catch {
3975
+ return c.notFound();
3976
+ }
3977
+ };
3978
+ app.get("*", spaFallback);
3979
+ }
3980
+ const PORT = Number(process.env.PORT) || 18473;
3981
+ const IS_DEV = process.env.NODE_ENV === "development" || process.argv.includes("--watch");
3982
+ const PLUGIN_WATCH_EXTS = [".rssany.js", ".rssany.ts"];
3983
+ function createApp() {
3984
+ const app = new Hono();
3985
+ app.use(
3986
+ "*",
3987
+ cors({
3988
+ origin: "*",
3989
+ allowMethods: ["GET", "POST", "PUT", "DELETE", "OPTIONS", "PATCH"],
3990
+ allowHeaders: ["Content-Type", "Authorization", "X-Admin-Token"]
3991
+ })
3992
+ );
3993
+ registerApiRoutes(app);
3994
+ registerAuthRoutes(app);
3995
+ registerAdminRoutes(app);
3996
+ registerRssRoutes(app);
3997
+ registerWebUiRoutes(app);
3998
+ return app;
3999
+ }
4000
+ function watchPlugins() {
4001
+ let reloadTimer = null;
4002
+ const debouncedReload = async () => {
4003
+ if (reloadTimer) clearTimeout(reloadTimer);
4004
+ reloadTimer = setTimeout(async () => {
4005
+ try {
4006
+ await initSources();
4007
+ } catch (err) {
4008
+ logger.error("plugin", "插件重新加载失败", { err: err instanceof Error ? err.message : String(err) });
4009
+ }
4010
+ }, 300);
4011
+ };
4012
+ for (const dir of [BUILTIN_PLUGINS_DIR, USER_PLUGINS_DIR]) {
4013
+ const watcher = watch(dir, { recursive: true }, (eventType, filename) => {
4014
+ if (!filename || !PLUGIN_WATCH_EXTS.some((ext) => filename.endsWith(ext))) return;
4015
+ if (eventType === "rename" || eventType === "change") debouncedReload();
4016
+ });
4017
+ watcher.on("error", (err) => {
4018
+ logger.warn("plugin", "插件目录监听错误", { dir, err: err.message });
4019
+ });
4020
+ }
4021
+ }
4022
+ async function main() {
4023
+ await initUserDir();
4024
+ await initSources();
4025
+ await initScheduler(CACHE_DIR);
4026
+ const app = createApp();
4027
+ const server = serve({ fetch: app.fetch, port: PORT, hostname: "0.0.0.0" });
4028
+ server.setMaxListeners(32);
4029
+ console.log(`服务已启动 http://127.0.0.1:${PORT}/(API + 静态前端,需先 pnpm run webui:build)`);
4030
+ const lanIp = Object.values(networkInterfaces()).flat().find((iface) => iface?.family === "IPv4" && !iface.internal)?.address;
4031
+ if (lanIp) console.log(`局域网访问 http://${lanIp}:${PORT}/`);
4032
+ if (IS_DEV) {
4033
+ watchPlugins();
4034
+ }
4035
+ }
4036
+ main();
4037
+ //# sourceMappingURL=index.js.map