rssany 0.1.5 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -5
- package/app/plugins/builtin/agi-eval-evaluation.rssany.js +1 -1
- package/app/plugins/builtin/brightdata-blog.rssany.js +1 -1
- package/app/plugins/builtin/five-radar.rssany.js +1 -1
- package/app/plugins/builtin/google-deepmind-research.rssany.js +1 -1
- package/app/plugins/builtin/opendrivelab-publications.rssany.js +1 -1
- package/app/plugins/builtin/pjlab-adg-publications.rssany.js +1 -1
- package/app/plugins/builtin/theinformation-briefings.rssany.js +150 -0
- package/app/plugins/builtin/x.rssany.js +192 -23
- package/app/plugins/builtin/zhipu-research.rssany.js +2 -2
- package/app/plugins/site.rssany.js +1 -0
- package/dist/index.js +347 -255
- package/dist/index.js.map +1 -1
- package/init/config.json +1 -1
- package/package.json +8 -8
- package/webui/build/200.html +6 -6
- package/webui/build/_app/immutable/assets/12.DfJcfUWl.css +1 -0
- package/webui/build/_app/immutable/assets/5.B-dPiwB7.css +1 -0
- package/webui/build/_app/immutable/assets/6.B27N7pdA.css +1 -0
- package/webui/build/_app/immutable/assets/8.Cgji2b15.css +1 -0
- package/webui/build/_app/immutable/assets/9.BsCIAvn3.css +1 -0
- package/webui/build/_app/immutable/assets/homeFeedPanelStore.CSvlNcpm.css +1 -0
- package/webui/build/_app/immutable/chunks/5LVkDJzw.js +1 -0
- package/webui/build/_app/immutable/chunks/Bns1MuyM.js +36 -0
- package/webui/build/_app/immutable/chunks/{D6VIKef0.js → Bu9HsS-V.js} +1 -1
- package/webui/build/_app/immutable/chunks/{Dbqx2mXq.js → CmjOpds-.js} +1 -1
- package/webui/build/_app/immutable/chunks/bvuf_jZd.js +36 -0
- package/webui/build/_app/immutable/entry/{app.XPso7q7g.js → app.BVkrDt5l.js} +2 -2
- package/webui/build/_app/immutable/entry/start.D3Q-BMMd.js +1 -0
- package/webui/build/_app/immutable/nodes/{0.BKTQePmA.js → 0.I1lQdWMl.js} +1 -1
- package/webui/build/_app/immutable/nodes/{1.BS3_Rfxm.js → 1.BiQQfx2j.js} +1 -1
- package/webui/build/_app/immutable/nodes/{10.CyyxDCIS.js → 10.CvfUsqsw.js} +1 -1
- package/webui/build/_app/immutable/nodes/{11.CtYgIaGj.js → 11.B4LHPNL6.js} +1 -1
- package/webui/build/_app/immutable/nodes/12.DVFJuIWI.js +1 -0
- package/webui/build/_app/immutable/nodes/{14.D5OEGPR2.js → 14.DfaAf0f8.js} +1 -1
- package/webui/build/_app/immutable/nodes/{15.B4dFN1Gk.js → 15.CMzkX9OK.js} +1 -1
- package/webui/build/_app/immutable/nodes/{16.M7ZII7tl.js → 16.zPgTQNze.js} +1 -1
- package/webui/build/_app/immutable/nodes/{18.Ba_qJjp6.js → 18.BIzqhTqv.js} +1 -1
- package/webui/build/_app/immutable/nodes/{3.7r8v7qkm.js → 3.B8Viux9S.js} +1 -1
- package/webui/build/_app/immutable/nodes/5.B6fR3n6J.js +2 -0
- package/webui/build/_app/immutable/nodes/{6.BDBqx-GY.js → 6.j2O5Mwjv.js} +1 -1
- package/webui/build/_app/immutable/nodes/{7.D5czsDmz.js → 7.Bd2USIrl.js} +1 -1
- package/webui/build/_app/immutable/nodes/{8.pjVNsCdV.js → 8.Bw_d63B_.js} +1 -1
- package/webui/build/_app/immutable/nodes/{9.CsARv1BH.js → 9.pMMi5PP6.js} +1 -1
- package/webui/build/_app/version.json +1 -1
- package/app/plugins/builtin/google.rssany.js +0 -187
- package/webui/build/_app/immutable/assets/12.Ct59LCqW.css +0 -1
- package/webui/build/_app/immutable/assets/5.ClehBQ0g.css +0 -1
- package/webui/build/_app/immutable/assets/6.DSJfjJwx.css +0 -1
- package/webui/build/_app/immutable/assets/8.Ba5_jYIY.css +0 -1
- package/webui/build/_app/immutable/assets/9.m-LCx_kl.css +0 -1
- package/webui/build/_app/immutable/assets/homeFeedPanelStore.iOmfP2qL.css +0 -1
- package/webui/build/_app/immutable/chunks/CZD-YNDw.js +0 -31
- package/webui/build/_app/immutable/chunks/DeX-oq5W.js +0 -41
- package/webui/build/_app/immutable/chunks/dhB8G5Is.js +0 -1
- package/webui/build/_app/immutable/entry/start.Db4snNCd.js +0 -1
- package/webui/build/_app/immutable/nodes/12.Cg8AeCSH.js +0 -1
- package/webui/build/_app/immutable/nodes/5.CHIzoGrb.js +0 -1
package/dist/index.js
CHANGED
|
@@ -9,7 +9,7 @@ import { join, dirname, basename, resolve, sep, relative } from "node:path";
|
|
|
9
9
|
import { promisify } from "node:util";
|
|
10
10
|
import puppeteerCore from "puppeteer-core";
|
|
11
11
|
import { parse, NodeType } from "node-html-parser";
|
|
12
|
-
import
|
|
12
|
+
import { DatabaseSync } from "node:sqlite";
|
|
13
13
|
import { mkdir, writeFile, copyFile, access, rename, readFile, readdir, stat, unlink } from "node:fs/promises";
|
|
14
14
|
import { fileURLToPath, pathToFileURL } from "node:url";
|
|
15
15
|
import { createHash } from "node:crypto";
|
|
@@ -292,16 +292,14 @@ async function initUserDir() {
|
|
|
292
292
|
}
|
|
293
293
|
const MAIN_DB_JOURNAL = (process.env.RSSANY_DB_JOURNAL ?? "wal").toLowerCase() === "delete" ? "DELETE" : "WAL";
|
|
294
294
|
let _db = null;
|
|
295
|
-
let _dbInit = null;
|
|
296
295
|
let _writeLock = Promise.resolve();
|
|
297
296
|
const MAIN_DB_LOCK_PATH = join(DATA_DIR, "rssany.db.lock");
|
|
298
297
|
function logCorruptDiagnostic(operation, err) {
|
|
299
|
-
const code = err?.code;
|
|
300
298
|
const msg = err instanceof Error ? err.message : String(err);
|
|
301
299
|
const lines = [
|
|
302
300
|
"[rssany db] 数据库可能损坏或并发冲突",
|
|
303
301
|
` 操作: ${operation}`,
|
|
304
|
-
` 错误: ${
|
|
302
|
+
` 错误: ${msg}`,
|
|
305
303
|
" 常见原因:",
|
|
306
304
|
" 1. 多进程同时打开同一库(例如 tsx --watch 与另一实例同时写)",
|
|
307
305
|
" 2. 异常退出后 WAL 未正常 checkpoint",
|
|
@@ -437,48 +435,40 @@ function mapRowsToDbItems(rows) {
|
|
|
437
435
|
return rows.map(toDbItem);
|
|
438
436
|
}
|
|
439
437
|
function isCorruptError(err) {
|
|
440
|
-
const code = err?.code;
|
|
441
438
|
const msg = err instanceof Error ? err.message : String(err);
|
|
442
|
-
return
|
|
439
|
+
return msg.includes("SQLITE_CORRUPT") || msg.includes("database disk image is malformed");
|
|
443
440
|
}
|
|
444
441
|
async function getDb() {
|
|
445
442
|
if (_db) return _db;
|
|
446
|
-
if (_dbInit) return _dbInit;
|
|
447
443
|
const dbPath = join(DATA_DIR, "rssany.db");
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
if (db) {
|
|
463
|
-
try {
|
|
464
|
-
db.close();
|
|
465
|
-
} catch {
|
|
466
|
-
}
|
|
467
|
-
db = null;
|
|
468
|
-
}
|
|
469
|
-
if (isCorruptError(err)) {
|
|
470
|
-
logCorruptDiagnostic("打开/初始化主库 (getDb)", err);
|
|
444
|
+
await mkdir(DATA_DIR, { recursive: true });
|
|
445
|
+
acquireDbLock(DATA_DIR);
|
|
446
|
+
try {
|
|
447
|
+
_db = new DatabaseSync(dbPath);
|
|
448
|
+
_db.exec(`PRAGMA journal_mode = ${MAIN_DB_JOURNAL}`);
|
|
449
|
+
_db.exec("PRAGMA synchronous = NORMAL");
|
|
450
|
+
initSchema(_db);
|
|
451
|
+
return _db;
|
|
452
|
+
} catch (err) {
|
|
453
|
+
releaseDbLock();
|
|
454
|
+
if (_db) {
|
|
455
|
+
try {
|
|
456
|
+
_db.close();
|
|
457
|
+
} catch {
|
|
471
458
|
}
|
|
472
|
-
|
|
459
|
+
_db = null;
|
|
473
460
|
}
|
|
474
|
-
|
|
475
|
-
|
|
461
|
+
if (isCorruptError(err)) {
|
|
462
|
+
logCorruptDiagnostic("打开/初始化主库 (getDb)", err);
|
|
463
|
+
}
|
|
464
|
+
throw err;
|
|
465
|
+
}
|
|
476
466
|
}
|
|
477
467
|
async function runIntegrityCheck() {
|
|
478
468
|
const db = await getDb();
|
|
479
469
|
try {
|
|
480
|
-
const
|
|
481
|
-
return
|
|
470
|
+
const result = db.prepare("PRAGMA integrity_check").get();
|
|
471
|
+
return result?.integrity_check ?? "unknown";
|
|
482
472
|
} catch (err) {
|
|
483
473
|
const msg = err instanceof Error ? err.message : String(err);
|
|
484
474
|
return `integrity_check 执行失败: ${msg}`;
|
|
@@ -486,7 +476,6 @@ async function runIntegrityCheck() {
|
|
|
486
476
|
}
|
|
487
477
|
const LOGS_DB_PATH = join(DATA_DIR, "logs.db");
|
|
488
478
|
let _logsDb = null;
|
|
489
|
-
let _logsDbInit = null;
|
|
490
479
|
function initLogsSchema(db) {
|
|
491
480
|
db.exec(`
|
|
492
481
|
CREATE TABLE IF NOT EXISTS logs (
|
|
@@ -504,17 +493,12 @@ function initLogsSchema(db) {
|
|
|
504
493
|
}
|
|
505
494
|
async function getLogsDb() {
|
|
506
495
|
if (_logsDb) return _logsDb;
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
initLogsSchema(db);
|
|
514
|
-
_logsDb = db;
|
|
515
|
-
return db;
|
|
516
|
-
})();
|
|
517
|
-
return _logsDbInit;
|
|
496
|
+
await mkdir(DATA_DIR, { recursive: true });
|
|
497
|
+
_logsDb = new DatabaseSync(LOGS_DB_PATH);
|
|
498
|
+
_logsDb.exec("PRAGMA journal_mode = WAL");
|
|
499
|
+
_logsDb.exec("PRAGMA synchronous = NORMAL");
|
|
500
|
+
initLogsSchema(_logsDb);
|
|
501
|
+
return _logsDb;
|
|
518
502
|
}
|
|
519
503
|
function initSchema(db) {
|
|
520
504
|
db.exec(`
|
|
@@ -588,8 +572,8 @@ function initSchema(db) {
|
|
|
588
572
|
END;
|
|
589
573
|
`);
|
|
590
574
|
try {
|
|
591
|
-
const
|
|
592
|
-
if (
|
|
575
|
+
const cols = db.prepare("PRAGMA table_info(items)").all().map((r) => r.name);
|
|
576
|
+
if (!cols.includes("image_url")) {
|
|
593
577
|
db.exec("ALTER TABLE items ADD COLUMN image_url TEXT");
|
|
594
578
|
}
|
|
595
579
|
} catch {
|
|
@@ -597,20 +581,25 @@ function initSchema(db) {
|
|
|
597
581
|
migrateItemsSourceUrlIfNeeded(db);
|
|
598
582
|
}
|
|
599
583
|
function migrateItemsSourceUrlIfNeeded(db) {
|
|
600
|
-
const
|
|
584
|
+
const pragmaResult = db.exec("PRAGMA user_version");
|
|
585
|
+
const v = pragmaResult?.values?.[0]?.[0] ?? 0;
|
|
601
586
|
if (v >= 2) return;
|
|
602
587
|
const rows = db.prepare("SELECT rowid, source_url FROM items").all();
|
|
603
|
-
const
|
|
604
|
-
|
|
588
|
+
const updateStmt = db.prepare("UPDATE items SET source_url = @next WHERE rowid = @rowid");
|
|
589
|
+
db.exec("BEGIN TRANSACTION");
|
|
590
|
+
try {
|
|
605
591
|
for (const r of rows) {
|
|
606
592
|
const next = canonicalHttpSourceRef(r.source_url);
|
|
607
593
|
if (next !== r.source_url) {
|
|
608
|
-
|
|
594
|
+
updateStmt.run({ next, rowid: r.rowid });
|
|
609
595
|
}
|
|
610
596
|
}
|
|
611
|
-
db.
|
|
612
|
-
|
|
613
|
-
|
|
597
|
+
db.exec("PRAGMA user_version = 2");
|
|
598
|
+
db.exec("COMMIT");
|
|
599
|
+
} catch (err) {
|
|
600
|
+
db.exec("ROLLBACK");
|
|
601
|
+
throw err;
|
|
602
|
+
}
|
|
614
603
|
}
|
|
615
604
|
async function upsertItems(items, sourceUrlOverride) {
|
|
616
605
|
if (items.length === 0) return { newCount: 0, newIds: /* @__PURE__ */ new Set() };
|
|
@@ -621,79 +610,72 @@ async function upsertItems(items, sourceUrlOverride) {
|
|
|
621
610
|
const sourceUrl = canonicalHttpSourceRef(raw);
|
|
622
611
|
return withWriteLock(async () => {
|
|
623
612
|
const db = await getDb();
|
|
624
|
-
const stmt = db.prepare(`
|
|
625
|
-
INSERT OR IGNORE INTO items (id, url, source_url, title, author, summary, image_url, tags, pub_date, fetched_at)
|
|
626
|
-
VALUES (@id, @url, @sourceUrl, @title, @author, @summary, @imageUrl, @tags, @pubDate, @fetchedAt)
|
|
627
|
-
`);
|
|
628
|
-
const selectExistingStmt = db.prepare(`
|
|
629
|
-
SELECT id, title, author, summary, image_url, pub_date, fetched_at
|
|
630
|
-
FROM items
|
|
631
|
-
WHERE id = @id
|
|
632
|
-
`);
|
|
633
|
-
const repairExistingStmt = db.prepare(`
|
|
634
|
-
UPDATE items
|
|
635
|
-
SET title = @title,
|
|
636
|
-
author = @author,
|
|
637
|
-
summary = @summary,
|
|
638
|
-
image_url = @imageUrl,
|
|
639
|
-
pub_date = @pubDate,
|
|
640
|
-
fetched_at = @fetchedAt
|
|
641
|
-
WHERE id = @id
|
|
642
|
-
`);
|
|
643
613
|
const now2 = (/* @__PURE__ */ new Date()).toISOString();
|
|
644
614
|
let newCount = 0;
|
|
645
615
|
const newIds = /* @__PURE__ */ new Set();
|
|
646
|
-
const
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
continue;
|
|
684
|
-
}
|
|
685
|
-
repairExistingStmt.run({
|
|
686
|
-
id: item.guid,
|
|
687
|
-
title: shouldRepairTitle ? nextTitle : existing.title,
|
|
688
|
-
author: shouldRepairAuthor ? nextAuthor : existing.author ?? null,
|
|
689
|
-
summary: shouldRepairSummary ? nextSummary : existing.summary,
|
|
690
|
-
imageUrl: shouldRepairImageUrl ? nextImageUrl : existing.image_url ?? null,
|
|
691
|
-
pubDate: shouldRepairPubDate ? nextPubDate : existing.pub_date,
|
|
692
|
-
fetchedAt: now2
|
|
693
|
-
});
|
|
616
|
+
const insertStmt = db.prepare(`
|
|
617
|
+
INSERT OR IGNORE INTO items (id, url, source_url, title, author, summary, image_url, tags, pub_date, fetched_at)
|
|
618
|
+
VALUES (@id, @url, @sourceUrl, @title, @author, @summary, @imageUrl, @tags, @pubDate, @fetchedAt)
|
|
619
|
+
`);
|
|
620
|
+
const selectExistingStmt = db.prepare(`
|
|
621
|
+
SELECT title, author, summary, image_url, pub_date, fetched_at
|
|
622
|
+
FROM items WHERE id = @id
|
|
623
|
+
`);
|
|
624
|
+
const updateStmt = db.prepare(`
|
|
625
|
+
UPDATE items SET title = @title, author = @author, summary = @summary,
|
|
626
|
+
image_url = @imageUrl, pub_date = @pubDate, fetched_at = @fetchedAt
|
|
627
|
+
WHERE id = @id
|
|
628
|
+
`);
|
|
629
|
+
for (const item of items) {
|
|
630
|
+
const nextTitle = normalizeText(item.title) || null;
|
|
631
|
+
const nextSummary = normalizeText(item.summary) || null;
|
|
632
|
+
const nextAuthorArr = normalizeAuthor(item.author);
|
|
633
|
+
const nextAuthor = nextAuthorArr?.length ? JSON.stringify(nextAuthorArr) : null;
|
|
634
|
+
const nextPubDate = pubDateToIsoOrNull(item.pubDate);
|
|
635
|
+
const nextTags = item.tags?.length ? JSON.stringify(item.tags) : null;
|
|
636
|
+
const nextImageUrl = typeof item.imageUrl === "string" && item.imageUrl.trim() ? item.imageUrl.trim() : null;
|
|
637
|
+
const info = insertStmt.run({
|
|
638
|
+
id: item.guid,
|
|
639
|
+
url: item.link,
|
|
640
|
+
sourceUrl,
|
|
641
|
+
title: nextTitle,
|
|
642
|
+
author: nextAuthor,
|
|
643
|
+
summary: nextSummary,
|
|
644
|
+
imageUrl: nextImageUrl,
|
|
645
|
+
tags: nextTags,
|
|
646
|
+
pubDate: nextPubDate,
|
|
647
|
+
fetchedAt: now2
|
|
648
|
+
});
|
|
649
|
+
newCount += Number(info.changes);
|
|
650
|
+
if (info.changes > 0) {
|
|
651
|
+
newIds.add(item.guid);
|
|
652
|
+
continue;
|
|
694
653
|
}
|
|
695
|
-
|
|
696
|
-
|
|
654
|
+
const existing = selectExistingStmt.get({ id: item.guid });
|
|
655
|
+
if (!existing) continue;
|
|
656
|
+
const shouldRepairTitle = !!nextTitle && !isDateOnlyTitle(nextTitle) && (isDateOnlyTitle(existing.title) || !normalizeText(existing.title));
|
|
657
|
+
const shouldRepairSummary = !!nextSummary && normalizeText(existing.summary ?? "").length < nextSummary.length;
|
|
658
|
+
const shouldRepairImageUrl = !!nextImageUrl && !existing.image_url?.trim();
|
|
659
|
+
const existingAuthorArr = parseAuthorFromDb(existing.author);
|
|
660
|
+
const shouldRepairAuthor = !!nextAuthorArr?.length && !existingAuthorArr?.length;
|
|
661
|
+
const existingPubDateMs = toMs(existing.pub_date);
|
|
662
|
+
const existingFetchedAtMs = toMs(existing.fetched_at);
|
|
663
|
+
const nextPubDateMs = toMs(nextPubDate);
|
|
664
|
+
const existingPubDateLooksFallback = existingPubDateMs != null && existingFetchedAtMs != null && Math.abs(existingPubDateMs - existingFetchedAtMs) <= 5 * 60 * 1e3;
|
|
665
|
+
const shouldRepairPubDate = nextPubDateMs != null && (existingPubDateMs == null || existingPubDateLooksFallback && nextPubDateMs < existingPubDateMs - 24 * 60 * 60 * 1e3);
|
|
666
|
+
if (!(shouldRepairTitle || shouldRepairSummary || shouldRepairImageUrl || shouldRepairAuthor || shouldRepairPubDate)) {
|
|
667
|
+
continue;
|
|
668
|
+
}
|
|
669
|
+
updateStmt.run({
|
|
670
|
+
id: item.guid,
|
|
671
|
+
title: shouldRepairTitle ? nextTitle : existing.title,
|
|
672
|
+
author: shouldRepairAuthor ? nextAuthor : existing.author ?? null,
|
|
673
|
+
summary: shouldRepairSummary ? nextSummary : existing.summary,
|
|
674
|
+
imageUrl: shouldRepairImageUrl ? nextImageUrl : existing.image_url ?? null,
|
|
675
|
+
pubDate: shouldRepairPubDate ? nextPubDate : existing.pub_date,
|
|
676
|
+
fetchedAt: now2
|
|
677
|
+
});
|
|
678
|
+
}
|
|
697
679
|
return { newCount, newIds };
|
|
698
680
|
});
|
|
699
681
|
}
|
|
@@ -701,15 +683,15 @@ async function updateItemContent(item) {
|
|
|
701
683
|
return withWriteLock(async () => {
|
|
702
684
|
const db = await getDb();
|
|
703
685
|
db.prepare(`
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
image_url
|
|
707
|
-
author
|
|
708
|
-
pub_date
|
|
709
|
-
tags
|
|
686
|
+
UPDATE items SET
|
|
687
|
+
content = COALESCE(content, @content),
|
|
688
|
+
image_url = COALESCE(@imageUrl, image_url),
|
|
689
|
+
author = COALESCE(@author, author),
|
|
690
|
+
pub_date = COALESCE(@pubDate, pub_date),
|
|
691
|
+
tags = @tags,
|
|
710
692
|
translations = COALESCE(@translations, translations)
|
|
711
|
-
|
|
712
|
-
|
|
693
|
+
WHERE id = @id
|
|
694
|
+
`).run({
|
|
713
695
|
id: item.guid,
|
|
714
696
|
content: item.content ?? null,
|
|
715
697
|
imageUrl: typeof item.imageUrl === "string" && item.imageUrl.trim() ? item.imageUrl.trim() : null,
|
|
@@ -734,6 +716,7 @@ async function queryFeedItems(sourceUrls, limit, offset, opts) {
|
|
|
734
716
|
expanded.forEach((url, i) => {
|
|
735
717
|
params[`u${i}`] = url;
|
|
736
718
|
});
|
|
719
|
+
const sqlParams = params;
|
|
737
720
|
if (opts?.since) {
|
|
738
721
|
conditions.push("COALESCE(pub_date, fetched_at) >= @since");
|
|
739
722
|
params.since = opts.since.length === 10 ? `${opts.since}T00:00:00.000Z` : opts.since;
|
|
@@ -750,11 +733,10 @@ async function queryFeedItems(sourceUrls, limit, offset, opts) {
|
|
|
750
733
|
}
|
|
751
734
|
const where = conditions.length ? `WHERE ${conditions.join(" AND ")}` : "";
|
|
752
735
|
const rows = db.prepare(`
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
`).all(params);
|
|
736
|
+
SELECT * FROM items ${where}
|
|
737
|
+
ORDER BY COALESCE(pub_date, fetched_at) DESC
|
|
738
|
+
LIMIT ${limit + 1} OFFSET ${offset}
|
|
739
|
+
`).all(sqlParams);
|
|
758
740
|
const hasMore = rows.length > limit;
|
|
759
741
|
const items = mapRowsToDbItems(hasMore ? rows.slice(0, limit) : rows);
|
|
760
742
|
return { items, hasMore };
|
|
@@ -763,22 +745,20 @@ async function queryItems(opts) {
|
|
|
763
745
|
const db = await getDb();
|
|
764
746
|
const { sourceUrl, sourceUrls, author, q, tags: tagsFilter, limit = 20, offset = 0, since, until } = opts;
|
|
765
747
|
const conditions = [];
|
|
766
|
-
const params = {
|
|
748
|
+
const params = {};
|
|
767
749
|
if (sourceUrl) {
|
|
768
750
|
const key = canonicalHttpSourceRef(sourceUrl);
|
|
769
|
-
if (!key) {
|
|
770
|
-
return { items: [], total: 0 };
|
|
771
|
-
}
|
|
751
|
+
if (!key) return { items: [], total: 0 };
|
|
772
752
|
conditions.push("i.source_url = @sourceUrl");
|
|
773
753
|
params.sourceUrl = key;
|
|
774
754
|
} else if (sourceUrls && sourceUrls.length > 0) {
|
|
775
755
|
const expanded = [...new Set(sourceUrls.map((s) => canonicalHttpSourceRef(s)).filter(Boolean))];
|
|
776
|
-
if (expanded.length === 0) {
|
|
777
|
-
return { items: [], total: 0 };
|
|
778
|
-
}
|
|
756
|
+
if (expanded.length === 0) return { items: [], total: 0 };
|
|
779
757
|
const placeholders = expanded.map((_, i) => `@src${i}`).join(", ");
|
|
780
758
|
conditions.push(`i.source_url IN (${placeholders})`);
|
|
781
|
-
expanded.forEach((s, i) =>
|
|
759
|
+
expanded.forEach((s, i) => {
|
|
760
|
+
params[`src${i}`] = s;
|
|
761
|
+
});
|
|
782
762
|
}
|
|
783
763
|
if (author && author.trim().length >= 2) {
|
|
784
764
|
conditions.push("instr(i.author, @author) > 0");
|
|
@@ -789,9 +769,9 @@ async function queryItems(opts) {
|
|
|
789
769
|
params.q = q;
|
|
790
770
|
}
|
|
791
771
|
if (tagsFilter && tagsFilter.length > 0) {
|
|
792
|
-
const trimmed = tagsFilter.filter((t) => typeof t === "string" && t.trim()).map((t) => t.trim());
|
|
772
|
+
const trimmed = tagsFilter.filter((t) => typeof t === "string" && t.trim().length > 0).map((t) => t.trim());
|
|
793
773
|
if (trimmed.length > 0) {
|
|
794
|
-
const tagConds = trimmed.map((_,
|
|
774
|
+
const tagConds = trimmed.map((_, idx) => `LOWER(TRIM(json_each.value)) = LOWER(@tag${idx})`).join(" OR ");
|
|
795
775
|
conditions.push(`i.tags IS NOT NULL AND EXISTS (SELECT 1 FROM json_each(i.tags) WHERE ${tagConds})`);
|
|
796
776
|
trimmed.forEach((t, i) => {
|
|
797
777
|
params[`tag${i}`] = t;
|
|
@@ -807,14 +787,19 @@ async function queryItems(opts) {
|
|
|
807
787
|
params.until = until.toISOString();
|
|
808
788
|
}
|
|
809
789
|
const where = conditions.length ? `WHERE ${conditions.join(" AND ")}` : "";
|
|
790
|
+
const sqlParams = params;
|
|
810
791
|
const rows = db.prepare(`
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
const { count } = db.prepare(`SELECT COUNT(*) as count FROM items i ${where}`).get(
|
|
817
|
-
return { items: mapRowsToDbItems(rows)
|
|
792
|
+
SELECT i.id, i.url, i.source_url, i.title, i.author, i.summary, i.content, i.tags, i.translations, i.pub_date, i.fetched_at, i.pushed_at
|
|
793
|
+
FROM items i ${where}
|
|
794
|
+
ORDER BY COALESCE(i.pub_date, i.fetched_at) DESC
|
|
795
|
+
LIMIT ${limit} OFFSET ${offset}
|
|
796
|
+
`).all(sqlParams);
|
|
797
|
+
const { count } = db.prepare(`SELECT COUNT(*) as count FROM items i ${where}`).get(sqlParams);
|
|
798
|
+
return { items: mapRowsToDbItems(rows.map((r) => {
|
|
799
|
+
const obj = {};
|
|
800
|
+
for (const [k, v] of Object.entries(r)) obj[k] = v;
|
|
801
|
+
return obj;
|
|
802
|
+
})), total: count };
|
|
818
803
|
}
|
|
819
804
|
async function removeTagFromAllItems(tag) {
|
|
820
805
|
const trimmed = String(tag ?? "").trim();
|
|
@@ -825,22 +810,19 @@ async function removeTagFromAllItems(tag) {
|
|
|
825
810
|
const rows = db.prepare("SELECT id, tags FROM items WHERE tags IS NOT NULL AND tags != ''").all();
|
|
826
811
|
const updateStmt = db.prepare("UPDATE items SET tags = @tags WHERE id = @id");
|
|
827
812
|
let count = 0;
|
|
828
|
-
const
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
continue;
|
|
835
|
-
}
|
|
836
|
-
const filtered = itemTags.filter((t) => String(t).trim().toLowerCase() !== targetLower);
|
|
837
|
-
if (filtered.length === itemTags.length) continue;
|
|
838
|
-
const nextTags = filtered.length > 0 ? JSON.stringify(filtered) : null;
|
|
839
|
-
updateStmt.run({ id: row.id, tags: nextTags });
|
|
840
|
-
count += 1;
|
|
813
|
+
for (const row of rows) {
|
|
814
|
+
let itemTags;
|
|
815
|
+
try {
|
|
816
|
+
itemTags = JSON.parse(row.tags);
|
|
817
|
+
} catch {
|
|
818
|
+
continue;
|
|
841
819
|
}
|
|
842
|
-
|
|
843
|
-
|
|
820
|
+
const filtered = itemTags.filter((t) => String(t).trim().toLowerCase() !== targetLower);
|
|
821
|
+
if (filtered.length === itemTags.length) continue;
|
|
822
|
+
const nextTags = filtered.length > 0 ? JSON.stringify(filtered) : null;
|
|
823
|
+
updateStmt.run({ id: row.id, tags: nextTags });
|
|
824
|
+
count += 1;
|
|
825
|
+
}
|
|
844
826
|
return count;
|
|
845
827
|
});
|
|
846
828
|
}
|
|
@@ -849,25 +831,19 @@ async function markPushed(ids) {
|
|
|
849
831
|
return withWriteLock(async () => {
|
|
850
832
|
const db = await getDb();
|
|
851
833
|
const now2 = (/* @__PURE__ */ new Date()).toISOString();
|
|
852
|
-
const
|
|
853
|
-
|
|
854
|
-
for (const id of list) stmt.run({ now: now2, id });
|
|
855
|
-
});
|
|
856
|
-
run(ids);
|
|
834
|
+
const placeholders = ids.map(() => "?").join(",");
|
|
835
|
+
db.prepare(`UPDATE items SET pushed_at = ? WHERE id IN (${placeholders})`).run(now2, ...ids);
|
|
857
836
|
});
|
|
858
837
|
}
|
|
859
838
|
async function deleteItem(id) {
|
|
860
839
|
if (!id?.trim()) return false;
|
|
861
840
|
return withWriteLock(async () => {
|
|
862
841
|
const db = await getDb();
|
|
863
|
-
const
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
return info.changes;
|
|
869
|
-
});
|
|
870
|
-
return run() > 0;
|
|
842
|
+
const row = db.prepare("SELECT rowid FROM items WHERE id = @id").get({ id: id.trim() });
|
|
843
|
+
if (!row) return false;
|
|
844
|
+
db.prepare("DELETE FROM items_fts WHERE rowid = @rowid").run({ rowid: row.rowid });
|
|
845
|
+
const info = db.prepare("DELETE FROM items WHERE id = @id").run({ id: id.trim() });
|
|
846
|
+
return Number(info.changes) > 0;
|
|
871
847
|
});
|
|
872
848
|
}
|
|
873
849
|
async function deleteItemsBySourceUrl(sourceUrl) {
|
|
@@ -877,29 +853,33 @@ async function deleteItemsBySourceUrl(sourceUrl) {
|
|
|
877
853
|
return withWriteLock(async () => {
|
|
878
854
|
const db = await getDb();
|
|
879
855
|
const info = db.prepare("DELETE FROM items WHERE source_url = @sourceUrl").run({ sourceUrl: key });
|
|
880
|
-
return info.changes;
|
|
856
|
+
return Number(info.changes);
|
|
881
857
|
});
|
|
882
858
|
}
|
|
883
859
|
async function getPendingPushItems(limit = 100) {
|
|
884
860
|
const db = await getDb();
|
|
885
861
|
const rows = db.prepare(`
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
return mapRowsToDbItems(rows)
|
|
862
|
+
SELECT * FROM items
|
|
863
|
+
WHERE pushed_at IS NULL AND content IS NOT NULL
|
|
864
|
+
ORDER BY fetched_at ASC
|
|
865
|
+
LIMIT ${limit}
|
|
866
|
+
`).all();
|
|
867
|
+
return mapRowsToDbItems(rows.map((r) => {
|
|
868
|
+
const obj = {};
|
|
869
|
+
for (const [k, v] of Object.entries(r)) obj[k] = v;
|
|
870
|
+
return obj;
|
|
871
|
+
}));
|
|
892
872
|
}
|
|
893
873
|
async function getSourceStats() {
|
|
894
874
|
const { mergeSourceStatsRows: mergeSourceStatsRows2 } = await Promise.resolve().then(() => httpSourceRef);
|
|
895
875
|
const db = await getDb();
|
|
896
|
-
const rows = db.prepare(
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
876
|
+
const rows = db.prepare(`
|
|
877
|
+
SELECT source_url,
|
|
878
|
+
COUNT(*) as count,
|
|
879
|
+
SUM(CASE WHEN julianday(fetched_at) >= julianday('now', '-7 days') THEN 1 ELSE 0 END) as count_7d,
|
|
880
|
+
MAX(COALESCE(pub_date, fetched_at)) as latest_at
|
|
881
|
+
FROM items GROUP BY source_url ORDER BY count DESC
|
|
882
|
+
`).all();
|
|
903
883
|
return mergeSourceStatsRows2(rows);
|
|
904
884
|
}
|
|
905
885
|
async function insertLog(entry) {
|
|
@@ -919,7 +899,7 @@ async function queryLogs(opts) {
|
|
|
919
899
|
const db = await getLogsDb();
|
|
920
900
|
const { level, category, limit = 50, offset = 0, since } = opts;
|
|
921
901
|
const conditions = [];
|
|
922
|
-
const params = {
|
|
902
|
+
const params = {};
|
|
923
903
|
if (level) {
|
|
924
904
|
conditions.push("level = @level");
|
|
925
905
|
params.level = level;
|
|
@@ -933,19 +913,30 @@ async function queryLogs(opts) {
|
|
|
933
913
|
params.since = since.toISOString();
|
|
934
914
|
}
|
|
935
915
|
const where = conditions.length ? `WHERE ${conditions.join(" AND ")}` : "";
|
|
916
|
+
const sqlParams = params;
|
|
936
917
|
const rows = db.prepare(`
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
const { count } = db.prepare(`SELECT COUNT(*) as count FROM logs ${where}`).get(
|
|
943
|
-
return {
|
|
918
|
+
SELECT id, level, category, message, payload, created_at
|
|
919
|
+
FROM logs ${where}
|
|
920
|
+
ORDER BY created_at DESC
|
|
921
|
+
LIMIT ${limit} OFFSET ${offset}
|
|
922
|
+
`).all(sqlParams);
|
|
923
|
+
const { count } = db.prepare(`SELECT COUNT(*) as count FROM logs ${where}`).get(sqlParams);
|
|
924
|
+
return {
|
|
925
|
+
items: rows.map((r) => ({
|
|
926
|
+
id: Number(r.id),
|
|
927
|
+
level: String(r.level),
|
|
928
|
+
category: String(r.category),
|
|
929
|
+
message: String(r.message),
|
|
930
|
+
payload: r.payload,
|
|
931
|
+
created_at: String(r.created_at)
|
|
932
|
+
})),
|
|
933
|
+
total: Number(count)
|
|
934
|
+
};
|
|
944
935
|
}
|
|
945
936
|
async function clearAllLogs() {
|
|
946
937
|
const db = await getLogsDb();
|
|
947
938
|
const r = db.prepare("DELETE FROM logs").run();
|
|
948
|
-
return r.changes;
|
|
939
|
+
return Number(r.changes);
|
|
949
940
|
}
|
|
950
941
|
async function getSystemTags() {
|
|
951
942
|
try {
|
|
@@ -2512,18 +2503,30 @@ function onFeedUpdated(fn) {
|
|
|
2512
2503
|
eventBus.on("feed:updated", fn);
|
|
2513
2504
|
return () => eventBus.off("feed:updated", fn);
|
|
2514
2505
|
}
|
|
2506
|
+
function migrateGatewayFromFile(j) {
|
|
2507
|
+
const g = j?.deliver?.gateway?.trim();
|
|
2508
|
+
if (g) return g;
|
|
2509
|
+
const u = j?.deliver?.url?.trim() ?? "";
|
|
2510
|
+
if (u) {
|
|
2511
|
+
return u.replace(/\/items\/?$/i, "").replace(/\/+$/, "").trim();
|
|
2512
|
+
}
|
|
2513
|
+
const s = j?.deliver?.sourcesUrl?.trim() ?? "";
|
|
2514
|
+
if (s) {
|
|
2515
|
+
return s.replace(/\/sources\/?$/i, "").replace(/\/+$/, "").trim();
|
|
2516
|
+
}
|
|
2517
|
+
return "";
|
|
2518
|
+
}
|
|
2515
2519
|
async function getDeliverConfig() {
|
|
2516
2520
|
try {
|
|
2517
2521
|
const raw = await readFile(CONFIG_PATH, "utf-8");
|
|
2518
2522
|
const j = JSON.parse(raw);
|
|
2519
|
-
const u = j?.deliver?.url;
|
|
2520
2523
|
const t = j?.deliver?.token;
|
|
2521
2524
|
return {
|
|
2522
|
-
|
|
2525
|
+
gateway: migrateGatewayFromFile(j),
|
|
2523
2526
|
token: typeof t === "string" ? t.trim() : ""
|
|
2524
2527
|
};
|
|
2525
2528
|
} catch {
|
|
2526
|
-
return {
|
|
2529
|
+
return { gateway: "", token: "" };
|
|
2527
2530
|
}
|
|
2528
2531
|
}
|
|
2529
2532
|
async function saveDeliverConfig(config) {
|
|
@@ -2533,13 +2536,11 @@ async function saveDeliverConfig(config) {
|
|
|
2533
2536
|
root = JSON.parse(raw);
|
|
2534
2537
|
} catch {
|
|
2535
2538
|
}
|
|
2536
|
-
const
|
|
2537
|
-
const base2 = typeof prev === "object" && prev !== null && !Array.isArray(prev) ? { ...prev } : {};
|
|
2538
|
-
const url = config.url.trim();
|
|
2539
|
+
const gateway = config.gateway.trim();
|
|
2539
2540
|
const token = config.token.trim();
|
|
2540
|
-
const next = {
|
|
2541
|
+
const next = {};
|
|
2542
|
+
if (gateway) next.gateway = gateway;
|
|
2541
2543
|
if (token) next.token = token;
|
|
2542
|
-
else delete next.token;
|
|
2543
2544
|
root.deliver = next;
|
|
2544
2545
|
await writeFile(CONFIG_PATH, JSON.stringify(root, null, 2) + "\n", "utf-8");
|
|
2545
2546
|
}
|
|
@@ -2557,6 +2558,11 @@ function feedItemsToPayload(items) {
|
|
|
2557
2558
|
translations: i.translations
|
|
2558
2559
|
}));
|
|
2559
2560
|
}
|
|
2561
|
+
function joinGatewayPath(gatewayBase, segment) {
|
|
2562
|
+
const base2 = gatewayBase.trim().replace(/\/+$/, "");
|
|
2563
|
+
if (!base2) return "";
|
|
2564
|
+
return `${base2}/${segment}`;
|
|
2565
|
+
}
|
|
2560
2566
|
async function postDeliverItems(url, sourceRef, items, options) {
|
|
2561
2567
|
if (!url.trim() || items.length === 0) return;
|
|
2562
2568
|
const body = JSON.stringify({ sourceRef, items: feedItemsToPayload(items) });
|
|
@@ -2585,6 +2591,52 @@ async function postDeliverItemsSafe(url, sourceRef, items, options) {
|
|
|
2585
2591
|
});
|
|
2586
2592
|
}
|
|
2587
2593
|
}
|
|
2594
|
+
async function postDeliverSources(url, sourcesJson, options) {
|
|
2595
|
+
if (!url.trim() || !sourcesJson.trim()) return;
|
|
2596
|
+
const headers = {
|
|
2597
|
+
"Content-Type": "application/json; charset=utf-8"
|
|
2598
|
+
};
|
|
2599
|
+
const t = options?.bearerToken?.trim();
|
|
2600
|
+
if (t) headers.Authorization = `Bearer ${t}`;
|
|
2601
|
+
const res = await fetch(url.trim(), {
|
|
2602
|
+
method: "POST",
|
|
2603
|
+
headers,
|
|
2604
|
+
body: sourcesJson,
|
|
2605
|
+
signal: AbortSignal.timeout(12e4)
|
|
2606
|
+
});
|
|
2607
|
+
if (!res.ok) {
|
|
2608
|
+
const text = await res.text().catch(() => "");
|
|
2609
|
+
throw new Error(`HTTP ${res.status}${text ? `: ${text.slice(0, 200)}` : ""}`);
|
|
2610
|
+
}
|
|
2611
|
+
}
|
|
2612
|
+
async function postDeliverSourcesSafe(url, sourcesJson, options) {
|
|
2613
|
+
try {
|
|
2614
|
+
await postDeliverSources(url, sourcesJson, options);
|
|
2615
|
+
} catch (err) {
|
|
2616
|
+
logger.warn("deliver", "信源配置投递失败", {
|
|
2617
|
+
err: err instanceof Error ? err.message : String(err)
|
|
2618
|
+
});
|
|
2619
|
+
}
|
|
2620
|
+
}
|
|
2621
|
+
async function postDeliverGatewayTest(gateway, body, options) {
|
|
2622
|
+
const url = joinGatewayPath(gateway, "test");
|
|
2623
|
+
if (!url) throw new Error("gateway 不能为空");
|
|
2624
|
+
const headers = {
|
|
2625
|
+
"Content-Type": "application/json; charset=utf-8"
|
|
2626
|
+
};
|
|
2627
|
+
const t = options?.bearerToken?.trim();
|
|
2628
|
+
if (t) headers.Authorization = `Bearer ${t}`;
|
|
2629
|
+
const res = await fetch(url, {
|
|
2630
|
+
method: "POST",
|
|
2631
|
+
headers,
|
|
2632
|
+
body: JSON.stringify(body),
|
|
2633
|
+
signal: AbortSignal.timeout(12e4)
|
|
2634
|
+
});
|
|
2635
|
+
if (!res.ok) {
|
|
2636
|
+
const text = await res.text().catch(() => "");
|
|
2637
|
+
throw new Error(`HTTP ${res.status}${text ? `: ${text.slice(0, 200)}` : ""}`);
|
|
2638
|
+
}
|
|
2639
|
+
}
|
|
2588
2640
|
function resolveHeadlessForFeeder(config) {
|
|
2589
2641
|
if (config.force === true) {
|
|
2590
2642
|
return config.headless === true ? true : false;
|
|
@@ -2642,7 +2694,7 @@ async function generateAndCache(listUrl, key, config, proxy) {
|
|
|
2642
2694
|
});
|
|
2643
2695
|
generatingKeys.delete(key);
|
|
2644
2696
|
logger.info("scraper", "抓取成功", { source_url: listUrl, count: items.length });
|
|
2645
|
-
const {
|
|
2697
|
+
const { gateway: deliverGateway, token: deliverToken } = await getDeliverConfig();
|
|
2646
2698
|
let newCount = 0;
|
|
2647
2699
|
let newIds = /* @__PURE__ */ new Set();
|
|
2648
2700
|
const upsertResult = await upsertItems(items).catch((err) => {
|
|
@@ -2672,8 +2724,8 @@ async function generateAndCache(listUrl, key, config, proxy) {
|
|
|
2672
2724
|
emitFeedUpdated({ sourceUrl: sourceRefStored, newCount: newCount - pipelineDroppedNew });
|
|
2673
2725
|
}
|
|
2674
2726
|
const out = items.filter((i) => !isPipelineDroppedItem(i));
|
|
2675
|
-
if (
|
|
2676
|
-
await postDeliverItemsSafe(
|
|
2727
|
+
if (deliverGateway.trim() && out.length > 0) {
|
|
2728
|
+
await postDeliverItemsSafe(joinGatewayPath(deliverGateway, "items"), sourceRefStored, out, {
|
|
2677
2729
|
bearerToken: deliverToken || void 0
|
|
2678
2730
|
});
|
|
2679
2731
|
}
|
|
@@ -2899,17 +2951,24 @@ const DEFAULT_REFRESH = "1day";
|
|
|
2899
2951
|
const SOURCES_CONCURRENCY = 1;
|
|
2900
2952
|
function createPullTask(ref, cacheDir, cronExpr) {
|
|
2901
2953
|
return async () => {
|
|
2902
|
-
|
|
2903
|
-
|
|
2904
|
-
|
|
2905
|
-
|
|
2906
|
-
});
|
|
2907
|
-
} catch (err) {
|
|
2908
|
-
throw err;
|
|
2909
|
-
}
|
|
2954
|
+
await getItems(ref, {
|
|
2955
|
+
cacheDir,
|
|
2956
|
+
cron: cronExpr
|
|
2957
|
+
});
|
|
2910
2958
|
};
|
|
2911
2959
|
}
|
|
2912
2960
|
const SOURCES_GROUP = "sources";
|
|
2961
|
+
async function deliverSourcesConfigIfConfigured() {
|
|
2962
|
+
const { gateway, token } = await getDeliverConfig();
|
|
2963
|
+
if (!gateway.trim()) return;
|
|
2964
|
+
let raw;
|
|
2965
|
+
try {
|
|
2966
|
+
raw = await getSourcesRaw();
|
|
2967
|
+
} catch {
|
|
2968
|
+
return;
|
|
2969
|
+
}
|
|
2970
|
+
await postDeliverSourcesSafe(joinGatewayPath(gateway, "sources"), raw, { bearerToken: token || void 0 });
|
|
2971
|
+
}
|
|
2913
2972
|
async function rescheduleSources(cacheDir, runNow2) {
|
|
2914
2973
|
unscheduleGroup(SOURCES_GROUP);
|
|
2915
2974
|
let sources;
|
|
@@ -2939,7 +2998,7 @@ async function initScheduler(cacheDir) {
|
|
|
2939
2998
|
const watcher = watch(SOURCES_CONFIG_PATH, () => {
|
|
2940
2999
|
if (debounceTimer) clearTimeout(debounceTimer);
|
|
2941
3000
|
debounceTimer = setTimeout(() => {
|
|
2942
|
-
rescheduleSources(cacheDir, false).catch(() => {
|
|
3001
|
+
void rescheduleSources(cacheDir, false).then(() => deliverSourcesConfigIfConfigured()).catch(() => {
|
|
2943
3002
|
});
|
|
2944
3003
|
}, 500);
|
|
2945
3004
|
});
|
|
@@ -3326,7 +3385,7 @@ function registerItemsRoutes(app) {
|
|
|
3326
3385
|
return c.json({ items: [], total: 0, hasMore: false });
|
|
3327
3386
|
}
|
|
3328
3387
|
const result = await queryItems({
|
|
3329
|
-
sourceUrl:
|
|
3388
|
+
sourceUrl: sourceUrls ? void 0 : effectiveSourceUrl ? canonicalHttpSourceRef(effectiveSourceUrl) : void 0,
|
|
3330
3389
|
sourceUrls,
|
|
3331
3390
|
author,
|
|
3332
3391
|
q,
|
|
@@ -3533,16 +3592,29 @@ function registerTopicsRoutes(app) {
|
|
|
3533
3592
|
}
|
|
3534
3593
|
function registerDeliverRoutes(app) {
|
|
3535
3594
|
app.get("/api/deliver", requireAdmin(), async (c) => {
|
|
3536
|
-
const {
|
|
3537
|
-
return c.json({
|
|
3595
|
+
const { gateway, token } = await getDeliverConfig();
|
|
3596
|
+
return c.json({ gateway, token });
|
|
3538
3597
|
});
|
|
3539
3598
|
app.put("/api/deliver", requireAdmin(), async (c) => {
|
|
3540
3599
|
try {
|
|
3541
3600
|
const body = await c.req.json();
|
|
3542
|
-
const
|
|
3543
|
-
const
|
|
3544
|
-
|
|
3545
|
-
|
|
3601
|
+
const prev = await getDeliverConfig();
|
|
3602
|
+
const explicitGateway = body != null && "gateway" in body;
|
|
3603
|
+
const explicitUrl = body != null && "url" in body;
|
|
3604
|
+
const explicitToken = body != null && "token" in body;
|
|
3605
|
+
let gateway = typeof body?.gateway === "string" ? body.gateway.trim() : "";
|
|
3606
|
+
if (!gateway && typeof body?.url === "string") {
|
|
3607
|
+
gateway = body.url.trim().replace(/\/items\/?$/i, "").replace(/\/+$/, "");
|
|
3608
|
+
}
|
|
3609
|
+
if (!explicitGateway && !explicitUrl) {
|
|
3610
|
+
gateway = prev.gateway;
|
|
3611
|
+
}
|
|
3612
|
+
let token = typeof body?.token === "string" ? body.token.trim() : "";
|
|
3613
|
+
if (!explicitToken) {
|
|
3614
|
+
token = prev.token;
|
|
3615
|
+
}
|
|
3616
|
+
await saveDeliverConfig({ gateway, token });
|
|
3617
|
+
return c.json({ ok: true, gateway, token });
|
|
3546
3618
|
} catch (err) {
|
|
3547
3619
|
return c.json({ ok: false, message: err instanceof Error ? err.message : String(err) }, 400);
|
|
3548
3620
|
}
|
|
@@ -3550,31 +3622,39 @@ function registerDeliverRoutes(app) {
|
|
|
3550
3622
|
app.post("/api/deliver/test", requireAdmin(), async (c) => {
|
|
3551
3623
|
try {
|
|
3552
3624
|
const body = await c.req.json();
|
|
3553
|
-
const
|
|
3554
|
-
|
|
3555
|
-
if (!
|
|
3625
|
+
const prev = await getDeliverConfig();
|
|
3626
|
+
let gateway = typeof body?.gateway === "string" ? body.gateway.trim() : "";
|
|
3627
|
+
if (!gateway && typeof body?.url === "string") {
|
|
3628
|
+
gateway = body.url.trim().replace(/\/items\/?$/i, "").replace(/\/+$/, "");
|
|
3629
|
+
}
|
|
3630
|
+
if (!gateway) gateway = prev.gateway;
|
|
3631
|
+
const token = typeof body?.token === "string" ? body.token.trim() : prev.token;
|
|
3632
|
+
if (!gateway.trim()) return c.json({ ok: false, message: "gateway 不能为空" }, 400);
|
|
3633
|
+
const now2 = Date.now();
|
|
3556
3634
|
const sample = {
|
|
3557
|
-
guid: "deliver-test-" +
|
|
3635
|
+
guid: "deliver-test-" + now2,
|
|
3558
3636
|
title: "投递连通性测试",
|
|
3559
3637
|
link: "https://example.com/rssany-deliver-test",
|
|
3560
|
-
pubDate:
|
|
3561
|
-
summary: "
|
|
3638
|
+
pubDate: /* @__PURE__ */ new Date(),
|
|
3639
|
+
summary: "若下游 /test 收到此条,说明 Gateway 可用。",
|
|
3640
|
+
sourceRef: "rssany-deliver-test"
|
|
3562
3641
|
};
|
|
3563
|
-
await
|
|
3564
|
-
|
|
3565
|
-
|
|
3566
|
-
|
|
3567
|
-
|
|
3568
|
-
|
|
3569
|
-
|
|
3570
|
-
|
|
3571
|
-
|
|
3572
|
-
|
|
3573
|
-
|
|
3574
|
-
|
|
3575
|
-
|
|
3576
|
-
|
|
3577
|
-
|
|
3642
|
+
const raw = await getSourcesRaw();
|
|
3643
|
+
let sourcesDoc;
|
|
3644
|
+
try {
|
|
3645
|
+
sourcesDoc = JSON.parse(raw);
|
|
3646
|
+
} catch {
|
|
3647
|
+
sourcesDoc = { sources: [] };
|
|
3648
|
+
}
|
|
3649
|
+
const payload = {
|
|
3650
|
+
rssanyConnectivityTest: true,
|
|
3651
|
+
items: {
|
|
3652
|
+
sourceRef: "rssany-deliver-test",
|
|
3653
|
+
items: feedItemsToPayload([sample])
|
|
3654
|
+
},
|
|
3655
|
+
sources: sourcesDoc
|
|
3656
|
+
};
|
|
3657
|
+
await postDeliverGatewayTest(gateway.trim(), payload, { bearerToken: token || void 0 });
|
|
3578
3658
|
return c.json({ ok: true });
|
|
3579
3659
|
} catch (err) {
|
|
3580
3660
|
return c.json({ ok: false, message: err instanceof Error ? err.message : String(err) }, 400);
|
|
@@ -4440,6 +4520,16 @@ function registerWebUiRoutes(app) {
|
|
|
4440
4520
|
};
|
|
4441
4521
|
app.get("*", spaFallback);
|
|
4442
4522
|
}
|
|
4523
|
+
const here = dirname(fileURLToPath(import.meta.url));
|
|
4524
|
+
function getAppVersion() {
|
|
4525
|
+
try {
|
|
4526
|
+
const pkgPath = join(here, "../package.json");
|
|
4527
|
+
const pkg = JSON.parse(readFileSync(pkgPath, "utf8"));
|
|
4528
|
+
return pkg.version ?? "unknown";
|
|
4529
|
+
} catch {
|
|
4530
|
+
return "unknown";
|
|
4531
|
+
}
|
|
4532
|
+
}
|
|
4443
4533
|
const PORT = Number(process.env.PORT) || 18473;
|
|
4444
4534
|
const IS_DEV = process.env.NODE_ENV === "development" || process.argv.includes("--watch");
|
|
4445
4535
|
const PLUGIN_WATCH_EXTS = [".rssany.js", ".rssany.ts"];
|
|
@@ -4489,7 +4579,9 @@ async function main() {
|
|
|
4489
4579
|
const app = createApp();
|
|
4490
4580
|
const server = serve({ fetch: app.fetch, port: PORT, hostname: "0.0.0.0" });
|
|
4491
4581
|
server.setMaxListeners(32);
|
|
4492
|
-
console.log(
|
|
4582
|
+
console.log(
|
|
4583
|
+
`RssAny ${getAppVersion()} 服务已启动 http://127.0.0.1:${PORT}/(API + 静态前端,需先 pnpm run webui:build)`
|
|
4584
|
+
);
|
|
4493
4585
|
const lanIp = Object.values(networkInterfaces()).flat().find((iface) => iface?.family === "IPv4" && !iface.internal)?.address;
|
|
4494
4586
|
if (lanIp) console.log(`局域网访问 http://${lanIp}:${PORT}/`);
|
|
4495
4587
|
if (IS_DEV) {
|