rssany 0.1.0 → 0.1.1

package/dist/index.js

@@ -12,10 +12,14 @@ import { parse, NodeType } from "node-html-parser";
 import Database from "better-sqlite3";
 import { mkdir, copyFile, access, rename, readFile, writeFile, readdir } from "node:fs/promises";
 import { fileURLToPath, pathToFileURL } from "node:url";
-import { createHash, randomUUID } from "node:crypto";
+import { createHash } from "node:crypto";
 import { JSDOM } from "jsdom";
 import { Readability } from "@mozilla/readability";
 import OpenAI from "openai";
+import RssParser from "rss-parser";
+import { HttpsProxyAgent } from "https-proxy-agent";
+import { ImapFlow } from "imapflow";
+import { simpleParser } from "mailparser";
 import { EventEmitter } from "node:events";
 import { CronExpressionParser } from "cron-parser";
 import { validate, schedule as schedule$1 } from "node-cron";
@@ -156,12 +160,9 @@ const SOURCES_CONFIG_PATH = join(USER_DIR, "sources.json");
 const TAGS_CONFIG_PATH = join(USER_DIR, "tags.json");
 const CONFIG_PATH = join(USER_DIR, "config.json");
 const LEGACY_SUBSCRIPTIONS_PATH = join(USER_DIR, "subscriptions.json");
-const BUILTIN_PLUGINS_DIR = join(PACKAGE_ROOT, "plugins");
+const BUILTIN_PLUGINS_DIR = join(PACKAGE_ROOT, "app/plugins/builtin");
 const USER_PLUGINS_DIR = join(USER_DIR, "plugins");
-const BUILTIN_SOURCES_DIR = join(BUILTIN_PLUGINS_DIR, "sources");
-const USER_SOURCES_DIR = join(USER_PLUGINS_DIR, "sources");
-const BUILTIN_ENRICH_DIR = join(BUILTIN_PLUGINS_DIR, "enrich");
-const USER_ENRICH_DIR = join(USER_PLUGINS_DIR, "enrich");
+const PLUGIN_SITE_TEMPLATE_PATH = join(PACKAGE_ROOT, "app/plugins/site.rssany.js");
 async function pathExists(p) {
   try {
     await access(p);
@@ -209,8 +210,6 @@ async function initUserDir() {
   await mkdir(DATA_DIR, { recursive: true });
   await mkdir(CACHE_DIR, { recursive: true });
   await mkdir(USER_PLUGINS_DIR, { recursive: true });
-  await mkdir(USER_SOURCES_DIR, { recursive: true });
-  await mkdir(USER_ENRICH_DIR, { recursive: true });
   await seedExampleConfigsIfMissing();
   if (!await pathExists(SOURCES_CONFIG_PATH) && await pathExists(LEGACY_SUBSCRIPTIONS_PATH)) {
     await migrateFile(LEGACY_SUBSCRIPTIONS_PATH, SOURCES_CONFIG_PATH);
@@ -225,17 +224,17 @@ function logCorruptDiagnostic(operation, err) {
   const code = err?.code;
   const msg = err instanceof Error ? err.message : String(err);
   const lines = [
-    "[rssany db] ??????????????",
-    `  ?????: ${operation}`,
-    `  ????: ${code ?? "unknown"} - ${msg}`,
-    "  ????????:",
-    "    1. ??????/????????????????????????? tsx --watch ?????????????????????????????????????????????",
-    "    2. ???????????????????WAL ????? checkpoint",
-    "    3. ????/?????/?????????????????????????",
-    "  ??:",
-    "    - ??????????????? --watch???????????????????????????????",
-    "    - ???????????? RSSANY_DB_JOURNAL=delete ?????? DELETE ???????????????????",
-    "    - ?????? .rssany/data/rssany.db ????? -wal???-shm???rssany.db.lock ??????"
+    "[rssany db] 数据库可能损坏或并发冲突",
+    `  操作: ${operation}`,
+    `  错误: ${code ?? "unknown"} - ${msg}`,
+    "  常见原因:",
+    "    1. 多进程同时打开同一库（例如 tsx --watch 与另一实例同时写）",
+    "    2. 异常退出后 WAL 未正常 checkpoint",
+    "    3. 磁盘/杀毒/同步盘导致文件不完整",
+    "  建议:",
+    "    - 避免多实例同时写库；开发时慎用 --watch 与后台任务并行",
+    "    - 可尝试 RSSANY_DB_JOURNAL=delete 使用 DELETE 模式降低多文件依赖",
+    "    - 备份后删除 .rssany/data/rssany.db 及同目录 -wal、-shm、rssany.db.lock 再启动"
   ];
   process.stderr.write(lines.join("\n") + "\n");
 }
@@ -274,7 +273,7 @@ function acquireDbLock(dbDir) {
       })();
       if (stillRunning) {
         throw new Error(
-          `???????????????????????????PID ${oldPid}??????????? tsx --watch ?????????????????????????????????? ${lockPath} ??????????`
+          `数据库已被其他进程占用（PID ${oldPid}）。请勿多开实例；若确认无其他进程，可删除 ${lockPath} 后重试（常见于 tsx --watch 未退出）`
         );
       }
     }
@@ -307,7 +306,7 @@ function withWriteLock(fn) {
     },
     (e) => {
       if (isCorruptError(e)) {
-        logCorruptDiagnostic("???????????? updateItemContent/upsertItems??", e);
+        logCorruptDiagnostic("withWriteLock 内 updateItemContent/upsertItems 等", e);
       }
       rejectOut(e);
       throw e;
@@ -315,7 +314,7 @@ function withWriteLock(fn) {
   );
   return out;
 }
-const DATE_ONLY_TITLE_RE = /^(?:jan|feb|mar|apr|may|jun|jul|aug|sep|sept|oct|nov|dec)\b[\s\d,??./-]*(?:st|nd|rd|th)?[\s\d,??./-]*$/i;
+const DATE_ONLY_TITLE_RE = /^(?:jan|feb|mar|apr|may|jun|jul|aug|sep|sept|oct|nov|dec)\b[\s\d,./-]*(?:st|nd|rd|th)?[\s\d,./-]*$/i;
 function normalizeText(text) {
   return (text ?? "").replace(/\s+/g, " ").trim();
 }
@@ -393,7 +392,7 @@ async function getDb() {
         db = null;
       }
       if (isCorruptError(err)) {
-        logCorruptDiagnostic("?????/??????????? (getDb)", err);
+        logCorruptDiagnostic("打开/初始化主库 (getDb)", err);
       }
       throw err;
     }
@@ -407,7 +406,7 @@ async function runIntegrityCheck() {
     return row?.integrity_check ?? "unknown";
   } catch (err) {
     const msg = err instanceof Error ? err.message : String(err);
-    return `integrity_check ???????: ${msg}`;
+    return `integrity_check 执行失败: ${msg}`;
   }
 }
 const LOGS_DB_PATH = join(DATA_DIR, "logs.db");
@@ -525,7 +524,7 @@ async function upsertItems(items, sourceUrlOverride) {
   if (items.length === 0) return { newCount: 0, newIds: /* @__PURE__ */ new Set() };
   const sourceUrl = items[0].sourceRef;
   if (!sourceUrl) {
-    throw new Error("upsertItems: ???????? item ???? sourceRef????????? sourceUrlOverride");
+    throw new Error("upsertItems: 每条 item 须有 sourceRef，或传入 sourceUrlOverride");
   }
   return withWriteLock(async () => {
     const db = await getDb();
@@ -647,7 +646,7 @@ async function queryFeedItems(sourceUrls, limit, offset, opts) {
   if (opts?.until) {
     conditions.push("COALESCE(pub_date, fetched_at) < @until");
     if (opts.until.length === 10) {
-      const d = /* @__PURE__ */ new Date(opts.until + "T12:00:00Z");
+      const d = /* @__PURE__ */ new Date(`${opts.until}T12:00:00Z`);
       d.setUTCDate(d.getUTCDate() + 1);
       params.until = d.toISOString();
     } else {
@@ -690,9 +689,7 @@ async function queryItems(opts) {
     const trimmed = tagsFilter.filter((t) => typeof t === "string" && t.trim()).map((t) => t.trim());
     if (trimmed.length > 0) {
       const tagConds = trimmed.map((_, i) => `LOWER(TRIM(json_each.value)) = LOWER(@tag${i})`).join(" OR ");
-      conditions.push(
-        `i.tags IS NOT NULL AND EXISTS (SELECT 1 FROM json_each(i.tags) WHERE ${tagConds})`
-      );
+      conditions.push(`i.tags IS NOT NULL AND EXISTS (SELECT 1 FROM json_each(i.tags) WHERE ${tagConds})`);
       trimmed.forEach((t, i) => {
         params[`tag${i}`] = t;
       });
@@ -1675,11 +1672,6 @@ function isValidSource(obj) {
   const s = obj;
   return typeof s.id === "string" && (typeof s.pattern === "string" || s.pattern instanceof RegExp) && typeof s.fetchItems === "function" && s.listUrlPattern === void 0;
 }
-function isValidEnrichPlugin(obj) {
-  if (obj == null || typeof obj !== "object") return false;
-  const p = obj;
-  return typeof p.id === "string" && typeof p.match === "function" && typeof p.enrichItem === "function";
-}
 async function loadSourcePluginsFromDir(dir, label) {
   const siteEntries = [];
   const sources = [];
@@ -1711,46 +1703,12 @@ async function loadSourcePluginsFromDir(dir, label) {
   }
   return { siteEntries, sources };
 }
-async function loadPluginsFromDir(dir, label, validator) {
-  const result = [];
-  let entries;
-  try {
-    const raw = await readdir(dir, { withFileTypes: true, encoding: "utf-8" });
-    entries = raw;
-  } catch {
-    return result;
-  }
-  for (const e of entries) {
-    const name = String(e.name);
-    if (!e.isFile()) continue;
-    if (!PLUGIN_EXTENSIONS.some((ext) => name.endsWith(ext))) continue;
-    const filePath = join(dir, name);
-    try {
-      const mod = await import(pathToFileURL(filePath).href);
-      const plugin = mod.default ?? mod;
-      if (validator(plugin)) {
-        result.push(plugin);
-      } else {
-        logger.warn("plugin", "插件接口不匹配，已跳过", { label, name });
-      }
-    } catch (err) {
-      logger.warn("plugin", "插件加载失败", { label, name, err: err instanceof Error ? err.message : String(err) });
-    }
-  }
-  return result;
-}
-async function loadFromSourcesOrRoot() {
-  const [builtinFromSources, userFromSources] = await Promise.all([
-    loadSourcePluginsFromDir(BUILTIN_SOURCES_DIR, "builtin:sources"),
-    loadSourcePluginsFromDir(USER_SOURCES_DIR, "user:sources")
-  ]);
-  const hasAny = builtinFromSources.siteEntries.length + builtinFromSources.sources.length + userFromSources.siteEntries.length + userFromSources.sources.length > 0;
-  if (hasAny) return { builtin: builtinFromSources, user: userFromSources };
-  const [builtinRoot, userRoot] = await Promise.all([
+async function loadBuiltinAndUser() {
+  const [builtin, user] = await Promise.all([
     loadSourcePluginsFromDir(BUILTIN_PLUGINS_DIR, "builtin"),
     loadSourcePluginsFromDir(USER_PLUGINS_DIR, "user")
   ]);
-  return { builtin: builtinRoot, user: userRoot };
+  return { builtin, user };
 }
 const pluginSitePaths = /* @__PURE__ */ new Map();
 function mergeSourcePluginPaths(siteIds, pathMap, builtinSources, userSources) {
@@ -1774,7 +1732,7 @@ function getPluginFilePath(id) {
   return pluginSitePaths.get(id);
 }
 async function loadSiteAndSourcePlugins() {
-  const { builtin, user } = await loadFromSourcesOrRoot();
+  const { builtin, user } = await loadBuiltinAndUser();
   const siteMap = /* @__PURE__ */ new Map();
   const pathMap = /* @__PURE__ */ new Map();
   for (const { site: s, filePath } of builtin.siteEntries) {
@@ -1797,73 +1755,6 @@ async function loadSiteAndSourcePlugins() {
   pathMap.forEach((path, id) => pluginSitePaths.set(id, path));
   return { sites: Array.from(siteMap.values()), sources: Array.from(sourceMap.values()) };
 }
-let registeredEnrichPlugins = [];
-async function loadEnrichPlugins() {
-  const [builtin, user] = await Promise.all([
-    loadPluginsFromDir(BUILTIN_ENRICH_DIR, "builtin:enrich", isValidEnrichPlugin),
-    loadPluginsFromDir(USER_ENRICH_DIR, "user:enrich", isValidEnrichPlugin)
-  ]);
-  const merged = /* @__PURE__ */ new Map();
-  for (const p of builtin) merged.set(p.id, p);
-  for (const p of user) {
-    if (merged.has(p.id)) logger.info("plugin", "用户 Enrich 插件覆盖同名内置", { pluginId: p.id });
-    merged.set(p.id, p);
-  }
-  const list = Array.from(merged.values());
-  list.sort((a, b) => (a.priority ?? 100) - (b.priority ?? 100));
-  registeredEnrichPlugins = list;
-  return list;
-}
-function getMatchedEnrichPlugin(item, ctx) {
-  return registeredEnrichPlugins.find((p) => p.match(item, ctx));
-}
-function buildEnrichContext(ctx) {
-  return {
-    cacheDir: ctx.cacheDir,
-    headless: ctx.headless,
-    proxy: ctx.proxy,
-    async fetchHtml(url, opts) {
-      const res = await fetchHtml(url, {
-        cacheDir: ctx.cacheDir,
-        useCache: false,
-        authFlow: void 0,
-        headless: ctx.headless,
-        proxy: ctx.proxy,
-        waitAfterLoadMs: opts?.waitMs,
-        purify: opts?.purify
-      });
-      return { html: res.body, finalUrl: res.finalUrl ?? url, status: res.status };
-    },
-    async extractItem(item, opts) {
-      const res = await fetchHtml(item.link, {
-        cacheDir: ctx.cacheDir,
-        useCache: false,
-        authFlow: void 0,
-        headless: ctx.headless,
-        proxy: ctx.proxy
-      });
-      if (res.status !== 200 && res.status !== 304) {
-        throw new Error(`默认正文提取失败: HTTP ${res.status} ${res.statusText} for ${item.link}`);
-      }
-      const extracted = await extractHtml(res.body, {
-        url: res.finalUrl ?? item.link,
-        cacheDir: ctx.cacheDir ?? void 0,
-        mode: "readability",
-        useCache: true,
-        cacheKey: opts?.cacheKey
-      });
-      const pubDate = extracted.pubDate != null ? typeof extracted.pubDate === "string" ? new Date(extracted.pubDate) : extracted.pubDate : item.pubDate;
-      return {
-        ...item,
-        author: normalizeAuthor(extracted.author ?? item.author),
-        title: extracted.title ?? item.title,
-        summary: extracted.summary ?? item.summary,
-        content: extracted.content ?? item.content,
-        pubDate
-      };
-    }
-  };
-}
 function buildSiteContext(site, ctx) {
   const proxy = ctx.proxy ?? site.proxy;
   const authFlow = toAuthFlow(site);
@@ -1871,6 +1762,7 @@ function buildSiteContext(site, ctx) {
     cacheDir: ctx.cacheDir,
     headless: ctx.headless,
     proxy,
+    deps: ctx.deps,
     async fetchHtml(url, opts) {
       const res = await fetchHtml(url, {
         cacheDir: ctx.cacheDir,
@@ -1930,10 +1822,7 @@ function createWebSource(site) {
     } : void 0,
     async fetchItems(sourceId, ctx) {
       return site.fetchItems(sourceId, buildSiteContext(site, ctx));
-    },
-    enrichItem: site.enrichItem ? async (item, ctx) => {
-      return site.enrichItem(item, buildSiteContext(site, ctx));
-    } : void 0
+    }
   };
 }
 const genericWebSource = {
@@ -1970,6 +1859,19 @@ function getPluginSites() {
 function getBestSite(url) {
   return getSiteByUrl(url, loadedSites);
 }
+const PLUGIN_HOST_DEPS = {
+  parseHtml: parse,
+  NodeType,
+  createHash,
+  RssParser,
+  HttpsProxyAgent,
+  ImapFlow,
+  simpleParser,
+  logger
+};
+function buildSourceContext(partial) {
+  return { ...partial, deps: PLUGIN_HOST_DEPS };
+}
 const registeredSources = [];
 function sourcePatternToRegex(pattern) {
   if (pattern instanceof RegExp) return pattern;
@@ -1992,10 +1894,7 @@ function getSource(sourceId) {
   return genericWebSource;
 }
 async function initSources() {
-  const [siteResult] = await Promise.all([
-    loadSiteAndSourcePlugins(),
-    loadEnrichPlugins()
-  ]);
+  const siteResult = await loadSiteAndSourcePlugins();
   const { sites, sources: sourcePlugins } = siteResult;
   setLoadedSites(sites);
   registeredSources.length = 0;
@@ -2378,25 +2277,180 @@ function onFeedUpdated(fn) {
   eventBus.on("feed:updated", fn);
   return () => eventBus.off("feed:updated", fn);
 }
-const DEFAULTS = {
-  concurrency: 2,
-  maxRetries: 2
-};
-async function loadEnrichConfig() {
-  let fileEnrich = {};
+async function getDeliverUrl() {
   try {
-    const raw = await readFile(join(USER_DIR, "config.json"), "utf-8");
-    const parsed = JSON.parse(raw);
-    if (parsed.enrich && typeof parsed.enrich === "object") {
-      fileEnrich = parsed.enrich;
-    }
+    const raw = await readFile(CONFIG_PATH, "utf-8");
+    const j = JSON.parse(raw);
+    const u = j?.deliver?.url;
+    return typeof u === "string" ? u.trim() : "";
+  } catch {
+    return "";
+  }
+}
+async function saveDeliverUrl(url) {
+  let root = {};
+  try {
+    const raw = await readFile(CONFIG_PATH, "utf-8");
+    root = JSON.parse(raw);
   } catch {
   }
+  root.deliver = { url: url.trim() };
+  await writeFile(CONFIG_PATH, JSON.stringify(root, null, 2) + "\n", "utf-8");
+}
+function feedItemsToPayload(items) {
+  return items.map((i) => ({
+    guid: i.guid,
+    title: i.title,
+    link: i.link,
+    pubDate: i.pubDate instanceof Date ? i.pubDate.toISOString() : (/* @__PURE__ */ new Date()).toISOString(),
+    author: i.author,
+    summary: i.summary,
+    content: i.content,
+    tags: i.tags,
+    sourceRef: i.sourceRef,
+    translations: i.translations
+  }));
+}
+async function postDeliverItems(url, sourceRef, items) {
+  if (!url.trim() || items.length === 0) return;
+  const body = JSON.stringify({ sourceRef, items: feedItemsToPayload(items) });
+  const res = await fetch(url.trim(), {
+    method: "POST",
+    headers: { "Content-Type": "application/json" },
+    body,
+    signal: AbortSignal.timeout(12e4)
+  });
+  if (!res.ok) {
+    const text = await res.text().catch(() => "");
+    throw new Error(`HTTP ${res.status}${text ? `: ${text.slice(0, 200)}` : ""}`);
+  }
+}
+async function postDeliverItemsSafe(url, sourceRef, items) {
+  try {
+    await postDeliverItems(url, sourceRef, items);
+  } catch (err) {
+    logger.warn("deliver", "投递失败", {
+      sourceRef,
+      count: items.length,
+      err: err instanceof Error ? err.message : String(err)
+    });
+  }
+}
+function buildChannelFromItems(listUrl, items, lng) {
+  const channel = {
+    title: items[0]?.author?.length ? `${items[0].author[0]} 的订阅` : "RSS 订阅",
+    link: listUrl,
+    description: `来自 ${listUrl} 的订阅`
+  };
+  if (lng) channel.language = lng;
+  return channel;
+}
+function toRssEntry(item, lng) {
+  const eff = getEffectiveItemFields(item, lng);
+  const hasContent = eff.content != null && eff.content !== "";
+  const desc = hasContent ? eff.content : eff.summary;
   return {
-    concurrency: Number(fileEnrich["concurrency"] ?? process.env.ENRICH_CONCURRENCY ?? DEFAULTS.concurrency),
-    maxRetries: Number(fileEnrich["maxRetries"] ?? process.env.ENRICH_MAX_RETRIES ?? DEFAULTS.maxRetries)
+    title: eff.title,
+    link: item.link,
+    description: desc,
+    guid: item.guid,
+    published: item.pubDate?.toISOString?.() ?? void 0,
+    imageUrl: item.imageUrl
   };
 }
+const generatingKeys = /* @__PURE__ */ new Map();
+const pipelineCtx = {
+  llm: { chatJson, chatText },
+  db: { getSystemTags }
+};
+async function runPipelineOnItem(item, ctx) {
+  return runPipeline(item, { ...pipelineCtx, ...ctx });
+}
+async function generateAndCache(listUrl, key, config) {
+  const { cacheDir = "cache", headless } = config;
+  const source = getSource(listUrl);
+  const ctx = buildSourceContext({ cacheDir, headless, proxy: config.proxy ?? source.proxy });
+  let items;
+  try {
+    items = await source.fetchItems(listUrl, ctx);
+  } catch (err) {
+    generatingKeys.delete(key);
+    const message = err instanceof Error ? err.message : String(err);
+    logger.error("scraper", "抓取失败", { source_url: listUrl, err: message });
+    throw err;
+  }
+  items.forEach((i) => {
+    i.sourceRef = listUrl;
+    i.author = normalizeAuthor(i.author);
+  });
+  generatingKeys.delete(key);
+  logger.info("scraper", "抓取成功", { source_url: listUrl, count: items.length });
+  const deliverUrl = await getDeliverUrl();
+  let newCount = 0;
+  let newIds = /* @__PURE__ */ new Set();
+  const upsertResult = await upsertItems(items).catch((err) => {
+    logger.warn("db", "upsertItems 失败", { source_url: listUrl, err: err instanceof Error ? err.message : String(err) });
+    return { newCount: 0, newIds: /* @__PURE__ */ new Set() };
+  });
+  newCount = upsertResult.newCount;
+  newIds = upsertResult.newIds;
+  let pipelineDroppedNew = 0;
+  const shouldRunPipelineRow = (guid) => newIds.has(guid);
+  for (let i = 0; i < items.length; i++) {
+    if (!shouldRunPipelineRow(items[i].guid)) continue;
+    const processed = await runPipelineOnItem(items[i], { sourceUrl: listUrl });
+    items[i] = processed;
+    if (isPipelineDroppedItem(processed)) {
+      await deleteItem(processed.guid).catch(
+        (err) => logger.warn("db", "质量过滤后删除条目失败", { source_url: listUrl, err: err instanceof Error ? err.message : String(err) })
+      );
+      pipelineDroppedNew++;
+    } else {
+      updateItemContent(processed).catch(
+        (err) => logger.warn("db", "updateItemContent 失败", { source_url: listUrl, err: err instanceof Error ? err.message : String(err) })
+      );
+    }
+  }
+  if (newCount > 0) {
+    emitFeedUpdated({ sourceUrl: listUrl, newCount: newCount - pipelineDroppedNew });
+  }
+  const out = items.filter((i) => !isPipelineDroppedItem(i));
+  if (deliverUrl && out.length > 0) {
+    await postDeliverItemsSafe(deliverUrl, listUrl, out);
+  }
+  return { items: out };
+}
+async function getItems(listUrl, config = {}) {
+  const source = getSource(listUrl);
+  const key = config.cron ? cacheKeyFromCron(listUrl, config.cron) : cacheKey(listUrl, config.refreshInterval ?? source.refreshInterval ?? "1day");
+  if (source.preCheck != null) {
+    try {
+      await source.preCheck(
+        buildSourceContext({
+          cacheDir: config.cacheDir ?? "cache",
+          headless: config.headless,
+          proxy: config.proxy ?? source.proxy
+        })
+      );
+    } catch (err) {
+      if (err instanceof AuthRequiredError) throw err;
+      throw err;
+    }
+  }
+  let task = config.force ? void 0 : generatingKeys.get(key);
+  if (!task) {
+    task = generateAndCache(listUrl, key, config);
+    if (!config.force) generatingKeys.set(key, task);
+  }
+  const { items } = await task;
+  return { items, fromCache: false };
+}
+function feedItemsToRssXml(items, listUrl, lng, opts) {
+  const channel = buildChannelFromItems(listUrl, items, lng);
+  if (opts?.channelTitle) channel.title = opts.channelTitle;
+  if (opts?.channelDesc) channel.description = opts.channelDesc;
+  return buildRssXml(channel, items.map((it) => toRssEntry(it, lng)));
+}
 const validateCron = validate;
 const tasks$1 = /* @__PURE__ */ new Map();
 const groups = /* @__PURE__ */ new Map();
@@ -2580,363 +2634,6 @@ function getGroupStats() {
   }
   return result;
 }
-const ENRICH_GROUP = "enrich";
-const MAX_STORED_TASKS = 200;
-const RETRY_DELAY_MS = 3e3;
-class EnrichQueue {
-  tasks = /* @__PURE__ */ new Map();
-  taskItems = /* @__PURE__ */ new Map();
-  taskCallbacks = /* @__PURE__ */ new Map();
-  configLoaded = false;
-  async ensureConfig() {
-    if (this.configLoaded) return { concurrency: 2, maxRetries: 2 };
-    const config = await loadEnrichConfig();
-    this.configLoaded = true;
-    logger.info("scraper", "配置加载完成", { concurrency: config.concurrency, maxRetries: config.maxRetries });
-    return config;
-  }
-  evictIfNeeded() {
-    if (this.tasks.size <= MAX_STORED_TASKS) return;
-    const ids = [...this.tasks.keys()];
-    for (const id of ids) {
-      if (this.tasks.get(id)?.status === "done") {
-        this.removeTask(id);
-        if (this.tasks.size <= MAX_STORED_TASKS) return;
-      }
-    }
-    if (this.tasks.size > MAX_STORED_TASKS) this.removeTask(ids[0]);
-  }
-  removeTask(id) {
-    this.tasks.delete(id);
-    this.taskItems.delete(id);
-    this.taskCallbacks.delete(id);
-  }
-  checkTaskComplete(taskId) {
-    const task = this.tasks.get(taskId);
-    const items = this.taskItems.get(taskId);
-    const callbacks = this.taskCallbacks.get(taskId);
-    if (!task || !items) return;
-    const allSettled = task.itemResults.every((r) => r.status === "done" || r.status === "failed");
-    if (!allSettled) return;
-    task.status = "done";
-    task.completedAt = (/* @__PURE__ */ new Date()).toISOString();
-    logger.info("scraper", "任务完成", {
-      source_url: task.sourceUrl,
-      taskId,
-      done: task.progress.done,
-      failed: task.progress.failed
-    });
-    Promise.resolve(callbacks?.onAllDone?.(items)).catch((err) => {
-      logger.warn("scraper", "onAllDone 回调异常", { taskId, err: err instanceof Error ? err.message : String(err) });
-    });
-  }
-  async submit(items, enrichFn, ctx, opts) {
-    const config = await this.ensureConfig();
-    const id = randomUUID();
-    const itemResults = items.map((_, i) => ({
-      index: i,
-      status: "pending",
-      retries: 0
-    }));
-    const task = {
-      id,
-      sourceUrl: opts.sourceUrl,
-      status: items.length === 0 ? "done" : "pending",
-      progress: { total: items.length, done: 0, failed: 0 },
-      itemResults,
-      createdAt: (/* @__PURE__ */ new Date()).toISOString(),
-      completedAt: items.length === 0 ? (/* @__PURE__ */ new Date()).toISOString() : void 0
-    };
-    const itemsCopy = [...items];
-    this.tasks.set(id, task);
-    this.taskItems.set(id, itemsCopy);
-    this.taskCallbacks.set(id, opts);
-    this.evictIfNeeded();
-    for (let i = 0; i < items.length; i++) {
-      const itemIndex = i;
-      const workId = `${id}-${i}`;
-      const taskFn = async () => {
-        const t = this.tasks.get(id);
-        const its = this.taskItems.get(id);
-        const cbs = this.taskCallbacks.get(id);
-        if (!t || !its || !cbs) return;
-        const itemResult = t.itemResults[itemIndex];
-        if (!itemResult) return;
-        itemResult.status = "running";
-        if (t.status === "pending") t.status = "running";
-        for (let r = 0; r <= config.maxRetries; r++) {
-          try {
-            const enriched = await enrichFn(its[itemIndex], ctx);
-            its[itemIndex] = enriched;
-            itemResult.item = enriched;
-            itemResult.status = "done";
-            t.progress.done++;
-            await Promise.resolve(cbs.onItemDone?.(enriched, itemIndex));
-            this.checkTaskComplete(id);
-            return;
-          } catch (err) {
-            const msg = err instanceof Error ? err.message : String(err);
-            if (r < config.maxRetries) {
-              logger.warn("scraper", "提取失败，重试中", {
-                source_url: t.sourceUrl,
-                item_url: its[itemIndex]?.link,
-                retries: r + 1,
-                maxRetries: config.maxRetries,
-                err: msg
-              });
-              await new Promise((resolve2) => setTimeout(resolve2, RETRY_DELAY_MS));
-            } else {
-              itemResult.status = "failed";
-              itemResult.error = msg;
-              t.progress.failed++;
-              logger.warn("scraper", "提取最终失败", {
-                source_url: t.sourceUrl,
-                item_url: its[itemIndex]?.link,
-                err: msg
-              });
-              const failedItem = { ...its[itemIndex], enrichFailed: true };
-              its[itemIndex] = failedItem;
-              await Promise.resolve(cbs.onItemDone?.(failedItem, itemIndex));
-              this.checkTaskComplete(id);
-            }
-          }
-        }
-      };
-      schedule(ENRICH_GROUP, workId, taskFn, { concurrency: config.concurrency }).catch(() => {
-      });
-    }
-    return id;
-  }
-  getTask(id) {
-    return this.tasks.get(id);
-  }
-  getTaskItems(id) {
-    return this.taskItems.get(id);
-  }
-}
-const enrichQueue = new EnrichQueue();
-async function getDeliverUrl() {
-  try {
-    const raw = await readFile(CONFIG_PATH, "utf-8");
-    const j = JSON.parse(raw);
-    const u = j?.deliver?.url;
-    return typeof u === "string" ? u.trim() : "";
-  } catch {
-    return "";
-  }
-}
-async function saveDeliverUrl(url) {
-  let root = {};
-  try {
-    const raw = await readFile(CONFIG_PATH, "utf-8");
-    root = JSON.parse(raw);
-  } catch {
-  }
-  root.deliver = { url: url.trim() };
-  await writeFile(CONFIG_PATH, JSON.stringify(root, null, 2) + "\n", "utf-8");
-}
-function feedItemsToPayload(items) {
-  return items.map((i) => ({
-    guid: i.guid,
-    title: i.title,
-    link: i.link,
-    pubDate: i.pubDate instanceof Date ? i.pubDate.toISOString() : (/* @__PURE__ */ new Date()).toISOString(),
-    author: i.author,
-    summary: i.summary,
-    content: i.content,
-    tags: i.tags,
-    sourceRef: i.sourceRef,
-    translations: i.translations
-  }));
-}
-async function postDeliverItems(url, sourceRef, items) {
-  if (!url.trim() || items.length === 0) return;
-  const body = JSON.stringify({ sourceRef, items: feedItemsToPayload(items) });
-  const res = await fetch(url.trim(), {
-    method: "POST",
-    headers: { "Content-Type": "application/json" },
-    body,
-    signal: AbortSignal.timeout(12e4)
-  });
-  if (!res.ok) {
-    const text = await res.text().catch(() => "");
-    throw new Error(`HTTP ${res.status}${text ? `: ${text.slice(0, 200)}` : ""}`);
-  }
-}
-async function postDeliverItemsSafe(url, sourceRef, items) {
-  try {
-    await postDeliverItems(url, sourceRef, items);
-  } catch (err) {
-    logger.warn("deliver", "投递失败", {
-      sourceRef,
-      count: items.length,
-      err: err instanceof Error ? err.message : String(err)
-    });
-  }
-}
-function buildChannelFromItems(listUrl, items, lng) {
-  const channel = {
-    title: items[0]?.author?.length ? `${items[0].author[0]} 的订阅` : "RSS 订阅",
-    link: listUrl,
-    description: `来自 ${listUrl} 的订阅`
-  };
-  if (lng) channel.language = lng;
-  return channel;
-}
-function toRssEntry(item, lng) {
-  const eff = getEffectiveItemFields(item, lng);
-  const hasContent = eff.content != null && eff.content !== "";
-  const desc = hasContent ? eff.content : eff.summary;
-  return {
-    title: eff.title,
-    link: item.link,
-    description: desc,
-    guid: item.guid,
-    published: item.pubDate?.toISOString?.() ?? void 0,
-    imageUrl: item.imageUrl
-  };
-}
-const generatingKeys = /* @__PURE__ */ new Map();
-const pipelineCtx = {
-  llm: { chatJson, chatText },
-  db: { getSystemTags }
-};
-async function runPipelineOnItem(item, ctx) {
-  return runPipeline(item, { ...pipelineCtx, ...ctx });
-}
-function buildEnrichFn(source, listUrl, ctx) {
-  const enrichCtx = buildEnrichContext(ctx);
-  enrichCtx.sourceUrl = listUrl;
-  return async (item) => {
-    let result = item;
-    if (source.enrichItem) {
-      result = await source.enrichItem(item, ctx);
-    }
-    const plugin = getMatchedEnrichPlugin(result, { sourceUrl: listUrl });
-    if (plugin) {
-      result = await plugin.enrichItem(result, enrichCtx);
-    }
-    return result;
-  };
-}
-async function generateAndCache(listUrl, key, config) {
-  const { cacheDir = "cache", includeContent = true, headless } = config;
-  const source = getSource(listUrl);
-  const ctx = { cacheDir, headless, proxy: config.proxy ?? source.proxy };
-  let items;
-  try {
-    items = await source.fetchItems(listUrl, ctx);
-  } catch (err) {
-    generatingKeys.delete(key);
-    const message = err instanceof Error ? err.message : String(err);
-    logger.error("scraper", "抓取失败", { source_url: listUrl, err: message });
-    throw err;
-  }
-  items.forEach((i) => {
-    i.sourceRef = listUrl;
-    i.author = normalizeAuthor(i.author);
-  });
-  generatingKeys.delete(key);
-  logger.info("scraper", "抓取成功", { source_url: listUrl, count: items.length });
-  const deliverUrl = await getDeliverUrl();
-  let newCount = 0;
-  let newIds = /* @__PURE__ */ new Set();
-  const upsertResult = await upsertItems(items).catch((err) => {
-    logger.warn("db", "upsertItems 失败", { source_url: listUrl, err: err instanceof Error ? err.message : String(err) });
-    return { newCount: 0, newIds: /* @__PURE__ */ new Set() };
-  });
-  newCount = upsertResult.newCount;
-  newIds = upsertResult.newIds;
-  let pipelineDroppedNew = 0;
-  const shouldRunPipelineRow = (guid) => newIds.has(guid);
-  const hasEnrich = source.enrichItem != null || items.some((i) => getMatchedEnrichPlugin(i, { sourceUrl: listUrl }));
-  if (!includeContent || items.length === 0 || !hasEnrich) {
-    for (let i = 0; i < items.length; i++) {
-      if (!shouldRunPipelineRow(items[i].guid)) continue;
-      const processed = await runPipelineOnItem(items[i], { sourceUrl: listUrl, isEnriched: false });
-      items[i] = processed;
-      if (isPipelineDroppedItem(processed)) {
-        await deleteItem(processed.guid).catch(
-          (err) => logger.warn("db", "质量过滤后删除条目失败", { source_url: listUrl, err: err instanceof Error ? err.message : String(err) })
-        );
-        pipelineDroppedNew++;
-      } else {
-        updateItemContent(processed).catch(
-          (err) => logger.warn("db", "updateItemContent 失败", { source_url: listUrl, err: err instanceof Error ? err.message : String(err) })
-        );
-      }
-    }
-    if (newCount > 0) {
-      emitFeedUpdated({ sourceUrl: listUrl, newCount: newCount - pipelineDroppedNew });
-    }
-    const out = items.filter((i) => !isPipelineDroppedItem(i));
-    if (deliverUrl && out.length > 0) {
-      await postDeliverItemsSafe(deliverUrl, listUrl, out);
-    }
-    return { items: out };
-  }
-  const enrichFn = (item, _ctx) => buildEnrichFn(source, listUrl, ctx)(item);
-  await enrichQueue.submit(
-    items,
-    enrichFn,
-    ctx,
-    {
-      sourceUrl: listUrl,
-      onItemDone: async (enrichedItem, index) => {
-        enrichedItem.sourceRef = listUrl;
-        const processed = shouldRunPipelineRow(enrichedItem.guid) ? await runPipelineOnItem(enrichedItem, { sourceUrl: listUrl, isEnriched: true }) : enrichedItem;
-        items[index] = processed;
-        if (isPipelineDroppedItem(processed)) {
-          await deleteItem(processed.guid).catch(
-            (err) => logger.warn("db", "质量过滤后删除条目失败", { source_url: listUrl, err: err instanceof Error ? err.message : String(err) })
-          );
-          pipelineDroppedNew++;
-        } else {
-          updateItemContent(processed).catch(
-            (err) => logger.warn("db", "updateItemContent 失败", { source_url: listUrl, err: err instanceof Error ? err.message : String(err) })
-          );
-        }
-      },
-      onAllDone: async () => {
-        for (let i = items.length - 1; i >= 0; i--) {
-          if (isPipelineDroppedItem(items[i])) items.splice(i, 1);
-        }
-        if (newCount > 0) {
-          emitFeedUpdated({ sourceUrl: listUrl, newCount: newCount - pipelineDroppedNew });
-        }
-        if (deliverUrl && items.length > 0) {
-          await postDeliverItemsSafe(deliverUrl, listUrl, items);
-        }
-      }
-    }
-  );
-  return { items };
-}
-async function getItems(listUrl, config = {}) {
-  const source = getSource(listUrl);
-  const key = config.cron ? cacheKeyFromCron(listUrl, config.cron) : cacheKey(listUrl, config.refreshInterval ?? source.refreshInterval ?? "1day");
-  if (source.preCheck != null) {
-    try {
-      await source.preCheck({ cacheDir: config.cacheDir ?? "cache", headless: config.headless, proxy: config.proxy ?? source.proxy });
-    } catch (err) {
-      if (err instanceof AuthRequiredError) throw err;
-      throw err;
-    }
-  }
-  let task = config.force ? void 0 : generatingKeys.get(key);
-  if (!task) {
-    task = generateAndCache(listUrl, key, config);
-    if (!config.force) generatingKeys.set(key, task);
-  }
-  const { items } = await task;
-  return { items, fromCache: false };
-}
-function feedItemsToRssXml(items, listUrl, lng, opts) {
-  const channel = buildChannelFromItems(listUrl, items, lng);
-  if (opts?.channelTitle) channel.title = opts.channelTitle;
-  if (opts?.channelDesc) channel.description = opts.channelDesc;
-  return buildRssXml(channel, items.map((it) => toRssEntry(it, lng)));
-}
 const DEFAULT_REFRESH = "1day";
 const SOURCES_CONCURRENCY = 5;
 function createPullTask(ref, cacheDir, cronExpr) {
@@ -3044,23 +2741,15 @@ function registerRssApiRoutes(app) {
     }
   });
 }
-function registerEnrichRoutes(app) {
-  app.get("/api/enrich/:taskId", (c) => {
-    const taskId = c.req.param("taskId");
-    const task = enrichQueue.getTask(taskId);
-    if (!task) return c.json({ error: "任务不存在或已过期" }, 404);
-    return c.json(task);
-  });
-}
 function registerSchedulerRoutes(app) {
   app.get("/api/scheduler/stats", requireAdmin(), (c) => {
     const stats = getGroupStats();
     return c.json(stats);
   });
 }
-const USER_SITE_TEMPLATE = join(BUILTIN_PLUGINS_DIR, "templates", "site.rssany.js");
 const SITE_TEMPLATE_FALLBACK = `/**
- * Site 插件模板（由管理页添加，位于 .rssany/plugins/sources/）
+ * Site 插件模板（由管理页添加，位于 .rssany/plugins/）
+ * HTML DOM 解析请用 ctx.deps.parseHtml，勿在插件内 import node_modules。
  */
 export default {
   id: "__PLUGIN_ID__",
@@ -3072,7 +2761,7 @@ export default {
       waitMs: 2000,
       purify: true,
     });
-    void html;
+    void ctx.deps.parseHtml(html);
     void finalUrl;
     return [];
   },
@@ -3111,12 +2800,11 @@ function registerPluginsRoutes(app) {
       return c.json({ error: "id 须为字母开头，仅含字母数字、下划线、连字符；不能为 generic 或 new" }, 400);
     }
     await mkdir(USER_PLUGINS_DIR, { recursive: true });
-    await mkdir(USER_SOURCES_DIR, { recursive: true });
-    const outPath = join(USER_SOURCES_DIR, `${id}.rssany.ts`);
+    const outPath = join(USER_PLUGINS_DIR, `${id}.rssany.js`);
     if (await fileExists(outPath)) return c.json({ error: "该 id 已存在同名文件" }, 409);
     let tpl = SITE_TEMPLATE_FALLBACK;
     try {
-      tpl = await readFile(USER_SITE_TEMPLATE, "utf-8");
+      tpl = await readFile(PLUGIN_SITE_TEMPLATE_PATH, "utf-8");
     } catch {
     }
     const content = tpl.replace(/__PLUGIN_ID__/g, id);
@@ -3134,7 +2822,6 @@ function registerPluginsRoutes(app) {
       kind: "site",
       id: s.id,
       listUrlPattern: typeof s.listUrlPattern === "string" ? s.listUrlPattern : String(s.listUrlPattern),
-      hasEnrich: !!s.enrichItem,
       hasAuth: !!(s.checkAuth && s.loginUrl)
     }));
     const siteIds = new Set(sites.map((p) => p.id));
@@ -3142,7 +2829,6 @@ function registerPluginsRoutes(app) {
       kind: "source",
       id: src.id,
       listUrlPattern: typeof src.pattern === "string" ? src.pattern : String(src.pattern),
-      hasEnrich: !!src.enrichItem,
       hasAuth: false
     }));
     return c.json([...sites, ...sources]);
@@ -3654,7 +3340,6 @@ function registerTasksRoutes(app) {
 function registerApiRoutes(app) {
   registerServerRoutes(app);
   registerRssApiRoutes(app);
-  registerEnrichRoutes(app);
   registerSchedulerRoutes(app);
   registerPluginsRoutes(app);
   registerPipelineRoutes(app);
@@ -3756,7 +3441,7 @@ function registerAdminRoutes(app) {
       const headlessParam = c.req.query("headless");
       const headless = headlessParam === "false" || headlessParam === "0" ? false : void 0;
       const source = getSource(url);
-      const ctx = { cacheDir: CACHE_DIR, headless, proxy: source.proxy };
+      const ctx = buildSourceContext({ cacheDir: CACHE_DIR, headless, proxy: source.proxy });
       const items = await source.fetchItems(url, ctx);
       const mode = source.id === "generic" ? "generic" : "plugin";
       return c.json({ items, url, mode, pluginId: source.id });
@@ -3776,18 +3461,6 @@ function registerAdminRoutes(app) {
       const headlessParam = c.req.query("headless");
       const headless = headlessParam === "false" || headlessParam === "0" ? false : void 0;
       const site = getBestSite(url);
-      if (site?.enrichItem) {
-        const siteCtx = buildSiteContext(site, { cacheDir: CACHE_DIR, headless });
-        const stub = { guid: url, title: "", link: url, pubDate: /* @__PURE__ */ new Date() };
-        const enriched = await site.enrichItem(stub, siteCtx);
-        return c.json({
-          title: enriched.title ?? null,
-          author: enriched.author ?? null,
-          pubDate: enriched.pubDate instanceof Date ? enriched.pubDate.toISOString() : enriched.pubDate ?? null,
-          content: enriched.content ?? null,
-          _extractor: site.id
-        });
-      }
       const proxy = site?.proxy;
       const result = await extractFromLink(url, {}, { timeoutMs: 6e4, headless, proxy });
       return c.json({