rssany 0.1.0 → 0.1.1

package/README.md

@@ -10,8 +10,8 @@
 ## 功能概览
 
 - **统一订阅**：在 `.rssany/sources.json` 中配置网站列表、标准 RSS、IMAP 邮件等，由调度器按 `refresh` 策略拉取。
-- **可插拔信源**：`plugins/sources/` 与 `.rssany/plugins/sources/` 中的 **Site** 插件（`.rssany.js` / `.rssany.ts`），自定义列表解析与详情规则。
-- **正文补全**：可选 **enrich** 插件拉全文；无 enrich 时入库后仍会跑 pipeline。
+- **可插拔信源**：`app/plugins/builtin/` 与 `.rssany/plugins/` 中的 **Site** 插件（`.rssany.js` / `.rssany.ts`），自定义列表解析与详情规则。
+- **正文与解析**：在信源 `fetchItems`（及需要的 `ctx.extractItem` 等）内完成；入库后跑 pipeline。
 - **固定 pipeline**：`app/pipeline/` 中打标签、翻译等，由 `.rssany/config.json` 的 `pipeline.steps` 开关（**不是**用户目录下的 pipeline 插件）。
 - **LLM 辅助**：解析、提取、标签、翻译等可按配置走 OpenAI 兼容接口。
 - **站点登录**：需登录的站点通过 Puppeteer 管理 Cookie（与产品用户账号无关）。
@@ -99,7 +99,7 @@ rssany
 
 重置数据（结束 `PORT` 监听进程并删除用户目录）：**`rssany reset`**（与仓库内 **`pnpm reset`** 相同逻辑；可在含 `.env` 的目录下执行以读取 `PORT` / `RSSANY_USER_DIR`）。
 
-用户数据在 **`~/.rssany/`**（Windows：`%USERPROFILE%\.rssany`），与工作目录无关。可选环境变量 **`RSSANY_USER_DIR`** 可指定其它路径。等价于 `node node_modules/rssany/dist/index.js`；CLI 名称为 `rssany`。内置 `plugins/`、`statics/`、`webui/build` 随包安装路径解析。
+用户数据在 **`~/.rssany/`**（Windows：`%USERPROFILE%\.rssany`），与工作目录无关。可选环境变量 **`RSSANY_USER_DIR`** 可指定其它路径。等价于 `node node_modules/rssany/dist/index.js`；CLI 名称为 `rssany`。内置 `app/plugins/builtin/`、`statics/`、`webui/build` 随包安装路径解析。
 
 ---
 
@@ -109,7 +109,6 @@ rssany
 sources.json / Site 插件
   → 调度器触发 fetchItems
   → upsertItems
-  → [可选] enrich 队列
   → pipeline（每条一次）
   → [可选] deliver.url POST（出站，非入站 API）
 ```
@@ -131,11 +130,7 @@ sources.json / Site 插件
 
 ### 信源插件（Site）
 
-放置于 `**plugins/sources/`** 或 `**.rssany/plugins/sources/**`，用户插件可与内置插件同 `id` 覆盖。最小约定包括 `id`、`listUrlPattern` 等（详见 `app/scraper/sources/web/site.ts`）。
-
-### Enrich 插件
-
-`**plugins/enrich/**`、`**.rssany/plugins/enrich/**`，按 enrich 管线加载。
+放置于 `**app/plugins/builtin/**` 或 `**.rssany/plugins/**`（扁平），用户插件可与内置插件同 `id` 覆盖。最小约定包括 `id`、`listUrlPattern` 等（详见 `app/scraper/sources/web/site.ts`）。
 
 ### Pipeline（固定代码）
 
@@ -175,7 +170,7 @@ sources.json / Site 插件
 
 ```
 ├── app/                 # 后端：路由、feeder、scraper、pipeline、mcp、db、auth…
-├── plugins/             # 内置信源 / enrich 等插件
+│   └── plugins/builtin/ # 内置信源 *.rssany.js
 └── webui/               # SvelteKit 前端
 
 ~/.rssany/               # 运行时用户数据（首次启动创建；或 RSSANY_USER_DIR）

package/{plugins/sources → app/plugins/builtin}/email.rssany.js

@@ -1,96 +1,92 @@
-// 内置 IMAP 邮件插件：匹配 imap://、imaps:// 协议 URL
-
-import { ImapFlow } from "imapflow";
-import { logger } from "../../app/core/logger/index.js";
-import { simpleParser } from "mailparser";
-import { createHash } from "node:crypto";
-
-function parseImapUrl(sourceId) {
-  const url = new URL(sourceId);
-  const host = url.hostname;
-  const port = url.port ? parseInt(url.port, 10) : 993;
-  const secure = url.protocol === "imaps:" || port === 993;
-  const user = decodeURIComponent(url.username);
-  const pass = decodeURIComponent(url.password);
-  const folder = decodeURIComponent(url.pathname.slice(1)) || "INBOX";
-  const limit = Math.max(1, parseInt(url.searchParams.get("limit") ?? "30", 10));
-  return { host, port, secure, user, pass, folder, limit };
-}
-
-function makeGuid(messageId, uid, host) {
-  const raw = messageId ?? `${uid}@${host}`;
-  return createHash("sha256").update(raw).digest("hex");
-}
-
-export default {
-  id: "__email__",
-  pattern: /^imaps?:\/\//,
-  priority: 0,
-  refreshInterval: "30min",
-  async fetchItems(sourceId, _ctx) {
-    const { host, port, secure, user, pass, folder, limit } = parseImapUrl(sourceId);
-    const client = new ImapFlow({
-      host,
-      port,
-      secure,
-      auth: { user, pass },
-      logger: false,
-    });
-
-    client.on("error", (err) => {
-      logger.error("source", "IMAP 连接异常", { err: err?.message, host, folder });
-    });
-
-    const items = [];
-    let connected = false;
-    try {
-      await client.connect();
-      connected = true;
-      const lock = await client.getMailboxLock(folder);
-      try {
-        const mailbox = client.mailbox;
-        if (mailbox === false) return [];
-        const total = mailbox.exists ?? 0;
-        if (total === 0) return [];
-        const start = Math.max(1, total - limit + 1);
-        for await (const msg of client.fetch(`${start}:*`, { source: true, envelope: true })) {
-          try {
-            if (msg.source === undefined || msg.envelope === undefined) continue;
-            const parsed = await simpleParser(msg.source);
-            const envelope = msg.envelope;
-            const guid = makeGuid(envelope.messageId, msg.uid, host);
-            const title = parsed.subject ?? envelope.subject ?? "(无主题)";
-            const fromAddr = envelope.from?.[0];
-            const authorRaw = fromAddr?.name || fromAddr?.address || undefined;
-            const author = authorRaw ? [authorRaw] : undefined;
-            const pubDate = parsed.date ?? envelope.date ?? new Date();
-            const link = `imap://${host}/${encodeURIComponent(folder)}#${msg.uid}`;
-            const htmlBody = typeof parsed.html === "string" ? parsed.html : undefined;
-            const textBody = typeof parsed.text === "string" ? parsed.text : undefined;
-            const content = htmlBody ?? (textBody ? `<pre>${textBody}</pre>` : undefined);
-            const summary = textBody?.slice(0, 300) || undefined;
-            items.push({ guid, title, link, pubDate, author, summary, content });
-          } catch (err) {
-            logger.warn("source", "解析单封邮件失败", { err: err?.message });
-          }
-        }
-      } finally {
-        lock.release();
-      }
-    } catch (err) {
-      logger.warn("source", "拉取 IMAP 邮件失败", { err: err?.message, host, folder });
-      return [];
-    } finally {
-      if (connected && client.usable) {
-        try {
-          await client.logout();
-        } catch (err) {
-          logger.warn("source", "IMAP 退出连接失败", { err: err?.message, host, folder });
-        }
-      } else {
-        client.close();
-      }
-    }
-    return items.sort((a, b) => b.pubDate.getTime() - a.pubDate.getTime());
-  },
-};
+// 内置 IMAP 邮件插件：匹配 imap://、imaps:// 协议 URL
+
+function parseImapUrl(sourceId) {
+  const url = new URL(sourceId);
+  const host = url.hostname;
+  const port = url.port ? parseInt(url.port, 10) : 993;
+  const secure = url.protocol === "imaps:" || port === 993;
+  const user = decodeURIComponent(url.username);
+  const pass = decodeURIComponent(url.password);
+  const folder = decodeURIComponent(url.pathname.slice(1)) || "INBOX";
+  const limit = Math.max(1, parseInt(url.searchParams.get("limit") ?? "30", 10));
+  return { host, port, secure, user, pass, folder, limit };
+}
+
+function makeGuid(messageId, uid, host, createHash) {
+  const raw = messageId ?? `${uid}@${host}`;
+  return createHash("sha256").update(raw).digest("hex");
+}
+
+export default {
+  id: "__email__",
+  pattern: /^imaps?:\/\//,
+  priority: 0,
+  refreshInterval: "30min",
+  async fetchItems(sourceId, ctx) {
+    const { deps } = ctx;
+    const { host, port, secure, user, pass, folder, limit } = parseImapUrl(sourceId);
+    const client = new deps.ImapFlow({
+      host,
+      port,
+      secure,
+      auth: { user, pass },
+      logger: false,
+    });
+
+    client.on("error", (err) => {
+      deps.logger.error("source", "IMAP 连接异常", { err: err?.message, host, folder });
+    });
+
+    const items = [];
+    let connected = false;
+    try {
+      await client.connect();
+      connected = true;
+      const lock = await client.getMailboxLock(folder);
+      try {
+        const mailbox = client.mailbox;
+        if (mailbox === false) return [];
+        const total = mailbox.exists ?? 0;
+        if (total === 0) return [];
+        const start = Math.max(1, total - limit + 1);
+        for await (const msg of client.fetch(`${start}:*`, { source: true, envelope: true })) {
+          try {
+            if (msg.source === undefined || msg.envelope === undefined) continue;
+            const parsed = await deps.simpleParser(msg.source);
+            const envelope = msg.envelope;
+            const guid = makeGuid(envelope.messageId, msg.uid, host, deps.createHash);
+            const title = parsed.subject ?? envelope.subject ?? "(无主题)";
+            const fromAddr = envelope.from?.[0];
+            const authorRaw = fromAddr?.name || fromAddr?.address || undefined;
+            const author = authorRaw ? [authorRaw] : undefined;
+            const pubDate = parsed.date ?? envelope.date ?? new Date();
+            const link = `imap://${host}/${encodeURIComponent(folder)}#${msg.uid}`;
+            const htmlBody = typeof parsed.html === "string" ? parsed.html : undefined;
+            const textBody = typeof parsed.text === "string" ? parsed.text : undefined;
+            const content = htmlBody ?? (textBody ? `<pre>${textBody}</pre>` : undefined);
+            const summary = textBody?.slice(0, 300) || undefined;
+            items.push({ guid, title, link, pubDate, author, summary, content });
+          } catch (err) {
+            deps.logger.warn("source", "解析单封邮件失败", { err: err?.message });
+          }
+        }
+      } finally {
+        lock.release();
+      }
+    } catch (err) {
+      deps.logger.warn("source", "拉取 IMAP 邮件失败", { err: err?.message, host, folder });
+      return [];
+    } finally {
+      if (connected && client.usable) {
+        try {
+          await client.logout();
+        } catch (err) {
+          deps.logger.warn("source", "IMAP 退出连接失败", { err: err?.message, host, folder });
+        }
+      } else {
+        client.close();
+      }
+    }
+    return items.sort((a, b) => b.pubDate.getTime() - a.pubDate.getTime());
+  },
+};

package/{plugins/sources → app/plugins/builtin}/rss.rssany.js

@@ -1,38 +1,13 @@
 // 内置 RSS/Atom/JSON Feed 插件：匹配 *rss*、*atom*、*.xml 等标准 Feed URL
 
-import Parser from "rss-parser";
-import { createHash } from "node:crypto";
-
 const UA = "RssAny/1.0 (+https://github.com/joohw/rssany)";
-const parser = new Parser({
-  timeout: 15_000,
-  headers: {
-    "User-Agent": UA,
-    Accept: "application/rss+xml,application/atom+xml,application/json,application/xml,text/xml,*/*",
-  },
-});
-
-function looksLikeFeed(url) {
-  const lower = url.toLowerCase();
-  return (
-    lower.includes("/feed") ||
-    lower.includes("/rss") ||
-    lower.includes("/atom") ||
-    lower.endsWith(".xml") ||
-    lower.endsWith(".rss") ||
-    lower.endsWith(".atom") ||
-    lower.includes("format=rss") ||
-    lower.includes("format=atom") ||
-    lower.includes("output=rss")
-  );
-}
 
-async function fetchFeed(url, proxy) {
-  const proxyToUse = proxy ?? process.env.HTTP_PROXY ?? process.env.HTTPS_PROXY;
+async function fetchFeed(url, ctx) {
+  const { deps } = ctx;
+  const proxyToUse = ctx.proxy ?? process.env.HTTP_PROXY ?? process.env.HTTPS_PROXY;
   if (proxyToUse) {
-    const { HttpsProxyAgent } = await import("https-proxy-agent");
-    const agent = new HttpsProxyAgent(proxyToUse);
-    const parserWithProxy = new Parser({
+    const agent = new deps.HttpsProxyAgent(proxyToUse);
+    const parserWithProxy = new deps.RssParser({
       timeout: 15_000,
       headers: {
         "User-Agent": UA,
@@ -42,6 +17,13 @@ async function fetchFeed(url, proxy) {
     });
     return parserWithProxy.parseURL(url);
   }
+  const parser = new deps.RssParser({
+    timeout: 15_000,
+    headers: {
+      "User-Agent": UA,
+      Accept: "application/rss+xml,application/atom+xml,application/json,application/xml,text/xml,*/*",
+    },
+  });
   return parser.parseURL(url);
 }
 
@@ -52,10 +34,11 @@ export default {
   priority: 20,
   refreshInterval: "1h",
   async fetchItems(sourceId, ctx) {
-    const feed = await fetchFeed(sourceId, ctx.proxy);
+    const { deps } = ctx;
+    const feed = await fetchFeed(sourceId, ctx);
     return (feed.items ?? []).map((item) => {
       const link = item.link ?? item.guid ?? sourceId;
-      const guid = item.guid ?? createHash("sha256").update(link).digest("hex");
+      const guid = item.guid ?? deps.createHash("sha256").update(link).digest("hex");
       const pubDate =
         item.pubDate != null
           ? new Date(item.pubDate)
@@ -81,3 +64,18 @@ export default {
     });
   },
 };
+
+function looksLikeFeed(url) {
+  const lower = url.toLowerCase();
+  return (
+    lower.includes("/feed") ||
+    lower.includes("/rss") ||
+    lower.includes("/atom") ||
+    lower.endsWith(".xml") ||
+    lower.endsWith(".rss") ||
+    lower.endsWith(".atom") ||
+    lower.includes("format=rss") ||
+    lower.includes("format=atom") ||
+    lower.includes("output=rss")
+  );
+}

package/{plugins/templates → app/plugins}/site.rssany.js

@@ -1,5 +1,5 @@
 /**
- * Site 插件模板（管理页「添加插件」会复制到 `.rssany/plugins/sources/{id}.rssany.ts`）
+ * Site 插件模板（管理页「添加插件」会复制到 `.rssany/plugins/{id}.rssany.js`）
  * 修改 `id` 后请与文件名保持一致。
  *
  * 接口说明：app/scraper/sources/web/site.ts
@@ -10,17 +10,16 @@ export default {
   listUrlPattern: "https://example.com/{segment}",
   refreshInterval: "1day",
 
-  /** sourceId 与订阅里 ref 一致；ctx 含 fetchHtml、extractItem 等 */
+  /** sourceId 与订阅里 ref 一致；ctx 含 fetchHtml、extractItem、deps（parseHtml 等） */
   async fetchItems(sourceId, ctx) {
     const { html, finalUrl } = await ctx.fetchHtml(sourceId, {
       waitMs: 2000,
       purify: true,
     });
-    void html;
+    const root = ctx.deps.parseHtml(html);
+    void root;
     void finalUrl;
-    // TODO: 解析列表页 HTML，产出 { title, link, summary?, pubDate? } 等 FeedItem
+    // TODO: 用 ctx.deps.parseHtml 解析列表页，产出 { title, link, summary?, pubDate? } 等 FeedItem
     return [];
   },
-
-  // enrichItem: async (item, ctx) => ctx.extractItem(item),
 };