rssany 0.1.0

package/package.json

@@ -0,0 +1,98 @@
+{
+  "name": "rssany",
+  "version": "0.1.0",
+  "description": "Universal RSS/Atom/JSON Feed pipeline — fetches, extracts, parses and converts any web content into consumable feeds with plugin support",
+  "author": "Joo",
+  "type": "module",
+  "main": "./dist/index.js",
+  "exports": {
+    ".": "./dist/index.js"
+  },
+  "bin": {
+    "rssany": "./bin/rssany.js"
+  },
+  "files": [
+    "dist",
+    "bin",
+    "plugins",
+    "statics",
+    "webui/build",
+    ".env.example",
+    "README.md",
+    "sources.example.json",
+    "config.examples.json",
+    "scripts/reset.mjs"
+  ],
+  "engines": {
+    "node": ">=20 <24"
+  },
+  "scripts": {
+    "build": "vite build",
+    "dev": "cross-env NODE_ENV=development NODE_OPTIONS=--no-deprecation tsx app/index.ts",
+    "dev:api": "cross-env NODE_ENV=development NODE_OPTIONS=--no-deprecation tsx app/index.ts",
+    "dev:all": "concurrently -k -n api,web -c blue,magenta \"pnpm dev\" \"pnpm webui:watch\"",
+    "start": "node dist/index.js",
+    "serve:route": "node scripts/serve-route.mjs",
+    "serve:app": "npx tsx app/index.ts",
+    "test": "vitest",
+    "test:run": "vitest run",
+    "lint": "eslint .",
+    "lint:fix": "eslint . --fix",
+    "typecheck": "tsc --noEmit",
+    "reset": "node scripts/reset.mjs",
+    "webui:install": "cd webui && npm install",
+    "webui:dev": "cd webui && pnpm run dev",
+    "webui:build": "cd webui && npm run build",
+    "webui:watch": "cd webui && npm run build:watch",
+    "build:all": "npm run build && npm run webui:build",
+    "prepublishOnly": "npm run build:all",
+    "docker:build": "bash scripts/docker-build.sh",
+    "docker:build:tag": "bash scripts/docker-build.sh"
+  },
+  "keywords": [
+    "rss",
+    "atom",
+    "json-feed",
+    "subscription",
+    "pipeline"
+  ],
+  "license": "MIT",
+  "devDependencies": {
+    "@eslint/js": "^9.15.0",
+    "@types/jsdom": "^21.1.7",
+    "@types/mailparser": "^3.4.6",
+    "@types/better-sqlite3": "^7.6.13",
+    "@types/node": "^25.2.0",
+    "@types/node-cron": "^3.0.11",
+    "@types/nodemailer": "^7.0.11",
+    "concurrently": "^9.2.1",
+    "cross-env": "^7.0.3",
+    "eslint": "^9.15.0",
+    "globals": "^15.12.0",
+    "tsx": "^4.19.0",
+    "typescript": "~5.6.0",
+    "typescript-eslint": "^8.15.0",
+    "vite": "^6.0.0",
+    "vitest": "^2.1.0"
+  },
+  "dependencies": {
+    "@hono/node-server": "^1.13.0",
+    "@mozilla/readability": "^0.6.0",
+    "better-sqlite3": "^12.6.2",
+    "cron-parser": "^5.0.0",
+    "dotenv": "^16.4.7",
+    "hono": "^4.6.0",
+    "https-proxy-agent": "^7.0.6",
+    "imapflow": "^1.2.10",
+    "jsdom": "^25.0.0",
+    "mailparser": "^3.9.3",
+    "marked": "^17.0.3",
+    "node-cron": "^3.0.3",
+    "node-html-parser": "^7.0.2",
+    "nodemailer": "^8.0.2",
+    "openai": "^4.76.1",
+    "puppeteer-core": "^24.36.0",
+    "rss-parser": "^3.13.0",
+    "zod": "^4.3.6"
+  }
+}

package/plugins/sources/email.rssany.js

@@ -0,0 +1,96 @@
+// 内置 IMAP 邮件插件：匹配 imap://、imaps:// 协议 URL
+
+import { ImapFlow } from "imapflow";
+import { logger } from "../../app/core/logger/index.js";
+import { simpleParser } from "mailparser";
+import { createHash } from "node:crypto";
+
+function parseImapUrl(sourceId) {
+  const url = new URL(sourceId);
+  const host = url.hostname;
+  const port = url.port ? parseInt(url.port, 10) : 993;
+  const secure = url.protocol === "imaps:" || port === 993;
+  const user = decodeURIComponent(url.username);
+  const pass = decodeURIComponent(url.password);
+  const folder = decodeURIComponent(url.pathname.slice(1)) || "INBOX";
+  const limit = Math.max(1, parseInt(url.searchParams.get("limit") ?? "30", 10));
+  return { host, port, secure, user, pass, folder, limit };
+}
+
+function makeGuid(messageId, uid, host) {
+  const raw = messageId ?? `${uid}@${host}`;
+  return createHash("sha256").update(raw).digest("hex");
+}
+
+export default {
+  id: "__email__",
+  pattern: /^imaps?:\/\//,
+  priority: 0,
+  refreshInterval: "30min",
+  async fetchItems(sourceId, _ctx) {
+    const { host, port, secure, user, pass, folder, limit } = parseImapUrl(sourceId);
+    const client = new ImapFlow({
+      host,
+      port,
+      secure,
+      auth: { user, pass },
+      logger: false,
+    });
+
+    client.on("error", (err) => {
+      logger.error("source", "IMAP 连接异常", { err: err?.message, host, folder });
+    });
+
+    const items = [];
+    let connected = false;
+    try {
+      await client.connect();
+      connected = true;
+      const lock = await client.getMailboxLock(folder);
+      try {
+        const mailbox = client.mailbox;
+        if (mailbox === false) return [];
+        const total = mailbox.exists ?? 0;
+        if (total === 0) return [];
+        const start = Math.max(1, total - limit + 1);
+        for await (const msg of client.fetch(`${start}:*`, { source: true, envelope: true })) {
+          try {
+            if (msg.source === undefined || msg.envelope === undefined) continue;
+            const parsed = await simpleParser(msg.source);
+            const envelope = msg.envelope;
+            const guid = makeGuid(envelope.messageId, msg.uid, host);
+            const title = parsed.subject ?? envelope.subject ?? "(无主题)";
+            const fromAddr = envelope.from?.[0];
+            const authorRaw = fromAddr?.name || fromAddr?.address || undefined;
+            const author = authorRaw ? [authorRaw] : undefined;
+            const pubDate = parsed.date ?? envelope.date ?? new Date();
+            const link = `imap://${host}/${encodeURIComponent(folder)}#${msg.uid}`;
+            const htmlBody = typeof parsed.html === "string" ? parsed.html : undefined;
+            const textBody = typeof parsed.text === "string" ? parsed.text : undefined;
+            const content = htmlBody ?? (textBody ? `<pre>${textBody}</pre>` : undefined);
+            const summary = textBody?.slice(0, 300) || undefined;
+            items.push({ guid, title, link, pubDate, author, summary, content });
+          } catch (err) {
+            logger.warn("source", "解析单封邮件失败", { err: err?.message });
+          }
+        }
+      } finally {
+        lock.release();
+      }
+    } catch (err) {
+      logger.warn("source", "拉取 IMAP 邮件失败", { err: err?.message, host, folder });
+      return [];
+    } finally {
+      if (connected && client.usable) {
+        try {
+          await client.logout();
+        } catch (err) {
+          logger.warn("source", "IMAP 退出连接失败", { err: err?.message, host, folder });
+        }
+      } else {
+        client.close();
+      }
+    }
+    return items.sort((a, b) => b.pubDate.getTime() - a.pubDate.getTime());
+  },
+};

package/plugins/sources/rss.rssany.js

@@ -0,0 +1,83 @@
+// 内置 RSS/Atom/JSON Feed 插件：匹配 *rss*、*atom*、*.xml 等标准 Feed URL
+
+import Parser from "rss-parser";
+import { createHash } from "node:crypto";
+
+const UA = "RssAny/1.0 (+https://github.com/joohw/rssany)";
+const parser = new Parser({
+  timeout: 15_000,
+  headers: {
+    "User-Agent": UA,
+    Accept: "application/rss+xml,application/atom+xml,application/json,application/xml,text/xml,*/*",
+  },
+});
+
+function looksLikeFeed(url) {
+  const lower = url.toLowerCase();
+  return (
+    lower.includes("/feed") ||
+    lower.includes("/rss") ||
+    lower.includes("/atom") ||
+    lower.endsWith(".xml") ||
+    lower.endsWith(".rss") ||
+    lower.endsWith(".atom") ||
+    lower.includes("format=rss") ||
+    lower.includes("format=atom") ||
+    lower.includes("output=rss")
+  );
+}
+
+async function fetchFeed(url, proxy) {
+  const proxyToUse = proxy ?? process.env.HTTP_PROXY ?? process.env.HTTPS_PROXY;
+  if (proxyToUse) {
+    const { HttpsProxyAgent } = await import("https-proxy-agent");
+    const agent = new HttpsProxyAgent(proxyToUse);
+    const parserWithProxy = new Parser({
+      timeout: 15_000,
+      headers: {
+        "User-Agent": UA,
+        Accept: "application/rss+xml,application/atom+xml,application/json,application/xml,text/xml,*/*",
+      },
+      requestOptions: { agent },
+    });
+    return parserWithProxy.parseURL(url);
+  }
+  return parser.parseURL(url);
+}
+
+export default {
+  id: "__rss__",
+  pattern: /^https?:\/\//,
+  match: looksLikeFeed,
+  priority: 20,
+  refreshInterval: "1h",
+  async fetchItems(sourceId, ctx) {
+    const feed = await fetchFeed(sourceId, ctx.proxy);
+    return (feed.items ?? []).map((item) => {
+      const link = item.link ?? item.guid ?? sourceId;
+      const guid = item.guid ?? createHash("sha256").update(link).digest("hex");
+      const pubDate =
+        item.pubDate != null
+          ? new Date(item.pubDate)
+          : item.isoDate != null
+            ? new Date(item.isoDate)
+            : new Date();
+      const authorRaw =
+        typeof item.creator === "string" ? item.creator : typeof item.author === "string" ? item.author : undefined;
+      const author = authorRaw ? [authorRaw] : undefined;
+      const summary =
+        typeof item.summary === "string" ? item.summary : typeof item.contentSnippet === "string" ? item.contentSnippet : undefined;
+      const content =
+        typeof item.content === "string" ? item.content : typeof item["content:encoded"] === "string" ? item["content:encoded"] : undefined;
+      return {
+        guid,
+        title: item.title ?? "",
+        link,
+        pubDate,
+        author,
+        summary,
+        content,
+      };
+    });
+  },
+};

package/plugins/templates/site.rssany.js

@@ -0,0 +1,26 @@
+/**
+ * Site 插件模板（管理页「添加插件」会复制到 `.rssany/plugins/sources/{id}.rssany.ts`）
+ * 修改 `id` 后请与文件名保持一致。
+ *
+ * 接口说明：app/scraper/sources/web/site.ts
+ */
+
+export default {
+  id: "__PLUGIN_ID__",
+  listUrlPattern: "https://example.com/{segment}",
+  refreshInterval: "1day",
+
+  /** sourceId 与订阅里 ref 一致；ctx 含 fetchHtml、extractItem 等 */
+  async fetchItems(sourceId, ctx) {
+    const { html, finalUrl } = await ctx.fetchHtml(sourceId, {
+      waitMs: 2000,
+      purify: true,
+    });
+    void html;
+    void finalUrl;
+    // TODO: 解析列表页 HTML，产出 { title, link, summary?, pubDate? } 等 FeedItem
+    return [];
+  },
+
+  // enrichItem: async (item, ctx) => ctx.extractItem(item),
+};

package/scripts/reset.mjs

@@ -0,0 +1,136 @@
+#!/usr/bin/env node
+/**
+ * 停止占用 HTTP 服务端口的进程，并删除用户数据目录（与 app 中 PORT / RSSANY_USER_DIR 约定一致）。
+ * 用法：pnpm reset  或  PORT=3000 pnpm reset
+ */
+
+import { execFileSync } from "node:child_process";
+import { existsSync, rmSync } from "node:fs";
+import { homedir } from "node:os";
+import { join } from "node:path";
+import { config } from "dotenv";
+
+config({ path: join(process.cwd(), ".env") });
+
+const DEFAULT_PORT = 18473;
+const port = Number(process.env.PORT) || DEFAULT_PORT;
+const userDirRaw = process.env.RSSANY_USER_DIR?.trim();
+const userDir =
+  userDirRaw && userDirRaw.length > 0 ? userDirRaw : join(homedir(), ".rssany");
+
+function pidsListeningWin32(p) {
+  const cmd = `Get-NetTCPConnection -LocalPort ${p} -State Listen -ErrorAction SilentlyContinue | ForEach-Object { $_.OwningProcess } | Sort-Object -Unique`;
+  try {
+    const out = execFileSync("powershell.exe", ["-NoProfile", "-Command", cmd], {
+      encoding: "utf8",
+      windowsHide: true,
+    });
+    return [
+      ...new Set(
+        out
+          .trim()
+          .split(/\r?\n/)
+          .map((s) => s.trim())
+          .filter(Boolean)
+          .map(Number)
+          .filter((n) => Number.isFinite(n) && n > 0),
+      ),
+    ];
+  } catch {
+    return [];
+  }
+}
+
+function pidsListeningNetstatWin32(p) {
+  const pids = new Set();
+  try {
+    const out = execFileSync("netstat", ["-ano"], { encoding: "utf8", windowsHide: true });
+    const needle = `:${p}`;
+    for (const line of out.split(/\r?\n/)) {
+      if (!line.includes("LISTENING") || !line.includes(needle)) continue;
+      const m = line.trim().match(/LISTENING\s+(\d+)\s*$/);
+      if (m) pids.add(Number(m[1]));
+    }
+  } catch {
+    // ignore
+  }
+  return [...pids];
+}
+
+function pidsListeningUnix(p) {
+  const argsList = [
+    ["-nP", `-iTCP:${p}`, "-sTCP:LISTEN", "-t"],
+    ["-ti", `:${p}`],
+  ];
+  for (const args of argsList) {
+    try {
+      const out = execFileSync("lsof", args, { encoding: "utf8" });
+      const pids = out
+        .trim()
+        .split(/\n/)
+        .map((s) => Number(s.trim()))
+        .filter((n) => Number.isFinite(n) && n > 0);
+      if (pids.length) return [...new Set(pids)];
+    } catch {
+      // try next
+    }
+  }
+  return [];
+}
+
+function killPidsWin32(pids) {
+  for (const pid of pids) {
+    if (pid === process.pid) continue;
+    try {
+      execFileSync("taskkill", ["/F", "/PID", String(pid)], { stdio: "inherit", windowsHide: true });
+      console.log(`已结束进程 PID ${pid}`);
+    } catch {
+      console.warn(`无法结束进程 PID ${pid}（可能已退出）`);
+    }
+  }
+}
+
+function killPidsUnix(pids) {
+  for (const pid of pids) {
+    if (pid === process.pid) continue;
+    try {
+      process.kill(pid, "SIGTERM");
+      console.log(`已发送 SIGTERM 至 PID ${pid}`);
+    } catch {
+      try {
+        process.kill(pid, "SIGKILL");
+        console.log(`已发送 SIGKILL 至 PID ${pid}`);
+      } catch {
+        console.warn(`无法结束进程 PID ${pid}`);
+      }
+    }
+  }
+}
+
+function main() {
+  console.log(`端口: ${port}（来自 PORT 或默认 ${DEFAULT_PORT}）`);
+  console.log(`用户数据目录: ${userDir}`);
+
+  let pids =
+    process.platform === "win32" ? pidsListeningWin32(port) : pidsListeningUnix(port);
+  if (process.platform === "win32" && pids.length === 0) {
+    pids = pidsListeningNetstatWin32(port);
+  }
+
+  if (pids.length === 0) {
+    console.log("未发现占用该端口的监听进程。");
+  } else {
+    console.log(`将结束占用端口的进程: ${pids.join(", ")}`);
+    if (process.platform === "win32") killPidsWin32(pids);
+    else killPidsUnix(pids);
+  }
+
+  if (!existsSync(userDir)) {
+    console.log("用户数据目录不存在，跳过删除。");
+    return;
+  }
+  rmSync(userDir, { recursive: true, force: true });
+  console.log("已删除用户数据目录。");
+}
+
+main();