rssany 0.1.6 → 0.3.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "rssany",
-  "version": "0.1.6",
+  "version": "0.3.0",
   "description": "Universal RSS/Atom/JSON Feed pipeline — fetches, extracts, parses and converts any web content into consumable feeds with plugin support",
   "author": "Joo",
   "type": "module",
@@ -21,14 +21,16 @@
     ".env.example",
     "README.md",
     "init",
-    "scripts/reset.mjs"
+    "scripts/reset.mjs",
+    "scripts/dev.mjs"
   ],
   "engines": {
     "node": ">=20 <24"
   },
   "scripts": {
     "build": "vite build",
-    "dev": "tsx app/index.ts",
+    "dev": "node scripts/dev.mjs",
+    "dev:backend": "cross-env PORT=3999 tsx app/index.ts",
     "start": "node dist/index.js",
     "serve:route": "node scripts/serve-route.mjs",
     "serve:app": "npx tsx app/index.ts",
@@ -39,14 +41,20 @@
     "typecheck": "tsc --noEmit",
     "reset": "node scripts/reset.mjs",
     "proxy-browser": "tsx scripts/proxy-browser.ts",
-    "webui:install": "cd webui && npm install",
-    "webui:dev": "cd webui && pnpm run dev",
-    "webui:build": "cd webui && npm run build",
-    "webui:watch": "cd webui && npm run build:watch",
+    "webui:install": "cd app/webui && npm install",
+    "webui:dev": "cd app/webui && npm run build:watch",
+    "webui:build": "cd app/webui && npm run build",
+    "webui:watch": "cd app/webui && npm run build:watch",
+    "dev:all": "npm run dev",
     "build:all": "npm run build && npm run webui:build",
     "prepublishOnly": "npm run build:all",
     "docker:build": "bash scripts/docker-build.sh",
-    "docker:build:tag": "bash scripts/docker-build.sh"
+    "docker:build:tag": "bash scripts/docker-build.sh",
+    "landing:install": "cd landing && npm install",
+    "landing:dev": "cd landing && npm run dev",
+    "landing:build": "cd landing && npm run build",
+    "deploy": "node scripts/deploy-landing.mjs",
+    "deploy:landing": "node scripts/deploy-landing.mjs"
   },
   "keywords": [
     "rss",
@@ -60,7 +68,6 @@
     "@eslint/js": "^9.15.0",
     "@types/jsdom": "^21.1.7",
     "@types/mailparser": "^3.4.6",
-    "@types/better-sqlite3": "^7.6.13",
     "@types/node": "^25.2.0",
     "@types/node-cron": "^3.0.11",
     "@types/nodemailer": "^7.0.11",
@@ -68,19 +75,19 @@
     "cross-env": "^7.0.3",
     "eslint": "^9.15.0",
     "globals": "^15.12.0",
+    "ssh2": "^1.17.0",
     "tsx": "^4.19.0",
     "typescript": "~5.6.0",
     "typescript-eslint": "^8.15.0",
-    "vite": "^6.0.0",
+    "vite": "^6.4.2",
     "vitest": "^2.1.0"
   },
   "dependencies": {
-    "@hono/node-server": "^1.13.0",
+    "@hono/node-server": "^1.19.10",
     "@mozilla/readability": "^0.6.0",
-    "better-sqlite3": "^12.6.2",
     "cron-parser": "^5.0.0",
     "dotenv": "^16.4.7",
-    "hono": "^4.6.0",
+    "hono": "^4.12.12",
     "https-proxy-agent": "^7.0.6",
     "imapflow": "^1.2.10",
     "jsdom": "^25.0.0",

package/scripts/dev.mjs

@@ -0,0 +1,114 @@
+#!/usr/bin/env node
+import { spawn, execFileSync } from "node:child_process";
+import { existsSync, statSync } from "node:fs";
+import { join } from "node:path";
+
+const root = join(import.meta.dirname, "..");
+const npmCmd = "npm";
+const webuiMarker = join(root, "app", "webui", "build", "200.html");
+const initialMarkerMtime = existsSync(webuiMarker) ? statSync(webuiMarker).mtimeMs : 0;
+
+const children = new Set();
+let shuttingDown = false;
+
+function spawnNpm(args, opts = {}) {
+  const child = spawn(npmCmd, args, {
+    cwd: root,
+    env: process.env,
+    stdio: opts.stdio ?? ["ignore", "pipe", "pipe"],
+    // Windows: Node 20.12+ / 22+ reject spawning .cmd without shell (CVE-2024-27980).
+    shell: process.platform === "win32",
+  });
+  children.add(child);
+  child.once("exit", () => children.delete(child));
+  return child;
+}
+
+function stopProcessTree(child) {
+  if (!child || child.killed) return;
+  if (process.platform === "win32") {
+    try {
+      execFileSync("taskkill.exe", ["/pid", String(child.pid), "/t", "/f"], { stdio: "ignore" });
+      return;
+    } catch {
+      // Fall through to plain kill.
+    }
+  }
+  child.kill("SIGTERM");
+}
+
+function shutdown(code = 0) {
+  if (shuttingDown) return;
+  shuttingDown = true;
+  for (const child of Array.from(children)) stopProcessTree(child);
+  process.exitCode = code;
+}
+
+function pipeOutput(child, label) {
+  child.stdout?.on("data", (chunk) => process.stdout.write(`[${label}] ${chunk}`));
+  child.stderr?.on("data", (chunk) => process.stderr.write(`[${label}] ${chunk}`));
+}
+
+function waitForInitialWebuiBuild(child) {
+  return new Promise((resolve, reject) => {
+    let done = false;
+    const finish = () => {
+      if (done) return;
+      done = true;
+      clearInterval(poll);
+      resolve();
+    };
+    const fail = (err) => {
+      if (done) return;
+      done = true;
+      clearInterval(poll);
+      reject(err);
+    };
+    const hasFreshMarker = () => {
+      if (!existsSync(webuiMarker)) return false;
+      return statSync(webuiMarker).mtimeMs > initialMarkerMtime;
+    };
+    const poll = setInterval(() => {
+      if (hasFreshMarker()) finish();
+    }, 500);
+    child.stdout?.on("data", (chunk) => {
+      const text = chunk.toString();
+      if (text.includes('Wrote site to "build"') || text.includes("Wrote site to 'build'")) {
+        finish();
+      }
+    });
+    child.once("exit", (code) => {
+      if (!done) fail(new Error(`webui watch exited before initial build completed (code ${code ?? "unknown"})`));
+    });
+  });
+}
+
+process.on("SIGINT", () => shutdown(0));
+process.on("SIGTERM", () => shutdown(0));
+
+console.log("Starting WebUI static build watcher...");
+const webui = spawnNpm(["run", "webui:watch"]);
+pipeOutput(webui, "webui");
+
+try {
+  await waitForInitialWebuiBuild(webui);
+  console.log("WebUI initial build is ready. Starting backend server...");
+  const backend = spawnNpm(["run", "dev:backend"]);
+  pipeOutput(backend, "backend");
+
+  backend.once("exit", (code) => {
+    if (!shuttingDown) {
+      console.log(`Backend exited (code ${code ?? "unknown"}). Stopping WebUI watcher...`);
+      shutdown(code ?? 0);
+    }
+  });
+  webui.once("exit", (code) => {
+    if (!shuttingDown) {
+      console.log(`WebUI watcher exited (code ${code ?? "unknown"}). Stopping backend...`);
+      shutdown(code ?? 0);
+    }
+  });
+} catch (err) {
+  console.error(err instanceof Error ? err.message : String(err));
+  shutdown(1);
+}

package/scripts/reset.mjs

@@ -1,7 +1,7 @@
 #!/usr/bin/env node
 /**
  * 停止占用 HTTP 服务端口的进程，并删除用户数据目录（与 app 中 PORT / RSSANY_USER_DIR 约定一致）。
- * 用法：pnpm reset  或  PORT=3000 pnpm reset
+ * 用法：npm run reset  或  PORT=3000 npm run reset
  */
 
 import { execFileSync } from "node:child_process";

package/app/plugins/builtin/google.rssany.js

@@ -1,187 +0,0 @@
-let _deps;
-
-// Google Search 插件：将搜索结果页转换为 FeedItem 列表（不含 enrich）
-
-
-function normalizeText(text) {
-  return (text ?? "").replace(/\s+/g, " ").trim();
-}
-
-function isGoogleHost(hostname) {
-  return /^([a-z0-9-]+\.)*google\.[a-z.]+$/i.test(hostname);
-}
-
-function resolveResultLink(rawHref, pageUrl) {
-  if (!rawHref) return null;
-  const href = rawHref.trim();
-  if (!href || href.startsWith("#") || href.startsWith("javascript:")) return null;
-
-  let url;
-  try {
-    url = new URL(href, pageUrl);
-  } catch {
-    return null;
-  }
-
-  if (isGoogleHost(url.hostname) && url.pathname === "/url") {
-    const q = url.searchParams.get("q") ?? url.searchParams.get("url");
-    if (!q) return null;
-    try {
-      const target = new URL(q);
-      return /^https?:$/i.test(target.protocol) ? target.href : null;
-    } catch {
-      return null;
-    }
-  }
-
-  if (!/^https?:$/i.test(url.protocol)) return null;
-  if (isGoogleHost(url.hostname)) return null;
-  return url.href;
-}
-
-function closestAnchor(node) {
-  let cur = node;
-  while (cur) {
-    if (cur.tagName?.toLowerCase() === "a") return cur;
-    cur = cur.parentNode ?? null;
-  }
-  return null;
-}
-
-function extractSnippet(startNode) {
-  const snippetSelectors = [".VwiC3b", ".IsZvec", ".MUxGbd", ".lyLwlc"];
-  let cur = startNode;
-  for (let i = 0; i < 6 && cur; i += 1) {
-    for (const sel of snippetSelectors) {
-      const el = cur.querySelector?.(sel);
-      const text = normalizeText(el?.textContent);
-      if (text) return text;
-    }
-    cur = cur.parentNode ?? null;
-  }
-  return "";
-}
-
-function looksLikeBlockedPage(root, html, finalUrl) {
-  const text = normalizeText(root.textContent).toLowerCase();
-  const body = (html ?? "").toLowerCase();
-  if (finalUrl.includes("/sorry/")) return true;
-  if (root.querySelector("#captcha-form, #recaptcha, .g-recaptcha")) return true;
-  if (body.includes("/httpservice/retry/enablejs")) return true;
-  if (body.includes("id=\"yvlrue\"") || body.includes("if you're having trouble accessing google search")) return true;
-  if (body.includes("sg_rel")) return true;
-  return text.includes("about this page") || text.includes("unusual traffic") || text.includes("captcha");
-}
-
-async function checkAuth(page, _url) {
-  try {
-    const currentUrl = page.url() || "";
-    if (currentUrl.includes("/sorry/") || currentUrl.includes("/httpservice/retry/enablejs")) return false;
-
-    const blockedDom = await page.$("#captcha-form, #recaptcha, .g-recaptcha, #yvlrue");
-    if (blockedDom) return false;
-
-    const bodyText = await page.evaluate(() => (document.body?.innerText ?? "").toLowerCase());
-    if (
-      bodyText.includes("unusual traffic") ||
-      bodyText.includes("about this page") ||
-      bodyText.includes("captcha") ||
-      bodyText.includes("if you're having trouble accessing google search")
-    ) {
-      return false;
-    }
-
-    // 首页或搜索页出现 q 输入框即可认为当前会话可用；若被风控会在上面的分支提前返回 false。
-    const searchBox = await page.$('textarea[name="q"], input[name="q"]');
-    return !!searchBox;
-  } catch {
-    return false;
-  }
-}
-
-function parseFromResultBlocks(root, pageUrl) {
-  const resultBlocks = root.querySelectorAll("#rso .MjjYud .A6K0A, #rso .MjjYud .tF2Cxc");
-  const seen = new Set();
-  const items = [];
-
-  for (const block of resultBlocks) {
-    const anchor =
-      block.querySelector('a[jsname="UWckNb"][href]') ??
-      block.querySelector(".yuRUbf a[href]") ??
-      block.querySelector("a[href]");
-    const link = resolveResultLink(anchor?.getAttribute("href"), pageUrl);
-    if (!link || seen.has(link)) continue;
-
-    const titleNode =
-      block.querySelector("h3.LC20lb, h3.DKV0Md, h3.MBeuO, h3") ??
-      anchor?.querySelector?.("h3");
-    const title = normalizeText(titleNode?.textContent);
-    if (!title) continue;
-
-    seen.add(link);
-    const summary = extractSnippet(block) || extractSnippet(titleNode ?? block) || title;
-    items.push({
-      guid: _deps.createHash("sha256").update(link).digest("hex"),
-      title,
-      link,
-      pubDate: new Date(),
-      author: "Google Search",
-      summary,
-    });
-  }
-
-  return items;
-}
-
-function parseFromHeadingFallback(root, pageUrl) {
-  const seen = new Set();
-  const items = [];
-  const titles = root.querySelectorAll("h3");
-  for (const h3 of titles) {
-    const title = normalizeText(h3.textContent);
-    if (!title) continue;
-    const anchor = closestAnchor(h3);
-    const link = resolveResultLink(anchor?.getAttribute("href"), pageUrl);
-    if (!link || seen.has(link)) continue;
-    seen.add(link);
-    const summary = extractSnippet(h3) || title;
-    items.push({
-      guid: _deps.createHash("sha256").update(link).digest("hex"),
-      title,
-      link,
-      pubDate: new Date(),
-      author: "Google Search",
-      summary,
-    });
-  }
-  return items;
-}
-
-async function fetchItems(sourceId, ctx) {
-  _deps = ctx.deps;
-  const { html, finalUrl } = await ctx.fetchHtml(sourceId, { waitMs: 2500 });
-  const root = _deps.parseHtml(html);
-  const pageUrl = new URL(finalUrl);
-
-  const fromBlocks = parseFromResultBlocks(root, pageUrl);
-  const items = fromBlocks.length > 0 ? fromBlocks : parseFromHeadingFallback(root, pageUrl);
-
-  if (items.length === 0) {
-    if (looksLikeBlockedPage(root, html, finalUrl)) {
-      throw new Error("[google] 命中 Google 验证页（reCAPTCHA/风控），当前会话无法稳定抓取搜索结果");
-    }
-    throw new Error("[google] 未解析到搜索结果，页面结构可能已变化");
-  }
-  return items;
-}
-
-export default {
-  id: "google-search",
-  listUrlPattern: /^https?:\/\/(www\.)?google\.[^/]+\/search(\?.*)?$/i,
-  fetchItems,
-  checkAuth,
-  loginUrl: "https://www.google.com/",
-  domain: "google.com",
-  loginTimeoutMs: 5 * 60 * 1000,
-  pollIntervalMs: 2000,
-};

package/init/config.json

@@ -1,17 +0,0 @@
-{
-    "globalProxy": "",
-    "pipeline": {
-        "steps": [
-            { "id": "tagger", "enabled": false },
-            { "id": "translator", "enabled": false }
-        ]
-    },
-    "deliver": {
-        "url": "",
-        "token": ""
-    },
-    "llm": {
-        "baseUrl": "https://api.openai.com/v1",
-        "model": "gpt-4o-mini"
-    }
-}