rssany 0.2.0 → 0.3.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "rssany",
-  "version": "0.2.0",
+  "version": "0.3.1",
   "description": "Universal RSS/Atom/JSON Feed pipeline — fetches, extracts, parses and converts any web content into consumable feeds with plugin support",
   "author": "Joo",
   "type": "module",
@@ -17,18 +17,19 @@
     "app/plugins/builtin",
     "app/plugins/site.rssany.js",
     "statics",
-    "webui/build",
+    "app/webui/build",
     ".env.example",
     "README.md",
     "init",
-    "scripts/reset.mjs"
+    "scripts/reset.mjs",
+    "scripts/dev.mjs"
   ],
   "engines": {
     "node": ">=20 <24"
   },
   "scripts": {
     "build": "vite build",
-"dev": "concurrently -k \"npm run dev:backend\" \"npm run webui:dev\"",
+    "dev": "node scripts/dev.mjs",
     "dev:backend": "cross-env PORT=3999 tsx app/index.ts",
     "start": "node dist/index.js",
     "serve:route": "node scripts/serve-route.mjs",
@@ -40,15 +41,20 @@
     "typecheck": "tsc --noEmit",
     "reset": "node scripts/reset.mjs",
     "proxy-browser": "tsx scripts/proxy-browser.ts",
-    "webui:install": "cd webui && npm install",
-    "webui:dev": "cd webui && npm run dev",
-    "webui:build": "cd webui && npm run build",
-    "webui:watch": "cd webui && npm run build:watch",
-    "dev:all": "concurrently -k \"npm run dev\" \"npm run webui:dev\"",
+    "webui:install": "cd app/webui && npm install",
+    "webui:dev": "cd app/webui && npm run build:watch",
+    "webui:build": "cd app/webui && npm run build",
+    "webui:watch": "cd app/webui && npm run build:watch",
+    "dev:all": "npm run dev",
     "build:all": "npm run build && npm run webui:build",
     "prepublishOnly": "npm run build:all",
     "docker:build": "bash scripts/docker-build.sh",
-    "docker:build:tag": "bash scripts/docker-build.sh"
+    "docker:build:tag": "bash scripts/docker-build.sh",
+    "landing:install": "cd landing && npm install",
+    "landing:dev": "cd landing && npm run dev",
+    "landing:build": "cd landing && npm run build",
+    "deploy": "node scripts/deploy-landing.mjs",
+    "deploy:landing": "node scripts/deploy-landing.mjs"
   },
   "keywords": [
     "rss",
@@ -60,7 +66,7 @@
   "license": "MIT",
   "devDependencies": {
     "@eslint/js": "^9.15.0",
-    "@types/jsdom": "^21.1.7",
+    "@types/jsdom": "^28.0.3",
     "@types/mailparser": "^3.4.6",
     "@types/node": "^25.2.0",
     "@types/node-cron": "^3.0.11",
@@ -69,6 +75,7 @@
     "cross-env": "^7.0.3",
     "eslint": "^9.15.0",
     "globals": "^15.12.0",
+    "ssh2": "^1.17.0",
     "tsx": "^4.19.0",
     "typescript": "~5.6.0",
     "typescript-eslint": "^8.15.0",
@@ -83,15 +90,14 @@
     "hono": "^4.12.12",
     "https-proxy-agent": "^7.0.6",
     "imapflow": "^1.2.10",
-    "jsdom": "^25.0.0",
+    "jsdom": "^29.1.1",
     "mailparser": "^3.9.3",
     "marked": "^17.0.3",
-    "node-cron": "^3.0.3",
+    "node-cron": "^4.2.1",
     "node-html-parser": "^7.0.2",
     "nodemailer": "^8.0.2",
-    "openai": "^4.76.1",
+    "openai": "^6.42.0",
     "puppeteer-core": "^24.36.0",
-    "rss-parser": "^3.13.0",
-    "zod": "^4.3.6"
+    "rss-parser": "^3.13.0"
   }
 }

package/scripts/dev.mjs

@@ -0,0 +1,114 @@
+#!/usr/bin/env node
+import { spawn, execFileSync } from "node:child_process";
+import { existsSync, statSync } from "node:fs";
+import { join } from "node:path";
+
+const root = join(import.meta.dirname, "..");
+const npmCmd = "npm";
+const webuiMarker = join(root, "app", "webui", "build", "200.html");
+const initialMarkerMtime = existsSync(webuiMarker) ? statSync(webuiMarker).mtimeMs : 0;
+
+const children = new Set();
+let shuttingDown = false;
+
+function spawnNpm(args, opts = {}) {
+  const child = spawn(npmCmd, args, {
+    cwd: root,
+    env: process.env,
+    stdio: opts.stdio ?? ["ignore", "pipe", "pipe"],
+    // Windows: Node 20.12+ / 22+ reject spawning .cmd without shell (CVE-2024-27980).
+    shell: process.platform === "win32",
+  });
+  children.add(child);
+  child.once("exit", () => children.delete(child));
+  return child;
+}
+
+function stopProcessTree(child) {
+  if (!child || child.killed) return;
+  if (process.platform === "win32") {
+    try {
+      execFileSync("taskkill.exe", ["/pid", String(child.pid), "/t", "/f"], { stdio: "ignore" });
+      return;
+    } catch {
+      // Fall through to plain kill.
+    }
+  }
+  child.kill("SIGTERM");
+}
+
+function shutdown(code = 0) {
+  if (shuttingDown) return;
+  shuttingDown = true;
+  for (const child of Array.from(children)) stopProcessTree(child);
+  process.exitCode = code;
+}
+
+function pipeOutput(child, label) {
+  child.stdout?.on("data", (chunk) => process.stdout.write(`[${label}] ${chunk}`));
+  child.stderr?.on("data", (chunk) => process.stderr.write(`[${label}] ${chunk}`));
+}
+
+function waitForInitialWebuiBuild(child) {
+  return new Promise((resolve, reject) => {
+    let done = false;
+    const finish = () => {
+      if (done) return;
+      done = true;
+      clearInterval(poll);
+      resolve();
+    };
+    const fail = (err) => {
+      if (done) return;
+      done = true;
+      clearInterval(poll);
+      reject(err);
+    };
+    const hasFreshMarker = () => {
+      if (!existsSync(webuiMarker)) return false;
+      return statSync(webuiMarker).mtimeMs > initialMarkerMtime;
+    };
+    const poll = setInterval(() => {
+      if (hasFreshMarker()) finish();
+    }, 500);
+    child.stdout?.on("data", (chunk) => {
+      const text = chunk.toString();
+      if (text.includes('Wrote site to "build"') || text.includes("Wrote site to 'build'")) {
+        finish();
+      }
+    });
+    child.once("exit", (code) => {
+      if (!done) fail(new Error(`webui watch exited before initial build completed (code ${code ?? "unknown"})`));
+    });
+  });
+}
+
+process.on("SIGINT", () => shutdown(0));
+process.on("SIGTERM", () => shutdown(0));
+
+console.log("Starting WebUI static build watcher...");
+const webui = spawnNpm(["run", "webui:watch"]);
+pipeOutput(webui, "webui");
+
+try {
+  await waitForInitialWebuiBuild(webui);
+  console.log("WebUI initial build is ready. Starting backend server...");
+  const backend = spawnNpm(["run", "dev:backend"]);
+  pipeOutput(backend, "backend");
+
+  backend.once("exit", (code) => {
+    if (!shuttingDown) {
+      console.log(`Backend exited (code ${code ?? "unknown"}). Stopping WebUI watcher...`);
+      shutdown(code ?? 0);
+    }
+  });
+  webui.once("exit", (code) => {
+    if (!shuttingDown) {
+      console.log(`WebUI watcher exited (code ${code ?? "unknown"}). Stopping backend...`);
+      shutdown(code ?? 0);
+    }
+  });
+} catch (err) {
+  console.error(err instanceof Error ? err.message : String(err));
+  shutdown(1);
+}

package/scripts/reset.mjs

@@ -1,7 +1,7 @@
 #!/usr/bin/env node
 /**
  * 停止占用 HTTP 服务端口的进程，并删除用户数据目录（与 app 中 PORT / RSSANY_USER_DIR 约定一致）。
- * 用法：pnpm reset  或  PORT=3000 pnpm reset
+ * 用法：npm run reset  或  PORT=3000 npm run reset
  */
 
 import { execFileSync } from "node:child_process";

package/init/config.json

@@ -1,17 +0,0 @@
-{
-    "globalProxy": "",
-    "pipeline": {
-        "steps": [
-            { "id": "tagger", "enabled": false },
-            { "id": "translator", "enabled": false }
-        ]
-    },
-    "deliver": {
-        "gateway": "",
-        "token": ""
-    },
-    "llm": {
-        "baseUrl": "https://api.openai.com/v1",
-        "model": "gpt-4o-mini"
-    }
-}

package/init/sources.json

@@ -1,353 +0,0 @@
-{
-  "sources": [
-    {
-      "ref": "https://www.theinformation.com/",
-      "label": "The Information",
-      "description": "科技商业深度报道",
-      "refresh": "1h",
-      "weight": 0.95
-    },
-    {
-      "ref": "https://techcrunch.com/feed/",
-      "label": "TechCrunch AI",
-      "description": "创业与产品资讯",
-      "refresh": "1h",
-      "weight": 0.95
-    },
-    {
-      "ref": "https://www.technologyreview.com/feed/",
-      "label": "MIT Technology Review",
-      "description": "MIT 科技评论",
-      "refresh": "1h",
-      "weight": 0.94
-    },
-    {
-      "ref": "https://venturebeat.com/category/ai/feed/",
-      "label": "VentureBeat AI频道",
-      "description": "VentureBeat AI频道",
-      "refresh": "1h",
-      "weight": 0.82
-    },
-    {
-      "ref": "https://rss.slashdot.org/Slashdot/slashdotDevelopers",
-      "label": "Slashdot开发者频道",
-      "description": "科技新闻聚合",
-      "refresh": "1h",
-      "weight": 0.62
-    },
-    {
-      "ref": "https://news.ycombinator.com/rss",
-      "label": "HN newest",
-      "description": "Hacker News 最新",
-      "refresh": "6h",
-      "weight": 0.92
-    },
-    {
-      "ref": "https://x.com/elonmusk",
-      "label": "elonmusk",
-      "description": "Elon Musk 推文",
-      "refresh": "1h",
-      "weight": 0.88
-    },
-    {
-      "ref": "https://x.com/AIatMeta",
-      "label": "Meta AI",
-      "description": "Meta AI 官方",
-      "refresh": "1h",
-      "weight": 0.88
-    },
-    {
-      "ref": "https://x.com/OpenAI",
-      "label": "OpenAI",
-      "description": "OpenAI 官方推特",
-      "refresh": "1h",
-      "weight": 0.94
-    },
-    {
-      "ref": "https://x.com/GoogleAI",
-      "label": "Google",
-      "description": "Google AI 官方",
-      "refresh": "1h",
-      "weight": 0.9
-    },
-    {
-      "ref": "https://x.com/Microsoft",
-      "label": "Microsoft",
-      "description": "微软官方",
-      "refresh": "1h",
-      "weight": 0.84
-    },
-    {
-      "ref": "https://x.com/NVIDIA",
-      "label": "NVIDIA",
-      "description": "英伟达官方",
-      "refresh": "1h",
-      "weight": 0.9
-    },
-    {
-      "ref": "https://x.com/NVIDIARobotics",
-      "label": "NVIDIA Robotics",
-      "description": "NVIDIA 机器人",
-      "refresh": "1h",
-      "weight": 0.8
-    },
-    {
-      "ref": "https://x.com/xai",
-      "label": "xAI",
-      "description": "xAI 官方",
-      "refresh": "1h",
-      "weight": 0.88
-    },
-    {
-      "ref": "https://x.com/Figure_robot",
-      "label": "Figure AI",
-      "description": "Figure 人形机器人",
-      "refresh": "1h",
-      "weight": 0.78
-    },
-    {
-      "ref": "https://rss.arxiv.org/rss/cs.CL",
-      "label": "arxiv cs.CL (NLP/LLM)",
-      "description": "NLP/大模型论文",
-      "refresh": "1day",
-      "weight": 0.96
-    },
-    {
-      "ref": "https://rss.arxiv.org/rss/cs.LG",
-      "label": "arxiv cs.LG (Machine Learning)",
-      "description": "机器学习论文",
-      "refresh": "1day",
-      "weight": 0.93
-    },
-    {
-      "ref": "https://x.com/turingcom",
-      "label": "Turing",
-      "description": "Turing 数据标注",
-      "refresh": "1h",
-      "weight": 0.52
-    },
-    {
-      "ref": "https://x.com/scale_ai",
-      "label": "Scale AI 博客",
-      "description": "Scale AI 数据标注",
-      "refresh": "1h",
-      "weight": 0.72
-    },
-    {
-      "ref": "https://x.com/labelbox",
-      "label": "Labelbox",
-      "description": "Labelbox 数据标注平台",
-      "refresh": "1h",
-      "weight": 0.54
-    },
-    {
-      "ref": "https://www.kognic.com/articles",
-      "label": "Kognic",
-      "description": "自动驾驶数据标注",
-      "refresh": "1h",
-      "weight": 0.56
-    },
-    {
-      "ref": "https://x.com/geoffreyhinton",
-      "label": "Geoffrey Hinton",
-      "description": "图灵奖得主，深度学习教父",
-      "refresh": "1h",
-      "weight": 0.9
-    },
-    {
-      "ref": "https://x.com/AndrewYNg",
-      "label": "Andrew Ng",
-      "description": "Coursera 创始人，AI 教育",
-      "refresh": "1h",
-      "weight": 0.9
-    },
-    {
-      "ref": "https://x.com/drfeifei",
-      "label": "李飞飞",
-      "description": "斯坦福 AI 实验室主任",
-      "refresh": "6h",
-      "proxy": "http://192.168.34.247:30002",
-      "weight": 0.95
-    },
-    {
-      "ref": "https://x.com/percyliang",
-      "label": "Percy Liang",
-      "description": "斯坦福 CRFM 主任",
-      "refresh": "1h",
-      "weight": 0.86
-    },
-    {
-      "ref": "https://x.com/ilyasut",
-      "label": "Ilya Sutskever",
-      "description": "OpenAI 联合创始人",
-      "refresh": "1h",
-      "weight": 0.9
-    },
-    {
-      "ref": "https://www.moonshot.ai/",
-      "label": "Moonshot",
-      "description": "月之暗面",
-      "refresh": "1day",
-      "weight": 0.85
-    },
-    {
-      "ref": "https://www.microsoft.com/en-us/research/blog/",
-      "label": "Microsoft Research",
-      "description": "微软研究院博客",
-      "refresh": "1day",
-      "weight": 0.88
-    },
-    {
-      "ref": "https://www.csail.mit.edu/research",
-      "label": "MIT CSAIL",
-      "description": "MIT 计算机与 AI 实验室",
-      "refresh": "1day",
-      "weight": 0.84
-    },
-    {
-      "ref": "https://www.anthropic.com/research",
-      "label": "Anthropic Research",
-      "description": "Anthropic 研究论文",
-      "refresh": "1day",
-      "weight": 0.95
-    },
-    {
-      "ref": "https://trendshift.io/github-trending-repositories?trending-range=1",
-      "label": "Trendshift热门仓库",
-      "description": "AI 趋势追踪",
-      "refresh": "1h",
-      "weight": 0.74
-    },
-    {
-      "ref": "https://tech.meituan.com/feed/",
-      "label": "美团技术",
-      "description": "美团技术博客",
-      "refresh": "1day",
-      "weight": 0.68
-    },
-    {
-      "ref": "https://research.google/",
-      "label": "Google Research",
-      "description": "Google 研究院",
-      "refresh": "1day",
-      "weight": 0.9
-    },
-    {
-      "ref": "https://research.google/resources/datasets/",
-      "label": "Google Datasets",
-      "description": "Google 公开数据集",
-      "refresh": "1day",
-      "weight": 0.76
-    },
-    {
-      "ref": "https://paperswithcode.co/",
-      "label": "Papers with Code",
-      "description": "论文与代码检索",
-      "refresh": "1day",
-      "weight": 0.88
-    },
-    {
-      "ref": "https://opendrivelab.com/EmbodiedAI",
-      "label": "OpenDriveLab Embodied",
-      "description": "具身智能与自动驾驶",
-      "refresh": "1day",
-      "weight": 0.8
-    },
-    {
-      "ref": "https://opendatalab.org.cn",
-      "label": "OpenDataLab",
-      "description": "开放数据平台",
-      "refresh": "1day",
-      "weight": 0.74
-    },
-    {
-      "ref": "https://mlcommons.org/research/",
-      "label": "MLCommons",
-      "description": "MLCommons 基准与研究",
-      "refresh": "1day",
-      "weight": 0.76
-    },
-    {
-      "ref": "https://mila.quebec/en",
-      "label": "Mila Quebec",
-      "description": "蒙特利尔 AI 研究所",
-      "refresh": "1day",
-      "weight": 0.82
-    },
-    {
-      "ref": "https://lmsys.org/projects/",
-      "label": "LMSys",
-      "description": "大模型排行榜",
-      "refresh": "1day",
-      "weight": 0.85
-    },
-    {
-      "ref": "https://huggingface.co/blog",
-      "label": "Hugging Face Blog",
-      "description": "Hugging Face 官方博客",
-      "refresh": "1day",
-      "weight": 0.9
-    },
-    {
-      "ref": "https://flageval.baai.ac.cn/#/news",
-      "label": "FlagEval",
-      "description": "智源大模型评测",
-      "refresh": "1day",
-      "weight": 0.7
-    },
-    {
-      "ref": "https://deepmind.google/blog/rss.xml",
-      "label": "DeepMind Research",
-      "description": "DeepMind 官方RSS",
-      "refresh": "1day",
-      "weight": 0.95
-    },
-    {
-      "ref": "https://dataverse.harvard.edu",
-      "label": "Harvard Dataverse",
-      "description": "哈佛学术数据集",
-      "refresh": "1day",
-      "weight": 0.68
-    },
-    {
-      "ref": "https://bair.berkeley.edu/blog/feed.xml",
-      "label": "Berkeley BAIR",
-      "description": "伯克利 AI 研究院",
-      "refresh": "1day",
-      "weight": 0.88
-    },
-    {
-      "ref": "https://ai.meta.com/results/?content_types%5B0%5D=publication",
-      "label": "Meta AI Publications",
-      "description": "Meta AI 论文",
-      "refresh": "1day",
-      "weight": 0.88
-    },
-    {
-      "ref": "https://rss.arxiv.org/rss/cs.AI",
-      "label": "arXiv cs.AI",
-      "description": "arXiv cs.AI",
-      "refresh": "1day",
-      "weight": 0.95
-    },
-    {
-      "ref": "https://openai.com/news/rss.xml",
-      "label": "OpenAI 博客",
-      "description": "OpenAI 官方博客",
-      "refresh": "1day",
-      "weight": 0.97
-    },
-    {
-      "ref": "https://www.reddit.com/r/LocalLLaMA.rss",
-      "label": "Reddit 开源大模型板块",
-      "refresh": "1day",
-      "weight": 0.82
-    },
-    {
-      "ref": "https://datahub.com/feed",
-      "label": "Datahub",
-      "refresh": "1day",
-      "weight": 0.94
-    }
-  ]
-}

package/statics/401.html

@@ -1,56 +0,0 @@
-<!DOCTYPE html>
-<html lang="zh-CN">
-<head>
-  <meta charset="utf-8">
-  <meta name="viewport" content="width=device-width, initial-scale=1">
-  <title>401 需要登录</title>
-  <style>
-    body { font-family: system-ui, sans-serif; max-width: 560px; margin: 2rem auto; padding: 0 1rem; }
-    h1 { font-size: 1.25rem; margin-bottom: 1rem; }
-    .url { word-break: break-all; background: #f5f5f5; padding: 0.5rem 0.75rem; border-radius: 4px; margin: 0.5rem 0 1rem; font-size: 0.875rem; }
-    .btn { padding: 0.5rem 1rem; background: #333; color: #fff; border: none; border-radius: 4px; cursor: pointer; font-size: 1rem; }
-    .btn:hover { background: #555; }
-    .btn:disabled { opacity: 0.6; cursor: not-allowed; }
-    .msg { margin-top: 1rem; font-size: 0.875rem; color: #666; }
-    .links { margin-top: 1.5rem; font-size: 0.875rem; }
-    .links a { color: #0969da; text-decoration: none; }
-    .links a:hover { text-decoration: underline; }
-  </style>
-</head>
-<body>
-  <h1>401 需要登录</h1>
-  <p>当前请求的订阅地址需要登录后才能拉取：</p>
-  <div class="url" id="failed-url">{{listUrl}}</div>
-  <p>请点击下方按钮，在<strong>弹出的有头浏览器窗口</strong>中完成该站点的登录，完成后关闭窗口并刷新本页或 RSS 地址即可。</p>
-  <button type="button" class="btn" id="btn-login" data-list-url="{{listUrl}}">打开有头登录页</button>
-  <p class="msg" id="api-msg"></p>
-  <p class="links"><a href="/">首页</a> · <a href="/plugins">已生效插件</a>（可对需登录站点批量打开登录页）</p>
-  <script>
-    (function(){
-      var btn = document.getElementById('btn-login');
-      var urlEl = document.getElementById('failed-url');
-      var msgEl = document.getElementById('api-msg');
-      var listUrl = btn.getAttribute('data-list-url');
-      if (!listUrl) {
-        urlEl.textContent = '（未知地址）';
-        btn.disabled = true;
-        return;
-      }
-      btn.onclick = function(){
-        btn.disabled = true;
-        msgEl.textContent = '正在打开登录窗口…';
-        fetch('/auth/ensure?url=' + encodeURIComponent(listUrl), { method: 'POST' })
-          .then(function(r){ return r.json(); })
-          .then(function(d){
-            msgEl.textContent = d.message || (d.ok ? '已打开登录窗口。' : '请求失败');
-            if (!d.ok) btn.disabled = false;
-          })
-          .catch(function(){
-            msgEl.textContent = '请求失败，请重试';
-            btn.disabled = false;
-          });
-      };
-    })();
-  </script>
-</body>
-</html>

package/statics/404.html

@@ -1,12 +0,0 @@
-<!DOCTYPE html>
-<html lang="zh-CN">
-<head>
-  <meta charset="utf-8">
-  <meta name="viewport" content="width=device-width, initial-scale=1">
-  <title>404 未找到</title>
-</head>
-<body>
-  <h1>404 未找到</h1>
-  <p>请求的 URL 无匹配的站点规则，请检查 URL 或添加对应插件。</p>
-</body>
-</html>