rssany 0.3.2 → 0.3.4

package/dist/index.js

@@ -5,14 +5,14 @@ import { serve } from "@hono/node-server";
 import { Hono } from "hono";
 import { cors } from "hono/cors";
 import { exec } from "node:child_process";
+import { createHash } from "node:crypto";
 import { join, dirname, basename, resolve, sep, relative } from "node:path";
 import { promisify } from "node:util";
 import puppeteerCore from "puppeteer-core";
 import { parse, NodeType } from "node-html-parser";
 import { DatabaseSync } from "node:sqlite";
-import { mkdir, writeFile, copyFile, access, rename, readFile, readdir, stat, unlink } from "node:fs/promises";
+import { mkdir, rename, writeFile, copyFile, access, readFile, readdir, stat, unlink } from "node:fs/promises";
 import { fileURLToPath, pathToFileURL } from "node:url";
-import { createHash } from "node:crypto";
 import { JSDOM } from "jsdom";
 import { Readability } from "@mozilla/readability";
 import OpenAI from "openai";
@@ -160,7 +160,7 @@ function markPipelineDrop(item) {
 function isPipelineDroppedItem(item) {
   return item.extra?.[PIPELINE_DROP_EXTRA_KEY] === true;
 }
-function canonicalHttpSourceRef(ref) {
+function canonicalHttpSourceRef(ref, opts = {}) {
   const t = ref.trim();
   if (!t) return t;
   if (!/^https?:\/\//i.test(t)) return t.toLowerCase();
@@ -172,7 +172,7 @@ function canonicalHttpSourceRef(ref) {
     if (path.length > 1 && path.endsWith("/")) {
       path = path.slice(0, -1);
     }
-    path = path.toLowerCase();
+    if (opts.lowerPathCase) path = path.toLowerCase();
     return `${protocol}//${host}${path}${u.search}${u.hash}`;
   } catch {
     return t.toLowerCase();
@@ -209,8 +209,48 @@ const httpSourceRef = /* @__PURE__ */ Object.freeze(/* @__PURE__ */ Object.defin
 const __dir = dirname(fileURLToPath(import.meta.url));
 const base = basename(__dir);
 const PACKAGE_ROOT = base === "app" || base === "dist" ? join(__dir, "..") : __dir;
-const envUserDir = process.env.RSSANY_USER_DIR?.trim();
-const USER_DIR = envUserDir && envUserDir.length > 0 ? envUserDir : join(homedir(), ".rssany");
+const LEGACY_HOME_DIR = ".rssany";
+function resolveNpmPrefixFromPackageRoot(packageRoot) {
+  const normalized = packageRoot.replace(/\\/g, "/");
+  const libSuffix = "/lib/node_modules/rssany";
+  if (normalized.endsWith(libSuffix)) {
+    return packageRoot.slice(0, packageRoot.length - libSuffix.length);
+  }
+  const flatSuffix = "/node_modules/rssany";
+  if (normalized.endsWith(flatSuffix)) {
+    return packageRoot.slice(0, packageRoot.length - flatSuffix.length);
+  }
+  return null;
+}
+function isGlobalNpmInstall(packageRoot) {
+  const normalized = packageRoot.replace(/\\/g, "/");
+  if (normalized.endsWith("/lib/node_modules/rssany")) return true;
+  const globalPatterns = [
+    /\/npm\/node_modules\/rssany$/,
+    /\/\.local\/lib\/node_modules\/rssany$/,
+    /\/\.local\/node_modules\/rssany$/,
+    /\/\.npm-global\/lib\/node_modules\/rssany$/,
+    /\/\.nvm\/versions\/node\/[^/]+\/lib\/node_modules\/rssany$/,
+    /\/\.fnm\/node-versions\/[^/]+\/installation\/lib\/node_modules\/rssany$/
+  ];
+  return globalPatterns.some((pattern) => pattern.test(normalized));
+}
+function resolveDefaultUserDir(packageRoot) {
+  const env = process.env.RSSANY_USER_DIR?.trim();
+  if (env) return env;
+  const npmPrefix = resolveNpmPrefixFromPackageRoot(packageRoot);
+  if (npmPrefix && isGlobalNpmInstall(packageRoot)) {
+    return join(npmPrefix, "var", "rssany");
+  }
+  if (!packageRoot.replace(/\\/g, "/").includes("/node_modules/")) {
+    return join(packageRoot, ".rssany");
+  }
+  return join(homedir(), LEGACY_HOME_DIR);
+}
+function getLegacyHomeUserDir() {
+  return join(homedir(), LEGACY_HOME_DIR);
+}
+const USER_DIR = resolveDefaultUserDir(PACKAGE_ROOT);
 const DATA_DIR = join(USER_DIR, "data");
 const CACHE_DIR = process.env.CACHE_DIR ?? join(USER_DIR, "cache");
 join(USER_DIR, "sites.json");
@@ -279,7 +319,24 @@ async function ensureUserDirPackageJsonForPlugins() {
     });
   }
 }
+async function migrateLegacyHomeUserDir() {
+  const legacy = getLegacyHomeUserDir();
+  if (USER_DIR === legacy) return;
+  if (await pathExists(USER_DIR)) return;
+  if (!await pathExists(legacy)) return;
+  try {
+    await rename(legacy, USER_DIR);
+    logger.info("config", "已从 ~/.rssany 迁移用户数据", { from: legacy, to: USER_DIR });
+  } catch (err) {
+    logger.warn("config", "从 ~/.rssany 迁移失败", {
+      from: legacy,
+      to: USER_DIR,
+      err: err instanceof Error ? err.message : String(err)
+    });
+  }
+}
 async function initUserDir() {
+  await migrateLegacyHomeUserDir();
   await mkdir(USER_DIR, { recursive: true });
   await mkdir(DATA_DIR, { recursive: true });
   await mkdir(CACHE_DIR, { recursive: true });
@@ -1081,9 +1138,14 @@ function launchArgs(config) {
   }
   return base2;
 }
-function getUserDataDir(cacheDir) {
+function proxyProfileName(proxy) {
+  if (!proxy) return "main";
+  const hash = createHash("sha1").update(proxy).digest("hex").slice(0, 12);
+  return `main_proxy_${hash}`;
+}
+function getUserDataDir(cacheDir, proxy) {
   if (!cacheDir) return void 0;
-  return join(cacheDir, "browser_data", "main");
+  return join(cacheDir, "browser_data", proxyProfileName(proxy));
 }
 function isAlreadyRunningError(e) {
   const msg = e instanceof Error ? e.message : String(e);
@@ -1174,13 +1236,141 @@ function isFrameDetachedError(e) {
   return /detached|Navigating frame was detached|Session closed/i.test(msg);
 }
 const sharedBrowsers = /* @__PURE__ */ new Map();
+const managedBrowsers = /* @__PURE__ */ new WeakSet();
+const infoPagePromises = /* @__PURE__ */ new WeakMap();
+const RSSANY_INFO_PAGE_PREFIX = "data:text/html;charset=utf-8,";
+function rssAnyInfoPageUrl() {
+  const html = `<!doctype html>
+<html lang="zh-CN">
+<head>
+  <meta charset="utf-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1">
+  <title>RssAny 浏览器</title>
+  <style>
+    :root {
+      color-scheme: light dark;
+      font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif;
+      background: #f7f7f5;
+      color: #202124;
+    }
+    body {
+      margin: 0;
+      min-height: 100vh;
+      display: grid;
+      place-items: center;
+      padding: 32px;
+      box-sizing: border-box;
+    }
+    main {
+      max-width: 680px;
+      border: 1px solid #d7d7d1;
+      border-radius: 8px;
+      background: #ffffff;
+      padding: 28px 32px;
+      box-shadow: 0 8px 24px rgba(0, 0, 0, 0.08);
+    }
+    h1 {
+      margin: 0 0 12px;
+      font-size: 24px;
+      line-height: 1.25;
+    }
+    p {
+      margin: 10px 0 0;
+      font-size: 15px;
+      line-height: 1.7;
+    }
+    code {
+      font-family: ui-monospace, SFMono-Regular, Consolas, "Liberation Mono", monospace;
+      font-size: 13px;
+      background: #f0f0ec;
+      padding: 2px 6px;
+      border-radius: 4px;
+    }
+    @media (prefers-color-scheme: dark) {
+      :root {
+        background: #181a1b;
+        color: #f1f1ed;
+      }
+      main {
+        background: #202325;
+        border-color: #3a3d3f;
+        box-shadow: none;
+      }
+      code {
+        background: #303335;
+      }
+    }
+  </style>
+</head>
+<body>
+  <main>
+    <h1>这是 RssAny 创建的浏览器</h1>
+    <p>RssAny 会复用这个浏览器执行订阅抓取、站点登录、页面解析和调试打开等任务。</p>
+    <p>抓取任务会临时打开自己的标签页，完成后自动关闭。请保留此页面，以便区分 RssAny 管理的浏览器窗口。</p>
+    <p>用户数据与 cookies 保存在 <code>.rssany/cache/browser_data</code> 对应的浏览器 profile 中。</p>
+  </main>
+</body>
+</html>`;
+  return `${RSSANY_INFO_PAGE_PREFIX}${encodeURIComponent(html)}`;
+}
+function isRssAnyInfoPage(page) {
+  return page.url().startsWith(RSSANY_INFO_PAGE_PREFIX);
+}
+function isBlankPage(page) {
+  const url = page.url();
+  return url === "about:blank" || url === "" || url.startsWith("chrome://newtab");
+}
+async function cleanupExtraBlankPages(browser) {
+  if (!isBrowserConnected(browser)) return;
+  const pages = await browser.pages().catch(() => []);
+  for (const page of pages) {
+    if (page.isClosed() || isRssAnyInfoPage(page) || !isBlankPage(page)) continue;
+    if (pages.length <= 1) continue;
+    await page.close().catch(() => {
+    });
+  }
+}
+async function ensureRssAnyInfoPage(browser) {
+  if (!isBrowserConnected(browser)) return;
+  const current = infoPagePromises.get(browser);
+  if (current) return current;
+  const promise = (async () => {
+    const pages = await browser.pages().catch(() => []);
+    if (pages.some((page2) => !page2.isClosed() && isRssAnyInfoPage(page2))) {
+      await cleanupExtraBlankPages(browser);
+      return;
+    }
+    const page = await browser.newPage();
+    await page.goto(rssAnyInfoPageUrl(), { waitUntil: "domcontentloaded", timeout: 1e4 }).catch(() => {
+    });
+    await cleanupExtraBlankPages(browser);
+  })().finally(() => {
+    infoPagePromises.delete(browser);
+  });
+  infoPagePromises.set(browser, promise);
+  return promise;
+}
+function setupRssAnyBrowserLifecycle(browser, headless) {
+  if (headless || managedBrowsers.has(browser)) return;
+  managedBrowsers.add(browser);
+  browser.on("targetcreated", () => {
+    setTimeout(() => {
+      cleanupExtraBlankPages(browser).catch(() => {
+      });
+    }, 2500);
+  });
+  browser.on("targetdestroyed", () => {
+    setTimeout(() => {
+      ensureRssAnyInfoPage(browser).catch(() => {
+      });
+    }, 300);
+  });
+}
 function browserKey(config) {
-  const wantHeadless = config.headless !== false;
   const executablePath = config.chromeExecutablePath ?? process.env.CHROME_PATH ?? findChromeExecutable() ?? "";
-  const userDataDir = getUserDataDir(config.cacheDir);
   const proxy = resolveProxy(config) ?? "";
+  const userDataDir = getUserDataDir(config.cacheDir, proxy);
   return JSON.stringify({
-    headless: wantHeadless,
     userDataDir: userDataDir ? resolve(userDataDir) : "",
     proxy,
     executablePath
@@ -1195,7 +1385,8 @@ async function launchBrowser(config) {
   if (!executablePath) {
     throw new Error("未找到 Chrome 可执行文件，请安装 Google Chrome 或设置 CHROME_PATH 环境变量");
   }
-  const userDataDir = getUserDataDir(config.cacheDir);
+  const proxy = resolveProxy(config);
+  const userDataDir = getUserDataDir(config.cacheDir, proxy);
   const maxRetries = 2;
   let lastErr;
   for (let attempt = 0; attempt <= maxRetries; attempt++) {
@@ -1211,7 +1402,7 @@ async function launchBrowser(config) {
       }
       return await puppeteerCore.launch({
         headless: wantHeadless,
-        args: launchArgs({ proxy: config.proxy, headless: wantHeadless }),
+        args: launchArgs({ proxy, headless: wantHeadless }),
         userDataDir,
         executablePath,
         ignoreDefaultArgs: ["--enable-automation"]
@@ -1224,7 +1415,7 @@ async function launchBrowser(config) {
       if (isAlreadyRunningError(e)) {
         const dir = userDataDir ?? "browser_data/main";
         throw new Error(
-          `Chrome 的 profile 目录已被占用（${dir}）。通常是因为上次未正常退出或同时运行了多个本服务实例。请关闭占用该目录的 Chrome 进程后重试，或设置环境变量 CACHE_DIR 使用不同缓存目录。`
+          `Chrome 的 profile 目录已被占用（${dir}）。通常是因为上次未正常退出，或另一个服务实例正在使用同一个缓存目录。请关闭占用该目录的 Chrome 进程后重试，或设置环境变量 CACHE_DIR 使用不同缓存目录。`
         );
       }
       throw e;
@@ -1233,24 +1424,53 @@ async function launchBrowser(config) {
   throw lastErr;
 }
 async function getOrCreateBrowser(config) {
-  const key = browserKey(config);
+  const normalizedConfig = { ...config, proxy: resolveProxy(config) };
+  const key = browserKey(normalizedConfig);
+  const wantHeadless = normalizedConfig.headless !== false;
   const current = sharedBrowsers.get(key);
-  if (isBrowserConnected(current?.browser)) {
-    return current.browser;
-  }
   if (current?.promise) {
-    return current.promise;
+    const browser = await current.promise;
+    if (isBrowserConnected(browser) && (wantHeadless || current.headless === false)) {
+      if (!wantHeadless) {
+        await ensureRssAnyInfoPage(browser);
+      }
+      return browser;
+    }
+    if (isBrowserConnected(browser)) {
+      await browser.close().catch(() => {
+      });
+    }
+    if (sharedBrowsers.get(key) === current) {
+      sharedBrowsers.delete(key);
+    }
+  } else if (isBrowserConnected(current?.browser)) {
+    if (wantHeadless || current.headless === false) {
+      if (!wantHeadless) {
+        await ensureRssAnyInfoPage(current.browser);
+      }
+      return current.browser;
+    }
+    await current.browser.close().catch(() => {
+    });
+    if (sharedBrowsers.get(key) === current) {
+      sharedBrowsers.delete(key);
+    }
   }
   const slot = {};
-  const promise = launchBrowser({ ...config, proxy: resolveProxy(config) }).then((browser) => {
+  const promise = launchBrowser(normalizedConfig).then((browser) => {
     slot.browser = browser;
     slot.promise = void 0;
+    slot.headless = wantHeadless;
+    setupRssAnyBrowserLifecycle(browser, wantHeadless);
     browser.once("disconnected", () => {
       if (sharedBrowsers.get(key)?.browser === browser) {
         sharedBrowsers.delete(key);
       }
     });
-    return browser;
+    if (wantHeadless) {
+      return browser;
+    }
+    return ensureRssAnyInfoPage(browser).then(() => browser);
   }).catch((err) => {
     if (sharedBrowsers.get(key) === slot) {
       sharedBrowsers.delete(key);
@@ -1284,30 +1504,39 @@ async function preCheckAuth(authFlow, cacheDir, opts) {
 }
 async function ensureAuth(authFlow, cacheDir, opts) {
   const { checkAuth, loginUrl, loginTimeoutMs = 60 * 1e3, pollIntervalMs = 2e3 } = authFlow;
-  const browser = await launchBrowser({ headless: false, cacheDir, proxy: resolveProxy(opts) });
+  const browser = await getOrCreateBrowser({ headless: false, cacheDir, proxy: resolveProxy(opts) });
+  const page = await browser.newPage();
   try {
-    const page = await browser.newPage();
-    try {
-      await setupPage(page, false);
-      await applyProxyAuthToPage(page, opts);
-      await page.goto(loginUrl, { waitUntil: "domcontentloaded", timeout: 6e4 });
-      await new Promise((resolve2) => setTimeout(resolve2, 3e3));
-      const authenticated = await checkAuth(page, page.url());
-      if (authenticated) return;
-      const startTime = Date.now();
-      while (Date.now() - startTime < loginTimeoutMs) {
-        await new Promise((resolve2) => setTimeout(resolve2, pollIntervalMs));
-        const authenticated2 = await checkAuth(page, page.url());
-        if (authenticated2) return;
-      }
-      throw new Error(`登录超时（${loginTimeoutMs}ms）`);
-    } finally {
-      await page.close().catch(() => {
-      });
+    await setupPage(page, false);
+    await applyProxyAuthToPage(page, opts);
+    await page.goto(loginUrl, { waitUntil: "domcontentloaded", timeout: 6e4 });
+    await new Promise((resolve2) => setTimeout(resolve2, 3e3));
+    const authenticated = await checkAuth(page, page.url());
+    if (authenticated) return;
+    const startTime = Date.now();
+    while (Date.now() - startTime < loginTimeoutMs) {
+      await new Promise((resolve2) => setTimeout(resolve2, pollIntervalMs));
+      const authenticated2 = await checkAuth(page, page.url());
+      if (authenticated2) return;
     }
+    throw new Error(`登录超时（${loginTimeoutMs}ms）`);
   } finally {
-    await browser.close().catch(() => {
+    await page.close().catch(() => {
+    });
+  }
+}
+async function openBrowserPage(url, cacheDir, opts) {
+  const browser = await getOrCreateBrowser({ headless: false, cacheDir, proxy: resolveProxy(opts) });
+  const page = await browser.newPage();
+  try {
+    await setupPage(page, false);
+    await applyProxyAuthToPage(page, opts);
+    await page.goto(url, { waitUntil: "domcontentloaded", timeout: 6e4 });
+    return page;
+  } catch (err) {
+    await page.close().catch(() => {
     });
+    throw err;
   }
 }
 async function fetchHtml(url, config = {}) {
@@ -2133,30 +2362,52 @@ async function initSources() {
 function resolveRef(src) {
   return src.ref ?? src.url ?? "";
 }
-async function readGlobalProxyFromConfig() {
-  try {
-    const raw = await readFile(CONFIG_PATH, "utf-8");
-    const j = JSON.parse(raw);
-    if (typeof j.globalProxy === "string") {
-      const t = j.globalProxy.trim();
-      return t.length > 0 ? t : void 0;
-    }
-  } catch {
+function normalizeProxyList(raw) {
+  if (!Array.isArray(raw)) return [];
+  const seen = /* @__PURE__ */ new Set();
+  const out = [];
+  for (const v of raw) {
+    if (typeof v !== "string") continue;
+    const t = v.trim();
+    if (!t || seen.has(t)) continue;
+    seen.add(t);
+    out.push(t);
   }
-  return void 0;
+  return out;
 }
-async function saveGlobalProxyToConfig(proxy) {
-  let root = {};
+async function readConfigRoot() {
   try {
     const raw = await readFile(CONFIG_PATH, "utf-8");
-    root = JSON.parse(raw);
+    return JSON.parse(raw);
   } catch {
+    return {};
   }
-  const t = proxy.trim();
-  if (t.length === 0) {
+}
+async function readProxySettingsFromConfig() {
+  const root = await readConfigRoot();
+  const globalProxy = typeof root.globalProxy === "string" ? root.globalProxy.trim() : "";
+  return {
+    globalProxy,
+    proxyList: normalizeProxyList(root.proxyList)
+  };
+}
+async function readGlobalProxyFromConfig() {
+  const { globalProxy } = await readProxySettingsFromConfig();
+  return globalProxy.length > 0 ? globalProxy : void 0;
+}
+async function saveProxySettingsToConfig(settings) {
+  const root = await readConfigRoot();
+  const globalProxy = settings.globalProxy.trim();
+  const proxyList = normalizeProxyList(settings.proxyList);
+  if (globalProxy) {
+    root.globalProxy = globalProxy;
+  } else {
     delete root.globalProxy;
+  }
+  if (proxyList.length > 0) {
+    root.proxyList = proxyList;
   } else {
-    root.globalProxy = t;
+    delete root.proxyList;
   }
   await writeFile(CONFIG_PATH, JSON.stringify(root, null, 2) + "\n", "utf-8");
 }
@@ -3533,23 +3784,7 @@ function registerSourcesRoutes(app) {
       const source = getSource(url);
       const merged = await getEffectiveProxyForListUrl(url, source);
       const proxy = resolveProxy({ proxy: merged });
-      void launchBrowser({ headless: false, cacheDir: CACHE_DIR, proxy }).then(async (browser) => {
-        try {
-          const page = await browser.newPage();
-          await applyProxyAuthToPage(page, { proxy: merged });
-          const realUserAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36";
-          await page.setUserAgent(realUserAgent);
-          await page.setViewport({ width: 1366, height: 960 });
-          await page.goto(url, { waitUntil: "domcontentloaded", timeout: 6e4 });
-          page.once("close", () => {
-            void browser.close().catch(() => {
-            });
-          });
-        } catch {
-          await browser.close().catch(() => {
-          });
-        }
-      }).catch(() => {
+      void openBrowserPage(url, CACHE_DIR, { proxy }).catch(() => {
       });
       return c.json({ ok: true, message: "已在爬虫浏览器中打开" });
     } catch {
@@ -3824,16 +4059,15 @@ function registerLlmRoutes(app) {
 }
 function registerProxySettingsRoutes(app) {
   app.get("/api/proxy", requireAdmin(), async (c) => {
-    const globalProxy = await readGlobalProxyFromConfig() ?? "";
-    return c.json({ globalProxy });
+    return c.json(await readProxySettingsFromConfig());
   });
   app.put("/api/proxy", requireAdmin(), async (c) => {
     try {
       const body = await c.req.json().catch(() => ({}));
       const globalProxy = typeof body.globalProxy === "string" ? body.globalProxy : "";
-      await saveGlobalProxyToConfig(globalProxy);
-      const saved = await readGlobalProxyFromConfig() ?? "";
-      return c.json({ ok: true, globalProxy: saved });
+      const proxyList = Array.isArray(body.proxyList) ? body.proxyList.filter((v) => typeof v === "string") : [];
+      await saveProxySettingsToConfig({ globalProxy, proxyList });
+      return c.json({ ok: true, ...await readProxySettingsFromConfig() });
     } catch (err) {
       return c.json(
         { ok: false, message: err instanceof Error ? err.message : String(err) },
@@ -4307,23 +4541,7 @@ function registerAuthRoutes(app) {
     if (!authFlow) return c.json({ ok: false, message: "该站点无需登录" }, 400);
     const { loginUrl } = authFlow;
     const proxy = await resolveProxyForSite(site);
-    void launchBrowser({ headless: false, cacheDir: CACHE_DIR, proxy: resolveProxy({ proxy }) }).then(async (browser) => {
-      try {
-        const page = await browser.newPage();
-        await applyProxyAuthToPage(page, { proxy });
-        const realUserAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36";
-        await page.setUserAgent(realUserAgent);
-        await page.setViewport({ width: 1366, height: 960 });
-        await page.goto(loginUrl, { waitUntil: "domcontentloaded", timeout: 6e4 });
-        page.once("close", () => {
-          void browser.close().catch(() => {
-          });
-        });
-      } catch {
-        await browser.close().catch(() => {
-        });
-      }
-    }).catch(() => {
+    void openBrowserPage(loginUrl, CACHE_DIR, { proxy: resolveProxy({ proxy }) }).catch(() => {
     });
     return c.json({ ok: true, message: "已打开登录页面" });
   });