rssany 0.1.6 → 0.3.0

package/app/plugins/builtin/xiaohongshu.rssany.js

@@ -1,3 +1,7 @@
+export const id = "xiaohongshu";
+export const name = "Xiaohongshu";
+export const listUrlPattern = /^https:\/\/(www\.)?xiaohongshu\.com\/user\/profile\/[^/?#]+\/?(?:[?#].*)?$/i;
+
 let _deps;
 
 // 小红书站点插件：用户主页列表抓取、笔记详情提取、认证流程
@@ -5,6 +9,41 @@ let _deps;
 
 
 const XHS_ORIGIN = "https://www.xiaohongshu.com";
+const XHS_NOTE_PATH_RE = /^\/(?:explore|discovery\/item)\/([0-9a-f]{24})\/?$/i;
+const XHS_NOTE_ID_RE = /^[0-9a-f]{24}$/i;
+const XHS_NOTE_ID_IN_IMG_RE = /xhscdn\.com\/\d+\/([0-9a-f]{24})/i;
+const XHS_PROFILE_USER_RE = /\/user\/profile\/([0-9a-f]{24})/i;
+
+
+function hashNoteGuid(noteId) {
+  return _deps.createHash("sha256").update(`xhs:note:${noteId}`).digest("hex");
+}
+
+
+function extractProfileUserId(url) {
+  const m = String(url).match(XHS_PROFILE_USER_RE);
+  return m?.[1]?.toLowerCase() ?? null;
+}
+
+
+function buildExploreLink(noteId, origin) {
+  return `${origin.replace(/\/$/, "")}/explore/${noteId}`;
+}
+
+
+function extractNoteIdFromSection(section, profileUserId) {
+  for (const img of section.querySelectorAll('img[src*="xhscdn"]')) {
+    const src = img.getAttribute("src")?.trim() ?? "";
+    const fromImg = src.match(XHS_NOTE_ID_IN_IMG_RE);
+    if (fromImg?.[1] && fromImg[1] !== profileUserId) return fromImg[1].toLowerCase();
+  }
+  const html = section.outerHTML ?? "";
+  for (const match of html.match(/[0-9a-f]{24}/gi) ?? []) {
+    const id = match.toLowerCase();
+    if (id !== profileUserId && XHS_NOTE_ID_RE.test(id)) return id;
+  }
+  return null;
+}
 
 
 function getOrigin(url) {
@@ -16,57 +55,99 @@ function getOrigin(url) {
 }
 
 
-function buildExploreLinkWithXsec(profileHref, origin) {
+function normalizeXhsUrl(href, origin) {
+  try {
+    const url = new URL(href.replace(/&/g, "&"), origin);
+    url.hash = "";
+    return url;
+  } catch {
+    return null;
+  }
+}
+
+
+function normalizeXhsItemLink(href, origin) {
+  const url = normalizeXhsUrl(href, origin);
+  if (!url) return null;
+
   try {
-    const fullUrl = new URL(profileHref.replace(/&/g, "&"), origin);
-    const pathSegs = fullUrl.pathname.split("/").filter(Boolean);
-    const noteId = pathSegs[pathSegs.length - 1];
-    if (!noteId || !/^[0-9a-f]+$/i.test(noteId)) return null;
-    const token = fullUrl.searchParams.get("xsec_token");
-    const source = fullUrl.searchParams.get("xsec_source") ?? "pc_user";
-    if (!token) return null;
-    const explore = new URL(`/explore/${noteId}`, origin);
-    explore.searchParams.set("xsec_token", token);
-    explore.searchParams.set("xsec_source", source);
-    return explore.href;
+    if (!/(^|\.)xiaohongshu\.com$/i.test(url.hostname)) return null;
+    const m = url.pathname.match(XHS_NOTE_PATH_RE);
+    if (!m?.[1]) return null;
+    return buildExploreLink(m[1].toLowerCase(), url.origin);
   } catch {
     return null;
   }
 }
 
 
+function extractRedirectItemLink(href, origin) {
+  const wrapper = normalizeXhsUrl(href, origin);
+  if (!wrapper) return null;
+  if (!/\/website-login\/error\/?$/i.test(wrapper.pathname)) return null;
+
+  const redirectPath = wrapper.searchParams.get("redirectPath");
+  if (!redirectPath) return null;
+  return normalizeXhsItemLink(redirectPath, origin);
+}
+
+
+function extractListItemLink(section, origin, profileUserId) {
+  const noteId = extractNoteIdFromSection(section, profileUserId);
+  if (noteId) return buildExploreLink(noteId, origin);
+
+  const anchors = section.querySelectorAll("a[href]");
+  const candidates = [];
+  for (const anchor of anchors) {
+    const href = anchor.getAttribute("href")?.trim();
+    if (!href) continue;
+
+    const direct = normalizeXhsItemLink(href, origin);
+    if (direct) candidates.push(direct);
+
+    const redirected = extractRedirectItemLink(href, origin);
+    if (redirected) candidates.push(redirected);
+  }
+  return candidates[0] ?? null;
+}
+
+
 function parseListHtml(html, url) {
   const root = _deps.parseHtml(html);
   const origin = getOrigin(url);
+  const profileUserId = extractProfileUserId(url);
   const feed = root.querySelector("#userPostedFeeds");
   if (!feed) return [];
-  const sections = feed.querySelectorAll("section[data-v-79abd645][data-index]");
+  const sections = feed.querySelectorAll("section[data-index]");
   const items = [];
+  const seenNoteIds = new Set();
   for (const section of sections) {
-    const profileWithToken = section.querySelector('a[href*="xsec_token="]');
-    const profileHref = profileWithToken?.getAttribute("href")?.trim();
-    let link;
-    if (profileHref && profileHref.includes("/user/profile/")) {
-      const withXsec = buildExploreLinkWithXsec(profileHref, origin);
-      if (withXsec) link = withXsec;
-      else link = new URL(profileHref.replace(/&/g, "&"), origin).href;
-    } else {
-      const linkEl = section.querySelector('a[href^="/explore/"]');
-      const href = linkEl?.getAttribute("href")?.trim();
-      if (!href) continue;
-      link = new URL(href, origin).href;
-    }
-    const titleEl = section.querySelector("span[data-v-51ec0135]");
-    const title = (titleEl?.textContent ?? "").trim() || "笔记";
-    const authorEl = section.querySelector('a[aria-current="page"] span');
+    const noteId = extractNoteIdFromSection(section, profileUserId);
+    const link = noteId
+      ? buildExploreLink(noteId, origin)
+      : extractListItemLink(section, origin, profileUserId);
+    if (!link) continue;
+    const dedupeKey = noteId ?? link;
+    if (seenNoteIds.has(dedupeKey)) continue;
+    seenNoteIds.add(dedupeKey);
+    const titleEl = section.querySelector("span[data-v-51ec0135]") ?? section.querySelector(".title span") ?? section.querySelector("span");
+    const title = (titleEl?.textContent ?? "").trim() || "Note";
+    const authorEl = section.querySelector('a[aria-current="page"] .name') ?? section.querySelector('a[aria-current="page"] span');
     const author = (authorEl?.textContent ?? "").trim() || undefined;
+    const imageEl = section.querySelector("img[data-xhs-img], img");
+    const image = imageEl?.getAttribute("src")?.trim() || undefined;
+    const summary = image ? undefined : title;
+    const guid = noteId ? hashNoteGuid(noteId) : _deps.createHash("sha256").update(link).digest("hex");
     items.push({
-      guid: _deps.createHash("sha256").update(link).digest("hex"),
+      guid,
       title,
       link,
       pubDate: new Date(),
       author,
-      summary: title,
+      summary,
+      imageUrl: image,
+      coverImg: image,
+      cover_img: image,
     });
   }
   return items;
@@ -240,9 +321,14 @@ function extractDetailHtml(html) {
 }
 
 
-async function fetchItems(sourceId, ctx) {
+export async function fetchItems(sourceId, ctx) {
   _deps = ctx.deps;
-  const { html, finalUrl } = await ctx.fetchHtml(sourceId);
+  const { html, finalUrl } = await ctx.fetchHtml(sourceId, {
+    waitMs: 3000,
+    waitForSelector: "#userPostedFeeds",
+    waitForSelectorTimeoutMs: 15000,
+    scrollBeforeSnapshot: { selector: "#userPostedFeeds", rounds: 8, pauseMs: 900 },
+  });
   return parseListHtml(html, finalUrl);
 }
 
@@ -258,26 +344,3 @@ async function enrichItem(item, ctx) {
     pubDate: detail.pubDate ?? item.pubDate,
   };
 }
-
-
-async function checkAuth(page, _url) {
-  try {
-    const loginButton = await page.$(".reds-button-new.login-btn.large.primary");
-    return loginButton == null;
-  } catch {
-    return false;
-  }
-}
-
-
-export default {
-  id: "xiaohongshu",
-  listUrlPattern: "https://xiaohongshu.com/user/profile/{userId}",
-  fetchItems,
-  enrichItem,
-  checkAuth,
-  loginUrl: "https://www.xiaohongshu.com/",
-  domain: "xiaohongshu.com",
-  loginTimeoutMs: 30 * 1000,
-  pollIntervalMs: 2000,
-};

package/app/plugins/builtin/zhipu-research.rssany.js

@@ -1,3 +1,7 @@
+export const id = "zhipu-research";
+export const name = "Zhipu Research";
+export const listUrlPattern = /^https:\/\/(www\.)?zhipuai\.cn\/zh\/research\/?(?:[?#].*)?$/i;
+
 let _deps;
 
 // 智谱研究页插件：仅抓取列表，不做正文 enrich（兼容净化后的 HTML）
@@ -113,7 +117,7 @@ function buildItemsFromBlogsItems(blogsItems) {
     const summary = normalizeText(blog.resume_zh ?? blog.resume_en ?? "");
     const createdAt = String(blog.createAt ?? "").trim();
     const pubDate = createdAt ? new Date(createdAt) : new Date();
-    const category = normalizeText(blog.tag_zh ?? blog.tag_en ?? "");
+    const _category = normalizeText(blog.tag_zh ?? blog.tag_en ?? "");
     items.push({
       guid: hashGuid(link),
       title,
@@ -263,7 +267,7 @@ function buildItemsFromLeafSequence(html, titleIdMap) {
   for (let i = 0; i < leafTexts.length; i += 1) {
     const dateText = leafTexts[i];
     if (!isDateText(dateText)) continue;
-    const category = i > 0 && RESEARCH_TAGS.has(leafTexts[i - 1]) ? leafTexts[i - 1] : undefined;
+    const _category = i > 0 && RESEARCH_TAGS.has(leafTexts[i - 1]) ? leafTexts[i - 1] : undefined;
 
     let title = "";
     let summary;
@@ -300,7 +304,7 @@ function buildItemsFromLeafSequence(html, titleIdMap) {
 }
 
 
-async function fetchItems(sourceId, ctx) {
+export async function fetchItems(sourceId, ctx) {
   _deps = ctx.deps;
   // 需要读取页面脚本里的 blogsItems（包含详情 id），因此这里禁用净化。
   const { html, finalUrl } = await ctx.fetchHtml(sourceId, { waitMs: 5000, purify: false });
@@ -325,10 +329,3 @@ async function fetchItems(sourceId, ctx) {
 
   throw new Error("[zhipu-research] 未解析到研究条目，页面结构可能已变化");
 }
-
-
-export default {
-  id: "zhipu-research",
-  listUrlPattern: ZHIPU_RESEARCH_URL,
-  fetchItems,
-};

package/app/plugins/site.rssany.js

@@ -1,25 +1,25 @@
-/**
- * Site 插件模板（管理页「添加插件」会复制到 `.rssany/plugins/{id}.rssany.js`）
- * 修改 `id` 后请与文件名保持一致。
- *
- * 接口说明：app/scraper/sources/web/site.ts
- */
-
-export default {
-  id: "__PLUGIN_ID__",
-  listUrlPattern: __LIST_URL_PATTERN__,
-  refreshInterval: "1day",
-
-  /** sourceId 与订阅里 ref 一致；ctx 含 fetchHtml、extractItem、deps（parseHtml 等） */
-  async fetchItems(sourceId, ctx) {
-    const { html, finalUrl } = await ctx.fetchHtml(sourceId, {
-      waitMs: 2000,
-      purify: true,
-    });
-    const root = ctx.deps.parseHtml(html);
-    void root;
-    void finalUrl;
-    // TODO: 用 ctx.deps.parseHtml 解析列表页，产出 { title, link, summary?, pubDate? } 等 FeedItem
-    return [];
-  },
-};
+/**
+ * Site plugin template. The admin UI copies this file to .rssany/plugins/{id}.rssany.js.
+ * Plugin protocol: named exports. No export default is required.
+ *
+ * Interface: app/scraper/sources/web/site.ts
+ */
+
+// Predefined fields stay together at the top.
+export const id = "__PLUGIN_ID__";
+export const name = "__PLUGIN_ID__";
+// eslint-disable-next-line no-undef
+export const listUrlPattern = __LIST_URL_PATTERN__;
+export const refreshInterval = "1day";
+
+export async function fetchItems(sourceId, ctx) {
+  const { html, finalUrl } = await ctx.fetchHtml(sourceId, {
+    waitMs: 2000,
+    purify: true,
+  });
+  const root = ctx.deps.parseHtml(html);
+  void root;
+  void finalUrl;
+  // TODO: Parse the list page and return FeedItem objects.
+  return [];
+}

package/{statics → app/statics}/README.md

@@ -1,7 +1,7 @@
-# statics
-
-静态 HTML 页面：home（首页）、401、404。
-
-- **home.html**：首页，含 Try This 示例链接；下方「需登录的站点」从 `/plugins` 拉取，每个站点可点击「打开登录页」调用 `POST /auth/ensure?siteId=...` 批量做登录。
-- **401.html**：需登录时返回；占位符 `{{listUrl}}` 由 router 注入为失败请求的订阅地址；页内「打开有头登录页」按钮调用 `POST /auth/ensure?url=...` 弹出有头浏览器完成登录。
-- **404.html**：无匹配站点时返回。
+# statics
+
+静态 HTML 页面：home（首页）、401、404。
+
+- **home.html**：首页，含 Try This 示例链接；下方「需登录的站点」从 `/plugins` 拉取，每个站点可点击「打开登录页」调用 `POST /auth/ensure?siteId=...` 批量做登录。
+- **401.html**：需登录时返回；占位符 `{{listUrl}}` 由 router 注入为失败请求的订阅地址；页内「打开有头登录页」按钮调用 `POST /auth/ensure?url=...` 弹出有头浏览器完成登录。
+- **404.html**：无匹配站点时返回。

package/bin/rssany.js

@@ -1,6 +1,226 @@
-#!/usr/bin/env node
-if (process.argv[2] === "reset") {
-  await import(new URL("../scripts/reset.mjs", import.meta.url));
-} else {
-  await import(new URL("../dist/index.js", import.meta.url));
-}
+#!/usr/bin/env node
+import { spawn } from "node:child_process";
+import { closeSync, openSync } from "node:fs";
+import { access, mkdir, readFile, rm, writeFile } from "node:fs/promises";
+import http from "node:http";
+import { homedir, networkInterfaces } from "node:os";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+
+const command = process.argv[2];
+const binDir = dirname(fileURLToPath(import.meta.url));
+const packageRoot = join(binDir, "..");
+const userDir = process.env.RSSANY_USER_DIR?.trim() || join(homedir(), ".rssany");
+const pidPath = join(userDir, "rssany.pid");
+const logPath = join(userDir, "rssany.log");
+const port = Number(process.env.PORT) || 18473;
+const serverOrigin = `http://127.0.0.1:${port}`;
+
+async function pathExists(path) {
+  try {
+    await access(path);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+async function readPid() {
+  try {
+    const raw = await readFile(pidPath, "utf-8");
+    const pid = Number(raw.trim());
+    return Number.isInteger(pid) && pid > 0 ? pid : null;
+  } catch {
+    return null;
+  }
+}
+
+function isProcessRunning(pid) {
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+function getLanUrl() {
+  const lanIp = Object.values(networkInterfaces())
+    .flat()
+    .find((iface) => iface?.family === "IPv4" && !iface.internal)?.address;
+  return lanIp ? `http://${lanIp}:${port}/` : null;
+}
+
+function printAddress(prefix = "RssAny 已启动") {
+  console.log(`${prefix}: http://127.0.0.1:${port}/`);
+  const lanUrl = getLanUrl();
+  if (lanUrl) console.log(`局域网访问: ${lanUrl}`);
+}
+
+function printUsage() {
+  console.log("用法: rssany <start|stop|reset|crawl>");
+  console.log("  rssany start  后台启动服务并输出访问地址");
+  console.log("  rssany stop   关闭后台服务并输出执行状态");
+  console.log("  rssany reset  重置本地数据");
+  console.log("  rssany crawl <ref>  按内部抓取链路拉取指定信源");
+}
+
+async function canConnectToServer() {
+  return new Promise((resolve) => {
+    const req = http.get(`${serverOrigin}/api/server-info`, (res) => {
+      res.resume();
+      resolve(true);
+    });
+    req.setTimeout(500, () => {
+      req.destroy();
+      resolve(false);
+    });
+    req.on("error", () => resolve(false));
+  });
+}
+
+async function waitForServer(timeoutMs = 5000) {
+  const startTime = Date.now();
+  while (Date.now() - startTime < timeoutMs) {
+    if (await canConnectToServer()) return true;
+    await new Promise((resolve) => setTimeout(resolve, 250));
+  }
+  return false;
+}
+
+async function start() {
+  await mkdir(userDir, { recursive: true });
+
+  const currentPid = await readPid();
+  if (currentPid && isProcessRunning(currentPid)) {
+    printAddress(`RssAny 已在运行 (pid ${currentPid})`);
+    return;
+  }
+
+  const entry = join(packageRoot, "dist", "index.js");
+  if (!(await pathExists(entry))) {
+    console.error("未找到 dist/index.js，请先构建项目或重新安装 rssany。");
+    process.exitCode = 1;
+    return;
+  }
+
+  const logFd = openSync(logPath, "a");
+  const child = spawn(process.execPath, [entry], {
+    cwd: process.cwd(),
+    detached: true,
+    env: process.env,
+    stdio: ["ignore", logFd, logFd],
+  });
+  closeSync(logFd);
+
+  await writeFile(pidPath, `${child.pid}\n`, "utf-8");
+  console.log(`日志: ${logPath}`);
+  if (await waitForServer()) {
+    child.unref();
+    printAddress(`RssAny 已启动 (pid ${child.pid})`);
+    return;
+  }
+
+  child.unref();
+  console.error(`RssAny 启动未完成，请查看日志: ${logPath}`);
+  process.exitCode = 1;
+}
+
+async function stop() {
+  const pid = await readPid();
+  if (!pid) {
+    console.log("RssAny 未运行：没有找到 pid 文件。");
+    return;
+  }
+
+  if (!isProcessRunning(pid)) {
+    await rm(pidPath, { force: true });
+    console.log(`RssAny 未运行：已清理失效 pid ${pid}。`);
+    return;
+  }
+
+  process.kill(pid, "SIGTERM");
+  await rm(pidPath, { force: true });
+  console.log(`RssAny 已发送停止信号 (pid ${pid})。`);
+}
+
+function readCrawlRef(args) {
+  const refFlagIndex = args.findIndex((arg) => arg === "--ref");
+  if (refFlagIndex >= 0) return args[refFlagIndex + 1]?.trim() || "";
+  const refEquals = args.find((arg) => arg.startsWith("--ref="));
+  if (refEquals) return refEquals.slice("--ref=".length).trim();
+  return args.find((arg) => !arg.startsWith("-"))?.trim() || "";
+}
+
+async function postJson(path, body) {
+  const res = await fetch(`${serverOrigin}${path}`, {
+    method: "POST",
+    headers: { "Content-Type": "application/json" },
+    body: JSON.stringify(body),
+  });
+  const data = await res.json().catch(() => ({}));
+  if (!res.ok) {
+    throw new Error(data.error || `HTTP ${res.status}`);
+  }
+  return data;
+}
+
+async function getJson(path) {
+  const res = await fetch(`${serverOrigin}${path}`);
+  const data = await res.json().catch(() => ({}));
+  if (!res.ok) {
+    throw new Error(data.error || `HTTP ${res.status}`);
+  }
+  return data;
+}
+
+async function pollTask(taskId, timeoutMs = 120000) {
+  const start = Date.now();
+  while (Date.now() - start < timeoutMs) {
+    const task = await getJson(`/api/tasks/${encodeURIComponent(taskId)}`);
+    if (task.status === "done") return task;
+    if (task.status === "error") {
+      throw new Error(task.error || "抓取失败");
+    }
+    await new Promise((resolve) => setTimeout(resolve, 800));
+  }
+  throw new Error("抓取超时");
+}
+
+async function crawl() {
+  const ref = readCrawlRef(process.argv.slice(3));
+  if (!ref) {
+    console.error("ref 不能为空。用法: rssany crawl <ref>");
+    process.exitCode = 1;
+    return;
+  }
+  if (!(await canConnectToServer())) {
+    console.error(`RssAny 服务未运行，请先执行 rssany start。目标: ${serverOrigin}`);
+    process.exitCode = 1;
+    return;
+  }
+  try {
+    const { taskId } = await postJson("/api/tasks", { type: "source-pull", ref });
+    if (!taskId) throw new Error("后端未返回 taskId");
+    console.log(`crawl 已提交: ${ref}`);
+    console.log(`task: ${taskId}`);
+    await pollTask(taskId);
+    console.log("crawl 完成");
+  } catch (err) {
+    console.error(err instanceof Error ? err.message : String(err));
+    process.exitCode = 1;
+  }
+}
+
+if (command === "reset") {
+  await import(new URL("../scripts/reset.mjs", import.meta.url));
+} else if (command === "start") {
+  await start();
+} else if (command === "stop") {
+  await stop();
+} else if (command === "crawl") {
+  await crawl();
+} else {
+  printUsage();
+  if (command) process.exitCode = 1;
+}