rssany 0.1.4 → 0.1.6

package/app/plugins/builtin/mit-csail-research.rssany.js

@@ -0,0 +1,208 @@
+let _deps;
+
+// MIT CSAIL Research plugin: warm up via homepage, then parse /research list items.
+
+
+
+const SITE_ID = "mit-csail-research";
+const CSAIL_HOME_URL = "https://www.csail.mit.edu/";
+const CSAIL_RESEARCH_PATH = "/research";
+const SUMMARY_SELECTOR = "div, p, span, h2, h3, h4, a";
+const BLOCKED_HINTS = [
+  "http error 403",
+  "request to access",
+  "access denied",
+  "request denied",
+  "body class=\"neterror\"",
+  "\u60a8\u672a\u83b7\u6388\u6743\uff0c\u65e0\u6cd5\u67e5\u770b\u6b64\u7f51\u9875\u3002",
+  "\u8bbf\u95ee <span>www.csail.mit.edu</span> \u7684\u8bf7\u6c42\u906d\u5230\u62d2\u7edd",
+];
+
+
+function normalizeText(text) {
+  return (text ?? "").replace(/\s+/g, " ").trim();
+}
+
+
+function hashGuid(input) {
+  return _deps.createHash("sha256").update(input).digest("hex");
+}
+
+
+function toAbsoluteHttpUrl(rawHref, baseUrl) {
+  if (!rawHref) return null;
+  const href = rawHref.trim();
+  if (!href || href.startsWith("#") || href.startsWith("javascript:") || href.startsWith("mailto:")) return null;
+  try {
+    const url = new URL(href, baseUrl);
+    if (!/^https?:$/i.test(url.protocol)) return null;
+    return url.href;
+  } catch {
+    return null;
+  }
+}
+
+
+function normalizeCategoryToken(text) {
+  return normalizeText(text).toLowerCase().replace(/[^a-z0-9]+/g, "");
+}
+
+
+function matchesRequestedCategory(cardCategory, queryCategory) {
+  const wanted = normalizeCategoryToken(queryCategory);
+  if (!wanted) return true;
+  const card = normalizeCategoryToken(cardCategory);
+  if (!card) return false;
+
+  if (wanted.includes("group")) return card.includes("group");
+  if (wanted.includes("center")) return card.includes("center");
+  if (wanted.includes("community")) return card.includes("community");
+  return card.includes(wanted);
+}
+
+
+function looksLikeBlockedPage(status, html) {
+  if (status >= 400) return true;
+  const body = (html ?? "").toLowerCase();
+  return BLOCKED_HINTS.some((hint) => body.includes(hint.toLowerCase()));
+}
+
+
+function resolveResearchUrlFromHome(homeHtml, homeUrl) {
+  const root = _deps.parseHtml(homeHtml);
+  const anchors = root.querySelectorAll("a[href]");
+  for (const anchor of anchors) {
+    const text = normalizeText(anchor.textContent).toLowerCase();
+    if (!text || !text.includes("research")) continue;
+    const link = toAbsoluteHttpUrl(anchor.getAttribute("href"), homeUrl);
+    if (!link) continue;
+    try {
+      const url = new URL(link);
+      if (!/(^|\.)csail\.mit\.edu$/i.test(url.hostname)) continue;
+      const pathname = url.pathname.replace(/\/+$/, "") || "/";
+      if (pathname !== CSAIL_RESEARCH_PATH) continue;
+      url.search = "";
+      url.hash = "";
+      return url.href;
+    } catch {
+      // ignore malformed link
+    }
+  }
+  return new URL(CSAIL_RESEARCH_PATH, homeUrl).href;
+}
+
+
+function getLeafTexts(node) {
+  return node
+    .querySelectorAll(SUMMARY_SELECTOR)
+    .filter((el) => el.querySelector(SUMMARY_SELECTOR) == null)
+    .map((el) => normalizeText(el.textContent))
+    .filter(Boolean);
+}
+
+
+function extractSummary(card, title, category) {
+  const texts = getLeafTexts(card)
+    .filter((t) => t !== title && t !== category)
+    .filter((t) => !/^lead$/i.test(t))
+    .filter((t) => !/^\+\s*\d+$/i.test(t))
+    .filter((t) => !/^\.{3}more$/i.test(t));
+
+  return texts.find((t) => t.length >= 20 && t.length <= 600);
+}
+
+
+function extractAuthor(card) {
+  const names = card.querySelectorAll("app-lead-bar a[href]")
+    .map((a) => normalizeText(a.textContent))
+    .filter(Boolean)
+    .filter((name) => name !== "...more");
+  const unique = [...new Set(names)];
+  return unique.length > 0 ? unique.join(", ") : undefined;
+}
+
+
+function parseCardItem(card, baseUrl, requestedCategory) {
+  const bookmark =
+    card.querySelector('a[rel="bookmark"][href]') ??
+    card.querySelector('a[href*="/research/"]');
+  if (!bookmark) return null;
+
+  const title = normalizeText(bookmark.querySelector("h2")?.textContent || bookmark.textContent);
+  if (!title) return null;
+
+  const link = toAbsoluteHttpUrl(bookmark.getAttribute("href"), baseUrl);
+  if (!link) return null;
+
+  const category = normalizeText(card.querySelector("h4")?.textContent);
+  if (!matchesRequestedCategory(category, requestedCategory)) return null;
+
+  const summary = extractSummary(card, title, category);
+  const author = extractAuthor(card);
+
+  return {
+    guid: hashGuid(link),
+    title,
+    link,
+    pubDate: new Date(),
+    author,
+    summary: summary || undefined,
+  };
+}
+
+
+function parseItems(html, finalUrl, requestedCategory) {
+  const root = _deps.parseHtml(html);
+  const cards = root.querySelectorAll("article");
+  const seen = new Set();
+  const items = [];
+
+  for (const card of cards) {
+    const item = parseCardItem(card, finalUrl, requestedCategory);
+    if (!item) continue;
+    if (seen.has(item.link)) continue;
+    seen.add(item.link);
+    items.push(item);
+  }
+
+  return items;
+}
+
+
+async function fetchItems(sourceId, ctx) {
+  _deps = ctx.deps;
+  const sourceUrl = new URL(sourceId);
+  const requestedCategory = sourceUrl.searchParams.get("category") ?? "";
+
+  const home = await ctx.fetchHtml(CSAIL_HOME_URL, { waitMs: 3000 });
+  if (looksLikeBlockedPage(home.status, home.html)) {
+    throw new Error(`[${SITE_ID}] \u8bbf\u95ee CSAIL \u9996\u9875\u88ab\u62d2\u7edd\uff08HTTP ${home.status}\uff09`);
+  }
+
+  const researchUrl = resolveResearchUrlFromHome(home.html, home.finalUrl || CSAIL_HOME_URL);
+  const research = await ctx.fetchHtml(researchUrl, { waitMs: 4500 });
+  if (looksLikeBlockedPage(research.status, research.html)) {
+    throw new Error(`[${SITE_ID}] \u8bbf\u95ee research \u5217\u8868\u88ab\u62d2\u7edd\uff08HTTP ${research.status}\uff09`);
+  }
+
+  let items = parseItems(research.html, research.finalUrl || researchUrl, requestedCategory);
+  if (items.length === 0) {
+    const retry = await ctx.fetchHtml(researchUrl, { waitMs: 6500 });
+    if (looksLikeBlockedPage(retry.status, retry.html)) {
+      throw new Error(`[${SITE_ID}] \u8bbf\u95ee research \u5217\u8868\u88ab\u62d2\u7edd\uff08HTTP ${retry.status}\uff09`);
+    }
+    items = parseItems(retry.html, retry.finalUrl || researchUrl, requestedCategory);
+  }
+  if (items.length === 0) {
+    const withCategory = requestedCategory ? ` (category=${requestedCategory})` : "";
+    throw new Error(`[${SITE_ID}] \u672a\u89e3\u6790\u5230\u7814\u7a76\u6761\u76ee${withCategory}\uff0c\u9875\u9762\u7ed3\u6784\u53ef\u80fd\u5df2\u53d8\u5316`);
+  }
+  return items;
+}
+
+
+export default {
+  id: SITE_ID,
+  listUrlPattern: /^https?:\/\/(www\.)?csail\.mit\.edu\/research(?:\?.*)?$/i,
+  fetchItems,
+};

package/app/plugins/builtin/moonshot.rssany.js

@@ -0,0 +1,127 @@
+let _deps;
+
+// Moonshot 官方站插件：抓取首页“最新研究”列表，输出 FeedItem（不含 enrich）
+
+
+
+const DATE_RE = /^(\d{4})-(\d{1,2})-(\d{1,2})$/;
+const RESEARCH_HEADING_RE = /最新研究|latest\s+research/i;
+
+
+function normalizeText(text) {
+  return (text ?? "").replace(/\s+/g, " ").trim();
+}
+
+
+function hashGuid(input) {
+  return _deps.createHash("sha256").update(input).digest("hex");
+}
+
+
+function toAbsoluteHttpUrl(rawHref, baseUrl) {
+  if (!rawHref) return null;
+  const href = rawHref.trim();
+  if (!href || href.startsWith("#") || href.startsWith("javascript:")) return null;
+  try {
+    const url = new URL(href, baseUrl);
+    if (!/^https?:$/i.test(url.protocol)) return null;
+    return url.href;
+  } catch {
+    return null;
+  }
+}
+
+
+function parseDate(dateText) {
+  const normalized = normalizeText(dateText);
+  const m = normalized.match(DATE_RE);
+  if (!m) return undefined;
+  const [, y, mm, dd] = m;
+  const date = new Date(Date.UTC(Number(y), Number(mm) - 1, Number(dd), 12, 0, 0));
+  if (Number.isNaN(date.getTime())) return undefined;
+  return date;
+}
+
+
+function extractTitleParts(anchor) {
+  const parts = anchor
+    .querySelectorAll("h2")
+    .map((el) => normalizeText(el.textContent))
+    .filter(Boolean);
+  if (parts.length > 0) return parts;
+
+  const fallback = normalizeText(anchor.textContent);
+  return fallback ? [fallback] : [];
+}
+
+
+function parseResearchItem(anchor, finalUrl) {
+  const link = toAbsoluteHttpUrl(anchor.getAttribute("href"), finalUrl);
+  if (!link) return null;
+
+  const titleParts = extractTitleParts(anchor);
+  if (titleParts.length === 0) return null;
+
+  const dateText = anchor
+    .querySelectorAll("p")
+    .map((p) => normalizeText(p.textContent))
+    .find((t) => DATE_RE.test(t));
+  if (!dateText) return null;
+
+  const pubDate = parseDate(dateText) ?? new Date();
+  const title = titleParts[0];
+  const subtitle = titleParts.length > 1 ? titleParts.slice(1).join(" ") : "";
+
+  return {
+    guid: hashGuid(link),
+    title,
+    link,
+    pubDate,
+    summary: subtitle || undefined,
+  };
+}
+
+
+function collectCandidateAnchors(root) {
+  const heading = root
+    .querySelectorAll("h1, h2, h3")
+    .find((node) => RESEARCH_HEADING_RE.test(normalizeText(node.textContent)));
+
+  if (heading?.parentNode && "querySelectorAll" in heading.parentNode) {
+    return heading.parentNode.querySelectorAll("a[href]");
+  }
+
+  return root.querySelectorAll("a[href]");
+}
+
+
+async function fetchItems(sourceId, ctx) {
+  _deps = ctx.deps;
+  const { html, finalUrl } = await ctx.fetchHtml(sourceId, { waitMs: 4500 });
+  const root = _deps.parseHtml(html);
+
+  const seen = new Set();
+  const items = [];
+  const anchors = collectCandidateAnchors(root);
+
+  for (const anchor of anchors) {
+    const item = parseResearchItem(anchor, finalUrl);
+    if (!item) continue;
+    if (seen.has(item.link)) continue;
+    seen.add(item.link);
+    items.push(item);
+  }
+
+  if (items.length === 0) {
+    throw new Error("[moonshot] 未解析到“最新研究”条目，页面结构可能已变化");
+  }
+
+  return items;
+}
+
+
+export default {
+  id: "moonshot",
+  listUrlPattern: /^https?:\/\/(www\.)?moonshot\.ai(?:\/[a-z]{2}(?:-[a-z]{2})?)?\/?(\?.*)?$/i,
+  fetchItems,
+};

package/app/plugins/builtin/opendatalab-news.rssany.js

@@ -0,0 +1,174 @@
+let _deps;
+
+
+const API_ROOT = "https://static.openxlab.org.cn/opendatalab/dynamics";
+const DETAIL_ROOT = "https://opendatalab.org.cn/news/details";
+
+const SOURCES = [
+  {
+    label: "featuredArticles",
+    url: `${API_ROOT}/featuredArticles/data.json?t=12`,
+    detailType: "article",
+    category: "精选文章",
+  },
+  {
+    label: "banner",
+    url: `${API_ROOT}/banner/data.json?t=1`,
+    detailType: "banner",
+    category: "轮播",
+  },
+  {
+    label: "AItalk",
+    url: `${API_ROOT}/talkArticles/AItalk/data.json?t=1`,
+    detailType: "AItalk",
+    category: "AI Talk",
+  },
+  {
+    label: "JStalk",
+    url: `${API_ROOT}/talkArticles/JStalk/data.json?t=1`,
+    detailType: "JStalk",
+    category: "解数 Talk",
+  },
+];
+
+function normalizeText(text) {
+  return (text ?? "").replace(/\s+/g, " ").trim();
+}
+
+function hashGuid(input) {
+  return _deps.createHash("sha256").update(input).digest("hex");
+}
+
+function parsePubDate(rawDate, rawStartTime) {
+  const dateText = normalizeText(rawDate);
+  const dateMatch = dateText.match(/(\d{4})[/-](\d{1,2})[/-](\d{1,2})/);
+  if (!dateMatch) return new Date();
+
+  const year = Number(dateMatch[1]);
+  const month = Number(dateMatch[2]);
+  const day = Number(dateMatch[3]);
+  const monthText = String(month).padStart(2, "0");
+  const dayText = String(day).padStart(2, "0");
+
+  const timeText = normalizeText(rawStartTime);
+  const timeMatch = timeText.match(/(\d{1,2}):(\d{2})/);
+
+  if (timeMatch) {
+    const hour = Number(timeMatch[1]);
+    const minute = Number(timeMatch[2]);
+    if ([year, month, day, hour, minute].every(Number.isFinite)) {
+      const hourText = String(hour).padStart(2, "0");
+      const minuteText = String(minute).padStart(2, "0");
+      return new Date(`${year}-${monthText}-${dayText}T${hourText}:${minuteText}:00+08:00`);
+    }
+  }
+
+  if ([year, month, day].every(Number.isFinite)) {
+    return new Date(`${year}-${monthText}-${dayText}T12:00:00+08:00`);
+  }
+
+  return new Date();
+}
+
+function buildSummary(item) {
+  const dateText = normalizeText(item?.date);
+  const start = normalizeText(item?.start_time);
+  const end = normalizeText(item?.end_time);
+  if (!dateText) return undefined;
+  if (start && end) return `直播时间: ${dateText} ${start}-${end}`;
+  return `发布时间: ${dateText}`;
+}
+
+async function fetchJson(url, label) {
+  let response;
+  try {
+    response = await fetch(url, {
+      headers: {
+        "Accept": "application/json,text/plain,*/*",
+        "User-Agent": "RssAny/1.0 (+https://github.com/rssany/rssany)",
+      },
+    });
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err);
+    throw new Error(`[opendatalab-news] 请求 ${label} 失败: ${message}`);
+  }
+
+  if (!response.ok) {
+    throw new Error(`[opendatalab-news] 请求 ${label} 失败: HTTP ${response.status}`);
+  }
+
+  let data;
+  try {
+    data = await response.json();
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err);
+    throw new Error(`[opendatalab-news] 解析 ${label} JSON 失败: ${message}`);
+  }
+
+  if (!Array.isArray(data)) {
+    throw new Error(`[opendatalab-news] ${label} 返回结构异常，期望数组`);
+  }
+
+  return data;
+}
+
+function mapItems(records, source) {
+  const items = [];
+
+  for (const raw of records) {
+    if (typeof raw !== "object" || raw == null) continue;
+
+    const id = Number(raw.id);
+    const title = normalizeText(raw.title);
+    if (!Number.isFinite(id) || !title) continue;
+
+    const link = `${DETAIL_ROOT}/${source.detailType}/${id}`;
+    const pubDate = parsePubDate(raw.date, raw.start_time);
+    const summary = buildSummary(raw);
+
+    items.push({
+      guid: hashGuid(link),
+      title,
+      link,
+      pubDate,
+      summary,
+      sourceId: "opendatalab-news",
+    });
+  }
+
+  return items;
+}
+
+async function fetchItems(_sourceId, _ctx) {
+  _deps = _ctx.deps;
+  const collected = [];
+
+  for (const source of SOURCES) {
+    const records = await fetchJson(source.url, source.label);
+    collected.push(...mapItems(records, source));
+  }
+
+  const deduped = [];
+  const seen = new Set();
+
+  for (const item of collected) {
+    if (seen.has(item.link)) continue;
+    seen.add(item.link);
+    deduped.push(item);
+  }
+
+  deduped.sort((a, b) => b.pubDate.getTime() - a.pubDate.getTime());
+
+  if (deduped.length === 0) {
+    throw new Error("[opendatalab-news] 未解析到条目，页面数据源可能已变化");
+  }
+
+  return deduped;
+}
+
+export default {
+  id: "opendatalab-news",
+  listUrlPattern: /^https?:\/\/(www\.)?opendatalab\.(org\.cn|com)\/news\/?(\?.*)?$/i,
+  refreshInterval: "1h",
+  fetchItems,
+};

package/app/plugins/builtin/opendatalab.rssany.js

@@ -0,0 +1,109 @@
+let _deps;
+
+
+const OPENDATALAB_ORIGIN = "https://opendatalab.org.cn";
+const OPENDATALAB_LIST_API = `${OPENDATALAB_ORIGIN}/datasets/api/v3/datasets/list`;
+
+function normalizeText(text) {
+  return (text ?? "").replace(/\s+/g, " ").trim();
+}
+
+function hashGuid(input) {
+  return _deps.createHash("sha256").update(input).digest("hex");
+}
+
+function toDate(value) {
+  const raw = typeof value === "string" ? Number(value) : value;
+  if (typeof raw !== "number" || !Number.isFinite(raw) || raw <= 0) return undefined;
+  const ms = raw < 1e12 ? raw * 1000 : raw;
+  const date = new Date(ms);
+  return Number.isNaN(date.getTime()) ? undefined : date;
+}
+
+function pickPubDate(record) {
+  const candidates = [record?.lastUpdateTime, record?.updatedAt, record?.publicTime, record?.createdAt];
+  for (const value of candidates) {
+    const parsed = toDate(value);
+    if (parsed) return parsed;
+  }
+  return new Date();
+}
+
+function toDatasetLink(name) {
+  const normalized = normalizeText(name);
+  if (!normalized) return null;
+  const parts = normalized.split("/").map((x) => normalizeText(x)).filter(Boolean);
+  if (parts.length < 2) return null;
+  const encodedPath = parts.map((x) => encodeURIComponent(x)).join("/");
+  return `${OPENDATALAB_ORIGIN}/${encodedPath}`;
+}
+
+function toFeedItem(record) {
+  if (!record || typeof record !== "object") return null;
+  const title = normalizeText(record.displayName || record.name || "");
+  const link = toDatasetLink(record.name);
+  if (!title || !link) return null;
+
+  const summary = normalizeText(record?.introduction?.zh || record?.introduction?.en || "");
+  const author = normalizeText(record?.createdBy?.name || record?.updatedBy || "");
+
+  return {
+    guid: hashGuid(link),
+    title,
+    link,
+    pubDate: pickPubDate(record),
+    author: author || undefined,
+    summary: summary || undefined,
+    sourceId: "opendatalab",
+  };
+}
+
+function parsePaginationFromSourceId(sourceId) {
+  const defaults = { pageNo: 1, pageSize: 30 };
+  try {
+    const url = new URL(sourceId);
+    const pageNo = Number(url.searchParams.get("pageNo") ?? defaults.pageNo);
+    const pageSize = Number(url.searchParams.get("pageSize") ?? defaults.pageSize);
+    const safePageNo = Number.isInteger(pageNo) && pageNo > 0 ? pageNo : defaults.pageNo;
+    const safePageSize = Number.isInteger(pageSize) && pageSize >= 1 && pageSize <= 100 ? pageSize : defaults.pageSize;
+    return { pageNo: safePageNo, pageSize: safePageSize };
+  } catch {
+    return defaults;
+  }
+}
+
+async function fetchItems(sourceId, ctx) {
+  _deps = ctx.deps;
+  const { pageNo, pageSize } = parsePaginationFromSourceId(sourceId);
+  const response = await fetch(OPENDATALAB_LIST_API, {
+    method: "POST",
+    headers: {
+      "Content-Type": "application/json",
+      "Accept": "application/json",
+    },
+    body: JSON.stringify({ pageNo, pageSize }),
+  });
+
+  if (!response.ok) {
+    throw new Error(`[opendatalab] 请求列表接口失败: HTTP ${response.status}`);
+  }
+
+  const payload = await response.json().catch(() => null);
+  const list = payload?.data?.list;
+  if (!Array.isArray(list)) {
+    const msg = normalizeText(payload?.msg) || "返回结构异常";
+    throw new Error(`[opendatalab] 列表接口响应不可用: ${msg}`);
+  }
+
+  const items = list.map((record) => toFeedItem(record)).filter(Boolean);
+  if (items.length === 0) {
+    throw new Error("[opendatalab] 未解析到条目，接口结构可能已变化");
+  }
+  return items;
+}
+
+export default {
+  id: "opendatalab",
+  listUrlPattern: /^https?:\/\/(www\.)?opendatalab\.(org\.cn|com)\/?(?:datasets\/?)?(?:\?.*)?$/i,
+  fetchItems,
+};