rssany 0.1.2 → 0.1.5

package/app/plugins/builtin/mila-quebec.rssany.js

@@ -0,0 +1,199 @@
+let _deps;
+
+// Mila (Quebec AI Institute) 新闻列表插件：支持首页 /en 与新闻页 /en/news
+
+
+
+const MILA_ORIGIN = "https://mila.quebec";
+const NEWS_PATH_RE = /^\/en\/news\/[^/?#]+\/?$/i;
+const NOISE_TITLE_RE = /^(read the (story|news)|see more news)$/i;
+const MONTH_TO_INDEX = {
+  jan: 0,
+  feb: 1,
+  mar: 2,
+  apr: 3,
+  may: 4,
+  jun: 5,
+  jul: 6,
+  aug: 7,
+  sep: 8,
+  oct: 9,
+  nov: 10,
+  dec: 11,
+};
+
+
+function normalizeText(text) {
+  return (text ?? "").replace(/\s+/g, " ").trim();
+}
+
+
+function hashGuid(input) {
+  return _deps.createHash("sha256").update(input).digest("hex");
+}
+
+
+function toAbsoluteHttpUrl(rawHref, baseUrl) {
+  if (!rawHref) return null;
+  const href = rawHref.trim();
+  if (!href || href.startsWith("#") || href.startsWith("javascript:")) return null;
+  try {
+    const url = new URL(href, baseUrl);
+    if (!/^https?:$/i.test(url.protocol)) return null;
+    return url.href;
+  } catch {
+    return null;
+  }
+}
+
+
+function isNewsArticleUrl(url) {
+  try {
+    return NEWS_PATH_RE.test(new URL(url).pathname);
+  } catch {
+    return false;
+  }
+}
+
+
+function extractDateFromText(text) {
+  const normalized = normalizeText(text).replace(/,/g, " ");
+  if (!normalized) return undefined;
+
+  const m = normalized.match(
+    /(?:^|\b)(\d{1,2})\s+(Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:t(?:ember)?)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\s+(\d{4})(?:\b|$)/i
+  );
+  if (!m) return undefined;
+
+  const day = Number(m[1]);
+  const monthIdx = MONTH_TO_INDEX[m[2].slice(0, 3).toLowerCase()];
+  const year = Number(m[3]);
+  if (monthIdx == null || !Number.isFinite(day) || !Number.isFinite(year)) return undefined;
+
+  const d = new Date(Date.UTC(year, monthIdx, day, 12, 0, 0));
+  if (Number.isNaN(d.getTime())) return undefined;
+  return d;
+}
+
+
+function extractDateNearNode(node) {
+  const timeDatetime = node.querySelector?.("time[datetime]")?.getAttribute("datetime");
+  if (timeDatetime) {
+    const d = new Date(timeDatetime);
+    if (!Number.isNaN(d.getTime())) return d;
+  }
+
+  let current = node;
+  for (let i = 0; i < 6 && current; i += 1) {
+    const parsed = extractDateFromText(current.textContent);
+    if (parsed) return parsed;
+    current = current.parentNode ?? null;
+  }
+  return undefined;
+}
+
+
+function scoreTitle(text) {
+  const normalized = normalizeText(text);
+  if (!normalized) return 0;
+  if (NOISE_TITLE_RE.test(normalized.toLowerCase())) return 1;
+  if (normalized.length < 5) return 2;
+  return 10 + Math.min(normalized.length, 120);
+}
+
+
+function titleFromUrl(link) {
+  try {
+    const slug = decodeURIComponent(new URL(link).pathname.split("/").filter(Boolean).pop() ?? "");
+    return normalizeText(slug.replace(/[-_]+/g, " "));
+  } catch {
+    return "Mila News";
+  }
+}
+
+
+function chooseSummary(node, title) {
+  let current = node;
+  for (let i = 0; i < 4 && current; i += 1) {
+    const candidates = current
+      .querySelectorAll?.("p")
+      ?.map((p) => normalizeText(p.textContent))
+      ?.filter(Boolean) ?? [];
+    for (const text of candidates) {
+      const lower = text.toLowerCase();
+      if (lower === title.toLowerCase()) continue;
+      if (NOISE_TITLE_RE.test(lower)) continue;
+      return text;
+    }
+    current = current.parentNode ?? null;
+  }
+  return undefined;
+}
+
+
+async function fetchItems(sourceId, ctx) {
+  _deps = ctx.deps;
+  const { html, finalUrl } = await ctx.fetchHtml(sourceId, { waitMs: 4000 });
+  const root = _deps.parseHtml(html);
+  const pageUrl = finalUrl || sourceId || MILA_ORIGIN;
+  const byLink = new Map();
+
+  for (const anchor of root.querySelectorAll("a[href]")) {
+    const link = toAbsoluteHttpUrl(anchor.getAttribute("href"), pageUrl);
+    if (!link || !isNewsArticleUrl(link)) continue;
+
+    const rawTitle = normalizeText(anchor.textContent);
+    const titleScore = scoreTitle(rawTitle);
+    const pubDate = extractDateNearNode(anchor);
+    const summary = chooseSummary(anchor, rawTitle || "");
+
+    const existing = byLink.get(link) ?? {
+      link,
+      title: "",
+      titleScore: 0,
+      pubDate: undefined,
+      summary: undefined,
+    };
+
+    if (titleScore > existing.titleScore) {
+      existing.title = rawTitle;
+      existing.titleScore = titleScore;
+    }
+    if (!existing.pubDate && pubDate) existing.pubDate = pubDate;
+    if (!existing.summary && summary) existing.summary = summary;
+
+    byLink.set(link, existing);
+  }
+
+  const items = Array.from(byLink.values()).map((entry) => {
+    const title = entry.title && !NOISE_TITLE_RE.test(entry.title.toLowerCase())
+      ? entry.title
+      : titleFromUrl(entry.link);
+    const summary = entry.summary && normalizeText(entry.summary) !== normalizeText(title)
+      ? entry.summary
+      : undefined;
+    return {
+      guid: hashGuid(entry.link),
+      title,
+      link: entry.link,
+      pubDate: entry.pubDate ?? new Date(),
+      author: "Mila",
+      summary,
+    };
+  });
+
+  items.sort((a, b) => b.pubDate.getTime() - a.pubDate.getTime());
+
+  if (items.length === 0) {
+    throw new Error("[mila-quebec] 未解析到新闻条目，页面结构可能已变化");
+  }
+
+  return items;
+}
+
+
+export default {
+  id: "mila-quebec",
+  listUrlPattern: /^https?:\/\/(www\.)?mila\.quebec\/en(?:\/news)?(?:\/)?(?:\?.*)?$/i,
+  fetchItems,
+};

package/app/plugins/builtin/mit-csail-research.rssany.js

@@ -0,0 +1,208 @@
+let _deps;
+
+// MIT CSAIL Research plugin: warm up via homepage, then parse /research list items.
+
+
+
+const SITE_ID = "mit-csail-research";
+const CSAIL_HOME_URL = "https://www.csail.mit.edu/";
+const CSAIL_RESEARCH_PATH = "/research";
+const SUMMARY_SELECTOR = "div, p, span, h2, h3, h4, a";
+const BLOCKED_HINTS = [
+  "http error 403",
+  "request to access",
+  "access denied",
+  "request denied",
+  "body class=\"neterror\"",
+  "\u60a8\u672a\u83b7\u6388\u6743\uff0c\u65e0\u6cd5\u67e5\u770b\u6b64\u7f51\u9875\u3002",
+  "\u8bbf\u95ee <span>www.csail.mit.edu</span> \u7684\u8bf7\u6c42\u906d\u5230\u62d2\u7edd",
+];
+
+
+function normalizeText(text) {
+  return (text ?? "").replace(/\s+/g, " ").trim();
+}
+
+
+function hashGuid(input) {
+  return _deps.createHash("sha256").update(input).digest("hex");
+}
+
+
+function toAbsoluteHttpUrl(rawHref, baseUrl) {
+  if (!rawHref) return null;
+  const href = rawHref.trim();
+  if (!href || href.startsWith("#") || href.startsWith("javascript:") || href.startsWith("mailto:")) return null;
+  try {
+    const url = new URL(href, baseUrl);
+    if (!/^https?:$/i.test(url.protocol)) return null;
+    return url.href;
+  } catch {
+    return null;
+  }
+}
+
+
+function normalizeCategoryToken(text) {
+  return normalizeText(text).toLowerCase().replace(/[^a-z0-9]+/g, "");
+}
+
+
+function matchesRequestedCategory(cardCategory, queryCategory) {
+  const wanted = normalizeCategoryToken(queryCategory);
+  if (!wanted) return true;
+  const card = normalizeCategoryToken(cardCategory);
+  if (!card) return false;
+
+  if (wanted.includes("group")) return card.includes("group");
+  if (wanted.includes("center")) return card.includes("center");
+  if (wanted.includes("community")) return card.includes("community");
+  return card.includes(wanted);
+}
+
+
+function looksLikeBlockedPage(status, html) {
+  if (status >= 400) return true;
+  const body = (html ?? "").toLowerCase();
+  return BLOCKED_HINTS.some((hint) => body.includes(hint.toLowerCase()));
+}
+
+
+function resolveResearchUrlFromHome(homeHtml, homeUrl) {
+  const root = _deps.parseHtml(homeHtml);
+  const anchors = root.querySelectorAll("a[href]");
+  for (const anchor of anchors) {
+    const text = normalizeText(anchor.textContent).toLowerCase();
+    if (!text || !text.includes("research")) continue;
+    const link = toAbsoluteHttpUrl(anchor.getAttribute("href"), homeUrl);
+    if (!link) continue;
+    try {
+      const url = new URL(link);
+      if (!/(^|\.)csail\.mit\.edu$/i.test(url.hostname)) continue;
+      const pathname = url.pathname.replace(/\/+$/, "") || "/";
+      if (pathname !== CSAIL_RESEARCH_PATH) continue;
+      url.search = "";
+      url.hash = "";
+      return url.href;
+    } catch {
+      // ignore malformed link
+    }
+  }
+  return new URL(CSAIL_RESEARCH_PATH, homeUrl).href;
+}
+
+
+function getLeafTexts(node) {
+  return node
+    .querySelectorAll(SUMMARY_SELECTOR)
+    .filter((el) => el.querySelector(SUMMARY_SELECTOR) == null)
+    .map((el) => normalizeText(el.textContent))
+    .filter(Boolean);
+}
+
+
+function extractSummary(card, title, category) {
+  const texts = getLeafTexts(card)
+    .filter((t) => t !== title && t !== category)
+    .filter((t) => !/^lead$/i.test(t))
+    .filter((t) => !/^\+\s*\d+$/i.test(t))
+    .filter((t) => !/^\.{3}more$/i.test(t));
+
+  return texts.find((t) => t.length >= 20 && t.length <= 600);
+}
+
+
+function extractAuthor(card) {
+  const names = card.querySelectorAll("app-lead-bar a[href]")
+    .map((a) => normalizeText(a.textContent))
+    .filter(Boolean)
+    .filter((name) => name !== "...more");
+  const unique = [...new Set(names)];
+  return unique.length > 0 ? unique.join(", ") : undefined;
+}
+
+
+function parseCardItem(card, baseUrl, requestedCategory) {
+  const bookmark =
+    card.querySelector('a[rel="bookmark"][href]') ??
+    card.querySelector('a[href*="/research/"]');
+  if (!bookmark) return null;
+
+  const title = normalizeText(bookmark.querySelector("h2")?.textContent || bookmark.textContent);
+  if (!title) return null;
+
+  const link = toAbsoluteHttpUrl(bookmark.getAttribute("href"), baseUrl);
+  if (!link) return null;
+
+  const category = normalizeText(card.querySelector("h4")?.textContent);
+  if (!matchesRequestedCategory(category, requestedCategory)) return null;
+
+  const summary = extractSummary(card, title, category);
+  const author = extractAuthor(card);
+
+  return {
+    guid: hashGuid(link),
+    title,
+    link,
+    pubDate: new Date(),
+    author,
+    summary: summary || undefined,
+  };
+}
+
+
+function parseItems(html, finalUrl, requestedCategory) {
+  const root = _deps.parseHtml(html);
+  const cards = root.querySelectorAll("article");
+  const seen = new Set();
+  const items = [];
+
+  for (const card of cards) {
+    const item = parseCardItem(card, finalUrl, requestedCategory);
+    if (!item) continue;
+    if (seen.has(item.link)) continue;
+    seen.add(item.link);
+    items.push(item);
+  }
+
+  return items;
+}
+
+
+async function fetchItems(sourceId, ctx) {
+  _deps = ctx.deps;
+  const sourceUrl = new URL(sourceId);
+  const requestedCategory = sourceUrl.searchParams.get("category") ?? "";
+
+  const home = await ctx.fetchHtml(CSAIL_HOME_URL, { waitMs: 3000 });
+  if (looksLikeBlockedPage(home.status, home.html)) {
+    throw new Error(`[${SITE_ID}] \u8bbf\u95ee CSAIL \u9996\u9875\u88ab\u62d2\u7edd\uff08HTTP ${home.status}\uff09`);
+  }
+
+  const researchUrl = resolveResearchUrlFromHome(home.html, home.finalUrl || CSAIL_HOME_URL);
+  const research = await ctx.fetchHtml(researchUrl, { waitMs: 4500 });
+  if (looksLikeBlockedPage(research.status, research.html)) {
+    throw new Error(`[${SITE_ID}] \u8bbf\u95ee research \u5217\u8868\u88ab\u62d2\u7edd\uff08HTTP ${research.status}\uff09`);
+  }
+
+  let items = parseItems(research.html, research.finalUrl || researchUrl, requestedCategory);
+  if (items.length === 0) {
+    const retry = await ctx.fetchHtml(researchUrl, { waitMs: 6500 });
+    if (looksLikeBlockedPage(retry.status, retry.html)) {
+      throw new Error(`[${SITE_ID}] \u8bbf\u95ee research \u5217\u8868\u88ab\u62d2\u7edd\uff08HTTP ${retry.status}\uff09`);
+    }
+    items = parseItems(retry.html, retry.finalUrl || researchUrl, requestedCategory);
+  }
+  if (items.length === 0) {
+    const withCategory = requestedCategory ? ` (category=${requestedCategory})` : "";
+    throw new Error(`[${SITE_ID}] \u672a\u89e3\u6790\u5230\u7814\u7a76\u6761\u76ee${withCategory}\uff0c\u9875\u9762\u7ed3\u6784\u53ef\u80fd\u5df2\u53d8\u5316`);
+  }
+  return items;
+}
+
+
+export default {
+  id: SITE_ID,
+  listUrlPattern: /^https?:\/\/(www\.)?csail\.mit\.edu\/research(?:\?.*)?$/i,
+  fetchItems,
+};

package/app/plugins/builtin/moonshot.rssany.js

@@ -0,0 +1,127 @@
+let _deps;
+
+// Moonshot 官方站插件：抓取首页“最新研究”列表，输出 FeedItem（不含 enrich）
+
+
+
+const DATE_RE = /^(\d{4})-(\d{1,2})-(\d{1,2})$/;
+const RESEARCH_HEADING_RE = /最新研究|latest\s+research/i;
+
+
+function normalizeText(text) {
+  return (text ?? "").replace(/\s+/g, " ").trim();
+}
+
+
+function hashGuid(input) {
+  return _deps.createHash("sha256").update(input).digest("hex");
+}
+
+
+function toAbsoluteHttpUrl(rawHref, baseUrl) {
+  if (!rawHref) return null;
+  const href = rawHref.trim();
+  if (!href || href.startsWith("#") || href.startsWith("javascript:")) return null;
+  try {
+    const url = new URL(href, baseUrl);
+    if (!/^https?:$/i.test(url.protocol)) return null;
+    return url.href;
+  } catch {
+    return null;
+  }
+}
+
+
+function parseDate(dateText) {
+  const normalized = normalizeText(dateText);
+  const m = normalized.match(DATE_RE);
+  if (!m) return undefined;
+  const [, y, mm, dd] = m;
+  const date = new Date(Date.UTC(Number(y), Number(mm) - 1, Number(dd), 12, 0, 0));
+  if (Number.isNaN(date.getTime())) return undefined;
+  return date;
+}
+
+
+function extractTitleParts(anchor) {
+  const parts = anchor
+    .querySelectorAll("h2")
+    .map((el) => normalizeText(el.textContent))
+    .filter(Boolean);
+  if (parts.length > 0) return parts;
+
+  const fallback = normalizeText(anchor.textContent);
+  return fallback ? [fallback] : [];
+}
+
+
+function parseResearchItem(anchor, finalUrl) {
+  const link = toAbsoluteHttpUrl(anchor.getAttribute("href"), finalUrl);
+  if (!link) return null;
+
+  const titleParts = extractTitleParts(anchor);
+  if (titleParts.length === 0) return null;
+
+  const dateText = anchor
+    .querySelectorAll("p")
+    .map((p) => normalizeText(p.textContent))
+    .find((t) => DATE_RE.test(t));
+  if (!dateText) return null;
+
+  const pubDate = parseDate(dateText) ?? new Date();
+  const title = titleParts[0];
+  const subtitle = titleParts.length > 1 ? titleParts.slice(1).join(" ") : "";
+
+  return {
+    guid: hashGuid(link),
+    title,
+    link,
+    pubDate,
+    summary: subtitle || undefined,
+  };
+}
+
+
+function collectCandidateAnchors(root) {
+  const heading = root
+    .querySelectorAll("h1, h2, h3")
+    .find((node) => RESEARCH_HEADING_RE.test(normalizeText(node.textContent)));
+
+  if (heading?.parentNode && "querySelectorAll" in heading.parentNode) {
+    return heading.parentNode.querySelectorAll("a[href]");
+  }
+
+  return root.querySelectorAll("a[href]");
+}
+
+
+async function fetchItems(sourceId, ctx) {
+  _deps = ctx.deps;
+  const { html, finalUrl } = await ctx.fetchHtml(sourceId, { waitMs: 4500 });
+  const root = _deps.parseHtml(html);
+
+  const seen = new Set();
+  const items = [];
+  const anchors = collectCandidateAnchors(root);
+
+  for (const anchor of anchors) {
+    const item = parseResearchItem(anchor, finalUrl);
+    if (!item) continue;
+    if (seen.has(item.link)) continue;
+    seen.add(item.link);
+    items.push(item);
+  }
+
+  if (items.length === 0) {
+    throw new Error("[moonshot] 未解析到“最新研究”条目，页面结构可能已变化");
+  }
+
+  return items;
+}
+
+
+export default {
+  id: "moonshot",
+  listUrlPattern: /^https?:\/\/(www\.)?moonshot\.ai(?:\/[a-z]{2}(?:-[a-z]{2})?)?\/?(\?.*)?$/i,
+  fetchItems,
+};

package/app/plugins/builtin/opendatalab-news.rssany.js

@@ -0,0 +1,174 @@
+let _deps;
+
+
+const API_ROOT = "https://static.openxlab.org.cn/opendatalab/dynamics";
+const DETAIL_ROOT = "https://opendatalab.org.cn/news/details";
+
+const SOURCES = [
+  {
+    label: "featuredArticles",
+    url: `${API_ROOT}/featuredArticles/data.json?t=12`,
+    detailType: "article",
+    category: "精选文章",
+  },
+  {
+    label: "banner",
+    url: `${API_ROOT}/banner/data.json?t=1`,
+    detailType: "banner",
+    category: "轮播",
+  },
+  {
+    label: "AItalk",
+    url: `${API_ROOT}/talkArticles/AItalk/data.json?t=1`,
+    detailType: "AItalk",
+    category: "AI Talk",
+  },
+  {
+    label: "JStalk",
+    url: `${API_ROOT}/talkArticles/JStalk/data.json?t=1`,
+    detailType: "JStalk",
+    category: "解数 Talk",
+  },
+];
+
+function normalizeText(text) {
+  return (text ?? "").replace(/\s+/g, " ").trim();
+}
+
+function hashGuid(input) {
+  return _deps.createHash("sha256").update(input).digest("hex");
+}
+
+function parsePubDate(rawDate, rawStartTime) {
+  const dateText = normalizeText(rawDate);
+  const dateMatch = dateText.match(/(\d{4})[/-](\d{1,2})[/-](\d{1,2})/);
+  if (!dateMatch) return new Date();
+
+  const year = Number(dateMatch[1]);
+  const month = Number(dateMatch[2]);
+  const day = Number(dateMatch[3]);
+  const monthText = String(month).padStart(2, "0");
+  const dayText = String(day).padStart(2, "0");
+
+  const timeText = normalizeText(rawStartTime);
+  const timeMatch = timeText.match(/(\d{1,2}):(\d{2})/);
+
+  if (timeMatch) {
+    const hour = Number(timeMatch[1]);
+    const minute = Number(timeMatch[2]);
+    if ([year, month, day, hour, minute].every(Number.isFinite)) {
+      const hourText = String(hour).padStart(2, "0");
+      const minuteText = String(minute).padStart(2, "0");
+      return new Date(`${year}-${monthText}-${dayText}T${hourText}:${minuteText}:00+08:00`);
+    }
+  }
+
+  if ([year, month, day].every(Number.isFinite)) {
+    return new Date(`${year}-${monthText}-${dayText}T12:00:00+08:00`);
+  }
+
+  return new Date();
+}
+
+function buildSummary(item) {
+  const dateText = normalizeText(item?.date);
+  const start = normalizeText(item?.start_time);
+  const end = normalizeText(item?.end_time);
+  if (!dateText) return undefined;
+  if (start && end) return `直播时间: ${dateText} ${start}-${end}`;
+  return `发布时间: ${dateText}`;
+}
+
+async function fetchJson(url, label) {
+  let response;
+  try {
+    response = await fetch(url, {
+      headers: {
+        "Accept": "application/json,text/plain,*/*",
+        "User-Agent": "RssAny/1.0 (+https://github.com/rssany/rssany)",
+      },
+    });
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err);
+    throw new Error(`[opendatalab-news] 请求 ${label} 失败: ${message}`);
+  }
+
+  if (!response.ok) {
+    throw new Error(`[opendatalab-news] 请求 ${label} 失败: HTTP ${response.status}`);
+  }
+
+  let data;
+  try {
+    data = await response.json();
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err);
+    throw new Error(`[opendatalab-news] 解析 ${label} JSON 失败: ${message}`);
+  }
+
+  if (!Array.isArray(data)) {
+    throw new Error(`[opendatalab-news] ${label} 返回结构异常，期望数组`);
+  }
+
+  return data;
+}
+
+function mapItems(records, source) {
+  const items = [];
+
+  for (const raw of records) {
+    if (typeof raw !== "object" || raw == null) continue;
+
+    const id = Number(raw.id);
+    const title = normalizeText(raw.title);
+    if (!Number.isFinite(id) || !title) continue;
+
+    const link = `${DETAIL_ROOT}/${source.detailType}/${id}`;
+    const pubDate = parsePubDate(raw.date, raw.start_time);
+    const summary = buildSummary(raw);
+
+    items.push({
+      guid: hashGuid(link),
+      title,
+      link,
+      pubDate,
+      summary,
+      sourceId: "opendatalab-news",
+    });
+  }
+
+  return items;
+}
+
+async function fetchItems(_sourceId, _ctx) {
+  _deps = _ctx.deps;
+  const collected = [];
+
+  for (const source of SOURCES) {
+    const records = await fetchJson(source.url, source.label);
+    collected.push(...mapItems(records, source));
+  }
+
+  const deduped = [];
+  const seen = new Set();
+
+  for (const item of collected) {
+    if (seen.has(item.link)) continue;
+    seen.add(item.link);
+    deduped.push(item);
+  }
+
+  deduped.sort((a, b) => b.pubDate.getTime() - a.pubDate.getTime());
+
+  if (deduped.length === 0) {
+    throw new Error("[opendatalab-news] 未解析到条目，页面数据源可能已变化");
+  }
+
+  return deduped;
+}
+
+export default {
+  id: "opendatalab-news",
+  listUrlPattern: /^https?:\/\/(www\.)?opendatalab\.(org\.cn|com)\/news\/?(\?.*)?$/i,
+  refreshInterval: "1h",
+  fetchItems,
+};