rssany 0.1.2 → 0.1.5

package/app/plugins/builtin/flageval-news.rssany.js

@@ -0,0 +1,118 @@
+let _deps;
+
+
+
+const FLAGEVAL_ORIGIN = "https://flageval.baai.ac.cn";
+const FLAGEVAL_NEWS_API = `${FLAGEVAL_ORIGIN}/api/news/?page=1&pageSize=10`;
+const MAX_PAGES = 5;
+
+
+function normalizeText(text) {
+  return (text ?? "").replace(/\s+/g, " ").trim();
+}
+
+
+function hashGuid(input) {
+  return _deps.createHash("sha256").update(input).digest("hex");
+}
+
+
+function parseDate(rawValue) {
+  const value = normalizeText(rawValue);
+  if (!value) return new Date();
+  const date = new Date(value);
+  return Number.isNaN(date.getTime()) ? new Date() : date;
+}
+
+
+function resolveLink(news) {
+  const linkTo = normalizeText(news.linkTo);
+  if (linkTo) {
+    try {
+      const url = new URL(linkTo);
+      if (url.protocol === "http:" || url.protocol === "https:") return url.href;
+    } catch {
+      // ignore invalid linkTo and use fallback
+    }
+  }
+  const id = String(news.id ?? "").trim();
+  return `${FLAGEVAL_ORIGIN}/#/news/detail/${encodeURIComponent(id || "unknown")}`;
+}
+
+
+async function fetchNewsPage(url) {
+  const response = await fetch(url, {
+    redirect: "follow",
+    headers: {
+      "User-Agent":
+        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
+      "Accept": "application/json,text/plain,*/*",
+      "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
+    },
+  });
+  if (!response.ok) {
+    throw new Error(`[flageval-news] 拉取新闻列表失败: HTTP ${response.status}`);
+  }
+  return await response.json();
+}
+
+
+async function fetchNewsList() {
+  const all = [];
+  let next = FLAGEVAL_NEWS_API;
+
+  for (let page = 0; page < MAX_PAGES && next; page += 1) {
+    const payload = await fetchNewsPage(next);
+    const results = Array.isArray(payload?.results) ? payload.results : [];
+    all.push(...results);
+    next = typeof payload?.next === "string" && payload.next ? payload.next : "";
+  }
+
+  return all;
+}
+
+
+async function fetchItems(_sourceId, _ctx) {
+  _deps = _ctx.deps;
+  const newsList = await fetchNewsList();
+  const seen = new Set();
+  const items = [];
+
+  for (const news of newsList) {
+    if (typeof news !== "object" || news == null) continue;
+
+    const title = normalizeText(news.title);
+    if (!title) continue;
+
+    const link = resolveLink(news);
+    const id = String(news.id ?? "").trim();
+    const guid = hashGuid(id ? `${id}|${link}` : link);
+    if (seen.has(guid)) continue;
+    seen.add(guid);
+
+    const summary = normalizeText(news.description) || undefined;
+    const pubDate = parseDate(news.publishedAt ?? news.updatedAt ?? news.createdAt);
+
+    items.push({
+      guid,
+      title,
+      link,
+      pubDate,
+      summary,
+      sourceId: "flageval-news",
+    });
+  }
+
+  if (items.length === 0) {
+    throw new Error("[flageval-news] 未解析到新闻条目，接口结构可能已变化");
+  }
+
+  return items;
+}
+
+
+export default {
+  id: "flageval-news",
+  listUrlPattern: /^https?:\/\/flageval\.baai\.ac\.cn\/#\/news(?:[/?].*)?$/i,
+  fetchItems,
+};

package/app/plugins/builtin/google-deepmind-research.rssany.js

@@ -0,0 +1,223 @@
+let _deps;
+
+// Google DeepMind Research 插件：抓取 research 页面中的最新研究条目（不做 enrich）
+
+
+
+const DEEPMIND_RESEARCH_URL = "https://deepmind.google/research/";
+const DEEPMIND_ORIGIN = "https://deepmind.google";
+const MONTH_TO_INDEX = {
+  january: 0,
+  february: 1,
+  march: 2,
+  april: 3,
+  may: 4,
+  june: 5,
+  july: 6,
+  august: 7,
+  september: 8,
+  october: 9,
+  november: 10,
+  december: 11,
+};
+
+
+function normalizeText(text) {
+  return (text ?? "").replace(/\s+/g, " ").trim();
+}
+
+
+function hashGuid(input) {
+  return _deps.createHash("sha256").update(input).digest("hex");
+}
+
+
+function toAbsoluteHttpUrl(rawHref, baseUrl) {
+  if (!rawHref) return null;
+  const href = rawHref.trim();
+  if (!href || href.startsWith("#") || href.startsWith("javascript:")) return null;
+  try {
+    const url = new URL(href, baseUrl);
+    if (!/^https?:$/i.test(url.protocol)) return null;
+    return url.href;
+  } catch {
+    return null;
+  }
+}
+
+
+function parsePubDate(rawDate) {
+  const normalized = normalizeText(rawDate);
+  if (!normalized) return undefined;
+
+  const monthYear = normalized.match(/^([A-Za-z]+)\s+(\d{4})$/);
+  if (monthYear) {
+    const monthName = monthYear[1]?.toLowerCase();
+    const year = Number(monthYear[2]);
+    const month = monthName ? MONTH_TO_INDEX[monthName] : undefined;
+    if (month != null && Number.isFinite(year)) {
+      return new Date(Date.UTC(year, month, 1, 12, 0, 0));
+    }
+  }
+
+  const direct = new Date(normalized);
+  if (!Number.isNaN(direct.getTime())) return direct;
+  return undefined;
+}
+
+
+function isResearchLink(link) {
+  let url;
+  try {
+    url = new URL(link);
+  } catch {
+    return false;
+  }
+
+  if (url.hostname === "deepmind.google") {
+    if (/^\/research\/publications\/\d+\/?$/i.test(url.pathname)) return true;
+    if (/^\/blog\/[^?#]+/i.test(url.pathname)) return true;
+    return false;
+  }
+
+  if (url.hostname === "blog.google") {
+    return /^\/technology\/google-deepmind\/[^?#]+/i.test(url.pathname);
+  }
+
+  return false;
+}
+
+
+function extractTitle(container) {
+  const heading =
+    container.querySelector("h1") ??
+    container.querySelector("h2") ??
+    container.querySelector("h3") ??
+    container.querySelector("h4");
+  const title = normalizeText(heading?.textContent);
+  if (title) return title;
+  return "";
+}
+
+
+function extractSummary(container, title) {
+  const summary = normalizeText(container.querySelector("p")?.textContent);
+  if (!summary) return undefined;
+  if (summary === title) return undefined;
+  return summary;
+}
+
+
+function isCallToActionTitle(title) {
+  return /^(learn|view|see|read|watch)\b/i.test(title);
+}
+
+
+function parseItemsFromArticles(root, baseUrl) {
+  const items = [];
+  const seen = new Set();
+  const articles = root.querySelectorAll("article");
+
+  for (const article of articles) {
+    const title = extractTitle(article);
+    if (!title || isCallToActionTitle(title)) continue;
+
+    const anchor = article.querySelector("a[href]");
+    const link = toAbsoluteHttpUrl(anchor?.getAttribute("href"), baseUrl);
+    if (!link || !isResearchLink(link)) continue;
+    if (seen.has(link)) continue;
+    seen.add(link);
+
+    const dateRaw =
+      article.querySelector("time")?.getAttribute("datetime") ??
+      article.querySelector("time")?.textContent ??
+      "";
+    const pubDate = parsePubDate(dateRaw) ?? new Date();
+    const summary = extractSummary(article, title);
+
+    items.push({
+      guid: hashGuid(link),
+      title,
+      link,
+      pubDate,
+      author: "Google DeepMind",
+      summary,
+      sourceId: "google-deepmind-research",
+    });
+  }
+
+  return items;
+}
+
+
+function findTitleAroundAnchor(anchor) {
+  const inlineHeading =
+    anchor.querySelector("h1, h2, h3, h4") ??
+    anchor.parentNode?.querySelector?.("h1, h2, h3, h4");
+  const title = normalizeText(inlineHeading?.textContent);
+  if (title) return title;
+
+  const text = normalizeText(anchor.textContent);
+  if (text && text.length >= 8 && !isCallToActionTitle(text)) return text;
+  return "";
+}
+
+
+function parseItemsFromAnchors(root, baseUrl) {
+  const items = [];
+  const seen = new Set();
+  const anchors = root.querySelectorAll("a[href]");
+
+  for (const anchor of anchors) {
+    const link = toAbsoluteHttpUrl(anchor.getAttribute("href"), baseUrl);
+    if (!link || !isResearchLink(link)) continue;
+    if (seen.has(link)) continue;
+
+    const title = findTitleAroundAnchor(anchor);
+    if (!title) continue;
+
+    const container = anchor.parentNode ?? anchor;
+    const dateRaw =
+      container.querySelector?.("time")?.getAttribute?.("datetime") ??
+      container.querySelector?.("time")?.textContent ??
+      "";
+    const pubDate = parsePubDate(dateRaw) ?? new Date();
+    const summary = extractSummary(container, title);
+
+    seen.add(link);
+    items.push({
+      guid: hashGuid(link),
+      title,
+      link,
+      pubDate,
+      author: "Google DeepMind",
+      summary,
+      sourceId: "google-deepmind-research",
+    });
+  }
+
+  return items;
+}
+
+
+async function fetchItems(sourceId, ctx) {
+  _deps = ctx.deps;
+  const { html, finalUrl } = await ctx.fetchHtml(sourceId, { waitMs: 4500 });
+  const root = _deps.parseHtml(html);
+  const baseUrl = finalUrl || DEEPMIND_ORIGIN;
+
+  const fromArticles = parseItemsFromArticles(root, baseUrl);
+  if (fromArticles.length > 0) return fromArticles;
+
+  const fromAnchors = parseItemsFromAnchors(root, baseUrl);
+  if (fromAnchors.length > 0) return fromAnchors;
+
+  throw new Error("[google-deepmind-research] 未解析到研究条目，页面结构可能已变化");
+}
+
+
+export default {
+  id: "google-deepmind-research",
+  listUrlPattern: /^https?:\/\/deepmind\.google\/research\/?(?:\?.*)?$/i,
+  fetchItems,
+};

package/app/plugins/builtin/google-research-datasets.rssany.js

@@ -0,0 +1,171 @@
+let _deps;
+
+
+
+const GOOGLE_RESEARCH_AUTHOR = "Google Research Datasets";
+const DATASETS_URL = "https://research.google/resources/datasets/";
+const MIN_SUMMARY_LENGTH = 24;
+
+
+function normalizeText(text) {
+  return (text ?? "").replace(/\s+/g, " ").trim();
+}
+
+
+function hashGuid(input) {
+  return _deps.createHash("sha256").update(input).digest("hex");
+}
+
+
+function isGoogleHost(hostname) {
+  return /^([a-z0-9-]+\.)*google\.[a-z.]+$/i.test(hostname);
+}
+
+
+function resolveHttpUrl(rawHref, baseUrl) {
+  if (!rawHref) return null;
+  const href = rawHref.trim();
+  if (!href || href.startsWith("#") || href.startsWith("javascript:")) return null;
+
+  try {
+    const url = new URL(href, baseUrl);
+    if (!/^https?:$/i.test(url.protocol)) return null;
+    return url;
+  } catch {
+    return null;
+  }
+}
+
+
+function resolveResultLink(rawHref, baseUrl) {
+  const url = resolveHttpUrl(rawHref, baseUrl);
+  if (!url) return null;
+
+  if (isGoogleHost(url.hostname) && url.pathname === "/url") {
+    const target = resolveHttpUrl(url.searchParams.get("q") ?? url.searchParams.get("url"), url.href);
+    return target?.href ?? null;
+  }
+  return url.href;
+}
+
+
+function dedupeTexts(texts) {
+  const out = [];
+  const seen = new Set();
+  for (const text of texts) {
+    const normalized = normalizeText(text);
+    if (!normalized) continue;
+    const key = normalized.toLowerCase();
+    if (seen.has(key)) continue;
+    seen.add(key);
+    out.push(normalized);
+  }
+  return out;
+}
+
+
+function extractLeafTexts(anchor) {
+  const leaves = anchor
+    .querySelectorAll("h1,h2,h3,h4,h5,h6,p,span,div")
+    .filter((node) => node.querySelector("h1,h2,h3,h4,h5,h6,p,span,div") == null)
+    .map((node) => normalizeText(node.textContent))
+    .filter(Boolean);
+  return dedupeTexts(leaves);
+}
+
+
+function parseYearFromText(text) {
+  if (!text) return undefined;
+  const matches = text.match(/\b20\d{2}\b/g) ?? [];
+  if (matches.length === 0) return undefined;
+  const currentYear = new Date().getUTCFullYear();
+  const years = matches
+    .map((x) => Number(x))
+    .filter((x) => Number.isFinite(x))
+    .filter((x) => x >= 2000 && x <= currentYear + 1);
+  if (years.length === 0) return undefined;
+  return Math.max(...years);
+}
+
+
+function buildItem(title, link, summary, index) {
+  const year = parseYearFromText(`${title} ${summary ?? ""}`);
+  const pubDate = year == null
+    ? new Date(Date.now() - index * 1000)
+    : new Date(Date.UTC(year, 0, 1, 12, 0, 0));
+  return {
+    guid: hashGuid(link),
+    title,
+    link,
+    pubDate,
+    author: GOOGLE_RESEARCH_AUTHOR,
+    summary: summary || undefined,
+  };
+}
+
+
+function parseFromPurifiedHtml(html, finalUrl) {
+  const root = _deps.parseHtml(html);
+  const anchors = root.querySelectorAll("a[href]");
+  const seenLinks = new Set();
+  const items = [];
+
+  for (const anchor of anchors) {
+    const link = resolveResultLink(anchor.getAttribute("href"), finalUrl);
+    if (!link || seenLinks.has(link)) continue;
+
+    const texts = extractLeafTexts(anchor);
+    if (texts.length < 2) continue;
+    const title = texts[0];
+    const summary = texts.find((text) => text !== title && text.length >= MIN_SUMMARY_LENGTH);
+    if (!title || !summary) continue;
+
+    seenLinks.add(link);
+    items.push(buildItem(title, link, summary, items.length));
+  }
+
+  return items;
+}
+
+
+function parseFromRawHtml(html, finalUrl) {
+  const root = _deps.parseHtml(html);
+  const anchors = root.querySelectorAll("a.row-card[href]");
+  const seenLinks = new Set();
+  const items = [];
+
+  for (const anchor of anchors) {
+    const link = resolveResultLink(anchor.getAttribute("href"), finalUrl);
+    if (!link || seenLinks.has(link)) continue;
+
+    const title = normalizeText(anchor.querySelector(".row-card__heading")?.textContent);
+    const summary = normalizeText(anchor.querySelector(".row-card__subheading__item")?.textContent);
+    if (!title || !summary) continue;
+
+    seenLinks.add(link);
+    items.push(buildItem(title, link, summary, items.length));
+  }
+
+  return items;
+}
+
+
+async function fetchItems(sourceId, ctx) {
+  _deps = ctx.deps;
+  const { html, finalUrl } = await ctx.fetchHtml(sourceId, { waitMs: 3500 });
+  const fromPurified = parseFromPurifiedHtml(html, finalUrl || sourceId || DATASETS_URL);
+  if (fromPurified.length > 0) return fromPurified;
+
+  const raw = await ctx.fetchHtml(sourceId, { waitMs: 3500, purify: false });
+  const fromRaw = parseFromRawHtml(raw.html, raw.finalUrl || sourceId || DATASETS_URL);
+  if (fromRaw.length > 0) return fromRaw;
+
+  throw new Error("[google-research-datasets] 未解析到数据集条目，页面结构可能已变化");
+}
+
+
+export default {
+  id: "google-research-datasets",
+  listUrlPattern: /^https?:\/\/research\.google\/resources\/datasets\/?(\?.*)?$/i,
+  fetchItems,
+};