rssany 0.1.4 → 0.1.5

package/app/plugins/builtin/google-deepmind-research.rssany.js

@@ -0,0 +1,223 @@
+let _deps;
+
+// Google DeepMind Research 插件：抓取 research 页面中的最新研究条目（不做 enrich）
+
+
+
+const DEEPMIND_RESEARCH_URL = "https://deepmind.google/research/";
+const DEEPMIND_ORIGIN = "https://deepmind.google";
+const MONTH_TO_INDEX = {
+  january: 0,
+  february: 1,
+  march: 2,
+  april: 3,
+  may: 4,
+  june: 5,
+  july: 6,
+  august: 7,
+  september: 8,
+  october: 9,
+  november: 10,
+  december: 11,
+};
+
+
+function normalizeText(text) {
+  return (text ?? "").replace(/\s+/g, " ").trim();
+}
+
+
+function hashGuid(input) {
+  return _deps.createHash("sha256").update(input).digest("hex");
+}
+
+
+function toAbsoluteHttpUrl(rawHref, baseUrl) {
+  if (!rawHref) return null;
+  const href = rawHref.trim();
+  if (!href || href.startsWith("#") || href.startsWith("javascript:")) return null;
+  try {
+    const url = new URL(href, baseUrl);
+    if (!/^https?:$/i.test(url.protocol)) return null;
+    return url.href;
+  } catch {
+    return null;
+  }
+}
+
+
+function parsePubDate(rawDate) {
+  const normalized = normalizeText(rawDate);
+  if (!normalized) return undefined;
+
+  const monthYear = normalized.match(/^([A-Za-z]+)\s+(\d{4})$/);
+  if (monthYear) {
+    const monthName = monthYear[1]?.toLowerCase();
+    const year = Number(monthYear[2]);
+    const month = monthName ? MONTH_TO_INDEX[monthName] : undefined;
+    if (month != null && Number.isFinite(year)) {
+      return new Date(Date.UTC(year, month, 1, 12, 0, 0));
+    }
+  }
+
+  const direct = new Date(normalized);
+  if (!Number.isNaN(direct.getTime())) return direct;
+  return undefined;
+}
+
+
+function isResearchLink(link) {
+  let url;
+  try {
+    url = new URL(link);
+  } catch {
+    return false;
+  }
+
+  if (url.hostname === "deepmind.google") {
+    if (/^\/research\/publications\/\d+\/?$/i.test(url.pathname)) return true;
+    if (/^\/blog\/[^?#]+/i.test(url.pathname)) return true;
+    return false;
+  }
+
+  if (url.hostname === "blog.google") {
+    return /^\/technology\/google-deepmind\/[^?#]+/i.test(url.pathname);
+  }
+
+  return false;
+}
+
+
+function extractTitle(container) {
+  const heading =
+    container.querySelector("h1") ??
+    container.querySelector("h2") ??
+    container.querySelector("h3") ??
+    container.querySelector("h4");
+  const title = normalizeText(heading?.textContent);
+  if (title) return title;
+  return "";
+}
+
+
+function extractSummary(container, title) {
+  const summary = normalizeText(container.querySelector("p")?.textContent);
+  if (!summary) return undefined;
+  if (summary === title) return undefined;
+  return summary;
+}
+
+
+function isCallToActionTitle(title) {
+  return /^(learn|view|see|read|watch)\b/i.test(title);
+}
+
+
+function parseItemsFromArticles(root, baseUrl) {
+  const items = [];
+  const seen = new Set();
+  const articles = root.querySelectorAll("article");
+
+  for (const article of articles) {
+    const title = extractTitle(article);
+    if (!title || isCallToActionTitle(title)) continue;
+
+    const anchor = article.querySelector("a[href]");
+    const link = toAbsoluteHttpUrl(anchor?.getAttribute("href"), baseUrl);
+    if (!link || !isResearchLink(link)) continue;
+    if (seen.has(link)) continue;
+    seen.add(link);
+
+    const dateRaw =
+      article.querySelector("time")?.getAttribute("datetime") ??
+      article.querySelector("time")?.textContent ??
+      "";
+    const pubDate = parsePubDate(dateRaw) ?? new Date();
+    const summary = extractSummary(article, title);
+
+    items.push({
+      guid: hashGuid(link),
+      title,
+      link,
+      pubDate,
+      author: "Google DeepMind",
+      summary,
+      sourceId: "google-deepmind-research",
+    });
+  }
+
+  return items;
+}
+
+
+function findTitleAroundAnchor(anchor) {
+  const inlineHeading =
+    anchor.querySelector("h1, h2, h3, h4") ??
+    anchor.parentNode?.querySelector?.("h1, h2, h3, h4");
+  const title = normalizeText(inlineHeading?.textContent);
+  if (title) return title;
+
+  const text = normalizeText(anchor.textContent);
+  if (text && text.length >= 8 && !isCallToActionTitle(text)) return text;
+  return "";
+}
+
+
+function parseItemsFromAnchors(root, baseUrl) {
+  const items = [];
+  const seen = new Set();
+  const anchors = root.querySelectorAll("a[href]");
+
+  for (const anchor of anchors) {
+    const link = toAbsoluteHttpUrl(anchor.getAttribute("href"), baseUrl);
+    if (!link || !isResearchLink(link)) continue;
+    if (seen.has(link)) continue;
+
+    const title = findTitleAroundAnchor(anchor);
+    if (!title) continue;
+
+    const container = anchor.parentNode ?? anchor;
+    const dateRaw =
+      container.querySelector?.("time")?.getAttribute?.("datetime") ??
+      container.querySelector?.("time")?.textContent ??
+      "";
+    const pubDate = parsePubDate(dateRaw) ?? new Date();
+    const summary = extractSummary(container, title);
+
+    seen.add(link);
+    items.push({
+      guid: hashGuid(link),
+      title,
+      link,
+      pubDate,
+      author: "Google DeepMind",
+      summary,
+      sourceId: "google-deepmind-research",
+    });
+  }
+
+  return items;
+}
+
+
+async function fetchItems(sourceId, ctx) {
+  _deps = ctx.deps;
+  const { html, finalUrl } = await ctx.fetchHtml(sourceId, { waitMs: 4500 });
+  const root = _deps.parseHtml(html);
+  const baseUrl = finalUrl || DEEPMIND_ORIGIN;
+
+  const fromArticles = parseItemsFromArticles(root, baseUrl);
+  if (fromArticles.length > 0) return fromArticles;
+
+  const fromAnchors = parseItemsFromAnchors(root, baseUrl);
+  if (fromAnchors.length > 0) return fromAnchors;
+
+  throw new Error("[google-deepmind-research] 未解析到研究条目，页面结构可能已变化");
+}
+
+
+export default {
+  id: "google-deepmind-research",
+  listUrlPattern: /^https?:\/\/deepmind\.google\/research\/?(?:\?.*)?$/i,
+  fetchItems,
+};

package/app/plugins/builtin/google-research-datasets.rssany.js

@@ -0,0 +1,171 @@
+let _deps;
+
+
+
+const GOOGLE_RESEARCH_AUTHOR = "Google Research Datasets";
+const DATASETS_URL = "https://research.google/resources/datasets/";
+const MIN_SUMMARY_LENGTH = 24;
+
+
+function normalizeText(text) {
+  return (text ?? "").replace(/\s+/g, " ").trim();
+}
+
+
+function hashGuid(input) {
+  return _deps.createHash("sha256").update(input).digest("hex");
+}
+
+
+function isGoogleHost(hostname) {
+  return /^([a-z0-9-]+\.)*google\.[a-z.]+$/i.test(hostname);
+}
+
+
+function resolveHttpUrl(rawHref, baseUrl) {
+  if (!rawHref) return null;
+  const href = rawHref.trim();
+  if (!href || href.startsWith("#") || href.startsWith("javascript:")) return null;
+
+  try {
+    const url = new URL(href, baseUrl);
+    if (!/^https?:$/i.test(url.protocol)) return null;
+    return url;
+  } catch {
+    return null;
+  }
+}
+
+
+function resolveResultLink(rawHref, baseUrl) {
+  const url = resolveHttpUrl(rawHref, baseUrl);
+  if (!url) return null;
+
+  if (isGoogleHost(url.hostname) && url.pathname === "/url") {
+    const target = resolveHttpUrl(url.searchParams.get("q") ?? url.searchParams.get("url"), url.href);
+    return target?.href ?? null;
+  }
+  return url.href;
+}
+
+
+function dedupeTexts(texts) {
+  const out = [];
+  const seen = new Set();
+  for (const text of texts) {
+    const normalized = normalizeText(text);
+    if (!normalized) continue;
+    const key = normalized.toLowerCase();
+    if (seen.has(key)) continue;
+    seen.add(key);
+    out.push(normalized);
+  }
+  return out;
+}
+
+
+function extractLeafTexts(anchor) {
+  const leaves = anchor
+    .querySelectorAll("h1,h2,h3,h4,h5,h6,p,span,div")
+    .filter((node) => node.querySelector("h1,h2,h3,h4,h5,h6,p,span,div") == null)
+    .map((node) => normalizeText(node.textContent))
+    .filter(Boolean);
+  return dedupeTexts(leaves);
+}
+
+
+function parseYearFromText(text) {
+  if (!text) return undefined;
+  const matches = text.match(/\b20\d{2}\b/g) ?? [];
+  if (matches.length === 0) return undefined;
+  const currentYear = new Date().getUTCFullYear();
+  const years = matches
+    .map((x) => Number(x))
+    .filter((x) => Number.isFinite(x))
+    .filter((x) => x >= 2000 && x <= currentYear + 1);
+  if (years.length === 0) return undefined;
+  return Math.max(...years);
+}
+
+
+function buildItem(title, link, summary, index) {
+  const year = parseYearFromText(`${title} ${summary ?? ""}`);
+  const pubDate = year == null
+    ? new Date(Date.now() - index * 1000)
+    : new Date(Date.UTC(year, 0, 1, 12, 0, 0));
+  return {
+    guid: hashGuid(link),
+    title,
+    link,
+    pubDate,
+    author: GOOGLE_RESEARCH_AUTHOR,
+    summary: summary || undefined,
+  };
+}
+
+
+function parseFromPurifiedHtml(html, finalUrl) {
+  const root = _deps.parseHtml(html);
+  const anchors = root.querySelectorAll("a[href]");
+  const seenLinks = new Set();
+  const items = [];
+
+  for (const anchor of anchors) {
+    const link = resolveResultLink(anchor.getAttribute("href"), finalUrl);
+    if (!link || seenLinks.has(link)) continue;
+
+    const texts = extractLeafTexts(anchor);
+    if (texts.length < 2) continue;
+    const title = texts[0];
+    const summary = texts.find((text) => text !== title && text.length >= MIN_SUMMARY_LENGTH);
+    if (!title || !summary) continue;
+
+    seenLinks.add(link);
+    items.push(buildItem(title, link, summary, items.length));
+  }
+
+  return items;
+}
+
+
+function parseFromRawHtml(html, finalUrl) {
+  const root = _deps.parseHtml(html);
+  const anchors = root.querySelectorAll("a.row-card[href]");
+  const seenLinks = new Set();
+  const items = [];
+
+  for (const anchor of anchors) {
+    const link = resolveResultLink(anchor.getAttribute("href"), finalUrl);
+    if (!link || seenLinks.has(link)) continue;
+
+    const title = normalizeText(anchor.querySelector(".row-card__heading")?.textContent);
+    const summary = normalizeText(anchor.querySelector(".row-card__subheading__item")?.textContent);
+    if (!title || !summary) continue;
+
+    seenLinks.add(link);
+    items.push(buildItem(title, link, summary, items.length));
+  }
+
+  return items;
+}
+
+
+async function fetchItems(sourceId, ctx) {
+  _deps = ctx.deps;
+  const { html, finalUrl } = await ctx.fetchHtml(sourceId, { waitMs: 3500 });
+  const fromPurified = parseFromPurifiedHtml(html, finalUrl || sourceId || DATASETS_URL);
+  if (fromPurified.length > 0) return fromPurified;
+
+  const raw = await ctx.fetchHtml(sourceId, { waitMs: 3500, purify: false });
+  const fromRaw = parseFromRawHtml(raw.html, raw.finalUrl || sourceId || DATASETS_URL);
+  if (fromRaw.length > 0) return fromRaw;
+
+  throw new Error("[google-research-datasets] 未解析到数据集条目，页面结构可能已变化");
+}
+
+
+export default {
+  id: "google-research-datasets",
+  listUrlPattern: /^https?:\/\/research\.google\/resources\/datasets\/?(\?.*)?$/i,
+  fetchItems,
+};

package/app/plugins/builtin/google-research.rssany.js

@@ -0,0 +1,220 @@
+let _deps;
+
+// Google Research 首页插件：抓取 research.google 首页中的最新博客/论文条目（不做 enrich）
+
+
+
+const BLOG_DETAIL_PATH_RE = /^\/blog\/(?!rss\/?$|label\/|\d{4}(?:\/|$))[^?#]+/i;
+const PUBS_DETAIL_PATH_RE = /^\/pubs\/(?!$|\/?$)[^?#]+/i;
+const CTA_TITLE_RE = /^(see|learn|explore|read|watch)\b/i;
+const MONTH_NAME_RE =
+  /(january|february|march|april|may|june|july|august|september|october|november|december)\s+(\d{1,2})(?:,\s*|\s+)(\d{4})/i;
+const MONTH_DAY_RE =
+  /(january|february|march|april|may|june|july|august|september|october|november|december)\s+(\d{1,2})\b/i;
+const MONTH_INDEX = {
+  january: 0,
+  february: 1,
+  march: 2,
+  april: 3,
+  may: 4,
+  june: 5,
+  july: 6,
+  august: 7,
+  september: 8,
+  october: 9,
+  november: 10,
+  december: 11,
+};
+
+
+function normalizeText(text) {
+  return (text ?? "")
+    .replace(/[\u200B-\u200D\uFEFF]/g, "")
+    .replace(/\s+/g, " ")
+    .trim();
+}
+
+
+function splitMeaningfulLines(text) {
+  return (text ?? "")
+    .split(/\n+/)
+    .map((line) => normalizeText(line))
+    .filter(Boolean)
+    .filter((line) => line !== "·");
+}
+
+
+function decodeSlug(slug) {
+  return normalizeText(
+    slug
+      .replace(/[-_]+/g, " ")
+      .replace(/\b([a-z])/g, (m) => m.toUpperCase())
+  );
+}
+
+
+function toAbsoluteHttpUrl(rawHref, pageUrl) {
+  if (!rawHref) return null;
+  const href = rawHref.trim();
+  if (!href || href.startsWith("#") || href.startsWith("javascript:")) return null;
+  try {
+    const url = new URL(href, pageUrl);
+    if (!/^https?:$/i.test(url.protocol)) return null;
+    return url;
+  } catch {
+    return null;
+  }
+}
+
+
+function isResearchItemPath(pathname) {
+  return BLOG_DETAIL_PATH_RE.test(pathname) || PUBS_DETAIL_PATH_RE.test(pathname);
+}
+
+
+function extractTitle(anchor, linkUrl) {
+  const heading = anchor.querySelector("h1, h2, h3, h4, h5, h6");
+  const headingText = normalizeText(heading?.textContent);
+  if (headingText) return headingText;
+
+  const spanText = normalizeText(anchor.querySelector("span")?.textContent);
+  if (spanText && spanText.length >= 6 && !MONTH_DAY_RE.test(spanText)) return spanText;
+
+  const lines = splitMeaningfulLines(anchor.textContent).filter((line) => {
+    if (line.length < 3) return false;
+    if (MONTH_NAME_RE.test(line) || MONTH_DAY_RE.test(line)) return false;
+    if (line.toUpperCase() === "BLOG" || line.toUpperCase() === "PUBLICATIONS") return false;
+    return true;
+  });
+  if (lines.length > 0) return lines[0];
+
+  const segments = linkUrl.pathname.split("/").filter(Boolean);
+  const slug = segments.at(-1) ?? "";
+  return decodeSlug(slug);
+}
+
+
+function parseDateFromText(text) {
+  const normalized = normalizeText(text);
+  const monthWithYear = normalized.match(MONTH_NAME_RE);
+  if (monthWithYear) {
+    const monthName = monthWithYear[1]?.toLowerCase();
+    const day = Number(monthWithYear[2]);
+    const year = Number(monthWithYear[3]);
+    const month = monthName ? MONTH_INDEX[monthName] : undefined;
+    if (month != null && Number.isFinite(day) && Number.isFinite(year)) {
+      return new Date(Date.UTC(year, month, day, 0, 0, 0));
+    }
+  }
+
+  const monthDayOnly = normalized.match(MONTH_DAY_RE);
+  if (monthDayOnly) {
+    const monthName = monthDayOnly[1]?.toLowerCase();
+    const day = Number(monthDayOnly[2]);
+    const year = new Date().getFullYear();
+    const month = monthName ? MONTH_INDEX[monthName] : undefined;
+    if (month != null && Number.isFinite(day)) {
+      return new Date(Date.UTC(year, month, day, 0, 0, 0));
+    }
+  }
+
+  const yearMatch = normalized.match(/\((19|20)\d{2}\)/);
+  if (yearMatch) {
+    const year = Number(yearMatch[0].replace(/[()]/g, ""));
+    if (year >= 1990 && year <= new Date().getFullYear() + 1) {
+      return new Date(Date.UTC(year, 0, 1, 0, 0, 0));
+    }
+  }
+  return undefined;
+}
+
+
+function extractSummary(anchor, title) {
+  const lines = splitMeaningfulLines(anchor.textContent).filter((line) => {
+    if (!line) return false;
+    if (line === title) return false;
+    if (MONTH_NAME_RE.test(line) || MONTH_DAY_RE.test(line)) return false;
+    return true;
+  });
+  if (lines.length === 0) return undefined;
+  const summary = normalizeText(lines.slice(0, 2).join(" · "));
+  return summary || undefined;
+}
+
+
+function collectContextText(anchor) {
+  let cur = anchor;
+  const chunks = [];
+  for (let i = 0; i < 4 && cur; i += 1) {
+    const text = normalizeText(cur.textContent);
+    if (text) chunks.push(text);
+    cur = cur.parentNode ?? null;
+  }
+  return chunks.join(" ");
+}
+
+
+function isCallToActionTitle(title) {
+  const normalized = normalizeText(title);
+  if (!normalized) return true;
+  if (!CTA_TITLE_RE.test(normalized)) return false;
+  return normalized.split(" ").length <= 8;
+}
+
+
+function parseItemsFromHome(html, pageUrl) {
+  const root = _deps.parseHtml(html);
+  const anchors = root.querySelectorAll("a[href]");
+  const seen = new Set();
+  const items = [];
+
+  for (const anchor of anchors) {
+    const linkUrl = toAbsoluteHttpUrl(anchor.getAttribute("href"), pageUrl);
+    if (!linkUrl) continue;
+    if (linkUrl.hostname !== "research.google") continue;
+    if (!isResearchItemPath(linkUrl.pathname)) continue;
+
+    const link = linkUrl.href;
+    if (seen.has(link)) continue;
+    seen.add(link);
+
+    const title = extractTitle(anchor, linkUrl);
+    if (!title) continue;
+    if (isCallToActionTitle(title)) continue;
+
+    const context = collectContextText(anchor);
+    const pubDate = parseDateFromText(context) ?? new Date();
+    const summary = extractSummary(anchor, title);
+
+    items.push({
+      guid: _deps.createHash("sha256").update(link).digest("hex"),
+      title,
+      link,
+      pubDate,
+      author: "Google Research",
+      summary,
+      sourceId: "google-research",
+    });
+  }
+
+  return items;
+}
+
+
+async function fetchItems(sourceId, ctx) {
+  _deps = ctx.deps;
+  const { html, finalUrl } = await ctx.fetchHtml(sourceId, { waitMs: 4000 });
+  const pageUrl = new URL(finalUrl);
+  const items = parseItemsFromHome(html, pageUrl);
+  if (items.length === 0) {
+    throw new Error("[google-research] 未解析到条目，页面结构可能已变化");
+  }
+  return items;
+}
+
+
+export default {
+  id: "google-research",
+  listUrlPattern: /^https?:\/\/research\.google\/?(?:\?.*)?$/i,
+  fetchItems,
+};