rssany 0.1.4 → 0.1.5

package/app/plugins/builtin/google.rssany.js

@@ -0,0 +1,187 @@
+let _deps;
+
+// Google Search 插件：将搜索结果页转换为 FeedItem 列表（不含 enrich）
+
+
+function normalizeText(text) {
+  return (text ?? "").replace(/\s+/g, " ").trim();
+}
+
+function isGoogleHost(hostname) {
+  return /^([a-z0-9-]+\.)*google\.[a-z.]+$/i.test(hostname);
+}
+
+function resolveResultLink(rawHref, pageUrl) {
+  if (!rawHref) return null;
+  const href = rawHref.trim();
+  if (!href || href.startsWith("#") || href.startsWith("javascript:")) return null;
+
+  let url;
+  try {
+    url = new URL(href, pageUrl);
+  } catch {
+    return null;
+  }
+
+  if (isGoogleHost(url.hostname) && url.pathname === "/url") {
+    const q = url.searchParams.get("q") ?? url.searchParams.get("url");
+    if (!q) return null;
+    try {
+      const target = new URL(q);
+      return /^https?:$/i.test(target.protocol) ? target.href : null;
+    } catch {
+      return null;
+    }
+  }
+
+  if (!/^https?:$/i.test(url.protocol)) return null;
+  if (isGoogleHost(url.hostname)) return null;
+  return url.href;
+}
+
+function closestAnchor(node) {
+  let cur = node;
+  while (cur) {
+    if (cur.tagName?.toLowerCase() === "a") return cur;
+    cur = cur.parentNode ?? null;
+  }
+  return null;
+}
+
+function extractSnippet(startNode) {
+  const snippetSelectors = [".VwiC3b", ".IsZvec", ".MUxGbd", ".lyLwlc"];
+  let cur = startNode;
+  for (let i = 0; i < 6 && cur; i += 1) {
+    for (const sel of snippetSelectors) {
+      const el = cur.querySelector?.(sel);
+      const text = normalizeText(el?.textContent);
+      if (text) return text;
+    }
+    cur = cur.parentNode ?? null;
+  }
+  return "";
+}
+
+function looksLikeBlockedPage(root, html, finalUrl) {
+  const text = normalizeText(root.textContent).toLowerCase();
+  const body = (html ?? "").toLowerCase();
+  if (finalUrl.includes("/sorry/")) return true;
+  if (root.querySelector("#captcha-form, #recaptcha, .g-recaptcha")) return true;
+  if (body.includes("/httpservice/retry/enablejs")) return true;
+  if (body.includes("id=\"yvlrue\"") || body.includes("if you're having trouble accessing google search")) return true;
+  if (body.includes("sg_rel")) return true;
+  return text.includes("about this page") || text.includes("unusual traffic") || text.includes("captcha");
+}
+
+async function checkAuth(page, _url) {
+  try {
+    const currentUrl = page.url() || "";
+    if (currentUrl.includes("/sorry/") || currentUrl.includes("/httpservice/retry/enablejs")) return false;
+
+    const blockedDom = await page.$("#captcha-form, #recaptcha, .g-recaptcha, #yvlrue");
+    if (blockedDom) return false;
+
+    const bodyText = await page.evaluate(() => (document.body?.innerText ?? "").toLowerCase());
+    if (
+      bodyText.includes("unusual traffic") ||
+      bodyText.includes("about this page") ||
+      bodyText.includes("captcha") ||
+      bodyText.includes("if you're having trouble accessing google search")
+    ) {
+      return false;
+    }
+
+    // 首页或搜索页出现 q 输入框即可认为当前会话可用；若被风控会在上面的分支提前返回 false。
+    const searchBox = await page.$('textarea[name="q"], input[name="q"]');
+    return !!searchBox;
+  } catch {
+    return false;
+  }
+}
+
+function parseFromResultBlocks(root, pageUrl) {
+  const resultBlocks = root.querySelectorAll("#rso .MjjYud .A6K0A, #rso .MjjYud .tF2Cxc");
+  const seen = new Set();
+  const items = [];
+
+  for (const block of resultBlocks) {
+    const anchor =
+      block.querySelector('a[jsname="UWckNb"][href]') ??
+      block.querySelector(".yuRUbf a[href]") ??
+      block.querySelector("a[href]");
+    const link = resolveResultLink(anchor?.getAttribute("href"), pageUrl);
+    if (!link || seen.has(link)) continue;
+
+    const titleNode =
+      block.querySelector("h3.LC20lb, h3.DKV0Md, h3.MBeuO, h3") ??
+      anchor?.querySelector?.("h3");
+    const title = normalizeText(titleNode?.textContent);
+    if (!title) continue;
+
+    seen.add(link);
+    const summary = extractSnippet(block) || extractSnippet(titleNode ?? block) || title;
+    items.push({
+      guid: _deps.createHash("sha256").update(link).digest("hex"),
+      title,
+      link,
+      pubDate: new Date(),
+      author: "Google Search",
+      summary,
+    });
+  }
+
+  return items;
+}
+
+function parseFromHeadingFallback(root, pageUrl) {
+  const seen = new Set();
+  const items = [];
+  const titles = root.querySelectorAll("h3");
+  for (const h3 of titles) {
+    const title = normalizeText(h3.textContent);
+    if (!title) continue;
+    const anchor = closestAnchor(h3);
+    const link = resolveResultLink(anchor?.getAttribute("href"), pageUrl);
+    if (!link || seen.has(link)) continue;
+    seen.add(link);
+    const summary = extractSnippet(h3) || title;
+    items.push({
+      guid: _deps.createHash("sha256").update(link).digest("hex"),
+      title,
+      link,
+      pubDate: new Date(),
+      author: "Google Search",
+      summary,
+    });
+  }
+  return items;
+}
+
+async function fetchItems(sourceId, ctx) {
+  _deps = ctx.deps;
+  const { html, finalUrl } = await ctx.fetchHtml(sourceId, { waitMs: 2500 });
+  const root = _deps.parseHtml(html);
+  const pageUrl = new URL(finalUrl);
+
+  const fromBlocks = parseFromResultBlocks(root, pageUrl);
+  const items = fromBlocks.length > 0 ? fromBlocks : parseFromHeadingFallback(root, pageUrl);
+
+  if (items.length === 0) {
+    if (looksLikeBlockedPage(root, html, finalUrl)) {
+      throw new Error("[google] 命中 Google 验证页（reCAPTCHA/风控），当前会话无法稳定抓取搜索结果");
+    }
+    throw new Error("[google] 未解析到搜索结果，页面结构可能已变化");
+  }
+  return items;
+}
+
+export default {
+  id: "google-search",
+  listUrlPattern: /^https?:\/\/(www\.)?google\.[^/]+\/search(\?.*)?$/i,
+  fetchItems,
+  checkAuth,
+  loginUrl: "https://www.google.com/",
+  domain: "google.com",
+  loginTimeoutMs: 5 * 60 * 1000,
+  pollIntervalMs: 2000,
+};

package/app/plugins/builtin/hacker-news-newest.rssany.js

@@ -0,0 +1,130 @@
+let _deps;
+
+// Hacker News newest 插件：解析 newest 列表页为 FeedItem（仅列表，不做正文 enrich）
+
+
+
+const HN_ORIGIN = "https://news.ycombinator.com";
+
+
+function normalizeText(text) {
+  return (text ?? "").replace(/\s+/g, " ").trim();
+}
+
+
+function toAbsoluteUrl(rawHref, baseUrl) {
+  const href = normalizeText(rawHref);
+  if (!href || href.startsWith("#") || href.startsWith("javascript:")) return null;
+  try {
+    const url = new URL(href, baseUrl);
+    if (!/^https?:$/i.test(url.protocol)) return null;
+    return url.href;
+  } catch {
+    return null;
+  }
+}
+
+
+function parsePubDate(rawTitle) {
+  const text = normalizeText(rawTitle);
+  if (!text) return new Date();
+
+  const parts = text.split(/\s+/);
+  const epochPart = parts[1];
+  if (epochPart && /^\d{10}$/.test(epochPart)) {
+    return new Date(Number(epochPart) * 1000);
+  }
+
+  const date = new Date(parts[0]);
+  return Number.isNaN(date.getTime()) ? new Date() : date;
+}
+
+
+function parseTitleLink(row, pageUrl, itemId) {
+  const anchors = row.querySelectorAll("a[href]");
+  for (const anchor of anchors) {
+    const title = normalizeText(anchor.textContent);
+    if (!title) continue;
+    const href = anchor.getAttribute("href") ?? "";
+    if (/^vote\?/i.test(href) || /^from\?site=/i.test(href)) continue;
+    const link = toAbsoluteUrl(href, pageUrl);
+    if (!link) continue;
+    return { title, link };
+  }
+
+  return {
+    title: `HN Item ${itemId}`,
+    link: new URL(`/item?id=${itemId}`, pageUrl).href,
+  };
+}
+
+
+function parseMeta(root, row, itemId) {
+  const itemHref = `item?id=${itemId}`;
+  const ageAnchors = root.querySelectorAll(`a[href="${itemHref}"]`);
+
+  let ageSpan = null;
+  for (const anchor of ageAnchors) {
+    const parent = anchor.parentNode;
+    if (!parent || parent.tagName?.toLowerCase() !== "span") continue;
+    const titleAttr = parent.getAttribute?.("title");
+    if (!titleAttr) continue;
+    ageSpan = parent;
+    break;
+  }
+
+  const pubDate = parsePubDate(ageSpan?.getAttribute?.("title"));
+  const metaContainer = ageSpan?.parentNode ?? null;
+  const author = normalizeText(metaContainer?.querySelector?.('a[href^="user?id="]')?.textContent) || undefined;
+  const scoreText = normalizeText(metaContainer?.querySelector?.(`span[id="score_${itemId}"]`)?.textContent);
+  const commentLinks = metaContainer?.querySelectorAll?.(`a[href="${itemHref}"]`) ?? [];
+  const commentText = normalizeText(commentLinks[commentLinks.length - 1]?.textContent);
+  const siteText = normalizeText(row.querySelector?.('a[href^="from?site="]')?.textContent);
+
+  const summaryParts = [siteText, scoreText, commentText].filter(Boolean);
+  const summary = summaryParts.length > 0 ? summaryParts.join(" | ") : undefined;
+
+  return { pubDate, author, summary };
+}
+
+
+async function fetchItems(sourceId, ctx) {
+  _deps = ctx.deps;
+  const { html, finalUrl } = await ctx.fetchHtml(sourceId, { waitMs: 3000 });
+  const root = _deps.parseHtml(html);
+  const pageUrl = new URL(finalUrl || sourceId, HN_ORIGIN);
+  const items = [];
+  const seen = new Set();
+
+  for (const row of root.querySelectorAll("tr[id]")) {
+    const itemId = normalizeText(row.getAttribute("id"));
+    if (!/^\d+$/.test(itemId) || seen.has(itemId)) continue;
+
+    const { title, link } = parseTitleLink(row, pageUrl, itemId);
+    const { pubDate, author, summary } = parseMeta(root, row, itemId);
+    seen.add(itemId);
+
+    items.push({
+      guid: _deps.createHash("sha256").update(`hn:${itemId}`).digest("hex"),
+      title: title || "(无标题)",
+      link,
+      pubDate,
+      author,
+      summary,
+    });
+  }
+
+  if (items.length === 0) {
+    throw new Error("[hacker-news-newest] 未解析到条目，页面结构可能已变化");
+  }
+
+  return items;
+}
+
+
+export default {
+  id: "hacker-news-newest",
+  listUrlPattern: /^https?:\/\/news\.ycombinator\.com\/newest\/?(\?.*)?$/i,
+  refreshInterval: "10min",
+  fetchItems,
+};

package/app/plugins/builtin/harvard-dataverse.rssany.js

@@ -0,0 +1,166 @@
+let _deps;
+
+
+const DATAVERSE_ORIGIN = "https://dataverse.harvard.edu";
+const DATAVERSE_SEARCH_API = `${DATAVERSE_ORIGIN}/api/search`;
+const DEFAULT_QUERY = "*";
+const DEFAULT_PER_PAGE = 30;
+const MAX_PER_PAGE = 100;
+
+function normalizeText(text) {
+  return (text ?? "").replace(/\s+/g, " ").trim();
+}
+
+function hashGuid(input) {
+  return _deps.createHash("sha256").update(input).digest("hex");
+}
+
+function parseDate(value) {
+  const text = normalizeText(value);
+  if (!text) return undefined;
+  const parsed = new Date(text);
+  return Number.isNaN(parsed.getTime()) ? undefined : parsed;
+}
+
+function toSafeInteger(value, fallback, { min = Number.NEGATIVE_INFINITY, max = Number.POSITIVE_INFINITY } = {}) {
+  const raw = typeof value === "string" ? Number(value) : value;
+  if (!Number.isInteger(raw)) return fallback;
+  if (raw < min || raw > max) return fallback;
+  return raw;
+}
+
+function toHttpUrl(rawUrl, baseUrl = DATAVERSE_ORIGIN) {
+  const text = normalizeText(rawUrl);
+  if (!text) return null;
+  try {
+    const url = new URL(text, baseUrl);
+    if (!/^https?:$/i.test(url.protocol)) return null;
+    return url.href;
+  } catch {
+    return null;
+  }
+}
+
+function extractAuthor(record) {
+  if (Array.isArray(record?.authors)) {
+    const authors = record.authors.map((x) => normalizeText(x)).filter(Boolean);
+    if (authors.length > 0) return authors.join(", ");
+  }
+  const publisher = normalizeText(record?.publisher);
+  if (publisher) return publisher;
+  const dataverseName = normalizeText(record?.name_of_dataverse);
+  return dataverseName || undefined;
+}
+
+function extractSubtree(sourceId) {
+  try {
+    const url = new URL(sourceId);
+    const fromQuery = normalizeText(url.searchParams.get("subtree"));
+    if (fromQuery) return fromQuery;
+
+    const dataverseAlias = normalizeText(url.searchParams.get("alias"));
+    if (url.pathname === "/dataverse.xhtml" && dataverseAlias) {
+      return dataverseAlias;
+    }
+
+    const match = url.pathname.match(/^\/dataverse\/([^/?#]+)\/?$/i);
+    if (!match) return undefined;
+    return decodeURIComponent(match[1]);
+  } catch {
+    return undefined;
+  }
+}
+
+function buildQuery(sourceId) {
+  const params = new URLSearchParams();
+  params.set("type", "dataset");
+  params.set("sort", "date");
+  params.set("order", "desc");
+
+  try {
+    const url = new URL(sourceId);
+    const q = normalizeText(url.searchParams.get("q")) || DEFAULT_QUERY;
+    const perPage = toSafeInteger(
+      url.searchParams.get("per_page") ?? url.searchParams.get("perPage"),
+      DEFAULT_PER_PAGE,
+      { min: 1, max: MAX_PER_PAGE }
+    );
+    const start = toSafeInteger(url.searchParams.get("start"), 0, { min: 0 });
+    const sort = normalizeText(url.searchParams.get("sort"));
+    const order = normalizeText(url.searchParams.get("order"));
+
+    params.set("q", q);
+    params.set("per_page", String(perPage));
+    params.set("start", String(start));
+    if (/^[A-Za-z_]+$/.test(sort)) params.set("sort", sort);
+    if (/^(asc|desc)$/i.test(order)) params.set("order", order.toLowerCase());
+  } catch {
+    params.set("q", DEFAULT_QUERY);
+    params.set("per_page", String(DEFAULT_PER_PAGE));
+    params.set("start", "0");
+  }
+
+  const subtree = extractSubtree(sourceId);
+  if (subtree) params.set("subtree", subtree);
+  return params;
+}
+
+function toFeedItem(record, index) {
+  if (!record || typeof record !== "object") return null;
+
+  const title = normalizeText(record.name);
+  const link = toHttpUrl(record.url);
+  if (!title || !link) return null;
+
+  const guidSeed = normalizeText(record.global_id) || link;
+  const pubDate =
+    parseDate(record.published_at) ??
+    parseDate(record.updatedAt) ??
+    parseDate(record.createdAt) ??
+    new Date(Date.now() - index * 1000);
+  const summary = normalizeText(record.description);
+
+  return {
+    guid: hashGuid(guidSeed),
+    title,
+    link,
+    pubDate,
+    author: extractAuthor(record),
+    summary: summary || undefined,
+    sourceId: "harvard-dataverse",
+  };
+}
+
+async function fetchItems(sourceId, ctx) {
+  _deps = ctx.deps;
+  const query = buildQuery(sourceId);
+  const apiUrl = `${DATAVERSE_SEARCH_API}?${query.toString()}`;
+  const response = await fetch(apiUrl, {
+    headers: {
+      Accept: "application/json",
+    },
+  });
+
+  if (!response.ok) {
+    throw new Error(`[harvard-dataverse] 请求搜索接口失败: HTTP ${response.status}`);
+  }
+
+  const payload = await response.json().catch(() => null);
+  const records = payload?.data?.items;
+  if (!Array.isArray(records)) {
+    throw new Error("[harvard-dataverse] 搜索接口响应结构异常");
+  }
+
+  const items = records.map((record, index) => toFeedItem(record, index)).filter(Boolean);
+  if (items.length === 0) {
+    throw new Error("[harvard-dataverse] 未解析到条目，接口结构可能已变化");
+  }
+  return items;
+}
+
+export default {
+  id: "harvard-dataverse",
+  listUrlPattern:
+    /^https?:\/\/dataverse\.harvard\.edu(?:\/?$|\/\?.*|\/dataverse\/[^/?#]+\/?(?:\?.*)?|\/dataverse\.xhtml(?:\?.*)?)$/i,
+  fetchItems,
+};

package/app/plugins/builtin/huaweicloud-bbs-blogs.rssany.js

@@ -0,0 +1,185 @@
+let _deps;
+
+// 华为云社区博客插件：抓取 https://bbs.huaweicloud.com/blogs 列表条目（默认仅列表，不做 enrich）
+
+
+
+const HUAWEICLOUD_ORIGIN = "https://bbs.huaweicloud.com";
+const BLOG_PATH_RE = /^\/blogs\/\d+$/;
+const DATE_RE = /(\d{4})[/-](\d{1,2})[/-](\d{1,2})/;
+
+
+function normalizeText(text) {
+  return (text ?? "").replace(/\s+/g, " ").trim();
+}
+
+
+function cleanTitle(text) {
+  return normalizeText(text).replace(/\s+HOT$/i, "");
+}
+
+
+function hashGuid(input) {
+  return _deps.createHash("sha256").update(input).digest("hex");
+}
+
+
+function toAbsoluteUrl(href, baseUrl) {
+  if (!href) return null;
+  try {
+    const url = new URL(href, baseUrl);
+    if (!/^https?:$/i.test(url.protocol)) return null;
+    return url.href;
+  } catch {
+    return null;
+  }
+}
+
+
+function getBlogPath(href, baseUrl) {
+  const absolute = toAbsoluteUrl(href, baseUrl);
+  if (!absolute) return null;
+  try {
+    const url = new URL(absolute);
+    const normalizedPath = url.pathname.replace(/\/+$/, "");
+    return BLOG_PATH_RE.test(normalizedPath) ? normalizedPath : null;
+  } catch {
+    return null;
+  }
+}
+
+
+function parseDate(text) {
+  const normalized = normalizeText(text);
+  const m = normalized.match(DATE_RE);
+  if (!m) return undefined;
+  const [, year, month, day] = m;
+  const iso = `${year}-${month.padStart(2, "0")}-${day.padStart(2, "0")}T00:00:00+08:00`;
+  const date = new Date(iso);
+  return Number.isNaN(date.getTime()) ? undefined : date;
+}
+
+
+function extractDate(card) {
+  const allText = normalizeText(card?.textContent);
+  return parseDate(allText);
+}
+
+
+function extractAuthor(card) {
+  if (!card) return undefined;
+  const authorAnchor = card.querySelector('a[id^="ydcomm_blog_author_"]') ??
+    card.querySelector('a[href^="/community/usersnew/"]');
+  return normalizeText(authorAnchor?.textContent) || undefined;
+}
+
+
+function extractSummary(card, title, linkPath, pageUrl) {
+  if (!card) return undefined;
+  let best = "";
+  for (const anchor of card.querySelectorAll("a[href]")) {
+    const href = anchor.getAttribute("href") || "";
+    const path = getBlogPath(href, pageUrl);
+    if (!path || path !== linkPath) continue;
+    const text = normalizeText(anchor.textContent);
+    if (!text) continue;
+    if (text === title) continue;
+    if (text.length > best.length) best = text;
+  }
+  return best || undefined;
+}
+
+
+function mapCardToFeedItem(card, pageUrl) {
+  const titleAnchor = card?.querySelector('a[id^="ydcomm_blog_title_"][href]') ??
+    card?.querySelector('a[title][href^="/blogs/"], a[title][href*="/blogs/"]');
+  const href = titleAnchor?.getAttribute("href") ?? "";
+  const linkPath = getBlogPath(href, pageUrl);
+  if (!linkPath) return null;
+
+  const link = toAbsoluteUrl(linkPath, pageUrl);
+  if (!link) return null;
+
+  const title = cleanTitle(titleAnchor?.getAttribute("title")) || cleanTitle(titleAnchor?.textContent);
+  if (!title) return null;
+
+  const pubDate = extractDate(card) ?? new Date();
+  const summary = extractSummary(card, title, linkPath, pageUrl);
+  const author = extractAuthor(card);
+
+  return {
+    guid: hashGuid(link),
+    title,
+    link,
+    pubDate,
+    summary: summary || undefined,
+    author,
+  };
+}
+
+
+function findCardRoot(node) {
+  let current = node ?? null;
+  for (let i = 0; i < 8 && current; i += 1) {
+    if (typeof current.getAttribute === "function") {
+      const id = current.getAttribute("id") || "";
+      if (id.startsWith("ydcomm_blog_content_")) return current;
+    }
+    current = current.parentNode ?? null;
+  }
+  return null;
+}
+
+
+function parseFromCardBlocks(root, pageUrl) {
+  const items = [];
+  const seen = new Set();
+  const cards = root.querySelectorAll('div[id^="ydcomm_blog_content_"]');
+  for (const card of cards) {
+    const item = mapCardToFeedItem(card, pageUrl);
+    if (!item || seen.has(item.link)) continue;
+    seen.add(item.link);
+    items.push(item);
+  }
+  return items;
+}
+
+
+function parseFromTitleAnchors(root, pageUrl) {
+  const items = [];
+  const seen = new Set();
+  const anchors = root.querySelectorAll('a[id^="ydcomm_blog_title_"][href]');
+  for (const anchor of anchors) {
+    const card = findCardRoot(anchor);
+    const item = mapCardToFeedItem(card ?? anchor.parentNode, pageUrl);
+    if (!item || seen.has(item.link)) continue;
+    seen.add(item.link);
+    items.push(item);
+  }
+  return items;
+}
+
+
+async function fetchItems(sourceId, ctx) {
+  _deps = ctx.deps;
+  const { html, finalUrl } = await ctx.fetchHtml(sourceId, { waitMs: 4500 });
+  const root = _deps.parseHtml(html);
+  const pageUrl = new URL(finalUrl || sourceId, HUAWEICLOUD_ORIGIN);
+
+  const itemsFromCards = parseFromCardBlocks(root, pageUrl);
+  const items = itemsFromCards.length > 0 ? itemsFromCards : parseFromTitleAnchors(root, pageUrl);
+
+  if (items.length === 0) {
+    throw new Error("[huaweicloud-bbs-blogs] 未解析到博客条目，页面结构可能已变化");
+  }
+
+  return items;
+}
+
+
+export default {
+  id: "huaweicloud-bbs-blogs",
+  listUrlPattern: /^https?:\/\/bbs\.huaweicloud\.com\/blogs\/?(\?.*)?$/i,
+  refreshInterval: "1h",
+  fetchItems,
+};