rssany 0.1.2 → 0.1.5

package/app/plugins/builtin/huaweicloud-bbs-blogs.rssany.js

@@ -0,0 +1,185 @@
+let _deps;
+
+// 华为云社区博客插件：抓取 https://bbs.huaweicloud.com/blogs 列表条目（默认仅列表，不做 enrich）
+
+
+
+const HUAWEICLOUD_ORIGIN = "https://bbs.huaweicloud.com";
+const BLOG_PATH_RE = /^\/blogs\/\d+$/;
+const DATE_RE = /(\d{4})[/-](\d{1,2})[/-](\d{1,2})/;
+
+
+function normalizeText(text) {
+  return (text ?? "").replace(/\s+/g, " ").trim();
+}
+
+
+function cleanTitle(text) {
+  return normalizeText(text).replace(/\s+HOT$/i, "");
+}
+
+
+function hashGuid(input) {
+  return _deps.createHash("sha256").update(input).digest("hex");
+}
+
+
+function toAbsoluteUrl(href, baseUrl) {
+  if (!href) return null;
+  try {
+    const url = new URL(href, baseUrl);
+    if (!/^https?:$/i.test(url.protocol)) return null;
+    return url.href;
+  } catch {
+    return null;
+  }
+}
+
+
+function getBlogPath(href, baseUrl) {
+  const absolute = toAbsoluteUrl(href, baseUrl);
+  if (!absolute) return null;
+  try {
+    const url = new URL(absolute);
+    const normalizedPath = url.pathname.replace(/\/+$/, "");
+    return BLOG_PATH_RE.test(normalizedPath) ? normalizedPath : null;
+  } catch {
+    return null;
+  }
+}
+
+
+function parseDate(text) {
+  const normalized = normalizeText(text);
+  const m = normalized.match(DATE_RE);
+  if (!m) return undefined;
+  const [, year, month, day] = m;
+  const iso = `${year}-${month.padStart(2, "0")}-${day.padStart(2, "0")}T00:00:00+08:00`;
+  const date = new Date(iso);
+  return Number.isNaN(date.getTime()) ? undefined : date;
+}
+
+
+function extractDate(card) {
+  const allText = normalizeText(card?.textContent);
+  return parseDate(allText);
+}
+
+
+function extractAuthor(card) {
+  if (!card) return undefined;
+  const authorAnchor = card.querySelector('a[id^="ydcomm_blog_author_"]') ??
+    card.querySelector('a[href^="/community/usersnew/"]');
+  return normalizeText(authorAnchor?.textContent) || undefined;
+}
+
+
+function extractSummary(card, title, linkPath, pageUrl) {
+  if (!card) return undefined;
+  let best = "";
+  for (const anchor of card.querySelectorAll("a[href]")) {
+    const href = anchor.getAttribute("href") || "";
+    const path = getBlogPath(href, pageUrl);
+    if (!path || path !== linkPath) continue;
+    const text = normalizeText(anchor.textContent);
+    if (!text) continue;
+    if (text === title) continue;
+    if (text.length > best.length) best = text;
+  }
+  return best || undefined;
+}
+
+
+function mapCardToFeedItem(card, pageUrl) {
+  const titleAnchor = card?.querySelector('a[id^="ydcomm_blog_title_"][href]') ??
+    card?.querySelector('a[title][href^="/blogs/"], a[title][href*="/blogs/"]');
+  const href = titleAnchor?.getAttribute("href") ?? "";
+  const linkPath = getBlogPath(href, pageUrl);
+  if (!linkPath) return null;
+
+  const link = toAbsoluteUrl(linkPath, pageUrl);
+  if (!link) return null;
+
+  const title = cleanTitle(titleAnchor?.getAttribute("title")) || cleanTitle(titleAnchor?.textContent);
+  if (!title) return null;
+
+  const pubDate = extractDate(card) ?? new Date();
+  const summary = extractSummary(card, title, linkPath, pageUrl);
+  const author = extractAuthor(card);
+
+  return {
+    guid: hashGuid(link),
+    title,
+    link,
+    pubDate,
+    summary: summary || undefined,
+    author,
+  };
+}
+
+
+function findCardRoot(node) {
+  let current = node ?? null;
+  for (let i = 0; i < 8 && current; i += 1) {
+    if (typeof current.getAttribute === "function") {
+      const id = current.getAttribute("id") || "";
+      if (id.startsWith("ydcomm_blog_content_")) return current;
+    }
+    current = current.parentNode ?? null;
+  }
+  return null;
+}
+
+
+function parseFromCardBlocks(root, pageUrl) {
+  const items = [];
+  const seen = new Set();
+  const cards = root.querySelectorAll('div[id^="ydcomm_blog_content_"]');
+  for (const card of cards) {
+    const item = mapCardToFeedItem(card, pageUrl);
+    if (!item || seen.has(item.link)) continue;
+    seen.add(item.link);
+    items.push(item);
+  }
+  return items;
+}
+
+
+function parseFromTitleAnchors(root, pageUrl) {
+  const items = [];
+  const seen = new Set();
+  const anchors = root.querySelectorAll('a[id^="ydcomm_blog_title_"][href]');
+  for (const anchor of anchors) {
+    const card = findCardRoot(anchor);
+    const item = mapCardToFeedItem(card ?? anchor.parentNode, pageUrl);
+    if (!item || seen.has(item.link)) continue;
+    seen.add(item.link);
+    items.push(item);
+  }
+  return items;
+}
+
+
+async function fetchItems(sourceId, ctx) {
+  _deps = ctx.deps;
+  const { html, finalUrl } = await ctx.fetchHtml(sourceId, { waitMs: 4500 });
+  const root = _deps.parseHtml(html);
+  const pageUrl = new URL(finalUrl || sourceId, HUAWEICLOUD_ORIGIN);
+
+  const itemsFromCards = parseFromCardBlocks(root, pageUrl);
+  const items = itemsFromCards.length > 0 ? itemsFromCards : parseFromTitleAnchors(root, pageUrl);
+
+  if (items.length === 0) {
+    throw new Error("[huaweicloud-bbs-blogs] 未解析到博客条目，页面结构可能已变化");
+  }
+
+  return items;
+}
+
+
+export default {
+  id: "huaweicloud-bbs-blogs",
+  listUrlPattern: /^https?:\/\/bbs\.huaweicloud\.com\/blogs\/?(\?.*)?$/i,
+  refreshInterval: "1h",
+  fetchItems,
+};

package/app/plugins/builtin/lingowhale.rssany.js

@@ -0,0 +1,119 @@
+// 语鲸 Open API 插件：将语鲸推荐文章 / 今日文章转换为条目列表
+//
+// sourceId 格式：
+//   lingowhale://articles  — 推荐文章（默认拉取 2 页，可用 ?pages=5 调整）
+//   lingowhale://today     — 今日文章（自动翻页拉完）
+//
+// 认证参数可内联于 URL query：
+//   lingowhale://articles?app_id=xxx&app_secret=yyy
+// 也可通过环境变量提供（优先级低于 URL 参数）：
+//   LINGOWHALE_APP_ID / LINGOWHALE_APP_SECRET
+
+
+const BASE_URL = "https://open.lingowhale.com/open-api/v1";
+
+
+function resolveCredentials(sourceId) {
+  let appId = process.env.LINGOWHALE_APP_ID ?? "";
+  let appSecret = process.env.LINGOWHALE_APP_SECRET ?? "";
+  try {
+    const parsed = new URL(sourceId.replace(/^lingowhale:/, "http://x"));
+    appId = parsed.searchParams.get("app_id") ?? appId;
+    appSecret = parsed.searchParams.get("app_secret") ?? appSecret;
+  } catch { /* fallback to env vars */ }
+  if (!appId || !appSecret) {
+    throw new Error(
+      "[LingowhalePlugin] 缺少认证信息：请在 sourceId 中提供 ?app_id=&app_secret= 参数，" +
+      "或设置环境变量 LINGOWHALE_APP_ID / LINGOWHALE_APP_SECRET"
+    );
+  }
+  return { appId, appSecret };
+}
+
+function resolveEndpoint(sourceId) {
+  return sourceId.includes("://today") ? "today" : "articles";
+}
+
+function resolveMaxPages(sourceId) {
+  try {
+    const parsed = new URL(sourceId.replace(/^lingowhale:/, "http://x"));
+    const pages = parseInt(parsed.searchParams.get("pages") ?? "", 10);
+    if (!isNaN(pages) && pages > 0) return pages;
+  } catch { /* ignore */ }
+  return 2;
+}
+
+function buildHeaders(appId, appSecret) {
+  return {
+    "X-App-ID": appId,
+    "X-App-Secret": appSecret,
+    "Content-Type": "application/json",
+    "Accept": "application/json",
+  };
+}
+
+function stripHl(text) {
+  return text.replace(/<\/?hl>/g, "");
+}
+
+function mapArticle(article) {
+  const link = article.orig_url ?? `https://lingowhale.com/article/${article.entry_id}`;
+  const rawSummary = article.abstract || article.description;
+  return {
+    guid: article.entry_id,
+    title: article.title ?? "(无标题)",
+    link,
+    pubDate: article.pub_time ? new Date(article.pub_time * 1000) : new Date(),
+    summary: rawSummary ? stripHl(rawSummary) : undefined,
+    content: article.html || undefined,
+  };
+}
+
+async function fetchArticles(appId, appSecret, maxPages) {
+  const headers = buildHeaders(appId, appSecret);
+  const items = [];
+  for (let page = 1; page <= maxPages; page++) {
+    const res = await fetch(`${BASE_URL}/articles?page=${page}&page_size=100`, { headers });
+    if (!res.ok) throw new Error(`[LingowhalePlugin] HTTP ${res.status} 拉取文章列表失败`);
+    const json = await res.json();
+    if (json.code !== 0) throw new Error(`[LingowhalePlugin] API 错误：${json.message}`);
+    const pageItems = json.data.items ?? [];
+    items.push(...pageItems.map(mapArticle));
+    const total = json.data.total ?? 0;
+    if (items.length >= total || pageItems.length === 0) break;
+  }
+  return items;
+}
+
+async function fetchTodayArticles(appId, appSecret) {
+  const headers = buildHeaders(appId, appSecret);
+  const items = [];
+  let cursor;
+  for (let round = 0; round < 10; round++) {
+    const cursorParam = cursor ? `&cursor=${encodeURIComponent(cursor)}` : "";
+    const res = await fetch(`${BASE_URL}/articles/today?page_size=20${cursorParam}`, { headers });
+    if (!res.ok) throw new Error(`[LingowhalePlugin] HTTP ${res.status} 拉取今日文章失败`);
+    const json = await res.json();
+    if (json.code !== 0) throw new Error(`[LingowhalePlugin] API 错误：${json.message}`);
+    const pageItems = json.data.items ?? [];
+    items.push(...pageItems.map(mapArticle));
+    cursor = json.data.nextCursor;
+    if (!cursor || pageItems.length === 0) break;
+  }
+  return items;
+}
+
+
+export default {
+  id: "lingowhale",
+  listUrlPattern: /^lingowhale:\/\//,
+  refreshInterval: "1h",
+
+  async fetchItems(sourceId, _ctx) {
+    const { appId, appSecret } = resolveCredentials(sourceId);
+    const endpoint = resolveEndpoint(sourceId);
+    const maxPages = resolveMaxPages(sourceId);
+    if (endpoint === "today") return fetchTodayArticles(appId, appSecret);
+    return fetchArticles(appId, appSecret, maxPages);
+  },
+};

package/app/plugins/builtin/meituan-tech.rssany.js

@@ -0,0 +1,130 @@
+let _deps;
+
+// 美团技术团队博客：https://tech.meituan.com/
+// 列表页解析 .post-container，可选 enrichItem 拉取正文
+
+
+function normalizeText(text) {
+  return (text ?? "").replace(/\s+/g, " ").trim();
+}
+
+function toAbsoluteUrl(href, baseUrl) {
+  if (!href) return null;
+  try {
+    const url = new URL(href, baseUrl);
+    if (!/^https?:$/i.test(url.protocol)) return null;
+    return url.href;
+  } catch {
+    return null;
+  }
+}
+
+/** 解析 "2026年03月13日" 或 "2026-03-13" */
+function parseDate(text) {
+  const raw = normalizeText(text);
+  const m = raw.match(/(\d{4})[年/-](\d{1,2})[月/-](\d{1,2})/);
+  if (!m) return undefined;
+  const [, y, mon, d] = m;
+  const iso = `${y}-${mon.padStart(2, "0")}-${d.padStart(2, "0")}T00:00:00+08:00`;
+  const date = new Date(iso);
+  return Number.isNaN(date.getTime()) ? undefined : date;
+}
+
+function hashGuid(link) {
+  return _deps.createHash("sha256").update(link).digest("hex");
+}
+
+async function fetchItems(sourceId, ctx) {
+  _deps = ctx.deps;
+  const { html, finalUrl } = await ctx.fetchHtml(sourceId, {
+    waitMs: 5000,
+    waitForSelector: ".post-container",
+    waitForSelectorTimeoutMs: 25000,
+  });
+  const root = _deps.parseHtml(html);
+  const baseUrl = finalUrl || sourceId;
+
+  const items = [];
+  const seen = new Set();
+
+  // 兼容 .row.post-container-wrapper > .col-md-6 > .post-container 或直接 .post-container
+  let containers = root.querySelectorAll(".post-container");
+  if (containers.length === 0) {
+    containers = root.querySelectorAll(".post-container-wrapper .post-container");
+  }
+  for (const el of containers) {
+    const titleEl = el.querySelector(".post-title a[href]");
+    let href = titleEl?.getAttribute("href");
+    if (!href) {
+      const moreLink = el.querySelector("a.more-link[href]");
+      if (moreLink) href = moreLink.getAttribute("href");
+    }
+    if (!href) continue;
+
+    const link = toAbsoluteUrl(href, baseUrl);
+    if (!link) continue;
+    try {
+      const u = new URL(link);
+      if (!u.hostname.endsWith("tech.meituan.com")) continue;
+    } catch {
+      continue;
+    }
+    if (seen.has(link)) continue;
+    seen.add(link);
+
+    const title = normalizeText(titleEl?.textContent) || normalizeText(el.querySelector(".post-title")?.textContent);
+    if (!title) continue;
+
+    const dateEl = el.querySelector(".m-post-date");
+    const pubDate = parseDate(dateEl?.textContent) ?? new Date();
+
+    const authorEl = el.querySelector(".m-post-nick");
+    const authorRaw = normalizeText(authorEl?.textContent);
+    const author = authorRaw ? [authorRaw] : undefined;
+
+    const summaryEl = el.querySelector(".post-content.post-expect");
+    let summary = "";
+    if (summaryEl) {
+      const clone = summaryEl.clone();
+      const moreLink = clone.querySelector("a.more-link");
+      if (moreLink) moreLink.remove();
+      summary = normalizeText(clone.textContent);
+    }
+
+    const tagLinks = el.querySelectorAll(".tag-links a[rel='tag']");
+    const categories = tagLinks.length
+      ? Array.from(tagLinks).map((a) => normalizeText(a.textContent)).filter(Boolean)
+      : undefined;
+
+    items.push({
+      guid: hashGuid(link),
+      title,
+      link,
+      pubDate,
+      author,
+      summary: summary || undefined,
+      categories,
+      sourceRef: sourceId,
+    });
+  }
+
+  if (items.length === 0) {
+    throw new Error("[meituan-tech] 未解析到文章条目，请检查列表页结构是否变化");
+  }
+
+  return items;
+}
+
+/** 可选：拉取详情页正文，使用默认 Readability 提取 */
+async function enrichItem(item, ctx) {
+  return ctx.extractItem(item);
+}
+
+export default {
+  id: "meituan-tech",
+  listUrlPattern: /^https?:\/\/(www\.)?tech\.meituan\.com(\/.*)?$/i,
+  detailUrlPattern: /^https?:\/\/(www\.)?tech\.meituan\.com\/\d{4}\/\d{2}\/\d{2}\/[^/]+\.html(?:\?.*)?$/i,
+  refreshInterval: "1day",
+  fetchItems,
+  enrichItem,
+};

package/app/plugins/builtin/meta-ai-publications.rssany.js

@@ -0,0 +1,221 @@
+let _deps;
+
+// Meta AI Publications 插件：抓取结果页中的 publication 条目（不做正文 enrich）
+
+
+
+const PUBLICATION_PATH_RE = /^\/research\/publications\/[^?#]+\/?$/i;
+const PUBLICATION_RESULTS_URL_RE =
+  /^https?:\/\/ai\.meta\.com\/results\/?\?.*content_types(?:%5B0%5D|\[0\])=publication(?:&.*)?$/i;
+const MONTH_TO_INDEX = {
+  january: 0,
+  february: 1,
+  march: 2,
+  april: 3,
+  may: 4,
+  june: 5,
+  july: 6,
+  august: 7,
+  september: 8,
+  october: 9,
+  november: 10,
+  december: 11,
+};
+
+
+function normalizeText(text) {
+  return (text ?? "")
+    .replace(/[\u200B-\u200D\uFEFF]/g, "")
+    .replace(/\s+/g, " ")
+    .trim();
+}
+
+
+function hashGuid(input) {
+  return _deps.createHash("sha256").update(input).digest("hex");
+}
+
+
+function toAbsolutePublicationUrl(rawHref, pageUrl) {
+  if (!rawHref) return null;
+  const href = rawHref.trim();
+  if (!href || href.startsWith("#") || href.startsWith("javascript:")) return null;
+  try {
+    const url = new URL(href, pageUrl);
+    if (!/^https?:$/i.test(url.protocol)) return null;
+    if (url.hostname !== "ai.meta.com") return null;
+    if (!PUBLICATION_PATH_RE.test(url.pathname)) return null;
+    return url.href;
+  } catch {
+    return null;
+  }
+}
+
+
+function parsePubDate(rawDate) {
+  const normalized = normalizeText(rawDate);
+  if (!normalized) return undefined;
+  const m = normalized.match(
+    /^(January|February|March|April|May|June|July|August|September|October|November|December)\s+(\d{1,2}),\s*(\d{4})$/i
+  );
+  if (!m) return undefined;
+  const monthName = m[1]?.toLowerCase();
+  const day = Number(m[2]);
+  const year = Number(m[3]);
+  const month = monthName ? MONTH_TO_INDEX[monthName] : undefined;
+  if (month == null || !Number.isFinite(day) || !Number.isFinite(year)) return undefined;
+  return new Date(Date.UTC(year, month, day, 0, 0, 0));
+}
+
+
+function decodeTitleFromLink(link) {
+  try {
+    const path = new URL(link).pathname;
+    const slug = path.split("/").filter(Boolean).at(-1) ?? "";
+    if (!slug) return "";
+    const decoded = decodeURIComponent(slug).replace(/[-_]+/g, " ");
+    return normalizeText(decoded.replace(/\b([a-z])/g, (m) => m.toUpperCase()));
+  } catch {
+    return "";
+  }
+}
+
+
+function looksLikeCategory(text) {
+  if (!text || text.length > 80) return false;
+  if (/[.!?]/.test(text)) return false;
+  if (/\d{4}/.test(text)) return false;
+  const letters = text.replace(/[^A-Za-z]/g, "");
+  if (!letters) return false;
+  const uppercase = letters.replace(/[^A-Z]/g, "").length;
+  const ratio = uppercase / letters.length;
+  return ratio >= 0.7 || text.includes("|");
+}
+
+
+function looksLikeAuthorLine(text) {
+  if (!text) return false;
+  if (text.length < 8 || text.length > 700) return false;
+  if (!text.includes(",")) return false;
+  if (/[.!?]/.test(text)) return false;
+  return /[A-Za-z]/.test(text);
+}
+
+
+function collectUniqueTexts(nodes) {
+  const out = [];
+  const seen = new Set();
+  for (const node of nodes) {
+    const text = normalizeText(node.textContent);
+    if (!text || seen.has(text)) continue;
+    seen.add(text);
+    out.push(text);
+  }
+  return out;
+}
+
+
+function findCardRoot(anchor) {
+  let cur = anchor;
+  let fallback = null;
+  for (let i = 0; i < 8 && cur; i += 1) {
+    const headings = collectUniqueTexts(cur.querySelectorAll?.("h1, h2, h3, h4, h5, h6") ?? []);
+    const paragraphs = collectUniqueTexts(cur.querySelectorAll?.("p") ?? []);
+    const hasDate = paragraphs.some((p) => parsePubDate(p) != null);
+    if (headings.length >= 2 && hasDate) return cur;
+    if (headings.length >= 1 && paragraphs.length >= 3) fallback = cur;
+    cur = cur.parentNode ?? null;
+  }
+  return fallback ?? anchor.parentNode ?? anchor;
+}
+
+
+function extractTitle(card, link) {
+  const headings = collectUniqueTexts(card.querySelectorAll("h1, h2, h3, h4, h5, h6"));
+  const candidates = headings
+    .filter((text) => !/^(publication|read the paper)$/i.test(text))
+    .filter((text) => !looksLikeCategory(text));
+  const chosen = candidates.find((text) => text.length >= 8) ?? candidates.at(-1) ?? "";
+  if (chosen) return chosen;
+  return decodeTitleFromLink(link);
+}
+
+
+function extractPubDateFromCard(card) {
+  const timeDatetime = card.querySelector("time[datetime]")?.getAttribute("datetime");
+  if (timeDatetime) {
+    const direct = new Date(timeDatetime);
+    if (!Number.isNaN(direct.getTime())) return direct;
+  }
+  const paragraphs = collectUniqueTexts(card.querySelectorAll("p, span, div"));
+  for (const text of paragraphs) {
+    const parsed = parsePubDate(text);
+    if (parsed) return parsed;
+  }
+  return undefined;
+}
+
+
+function extractSummary(card, title) {
+  const paragraphs = collectUniqueTexts(card.querySelectorAll("p"));
+  const candidates = paragraphs
+    .filter((text) => !/^(publication|read the paper)$/i.test(text))
+    .filter((text) => parsePubDate(text) == null)
+    .filter((text) => !looksLikeAuthorLine(text))
+    .filter((text) => !looksLikeCategory(text))
+    .filter((text) => text !== title);
+  const summary = candidates.find((text) => text.length >= 40) ?? "";
+  return summary || undefined;
+}
+
+
+async function fetchItems(sourceId, ctx) {
+  _deps = ctx.deps;
+  const { html, finalUrl, status } = await ctx.fetchHtml(sourceId, { waitMs: 3500, purify: false });
+  if (status >= 400) {
+    throw new Error(`[meta-ai-publications] 抓取失败: HTTP ${status}`);
+  }
+
+  const root = _deps.parseHtml(html);
+  const pageUrl = finalUrl || sourceId;
+  const anchors = root.querySelectorAll('a[href*="/research/publications/"]');
+  const seen = new Set();
+  const items = [];
+
+  for (const anchor of anchors) {
+    const link = toAbsolutePublicationUrl(anchor.getAttribute("href"), pageUrl);
+    if (!link || seen.has(link)) continue;
+    seen.add(link);
+
+    const card = findCardRoot(anchor);
+    const title = extractTitle(card, link);
+    if (!title) continue;
+
+    const pubDate = extractPubDateFromCard(card) ?? new Date();
+    const summary = extractSummary(card, title);
+
+    items.push({
+      guid: hashGuid(link),
+      title,
+      link,
+      pubDate,
+      author: "Meta AI",
+      summary,
+      sourceId: "meta-ai-publications",
+    });
+  }
+
+  if (items.length === 0) {
+    throw new Error(
+      "[meta-ai-publications] 未解析到 publication 条目，可能是页面结构变化，或需要更长等待时间/可用浏览器会话"
+    );
+  }
+  return items;
+}
+
+
+export default {
+  id: "meta-ai-publications",
+  listUrlPattern: PUBLICATION_RESULTS_URL_RE,
+  fetchItems,
+};