rssany 0.1.4 → 0.1.5

package/app/plugins/builtin/lingowhale.rssany.js

@@ -0,0 +1,119 @@
+// 语鲸 Open API 插件：将语鲸推荐文章 / 今日文章转换为条目列表
+//
+// sourceId 格式：
+//   lingowhale://articles  — 推荐文章（默认拉取 2 页，可用 ?pages=5 调整）
+//   lingowhale://today     — 今日文章（自动翻页拉完）
+//
+// 认证参数可内联于 URL query：
+//   lingowhale://articles?app_id=xxx&app_secret=yyy
+// 也可通过环境变量提供（优先级低于 URL 参数）：
+//   LINGOWHALE_APP_ID / LINGOWHALE_APP_SECRET
+
+
+const BASE_URL = "https://open.lingowhale.com/open-api/v1";
+
+
+function resolveCredentials(sourceId) {
+  let appId = process.env.LINGOWHALE_APP_ID ?? "";
+  let appSecret = process.env.LINGOWHALE_APP_SECRET ?? "";
+  try {
+    const parsed = new URL(sourceId.replace(/^lingowhale:/, "http://x"));
+    appId = parsed.searchParams.get("app_id") ?? appId;
+    appSecret = parsed.searchParams.get("app_secret") ?? appSecret;
+  } catch { /* fallback to env vars */ }
+  if (!appId || !appSecret) {
+    throw new Error(
+      "[LingowhalePlugin] 缺少认证信息：请在 sourceId 中提供 ?app_id=&app_secret= 参数，" +
+      "或设置环境变量 LINGOWHALE_APP_ID / LINGOWHALE_APP_SECRET"
+    );
+  }
+  return { appId, appSecret };
+}
+
+function resolveEndpoint(sourceId) {
+  return sourceId.includes("://today") ? "today" : "articles";
+}
+
+function resolveMaxPages(sourceId) {
+  try {
+    const parsed = new URL(sourceId.replace(/^lingowhale:/, "http://x"));
+    const pages = parseInt(parsed.searchParams.get("pages") ?? "", 10);
+    if (!isNaN(pages) && pages > 0) return pages;
+  } catch { /* ignore */ }
+  return 2;
+}
+
+function buildHeaders(appId, appSecret) {
+  return {
+    "X-App-ID": appId,
+    "X-App-Secret": appSecret,
+    "Content-Type": "application/json",
+    "Accept": "application/json",
+  };
+}
+
+function stripHl(text) {
+  return text.replace(/<\/?hl>/g, "");
+}
+
+function mapArticle(article) {
+  const link = article.orig_url ?? `https://lingowhale.com/article/${article.entry_id}`;
+  const rawSummary = article.abstract || article.description;
+  return {
+    guid: article.entry_id,
+    title: article.title ?? "(无标题)",
+    link,
+    pubDate: article.pub_time ? new Date(article.pub_time * 1000) : new Date(),
+    summary: rawSummary ? stripHl(rawSummary) : undefined,
+    content: article.html || undefined,
+  };
+}
+
+async function fetchArticles(appId, appSecret, maxPages) {
+  const headers = buildHeaders(appId, appSecret);
+  const items = [];
+  for (let page = 1; page <= maxPages; page++) {
+    const res = await fetch(`${BASE_URL}/articles?page=${page}&page_size=100`, { headers });
+    if (!res.ok) throw new Error(`[LingowhalePlugin] HTTP ${res.status} 拉取文章列表失败`);
+    const json = await res.json();
+    if (json.code !== 0) throw new Error(`[LingowhalePlugin] API 错误：${json.message}`);
+    const pageItems = json.data.items ?? [];
+    items.push(...pageItems.map(mapArticle));
+    const total = json.data.total ?? 0;
+    if (items.length >= total || pageItems.length === 0) break;
+  }
+  return items;
+}
+
+async function fetchTodayArticles(appId, appSecret) {
+  const headers = buildHeaders(appId, appSecret);
+  const items = [];
+  let cursor;
+  for (let round = 0; round < 10; round++) {
+    const cursorParam = cursor ? `&cursor=${encodeURIComponent(cursor)}` : "";
+    const res = await fetch(`${BASE_URL}/articles/today?page_size=20${cursorParam}`, { headers });
+    if (!res.ok) throw new Error(`[LingowhalePlugin] HTTP ${res.status} 拉取今日文章失败`);
+    const json = await res.json();
+    if (json.code !== 0) throw new Error(`[LingowhalePlugin] API 错误：${json.message}`);
+    const pageItems = json.data.items ?? [];
+    items.push(...pageItems.map(mapArticle));
+    cursor = json.data.nextCursor;
+    if (!cursor || pageItems.length === 0) break;
+  }
+  return items;
+}
+
+
+export default {
+  id: "lingowhale",
+  listUrlPattern: /^lingowhale:\/\//,
+  refreshInterval: "1h",
+
+  async fetchItems(sourceId, _ctx) {
+    const { appId, appSecret } = resolveCredentials(sourceId);
+    const endpoint = resolveEndpoint(sourceId);
+    const maxPages = resolveMaxPages(sourceId);
+    if (endpoint === "today") return fetchTodayArticles(appId, appSecret);
+    return fetchArticles(appId, appSecret, maxPages);
+  },
+};

package/app/plugins/builtin/meituan-tech.rssany.js

@@ -0,0 +1,130 @@
+let _deps;
+
+// 美团技术团队博客：https://tech.meituan.com/
+// 列表页解析 .post-container，可选 enrichItem 拉取正文
+
+
+function normalizeText(text) {
+  return (text ?? "").replace(/\s+/g, " ").trim();
+}
+
+function toAbsoluteUrl(href, baseUrl) {
+  if (!href) return null;
+  try {
+    const url = new URL(href, baseUrl);
+    if (!/^https?:$/i.test(url.protocol)) return null;
+    return url.href;
+  } catch {
+    return null;
+  }
+}
+
+/** 解析 "2026年03月13日" 或 "2026-03-13" */
+function parseDate(text) {
+  const raw = normalizeText(text);
+  const m = raw.match(/(\d{4})[年/-](\d{1,2})[月/-](\d{1,2})/);
+  if (!m) return undefined;
+  const [, y, mon, d] = m;
+  const iso = `${y}-${mon.padStart(2, "0")}-${d.padStart(2, "0")}T00:00:00+08:00`;
+  const date = new Date(iso);
+  return Number.isNaN(date.getTime()) ? undefined : date;
+}
+
+function hashGuid(link) {
+  return _deps.createHash("sha256").update(link).digest("hex");
+}
+
+async function fetchItems(sourceId, ctx) {
+  _deps = ctx.deps;
+  const { html, finalUrl } = await ctx.fetchHtml(sourceId, {
+    waitMs: 5000,
+    waitForSelector: ".post-container",
+    waitForSelectorTimeoutMs: 25000,
+  });
+  const root = _deps.parseHtml(html);
+  const baseUrl = finalUrl || sourceId;
+
+  const items = [];
+  const seen = new Set();
+
+  // 兼容 .row.post-container-wrapper > .col-md-6 > .post-container 或直接 .post-container
+  let containers = root.querySelectorAll(".post-container");
+  if (containers.length === 0) {
+    containers = root.querySelectorAll(".post-container-wrapper .post-container");
+  }
+  for (const el of containers) {
+    const titleEl = el.querySelector(".post-title a[href]");
+    let href = titleEl?.getAttribute("href");
+    if (!href) {
+      const moreLink = el.querySelector("a.more-link[href]");
+      if (moreLink) href = moreLink.getAttribute("href");
+    }
+    if (!href) continue;
+
+    const link = toAbsoluteUrl(href, baseUrl);
+    if (!link) continue;
+    try {
+      const u = new URL(link);
+      if (!u.hostname.endsWith("tech.meituan.com")) continue;
+    } catch {
+      continue;
+    }
+    if (seen.has(link)) continue;
+    seen.add(link);
+
+    const title = normalizeText(titleEl?.textContent) || normalizeText(el.querySelector(".post-title")?.textContent);
+    if (!title) continue;
+
+    const dateEl = el.querySelector(".m-post-date");
+    const pubDate = parseDate(dateEl?.textContent) ?? new Date();
+
+    const authorEl = el.querySelector(".m-post-nick");
+    const authorRaw = normalizeText(authorEl?.textContent);
+    const author = authorRaw ? [authorRaw] : undefined;
+
+    const summaryEl = el.querySelector(".post-content.post-expect");
+    let summary = "";
+    if (summaryEl) {
+      const clone = summaryEl.clone();
+      const moreLink = clone.querySelector("a.more-link");
+      if (moreLink) moreLink.remove();
+      summary = normalizeText(clone.textContent);
+    }
+
+    const tagLinks = el.querySelectorAll(".tag-links a[rel='tag']");
+    const categories = tagLinks.length
+      ? Array.from(tagLinks).map((a) => normalizeText(a.textContent)).filter(Boolean)
+      : undefined;
+
+    items.push({
+      guid: hashGuid(link),
+      title,
+      link,
+      pubDate,
+      author,
+      summary: summary || undefined,
+      categories,
+      sourceRef: sourceId,
+    });
+  }
+
+  if (items.length === 0) {
+    throw new Error("[meituan-tech] 未解析到文章条目，请检查列表页结构是否变化");
+  }
+
+  return items;
+}
+
+/** 可选：拉取详情页正文，使用默认 Readability 提取 */
+async function enrichItem(item, ctx) {
+  return ctx.extractItem(item);
+}
+
+export default {
+  id: "meituan-tech",
+  listUrlPattern: /^https?:\/\/(www\.)?tech\.meituan\.com(\/.*)?$/i,
+  detailUrlPattern: /^https?:\/\/(www\.)?tech\.meituan\.com\/\d{4}\/\d{2}\/\d{2}\/[^/]+\.html(?:\?.*)?$/i,
+  refreshInterval: "1day",
+  fetchItems,
+  enrichItem,
+};

package/app/plugins/builtin/meta-ai-publications.rssany.js

@@ -0,0 +1,221 @@
+let _deps;
+
+// Meta AI Publications 插件：抓取结果页中的 publication 条目（不做正文 enrich）
+
+
+
+const PUBLICATION_PATH_RE = /^\/research\/publications\/[^?#]+\/?$/i;
+const PUBLICATION_RESULTS_URL_RE =
+  /^https?:\/\/ai\.meta\.com\/results\/?\?.*content_types(?:%5B0%5D|\[0\])=publication(?:&.*)?$/i;
+const MONTH_TO_INDEX = {
+  january: 0,
+  february: 1,
+  march: 2,
+  april: 3,
+  may: 4,
+  june: 5,
+  july: 6,
+  august: 7,
+  september: 8,
+  october: 9,
+  november: 10,
+  december: 11,
+};
+
+
+function normalizeText(text) {
+  return (text ?? "")
+    .replace(/[\u200B-\u200D\uFEFF]/g, "")
+    .replace(/\s+/g, " ")
+    .trim();
+}
+
+
+function hashGuid(input) {
+  return _deps.createHash("sha256").update(input).digest("hex");
+}
+
+
+function toAbsolutePublicationUrl(rawHref, pageUrl) {
+  if (!rawHref) return null;
+  const href = rawHref.trim();
+  if (!href || href.startsWith("#") || href.startsWith("javascript:")) return null;
+  try {
+    const url = new URL(href, pageUrl);
+    if (!/^https?:$/i.test(url.protocol)) return null;
+    if (url.hostname !== "ai.meta.com") return null;
+    if (!PUBLICATION_PATH_RE.test(url.pathname)) return null;
+    return url.href;
+  } catch {
+    return null;
+  }
+}
+
+
+function parsePubDate(rawDate) {
+  const normalized = normalizeText(rawDate);
+  if (!normalized) return undefined;
+  const m = normalized.match(
+    /^(January|February|March|April|May|June|July|August|September|October|November|December)\s+(\d{1,2}),\s*(\d{4})$/i
+  );
+  if (!m) return undefined;
+  const monthName = m[1]?.toLowerCase();
+  const day = Number(m[2]);
+  const year = Number(m[3]);
+  const month = monthName ? MONTH_TO_INDEX[monthName] : undefined;
+  if (month == null || !Number.isFinite(day) || !Number.isFinite(year)) return undefined;
+  return new Date(Date.UTC(year, month, day, 0, 0, 0));
+}
+
+
+function decodeTitleFromLink(link) {
+  try {
+    const path = new URL(link).pathname;
+    const slug = path.split("/").filter(Boolean).at(-1) ?? "";
+    if (!slug) return "";
+    const decoded = decodeURIComponent(slug).replace(/[-_]+/g, " ");
+    return normalizeText(decoded.replace(/\b([a-z])/g, (m) => m.toUpperCase()));
+  } catch {
+    return "";
+  }
+}
+
+
+function looksLikeCategory(text) {
+  if (!text || text.length > 80) return false;
+  if (/[.!?]/.test(text)) return false;
+  if (/\d{4}/.test(text)) return false;
+  const letters = text.replace(/[^A-Za-z]/g, "");
+  if (!letters) return false;
+  const uppercase = letters.replace(/[^A-Z]/g, "").length;
+  const ratio = uppercase / letters.length;
+  return ratio >= 0.7 || text.includes("|");
+}
+
+
+function looksLikeAuthorLine(text) {
+  if (!text) return false;
+  if (text.length < 8 || text.length > 700) return false;
+  if (!text.includes(",")) return false;
+  if (/[.!?]/.test(text)) return false;
+  return /[A-Za-z]/.test(text);
+}
+
+
+function collectUniqueTexts(nodes) {
+  const out = [];
+  const seen = new Set();
+  for (const node of nodes) {
+    const text = normalizeText(node.textContent);
+    if (!text || seen.has(text)) continue;
+    seen.add(text);
+    out.push(text);
+  }
+  return out;
+}
+
+
+function findCardRoot(anchor) {
+  let cur = anchor;
+  let fallback = null;
+  for (let i = 0; i < 8 && cur; i += 1) {
+    const headings = collectUniqueTexts(cur.querySelectorAll?.("h1, h2, h3, h4, h5, h6") ?? []);
+    const paragraphs = collectUniqueTexts(cur.querySelectorAll?.("p") ?? []);
+    const hasDate = paragraphs.some((p) => parsePubDate(p) != null);
+    if (headings.length >= 2 && hasDate) return cur;
+    if (headings.length >= 1 && paragraphs.length >= 3) fallback = cur;
+    cur = cur.parentNode ?? null;
+  }
+  return fallback ?? anchor.parentNode ?? anchor;
+}
+
+
+function extractTitle(card, link) {
+  const headings = collectUniqueTexts(card.querySelectorAll("h1, h2, h3, h4, h5, h6"));
+  const candidates = headings
+    .filter((text) => !/^(publication|read the paper)$/i.test(text))
+    .filter((text) => !looksLikeCategory(text));
+  const chosen = candidates.find((text) => text.length >= 8) ?? candidates.at(-1) ?? "";
+  if (chosen) return chosen;
+  return decodeTitleFromLink(link);
+}
+
+
+function extractPubDateFromCard(card) {
+  const timeDatetime = card.querySelector("time[datetime]")?.getAttribute("datetime");
+  if (timeDatetime) {
+    const direct = new Date(timeDatetime);
+    if (!Number.isNaN(direct.getTime())) return direct;
+  }
+  const paragraphs = collectUniqueTexts(card.querySelectorAll("p, span, div"));
+  for (const text of paragraphs) {
+    const parsed = parsePubDate(text);
+    if (parsed) return parsed;
+  }
+  return undefined;
+}
+
+
+function extractSummary(card, title) {
+  const paragraphs = collectUniqueTexts(card.querySelectorAll("p"));
+  const candidates = paragraphs
+    .filter((text) => !/^(publication|read the paper)$/i.test(text))
+    .filter((text) => parsePubDate(text) == null)
+    .filter((text) => !looksLikeAuthorLine(text))
+    .filter((text) => !looksLikeCategory(text))
+    .filter((text) => text !== title);
+  const summary = candidates.find((text) => text.length >= 40) ?? "";
+  return summary || undefined;
+}
+
+
+async function fetchItems(sourceId, ctx) {
+  _deps = ctx.deps;
+  const { html, finalUrl, status } = await ctx.fetchHtml(sourceId, { waitMs: 3500, purify: false });
+  if (status >= 400) {
+    throw new Error(`[meta-ai-publications] 抓取失败: HTTP ${status}`);
+  }
+
+  const root = _deps.parseHtml(html);
+  const pageUrl = finalUrl || sourceId;
+  const anchors = root.querySelectorAll('a[href*="/research/publications/"]');
+  const seen = new Set();
+  const items = [];
+
+  for (const anchor of anchors) {
+    const link = toAbsolutePublicationUrl(anchor.getAttribute("href"), pageUrl);
+    if (!link || seen.has(link)) continue;
+    seen.add(link);
+
+    const card = findCardRoot(anchor);
+    const title = extractTitle(card, link);
+    if (!title) continue;
+
+    const pubDate = extractPubDateFromCard(card) ?? new Date();
+    const summary = extractSummary(card, title);
+
+    items.push({
+      guid: hashGuid(link),
+      title,
+      link,
+      pubDate,
+      author: "Meta AI",
+      summary,
+      sourceId: "meta-ai-publications",
+    });
+  }
+
+  if (items.length === 0) {
+    throw new Error(
+      "[meta-ai-publications] 未解析到 publication 条目，可能是页面结构变化，或需要更长等待时间/可用浏览器会话"
+    );
+  }
+  return items;
+}
+
+
+export default {
+  id: "meta-ai-publications",
+  listUrlPattern: PUBLICATION_RESULTS_URL_RE,
+  fetchItems,
+};

package/app/plugins/builtin/mila-quebec.rssany.js

@@ -0,0 +1,199 @@
+let _deps;
+
+// Mila (Quebec AI Institute) 新闻列表插件：支持首页 /en 与新闻页 /en/news
+
+
+
+const MILA_ORIGIN = "https://mila.quebec";
+const NEWS_PATH_RE = /^\/en\/news\/[^/?#]+\/?$/i;
+const NOISE_TITLE_RE = /^(read the (story|news)|see more news)$/i;
+const MONTH_TO_INDEX = {
+  jan: 0,
+  feb: 1,
+  mar: 2,
+  apr: 3,
+  may: 4,
+  jun: 5,
+  jul: 6,
+  aug: 7,
+  sep: 8,
+  oct: 9,
+  nov: 10,
+  dec: 11,
+};
+
+
+function normalizeText(text) {
+  return (text ?? "").replace(/\s+/g, " ").trim();
+}
+
+
+function hashGuid(input) {
+  return _deps.createHash("sha256").update(input).digest("hex");
+}
+
+
+function toAbsoluteHttpUrl(rawHref, baseUrl) {
+  if (!rawHref) return null;
+  const href = rawHref.trim();
+  if (!href || href.startsWith("#") || href.startsWith("javascript:")) return null;
+  try {
+    const url = new URL(href, baseUrl);
+    if (!/^https?:$/i.test(url.protocol)) return null;
+    return url.href;
+  } catch {
+    return null;
+  }
+}
+
+
+function isNewsArticleUrl(url) {
+  try {
+    return NEWS_PATH_RE.test(new URL(url).pathname);
+  } catch {
+    return false;
+  }
+}
+
+
+function extractDateFromText(text) {
+  const normalized = normalizeText(text).replace(/,/g, " ");
+  if (!normalized) return undefined;
+
+  const m = normalized.match(
+    /(?:^|\b)(\d{1,2})\s+(Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:t(?:ember)?)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\s+(\d{4})(?:\b|$)/i
+  );
+  if (!m) return undefined;
+
+  const day = Number(m[1]);
+  const monthIdx = MONTH_TO_INDEX[m[2].slice(0, 3).toLowerCase()];
+  const year = Number(m[3]);
+  if (monthIdx == null || !Number.isFinite(day) || !Number.isFinite(year)) return undefined;
+
+  const d = new Date(Date.UTC(year, monthIdx, day, 12, 0, 0));
+  if (Number.isNaN(d.getTime())) return undefined;
+  return d;
+}
+
+
+function extractDateNearNode(node) {
+  const timeDatetime = node.querySelector?.("time[datetime]")?.getAttribute("datetime");
+  if (timeDatetime) {
+    const d = new Date(timeDatetime);
+    if (!Number.isNaN(d.getTime())) return d;
+  }
+
+  let current = node;
+  for (let i = 0; i < 6 && current; i += 1) {
+    const parsed = extractDateFromText(current.textContent);
+    if (parsed) return parsed;
+    current = current.parentNode ?? null;
+  }
+  return undefined;
+}
+
+
+function scoreTitle(text) {
+  const normalized = normalizeText(text);
+  if (!normalized) return 0;
+  if (NOISE_TITLE_RE.test(normalized.toLowerCase())) return 1;
+  if (normalized.length < 5) return 2;
+  return 10 + Math.min(normalized.length, 120);
+}
+
+
+function titleFromUrl(link) {
+  try {
+    const slug = decodeURIComponent(new URL(link).pathname.split("/").filter(Boolean).pop() ?? "");
+    return normalizeText(slug.replace(/[-_]+/g, " "));
+  } catch {
+    return "Mila News";
+  }
+}
+
+
+function chooseSummary(node, title) {
+  let current = node;
+  for (let i = 0; i < 4 && current; i += 1) {
+    const candidates = current
+      .querySelectorAll?.("p")
+      ?.map((p) => normalizeText(p.textContent))
+      ?.filter(Boolean) ?? [];
+    for (const text of candidates) {
+      const lower = text.toLowerCase();
+      if (lower === title.toLowerCase()) continue;
+      if (NOISE_TITLE_RE.test(lower)) continue;
+      return text;
+    }
+    current = current.parentNode ?? null;
+  }
+  return undefined;
+}
+
+
+async function fetchItems(sourceId, ctx) {
+  _deps = ctx.deps;
+  const { html, finalUrl } = await ctx.fetchHtml(sourceId, { waitMs: 4000 });
+  const root = _deps.parseHtml(html);
+  const pageUrl = finalUrl || sourceId || MILA_ORIGIN;
+  const byLink = new Map();
+
+  for (const anchor of root.querySelectorAll("a[href]")) {
+    const link = toAbsoluteHttpUrl(anchor.getAttribute("href"), pageUrl);
+    if (!link || !isNewsArticleUrl(link)) continue;
+
+    const rawTitle = normalizeText(anchor.textContent);
+    const titleScore = scoreTitle(rawTitle);
+    const pubDate = extractDateNearNode(anchor);
+    const summary = chooseSummary(anchor, rawTitle || "");
+
+    const existing = byLink.get(link) ?? {
+      link,
+      title: "",
+      titleScore: 0,
+      pubDate: undefined,
+      summary: undefined,
+    };
+
+    if (titleScore > existing.titleScore) {
+      existing.title = rawTitle;
+      existing.titleScore = titleScore;
+    }
+    if (!existing.pubDate && pubDate) existing.pubDate = pubDate;
+    if (!existing.summary && summary) existing.summary = summary;
+
+    byLink.set(link, existing);
+  }
+
+  const items = Array.from(byLink.values()).map((entry) => {
+    const title = entry.title && !NOISE_TITLE_RE.test(entry.title.toLowerCase())
+      ? entry.title
+      : titleFromUrl(entry.link);
+    const summary = entry.summary && normalizeText(entry.summary) !== normalizeText(title)
+      ? entry.summary
+      : undefined;
+    return {
+      guid: hashGuid(entry.link),
+      title,
+      link: entry.link,
+      pubDate: entry.pubDate ?? new Date(),
+      author: "Mila",
+      summary,
+    };
+  });
+
+  items.sort((a, b) => b.pubDate.getTime() - a.pubDate.getTime());
+
+  if (items.length === 0) {
+    throw new Error("[mila-quebec] 未解析到新闻条目，页面结构可能已变化");
+  }
+
+  return items;
+}
+
+
+export default {
+  id: "mila-quebec",
+  listUrlPattern: /^https?:\/\/(www\.)?mila\.quebec\/en(?:\/news)?(?:\/)?(?:\?.*)?$/i,
+  fetchItems,
+};