rssany 0.1.2 → 0.1.5

package/app/plugins/builtin/baidu-research.rssany.js

@@ -0,0 +1,222 @@
+let _deps;
+
+// Baidu Research 插件：抓取 Blog 列表条目（不做正文 enrich）
+
+
+
+const BLOG_ITEM_PATH_RE = /^\/Blog\/index-view(?:\/)?$/i;
+const MONTH_TO_INDEX = {
+  jan: 0,
+  feb: 1,
+  mar: 2,
+  apr: 3,
+  may: 4,
+  jun: 5,
+  jul: 6,
+  aug: 7,
+  sep: 8,
+  oct: 9,
+  nov: 10,
+  dec: 11,
+};
+const TEXT_NODE_SELECTORS = "div, p, span, h1, h2, h3, h4, h5, h6, strong, em";
+const MONTH_DAY_TEXT_RE = /^([A-Za-z]{3,9})\s+\d{1,2}(?:st|nd|rd|th)?(?:\s*[，,]\s*\d{0,4})?$/i;
+
+
+function normalizeText(text) {
+  return (text ?? "").replace(/\s+/g, " ").trim();
+}
+
+
+function hashGuid(input) {
+  return _deps.createHash("sha256").update(input).digest("hex");
+}
+
+
+function toAbsoluteHttpUrl(rawHref, baseUrl) {
+  if (!rawHref) return null;
+  const href = rawHref.trim();
+  if (!href || href.startsWith("#") || href.startsWith("javascript:")) return null;
+  try {
+    const url = new URL(href, baseUrl);
+    if (!/^https?:$/i.test(url.protocol)) return null;
+    return url.href;
+  } catch {
+    return null;
+  }
+}
+
+
+function inferYearFromTexts(texts) {
+  for (const text of texts) {
+    const m = normalizeText(text).match(/\b(19|20)\d{2}\b/);
+    if (!m) continue;
+    const year = Number(m[0]);
+    if (year >= 1990 && year <= new Date().getUTCFullYear() + 1) {
+      return year;
+    }
+  }
+  return undefined;
+}
+
+
+function parsePubDate(rawText, fallbackTexts) {
+  const text = normalizeText(rawText).replace("，", ",");
+  if (!text) return undefined;
+
+  const monthMatch = text.match(
+    /^([A-Za-z]{3,9})\s+(\d{1,2})(?:st|nd|rd|th)?(?:\s*,\s*)?(?:(\d{4}))?$/
+  );
+  if (monthMatch) {
+    const month = MONTH_TO_INDEX[monthMatch[1].slice(0, 3).toLowerCase()];
+    if (month != null) {
+      const day = Number(monthMatch[2]);
+      const year = monthMatch[3] ? Number(monthMatch[3]) : inferYearFromTexts(fallbackTexts);
+      if (!year) return undefined;
+      const parsed = new Date(Date.UTC(year, month, day, 12, 0, 0));
+      if (!Number.isNaN(parsed.getTime())) return parsed;
+    }
+  }
+
+  const numericMatch = text.match(/^(\d{4})[/-](\d{1,2})[/-](\d{1,2})$/);
+  if (numericMatch) {
+    const year = Number(numericMatch[1]);
+    const month = Number(numericMatch[2]) - 1;
+    const day = Number(numericMatch[3]);
+    const parsed = new Date(Date.UTC(year, month, day, 12, 0, 0));
+    if (!Number.isNaN(parsed.getTime())) return parsed;
+  }
+
+  return undefined;
+}
+
+
+function isDateLikeText(text) {
+  return MONTH_DAY_TEXT_RE.test(normalizeText(text));
+}
+
+
+function getLeafTexts(anchor) {
+  const all = anchor.querySelectorAll(TEXT_NODE_SELECTORS);
+  const out = [];
+  const seen = new Set();
+  for (const el of all) {
+    if (el.querySelector(TEXT_NODE_SELECTORS) != null) continue;
+    const text = normalizeText(el.textContent);
+    if (!text || text === "MORE") continue;
+    if (seen.has(text)) continue;
+    seen.add(text);
+    out.push(text);
+  }
+  return out;
+}
+
+
+function parseAnchorItem(anchor, finalUrl) {
+  const link = toAbsoluteHttpUrl(anchor.getAttribute("href"), finalUrl);
+  if (!link) return null;
+
+  const linkUrl = new URL(link);
+  if (linkUrl.hostname !== "research.baidu.com") return null;
+  if (!BLOG_ITEM_PATH_RE.test(linkUrl.pathname)) return null;
+  if (!/^\d+$/.test(linkUrl.searchParams.get("id") ?? "")) return null;
+
+  const texts = getLeafTexts(anchor);
+  if (texts.length === 0) return null;
+
+  const datedTexts = texts.map((text) => ({ text, date: parsePubDate(text, texts) }));
+  const dateIndex = datedTexts.findIndex((x) => x.date != null);
+  const pubDate = dateIndex >= 0 ? datedTexts[dateIndex].date : undefined;
+
+  const nonDateTexts = datedTexts
+    .filter((x) => x.date == null && !isDateLikeText(x.text))
+    .map((x) => x.text);
+  const titleCandidates = nonDateTexts.filter((text) => text.length >= 6);
+  const looksLikeSummary = (text) => text.endsWith("...") || text.split(/\s+/).length >= 22;
+
+  let title = "";
+  if (titleCandidates.length > 0) {
+    title = [...titleCandidates]
+      .sort((a, b) => {
+        const aPenalty = looksLikeSummary(a) ? 1 : 0;
+        const bPenalty = looksLikeSummary(b) ? 1 : 0;
+        if (aPenalty !== bPenalty) return aPenalty - bPenalty;
+        return a.length - b.length;
+      })[0];
+  }
+  if (!title && dateIndex > 0) {
+    title = datedTexts.slice(0, dateIndex).map((x) => x.text).find((text) => text.length >= 6) ?? "";
+  }
+  if (!title) return null;
+
+  const summary = nonDateTexts.find(
+    (text) => text !== title && (text.endsWith("...") || text.length >= 40)
+  );
+
+  return {
+    guid: hashGuid(link),
+    title,
+    link,
+    pubDate: pubDate ?? new Date(),
+    author: "Baidu Research",
+    summary: summary || undefined,
+    sourceId: "baidu-research",
+  };
+}
+
+
+function parseBlogItems(html, finalUrl) {
+  const root = _deps.parseHtml(html);
+  const anchors = root.querySelectorAll("a[href]");
+  const seen = new Set();
+  const items = [];
+
+  for (const anchor of anchors) {
+    const item = parseAnchorItem(anchor, finalUrl);
+    if (!item) continue;
+    if (seen.has(item.link)) continue;
+    seen.add(item.link);
+    items.push(item);
+  }
+
+  return items;
+}
+
+
+function mergeByLink(itemsA, itemsB) {
+  const byLink = new Map();
+  for (const item of [...itemsA, ...itemsB]) {
+    if (byLink.has(item.link)) continue;
+    byLink.set(item.link, item);
+  }
+  return Array.from(byLink.values());
+}
+
+
+async function fetchItems(sourceId, ctx) {
+  _deps = ctx.deps;
+  const primary = await ctx.fetchHtml(sourceId, { waitMs: 4500 });
+  let items = parseBlogItems(primary.html, primary.finalUrl || sourceId);
+
+  const primaryUrl = new URL(primary.finalUrl || sourceId);
+  if (items.length < 5 && primaryUrl.hostname === "research.baidu.com") {
+    const blogUrl = new URL("/Blog", primaryUrl).href;
+    if (blogUrl !== primaryUrl.href) {
+      const blogPage = await ctx.fetchHtml(blogUrl, { waitMs: 4500 });
+      items = mergeByLink(items, parseBlogItems(blogPage.html, blogPage.finalUrl || blogUrl));
+    }
+  }
+
+  if (items.length === 0) {
+    throw new Error("[baidu-research] 未解析到 Blog 条目，页面结构可能已变化");
+  }
+
+  return items;
+}
+
+
+export default {
+  id: "baidu-research",
+  listUrlPattern: /^https?:\/\/research\.baidu\.com\/(?:(?:Index|Blog)\/?)?(?:\?.*)?$/i,
+  fetchItems,
+};

package/app/plugins/builtin/brightdata-blog.rssany.js

@@ -0,0 +1,301 @@
+let _deps;
+
+// Bright Data 博客插件：优先解析站点 RSS feed，失败时回退解析列表页（不做正文 enrich）
+
+
+
+const BRIGHTDATA_ORIGIN = "https://brightdata.com";
+const LIST_URL_RE =
+  /^https?:\/\/(?:www\.)?brightdata\.com\/blog(?:\/(?:page\/\d+|[a-z0-9-]+(?:\/page\/\d+)?)?)?\/?(?:\?.*)?$/i;
+const ARTICLE_PATH_RE = /^\/blog\/([^/?#/]+)\/([^/?#/]+)\/?$/i;
+const MIN_READ_RE = /^\d+\s*min\s*read$/i;
+
+
+function normalizeText(text) {
+  return (text ?? "").replace(/\s+/g, " ").trim();
+}
+
+
+function hashGuid(input) {
+  return _deps.createHash("sha256").update(input).digest("hex");
+}
+
+
+function toAbsoluteHttpUrl(rawHref, baseUrl) {
+  if (!rawHref) return null;
+  const href = rawHref.trim();
+  if (!href || href.startsWith("#") || href.startsWith("javascript:")) return null;
+  try {
+    const url = new URL(href, baseUrl);
+    if (!/^https?:$/i.test(url.protocol)) return null;
+    return url.href;
+  } catch {
+    return null;
+  }
+}
+
+
+function parsePubDate(raw) {
+  if (!raw) return undefined;
+  const date = new Date(raw);
+  return Number.isNaN(date.getTime()) ? undefined : date;
+}
+
+
+function isBrightDataArticleUrl(urlText) {
+  try {
+    const url = new URL(urlText);
+    if (!/(^|\.)brightdata\.com$/i.test(url.hostname)) return false;
+    const m = url.pathname.match(ARTICLE_PATH_RE);
+    if (!m) return false;
+    return m[1].toLowerCase() !== "page" && m[1].toLowerCase() !== "feed";
+  } catch {
+    return false;
+  }
+}
+
+
+function formatCategory(raw) {
+  return raw
+    .split("-")
+    .map((part) => part ? part[0].toUpperCase() + part.slice(1) : "")
+    .filter(Boolean)
+    .join(" ");
+}
+
+
+function extractCategoryFromLink(link) {
+  try {
+    const url = new URL(link);
+    const m = url.pathname.match(ARTICLE_PATH_RE);
+    if (!m) return undefined;
+    const category = formatCategory(m[1]);
+    return category || undefined;
+  } catch {
+    return undefined;
+  }
+}
+
+
+function uniqueTexts(values) {
+  const seen = new Set();
+  const out = [];
+  for (const value of values) {
+    if (!value || seen.has(value)) continue;
+    seen.add(value);
+    out.push(value);
+  }
+  return out;
+}
+
+
+function queryText(node, selectors) {
+  for (const selector of selectors) {
+    try {
+      const text = normalizeText(node.querySelector(selector)?.textContent);
+      if (text) return text;
+    } catch {
+      // ignore unsupported selectors
+    }
+  }
+  return "";
+}
+
+
+function deriveFeedUrlFromListUrl(sourceId) {
+  try {
+    const url = new URL(sourceId, BRIGHTDATA_ORIGIN);
+    const parts = url.pathname.split("/").filter(Boolean);
+    if (parts[0] !== "blog") return new URL("/blog/feed/", url.origin).href;
+
+    if (parts.length >= 2 && parts[1].toLowerCase() !== "page") {
+      return new URL(`/blog/${parts[1]}/feed/`, url.origin).href;
+    }
+    return new URL("/blog/feed/", url.origin).href;
+  } catch {
+    return null;
+  }
+}
+
+
+function resolveFeedUrl(root, pageUrl) {
+  const feedHref = root
+    .querySelector('link[rel="alternate"][type="application/rss+xml"][href], link[href*="/feed/"][type="application/rss+xml"]')
+    ?.getAttribute("href");
+  return toAbsoluteHttpUrl(feedHref, pageUrl);
+}
+
+
+async function fetchFeedItems(feedUrl) {
+  const res = await fetch(feedUrl, {
+    redirect: "follow",
+    headers: {
+      "User-Agent":
+        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
+      "Accept-Language": "en-US,en;q=0.9",
+    },
+  });
+  if (!res.ok) {
+    throw new Error(`[brightdata-blog] 获取 RSS feed 失败: ${res.status}`);
+  }
+
+  const xml = await res.text();
+  const root = _deps.parseHtml(xml);
+  const items = [];
+  const seen = new Set();
+
+  for (const entry of root.querySelectorAll("item")) {
+    const title = queryText(entry, ["title"]);
+    const link = toAbsoluteHttpUrl(queryText(entry, ["link"]), feedUrl);
+    if (!title || !link || !isBrightDataArticleUrl(link) || seen.has(link)) continue;
+    seen.add(link);
+
+    const summary = queryText(entry, ["description"]);
+    const author = queryText(entry, ["dc\\:creator", "creator", "author"]);
+    const pubDateRaw = queryText(entry, ["pubDate", "published", "updated", "dc\\:date"]);
+    const pubDate = parsePubDate(pubDateRaw) ?? new Date();
+
+    const categories = entry
+      .querySelectorAll("category")
+      .map((node) => normalizeText(node.textContent))
+      .filter(Boolean);
+    const fallbackCategory = extractCategoryFromLink(link);
+    const finalCategories = categories.length > 0
+      ? uniqueTexts(categories)
+      : (fallbackCategory ? [fallbackCategory] : undefined);
+
+    items.push({
+      guid: hashGuid(link),
+      title,
+      link,
+      pubDate,
+      author: author || undefined,
+      summary: summary || undefined,
+    });
+  }
+  return items;
+}
+
+
+function collectLeafTexts(anchor) {
+  const nodes = anchor.querySelectorAll("h1, h2, h3, h4, h5, h6, p, span, div");
+  const texts = [];
+  for (const node of nodes) {
+    if (node.querySelector("h1, h2, h3, h4, h5, h6, p, span, div")) continue;
+    const text = normalizeText(node.textContent);
+    if (/<[^>]+>/.test(text)) continue;
+    if (text) texts.push(text);
+  }
+  return uniqueTexts(texts);
+}
+
+
+function isMetaText(text) {
+  const normalized = normalizeText(text).toLowerCase();
+  if (!normalized) return true;
+  if (MIN_READ_RE.test(normalized)) return true;
+  return (
+    normalized === "editor's pick" ||
+    normalized === "latest articles" ||
+    normalized === "all categories" ||
+    normalized.includes("min read")
+  );
+}
+
+
+function extractTitle(anchor, texts, category) {
+  const categoryText = normalizeText(category).toLowerCase();
+  const heading = normalizeText(anchor.querySelector("h1, h2, h3, h4, h5, h6")?.textContent);
+  if (heading && !isMetaText(heading) && heading.toLowerCase() !== categoryText) return heading;
+
+  for (const text of texts) {
+    if (isMetaText(text)) continue;
+    if (categoryText && text.toLowerCase() === categoryText) continue;
+    if (text.length < 12) continue;
+    return text;
+  }
+  return "";
+}
+
+
+function extractSummary(title, texts) {
+  for (const text of texts) {
+    if (!text || text === title || isMetaText(text)) continue;
+    if (text.length < 20) continue;
+    return text;
+  }
+  return "";
+}
+
+
+function extractAuthor(anchor) {
+  const images = anchor.querySelectorAll("img[alt]");
+  for (const image of images) {
+    const alt = normalizeText(image.getAttribute("alt"));
+    if (!alt) continue;
+    if (/^[A-Z][A-Za-z.'-]+(?:\s+[A-Z][A-Za-z.'-]+){1,2}$/.test(alt)) return alt;
+  }
+  return "";
+}
+
+
+function parseHtmlItems(root, baseUrl) {
+  const items = [];
+  const seen = new Set();
+  for (const anchor of root.querySelectorAll("a[href]")) {
+    const link = toAbsoluteHttpUrl(anchor.getAttribute("href"), baseUrl);
+    if (!link || !isBrightDataArticleUrl(link) || seen.has(link)) continue;
+    const category = extractCategoryFromLink(link);
+
+    const texts = collectLeafTexts(anchor);
+    const title = extractTitle(anchor, texts, category);
+    if (!title) continue;
+
+    seen.add(link);
+    const summary = extractSummary(title, texts);
+    const author = extractAuthor(anchor);
+
+    items.push({
+      guid: hashGuid(link),
+      title,
+      link,
+      pubDate: new Date(),
+      author: author || undefined,
+      summary: summary || undefined,
+    });
+  }
+  return items;
+}
+
+
+async function fetchItems(sourceId, ctx) {
+  _deps = ctx.deps;
+  const { html, finalUrl } = await ctx.fetchHtml(sourceId, { waitMs: 3500 });
+  const root = _deps.parseHtml(html);
+  const pageUrl = finalUrl || sourceId || BRIGHTDATA_ORIGIN;
+
+  const discoveredFeedUrl = resolveFeedUrl(root, pageUrl);
+  const fallbackFeedUrl = deriveFeedUrlFromListUrl(pageUrl);
+  const feedUrls = uniqueTexts([discoveredFeedUrl, fallbackFeedUrl]);
+  for (const feedUrl of feedUrls) {
+    try {
+      const fromFeed = await fetchFeedItems(feedUrl);
+      if (fromFeed.length > 0) return fromFeed;
+    } catch {
+      // feed 失败时回退 HTML 解析
+    }
+  }
+
+  const fromHtml = parseHtmlItems(root, pageUrl);
+  if (fromHtml.length === 0) {
+    throw new Error("[brightdata-blog] 未解析到文章条目，页面结构可能已变化");
+  }
+  return fromHtml;
+}
+
+
+export default {
+  id: "brightdata-blog",
+  listUrlPattern: LIST_URL_RE,
+  fetchItems,
+};