rssany 0.1.4 → 0.1.5

package/app/plugins/builtin/baai-wudao-paper-article.rssany.js

@@ -0,0 +1,185 @@
+let _deps;
+
+
+const SITE_ID = "baai-wudao-paper-article";
+const OWNER = "BAAI-WuDao";
+const REPO = "Paper-Article";
+const README_PATH = "README.md";
+const README_RAW_URL = `https://raw.githubusercontent.com/${OWNER}/${REPO}/main/${README_PATH}`;
+const README_COMMITS_API_URL = `https://api.github.com/repos/${OWNER}/${REPO}/commits?path=${encodeURIComponent(
+  README_PATH
+)}&per_page=1`;
+
+function normalizeText(text) {
+  return (text ?? "").replace(/\s+/g, " ").trim();
+}
+
+function hashGuid(input) {
+  return _deps.createHash("sha256").update(input).digest("hex");
+}
+
+function cleanUrl(raw) {
+  const text = normalizeText(raw).replace(/[)>.,;!?]+$/g, "");
+  try {
+    const url = new URL(text);
+    if (!/^https?:$/i.test(url.protocol)) return null;
+    return url.href;
+  } catch {
+    return null;
+  }
+}
+
+async function fetchJson(url) {
+  const response = await fetch(url, {
+    headers: {
+      Accept: "application/vnd.github+json",
+      "User-Agent": "RssAny/1.0 (+https://github.com/rssany/rssany)",
+    },
+  });
+  if (!response.ok) {
+    throw new Error(`HTTP ${response.status}`);
+  }
+  try {
+    return await response.json();
+  } catch {
+    throw new Error("接口返回非 JSON 数据");
+  }
+}
+
+async function fetchReadmeMarkdown() {
+  const response = await fetch(README_RAW_URL, {
+    headers: {
+      Accept: "text/plain",
+      "User-Agent": "RssAny/1.0 (+https://github.com/rssany/rssany)",
+    },
+  });
+  if (!response.ok) {
+    throw new Error(`HTTP ${response.status}`);
+  }
+  const markdown = await response.text();
+  if (!normalizeText(markdown)) {
+    throw new Error("README 内容为空");
+  }
+  return markdown;
+}
+
+async function fetchReadmeUpdatedAt() {
+  try {
+    const payload = await fetchJson(README_COMMITS_API_URL);
+    const first = Array.isArray(payload) ? payload[0] : undefined;
+    const rawDate =
+      first?.commit?.committer?.date ??
+      first?.commit?.author?.date ??
+      first?.committer?.date ??
+      first?.author?.date;
+    const parsed = rawDate ? new Date(rawDate) : undefined;
+    if (parsed && !Number.isNaN(parsed.getTime())) return parsed;
+  } catch {
+    // 忽略日期接口异常，回退到当前时间保证插件可用
+  }
+  return new Date();
+}
+
+function parseSectionName(line) {
+  const m = normalizeText(line).match(/^###\s+(.+)$/);
+  if (!m) return "";
+  const name = normalizeText(m[1]).toLowerCase();
+  if (name.includes("paper")) return "Paper";
+  if (name.includes("article")) return "Article";
+  return "";
+}
+
+function parseTitleLine(line) {
+  const text = normalizeText(line);
+  const m = text.match(/^\*\s+\*\*(.+?)\*\*\s*$/);
+  if (!m) return "";
+  return normalizeText(m[1]);
+}
+
+function parseLinkLine(line) {
+  const text = normalizeText(line);
+  if (!text) return null;
+  const directMatch = text.match(/链接[:：]\s*(https?:\/\/\S+)/i);
+  if (directMatch) return cleanUrl(directMatch[1]);
+  const urlMatch = text.match(/(https?:\/\/\S+)/i);
+  if (urlMatch) return cleanUrl(urlMatch[1]);
+  return null;
+}
+
+function parseItemsFromReadme(markdown, baseDate) {
+  const lines = markdown.split(/\r?\n/);
+  const items = [];
+  const seenLinks = new Set();
+  let currentSection = "";
+
+  for (let i = 0; i < lines.length; i += 1) {
+    const line = lines[i];
+    const section = parseSectionName(line);
+    if (section) {
+      currentSection = section;
+      continue;
+    }
+
+    const title = parseTitleLine(line);
+    if (!title) continue;
+
+    let link = null;
+    for (let j = i + 1; j < Math.min(lines.length, i + 8); j += 1) {
+      link = parseLinkLine(lines[j]);
+      if (link) break;
+      if (parseTitleLine(lines[j]) || parseSectionName(lines[j])) break;
+    }
+    if (!link || seenLinks.has(link)) continue;
+    seenLinks.add(link);
+
+    const pubDate = new Date(baseDate.getTime() - items.length * 1000);
+    const category = currentSection || "Paper-Article";
+    const summary =
+      category === "Paper"
+        ? "BAAI-WuDao Paper collection"
+        : category === "Article"
+          ? "BAAI-WuDao related article"
+          : "BAAI-WuDao Paper-Article repository";
+
+    items.push({
+      guid: hashGuid(link),
+      title,
+      link,
+      pubDate,
+      summary,
+      sourceId: SITE_ID,
+    });
+  }
+
+  return items;
+}
+
+async function fetchItems(sourceId, _ctx) {
+  _deps = _ctx.deps;
+  let sourceUrl;
+  try {
+    sourceUrl = new URL(sourceId);
+  } catch {
+    throw new Error(`[${SITE_ID}] 无效 URL: ${sourceId}`);
+  }
+
+  if (
+    sourceUrl.hostname !== "github.com" ||
+    sourceUrl.pathname.replace(/\/+$/, "") !== `/${OWNER}/${REPO}`
+  ) {
+    throw new Error(`[${SITE_ID}] 仅支持仓库 URL: https://github.com/${OWNER}/${REPO}`);
+  }
+
+  const [markdown, baseDate] = await Promise.all([fetchReadmeMarkdown(), fetchReadmeUpdatedAt()]);
+  const items = parseItemsFromReadme(markdown, baseDate);
+  if (items.length === 0) {
+    throw new Error(`[${SITE_ID}] 未解析到条目，README 结构可能已变化`);
+  }
+  return items;
+}
+
+export default {
+  id: SITE_ID,
+  listUrlPattern: /^https?:\/\/(www\.)?github\.com\/BAAI-WuDao\/Paper-Article\/?(?:\?.*)?$/i,
+  fetchItems,
+};

package/app/plugins/builtin/baaidata-csdn.rssany.js

@@ -0,0 +1,242 @@
+let _deps;
+
+
+const ARTICLE_PATH_RE = /^\/([0-9a-f]{24})\.html$/i;
+const DATE_RE = /(\d{4})-(\d{1,2})-(\d{1,2})(?:\s+(\d{1,2}):(\d{2})(?::(\d{2}))?)?/;
+
+function normalizeText(text) {
+  return (text ?? "").replace(/\s+/g, " ").trim();
+}
+
+function hashGuid(input) {
+  return _deps.createHash("sha256").update(input).digest("hex");
+}
+
+function toAbsoluteUrl(rawHref, pageUrl) {
+  if (!rawHref) return null;
+  try {
+    const url = new URL(rawHref, pageUrl);
+    if (!/^https?:$/i.test(url.protocol)) return null;
+    return url.href;
+  } catch {
+    return null;
+  }
+}
+
+function parsePubDate(rawText) {
+  const text = normalizeText(rawText);
+  const match = text.match(DATE_RE);
+  if (!match) return new Date();
+  const [, y, mm, dd, hh, min, sec] = match;
+  const iso = `${y}-${mm.padStart(2, "0")}-${dd.padStart(2, "0")}T${(hh ?? "00").padStart(2, "0")}:${(min ?? "00").padStart(2, "0")}:${(sec ?? "00").padStart(2, "0")}+08:00`;
+  const parsed = new Date(iso);
+  return Number.isNaN(parsed.getTime()) ? new Date() : parsed;
+}
+
+function findNearbyDateText(anchor) {
+  let current = anchor;
+  for (let i = 0; i < 6 && current; i += 1) {
+    const text = normalizeText(current.textContent);
+    if (DATE_RE.test(text)) return text;
+    current = current.parentNode ?? null;
+  }
+  return "";
+}
+
+function isNoiseTitle(title) {
+  const text = normalizeText(title);
+  if (!text) return true;
+  if (text === "暂无图片" || text === "登录" || text === "加入社区") return true;
+  return false;
+}
+
+function pickBestTitle(candidates) {
+  const normalized = [...new Set(candidates.map((x) => normalizeText(x)).filter(Boolean))];
+  const preferred = normalized
+    .filter((text) => !isNoiseTitle(text))
+    .filter((text) => text.length >= 6 && text.length <= 80)
+    .sort((a, b) => a.length - b.length);
+  if (preferred.length > 0) return preferred[0];
+
+  const fallback = normalized
+    .filter((text) => !isNoiseTitle(text))
+    .filter((text) => text.length >= 6)
+    .sort((a, b) => a.length - b.length);
+  if (fallback.length > 0) return fallback[0];
+
+  return "";
+}
+
+function parseDomItems(html, finalUrl) {
+  const root = _deps.parseHtml(html);
+  const grouped = new Map();
+  const items = [];
+  const anchors = root.querySelectorAll("a[href]");
+
+  for (const anchor of anchors) {
+    const href = toAbsoluteUrl(anchor.getAttribute("href"), finalUrl);
+    if (!href) continue;
+    const url = new URL(href);
+    if (url.hostname !== "baaidata.csdn.net") continue;
+    if (!ARTICLE_PATH_RE.test(url.pathname)) continue;
+
+    const canonicalLink = `${url.origin}${url.pathname}`;
+    const group = grouped.get(canonicalLink) ?? { anchors: [], titles: [] };
+    group.anchors.push(anchor);
+    group.titles.push(anchor.textContent ?? "");
+    grouped.set(canonicalLink, group);
+  }
+
+  for (const [canonicalLink, group] of grouped) {
+    const title = pickBestTitle(group.titles);
+    if (!title) continue;
+
+    let dateText = "";
+    for (const anchor of group.anchors) {
+      const found = findNearbyDateText(anchor);
+      if (found) {
+        dateText = found;
+        break;
+      }
+    }
+
+    items.push({
+      guid: hashGuid(canonicalLink),
+      title,
+      link: canonicalLink,
+      pubDate: parsePubDate(dateText),
+      author: "智源数据社区",
+      sourceId: "baaidata-csdn",
+    });
+  }
+
+  items.sort((a, b) => b.pubDate.getTime() - a.pubDate.getTime());
+  return items;
+}
+
+function findJsonObjectEnd(raw, startIndex) {
+  let depth = 0;
+  let inString = false;
+  let escaped = false;
+
+  for (let i = startIndex; i < raw.length; i += 1) {
+    const ch = raw[i];
+    if (inString) {
+      if (escaped) {
+        escaped = false;
+        continue;
+      }
+      if (ch === "\\") {
+        escaped = true;
+        continue;
+      }
+      if (ch === "\"") inString = false;
+      continue;
+    }
+    if (ch === "\"") {
+      inString = true;
+      continue;
+    }
+    if (ch === "{") depth += 1;
+    if (ch === "}") {
+      depth -= 1;
+      if (depth === 0) return i;
+    }
+  }
+
+  return -1;
+}
+
+function extractInitialState(html) {
+  const marker = "window.__INITIAL_STATE__=";
+  const markerIndex = html.indexOf(marker);
+  if (markerIndex < 0) return null;
+
+  const objStart = html.indexOf("{", markerIndex + marker.length);
+  if (objStart < 0) return null;
+
+  const objEnd = findJsonObjectEnd(html, objStart);
+  if (objEnd < 0) return null;
+
+  const jsonRaw = html.slice(objStart, objEnd + 1);
+  try {
+    const parsed = JSON.parse(jsonRaw);
+    return parsed && typeof parsed === "object" ? parsed : null;
+  } catch {
+    return null;
+  }
+}
+
+function toItemFromState(raw, finalUrl) {
+  if (!raw || typeof raw !== "object") return null;
+  const entry = raw.content && typeof raw.content === "object" ? raw.content : raw;
+
+  const id = normalizeText(entry.id ?? "");
+  const title = normalizeText(entry.name ?? entry.title ?? "");
+  if (!title) return null;
+
+  const rawLink = normalizeText(raw.contentUrl ?? entry.contentUrl ?? entry.pagePath ?? "");
+  const fallbackLink = id ? `/${id}.html` : "";
+  const link = toAbsoluteUrl(rawLink || fallbackLink, finalUrl);
+  if (!link) return null;
+
+  const canonical = new URL(link);
+  const canonicalLink = `${canonical.origin}${canonical.pathname}`;
+
+  const summary = normalizeText(entry.desc ?? raw.desc ?? "");
+  const author = normalizeText(raw.nickname ?? raw.username ?? raw.user?.nickname ?? "");
+  const pubDate = parsePubDate(entry.createdTime ?? raw.createdTime ?? "");
+
+  return {
+    guid: hashGuid(canonicalLink),
+    title,
+    link: canonicalLink,
+    pubDate,
+    author: author || "智源数据社区",
+    summary: summary || undefined,
+    sourceId: "baaidata-csdn",
+  };
+}
+
+function parseStateItems(html, finalUrl) {
+  const state = extractInitialState(html);
+  if (!state) return [];
+
+  const latest = Array.isArray(state?.articleContent?.latest) ? state.articleContent.latest : [];
+  const hot = Array.isArray(state?.articleContent?.hot) ? state.articleContent.hot : [];
+  const headlines = Array.isArray(state?.headlines) ? state.headlines : [];
+
+  const merged = [...latest, ...hot, ...headlines];
+  const seen = new Set();
+  const items = [];
+
+  for (const raw of merged) {
+    const item = toItemFromState(raw, finalUrl);
+    if (!item) continue;
+    if (seen.has(item.link)) continue;
+    seen.add(item.link);
+    items.push(item);
+  }
+
+  items.sort((a, b) => b.pubDate.getTime() - a.pubDate.getTime());
+  return items;
+}
+
+async function fetchItems(sourceId, ctx) {
+  _deps = ctx.deps;
+  const rendered = await ctx.fetchHtml(sourceId, { waitMs: 3500 });
+  const fromDom = parseDomItems(rendered.html, rendered.finalUrl);
+  if (fromDom.length > 0) return fromDom;
+
+  const raw = await ctx.fetchHtml(sourceId, { waitMs: 3500, purify: false });
+  const fromState = parseStateItems(raw.html, raw.finalUrl);
+  if (fromState.length > 0) return fromState;
+
+  throw new Error("[baaidata-csdn] 未解析到条目，页面结构可能已变化");
+}
+
+export default {
+  id: "baaidata-csdn",
+  listUrlPattern: /^https?:\/\/baaidata\.csdn\.net\/?(?:\?.*)?$/i,
+  fetchItems,
+};

package/app/plugins/builtin/baidu-research.rssany.js

@@ -0,0 +1,222 @@
+let _deps;
+
+// Baidu Research 插件：抓取 Blog 列表条目（不做正文 enrich）
+
+
+
+const BLOG_ITEM_PATH_RE = /^\/Blog\/index-view(?:\/)?$/i;
+const MONTH_TO_INDEX = {
+  jan: 0,
+  feb: 1,
+  mar: 2,
+  apr: 3,
+  may: 4,
+  jun: 5,
+  jul: 6,
+  aug: 7,
+  sep: 8,
+  oct: 9,
+  nov: 10,
+  dec: 11,
+};
+const TEXT_NODE_SELECTORS = "div, p, span, h1, h2, h3, h4, h5, h6, strong, em";
+const MONTH_DAY_TEXT_RE = /^([A-Za-z]{3,9})\s+\d{1,2}(?:st|nd|rd|th)?(?:\s*[，,]\s*\d{0,4})?$/i;
+
+
+function normalizeText(text) {
+  return (text ?? "").replace(/\s+/g, " ").trim();
+}
+
+
+function hashGuid(input) {
+  return _deps.createHash("sha256").update(input).digest("hex");
+}
+
+
+function toAbsoluteHttpUrl(rawHref, baseUrl) {
+  if (!rawHref) return null;
+  const href = rawHref.trim();
+  if (!href || href.startsWith("#") || href.startsWith("javascript:")) return null;
+  try {
+    const url = new URL(href, baseUrl);
+    if (!/^https?:$/i.test(url.protocol)) return null;
+    return url.href;
+  } catch {
+    return null;
+  }
+}
+
+
+function inferYearFromTexts(texts) {
+  for (const text of texts) {
+    const m = normalizeText(text).match(/\b(19|20)\d{2}\b/);
+    if (!m) continue;
+    const year = Number(m[0]);
+    if (year >= 1990 && year <= new Date().getUTCFullYear() + 1) {
+      return year;
+    }
+  }
+  return undefined;
+}
+
+
+function parsePubDate(rawText, fallbackTexts) {
+  const text = normalizeText(rawText).replace("，", ",");
+  if (!text) return undefined;
+
+  const monthMatch = text.match(
+    /^([A-Za-z]{3,9})\s+(\d{1,2})(?:st|nd|rd|th)?(?:\s*,\s*)?(?:(\d{4}))?$/
+  );
+  if (monthMatch) {
+    const month = MONTH_TO_INDEX[monthMatch[1].slice(0, 3).toLowerCase()];
+    if (month != null) {
+      const day = Number(monthMatch[2]);
+      const year = monthMatch[3] ? Number(monthMatch[3]) : inferYearFromTexts(fallbackTexts);
+      if (!year) return undefined;
+      const parsed = new Date(Date.UTC(year, month, day, 12, 0, 0));
+      if (!Number.isNaN(parsed.getTime())) return parsed;
+    }
+  }
+
+  const numericMatch = text.match(/^(\d{4})[/-](\d{1,2})[/-](\d{1,2})$/);
+  if (numericMatch) {
+    const year = Number(numericMatch[1]);
+    const month = Number(numericMatch[2]) - 1;
+    const day = Number(numericMatch[3]);
+    const parsed = new Date(Date.UTC(year, month, day, 12, 0, 0));
+    if (!Number.isNaN(parsed.getTime())) return parsed;
+  }
+
+  return undefined;
+}
+
+
+function isDateLikeText(text) {
+  return MONTH_DAY_TEXT_RE.test(normalizeText(text));
+}
+
+
+function getLeafTexts(anchor) {
+  const all = anchor.querySelectorAll(TEXT_NODE_SELECTORS);
+  const out = [];
+  const seen = new Set();
+  for (const el of all) {
+    if (el.querySelector(TEXT_NODE_SELECTORS) != null) continue;
+    const text = normalizeText(el.textContent);
+    if (!text || text === "MORE") continue;
+    if (seen.has(text)) continue;
+    seen.add(text);
+    out.push(text);
+  }
+  return out;
+}
+
+
+function parseAnchorItem(anchor, finalUrl) {
+  const link = toAbsoluteHttpUrl(anchor.getAttribute("href"), finalUrl);
+  if (!link) return null;
+
+  const linkUrl = new URL(link);
+  if (linkUrl.hostname !== "research.baidu.com") return null;
+  if (!BLOG_ITEM_PATH_RE.test(linkUrl.pathname)) return null;
+  if (!/^\d+$/.test(linkUrl.searchParams.get("id") ?? "")) return null;
+
+  const texts = getLeafTexts(anchor);
+  if (texts.length === 0) return null;
+
+  const datedTexts = texts.map((text) => ({ text, date: parsePubDate(text, texts) }));
+  const dateIndex = datedTexts.findIndex((x) => x.date != null);
+  const pubDate = dateIndex >= 0 ? datedTexts[dateIndex].date : undefined;
+
+  const nonDateTexts = datedTexts
+    .filter((x) => x.date == null && !isDateLikeText(x.text))
+    .map((x) => x.text);
+  const titleCandidates = nonDateTexts.filter((text) => text.length >= 6);
+  const looksLikeSummary = (text) => text.endsWith("...") || text.split(/\s+/).length >= 22;
+
+  let title = "";
+  if (titleCandidates.length > 0) {
+    title = [...titleCandidates]
+      .sort((a, b) => {
+        const aPenalty = looksLikeSummary(a) ? 1 : 0;
+        const bPenalty = looksLikeSummary(b) ? 1 : 0;
+        if (aPenalty !== bPenalty) return aPenalty - bPenalty;
+        return a.length - b.length;
+      })[0];
+  }
+  if (!title && dateIndex > 0) {
+    title = datedTexts.slice(0, dateIndex).map((x) => x.text).find((text) => text.length >= 6) ?? "";
+  }
+  if (!title) return null;
+
+  const summary = nonDateTexts.find(
+    (text) => text !== title && (text.endsWith("...") || text.length >= 40)
+  );
+
+  return {
+    guid: hashGuid(link),
+    title,
+    link,
+    pubDate: pubDate ?? new Date(),
+    author: "Baidu Research",
+    summary: summary || undefined,
+    sourceId: "baidu-research",
+  };
+}
+
+
+function parseBlogItems(html, finalUrl) {
+  const root = _deps.parseHtml(html);
+  const anchors = root.querySelectorAll("a[href]");
+  const seen = new Set();
+  const items = [];
+
+  for (const anchor of anchors) {
+    const item = parseAnchorItem(anchor, finalUrl);
+    if (!item) continue;
+    if (seen.has(item.link)) continue;
+    seen.add(item.link);
+    items.push(item);
+  }
+
+  return items;
+}
+
+
+function mergeByLink(itemsA, itemsB) {
+  const byLink = new Map();
+  for (const item of [...itemsA, ...itemsB]) {
+    if (byLink.has(item.link)) continue;
+    byLink.set(item.link, item);
+  }
+  return Array.from(byLink.values());
+}
+
+
+async function fetchItems(sourceId, ctx) {
+  _deps = ctx.deps;
+  const primary = await ctx.fetchHtml(sourceId, { waitMs: 4500 });
+  let items = parseBlogItems(primary.html, primary.finalUrl || sourceId);
+
+  const primaryUrl = new URL(primary.finalUrl || sourceId);
+  if (items.length < 5 && primaryUrl.hostname === "research.baidu.com") {
+    const blogUrl = new URL("/Blog", primaryUrl).href;
+    if (blogUrl !== primaryUrl.href) {
+      const blogPage = await ctx.fetchHtml(blogUrl, { waitMs: 4500 });
+      items = mergeByLink(items, parseBlogItems(blogPage.html, blogPage.finalUrl || blogUrl));
+    }
+  }
+
+  if (items.length === 0) {
+    throw new Error("[baidu-research] 未解析到 Blog 条目，页面结构可能已变化");
+  }
+
+  return items;
+}
+
+
+export default {
+  id: "baidu-research",
+  listUrlPattern: /^https?:\/\/research\.baidu\.com\/(?:(?:Index|Blog)\/?)?(?:\?.*)?$/i,
+  fetchItems,
+};