rssany 0.1.2 → 0.1.5

package/app/plugins/builtin/opendrivelab.rssany.js

@@ -0,0 +1,333 @@
+let _deps;
+
+// OpenDriveLab 首页插件：解析首页展示内容并输出 FeedItem（不含 enrich）
+
+
+const SITE_ID = "opendrivelab";
+const NAVIGATION_TITLES = new Set([
+  "news",
+  "recruit",
+  "research",
+  "publication",
+  "dataset",
+  "event",
+  "more",
+  "team",
+  "sponsor",
+  "opendrivelab",
+  "embodied ai",
+  "autonomous driving",
+]);
+const ACTION_LABELS = new Set([
+  "paper",
+  "page",
+  "blog",
+  "code",
+  "github",
+  "dataset",
+  "demo",
+  "video",
+  "poster",
+  "slides",
+  "community",
+  "cite",
+  "checkout at mmlab.hk/mm-hand",
+]);
+const NAVIGATION_PATHS = new Set([
+  "/",
+  "/embodiedai",
+  "/autonomousdriving",
+  "/publications",
+  "/events",
+  "/team",
+  "/recruit",
+  "/ccai9025",
+]);
+
+function normalizeText(text) {
+  return (text ?? "").replace(/\s+/g, " ").trim();
+}
+
+function hashGuid(input) {
+  return _deps.createHash("sha256").update(input).digest("hex");
+}
+
+function toAbsoluteHttpUrl(rawHref, baseUrl) {
+  if (!rawHref) return null;
+  const href = rawHref.trim();
+  if (!href || href.startsWith("#") || href.startsWith("javascript:") || href.startsWith("mailto:")) return null;
+  try {
+    const url = new URL(href, baseUrl);
+    if (!/^https?:$/i.test(url.protocol)) return null;
+    return url.href;
+  } catch {
+    return null;
+  }
+}
+
+function normalizePath(pathname) {
+  if (!pathname) return "/";
+  const trimmed = pathname.replace(/\/+$/, "");
+  return (trimmed || "/").toLowerCase();
+}
+
+function isBlockedPage(root, html, finalUrl) {
+  const text = normalizeText(root.textContent).toLowerCase();
+  const body = (html ?? "").toLowerCase();
+  const url = (finalUrl ?? "").toLowerCase();
+  if (url.includes("/cdn-cgi/challenge")) return true;
+  if (body.includes("__cf_chl_opt")) return true;
+  if (body.includes("/cdn-cgi/challenge-platform")) return true;
+  if (text.includes("just a moment")) return true;
+  if (text.includes("checking your browser")) return true;
+  if (text.includes("attention required")) return true;
+  return text.includes("captcha");
+}
+
+function isNoiseTitle(text) {
+  const title = normalizeText(text);
+  if (!title) return true;
+  const lower = title.toLowerCase();
+  if (NAVIGATION_TITLES.has(lower)) return true;
+  if (/^\d+\s*\/\s*\d+$/.test(lower)) return true;
+  if (title.length < 8) return true;
+  return false;
+}
+
+function isActionLabel(text) {
+  const lower = normalizeText(text).toLowerCase();
+  if (!lower) return true;
+  if (ACTION_LABELS.has(lower)) return true;
+  if (/(best paper|award|finalist|position paper)/i.test(lower)) return true;
+  return false;
+}
+
+function findContentContainer(node) {
+  let current = node;
+  for (let i = 0; i < 8 && current; i += 1) {
+    if (current.nodeType !== _deps.NodeType.ELEMENT_NODE) {
+      current = current.parentNode ?? null;
+      continue;
+    }
+    const anchors = current.querySelectorAll?.("a[href]") ?? [];
+    if (anchors.length >= 1 && anchors.length <= 20) return current;
+    current = current.parentNode ?? null;
+  }
+  return node.parentNode ?? node;
+}
+
+function parseDateFromText(text) {
+  const normalized = normalizeText(text);
+  if (!normalized) return undefined;
+
+  let m = normalized.match(/\b(20\d{2})[./-](\d{1,2})[./-](\d{1,2})\b/);
+  if (m) {
+    const [, y, mm, dd] = m;
+    const date = new Date(Date.UTC(Number(y), Number(mm) - 1, Number(dd), 12, 0, 0));
+    if (!Number.isNaN(date.getTime())) return date;
+  }
+
+  m = normalized.match(/\b(January|February|March|April|May|June|July|August|September|October|November|December)\s*,?\s*(20\d{2})\b/i);
+  if (m) {
+    const monthMap = {
+      january: 0,
+      february: 1,
+      march: 2,
+      april: 3,
+      may: 4,
+      june: 5,
+      july: 6,
+      august: 7,
+      september: 8,
+      october: 9,
+      november: 10,
+      december: 11,
+    };
+    const monthIndex = monthMap[m[1].toLowerCase()];
+    const year = Number(m[2]);
+    const date = new Date(Date.UTC(year, monthIndex, 1, 12, 0, 0));
+    if (!Number.isNaN(date.getTime())) return date;
+  }
+
+  m = normalized.match(/\b(20\d{2})\b/);
+  if (m) {
+    const year = Number(m[1]);
+    const date = new Date(Date.UTC(year, 0, 1, 12, 0, 0));
+    if (!Number.isNaN(date.getTime())) return date;
+  }
+  return undefined;
+}
+
+function parseDateFromLink(link) {
+  try {
+    const url = new URL(link);
+    if (!/(^|\.)arxiv\.org$/i.test(url.hostname)) return undefined;
+    const m = url.pathname.match(/\/abs\/(\d{2})(\d{2})\.\d+/);
+    if (!m) return undefined;
+    const year = 2000 + Number(m[1]);
+    const month = Number(m[2]);
+    if (month < 1 || month > 12) return undefined;
+    const date = new Date(Date.UTC(year, month - 1, 1, 12, 0, 0));
+    return Number.isNaN(date.getTime()) ? undefined : date;
+  } catch {
+    return undefined;
+  }
+}
+
+function pickPrimaryLink(container, title, baseUrl) {
+  const anchors = container.querySelectorAll("a[href]");
+  const candidates = [];
+
+  for (const anchor of anchors) {
+    const text = normalizeText(anchor.textContent);
+    if (!text) continue;
+    const link = toAbsoluteHttpUrl(anchor.getAttribute("href"), baseUrl);
+    if (!link) continue;
+    const path = normalizePath(new URL(link).pathname);
+    candidates.push({ text, textLower: text.toLowerCase(), link, path });
+  }
+
+  if (candidates.length === 0) return null;
+
+  const normalizedTitle = normalizeText(title).toLowerCase();
+  const titleMatch = candidates.find((item) => item.textLower === normalizedTitle);
+  if (titleMatch) return titleMatch.link;
+
+  const actionMatch = candidates.find((item) => ACTION_LABELS.has(item.textLower));
+  if (actionMatch) return actionMatch.link;
+
+  const nonNav = candidates.find((item) => !NAVIGATION_PATHS.has(item.path));
+  return (nonNav ?? candidates[0]).link;
+}
+
+function pickSummary(container, title) {
+  const texts = [];
+  for (const selector of ["i", "p", "h2", "h3", "span"]) {
+    for (const node of container.querySelectorAll(selector)) {
+      const text = normalizeText(node.textContent);
+      if (!text || text === title) continue;
+      if (isActionLabel(text)) continue;
+      if (/^\d+\s*\/\s*\d+$/.test(text)) continue;
+      if (parseDateFromText(text) && text.length <= 24) continue;
+      texts.push(text);
+    }
+  }
+
+  const unique = [...new Set(texts)];
+  return unique.find((text) => text.length >= 20 && text.length <= 400);
+}
+
+function extractPubDate(headingNode, container, link) {
+  const texts = [];
+  const aroundHeading = normalizeText(headingNode.parentNode?.textContent);
+  if (aroundHeading) texts.push(aroundHeading);
+  const containerText = normalizeText(container.textContent);
+  if (containerText) texts.push(containerText);
+
+  let cursor = container.parentNode ?? null;
+  for (let i = 0; i < 3 && cursor; i += 1) {
+    const t = normalizeText(cursor.textContent);
+    if (t && t.length <= 3000) texts.push(t);
+    cursor = cursor.parentNode ?? null;
+  }
+
+  for (const text of texts) {
+    const date = parseDateFromText(text);
+    if (date) return date;
+  }
+
+  return parseDateFromLink(link) ?? new Date();
+}
+
+function toFeedItem({ title, link, pubDate, summary }) {
+  return {
+    guid: hashGuid(link),
+    title,
+    link,
+    pubDate,
+    author: "OpenDriveLab",
+    summary: summary || undefined,
+    sourceId: SITE_ID,
+  };
+}
+
+function parseFromHeadings(root, finalUrl, seen) {
+  const items = [];
+  const headings = root.querySelectorAll("h1, h2, h3");
+
+  for (const heading of headings) {
+    const title = normalizeText(heading.textContent);
+    if (isNoiseTitle(title)) continue;
+
+    const container = findContentContainer(heading);
+    const link = pickPrimaryLink(container, title, finalUrl);
+    if (!link || seen.has(link)) continue;
+
+    seen.add(link);
+    items.push(
+      toFeedItem({
+        title,
+        link,
+        summary: pickSummary(container, title),
+        pubDate: extractPubDate(heading, container, link),
+      }),
+    );
+  }
+
+  return items;
+}
+
+function parseFromTitleAnchors(root, finalUrl, seen) {
+  const items = [];
+  const anchors = root.querySelectorAll("a[href]");
+
+  for (const anchor of anchors) {
+    const title = normalizeText(anchor.textContent);
+    if (!title || title.length < 20) continue;
+    if (isNoiseTitle(title) || isActionLabel(title)) continue;
+
+    const link = toAbsoluteHttpUrl(anchor.getAttribute("href"), finalUrl);
+    if (!link || seen.has(link)) continue;
+
+    const path = normalizePath(new URL(link).pathname);
+    if (NAVIGATION_PATHS.has(path)) continue;
+
+    const container = findContentContainer(anchor);
+    seen.add(link);
+    items.push(
+      toFeedItem({
+        title,
+        link,
+        summary: pickSummary(container, title),
+        pubDate: extractPubDate(anchor, container, link),
+      }),
+    );
+  }
+
+  return items;
+}
+
+async function fetchItems(sourceId, ctx) {
+  _deps = ctx.deps;
+  const { html, finalUrl } = await ctx.fetchHtml(sourceId, { waitMs: 4500 });
+  const root = _deps.parseHtml(html);
+
+  const seenLinks = new Set();
+  const items = [
+    ...parseFromHeadings(root, finalUrl, seenLinks),
+    ...parseFromTitleAnchors(root, finalUrl, seenLinks),
+  ];
+
+  if (items.length > 0) return items;
+
+  if (isBlockedPage(root, html, finalUrl)) {
+    throw new Error(`[${SITE_ID}] 命中站点风控验证页，当前会话无法稳定抓取`);
+  }
+  throw new Error(`[${SITE_ID}] 未解析到首页条目，页面结构可能已变化`);
+}
+
+export default {
+  id: SITE_ID,
+  listUrlPattern: /^https?:\/\/(www\.)?opendrivelab\.com\/?(\?.*)?$/i,
+  fetchItems,
+};

package/app/plugins/builtin/paperswithcode.rssany.js

@@ -0,0 +1,227 @@
+let _deps;
+
+
+
+const SITE_ID = "paperswithcode";
+const API_ORIGIN = "https://paperswithcode.co";
+const DEFAULT_TRENDING_LIMIT = 30;
+const DEFAULT_MAX_AGE_DAYS = 180;
+const DEFAULT_LATEST_PAGE_SIZE = 30;
+
+
+function normalizeText(text) {
+  return (text ?? "").replace(/\s+/g, " ").trim();
+}
+
+
+function hashGuid(input) {
+  return _deps.createHash("sha256").update(input).digest("hex");
+}
+
+
+function toValidDate(raw) {
+  const text = normalizeText(raw);
+  if (!text) return new Date();
+  const parsed = new Date(text);
+  return Number.isNaN(parsed.getTime()) ? new Date() : parsed;
+}
+
+
+function parsePositiveInt(raw, fallback, { min = 1, max = 200 } = {}) {
+  const n = Number.parseInt(normalizeText(raw), 10);
+  if (Number.isNaN(n)) return fallback;
+  return Math.min(max, Math.max(min, n));
+}
+
+
+function resolveMode(sourceUrl) {
+  const sort = normalizeText(sourceUrl.searchParams.get("sort")).toLowerCase();
+  if (["latest", "new", "date", "date_published"].includes(sort)) return "latest";
+  if (["trending", "hot", "popular"].includes(sort)) return "trending";
+  return "trending";
+}
+
+
+function toPaperLink(arxivId) {
+  const id = normalizeText(arxivId);
+  if (!id) return null;
+  return `${API_ORIGIN}/paper/${encodeURIComponent(id)}`;
+}
+
+
+function joinCategories(tasks) {
+  if (!Array.isArray(tasks)) return undefined;
+  const values = tasks
+    .map((task) => {
+      if (typeof task === "string") return normalizeText(task);
+      if (task && typeof task === "object") return normalizeText(task.name);
+      return "";
+    })
+    .filter(Boolean);
+  return values.length > 0 ? values : undefined;
+}
+
+
+function buildSummaryFromTrending(item) {
+  const repoName = normalizeText(item?.repository?.name);
+  const repoOwner = normalizeText(item?.repository?.owner);
+  const stars = Number(item?.repository?.num_stars ?? 0);
+  const tasks = joinCategories(item?.tasks);
+  const parts = [];
+
+  if (repoOwner && repoName) {
+    parts.push(`Repo: ${repoOwner}/${repoName}`);
+  }
+  if (stars > 0) {
+    parts.push(`Stars: ${stars}`);
+  }
+  if (tasks?.length) {
+    parts.push(`Tasks: ${tasks.join(", ")}`);
+  }
+  return parts.length ? parts.join(" | ") : undefined;
+}
+
+
+function mapTrendingItem(item) {
+  const title = normalizeText(item?.title);
+  const link = toPaperLink(item?.arxiv_id);
+  if (!title || !link) return null;
+
+  return {
+    guid: hashGuid(link),
+    title,
+    link,
+    pubDate: toValidDate(item?.date_published),
+    summary: buildSummaryFromTrending(item),
+    sourceId: SITE_ID,
+  };
+}
+
+
+function mapLatestItem(item) {
+  const title = normalizeText(item?.title);
+  const link = toPaperLink(item?.arxiv_id) || normalizeText(item?.url_abs);
+  if (!title || !link) return null;
+
+  const summary = normalizeText(item?.abstract);
+  const firstAuthor = Array.isArray(item?.authors) ? normalizeText(item.authors[0]) : "";
+
+  return {
+    guid: hashGuid(link),
+    title,
+    link,
+    pubDate: toValidDate(item?.published),
+    author: firstAuthor || undefined,
+    summary: summary || undefined,
+    sourceId: SITE_ID,
+  };
+}
+
+
+async function fetchJson(url) {
+  const res = await fetch(url, {
+    headers: {
+      Accept: "application/json",
+      "User-Agent":
+        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
+    },
+  });
+  if (!res.ok) {
+    throw new Error(`HTTP ${res.status}`);
+  }
+  try {
+    return await res.json();
+  } catch {
+    throw new Error("接口返回非 JSON 数据");
+  }
+}
+
+
+function dedupeByLink(items) {
+  const seen = new Set();
+  const output = [];
+  for (const item of items) {
+    if (!item || !item.link) continue;
+    if (seen.has(item.link)) continue;
+    seen.add(item.link);
+    output.push(item);
+  }
+  return output;
+}
+
+
+async function fetchTrendingItems(sourceUrl) {
+  const limit = parsePositiveInt(sourceUrl.searchParams.get("limit"), DEFAULT_TRENDING_LIMIT, { max: 100 });
+  const maxAgeDays = parsePositiveInt(
+    sourceUrl.searchParams.get("max_age_days"),
+    DEFAULT_MAX_AGE_DAYS,
+    { max: 3650 }
+  );
+
+  const apiUrl = new URL("/api/v1/papers/trending", API_ORIGIN);
+  apiUrl.searchParams.set("limit", String(limit));
+  apiUrl.searchParams.set("max_age_days", String(maxAgeDays));
+
+  const payload = await fetchJson(apiUrl);
+  const rows = Array.isArray(payload) ? payload : [];
+  return dedupeByLink(rows.map(mapTrendingItem).filter(Boolean));
+}
+
+
+async function fetchLatestItems(sourceUrl) {
+  const pageSize = parsePositiveInt(sourceUrl.searchParams.get("page_size"), DEFAULT_LATEST_PAGE_SIZE, {
+    max: 100,
+  });
+  const page = parsePositiveInt(sourceUrl.searchParams.get("page"), 1, { max: 1000 });
+
+  const apiUrl = new URL("/api/v1/papers/", API_ORIGIN);
+  apiUrl.searchParams.set("page", String(page));
+  apiUrl.searchParams.set("page_size", String(pageSize));
+  apiUrl.searchParams.set("order_by", "date_published");
+  apiUrl.searchParams.set("order_dir", "desc");
+  apiUrl.searchParams.set("include_resources", "true");
+
+  const payload = await fetchJson(apiUrl);
+  const rows = Array.isArray(payload?.results) ? payload.results : [];
+  return dedupeByLink(rows.map(mapLatestItem).filter(Boolean));
+}
+
+
+async function fetchItems(sourceId, _ctx) {
+  _deps = _ctx.deps;
+  let sourceUrl;
+  try {
+    sourceUrl = new URL(sourceId);
+  } catch {
+    throw new Error(`[${SITE_ID}] 无效 URL: ${sourceId}`);
+  }
+
+  const mode = resolveMode(sourceUrl);
+  const errors = [];
+
+  const tryMode = async (m) => {
+    try {
+      return m === "latest" ? await fetchLatestItems(sourceUrl) : await fetchTrendingItems(sourceUrl);
+    } catch (err) {
+      const msg = err instanceof Error ? err.message : String(err);
+      errors.push(`${m}: ${msg}`);
+      return [];
+    }
+  };
+
+  const primary = await tryMode(mode);
+  if (primary.length > 0) return primary;
+
+  const fallbackMode = mode === "trending" ? "latest" : "trending";
+  const fallback = await tryMode(fallbackMode);
+  if (fallback.length > 0) return fallback;
+
+  throw new Error(`[${SITE_ID}] 未解析到条目（${errors.join(" | ")}）`);
+}
+
+
+export default {
+  id: SITE_ID,
+  listUrlPattern: /^https?:\/\/(www\.)?paperswithcode\.(co|com)(?:\/(?:papers)?\/?)?(?:\?.*)?$/i,
+  fetchItems,
+};