rssany 0.1.2 → 0.1.5

package/app/plugins/builtin/opendatalab.rssany.js

@@ -0,0 +1,109 @@
+let _deps;
+
+
+const OPENDATALAB_ORIGIN = "https://opendatalab.org.cn";
+const OPENDATALAB_LIST_API = `${OPENDATALAB_ORIGIN}/datasets/api/v3/datasets/list`;
+
+function normalizeText(text) {
+  return (text ?? "").replace(/\s+/g, " ").trim();
+}
+
+function hashGuid(input) {
+  return _deps.createHash("sha256").update(input).digest("hex");
+}
+
+function toDate(value) {
+  const raw = typeof value === "string" ? Number(value) : value;
+  if (typeof raw !== "number" || !Number.isFinite(raw) || raw <= 0) return undefined;
+  const ms = raw < 1e12 ? raw * 1000 : raw;
+  const date = new Date(ms);
+  return Number.isNaN(date.getTime()) ? undefined : date;
+}
+
+function pickPubDate(record) {
+  const candidates = [record?.lastUpdateTime, record?.updatedAt, record?.publicTime, record?.createdAt];
+  for (const value of candidates) {
+    const parsed = toDate(value);
+    if (parsed) return parsed;
+  }
+  return new Date();
+}
+
+function toDatasetLink(name) {
+  const normalized = normalizeText(name);
+  if (!normalized) return null;
+  const parts = normalized.split("/").map((x) => normalizeText(x)).filter(Boolean);
+  if (parts.length < 2) return null;
+  const encodedPath = parts.map((x) => encodeURIComponent(x)).join("/");
+  return `${OPENDATALAB_ORIGIN}/${encodedPath}`;
+}
+
+function toFeedItem(record) {
+  if (!record || typeof record !== "object") return null;
+  const title = normalizeText(record.displayName || record.name || "");
+  const link = toDatasetLink(record.name);
+  if (!title || !link) return null;
+
+  const summary = normalizeText(record?.introduction?.zh || record?.introduction?.en || "");
+  const author = normalizeText(record?.createdBy?.name || record?.updatedBy || "");
+
+  return {
+    guid: hashGuid(link),
+    title,
+    link,
+    pubDate: pickPubDate(record),
+    author: author || undefined,
+    summary: summary || undefined,
+    sourceId: "opendatalab",
+  };
+}
+
+function parsePaginationFromSourceId(sourceId) {
+  const defaults = { pageNo: 1, pageSize: 30 };
+  try {
+    const url = new URL(sourceId);
+    const pageNo = Number(url.searchParams.get("pageNo") ?? defaults.pageNo);
+    const pageSize = Number(url.searchParams.get("pageSize") ?? defaults.pageSize);
+    const safePageNo = Number.isInteger(pageNo) && pageNo > 0 ? pageNo : defaults.pageNo;
+    const safePageSize = Number.isInteger(pageSize) && pageSize >= 1 && pageSize <= 100 ? pageSize : defaults.pageSize;
+    return { pageNo: safePageNo, pageSize: safePageSize };
+  } catch {
+    return defaults;
+  }
+}
+
+async function fetchItems(sourceId, ctx) {
+  _deps = ctx.deps;
+  const { pageNo, pageSize } = parsePaginationFromSourceId(sourceId);
+  const response = await fetch(OPENDATALAB_LIST_API, {
+    method: "POST",
+    headers: {
+      "Content-Type": "application/json",
+      "Accept": "application/json",
+    },
+    body: JSON.stringify({ pageNo, pageSize }),
+  });
+
+  if (!response.ok) {
+    throw new Error(`[opendatalab] 请求列表接口失败: HTTP ${response.status}`);
+  }
+
+  const payload = await response.json().catch(() => null);
+  const list = payload?.data?.list;
+  if (!Array.isArray(list)) {
+    const msg = normalizeText(payload?.msg) || "返回结构异常";
+    throw new Error(`[opendatalab] 列表接口响应不可用: ${msg}`);
+  }
+
+  const items = list.map((record) => toFeedItem(record)).filter(Boolean);
+  if (items.length === 0) {
+    throw new Error("[opendatalab] 未解析到条目，接口结构可能已变化");
+  }
+  return items;
+}
+
+export default {
+  id: "opendatalab",
+  listUrlPattern: /^https?:\/\/(www\.)?opendatalab\.(org\.cn|com)\/?(?:datasets\/?)?(?:\?.*)?$/i,
+  fetchItems,
+};

package/app/plugins/builtin/opendrivelab-autonomous-driving.rssany.js

@@ -0,0 +1,114 @@
+let _deps;
+
+// OpenDriveLab Autonomous Driving 插件：抓取时间线条目并输出 FeedItem（不含 enrich）
+
+
+function normalizeText(text) {
+  return (text ?? "").replace(/\s+/g, " ").trim();
+}
+
+function hashGuid(input) {
+  return _deps.createHash("sha256").update(input).digest("hex");
+}
+
+function toAbsoluteHttpUrl(rawHref, baseUrl) {
+  if (!rawHref) return null;
+  const href = rawHref.trim();
+  if (!href || href.startsWith("#") || href.startsWith("javascript:")) return null;
+  try {
+    const url = new URL(href, baseUrl);
+    if (!/^https?:$/i.test(url.protocol)) return null;
+    return url.href;
+  } catch {
+    return null;
+  }
+}
+
+function parsePubDate(dateText) {
+  const normalized = normalizeText(dateText);
+  const m = normalized.match(/(\d{4})[./-](\d{1,2})[./-](\d{1,2})/);
+  if (!m) return new Date();
+  const year = Number(m[1]);
+  const month = Number(m[2]);
+  const day = Number(m[3]);
+  if (!Number.isFinite(year) || !Number.isFinite(month) || !Number.isFinite(day)) {
+    return new Date();
+  }
+  // 统一使用 UTC 中午，避免仅日期时的时区偏移问题。
+  return new Date(Date.UTC(year, month - 1, day, 12, 0, 0));
+}
+
+const AUX_LINK_TEXTS = new Set([
+  "paper",
+  "page",
+  "github",
+  "dataset",
+  "hugging face",
+  "video",
+  "blog",
+  "poster",
+  "slides",
+  "arxiv",
+  "code",
+  "demo",
+  "解讀",
+]);
+
+function findTitleAnchor(li, finalUrl) {
+  const anchors = li.querySelectorAll("a[href]");
+  let fallback = null;
+
+  for (const anchor of anchors) {
+    const title = normalizeText(anchor.textContent);
+    const link = toAbsoluteHttpUrl(anchor.getAttribute("href"), finalUrl);
+    if (!title || !link) continue;
+
+    if (!fallback) fallback = { title, link };
+    if (AUX_LINK_TEXTS.has(title.toLowerCase())) continue;
+    if (title.length < 8) continue;
+    return { title, link };
+  }
+
+  return fallback;
+}
+
+async function fetchItems(sourceId, ctx) {
+  _deps = ctx.deps;
+  const { html, finalUrl } = await ctx.fetchHtml(sourceId, { waitMs: 3500 });
+  const root = _deps.parseHtml(html);
+
+  const seenLinks = new Set();
+  const items = [];
+  const rows = root.querySelectorAll("li");
+  for (const row of rows) {
+    const dateText = normalizeText(row.querySelector("time")?.textContent);
+    if (!dateText) continue;
+
+    const titleAnchor = findTitleAnchor(row, finalUrl);
+    if (!titleAnchor) continue;
+    if (seenLinks.has(titleAnchor.link)) continue;
+    seenLinks.add(titleAnchor.link);
+
+    const summaryText = normalizeText(row.querySelector("i")?.textContent);
+    items.push({
+      guid: hashGuid(titleAnchor.link),
+      title: titleAnchor.title,
+      link: titleAnchor.link,
+      pubDate: parsePubDate(dateText),
+      author: "OpenDriveLab",
+      summary: summaryText || undefined,
+      sourceId: "opendrivelab-autonomous-driving",
+    });
+  }
+
+  if (items.length === 0) {
+    throw new Error("[opendrivelab-autonomous-driving] 未解析到条目，页面结构可能已变化");
+  }
+  return items;
+}
+
+export default {
+  id: "opendrivelab-autonomous-driving",
+  listUrlPattern: /^https?:\/\/(www\.)?opendrivelab\.com\/AutonomousDriving\/?(\?.*)?$/i,
+  fetchItems,
+};

package/app/plugins/builtin/opendrivelab-embodiedai.rssany.js

@@ -0,0 +1,114 @@
+let _deps;
+
+
+const SITE_ID = "opendrivelab-embodiedai";
+const DATE_RE = /\b(20\d{2})[./-](\d{1,2})[./-](\d{1,2})\b/;
+const ACTION_LINK_LABELS = new Set([
+  "paper",
+  "page",
+  "blog",
+  "github",
+  "video",
+  "dataset",
+  "challenge",
+  "hugging face",
+  "hardware guide",
+]);
+
+function normalizeText(text) {
+  return (text ?? "").replace(/\s+/g, " ").trim();
+}
+
+function hashGuid(input) {
+  return _deps.createHash("sha256").update(input).digest("hex");
+}
+
+function toAbsoluteHttpUrl(rawHref, baseUrl) {
+  if (!rawHref) return null;
+  const href = rawHref.trim();
+  if (!href || href.startsWith("#") || href.startsWith("javascript:")) return null;
+  try {
+    const url = new URL(href, baseUrl);
+    if (!/^https?:$/i.test(url.protocol)) return null;
+    return url.href;
+  } catch {
+    return null;
+  }
+}
+
+function parseDate(dateText) {
+  const text = normalizeText(dateText);
+  const m = text.match(DATE_RE);
+  if (!m) return undefined;
+  const [, y, mm, dd] = m;
+  const date = new Date(Date.UTC(Number(y), Number(mm) - 1, Number(dd), 12, 0, 0));
+  return Number.isNaN(date.getTime()) ? undefined : date;
+}
+
+function isActionLabel(text) {
+  const normalized = normalizeText(text).toLowerCase();
+  return ACTION_LINK_LABELS.has(normalized);
+}
+
+function findTitleAnchor(liNode, finalUrl) {
+  const anchors = liNode.querySelectorAll("a[href]");
+  let fallback = null;
+
+  for (const anchor of anchors) {
+    const title = normalizeText(anchor.textContent);
+    if (!title || isActionLabel(title)) continue;
+    const link = toAbsoluteHttpUrl(anchor.getAttribute("href"), finalUrl);
+    if (!link) continue;
+    if (!fallback) fallback = { anchor, link, title };
+    if (title.length >= 12) return { anchor, link, title };
+  }
+
+  return fallback;
+}
+
+function buildItemsFromHtml(html, finalUrl) {
+  const root = _deps.parseHtml(html);
+  const items = [];
+  const seen = new Set();
+  const liNodes = root.querySelectorAll("li");
+
+  for (const li of liNodes) {
+    const dateText = normalizeText(li.querySelector("time")?.textContent);
+    if (!dateText) continue;
+
+    const titleAnchor = findTitleAnchor(li, finalUrl);
+    if (!titleAnchor) continue;
+    if (seen.has(titleAnchor.link)) continue;
+
+    const summary = normalizeText(li.querySelector("i")?.textContent);
+    items.push({
+      guid: hashGuid(titleAnchor.link),
+      title: titleAnchor.title,
+      link: titleAnchor.link,
+      pubDate: parseDate(dateText) ?? new Date(),
+      summary: summary || undefined,
+    });
+    seen.add(titleAnchor.link);
+  }
+
+  return items;
+}
+
+async function fetchItems(sourceId, ctx) {
+  _deps = ctx.deps;
+  const { html, finalUrl } = await ctx.fetchHtml(sourceId, { waitMs: 3500 });
+  const items = buildItemsFromHtml(html, finalUrl);
+  if (items.length > 0) return items;
+
+  const text = normalizeText(_deps.parseHtml(html).textContent).toLowerCase();
+  if (text.includes("just a moment") || text.includes("checking your browser")) {
+    throw new Error(`[${SITE_ID}] 命中站点风控验证页，当前会话无法稳定抓取`);
+  }
+  throw new Error(`[${SITE_ID}] 未解析到 Embodied AI 条目，页面结构可能已变化`);
+}
+
+export default {
+  id: SITE_ID,
+  listUrlPattern: /^https?:\/\/(www\.)?opendrivelab\.com\/EmbodiedAI\/?(\?.*)?$/i,
+  fetchItems,
+};

package/app/plugins/builtin/opendrivelab-publications.rssany.js

@@ -0,0 +1,130 @@
+let _deps;
+
+
+
+const VENUE_HINT_RE = /(20\d{2}|cvpr|iccv|eccv|neurips|iclr|aaai|ijcv|tpami|icra|rss|corl|preprint|arxiv)/i;
+const BLOCKED_HOST_RE = /(^|\.)scholar\.google\.com$|(^|\.)github\.com$|(^|\.)img\.shields\.io$|(^|\.)youtube\.com$|(^|\.)youtu\.be$|(^|\.)zhihu\.com$|(^|\.)mp\.weixin\.qq\.com$|(^|\.)cvpr\d{4}\.thecvf\.com$/i;
+const AWARD_RE = /\baward\b/i;
+
+
+function normalizeText(text) {
+  return (text ?? "").replace(/\s+/g, " ").trim();
+}
+
+
+function toHttpUrl(rawHref, baseUrl) {
+  if (!rawHref) return null;
+  try {
+    const url = new URL(rawHref, baseUrl);
+    if (!/^https?:$/i.test(url.protocol)) return null;
+    return url;
+  } catch {
+    return null;
+  }
+}
+
+
+function hashGuid(input) {
+  return _deps.createHash("sha256").update(input).digest("hex");
+}
+
+
+function isLikelyPaperTitle(title) {
+  if (!title || title.length < 20) return false;
+  if (AWARD_RE.test(title)) return false;
+  const words = title.match(/[A-Za-z0-9][A-Za-z0-9-]*/g) ?? [];
+  return words.length >= 4;
+}
+
+
+function findYear(text) {
+  const m = normalizeText(text).match(/\b(20\d{2})\b/);
+  if (!m) return undefined;
+  const year = Number(m[1]);
+  if (year < 2000 || year > 2099) return undefined;
+  return year;
+}
+
+
+function extractContext(anchor) {
+  let node = anchor;
+  let summary;
+  let category;
+  let year;
+
+  for (let i = 0; i < 8 && node; i += 1) {
+    if (!summary) {
+      const summaryNode = node.querySelector?.("i");
+      const summaryText = normalizeText(summaryNode?.textContent);
+      if (summaryText && summaryText.length >= 24) summary = summaryText;
+    }
+
+    const spanTexts = (node.querySelectorAll?.("span") ?? [])
+      .map((el) => normalizeText(el.textContent))
+      .filter(Boolean);
+
+    if (!category) {
+      category = spanTexts.find((text) => VENUE_HINT_RE.test(text) && !AWARD_RE.test(text));
+    }
+    if (year == null) {
+      for (const text of spanTexts) {
+        const parsed = findYear(text);
+        if (parsed != null) {
+          year = parsed;
+          break;
+        }
+      }
+    }
+
+    node = node.parentNode ?? null;
+  }
+
+  return { summary, category, year };
+}
+
+
+async function fetchItems(sourceId, ctx) {
+  _deps = ctx.deps;
+  const { html, finalUrl } = await ctx.fetchHtml(sourceId, { waitMs: 3500 });
+  const root = _deps.parseHtml(html);
+  const main = root.querySelector("main") ?? root;
+  const anchors = main.querySelectorAll("a[href]");
+  const seen = new Set();
+  const items = [];
+
+  for (const anchor of anchors) {
+    const title = normalizeText(anchor.textContent);
+    if (!isLikelyPaperTitle(title)) continue;
+
+    const url = toHttpUrl(anchor.getAttribute("href"), finalUrl);
+    if (!url) continue;
+    if (BLOCKED_HOST_RE.test(url.hostname)) continue;
+
+    const link = url.href;
+    if (seen.has(link)) continue;
+    seen.add(link);
+
+    const { summary, category, year } = extractContext(anchor);
+    const pubDate = year != null ? new Date(Date.UTC(year, 0, 1)) : new Date();
+
+    items.push({
+      guid: hashGuid(link),
+      title,
+      link,
+      pubDate,
+      summary: summary || undefined,
+    });
+  }
+
+  if (items.length === 0) {
+    throw new Error("[opendrivelab-publications] 未解析到论文条目，页面结构可能已变化");
+  }
+  return items;
+}
+
+
+export default {
+  id: "opendrivelab-publications",
+  listUrlPattern: /^https?:\/\/(www\.)?opendrivelab\.com\/publications\/?(\?.*)?$/i,
+  fetchItems,
+};