rssany 0.1.4 → 0.1.6

package/app/plugins/builtin/sensetime-tech-achievements.rssany.js

@@ -0,0 +1,154 @@
+let _deps;
+
+
+const SITE_ID = "sensetime-tech-achievements";
+const DATE_RE = /\b(20\d{2})-(\d{1,2})-(\d{1,2})\b/;
+
+function normalizeText(text) {
+  return (text ?? "").replace(/\s+/g, " ").trim();
+}
+
+function hashGuid(input) {
+  return _deps.createHash("sha256").update(input).digest("hex");
+}
+
+function toAbsoluteUrl(rawHref, baseUrl) {
+  if (!rawHref) return null;
+  const href = rawHref.trim();
+  if (!href || href.startsWith("#") || href.startsWith("javascript:")) return null;
+  try {
+    const url = new URL(href, baseUrl);
+    if (!/^https?:$/i.test(url.protocol)) return null;
+    return url.href;
+  } catch {
+    return null;
+  }
+}
+
+function parseDate(dateText) {
+  const text = normalizeText(dateText);
+  const m = text.match(DATE_RE);
+  if (!m) return undefined;
+  const [, y, mm, dd] = m;
+  return new Date(`${y}-${mm.padStart(2, "0")}-${dd.padStart(2, "0")}T00:00:00.000Z`);
+}
+
+function extractDateText(anchor) {
+  let current = anchor;
+  for (let i = 0; i < 8 && current; i += 1) {
+    const text = normalizeText(current.textContent);
+    const m = text.match(DATE_RE);
+    if (m) return m[0];
+    current = current.parentNode ?? null;
+  }
+  return "";
+}
+
+function extractTags(anchor, title, dateText) {
+  let current = anchor;
+  for (let i = 0; i < 6 && current; i += 1) {
+    const spans = current.querySelectorAll?.("span") ?? [];
+    const tags = spans
+      .map((s) => normalizeText(s.textContent))
+      .filter(Boolean)
+      .filter((x) => x !== title && x !== dateText)
+      .filter((x) => !DATE_RE.test(x));
+    if (tags.length > 0) return Array.from(new Set(tags));
+    current = current.parentNode ?? null;
+  }
+  return [];
+}
+
+function parseItemsFromHtml(html, finalUrl) {
+  const root = _deps.parseHtml(html);
+  const anchors = root.querySelectorAll('a[href*="/technology-new-detail/"]');
+  const seen = new Set();
+  const items = [];
+
+  for (const anchor of anchors) {
+    const link = toAbsoluteUrl(anchor.getAttribute("href"), finalUrl);
+    if (!link || seen.has(link)) continue;
+
+    const title = normalizeText(anchor.textContent);
+    if (!title) continue;
+
+    seen.add(link);
+    const dateText = extractDateText(anchor);
+    const tags = extractTags(anchor, title, dateText);
+    const summary = [dateText, tags.join(" / ")].filter(Boolean).join(" | ");
+    const pubDate = parseDate(dateText) ?? new Date();
+
+    items.push({
+      guid: hashGuid(link),
+      title,
+      link,
+      pubDate,
+      summary: summary || undefined,
+    });
+  }
+
+  return items;
+}
+
+async function fetchItemsFromApi(finalUrl) {
+  const origin = new URL(finalUrl).origin;
+  const apiUrl = new URL("/rest/v1/contents/1/getlistbyparam/48/1/20/0/0?scene=1", origin);
+
+  const res = await fetch(apiUrl, {
+    headers: {
+      Accept: "application/json,text/plain,*/*",
+      "User-Agent":
+        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
+    },
+  });
+  if (!res.ok) return [];
+
+  const data = await res.json();
+  const list = Array.isArray(data?.data?.lists) ? data.data.lists : [];
+  const items = [];
+  const seen = new Set();
+
+  for (const row of list) {
+    const contentId = String(row?.contentId ?? "").trim();
+    const title = normalizeText(row?.title);
+    if (!contentId || !title) continue;
+    const link = new URL(`/cn/technology-new-detail/${contentId}?categoryId=48`, origin).href;
+    if (seen.has(link)) continue;
+    seen.add(link);
+
+    const dateText = normalizeText(row?.createTime);
+    const tags = Array.isArray(row?.tagnames)
+      ? row.tagnames.map((x) => normalizeText(x)).filter(Boolean)
+      : [];
+    const summary = [dateText, tags.join(" / ")].filter(Boolean).join(" | ");
+    const pubDate = parseDate(dateText) ?? new Date();
+
+    items.push({
+      guid: hashGuid(link),
+      title,
+      link,
+      pubDate,
+      summary: summary || undefined,
+    });
+  }
+
+  return items;
+}
+
+async function fetchItems(sourceId, ctx) {
+  _deps = ctx.deps;
+  const { html, finalUrl } = await ctx.fetchHtml(sourceId, { waitMs: 3500 });
+  const items = parseItemsFromHtml(html, finalUrl);
+  if (items.length > 0) return items;
+
+  const fallbackItems = await fetchItemsFromApi(finalUrl);
+  if (fallbackItems.length > 0) return fallbackItems;
+
+  throw new Error(`[${SITE_ID}] 未解析到学术成果条目，页面结构或接口可能已变化`);
+}
+
+export default {
+  id: SITE_ID,
+  listUrlPattern: /^https?:\/\/(www\.)?sensetime\.com\/cn\/technology-achievements(\?.*)?$/i,
+  fetchItems,
+};

package/app/plugins/builtin/supervisely-blog.rssany.js

@@ -0,0 +1,159 @@
+let _deps;
+
+// Supervisely Blog 插件：抓取列表页并解析为 FeedItem（不做正文 enrich）
+
+
+
+const SUPERVISELY_ORIGIN = "https://supervisely.com";
+const MONTH_INDEX = {
+  jan: 0,
+  feb: 1,
+  mar: 2,
+  apr: 3,
+  may: 4,
+  jun: 5,
+  jul: 6,
+  aug: 7,
+  sep: 8,
+  oct: 9,
+  nov: 10,
+  dec: 11,
+};
+
+
+function normalizeText(text) {
+  return (text ?? "").replace(/\s+/g, " ").trim();
+}
+
+
+function hashGuid(input) {
+  return _deps.createHash("sha256").update(input).digest("hex");
+}
+
+
+function toAbsoluteUrl(href, baseUrl) {
+  if (!href) return null;
+  try {
+    const url = new URL(href, baseUrl);
+    if (!/^https?:$/i.test(url.protocol)) return null;
+    return url.href;
+  } catch {
+    return null;
+  }
+}
+
+
+function parsePubDate(rawText) {
+  const text = normalizeText(rawText);
+  const m = text.match(/^([A-Za-z]{3,9})\s+(\d{1,2}),\s*(\d{4})$/);
+  if (!m) return undefined;
+  const month = MONTH_INDEX[m[1].slice(0, 3).toLowerCase()];
+  if (month == null) return undefined;
+  const day = Number(m[2]);
+  const year = Number(m[3]);
+  if (!Number.isInteger(day) || !Number.isInteger(year)) return undefined;
+  return new Date(Date.UTC(year, month, day, 0, 0, 0));
+}
+
+
+function looksLikeBlogLink(link) {
+  try {
+    const u = new URL(link);
+    return /^\/blog\/[^/]+\/?$/i.test(u.pathname);
+  } catch {
+    return false;
+  }
+}
+
+
+function findAncestor(node, maxDepth) {
+  let current = node?.parentNode ?? null;
+  for (let i = 0; i < maxDepth && current; i += 1) {
+    if (current.querySelector?.("time")) return current;
+    current = current.parentNode ?? null;
+  }
+  return node?.parentNode ?? null;
+}
+
+
+function buildFeedItem({ title, link, summary, author, pubDate }) {
+  return {
+    guid: hashGuid(link),
+    title,
+    link,
+    pubDate: pubDate ?? new Date(),
+    author: author || undefined,
+    summary: summary || undefined,
+  };
+}
+
+
+function parseFromCards(root, baseUrl) {
+  const seen = new Set();
+  const items = [];
+  const cards = root.querySelectorAll("div.blog-card");
+
+  for (const card of cards) {
+    const titleAnchor = card.querySelector("h4 a[href]");
+    const title = normalizeText(titleAnchor?.textContent);
+    const link = toAbsoluteUrl(titleAnchor?.getAttribute("href"), baseUrl);
+    if (!title || !link || !looksLikeBlogLink(link) || seen.has(link)) continue;
+
+    const summary = normalizeText(card.querySelector("p")?.textContent);
+    const author = normalizeText(card.querySelector('b[rel="author"], address b')?.textContent);
+    const pubDateText = normalizeText(card.querySelector("time")?.textContent);
+    const pubDate = parsePubDate(pubDateText);
+
+    seen.add(link);
+    items.push(buildFeedItem({ title, link, summary, author, pubDate }));
+  }
+
+  return items;
+}
+
+
+function parseFromHeadingFallback(root, baseUrl) {
+  const seen = new Set();
+  const items = [];
+  const anchors = root.querySelectorAll('h4 a[href*="/blog/"]');
+
+  for (const anchor of anchors) {
+    const title = normalizeText(anchor.textContent);
+    const link = toAbsoluteUrl(anchor.getAttribute("href"), baseUrl);
+    if (!title || !link || !looksLikeBlogLink(link) || seen.has(link)) continue;
+
+    const container = findAncestor(anchor, 7);
+    const summary = normalizeText(container?.querySelector("p")?.textContent);
+    const author = normalizeText(container?.querySelector('b[rel="author"], address b')?.textContent);
+    const pubDateText = normalizeText(container?.querySelector("time")?.textContent);
+    const pubDate = parsePubDate(pubDateText);
+
+    seen.add(link);
+    items.push(buildFeedItem({ title, link, summary, author, pubDate }));
+  }
+
+  return items;
+}
+
+
+async function fetchItems(sourceId, ctx) {
+  _deps = ctx.deps;
+  const { html, finalUrl } = await ctx.fetchHtml(sourceId, { waitMs: 3500 });
+  const root = _deps.parseHtml(html);
+  const baseUrl = finalUrl || SUPERVISELY_ORIGIN;
+
+  const fromCards = parseFromCards(root, baseUrl);
+  const items = fromCards.length > 0 ? fromCards : parseFromHeadingFallback(root, baseUrl);
+
+  if (items.length === 0) {
+    throw new Error("[supervisely-blog] 未解析到文章条目，页面结构可能已变化");
+  }
+  return items;
+}
+
+
+export default {
+  id: "supervisely-blog",
+  listUrlPattern: /^https?:\/\/(www\.)?supervisely\.com\/blog\/?(?:\?.*)?$/i,
+  fetchItems,
+};

package/app/plugins/builtin/theinformation-briefings.rssany.js

@@ -0,0 +1,136 @@
+let _deps;
+
+// The Information — Briefings 列表页：https://www.theinformation.com/briefings
+// 结构：.content-feed .article.briefing.feed-item，标题 h3.title a，摘要 .briefing-dek，时间 .authors
+
+const ORIGIN = "https://www.theinformation.com";
+const LIST_URL_RE =
+  /^https?:\/\/(www\.)?theinformation\.com\/briefings\/?(\?.*)?$/i;
+
+
+function normalizeText(text) {
+  return (text ?? "").replace(/\s+/g, " ").trim();
+}
+
+
+function hashGuid(input) {
+  return _deps.createHash("sha256").update(input).digest("hex");
+}
+
+
+function toAbsoluteHttpUrl(rawHref, baseUrl) {
+  if (!rawHref) return null;
+  const href = rawHref.trim();
+  if (!href || href.startsWith("#") || href.startsWith("javascript:")) return null;
+  try {
+    const url = new URL(href, baseUrl);
+    if (!/^https?:$/i.test(url.protocol)) return null;
+    return url.href;
+  } catch {
+    return null;
+  }
+}
+
+
+function pad2(n) {
+  return String(n).padStart(2, "0");
+}
+
+
+/** .authors 文本：Apr 14, 2026 · 5:41am PDT（可含 · 1 comment）；Node 不能可靠解析 PDT 缩写，手动换算 offset */
+function parseBriefingAuthorsDate(raw) {
+  let t = normalizeText(raw);
+  t = t.replace(/\s*·\s*\d+\s+comments?\s*$/i, "").trim();
+
+  const m = t.match(
+    /^(.+?\d{4})\s*·\s*(\d{1,2}:\d{2}\s*(?:am|pm))\s*(PDT|PST|PT)\s*$/i
+  );
+  if (m) {
+    const datePart = m[1].trim();
+    const timePart = m[2].trim();
+    const tz = m[3].toUpperCase();
+    const offset = tz === "PDT" ? "-07:00" : "-08:00";
+
+    const hm = timePart.match(/(\d{1,2}):(\d{2})\s*(am|pm)/i);
+    const d0 = new Date(datePart);
+    if (hm && !Number.isNaN(d0.getTime())) {
+      let h = Number(hm[1]);
+      const min = Number(hm[2]);
+      const ap = hm[3].toLowerCase();
+      if (ap === "pm" && h < 12) h += 12;
+      if (ap === "am" && h === 12) h = 0;
+      const y = d0.getFullYear();
+      const mo = d0.getMonth() + 1;
+      const da = d0.getDate();
+      const iso = `${y}-${pad2(mo)}-${pad2(da)}T${pad2(h)}:${pad2(min)}:00${offset}`;
+      const out = new Date(iso);
+      if (!Number.isNaN(out.getTime())) return out;
+    }
+  }
+
+  const first = t.split("·")[0].trim();
+  const fallback = new Date(first);
+  return Number.isNaN(fallback.getTime()) ? new Date() : fallback;
+}
+
+
+function parseBriefingItems(html, pageUrl) {
+  const root = _deps.parseHtml(html);
+  const items = [];
+  const seen = new Set();
+
+  for (const node of root.querySelectorAll(".content-feed .article.briefing.feed-item")) {
+    const linkEl = node.querySelector("h3.title a[href]");
+    if (!linkEl) continue;
+
+    const title = normalizeText(linkEl.textContent);
+    const link = toAbsoluteHttpUrl(linkEl.getAttribute("href"), pageUrl);
+    if (!title || !link || seen.has(link)) continue;
+    seen.add(link);
+
+    const authorsText = normalizeText(node.querySelector(".authors")?.textContent ?? "");
+    const pubDate = parseBriefingAuthorsDate(authorsText);
+    const summary = normalizeText(node.querySelector(".briefing-dek")?.textContent ?? "") || undefined;
+
+    items.push({
+      guid: hashGuid(link),
+      title,
+      link,
+      pubDate,
+      summary,
+    });
+  }
+
+  return items;
+}
+
+
+async function fetchItems(sourceId, ctx) {
+  _deps = ctx.deps;
+  const { html, finalUrl, status } = await ctx.fetchHtml(sourceId, {
+    waitMs: 5000,
+    waitForSelector: ".content-feed .article.briefing",
+    waitForSelectorTimeoutMs: 25_000,
+  });
+
+  const pageUrl = finalUrl || sourceId || ORIGIN;
+  const items = parseBriefingItems(html, pageUrl);
+
+  if (items.length === 0) {
+    const hint = status && status >= 400 ? ` HTTP ${status}` : "";
+    throw new Error(
+      `[theinformation-briefings] 未解析到条目，页面结构可能已变化或需登录后抓取。${hint}`
+    );
+  }
+
+  items.sort((a, b) => b.pubDate.getTime() - a.pubDate.getTime());
+  return items;
+}
+
+
+export default {
+  id: "theinformation-briefings",
+  listUrlPattern: LIST_URL_RE,
+  refreshInterval: "1h",
+  fetchItems,
+};

package/app/plugins/builtin/uci-ml-repository.rssany.js

@@ -0,0 +1,111 @@
+let _deps;
+
+
+const UCI_ORIGIN = "https://archive.ics.uci.edu";
+
+function normalizeText(text) {
+  return (text ?? "").replace(/\s+/g, " ").trim();
+}
+
+function hashGuid(input) {
+  return _deps.createHash("sha256").update(input).digest("hex");
+}
+
+function resolveDatasetLink(rawHref, baseUrl) {
+  const href = normalizeText(rawHref);
+  if (!href || href.startsWith("#") || href.startsWith("javascript:")) return null;
+  try {
+    const url = new URL(href, baseUrl);
+    if (!/^https?:$/i.test(url.protocol)) return null;
+    if (url.hostname !== "archive.ics.uci.edu") return null;
+    if (!/^\/dataset\/\d+\/[^/?#]+$/i.test(url.pathname)) return null;
+    url.search = "";
+    url.hash = "";
+    return url.href;
+  } catch {
+    return null;
+  }
+}
+
+function pickSummaryNearNode(node) {
+  let cur = node;
+  for (let i = 0; i < 6 && cur; i += 1) {
+    const p = cur.querySelector?.("p");
+    const summary = normalizeText(p?.textContent);
+    if (summary) return summary;
+    cur = cur.parentNode ?? null;
+  }
+  return "";
+}
+
+function buildItem({ title, link, summary, index }) {
+  return {
+    guid: hashGuid(link),
+    title,
+    link,
+    pubDate: new Date(Date.now() - index * 1000),
+    summary: summary || undefined,
+    sourceId: "uci-ml-repository",
+  };
+}
+
+function parseFromHeadingAnchors(root, baseUrl) {
+  const anchors = root.querySelectorAll('h2 a[href^="/dataset/"]');
+  const items = [];
+  const seen = new Set();
+
+  for (const anchor of anchors) {
+    const link = resolveDatasetLink(anchor.getAttribute("href"), baseUrl);
+    if (!link || seen.has(link)) continue;
+
+    const title = normalizeText(anchor.textContent);
+    if (!title) continue;
+
+    const summary = pickSummaryNearNode(anchor.parentNode ?? anchor);
+    seen.add(link);
+    items.push(buildItem({ title, link, summary, index: items.length }));
+  }
+  return items;
+}
+
+function parseFromGenericAnchors(root, baseUrl) {
+  const anchors = root.querySelectorAll('a[href^="/dataset/"]');
+  const items = [];
+  const seen = new Set();
+
+  for (const anchor of anchors) {
+    const link = resolveDatasetLink(anchor.getAttribute("href"), baseUrl);
+    if (!link || seen.has(link)) continue;
+
+    const titleFromText = normalizeText(anchor.textContent);
+    const titleFromImage = normalizeText(anchor.querySelector("img")?.getAttribute("alt"));
+    const title = titleFromText || titleFromImage;
+    if (!title) continue;
+
+    const summary = pickSummaryNearNode(anchor.parentNode ?? anchor);
+    seen.add(link);
+    items.push(buildItem({ title, link, summary, index: items.length }));
+  }
+  return items;
+}
+
+async function fetchItems(sourceId, ctx) {
+  _deps = ctx.deps;
+  const { html, finalUrl } = await ctx.fetchHtml(sourceId, { waitMs: 4000 });
+  const baseUrl = finalUrl || sourceId || UCI_ORIGIN;
+  const root = _deps.parseHtml(html);
+
+  const byHeading = parseFromHeadingAnchors(root, baseUrl);
+  if (byHeading.length > 0) return byHeading;
+
+  const fallback = parseFromGenericAnchors(root, baseUrl);
+  if (fallback.length > 0) return fallback;
+
+  throw new Error("[uci-ml-repository] 未解析到数据集条目，页面结构可能已变化");
+}
+
+export default {
+  id: "uci-ml-repository",
+  listUrlPattern: /^https?:\/\/archive\.ics\.uci\.edu(?:\/(?:datasets\/?)?)?(?:\?.*)?$/i,
+  fetchItems,
+};

package/app/plugins/builtin/venturebeat.rssany.js

@@ -0,0 +1,97 @@
+let _deps;
+
+// VentureBeat 插件：通过官方 RSS Feed 拉取列表，规避首页安全检查页
+
+
+
+
+function normalizeText(text) {
+  return (text ?? "").replace(/\s+/g, " ").trim();
+}
+
+
+function stripHtml(text) {
+  return normalizeText((text ?? "").replace(/<[^>]*>/g, " "));
+}
+
+
+function toValidDate(raw) {
+  if (!raw) return new Date();
+  const date = new Date(raw);
+  return Number.isNaN(date.getTime()) ? new Date() : date;
+}
+
+
+function toFeedUrl(sourceId) {
+  const url = new URL(sourceId);
+  const path = url.pathname.replace(/\/+$/, "");
+  if (path.endsWith("/feed")) return url.href;
+  url.pathname = path ? `${path}/feed/` : "/feed/";
+  url.search = "";
+  url.hash = "";
+  return url.href;
+}
+
+
+function mapFeedItem(item) {
+  const link = normalizeText(item.link ?? "");
+  if (!/^https?:\/\//i.test(link)) return null;
+
+  const title = normalizeText(item.title ?? "");
+  const pubDate = toValidDate(item.isoDate ?? item.pubDate);
+  const summary = normalizeText(item.contentSnippet ?? "") || stripHtml(item.summary ?? item.content ?? "");
+  const author = normalizeText(item.creator ?? item.author ?? "") || undefined;
+
+  return {
+    guid: _deps.createHash("sha256").update(link).digest("hex"),
+    title: title || "(无标题)",
+    link,
+    pubDate,
+    author,
+    summary: summary || undefined,
+  };
+}
+
+
+async function fetchItems(sourceId, _ctx) {
+  _deps = _ctx.deps;
+  const parser = new _deps.RssParser({
+    timeout: 15_000,
+    headers: {
+      "User-Agent": "RssAny/1.0 (+https://github.com/rssany/rssany)",
+      Accept: "application/rss+xml,application/atom+xml,application/xml,text/xml,*/*",
+    },
+  });
+  const feedUrl = toFeedUrl(sourceId);
+  let feed;
+  try {
+    feed = await parser.parseURL(feedUrl);
+  } catch (err) {
+    const msg = err instanceof Error ? err.message : String(err);
+    throw new Error(`[venturebeat] 抓取 feed 失败: ${feedUrl} (${msg})`);
+  }
+
+  const seen = new Set();
+  const items = [];
+  for (const item of feed.items ?? []) {
+    const mapped = mapFeedItem(item);
+    if (!mapped) continue;
+    if (seen.has(mapped.link)) continue;
+    seen.add(mapped.link);
+    items.push(mapped);
+  }
+
+  if (items.length === 0) {
+    throw new Error(`[venturebeat] 未解析到条目: ${feedUrl}`);
+  }
+
+  return items;
+}
+
+
+export default {
+  id: "venturebeat",
+  listUrlPattern: /^https?:\/\/(www\.)?venturebeat\.com\/?(\?.*)?$/i,
+  refreshInterval: "1h",
+  fetchItems,
+};