rssany 0.1.6 → 0.3.0

package/app/plugins/builtin/rss.rssany.js

@@ -1,6 +1,12 @@
 // 内置 RSS/Atom/JSON Feed：通过浏览器（Puppeteer）拉取 Feed URL，再用 rss-parser 解析；
 // 与站点插件一致走 Chrome，便于应对需浏览器环境或代理的场景；XML 使用 HTTP 响应原文（useHttpResponseBody）。
-
+export const id = "__rss__";
+export const name = "RSS Feed";
+export const pattern = /^https:\/\//;
+export const match = looksLikeFeed;
+export const priority = 20;
+export const refreshInterval = "1h";
+
 const UA = "RssAny/1.0 (+https://github.com/joohw/rssany)";
 
 const IMAGE_TYPE_RE = /^image\//i;
@@ -76,7 +82,7 @@ function extractItemImageUrl(item) {
     firstImgSrcFromHtml(item.summary) ||
     firstImgSrcFromHtml(item["content:encoded"]) ||
     firstImgSrcFromHtml(item.contentSnippet);
-  if (fromHtml && /^https?:\/\//i.test(fromHtml)) {
+  if (fromHtml && /^https:\/\//i.test(fromHtml)) {
     return fromHtml;
   }
 
@@ -102,13 +108,7 @@ async function fetchFeedXml(url, ctx) {
   return html;
 }
 
-export default {
-  id: "__rss__",
-  pattern: /^https?:\/\//,
-  match: looksLikeFeed,
-  priority: 20,
-  refreshInterval: "1h",
-  async fetchItems(sourceId, ctx) {
+export async function fetchItems(sourceId, ctx) {
     const { deps } = ctx;
     const xml = await fetchFeedXml(sourceId, ctx);
     const parser = new deps.RssParser({
@@ -155,9 +155,8 @@ export default {
       if (!imageUrl) return base;
       return { ...base, imageUrl, cover_img: imageUrl };
     });
-  },
-};
-
+  }
+
 function looksLikeFeed(url) {
   const lower = url.toLowerCase();
   return (

package/app/plugins/builtin/selectdataset.rssany.js

@@ -1,3 +1,7 @@
+export const id = "selectdataset";
+export const name = "Selectdataset";
+export const listUrlPattern = /^https:\/\/(www\.)?selectdataset\.com\/(?:$|\?.*|search(?:\?.*)?|subject(?:\?.*)?)$/i;
+
 let _deps;
 
 // SelectDataset 插件：解析首页/搜索页 Nuxt payload，输出数据集条目（不含 enrich）
@@ -66,7 +70,7 @@ function parseFromAnchorDom(html, finalUrl) {
     let link = null;
     try {
       const url = new URL(href, baseUrl);
-      if (!/^https?:$/i.test(url.protocol)) continue;
+      if (!/^https:$/i.test(url.protocol)) continue;
       if (!/\/dataset\/[A-Za-z0-9]{16,}/.test(url.pathname)) continue;
       link = url.href;
     } catch {
@@ -184,7 +188,7 @@ function parseFromNuxtPayload(html) {
 }
 
 
-async function fetchItems(sourceId, ctx) {
+export async function fetchItems(sourceId, ctx) {
   _deps = ctx.deps;
   const { html, finalUrl } = await ctx.fetchHtml(sourceId, { waitMs: 3500 });
   const fromAnchorDom = parseFromAnchorDom(html, finalUrl);
@@ -198,9 +202,3 @@ async function fetchItems(sourceId, ctx) {
   throw new Error("[selectdataset] 未解析到数据集条目，页面结构可能已变化");
 }
 
-
-export default {
-  id: "selectdataset",
-  listUrlPattern: /^https?:\/\/(www\.)?selectdataset\.com\/(?:$|\?.*|search(?:\?.*)?|subject(?:\?.*)?)$/i,
-  fetchItems,
-};

package/app/plugins/builtin/sensetime-tech-achievements.rssany.js

@@ -1,7 +1,11 @@
+export const id = "sensetime-tech-achievements";
+export const name = "Sensetime Tech Achievements";
+export const listUrlPattern = /^https:\/\/(www\.)?sensetime\.com\/cn\/technology-achievements(\?.*)?$/i;
+
 let _deps;
 
 
-const SITE_ID = "sensetime-tech-achievements";
+const SITE_ID = id;
 const DATE_RE = /\b(20\d{2})-(\d{1,2})-(\d{1,2})\b/;
 
 function normalizeText(text) {
@@ -18,7 +22,7 @@ function toAbsoluteUrl(rawHref, baseUrl) {
   if (!href || href.startsWith("#") || href.startsWith("javascript:")) return null;
   try {
     const url = new URL(href, baseUrl);
-    if (!/^https?:$/i.test(url.protocol)) return null;
+    if (!/^https:$/i.test(url.protocol)) return null;
     return url.href;
   } catch {
     return null;
@@ -135,7 +139,7 @@ async function fetchItemsFromApi(finalUrl) {
   return items;
 }
 
-async function fetchItems(sourceId, ctx) {
+export async function fetchItems(sourceId, ctx) {
   _deps = ctx.deps;
   const { html, finalUrl } = await ctx.fetchHtml(sourceId, { waitMs: 3500 });
   const items = parseItemsFromHtml(html, finalUrl);
@@ -147,8 +151,3 @@ async function fetchItems(sourceId, ctx) {
   throw new Error(`[${SITE_ID}] 未解析到学术成果条目，页面结构或接口可能已变化`);
 }
 
-export default {
-  id: SITE_ID,
-  listUrlPattern: /^https?:\/\/(www\.)?sensetime\.com\/cn\/technology-achievements(\?.*)?$/i,
-  fetchItems,
-};

package/app/plugins/builtin/supervisely-blog.rssany.js

@@ -1,3 +1,7 @@
+export const id = "supervisely-blog";
+export const name = "Supervisely Blog";
+export const listUrlPattern = /^https:\/\/(www\.)?supervisely\.com\/blog\/?(?:\?.*)?$/i;
+
 let _deps;
 
 // Supervisely Blog 插件：抓取列表页并解析为 FeedItem（不做正文 enrich）
@@ -35,7 +39,7 @@ function toAbsoluteUrl(href, baseUrl) {
   if (!href) return null;
   try {
     const url = new URL(href, baseUrl);
-    if (!/^https?:$/i.test(url.protocol)) return null;
+    if (!/^https:$/i.test(url.protocol)) return null;
     return url.href;
   } catch {
     return null;
@@ -136,7 +140,7 @@ function parseFromHeadingFallback(root, baseUrl) {
 }
 
 
-async function fetchItems(sourceId, ctx) {
+export async function fetchItems(sourceId, ctx) {
   _deps = ctx.deps;
   const { html, finalUrl } = await ctx.fetchHtml(sourceId, { waitMs: 3500 });
   const root = _deps.parseHtml(html);
@@ -151,9 +155,3 @@ async function fetchItems(sourceId, ctx) {
   return items;
 }
 
-
-export default {
-  id: "supervisely-blog",
-  listUrlPattern: /^https?:\/\/(www\.)?supervisely\.com\/blog\/?(?:\?.*)?$/i,
-  fetchItems,
-};

package/app/plugins/builtin/theinformation-briefings.rssany.js

@@ -1,136 +1,144 @@
-let _deps;
-
-// The Information — Briefings 列表页：https://www.theinformation.com/briefings
-// 结构：.content-feed .article.briefing.feed-item，标题 h3.title a，摘要 .briefing-dek，时间 .authors
-
-const ORIGIN = "https://www.theinformation.com";
-const LIST_URL_RE =
-  /^https?:\/\/(www\.)?theinformation\.com\/briefings\/?(\?.*)?$/i;
-
-
-function normalizeText(text) {
-  return (text ?? "").replace(/\s+/g, " ").trim();
-}
-
-
-function hashGuid(input) {
-  return _deps.createHash("sha256").update(input).digest("hex");
-}
-
-
-function toAbsoluteHttpUrl(rawHref, baseUrl) {
-  if (!rawHref) return null;
-  const href = rawHref.trim();
-  if (!href || href.startsWith("#") || href.startsWith("javascript:")) return null;
-  try {
-    const url = new URL(href, baseUrl);
-    if (!/^https?:$/i.test(url.protocol)) return null;
-    return url.href;
-  } catch {
-    return null;
-  }
-}
-
-
-function pad2(n) {
-  return String(n).padStart(2, "0");
-}
-
-
-/** .authors 文本：Apr 14, 2026 · 5:41am PDT（可含 · 1 comment）；Node 不能可靠解析 PDT 缩写，手动换算 offset */
-function parseBriefingAuthorsDate(raw) {
-  let t = normalizeText(raw);
-  t = t.replace(/\s*·\s*\d+\s+comments?\s*$/i, "").trim();
-
-  const m = t.match(
-    /^(.+?\d{4})\s*·\s*(\d{1,2}:\d{2}\s*(?:am|pm))\s*(PDT|PST|PT)\s*$/i
-  );
-  if (m) {
-    const datePart = m[1].trim();
-    const timePart = m[2].trim();
-    const tz = m[3].toUpperCase();
-    const offset = tz === "PDT" ? "-07:00" : "-08:00";
-
-    const hm = timePart.match(/(\d{1,2}):(\d{2})\s*(am|pm)/i);
-    const d0 = new Date(datePart);
-    if (hm && !Number.isNaN(d0.getTime())) {
-      let h = Number(hm[1]);
-      const min = Number(hm[2]);
-      const ap = hm[3].toLowerCase();
-      if (ap === "pm" && h < 12) h += 12;
-      if (ap === "am" && h === 12) h = 0;
-      const y = d0.getFullYear();
-      const mo = d0.getMonth() + 1;
-      const da = d0.getDate();
-      const iso = `${y}-${pad2(mo)}-${pad2(da)}T${pad2(h)}:${pad2(min)}:00${offset}`;
-      const out = new Date(iso);
-      if (!Number.isNaN(out.getTime())) return out;
-    }
-  }
-
-  const first = t.split("·")[0].trim();
-  const fallback = new Date(first);
-  return Number.isNaN(fallback.getTime()) ? new Date() : fallback;
-}
-
-
-function parseBriefingItems(html, pageUrl) {
-  const root = _deps.parseHtml(html);
-  const items = [];
-  const seen = new Set();
-
-  for (const node of root.querySelectorAll(".content-feed .article.briefing.feed-item")) {
-    const linkEl = node.querySelector("h3.title a[href]");
-    if (!linkEl) continue;
-
-    const title = normalizeText(linkEl.textContent);
-    const link = toAbsoluteHttpUrl(linkEl.getAttribute("href"), pageUrl);
-    if (!title || !link || seen.has(link)) continue;
-    seen.add(link);
-
-    const authorsText = normalizeText(node.querySelector(".authors")?.textContent ?? "");
-    const pubDate = parseBriefingAuthorsDate(authorsText);
-    const summary = normalizeText(node.querySelector(".briefing-dek")?.textContent ?? "") || undefined;
-
-    items.push({
-      guid: hashGuid(link),
-      title,
-      link,
-      pubDate,
-      summary,
-    });
-  }
-
-  return items;
-}
-
-
-async function fetchItems(sourceId, ctx) {
-  _deps = ctx.deps;
-  const { html, finalUrl, status } = await ctx.fetchHtml(sourceId, {
-    waitMs: 5000,
-    waitForSelector: ".content-feed .article.briefing",
-    waitForSelectorTimeoutMs: 25_000,
-  });
-
-  const pageUrl = finalUrl || sourceId || ORIGIN;
-  const items = parseBriefingItems(html, pageUrl);
-
-  if (items.length === 0) {
-    const hint = status && status >= 400 ? ` HTTP ${status}` : "";
-    throw new Error(
-      `[theinformation-briefings] 未解析到条目，页面结构可能已变化或需登录后抓取。${hint}`
-    );
-  }
-
-  items.sort((a, b) => b.pubDate.getTime() - a.pubDate.getTime());
-  return items;
-}
-
-
-export default {
-  id: "theinformation-briefings",
-  listUrlPattern: LIST_URL_RE,
-  refreshInterval: "1h",
-  fetchItems,
-};
+export const id = "theinformation";
+export const name = "Theinformation";
+export const listUrlPattern = /^https:\/\/(www\.)?theinformation\.com\/(briefings|features\/[^/]+)\/?(\?.*)?$/i;
+export const refreshInterval = "1h";
+
+let _deps;
+
+// The Information — AI Agenda 和 Briefings 列表页
+// 当前结构：.article.feed-item，标题 h3.title a，分类 .category-content a，作者 .authors，摘要 .recent-excerpt .long-excerpt
+
+const ORIGIN = "https://www.theinformation.com";
+
+function normalizeText(text) {
+  return (text ?? "").replace(/\s+/g, " ").trim();
+}
+
+
+function hashGuid(input) {
+  return _deps.createHash("sha256").update(input).digest("hex");
+}
+
+
+function toAbsoluteHttpUrl(rawHref, baseUrl) {
+  if (!rawHref) return null;
+  const href = rawHref.trim();
+  if (!href || href.startsWith("#") || href.startsWith("javascript:")) return null;
+  try {
+    const url = new URL(href, baseUrl);
+    if (!/^https:$/i.test(url.protocol)) return null;
+    return url.href;
+  } catch {
+    return null;
+  }
+}
+
+
+function pad2(n) {
+  return String(n).padStart(2, "0");
+}
+
+
+/** .authors 文本：By Author · Apr 14, 2026 · 7:52am PDT */
+function parseAuthorsDate(raw) {
+  let t = normalizeText(raw);
+  t = t.replace(/\s*·\s*\d+\s+comments?\s*$/i, "").trim();
+
+  const m = t.match(
+    /^By\s+(.+?)\s*·\s*(.+?\d{4})\s*·\s*(\d{1,2}:\d{2}\s*(?:am|pm))\s*(PDT|PST|PT)\s*$/i
+  );
+  if (m) {
+    const author = m[1].trim();
+    const datePart = m[2].trim();
+    const timePart = m[3].trim();
+    const tz = m[4].toUpperCase();
+    const offset = tz === "PDT" ? "-07:00" : "-08:00";
+
+    const hm = timePart.match(/(\d{1,2}):(\d{2})\s*(am|pm)/i);
+    const d0 = new Date(datePart);
+    if (hm && !Number.isNaN(d0.getTime())) {
+      let h = Number(hm[1]);
+      const min = Number(hm[2]);
+      const ap = hm[3].toLowerCase();
+      if (ap === "pm" && h < 12) h += 12;
+      if (ap === "am" && h === 12) h = 0;
+      const y = d0.getFullYear();
+      const mo = d0.getMonth() + 1;
+      const da = d0.getDate();
+      const iso = `${y}-${pad2(mo)}-${pad2(da)}T${pad2(h)}:${pad2(min)}:00${offset}`;
+      const pubDate = new Date(iso);
+      if (!Number.isNaN(pubDate.getTime())) return { author, pubDate };
+    }
+  }
+
+  const authorMatch = t.match(/^By\s+(.+?)\s*·/i);
+  const author = authorMatch ? authorMatch[1].trim() : undefined;
+  const dateStr = t.replace(/^By\s+.*?\s*·\s*/, "").trim();
+  const pubDate = new Date(dateStr);
+  return { author, pubDate: Number.isNaN(pubDate.getTime()) ? new Date() : pubDate };
+}
+
+
+function parseFeedItems(html, pageUrl) {
+  const root = _deps.parseHtml(html);
+  const items = [];
+  const seen = new Set();
+
+  for (const node of root.querySelectorAll(".article.feed-item")) {
+    const linkEl = node.querySelector("h3.title a[href]");
+    if (!linkEl) continue;
+
+    const title = normalizeText(linkEl.textContent);
+    const link = toAbsoluteHttpUrl(linkEl.getAttribute("href"), pageUrl);
+    if (!title || !link || seen.has(link)) continue;
+    seen.add(link);
+
+    const authorsText = normalizeText(node.querySelector(".authors")?.textContent ?? "");
+    const { author, pubDate } = parseAuthorsDate(authorsText);
+
+    const summary = normalizeText(
+      node.querySelector(".recent-excerpt .long-excerpt")?.textContent ??
+      node.querySelector(".recent-excerpt")?.textContent ??
+      node.querySelector(".short-excerpt")?.textContent ??
+      ""
+    ) || undefined;
+
+    const categoryEl = node.querySelector(".category-content a");
+    const category = categoryEl ? normalizeText(categoryEl.textContent) : undefined;
+
+    items.push({
+      guid: hashGuid(link),
+      title,
+      link,
+      pubDate,
+      author,
+      summary,
+      categories: category ? [category] : undefined,
+    });
+  }
+
+  return items;
+}
+
+
+export async function fetchItems(sourceId, ctx) {
+  _deps = ctx.deps;
+  const { html, finalUrl, status } = await ctx.fetchHtml(sourceId, {
+    waitMs: 5000,
+    waitForSelector: ".article.feed-item",
+    waitForSelectorTimeoutMs: 25_000,
+  });
+
+  const pageUrl = finalUrl || sourceId || ORIGIN;
+  const items = parseFeedItems(html, pageUrl);
+
+  if (items.length === 0) {
+    const hint = status && status >= 400 ? ` HTTP ${status}` : "";
+    throw new Error(
+      `[theinformation] 未解析到条目，页面结构可能已变化或需登录后抓取。${hint}`
+    );
+  }
+
+  items.sort((a, b) => b.pubDate.getTime() - a.pubDate.getTime());
+  return items;
+}

package/app/plugins/builtin/uci-ml-repository.rssany.js

@@ -1,3 +1,7 @@
+export const id = "uci-ml-repository";
+export const name = "UCI Ml Repository";
+export const listUrlPattern = /^https:\/\/archive\.ics\.uci\.edu(?:\/(?:datasets\/?)?)?(?:\?.*)?$/i;
+
 let _deps;
 
 
@@ -16,7 +20,7 @@ function resolveDatasetLink(rawHref, baseUrl) {
   if (!href || href.startsWith("#") || href.startsWith("javascript:")) return null;
   try {
     const url = new URL(href, baseUrl);
-    if (!/^https?:$/i.test(url.protocol)) return null;
+    if (!/^https:$/i.test(url.protocol)) return null;
     if (url.hostname !== "archive.ics.uci.edu") return null;
     if (!/^\/dataset\/\d+\/[^/?#]+$/i.test(url.pathname)) return null;
     url.search = "";
@@ -89,7 +93,7 @@ function parseFromGenericAnchors(root, baseUrl) {
   return items;
 }
 
-async function fetchItems(sourceId, ctx) {
+export async function fetchItems(sourceId, ctx) {
   _deps = ctx.deps;
   const { html, finalUrl } = await ctx.fetchHtml(sourceId, { waitMs: 4000 });
   const baseUrl = finalUrl || sourceId || UCI_ORIGIN;
@@ -104,8 +108,3 @@ async function fetchItems(sourceId, ctx) {
   throw new Error("[uci-ml-repository] 未解析到数据集条目，页面结构可能已变化");
 }
 
-export default {
-  id: "uci-ml-repository",
-  listUrlPattern: /^https?:\/\/archive\.ics\.uci\.edu(?:\/(?:datasets\/?)?)?(?:\?.*)?$/i,
-  fetchItems,
-};

package/app/plugins/builtin/venturebeat.rssany.js

@@ -1,3 +1,8 @@
+export const id = "venturebeat";
+export const name = "Venturebeat";
+export const listUrlPattern = /^https:\/\/(www\.)?venturebeat\.com\/?(\?.*)?$/i;
+export const refreshInterval = "1h";
+
 let _deps;
 
 // VentureBeat 插件：通过官方 RSS Feed 拉取列表，规避首页安全检查页
@@ -35,7 +40,7 @@ function toFeedUrl(sourceId) {
 
 function mapFeedItem(item) {
   const link = normalizeText(item.link ?? "");
-  if (!/^https?:\/\//i.test(link)) return null;
+  if (!/^https:\/\//i.test(link)) return null;
 
   const title = normalizeText(item.title ?? "");
   const pubDate = toValidDate(item.isoDate ?? item.pubDate);
@@ -53,7 +58,7 @@ function mapFeedItem(item) {
 }
 
 
-async function fetchItems(sourceId, _ctx) {
+export async function fetchItems(sourceId, _ctx) {
   _deps = _ctx.deps;
   const parser = new _deps.RssParser({
     timeout: 15_000,
@@ -88,10 +93,3 @@ async function fetchItems(sourceId, _ctx) {
   return items;
 }
 
-
-export default {
-  id: "venturebeat",
-  listUrlPattern: /^https?:\/\/(www\.)?venturebeat\.com\/?(\?.*)?$/i,
-  refreshInterval: "1h",
-  fetchItems,
-};

package/app/plugins/builtin/worldlabs.rssany.js

@@ -1,3 +1,7 @@
+export const id = "worldlabs";
+export const name = "Worldlabs";
+export const listUrlPattern = /^https:\/\/(www\.)?worldlabs\.ai\/blog(\?.*)?$/i;
+
 let _deps;
 
 // World Labs 博客插件：抓取 Research & Insights 列表页，输出 FeedItem（不含 enrich）
@@ -39,7 +43,7 @@ function toAbsoluteHttpUrl(rawHref, baseUrl) {
   if (!href || href.startsWith("#") || href.startsWith("javascript:")) return null;
   try {
     const url = new URL(href, baseUrl);
-    if (!/^https?:$/i.test(url.protocol)) return null;
+    if (!/^https:$/i.test(url.protocol)) return null;
     return url.href;
   } catch {
     return null;
@@ -99,7 +103,7 @@ function parseCard(anchor, finalUrl) {
 }
 
 
-async function fetchItems(sourceId, ctx) {
+export async function fetchItems(sourceId, ctx) {
   _deps = ctx.deps;
   const { html, finalUrl } = await ctx.fetchHtml(sourceId, { waitMs: 3500 });
   const root = _deps.parseHtml(html);
@@ -121,9 +125,3 @@ async function fetchItems(sourceId, ctx) {
   return items;
 }
 
-
-export default {
-  id: "worldlabs",
-  listUrlPattern: /^https?:\/\/(www\.)?worldlabs\.ai\/blog(\?.*)?$/i,
-  fetchItems,
-};

package/app/plugins/builtin/x.rssany.js

@@ -1,3 +1,7 @@
+export const id = "x";
+export const name = "X";
+export const listUrlPattern = /^https:\/\/(www\.)?x\.com\/[^/?#]+\/?(?:[?#].*)?$/i;
+
 let _deps;
 
 // X (Twitter) 站点插件：用户主页列表抓取与解析
@@ -161,13 +165,13 @@ function extractMediaUrl(article) {
   const video = article.querySelector("video[poster]");
   if (video) {
     const poster = video.getAttribute("poster");
-    if (poster && /^https?:\/\//i.test(poster)) return poster;
+    if (poster && /^https:\/\//i.test(poster)) return poster;
   }
   for (const img of article.querySelectorAll(
     '[data-testid="card.wrapper"] img[src*="twimg.com/card_img"], [data-testid="card.wrapper"] img[src*="pbs.twimg.com/card_img"]',
   )) {
     const src = img.getAttribute("src");
-    if (src && /^https?:\/\//i.test(src) && !/profile_images/i.test(src)) {
+    if (src && /^https:\/\//i.test(src) && !/profile_images/i.test(src)) {
       return normalizeCardImgUrl(src);
     }
   }
@@ -288,7 +292,7 @@ function entriesToFeedItems(entries) {
 }
 
 
-async function fetchItems(sourceId, ctx) {
+export async function fetchItems(sourceId, ctx) {
   _deps = ctx.deps;
   const { html, finalUrl } = await ctx.fetchHtml(sourceId, { waitMs: 6000 });
   const root = _deps.parseHtml(html);
@@ -320,9 +324,3 @@ async function fetchItems(sourceId, ctx) {
   throw new Error(`[X] ${message}`);
 }
 
-
-export default {
-  id: "x",
-  listUrlPattern: "https://x.com/{username}",
-  fetchItems,
-};