rssany 0.1.4 → 0.1.5

package/app/plugins/builtin/sensetime-tech-achievements.rssany.js

@@ -0,0 +1,154 @@
+let _deps;
+
+
+const SITE_ID = "sensetime-tech-achievements";
+const DATE_RE = /\b(20\d{2})-(\d{1,2})-(\d{1,2})\b/;
+
+function normalizeText(text) {
+  return (text ?? "").replace(/\s+/g, " ").trim();
+}
+
+function hashGuid(input) {
+  return _deps.createHash("sha256").update(input).digest("hex");
+}
+
+function toAbsoluteUrl(rawHref, baseUrl) {
+  if (!rawHref) return null;
+  const href = rawHref.trim();
+  if (!href || href.startsWith("#") || href.startsWith("javascript:")) return null;
+  try {
+    const url = new URL(href, baseUrl);
+    if (!/^https?:$/i.test(url.protocol)) return null;
+    return url.href;
+  } catch {
+    return null;
+  }
+}
+
+function parseDate(dateText) {
+  const text = normalizeText(dateText);
+  const m = text.match(DATE_RE);
+  if (!m) return undefined;
+  const [, y, mm, dd] = m;
+  return new Date(`${y}-${mm.padStart(2, "0")}-${dd.padStart(2, "0")}T00:00:00.000Z`);
+}
+
+function extractDateText(anchor) {
+  let current = anchor;
+  for (let i = 0; i < 8 && current; i += 1) {
+    const text = normalizeText(current.textContent);
+    const m = text.match(DATE_RE);
+    if (m) return m[0];
+    current = current.parentNode ?? null;
+  }
+  return "";
+}
+
+function extractTags(anchor, title, dateText) {
+  let current = anchor;
+  for (let i = 0; i < 6 && current; i += 1) {
+    const spans = current.querySelectorAll?.("span") ?? [];
+    const tags = spans
+      .map((s) => normalizeText(s.textContent))
+      .filter(Boolean)
+      .filter((x) => x !== title && x !== dateText)
+      .filter((x) => !DATE_RE.test(x));
+    if (tags.length > 0) return Array.from(new Set(tags));
+    current = current.parentNode ?? null;
+  }
+  return [];
+}
+
+function parseItemsFromHtml(html, finalUrl) {
+  const root = _deps.parseHtml(html);
+  const anchors = root.querySelectorAll('a[href*="/technology-new-detail/"]');
+  const seen = new Set();
+  const items = [];
+
+  for (const anchor of anchors) {
+    const link = toAbsoluteUrl(anchor.getAttribute("href"), finalUrl);
+    if (!link || seen.has(link)) continue;
+
+    const title = normalizeText(anchor.textContent);
+    if (!title) continue;
+
+    seen.add(link);
+    const dateText = extractDateText(anchor);
+    const tags = extractTags(anchor, title, dateText);
+    const summary = [dateText, tags.join(" / ")].filter(Boolean).join(" | ");
+    const pubDate = parseDate(dateText) ?? new Date();
+
+    items.push({
+      guid: hashGuid(link),
+      title,
+      link,
+      pubDate,
+      summary: summary || undefined,
+    });
+  }
+
+  return items;
+}
+
+async function fetchItemsFromApi(finalUrl) {
+  const origin = new URL(finalUrl).origin;
+  const apiUrl = new URL("/rest/v1/contents/1/getlistbyparam/48/1/20/0/0?scene=1", origin);
+
+  const res = await fetch(apiUrl, {
+    headers: {
+      Accept: "application/json,text/plain,*/*",
+      "User-Agent":
+        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
+    },
+  });
+  if (!res.ok) return [];
+
+  const data = await res.json();
+  const list = Array.isArray(data?.data?.lists) ? data.data.lists : [];
+  const items = [];
+  const seen = new Set();
+
+  for (const row of list) {
+    const contentId = String(row?.contentId ?? "").trim();
+    const title = normalizeText(row?.title);
+    if (!contentId || !title) continue;
+    const link = new URL(`/cn/technology-new-detail/${contentId}?categoryId=48`, origin).href;
+    if (seen.has(link)) continue;
+    seen.add(link);
+
+    const dateText = normalizeText(row?.createTime);
+    const tags = Array.isArray(row?.tagnames)
+      ? row.tagnames.map((x) => normalizeText(x)).filter(Boolean)
+      : [];
+    const summary = [dateText, tags.join(" / ")].filter(Boolean).join(" | ");
+    const pubDate = parseDate(dateText) ?? new Date();
+
+    items.push({
+      guid: hashGuid(link),
+      title,
+      link,
+      pubDate,
+      summary: summary || undefined,
+    });
+  }
+
+  return items;
+}
+
+async function fetchItems(sourceId, ctx) {
+  _deps = ctx.deps;
+  const { html, finalUrl } = await ctx.fetchHtml(sourceId, { waitMs: 3500 });
+  const items = parseItemsFromHtml(html, finalUrl);
+  if (items.length > 0) return items;
+
+  const fallbackItems = await fetchItemsFromApi(finalUrl);
+  if (fallbackItems.length > 0) return fallbackItems;
+
+  throw new Error(`[${SITE_ID}] 未解析到学术成果条目，页面结构或接口可能已变化`);
+}
+
+export default {
+  id: SITE_ID,
+  listUrlPattern: /^https?:\/\/(www\.)?sensetime\.com\/cn\/technology-achievements(\?.*)?$/i,
+  fetchItems,
+};

package/app/plugins/builtin/supervisely-blog.rssany.js

@@ -0,0 +1,159 @@
+let _deps;
+
+// Supervisely Blog 插件：抓取列表页并解析为 FeedItem（不做正文 enrich）
+
+
+
+const SUPERVISELY_ORIGIN = "https://supervisely.com";
+const MONTH_INDEX = {
+  jan: 0,
+  feb: 1,
+  mar: 2,
+  apr: 3,
+  may: 4,
+  jun: 5,
+  jul: 6,
+  aug: 7,
+  sep: 8,
+  oct: 9,
+  nov: 10,
+  dec: 11,
+};
+
+
+function normalizeText(text) {
+  return (text ?? "").replace(/\s+/g, " ").trim();
+}
+
+
+function hashGuid(input) {
+  return _deps.createHash("sha256").update(input).digest("hex");
+}
+
+
+function toAbsoluteUrl(href, baseUrl) {
+  if (!href) return null;
+  try {
+    const url = new URL(href, baseUrl);
+    if (!/^https?:$/i.test(url.protocol)) return null;
+    return url.href;
+  } catch {
+    return null;
+  }
+}
+
+
+function parsePubDate(rawText) {
+  const text = normalizeText(rawText);
+  const m = text.match(/^([A-Za-z]{3,9})\s+(\d{1,2}),\s*(\d{4})$/);
+  if (!m) return undefined;
+  const month = MONTH_INDEX[m[1].slice(0, 3).toLowerCase()];
+  if (month == null) return undefined;
+  const day = Number(m[2]);
+  const year = Number(m[3]);
+  if (!Number.isInteger(day) || !Number.isInteger(year)) return undefined;
+  return new Date(Date.UTC(year, month, day, 0, 0, 0));
+}
+
+
+function looksLikeBlogLink(link) {
+  try {
+    const u = new URL(link);
+    return /^\/blog\/[^/]+\/?$/i.test(u.pathname);
+  } catch {
+    return false;
+  }
+}
+
+
+function findAncestor(node, maxDepth) {
+  let current = node?.parentNode ?? null;
+  for (let i = 0; i < maxDepth && current; i += 1) {
+    if (current.querySelector?.("time")) return current;
+    current = current.parentNode ?? null;
+  }
+  return node?.parentNode ?? null;
+}
+
+
+function buildFeedItem({ title, link, summary, author, pubDate }) {
+  return {
+    guid: hashGuid(link),
+    title,
+    link,
+    pubDate: pubDate ?? new Date(),
+    author: author || undefined,
+    summary: summary || undefined,
+  };
+}
+
+
+function parseFromCards(root, baseUrl) {
+  const seen = new Set();
+  const items = [];
+  const cards = root.querySelectorAll("div.blog-card");
+
+  for (const card of cards) {
+    const titleAnchor = card.querySelector("h4 a[href]");
+    const title = normalizeText(titleAnchor?.textContent);
+    const link = toAbsoluteUrl(titleAnchor?.getAttribute("href"), baseUrl);
+    if (!title || !link || !looksLikeBlogLink(link) || seen.has(link)) continue;
+
+    const summary = normalizeText(card.querySelector("p")?.textContent);
+    const author = normalizeText(card.querySelector('b[rel="author"], address b')?.textContent);
+    const pubDateText = normalizeText(card.querySelector("time")?.textContent);
+    const pubDate = parsePubDate(pubDateText);
+
+    seen.add(link);
+    items.push(buildFeedItem({ title, link, summary, author, pubDate }));
+  }
+
+  return items;
+}
+
+
+function parseFromHeadingFallback(root, baseUrl) {
+  const seen = new Set();
+  const items = [];
+  const anchors = root.querySelectorAll('h4 a[href*="/blog/"]');
+
+  for (const anchor of anchors) {
+    const title = normalizeText(anchor.textContent);
+    const link = toAbsoluteUrl(anchor.getAttribute("href"), baseUrl);
+    if (!title || !link || !looksLikeBlogLink(link) || seen.has(link)) continue;
+
+    const container = findAncestor(anchor, 7);
+    const summary = normalizeText(container?.querySelector("p")?.textContent);
+    const author = normalizeText(container?.querySelector('b[rel="author"], address b')?.textContent);
+    const pubDateText = normalizeText(container?.querySelector("time")?.textContent);
+    const pubDate = parsePubDate(pubDateText);
+
+    seen.add(link);
+    items.push(buildFeedItem({ title, link, summary, author, pubDate }));
+  }
+
+  return items;
+}
+
+
+async function fetchItems(sourceId, ctx) {
+  _deps = ctx.deps;
+  const { html, finalUrl } = await ctx.fetchHtml(sourceId, { waitMs: 3500 });
+  const root = _deps.parseHtml(html);
+  const baseUrl = finalUrl || SUPERVISELY_ORIGIN;
+
+  const fromCards = parseFromCards(root, baseUrl);
+  const items = fromCards.length > 0 ? fromCards : parseFromHeadingFallback(root, baseUrl);
+
+  if (items.length === 0) {
+    throw new Error("[supervisely-blog] 未解析到文章条目，页面结构可能已变化");
+  }
+  return items;
+}
+
+
+export default {
+  id: "supervisely-blog",
+  listUrlPattern: /^https?:\/\/(www\.)?supervisely\.com\/blog\/?(?:\?.*)?$/i,
+  fetchItems,
+};

package/app/plugins/builtin/uci-ml-repository.rssany.js

@@ -0,0 +1,111 @@
+let _deps;
+
+
+const UCI_ORIGIN = "https://archive.ics.uci.edu";
+
+function normalizeText(text) {
+  return (text ?? "").replace(/\s+/g, " ").trim();
+}
+
+function hashGuid(input) {
+  return _deps.createHash("sha256").update(input).digest("hex");
+}
+
+function resolveDatasetLink(rawHref, baseUrl) {
+  const href = normalizeText(rawHref);
+  if (!href || href.startsWith("#") || href.startsWith("javascript:")) return null;
+  try {
+    const url = new URL(href, baseUrl);
+    if (!/^https?:$/i.test(url.protocol)) return null;
+    if (url.hostname !== "archive.ics.uci.edu") return null;
+    if (!/^\/dataset\/\d+\/[^/?#]+$/i.test(url.pathname)) return null;
+    url.search = "";
+    url.hash = "";
+    return url.href;
+  } catch {
+    return null;
+  }
+}
+
+function pickSummaryNearNode(node) {
+  let cur = node;
+  for (let i = 0; i < 6 && cur; i += 1) {
+    const p = cur.querySelector?.("p");
+    const summary = normalizeText(p?.textContent);
+    if (summary) return summary;
+    cur = cur.parentNode ?? null;
+  }
+  return "";
+}
+
+function buildItem({ title, link, summary, index }) {
+  return {
+    guid: hashGuid(link),
+    title,
+    link,
+    pubDate: new Date(Date.now() - index * 1000),
+    summary: summary || undefined,
+    sourceId: "uci-ml-repository",
+  };
+}
+
+function parseFromHeadingAnchors(root, baseUrl) {
+  const anchors = root.querySelectorAll('h2 a[href^="/dataset/"]');
+  const items = [];
+  const seen = new Set();
+
+  for (const anchor of anchors) {
+    const link = resolveDatasetLink(anchor.getAttribute("href"), baseUrl);
+    if (!link || seen.has(link)) continue;
+
+    const title = normalizeText(anchor.textContent);
+    if (!title) continue;
+
+    const summary = pickSummaryNearNode(anchor.parentNode ?? anchor);
+    seen.add(link);
+    items.push(buildItem({ title, link, summary, index: items.length }));
+  }
+  return items;
+}
+
+function parseFromGenericAnchors(root, baseUrl) {
+  const anchors = root.querySelectorAll('a[href^="/dataset/"]');
+  const items = [];
+  const seen = new Set();
+
+  for (const anchor of anchors) {
+    const link = resolveDatasetLink(anchor.getAttribute("href"), baseUrl);
+    if (!link || seen.has(link)) continue;
+
+    const titleFromText = normalizeText(anchor.textContent);
+    const titleFromImage = normalizeText(anchor.querySelector("img")?.getAttribute("alt"));
+    const title = titleFromText || titleFromImage;
+    if (!title) continue;
+
+    const summary = pickSummaryNearNode(anchor.parentNode ?? anchor);
+    seen.add(link);
+    items.push(buildItem({ title, link, summary, index: items.length }));
+  }
+  return items;
+}
+
+async function fetchItems(sourceId, ctx) {
+  _deps = ctx.deps;
+  const { html, finalUrl } = await ctx.fetchHtml(sourceId, { waitMs: 4000 });
+  const baseUrl = finalUrl || sourceId || UCI_ORIGIN;
+  const root = _deps.parseHtml(html);
+
+  const byHeading = parseFromHeadingAnchors(root, baseUrl);
+  if (byHeading.length > 0) return byHeading;
+
+  const fallback = parseFromGenericAnchors(root, baseUrl);
+  if (fallback.length > 0) return fallback;
+
+  throw new Error("[uci-ml-repository] 未解析到数据集条目，页面结构可能已变化");
+}
+
+export default {
+  id: "uci-ml-repository",
+  listUrlPattern: /^https?:\/\/archive\.ics\.uci\.edu(?:\/(?:datasets\/?)?)?(?:\?.*)?$/i,
+  fetchItems,
+};

package/app/plugins/builtin/venturebeat.rssany.js

@@ -0,0 +1,97 @@
+let _deps;
+
+// VentureBeat 插件：通过官方 RSS Feed 拉取列表，规避首页安全检查页
+
+
+
+
+function normalizeText(text) {
+  return (text ?? "").replace(/\s+/g, " ").trim();
+}
+
+
+function stripHtml(text) {
+  return normalizeText((text ?? "").replace(/<[^>]*>/g, " "));
+}
+
+
+function toValidDate(raw) {
+  if (!raw) return new Date();
+  const date = new Date(raw);
+  return Number.isNaN(date.getTime()) ? new Date() : date;
+}
+
+
+function toFeedUrl(sourceId) {
+  const url = new URL(sourceId);
+  const path = url.pathname.replace(/\/+$/, "");
+  if (path.endsWith("/feed")) return url.href;
+  url.pathname = path ? `${path}/feed/` : "/feed/";
+  url.search = "";
+  url.hash = "";
+  return url.href;
+}
+
+
+function mapFeedItem(item) {
+  const link = normalizeText(item.link ?? "");
+  if (!/^https?:\/\//i.test(link)) return null;
+
+  const title = normalizeText(item.title ?? "");
+  const pubDate = toValidDate(item.isoDate ?? item.pubDate);
+  const summary = normalizeText(item.contentSnippet ?? "") || stripHtml(item.summary ?? item.content ?? "");
+  const author = normalizeText(item.creator ?? item.author ?? "") || undefined;
+
+  return {
+    guid: _deps.createHash("sha256").update(link).digest("hex"),
+    title: title || "(无标题)",
+    link,
+    pubDate,
+    author,
+    summary: summary || undefined,
+  };
+}
+
+
+async function fetchItems(sourceId, _ctx) {
+  _deps = _ctx.deps;
+  const parser = new _deps.RssParser({
+    timeout: 15_000,
+    headers: {
+      "User-Agent": "RssAny/1.0 (+https://github.com/rssany/rssany)",
+      Accept: "application/rss+xml,application/atom+xml,application/xml,text/xml,*/*",
+    },
+  });
+  const feedUrl = toFeedUrl(sourceId);
+  let feed;
+  try {
+    feed = await parser.parseURL(feedUrl);
+  } catch (err) {
+    const msg = err instanceof Error ? err.message : String(err);
+    throw new Error(`[venturebeat] 抓取 feed 失败: ${feedUrl} (${msg})`);
+  }
+
+  const seen = new Set();
+  const items = [];
+  for (const item of feed.items ?? []) {
+    const mapped = mapFeedItem(item);
+    if (!mapped) continue;
+    if (seen.has(mapped.link)) continue;
+    seen.add(mapped.link);
+    items.push(mapped);
+  }
+
+  if (items.length === 0) {
+    throw new Error(`[venturebeat] 未解析到条目: ${feedUrl}`);
+  }
+
+  return items;
+}
+
+
+export default {
+  id: "venturebeat",
+  listUrlPattern: /^https?:\/\/(www\.)?venturebeat\.com\/?(\?.*)?$/i,
+  refreshInterval: "1h",
+  fetchItems,
+};

package/app/plugins/builtin/worldlabs.rssany.js

@@ -0,0 +1,129 @@
+let _deps;
+
+// World Labs 博客插件：抓取 Research & Insights 列表页，输出 FeedItem（不含 enrich）
+
+
+
+const MONTH_NAME =
+  "January|February|March|April|May|June|July|August|September|October|November|December";
+const DATE_RE = new RegExp(`\\b(${MONTH_NAME})\\s+\\d{1,2},\\s+\\d{4}\\b`, "i");
+const MONTH_INDEX = {
+  january: 0,
+  february: 1,
+  march: 2,
+  april: 3,
+  may: 4,
+  june: 5,
+  july: 6,
+  august: 7,
+  september: 8,
+  october: 9,
+  november: 10,
+  december: 11,
+};
+
+
+function normalizeText(text) {
+  return (text ?? "").replace(/\s+/g, " ").trim();
+}
+
+
+function hashGuid(input) {
+  return _deps.createHash("sha256").update(input).digest("hex");
+}
+
+
+function toAbsoluteHttpUrl(rawHref, baseUrl) {
+  if (!rawHref) return null;
+  const href = rawHref.trim();
+  if (!href || href.startsWith("#") || href.startsWith("javascript:")) return null;
+  try {
+    const url = new URL(href, baseUrl);
+    if (!/^https?:$/i.test(url.protocol)) return null;
+    return url.href;
+  } catch {
+    return null;
+  }
+}
+
+
+function parseDateAndAuthor(metaText) {
+  const text = normalizeText(metaText);
+  const m = text.match(DATE_RE);
+  if (!m) return { pubDate: new Date(), author: undefined };
+
+  const dateText = m[0];
+  const parts = dateText.match(/^(?<month>[A-Za-z]+)\s+(?<day>\d{1,2}),\s*(?<year>\d{4})$/);
+  let date = new Date();
+  if (parts?.groups) {
+    const month = MONTH_INDEX[parts.groups.month.toLowerCase()];
+    const day = Number(parts.groups.day);
+    const year = Number(parts.groups.year);
+    if (month != null && Number.isFinite(day) && Number.isFinite(year)) {
+      // 统一用 UTC 中午，避免仅有日期时因时区导致前后一天偏移。
+      date = new Date(Date.UTC(year, month, day, 12, 0, 0));
+    }
+  }
+  const authorText = normalizeText(text.slice(m.index + dateText.length)).replace(/^[|/\-•·,:]+/, "").trim();
+
+  return {
+    pubDate: Number.isNaN(date.getTime()) ? new Date() : date,
+    author: authorText || undefined,
+  };
+}
+
+
+function parseCard(anchor, finalUrl) {
+  const title = normalizeText(anchor.querySelector("h2, h3")?.textContent);
+  if (!title) return null;
+
+  const link = toAbsoluteHttpUrl(anchor.getAttribute("href"), finalUrl);
+  if (!link) return null;
+
+  const paragraphTexts = anchor
+    .querySelectorAll("p")
+    .map((p) => normalizeText(p.textContent))
+    .filter(Boolean);
+  const metaText = paragraphTexts.find((t) => DATE_RE.test(t)) ?? paragraphTexts[0] ?? "";
+  const { pubDate, author } = parseDateAndAuthor(metaText);
+  const summary = paragraphTexts.find((t) => t !== metaText && !DATE_RE.test(t));
+
+  return {
+    guid: hashGuid(link),
+    title,
+    link,
+    pubDate,
+    author,
+    summary: summary || undefined,
+  };
+}
+
+
+async function fetchItems(sourceId, ctx) {
+  _deps = ctx.deps;
+  const { html, finalUrl } = await ctx.fetchHtml(sourceId, { waitMs: 3500 });
+  const root = _deps.parseHtml(html);
+
+  const seen = new Set();
+  const items = [];
+  const anchors = root.querySelectorAll("a[href]");
+  for (const anchor of anchors) {
+    const item = parseCard(anchor, finalUrl);
+    if (!item) continue;
+    if (seen.has(item.link)) continue;
+    seen.add(item.link);
+    items.push(item);
+  }
+
+  if (items.length === 0) {
+    throw new Error("[worldlabs] 未解析到条目，页面结构可能已变化");
+  }
+  return items;
+}
+
+
+export default {
+  id: "worldlabs",
+  listUrlPattern: /^https?:\/\/(www\.)?worldlabs\.ai\/blog(\?.*)?$/i,
+  fetchItems,
+};