rssany 0.1.4 → 0.1.5

package/app/plugins/builtin/agi-eval-evaluation.rssany.js

@@ -0,0 +1,188 @@
+let _deps;
+
+
+const DEFAULT_ORIGIN = "https://agi-eval.org";
+const DEFAULT_SOURCES = ["PUBLIC", "PRIVATE"];
+const VALID_SOURCES = new Set(DEFAULT_SOURCES);
+
+function normalizeText(text) {
+  return (text ?? "").replace(/\s+/g, " ").trim();
+}
+
+function hashGuid(input) {
+  return _deps.createHash("sha256").update(input).digest("hex");
+}
+
+function parseDate(value) {
+  const text = normalizeText(value);
+  if (!text) return undefined;
+  const date = new Date(text);
+  return Number.isNaN(date.getTime()) ? undefined : date;
+}
+
+function clampText(text, maxLen = 300) {
+  if (text.length <= maxLen) return text;
+  return `${text.slice(0, maxLen - 1).trim()}…`;
+}
+
+function pickOrigin(sourceId) {
+  try {
+    const url = new URL(sourceId);
+    if (/^https?:$/i.test(url.protocol)) return url.origin;
+  } catch {
+    // ignore
+  }
+  return DEFAULT_ORIGIN;
+}
+
+function parseSources(sourceId) {
+  const found = [];
+  try {
+    const url = new URL(sourceId);
+    const fromQuery = [
+      url.searchParams.get("source"),
+      url.searchParams.get("sources"),
+    ];
+    for (const value of fromQuery) {
+      if (!value) continue;
+      found.push(...value.split(/[,+\s|/]+/g));
+    }
+
+    const pathMatch = url.pathname.match(/^\/evaluation\/home\/([^/?#]+)/i);
+    if (pathMatch) {
+      found.push(...decodeURIComponent(pathMatch[1]).split(/[,+\s|/]+/g));
+    }
+  } catch {
+    // ignore
+  }
+
+  const picked = [];
+  const seen = new Set();
+  for (const raw of found) {
+    const source = normalizeText(raw).toUpperCase();
+    if (!VALID_SOURCES.has(source) || seen.has(source)) continue;
+    seen.add(source);
+    picked.push(source);
+  }
+  return picked.length > 0 ? picked : [...DEFAULT_SOURCES];
+}
+
+async function fetchBySource(origin, source) {
+  const response = await fetch(`${origin}/commWebApi/evaluation/home`, {
+    method: "POST",
+    headers: {
+      "Content-Type": "application/json",
+      "Accept": "application/json",
+    },
+    body: JSON.stringify({ source }),
+  });
+
+  if (!response.ok) {
+    throw new Error(`[agi-eval-evaluation] source=${source} 请求失败: HTTP ${response.status}`);
+  }
+
+  const payload = await response.json().catch(() => null);
+  if (!payload || payload.rescode !== 0) {
+    const message = normalizeText(payload?.msg) || "接口返回异常";
+    throw new Error(`[agi-eval-evaluation] source=${source} 请求失败: ${message}`);
+  }
+
+  const list = payload?.data?.evaluationList;
+  if (!Array.isArray(list)) {
+    throw new Error(`[agi-eval-evaluation] source=${source} 响应结构异常`);
+  }
+  return list;
+}
+
+function buildLink(origin, record) {
+  const id = String(record?.id ?? "").trim();
+  const name = normalizeText(record?.name);
+  if (id && name) {
+    return `${origin}/evaluation/${encodeURIComponent(name)}?id=${encodeURIComponent(id)}`;
+  }
+  if (id) {
+    return `${origin}/evaluation/detail?id=${encodeURIComponent(id)}`;
+  }
+  return `${origin}/evaluation/home`;
+}
+
+function pickPubDate(record) {
+  const candidates = [record?.publishTime, record?.createdAt, record?.updatedAt];
+  for (const value of candidates) {
+    const date = parseDate(value);
+    if (date) return date;
+  }
+  return new Date();
+}
+
+function buildSummary(record) {
+  const description = normalizeText(
+    record?.description ||
+      record?.introduction ||
+      record?.zhData?.description ||
+      record?.enData?.description ||
+      ""
+  );
+  const detail = description || "AGI-Eval 评测条目";
+  const views = Number(record?.views);
+  if (Number.isFinite(views) && views >= 0) {
+    return clampText(`${detail} | 浏览量 ${views}`);
+  }
+  return clampText(detail);
+}
+
+function toFeedItem(record, origin, source) {
+  if (!record || typeof record !== "object") return null;
+  const title = normalizeText(record.name);
+  if (!title) return null;
+
+  const link = buildLink(origin, record);
+  const id = String(record.id ?? "").trim();
+  const guidSeed = id ? `agi-eval:${id}` : link;
+
+  return {
+    guid: hashGuid(guidSeed),
+    title,
+    link,
+    pubDate: pickPubDate(record),
+    author: "AGI-Eval",
+    summary: buildSummary(record),
+    sourceId: "agi-eval-evaluation",
+  };
+}
+
+async function fetchItems(sourceId, ctx) {
+  _deps = ctx.deps;
+  const origin = pickOrigin(sourceId);
+  const sources = parseSources(sourceId);
+  const rows = [];
+
+  for (const source of sources) {
+    const list = await fetchBySource(origin, source);
+    for (const record of list) {
+      rows.push({ source, record });
+    }
+  }
+
+  const seen = new Set();
+  const items = [];
+  for (const { source, record } of rows) {
+    const item = toFeedItem(record, origin, source);
+    if (!item || seen.has(item.guid)) continue;
+    seen.add(item.guid);
+    items.push(item);
+  }
+
+  if (items.length === 0) {
+    throw new Error("[agi-eval-evaluation] 未解析到条目，接口结构可能已变化");
+  }
+
+  items.sort((a, b) => b.pubDate.getTime() - a.pubDate.getTime());
+  return items;
+}
+
+export default {
+  id: "agi-eval-evaluation",
+  listUrlPattern: /^https?:\/\/agi-eval\.(org|cn)\/evaluation\/home(?:\/[^/?#]+)?\/?(?:\?.*)?$/i,
+  fetchItems,
+};

package/app/plugins/builtin/amii-research-talent.rssany.js

@@ -0,0 +1,73 @@
+let _deps;
+
+// Amii Research & Talent 插件：抓取人物卡片列表（不做正文 enrich）
+
+
+const PEOPLE_PATH_RE = /^\/people\/[^/?#]+\/?$/i;
+
+function normalizeText(text) {
+  return (text ?? "").replace(/\s+/g, " ").trim();
+}
+
+function hashGuid(input) {
+  return _deps.createHash("sha256").update(input).digest("hex");
+}
+
+function resolvePeopleLink(rawHref, pageUrl) {
+  if (!rawHref) return null;
+  try {
+    const url = new URL(rawHref, pageUrl);
+    if (!/^https?:$/i.test(url.protocol)) return null;
+    if (!PEOPLE_PATH_RE.test(url.pathname)) return null;
+    return url.href;
+  } catch {
+    return null;
+  }
+}
+
+function extractPeopleItems(root, pageUrl) {
+  const anchors = root.querySelectorAll('a[href*="/people/"]');
+  const seen = new Set();
+  const items = [];
+
+  for (const anchor of anchors) {
+    const link = resolvePeopleLink(anchor.getAttribute("href"), pageUrl);
+    if (!link || seen.has(link)) continue;
+
+    const title = normalizeText(anchor.querySelector("h3")?.textContent);
+    if (!title) continue;
+
+    const summary = normalizeText(anchor.querySelector("p")?.textContent);
+    seen.add(link);
+    items.push({
+      guid: hashGuid(link),
+      title,
+      link,
+      pubDate: new Date(),
+      author: "Amii",
+      summary: summary || undefined,
+    });
+  }
+
+  return items;
+}
+
+async function fetchItems(sourceId, ctx) {
+  _deps = ctx.deps;
+  const { html, finalUrl } = await ctx.fetchHtml(sourceId, { waitMs: 3000 });
+  const root = _deps.parseHtml(html);
+  const pageUrl = new URL(finalUrl);
+
+  const items = extractPeopleItems(root, pageUrl);
+  if (items.length === 0) {
+    throw new Error("[amii-research-talent] 未解析到人物条目，页面结构可能已变化");
+  }
+
+  return items;
+}
+
+export default {
+  id: "amii-research-talent",
+  listUrlPattern: /^https?:\/\/(www\.)?amii\.ca\/research-talent\/?(\?.*)?$/i,
+  fetchItems,
+};

package/app/plugins/builtin/anthropic-research.rssany.js

@@ -0,0 +1,155 @@
+let _deps;
+
+// Anthropic Research 插件：抓取研究页列表条目（不含 enrich）
+
+
+
+const ANTHROPIC_ORIGIN = "https://www.anthropic.com";
+const MONTH_TO_INDEX = {
+  jan: 0,
+  feb: 1,
+  mar: 2,
+  apr: 3,
+  may: 4,
+  jun: 5,
+  jul: 6,
+  aug: 7,
+  sep: 8,
+  oct: 9,
+  nov: 10,
+  dec: 11,
+};
+
+
+function normalizeText(text) {
+  return (text ?? "").replace(/\s+/g, " ").trim();
+}
+
+
+function hashGuid(input) {
+  return _deps.createHash("sha256").update(input).digest("hex");
+}
+
+
+function toAbsoluteHttpUrl(rawHref, baseUrl) {
+  if (!rawHref) return null;
+  const href = rawHref.trim();
+  if (!href || href.startsWith("#") || href.startsWith("javascript:")) return null;
+  try {
+    const url = new URL(href, baseUrl);
+    if (!/^https?:$/i.test(url.protocol)) return null;
+    return url.href;
+  } catch {
+    return null;
+  }
+}
+
+
+function isResearchArticlePath(pathname) {
+  if (!pathname.startsWith("/research/")) return false;
+  if (pathname.startsWith("/research/team/")) return false;
+  return pathname.length > "/research/".length;
+}
+
+
+function parsePubDate(dateText) {
+  const normalized = normalizeText(dateText);
+  if (!normalized) return undefined;
+
+  const m = normalized.match(/^([A-Za-z]{3,9})\s+(\d{1,2}),\s*(\d{4})$/);
+  if (m) {
+    const month = MONTH_TO_INDEX[m[1].slice(0, 3).toLowerCase()];
+    if (month != null) {
+      const day = Number(m[2]);
+      const year = Number(m[3]);
+      const d = new Date(Date.UTC(year, month, day, 12, 0, 0));
+      if (!Number.isNaN(d.getTime())) return d;
+    }
+  }
+
+  const direct = new Date(normalized);
+  if (!Number.isNaN(direct.getTime())) return direct;
+  return undefined;
+}
+
+
+function extractTitle(anchor) {
+  const heading =
+    anchor.querySelector("h2") ??
+    anchor.querySelector("h3") ??
+    anchor.querySelector("h4");
+  const headingText = normalizeText(heading?.textContent);
+  if (headingText) return headingText;
+
+  const spans = anchor
+    .querySelectorAll("span")
+    .map((s) => normalizeText(s.textContent))
+    .filter(Boolean);
+  if (spans.length > 0) return spans[spans.length - 1];
+
+  return normalizeText(anchor.textContent);
+}
+
+
+function extractSummary(anchor, title) {
+  const summary = normalizeText(anchor.querySelector("p")?.textContent);
+  if (summary && summary !== title) return summary;
+  return undefined;
+}
+
+
+function parseAnchorItem(anchor, finalUrl) {
+  const link = toAbsoluteHttpUrl(anchor.getAttribute("href"), finalUrl);
+  if (!link) return null;
+
+  const pathname = new URL(link).pathname;
+  if (!isResearchArticlePath(pathname)) return null;
+
+  const title = extractTitle(anchor);
+  if (!title) return null;
+
+  const dateText = normalizeText(anchor.querySelector("time")?.textContent);
+  const pubDate = parsePubDate(dateText) ?? new Date();
+  const summary = extractSummary(anchor, title);
+
+  return {
+    guid: hashGuid(link),
+    title,
+    link,
+    pubDate,
+    author: "Anthropic",
+    summary,
+  };
+}
+
+
+async function fetchItems(sourceId, ctx) {
+  _deps = ctx.deps;
+  const { html, finalUrl } = await ctx.fetchHtml(sourceId, { waitMs: 4500 });
+  const root = _deps.parseHtml(html);
+
+  const seen = new Set();
+  const items = [];
+  const anchors = root.querySelectorAll("a[href]");
+
+  for (const anchor of anchors) {
+    const item = parseAnchorItem(anchor, finalUrl || ANTHROPIC_ORIGIN);
+    if (!item) continue;
+    if (seen.has(item.link)) continue;
+    seen.add(item.link);
+    items.push(item);
+  }
+
+  if (items.length === 0) {
+    throw new Error("[anthropic-research] 未解析到研究条目，页面结构可能已变化");
+  }
+
+  return items;
+}
+
+
+export default {
+  id: "anthropic-research",
+  listUrlPattern: /^https?:\/\/(www\.)?anthropic\.com\/research(?:\/)?(\?.*)?$/i,
+  fetchItems,
+};

package/app/plugins/builtin/appen-resources.rssany.js

@@ -0,0 +1,155 @@
+let _deps;
+
+
+const APPEN_ORIGIN = "https://www.appen.com";
+const IGNORED_TEXTS = new Set([
+  "learn more",
+  "view all",
+  "browse catalog",
+  "blog",
+  "case studies",
+  "white papers, reports and ebooks",
+  "white papers, reports and ebooks",
+]);
+
+function normalizeText(text) {
+  return (text ?? "").replace(/\s+/g, " ").trim();
+}
+
+function hashGuid(input) {
+  return _deps.createHash("sha256").update(input).digest("hex");
+}
+
+function resolveHttpUrl(href, baseUrl) {
+  if (!href) return null;
+  const raw = href.trim();
+  if (!raw || raw.startsWith("#") || raw.startsWith("javascript:")) return null;
+  try {
+    const url = new URL(raw, baseUrl);
+    if (!/^https?:$/i.test(url.protocol)) return null;
+    return url;
+  } catch {
+    return null;
+  }
+}
+
+function getResourceType(pathname) {
+  if (/^\/blog\/[^/?#]+\/?$/i.test(pathname)) return { label: "Blog", author: "Appen Blog" };
+  if (/^\/case-studies\/[^/?#]+\/?$/i.test(pathname)) return { label: "Case Study", author: "Appen Case Studies" };
+  if (/^\/whitepapers\/[^/?#]+\/?$/i.test(pathname)) return { label: "Whitepaper", author: "Appen Whitepapers" };
+  return null;
+}
+
+function dedupeTexts(texts) {
+  const out = [];
+  const seen = new Set();
+  for (const text of texts) {
+    if (!text) continue;
+    const key = text.toLowerCase();
+    if (seen.has(key)) continue;
+    seen.add(key);
+    out.push(text);
+  }
+  return out;
+}
+
+function extractLeafTexts(anchor) {
+  const nodes = anchor.querySelectorAll("h1,h2,h3,h4,h5,h6,p,span,div");
+  const leafs = nodes
+    .filter((node) => node.querySelector("h1,h2,h3,h4,h5,h6,p,span,div") == null)
+    .map((node) => normalizeText(node.textContent))
+    .filter(Boolean)
+    .filter((text) => !IGNORED_TEXTS.has(text.toLowerCase()));
+  return dedupeTexts(leafs);
+}
+
+function pickTitle(anchor, leafTexts) {
+  const heading = normalizeText(anchor.querySelector("h1,h2,h3,h4,h5,h6")?.textContent);
+  if (heading && !IGNORED_TEXTS.has(heading.toLowerCase())) return heading;
+  const longEnough = leafTexts.filter((text) => text.length >= 12);
+  if (longEnough.length > 0) {
+    return longEnough.slice().sort((a, b) => b.length - a.length)[0];
+  }
+  return leafTexts[0] ?? "";
+}
+
+function pickSummary(leafTexts, title) {
+  const summaryCandidates = leafTexts
+    .filter((text) => text !== title)
+    .filter((text) => text.length >= 24);
+  if (summaryCandidates.length === 0) return undefined;
+  return summaryCandidates.slice().sort((a, b) => b.length - a.length)[0];
+}
+
+function parsePagePublishedDate(html) {
+  const match = html.match(/Last Published:\s*([^-<]+GMT\+0000)/i);
+  if (!match) return undefined;
+  const date = new Date(match[1].trim());
+  return Number.isNaN(date.getTime()) ? undefined : date;
+}
+
+function parseDateFromSlug(pathname) {
+  const match = pathname.match(/(?:^|[-/])(20\d{2})(?:[-/]|$)/);
+  if (!match) return undefined;
+  const year = Number(match[1]);
+  if (!Number.isFinite(year) || year < 2000 || year > 2100) return undefined;
+  return new Date(Date.UTC(year, 0, 1));
+}
+
+function upsertItem(itemsByLink, candidate) {
+  const previous = itemsByLink.get(candidate.link);
+  if (!previous) {
+    itemsByLink.set(candidate.link, candidate);
+    return;
+  }
+  const score = (item) => (item.summary ? 2 : 0) + (item.title.length >= 20 ? 1 : 0);
+  if (score(candidate) > score(previous)) {
+    itemsByLink.set(candidate.link, candidate);
+  }
+}
+
+async function fetchItems(sourceId, ctx) {
+  _deps = ctx.deps;
+  const { html, finalUrl } = await ctx.fetchHtml(sourceId, { waitMs: 4500 });
+  const root = _deps.parseHtml(html);
+  const baseUrl = finalUrl || sourceId || APPEN_ORIGIN;
+  const pagePublishedDate = parsePagePublishedDate(html);
+  const fallbackDate = pagePublishedDate ?? new Date();
+  const itemsByLink = new Map();
+
+  const anchors = root.querySelectorAll("a[href]");
+  for (const anchor of anchors) {
+    const url = resolveHttpUrl(anchor.getAttribute("href"), baseUrl);
+    if (!url) continue;
+
+    const resourceType = getResourceType(url.pathname);
+    if (!resourceType) continue;
+
+    const leafTexts = extractLeafTexts(anchor);
+    const title = pickTitle(anchor, leafTexts);
+    if (!title) continue;
+
+    const summary = pickSummary(leafTexts, title);
+    const inferredDate = parseDateFromSlug(url.pathname);
+    upsertItem(itemsByLink, {
+      guid: hashGuid(url.href),
+      title,
+      link: url.href,
+      pubDate: inferredDate ?? fallbackDate,
+      author: resourceType.author,
+      summary,
+    });
+  }
+
+  const items = Array.from(itemsByLink.values());
+  if (items.length === 0) {
+    throw new Error("[appen-resources] 未解析到资源条目，页面结构可能已变化");
+  }
+  return items;
+}
+
+export default {
+  id: "appen-resources",
+  listUrlPattern: /^https?:\/\/(www\.)?appen\.com\/resources\/?(\?.*)?$/i,
+  fetchItems,
+};