rssany 0.1.2 → 0.1.5

package/app/plugins/builtin/pjlab-adg-publications.rssany.js

@@ -0,0 +1,202 @@
+let _deps;
+
+
+
+const SITE_ID = "pjlab-adg-publications";
+
+
+function normalizeText(text) {
+  return (text ?? "")
+    .replace(/\u00a0/g, " ")
+    .replace(/\s+/g, " ")
+    .trim();
+}
+
+
+function hashGuid(input) {
+  return _deps.createHash("sha256").update(input).digest("hex");
+}
+
+
+function parseYear(raw) {
+  const text = normalizeText(raw);
+  const match = text.match(/\b(19|20)\d{2}\b/);
+  if (!match) return undefined;
+  const year = Number(match[0]);
+  return Number.isFinite(year) ? year : undefined;
+}
+
+
+function toAbsoluteLink(rawHref, baseUrl) {
+  if (!rawHref) return null;
+  const href = rawHref.trim();
+  if (!href || href.startsWith("#") || href.startsWith("javascript:")) return null;
+  try {
+    const url = new URL(href, baseUrl);
+    if (!/^https?:$/i.test(url.protocol)) return null;
+    return url.href;
+  } catch {
+    return null;
+  }
+}
+
+
+function pickBestLink(detailNode, pageUrl, entryId) {
+  const linkNodes = detailNode.querySelectorAll(".links a[href], a[href]");
+  const candidates = [];
+  for (const node of linkNodes) {
+    const link = toAbsoluteLink(node.getAttribute("href"), pageUrl);
+    if (!link) continue;
+    const label = normalizeText(node.textContent).toLowerCase();
+    candidates.push({ link, label });
+  }
+
+  const preferHtml = candidates.find((x) => x.label === "html" || x.label === "arxiv" || x.label === "doi");
+  if (preferHtml) return preferHtml.link;
+
+  const preferPdf = candidates.find((x) => x.label === "pdf");
+  if (preferPdf) return preferPdf.link;
+
+  if (candidates.length > 0) return candidates[0].link;
+  if (entryId) return `${pageUrl}#${encodeURIComponent(entryId)}`;
+  return pageUrl;
+}
+
+
+function buildSummary(author, periodical) {
+  const chunks = [author, periodical].map((x) => normalizeText(x)).filter(Boolean);
+  if (chunks.length === 0) return undefined;
+  return chunks.join(" | ");
+}
+
+
+function elementChildren(node) {
+  return node.childNodes.filter((child) => child.tagName != null);
+}
+
+
+function directDivChildren(node) {
+  return elementChildren(node).filter((child) => child.tagName?.toLowerCase() === "div");
+}
+
+
+function extractTitle(detailNode) {
+  const fromClass = normalizeText(detailNode.querySelector(".title")?.textContent);
+  if (fromClass) return fromClass;
+
+  const divs = directDivChildren(detailNode);
+  for (const div of divs) {
+    if (div.querySelector("em")) continue;
+    if (div.querySelector("a[href]")) continue;
+    const text = normalizeText(div.textContent);
+    if (!text) continue;
+    if (text.length < 8) continue;
+    return text;
+  }
+  return "";
+}
+
+
+function extractAuthor(detailNode, title) {
+  const fromClass = normalizeText(detailNode.querySelector(".author")?.textContent);
+  if (fromClass) return fromClass;
+
+  const divs = directDivChildren(detailNode);
+  const textDivs = divs
+    .filter((div) => !div.querySelector("em"))
+    .filter((div) => !div.querySelector("a[href]"))
+    .map((div) => normalizeText(div.textContent))
+    .filter(Boolean);
+  const candidate = textDivs.find((text) => text !== title);
+  return candidate || undefined;
+}
+
+
+function extractPeriodical(detailNode) {
+  const fromClass = normalizeText(
+    (detailNode.querySelector(".periodical em") ?? detailNode.querySelector(".periodical"))?.textContent
+  );
+  if (fromClass) return fromClass;
+
+  const divs = directDivChildren(detailNode);
+  for (const div of divs) {
+    const emText = normalizeText(div.querySelector("em")?.textContent);
+    if (emText) return emText;
+  }
+  return "";
+}
+
+
+function parseOneEntry(liNode, currentYear, pageUrl) {
+  const detailNode = liNode.querySelector("div[id]") ?? liNode.querySelector("div");
+  if (!detailNode) return null;
+
+  const entryId = normalizeText(detailNode.getAttribute("id"));
+  const title = extractTitle(detailNode);
+  if (!title) return null;
+
+  const author = extractAuthor(detailNode, title);
+  const periodical = extractPeriodical(detailNode);
+  const fallbackYear = parseYear(`${periodical} ${detailNode.textContent}`);
+  const finalYear = currentYear ?? fallbackYear;
+  const pubDate = finalYear != null ? new Date(Date.UTC(finalYear, 0, 1, 0, 0, 0)) : new Date();
+  const badge = normalizeText((liNode.querySelector(".abbr .badge") ?? liNode.querySelector("abbr"))?.textContent) || undefined;
+  const link = pickBestLink(detailNode, pageUrl, entryId);
+  const guidSeed = entryId || link || `${title}|${author ?? ""}|${finalYear ?? ""}`;
+
+  return {
+    guid: hashGuid(guidSeed),
+    title,
+    link,
+    pubDate,
+    author,
+    summary: buildSummary(author, periodical),
+    sourceId: SITE_ID,
+  };
+}
+
+
+function parseItems(html, finalUrl) {
+  const root = _deps.parseHtml(html);
+  const container = root.querySelector("article") ?? root;
+  const items = [];
+  const seenGuid = new Set();
+  let currentYear;
+
+  const stream = container.querySelectorAll("h2, li");
+  for (const node of stream) {
+    const tag = node.tagName?.toLowerCase();
+    if (tag === "h2") {
+      const year = parseYear(node.textContent);
+      if (year != null) currentYear = year;
+      continue;
+    }
+    if (tag !== "li") continue;
+
+    const item = parseOneEntry(node, currentYear, finalUrl);
+    if (!item) continue;
+    if (seenGuid.has(item.guid)) continue;
+    seenGuid.add(item.guid);
+    items.push(item);
+  }
+  return items;
+}
+
+
+async function fetchItems(sourceId, ctx) {
+  _deps = ctx.deps;
+  const { html, finalUrl } = await ctx.fetchHtml(sourceId, { waitMs: 3500 });
+  const pageUrl = finalUrl || sourceId;
+  const items = parseItems(html, pageUrl);
+  if (items.length === 0) {
+    throw new Error("[pjlab-adg-publications] 未解析到论文条目，页面结构可能已变化");
+  }
+  return items;
+}
+
+
+export default {
+  id: SITE_ID,
+  listUrlPattern: /^https?:\/\/pjlab-adg\.github\.io\/publications\/?(\?.*)?$/i,
+  fetchItems,
+};

package/app/plugins/builtin/rss.rssany.js

@@ -12,6 +12,16 @@ function trimUrl(s) {
   return t || undefined;
 }
 
+/** rss-parser 常把多位作者压成一段逗号（或中文逗号）分隔文本，拆成数组入库。 */
+function authorsFromCommaText(authorRaw) {
+  if (typeof authorRaw !== "string") return undefined;
+  const parts = authorRaw
+    .split(/[,，]/)
+    .map((s) => s.trim())
+    .filter(Boolean);
+  return parts.length > 0 ? parts : undefined;
+}
+
 /** 从 rss-parser 条目上尽量取出配图 URL（入库用 imageUrl，与 Gateway 的 cover_img 对齐）。 */
 function extractItemImageUrl(item) {
   const enc = item.enclosure;
@@ -127,7 +137,7 @@ export default {
             : new Date();
       const authorRaw =
         typeof item.creator === "string" ? item.creator : typeof item.author === "string" ? item.author : undefined;
-      const author = authorRaw ? [authorRaw] : undefined;
+      const author = authorsFromCommaText(authorRaw);
       const summary =
         typeof item.summary === "string" ? item.summary : typeof item.contentSnippet === "string" ? item.contentSnippet : undefined;
       const content =

package/app/plugins/builtin/selectdataset.rssany.js

@@ -0,0 +1,206 @@
+let _deps;
+
+// SelectDataset 插件：解析首页/搜索页 Nuxt payload，输出数据集条目（不含 enrich）
+
+
+
+const SELECT_DATASET_ORIGIN = "https://www.selectdataset.com";
+
+
+function normalizeText(text) {
+  return (text ?? "").replace(/\s+/g, " ").trim();
+}
+
+
+function hashGuid(input) {
+  return _deps.createHash("sha256").update(input).digest("hex");
+}
+
+
+function parseDate(dateText) {
+  const text = normalizeText(dateText);
+  if (!text || text === "0") return new Date();
+  const m = text.match(
+    /^(\d{4})-(\d{1,2})-(\d{1,2})(?:[ T](\d{1,2}):(\d{1,2})(?::(\d{1,2}))?)?$/
+  );
+  if (!m) {
+    const fallback = new Date(text);
+    return Number.isNaN(fallback.getTime()) ? new Date() : fallback;
+  }
+  const [, y, mm, dd, hh = "0", mi = "0", ss = "0"] = m;
+  // 站点时间以中国时区为主，显式补 +08:00 避免环境时区影响排序。
+  const withTimezone = `${y}-${mm.padStart(2, "0")}-${dd.padStart(2, "0")}T` +
+    `${hh.padStart(2, "0")}:${mi.padStart(2, "0")}:${ss.padStart(2, "0")}+08:00`;
+  const parsed = new Date(withTimezone);
+  return Number.isNaN(parsed.getTime()) ? new Date() : parsed;
+}
+
+
+function toAbsoluteDatasetUrl(datasetId) {
+  const id = normalizeText(datasetId);
+  if (!id) return null;
+  return `${SELECT_DATASET_ORIGIN}/dataset/${id}`;
+}
+
+
+function dedupeItems(items) {
+  const seen = new Set();
+  const out = [];
+  for (const item of items) {
+    if (!item?.link || seen.has(item.link)) continue;
+    seen.add(item.link);
+    out.push(item);
+  }
+  return out;
+}
+
+
+function parseFromAnchorDom(html, finalUrl) {
+  const root = _deps.parseHtml(html);
+  const baseUrl = finalUrl || SELECT_DATASET_ORIGIN;
+  const items = [];
+
+  for (const anchor of root.querySelectorAll('a[href*="/dataset/"]')) {
+    const href = anchor.getAttribute("href");
+    if (!href) continue;
+    let link = null;
+    try {
+      const url = new URL(href, baseUrl);
+      if (!/^https?:$/i.test(url.protocol)) continue;
+      if (!/\/dataset\/[A-Za-z0-9]{16,}/.test(url.pathname)) continue;
+      link = url.href;
+    } catch {
+      continue;
+    }
+    const title = normalizeText(anchor.textContent);
+    if (!title) continue;
+
+    items.push({
+      guid: hashGuid(link),
+      title,
+      link,
+      pubDate: new Date(),
+      summary: undefined,
+    });
+  }
+
+  return dedupeItems(items);
+}
+
+
+function createNuxtResolver(table) {
+  const cache = new Map();
+  const inProgress = new Set();
+
+  function decodeRef(index) {
+    if (cache.has(index)) return cache.get(index);
+    if (inProgress.has(index)) return undefined;
+    inProgress.add(index);
+    const decoded = decodeValue(table[index]);
+    inProgress.delete(index);
+    cache.set(index, decoded);
+    return decoded;
+  }
+
+  function decodeValue(value) {
+    if (typeof value === "number") {
+      if (Number.isInteger(value) && value >= 0 && value < table.length) {
+        return decodeRef(value);
+      }
+      return value;
+    }
+    if (value == null || typeof value !== "object") return value;
+
+    if (Array.isArray(value)) {
+      if (value.length === 2 && (value[0] === "Reactive" || value[0] === "ShallowReactive")) {
+        return decodeValue(value[1]);
+      }
+      if (value.length === 2 && value[0] === "Set") {
+        const raw = decodeValue(value[1]);
+        return Array.isArray(raw) ? raw : [];
+      }
+      return value.map((x) => decodeValue(x));
+    }
+
+    const out = {};
+    for (const [key, v] of Object.entries(value)) {
+      out[key] = decodeValue(v);
+    }
+    return out;
+  }
+
+  return { decodeRef };
+}
+
+
+function toFeedItem(record) {
+  const title = normalizeText(record.dataset_name);
+  const link = toAbsoluteDatasetUrl(record.id);
+  if (!title || !link) return null;
+
+  const summary = normalizeText(record.dataset_desc);
+  const author = normalizeText(record.ext_host_name);
+  return {
+    guid: hashGuid(link),
+    title,
+    link,
+    pubDate: parseDate(record.date_index_update || record.date_dataset_update),
+    author: author || undefined,
+    summary: summary || undefined,
+  };
+}
+
+
+function parseFromNuxtPayload(html) {
+  const root = _deps.parseHtml(html);
+  const payload = root.querySelector("#__NUXT_DATA__")?.textContent;
+  if (!payload) return [];
+
+  let table;
+  try {
+    table = JSON.parse(payload);
+  } catch {
+    return [];
+  }
+  if (!Array.isArray(table)) return [];
+
+  const { decodeRef } = createNuxtResolver(table);
+  const items = [];
+
+  for (let i = 0; i < table.length; i += 1) {
+    const entry = table[i];
+    if (entry == null || typeof entry !== "object" || Array.isArray(entry)) continue;
+    if (!("dataset_name" in entry) || !("id" in entry)) continue;
+
+    const decoded = decodeRef(i);
+    if (!decoded || typeof decoded !== "object") continue;
+    const item = toFeedItem(decoded);
+    if (item) items.push(item);
+  }
+
+  const deduped = dedupeItems(items);
+  deduped.sort((a, b) => b.pubDate.getTime() - a.pubDate.getTime());
+  return deduped;
+}
+
+
+async function fetchItems(sourceId, ctx) {
+  _deps = ctx.deps;
+  const { html, finalUrl } = await ctx.fetchHtml(sourceId, { waitMs: 3500 });
+  const fromAnchorDom = parseFromAnchorDom(html, finalUrl);
+  if (fromAnchorDom.length > 0) return fromAnchorDom;
+
+  // 净化 HTML 下无稳定数据集链接时，回退到未净化 payload 取 dataset id 与更新时间。
+  const raw = await ctx.fetchHtml(sourceId, { waitMs: 3500, purify: false });
+  const fromPayload = parseFromNuxtPayload(raw.html);
+  if (fromPayload.length > 0) return fromPayload;
+
+  throw new Error("[selectdataset] 未解析到数据集条目，页面结构可能已变化");
+}
+
+
+export default {
+  id: "selectdataset",
+  listUrlPattern: /^https?:\/\/(www\.)?selectdataset\.com\/(?:$|\?.*|search(?:\?.*)?|subject(?:\?.*)?)$/i,
+  fetchItems,
+};

package/app/plugins/builtin/sensetime-tech-achievements.rssany.js

@@ -0,0 +1,154 @@
+let _deps;
+
+
+const SITE_ID = "sensetime-tech-achievements";
+const DATE_RE = /\b(20\d{2})-(\d{1,2})-(\d{1,2})\b/;
+
+function normalizeText(text) {
+  return (text ?? "").replace(/\s+/g, " ").trim();
+}
+
+function hashGuid(input) {
+  return _deps.createHash("sha256").update(input).digest("hex");
+}
+
+function toAbsoluteUrl(rawHref, baseUrl) {
+  if (!rawHref) return null;
+  const href = rawHref.trim();
+  if (!href || href.startsWith("#") || href.startsWith("javascript:")) return null;
+  try {
+    const url = new URL(href, baseUrl);
+    if (!/^https?:$/i.test(url.protocol)) return null;
+    return url.href;
+  } catch {
+    return null;
+  }
+}
+
+function parseDate(dateText) {
+  const text = normalizeText(dateText);
+  const m = text.match(DATE_RE);
+  if (!m) return undefined;
+  const [, y, mm, dd] = m;
+  return new Date(`${y}-${mm.padStart(2, "0")}-${dd.padStart(2, "0")}T00:00:00.000Z`);
+}
+
+function extractDateText(anchor) {
+  let current = anchor;
+  for (let i = 0; i < 8 && current; i += 1) {
+    const text = normalizeText(current.textContent);
+    const m = text.match(DATE_RE);
+    if (m) return m[0];
+    current = current.parentNode ?? null;
+  }
+  return "";
+}
+
+function extractTags(anchor, title, dateText) {
+  let current = anchor;
+  for (let i = 0; i < 6 && current; i += 1) {
+    const spans = current.querySelectorAll?.("span") ?? [];
+    const tags = spans
+      .map((s) => normalizeText(s.textContent))
+      .filter(Boolean)
+      .filter((x) => x !== title && x !== dateText)
+      .filter((x) => !DATE_RE.test(x));
+    if (tags.length > 0) return Array.from(new Set(tags));
+    current = current.parentNode ?? null;
+  }
+  return [];
+}
+
+function parseItemsFromHtml(html, finalUrl) {
+  const root = _deps.parseHtml(html);
+  const anchors = root.querySelectorAll('a[href*="/technology-new-detail/"]');
+  const seen = new Set();
+  const items = [];
+
+  for (const anchor of anchors) {
+    const link = toAbsoluteUrl(anchor.getAttribute("href"), finalUrl);
+    if (!link || seen.has(link)) continue;
+
+    const title = normalizeText(anchor.textContent);
+    if (!title) continue;
+
+    seen.add(link);
+    const dateText = extractDateText(anchor);
+    const tags = extractTags(anchor, title, dateText);
+    const summary = [dateText, tags.join(" / ")].filter(Boolean).join(" | ");
+    const pubDate = parseDate(dateText) ?? new Date();
+
+    items.push({
+      guid: hashGuid(link),
+      title,
+      link,
+      pubDate,
+      summary: summary || undefined,
+    });
+  }
+
+  return items;
+}
+
+async function fetchItemsFromApi(finalUrl) {
+  const origin = new URL(finalUrl).origin;
+  const apiUrl = new URL("/rest/v1/contents/1/getlistbyparam/48/1/20/0/0?scene=1", origin);
+
+  const res = await fetch(apiUrl, {
+    headers: {
+      Accept: "application/json,text/plain,*/*",
+      "User-Agent":
+        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
+    },
+  });
+  if (!res.ok) return [];
+
+  const data = await res.json();
+  const list = Array.isArray(data?.data?.lists) ? data.data.lists : [];
+  const items = [];
+  const seen = new Set();
+
+  for (const row of list) {
+    const contentId = String(row?.contentId ?? "").trim();
+    const title = normalizeText(row?.title);
+    if (!contentId || !title) continue;
+    const link = new URL(`/cn/technology-new-detail/${contentId}?categoryId=48`, origin).href;
+    if (seen.has(link)) continue;
+    seen.add(link);
+
+    const dateText = normalizeText(row?.createTime);
+    const tags = Array.isArray(row?.tagnames)
+      ? row.tagnames.map((x) => normalizeText(x)).filter(Boolean)
+      : [];
+    const summary = [dateText, tags.join(" / ")].filter(Boolean).join(" | ");
+    const pubDate = parseDate(dateText) ?? new Date();
+
+    items.push({
+      guid: hashGuid(link),
+      title,
+      link,
+      pubDate,
+      summary: summary || undefined,
+    });
+  }
+
+  return items;
+}
+
+async function fetchItems(sourceId, ctx) {
+  _deps = ctx.deps;
+  const { html, finalUrl } = await ctx.fetchHtml(sourceId, { waitMs: 3500 });
+  const items = parseItemsFromHtml(html, finalUrl);
+  if (items.length > 0) return items;
+
+  const fallbackItems = await fetchItemsFromApi(finalUrl);
+  if (fallbackItems.length > 0) return fallbackItems;
+
+  throw new Error(`[${SITE_ID}] 未解析到学术成果条目，页面结构或接口可能已变化`);
+}
+
+export default {
+  id: SITE_ID,
+  listUrlPattern: /^https?:\/\/(www\.)?sensetime\.com\/cn\/technology-achievements(\?.*)?$/i,
+  fetchItems,
+};