rssany 0.1.2 → 0.1.5

package/app/plugins/builtin/xiaohongshu.rssany.js

@@ -0,0 +1,283 @@
+let _deps;
+
+// 小红书站点插件：用户主页列表抓取、笔记详情提取、认证流程
+
+
+
+const XHS_ORIGIN = "https://www.xiaohongshu.com";
+
+
+function getOrigin(url) {
+  try {
+    return new URL(url).origin;
+  } catch {
+    return XHS_ORIGIN;
+  }
+}
+
+
+function buildExploreLinkWithXsec(profileHref, origin) {
+  try {
+    const fullUrl = new URL(profileHref.replace(/&/g, "&"), origin);
+    const pathSegs = fullUrl.pathname.split("/").filter(Boolean);
+    const noteId = pathSegs[pathSegs.length - 1];
+    if (!noteId || !/^[0-9a-f]+$/i.test(noteId)) return null;
+    const token = fullUrl.searchParams.get("xsec_token");
+    const source = fullUrl.searchParams.get("xsec_source") ?? "pc_user";
+    if (!token) return null;
+    const explore = new URL(`/explore/${noteId}`, origin);
+    explore.searchParams.set("xsec_token", token);
+    explore.searchParams.set("xsec_source", source);
+    return explore.href;
+  } catch {
+    return null;
+  }
+}
+
+
+function parseListHtml(html, url) {
+  const root = _deps.parseHtml(html);
+  const origin = getOrigin(url);
+  const feed = root.querySelector("#userPostedFeeds");
+  if (!feed) return [];
+  const sections = feed.querySelectorAll("section[data-v-79abd645][data-index]");
+  const items = [];
+  for (const section of sections) {
+    const profileWithToken = section.querySelector('a[href*="xsec_token="]');
+    const profileHref = profileWithToken?.getAttribute("href")?.trim();
+    let link;
+    if (profileHref && profileHref.includes("/user/profile/")) {
+      const withXsec = buildExploreLinkWithXsec(profileHref, origin);
+      if (withXsec) link = withXsec;
+      else link = new URL(profileHref.replace(/&/g, "&"), origin).href;
+    } else {
+      const linkEl = section.querySelector('a[href^="/explore/"]');
+      const href = linkEl?.getAttribute("href")?.trim();
+      if (!href) continue;
+      link = new URL(href, origin).href;
+    }
+    const titleEl = section.querySelector("span[data-v-51ec0135]");
+    const title = (titleEl?.textContent ?? "").trim() || "笔记";
+    const authorEl = section.querySelector('a[aria-current="page"] span');
+    const author = (authorEl?.textContent ?? "").trim() || undefined;
+    items.push({
+      guid: _deps.createHash("sha256").update(link).digest("hex"),
+      title,
+      link,
+      pubDate: new Date(),
+      author,
+      summary: title,
+    });
+  }
+  return items;
+}
+
+
+function descToMarkdown(descEl) {
+  if (!descEl) return "";
+  const noteText = descEl.querySelector(".note-text");
+  if (!noteText) {
+    return (descEl.textContent ?? "").trim();
+  }
+  const parts = [];
+  for (const node of noteText.childNodes) {
+    if (node.nodeType === 3) {
+      const text = (node.textContent ?? "").trim();
+      if (text) parts.push(text);
+    } else if (node.nodeType === 1) {
+      const el = node;
+      const tagName = el.tagName?.toLowerCase();
+      if (tagName === "img") {
+        const alt = el.getAttribute("alt") || "";
+        if (alt) parts.push(alt);
+      } else if (tagName === "a" && el.classList?.contains("tag")) {
+        const txt = (el.textContent ?? "").trim();
+        if (txt) parts.push(txt);
+      } else {
+        const txt = (el.textContent ?? "").trim();
+        if (txt) parts.push(txt);
+      }
+    }
+  }
+  let result = parts.join(" ").replace(/\s+/g, " ").trim();
+  if (!result) result = (descEl.textContent ?? "").trim();
+  if (!result && descEl.parentNode) result = (descEl.parentNode.textContent ?? "").trim();
+  return result;
+}
+
+
+function extractUrl(val) {
+  if (!val) return null;
+  const decoded = val.replace(/"/g, '"').replace(/&/g, "&");
+  const m = decoded.match(/url\s*\(\s*["']?([^"')]+)["']?\s*\)/);
+  if (m) {
+    let url = m[1].trim();
+    url = url.replace(/^["']|["']$/g, "");
+    return url || null;
+  }
+  return null;
+}
+
+
+function collectNoteImages(root) {
+  const urls = [];
+  const seen = new Set();
+  const add = (url) => {
+    const u = (url || "").trim();
+    if (u && !seen.has(u) && (u.startsWith("http") || u.startsWith("//"))) {
+      seen.add(u);
+      urls.push(u.startsWith("//") ? "https:" + u : u);
+    }
+  };
+  const imgs = root.querySelectorAll(".img-container img, .note-slider-img img, .note-slider img, .xhs-webplayer img, .note-content img, [class*='note-detail'] img, .media-container img, .video-player-media img");
+  for (const el of imgs) {
+    const src = el.getAttribute("src") || el.getAttribute("data-src") || el.getAttribute("data-lazy-src");
+    if (src) add(src);
+  }
+  const posterSelectors = ["xg-poster", "[class*='xgplayer-poster']", ".player-container [style*='background-image']", ".render-ssr-image [style*='background-image']", "[class*='player-container'] [style*='background-image']", ".video-player-media [style*='background-image']", ".media-container [style*='background-image']"];
+  for (const sel of posterSelectors) {
+    const els = root.querySelectorAll(sel);
+    for (const el of els) {
+      const style = el.getAttribute("style");
+      const url = extractUrl(style ?? "");
+      if (url) add(url);
+    }
+  }
+  const anyBg = root.querySelectorAll("[style*='background-image']");
+  for (const el of anyBg) {
+    const url = extractUrl(el.getAttribute("style") ?? "");
+    if (url && (url.includes("xhscdn") || url.includes("sns-webpic") || url.includes("sns-avatar"))) add(url);
+  }
+  return urls;
+}
+
+
+function parseNoteDate(dateEl) {
+  const text = (dateEl?.textContent ?? "").trim();
+  if (!text) return undefined;
+  const now = new Date();
+  const published = text.match(/发布于\s*(\d{4})-(\d{1,2})-(\d{1,2})/);
+  if (published) {
+    const [, y, m, d] = published;
+    return new Date(`${y}-${m.padStart(2, "0")}-${d.padStart(2, "0")}T12:00:00.000Z`);
+  }
+  const edited = text.match(/编辑于\s*(\d{1,2})-(\d{1,2})/);
+  if (edited) {
+    const [, m, d] = edited;
+    let year = now.getFullYear();
+    const month = parseInt(m, 10);
+    const day = parseInt(d, 10);
+    const built = new Date(year, month - 1, day);
+    if (built > now) year -= 1;
+    return new Date(`${year}-${String(month).padStart(2, "0")}-${String(day).padStart(2, "0")}T12:00:00.000Z`);
+  }
+  const relativeMatch = text.match(/(编辑于|发布于)\s*(\d+)\s*(分钟|小时|天|周|个月)前/);
+  if (relativeMatch) {
+    const [, , amount, unit] = relativeMatch;
+    const num = parseInt(amount, 10);
+    const msMap = { 分钟: 60_000, 小时: 3_600_000, 天: 86_400_000, 周: 604_800_000, 个月: 2_592_000_000 };
+    return new Date(now.getTime() - (num * (msMap[unit] ?? 0)));
+  }
+  return undefined;
+}
+
+
+function extractDetailHtml(html) {
+  const root = _deps.parseHtml(html);
+  // 作者
+  let authorEl = null;
+  const authorSelectors = [
+    ".author .info a.name .username",
+    ".info a.name .username",
+    ".info .username",
+    "a.name .username",
+    ".author-container .username",
+    ".author .username",
+  ];
+  for (const sel of authorSelectors) {
+    authorEl = root.querySelector(sel);
+    if (authorEl) break;
+  }
+  if (!authorEl) {
+    const containers = root.querySelectorAll(".author, .author-container, .interaction-container, .info");
+    for (const c of containers) {
+      const u = c.querySelector("a.name .username") ?? c.querySelector(".username");
+      if (u) { authorEl = u; break; }
+    }
+  }
+  if (!authorEl) {
+    for (const u of root.querySelectorAll(".username")) {
+      let p = u.parentNode ?? null;
+      for (let i = 0; i < 5 && p; i++) {
+        const cls = p.getAttribute?.("class") || "";
+        if (typeof cls === "string" && (cls.includes("name") || cls.includes("info") || cls.includes("author"))) {
+          authorEl = u; break;
+        }
+        p = p.parentNode ?? null;
+      }
+      if (authorEl) break;
+    }
+  }
+  const author = (authorEl?.textContent ?? "").trim() || undefined;
+  // 标题
+  const titleEl = root.querySelector("#detail-title") ?? root.querySelector(".note-content .title") ?? root.querySelector("h1.title");
+  const title = (titleEl?.textContent ?? "").trim() || undefined;
+  // 正文
+  const descEl = root.querySelector("#detail-desc") ?? root.querySelector(".note-content .desc") ?? root.querySelector(".desc");
+  const descText = descToMarkdown(descEl);
+  const imgUrls = collectNoteImages(root);
+  const imgMd = imgUrls.length > 0 ? imgUrls.map((u) => `\n\n![](${u})`).join("") : "";
+  let content = (descText + imgMd).trim() || title || imgMd.trim() || undefined;
+  // 发布时间
+  let dateEl = root.querySelector(".bottom-container span.date") ?? root.querySelector(".bottom-container .date");
+  if (!dateEl) {
+    for (const span of root.querySelectorAll("span")) {
+      if (/(编辑于|发布于)/.test(span.textContent ?? "")) { dateEl = span; break; }
+    }
+  }
+  const pubDate = parseNoteDate(dateEl);
+  return { author, title, content, pubDate };
+}
+
+
+async function fetchItems(sourceId, ctx) {
+  _deps = ctx.deps;
+  const { html, finalUrl } = await ctx.fetchHtml(sourceId);
+  return parseListHtml(html, finalUrl);
+}
+
+
+async function enrichItem(item, ctx) {
+  const { html } = await ctx.fetchHtml(item.link);
+  const detail = extractDetailHtml(html);
+  return {
+    ...item,
+    author: detail.author ?? item.author,
+    title: detail.title ?? item.title,
+    content: detail.content ? `<p>${detail.content.replace(/\n\n/g, "</p><p>")}</p>` : undefined,
+    pubDate: detail.pubDate ?? item.pubDate,
+  };
+}
+
+
+async function checkAuth(page, _url) {
+  try {
+    const loginButton = await page.$(".reds-button-new.login-btn.large.primary");
+    return loginButton == null;
+  } catch {
+    return false;
+  }
+}
+
+
+export default {
+  id: "xiaohongshu",
+  listUrlPattern: "https://xiaohongshu.com/user/profile/{userId}",
+  fetchItems,
+  enrichItem,
+  checkAuth,
+  loginUrl: "https://www.xiaohongshu.com/",
+  domain: "xiaohongshu.com",
+  loginTimeoutMs: 30 * 1000,
+  pollIntervalMs: 2000,
+};

package/app/plugins/builtin/zhipu-research.rssany.js

@@ -0,0 +1,334 @@
+let _deps;
+
+// 智谱研究页插件：仅抓取列表，不做正文 enrich（兼容净化后的 HTML）
+
+
+
+const ZHIPU_RESEARCH_URL = "https://www.zhipuai.cn/zh/research";
+const ZHIPU_ORIGIN = "https://www.zhipuai.cn";
+const DATE_RE = /^\d{4}[/-]\d{1,2}[/-]\d{1,2}$/;
+const RESEARCH_TAGS = new Set(["多模态", "语言模型", "基座模型", "推理模型", "Agent", "代码模型"]);
+
+
+function normalizeText(text) {
+  return (text ?? "").replace(/\s+/g, " ").trim();
+}
+
+
+function hashGuid(input) {
+  return _deps.createHash("sha256").update(input).digest("hex");
+}
+
+
+function parseDate(dateText) {
+  if (!dateText) return undefined;
+  const normalized = normalizeText(dateText);
+  const m = normalized.match(/(\d{4})[/-](\d{1,2})[/-](\d{1,2})/);
+  if (!m) return undefined;
+  const [, y, mm, dd] = m;
+  return new Date(`${y}-${mm.padStart(2, "0")}-${dd.padStart(2, "0")}T00:00:00.000Z`);
+}
+
+
+function findArrayEnd(raw, startIndex) {
+  let inString = false;
+  let escaped = false;
+  let depth = 0;
+  for (let i = startIndex; i < raw.length; i += 1) {
+    const ch = raw[i];
+    if (inString) {
+      if (escaped) {
+        escaped = false;
+        continue;
+      }
+      if (ch === "\\") {
+        escaped = true;
+        continue;
+      }
+      if (ch === "\"") {
+        inString = false;
+      }
+      continue;
+    }
+    if (ch === "\"") {
+      inString = true;
+      continue;
+    }
+    if (ch === "[") depth += 1;
+    if (ch === "]") {
+      depth -= 1;
+      if (depth === 0) return i;
+    }
+  }
+  return -1;
+}
+
+
+function extractBlogsItems(html) {
+  const normalized = html.replace(/\\"/g, "\"").replace(/\\\\/g, "\\");
+  const marker = "\"blogsItems\":";
+  const markerIndex = normalized.indexOf(marker);
+  if (markerIndex < 0) return [];
+  const arrayStart = normalized.indexOf("[", markerIndex + marker.length);
+  if (arrayStart < 0) return [];
+  const arrayEnd = findArrayEnd(normalized, arrayStart);
+  if (arrayEnd < 0) return [];
+  const arrayRaw = normalized.slice(arrayStart, arrayEnd + 1);
+  try {
+    const parsed = JSON.parse(arrayRaw);
+    return Array.isArray(parsed) ? parsed : [];
+  } catch {
+    return [];
+  }
+}
+
+
+async function fetchRawHtml(url) {
+  try {
+    const res = await fetch(url, {
+      redirect: "follow",
+      headers: {
+        "User-Agent":
+          "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
+        "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
+      },
+    });
+    if (!res.ok) return undefined;
+    return await res.text();
+  } catch {
+    return undefined;
+  }
+}
+
+
+function buildItemsFromBlogsItems(blogsItems) {
+  const items = [];
+  for (const blog of blogsItems) {
+    if (typeof blog !== "object" || blog == null) continue;
+    const id = String(blog.id ?? "").trim();
+    if (!id) continue;
+    const title = normalizeText(blog.title_zh ?? blog.title_en ?? "");
+    if (!title) continue;
+    const link = `${ZHIPU_ORIGIN}/zh/research/${id}`;
+    const summary = normalizeText(blog.resume_zh ?? blog.resume_en ?? "");
+    const createdAt = String(blog.createAt ?? "").trim();
+    const pubDate = createdAt ? new Date(createdAt) : new Date();
+    const category = normalizeText(blog.tag_zh ?? blog.tag_en ?? "");
+    items.push({
+      guid: hashGuid(link),
+      title,
+      link,
+      pubDate: Number.isNaN(pubDate.getTime()) ? new Date() : pubDate,
+      summary: summary || undefined,
+    });
+  }
+  return items;
+}
+
+
+function buildTitleIdMap(blogsItems) {
+  const map = new Map();
+  for (const blog of blogsItems) {
+    if (typeof blog !== "object" || blog == null) continue;
+    const id = String(blog.id ?? "").trim();
+    const title = normalizeText(blog.title_zh ?? blog.title_en ?? "");
+    if (!id || !title) continue;
+    map.set(title, id);
+  }
+  return map;
+}
+
+
+function isDateText(text) {
+  return DATE_RE.test(normalizeText(text));
+}
+
+
+function uniqueTexts(texts) {
+  const out = [];
+  const seen = new Set();
+  for (const t of texts) {
+    if (!t || seen.has(t)) continue;
+    seen.add(t);
+    out.push(t);
+  }
+  return out;
+}
+
+
+function getLeafTexts(node) {
+  const leafs = node
+    .querySelectorAll("div, span, p, h1, h2, h3")
+    .filter((el) => el.querySelector("div, span, p, h1, h2, h3") == null)
+    .map((el) => normalizeText(el.textContent))
+    .filter(Boolean)
+    .filter((t) => !t.includes("没有更多"))
+    .filter((t) => !t.includes("加载更多"));
+  return uniqueTexts(leafs);
+}
+
+
+function countDateNodes(node) {
+  return node
+    .querySelectorAll("p")
+    .map((p) => normalizeText(p.textContent))
+    .filter((t) => isDateText(t))
+    .length;
+}
+
+
+function findCardRootFromDateNode(dateNode) {
+  let current = dateNode?.parentNode ?? null;
+  let candidate = null;
+  for (let i = 0; i < 8 && current; i += 1) {
+    if (current.nodeType !== _deps.NodeType.ELEMENT_NODE) {
+      current = current.parentNode ?? null;
+      continue;
+    }
+    const dateCount = countDateNodes(current);
+    if (dateCount === 1) {
+      const leafs = getLeafTexts(current);
+      const hasTitleCandidate = leafs.some((t) => !isDateText(t) && !RESEARCH_TAGS.has(t) && t.length >= 6);
+      if (hasTitleCandidate) candidate = current;
+    }
+    if (dateCount > 1) break;
+    current = current.parentNode ?? null;
+  }
+  return candidate;
+}
+
+
+function parseCardItem(card, dateText, titleIdMap) {
+  const leafs = getLeafTexts(card);
+  const category = leafs.find((t) => RESEARCH_TAGS.has(t));
+  const textCandidates = leafs
+    .filter((t) => !isDateText(t))
+    .filter((t) => !RESEARCH_TAGS.has(t))
+    .filter((t) => t !== "时间排序" && t !== "研究");
+  if (textCandidates.length === 0) return null;
+  const title = textCandidates[0];
+  const summary = textCandidates[1];
+  const id = titleIdMap.get(title);
+  const link = id
+    ? `${ZHIPU_ORIGIN}/zh/research/${id}`
+    : `${ZHIPU_RESEARCH_URL}#${encodeURIComponent(title)}`;
+  const pubDate = parseDate(dateText) ?? new Date();
+  return {
+    guid: hashGuid(id ? link : `${title}|${normalizeText(dateText)}`),
+    title,
+    link,
+    pubDate,
+    summary: summary || undefined,
+    categories: category ? [category] : undefined,
+  };
+}
+
+
+function buildItemsFromDom(html, titleIdMap) {
+  const root = _deps.parseHtml(html);
+  const dateNodes = root.querySelectorAll("p")
+    .map((p) => ({ node: p, dateText: normalizeText(p.textContent) }))
+    .filter((x) => isDateText(x.dateText));
+  const seen = new Set();
+  const items = [];
+  for (const { node, dateText } of dateNodes) {
+    const card = findCardRootFromDateNode(node);
+    if (!card) continue;
+    const parsed = parseCardItem(card, dateText, titleIdMap);
+    if (!parsed) continue;
+    const key = `${parsed.title}|${parsed.pubDate.toISOString()}`;
+    if (seen.has(key)) continue;
+    seen.add(key);
+    items.push(parsed);
+  }
+  return items;
+}
+
+
+function buildItemsFromLeafSequence(html, titleIdMap) {
+  const root = _deps.parseHtml(html);
+  const leafTexts = uniqueTexts(
+    root
+      .querySelectorAll("div, span, p, h1, h2, h3")
+      .filter((el) => el.querySelector("div, span, p, h1, h2, h3") == null)
+      .map((el) => normalizeText(el.textContent))
+      .filter(Boolean)
+      .filter((t) => !t.includes("没有更多"))
+      .filter((t) => !t.includes("加载更多"))
+      .filter((t) => t !== "研究" && t !== "时间排序")
+  );
+
+  const items = [];
+  const seen = new Set();
+  for (let i = 0; i < leafTexts.length; i += 1) {
+    const dateText = leafTexts[i];
+    if (!isDateText(dateText)) continue;
+    const category = i > 0 && RESEARCH_TAGS.has(leafTexts[i - 1]) ? leafTexts[i - 1] : undefined;
+
+    let title = "";
+    let summary;
+    for (let j = i + 1; j < leafTexts.length; j += 1) {
+      const t = leafTexts[j];
+      if (isDateText(t)) break;
+      if (RESEARCH_TAGS.has(t)) continue;
+      if (!title) {
+        title = t;
+        continue;
+      }
+      summary = t;
+      break;
+    }
+    if (!title || title.length < 4) continue;
+
+    const id = titleIdMap.get(title);
+    const link = id
+      ? `${ZHIPU_ORIGIN}/zh/research/${id}`
+      : `${ZHIPU_RESEARCH_URL}#${encodeURIComponent(title)}`;
+    const pubDate = parseDate(dateText) ?? new Date();
+    const key = `${title}|${pubDate.toISOString()}`;
+    if (seen.has(key)) continue;
+    seen.add(key);
+    items.push({
+      guid: hashGuid(id ? link : `${title}|${normalizeText(dateText)}`),
+      title,
+      link,
+      pubDate,
+      summary: summary || undefined,
+    });
+  }
+  return items;
+}
+
+
+async function fetchItems(sourceId, ctx) {
+  _deps = ctx.deps;
+  // 需要读取页面脚本里的 blogsItems（包含详情 id），因此这里禁用净化。
+  const { html, finalUrl } = await ctx.fetchHtml(sourceId, { waitMs: 5000, purify: false });
+
+  let blogsItems = extractBlogsItems(html);
+  if (blogsItems.length === 0) {
+    const rawHtml = await fetchRawHtml(finalUrl || sourceId);
+    if (rawHtml) {
+      blogsItems = extractBlogsItems(rawHtml);
+    }
+  }
+
+  const parsedFromScript = buildItemsFromBlogsItems(blogsItems);
+  if (parsedFromScript.length > 0) return parsedFromScript;
+
+  const titleIdMap = buildTitleIdMap(blogsItems);
+  const parsedFromDom = buildItemsFromDom(html, titleIdMap);
+  if (parsedFromDom.length > 0) return parsedFromDom;
+
+  const parsedFromLeafs = buildItemsFromLeafSequence(html, titleIdMap);
+  if (parsedFromLeafs.length > 0) return parsedFromLeafs;
+
+  throw new Error("[zhipu-research] 未解析到研究条目，页面结构可能已变化");
+}
+
+
+export default {
+  id: "zhipu-research",
+  listUrlPattern: ZHIPU_RESEARCH_URL,
+  fetchItems,
+};