rssany 0.1.2 → 0.1.5

package/app/plugins/builtin/bytedance-seed-research.rssany.js

@@ -0,0 +1,231 @@
+let _deps;
+
+// ByteDance Seed 研究页插件：抓取研究论文与动态条目（不含 enrich）
+
+
+
+const DEFAULT_ORIGIN = "https://seed.bytedance.com";
+const ROUTER_DATA_MARKER = "window._ROUTER_DATA = ";
+const RESEARCH_PAGE_KEY = "(locale$)/research/page";
+
+
+function normalizeText(text) {
+  return (text ?? "").replace(/\s+/g, " ").trim();
+}
+
+
+function hashGuid(input) {
+  return _deps.createHash("sha256").update(input).digest("hex");
+}
+
+
+function toOrigin(url) {
+  try {
+    return new URL(url).origin;
+  } catch {
+    return DEFAULT_ORIGIN;
+  }
+}
+
+
+function detectLocale(url) {
+  try {
+    const path = new URL(url).pathname.toLowerCase();
+    if (path.startsWith("/en/")) return "en";
+  } catch {
+    // ignore
+  }
+  return "zh";
+}
+
+
+function pickLocalized(zhValue, enValue, locale) {
+  const zh = normalizeText(zhValue);
+  const en = normalizeText(enValue);
+  if (locale === "zh") return zh || en;
+  return en || zh;
+}
+
+
+function parsePublishDate(raw) {
+  const num = Number(raw);
+  if (Number.isFinite(num) && num > 0) {
+    const date = new Date(num);
+    if (!Number.isNaN(date.getTime())) return date;
+  }
+
+  const direct = new Date(String(raw ?? ""));
+  if (!Number.isNaN(direct.getTime())) return direct;
+  return new Date();
+}
+
+
+function extractBalancedJson(raw, marker) {
+  const markerIndex = raw.indexOf(marker);
+  if (markerIndex < 0) return undefined;
+
+  const start = raw.indexOf("{", markerIndex + marker.length);
+  if (start < 0) return undefined;
+
+  let inString = false;
+  let escaped = false;
+  let depth = 0;
+
+  for (let i = start; i < raw.length; i += 1) {
+    const ch = raw[i];
+    if (inString) {
+      if (escaped) {
+        escaped = false;
+        continue;
+      }
+      if (ch === "\\") {
+        escaped = true;
+        continue;
+      }
+      if (ch === "\"") inString = false;
+      continue;
+    }
+
+    if (ch === "\"") {
+      inString = true;
+      continue;
+    }
+    if (ch === "{") depth += 1;
+    if (ch === "}") {
+      depth -= 1;
+      if (depth === 0) {
+        return raw.slice(start, i + 1);
+      }
+    }
+  }
+
+  return undefined;
+}
+
+
+function parseRouterData(html) {
+  const raw = extractBalancedJson(html, ROUTER_DATA_MARKER);
+  if (!raw) return undefined;
+  try {
+    return JSON.parse(raw);
+  } catch {
+    return undefined;
+  }
+}
+
+
+function parseModernData(html) {
+  const m = html.match(/<script[^>]*id="__MODERN_DATA__"[^>]*>([\s\S]*?)<\/script>/i);
+  if (!m) return undefined;
+  try {
+    return JSON.parse(m[1]);
+  } catch {
+    return undefined;
+  }
+}
+
+
+function getResearchPayload(html) {
+  const routerData = parseRouterData(html);
+  const fromRouter = routerData?.loaderData?.[RESEARCH_PAGE_KEY];
+  if (fromRouter && typeof fromRouter === "object") return fromRouter;
+
+  const modernData = parseModernData(html);
+  if (modernData?.data && typeof modernData.data === "object") return modernData.data;
+  if (modernData && typeof modernData === "object") return modernData;
+
+  return undefined;
+}
+
+
+function buildItem(entry, kind, locale, origin) {
+  const meta = entry?.ArticleMeta ?? {};
+  const zh = entry?.ArticleSubContentZh ?? {};
+  const en = entry?.ArticleSubContentEn ?? {};
+
+  const title = pickLocalized(zh.Title ?? zh.ShortTitle, en.Title ?? en.ShortTitle, locale);
+  if (!title) return null;
+
+  const abstract = pickLocalized(zh.Abstract, en.Abstract, locale);
+  const titleKey = pickLocalized(zh.TitleKey, en.TitleKey, locale);
+  const articleId = String(meta.ArticleID ?? meta.ID ?? "").trim();
+
+  let link;
+  if (titleKey) {
+    const path = kind === "public_papers"
+      ? `/${locale}/public_papers/${encodeURIComponent(titleKey)}`
+      : `/${locale}/blog/${encodeURIComponent(titleKey)}`;
+    link = new URL(path, origin).href;
+  } else {
+    const fallback = articleId || title;
+    link = `${origin}/${locale}/research#${encodeURIComponent(fallback)}`;
+  }
+
+  const author = normalizeText(meta.Author);
+  const pubDate = parsePublishDate(meta.PublishDate);
+
+  return {
+    guid: hashGuid(`${kind}|${articleId || titleKey || title}`),
+    title,
+    link,
+    pubDate,
+    author: author || undefined,
+    summary: abstract || undefined,
+  };
+}
+
+
+function dedupeAndSort(items) {
+  const out = [];
+  const seen = new Set();
+  for (const item of items) {
+    const key = item.link || item.guid;
+    if (seen.has(key)) continue;
+    seen.add(key);
+    out.push(item);
+  }
+  out.sort((a, b) => b.pubDate.getTime() - a.pubDate.getTime());
+  return out;
+}
+
+
+async function fetchItems(sourceId, ctx) {
+  _deps = ctx.deps;
+  // 该站点条目核心数据在脚本 JSON 中，需关闭 purify 才能读取。
+  const { html, finalUrl } = await ctx.fetchHtml(sourceId, { waitMs: 4500, purify: false });
+  const pageUrl = finalUrl || sourceId;
+  const locale = detectLocale(pageUrl);
+  const origin = toOrigin(pageUrl);
+  const payload = getResearchPayload(html);
+
+  if (!payload) {
+    throw new Error("[bytedance-seed-research] 未找到研究页数据块（window._ROUTER_DATA / __MODERN_DATA__）");
+  }
+
+  const articleList = Array.isArray(payload.article_list) ? payload.article_list : [];
+  const feedList = Array.isArray(payload.feedList) ? payload.feedList : [];
+
+  const parsed = [];
+  for (const entry of articleList) {
+    const item = buildItem(entry, "public_papers", locale, origin);
+    if (item) parsed.push(item);
+  }
+  for (const entry of feedList) {
+    const item = buildItem(entry, "blog", locale, origin);
+    if (item) parsed.push(item);
+  }
+
+  const items = dedupeAndSort(parsed);
+  if (items.length === 0) {
+    throw new Error("[bytedance-seed-research] 研究页数据存在，但未解析到有效条目");
+  }
+
+  return items;
+}
+
+
+export default {
+  id: "bytedance-seed-research",
+  listUrlPattern: /^https?:\/\/seed\.bytedance\.com\/(zh|en)\/research(?:\/)?(\?.*)?$/i,
+  fetchItems,
+};

package/app/plugins/builtin/five-radar.rssany.js

@@ -0,0 +1,490 @@
+let _deps;
+
+
+
+const SITE_ID = "five-radar";
+const DEFAULT_PAGE_SIZE = 30;
+const THEME_PAGE_SIZE = 20;
+const THEME_PAGE_SIZE_CANDIDATES = [20, 10, 5];
+const DEFAULT_MAX_PAGES = 3;
+const MAX_PAGES_LIMIT = 10;
+const THEME_PAGE_RE = /^\/([A-Za-z0-9_-]+)\/news\/?$/;
+const ALL_TAB_KEYS = [
+  "dataresourceregistration",
+  "dataresources",
+  "publicdatadevelop",
+  "dataproducts",
+  "dplists",
+  "dataproperty",
+  "dataintellectualproperty",
+  "internationaldatamarket",
+  "dataassets",
+  "dataopensource",
+];
+const KNOWN_TITLE_EN = new Set([
+  "classicproject",
+  "dataassets",
+  "dataassetvalue",
+  "dataintellectualproperty",
+  "dataopensource",
+  "dataproducts",
+  "dataproperty",
+  "dataresourceregistration",
+  "dataresources",
+  "dplists",
+  "internationaldatamarket",
+  "latestbusiness",
+  "latesttech",
+  "latesttrend",
+  "publicdatadevelop",
+  "salon",
+  "techforum",
+]);
+const TAB_ALIASES = new Map([
+  ["公共数据", "dataresourceregistration"],
+  ["公共数据资源登记", "dataresourceregistration"],
+  ["资源入表", "dataresources"],
+  ["数据资源入表", "dataresources"],
+  ["授权运营", "publicdatadevelop"],
+  ["公共数据授权运营", "publicdatadevelop"],
+  ["产品交易", "dataproducts"],
+  ["数据产品交易", "dataproducts"],
+  ["产品上架", "dplists"],
+  ["数据产品上架", "dplists"],
+  ["推荐", "recommend"],
+  ["产权登记", "dataproperty"],
+  ["数据产权登记", "dataproperty"],
+  ["知识产权", "dataintellectualproperty"],
+  ["数据知识产权登记", "dataintellectualproperty"],
+  ["商业市场", "internationaldatamarket"],
+  ["商业数据市场", "internationaldatamarket"],
+  ["资产融资", "dataassets"],
+  ["数据资产融资", "dataassets"],
+  ["开源市场", "dataopensource"],
+  ["开源数据市场", "dataopensource"],
+  ["全部", "all"],
+  ["所有目录", "all"],
+  ["all", "all"],
+  ["alltabs", "all"],
+  ["all-tabs", "all"],
+  ["最新", "latest"],
+  ["latest", "latest"],
+  ["new", "latest"],
+]);
+
+
+function normalizeText(text) {
+  return (text ?? "").replace(/\s+/g, " ").trim();
+}
+
+
+function hashGuid(input) {
+  return _deps.createHash("sha256").update(input).digest("hex");
+}
+
+
+function parsePublishedAt(input) {
+  const text = normalizeText(input);
+  if (!text) return new Date();
+
+  const m = text.match(
+    /^(\d{4})-(\d{1,2})-(\d{1,2})(?:[ T](\d{1,2}):(\d{1,2})(?::(\d{1,2}))?)?$/
+  );
+  if (m) {
+    const [, y, mm, dd, hh = "0", mi = "0", ss = "0"] = m;
+    const iso =
+      `${y}-${mm.padStart(2, "0")}-${dd.padStart(2, "0")}T` +
+      `${hh.padStart(2, "0")}:${mi.padStart(2, "0")}:${ss.padStart(2, "0")}+08:00`;
+    const parsed = new Date(iso);
+    if (!Number.isNaN(parsed.getTime())) return parsed;
+  }
+
+  const fallback = new Date(text);
+  return Number.isNaN(fallback.getTime()) ? new Date() : fallback;
+}
+
+
+function normalizeThemeToken(input) {
+  return normalizeText(input).toLowerCase();
+}
+
+
+function isChineseToken(input) {
+  return /[\u3400-\u9fff]/.test(input);
+}
+
+
+function decodeUrlToken(input) {
+  const text = normalizeText(input);
+  if (!text) return "";
+  try {
+    return decodeURIComponent(text);
+  } catch {
+    return text;
+  }
+}
+
+
+function resolveMaxPages(sourceUrl) {
+  const hashToken = normalizeText(sourceUrl.hash.replace(/^#/, ""));
+  const hashParams = new URLSearchParams(hashToken.startsWith("?") ? hashToken : `?${hashToken}`);
+
+  const raw =
+    sourceUrl.searchParams.get("pages") ||
+    sourceUrl.searchParams.get("maxPages") ||
+    hashParams.get("pages") ||
+    hashParams.get("maxPages");
+  const n = Number.parseInt(normalizeText(raw), 10);
+  if (Number.isNaN(n)) return DEFAULT_MAX_PAGES;
+  return Math.min(MAX_PAGES_LIMIT, Math.max(1, n));
+}
+
+
+function getThemeTokenFromHash(hash) {
+  const raw = decodeUrlToken(hash.replace(/^#/, ""));
+  if (!raw) return "";
+
+  const maybeQuery = raw.startsWith("?") ? raw : `?${raw}`;
+  const params = new URLSearchParams(maybeQuery);
+  const queryToken =
+    params.get("theme") ||
+    params.get("title_en") ||
+    params.get("tab") ||
+    params.get("category");
+  if (normalizeText(queryToken)) return normalizeText(queryToken);
+
+  const fallbackToken = normalizeText(raw.split("&")[0] ?? "");
+  return fallbackToken;
+}
+
+
+function getThemeTokenFromUrl(sourceUrl) {
+  const queryToken =
+    sourceUrl.searchParams.get("theme") ||
+    sourceUrl.searchParams.get("title_en") ||
+    sourceUrl.searchParams.get("tab") ||
+    sourceUrl.searchParams.get("category");
+  if (normalizeText(queryToken)) return normalizeText(queryToken);
+
+  const hashToken = getThemeTokenFromHash(sourceUrl.hash || "");
+  if (hashToken) return hashToken;
+
+  const m = sourceUrl.pathname.match(THEME_PAGE_RE);
+  return m?.[1] ? normalizeText(m[1]) : "";
+}
+
+
+function resolveThemeOptions(sourceUrl) {
+  const tokenFromUrl = getThemeTokenFromUrl(sourceUrl);
+  const tokenRaw = decodeUrlToken(tokenFromUrl);
+  if (!tokenRaw) {
+    return { mode: "all_tabs", label: "全部目录" };
+  }
+
+  const token = normalizeThemeToken(tokenRaw);
+  const alias = TAB_ALIASES.get(tokenRaw) ?? TAB_ALIASES.get(normalizeText(tokenRaw));
+  const resolved = alias ?? token;
+
+  if (resolved === "recommend") {
+    return {
+      mode: "theme",
+      titleEn: "",
+      isRecommend: 1,
+      label: "推荐",
+    };
+  }
+
+  if (resolved === "all") {
+    return {
+      mode: "all_tabs",
+      label: "全部目录",
+    };
+  }
+
+  if (resolved === "latest") {
+    return {
+      mode: "latest",
+      label: "latest",
+    };
+  }
+
+  if (KNOWN_TITLE_EN.has(resolved)) {
+    return {
+      mode: "theme",
+      titleEn: resolved,
+      isRecommend: 0,
+      label: tokenRaw,
+    };
+  }
+
+  if (isChineseToken(tokenRaw)) {
+    throw new Error(
+      `[${SITE_ID}] 不支持的目录: ${tokenRaw}。` +
+      "可用中文目录示例：公共数据、资源入表、授权运营、产品交易、产品上架、推荐、产权登记、知识产权、商业市场、资产融资、开源市场"
+    );
+  }
+
+  return {
+    mode: "theme",
+    titleEn: resolved,
+    isRecommend: 0,
+    label: tokenRaw,
+  };
+}
+
+
+function toAbsoluteLink(rawUrl, origin, row) {
+  const text = normalizeText(rawUrl);
+  if (text) {
+    try {
+      const url = new URL(text, origin);
+      if (/^https?:$/i.test(url.protocol) && !isHomepageLink(url)) return url.href;
+    } catch {
+      // ignore malformed url from upstream API
+    }
+  }
+
+  const idText = String(row?.id ?? "").trim();
+  if (idText) {
+    const detailUrl = new URL("/detail", origin);
+    detailUrl.searchParams.set("id", idText);
+    return detailUrl.href;
+  }
+  return null;
+}
+
+
+function isHomepageLink(url) {
+  return url.pathname === "/" && !url.search && !url.hash;
+}
+
+
+function mapRowToFeedItem(row, origin) {
+  const title = normalizeText(row?.title);
+  const link = toAbsoluteLink(row?.url, origin, row);
+  if (!title || !link) return null;
+
+  const summary = normalizeText(row?.summary);
+  const author = normalizeText(row?.source || row?.author);
+  const category = normalizeText(row?.theme?.title);
+  const idText = String(row?.id ?? "").trim();
+
+  return {
+    guid: idText ? `${SITE_ID}-${idText}` : hashGuid(link),
+    title,
+    link,
+    pubDate: parsePublishedAt(row?.published_at),
+    author: author || undefined,
+    summary: summary || undefined,
+  };
+}
+
+
+async function fetchNewsRows(origin, pageSize = DEFAULT_PAGE_SIZE) {
+  const apiUrl = new URL("/api/news/list", origin);
+  apiUrl.searchParams.set("page", "1");
+  apiUrl.searchParams.set("page_size", String(pageSize));
+  apiUrl.searchParams.set("key", "new");
+
+  const res = await fetch(apiUrl, {
+    headers: {
+      Accept: "application/json,text/plain,*/*",
+      "User-Agent":
+        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
+    },
+  });
+  if (!res.ok) {
+    throw new Error(`[${SITE_ID}] 拉取新闻接口失败: HTTP ${res.status}`);
+  }
+
+  let payload;
+  try {
+    payload = await res.json();
+  } catch {
+    throw new Error(`[${SITE_ID}] 新闻接口返回非 JSON 数据`);
+  }
+
+  const rows = Array.isArray(payload?.data) ? payload.data : [];
+  return rows;
+}
+
+
+async function fetchThemeRowsPage(origin, opts, page, pageSize = THEME_PAGE_SIZE) {
+  const apiUrl = new URL("/api/themes/news-list", origin);
+  apiUrl.searchParams.set("page", String(page));
+  apiUrl.searchParams.set("page_size", String(pageSize));
+  apiUrl.searchParams.set("title_en", opts.titleEn ?? "");
+  apiUrl.searchParams.set("is_recommend", String(opts.isRecommend ?? 0));
+
+  const res = await fetch(apiUrl, {
+    headers: {
+      Accept: "application/json,text/plain,*/*",
+      "User-Agent":
+        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
+    },
+  });
+  if (!res.ok) {
+    throw new Error(
+      `[${SITE_ID}] 拉取目录接口失败: HTTP ${res.status} (title_en=${opts.titleEn || "recommend"})`
+    );
+  }
+
+  let payload;
+  try {
+    payload = await res.json();
+  } catch {
+    throw new Error(`[${SITE_ID}] 目录接口返回非 JSON 数据`);
+  }
+
+  if (payload?.status === 422 || payload?.success === false) {
+    const message = normalizeText(payload?.error?.message || payload?.message || "unknown error");
+    const err = new Error(
+      `[${SITE_ID}] 目录接口返回错误 (title_en=${opts.titleEn || "recommend"}): ${message}`
+    );
+    if (isPageSizeValidationError(message)) err.name = "PageSizeValidationError";
+    throw err;
+  }
+
+  const rows = Array.isArray(payload?.data) ? payload.data : [];
+  const currentPage = Number.parseInt(String(payload?.pagination?.current_page ?? page), 10) || page;
+  const totalPages = Number.parseInt(String(payload?.pagination?.total_pages ?? currentPage), 10) || currentPage;
+
+  return { rows, currentPage, totalPages };
+}
+
+
+async function fetchThemeRowsPaged(origin, opts, pageSize = THEME_PAGE_SIZE, maxPages = DEFAULT_MAX_PAGES) {
+  const candidates = dedupeValidPageSizes([pageSize, ...THEME_PAGE_SIZE_CANDIDATES]);
+  let lastError;
+
+  for (const candidate of candidates) {
+    try {
+      return await fetchThemeRowsPagedWithPageSize(origin, opts, candidate, maxPages);
+    } catch (err) {
+      lastError = err;
+      if (!isPageSizeValidationErr(err)) throw err;
+    }
+  }
+
+  throw lastError ?? new Error(`[${SITE_ID}] 目录抓取失败: 未知错误`);
+}
+
+
+function dedupeValidPageSizes(sizes) {
+  const out = [];
+  for (const size of sizes) {
+    if (!Number.isInteger(size) || size <= 0) continue;
+    if (!out.includes(size)) out.push(size);
+  }
+  return out.length > 0 ? out : [THEME_PAGE_SIZE];
+}
+
+
+function isPageSizeValidationError(message) {
+  const text = normalizeText(message).toLowerCase();
+  return text.includes("page_size") && text.includes("validation.max.numeric");
+}
+
+
+function isPageSizeValidationErr(err) {
+  return err instanceof Error && err.name === "PageSizeValidationError";
+}
+
+
+async function fetchThemeRowsPagedWithPageSize(
+  origin,
+  opts,
+  pageSize = THEME_PAGE_SIZE,
+  maxPages = DEFAULT_MAX_PAGES
+) {
+  const out = [];
+  const limit = Math.min(MAX_PAGES_LIMIT, Math.max(1, maxPages));
+
+  for (let page = 1; page <= limit; page += 1) {
+    const { rows, currentPage, totalPages } = await fetchThemeRowsPage(origin, opts, page, pageSize);
+    out.push(...rows);
+    if (rows.length === 0 || currentPage >= totalPages) break;
+  }
+
+  return out;
+}
+
+
+async function fetchAllThemeRows(origin, pageSize = THEME_PAGE_SIZE, maxPages = DEFAULT_MAX_PAGES) {
+  const settled = await Promise.allSettled(
+    ALL_TAB_KEYS.map((titleEn) =>
+      fetchThemeRowsPaged(origin, { titleEn, isRecommend: 0, label: titleEn }, pageSize, maxPages)
+    )
+  );
+
+  const rows = [];
+  const failedKeys = [];
+  for (let i = 0; i < settled.length; i += 1) {
+    const result = settled[i];
+    const key = ALL_TAB_KEYS[i];
+    if (result.status === "fulfilled") {
+      rows.push(...result.value);
+    } else {
+      failedKeys.push(`${key}: ${result.reason instanceof Error ? result.reason.message : String(result.reason)}`);
+    }
+  }
+
+  if (rows.length === 0) {
+    throw new Error(
+      `[${SITE_ID}] 全目录抓取失败：` +
+      (failedKeys.length > 0 ? failedKeys.join(" | ") : "没有可用数据")
+    );
+  }
+  return rows;
+}
+
+
+async function fetchItems(sourceId, _ctx) {
+  _deps = _ctx.deps;
+  let sourceUrl;
+  try {
+    sourceUrl = new URL(sourceId);
+  } catch {
+    throw new Error(`[${SITE_ID}] 无效 URL: ${sourceId}`);
+  }
+  if (!/^https?:$/i.test(sourceUrl.protocol)) {
+    throw new Error(`[${SITE_ID}] 仅支持 http/https URL`);
+  }
+
+  const themeOptions = resolveThemeOptions(sourceUrl);
+  const maxPages = resolveMaxPages(sourceUrl);
+  let rows;
+  if (themeOptions.mode === "theme") {
+    rows = await fetchThemeRowsPaged(sourceUrl.origin, themeOptions, THEME_PAGE_SIZE, maxPages);
+  } else if (themeOptions.mode === "all_tabs") {
+    rows = await fetchAllThemeRows(sourceUrl.origin, THEME_PAGE_SIZE, maxPages);
+  } else {
+    rows = await fetchNewsRows(sourceUrl.origin);
+  }
+  const items = [];
+  const seenLinks = new Set();
+
+  for (const row of rows) {
+    const item = mapRowToFeedItem(row, sourceUrl.origin);
+    if (!item || seenLinks.has(item.link)) continue;
+    seenLinks.add(item.link);
+    items.push(item);
+  }
+
+  items.sort((a, b) => b.pubDate.getTime() - a.pubDate.getTime());
+  if (items.length === 0) {
+    if (themeOptions.mode === "theme") {
+      throw new Error(`[${SITE_ID}] 目录 ${themeOptions.label} 未解析到条目`);
+    }
+    throw new Error(`[${SITE_ID}] 未解析到条目，接口结构可能已变化`);
+  }
+  return items;
+}
+
+
+export default {
+  id: SITE_ID,
+  refreshInterval: "10min",
+  listUrlPattern: /^https?:\/\/(?:www\.)?5radar\.com(?:\/[A-Za-z0-9_-]+\/news)?\/?(?:[?#].*)?$/i,
+  fetchItems,
+};