webforai 2.1.1 → 4.0.0

package/dist/cjs/adapters/index.js

@@ -0,0 +1,86 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var adapters_exports = {};
+__export(adapters_exports, {
+  BUILTIN_ADAPTERS: () => BUILTIN_ADAPTERS,
+  DECLARATIVE_ADAPTERS: () => import_declarative2.DECLARATIVE_ADAPTERS,
+  adapterById: () => adapterById,
+  buildPageSignature: () => import_registry2.buildPageSignature,
+  defineSelectorAdapter: () => import_registry2.defineSelectorAdapter,
+  hackernewsAdapter: () => import_hackernews2.hackernewsAdapter,
+  removeAll: () => import_registry2.removeAll,
+  resolveAdapter: () => import_registry2.resolveAdapter,
+  runAdapters: () => runAdapters,
+  selectFirst: () => import_registry2.selectFirst,
+  youtubeAdapter: () => import_youtube2.youtubeAdapter,
+  youtubeCaptionTracks: () => import_youtube2.youtubeCaptionTracks
+});
+module.exports = __toCommonJS(adapters_exports);
+var import_sanitize = require("../extractors/lib/sanitize");
+var import_hast_fast = require("../utils/hast-fast");
+var import_registry = require("./registry");
+var import_declarative = require("./sites/declarative");
+var import_hackernews = require("./sites/hackernews");
+var import_youtube = require("./sites/youtube");
+var import_registry2 = require("./registry");
+var import_declarative2 = require("./sites/declarative");
+var import_hackernews2 = require("./sites/hackernews");
+var import_youtube2 = require("./sites/youtube");
+const BUILTIN_ADAPTERS = [import_youtube.youtubeAdapter, import_hackernews.hackernewsAdapter, ...import_declarative.DECLARATIVE_ADAPTERS];
+const adapterById = (id) => BUILTIN_ADAPTERS.find((adapter) => adapter.id === id);
+const MIN_ADAPTER_TEXT = 40;
+const runAdapters = (context, adapters = BUILTIN_ADAPTERS) => {
+  let matched = (0, import_registry.resolveAdapterByHost)(context, adapters);
+  let withSignature = context;
+  if (!matched) {
+    withSignature = { ...context, signature: context.signature ?? (0, import_registry.buildPageSignature)(context.hast) };
+    matched = (0, import_registry.resolveAdapterByFingerprint)(withSignature, adapters);
+  }
+  const adapter = matched;
+  if (!adapter) {
+    return void 0;
+  }
+  const target = withSignature.owned ? withSignature : { ...withSignature, hast: (0, import_hast_fast.cloneHast)(withSignature.hast), owned: true };
+  let extracted;
+  try {
+    extracted = adapter.extract(target);
+  } catch {
+    return void 0;
+  }
+  if (!extracted) {
+    return void 0;
+  }
+  (0, import_sanitize.stripNonContent)(extracted);
+  return new import_hast_fast.MetricsCollector().textLength(extracted) >= MIN_ADAPTER_TEXT ? extracted : void 0;
+};
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  BUILTIN_ADAPTERS,
+  DECLARATIVE_ADAPTERS,
+  adapterById,
+  buildPageSignature,
+  defineSelectorAdapter,
+  hackernewsAdapter,
+  removeAll,
+  resolveAdapter,
+  runAdapters,
+  selectFirst,
+  youtubeAdapter,
+  youtubeCaptionTracks
+});

package/dist/cjs/adapters/registry.js

@@ -0,0 +1,178 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var registry_exports = {};
+__export(registry_exports, {
+  buildPageSignature: () => buildPageSignature,
+  defineSelectorAdapter: () => defineSelectorAdapter,
+  removeAll: () => removeAll,
+  resolveAdapter: () => resolveAdapter,
+  resolveAdapterByFingerprint: () => resolveAdapterByFingerprint,
+  resolveAdapterByHost: () => resolveAdapterByHost,
+  selectFirst: () => selectFirst
+});
+module.exports = __toCommonJS(registry_exports);
+var import_hast_util_select = require("hast-util-select");
+var import_hast_fast = require("../utils/hast-fast");
+const EMPTY_SIGNATURE = {
+  classes: /* @__PURE__ */ new Set(),
+  ids: /* @__PURE__ */ new Set(),
+  tags: /* @__PURE__ */ new Set(),
+  generator: "",
+  meta: ""
+};
+const buildPageSignature = (root) => {
+  const classes = /* @__PURE__ */ new Set();
+  const ids = /* @__PURE__ */ new Set();
+  const tags = /* @__PURE__ */ new Set();
+  const generatorParts = [];
+  const metaParts = [];
+  (0, import_hast_fast.walk)(root, (node) => {
+    if (!(0, import_hast_fast.isElement)(node)) {
+      return;
+    }
+    tags.add(node.tagName);
+    for (const name of (0, import_hast_fast.classList)(node)) {
+      classes.add(name);
+    }
+    const id = (0, import_hast_fast.stringProperty)(node, "id");
+    if (id) {
+      ids.add(id);
+    }
+    if (node.tagName === "meta") {
+      const key = (0, import_hast_fast.stringProperty)(node, "name") ?? (0, import_hast_fast.stringProperty)(node, "property") ?? "";
+      const content = (0, import_hast_fast.stringProperty)(node, "content") ?? "";
+      metaParts.push(`${key}=${content}`);
+      if (key.toLowerCase() === "generator") {
+        generatorParts.push(content);
+      }
+    }
+  });
+  return {
+    classes,
+    ids,
+    tags,
+    generator: generatorParts.join(" ").toLowerCase(),
+    meta: metaParts.join("\n").toLowerCase()
+  };
+};
+const DEFAULT_MIN_LENGTH = 40;
+const hostOf = (url) => {
+  if (!url) {
+    return void 0;
+  }
+  try {
+    return new URL(url).hostname;
+  } catch {
+    return void 0;
+  }
+};
+const defineSelectorAdapter = (spec) => ({
+  id: spec.id,
+  matches: (context) => {
+    const host = hostOf(context.url);
+    return Boolean(host && spec.hosts?.test(host));
+  },
+  matchesFingerprint: (context) => {
+    const signature = context.signature ?? EMPTY_SIGNATURE;
+    return spec.fingerprint?.(signature) ?? false;
+  },
+  extract: (context) => {
+    for (const selector of spec.content) {
+      const content = tryContentSelector(selector, spec, context);
+      if (content) {
+        return content;
+      }
+    }
+    return void 0;
+  }
+});
+const tryContentSelector = (selector, spec, context) => {
+  const found = (0, import_hast_util_select.selectAll)(selector, context.hast);
+  if (found.length === 0) {
+    return void 0;
+  }
+  for (const element of found) {
+    removeAll(element, spec.remove);
+  }
+  const total = found.reduce((sum, element) => sum + new import_hast_fast.MetricsCollector().textLength(element), 0);
+  if (total < (spec.minLength ?? DEFAULT_MIN_LENGTH)) {
+    return void 0;
+  }
+  const decorated = spec.decorate?.(found, context);
+  if (decorated) {
+    return decorated;
+  }
+  return found.length === 1 ? found[0] : { type: "root", children: found };
+};
+const removeAll = (root, selectors) => {
+  if (!selectors || selectors.length === 0) {
+    return;
+  }
+  const doomed = /* @__PURE__ */ new Set();
+  for (const selector of selectors) {
+    for (const element of (0, import_hast_util_select.selectAll)(selector, root)) {
+      doomed.add(element);
+    }
+  }
+  if (doomed.size > 0) {
+    (0, import_hast_fast.pruneInPlace)(root, (node) => !((0, import_hast_fast.isElement)(node) && doomed.has(node)));
+  }
+};
+const selectFirst = (root, selectors) => {
+  for (const selector of selectors) {
+    const found = (0, import_hast_util_select.select)(selector, root);
+    if (found) {
+      return found;
+    }
+  }
+  return void 0;
+};
+const resolveAdapterByHost = (context, adapters) => {
+  for (const adapter of adapters) {
+    try {
+      if (adapter.matches(context)) {
+        return adapter;
+      }
+    } catch {
+    }
+  }
+  return void 0;
+};
+const resolveAdapterByFingerprint = (context, adapters) => {
+  for (const adapter of adapters) {
+    try {
+      if (adapter.matchesFingerprint?.(context)) {
+        return adapter;
+      }
+    } catch {
+    }
+  }
+  return void 0;
+};
+const resolveAdapter = (context, adapters) => resolveAdapterByHost(context, adapters) ?? resolveAdapterByFingerprint(context, adapters);
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  buildPageSignature,
+  defineSelectorAdapter,
+  removeAll,
+  resolveAdapter,
+  resolveAdapterByFingerprint,
+  resolveAdapterByHost,
+  selectFirst
+});

package/dist/cjs/adapters/sites/declarative.js

@@ -0,0 +1,202 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var declarative_exports = {};
+__export(declarative_exports, {
+  DECLARATIVE_ADAPTERS: () => DECLARATIVE_ADAPTERS,
+  declarativeAdapterById: () => declarativeAdapterById
+});
+module.exports = __toCommonJS(declarative_exports);
+var import_registry = require("../registry");
+const COMMON_NOISE = [
+  ".sr-only",
+  "[aria-hidden='true'][class*='icon']",
+  "button",
+  ".social-share",
+  ".share-buttons",
+  ".newsletter-signup"
+];
+const SPECS = [
+  {
+    id: "github",
+    hosts: /(^|\.)github\.com$/,
+    // Issues and PRs expose a testid; repository landing pages render the README instead.
+    content: ["[data-testid='issue-body']", "article.markdown-body", ".markdown-body"],
+    remove: [".js-comment-edit-history", ".Details-content--hidden", "clipboard-copy", ...COMMON_NOISE],
+    minLength: 30
+  },
+  {
+    id: "stackoverflow",
+    hosts: /(^|\.)(stackoverflow\.com|stackexchange\.com|superuser\.com|serverfault\.com|askubuntu\.com)$/,
+    content: ["#mainbar", ".question, .answer"],
+    remove: [
+      ".js-post-menu",
+      ".post-menu",
+      ".user-info",
+      ".votecell",
+      ".comments-link",
+      ".js-add-link",
+      ...COMMON_NOISE
+    ],
+    minLength: 100
+  },
+  {
+    id: "npm",
+    hosts: /(^|\.)npmjs\.com$/,
+    content: ["#readme", "#package-tab-readme", "[class*='markdown']"],
+    remove: COMMON_NOISE,
+    minLength: 50
+  },
+  {
+    id: "zenn",
+    hosts: /(^|\.)zenn\.dev$/,
+    content: [".znc"],
+    remove: COMMON_NOISE,
+    minLength: 50
+  },
+  {
+    id: "qiita",
+    hosts: /(^|\.)qiita\.com$/,
+    content: [".it-MdContent", "[class*='mdContent']", "article"],
+    remove: [".code-copy", ...COMMON_NOISE],
+    minLength: 50
+  },
+  {
+    id: "medium",
+    hosts: /(^|\.)medium\.com$/,
+    // Medium also powers custom domains; the fingerprint catches those.
+    fingerprint: (signature) => signature.meta.includes("al:android:app_name=medium"),
+    content: ["article section", "article", "[data-testid='storyContent']"],
+    remove: ["[data-testid='audioPlayButton']", "[role='separator']", ...COMMON_NOISE],
+    minLength: 200
+  },
+  {
+    id: "substack",
+    hosts: /(^|\.)substack\.com$/,
+    // Substack's custom domains are far more common than *.substack.com.
+    fingerprint: (signature) => signature.classes.has("available-content"),
+    content: [".available-content", ".body.markup", ".post-content"],
+    remove: [".subscription-widget-wrap", ".button-wrapper", ".paywall", ...COMMON_NOISE],
+    minLength: 200
+  },
+  {
+    id: "note",
+    hosts: /(^|\.)note\.com$/,
+    content: [".note-common-styles__textnote-body", "[class*='textnote-body']"],
+    remove: COMMON_NOISE,
+    minLength: 50
+  },
+  {
+    id: "hatena",
+    hosts: /(^|\.)(hatenablog\.com|hatenadiary\.(com|jp)|hateblo\.jp|hatenastaff\.com)$/,
+    fingerprint: (signature) => signature.classes.has("hatenablog-entry") || signature.classes.has("hatena-body"),
+    content: [".entry-content"],
+    remove: [".entry-footer", ".social-buttons", ".hatena-asin-detail", ...COMMON_NOISE],
+    minLength: 100
+  },
+  {
+    id: "wordpress",
+    // Deliberately fingerprint-only: WordPress powers a large share of the web on its own
+    // domains, so keying off a hostname would cover almost none of it.
+    fingerprint: (signature) => signature.generator.includes("wordpress") || signature.classes.has("wp-block-post-content") || signature.classes.has("wp-singular"),
+    content: [".wp-block-post-content", ".entry-content", ".post-content", "article .content"],
+    remove: [".sharedaddy", ".jp-relatedposts", ".wp-block-comments", "#comments", ...COMMON_NOISE],
+    minLength: 150
+  },
+  {
+    id: "wikipedia",
+    hosts: /(^|\.)(wikipedia\.org|wikimedia\.org|wikibooks\.org|wiktionary\.org|wikisource\.org)$/,
+    fingerprint: (signature) => signature.classes.has("mw-parser-output"),
+    content: [".mw-parser-output", "#mw-content-text"],
+    remove: [
+      ".mw-editsection",
+      ".navbox",
+      ".navbox-inner",
+      ".vertical-navbox",
+      ".sidebar",
+      ".side-box",
+      ".metadata",
+      ".reflist",
+      ".refbegin",
+      ".sistersitebox",
+      ".ambox",
+      ".hatnote",
+      ".shortdescription",
+      ".mw-jump-link",
+      ".printfooter",
+      ".catlinks",
+      "#toc",
+      ".toc",
+      "table.mbox-small",
+      ...COMMON_NOISE
+    ],
+    minLength: 200
+  },
+  {
+    id: "reddit",
+    hosts: /(^|\.)reddit\.com$/,
+    content: ["shreddit-post [slot='text-body']", "[data-test-id='post-content']", ".Post"],
+    remove: ["faceplate-tracker[noun='comment_action_bar']", "shreddit-comment-action-row", ...COMMON_NOISE],
+    // Reddit renders most of a thread into shadow roots that never reach the serialised HTML,
+    // so a weak match here is worse than generic extraction. Decline unless a substantial
+    // post body is actually present in the document.
+    minLength: 600
+  },
+  {
+    id: "docusaurus",
+    fingerprint: (signature) => signature.classes.has("theme-doc-markdown") || signature.classes.has("docusaurus-mt-lg"),
+    content: [".theme-doc-markdown", "article .markdown"],
+    remove: [".theme-doc-toc-mobile", ".pagination-nav", ".theme-doc-version-banner", ...COMMON_NOISE],
+    minLength: 150
+  },
+  {
+    id: "vitepress",
+    fingerprint: (signature) => signature.classes.has("vp-doc") || signature.classes.has("VPDoc"),
+    content: [".vp-doc > div", ".vp-doc", ".VPDoc .content-container"],
+    remove: [".VPDocFooter", ".vp-doc-footer", ".header-anchor", ...COMMON_NOISE],
+    minLength: 150
+  },
+  {
+    id: "mkdocs",
+    fingerprint: (signature) => signature.classes.has("md-content__inner"),
+    content: [".md-content__inner", "[data-md-component='content']"],
+    remove: [".md-source-file", ".md-feedback", ".headerlink", ...COMMON_NOISE],
+    minLength: 150
+  },
+  {
+    id: "readthedocs",
+    fingerprint: (signature) => signature.classes.has("rst-content"),
+    content: [".rst-content", "[role='main']"],
+    remove: [".headerlink", ".rst-footer-buttons", ".wy-breadcrumbs", ...COMMON_NOISE],
+    minLength: 150
+  },
+  {
+    id: "gitbook",
+    fingerprint: (signature) => signature.classes.has("gitbook-root"),
+    content: ["[data-testid='page.contentEditor']", "main article"],
+    remove: COMMON_NOISE,
+    minLength: 150
+  }
+];
+const DECLARATIVE_ADAPTERS = SPECS.map(import_registry.defineSelectorAdapter);
+const declarativeAdapterById = (id) => DECLARATIVE_ADAPTERS.find((adapter) => adapter.id === id);
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  DECLARATIVE_ADAPTERS,
+  declarativeAdapterById
+});

package/dist/cjs/adapters/sites/hackernews.js

@@ -0,0 +1,117 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var hackernews_exports = {};
+__export(hackernews_exports, {
+  hackernewsAdapter: () => hackernewsAdapter
+});
+module.exports = __toCommonJS(hackernews_exports);
+var import_hast_util_select = require("hast-util-select");
+var import_hast_fast = require("../../utils/hast-fast");
+const MAX_QUOTE_DEPTH = 6;
+const text = (value) => ({ type: "text", value });
+const element = (tagName, children) => ({
+  type: "element",
+  tagName,
+  properties: {},
+  children
+});
+const commentDepth = (row) => {
+  const cell = (0, import_hast_util_select.select)("td.ind", row);
+  if (!cell) {
+    return 0;
+  }
+  const indent = (0, import_hast_fast.numericProperty)(cell, "indent");
+  if (indent !== void 0) {
+    return indent;
+  }
+  const spacer = (0, import_hast_util_select.select)("img", cell);
+  const width = spacer ? (0, import_hast_fast.numericProperty)(spacer, "width") : void 0;
+  return width === void 0 ? 0 : Math.round(width / 40);
+};
+const nest = (content, depth) => {
+  let current = content;
+  for (let level = 0; level < Math.min(depth, MAX_QUOTE_DEPTH); level++) {
+    current = [element("blockquote", current)];
+  }
+  return current[0];
+};
+const storyNodes = (root) => {
+  const nodes = [];
+  const titleLink = (0, import_hast_util_select.select)(".titleline > a", root);
+  if (titleLink) {
+    nodes.push(element("h1", [...titleLink.children]));
+    const href = (0, import_hast_fast.stringProperty)(titleLink, "href");
+    if (href && /^https?:/.test(href)) {
+      nodes.push(element("p", [{ ...titleLink, children: [text(href)] }]));
+    }
+  }
+  const topText = (0, import_hast_util_select.select)(".toptext", root);
+  if (topText) {
+    nodes.push(element("div", [...topText.children]));
+  }
+  return nodes;
+};
+const commentNodes = (root) => {
+  const nodes = [];
+  for (const row of (0, import_hast_util_select.selectAll)("tr.comtr, tr.athing.comtr", root)) {
+    const body = (0, import_hast_util_select.select)(".commtext", row);
+    if (!body) {
+      continue;
+    }
+    const author = (0, import_hast_util_select.select)(".hnuser", row);
+    const header = author ? [element("strong", [...author.children])] : [];
+    const content = [
+      ...header.length > 0 ? [element("p", header)] : [],
+      element("div", [...body.children])
+    ];
+    nodes.push(nest(content, commentDepth(row)));
+  }
+  return nodes;
+};
+const hackernewsAdapter = {
+  id: "hackernews",
+  matches: ({ url }) => {
+    if (!url) {
+      return false;
+    }
+    try {
+      return /(^|\.)ycombinator\.com$/.test(new URL(url).hostname);
+    } catch {
+      return false;
+    }
+  },
+  matchesFingerprint: ({ signature }) => Boolean(signature?.ids.has("hnmain") || signature?.classes.has("fatitem")),
+  extract: ({ hast }) => {
+    const story = storyNodes(hast);
+    const comments = commentNodes(hast);
+    if (story.length === 0 && comments.length === 0) {
+      return void 0;
+    }
+    const children = [...story];
+    if (comments.length > 0) {
+      children.push(element("h2", [text("Comments")]));
+      children.push(...comments);
+    }
+    return { type: "root", children };
+  }
+};
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  hackernewsAdapter
+});