webforai 2.1.0 → 3.0.0

package/dist/html-to-mdast.js

@@ -8,18 +8,26 @@ import { customCodeHandler } from "./mdast-handlers/custom-code-handler.js";
 import { customDivHandler } from "./mdast-handlers/custom-div-handler.js";
 import { customImgHandler } from "./mdast-handlers/custom-img-handler.js";
 import { customTableHandler } from "./mdast-handlers/custom-table-handler.js";
+import { definitionListHandler } from "./mdast-handlers/definition-list-handler.js";
 import { mathHandler } from "./mdast-handlers/math-handler.js";
+import { supSubHandler } from "./mdast-handlers/sup-sub-handler.js";
+import { normalizeHast } from "./normalize/index.js";
 import { getLangFromHast, getLangFromStr, getUrlFromHast } from "./utils/hast-utils.js";
 var htmlToMdast = (htmlOrHast, options) => {
   const { extractors, url: defaultUrl, lang: defaultLang } = options || {};
   const [lang, hast] = typeof htmlOrHast === "string" ? [defaultLang || getLangFromStr(htmlOrHast), fromHtml(htmlOrHast, { fragment: true })] : [defaultLang || getLangFromHast(htmlOrHast), htmlOrHast];
+  const isOwned = typeof htmlOrHast === "string" || (options?.owned ?? false);
   const url = defaultUrl || getUrlFromHast(hast);
-  const extractedHast = pipeExtractors({ hast, lang, url }, extractors);
+  const extractedHast = pipeExtractors({ hast, lang, url, owned: isOwned }, extractors);
+  normalizeHast(extractedHast, options?.normalize);
   const mdast = toMdast(extractedHast, {
     handlers: {
       math: mathHandler,
       div: customDivHandler,
       pre: customCodeHandler,
+      dl: definitionListHandler,
+      sup: supSubHandler("sup"),
+      sub: supSubHandler("sub"),
       a: customAHandler({ asText: options?.linkAsText }),
       img: customImgHandler({ hideImage: options?.hideImage }),
       table: customTableHandler({ asText: options?.tableAsText })

package/dist/index.js

@@ -1,17 +1,48 @@
 // src/index.ts
-import { htmlToMarkdown } from "./html-to-markdown.js";
+import {
+  htmlToMarkdown,
+  htmlToMarkdownWithMetadata
+} from "./html-to-markdown.js";
 import { mdastSplitter } from "./md-splitter.js";
 import { htmlToMdast } from "./html-to-mdast.js";
 import { mdastToMarkdown } from "./mdast-to-markdown.js";
 import {
   pipeExtractors,
-  takumiExtractor
+  autoExtractor,
+  createAutoExtractor,
+  takumiExtractor,
+  minimalFilter
 } from "./extractors/index.js";
+import {
+  BUILTIN_ADAPTERS,
+  adapterById,
+  buildPageSignature,
+  defineSelectorAdapter,
+  resolveAdapter,
+  runAdapters,
+  youtubeCaptionTracks
+} from "./adapters/index.js";
+import { extractMetadata, toFrontmatter } from "./metadata/index.js";
+import { normalizeHast } from "./normalize/index.js";
 export {
+  BUILTIN_ADAPTERS,
+  adapterById,
+  autoExtractor,
+  buildPageSignature,
+  createAutoExtractor,
+  defineSelectorAdapter,
+  extractMetadata,
   htmlToMarkdown,
+  htmlToMarkdownWithMetadata,
   htmlToMdast,
   mdastSplitter,
   mdastToMarkdown,
+  minimalFilter,
+  normalizeHast,
   pipeExtractors,
-  takumiExtractor
+  resolveAdapter,
+  runAdapters,
+  takumiExtractor,
+  toFrontmatter,
+  youtubeCaptionTracks
 };

package/dist/mdast-handlers/custom-a-handler.js

@@ -1,21 +1,37 @@
 // src/mdast-handlers/custom-a-handler.ts
 import { defaultHandlers } from "hast-util-to-mdast";
 import { toString as hastToString } from "hast-util-to-string";
+import { stringProperty } from "../utils/hast-fast.js";
+var ALLOWED_SCHEMES = /* @__PURE__ */ new Set(["http:", "https:", "mailto:", "tel:", "ftp:", "ftps:"]);
+var isFollowableHref = (href) => {
+  const trimmed = href?.trim();
+  if (!trimmed) {
+    return false;
+  }
+  const scheme = trimmed.match(/^[a-z][a-z0-9+.-]*:/i)?.[0];
+  if (!scheme) {
+    return true;
+  }
+  return ALLOWED_SCHEMES.has(scheme.toLowerCase());
+};
 var customAHandler = (options) => (state, node) => {
-  if (options?.asText) {
+  const asPlainText = () => {
     const text = hastToString(node);
-    if (3 >= text.length) {
+    if (text.length === 0) {
       return void 0;
     }
-    const link2 = { type: "text", value: text };
-    state.patch(node, link2);
-    return link2;
+    const result = { type: "text", value: text };
+    state.patch(node, result);
+    return result;
+  };
+  if (options?.asText) {
+    return hastToString(node).length <= 3 ? void 0 : asPlainText();
   }
-  const link = defaultHandlers.a(state, node);
-  if (link.children.length > 0) {
-    return link;
+  if (!isFollowableHref(stringProperty(node, "href"))) {
+    return asPlainText();
   }
-  return void 0;
+  const link = defaultHandlers.a(state, node);
+  return link.children.length > 0 ? link : void 0;
 };
 export {
   customAHandler

package/dist/mdast-handlers/custom-img-handler.js

@@ -1,10 +1,32 @@
 // src/mdast-handlers/custom-img-handler.ts
 import { defaultHandlers } from "hast-util-to-mdast";
+import { stringProperty } from "../utils/hast-fast.js";
+var MAX_DATA_URL_LENGTH = 256;
+var hasNoUsableSource = (node) => {
+  const src = stringProperty(node, "src")?.trim();
+  if (!src) {
+    return true;
+  }
+  if (src.startsWith("data:")) {
+    return src.length > MAX_DATA_URL_LENGTH;
+  }
+  return false;
+};
 var customImgHandler = (options) => (state, node) => {
   if (options?.hideImage) {
     return void 0;
   }
-  return defaultHandlers.image(state, node);
+  if (hasNoUsableSource(node)) {
+    return void 0;
+  }
+  const image = defaultHandlers.image(state, node);
+  if (!image.alt) {
+    const title = stringProperty(node, "title");
+    if (title) {
+      image.alt = title;
+    }
+  }
+  return image;
 };
 export {
   customImgHandler

package/dist/mdast-handlers/custom-table-handler.js

@@ -1,12 +1,28 @@
 // src/mdast-handlers/custom-table-handler.ts
 import { defaultHandlers } from "hast-util-to-mdast";
 import { toText } from "hast-util-to-text";
+import { isElement, numericProperty, walk } from "../utils/hast-fast.js";
+var MAX_SPAN = 1e3;
+var clampSpans = (node) => {
+  walk(node, (candidate) => {
+    if (!isElement(candidate) || candidate.tagName !== "td" && candidate.tagName !== "th") {
+      return;
+    }
+    for (const attribute of ["colSpan", "rowSpan"]) {
+      const span = numericProperty(candidate, attribute);
+      if (span !== void 0 && span > MAX_SPAN) {
+        candidate.properties[attribute] = MAX_SPAN;
+      }
+    }
+  });
+};
 var customTableHandler = (options) => (state, node) => {
   if (options?.asText) {
     const paragraph = { type: "paragraph", children: [{ type: "text", value: toText(node) }] };
     state.patch(node, paragraph);
     return paragraph;
   }
+  clampSpans(node);
   return defaultHandlers.table(state, node);
 };
 export {

package/dist/mdast-handlers/definition-list-handler.js

@@ -0,0 +1,76 @@
+// src/mdast-handlers/definition-list-handler.ts
+var hoistBoundaryWhitespace = (nodes) => {
+  const result = [];
+  for (const node of nodes) {
+    if (node.type !== "emphasis" && node.type !== "strong") {
+      result.push(node);
+      continue;
+    }
+    node.children = hoistBoundaryWhitespace(node.children);
+    const first = node.children[0];
+    if (first?.type === "text" && /^\s/.test(first.value)) {
+      result.push({ type: "text", value: " " });
+      first.value = first.value.replace(/^\s+/, "");
+    }
+    let trailing = false;
+    const last = node.children[node.children.length - 1];
+    if (last?.type === "text" && /\s$/.test(last.value)) {
+      last.value = last.value.replace(/\s+$/, "");
+      trailing = true;
+    }
+    node.children = node.children.filter((child) => !(child.type === "text" && child.value.length === 0));
+    if (node.children.length > 0) {
+      result.push(node);
+    }
+    if (trailing) {
+      result.push({ type: "text", value: " " });
+    }
+  }
+  return result;
+};
+var definitionListHandler = (state, node) => {
+  const items = [];
+  let current;
+  const flush = () => {
+    if (!current) {
+      return;
+    }
+    const heading = {
+      type: "paragraph",
+      children: current.term.length > 0 ? [{ type: "strong", children: current.term }] : []
+    };
+    items.push({ type: "listItem", spread: false, children: [heading, ...current.definitions] });
+    current = void 0;
+  };
+  for (const child of node.children) {
+    if (child.type !== "element") {
+      continue;
+    }
+    const element = child;
+    if (element.tagName === "dt") {
+      if (current && current.definitions.length > 0) {
+        flush();
+      }
+      const phrasing = hoistBoundaryWhitespace(state.all(element));
+      current = current ? { term: [...current.term, { type: "text", value: ", " }, ...phrasing], definitions: [] } : { term: phrasing, definitions: [] };
+      continue;
+    }
+    if (element.tagName === "dd") {
+      if (!current) {
+        current = { term: [], definitions: [] };
+      }
+      const content = state.toFlow(state.all(element));
+      current.definitions.push(...content);
+    }
+  }
+  flush();
+  if (items.length === 0) {
+    return void 0;
+  }
+  const result = { type: "list", ordered: false, spread: false, children: items };
+  state.patch(node, result);
+  return result;
+};
+export {
+  definitionListHandler
+};

package/dist/mdast-handlers/math-handler.js

@@ -1,10 +1,19 @@
 // src/mdast-handlers/math-handler.ts
 import { toHtml } from "hast-util-to-html";
 import { MathMLToLaTeX } from "mathml-to-latex";
+import { stringProperty } from "../utils/hast-fast.js";
+var readAnnotation = (node) => {
+  const alttext = stringProperty(node, "alttext");
+  if (!alttext) {
+    return void 0;
+  }
+  const unwrapped = alttext.trim().replace(/^\{\\displaystyle\s*([\s\S]*)\}$/, "$1").trim();
+  return unwrapped.length > 0 ? unwrapped : void 0;
+};
+var isDisplayMath = (node) => stringProperty(node, "display") === "block";
 var mathHandler = (state, node) => {
-  const mathMl = toHtml(node);
-  const latex = MathMLToLaTeX.convert(mathMl);
-  const result = { type: "inlineMath", value: latex };
+  const latex = readAnnotation(node) ?? MathMLToLaTeX.convert(toHtml(node));
+  const result = isDisplayMath(node) ? { type: "math", value: latex, meta: null } : { type: "inlineMath", value: latex };
   state.patch(node, result);
   return result;
 };

package/dist/mdast-handlers/sup-sub-handler.js

@@ -0,0 +1,16 @@
+// src/mdast-handlers/sup-sub-handler.ts
+import { toString as hastToString } from "hast-util-to-string";
+var MAX_ANNOTATION_LENGTH = 32;
+var supSubHandler = (tagName) => (state, node) => {
+  const inner = hastToString(node).trim();
+  if (inner.length === 0 || inner.length > MAX_ANNOTATION_LENGTH) {
+    return state.all(node);
+  }
+  const escaped = inner.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;");
+  const result = { type: "html", value: `<${tagName}>${escaped}</${tagName}>` };
+  state.patch(node, result);
+  return result;
+};
+export {
+  supSubHandler
+};

package/dist/metadata/index.js

@@ -0,0 +1,198 @@
+// src/metadata/index.ts
+import { collectElements, findElement, isElement, stringProperty, walk } from "../utils/hast-fast.js";
+var readMetaTags = (tree) => {
+  const tags = /* @__PURE__ */ new Map();
+  for (const meta of collectElements(tree, (element) => element.tagName === "meta")) {
+    const key = stringProperty(meta, "property") ?? stringProperty(meta, "name") ?? stringProperty(meta, "itemprop");
+    const content = stringProperty(meta, "content");
+    if (!(key && content)) {
+      continue;
+    }
+    const normalized = key.toLowerCase();
+    if (!tags.has(normalized)) {
+      tags.set(normalized, content);
+    }
+  }
+  return tags;
+};
+var readJsonLd = (tree) => {
+  const entries = [];
+  for (const script of collectElements(tree, (element) => element.tagName === "script")) {
+    const type = stringProperty(script, "type");
+    if (type?.toLowerCase() !== "application/ld+json") {
+      continue;
+    }
+    const source = script.children.map((child) => child.type === "text" ? child.value : "").join("").trim();
+    if (source.length === 0) {
+      continue;
+    }
+    let parsed;
+    try {
+      parsed = JSON.parse(source);
+    } catch {
+      continue;
+    }
+    for (const entry of flattenJsonLd(parsed)) {
+      entries.push(entry);
+    }
+  }
+  return entries;
+};
+var flattenJsonLd = (value) => {
+  if (Array.isArray(value)) {
+    return value.flatMap(flattenJsonLd);
+  }
+  if (typeof value !== "object" || value === null) {
+    return [];
+  }
+  const record = value;
+  const graph = record["@graph"];
+  if (graph) {
+    return flattenJsonLd(graph);
+  }
+  return [record];
+};
+var ARTICLE_TYPES = /* @__PURE__ */ new Set([
+  "article",
+  "newsarticle",
+  "blogposting",
+  "techarticle",
+  "scholarlyarticle",
+  "report",
+  "webpage"
+]);
+var jsonLdType = (entry) => {
+  const type = entry["@type"];
+  if (typeof type === "string") {
+    return [type.toLowerCase()];
+  }
+  if (Array.isArray(type)) {
+    return type.filter((value) => typeof value === "string").map((value) => value.toLowerCase());
+  }
+  return [];
+};
+var readName = (value) => {
+  if (typeof value === "string") {
+    return value;
+  }
+  if (Array.isArray(value)) {
+    for (const entry of value) {
+      const name = readName(entry);
+      if (name) {
+        return name;
+      }
+    }
+    return void 0;
+  }
+  if (typeof value === "object" && value !== null) {
+    const name = value.name;
+    return typeof name === "string" ? name : void 0;
+  }
+  return void 0;
+};
+var asString = (value) => typeof value === "string" && value.length > 0 ? value : void 0;
+var MAX_TITLE_LENGTH = 160;
+var asTitle = (value) => {
+  const cleaned = clean(value);
+  return cleaned && cleaned.length <= MAX_TITLE_LENGTH ? cleaned : void 0;
+};
+var clean = (value) => {
+  if (value === void 0) {
+    return void 0;
+  }
+  const trimmed = value.replace(/\s+/g, " ").trim();
+  return trimmed.length > 0 ? trimmed : void 0;
+};
+var documentTitle = (tree) => {
+  const title = findElement(tree, (element) => element.tagName === "title");
+  if (!title) {
+    return void 0;
+  }
+  return title.children.map((child) => child.type === "text" ? child.value : "").join("");
+};
+var firstHeading = (tree) => {
+  const heading = findElement(tree, (element) => element.tagName === "h1");
+  if (!heading) {
+    return void 0;
+  }
+  let text = "";
+  walk(heading, (node) => {
+    if (node.type === "text") {
+      text += node.value;
+    }
+  });
+  return text;
+};
+var linkHref = (tree, rel) => {
+  const link = findElement(
+    tree,
+    (element) => element.tagName === "link" && (stringProperty(element, "rel") ?? "").toLowerCase().includes(rel)
+  );
+  return link ? stringProperty(link, "href") : void 0;
+};
+var documentLang = (tree) => {
+  const html = findElement(tree, (element) => element.tagName === "html");
+  if (html) {
+    const declared = stringProperty(html, "lang") ?? stringProperty(html, "xml:lang");
+    if (declared) {
+      return declared;
+    }
+  }
+  if (isElement(tree)) {
+    const own = stringProperty(tree, "lang");
+    if (own) {
+      return own;
+    }
+  }
+  const tagged = findElement(tree, (element) => Boolean(stringProperty(element, "lang")));
+  return tagged ? stringProperty(tagged, "lang") : void 0;
+};
+var extractMetadata = (tree) => {
+  const meta = readMetaTags(tree);
+  const jsonLd = readJsonLd(tree);
+  const article = jsonLd.find((entry) => jsonLdType(entry).some((type) => ARTICLE_TYPES.has(type))) ?? jsonLd[0];
+  const get = (...keys) => {
+    for (const key of keys) {
+      const value = meta.get(key);
+      if (value) {
+        return value;
+      }
+    }
+    return void 0;
+  };
+  const title = asTitle(get("og:title", "twitter:title")) ?? asTitle(asString(article?.headline)) ?? asTitle(asString(article?.name)) ?? asTitle(documentTitle(tree)) ?? asTitle(firstHeading(tree));
+  return {
+    title,
+    description: clean(get("og:description", "twitter:description", "description")) ?? clean(asString(article?.description)),
+    author: clean(readName(article?.author)) ?? clean(get("author", "article:author", "twitter:creator")),
+    published: clean(asString(article?.datePublished)) ?? clean(get("article:published_time", "date", "pubdate")),
+    modified: clean(asString(article?.dateModified)) ?? clean(get("article:modified_time", "og:updated_time")),
+    siteName: clean(get("og:site_name", "application-name")) ?? clean(readName(article?.publisher)),
+    canonicalUrl: clean(linkHref(tree, "canonical")) ?? clean(get("og:url", "twitter:url")),
+    lang: clean(documentLang(tree)),
+    image: clean(get("og:image", "twitter:image", "twitter:image:src")),
+    type: clean(get("og:type"))
+  };
+};
+var yamlString = (value) => `"${value.replace(/\\/g, "\\\\").replace(/"/g, '\\"')}"`;
+var toFrontmatter = (metadata) => {
+  const lines = [];
+  for (const [key, value] of Object.entries(metadata)) {
+    if (typeof value === "string" && value.length > 0) {
+      lines.push(`${key}: ${yamlString(value)}`);
+    }
+  }
+  if (lines.length === 0) {
+    return "";
+  }
+  return `---
+${lines.join("\n")}
+---
+
+`;
+};
+export {
+  extractMetadata,
+  readJsonLd,
+  toFrontmatter
+};

package/dist/normalize/index.js

@@ -0,0 +1,149 @@
+// src/normalize/index.ts
+import {
+  classList,
+  isElement,
+  numericProperty,
+  property,
+  pruneInPlace,
+  stringProperty,
+  walk
+} from "../utils/hast-fast.js";
+var LAZY_SRC_ATTRIBUTES = [
+  "data-src",
+  "data-original",
+  "data-lazy-src",
+  "data-original-src",
+  "data-actualsrc",
+  "data-hi-res-src",
+  "data-echo"
+];
+var PLACEHOLDER_SRC = /^$|^data:image\/|^about:blank$|spacer\.gif$|blank\.(gif|png)$/i;
+var readSrcset = (element, attribute) => {
+  const raw = property(element, attribute);
+  if (typeof raw === "string") {
+    return raw;
+  }
+  if (Array.isArray(raw)) {
+    return raw.join(", ");
+  }
+  return void 0;
+};
+var bestFromSrcset = (srcset) => {
+  let best;
+  for (const entry of srcset.split(",")) {
+    const parts = entry.trim().split(/\s+/);
+    const url = parts[0];
+    if (!url) {
+      continue;
+    }
+    const descriptor = parts[1];
+    let weight = 1;
+    if (descriptor?.endsWith("w")) {
+      weight = Number.parseFloat(descriptor) || 1;
+    } else if (descriptor?.endsWith("x")) {
+      weight = (Number.parseFloat(descriptor) || 1) * 1e3;
+    }
+    if (!best || weight > best.weight) {
+      best = { url, weight };
+    }
+  }
+  return best?.url;
+};
+var resolveImageSources = (tree) => {
+  walk(tree, (node) => {
+    if (!isElement(node) || node.tagName !== "img") {
+      return;
+    }
+    const current = stringProperty(node, "src");
+    if (current && !PLACEHOLDER_SRC.test(current)) {
+      return;
+    }
+    for (const attribute of LAZY_SRC_ATTRIBUTES) {
+      const candidate = stringProperty(node, attribute);
+      if (candidate && !PLACEHOLDER_SRC.test(candidate)) {
+        node.properties.src = candidate;
+        return;
+      }
+    }
+    for (const attribute of ["srcSet", "srcset", "data-srcset"]) {
+      const srcset = readSrcset(node, attribute);
+      const best = srcset ? bestFromSrcset(srcset) : void 0;
+      if (best) {
+        node.properties.src = best;
+        return;
+      }
+    }
+  });
+};
+var katexDuplicates = (node) => {
+  const classes = classList(node);
+  if (!(classes.includes("katex") || classes.includes("katex-display"))) {
+    return [];
+  }
+  const hasMathml = node.children.some((child) => isElement(child) && classList(child).includes("katex-mathml"));
+  if (!hasMathml) {
+    return [];
+  }
+  return node.children.filter((child) => isElement(child) && classList(child).includes("katex-html"));
+};
+var mediawikiDuplicates = (node) => {
+  if (!classList(node).includes("mwe-math-element")) {
+    return [];
+  }
+  return node.children.filter(
+    (child) => isElement(child) && classList(child).some((name) => name.startsWith("mwe-math-fallback-image"))
+  );
+};
+var mathjaxDuplicates = (node) => {
+  if (node.tagName !== "mjx-container") {
+    return [];
+  }
+  return node.children.filter((child) => isElement(child) && child.tagName !== "mjx-assistive-mml");
+};
+var dedupeRenderedMath = (tree) => {
+  const doomed = /* @__PURE__ */ new Set();
+  walk(tree, (node) => {
+    if (!isElement(node)) {
+      return;
+    }
+    for (const duplicate of [...katexDuplicates(node), ...mathjaxDuplicates(node), ...mediawikiDuplicates(node)]) {
+      doomed.add(duplicate);
+    }
+  });
+  if (doomed.size > 0) {
+    pruneInPlace(tree, (node) => !(isElement(node) && doomed.has(node)));
+  }
+};
+var promoteAriaHeadings = (tree) => {
+  walk(tree, (node) => {
+    if (!isElement(node) || stringProperty(node, "role") !== "heading") {
+      return;
+    }
+    if (/^h[1-6]$/.test(node.tagName)) {
+      return;
+    }
+    const level = numericProperty(node, "aria-level") ?? 2;
+    const clamped = Math.min(6, Math.max(1, Math.round(level)));
+    node.tagName = `h${clamped}`;
+  });
+};
+var normalizeHast = (tree, options = {}) => {
+  const { images = true, math = true, ariaHeadings = true } = options;
+  if (images) {
+    resolveImageSources(tree);
+  }
+  if (math) {
+    dedupeRenderedMath(tree);
+  }
+  if (ariaHeadings) {
+    promoteAriaHeadings(tree);
+  }
+  return tree;
+};
+export {
+  bestFromSrcset,
+  dedupeRenderedMath,
+  normalizeHast,
+  promoteAriaHeadings,
+  resolveImageSources
+};