webforai 3.0.0 → 4.1.0

package/dist/md-splitter.js

@@ -3,15 +3,13 @@ import { mdastToMarkdown } from "./mdast-to-markdown.js";
 import { chunk } from "./utils/common.js";
 import { internalType, unwarpRoot, warpRoot } from "./utils/mdast-utils.js";
 var PRIORITY_SPLITTERS = ["h1", "h2", "h3", "h4", "h5", "h6", "list", "table", "code"];
-var getSplitterGenerator = function* () {
-  for (const splitter of PRIORITY_SPLITTERS) {
-    yield splitter;
-  }
-};
-var _mdastSplitter = async (contents, checker, splitterGenerator) => {
-  const splitter = splitterGenerator.next().value;
+var _mdastSplitter = async (contents, checker, priority, signal) => {
+  signal?.throwIfAborted();
+  const splitter = PRIORITY_SPLITTERS[priority];
   const markdown = mdastToMarkdown(warpRoot(contents));
-  if (await checker(markdown) || contents.length === 1) {
+  const accepted = await checker(markdown);
+  signal?.throwIfAborted();
+  if (accepted || contents.length === 1) {
     return [contents];
   }
   const chunked = splitter ? contents.reduce((acc, content) => {
@@ -22,11 +20,26 @@ var _mdastSplitter = async (contents, checker, splitterGenerator) => {
     acc[acc.length - 1].push(content);
     return acc;
   }, []) : chunk(contents, Math.ceil(contents.length / 2));
-  const splitting = chunked.map((chunk2) => _mdastSplitter(chunk2, checker, splitterGenerator));
+  const splitting = chunked.map((chunk2) => _mdastSplitter(chunk2, checker, priority + 1, signal));
   return Promise.all(splitting).then((chunks) => chunks.flat());
 };
-var mdastSplitter = (mdast, checker, _options) => {
-  return _mdastSplitter(unwarpRoot(mdast), checker, getSplitterGenerator());
+var mdastSplitter = async (mdast, checker, options) => {
+  const signal = options?.signal;
+  if (!signal) {
+    return _mdastSplitter(unwarpRoot(mdast), checker, 0);
+  }
+  signal.throwIfAborted();
+  let onAbort = () => {
+  };
+  const aborted = new Promise((_, reject) => {
+    onAbort = () => reject(signal.reason);
+    signal.addEventListener("abort", onAbort, { once: true });
+  });
+  try {
+    return await Promise.race([_mdastSplitter(unwarpRoot(mdast), checker, 0, signal), aborted]);
+  } finally {
+    signal.removeEventListener("abort", onAbort);
+  }
 };
 export {
   mdastSplitter

package/dist/mdast-handlers/custom-code-handler.js

@@ -1,20 +1,41 @@
 // src/mdast-handlers/custom-code-handler.ts
-import { toText } from "hast-util-to-text";
 import { trimTrailingLines } from "trim-trailing-lines";
 import { detectLanguage } from "../utils/detect-code-lang.js";
-var LANGUAGE_MATCH_REGEX = [/language-(\w+)/, /highlight-source-(\w+)/, /CodeBlock--language-(\w+)/];
+import { classList, isElement } from "../utils/hast-fast.js";
+import { preText } from "../utils/pre-text.js";
+import { cleanTwoslash } from "./twoslash.js";
+var LANGUAGE_MATCH_REGEX = [/^language-(\S+)$/, /^highlight-source-(\S+)$/, /^CodeBlock--language-(\S+)$/];
+var LANGUAGE_ATTRIBUTES = ["dataLanguage", "dataLang"];
+var attributeLanguage = (element) => {
+  for (const name of LANGUAGE_ATTRIBUTES) {
+    const value = element.properties?.[name];
+    if (typeof value === "string" && /^[\w+#.-]+$/.test(value.trim()) && value.trim() !== "plaintext") {
+      return value.trim();
+    }
+  }
+  return void 0;
+};
+var codeLanguage = (...elements) => elements.flatMap(classList).map((className) => LANGUAGE_MATCH_REGEX.map((regex) => className.match(regex)).find(Boolean)?.[1]).find(Boolean) ?? elements.map(attributeLanguage).find(Boolean);
+var codeMeta = (pre) => {
+  const title = pre.properties?.dataTitle;
+  if (typeof title !== "string") {
+    return null;
+  }
+  const clean = title.replace(/["\r\n]+/g, " ").trim();
+  return clean ? `title="${clean}"` : null;
+};
 var customCodeHandler = (state, node) => {
-  const classNames = node.properties?.className || [];
-  const codeValue = trimTrailingLines(toText(node)).trim();
-  const classLang = classNames.map((className) => {
-    const match = LANGUAGE_MATCH_REGEX.map((regex) => className.match(regex)).find((match2) => match2);
-    return match?.[1];
-  }).find((className) => className);
+  const twoslashLang = cleanTwoslash(node);
+  const code = node.children.find((child) => isElement(child) && child.tagName === "code");
+  const codeValue = trimTrailingLines(preText(node));
+  const classLang = twoslashLang || codeLanguage(...isElement(code) ? [code, node] : [node]);
   const lang = classLang || detectLanguage(codeValue) || null;
-  const result = { type: "code", lang, meta: null, value: codeValue };
+  const result = { type: "code", lang, meta: codeMeta(node), value: codeValue };
   state.patch(node, result);
   return result;
 };
 export {
+  codeLanguage,
+  codeMeta,
   customCodeHandler
 };

package/dist/mdast-handlers/custom-div-handler.js

@@ -2,49 +2,28 @@
 import { select } from "hast-util-select";
 import { defaultHandlers } from "hast-util-to-mdast";
 import { toString as hastToString } from "hast-util-to-string";
-import { toText } from "hast-util-to-text";
 import { trimTrailingLines } from "trim-trailing-lines";
 import { detectLanguage } from "../utils/detect-code-lang.js";
+import { classList, collectElements, isElement } from "../utils/hast-fast.js";
+import { preText } from "../utils/pre-text.js";
+import { codeLanguage, codeMeta } from "./custom-code-handler.js";
 var CODE_BLOCK_REGEX = /highlight-source|language-|codegroup|codeblock|code-block/i;
 var CODE_FILENAME_SELECTORS = "[class*='fileName'],[class*='fileName'],[class*='title'],[class*='Title']";
-var LANGUAGE_MATCH_REGEX = [/language-(\w+)/, /highlight-source-(\w+)/, /CodeBlock--language-(\w+)/];
-var findRecursive = (array, condition, maxDepth = 3) => {
-  if (maxDepth <= 0) {
-    return null;
-  }
-  for (const value of array) {
-    const result = condition(value);
-    if (Array.isArray(result)) {
-      return findRecursive(result, condition, maxDepth - 1);
-    }
-    if (result) {
-      return value;
-    }
-  }
-  return null;
-};
 var customDivHandler = (state, node) => {
-  const classNames = Array.isArray(node.properties.className) ? node.properties.className : [];
-  const codeBlock = findRecursive(node.children, (child) => {
-    if (child.type !== "element") {
-      return false;
-    }
-    if (child.tagName === "pre") {
-      return true;
-    }
-    return child.children.filter((child2) => child2.type === "element");
-  });
-  if (codeBlock && classNames.some((className) => CODE_BLOCK_REGEX.test(className))) {
-    const codeBlockClassNames = codeBlock.type === "element" ? codeBlock.properties.className ?? [] : [];
-    const codeValue = trimTrailingLines(toText(codeBlock)).trim();
+  const classNames = classList(node);
+  if (!classNames.some((className) => CODE_BLOCK_REGEX.test(className))) {
+    return defaultHandlers.div(state, node);
+  }
+  const codeBlocks = collectElements(node, (element) => element.tagName === "pre");
+  const codeBlock = codeBlocks.length === 1 ? codeBlocks[0] : void 0;
+  if (codeBlock) {
+    const codeValue = trimTrailingLines(preText(codeBlock));
     const filenameElement = select(CODE_FILENAME_SELECTORS, node);
     const fileLang = filenameElement ? hastToString(filenameElement).match(/\.(\w+)$/)?.[1] : null;
-    const classLang = [...classNames, ...codeBlockClassNames].map((className) => {
-      const match = LANGUAGE_MATCH_REGEX.map((regex) => className.match(regex)).find((match2) => match2);
-      return match?.[1];
-    }).find((className) => className);
+    const code = codeBlock.children.find((child) => isElement(child) && child.tagName === "code");
+    const classLang = codeLanguage(...isElement(code) ? [code, codeBlock, node] : [codeBlock, node]);
     const lang = fileLang || classLang || detectLanguage(codeValue) || null;
-    const result = { type: "code", lang, meta: null, value: codeValue };
+    const result = { type: "code", lang, meta: codeMeta(codeBlock), value: codeValue };
     state.patch(node, result);
     return result;
   }

package/dist/mdast-handlers/custom-img-handler.js

@@ -12,11 +12,20 @@ var hasNoUsableSource = (node) => {
   }
   return false;
 };
+var PLACEHOLDER_FILE = /^(?:[\w-]*[-_.])?placeholder[\w-]*\.(?:gif|png|svg|webp|jpe?g)$|^(?:spacer|blank|transparent|pixel)\.(?:gif|png)$/i;
+var isUndescribedPlaceholder = (node) => {
+  if (stringProperty(node, "alt")?.trim() || stringProperty(node, "title")?.trim()) {
+    return false;
+  }
+  const src = stringProperty(node, "src") ?? "";
+  const file = src.split(/[?#]/)[0].split("/").pop() ?? "";
+  return PLACEHOLDER_FILE.test(file);
+};
 var customImgHandler = (options) => (state, node) => {
   if (options?.hideImage) {
     return void 0;
   }
-  if (hasNoUsableSource(node)) {
+  if (hasNoUsableSource(node) || isUndescribedPlaceholder(node)) {
     return void 0;
   }
   const image = defaultHandlers.image(state, node);

package/dist/mdast-handlers/custom-table-handler.js

@@ -1,7 +1,8 @@
 // src/mdast-handlers/custom-table-handler.ts
 import { defaultHandlers } from "hast-util-to-mdast";
 import { toText } from "hast-util-to-text";
-import { isElement, numericProperty, walk } from "../utils/hast-fast.js";
+import { findElement, isElement, numericProperty, walk } from "../utils/hast-fast.js";
+import { flattenTableCells, tableHasCodeSamples, unfoldTable } from "./table-cell-flatten.js";
 var MAX_SPAN = 1e3;
 var clampSpans = (node) => {
   walk(node, (candidate) => {
@@ -16,6 +17,14 @@ var clampSpans = (node) => {
     }
   });
 };
+var startsWithHeaderRow = (table) => {
+  const row = findElement(table, (element) => element.tagName === "tr");
+  if (!row) {
+    return false;
+  }
+  const cells = row.children.filter(isElement);
+  return cells.length > 0 && cells.every((cell) => cell.tagName === "th");
+};
 var customTableHandler = (options) => (state, node) => {
   if (options?.asText) {
     const paragraph = { type: "paragraph", children: [{ type: "text", value: toText(node) }] };
@@ -23,7 +32,15 @@ var customTableHandler = (options) => (state, node) => {
     return paragraph;
   }
   clampSpans(node);
-  return defaultHandlers.table(state, node);
+  const result = defaultHandlers.table(state, node);
+  if (!result || Array.isArray(result) || result.type !== "table") {
+    return result;
+  }
+  if (tableHasCodeSamples(result)) {
+    return unfoldTable(result, startsWithHeaderRow(node));
+  }
+  flattenTableCells(result);
+  return result;
 };
 export {
   customTableHandler

package/dist/mdast-handlers/definition-list-handler.js

@@ -28,6 +28,20 @@ var hoistBoundaryWhitespace = (nodes) => {
   }
   return result;
 };
+var termsAndDefinitions = (parent) => {
+  const found = [];
+  for (const child of parent.children) {
+    if (child.type !== "element") {
+      continue;
+    }
+    if (child.tagName === "dt" || child.tagName === "dd") {
+      found.push(child);
+    } else if (child.tagName === "div") {
+      found.push(...termsAndDefinitions(child));
+    }
+  }
+  return found;
+};
 var definitionListHandler = (state, node) => {
   const items = [];
   let current;
@@ -42,11 +56,7 @@ var definitionListHandler = (state, node) => {
     items.push({ type: "listItem", spread: false, children: [heading, ...current.definitions] });
     current = void 0;
   };
-  for (const child of node.children) {
-    if (child.type !== "element") {
-      continue;
-    }
-    const element = child;
+  for (const element of termsAndDefinitions(node)) {
     if (element.tagName === "dt") {
       if (current && current.definitions.length > 0) {
         flush();
@@ -65,7 +75,8 @@ var definitionListHandler = (state, node) => {
   }
   flush();
   if (items.length === 0) {
-    return void 0;
+    const fallback = state.toFlow(state.all(node));
+    return fallback.length > 0 ? fallback : void 0;
   }
   const result = { type: "list", ordered: false, spread: false, children: items };
   state.patch(node, result);

package/dist/mdast-handlers/table-cell-flatten.js

@@ -0,0 +1,170 @@
+// src/mdast-handlers/table-cell-flatten.ts
+var PHRASING_TYPES = /* @__PURE__ */ new Set([
+  "text",
+  "emphasis",
+  "strong",
+  "delete",
+  "inlineCode",
+  "link",
+  "linkReference",
+  "image",
+  "imageReference",
+  "html",
+  "footnoteReference",
+  "inlineMath"
+]);
+var PHRASING_PARENTS = /* @__PURE__ */ new Set(["emphasis", "strong", "delete", "link", "linkReference"]);
+var lineBreak = () => ({ type: "html", value: "<br>" });
+var needsFlattening = (nodes) => nodes.some((node) => {
+  if (node.type === "break") {
+    return true;
+  }
+  if (!PHRASING_TYPES.has(node.type)) {
+    return true;
+  }
+  if ((node.type === "text" || node.type === "inlineCode") && node.value.includes("\n")) {
+    return true;
+  }
+  return PHRASING_PARENTS.has(node.type) && "children" in node && needsFlattening(node.children);
+});
+var CellBuilder = class _CellBuilder {
+  constructor() {
+    this.out = [];
+    this.#pendingBlock = false;
+  }
+  #pendingBlock;
+  /** Starts a new visual line when the previous block left content on the current one. */
+  block() {
+    const last = this.out.at(-1);
+    if (this.#pendingBlock && last && !(last.type === "html" && last.value === "<br>")) {
+      this.out.push(lineBreak());
+    }
+    this.#pendingBlock = true;
+  }
+  add(nodes) {
+    for (const node of nodes) {
+      this.#addOne(node);
+    }
+  }
+  #addOne(node) {
+    switch (node.type) {
+      case "break":
+        this.out.push(lineBreak());
+        return;
+      case "text":
+        this.out.push({ type: "text", value: node.value.replace(/\s*\n\s*/g, " ") });
+        return;
+      case "inlineCode":
+        this.#codeLines(node.value);
+        return;
+      case "code":
+        this.block();
+        this.#codeLines(node.value);
+        return;
+      case "math":
+        this.block();
+        this.out.push({ type: "inlineMath", value: node.value.replace(/\s*\n\s*/g, " ") });
+        return;
+      case "list":
+        node.children.forEach((item, index) => {
+          this.block();
+          const marker = node.ordered ? `${(node.start ?? 1) + index}. ` : "- ";
+          this.out.push({ type: "text", value: marker });
+          this.#pendingBlock = false;
+          this.add(item.children);
+        });
+        return;
+      case "thematicBreak":
+        return;
+      default:
+        this.#addContainer(node);
+    }
+  }
+  #addContainer(node) {
+    if (PHRASING_PARENTS.has(node.type) && "children" in node) {
+      const inner = new _CellBuilder();
+      inner.add(node.children);
+      this.out.push({ ...node, children: inner.out });
+      return;
+    }
+    if (PHRASING_TYPES.has(node.type)) {
+      this.out.push(node);
+      return;
+    }
+    if ("children" in node) {
+      this.block();
+      this.add(node.children);
+    }
+  }
+  #codeLines(value) {
+    value.split("\n").forEach((line, index) => {
+      if (index > 0) {
+        this.out.push(lineBreak());
+      }
+      if (line.trim().length > 0) {
+        this.out.push({ type: "inlineCode", value: line });
+      }
+    });
+  }
+};
+var flattenTableCells = (table) => {
+  for (const row of table.children) {
+    for (const cell of row.children) {
+      if (!needsFlattening(cell.children)) {
+        continue;
+      }
+      const builder = new CellBuilder();
+      builder.add(cell.children);
+      cell.children = builder.out;
+    }
+  }
+};
+var hasMultilineCode = (nodes) => nodes.some(
+  (node) => node.type === "code" && node.value.includes("\n") || "children" in node && hasMultilineCode(node.children)
+);
+var tableHasCodeSamples = (table) => table.children.some((row) => row.children.some((cell) => hasMultilineCode(cell.children)));
+var unfoldTable = (table, hasHeader) => {
+  const [first, ...rest] = table.children;
+  const labelled = hasHeader && rest.length > 0;
+  const labels = labelled && first ? first.children.map((cell) => cell.children) : [];
+  const rows = labelled ? rest : table.children;
+  const out = [];
+  rows.forEach((row, rowIndex) => {
+    if (rowIndex > 0) {
+      out.push({ type: "thematicBreak" });
+    }
+    row.children.forEach((cell, cellIndex) => {
+      const label = labels[cellIndex];
+      if (label && label.length > 0) {
+        out.push({ type: "paragraph", children: [{ type: "strong", children: label }] });
+      }
+      out.push(...asBlocks(cell.children));
+    });
+  });
+  return out;
+};
+var asBlocks = (nodes) => {
+  const blocks = [];
+  let run = [];
+  const flush = () => {
+    if (run.some((node) => node.type !== "text" || node.value.trim().length > 0)) {
+      blocks.push({ type: "paragraph", children: run });
+    }
+    run = [];
+  };
+  for (const node of nodes) {
+    if (PHRASING_TYPES.has(node.type) || node.type === "break") {
+      run.push(node);
+    } else {
+      flush();
+      blocks.push(node);
+    }
+  }
+  flush();
+  return blocks;
+};
+export {
+  flattenTableCells,
+  tableHasCodeSamples,
+  unfoldTable
+};

package/dist/mdast-handlers/twoslash.js

@@ -0,0 +1,48 @@
+// src/mdast-handlers/twoslash.ts
+import { classList, isElement } from "../utils/hast-fast.js";
+import { preText } from "../utils/pre-text.js";
+var cleanTwoslash = (pre) => {
+  if (!classList(pre).includes("twoslash")) {
+    return void 0;
+  }
+  let language;
+  const clean = (element) => {
+    element.children = element.children.flatMap((child) => {
+      if (!isElement(child)) {
+        return [child];
+      }
+      const classes = classList(child);
+      if (classes.includes("language-id")) {
+        language ||= preText(child).trim() || void 0;
+        return [];
+      }
+      if (classes.includes("playground-link") || classes.includes("error-behind")) {
+        return [];
+      }
+      if (classes.includes("error")) {
+        return [errorLine(child)];
+      }
+      clean(child);
+      return [child];
+    });
+  };
+  clean(pre);
+  return language && /^[\w+#.-]+$/.test(language) ? language : void 0;
+};
+var errorLine = (error) => {
+  let code = "";
+  let message = "";
+  for (const child of error.children) {
+    if (isElement(child) && classList(child).includes("code")) {
+      code = preText(child).trim();
+    } else {
+      message += isElement(child) ? preText(child) : child.type === "text" ? child.value : "";
+    }
+  }
+  const prefix = /^\d+$/.test(code) ? `error TS${code}: ` : "error: ";
+  const lines = `${prefix}${message.trim()}`.split("\n").map((line) => `// ${line}`);
+  return { type: "element", tagName: "div", properties: {}, children: [{ type: "text", value: lines.join("\n") }] };
+};
+export {
+  cleanTwoslash
+};

package/dist/mdast-to-markdown.js

@@ -10,11 +10,8 @@ var DEFAULT_MDAST_TO_MARKDOWN_OPTIONS = {
 };
 var mdastToMarkdown = (mdast, options) => {
   const { baseUrl, ...toMarkdownOptions } = { ...DEFAULT_MDAST_TO_MARKDOWN_OPTIONS, ...options };
-  let markdown = toMarkdown(warpRoot(mdast), toMarkdownOptions).replace(/\*\*\*\*/g, "");
-  if (baseUrl) {
-    markdown = linkReplacer(markdown, baseUrl);
-  }
-  return markdown;
+  const tree = warpRoot(mdast);
+  return toMarkdown(baseUrl ? linkReplacer(tree, baseUrl) : tree, toMarkdownOptions);
 };
 export {
   DEFAULT_MDAST_TO_MARKDOWN_OPTIONS,

package/dist/metadata/index.js

@@ -92,9 +92,10 @@ var readName = (value) => {
 };
 var asString = (value) => typeof value === "string" && value.length > 0 ? value : void 0;
 var MAX_TITLE_LENGTH = 160;
-var asTitle = (value) => {
+var MAX_DOCUMENT_TITLE_LENGTH = 300;
+var asTitle = (value, maxLength = MAX_TITLE_LENGTH) => {
   const cleaned = clean(value);
-  return cleaned && cleaned.length <= MAX_TITLE_LENGTH ? cleaned : void 0;
+  return cleaned && cleaned.length <= maxLength ? cleaned : void 0;
 };
 var clean = (value) => {
   if (value === void 0) {
@@ -160,7 +161,7 @@ var extractMetadata = (tree) => {
     }
     return void 0;
   };
-  const title = asTitle(get("og:title", "twitter:title")) ?? asTitle(asString(article?.headline)) ?? asTitle(asString(article?.name)) ?? asTitle(documentTitle(tree)) ?? asTitle(firstHeading(tree));
+  const title = asTitle(get("og:title", "twitter:title")) ?? asTitle(asString(article?.headline)) ?? asTitle(asString(article?.name)) ?? asTitle(documentTitle(tree), MAX_DOCUMENT_TITLE_LENGTH) ?? asTitle(firstHeading(tree));
   return {
     title,
     description: clean(get("og:description", "twitter:description", "description")) ?? clean(asString(article?.description)),

package/dist/normalize/index.js

@@ -1,6 +1,7 @@
 // src/normalize/index.ts
 import {
   classList,
+  findElement,
   isElement,
   numericProperty,
   property,
@@ -80,14 +81,16 @@ var katexDuplicates = (node) => {
   if (!(classes.includes("katex") || classes.includes("katex-display"))) {
     return [];
   }
-  const hasMathml = node.children.some((child) => isElement(child) && classList(child).includes("katex-mathml"));
+  const hasMathml = node.children.some(
+    (child) => isElement(child) && classList(child).includes("katex-mathml") && hasMathSource(child)
+  );
   if (!hasMathml) {
     return [];
   }
   return node.children.filter((child) => isElement(child) && classList(child).includes("katex-html"));
 };
 var mediawikiDuplicates = (node) => {
-  if (!classList(node).includes("mwe-math-element")) {
+  if (!classList(node).includes("mwe-math-element") || !hasMathSource(node)) {
     return [];
   }
   return node.children.filter(
@@ -98,8 +101,15 @@ var mathjaxDuplicates = (node) => {
   if (node.tagName !== "mjx-container") {
     return [];
   }
+  const hasMathml = node.children.some(
+    (child) => isElement(child) && child.tagName === "mjx-assistive-mml" && hasMathSource(child)
+  );
+  if (!hasMathml) {
+    return [];
+  }
   return node.children.filter((child) => isElement(child) && child.tagName !== "mjx-assistive-mml");
 };
+var hasMathSource = (node) => Boolean(findElement(node, (element) => element.tagName === "math" && element.children.length > 0));
 var dedupeRenderedMath = (tree) => {
   const doomed = /* @__PURE__ */ new Set();
   walk(tree, (node) => {