webforai 4.0.0 → 4.1.1

package/dist/cjs/mdast-handlers/custom-table-handler.js

@@ -24,6 +24,7 @@ module.exports = __toCommonJS(custom_table_handler_exports);
 var import_hast_util_to_mdast = require("hast-util-to-mdast");
 var import_hast_util_to_text = require("hast-util-to-text");
 var import_hast_fast = require("../utils/hast-fast");
+var import_table_cell_flatten = require("./table-cell-flatten");
 const MAX_SPAN = 1e3;
 const clampSpans = (node) => {
   (0, import_hast_fast.walk)(node, (candidate) => {
@@ -38,6 +39,14 @@ const clampSpans = (node) => {
     }
   });
 };
+const startsWithHeaderRow = (table) => {
+  const row = (0, import_hast_fast.findElement)(table, (element) => element.tagName === "tr");
+  if (!row) {
+    return false;
+  }
+  const cells = row.children.filter(import_hast_fast.isElement);
+  return cells.length > 0 && cells.every((cell) => cell.tagName === "th");
+};
 const customTableHandler = (options) => (state, node) => {
   if (options?.asText) {
     const paragraph = { type: "paragraph", children: [{ type: "text", value: (0, import_hast_util_to_text.toText)(node) }] };
@@ -45,7 +54,15 @@ const customTableHandler = (options) => (state, node) => {
     return paragraph;
   }
   clampSpans(node);
-  return import_hast_util_to_mdast.defaultHandlers.table(state, node);
+  const result = import_hast_util_to_mdast.defaultHandlers.table(state, node);
+  if (!result || Array.isArray(result) || result.type !== "table") {
+    return result;
+  }
+  if ((0, import_table_cell_flatten.tableHasCodeSamples)(result)) {
+    return (0, import_table_cell_flatten.unfoldTable)(result, startsWithHeaderRow(node));
+  }
+  (0, import_table_cell_flatten.flattenTableCells)(result);
+  return result;
 };
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {

package/dist/cjs/mdast-handlers/table-cell-flatten.js

@@ -0,0 +1,195 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var table_cell_flatten_exports = {};
+__export(table_cell_flatten_exports, {
+  flattenTableCells: () => flattenTableCells,
+  tableHasCodeSamples: () => tableHasCodeSamples,
+  unfoldTable: () => unfoldTable
+});
+module.exports = __toCommonJS(table_cell_flatten_exports);
+const PHRASING_TYPES = /* @__PURE__ */ new Set([
+  "text",
+  "emphasis",
+  "strong",
+  "delete",
+  "inlineCode",
+  "link",
+  "linkReference",
+  "image",
+  "imageReference",
+  "html",
+  "footnoteReference",
+  "inlineMath"
+]);
+const PHRASING_PARENTS = /* @__PURE__ */ new Set(["emphasis", "strong", "delete", "link", "linkReference"]);
+const lineBreak = () => ({ type: "html", value: "<br>" });
+const needsFlattening = (nodes) => nodes.some((node) => {
+  if (node.type === "break") {
+    return true;
+  }
+  if (!PHRASING_TYPES.has(node.type)) {
+    return true;
+  }
+  if ((node.type === "text" || node.type === "inlineCode") && node.value.includes("\n")) {
+    return true;
+  }
+  return PHRASING_PARENTS.has(node.type) && "children" in node && needsFlattening(node.children);
+});
+class CellBuilder {
+  constructor() {
+    this.out = [];
+    this.#pendingBlock = false;
+  }
+  #pendingBlock;
+  /** Starts a new visual line when the previous block left content on the current one. */
+  block() {
+    const last = this.out.at(-1);
+    if (this.#pendingBlock && last && !(last.type === "html" && last.value === "<br>")) {
+      this.out.push(lineBreak());
+    }
+    this.#pendingBlock = true;
+  }
+  add(nodes) {
+    for (const node of nodes) {
+      this.#addOne(node);
+    }
+  }
+  #addOne(node) {
+    switch (node.type) {
+      case "break":
+        this.out.push(lineBreak());
+        return;
+      case "text":
+        this.out.push({ type: "text", value: node.value.replace(/\s*\n\s*/g, " ") });
+        return;
+      case "inlineCode":
+        this.#codeLines(node.value);
+        return;
+      case "code":
+        this.block();
+        this.#codeLines(node.value);
+        return;
+      case "math":
+        this.block();
+        this.out.push({ type: "inlineMath", value: node.value.replace(/\s*\n\s*/g, " ") });
+        return;
+      case "list":
+        node.children.forEach((item, index) => {
+          this.block();
+          const marker = node.ordered ? `${(node.start ?? 1) + index}. ` : "- ";
+          this.out.push({ type: "text", value: marker });
+          this.#pendingBlock = false;
+          this.add(item.children);
+        });
+        return;
+      case "thematicBreak":
+        return;
+      default:
+        this.#addContainer(node);
+    }
+  }
+  #addContainer(node) {
+    if (PHRASING_PARENTS.has(node.type) && "children" in node) {
+      const inner = new CellBuilder();
+      inner.add(node.children);
+      this.out.push({ ...node, children: inner.out });
+      return;
+    }
+    if (PHRASING_TYPES.has(node.type)) {
+      this.out.push(node);
+      return;
+    }
+    if ("children" in node) {
+      this.block();
+      this.add(node.children);
+    }
+  }
+  #codeLines(value) {
+    value.split("\n").forEach((line, index) => {
+      if (index > 0) {
+        this.out.push(lineBreak());
+      }
+      if (line.trim().length > 0) {
+        this.out.push({ type: "inlineCode", value: line });
+      }
+    });
+  }
+}
+const flattenTableCells = (table) => {
+  for (const row of table.children) {
+    for (const cell of row.children) {
+      if (!needsFlattening(cell.children)) {
+        continue;
+      }
+      const builder = new CellBuilder();
+      builder.add(cell.children);
+      cell.children = builder.out;
+    }
+  }
+};
+const hasMultilineCode = (nodes) => nodes.some(
+  (node) => node.type === "code" && node.value.includes("\n") || "children" in node && hasMultilineCode(node.children)
+);
+const tableHasCodeSamples = (table) => table.children.some((row) => row.children.some((cell) => hasMultilineCode(cell.children)));
+const unfoldTable = (table, hasHeader) => {
+  const [first, ...rest] = table.children;
+  const labelled = hasHeader && rest.length > 0;
+  const labels = labelled && first ? first.children.map((cell) => cell.children) : [];
+  const rows = labelled ? rest : table.children;
+  const out = [];
+  rows.forEach((row, rowIndex) => {
+    if (rowIndex > 0) {
+      out.push({ type: "thematicBreak" });
+    }
+    row.children.forEach((cell, cellIndex) => {
+      const label = labels[cellIndex];
+      if (label && label.length > 0) {
+        out.push({ type: "paragraph", children: [{ type: "strong", children: label }] });
+      }
+      out.push(...asBlocks(cell.children));
+    });
+  });
+  return out;
+};
+const asBlocks = (nodes) => {
+  const blocks = [];
+  let run = [];
+  const flush = () => {
+    if (run.some((node) => node.type !== "text" || node.value.trim().length > 0)) {
+      blocks.push({ type: "paragraph", children: run });
+    }
+    run = [];
+  };
+  for (const node of nodes) {
+    if (PHRASING_TYPES.has(node.type) || node.type === "break") {
+      run.push(node);
+    } else {
+      flush();
+      blocks.push(node);
+    }
+  }
+  flush();
+  return blocks;
+};
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  flattenTableCells,
+  tableHasCodeSamples,
+  unfoldTable
+});

package/dist/cjs/mdast-handlers/twoslash.js

@@ -0,0 +1,71 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var twoslash_exports = {};
+__export(twoslash_exports, {
+  cleanTwoslash: () => cleanTwoslash
+});
+module.exports = __toCommonJS(twoslash_exports);
+var import_hast_fast = require("../utils/hast-fast");
+var import_pre_text = require("../utils/pre-text");
+const cleanTwoslash = (pre) => {
+  if (!(0, import_hast_fast.classList)(pre).includes("twoslash")) {
+    return void 0;
+  }
+  let language;
+  const clean = (element) => {
+    element.children = element.children.flatMap((child) => {
+      if (!(0, import_hast_fast.isElement)(child)) {
+        return [child];
+      }
+      const classes = (0, import_hast_fast.classList)(child);
+      if (classes.includes("language-id")) {
+        language ||= (0, import_pre_text.preText)(child).trim() || void 0;
+        return [];
+      }
+      if (classes.includes("playground-link") || classes.includes("error-behind")) {
+        return [];
+      }
+      if (classes.includes("error")) {
+        return [errorLine(child)];
+      }
+      clean(child);
+      return [child];
+    });
+  };
+  clean(pre);
+  return language && /^[\w+#.-]+$/.test(language) ? language : void 0;
+};
+const errorLine = (error) => {
+  let code = "";
+  let message = "";
+  for (const child of error.children) {
+    if ((0, import_hast_fast.isElement)(child) && (0, import_hast_fast.classList)(child).includes("code")) {
+      code = (0, import_pre_text.preText)(child).trim();
+    } else {
+      message += (0, import_hast_fast.isElement)(child) ? (0, import_pre_text.preText)(child) : child.type === "text" ? child.value : "";
+    }
+  }
+  const prefix = /^\d+$/.test(code) ? `error TS${code}: ` : "error: ";
+  const lines = `${prefix}${message.trim()}`.split("\n").map((line) => `// ${line}`);
+  return { type: "element", tagName: "div", properties: {}, children: [{ type: "text", value: lines.join("\n") }] };
+};
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  cleanTwoslash
+});

package/dist/cjs/metadata/index.js

@@ -116,9 +116,10 @@ const readName = (value) => {
 };
 const asString = (value) => typeof value === "string" && value.length > 0 ? value : void 0;
 const MAX_TITLE_LENGTH = 160;
-const asTitle = (value) => {
+const MAX_DOCUMENT_TITLE_LENGTH = 300;
+const asTitle = (value, maxLength = MAX_TITLE_LENGTH) => {
   const cleaned = clean(value);
-  return cleaned && cleaned.length <= MAX_TITLE_LENGTH ? cleaned : void 0;
+  return cleaned && cleaned.length <= maxLength ? cleaned : void 0;
 };
 const clean = (value) => {
   if (value === void 0) {
@@ -184,7 +185,7 @@ const extractMetadata = (tree) => {
     }
     return void 0;
   };
-  const title = asTitle(get("og:title", "twitter:title")) ?? asTitle(asString(article?.headline)) ?? asTitle(asString(article?.name)) ?? asTitle(documentTitle(tree)) ?? asTitle(firstHeading(tree));
+  const title = asTitle(get("og:title", "twitter:title")) ?? asTitle(asString(article?.headline)) ?? asTitle(asString(article?.name)) ?? asTitle(documentTitle(tree), MAX_DOCUMENT_TITLE_LENGTH) ?? asTitle(firstHeading(tree));
   return {
     title,
     description: clean(get("og:description", "twitter:description", "description")) ?? clean(asString(article?.description)),

package/dist/cjs/utils/hast-fast.js

@@ -136,7 +136,7 @@ const pruneInPlace = (tree, keep) => {
     let write = 0;
     for (let read = 0; read < children.length; read++) {
       const child = children[read];
-      if (!keep(child)) {
+      if (!keep(child, parent)) {
         continue;
       }
       children[write++] = child;

package/dist/cjs/utils/parse-html.js

@@ -0,0 +1,113 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var parse_html_exports = {};
+__export(parse_html_exports, {
+  LARGE_DOCUMENT_CHARS: () => LARGE_DOCUMENT_CHARS,
+  MAX_ELEMENT_DEPTH: () => MAX_ELEMENT_DEPTH,
+  parseHtml: () => parseHtml,
+  stripScriptBodies: () => stripScriptBodies
+});
+module.exports = __toCommonJS(parse_html_exports);
+var import_hast_util_from_parse5 = require("hast-util-from-parse5");
+var import_parse5 = require("parse5");
+const MAX_ELEMENT_DEPTH = 256;
+const isParent = (node) => "childNodes" in node;
+const LINE_ELEMENTS = /* @__PURE__ */ new Set([
+  "p",
+  "div",
+  "li",
+  "tr",
+  "td",
+  "th",
+  "h1",
+  "h2",
+  "h3",
+  "h4",
+  "h5",
+  "h6",
+  "br",
+  "section",
+  "article",
+  "pre",
+  "blockquote"
+]);
+const textNode = (value, parentNode) => ({ nodeName: "#text", value, parentNode });
+const coalesceText = (parent) => {
+  const merged = [];
+  for (const child of parent.childNodes) {
+    const previous = merged.at(-1);
+    if (child.nodeName === "#text" && previous?.nodeName === "#text") {
+      previous.value += child.value;
+    } else {
+      merged.push(child);
+    }
+  }
+  parent.childNodes = merged;
+};
+const flattenDeep = (root, maxDepth) => {
+  const stack = [[root, 0]];
+  while (stack.length > 0) {
+    const [parent, depth] = stack.pop();
+    const children = parent.childNodes;
+    let unwrapped = false;
+    for (let index = 0; index < children.length; index++) {
+      const child = children[index];
+      if (!isParent(child)) {
+        continue;
+      }
+      if (depth + 1 > maxDepth && "tagName" in child) {
+        const replacement = [...child.childNodes];
+        if (LINE_ELEMENTS.has(child.tagName)) {
+          replacement.unshift(textNode("\n", parent));
+          replacement.push(textNode("\n", parent));
+        }
+        for (const node of replacement) {
+          node.parentNode = parent;
+        }
+        children.splice(index, 1, ...replacement);
+        index -= 1;
+        unwrapped = true;
+        continue;
+      }
+      stack.push([child, depth + 1]);
+    }
+    if (unwrapped) {
+      coalesceText(parent);
+    }
+  }
+};
+const SCRIPT_OR_STYLE = /<(script|style)\b([^>]*)>[\s\S]*?<\/\1\s*>/gi;
+const LARGE_DOCUMENT_CHARS = 2e6;
+const stripScriptBodies = (html) => html.replace(
+  SCRIPT_OR_STYLE,
+  (match, tag, attributes) => /application\/ld\+json/i.test(attributes) ? match : `<${tag}${attributes}></${tag}>`
+);
+const parseHtml = (html, options = {}) => {
+  const source = html.length > LARGE_DOCUMENT_CHARS ? stripScriptBodies(html) : html;
+  const document = options.fragment ? (0, import_parse5.parseFragment)(source, { scriptingEnabled: false }) : (0, import_parse5.parse)(source, { scriptingEnabled: false });
+  flattenDeep(document, MAX_ELEMENT_DEPTH);
+  return (0, import_hast_util_from_parse5.fromParse5)(document);
+};
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  LARGE_DOCUMENT_CHARS,
+  MAX_ELEMENT_DEPTH,
+  parseHtml,
+  stripScriptBodies
+});

package/dist/cjs/utils/pre-text.js

@@ -0,0 +1,104 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var pre_text_exports = {};
+__export(pre_text_exports, {
+  preText: () => preText
+});
+module.exports = __toCommonJS(pre_text_exports);
+const LINE_BLOCK_TAGS = /* @__PURE__ */ new Set([
+  "address",
+  "article",
+  "aside",
+  "blockquote",
+  "dd",
+  "div",
+  "dl",
+  "dt",
+  "figcaption",
+  "figure",
+  "footer",
+  "form",
+  "h1",
+  "h2",
+  "h3",
+  "h4",
+  "h5",
+  "h6",
+  "header",
+  "hr",
+  "li",
+  "main",
+  "nav",
+  "ol",
+  "p",
+  "pre",
+  "section",
+  "table",
+  "tr",
+  "ul"
+]);
+const INLINE_CONTEXT_TAGS = /* @__PURE__ */ new Set(["span", "a", "b", "i", "em", "strong", "mark", "small", "u", "s", "label"]);
+const preText = (node) => {
+  let out = "";
+  const ensureLineStart = () => {
+    if (out.length > 0 && out.charCodeAt(out.length - 1) !== 10) {
+      out += "\n";
+    }
+  };
+  const visitChildren = (parent, inline) => {
+    let cells = 0;
+    for (const child of parent.children) {
+      const isCell = child.type === "element" && (child.tagName === "td" || child.tagName === "th");
+      visit(child, isCell ? cells++ : 0, inline);
+    }
+  };
+  const visitElement = (element, cellIndex, inline) => {
+    const tag = element.tagName;
+    if (tag === "br") {
+      out += "\n";
+      return;
+    }
+    if (cellIndex > 0) {
+      out += "	";
+    }
+    const block = !inline && LINE_BLOCK_TAGS.has(tag);
+    if (block) {
+      ensureLineStart();
+    }
+    visitChildren(element, inline || INLINE_CONTEXT_TAGS.has(tag));
+    if (block) {
+      ensureLineStart();
+    }
+  };
+  const visit = (current, cellIndex, inline) => {
+    if (current.type === "text") {
+      out += current.value;
+    } else if (current.type === "element") {
+      visitElement(current, cellIndex, inline);
+    } else if (current.type === "root") {
+      visitChildren(current, inline);
+    }
+  };
+  visit(node, 0, false);
+  return out;
+};
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  preText
+});

package/dist/extract-mdast.js

@@ -10,6 +10,61 @@ var emptyDeclarationFilter = (node) => {
   }
   return true;
 };
+var trimEdge = (children, edge) => {
+  const index = edge === "start" ? 0 : children.length - 1;
+  const node = children[index];
+  if (node?.type !== "text") {
+    return children;
+  }
+  const value = edge === "start" ? node.value.trimStart() : node.value.trimEnd();
+  const copy = [...children];
+  if (value.length === 0) {
+    copy.splice(index, 1);
+  } else {
+    copy[index] = { ...node, value };
+  }
+  return copy;
+};
+var liftDisplayMath = (paragraph) => {
+  if (!paragraph.children.some((child) => child.type === "math")) {
+    return void 0;
+  }
+  const out = [];
+  let run = [];
+  const flush = () => {
+    const trimmed = trimEdge(trimEdge(run, "start"), "end");
+    if (trimmed.length > 0) {
+      out.push({ ...paragraph, children: trimmed });
+    }
+    run = [];
+  };
+  for (const child of paragraph.children) {
+    if (child.type === "math") {
+      flush();
+      out.push(child);
+    } else {
+      run.push(child);
+    }
+  }
+  flush();
+  return out;
+};
+var liftDisplayMathIn = (node) => {
+  if (!("children" in node)) {
+    return;
+  }
+  const parent = node;
+  for (let index = 0; index < parent.children.length; index++) {
+    const child = parent.children[index];
+    const lifted = child.type === "paragraph" ? liftDisplayMath(child) : void 0;
+    if (lifted) {
+      parent.children.splice(index, 1, ...lifted);
+      index += lifted.length - 1;
+      continue;
+    }
+    liftDisplayMathIn(child);
+  }
+};
 var extractMdast = (node) => {
   const extracted = filter(node, (node2) => {
     if (!emptyDeclarationFilter(node2)) {
@@ -17,6 +72,9 @@ var extractMdast = (node) => {
     }
     return true;
   });
+  if (extracted) {
+    liftDisplayMathIn(extracted);
+  }
   return extracted;
 };
 export {

package/dist/extractors/index.js

@@ -3,11 +3,14 @@ import {
   pipeExtractors
 } from "./pipeExtractors.js";
 import { takumiExtractor } from "./presets/takumi.js";
+import { createKiwameExtractor, kiwameExtractor } from "./presets/kiwame.js";
 import { minimalFilter } from "./presets/minimal-filter.js";
 import { autoExtractor, createAutoExtractor } from "./presets/auto.js";
 export {
   autoExtractor,
   createAutoExtractor,
+  createKiwameExtractor,
+  kiwameExtractor,
   minimalFilter,
   pipeExtractors,
   takumiExtractor