webforai 2.1.1 → 4.0.0

package/dist/mdast-handlers/custom-a-handler.js

@@ -1,21 +1,37 @@
 // src/mdast-handlers/custom-a-handler.ts
 import { defaultHandlers } from "hast-util-to-mdast";
 import { toString as hastToString } from "hast-util-to-string";
+import { stringProperty } from "../utils/hast-fast.js";
+var ALLOWED_SCHEMES = /* @__PURE__ */ new Set(["http:", "https:", "mailto:", "tel:", "ftp:", "ftps:"]);
+var isFollowableHref = (href) => {
+  const trimmed = href?.trim();
+  if (!trimmed) {
+    return false;
+  }
+  const scheme = trimmed.match(/^[a-z][a-z0-9+.-]*:/i)?.[0];
+  if (!scheme) {
+    return true;
+  }
+  return ALLOWED_SCHEMES.has(scheme.toLowerCase());
+};
 var customAHandler = (options) => (state, node) => {
-  if (options?.asText) {
+  const asPlainText = () => {
     const text = hastToString(node);
-    if (3 >= text.length) {
+    if (text.length === 0) {
       return void 0;
     }
-    const link2 = { type: "text", value: text };
-    state.patch(node, link2);
-    return link2;
+    const result = { type: "text", value: text };
+    state.patch(node, result);
+    return result;
+  };
+  if (options?.asText) {
+    return hastToString(node).length <= 3 ? void 0 : asPlainText();
   }
-  const link = defaultHandlers.a(state, node);
-  if (link.children.length > 0) {
-    return link;
+  if (!isFollowableHref(stringProperty(node, "href"))) {
+    return asPlainText();
   }
-  return void 0;
+  const link = defaultHandlers.a(state, node);
+  return link.children.length > 0 ? link : void 0;
 };
 export {
   customAHandler

package/dist/mdast-handlers/custom-code-handler.js

@@ -2,19 +2,29 @@
 import { toText } from "hast-util-to-text";
 import { trimTrailingLines } from "trim-trailing-lines";
 import { detectLanguage } from "../utils/detect-code-lang.js";
-var LANGUAGE_MATCH_REGEX = [/language-(\w+)/, /highlight-source-(\w+)/, /CodeBlock--language-(\w+)/];
+import { classList, isElement } from "../utils/hast-fast.js";
+var LANGUAGE_MATCH_REGEX = [/^language-(\S+)$/, /^highlight-source-(\S+)$/, /^CodeBlock--language-(\S+)$/];
+var LANGUAGE_ATTRIBUTES = ["dataLanguage", "dataLang"];
+var attributeLanguage = (element) => {
+  for (const name of LANGUAGE_ATTRIBUTES) {
+    const value = element.properties?.[name];
+    if (typeof value === "string" && /^[\w+#.-]+$/.test(value.trim()) && value.trim() !== "plaintext") {
+      return value.trim();
+    }
+  }
+  return void 0;
+};
+var codeLanguage = (...elements) => elements.flatMap(classList).map((className) => LANGUAGE_MATCH_REGEX.map((regex) => className.match(regex)).find(Boolean)?.[1]).find(Boolean) ?? elements.map(attributeLanguage).find(Boolean);
 var customCodeHandler = (state, node) => {
-  const classNames = node.properties?.className || [];
-  const codeValue = trimTrailingLines(toText(node)).trim();
-  const classLang = classNames.map((className) => {
-    const match = LANGUAGE_MATCH_REGEX.map((regex) => className.match(regex)).find((match2) => match2);
-    return match?.[1];
-  }).find((className) => className);
+  const code = node.children.find((child) => isElement(child) && child.tagName === "code");
+  const codeValue = trimTrailingLines(toText(node));
+  const classLang = codeLanguage(...isElement(code) ? [code, node] : [node]);
   const lang = classLang || detectLanguage(codeValue) || null;
   const result = { type: "code", lang, meta: null, value: codeValue };
   state.patch(node, result);
   return result;
 };
 export {
+  codeLanguage,
   customCodeHandler
 };

package/dist/mdast-handlers/custom-div-handler.js

@@ -5,44 +5,23 @@ import { toString as hastToString } from "hast-util-to-string";
 import { toText } from "hast-util-to-text";
 import { trimTrailingLines } from "trim-trailing-lines";
 import { detectLanguage } from "../utils/detect-code-lang.js";
+import { classList, collectElements, isElement } from "../utils/hast-fast.js";
+import { codeLanguage } from "./custom-code-handler.js";
 var CODE_BLOCK_REGEX = /highlight-source|language-|codegroup|codeblock|code-block/i;
 var CODE_FILENAME_SELECTORS = "[class*='fileName'],[class*='fileName'],[class*='title'],[class*='Title']";
-var LANGUAGE_MATCH_REGEX = [/language-(\w+)/, /highlight-source-(\w+)/, /CodeBlock--language-(\w+)/];
-var findRecursive = (array, condition, maxDepth = 3) => {
-  if (maxDepth <= 0) {
-    return null;
-  }
-  for (const value of array) {
-    const result = condition(value);
-    if (Array.isArray(result)) {
-      return findRecursive(result, condition, maxDepth - 1);
-    }
-    if (result) {
-      return value;
-    }
-  }
-  return null;
-};
 var customDivHandler = (state, node) => {
-  const classNames = Array.isArray(node.properties.className) ? node.properties.className : [];
-  const codeBlock = findRecursive(node.children, (child) => {
-    if (child.type !== "element") {
-      return false;
-    }
-    if (child.tagName === "pre") {
-      return true;
-    }
-    return child.children.filter((child2) => child2.type === "element");
-  });
-  if (codeBlock && classNames.some((className) => CODE_BLOCK_REGEX.test(className))) {
-    const codeBlockClassNames = codeBlock.type === "element" ? codeBlock.properties.className ?? [] : [];
-    const codeValue = trimTrailingLines(toText(codeBlock)).trim();
+  const classNames = classList(node);
+  if (!classNames.some((className) => CODE_BLOCK_REGEX.test(className))) {
+    return defaultHandlers.div(state, node);
+  }
+  const codeBlocks = collectElements(node, (element) => element.tagName === "pre");
+  const codeBlock = codeBlocks.length === 1 ? codeBlocks[0] : void 0;
+  if (codeBlock) {
+    const codeValue = trimTrailingLines(toText(codeBlock));
     const filenameElement = select(CODE_FILENAME_SELECTORS, node);
     const fileLang = filenameElement ? hastToString(filenameElement).match(/\.(\w+)$/)?.[1] : null;
-    const classLang = [...classNames, ...codeBlockClassNames].map((className) => {
-      const match = LANGUAGE_MATCH_REGEX.map((regex) => className.match(regex)).find((match2) => match2);
-      return match?.[1];
-    }).find((className) => className);
+    const code = codeBlock.children.find((child) => isElement(child) && child.tagName === "code");
+    const classLang = codeLanguage(...isElement(code) ? [code, codeBlock, node] : [codeBlock, node]);
     const lang = fileLang || classLang || detectLanguage(codeValue) || null;
     const result = { type: "code", lang, meta: null, value: codeValue };
     state.patch(node, result);

package/dist/mdast-handlers/custom-img-handler.js

@@ -1,10 +1,32 @@
 // src/mdast-handlers/custom-img-handler.ts
 import { defaultHandlers } from "hast-util-to-mdast";
+import { stringProperty } from "../utils/hast-fast.js";
+var MAX_DATA_URL_LENGTH = 256;
+var hasNoUsableSource = (node) => {
+  const src = stringProperty(node, "src")?.trim();
+  if (!src) {
+    return true;
+  }
+  if (src.startsWith("data:")) {
+    return src.length > MAX_DATA_URL_LENGTH;
+  }
+  return false;
+};
 var customImgHandler = (options) => (state, node) => {
   if (options?.hideImage) {
     return void 0;
   }
-  return defaultHandlers.image(state, node);
+  if (hasNoUsableSource(node)) {
+    return void 0;
+  }
+  const image = defaultHandlers.image(state, node);
+  if (!image.alt) {
+    const title = stringProperty(node, "title");
+    if (title) {
+      image.alt = title;
+    }
+  }
+  return image;
 };
 export {
   customImgHandler

package/dist/mdast-handlers/custom-table-handler.js

@@ -1,12 +1,28 @@
 // src/mdast-handlers/custom-table-handler.ts
 import { defaultHandlers } from "hast-util-to-mdast";
 import { toText } from "hast-util-to-text";
+import { isElement, numericProperty, walk } from "../utils/hast-fast.js";
+var MAX_SPAN = 1e3;
+var clampSpans = (node) => {
+  walk(node, (candidate) => {
+    if (!isElement(candidate) || candidate.tagName !== "td" && candidate.tagName !== "th") {
+      return;
+    }
+    for (const attribute of ["colSpan", "rowSpan"]) {
+      const span = numericProperty(candidate, attribute);
+      if (span !== void 0 && span > MAX_SPAN) {
+        candidate.properties[attribute] = MAX_SPAN;
+      }
+    }
+  });
+};
 var customTableHandler = (options) => (state, node) => {
   if (options?.asText) {
     const paragraph = { type: "paragraph", children: [{ type: "text", value: toText(node) }] };
     state.patch(node, paragraph);
     return paragraph;
   }
+  clampSpans(node);
   return defaultHandlers.table(state, node);
 };
 export {

package/dist/mdast-handlers/definition-list-handler.js

@@ -0,0 +1,87 @@
+// src/mdast-handlers/definition-list-handler.ts
+var hoistBoundaryWhitespace = (nodes) => {
+  const result = [];
+  for (const node of nodes) {
+    if (node.type !== "emphasis" && node.type !== "strong") {
+      result.push(node);
+      continue;
+    }
+    node.children = hoistBoundaryWhitespace(node.children);
+    const first = node.children[0];
+    if (first?.type === "text" && /^\s/.test(first.value)) {
+      result.push({ type: "text", value: " " });
+      first.value = first.value.replace(/^\s+/, "");
+    }
+    let trailing = false;
+    const last = node.children[node.children.length - 1];
+    if (last?.type === "text" && /\s$/.test(last.value)) {
+      last.value = last.value.replace(/\s+$/, "");
+      trailing = true;
+    }
+    node.children = node.children.filter((child) => !(child.type === "text" && child.value.length === 0));
+    if (node.children.length > 0) {
+      result.push(node);
+    }
+    if (trailing) {
+      result.push({ type: "text", value: " " });
+    }
+  }
+  return result;
+};
+var termsAndDefinitions = (parent) => {
+  const found = [];
+  for (const child of parent.children) {
+    if (child.type !== "element") {
+      continue;
+    }
+    if (child.tagName === "dt" || child.tagName === "dd") {
+      found.push(child);
+    } else if (child.tagName === "div") {
+      found.push(...termsAndDefinitions(child));
+    }
+  }
+  return found;
+};
+var definitionListHandler = (state, node) => {
+  const items = [];
+  let current;
+  const flush = () => {
+    if (!current) {
+      return;
+    }
+    const heading = {
+      type: "paragraph",
+      children: current.term.length > 0 ? [{ type: "strong", children: current.term }] : []
+    };
+    items.push({ type: "listItem", spread: false, children: [heading, ...current.definitions] });
+    current = void 0;
+  };
+  for (const element of termsAndDefinitions(node)) {
+    if (element.tagName === "dt") {
+      if (current && current.definitions.length > 0) {
+        flush();
+      }
+      const phrasing = hoistBoundaryWhitespace(state.all(element));
+      current = current ? { term: [...current.term, { type: "text", value: ", " }, ...phrasing], definitions: [] } : { term: phrasing, definitions: [] };
+      continue;
+    }
+    if (element.tagName === "dd") {
+      if (!current) {
+        current = { term: [], definitions: [] };
+      }
+      const content = state.toFlow(state.all(element));
+      current.definitions.push(...content);
+    }
+  }
+  flush();
+  if (items.length === 0) {
+    const fallback = state.toFlow(state.all(node));
+    return fallback.length > 0 ? fallback : void 0;
+  }
+  const result = { type: "list", ordered: false, spread: false, children: items };
+  state.patch(node, result);
+  return result;
+};
+export {
+  definitionListHandler
+};

package/dist/mdast-handlers/math-handler.js

@@ -1,10 +1,19 @@
 // src/mdast-handlers/math-handler.ts
 import { toHtml } from "hast-util-to-html";
 import { MathMLToLaTeX } from "mathml-to-latex";
+import { stringProperty } from "../utils/hast-fast.js";
+var readAnnotation = (node) => {
+  const alttext = stringProperty(node, "alttext");
+  if (!alttext) {
+    return void 0;
+  }
+  const unwrapped = alttext.trim().replace(/^\{\\displaystyle\s*([\s\S]*)\}$/, "$1").trim();
+  return unwrapped.length > 0 ? unwrapped : void 0;
+};
+var isDisplayMath = (node) => stringProperty(node, "display") === "block";
 var mathHandler = (state, node) => {
-  const mathMl = toHtml(node);
-  const latex = MathMLToLaTeX.convert(mathMl);
-  const result = { type: "inlineMath", value: latex };
+  const latex = readAnnotation(node) ?? MathMLToLaTeX.convert(toHtml(node));
+  const result = isDisplayMath(node) ? { type: "math", value: latex, meta: null } : { type: "inlineMath", value: latex };
   state.patch(node, result);
   return result;
 };

package/dist/mdast-handlers/sup-sub-handler.js

@@ -0,0 +1,16 @@
+// src/mdast-handlers/sup-sub-handler.ts
+import { toString as hastToString } from "hast-util-to-string";
+var MAX_ANNOTATION_LENGTH = 32;
+var supSubHandler = (tagName) => (state, node) => {
+  const inner = hastToString(node).trim();
+  if (inner.length === 0 || inner.length > MAX_ANNOTATION_LENGTH) {
+    return state.all(node);
+  }
+  const escaped = inner.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;");
+  const result = { type: "html", value: `<${tagName}>${escaped}</${tagName}>` };
+  state.patch(node, result);
+  return result;
+};
+export {
+  supSubHandler
+};

package/dist/mdast-to-markdown.js

@@ -10,11 +10,8 @@ var DEFAULT_MDAST_TO_MARKDOWN_OPTIONS = {
 };
 var mdastToMarkdown = (mdast, options) => {
   const { baseUrl, ...toMarkdownOptions } = { ...DEFAULT_MDAST_TO_MARKDOWN_OPTIONS, ...options };
-  let markdown = toMarkdown(warpRoot(mdast), toMarkdownOptions).replace(/\*\*\*\*/g, "");
-  if (baseUrl) {
-    markdown = linkReplacer(markdown, baseUrl);
-  }
-  return markdown;
+  const tree = warpRoot(mdast);
+  return toMarkdown(baseUrl ? linkReplacer(tree, baseUrl) : tree, toMarkdownOptions);
 };
 export {
   DEFAULT_MDAST_TO_MARKDOWN_OPTIONS,

package/dist/metadata/index.js

@@ -0,0 +1,198 @@
+// src/metadata/index.ts
+import { collectElements, findElement, isElement, stringProperty, walk } from "../utils/hast-fast.js";
+var readMetaTags = (tree) => {
+  const tags = /* @__PURE__ */ new Map();
+  for (const meta of collectElements(tree, (element) => element.tagName === "meta")) {
+    const key = stringProperty(meta, "property") ?? stringProperty(meta, "name") ?? stringProperty(meta, "itemprop");
+    const content = stringProperty(meta, "content");
+    if (!(key && content)) {
+      continue;
+    }
+    const normalized = key.toLowerCase();
+    if (!tags.has(normalized)) {
+      tags.set(normalized, content);
+    }
+  }
+  return tags;
+};
+var readJsonLd = (tree) => {
+  const entries = [];
+  for (const script of collectElements(tree, (element) => element.tagName === "script")) {
+    const type = stringProperty(script, "type");
+    if (type?.toLowerCase() !== "application/ld+json") {
+      continue;
+    }
+    const source = script.children.map((child) => child.type === "text" ? child.value : "").join("").trim();
+    if (source.length === 0) {
+      continue;
+    }
+    let parsed;
+    try {
+      parsed = JSON.parse(source);
+    } catch {
+      continue;
+    }
+    for (const entry of flattenJsonLd(parsed)) {
+      entries.push(entry);
+    }
+  }
+  return entries;
+};
+var flattenJsonLd = (value) => {
+  if (Array.isArray(value)) {
+    return value.flatMap(flattenJsonLd);
+  }
+  if (typeof value !== "object" || value === null) {
+    return [];
+  }
+  const record = value;
+  const graph = record["@graph"];
+  if (graph) {
+    return flattenJsonLd(graph);
+  }
+  return [record];
+};
+var ARTICLE_TYPES = /* @__PURE__ */ new Set([
+  "article",
+  "newsarticle",
+  "blogposting",
+  "techarticle",
+  "scholarlyarticle",
+  "report",
+  "webpage"
+]);
+var jsonLdType = (entry) => {
+  const type = entry["@type"];
+  if (typeof type === "string") {
+    return [type.toLowerCase()];
+  }
+  if (Array.isArray(type)) {
+    return type.filter((value) => typeof value === "string").map((value) => value.toLowerCase());
+  }
+  return [];
+};
+var readName = (value) => {
+  if (typeof value === "string") {
+    return value;
+  }
+  if (Array.isArray(value)) {
+    for (const entry of value) {
+      const name = readName(entry);
+      if (name) {
+        return name;
+      }
+    }
+    return void 0;
+  }
+  if (typeof value === "object" && value !== null) {
+    const name = value.name;
+    return typeof name === "string" ? name : void 0;
+  }
+  return void 0;
+};
+var asString = (value) => typeof value === "string" && value.length > 0 ? value : void 0;
+var MAX_TITLE_LENGTH = 160;
+var asTitle = (value) => {
+  const cleaned = clean(value);
+  return cleaned && cleaned.length <= MAX_TITLE_LENGTH ? cleaned : void 0;
+};
+var clean = (value) => {
+  if (value === void 0) {
+    return void 0;
+  }
+  const trimmed = value.replace(/\s+/g, " ").trim();
+  return trimmed.length > 0 ? trimmed : void 0;
+};
+var documentTitle = (tree) => {
+  const title = findElement(tree, (element) => element.tagName === "title");
+  if (!title) {
+    return void 0;
+  }
+  return title.children.map((child) => child.type === "text" ? child.value : "").join("");
+};
+var firstHeading = (tree) => {
+  const heading = findElement(tree, (element) => element.tagName === "h1");
+  if (!heading) {
+    return void 0;
+  }
+  let text = "";
+  walk(heading, (node) => {
+    if (node.type === "text") {
+      text += node.value;
+    }
+  });
+  return text;
+};
+var linkHref = (tree, rel) => {
+  const link = findElement(
+    tree,
+    (element) => element.tagName === "link" && (stringProperty(element, "rel") ?? "").toLowerCase().includes(rel)
+  );
+  return link ? stringProperty(link, "href") : void 0;
+};
+var documentLang = (tree) => {
+  const html = findElement(tree, (element) => element.tagName === "html");
+  if (html) {
+    const declared = stringProperty(html, "lang") ?? stringProperty(html, "xml:lang");
+    if (declared) {
+      return declared;
+    }
+  }
+  if (isElement(tree)) {
+    const own = stringProperty(tree, "lang");
+    if (own) {
+      return own;
+    }
+  }
+  const tagged = findElement(tree, (element) => Boolean(stringProperty(element, "lang")));
+  return tagged ? stringProperty(tagged, "lang") : void 0;
+};
+var extractMetadata = (tree) => {
+  const meta = readMetaTags(tree);
+  const jsonLd = readJsonLd(tree);
+  const article = jsonLd.find((entry) => jsonLdType(entry).some((type) => ARTICLE_TYPES.has(type))) ?? jsonLd[0];
+  const get = (...keys) => {
+    for (const key of keys) {
+      const value = meta.get(key);
+      if (value) {
+        return value;
+      }
+    }
+    return void 0;
+  };
+  const title = asTitle(get("og:title", "twitter:title")) ?? asTitle(asString(article?.headline)) ?? asTitle(asString(article?.name)) ?? asTitle(documentTitle(tree)) ?? asTitle(firstHeading(tree));
+  return {
+    title,
+    description: clean(get("og:description", "twitter:description", "description")) ?? clean(asString(article?.description)),
+    author: clean(readName(article?.author)) ?? clean(get("author", "article:author", "twitter:creator")),
+    published: clean(asString(article?.datePublished)) ?? clean(get("article:published_time", "date", "pubdate")),
+    modified: clean(asString(article?.dateModified)) ?? clean(get("article:modified_time", "og:updated_time")),
+    siteName: clean(get("og:site_name", "application-name")) ?? clean(readName(article?.publisher)),
+    canonicalUrl: clean(linkHref(tree, "canonical")) ?? clean(get("og:url", "twitter:url")),
+    lang: clean(documentLang(tree)),
+    image: clean(get("og:image", "twitter:image", "twitter:image:src")),
+    type: clean(get("og:type"))
+  };
+};
+var yamlString = (value) => `"${value.replace(/\\/g, "\\\\").replace(/"/g, '\\"')}"`;
+var toFrontmatter = (metadata) => {
+  const lines = [];
+  for (const [key, value] of Object.entries(metadata)) {
+    if (typeof value === "string" && value.length > 0) {
+      lines.push(`${key}: ${yamlString(value)}`);
+    }
+  }
+  if (lines.length === 0) {
+    return "";
+  }
+  return `---
+${lines.join("\n")}
+---
+
+`;
+};
+export {
+  extractMetadata,
+  readJsonLd,
+  toFrontmatter
+};