webforai 2.1.1 → 4.0.0

package/dist/cjs/html-to-markdown.js

@@ -18,18 +18,55 @@ var __copyProps = (to, from, except, desc) => {
 var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
 var html_to_markdown_exports = {};
 __export(html_to_markdown_exports, {
-  htmlToMarkdown: () => htmlToMarkdown
+  htmlToMarkdown: () => htmlToMarkdown,
+  htmlToMarkdownWithMetadata: () => htmlToMarkdownWithMetadata
 });
 module.exports = __toCommonJS(html_to_markdown_exports);
+var import_hast_util_from_html = require("hast-util-from-html");
 var import_html_to_mdast = require("./html-to-mdast");
 var import_mdast_to_markdown = require("./mdast-to-markdown");
-const htmlToMarkdown = (htmlOrHast, options) => {
-  const { baseUrl, formatting: toMarkdownOptions, ...toMdastOptions } = options || {};
-  const mdast = (0, import_html_to_mdast.htmlToMdast)(htmlOrHast, toMdastOptions);
-  const markdown = (0, import_mdast_to_markdown.mdastToMarkdown)(mdast, { baseUrl, ...toMarkdownOptions });
-  return markdown;
+var import_metadata = require("./metadata");
+var import_hast_utils = require("./utils/hast-utils");
+const htmlToMarkdownWithMetadata = (htmlOrHast, options) => {
+  const { baseUrl, frontmatter, title, formatting: toMarkdownOptions, ...toMdastOptions } = options || {};
+  const hast = typeof htmlOrHast === "string" ? (0, import_hast_util_from_html.fromHtml)(htmlOrHast, { fragment: true }) : htmlOrHast;
+  const isOwned = typeof htmlOrHast === "string";
+  const declaredLang = typeof htmlOrHast === "string" ? (0, import_hast_utils.getLangFromStr)(htmlOrHast) : void 0;
+  const metadata = (0, import_metadata.extractMetadata)(hast);
+  if (declaredLang && !metadata.lang) {
+    metadata.lang = declaredLang;
+  }
+  const mdast = (0, import_html_to_mdast.htmlToMdast)(hast, {
+    ...toMdastOptions,
+    owned: isOwned,
+    lang: toMdastOptions.lang ?? metadata.lang,
+    url: toMdastOptions.url ?? metadata.canonicalUrl
+  });
+  const body = (0, import_mdast_to_markdown.mdastToMarkdown)(mdast, { baseUrl, ...toMarkdownOptions });
+  const titled = title === false ? body : withTitle(body, metadata.title);
+  const markdown = frontmatter ? `${(0, import_metadata.toFrontmatter)(metadata)}${titled}` : titled;
+  return { markdown, metadata };
+};
+const TITLE_LOOKAHEAD = 400;
+const withTitle = (body, title) => {
+  if (!title) {
+    return body;
+  }
+  const opening = body.slice(0, TITLE_LOOKAHEAD);
+  if (/^#\s+\S/m.test(opening)) {
+    return body;
+  }
+  if (normalizeForCompare(opening).includes(normalizeForCompare(title))) {
+    return body;
+  }
+  return `# ${title}
+
+${body}`;
 };
+const normalizeForCompare = (value) => value.replace(/\s+/g, " ").trim().toLowerCase();
+const htmlToMarkdown = (htmlOrHast, options) => htmlToMarkdownWithMetadata(htmlOrHast, options).markdown;
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
-  htmlToMarkdown
+  htmlToMarkdown,
+  htmlToMarkdownWithMetadata
 });

package/dist/cjs/html-to-mdast.js

@@ -30,18 +30,27 @@ var import_custom_code_handler = require("./mdast-handlers/custom-code-handler")
 var import_custom_div_handler = require("./mdast-handlers/custom-div-handler");
 var import_custom_img_handler = require("./mdast-handlers/custom-img-handler");
 var import_custom_table_handler = require("./mdast-handlers/custom-table-handler");
+var import_definition_list_handler = require("./mdast-handlers/definition-list-handler");
 var import_math_handler = require("./mdast-handlers/math-handler");
+var import_sup_sub_handler = require("./mdast-handlers/sup-sub-handler");
+var import_normalize = require("./normalize");
+var import_hast_fast = require("./utils/hast-fast");
 var import_hast_utils = require("./utils/hast-utils");
 const htmlToMdast = (htmlOrHast, options) => {
   const { extractors, url: defaultUrl, lang: defaultLang } = options || {};
   const [lang, hast] = typeof htmlOrHast === "string" ? [defaultLang || (0, import_hast_utils.getLangFromStr)(htmlOrHast), (0, import_hast_util_from_html.fromHtml)(htmlOrHast, { fragment: true })] : [defaultLang || (0, import_hast_utils.getLangFromHast)(htmlOrHast), htmlOrHast];
+  const isOwned = typeof htmlOrHast === "string" || (options?.owned ?? false);
   const url = defaultUrl || (0, import_hast_utils.getUrlFromHast)(hast);
-  const extractedHast = (0, import_extractors.pipeExtractors)({ hast, lang, url }, extractors);
+  const extractedHast = (0, import_extractors.pipeExtractors)({ hast: isOwned ? hast : (0, import_hast_fast.cloneHast)(hast), lang, url, owned: true }, extractors);
+  (0, import_normalize.normalizeHast)(extractedHast, options?.normalize);
   const mdast = (0, import_hast_util_to_mdast.toMdast)(extractedHast, {
     handlers: {
       math: import_math_handler.mathHandler,
       div: import_custom_div_handler.customDivHandler,
       pre: import_custom_code_handler.customCodeHandler,
+      dl: import_definition_list_handler.definitionListHandler,
+      sup: (0, import_sup_sub_handler.supSubHandler)("sup"),
+      sub: (0, import_sup_sub_handler.supSubHandler)("sub"),
       a: (0, import_custom_a_handler.customAHandler)({ asText: options?.linkAsText }),
       img: (0, import_custom_img_handler.customImgHandler)({ hideImage: options?.hideImage }),
       table: (0, import_custom_table_handler.customTableHandler)({ asText: options?.tableAsText })

package/dist/cjs/index.js

@@ -18,13 +18,31 @@ var __copyProps = (to, from, except, desc) => {
 var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
 var src_exports = {};
 __export(src_exports, {
+  BUILTIN_ADAPTERS: () => import_adapters.BUILTIN_ADAPTERS,
+  MAX_META_REFRESH_DELAY_SECONDS: () => import_extract_meta_refresh.MAX_META_REFRESH_DELAY_SECONDS,
+  SHELL_EMPTY_THRESHOLD: () => import_detect_client_shell.SHELL_EMPTY_THRESHOLD,
+  SHELL_SUSPICION_THRESHOLD: () => import_detect_client_shell.SHELL_SUSPICION_THRESHOLD,
+  adapterById: () => import_adapters.adapterById,
+  autoExtractor: () => import_extractors.autoExtractor,
+  buildPageSignature: () => import_adapters.buildPageSignature,
+  createAutoExtractor: () => import_extractors.createAutoExtractor,
+  defineSelectorAdapter: () => import_adapters.defineSelectorAdapter,
+  detectClientShell: () => import_detect_client_shell.detectClientShell,
+  extractMetaRefresh: () => import_extract_meta_refresh.extractMetaRefresh,
+  extractMetadata: () => import_metadata.extractMetadata,
   htmlToMarkdown: () => import_html_to_markdown.htmlToMarkdown,
+  htmlToMarkdownWithMetadata: () => import_html_to_markdown.htmlToMarkdownWithMetadata,
   htmlToMdast: () => import_html_to_mdast.htmlToMdast,
   mdastSplitter: () => import_md_splitter.mdastSplitter,
   mdastToMarkdown: () => import_mdast_to_markdown.mdastToMarkdown,
   minimalFilter: () => import_extractors.minimalFilter,
+  normalizeHast: () => import_normalize.normalizeHast,
   pipeExtractors: () => import_extractors.pipeExtractors,
-  takumiExtractor: () => import_extractors.takumiExtractor
+  resolveAdapter: () => import_adapters.resolveAdapter,
+  runAdapters: () => import_adapters.runAdapters,
+  takumiExtractor: () => import_extractors.takumiExtractor,
+  toFrontmatter: () => import_metadata.toFrontmatter,
+  youtubeCaptionTracks: () => import_adapters.youtubeCaptionTracks
 });
 module.exports = __toCommonJS(src_exports);
 var import_html_to_markdown = require("./html-to-markdown");
@@ -32,13 +50,36 @@ var import_md_splitter = require("./md-splitter");
 var import_html_to_mdast = require("./html-to-mdast");
 var import_mdast_to_markdown = require("./mdast-to-markdown");
 var import_extractors = require("./extractors");
+var import_adapters = require("./adapters");
+var import_detect_client_shell = require("./detect-client-shell");
+var import_extract_meta_refresh = require("./extract-meta-refresh");
+var import_metadata = require("./metadata");
+var import_normalize = require("./normalize");
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
+  BUILTIN_ADAPTERS,
+  MAX_META_REFRESH_DELAY_SECONDS,
+  SHELL_EMPTY_THRESHOLD,
+  SHELL_SUSPICION_THRESHOLD,
+  adapterById,
+  autoExtractor,
+  buildPageSignature,
+  createAutoExtractor,
+  defineSelectorAdapter,
+  detectClientShell,
+  extractMetaRefresh,
+  extractMetadata,
   htmlToMarkdown,
+  htmlToMarkdownWithMetadata,
   htmlToMdast,
   mdastSplitter,
   mdastToMarkdown,
   minimalFilter,
+  normalizeHast,
   pipeExtractors,
-  takumiExtractor
+  resolveAdapter,
+  runAdapters,
+  takumiExtractor,
+  toFrontmatter,
+  youtubeCaptionTracks
 });

package/dist/cjs/link-replacer.js

@@ -21,19 +21,25 @@ __export(link_replacer_exports, {
   linkReplacer: () => linkReplacer
 });
 module.exports = __toCommonJS(link_replacer_exports);
-const linkReplacer = (markdown, base) => {
-  const regex = /(!?\[.*?\]\()([^)\s]+)(\))/g;
-  return markdown.replace(regex, (match, pre, url, post) => {
-    if (/^(https?:|#)/.test(url)) {
-      return match;
-    }
-    try {
-      const absoluteUrl = new URL(url, base).href;
-      return `${pre}${absoluteUrl}${post}`;
-    } catch {
-      return match;
-    }
-  });
+const absoluteUrl = (url, base) => {
+  if (/^(?:[a-z][a-z\d+.-]*:|#)/i.test(url)) {
+    return url;
+  }
+  try {
+    return new URL(url, base).href;
+  } catch {
+    return url;
+  }
+};
+const linkReplacer = (node, base) => {
+  let result = node;
+  if (node.type === "link" || node.type === "image" || node.type === "definition") {
+    result = { ...node, url: absoluteUrl(node.url, base) };
+  }
+  if ("children" in result) {
+    return { ...result, children: result.children.map((child) => linkReplacer(child, base)) };
+  }
+  return result;
 };
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {

package/dist/cjs/loaders/cf-puppeteer.js

@@ -32,15 +32,22 @@ __export(cf_puppeteer_exports, {
 });
 module.exports = __toCommonJS(cf_puppeteer_exports);
 var import_puppeteer = __toESM(require("@cloudflare/puppeteer"), 1);
-const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
 const loadHtml = async (url, ctx) => {
   const browser = await import_puppeteer.default.launch(ctx);
-  const page = await browser.newPage();
-  await page.goto(url);
-  const html = await page.content();
-  await Promise.race([page.waitForNetworkIdle(), sleep(1e4)]);
-  await page.close();
-  return html;
+  try {
+    const page = await browser.newPage();
+    await page.goto(url);
+    try {
+      await page.waitForNetworkIdle({ timeout: 1e4 });
+    } catch (error) {
+      if (!(error instanceof import_puppeteer.TimeoutError)) {
+        throw error;
+      }
+    }
+    return await page.content();
+  } finally {
+    await browser.close();
+  }
 };
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {

package/dist/cjs/loaders/fetch.js

@@ -22,10 +22,22 @@ __export(fetch_exports, {
   loadHtml: () => loadHtml
 });
 module.exports = __toCommonJS(fetch_exports);
+var import_extract_meta_refresh = require("../extract-meta-refresh");
 const USER_AGENT = "mozilla/5.0 (windows nt 10.0; win64; x64) applewebkit/537.36 (khtml, like gecko) chrome/125.0.0.0 safari/537.36";
+const MAX_META_REFRESH_HOPS = 3;
 const loadHtml = async (url, userAgent = USER_AGENT) => {
-  const response = await fetch(url, { headers: { "User-Agent": userAgent } });
-  return response.text();
+  let current = url;
+  let html = "";
+  for (let hop = 0; hop <= MAX_META_REFRESH_HOPS; hop++) {
+    const response = await fetch(current, { headers: { "User-Agent": userAgent } });
+    html = await response.text();
+    const refresh = (0, import_extract_meta_refresh.extractMetaRefresh)(html, response.url || current);
+    if (!refresh) {
+      break;
+    }
+    current = refresh.url;
+  }
+  return html;
 };
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {

package/dist/cjs/loaders/playwright.js

@@ -35,40 +35,42 @@ const SUPER_BYPASS_DEVICE = {
 const loadHtml = async (url, options) => {
   const { browser, waitUntil, timeout, superBypassMode } = options ?? {};
   const _browser = browser ?? await import_playwright_core.chromium.launch({ headless: true });
-  const context = await _browser.newContext(superBypassMode ? SUPER_BYPASS_DEVICE : import_playwright_core.devices["Desktop Chrome"]);
-  if (superBypassMode) {
-    await context.addInitScript(() => {
-      Object.defineProperty(navigator, "webdriver", { get: () => void 0 });
-      Object.defineProperty(navigator, "languages", { get: () => ["en-US", "en"] });
-      Object.defineProperty(navigator, "plugins", { get: () => [1, 2, 3, 4, 5] });
-    });
-  }
-  const page = await context.newPage();
-  if (superBypassMode) {
-    await page.route("**/*.js", (route) => {
-      if (route.request().url().includes("captcha-delivery")) {
-        return route.abort();
+  let context;
+  try {
+    context = await _browser.newContext(superBypassMode ? SUPER_BYPASS_DEVICE : import_playwright_core.devices["Desktop Chrome"]);
+    if (superBypassMode) {
+      await context.addInitScript(() => {
+        Object.defineProperty(navigator, "webdriver", { get: () => void 0 });
+        Object.defineProperty(navigator, "languages", { get: () => ["en-US", "en"] });
+        Object.defineProperty(navigator, "plugins", { get: () => [1, 2, 3, 4, 5] });
+      });
+    }
+    const page = await context.newPage();
+    if (superBypassMode) {
+      await page.route("**/*.js", (route) => {
+        if (route.request().url().includes("captcha-delivery")) {
+          return route.abort();
+        }
+        return route.continue();
+      });
+    }
+    await page.goto(url, { waitUntil: waitUntil ?? "load", timeout });
+    await page.evaluate(() => {
+      const elements = document.querySelectorAll("*");
+      for (const element of elements) {
+        const rect = element.getBoundingClientRect();
+        element.setAttribute("data-rwidth", rect.width.toString());
+        element.setAttribute("data-rheight", rect.height.toString());
       }
-      return route.continue();
     });
-  }
-  await page.goto(url, { waitUntil: waitUntil ?? "load", timeout });
-  await page.evaluate(() => {
-    const elements = document.querySelectorAll("*");
-    for (const element of elements) {
-      const rect = element.getBoundingClientRect();
-      element.setAttribute("data-rwidth", rect.width.toString());
-      element.setAttribute("data-rheight", rect.height.toString());
+    return await page.content();
+  } finally {
+    if (browser) {
+      await context?.close();
+    } else {
+      await _browser.close();
     }
-  });
-  const html = await page.content();
-  await page.close();
-  if (browser) {
-    await context.close();
-  } else {
-    await _browser.close();
   }
-  return html;
 };
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {

package/dist/cjs/loaders/puppeteer.js

@@ -32,7 +32,6 @@ __export(puppeteer_exports, {
 });
 module.exports = __toCommonJS(puppeteer_exports);
 var import_puppeteer = __toESM(require("puppeteer"), 1);
-const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
 const loadHtml = async (url, ctx) => {
   const browser = await import_puppeteer.default.launch(
     ctx || {
@@ -40,12 +39,20 @@ const loadHtml = async (url, ctx) => {
       args: ["--no-sandbox", "--disable-setuid-sandbox"]
     }
   );
-  const page = await browser.newPage();
-  await page.goto(url);
-  const html = await page.content();
-  await Promise.race([page.waitForNetworkIdle(), sleep(1e4)]);
-  await browser.close();
-  return html;
+  try {
+    const page = await browser.newPage();
+    await page.goto(url);
+    try {
+      await page.waitForNetworkIdle({ timeout: 1e4 });
+    } catch (error) {
+      if (!(error instanceof import_puppeteer.TimeoutError)) {
+        throw error;
+      }
+    }
+    return await page.content();
+  } finally {
+    await browser.close();
+  }
 };
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {

package/dist/cjs/md-splitter.js

@@ -25,15 +25,13 @@ var import_mdast_to_markdown = require("./mdast-to-markdown");
 var import_common = require("./utils/common");
 var import_mdast_utils = require("./utils/mdast-utils");
 const PRIORITY_SPLITTERS = ["h1", "h2", "h3", "h4", "h5", "h6", "list", "table", "code"];
-const getSplitterGenerator = function* () {
-  for (const splitter of PRIORITY_SPLITTERS) {
-    yield splitter;
-  }
-};
-const _mdastSplitter = async (contents, checker, splitterGenerator) => {
-  const splitter = splitterGenerator.next().value;
+const _mdastSplitter = async (contents, checker, priority, signal) => {
+  signal?.throwIfAborted();
+  const splitter = PRIORITY_SPLITTERS[priority];
   const markdown = (0, import_mdast_to_markdown.mdastToMarkdown)((0, import_mdast_utils.warpRoot)(contents));
-  if (await checker(markdown) || contents.length === 1) {
+  const accepted = await checker(markdown);
+  signal?.throwIfAborted();
+  if (accepted || contents.length === 1) {
     return [contents];
   }
   const chunked = splitter ? contents.reduce((acc, content) => {
@@ -44,11 +42,26 @@ const _mdastSplitter = async (contents, checker, splitterGenerator) => {
     acc[acc.length - 1].push(content);
     return acc;
   }, []) : (0, import_common.chunk)(contents, Math.ceil(contents.length / 2));
-  const splitting = chunked.map((chunk2) => _mdastSplitter(chunk2, checker, splitterGenerator));
+  const splitting = chunked.map((chunk2) => _mdastSplitter(chunk2, checker, priority + 1, signal));
   return Promise.all(splitting).then((chunks) => chunks.flat());
 };
-const mdastSplitter = (mdast, checker, _options) => {
-  return _mdastSplitter((0, import_mdast_utils.unwarpRoot)(mdast), checker, getSplitterGenerator());
+const mdastSplitter = async (mdast, checker, options) => {
+  const signal = options?.signal;
+  if (!signal) {
+    return _mdastSplitter((0, import_mdast_utils.unwarpRoot)(mdast), checker, 0);
+  }
+  signal.throwIfAborted();
+  let onAbort = () => {
+  };
+  const aborted = new Promise((_, reject) => {
+    onAbort = () => reject(signal.reason);
+    signal.addEventListener("abort", onAbort, { once: true });
+  });
+  try {
+    return await Promise.race([_mdastSplitter((0, import_mdast_utils.unwarpRoot)(mdast), checker, 0, signal), aborted]);
+  } finally {
+    signal.removeEventListener("abort", onAbort);
+  }
 };
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {

package/dist/cjs/mdast-handlers/custom-a-handler.js

@@ -23,21 +23,37 @@ __export(custom_a_handler_exports, {
 module.exports = __toCommonJS(custom_a_handler_exports);
 var import_hast_util_to_mdast = require("hast-util-to-mdast");
 var import_hast_util_to_string = require("hast-util-to-string");
+var import_hast_fast = require("../utils/hast-fast");
+const ALLOWED_SCHEMES = /* @__PURE__ */ new Set(["http:", "https:", "mailto:", "tel:", "ftp:", "ftps:"]);
+const isFollowableHref = (href) => {
+  const trimmed = href?.trim();
+  if (!trimmed) {
+    return false;
+  }
+  const scheme = trimmed.match(/^[a-z][a-z0-9+.-]*:/i)?.[0];
+  if (!scheme) {
+    return true;
+  }
+  return ALLOWED_SCHEMES.has(scheme.toLowerCase());
+};
 const customAHandler = (options) => (state, node) => {
-  if (options?.asText) {
+  const asPlainText = () => {
     const text = (0, import_hast_util_to_string.toString)(node);
-    if (3 >= text.length) {
+    if (text.length === 0) {
       return void 0;
     }
-    const link2 = { type: "text", value: text };
-    state.patch(node, link2);
-    return link2;
+    const result = { type: "text", value: text };
+    state.patch(node, result);
+    return result;
+  };
+  if (options?.asText) {
+    return (0, import_hast_util_to_string.toString)(node).length <= 3 ? void 0 : asPlainText();
   }
-  const link = import_hast_util_to_mdast.defaultHandlers.a(state, node);
-  if (link.children.length > 0) {
-    return link;
+  if (!isFollowableHref((0, import_hast_fast.stringProperty)(node, "href"))) {
+    return asPlainText();
   }
-  return void 0;
+  const link = import_hast_util_to_mdast.defaultHandlers.a(state, node);
+  return link.children.length > 0 ? link : void 0;
 };
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {

package/dist/cjs/mdast-handlers/custom-code-handler.js

@@ -18,20 +18,30 @@ var __copyProps = (to, from, except, desc) => {
 var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
 var custom_code_handler_exports = {};
 __export(custom_code_handler_exports, {
+  codeLanguage: () => codeLanguage,
   customCodeHandler: () => customCodeHandler
 });
 module.exports = __toCommonJS(custom_code_handler_exports);
 var import_hast_util_to_text = require("hast-util-to-text");
 var import_trim_trailing_lines = require("trim-trailing-lines");
 var import_detect_code_lang = require("../utils/detect-code-lang");
-const LANGUAGE_MATCH_REGEX = [/language-(\w+)/, /highlight-source-(\w+)/, /CodeBlock--language-(\w+)/];
+var import_hast_fast = require("../utils/hast-fast");
+const LANGUAGE_MATCH_REGEX = [/^language-(\S+)$/, /^highlight-source-(\S+)$/, /^CodeBlock--language-(\S+)$/];
+const LANGUAGE_ATTRIBUTES = ["dataLanguage", "dataLang"];
+const attributeLanguage = (element) => {
+  for (const name of LANGUAGE_ATTRIBUTES) {
+    const value = element.properties?.[name];
+    if (typeof value === "string" && /^[\w+#.-]+$/.test(value.trim()) && value.trim() !== "plaintext") {
+      return value.trim();
+    }
+  }
+  return void 0;
+};
+const codeLanguage = (...elements) => elements.flatMap(import_hast_fast.classList).map((className) => LANGUAGE_MATCH_REGEX.map((regex) => className.match(regex)).find(Boolean)?.[1]).find(Boolean) ?? elements.map(attributeLanguage).find(Boolean);
 const customCodeHandler = (state, node) => {
-  const classNames = node.properties?.className || [];
-  const codeValue = (0, import_trim_trailing_lines.trimTrailingLines)((0, import_hast_util_to_text.toText)(node)).trim();
-  const classLang = classNames.map((className) => {
-    const match = LANGUAGE_MATCH_REGEX.map((regex) => className.match(regex)).find((match2) => match2);
-    return match?.[1];
-  }).find((className) => className);
+  const code = node.children.find((child) => (0, import_hast_fast.isElement)(child) && child.tagName === "code");
+  const codeValue = (0, import_trim_trailing_lines.trimTrailingLines)((0, import_hast_util_to_text.toText)(node));
+  const classLang = codeLanguage(...(0, import_hast_fast.isElement)(code) ? [code, node] : [node]);
   const lang = classLang || (0, import_detect_code_lang.detectLanguage)(codeValue) || null;
   const result = { type: "code", lang, meta: null, value: codeValue };
   state.patch(node, result);
@@ -39,5 +49,6 @@ const customCodeHandler = (state, node) => {
 };
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
+  codeLanguage,
   customCodeHandler
 });

package/dist/cjs/mdast-handlers/custom-div-handler.js

@@ -27,44 +27,23 @@ var import_hast_util_to_string = require("hast-util-to-string");
 var import_hast_util_to_text = require("hast-util-to-text");
 var import_trim_trailing_lines = require("trim-trailing-lines");
 var import_detect_code_lang = require("../utils/detect-code-lang");
+var import_hast_fast = require("../utils/hast-fast");
+var import_custom_code_handler = require("./custom-code-handler");
 const CODE_BLOCK_REGEX = /highlight-source|language-|codegroup|codeblock|code-block/i;
 const CODE_FILENAME_SELECTORS = "[class*='fileName'],[class*='fileName'],[class*='title'],[class*='Title']";
-const LANGUAGE_MATCH_REGEX = [/language-(\w+)/, /highlight-source-(\w+)/, /CodeBlock--language-(\w+)/];
-const findRecursive = (array, condition, maxDepth = 3) => {
-  if (maxDepth <= 0) {
-    return null;
-  }
-  for (const value of array) {
-    const result = condition(value);
-    if (Array.isArray(result)) {
-      return findRecursive(result, condition, maxDepth - 1);
-    }
-    if (result) {
-      return value;
-    }
-  }
-  return null;
-};
 const customDivHandler = (state, node) => {
-  const classNames = Array.isArray(node.properties.className) ? node.properties.className : [];
-  const codeBlock = findRecursive(node.children, (child) => {
-    if (child.type !== "element") {
-      return false;
-    }
-    if (child.tagName === "pre") {
-      return true;
-    }
-    return child.children.filter((child2) => child2.type === "element");
-  });
-  if (codeBlock && classNames.some((className) => CODE_BLOCK_REGEX.test(className))) {
-    const codeBlockClassNames = codeBlock.type === "element" ? codeBlock.properties.className ?? [] : [];
-    const codeValue = (0, import_trim_trailing_lines.trimTrailingLines)((0, import_hast_util_to_text.toText)(codeBlock)).trim();
+  const classNames = (0, import_hast_fast.classList)(node);
+  if (!classNames.some((className) => CODE_BLOCK_REGEX.test(className))) {
+    return import_hast_util_to_mdast.defaultHandlers.div(state, node);
+  }
+  const codeBlocks = (0, import_hast_fast.collectElements)(node, (element) => element.tagName === "pre");
+  const codeBlock = codeBlocks.length === 1 ? codeBlocks[0] : void 0;
+  if (codeBlock) {
+    const codeValue = (0, import_trim_trailing_lines.trimTrailingLines)((0, import_hast_util_to_text.toText)(codeBlock));
     const filenameElement = (0, import_hast_util_select.select)(CODE_FILENAME_SELECTORS, node);
     const fileLang = filenameElement ? (0, import_hast_util_to_string.toString)(filenameElement).match(/\.(\w+)$/)?.[1] : null;
-    const classLang = [...classNames, ...codeBlockClassNames].map((className) => {
-      const match = LANGUAGE_MATCH_REGEX.map((regex) => className.match(regex)).find((match2) => match2);
-      return match?.[1];
-    }).find((className) => className);
+    const code = codeBlock.children.find((child) => (0, import_hast_fast.isElement)(child) && child.tagName === "code");
+    const classLang = (0, import_custom_code_handler.codeLanguage)(...(0, import_hast_fast.isElement)(code) ? [code, codeBlock, node] : [codeBlock, node]);
     const lang = fileLang || classLang || (0, import_detect_code_lang.detectLanguage)(codeValue) || null;
     const result = { type: "code", lang, meta: null, value: codeValue };
     state.patch(node, result);

package/dist/cjs/mdast-handlers/custom-img-handler.js

@@ -22,11 +22,33 @@ __export(custom_img_handler_exports, {
 });
 module.exports = __toCommonJS(custom_img_handler_exports);
 var import_hast_util_to_mdast = require("hast-util-to-mdast");
+var import_hast_fast = require("../utils/hast-fast");
+const MAX_DATA_URL_LENGTH = 256;
+const hasNoUsableSource = (node) => {
+  const src = (0, import_hast_fast.stringProperty)(node, "src")?.trim();
+  if (!src) {
+    return true;
+  }
+  if (src.startsWith("data:")) {
+    return src.length > MAX_DATA_URL_LENGTH;
+  }
+  return false;
+};
 const customImgHandler = (options) => (state, node) => {
   if (options?.hideImage) {
     return void 0;
   }
-  return import_hast_util_to_mdast.defaultHandlers.image(state, node);
+  if (hasNoUsableSource(node)) {
+    return void 0;
+  }
+  const image = import_hast_util_to_mdast.defaultHandlers.image(state, node);
+  if (!image.alt) {
+    const title = (0, import_hast_fast.stringProperty)(node, "title");
+    if (title) {
+      image.alt = title;
+    }
+  }
+  return image;
 };
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {

package/dist/cjs/mdast-handlers/custom-table-handler.js

@@ -23,12 +23,28 @@ __export(custom_table_handler_exports, {
 module.exports = __toCommonJS(custom_table_handler_exports);
 var import_hast_util_to_mdast = require("hast-util-to-mdast");
 var import_hast_util_to_text = require("hast-util-to-text");
+var import_hast_fast = require("../utils/hast-fast");
+const MAX_SPAN = 1e3;
+const clampSpans = (node) => {
+  (0, import_hast_fast.walk)(node, (candidate) => {
+    if (!(0, import_hast_fast.isElement)(candidate) || candidate.tagName !== "td" && candidate.tagName !== "th") {
+      return;
+    }
+    for (const attribute of ["colSpan", "rowSpan"]) {
+      const span = (0, import_hast_fast.numericProperty)(candidate, attribute);
+      if (span !== void 0 && span > MAX_SPAN) {
+        candidate.properties[attribute] = MAX_SPAN;
+      }
+    }
+  });
+};
 const customTableHandler = (options) => (state, node) => {
   if (options?.asText) {
     const paragraph = { type: "paragraph", children: [{ type: "text", value: (0, import_hast_util_to_text.toText)(node) }] };
     state.patch(node, paragraph);
     return paragraph;
   }
+  clampSpans(node);
   return import_hast_util_to_mdast.defaultHandlers.table(state, node);
 };
 // Annotate the CommonJS export names for ESM import in node: