webforai 3.0.0 → 4.0.0

package/dist/cjs/detect-client-shell.js

@@ -0,0 +1,78 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var detect_client_shell_exports = {};
+__export(detect_client_shell_exports, {
+  SHELL_EMPTY_THRESHOLD: () => SHELL_EMPTY_THRESHOLD,
+  SHELL_SUSPICION_THRESHOLD: () => SHELL_SUSPICION_THRESHOLD,
+  detectClientShell: () => detectClientShell
+});
+module.exports = __toCommonJS(detect_client_shell_exports);
+const SHELL_SUSPICION_THRESHOLD = 300;
+const SHELL_EMPTY_THRESHOLD = 50;
+const BODY_PATTERN = /<body[^>]*>([\s\S]*?)<\/body>/i;
+const HEAD_PATTERN = /<head[^>]*>[\s\S]*?<\/head>/gi;
+const NOSCRIPT_PATTERN = /<noscript[^>]*>([\s\S]*?)<\/noscript>/gi;
+const INVISIBLE_ELEMENT_PATTERN = /<(script|style|template|svg|noscript)[^>]*>[\s\S]*?<\/\1>/gi;
+const COMMENT_PATTERN = /<!--[\s\S]*?-->/g;
+const TAG_PATTERN = /<[^>]+>/g;
+const MOUNT_POINT_PATTERN = /<[a-z][^>]*\sid=["'](?:root|app|__next|___gatsby|__nuxt|q-app|svelte|react-root)["']|<app-root[\s>]|\sdata-reactroot[\s>=]/i;
+const CHALLENGE_PATTERN = /cf-chl|challenge-platform|cf_chl_opt|just a moment|attention required|checking your browser|_incapsula_|px-captcha|datadome/i;
+const ENTITIES = {
+  "&nbsp;": " ",
+  "&amp;": "&",
+  "&lt;": "<",
+  "&gt;": ">",
+  "&quot;": '"',
+  "&#39;": "'"
+};
+const visibleTextOf = (markup) => markup.replace(COMMENT_PATTERN, " ").replace(INVISIBLE_ELEMENT_PATTERN, " ").replace(TAG_PATTERN, " ").replace(/&(?:nbsp|amp|lt|gt|quot|#39);/g, (entity) => ENTITIES[entity] ?? " ").replace(/\s+/g, " ").trim();
+const noscriptMentionsJavascript = (html) => {
+  for (const match of html.matchAll(NOSCRIPT_PATTERN)) {
+    if (/javascript/i.test(match[1] ?? "")) {
+      return true;
+    }
+  }
+  return false;
+};
+const detectClientShell = (html) => {
+  const body = BODY_PATTERN.exec(html)?.[1] ?? html.replace(HEAD_PATTERN, " ");
+  const visibleTextLength = visibleTextOf(body).length;
+  if (visibleTextLength >= SHELL_SUSPICION_THRESHOLD) {
+    return { isShell: false, visibleTextLength };
+  }
+  if (CHALLENGE_PATTERN.test(html)) {
+    return { isShell: true, reason: "anti-bot-challenge", visibleTextLength };
+  }
+  if (MOUNT_POINT_PATTERN.test(body)) {
+    return { isShell: true, reason: "spa-shell", visibleTextLength };
+  }
+  if (noscriptMentionsJavascript(html)) {
+    return { isShell: true, reason: "noscript-only", visibleTextLength };
+  }
+  if (visibleTextLength < SHELL_EMPTY_THRESHOLD) {
+    return { isShell: true, reason: "empty-body", visibleTextLength };
+  }
+  return { isShell: false, visibleTextLength };
+};
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  SHELL_EMPTY_THRESHOLD,
+  SHELL_SUSPICION_THRESHOLD,
+  detectClientShell
+});

package/dist/cjs/extract-meta-refresh.js

@@ -0,0 +1,71 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var extract_meta_refresh_exports = {};
+__export(extract_meta_refresh_exports, {
+  MAX_META_REFRESH_DELAY_SECONDS: () => MAX_META_REFRESH_DELAY_SECONDS,
+  extractMetaRefresh: () => extractMetaRefresh
+});
+module.exports = __toCommonJS(extract_meta_refresh_exports);
+const MAX_META_REFRESH_DELAY_SECONDS = 10;
+const META_TAG_PATTERN = /<meta\b[^>]*>/gi;
+const HTTP_EQUIV_REFRESH_PATTERN = /\bhttp-equiv\s*=\s*("\s*refresh\s*"|'\s*refresh\s*'|refresh\b)/i;
+const CONTENT_ATTR_PATTERN = /\bcontent\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s>]+))/i;
+const CONTENT_VALUE_PATTERN = /^\s*(\d+(?:\.\d+)?)\s*[;,]\s*url\s*=\s*(?:"([^"]*)"|'([^']*)'|(.+?))\s*$/i;
+const resolveTarget = (raw, baseUrl) => {
+  try {
+    return baseUrl === void 0 ? new URL(raw) : new URL(raw, baseUrl);
+  } catch {
+    return void 0;
+  }
+};
+const extractMetaRefresh = (html, baseUrl) => {
+  for (const [tag] of html.matchAll(META_TAG_PATTERN)) {
+    if (!HTTP_EQUIV_REFRESH_PATTERN.test(tag)) {
+      continue;
+    }
+    const contentMatch = CONTENT_ATTR_PATTERN.exec(tag);
+    const content = contentMatch?.[1] ?? contentMatch?.[2] ?? contentMatch?.[3];
+    if (!content) {
+      continue;
+    }
+    const value = CONTENT_VALUE_PATTERN.exec(content);
+    if (!value) {
+      continue;
+    }
+    const delaySeconds = Number.parseFloat(value[1] ?? "");
+    const raw = (value[2] ?? value[3] ?? value[4] ?? "").trim();
+    if (!raw || delaySeconds > MAX_META_REFRESH_DELAY_SECONDS) {
+      continue;
+    }
+    const target = resolveTarget(raw, baseUrl);
+    if (!target || target.protocol !== "http:" && target.protocol !== "https:") {
+      continue;
+    }
+    if (baseUrl !== void 0 && resolveTarget(baseUrl, void 0)?.href === target.href) {
+      continue;
+    }
+    return { url: target.href, delaySeconds };
+  }
+  return void 0;
+};
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  MAX_META_REFRESH_DELAY_SECONDS,
+  extractMetaRefresh
+});

package/dist/cjs/extractors/lib/sanitize.js

@@ -383,7 +383,11 @@ const countAnchors = (element) => {
   }
   return count;
 };
+const LITERAL_TEXT_TAGS = /* @__PURE__ */ new Set(["code", "kbd", "samp", "var", "pre"]);
 const isMidArticleWidget = (element) => {
+  if (LITERAL_TEXT_TAGS.has(element.tagName)) {
+    return false;
+  }
   if (element.tagName === "form" || element.tagName === "fieldset") {
     return true;
   }

package/dist/cjs/html-to-mdast.js

@@ -34,13 +34,14 @@ var import_definition_list_handler = require("./mdast-handlers/definition-list-h
 var import_math_handler = require("./mdast-handlers/math-handler");
 var import_sup_sub_handler = require("./mdast-handlers/sup-sub-handler");
 var import_normalize = require("./normalize");
+var import_hast_fast = require("./utils/hast-fast");
 var import_hast_utils = require("./utils/hast-utils");
 const htmlToMdast = (htmlOrHast, options) => {
   const { extractors, url: defaultUrl, lang: defaultLang } = options || {};
   const [lang, hast] = typeof htmlOrHast === "string" ? [defaultLang || (0, import_hast_utils.getLangFromStr)(htmlOrHast), (0, import_hast_util_from_html.fromHtml)(htmlOrHast, { fragment: true })] : [defaultLang || (0, import_hast_utils.getLangFromHast)(htmlOrHast), htmlOrHast];
   const isOwned = typeof htmlOrHast === "string" || (options?.owned ?? false);
   const url = defaultUrl || (0, import_hast_utils.getUrlFromHast)(hast);
-  const extractedHast = (0, import_extractors.pipeExtractors)({ hast, lang, url, owned: isOwned }, extractors);
+  const extractedHast = (0, import_extractors.pipeExtractors)({ hast: isOwned ? hast : (0, import_hast_fast.cloneHast)(hast), lang, url, owned: true }, extractors);
   (0, import_normalize.normalizeHast)(extractedHast, options?.normalize);
   const mdast = (0, import_hast_util_to_mdast.toMdast)(extractedHast, {
     handlers: {

package/dist/cjs/index.js

@@ -19,11 +19,16 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
 var src_exports = {};
 __export(src_exports, {
   BUILTIN_ADAPTERS: () => import_adapters.BUILTIN_ADAPTERS,
+  MAX_META_REFRESH_DELAY_SECONDS: () => import_extract_meta_refresh.MAX_META_REFRESH_DELAY_SECONDS,
+  SHELL_EMPTY_THRESHOLD: () => import_detect_client_shell.SHELL_EMPTY_THRESHOLD,
+  SHELL_SUSPICION_THRESHOLD: () => import_detect_client_shell.SHELL_SUSPICION_THRESHOLD,
   adapterById: () => import_adapters.adapterById,
   autoExtractor: () => import_extractors.autoExtractor,
   buildPageSignature: () => import_adapters.buildPageSignature,
   createAutoExtractor: () => import_extractors.createAutoExtractor,
   defineSelectorAdapter: () => import_adapters.defineSelectorAdapter,
+  detectClientShell: () => import_detect_client_shell.detectClientShell,
+  extractMetaRefresh: () => import_extract_meta_refresh.extractMetaRefresh,
   extractMetadata: () => import_metadata.extractMetadata,
   htmlToMarkdown: () => import_html_to_markdown.htmlToMarkdown,
   htmlToMarkdownWithMetadata: () => import_html_to_markdown.htmlToMarkdownWithMetadata,
@@ -46,16 +51,23 @@ var import_html_to_mdast = require("./html-to-mdast");
 var import_mdast_to_markdown = require("./mdast-to-markdown");
 var import_extractors = require("./extractors");
 var import_adapters = require("./adapters");
+var import_detect_client_shell = require("./detect-client-shell");
+var import_extract_meta_refresh = require("./extract-meta-refresh");
 var import_metadata = require("./metadata");
 var import_normalize = require("./normalize");
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
   BUILTIN_ADAPTERS,
+  MAX_META_REFRESH_DELAY_SECONDS,
+  SHELL_EMPTY_THRESHOLD,
+  SHELL_SUSPICION_THRESHOLD,
   adapterById,
   autoExtractor,
   buildPageSignature,
   createAutoExtractor,
   defineSelectorAdapter,
+  detectClientShell,
+  extractMetaRefresh,
   extractMetadata,
   htmlToMarkdown,
   htmlToMarkdownWithMetadata,

package/dist/cjs/link-replacer.js

@@ -21,19 +21,25 @@ __export(link_replacer_exports, {
   linkReplacer: () => linkReplacer
 });
 module.exports = __toCommonJS(link_replacer_exports);
-const linkReplacer = (markdown, base) => {
-  const regex = /(!?\[.*?\]\()([^)\s]+)(\))/g;
-  return markdown.replace(regex, (match, pre, url, post) => {
-    if (/^(https?:|#)/.test(url)) {
-      return match;
-    }
-    try {
-      const absoluteUrl = new URL(url, base).href;
-      return `${pre}${absoluteUrl}${post}`;
-    } catch {
-      return match;
-    }
-  });
+const absoluteUrl = (url, base) => {
+  if (/^(?:[a-z][a-z\d+.-]*:|#)/i.test(url)) {
+    return url;
+  }
+  try {
+    return new URL(url, base).href;
+  } catch {
+    return url;
+  }
+};
+const linkReplacer = (node, base) => {
+  let result = node;
+  if (node.type === "link" || node.type === "image" || node.type === "definition") {
+    result = { ...node, url: absoluteUrl(node.url, base) };
+  }
+  if ("children" in result) {
+    return { ...result, children: result.children.map((child) => linkReplacer(child, base)) };
+  }
+  return result;
 };
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {

package/dist/cjs/loaders/cf-puppeteer.js

@@ -32,15 +32,22 @@ __export(cf_puppeteer_exports, {
 });
 module.exports = __toCommonJS(cf_puppeteer_exports);
 var import_puppeteer = __toESM(require("@cloudflare/puppeteer"), 1);
-const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
 const loadHtml = async (url, ctx) => {
   const browser = await import_puppeteer.default.launch(ctx);
-  const page = await browser.newPage();
-  await page.goto(url);
-  const html = await page.content();
-  await Promise.race([page.waitForNetworkIdle(), sleep(1e4)]);
-  await page.close();
-  return html;
+  try {
+    const page = await browser.newPage();
+    await page.goto(url);
+    try {
+      await page.waitForNetworkIdle({ timeout: 1e4 });
+    } catch (error) {
+      if (!(error instanceof import_puppeteer.TimeoutError)) {
+        throw error;
+      }
+    }
+    return await page.content();
+  } finally {
+    await browser.close();
+  }
 };
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {

package/dist/cjs/loaders/fetch.js

@@ -22,10 +22,22 @@ __export(fetch_exports, {
   loadHtml: () => loadHtml
 });
 module.exports = __toCommonJS(fetch_exports);
+var import_extract_meta_refresh = require("../extract-meta-refresh");
 const USER_AGENT = "mozilla/5.0 (windows nt 10.0; win64; x64) applewebkit/537.36 (khtml, like gecko) chrome/125.0.0.0 safari/537.36";
+const MAX_META_REFRESH_HOPS = 3;
 const loadHtml = async (url, userAgent = USER_AGENT) => {
-  const response = await fetch(url, { headers: { "User-Agent": userAgent } });
-  return response.text();
+  let current = url;
+  let html = "";
+  for (let hop = 0; hop <= MAX_META_REFRESH_HOPS; hop++) {
+    const response = await fetch(current, { headers: { "User-Agent": userAgent } });
+    html = await response.text();
+    const refresh = (0, import_extract_meta_refresh.extractMetaRefresh)(html, response.url || current);
+    if (!refresh) {
+      break;
+    }
+    current = refresh.url;
+  }
+  return html;
 };
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {

package/dist/cjs/loaders/playwright.js

@@ -35,40 +35,42 @@ const SUPER_BYPASS_DEVICE = {
 const loadHtml = async (url, options) => {
   const { browser, waitUntil, timeout, superBypassMode } = options ?? {};
   const _browser = browser ?? await import_playwright_core.chromium.launch({ headless: true });
-  const context = await _browser.newContext(superBypassMode ? SUPER_BYPASS_DEVICE : import_playwright_core.devices["Desktop Chrome"]);
-  if (superBypassMode) {
-    await context.addInitScript(() => {
-      Object.defineProperty(navigator, "webdriver", { get: () => void 0 });
-      Object.defineProperty(navigator, "languages", { get: () => ["en-US", "en"] });
-      Object.defineProperty(navigator, "plugins", { get: () => [1, 2, 3, 4, 5] });
-    });
-  }
-  const page = await context.newPage();
-  if (superBypassMode) {
-    await page.route("**/*.js", (route) => {
-      if (route.request().url().includes("captcha-delivery")) {
-        return route.abort();
+  let context;
+  try {
+    context = await _browser.newContext(superBypassMode ? SUPER_BYPASS_DEVICE : import_playwright_core.devices["Desktop Chrome"]);
+    if (superBypassMode) {
+      await context.addInitScript(() => {
+        Object.defineProperty(navigator, "webdriver", { get: () => void 0 });
+        Object.defineProperty(navigator, "languages", { get: () => ["en-US", "en"] });
+        Object.defineProperty(navigator, "plugins", { get: () => [1, 2, 3, 4, 5] });
+      });
+    }
+    const page = await context.newPage();
+    if (superBypassMode) {
+      await page.route("**/*.js", (route) => {
+        if (route.request().url().includes("captcha-delivery")) {
+          return route.abort();
+        }
+        return route.continue();
+      });
+    }
+    await page.goto(url, { waitUntil: waitUntil ?? "load", timeout });
+    await page.evaluate(() => {
+      const elements = document.querySelectorAll("*");
+      for (const element of elements) {
+        const rect = element.getBoundingClientRect();
+        element.setAttribute("data-rwidth", rect.width.toString());
+        element.setAttribute("data-rheight", rect.height.toString());
       }
-      return route.continue();
     });
-  }
-  await page.goto(url, { waitUntil: waitUntil ?? "load", timeout });
-  await page.evaluate(() => {
-    const elements = document.querySelectorAll("*");
-    for (const element of elements) {
-      const rect = element.getBoundingClientRect();
-      element.setAttribute("data-rwidth", rect.width.toString());
-      element.setAttribute("data-rheight", rect.height.toString());
+    return await page.content();
+  } finally {
+    if (browser) {
+      await context?.close();
+    } else {
+      await _browser.close();
     }
-  });
-  const html = await page.content();
-  await page.close();
-  if (browser) {
-    await context.close();
-  } else {
-    await _browser.close();
   }
-  return html;
 };
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {

package/dist/cjs/loaders/puppeteer.js

@@ -32,7 +32,6 @@ __export(puppeteer_exports, {
 });
 module.exports = __toCommonJS(puppeteer_exports);
 var import_puppeteer = __toESM(require("puppeteer"), 1);
-const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
 const loadHtml = async (url, ctx) => {
   const browser = await import_puppeteer.default.launch(
     ctx || {
@@ -40,12 +39,20 @@ const loadHtml = async (url, ctx) => {
       args: ["--no-sandbox", "--disable-setuid-sandbox"]
     }
   );
-  const page = await browser.newPage();
-  await page.goto(url);
-  const html = await page.content();
-  await Promise.race([page.waitForNetworkIdle(), sleep(1e4)]);
-  await browser.close();
-  return html;
+  try {
+    const page = await browser.newPage();
+    await page.goto(url);
+    try {
+      await page.waitForNetworkIdle({ timeout: 1e4 });
+    } catch (error) {
+      if (!(error instanceof import_puppeteer.TimeoutError)) {
+        throw error;
+      }
+    }
+    return await page.content();
+  } finally {
+    await browser.close();
+  }
 };
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {

package/dist/cjs/md-splitter.js

@@ -25,15 +25,13 @@ var import_mdast_to_markdown = require("./mdast-to-markdown");
 var import_common = require("./utils/common");
 var import_mdast_utils = require("./utils/mdast-utils");
 const PRIORITY_SPLITTERS = ["h1", "h2", "h3", "h4", "h5", "h6", "list", "table", "code"];
-const getSplitterGenerator = function* () {
-  for (const splitter of PRIORITY_SPLITTERS) {
-    yield splitter;
-  }
-};
-const _mdastSplitter = async (contents, checker, splitterGenerator) => {
-  const splitter = splitterGenerator.next().value;
+const _mdastSplitter = async (contents, checker, priority, signal) => {
+  signal?.throwIfAborted();
+  const splitter = PRIORITY_SPLITTERS[priority];
   const markdown = (0, import_mdast_to_markdown.mdastToMarkdown)((0, import_mdast_utils.warpRoot)(contents));
-  if (await checker(markdown) || contents.length === 1) {
+  const accepted = await checker(markdown);
+  signal?.throwIfAborted();
+  if (accepted || contents.length === 1) {
     return [contents];
   }
   const chunked = splitter ? contents.reduce((acc, content) => {
@@ -44,11 +42,26 @@ const _mdastSplitter = async (contents, checker, splitterGenerator) => {
     acc[acc.length - 1].push(content);
     return acc;
   }, []) : (0, import_common.chunk)(contents, Math.ceil(contents.length / 2));
-  const splitting = chunked.map((chunk2) => _mdastSplitter(chunk2, checker, splitterGenerator));
+  const splitting = chunked.map((chunk2) => _mdastSplitter(chunk2, checker, priority + 1, signal));
   return Promise.all(splitting).then((chunks) => chunks.flat());
 };
-const mdastSplitter = (mdast, checker, _options) => {
-  return _mdastSplitter((0, import_mdast_utils.unwarpRoot)(mdast), checker, getSplitterGenerator());
+const mdastSplitter = async (mdast, checker, options) => {
+  const signal = options?.signal;
+  if (!signal) {
+    return _mdastSplitter((0, import_mdast_utils.unwarpRoot)(mdast), checker, 0);
+  }
+  signal.throwIfAborted();
+  let onAbort = () => {
+  };
+  const aborted = new Promise((_, reject) => {
+    onAbort = () => reject(signal.reason);
+    signal.addEventListener("abort", onAbort, { once: true });
+  });
+  try {
+    return await Promise.race([_mdastSplitter((0, import_mdast_utils.unwarpRoot)(mdast), checker, 0, signal), aborted]);
+  } finally {
+    signal.removeEventListener("abort", onAbort);
+  }
 };
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {

package/dist/cjs/mdast-handlers/custom-code-handler.js

@@ -18,20 +18,30 @@ var __copyProps = (to, from, except, desc) => {
 var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
 var custom_code_handler_exports = {};
 __export(custom_code_handler_exports, {
+  codeLanguage: () => codeLanguage,
   customCodeHandler: () => customCodeHandler
 });
 module.exports = __toCommonJS(custom_code_handler_exports);
 var import_hast_util_to_text = require("hast-util-to-text");
 var import_trim_trailing_lines = require("trim-trailing-lines");
 var import_detect_code_lang = require("../utils/detect-code-lang");
-const LANGUAGE_MATCH_REGEX = [/language-(\w+)/, /highlight-source-(\w+)/, /CodeBlock--language-(\w+)/];
+var import_hast_fast = require("../utils/hast-fast");
+const LANGUAGE_MATCH_REGEX = [/^language-(\S+)$/, /^highlight-source-(\S+)$/, /^CodeBlock--language-(\S+)$/];
+const LANGUAGE_ATTRIBUTES = ["dataLanguage", "dataLang"];
+const attributeLanguage = (element) => {
+  for (const name of LANGUAGE_ATTRIBUTES) {
+    const value = element.properties?.[name];
+    if (typeof value === "string" && /^[\w+#.-]+$/.test(value.trim()) && value.trim() !== "plaintext") {
+      return value.trim();
+    }
+  }
+  return void 0;
+};
+const codeLanguage = (...elements) => elements.flatMap(import_hast_fast.classList).map((className) => LANGUAGE_MATCH_REGEX.map((regex) => className.match(regex)).find(Boolean)?.[1]).find(Boolean) ?? elements.map(attributeLanguage).find(Boolean);
 const customCodeHandler = (state, node) => {
-  const classNames = node.properties?.className || [];
-  const codeValue = (0, import_trim_trailing_lines.trimTrailingLines)((0, import_hast_util_to_text.toText)(node)).trim();
-  const classLang = classNames.map((className) => {
-    const match = LANGUAGE_MATCH_REGEX.map((regex) => className.match(regex)).find((match2) => match2);
-    return match?.[1];
-  }).find((className) => className);
+  const code = node.children.find((child) => (0, import_hast_fast.isElement)(child) && child.tagName === "code");
+  const codeValue = (0, import_trim_trailing_lines.trimTrailingLines)((0, import_hast_util_to_text.toText)(node));
+  const classLang = codeLanguage(...(0, import_hast_fast.isElement)(code) ? [code, node] : [node]);
   const lang = classLang || (0, import_detect_code_lang.detectLanguage)(codeValue) || null;
   const result = { type: "code", lang, meta: null, value: codeValue };
   state.patch(node, result);
@@ -39,5 +49,6 @@ const customCodeHandler = (state, node) => {
 };
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
+  codeLanguage,
   customCodeHandler
 });

package/dist/cjs/mdast-handlers/custom-div-handler.js

@@ -27,44 +27,23 @@ var import_hast_util_to_string = require("hast-util-to-string");
 var import_hast_util_to_text = require("hast-util-to-text");
 var import_trim_trailing_lines = require("trim-trailing-lines");
 var import_detect_code_lang = require("../utils/detect-code-lang");
+var import_hast_fast = require("../utils/hast-fast");
+var import_custom_code_handler = require("./custom-code-handler");
 const CODE_BLOCK_REGEX = /highlight-source|language-|codegroup|codeblock|code-block/i;
 const CODE_FILENAME_SELECTORS = "[class*='fileName'],[class*='fileName'],[class*='title'],[class*='Title']";
-const LANGUAGE_MATCH_REGEX = [/language-(\w+)/, /highlight-source-(\w+)/, /CodeBlock--language-(\w+)/];
-const findRecursive = (array, condition, maxDepth = 3) => {
-  if (maxDepth <= 0) {
-    return null;
-  }
-  for (const value of array) {
-    const result = condition(value);
-    if (Array.isArray(result)) {
-      return findRecursive(result, condition, maxDepth - 1);
-    }
-    if (result) {
-      return value;
-    }
-  }
-  return null;
-};
 const customDivHandler = (state, node) => {
-  const classNames = Array.isArray(node.properties.className) ? node.properties.className : [];
-  const codeBlock = findRecursive(node.children, (child) => {
-    if (child.type !== "element") {
-      return false;
-    }
-    if (child.tagName === "pre") {
-      return true;
-    }
-    return child.children.filter((child2) => child2.type === "element");
-  });
-  if (codeBlock && classNames.some((className) => CODE_BLOCK_REGEX.test(className))) {
-    const codeBlockClassNames = codeBlock.type === "element" ? codeBlock.properties.className ?? [] : [];
-    const codeValue = (0, import_trim_trailing_lines.trimTrailingLines)((0, import_hast_util_to_text.toText)(codeBlock)).trim();
+  const classNames = (0, import_hast_fast.classList)(node);
+  if (!classNames.some((className) => CODE_BLOCK_REGEX.test(className))) {
+    return import_hast_util_to_mdast.defaultHandlers.div(state, node);
+  }
+  const codeBlocks = (0, import_hast_fast.collectElements)(node, (element) => element.tagName === "pre");
+  const codeBlock = codeBlocks.length === 1 ? codeBlocks[0] : void 0;
+  if (codeBlock) {
+    const codeValue = (0, import_trim_trailing_lines.trimTrailingLines)((0, import_hast_util_to_text.toText)(codeBlock));
     const filenameElement = (0, import_hast_util_select.select)(CODE_FILENAME_SELECTORS, node);
     const fileLang = filenameElement ? (0, import_hast_util_to_string.toString)(filenameElement).match(/\.(\w+)$/)?.[1] : null;
-    const classLang = [...classNames, ...codeBlockClassNames].map((className) => {
-      const match = LANGUAGE_MATCH_REGEX.map((regex) => className.match(regex)).find((match2) => match2);
-      return match?.[1];
-    }).find((className) => className);
+    const code = codeBlock.children.find((child) => (0, import_hast_fast.isElement)(child) && child.tagName === "code");
+    const classLang = (0, import_custom_code_handler.codeLanguage)(...(0, import_hast_fast.isElement)(code) ? [code, codeBlock, node] : [codeBlock, node]);
     const lang = fileLang || classLang || (0, import_detect_code_lang.detectLanguage)(codeValue) || null;
     const result = { type: "code", lang, meta: null, value: codeValue };
     state.patch(node, result);

package/dist/cjs/mdast-handlers/definition-list-handler.js

@@ -50,6 +50,20 @@ const hoistBoundaryWhitespace = (nodes) => {
   }
   return result;
 };
+const termsAndDefinitions = (parent) => {
+  const found = [];
+  for (const child of parent.children) {
+    if (child.type !== "element") {
+      continue;
+    }
+    if (child.tagName === "dt" || child.tagName === "dd") {
+      found.push(child);
+    } else if (child.tagName === "div") {
+      found.push(...termsAndDefinitions(child));
+    }
+  }
+  return found;
+};
 const definitionListHandler = (state, node) => {
   const items = [];
   let current;
@@ -64,11 +78,7 @@ const definitionListHandler = (state, node) => {
     items.push({ type: "listItem", spread: false, children: [heading, ...current.definitions] });
     current = void 0;
   };
-  for (const child of node.children) {
-    if (child.type !== "element") {
-      continue;
-    }
-    const element = child;
+  for (const element of termsAndDefinitions(node)) {
     if (element.tagName === "dt") {
       if (current && current.definitions.length > 0) {
         flush();
@@ -87,7 +97,8 @@ const definitionListHandler = (state, node) => {
   }
   flush();
   if (items.length === 0) {
-    return void 0;
+    const fallback = state.toFlow(state.all(node));
+    return fallback.length > 0 ? fallback : void 0;
   }
   const result = { type: "list", ordered: false, spread: false, children: items };
   state.patch(node, result);

package/dist/cjs/mdast-to-markdown.js

@@ -33,11 +33,8 @@ const DEFAULT_MDAST_TO_MARKDOWN_OPTIONS = {
 };
 const mdastToMarkdown = (mdast, options) => {
   const { baseUrl, ...toMarkdownOptions } = { ...DEFAULT_MDAST_TO_MARKDOWN_OPTIONS, ...options };
-  let markdown = (0, import_mdast_util_to_markdown.toMarkdown)((0, import_mdast_utils.warpRoot)(mdast), toMarkdownOptions).replace(/\*\*\*\*/g, "");
-  if (baseUrl) {
-    markdown = (0, import_link_replacer.linkReplacer)(markdown, baseUrl);
-  }
-  return markdown;
+  const tree = (0, import_mdast_utils.warpRoot)(mdast);
+  return (0, import_mdast_util_to_markdown.toMarkdown)(baseUrl ? (0, import_link_replacer.linkReplacer)(tree, baseUrl) : tree, toMarkdownOptions);
 };
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {