webforai 4.2.0 → 4.3.0

package/dist/cjs/adapters/sites/youtube.js

@@ -18,6 +18,7 @@ var __copyProps = (to, from, except, desc) => {
 var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
 var youtube_exports = {};
 __export(youtube_exports, {
+  PLAYER_RESPONSE_VARIABLE: () => PLAYER_RESPONSE_VARIABLE,
   extractAssignedJson: () => extractAssignedJson,
   isYoutubeElement: () => isYoutubeElement,
   youtubeAdapter: () => youtubeAdapter,
@@ -69,12 +70,13 @@ const extractAssignedJson = (source, name) => {
 const scriptText = (root) => (0, import_hast_fast.collectElements)(root, (element2) => element2.tagName === "script").map(
   (script) => script.children.map((child) => child.type === "text" ? child.value : "").join("")
 );
+const PLAYER_RESPONSE_VARIABLE = "ytInitialPlayerResponse";
 const findPlayerResponse = (root) => {
   for (const source of scriptText(root)) {
-    if (!source.includes("ytInitialPlayerResponse")) {
+    if (!source.includes(PLAYER_RESPONSE_VARIABLE)) {
       continue;
     }
-    const parsed = extractAssignedJson(source, "ytInitialPlayerResponse");
+    const parsed = extractAssignedJson(source, PLAYER_RESPONSE_VARIABLE);
     if (parsed && typeof parsed === "object") {
       return parsed;
     }
@@ -184,6 +186,7 @@ const youtubeAdapter = {
 const isYoutubeElement = import_hast_fast.isElement;
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
+  PLAYER_RESPONSE_VARIABLE,
   extractAssignedJson,
   isYoutubeElement,
   youtubeAdapter,

package/dist/cjs/detect-client-shell.js

@@ -18,6 +18,7 @@ var __copyProps = (to, from, except, desc) => {
 var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
 var detect_client_shell_exports = {};
 __export(detect_client_shell_exports, {
+  CHALLENGE_PATTERN: () => CHALLENGE_PATTERN,
   SHELL_EMPTY_THRESHOLD: () => SHELL_EMPTY_THRESHOLD,
   SHELL_SUSPICION_THRESHOLD: () => SHELL_SUSPICION_THRESHOLD,
   detectClientShell: () => detectClientShell
@@ -72,6 +73,7 @@ const detectClientShell = (html) => {
 };
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
+  CHALLENGE_PATTERN,
   SHELL_EMPTY_THRESHOLD,
   SHELL_SUSPICION_THRESHOLD,
   detectClientShell

package/dist/cjs/extractors/lib/terminators.js

@@ -26,6 +26,8 @@ var import_hast_fast = require("../../utils/hast-fast");
 const TERMINATOR_PATTERNS = [
   /^was this (page|article|document) helpful/i,
   /^thanks for the feedback/i,
+  // Site-generator credit closing the page ("Powered by Docsify.js", "Powered by GitBook").
+  /^powered by\b/i,
   /^(this page was )?last (modified|updated) (on|at|:)?\s/i,
   /^learn how to contribute/i,
   /^help (us )?improve\b/i,

package/dist/cjs/index.js

@@ -49,6 +49,7 @@ __export(src_exports, {
   readabilityExtractor: () => import_extractors.readabilityExtractor,
   resolveAdapter: () => import_adapters.resolveAdapter,
   runAdapters: () => import_adapters.runAdapters,
+  stripScriptBodies: () => import_parse_html.stripScriptBodies,
   takumiExtractor: () => import_extractors.takumiExtractor,
   toFrontmatter: () => import_metadata.toFrontmatter,
   youtubeCaptionTracks: () => import_adapters.youtubeCaptionTracks
@@ -64,6 +65,7 @@ var import_detect_client_shell = require("./detect-client-shell");
 var import_extract_meta_refresh = require("./extract-meta-refresh");
 var import_metadata = require("./metadata");
 var import_normalize = require("./normalize");
+var import_parse_html = require("./utils/parse-html");
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
   BUILTIN_ADAPTERS,
@@ -97,6 +99,7 @@ var import_normalize = require("./normalize");
   readabilityExtractor,
   resolveAdapter,
   runAdapters,
+  stripScriptBodies,
   takumiExtractor,
   toFrontmatter,
   youtubeCaptionTracks

package/dist/cjs/mdast-handlers/custom-code-handler.js

@@ -29,17 +29,46 @@ var import_hast_fast = require("../utils/hast-fast");
 var import_pre_text = require("../utils/pre-text");
 var import_twoslash = require("./twoslash");
 const LANGUAGE_MATCH_REGEX = [/^language-(\S+)$/, /^highlight-source-(\S+)$/, /^CodeBlock--language-(\S+)$/];
+const LANGUAGE_TOKEN = /^[\w+#.-]+$/;
+const NO_LANGUAGE = /* @__PURE__ */ new Set(["plain", "plaintext", "text", "none", "nohighlight"]);
 const LANGUAGE_ATTRIBUTES = ["dataLanguage", "dataLang"];
 const attributeLanguage = (element) => {
   for (const name of LANGUAGE_ATTRIBUTES) {
     const value = element.properties?.[name];
-    if (typeof value === "string" && /^[\w+#.-]+$/.test(value.trim()) && value.trim() !== "plaintext") {
+    if (typeof value === "string" && LANGUAGE_TOKEN.test(value.trim()) && value.trim() !== "plaintext") {
       return value.trim();
     }
   }
   return void 0;
 };
-const codeLanguage = (...elements) => elements.flatMap(import_hast_fast.classList).map((className) => LANGUAGE_MATCH_REGEX.map((regex) => className.match(regex)).find(Boolean)?.[1]).find(Boolean) ?? elements.map(attributeLanguage).find(Boolean);
+const SANDPACK_LANGUAGES = /* @__PURE__ */ new Set([
+  "javascript",
+  "typescript",
+  "jsx",
+  "tsx",
+  "html",
+  "css",
+  "less",
+  "scss",
+  "sass",
+  "json",
+  "markdown",
+  "python",
+  "vue",
+  "svelte"
+]);
+const conventionLanguage = (element) => {
+  const classes = (0, import_hast_fast.classList)(element);
+  const brushAt = classes.findIndex((name) => name.startsWith("brush:"));
+  if (brushAt >= 0) {
+    const brush = classes[brushAt] === "brush:" ? classes[brushAt + 1] : classes[brushAt].slice("brush:".length);
+    const lang = brush?.toLowerCase();
+    return lang && LANGUAGE_TOKEN.test(lang) && !NO_LANGUAGE.has(lang) ? lang : void 0;
+  }
+  const sandpack = classes.find((name) => name.startsWith("sp-") && SANDPACK_LANGUAGES.has(name.slice(3)));
+  return sandpack?.slice(3);
+};
+const codeLanguage = (...elements) => elements.flatMap(import_hast_fast.classList).map((className) => LANGUAGE_MATCH_REGEX.map((regex) => className.match(regex)).find(Boolean)?.[1]).find(Boolean) ?? elements.map(attributeLanguage).find(Boolean) ?? elements.map(conventionLanguage).find(Boolean);
 const codeMeta = (pre) => {
   const title = pre.properties?.dataTitle;
   if (typeof title !== "string") {

package/dist/cjs/normalize/index.js

@@ -20,11 +20,13 @@ var normalize_exports = {};
 __export(normalize_exports, {
   bestFromSrcset: () => bestFromSrcset,
   dedupeRenderedMath: () => dedupeRenderedMath,
+  foldCodeLabels: () => foldCodeLabels,
   normalizeHast: () => normalizeHast,
   promoteAriaHeadings: () => promoteAriaHeadings,
   removeIconFonts: () => removeIconFonts,
   resolveImageSources: () => resolveImageSources,
-  separateAdjacentInlines: () => separateAdjacentInlines
+  separateAdjacentInlines: () => separateAdjacentInlines,
+  unwrapHeadingAnchors: () => unwrapHeadingAnchors
 });
 module.exports = __toCommonJS(normalize_exports);
 var import_hast_fast = require("../utils/hast-fast");
@@ -164,6 +166,82 @@ const removeIconFonts = (tree) => {
     (0, import_hast_fast.pruneInPlace)(tree, (node) => !isIcon(node));
   }
 };
+const textOfNode = (node) => node.type === "text" ? node.value : "children" in node ? node.children.map(textOfNode).join("") : "";
+const unwrapHeadingAnchors = (tree) => {
+  (0, import_hast_fast.walk)(tree, (node) => {
+    if (!((0, import_hast_fast.isElement)(node) && /^h[1-6]$/.test(node.tagName))) {
+      return;
+    }
+    const title = textOfNode(node).trim();
+    if (title === "") {
+      return;
+    }
+    const anchorAt = node.children.findIndex(
+      (child) => (0, import_hast_fast.isElement)(child) && child.tagName === "a" && (0, import_hast_fast.stringProperty)(child, "href")?.startsWith("#") && textOfNode(child).trim() === title
+    );
+    const anchor = node.children[anchorAt];
+    if ((0, import_hast_fast.isElement)(anchor)) {
+      node.children.splice(anchorAt, 1, ...anchor.children);
+    }
+  });
+};
+const LABEL_LANGUAGES = /* @__PURE__ */ new Set([
+  "js",
+  "javascript",
+  "ts",
+  "typescript",
+  "jsx",
+  "tsx",
+  "html",
+  "css",
+  "scss",
+  "json",
+  "bash",
+  "sh",
+  "shell",
+  "console",
+  "python",
+  "py",
+  "sql",
+  "xml",
+  "yaml",
+  "toml",
+  "http",
+  "rust",
+  "go",
+  "java",
+  "kotlin",
+  "swift",
+  "c",
+  "cpp",
+  "md",
+  "markdown",
+  "diff"
+]);
+const LABEL_NO_LANGUAGE = /* @__PURE__ */ new Set(["plain", "plaintext", "text"]);
+const foldCodeLabels = (tree) => {
+  (0, import_hast_fast.walk)(tree, (node) => {
+    if (!(0, import_hast_fast.isElement)(node) || node.tagName === "pre") {
+      return;
+    }
+    const elements = node.children.filter(import_hast_fast.isElement);
+    if (elements.length !== 2 || node.children.some((child) => child.type === "text" && child.value.trim() !== "")) {
+      return;
+    }
+    const [label, pre] = elements;
+    if (pre.tagName !== "pre" || /^(h[1-6]|pre)$/.test(label.tagName) || (0, import_hast_fast.findElement)(label, (el) => el.tagName === "pre")) {
+      return;
+    }
+    const name = textOfNode(label).trim().toLowerCase();
+    if (!(LABEL_LANGUAGES.has(name) || LABEL_NO_LANGUAGE.has(name))) {
+      return;
+    }
+    if (LABEL_LANGUAGES.has(name) && pre.properties.dataLanguage === void 0) {
+      pre.properties.dataLanguage = name;
+    }
+    node.children = node.children.filter((child) => child !== label);
+  });
+};
 const SPACED_INLINE = /* @__PURE__ */ new Set(["span", "time", "label"]);
 const SCRIPTS = [
   ["latin", /\p{Script=Latin}/u],
@@ -180,7 +258,6 @@ const needsSpace = (left, right) => {
   return a !== void 0 && b !== void 0 && a !== b;
 };
 const NO_SPACING = /* @__PURE__ */ new Set(["pre", "code", "kbd", "samp", "ruby", "math", "svg"]);
-const textOfNode = (node) => node.type === "text" ? node.value : "children" in node ? node.children.map(textOfNode).join("") : "";
 const separateAdjacentInlines = (tree) => {
   const visit = (node) => {
     if (!("children" in node) || (0, import_hast_fast.isElement)(node) && NO_SPACING.has(node.tagName)) {
@@ -201,7 +278,15 @@ const separateAdjacentInlines = (tree) => {
   visit(tree);
 };
 const normalizeHast = (tree, options = {}) => {
-  const { images = true, math = true, ariaHeadings = true, iconFonts = true, inlineSpacing = true } = options;
+  const {
+    images = true,
+    math = true,
+    ariaHeadings = true,
+    iconFonts = true,
+    headingAnchors = true,
+    codeLabels = true,
+    inlineSpacing = true
+  } = options;
   if (images) {
     resolveImageSources(tree);
   }
@@ -214,6 +299,12 @@ const normalizeHast = (tree, options = {}) => {
   if (iconFonts) {
     removeIconFonts(tree);
   }
+  if (headingAnchors) {
+    unwrapHeadingAnchors(tree);
+  }
+  if (codeLabels) {
+    foldCodeLabels(tree);
+  }
   if (inlineSpacing) {
     separateAdjacentInlines(tree);
   }
@@ -223,9 +314,11 @@ const normalizeHast = (tree, options = {}) => {
 0 && (module.exports = {
   bestFromSrcset,
   dedupeRenderedMath,
+  foldCodeLabels,
   normalizeHast,
   promoteAriaHeadings,
   removeIconFonts,
   resolveImageSources,
-  separateAdjacentInlines
+  separateAdjacentInlines,
+  unwrapHeadingAnchors
 });

package/dist/cjs/utils/detect-code-lang.js

@@ -62,11 +62,10 @@ const languages = [
   ["json", [/\b(true|false|null|\{})\b|\"[^"]+\":/g, 10]],
   ["yaml", [/^(\s+)?[a-z][a-z0-9]*:/gim, 10]]
 ];
-const detectLanguage = (code) => {
-  return languages.map(
-    ([lang, ...features]) => [lang, features.reduce((acc, [match, score]) => acc + [...code.matchAll(match)].length * score, 0)]
-  ).filter(([_, score]) => score > 20).sort((a, b) => b[1] - a[1])[0]?.[0] || "plain";
-};
+const MIN_CONFIDENT_SCORE = 400;
+const detectLanguage = (code) => languages.map(
+  ([lang, ...features]) => [lang, features.reduce((acc, [match, score]) => acc + [...code.matchAll(match)].length * score, 0)]
+).filter(([_, score]) => score >= MIN_CONFIDENT_SCORE).sort((a, b) => b[1] - a[1])[0]?.[0];
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
   detectLanguage

package/dist/cjs/utils/parse-html.js

@@ -26,6 +26,8 @@ __export(parse_html_exports, {
 module.exports = __toCommonJS(parse_html_exports);
 var import_hast_util_from_parse5 = require("hast-util-from-parse5");
 var import_parse5 = require("parse5");
+var import_youtube = require("../adapters/sites/youtube");
+var import_detect_client_shell = require("../detect-client-shell");
 const MAX_ELEMENT_DEPTH = 256;
 const isParent = (node) => "childNodes" in node;
 const LINE_ELEMENTS = /* @__PURE__ */ new Set([
@@ -92,12 +94,78 @@ const flattenDeep = (root, maxDepth) => {
     }
   }
 };
-const SCRIPT_OR_STYLE = /<(script|style)\b([^>]*)>[\s\S]*?<\/\1\s*>/gi;
+const SCRIPT_OR_STYLE_START = /<!--|<(script|style)(?=[\s/>])/gi;
+const CLOSING_TAG = {
+  script: /<\/script[\s/>]/gi,
+  style: /<\/style[\s/>]/gi
+};
+const startTagEnd = (html, from) => {
+  let index = from;
+  while (index < html.length) {
+    const char = html[index];
+    if (char === ">") {
+      return index + 1;
+    }
+    if (char === "=") {
+      let value = index + 1;
+      while (value < html.length && /\s/.test(html[value] ?? "")) {
+        value++;
+      }
+      const quote = html[value];
+      if (quote === '"' || quote === "'") {
+        const close = html.indexOf(quote, value + 1);
+        if (close < 0) {
+          return -1;
+        }
+        index = close + 1;
+        continue;
+      }
+      index = value;
+      continue;
+    }
+    index++;
+  }
+  return -1;
+};
+const isReadScript = (attributes, element) => /application\/ld\+json/i.test(attributes) || element.includes(import_youtube.PLAYER_RESPONSE_VARIABLE) || import_detect_client_shell.CHALLENGE_PATTERN.test(element);
 const LARGE_DOCUMENT_CHARS = 2e6;
-const stripScriptBodies = (html) => html.replace(
-  SCRIPT_OR_STYLE,
-  (match, tag, attributes) => /application\/ld\+json/i.test(attributes) ? match : `<${tag}${attributes}></${tag}>`
-);
+const stripScriptBodies = (html) => {
+  const out = [];
+  let copied = 0;
+  const start = new RegExp(SCRIPT_OR_STYLE_START);
+  for (let match = start.exec(html); match; match = start.exec(html)) {
+    if (match[1] === void 0) {
+      const commentEnd = html.indexOf("-->", match.index + 4);
+      if (commentEnd < 0) {
+        break;
+      }
+      start.lastIndex = commentEnd + 3;
+      continue;
+    }
+    const tag = match[1].toLowerCase();
+    const bodyStart = startTagEnd(html, match.index + match[0].length);
+    if (bodyStart < 0) {
+      break;
+    }
+    const closing = CLOSING_TAG[tag];
+    closing.lastIndex = bodyStart;
+    const close = closing.exec(html);
+    if (!close) {
+      break;
+    }
+    const tagEnd = html.indexOf(">", close.index);
+    const elementEnd = tagEnd < 0 ? html.length : tagEnd + 1;
+    const element = html.slice(match.index, elementEnd);
+    const openingTag = html.slice(match.index, bodyStart);
+    if (!(tag === "script" && isReadScript(openingTag, element))) {
+      out.push(html.slice(copied, bodyStart), html.slice(close.index, elementEnd));
+      copied = elementEnd;
+    }
+    start.lastIndex = elementEnd;
+  }
+  out.push(html.slice(copied));
+  return out.join("");
+};
 const parseHtml = (html, options = {}) => {
   const source = html.length > LARGE_DOCUMENT_CHARS ? stripScriptBodies(html) : html;
   const document = options.fragment ? (0, import_parse5.parseFragment)(source, { scriptingEnabled: false }) : (0, import_parse5.parse)(source, { scriptingEnabled: false });

package/dist/detect-client-shell.js

@@ -47,6 +47,7 @@ var detectClientShell = (html) => {
   return { isShell: false, visibleTextLength };
 };
 export {
+  CHALLENGE_PATTERN,
   SHELL_EMPTY_THRESHOLD,
   SHELL_SUSPICION_THRESHOLD,
   detectClientShell

package/dist/extractors/lib/terminators.js

@@ -3,6 +3,8 @@ import { isElement, isParent, pruneInPlace } from "../../utils/hast-fast.js";
 var TERMINATOR_PATTERNS = [
   /^was this (page|article|document) helpful/i,
   /^thanks for the feedback/i,
+  // Site-generator credit closing the page ("Powered by Docsify.js", "Powered by GitBook").
+  /^powered by\b/i,
   /^(this page was )?last (modified|updated) (on|at|:)?\s/i,
   /^learn how to contribute/i,
   /^help (us )?improve\b/i,

package/dist/index.js

@@ -42,6 +42,7 @@ import {
 } from "./extract-meta-refresh.js";
 import { extractMetadata, toFrontmatter } from "./metadata/index.js";
 import { normalizeHast } from "./normalize/index.js";
+import { stripScriptBodies } from "./utils/parse-html.js";
 export {
   BUILTIN_ADAPTERS,
   EXTRACTOR_PRESETS,
@@ -74,6 +75,7 @@ export {
   readabilityExtractor,
   resolveAdapter,
   runAdapters,
+  stripScriptBodies,
   takumiExtractor,
   toFrontmatter,
   youtubeCaptionTracks

package/dist/mdast-handlers/custom-code-handler.js

@@ -5,17 +5,46 @@ import { classList, isElement } from "../utils/hast-fast.js";
 import { preText } from "../utils/pre-text.js";
 import { cleanTwoslash } from "./twoslash.js";
 var LANGUAGE_MATCH_REGEX = [/^language-(\S+)$/, /^highlight-source-(\S+)$/, /^CodeBlock--language-(\S+)$/];
+var LANGUAGE_TOKEN = /^[\w+#.-]+$/;
+var NO_LANGUAGE = /* @__PURE__ */ new Set(["plain", "plaintext", "text", "none", "nohighlight"]);
 var LANGUAGE_ATTRIBUTES = ["dataLanguage", "dataLang"];
 var attributeLanguage = (element) => {
   for (const name of LANGUAGE_ATTRIBUTES) {
     const value = element.properties?.[name];
-    if (typeof value === "string" && /^[\w+#.-]+$/.test(value.trim()) && value.trim() !== "plaintext") {
+    if (typeof value === "string" && LANGUAGE_TOKEN.test(value.trim()) && value.trim() !== "plaintext") {
       return value.trim();
     }
   }
   return void 0;
 };
-var codeLanguage = (...elements) => elements.flatMap(classList).map((className) => LANGUAGE_MATCH_REGEX.map((regex) => className.match(regex)).find(Boolean)?.[1]).find(Boolean) ?? elements.map(attributeLanguage).find(Boolean);
+var SANDPACK_LANGUAGES = /* @__PURE__ */ new Set([
+  "javascript",
+  "typescript",
+  "jsx",
+  "tsx",
+  "html",
+  "css",
+  "less",
+  "scss",
+  "sass",
+  "json",
+  "markdown",
+  "python",
+  "vue",
+  "svelte"
+]);
+var conventionLanguage = (element) => {
+  const classes = classList(element);
+  const brushAt = classes.findIndex((name) => name.startsWith("brush:"));
+  if (brushAt >= 0) {
+    const brush = classes[brushAt] === "brush:" ? classes[brushAt + 1] : classes[brushAt].slice("brush:".length);
+    const lang = brush?.toLowerCase();
+    return lang && LANGUAGE_TOKEN.test(lang) && !NO_LANGUAGE.has(lang) ? lang : void 0;
+  }
+  const sandpack = classes.find((name) => name.startsWith("sp-") && SANDPACK_LANGUAGES.has(name.slice(3)));
+  return sandpack?.slice(3);
+};
+var codeLanguage = (...elements) => elements.flatMap(classList).map((className) => LANGUAGE_MATCH_REGEX.map((regex) => className.match(regex)).find(Boolean)?.[1]).find(Boolean) ?? elements.map(attributeLanguage).find(Boolean) ?? elements.map(conventionLanguage).find(Boolean);
 var codeMeta = (pre) => {
   const title = pre.properties?.dataTitle;
   if (typeof title !== "string") {

package/dist/normalize/index.js

@@ -145,6 +145,82 @@ var removeIconFonts = (tree) => {
     pruneInPlace(tree, (node) => !isIcon(node));
   }
 };
+var textOfNode = (node) => node.type === "text" ? node.value : "children" in node ? node.children.map(textOfNode).join("") : "";
+var unwrapHeadingAnchors = (tree) => {
+  walk(tree, (node) => {
+    if (!(isElement(node) && /^h[1-6]$/.test(node.tagName))) {
+      return;
+    }
+    const title = textOfNode(node).trim();
+    if (title === "") {
+      return;
+    }
+    const anchorAt = node.children.findIndex(
+      (child) => isElement(child) && child.tagName === "a" && stringProperty(child, "href")?.startsWith("#") && textOfNode(child).trim() === title
+    );
+    const anchor = node.children[anchorAt];
+    if (isElement(anchor)) {
+      node.children.splice(anchorAt, 1, ...anchor.children);
+    }
+  });
+};
+var LABEL_LANGUAGES = /* @__PURE__ */ new Set([
+  "js",
+  "javascript",
+  "ts",
+  "typescript",
+  "jsx",
+  "tsx",
+  "html",
+  "css",
+  "scss",
+  "json",
+  "bash",
+  "sh",
+  "shell",
+  "console",
+  "python",
+  "py",
+  "sql",
+  "xml",
+  "yaml",
+  "toml",
+  "http",
+  "rust",
+  "go",
+  "java",
+  "kotlin",
+  "swift",
+  "c",
+  "cpp",
+  "md",
+  "markdown",
+  "diff"
+]);
+var LABEL_NO_LANGUAGE = /* @__PURE__ */ new Set(["plain", "plaintext", "text"]);
+var foldCodeLabels = (tree) => {
+  walk(tree, (node) => {
+    if (!isElement(node) || node.tagName === "pre") {
+      return;
+    }
+    const elements = node.children.filter(isElement);
+    if (elements.length !== 2 || node.children.some((child) => child.type === "text" && child.value.trim() !== "")) {
+      return;
+    }
+    const [label, pre] = elements;
+    if (pre.tagName !== "pre" || /^(h[1-6]|pre)$/.test(label.tagName) || findElement(label, (el) => el.tagName === "pre")) {
+      return;
+    }
+    const name = textOfNode(label).trim().toLowerCase();
+    if (!(LABEL_LANGUAGES.has(name) || LABEL_NO_LANGUAGE.has(name))) {
+      return;
+    }
+    if (LABEL_LANGUAGES.has(name) && pre.properties.dataLanguage === void 0) {
+      pre.properties.dataLanguage = name;
+    }
+    node.children = node.children.filter((child) => child !== label);
+  });
+};
 var SPACED_INLINE = /* @__PURE__ */ new Set(["span", "time", "label"]);
 var SCRIPTS = [
   ["latin", /\p{Script=Latin}/u],
@@ -161,7 +237,6 @@ var needsSpace = (left, right) => {
   return a !== void 0 && b !== void 0 && a !== b;
 };
 var NO_SPACING = /* @__PURE__ */ new Set(["pre", "code", "kbd", "samp", "ruby", "math", "svg"]);
-var textOfNode = (node) => node.type === "text" ? node.value : "children" in node ? node.children.map(textOfNode).join("") : "";
 var separateAdjacentInlines = (tree) => {
   const visit = (node) => {
     if (!("children" in node) || isElement(node) && NO_SPACING.has(node.tagName)) {
@@ -182,7 +257,15 @@ var separateAdjacentInlines = (tree) => {
   visit(tree);
 };
 var normalizeHast = (tree, options = {}) => {
-  const { images = true, math = true, ariaHeadings = true, iconFonts = true, inlineSpacing = true } = options;
+  const {
+    images = true,
+    math = true,
+    ariaHeadings = true,
+    iconFonts = true,
+    headingAnchors = true,
+    codeLabels = true,
+    inlineSpacing = true
+  } = options;
   if (images) {
     resolveImageSources(tree);
   }
@@ -195,6 +278,12 @@ var normalizeHast = (tree, options = {}) => {
   if (iconFonts) {
     removeIconFonts(tree);
   }
+  if (headingAnchors) {
+    unwrapHeadingAnchors(tree);
+  }
+  if (codeLabels) {
+    foldCodeLabels(tree);
+  }
   if (inlineSpacing) {
     separateAdjacentInlines(tree);
   }
@@ -203,9 +292,11 @@ var normalizeHast = (tree, options = {}) => {
 export {
   bestFromSrcset,
   dedupeRenderedMath,
+  foldCodeLabels,
   normalizeHast,
   promoteAriaHeadings,
   removeIconFonts,
   resolveImageSources,
-  separateAdjacentInlines
+  separateAdjacentInlines,
+  unwrapHeadingAnchors
 };

package/dist/types/adapters/sites/youtube.d.ts

@@ -20,6 +20,8 @@ export interface CaptionTrack {
     isAutomatic: boolean;
 }
 export declare const extractAssignedJson: (source: string, name: string) => unknown;
+/** The script variable holding a watch page's title, description and caption tracks. */
+export declare const PLAYER_RESPONSE_VARIABLE = "ytInitialPlayerResponse";
 /**
  * Lists the caption tracks advertised by a YouTube watch page.
  *

package/dist/types/adapters/sites/youtube.js.map

@@ -1 +1 @@
-{"version":3,"file":"youtube.js","sourceRoot":"","sources":["../../../../src/adapters/sites/youtube.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAIH,OAAO,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAC;AAoBnE;;;;;GAKG;AACH;;;;;GAKG;AACH,MAAM,aAAa,GAAG,CAAC,MAAc,EAAE,KAAa,EAAU,EAAE;IAC/D,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,QAAQ,GAAG,KAAK,CAAC;IACrB,IAAI,OAAO,GAAG,KAAK,CAAC;IAEpB,KAAK,IAAI,KAAK,GAAG,KAAK,EAAE,KAAK,GAAG,MAAM,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;QACxD,MAAM,SAAS,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;QAEhC,IAAI,OAAO,EAAE,CAAC;YACb,OAAO,GAAG,KAAK,CAAC;QACjB,CAAC;aAAM,IAAI,SAAS,KAAK,IAAI,EAAE,CAAC;YAC/B,OAAO,GAAG,IAAI,CAAC;QAChB,CAAC;aAAM,IAAI,SAAS,KAAK,GAAG,EAAE,CAAC;YAC9B,QAAQ,GAAG,CAAC,QAAQ,CAAC;QACtB,CAAC;aAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;YACtB,IAAI,SAAS,KAAK,GAAG,EAAE,CAAC;gBACvB,KAAK,IAAI,CAAC,CAAC;YACZ,CAAC;iBAAM,IAAI,SAAS,KAAK,GAAG,IAAI,EAAE,KAAK,KAAK,CAAC,EAAE,CAAC;gBAC/C,OAAO,KAAK,GAAG,CAAC,CAAC;YAClB,CAAC;QACF,CAAC;IACF,CAAC;IAED,OAAO,CAAC,CAAC,CAAC;AACX,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,MAAc,EAAE,IAAY,EAAW,EAAE;IAC5E,MAAM,WAAW,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,IAAI,IAAI,CAAC,CAAC;IAChD,IAAI,WAAW,KAAK,CAAC,CAAC,EAAE,CAAC;QACxB,OAAO,SAAS,CAAC;IAClB,CAAC;IAED,MAAM,KAAK,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;IAC/C,IAAI,KAAK,KAAK,CAAC,CAAC,EAAE,CAAC;QAClB,OAAO,SAAS,CAAC;IAClB,CAAC;IAED,MAAM,GAAG,GAAG,aAAa,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;IACzC,IAAI,GAAG,KAAK,CAAC,CAAC,EAAE,CAAC;QAChB,OAAO,SAAS,CAAC;IAClB,CAAC;IAED,IAAI,CAAC;QACJ,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC;IAC7C,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,SAAS,CAAC;IAClB,CAAC;AACF,CAAC,CAAC;AAEF,MAAM,UAAU,GAAG,CAAC,IAAU,EAAY,EAAE,CAC3C,eAAe,CAAC,IAAI,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAC/E,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CACnF,CAAC;AAEH,MAAM,kBAAkB,GAAG,CAAC,IAAU,EAAuC,EAAE;IAC9E,KAAK,MAAM,MAAM,IAAI,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;QACvC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,yBAAyB,CAAC,EAAE,CAAC;YACjD,SAAS;QACV,CAAC;QACD,MAAM,MAAM,GAAG,mBAAmB,CAAC,MAAM,EAAE,yBAAyB,CAAC,CAAC;QACtE,IAAI,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;YAC1C,OAAO,MAAiC,CAAC;QAC1C,CAAC;IACF,CAAC;IACD,OAAO,SAAS,CAAC;AAClB,CAAC,CAAC;AAEF;;;;GAIG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,IAAU,EAAkB,EAAE;IAClE,MAAM,MAAM,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAC;IACxC,MAAM,QAAQ,GAAI,MAAM,EAAE,QAAgD,EAAE,+BAEhE,CAAC;IACb,MAAM,MAAM,GAAG,QAAQ,EAAE,aAAa,CAAC;IAEvC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QAC5B,OAAO,EAAE,CAAC;IACX,CAAC;IAED,OAAO,MAAM,CAAC,OAAO,CAAC,CAAC,KAA8B,EAAE,EAAE;QACxD,MAAM,GAAG,GAAG,OAAO,KAAK,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC;QAC1E,MAAM,YAAY,GAAG,OAAO,KAAK,CAAC,YAAY,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC,SAAS,CAAC;QAC7F,IAAI,CAAC,CAAC,GAAG,IAAI,YAAY,CAAC,EAAE,CAAC;YAC5B,OAAO,EAAE,CAAC;QACX,CAAC;QACD,MAAM,IAAI,GAAI,KAAK,CAAC,IAA4C,EAAE,UAAU,CAAC;QAC7E,OAAO;YACN;gBACC,GAAG;gBACH,YAAY;gBACZ,IAAI,EAAE,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS;gBACjD,WAAW,EAAE,KAAK,CAAC,IAAI,KAAK,KAAK;aACjC;SACD,CAAC;IACH,CAAC,CAAC,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,IAAI,GAAG,CAAC,KAAa,EAAkB,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;AAE1E,MAAM,OAAO,GAAG,CAAC,OAAe,EAAE,QAA0B,EAAW,EAAE,CAAC,CAAC;IAC1E,IAAI,EAAE,SAAS;IACf,OAAO;IACP,UAAU,EAAE,EAAE;IACd,QAAQ;CACR,CAAC,CAAC;AAEH,MAAM,OAAO,GAAG,CAAC,KAAa,EAAE,KAAa,EAAW,EAAE,CAAC,OAAO,CAAC,IAAI,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;AAE/F,MAAM,SAAS,GAAG,CAAC,KAAa,EAAW,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;AAE1E,4EAA4E;AAC5E,MAAM,gBAAgB,GAAG,CAAC,WAAmB,EAAa,EAAE,CAC3D,WAAW;KACT,KAAK,CAAC,QAAQ,CAAC;KACf,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;KAC5B,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;KACnC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CACd,OAAO,CACN,GAAG,EACH,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAC1G,CACD,CAAC;AAEJ,MAAM,cAAc,GAAG,CAAC,OAAe,EAAU,EAAE;IAClD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,IAAI,CAAC,CAAC;IACzC,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;IAClD,MAAM,SAAS,GAAG,OAAO,GAAG,EAAE,CAAC;IAC/B,MAAM,GAAG,GAAG,CAAC,KAAa,EAAU,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IACtE,OAAO,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,KAAK,IAAI,GAAG,CAAC,OAAO,CAAC,IAAI,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,OAAO,IAAI,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;AAClG,CAAC,CAAC;AAEF,MAAM,YAAY,GAAG,CAAC,OAAqB,EAAuB,EAAE;IACnE,MAAM,KAAK,GAAc,EAAE,CAAC;IAE5B,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QACpB,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,YAAY,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IACjE,CAAC;IACD,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;IAC9C,IAAI,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;QAC7C,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,aAAa,cAAc,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAC3E,CAAC;IACD,MAAM,KAAK,GAAG,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IACxC,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;QACzC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,UAAU,KAAK,CAAC,cAAc,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAC9E,CAAC;IAED,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;AAC5D,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,cAAc,GAAgB;IAC1C,EAAE,EAAE,SAAS;IAEb,OAAO,EAAE,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE;QACpB,IAAI,CAAC,GAAG,EAAE,CAAC;YACV,OAAO,KAAK,CAAC;QACd,CAAC;QACD,IAAI,CAAC;YACJ,OAAO,uDAAuD,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC;QAC5F,CAAC;QAAC,MAAM,CAAC;YACR,OAAO,KAAK,CAAC;QACd,CAAC;IACF,CAAC;IAED,6FAA6F;IAC7F,iEAAiE;IACjE,kBAAkB,EAAE,CAAC,EAAE,SAAS,EAAE,EAAE,EAAE,CACrC,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,SAAS,EAAE,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;IAEjF,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE;QACrB,MAAM,MAAM,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAC;QACxC,MAAM,OAAO,GAAG,MAAM,EAAE,YAAwC,CAAC;QAEjE,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,CAAC;YACrB,OAAO,SAAS,CAAC;QAClB,CAAC;QAED,MAAM,QAAQ,GAAkB,CAAC,OAAO,CAAC,CAAC,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;QAE5D,MAAM,QAAQ,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;QACvC,IAAI,QAAQ,EAAE,CAAC;YACd,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACzB,CAAC;QAED,MAAM,WAAW,GAAG,OAAO,CAAC,gBAAgB,EAAE,IAAI,EAAE,CAAC;QACrD,IAAI,WAAW,EAAE,CAAC;YACjB,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,EAAE,aAAa,CAAC,CAAC,CAAC;YACzC,QAAQ,CAAC,IAAI,CAAC,GAAG,gBAAgB,CAAC,WAAW,CAAC,CAAC,CAAC;QACjD,CAAC;QAED,MAAM,MAAM,GAAG,oBAAoB,CAAC,IAAI,CAAC,CAAC;QAC1C,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvB,MAAM,SAAS,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YACjF,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,uBAAuB,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;QACzE,CAAC;QAED,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAiB,CAAC;IAClD,CAAC;CACD,CAAC;AAEF,iFAAiF;AACjF,MAAM,CAAC,MAAM,gBAAgB,GAAG,SAAS,CAAC"}
+{"version":3,"file":"youtube.js","sourceRoot":"","sources":["../../../../src/adapters/sites/youtube.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAIH,OAAO,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAC;AAoBnE;;;;;GAKG;AACH;;;;;GAKG;AACH,MAAM,aAAa,GAAG,CAAC,MAAc,EAAE,KAAa,EAAU,EAAE;IAC/D,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,QAAQ,GAAG,KAAK,CAAC;IACrB,IAAI,OAAO,GAAG,KAAK,CAAC;IAEpB,KAAK,IAAI,KAAK,GAAG,KAAK,EAAE,KAAK,GAAG,MAAM,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;QACxD,MAAM,SAAS,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;QAEhC,IAAI,OAAO,EAAE,CAAC;YACb,OAAO,GAAG,KAAK,CAAC;QACjB,CAAC;aAAM,IAAI,SAAS,KAAK,IAAI,EAAE,CAAC;YAC/B,OAAO,GAAG,IAAI,CAAC;QAChB,CAAC;aAAM,IAAI,SAAS,KAAK,GAAG,EAAE,CAAC;YAC9B,QAAQ,GAAG,CAAC,QAAQ,CAAC;QACtB,CAAC;aAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;YACtB,IAAI,SAAS,KAAK,GAAG,EAAE,CAAC;gBACvB,KAAK,IAAI,CAAC,CAAC;YACZ,CAAC;iBAAM,IAAI,SAAS,KAAK,GAAG,IAAI,EAAE,KAAK,KAAK,CAAC,EAAE,CAAC;gBAC/C,OAAO,KAAK,GAAG,CAAC,CAAC;YAClB,CAAC;QACF,CAAC;IACF,CAAC;IAED,OAAO,CAAC,CAAC,CAAC;AACX,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,MAAc,EAAE,IAAY,EAAW,EAAE;IAC5E,MAAM,WAAW,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,IAAI,IAAI,CAAC,CAAC;IAChD,IAAI,WAAW,KAAK,CAAC,CAAC,EAAE,CAAC;QACxB,OAAO,SAAS,CAAC;IAClB,CAAC;IAED,MAAM,KAAK,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;IAC/C,IAAI,KAAK,KAAK,CAAC,CAAC,EAAE,CAAC;QAClB,OAAO,SAAS,CAAC;IAClB,CAAC;IAED,MAAM,GAAG,GAAG,aAAa,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;IACzC,IAAI,GAAG,KAAK,CAAC,CAAC,EAAE,CAAC;QAChB,OAAO,SAAS,CAAC;IAClB,CAAC;IAED,IAAI,CAAC;QACJ,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC;IAC7C,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,SAAS,CAAC;IAClB,CAAC;AACF,CAAC,CAAC;AAEF,MAAM,UAAU,GAAG,CAAC,IAAU,EAAY,EAAE,CAC3C,eAAe,CAAC,IAAI,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAC/E,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CACnF,CAAC;AAEH,wFAAwF;AACxF,MAAM,CAAC,MAAM,wBAAwB,GAAG,yBAAyB,CAAC;AAElE,MAAM,kBAAkB,GAAG,CAAC,IAAU,EAAuC,EAAE;IAC9E,KAAK,MAAM,MAAM,IAAI,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;QACvC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,wBAAwB,CAAC,EAAE,CAAC;YAChD,SAAS;QACV,CAAC;QACD,MAAM,MAAM,GAAG,mBAAmB,CAAC,MAAM,EAAE,wBAAwB,CAAC,CAAC;QACrE,IAAI,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;YAC1C,OAAO,MAAiC,CAAC;QAC1C,CAAC;IACF,CAAC;IACD,OAAO,SAAS,CAAC;AAClB,CAAC,CAAC;AAEF;;;;GAIG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,IAAU,EAAkB,EAAE;IAClE,MAAM,MAAM,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAC;IACxC,MAAM,QAAQ,GAAI,MAAM,EAAE,QAAgD,EAAE,+BAEhE,CAAC;IACb,MAAM,MAAM,GAAG,QAAQ,EAAE,aAAa,CAAC;IAEvC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QAC5B,OAAO,EAAE,CAAC;IACX,CAAC;IAED,OAAO,MAAM,CAAC,OAAO,CAAC,CAAC,KAA8B,EAAE,EAAE;QACxD,MAAM,GAAG,GAAG,OAAO,KAAK,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC;QAC1E,MAAM,YAAY,GAAG,OAAO,KAAK,CAAC,YAAY,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC,SAAS,CAAC;QAC7F,IAAI,CAAC,CAAC,GAAG,IAAI,YAAY,CAAC,EAAE,CAAC;YAC5B,OAAO,EAAE,CAAC;QACX,CAAC;QACD,MAAM,IAAI,GAAI,KAAK,CAAC,IAA4C,EAAE,UAAU,CAAC;QAC7E,OAAO;YACN;gBACC,GAAG;gBACH,YAAY;gBACZ,IAAI,EAAE,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS;gBACjD,WAAW,EAAE,KAAK,CAAC,IAAI,KAAK,KAAK;aACjC;SACD,CAAC;IACH,CAAC,CAAC,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,IAAI,GAAG,CAAC,KAAa,EAAkB,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;AAE1E,MAAM,OAAO,GAAG,CAAC,OAAe,EAAE,QAA0B,EAAW,EAAE,CAAC,CAAC;IAC1E,IAAI,EAAE,SAAS;IACf,OAAO;IACP,UAAU,EAAE,EAAE;IACd,QAAQ;CACR,CAAC,CAAC;AAEH,MAAM,OAAO,GAAG,CAAC,KAAa,EAAE,KAAa,EAAW,EAAE,CAAC,OAAO,CAAC,IAAI,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;AAE/F,MAAM,SAAS,GAAG,CAAC,KAAa,EAAW,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;AAE1E,4EAA4E;AAC5E,MAAM,gBAAgB,GAAG,CAAC,WAAmB,EAAa,EAAE,CAC3D,WAAW;KACT,KAAK,CAAC,QAAQ,CAAC;KACf,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;KAC5B,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;KACnC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CACd,OAAO,CACN,GAAG,EACH,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAC1G,CACD,CAAC;AAEJ,MAAM,cAAc,GAAG,CAAC,OAAe,EAAU,EAAE;IAClD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,IAAI,CAAC,CAAC;IACzC,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;IAClD,MAAM,SAAS,GAAG,OAAO,GAAG,EAAE,CAAC;IAC/B,MAAM,GAAG,GAAG,CAAC,KAAa,EAAU,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IACtE,OAAO,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,KAAK,IAAI,GAAG,CAAC,OAAO,CAAC,IAAI,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,OAAO,IAAI,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;AAClG,CAAC,CAAC;AAEF,MAAM,YAAY,GAAG,CAAC,OAAqB,EAAuB,EAAE;IACnE,MAAM,KAAK,GAAc,EAAE,CAAC;IAE5B,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QACpB,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,YAAY,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IACjE,CAAC;IACD,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;IAC9C,IAAI,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;QAC7C,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,aAAa,cAAc,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAC3E,CAAC;IACD,MAAM,KAAK,GAAG,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IACxC,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;QACzC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,UAAU,KAAK,CAAC,cAAc,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAC9E,CAAC;IAED,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;AAC5D,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,cAAc,GAAgB;IAC1C,EAAE,EAAE,SAAS;IAEb,OAAO,EAAE,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE;QACpB,IAAI,CAAC,GAAG,EAAE,CAAC;YACV,OAAO,KAAK,CAAC;QACd,CAAC;QACD,IAAI,CAAC;YACJ,OAAO,uDAAuD,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC;QAC5F,CAAC;QAAC,MAAM,CAAC;YACR,OAAO,KAAK,CAAC;QACd,CAAC;IACF,CAAC;IAED,6FAA6F;IAC7F,iEAAiE;IACjE,kBAAkB,EAAE,CAAC,EAAE,SAAS,EAAE,EAAE,EAAE,CACrC,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,SAAS,EAAE,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;IAEjF,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE;QACrB,MAAM,MAAM,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAC;QACxC,MAAM,OAAO,GAAG,MAAM,EAAE,YAAwC,CAAC;QAEjE,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,CAAC;YACrB,OAAO,SAAS,CAAC;QAClB,CAAC;QAED,MAAM,QAAQ,GAAkB,CAAC,OAAO,CAAC,CAAC,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;QAE5D,MAAM,QAAQ,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;QACvC,IAAI,QAAQ,EAAE,CAAC;YACd,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACzB,CAAC;QAED,MAAM,WAAW,GAAG,OAAO,CAAC,gBAAgB,EAAE,IAAI,EAAE,CAAC;QACrD,IAAI,WAAW,EAAE,CAAC;YACjB,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,EAAE,aAAa,CAAC,CAAC,CAAC;YACzC,QAAQ,CAAC,IAAI,CAAC,GAAG,gBAAgB,CAAC,WAAW,CAAC,CAAC,CAAC;QACjD,CAAC;QAED,MAAM,MAAM,GAAG,oBAAoB,CAAC,IAAI,CAAC,CAAC;QAC1C,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvB,MAAM,SAAS,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YACjF,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,uBAAuB,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;QACzE,CAAC;QAED,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAiB,CAAC;IAClD,CAAC;CACD,CAAC;AAEF,iFAAiF;AACjF,MAAM,CAAC,MAAM,gBAAgB,GAAG,SAAS,CAAC"}

package/dist/types/detect-client-shell.d.ts

@@ -21,4 +21,6 @@ export interface ClientShellVerdict {
 export declare const SHELL_SUSPICION_THRESHOLD = 300;
 /** Below this length the document is a shell even without a framework marker. */
 export declare const SHELL_EMPTY_THRESHOLD = 50;
+/** Fingerprints of bot-challenge interstitials (Cloudflare, Imperva, PerimeterX, DataDome). */
+export declare const CHALLENGE_PATTERN: RegExp;
 export declare const detectClientShell: (html: string) => ClientShellVerdict;