webforai 4.2.0 → 4.3.0

package/dist/bin.js

@@ -6,7 +6,7 @@ import { CommanderError, program } from "commander";
 // package.json
 var package_default = {
   name: "webforai",
-  version: "4.2.0",
+  version: "4.3.0",
   description: "Convert web pages and local HTML to clean, LLM-ready Markdown: library, CLI (npx webforai) and hosted-API client.",
   author: "inaridiy",
   license: "Apache-2.0",
@@ -1360,8 +1360,440 @@ import path5 from "node:path";
 // src/utils/parse-html.ts
 import { fromParse5 } from "hast-util-from-parse5";
 import { parse, parseFragment } from "parse5";
+
+// src/utils/hast-fast.ts
+var EMPTY_METRICS = { text: 0, link: 0, commas: 0, paragraphs: 0, elements: 0 };
+var COMMA_REGEX = /[,、，،]/g;
+var PARAGRAPH_LIKE = /* @__PURE__ */ new Set(["p", "pre", "li", "blockquote", "figcaption", "dd", "dt"]);
+var NON_PROSE_TAGS = /* @__PURE__ */ new Set(["script", "style", "noscript", "template", "svg", "math", "iframe", "object"]);
+var isElement = (node) => typeof node === "object" && node !== null && node.type === "element";
+var isParent = (node) => typeof node === "object" && node !== null && Array.isArray(node.children);
+var classList = (element4) => {
+  const className = element4.properties?.className;
+  if (Array.isArray(className)) {
+    return className;
+  }
+  if (typeof className === "string") {
+    return className.split(/\s+/).filter(Boolean);
+  }
+  return [];
+};
+var matchString = (element4) => {
+  const id = element4.properties?.id;
+  const classes = classList(element4);
+  if (!id && classes.length === 0) {
+    return element4.tagName;
+  }
+  return `${element4.tagName} ${typeof id === "string" ? id : ""} ${classes.join(" ")}`;
+};
+var MetricsCollector = class {
+  #cache = /* @__PURE__ */ new WeakMap();
+  metrics(node) {
+    if (node.type === "text") {
+      const value = node.value;
+      if (value.trim().length === 0) {
+        return EMPTY_METRICS;
+      }
+      return {
+        text: value.length,
+        link: 0,
+        commas: (value.match(COMMA_REGEX) ?? []).length,
+        paragraphs: 0,
+        elements: 0
+      };
+    }
+    if (!isParent(node)) {
+      return EMPTY_METRICS;
+    }
+    const cached = this.#cache.get(node);
+    if (cached) {
+      return cached;
+    }
+    if (isElement(node) && NON_PROSE_TAGS.has(node.tagName)) {
+      this.#cache.set(node, EMPTY_METRICS);
+      return EMPTY_METRICS;
+    }
+    let text7 = 0;
+    let link = 0;
+    let commas = 0;
+    let paragraphs = 0;
+    let elements = 0;
+    for (const child of node.children) {
+      const childMetrics = this.metrics(child);
+      text7 += childMetrics.text;
+      link += childMetrics.link;
+      commas += childMetrics.commas;
+      paragraphs += childMetrics.paragraphs;
+      elements += childMetrics.elements;
+      if (isElement(child)) {
+        elements += 1;
+        if (PARAGRAPH_LIKE.has(child.tagName)) {
+          paragraphs += 1;
+        }
+      }
+    }
+    if (isElement(node) && node.tagName === "a") {
+      link = text7;
+    }
+    const metrics = { text: text7, link, commas, paragraphs, elements };
+    this.#cache.set(node, metrics);
+    return metrics;
+  }
+  textLength(node) {
+    return this.metrics(node).text;
+  }
+  /** Fraction of a subtree's text that is link anchor text. `0` for empty subtrees. */
+  linkDensity(node) {
+    const { text: text7, link } = this.metrics(node);
+    if (text7 === 0) {
+      return 0;
+    }
+    return Math.min(1, link / text7);
+  }
+};
+var pruneInPlace = (tree, keep) => {
+  if (!isParent(tree)) {
+    return tree;
+  }
+  const stack = [tree];
+  while (stack.length > 0) {
+    const parent = stack.pop();
+    const children = parent.children;
+    let write = 0;
+    for (let read = 0; read < children.length; read++) {
+      const child = children[read];
+      if (!keep(child, parent)) {
+        continue;
+      }
+      children[write++] = child;
+      if (isParent(child)) {
+        stack.push(child);
+      }
+    }
+    children.length = write;
+  }
+  return tree;
+};
+var walk = (tree, visit) => {
+  const stack = [{ node: tree, parent: void 0 }];
+  while (stack.length > 0) {
+    const { node, parent } = stack.pop();
+    if (visit(node, parent) === false) {
+      continue;
+    }
+    if (isParent(node)) {
+      for (let i = node.children.length - 1; i >= 0; i--) {
+        stack.push({ node: node.children[i], parent: node });
+      }
+    }
+  }
+};
+var collectElements = (tree, predicate) => {
+  const found = [];
+  walk(tree, (node) => {
+    if (isElement(node) && predicate(node)) {
+      found.push(node);
+    }
+  });
+  return found;
+};
+var findElement = (tree, predicate) => {
+  let found;
+  walk(tree, (node) => {
+    if (found) {
+      return false;
+    }
+    if (isElement(node) && predicate(node)) {
+      found = node;
+      return false;
+    }
+  });
+  return found;
+};
+var cloneHast = (tree) => {
+  try {
+    return structuredClone(tree);
+  } catch {
+    return manualClone(tree);
+  }
+};
+var manualClone = (node) => {
+  if (Array.isArray(node)) {
+    return node.map(manualClone);
+  }
+  if (typeof node !== "object" || node === null) {
+    return node;
+  }
+  const copy = {};
+  for (const [key, value] of Object.entries(node)) {
+    copy[key] = manualClone(value);
+  }
+  return copy;
+};
+var property = (element4, key) => {
+  const properties = element4.properties;
+  if (!properties) {
+    return void 0;
+  }
+  const direct = properties[key];
+  if (direct !== void 0) {
+    return direct;
+  }
+  if (!(key.includes("-") || key.includes(":"))) {
+    return void 0;
+  }
+  return properties[camelCaseAttribute(key)];
+};
+var camelCaseAttribute = (key) => key.replace(/[-:]([a-z])/g, (_, character) => character.toUpperCase());
+var numericProperty = (element4, key) => {
+  const raw = property(element4, key);
+  if (typeof raw === "number") {
+    return Number.isFinite(raw) ? raw : void 0;
+  }
+  if (typeof raw === "string" && raw.length > 0) {
+    const parsed = Number.parseFloat(raw);
+    return Number.isFinite(parsed) ? parsed : void 0;
+  }
+  return void 0;
+};
+var stringProperty = (element4, key) => {
+  const raw = property(element4, key);
+  if (typeof raw === "string") {
+    return raw;
+  }
+  if (Array.isArray(raw)) {
+    return raw.join(" ");
+  }
+  if (raw === true) {
+    return "";
+  }
+  return void 0;
+};
+var isTruthyAttribute = (element4, key) => {
+  const raw = property(element4, key);
+  if (raw === void 0 || raw === null || raw === false) {
+    return false;
+  }
+  if (raw === true) {
+    return true;
+  }
+  if (typeof raw === "string") {
+    return raw.toLowerCase() !== "false";
+  }
+  return true;
+};
+
+// src/adapters/sites/youtube.ts
+var findObjectEnd = (source, start) => {
+  let depth = 0;
+  let inString = false;
+  let escaped = false;
+  for (let index = start; index < source.length; index++) {
+    const character = source[index];
+    if (escaped) {
+      escaped = false;
+    } else if (character === "\\") {
+      escaped = true;
+    } else if (character === '"') {
+      inString = !inString;
+    } else if (!inString) {
+      if (character === "{") {
+        depth += 1;
+      } else if (character === "}" && --depth === 0) {
+        return index + 1;
+      }
+    }
+  }
+  return -1;
+};
+var extractAssignedJson = (source, name) => {
+  const markerIndex = source.indexOf(`${name} =`);
+  if (markerIndex === -1) {
+    return void 0;
+  }
+  const start = source.indexOf("{", markerIndex);
+  if (start === -1) {
+    return void 0;
+  }
+  const end = findObjectEnd(source, start);
+  if (end === -1) {
+    return void 0;
+  }
+  try {
+    return JSON.parse(source.slice(start, end));
+  } catch {
+    return void 0;
+  }
+};
+var scriptText = (root) => collectElements(root, (element4) => element4.tagName === "script").map(
+  (script) => script.children.map((child) => child.type === "text" ? child.value : "").join("")
+);
+var PLAYER_RESPONSE_VARIABLE = "ytInitialPlayerResponse";
+var findPlayerResponse = (root) => {
+  for (const source of scriptText(root)) {
+    if (!source.includes(PLAYER_RESPONSE_VARIABLE)) {
+      continue;
+    }
+    const parsed = extractAssignedJson(source, PLAYER_RESPONSE_VARIABLE);
+    if (parsed && typeof parsed === "object") {
+      return parsed;
+    }
+  }
+  return void 0;
+};
+var youtubeCaptionTracks = (root) => {
+  const player = findPlayerResponse(root);
+  const renderer = player?.captions?.playerCaptionsTracklistRenderer;
+  const tracks = renderer?.captionTracks;
+  if (!Array.isArray(tracks)) {
+    return [];
+  }
+  return tracks.flatMap((track) => {
+    const url = typeof track.baseUrl === "string" ? track.baseUrl : void 0;
+    const languageCode = typeof track.languageCode === "string" ? track.languageCode : void 0;
+    if (!(url && languageCode)) {
+      return [];
+    }
+    const name = track.name?.simpleText;
+    return [
+      {
+        url,
+        languageCode,
+        name: typeof name === "string" ? name : void 0,
+        isAutomatic: track.kind === "asr"
+      }
+    ];
+  });
+};
+var text3 = (value) => ({ type: "text", value });
+var element = (tagName, children) => ({
+  type: "element",
+  tagName,
+  properties: {},
+  children
+});
+var heading = (depth, value) => element(`h${depth}`, [text3(value)]);
+var paragraph = (value) => element("p", [text3(value)]);
+var descriptionNodes = (description) => description.split(/\n{2,}/).map((block) => block.trim()).filter((block) => block.length > 0).map(
+  (block) => element(
+    "p",
+    block.split("\n").flatMap((line, index) => index === 0 ? [text3(line)] : [element("br", []), text3(line)])
+  )
+);
+var formatDuration = (seconds) => {
+  const hours = Math.floor(seconds / 3600);
+  const minutes = Math.floor(seconds % 3600 / 60);
+  const remainder = seconds % 60;
+  const pad = (value) => String(value).padStart(2, "0");
+  return hours > 0 ? `${hours}:${pad(minutes)}:${pad(remainder)}` : `${minutes}:${pad(remainder)}`;
+};
+var metadataList = (details) => {
+  const items = [];
+  if (details.author) {
+    items.push(element("li", [text3(`Channel: ${details.author}`)]));
+  }
+  const seconds = Number(details.lengthSeconds);
+  if (Number.isFinite(seconds) && seconds > 0) {
+    items.push(element("li", [text3(`Duration: ${formatDuration(seconds)}`)]));
+  }
+  const views = Number(details.viewCount);
+  if (Number.isFinite(views) && views > 0) {
+    items.push(element("li", [text3(`Views: ${views.toLocaleString("en-US")}`)]));
+  }
+  return items.length > 0 ? element("ul", items) : void 0;
+};
+var youtubeAdapter = {
+  id: "youtube",
+  matches: ({ url }) => {
+    if (!url) {
+      return false;
+    }
+    try {
+      return /(^|\.)(youtube\.com|youtu\.be|youtube-nocookie\.com)$/.test(new URL(url).hostname);
+    } catch {
+      return false;
+    }
+  },
+  // Scanning every script body for the JSON blob would cost a full traversal on every page, so
+  // the fallback keys off markers the signature already collected.
+  matchesFingerprint: ({ signature }) => Boolean(signature?.ids.has("movie_player") || signature?.classes.has("ytd-app")),
+  extract: ({ hast }) => {
+    const player = findPlayerResponse(hast);
+    const details = player?.videoDetails;
+    if (!details?.title) {
+      return void 0;
+    }
+    const children = [heading(1, details.title)];
+    const metadata = metadataList(details);
+    if (metadata) {
+      children.push(metadata);
+    }
+    const description = details.shortDescription?.trim();
+    if (description) {
+      children.push(heading(2, "Description"));
+      children.push(...descriptionNodes(description));
+    }
+    const tracks = youtubeCaptionTracks(hast);
+    if (tracks.length > 0) {
+      const languages2 = [...new Set(tracks.map((track) => track.languageCode))].sort();
+      children.push(paragraph(`Captions available: ${languages2.join(", ")}`));
+    }
+    return { type: "root", children };
+  }
+};
+
+// src/detect-client-shell.ts
+var SHELL_SUSPICION_THRESHOLD = 300;
+var SHELL_EMPTY_THRESHOLD = 50;
+var BODY_PATTERN = /<body[^>]*>([\s\S]*?)<\/body>/i;
+var HEAD_PATTERN = /<head[^>]*>[\s\S]*?<\/head>/gi;
+var NOSCRIPT_PATTERN = /<noscript[^>]*>([\s\S]*?)<\/noscript>/gi;
+var INVISIBLE_ELEMENT_PATTERN = /<(script|style|template|svg|noscript)[^>]*>[\s\S]*?<\/\1>/gi;
+var COMMENT_PATTERN = /<!--[\s\S]*?-->/g;
+var TAG_PATTERN = /<[^>]+>/g;
+var MOUNT_POINT_PATTERN = /<[a-z][^>]*\sid=["'](?:root|app|__next|___gatsby|__nuxt|q-app|svelte|react-root)["']|<app-root[\s>]|\sdata-reactroot[\s>=]/i;
+var CHALLENGE_PATTERN = /cf-chl|challenge-platform|cf_chl_opt|just a moment|attention required|checking your browser|_incapsula_|px-captcha|datadome/i;
+var ENTITIES = {
+  "&nbsp;": " ",
+  "&amp;": "&",
+  "&lt;": "<",
+  "&gt;": ">",
+  "&quot;": '"',
+  "&#39;": "'"
+};
+var visibleTextOf = (markup) => markup.replace(COMMENT_PATTERN, " ").replace(INVISIBLE_ELEMENT_PATTERN, " ").replace(TAG_PATTERN, " ").replace(/&(?:nbsp|amp|lt|gt|quot|#39);/g, (entity) => ENTITIES[entity] ?? " ").replace(/\s+/g, " ").trim();
+var noscriptMentionsJavascript = (html) => {
+  for (const match of html.matchAll(NOSCRIPT_PATTERN)) {
+    if (/javascript/i.test(match[1] ?? "")) {
+      return true;
+    }
+  }
+  return false;
+};
+var detectClientShell = (html) => {
+  const body = BODY_PATTERN.exec(html)?.[1] ?? html.replace(HEAD_PATTERN, " ");
+  const visibleTextLength = visibleTextOf(body).length;
+  if (visibleTextLength >= SHELL_SUSPICION_THRESHOLD) {
+    return { isShell: false, visibleTextLength };
+  }
+  if (CHALLENGE_PATTERN.test(html)) {
+    return { isShell: true, reason: "anti-bot-challenge", visibleTextLength };
+  }
+  if (MOUNT_POINT_PATTERN.test(body)) {
+    return { isShell: true, reason: "spa-shell", visibleTextLength };
+  }
+  if (noscriptMentionsJavascript(html)) {
+    return { isShell: true, reason: "noscript-only", visibleTextLength };
+  }
+  if (visibleTextLength < SHELL_EMPTY_THRESHOLD) {
+    return { isShell: true, reason: "empty-body", visibleTextLength };
+  }
+  return { isShell: false, visibleTextLength };
+};
+
+// src/utils/parse-html.ts
 var MAX_ELEMENT_DEPTH = 256;
-var isParent = (node) => "childNodes" in node;
+var isParent2 = (node) => "childNodes" in node;
 var LINE_ELEMENTS = /* @__PURE__ */ new Set([
   "p",
   "div",
@@ -1402,7 +1834,7 @@ var flattenDeep = (root, maxDepth) => {
     let unwrapped = false;
     for (let index = 0; index < children.length; index++) {
       const child = children[index];
-      if (!isParent(child)) {
+      if (!isParent2(child)) {
         continue;
       }
       if (depth + 1 > maxDepth && "tagName" in child) {
@@ -1426,12 +1858,78 @@ var flattenDeep = (root, maxDepth) => {
     }
   }
 };
-var SCRIPT_OR_STYLE = /<(script|style)\b([^>]*)>[\s\S]*?<\/\1\s*>/gi;
+var SCRIPT_OR_STYLE_START = /<!--|<(script|style)(?=[\s/>])/gi;
+var CLOSING_TAG = {
+  script: /<\/script[\s/>]/gi,
+  style: /<\/style[\s/>]/gi
+};
+var startTagEnd = (html, from) => {
+  let index = from;
+  while (index < html.length) {
+    const char = html[index];
+    if (char === ">") {
+      return index + 1;
+    }
+    if (char === "=") {
+      let value = index + 1;
+      while (value < html.length && /\s/.test(html[value] ?? "")) {
+        value++;
+      }
+      const quote = html[value];
+      if (quote === '"' || quote === "'") {
+        const close = html.indexOf(quote, value + 1);
+        if (close < 0) {
+          return -1;
+        }
+        index = close + 1;
+        continue;
+      }
+      index = value;
+      continue;
+    }
+    index++;
+  }
+  return -1;
+};
+var isReadScript = (attributes, element4) => /application\/ld\+json/i.test(attributes) || element4.includes(PLAYER_RESPONSE_VARIABLE) || CHALLENGE_PATTERN.test(element4);
 var LARGE_DOCUMENT_CHARS = 2e6;
-var stripScriptBodies = (html) => html.replace(
-  SCRIPT_OR_STYLE,
-  (match, tag, attributes) => /application\/ld\+json/i.test(attributes) ? match : `<${tag}${attributes}></${tag}>`
-);
+var stripScriptBodies = (html) => {
+  const out = [];
+  let copied = 0;
+  const start = new RegExp(SCRIPT_OR_STYLE_START);
+  for (let match = start.exec(html); match; match = start.exec(html)) {
+    if (match[1] === void 0) {
+      const commentEnd = html.indexOf("-->", match.index + 4);
+      if (commentEnd < 0) {
+        break;
+      }
+      start.lastIndex = commentEnd + 3;
+      continue;
+    }
+    const tag = match[1].toLowerCase();
+    const bodyStart = startTagEnd(html, match.index + match[0].length);
+    if (bodyStart < 0) {
+      break;
+    }
+    const closing = CLOSING_TAG[tag];
+    closing.lastIndex = bodyStart;
+    const close = closing.exec(html);
+    if (!close) {
+      break;
+    }
+    const tagEnd = html.indexOf(">", close.index);
+    const elementEnd = tagEnd < 0 ? html.length : tagEnd + 1;
+    const element4 = html.slice(match.index, elementEnd);
+    const openingTag = html.slice(match.index, bodyStart);
+    if (!(tag === "script" && isReadScript(openingTag, element4))) {
+      out.push(html.slice(copied, bodyStart), html.slice(close.index, elementEnd));
+      copied = elementEnd;
+    }
+    start.lastIndex = elementEnd;
+  }
+  out.push(html.slice(copied));
+  return out.join("");
+};
 var parseHtml = (html, options = {}) => {
   const source = html.length > LARGE_DOCUMENT_CHARS ? stripScriptBodies(html) : html;
   const document2 = options.fragment ? parseFragment(source, { scriptingEnabled: false }) : parse(source, { scriptingEnabled: false });
@@ -1617,246 +2115,24 @@ var hoistAttentionWhitespace = (node) => {
     if (lead) {
       index += spaceBeside(parent.children, index, "before");
     }
-    if (trail) {
-      spaceBeside(parent.children, index, "after");
-    }
-  }
-};
-var extractMdast = (node) => {
-  const extracted = filter(node, (node2) => {
-    if (!emptyDeclarationFilter(node2)) {
-      return false;
-    }
-    return true;
-  });
-  if (extracted) {
-    liftDisplayMathIn(extracted);
-    tidyBreaks(extracted);
-    hoistAttentionWhitespace(extracted);
-  }
-  return extracted;
-};
-
-// src/utils/hast-fast.ts
-var EMPTY_METRICS = { text: 0, link: 0, commas: 0, paragraphs: 0, elements: 0 };
-var COMMA_REGEX = /[,、，،]/g;
-var PARAGRAPH_LIKE = /* @__PURE__ */ new Set(["p", "pre", "li", "blockquote", "figcaption", "dd", "dt"]);
-var NON_PROSE_TAGS = /* @__PURE__ */ new Set(["script", "style", "noscript", "template", "svg", "math", "iframe", "object"]);
-var isElement = (node) => typeof node === "object" && node !== null && node.type === "element";
-var isParent2 = (node) => typeof node === "object" && node !== null && Array.isArray(node.children);
-var classList = (element4) => {
-  const className = element4.properties?.className;
-  if (Array.isArray(className)) {
-    return className;
-  }
-  if (typeof className === "string") {
-    return className.split(/\s+/).filter(Boolean);
-  }
-  return [];
-};
-var matchString = (element4) => {
-  const id = element4.properties?.id;
-  const classes = classList(element4);
-  if (!id && classes.length === 0) {
-    return element4.tagName;
-  }
-  return `${element4.tagName} ${typeof id === "string" ? id : ""} ${classes.join(" ")}`;
-};
-var MetricsCollector = class {
-  #cache = /* @__PURE__ */ new WeakMap();
-  metrics(node) {
-    if (node.type === "text") {
-      const value = node.value;
-      if (value.trim().length === 0) {
-        return EMPTY_METRICS;
-      }
-      return {
-        text: value.length,
-        link: 0,
-        commas: (value.match(COMMA_REGEX) ?? []).length,
-        paragraphs: 0,
-        elements: 0
-      };
-    }
-    if (!isParent2(node)) {
-      return EMPTY_METRICS;
-    }
-    const cached = this.#cache.get(node);
-    if (cached) {
-      return cached;
-    }
-    if (isElement(node) && NON_PROSE_TAGS.has(node.tagName)) {
-      this.#cache.set(node, EMPTY_METRICS);
-      return EMPTY_METRICS;
-    }
-    let text7 = 0;
-    let link = 0;
-    let commas = 0;
-    let paragraphs = 0;
-    let elements = 0;
-    for (const child of node.children) {
-      const childMetrics = this.metrics(child);
-      text7 += childMetrics.text;
-      link += childMetrics.link;
-      commas += childMetrics.commas;
-      paragraphs += childMetrics.paragraphs;
-      elements += childMetrics.elements;
-      if (isElement(child)) {
-        elements += 1;
-        if (PARAGRAPH_LIKE.has(child.tagName)) {
-          paragraphs += 1;
-        }
-      }
-    }
-    if (isElement(node) && node.tagName === "a") {
-      link = text7;
-    }
-    const metrics = { text: text7, link, commas, paragraphs, elements };
-    this.#cache.set(node, metrics);
-    return metrics;
-  }
-  textLength(node) {
-    return this.metrics(node).text;
-  }
-  /** Fraction of a subtree's text that is link anchor text. `0` for empty subtrees. */
-  linkDensity(node) {
-    const { text: text7, link } = this.metrics(node);
-    if (text7 === 0) {
-      return 0;
-    }
-    return Math.min(1, link / text7);
-  }
-};
-var pruneInPlace = (tree, keep) => {
-  if (!isParent2(tree)) {
-    return tree;
-  }
-  const stack = [tree];
-  while (stack.length > 0) {
-    const parent = stack.pop();
-    const children = parent.children;
-    let write = 0;
-    for (let read = 0; read < children.length; read++) {
-      const child = children[read];
-      if (!keep(child, parent)) {
-        continue;
-      }
-      children[write++] = child;
-      if (isParent2(child)) {
-        stack.push(child);
-      }
-    }
-    children.length = write;
-  }
-  return tree;
-};
-var walk = (tree, visit) => {
-  const stack = [{ node: tree, parent: void 0 }];
-  while (stack.length > 0) {
-    const { node, parent } = stack.pop();
-    if (visit(node, parent) === false) {
-      continue;
-    }
-    if (isParent2(node)) {
-      for (let i = node.children.length - 1; i >= 0; i--) {
-        stack.push({ node: node.children[i], parent: node });
-      }
-    }
-  }
-};
-var collectElements = (tree, predicate) => {
-  const found = [];
-  walk(tree, (node) => {
-    if (isElement(node) && predicate(node)) {
-      found.push(node);
-    }
-  });
-  return found;
-};
-var findElement = (tree, predicate) => {
-  let found;
-  walk(tree, (node) => {
-    if (found) {
-      return false;
-    }
-    if (isElement(node) && predicate(node)) {
-      found = node;
-      return false;
-    }
-  });
-  return found;
-};
-var cloneHast = (tree) => {
-  try {
-    return structuredClone(tree);
-  } catch {
-    return manualClone(tree);
-  }
-};
-var manualClone = (node) => {
-  if (Array.isArray(node)) {
-    return node.map(manualClone);
-  }
-  if (typeof node !== "object" || node === null) {
-    return node;
-  }
-  const copy = {};
-  for (const [key, value] of Object.entries(node)) {
-    copy[key] = manualClone(value);
-  }
-  return copy;
-};
-var property = (element4, key) => {
-  const properties = element4.properties;
-  if (!properties) {
-    return void 0;
-  }
-  const direct = properties[key];
-  if (direct !== void 0) {
-    return direct;
-  }
-  if (!(key.includes("-") || key.includes(":"))) {
-    return void 0;
-  }
-  return properties[camelCaseAttribute(key)];
-};
-var camelCaseAttribute = (key) => key.replace(/[-:]([a-z])/g, (_, character) => character.toUpperCase());
-var numericProperty = (element4, key) => {
-  const raw = property(element4, key);
-  if (typeof raw === "number") {
-    return Number.isFinite(raw) ? raw : void 0;
-  }
-  if (typeof raw === "string" && raw.length > 0) {
-    const parsed = Number.parseFloat(raw);
-    return Number.isFinite(parsed) ? parsed : void 0;
-  }
-  return void 0;
-};
-var stringProperty = (element4, key) => {
-  const raw = property(element4, key);
-  if (typeof raw === "string") {
-    return raw;
-  }
-  if (Array.isArray(raw)) {
-    return raw.join(" ");
-  }
-  if (raw === true) {
-    return "";
-  }
-  return void 0;
-};
-var isTruthyAttribute = (element4, key) => {
-  const raw = property(element4, key);
-  if (raw === void 0 || raw === null || raw === false) {
-    return false;
+    if (trail) {
+      spaceBeside(parent.children, index, "after");
+    }
   }
-  if (raw === true) {
+};
+var extractMdast = (node) => {
+  const extracted = filter(node, (node2) => {
+    if (!emptyDeclarationFilter(node2)) {
+      return false;
+    }
     return true;
+  });
+  if (extracted) {
+    liftDisplayMathIn(extracted);
+    tidyBreaks(extracted);
+    hoistAttentionWhitespace(extracted);
   }
-  if (typeof raw === "string") {
-    return raw.toLowerCase() !== "false";
-  }
-  return true;
+  return extracted;
 };
 
 // src/extractors/lib/code-tabs.ts
@@ -3057,8 +3333,8 @@ var DECLARATIVE_ADAPTERS = SPECS.map(defineSelectorAdapter);
 // src/adapters/sites/hackernews.ts
 import { select as select4, selectAll as selectAll2 } from "hast-util-select";
 var MAX_QUOTE_DEPTH = 6;
-var text3 = (value) => ({ type: "text", value });
-var element = (tagName, children) => ({
+var text4 = (value) => ({ type: "text", value });
+var element2 = (tagName, children) => ({
   type: "element",
   tagName,
   properties: {},
@@ -3080,7 +3356,7 @@ var commentDepth = (row) => {
 var nest = (content, depth) => {
   let current = content;
   for (let level = 0; level < Math.min(depth, MAX_QUOTE_DEPTH); level++) {
-    current = [element("blockquote", current)];
+    current = [element2("blockquote", current)];
   }
   return current[0];
 };
@@ -3088,15 +3364,15 @@ var storyNodes = (root) => {
   const nodes = [];
   const titleLink = select4(".titleline > a", root);
   if (titleLink) {
-    nodes.push(element("h1", [...titleLink.children]));
+    nodes.push(element2("h1", [...titleLink.children]));
     const href = stringProperty(titleLink, "href");
     if (href && /^https?:/.test(href)) {
-      nodes.push(element("p", [{ ...titleLink, children: [text3(href)] }]));
+      nodes.push(element2("p", [{ ...titleLink, children: [text4(href)] }]));
     }
   }
   const topText = select4(".toptext", root);
   if (topText) {
-    nodes.push(element("div", [...topText.children]));
+    nodes.push(element2("div", [...topText.children]));
   }
   return nodes;
 };
@@ -3108,10 +3384,10 @@ var commentNodes = (root) => {
       continue;
     }
     const author = select4(".hnuser", row);
-    const header = author ? [element("strong", [...author.children])] : [];
+    const header = author ? [element2("strong", [...author.children])] : [];
     const content = [
-      ...header.length > 0 ? [element("p", header)] : [],
-      element("div", [...body.children])
+      ...header.length > 0 ? [element2("p", header)] : [],
+      element2("div", [...body.children])
     ];
     nodes.push(nest(content, commentDepth(row)));
   }
@@ -3138,171 +3414,13 @@ var hackernewsAdapter = {
     }
     const children = [...story];
     if (comments.length > 0) {
-      children.push(element("h2", [text3("Comments")]));
+      children.push(element2("h2", [text4("Comments")]));
       children.push(...comments);
     }
     return { type: "root", children };
   }
 };
 
-// src/adapters/sites/youtube.ts
-var findObjectEnd = (source, start) => {
-  let depth = 0;
-  let inString = false;
-  let escaped = false;
-  for (let index = start; index < source.length; index++) {
-    const character = source[index];
-    if (escaped) {
-      escaped = false;
-    } else if (character === "\\") {
-      escaped = true;
-    } else if (character === '"') {
-      inString = !inString;
-    } else if (!inString) {
-      if (character === "{") {
-        depth += 1;
-      } else if (character === "}" && --depth === 0) {
-        return index + 1;
-      }
-    }
-  }
-  return -1;
-};
-var extractAssignedJson = (source, name) => {
-  const markerIndex = source.indexOf(`${name} =`);
-  if (markerIndex === -1) {
-    return void 0;
-  }
-  const start = source.indexOf("{", markerIndex);
-  if (start === -1) {
-    return void 0;
-  }
-  const end = findObjectEnd(source, start);
-  if (end === -1) {
-    return void 0;
-  }
-  try {
-    return JSON.parse(source.slice(start, end));
-  } catch {
-    return void 0;
-  }
-};
-var scriptText = (root) => collectElements(root, (element4) => element4.tagName === "script").map(
-  (script) => script.children.map((child) => child.type === "text" ? child.value : "").join("")
-);
-var findPlayerResponse = (root) => {
-  for (const source of scriptText(root)) {
-    if (!source.includes("ytInitialPlayerResponse")) {
-      continue;
-    }
-    const parsed = extractAssignedJson(source, "ytInitialPlayerResponse");
-    if (parsed && typeof parsed === "object") {
-      return parsed;
-    }
-  }
-  return void 0;
-};
-var youtubeCaptionTracks = (root) => {
-  const player = findPlayerResponse(root);
-  const renderer = player?.captions?.playerCaptionsTracklistRenderer;
-  const tracks = renderer?.captionTracks;
-  if (!Array.isArray(tracks)) {
-    return [];
-  }
-  return tracks.flatMap((track) => {
-    const url = typeof track.baseUrl === "string" ? track.baseUrl : void 0;
-    const languageCode = typeof track.languageCode === "string" ? track.languageCode : void 0;
-    if (!(url && languageCode)) {
-      return [];
-    }
-    const name = track.name?.simpleText;
-    return [
-      {
-        url,
-        languageCode,
-        name: typeof name === "string" ? name : void 0,
-        isAutomatic: track.kind === "asr"
-      }
-    ];
-  });
-};
-var text4 = (value) => ({ type: "text", value });
-var element2 = (tagName, children) => ({
-  type: "element",
-  tagName,
-  properties: {},
-  children
-});
-var heading = (depth, value) => element2(`h${depth}`, [text4(value)]);
-var paragraph = (value) => element2("p", [text4(value)]);
-var descriptionNodes = (description) => description.split(/\n{2,}/).map((block) => block.trim()).filter((block) => block.length > 0).map(
-  (block) => element2(
-    "p",
-    block.split("\n").flatMap((line, index) => index === 0 ? [text4(line)] : [element2("br", []), text4(line)])
-  )
-);
-var formatDuration = (seconds) => {
-  const hours = Math.floor(seconds / 3600);
-  const minutes = Math.floor(seconds % 3600 / 60);
-  const remainder = seconds % 60;
-  const pad = (value) => String(value).padStart(2, "0");
-  return hours > 0 ? `${hours}:${pad(minutes)}:${pad(remainder)}` : `${minutes}:${pad(remainder)}`;
-};
-var metadataList = (details) => {
-  const items = [];
-  if (details.author) {
-    items.push(element2("li", [text4(`Channel: ${details.author}`)]));
-  }
-  const seconds = Number(details.lengthSeconds);
-  if (Number.isFinite(seconds) && seconds > 0) {
-    items.push(element2("li", [text4(`Duration: ${formatDuration(seconds)}`)]));
-  }
-  const views = Number(details.viewCount);
-  if (Number.isFinite(views) && views > 0) {
-    items.push(element2("li", [text4(`Views: ${views.toLocaleString("en-US")}`)]));
-  }
-  return items.length > 0 ? element2("ul", items) : void 0;
-};
-var youtubeAdapter = {
-  id: "youtube",
-  matches: ({ url }) => {
-    if (!url) {
-      return false;
-    }
-    try {
-      return /(^|\.)(youtube\.com|youtu\.be|youtube-nocookie\.com)$/.test(new URL(url).hostname);
-    } catch {
-      return false;
-    }
-  },
-  // Scanning every script body for the JSON blob would cost a full traversal on every page, so
-  // the fallback keys off markers the signature already collected.
-  matchesFingerprint: ({ signature }) => Boolean(signature?.ids.has("movie_player") || signature?.classes.has("ytd-app")),
-  extract: ({ hast }) => {
-    const player = findPlayerResponse(hast);
-    const details = player?.videoDetails;
-    if (!details?.title) {
-      return void 0;
-    }
-    const children = [heading(1, details.title)];
-    const metadata = metadataList(details);
-    if (metadata) {
-      children.push(metadata);
-    }
-    const description = details.shortDescription?.trim();
-    if (description) {
-      children.push(heading(2, "Description"));
-      children.push(...descriptionNodes(description));
-    }
-    const tracks = youtubeCaptionTracks(hast);
-    if (tracks.length > 0) {
-      const languages2 = [...new Set(tracks.map((track) => track.languageCode))].sort();
-      children.push(paragraph(`Captions available: ${languages2.join(", ")}`));
-    }
-    return { type: "root", children };
-  }
-};
-
 // src/adapters/index.ts
 var BUILTIN_ADAPTERS = [youtubeAdapter, hackernewsAdapter, ...DECLARATIVE_ADAPTERS];
 var MIN_ADAPTER_TEXT = 40;
@@ -4826,6 +4944,8 @@ var NEURAL_MODEL = { "kind": "block-gru", "buckets": 1024, "emb": 16, "hidden":
 var TERMINATOR_PATTERNS = [
   /^was this (page|article|document) helpful/i,
   /^thanks for the feedback/i,
+  // Site-generator credit closing the page ("Powered by Docsify.js", "Powered by GitBook").
+  /^powered by\b/i,
   /^(this page was )?last (modified|updated) (on|at|:)?\s/i,
   /^learn how to contribute/i,
   /^help (us )?improve\b/i,
@@ -4855,7 +4975,7 @@ var textOf3 = (node) => {
   if (node.type === "text") {
     return node.value;
   }
-  if (!isParent2(node)) {
+  if (!isParent(node)) {
     return "";
   }
   let text7 = "";
@@ -4873,7 +4993,7 @@ var truncateTrailingBoilerplate = (tree) => {
       total += node.value.trim().length;
       return;
     }
-    if (isParent2(node)) {
+    if (isParent(node)) {
       for (const child of node.children) {
         measure(child);
       }
@@ -4886,7 +5006,7 @@ var truncateTrailingBoilerplate = (tree) => {
   const tailStart = total * TAIL_START_FRACTION;
   let cutoff = -1;
   const find = (node) => {
-    if (cutoff !== -1 || !isParent2(node)) {
+    if (cutoff !== -1 || !isParent(node)) {
       return;
     }
     for (const child of node.children) {
@@ -5846,11 +5966,10 @@ var languages = [
   ["json", [/\b(true|false|null|\{})\b|\"[^"]+\":/g, 10]],
   ["yaml", [/^(\s+)?[a-z][a-z0-9]*:/gim, 10]]
 ];
-var detectLanguage = (code) => {
-  return languages.map(
-    ([lang, ...features]) => [lang, features.reduce((acc, [match, score]) => acc + [...code.matchAll(match)].length * score, 0)]
-  ).filter(([_, score]) => score > 20).sort((a, b) => b[1] - a[1])[0]?.[0] || "plain";
-};
+var MIN_CONFIDENT_SCORE = 400;
+var detectLanguage = (code) => languages.map(
+  ([lang, ...features]) => [lang, features.reduce((acc, [match, score]) => acc + [...code.matchAll(match)].length * score, 0)]
+).filter(([_, score]) => score >= MIN_CONFIDENT_SCORE).sort((a, b) => b[1] - a[1])[0]?.[0];
 
 // src/utils/pre-text.ts
 var LINE_BLOCK_TAGS = /* @__PURE__ */ new Set([
@@ -5977,17 +6096,46 @@ var errorLine = (error) => {
 
 // src/mdast-handlers/custom-code-handler.ts
 var LANGUAGE_MATCH_REGEX = [/^language-(\S+)$/, /^highlight-source-(\S+)$/, /^CodeBlock--language-(\S+)$/];
+var LANGUAGE_TOKEN = /^[\w+#.-]+$/;
+var NO_LANGUAGE = /* @__PURE__ */ new Set(["plain", "plaintext", "text", "none", "nohighlight"]);
 var LANGUAGE_ATTRIBUTES = ["dataLanguage", "dataLang"];
 var attributeLanguage = (element4) => {
   for (const name of LANGUAGE_ATTRIBUTES) {
     const value = element4.properties?.[name];
-    if (typeof value === "string" && /^[\w+#.-]+$/.test(value.trim()) && value.trim() !== "plaintext") {
+    if (typeof value === "string" && LANGUAGE_TOKEN.test(value.trim()) && value.trim() !== "plaintext") {
       return value.trim();
     }
   }
   return void 0;
 };
-var codeLanguage = (...elements) => elements.flatMap(classList).map((className) => LANGUAGE_MATCH_REGEX.map((regex) => className.match(regex)).find(Boolean)?.[1]).find(Boolean) ?? elements.map(attributeLanguage).find(Boolean);
+var SANDPACK_LANGUAGES = /* @__PURE__ */ new Set([
+  "javascript",
+  "typescript",
+  "jsx",
+  "tsx",
+  "html",
+  "css",
+  "less",
+  "scss",
+  "sass",
+  "json",
+  "markdown",
+  "python",
+  "vue",
+  "svelte"
+]);
+var conventionLanguage = (element4) => {
+  const classes = classList(element4);
+  const brushAt = classes.findIndex((name) => name.startsWith("brush:"));
+  if (brushAt >= 0) {
+    const brush = classes[brushAt] === "brush:" ? classes[brushAt + 1] : classes[brushAt].slice("brush:".length);
+    const lang = brush?.toLowerCase();
+    return lang && LANGUAGE_TOKEN.test(lang) && !NO_LANGUAGE.has(lang) ? lang : void 0;
+  }
+  const sandpack = classes.find((name) => name.startsWith("sp-") && SANDPACK_LANGUAGES.has(name.slice(3)));
+  return sandpack?.slice(3);
+};
+var codeLanguage = (...elements) => elements.flatMap(classList).map((className) => LANGUAGE_MATCH_REGEX.map((regex) => className.match(regex)).find(Boolean)?.[1]).find(Boolean) ?? elements.map(attributeLanguage).find(Boolean) ?? elements.map(conventionLanguage).find(Boolean);
 var codeMeta = (pre) => {
   const title = pre.properties?.dataTitle;
   if (typeof title !== "string") {
@@ -6568,6 +6716,82 @@ var removeIconFonts = (tree) => {
     pruneInPlace(tree, (node) => !isIcon(node));
   }
 };
+var textOfNode = (node) => node.type === "text" ? node.value : "children" in node ? node.children.map(textOfNode).join("") : "";
+var unwrapHeadingAnchors = (tree) => {
+  walk(tree, (node) => {
+    if (!(isElement(node) && /^h[1-6]$/.test(node.tagName))) {
+      return;
+    }
+    const title = textOfNode(node).trim();
+    if (title === "") {
+      return;
+    }
+    const anchorAt = node.children.findIndex(
+      (child) => isElement(child) && child.tagName === "a" && stringProperty(child, "href")?.startsWith("#") && textOfNode(child).trim() === title
+    );
+    const anchor = node.children[anchorAt];
+    if (isElement(anchor)) {
+      node.children.splice(anchorAt, 1, ...anchor.children);
+    }
+  });
+};
+var LABEL_LANGUAGES = /* @__PURE__ */ new Set([
+  "js",
+  "javascript",
+  "ts",
+  "typescript",
+  "jsx",
+  "tsx",
+  "html",
+  "css",
+  "scss",
+  "json",
+  "bash",
+  "sh",
+  "shell",
+  "console",
+  "python",
+  "py",
+  "sql",
+  "xml",
+  "yaml",
+  "toml",
+  "http",
+  "rust",
+  "go",
+  "java",
+  "kotlin",
+  "swift",
+  "c",
+  "cpp",
+  "md",
+  "markdown",
+  "diff"
+]);
+var LABEL_NO_LANGUAGE = /* @__PURE__ */ new Set(["plain", "plaintext", "text"]);
+var foldCodeLabels = (tree) => {
+  walk(tree, (node) => {
+    if (!isElement(node) || node.tagName === "pre") {
+      return;
+    }
+    const elements = node.children.filter(isElement);
+    if (elements.length !== 2 || node.children.some((child) => child.type === "text" && child.value.trim() !== "")) {
+      return;
+    }
+    const [label, pre] = elements;
+    if (pre.tagName !== "pre" || /^(h[1-6]|pre)$/.test(label.tagName) || findElement(label, (el) => el.tagName === "pre")) {
+      return;
+    }
+    const name = textOfNode(label).trim().toLowerCase();
+    if (!(LABEL_LANGUAGES.has(name) || LABEL_NO_LANGUAGE.has(name))) {
+      return;
+    }
+    if (LABEL_LANGUAGES.has(name) && pre.properties.dataLanguage === void 0) {
+      pre.properties.dataLanguage = name;
+    }
+    node.children = node.children.filter((child) => child !== label);
+  });
+};
 var SPACED_INLINE = /* @__PURE__ */ new Set(["span", "time", "label"]);
 var SCRIPTS = [
   ["latin", /\p{Script=Latin}/u],
@@ -6584,7 +6808,6 @@ var needsSpace = (left, right) => {
   return a !== void 0 && b !== void 0 && a !== b;
 };
 var NO_SPACING = /* @__PURE__ */ new Set(["pre", "code", "kbd", "samp", "ruby", "math", "svg"]);
-var textOfNode = (node) => node.type === "text" ? node.value : "children" in node ? node.children.map(textOfNode).join("") : "";
 var separateAdjacentInlines = (tree) => {
   const visit = (node) => {
     if (!("children" in node) || isElement(node) && NO_SPACING.has(node.tagName)) {
@@ -6605,7 +6828,15 @@ var separateAdjacentInlines = (tree) => {
   visit(tree);
 };
 var normalizeHast = (tree, options = {}) => {
-  const { images = true, math = true, ariaHeadings = true, iconFonts = true, inlineSpacing = true } = options;
+  const {
+    images = true,
+    math = true,
+    ariaHeadings = true,
+    iconFonts = true,
+    headingAnchors = true,
+    codeLabels = true,
+    inlineSpacing = true
+  } = options;
   if (images) {
     resolveImageSources(tree);
   }
@@ -6618,6 +6849,12 @@ var normalizeHast = (tree, options = {}) => {
   if (iconFonts) {
     removeIconFonts(tree);
   }
+  if (headingAnchors) {
+    unwrapHeadingAnchors(tree);
+  }
+  if (codeLabels) {
+    foldCodeLabels(tree);
+  }
   if (inlineSpacing) {
     separateAdjacentInlines(tree);
   }
@@ -7030,55 +7267,6 @@ var mentionsTitle = (opening, title) => {
 };
 var normalizeForCompare = (value) => value.replace(/\s+/g, " ").trim().toLowerCase();
 
-// src/detect-client-shell.ts
-var SHELL_SUSPICION_THRESHOLD = 300;
-var SHELL_EMPTY_THRESHOLD = 50;
-var BODY_PATTERN = /<body[^>]*>([\s\S]*?)<\/body>/i;
-var HEAD_PATTERN = /<head[^>]*>[\s\S]*?<\/head>/gi;
-var NOSCRIPT_PATTERN = /<noscript[^>]*>([\s\S]*?)<\/noscript>/gi;
-var INVISIBLE_ELEMENT_PATTERN = /<(script|style|template|svg|noscript)[^>]*>[\s\S]*?<\/\1>/gi;
-var COMMENT_PATTERN = /<!--[\s\S]*?-->/g;
-var TAG_PATTERN = /<[^>]+>/g;
-var MOUNT_POINT_PATTERN = /<[a-z][^>]*\sid=["'](?:root|app|__next|___gatsby|__nuxt|q-app|svelte|react-root)["']|<app-root[\s>]|\sdata-reactroot[\s>=]/i;
-var CHALLENGE_PATTERN = /cf-chl|challenge-platform|cf_chl_opt|just a moment|attention required|checking your browser|_incapsula_|px-captcha|datadome/i;
-var ENTITIES = {
-  "&nbsp;": " ",
-  "&amp;": "&",
-  "&lt;": "<",
-  "&gt;": ">",
-  "&quot;": '"',
-  "&#39;": "'"
-};
-var visibleTextOf = (markup) => markup.replace(COMMENT_PATTERN, " ").replace(INVISIBLE_ELEMENT_PATTERN, " ").replace(TAG_PATTERN, " ").replace(/&(?:nbsp|amp|lt|gt|quot|#39);/g, (entity) => ENTITIES[entity] ?? " ").replace(/\s+/g, " ").trim();
-var noscriptMentionsJavascript = (html) => {
-  for (const match of html.matchAll(NOSCRIPT_PATTERN)) {
-    if (/javascript/i.test(match[1] ?? "")) {
-      return true;
-    }
-  }
-  return false;
-};
-var detectClientShell = (html) => {
-  const body = BODY_PATTERN.exec(html)?.[1] ?? html.replace(HEAD_PATTERN, " ");
-  const visibleTextLength = visibleTextOf(body).length;
-  if (visibleTextLength >= SHELL_SUSPICION_THRESHOLD) {
-    return { isShell: false, visibleTextLength };
-  }
-  if (CHALLENGE_PATTERN.test(html)) {
-    return { isShell: true, reason: "anti-bot-challenge", visibleTextLength };
-  }
-  if (MOUNT_POINT_PATTERN.test(body)) {
-    return { isShell: true, reason: "spa-shell", visibleTextLength };
-  }
-  if (noscriptMentionsJavascript(html)) {
-    return { isShell: true, reason: "noscript-only", visibleTextLength };
-  }
-  if (visibleTextLength < SHELL_EMPTY_THRESHOLD) {
-    return { isShell: true, reason: "empty-body", visibleTextLength };
-  }
-  return { isShell: false, visibleTextLength };
-};
-
 // src/extract-meta-refresh.ts
 var MAX_META_REFRESH_DELAY_SECONDS = 10;
 var META_TAG_PATTERN = /<meta\b[^>]*>/gi;