switchroom 0.21.9 → 0.21.10

package/telegram-plugin/render/parse.ts

@@ -12,7 +12,11 @@
 //     `source.slice(node.start, node.end)` round-trips to the source text.
 //   - Never lose text. Any mdast node type outside the supported palette
 //     degrades to a `plain` inline (or a paragraph wrapping one) carrying the
-//     raw source slice, rather than being dropped.
+//     raw source slice, rather than being dropped. The ONE deliberate
+//     exception is raw-HTML MARKUP: an unrecognised tag has its angle-bracket
+//     token dropped while its content is kept and rendered (see the raw-HTML
+//     note below and `html-fold.ts`). Content is never lost; unguaranteeable
+//     markup is.
 //
 // Underline vs bold (`__…__` vs `**…**`):
 //   Telegram's Bot API 10.1 rich markdown reads a `__…__` double-underscore run
@@ -56,6 +60,25 @@
 //   marker characters differ, and they are never part of quoted content. The
 //   set of line-start offsets that carried the marker is threaded into
 //   `foldBlock` so the matching blockquote nodes get `expandable: true`.
+//
+//   The rewrite is FENCE-AWARE. Every other fold reads the ORIGINAL `markdown`
+//   through `slice()`, so the rewritten bytes never escape — except for the
+//   `code` fold, which must use mdast's `node.value` (the dedented, fence-
+//   stripped content, which offsets alone cannot reconstruct without
+//   re-implementing micromark's fence parser). That made a fenced block
+//   containing a line starting `**>` ship silently corrupted as `  >` —
+//   anyone documenting the legacy syntax got their code altered. Skipping
+//   fenced regions in the pre-pass fixes it at the one place the rewrite is
+//   decided, keeps `code.value` trustworthy for every consumer, and preserves
+//   the length invariant trivially (a skipped line is copied verbatim).
+//
+// Raw HTML handling:
+//   mdast `html` nodes (inline `<b>`/`<a href>` runs and whole HTML blocks)
+//   used to fall through to `plain`, which the renderer `escapeMarkdown`s —
+//   escaping `=` and shipping `<a href\="…">`. They now go through
+//   `html-fold.ts`'s three-bucket policy: fold to the native IR construct,
+//   pass through raw (wire-verified allowlist), or drop the markup and keep
+//   the content. See that module's header for the rationale.
 
 import { fromMarkdown } from "mdast-util-from-markdown";
 import { gfm } from "micromark-extension-gfm";
@@ -78,6 +101,19 @@ import type {
   TableCell,
   TableRow,
 } from "./ir.js";
+import {
+  HTML_FOLD_TAGS,
+  escapeHtmlLiteral,
+  hrefOf,
+  isBlockLevelTag,
+  isPassthroughTag,
+  isVoidTag,
+  languageOf,
+  matchedTagOffsets,
+  tokenizeHtml,
+  type HtmlFoldTarget,
+  type HtmlTagInfo,
+} from "./html-fold.js";
 
 /** Copy UTF-16 offsets off an mdast node. Falls back to 0-length when a
  *  synthesized node lacks a position (from-markdown always sets one, but the
@@ -125,13 +161,42 @@ function isTgInlineEntityHref(href: string): boolean {
   return TG_INLINE_ENTITY_HREFS.some((base) => h === base || h.startsWith(`${base}?`));
 }
 
+/** A top-level fenced-code delimiter: 3+ backticks or tildes, indented 0–3
+ *  spaces. Deeper indentation is not a fence, and a fence nested inside a
+ *  blockquote / list item is prefixed by its container's markup, so this only
+ *  ever tracks TOP-LEVEL fences.
+ *
+ *  KNOWN LIMITATION (pre-existing, not a safety property): tracking only
+ *  top-level fences does NOT establish that every column-0 `**>` line outside
+ *  one is a real blockquote candidate. Counterexample:
+ *
+ *      > ```
+ *      **> lazy
+ *      > ```
+ *
+ *  The `> ``` ` opener is inside a blockquote so it never matches here; the
+ *  column-0 `**> lazy` line is rewritten to `  > lazy` and the `**` is eaten.
+ *  CommonMark forbids lazy continuation into fenced code, so that line should
+ *  CLOSE the blockquote and stay literal. Fixing it needs real container
+ *  tracking (blockquote/list prefixes), not a wider fence regex. The case is
+ *  pinned by a test in `tests/render/html-dialect.test.ts` so any future
+ *  container-aware rewrite has to decide it deliberately. */
+const FENCE_DELIM_RE = /^ {0,3}(`{3,}|~{3,})/;
+
 /** Pre-transform expandable-blockquote markers so mdast can parse them as
  *  ordinary blockquotes, WITHOUT shifting any source offset. Each line that
- *  opens with `**>` has its two `*` characters replaced by two spaces
- *  (`**>` → `  >`), which micromark reads as a normal (optionally
- *  1–3-space-indented) blockquote line. Returns the rewritten text plus the
- *  set of line-start offsets that carried the marker — `foldBlock` uses that
- *  set to flip `expandable: true` on the produced blockquote nodes. */
+ *  opens with `**>` — and is NOT inside a fenced code block — has its two `*`
+ *  characters replaced by two spaces (`**>` → `  >`), which micromark reads as
+ *  a normal (optionally 1–3-space-indented) blockquote line. Returns the
+ *  rewritten text plus the set of line-start offsets that carried the marker —
+ *  `foldBlock` uses that set to flip `expandable: true` on the produced
+ *  blockquote nodes.
+ *
+ *  Fence tracking mirrors CommonMark: an opener is 3+ backticks/tildes at
+ *  0–3 spaces of indent; the block closes on a delimiter of the SAME character
+ *  that is at least as long and carries nothing but whitespace after it, or at
+ *  end of document. Inside such a block every line is copied verbatim, so a
+ *  documented `**> …` example survives byte-identical into `code.value`. */
 function markExpandableQuotes(markdown: string): {
   text: string;
   expandableLineStarts: Set<number>;
@@ -139,8 +204,38 @@ function markExpandableQuotes(markdown: string): {
   const expandableLineStarts = new Set<number>();
   let out = "";
   let offset = 0;
+  /** The open fence's delimiter run, or null outside a fenced block. */
+  let openFence: string | null = null;
   // Split keeping the trailing newline on each line so offsets are exact.
   for (const line of markdown.split(/(?<=\n)/)) {
+    const body = line.replace(/\r?\n$/, "");
+    const fence = FENCE_DELIM_RE.exec(body)?.[1] ?? null;
+    if (openFence !== null) {
+      // Inside a fence: copy verbatim, and close on a matching delimiter.
+      out += line;
+      if (
+        fence !== null &&
+        fence[0] === openFence[0] &&
+        fence.length >= openFence.length &&
+        body.slice(body.indexOf(fence) + fence.length).trim() === ""
+      ) {
+        openFence = null;
+      }
+      offset += line.length;
+      continue;
+    }
+    if (fence !== null) {
+      // A backtick info string may not contain a backtick; such a line is not
+      // a fence opener at all (CommonMark), so only treat it as one when the
+      // rest of the line is clean.
+      const rest = body.slice(body.indexOf(fence) + fence.length);
+      if (!(fence[0] === "`" && rest.includes("`"))) {
+        openFence = fence;
+        out += line;
+        offset += line.length;
+        continue;
+      }
+    }
     if (EXPANDABLE_MARKER_RE.test(line)) {
       expandableLineStarts.add(offset);
       // Replace the leading two `*` chars with two spaces; the `>` and
@@ -168,7 +263,7 @@ function foldAlign(a: AlignType | undefined): "left" | "center" | "right" | null
   return a ?? null;
 }
 
-function foldInline(node: PhrasingContent, source: string): Inline {
+function foldInline(node: PhrasingContent, source: string, ctx: FoldCtx): Inline {
   switch (node.type) {
     case "text":
       return { type: "plain", text: node.value, ...pos(node) };
@@ -179,21 +274,21 @@ function foldInline(node: PhrasingContent, source: string): Inline {
       const underline = source.slice(p.start, p.start + 2) === "__";
       return {
         type: underline ? "underline" : "bold",
-        children: foldInlineChildren(node, source),
+        children: foldInlineChildren(node, source, ctx),
         ...p,
       };
     }
     case "emphasis":
-      return { type: "italic", children: foldInlineChildren(node, source), ...pos(node) };
+      return { type: "italic", children: foldInlineChildren(node, source, ctx), ...pos(node) };
     case "delete":
-      return { type: "strike", children: foldInlineChildren(node, source), ...pos(node) };
+      return { type: "strike", children: foldInlineChildren(node, source, ctx), ...pos(node) };
     case "inlineCode":
       return { type: "code", text: node.value, ...pos(node) };
     case "link":
       return {
         type: "link",
         href: node.url,
-        children: foldInlineChildren(node, source),
+        children: foldInlineChildren(node, source, ctx),
         ...pos(node),
       };
     case "image": {
@@ -290,42 +385,444 @@ function expandPlainNode(node: PlainNode, source: string): Inline[] {
   return out;
 }
 
+// ---------------------------------------------------------------------------
+// Raw-HTML tag folding (see html-fold.ts for the policy and its rationale)
+// ---------------------------------------------------------------------------
+
+/** One element of the stream the HTML matcher walks: either a classified HTML
+ *  token or an already-folded IR inline. */
+type HtmlFoldItem =
+  | { kind: "tag"; tag: HtmlTagInfo; start: number; end: number }
+  | { kind: "inline"; node: Inline };
+
+/** Index of the token that closes `name`, honouring same-name nesting, or -1. */
+function findClosingTag(items: HtmlFoldItem[], from: number, name: string): number {
+  let depth = 0;
+  for (let i = from; i < items.length; i++) {
+    const it = items[i];
+    if (it.kind !== "tag" || it.tag.name !== name) continue;
+    if (it.tag.kind === "open") depth++;
+    else if (it.tag.kind === "close") {
+      if (depth === 0) return i;
+      depth--;
+    }
+  }
+  return -1;
+}
+
+/** Flatten a folded run to literal text, or null when any child carries
+ *  structure a code span cannot represent. Used for `<code>`. Deliberately
+ *  accepts ONLY `plain`: an inner `code` node's own backticks are not in its
+ *  `text`, so folding it in would silently DELETE them
+ *  (`<code>a `b` c</code>` -> `` `a b c` ``), and a `raw` node's wire bytes
+ *  would be swallowed into a literal span. Both degrade to the children
+ *  instead, which preserves every byte. */
+function inlineLiteralText(nodes: Inline[]): string | null {
+  let out = "";
+  for (const n of nodes) {
+    if (n.type !== "plain") return null;
+    out += n.text;
+  }
+  return out;
+}
+
+/** Where a folded run is being emitted. `noBreaks` marks a container whose
+ *  content must stay on ONE line — a heading or a table cell — so a `<br>` or
+ *  a structural separator has to degrade to a space instead of a line break,
+ *  and a `<pre>` to an inline code span instead of a fence. Without this a
+ *  `# a<br>b` heading emitted `# a  \nb`, which ends the heading block and
+ *  drops `b` out of it. */
+interface FoldCtx {
+  noBreaks?: boolean;
+  /** Source offsets of every HTML token the DOCUMENT-level pass found a
+   *  partner for (`matchedTagOffsets`). A token missing from this set is
+   *  unmatched — prose, not markup. Empty means "no HTML in this document",
+   *  in which case nothing consults it. */
+  matched: ReadonlySet<number>;
+}
+
+/** Separator nodes this module synthesized while degrading structural markup.
+ *  Tracked by identity so `normalizeSeparators` can collapse and trim OUR
+ *  separators without touching a hard break the author actually wrote with
+ *  `<br>` (`a<br><br>b` must keep both). */
+const structuralSeparators = new WeakSet<Inline>();
+
+/** The whitespace a degraded block-level tag leaves behind. A GFM HARD break
+ *  (two spaces + newline), not a bare `\n`: the renderer runs AFTER
+ *  `normalizeParagraphBreaks`, so a lone `\n` would never be promoted and
+ *  Telegram collapses it to a space. */
+function mkSeparator(ctx: FoldCtx, start: number, end: number): Inline {
+  const node: Inline = {
+    type: "raw",
+    text: ctx.noBreaks === true ? " " : "  \n",
+    start,
+    end,
+  };
+  structuralSeparators.add(node);
+  return node;
+}
+
+/** Drop leading/trailing synthesized separators and collapse runs of them, so
+ *  a degrade never contributes stray whitespace at the edges of a block
+ *  (`<div></div>` must still render to nothing, and a `<pre>` fence must not
+ *  leave a dangling newline). */
+function normalizeSeparators(nodes: Inline[]): Inline[] {
+  const out: Inline[] = [];
+  for (const node of nodes) {
+    if (!structuralSeparators.has(node)) {
+      out.push(node);
+      continue;
+    }
+    if (out.length === 0) continue;
+    if (structuralSeparators.has(out[out.length - 1])) continue;
+    out.push(node);
+  }
+  while (out.length > 0 && structuralSeparators.has(out[out.length - 1])) out.pop();
+  return out;
+}
+
+/** An unmatched marker or unclassifiable token kept as LITERAL text, with its
+ *  angle brackets HTML-entity-escaped so the wire sees prose, not a tag. */
+function literalTag(tag: HtmlTagInfo, start: number, end: number): Inline {
+  return { type: "plain", text: escapeHtmlLiteral(tag.raw), start, end };
+}
+
+/**
+ * Apply the HTML dialect policy over a mixed tag/inline stream.
+ *
+ * `active` carries the constructs already open in an ANCESTOR fold. Markdown
+ * cannot nest same-kind emphasis — `**a **b** c**` is asterisk soup on the
+ * reader's screen, not nested bold — so an emphasis tag whose target is
+ * already active degrades to its children. Combined with the single-child
+ * unwrap in `buildFoldedTag`, `<b>a <b>b</b> c</b>` and `<b>**already**</b>`
+ * both come out as ONE bold run. A nested `<a>` is NOT unwrapped that way: its
+ * href is content, so it degrades to escaped literal markup instead of being
+ * silently deleted.
+ *
+ * See `html-fold.ts` for the bucket policy and the matched-vs-unmatched
+ * discriminator this implements.
+ */
+function foldHtmlItems(
+  items: HtmlFoldItem[],
+  active: ReadonlySet<HtmlFoldTarget>,
+  ctx: FoldCtx,
+): Inline[] {
+  const out: Inline[] = [];
+  let i = 0;
+  while (i < items.length) {
+    const it = items[i];
+    if (it.kind === "inline") {
+      out.push(it.node);
+      i++;
+      continue;
+    }
+    const tag = it.tag;
+
+    // A comment renders nothing anywhere: dropping it loses no content.
+    if (tag.kind === "comment") {
+      i++;
+      continue;
+    }
+
+    // `<br>` / `<br/>` — a real line break (a space where breaks are illegal).
+    if (HTML_FOLD_TAGS[tag.name] === "break" && tag.kind !== "close") {
+      out.push(
+        ctx.noBreaks === true
+          ? { type: "raw", text: " ", start: it.start, end: it.end }
+          : { type: "raw", text: "  \n", start: it.start, end: it.end },
+      );
+      i++;
+      continue;
+    }
+
+    // Void / self-closing tags delimit no content, so dropping their markup
+    // cannot lose text. Block-level ones still leave a separator.
+    if (tag.kind === "selfclose" || (tag.kind === "open" && isVoidTag(tag.name))) {
+      if (isPassthroughTag(tag)) {
+        out.push({ type: "raw", text: tag.raw, start: it.start, end: it.end });
+      } else if (isBlockLevelTag(tag.name)) {
+        out.push(mkSeparator(ctx, it.start, it.end));
+      }
+      i++;
+      continue;
+    }
+
+    // Bucket 2: the wire-verified allowlist, now BALANCE-CHECKED. Emitting an
+    // unmatched `<u>` raw is precisely the `unclosed start tag` 400 the fold
+    // policy exists to prevent — and the fallback resends the whole message as
+    // PLAIN TEXT, so one stray marker in relayed content degrades the entire
+    // reply. The check is document-level because a `<details>` open and close
+    // legitimately land in different mdast blocks.
+    if (isPassthroughTag(tag) && (tag.kind === "open" || tag.kind === "close")) {
+      out.push(
+        ctx.matched.has(it.start)
+          ? { type: "raw", text: tag.raw, start: it.start, end: it.end }
+          : literalTag(tag, it.start, it.end),
+      );
+      i++;
+      continue;
+    }
+
+    // Everything else must be BALANCED to count as markup.
+    if (tag.kind === "open") {
+      const close = findClosingTag(items, i + 1, tag.name);
+      const closeItem = close >= 0 ? items[close] : undefined;
+      if (closeItem !== undefined && closeItem.kind === "tag") {
+        out.push(
+          ...foldMatchedTag(tag, closeItem, items.slice(i + 1, close), active, ctx, {
+            start: it.start,
+            end: closeItem.end,
+          }),
+        );
+        i = close + 1;
+        continue;
+      }
+    }
+
+    // Matched somewhere ELSE in the document (a `<div>` whose `</div>` is a
+    // separate mdast block) — markup, so drop it, but leave a separator for a
+    // block-level name so the neighbouring blocks do not glue.
+    if (ctx.matched.has(it.start) && tag.kind !== "other") {
+      if (isBlockLevelTag(tag.name)) out.push(mkSeparator(ctx, it.start, it.end));
+      i++;
+      continue;
+    }
+
+    // Unmatched open, stray close, or an unclassifiable token: PROSE.
+    out.push(literalTag(tag, it.start, it.end));
+    i++;
+  }
+  return normalizeSeparators(out);
+}
+
+/** Emit a tag whose closing partner was found. */
+function foldMatchedTag(
+  tag: HtmlTagInfo,
+  closeTag: HtmlFoldItem & { kind: "tag" },
+  innerItems: HtmlFoldItem[],
+  active: ReadonlySet<HtmlFoldTarget>,
+  ctx: FoldCtx,
+  span: Pos,
+): Inline[] {
+  const target = HTML_FOLD_TAGS[tag.name];
+
+  // `<pre>` — the one BLOCK-level fold (see html-fold.ts).
+  if (target === "pre") {
+    const folded = buildPreBlock(innerItems, ctx, span);
+    if (folded !== null) return folded;
+  }
+
+  if (target !== undefined && target !== "break" && target !== "pre") {
+    const nested = new Set(active);
+    nested.add(target);
+    const inner = foldHtmlItems(innerItems, nested, ctx);
+    // A LINK cannot nest in markdown, and its href is content — degrading to
+    // the children would silently delete the inner URL. Keep the markup as
+    // escaped literal text instead.
+    if (active.has(target)) {
+      return target === "link"
+        ? [
+            literalTag(tag, span.start, span.start + tag.raw.length),
+            ...inner,
+            literalTag(closeTag.tag, closeTag.start, closeTag.end),
+          ]
+        : inner;
+    }
+    const folded = buildFoldedTag(target, tag, inner, span.start, span.end);
+    // A tag we cannot represent faithfully (an `<a>` with no href, a `<code>`
+    // wrapping structure) degrades to its CONTENT — nothing is lost.
+    return folded !== null ? [folded] : inner;
+  }
+
+  // Unknown but BALANCED: markup dropped, content kept. A block-level name
+  // leaves a separator so its neighbours cannot glue together.
+  const inner = foldHtmlItems(innerItems, active, ctx);
+  if (!isBlockLevelTag(tag.name)) return inner;
+  return [
+    mkSeparator(ctx, span.start, span.start + tag.raw.length),
+    ...inner,
+    mkSeparator(ctx, closeTag.start, closeTag.end),
+  ];
+}
+
+/** Build the fenced-code equivalent of a matched `<pre>…</pre>`, or null when
+ *  the content is not literal text a fence can carry (in which case the caller
+ *  falls back to the ordinary degrade). An optional single `<code …>` wrapper
+ *  is unwrapped, and its `class="language-…"` becomes the fence info string —
+ *  exactly the shape Telegram's own Rich HTML reference documents. */
+function buildPreBlock(
+  innerItems: HtmlFoldItem[],
+  ctx: FoldCtx,
+  span: Pos,
+): Inline[] | null {
+  let body = innerItems;
+  let language: string | null = null;
+  const first = body[0];
+  const last = body[body.length - 1];
+  if (
+    body.length >= 2 &&
+    first?.kind === "tag" &&
+    first.tag.kind === "open" &&
+    first.tag.name === "code" &&
+    last?.kind === "tag" &&
+    last.tag.kind === "close" &&
+    last.tag.name === "code"
+  ) {
+    language = languageOf(first.tag);
+    body = body.slice(1, -1);
+  }
+  let text = "";
+  for (const item of body) {
+    if (item.kind !== "inline" || item.node.type !== "plain") return null;
+    text += item.node.text;
+  }
+  // `<pre>` content routinely starts on the line after the tag and ends on the
+  // line before the close; those layout newlines are not content.
+  text = text.replace(/^\r?\n/, "").replace(/[ \t]*\r?\n[ \t]*$/, "");
+  // A fence cannot survive inside a heading or a table cell — degrade to an
+  // inline code span there, with the newlines flattened.
+  if (ctx.noBreaks === true) {
+    return [{ type: "code", text: text.replace(/\s*\r?\n\s*/g, " "), ...span }];
+  }
+  const fence = "```";
+  return [
+    mkSeparator(ctx, span.start, span.start),
+    {
+      type: "raw",
+      text: `${fence}${language ?? ""}\n${text}\n${fence}`,
+      ...span,
+    },
+    mkSeparator(ctx, span.end, span.end),
+  ];
+}
+
+function buildFoldedTag(
+  target: HtmlFoldTarget,
+  tag: HtmlTagInfo,
+  children: Inline[],
+  start: number,
+  end: number,
+): Inline | null {
+  switch (target) {
+    case "bold":
+    case "italic":
+    case "strike":
+      // Already exactly that construct (`<b>**already**</b>`) — re-wrapping
+      // would emit `****already****`, which reads as literal asterisks.
+      if (children.length === 1 && children[0].type === target) return children[0];
+      return { type: target, children, start, end };
+    case "code": {
+      const text = inlineLiteralText(children);
+      return text === null ? null : { type: "code", text, start, end };
+    }
+    case "link": {
+      const href = hrefOf(tag);
+      return href === null ? null : { type: "link", href, children, start, end };
+    }
+    default:
+      return null;
+  }
+}
+
+/** Turn a raw HTML string into fold items: tokens stay tokens, the text
+ *  between them becomes `plain` inlines (with spoiler/highlight recognition,
+ *  matching how prose is treated everywhere else). */
+function htmlStringToItems(raw: string, base: number, source: string): HtmlFoldItem[] {
+  return tokenizeHtml(raw, base).flatMap<HtmlFoldItem>((piece) =>
+    piece.kind === "tag"
+      ? [{ kind: "tag", tag: piece.tag, start: piece.start, end: piece.end }]
+      : expandPlainNode(mkPlain(piece.text, piece.start, piece.end), source).map(
+          (node) => ({ kind: "inline", node }) as HtmlFoldItem,
+        ),
+  );
+}
+
 /** Fold a run of mdast phrasing children into IR inlines, then run the
- *  post-parse spoiler/highlight recognition over the produced `plain` nodes. */
-function buildInlines(children: ReadonlyArray<MdastNode>, source: string): Inline[] {
-  return children
-    .map((c) => foldInline(c as PhrasingContent, source))
-    .flatMap((n) => (n.type === "plain" ? expandPlainNode(n, source) : [n]));
+ *  post-parse spoiler/highlight recognition over the produced `plain` nodes,
+ *  then apply the raw-HTML tag policy across the whole run (open/close markers
+ *  are SIBLING mdast `html` nodes, never a parent, so matching has to happen
+ *  at this level). */
+function buildInlines(
+  children: ReadonlyArray<MdastNode>,
+  source: string,
+  ctx: FoldCtx,
+): Inline[] {
+  const items: HtmlFoldItem[] = [];
+  let sawHtml = false;
+  for (const child of children) {
+    if ((child as PhrasingContent).type === "html") {
+      sawHtml = true;
+      const p = pos(child);
+      items.push(...htmlStringToItems(slice(source, child), p.start, source));
+      continue;
+    }
+    const folded = foldInline(child as PhrasingContent, source, ctx);
+    const expanded = folded.type === "plain" ? expandPlainNode(folded, source) : [folded];
+    for (const node of expanded) items.push({ kind: "inline", node });
+  }
+  if (!sawHtml) return items.map((it) => (it as { node: Inline }).node);
+  return foldHtmlItems(items, new Set(), ctx);
 }
 
-function foldInlineChildren(node: MdastParent, source: string): Inline[] {
-  return buildInlines(node.children, source);
+function foldInlineChildren(
+  node: MdastParent,
+  source: string,
+  ctx: FoldCtx,
+): Inline[] {
+  return buildInlines(node.children, source, ctx);
 }
 
 function foldTableRow(
   row: Extract<RootContent, { type: "tableRow" }>,
   source: string,
+  ctx: FoldCtx,
 ): TableRow {
   const cells: TableCell[] = row.children.map((cell) => ({
-    children: buildInlines(cell.children, source),
+    // A table cell is one line on the wire — a `<br>` or a structural degrade
+    // inside it must not emit a newline (it would end the row).
+    children: buildInlines(cell.children, source, { ...ctx, noBreaks: true }),
     ...pos(cell),
   }));
   return { cells, ...pos(row) };
 }
 
+/** True for a block that would render to the empty string. Only the raw-HTML
+ *  fold can produce one — a block that is nothing but markup we dropped
+ *  (`<!-- comment -->`, `<hr/>`, `<div></div>` on their own line). Left in,
+ *  each would contribute a spurious `\n\n` separator to the rendered document,
+ *  so they are filtered out at the point blocks are collected. */
+function isEmptyBlock(block: Block): boolean {
+  return block.type === "paragraph" && block.children.length === 0;
+}
+
+/** Fold a run of mdast block nodes, dropping any that render to nothing. */
+function foldBlocks(
+  nodes: ReadonlyArray<RootContent>,
+  source: string,
+  expandableLineStarts: Set<number>,
+  ctx: FoldCtx,
+): Block[] {
+  return nodes
+    .map((n) => foldBlock(n, source, expandableLineStarts, ctx))
+    .filter((b) => !isEmptyBlock(b));
+}
+
 function foldBlock(
   node: RootContent,
   source: string,
   expandableLineStarts: Set<number>,
+  ctx: FoldCtx,
 ): Block {
   switch (node.type) {
     case "paragraph":
-      return { type: "paragraph", children: foldInlineChildren(node, source), ...pos(node) };
+      return { type: "paragraph", children: foldInlineChildren(node, source, ctx), ...pos(node) };
     case "heading":
       return {
         type: "heading",
         level: node.depth,
-        children: foldInlineChildren(node, source),
+        // A heading is a single `#…` LINE: an embedded newline would end the
+        // block and orphan the rest of the text (`# a<br>b`).
+        children: foldInlineChildren(node, source, { ...ctx, noBreaks: true }),
         ...pos(node),
       };
     case "blockquote": {
@@ -335,7 +832,7 @@ function foldBlock(
       const expandable = expandableLineStarts.has(lineStart(source, p.start));
       return {
         type: "blockquote",
-        children: node.children.map((c) => foldBlock(c, source, expandableLineStarts)),
+        children: foldBlocks(node.children, source, expandableLineStarts, ctx),
         expandable,
         ...p,
       };
@@ -349,7 +846,7 @@ function foldBlock(
       };
     case "list": {
       const items: ListItem[] = node.children.map((li) => ({
-        children: li.children.map((c) => foldBlock(c, source, expandableLineStarts)),
+        children: foldBlocks(li.children, source, expandableLineStarts, ctx),
         checked: li.checked ?? null,
         // mdast `listItem.spread`: whether this item's block children are
         // separated by a blank line. Tight (false) keeps a paragraph and its
@@ -374,8 +871,8 @@ function foldBlock(
       const [headerRow, ...bodyRows] = rows;
       return {
         type: "table",
-        header: foldTableRow(headerRow, source),
-        rows: bodyRows.map((r) => foldTableRow(r, source)),
+        header: foldTableRow(headerRow, source, ctx),
+        rows: bodyRows.map((r) => foldTableRow(r, source, ctx)),
         align: (node.align ?? []).map(foldAlign),
         ...pos(node),
       };
@@ -391,7 +888,26 @@ function foldBlock(
         children: [{ type: "raw", text: slice(source, node), ...pos(node) }],
         ...pos(node),
       };
-    // Not in the palette (html, definition, …): degrade to a paragraph
+    // A raw HTML BLOCK (`<details>…`, `<div>…`, a bare `<b>alone</b>` line).
+    // mdast hands the whole block over as one opaque string, so tokenize it
+    // and run the same three-bucket policy the inline path uses: the
+    // wire-verified allowlist passes through raw (which is also what stops
+    // `escapeMarkdown` mangling `<details open="x">` into `open\="x"`),
+    // markdown-equivalent tags fold, everything else drops its markup and
+    // keeps its content.
+    case "html": {
+      const p = pos(node);
+      return {
+        type: "paragraph",
+        children: foldHtmlItems(
+          htmlStringToItems(slice(source, node), p.start, source),
+          new Set(),
+          ctx,
+        ),
+        ...p,
+      };
+    }
+    // Not in the palette (definition, …): degrade to a paragraph
     // carrying the raw source slice so no content is dropped.
     default:
       return {
@@ -415,8 +931,33 @@ export function parse(markdown: string): Document {
     mdastExtensions: [gfmFromMarkdown()],
   });
   return {
-    blocks: tree.children.map((child) =>
-      foldBlock(child, markdown, expandableLineStarts),
-    ),
+    blocks: foldBlocks(tree.children, markdown, expandableLineStarts, {
+      matched: matchedTagOffsets(collectHtmlTokens(tree.children, markdown)),
+    }),
+  };
+}
+
+/** Every HTML token in the tree, in document order, for the balance pass.
+ *  Walks mdast `html` nodes only — a tag inside a code fence or a code span is
+ *  not markup and must not balance anything. */
+function collectHtmlTokens(
+  nodes: ReadonlyArray<MdastNode>,
+  source: string,
+): { tag: HtmlTagInfo; start: number }[] {
+  const out: { tag: HtmlTagInfo; start: number }[] = [];
+  const walk = (list: ReadonlyArray<MdastNode>): void => {
+    for (const node of list) {
+      if (node.type === "html") {
+        const p = pos(node);
+        for (const piece of tokenizeHtml(slice(source, node), p.start)) {
+          if (piece.kind === "tag") out.push({ tag: piece.tag, start: piece.start });
+        }
+        continue;
+      }
+      const children = (node as MdastParent).children;
+      if (Array.isArray(children)) walk(children);
+    }
   };
+  walk(nodes);
+  return out;
 }