switchroom 0.21.9 → 0.21.10

package/telegram-plugin/format.ts

@@ -99,9 +99,26 @@ export function codeSpanSafe(s: string): string {
  * `\` first (so we never double-escape a following escape), then BOTH `(` and
  * `)` — the whole URL is preserved balanced and micromark decodes it back to
  * the original href on round-trip. Bot API 10.1 lists `(`/`)` as escapable.
+ *
+ * WHITESPACE and control characters are PERCENT-ENCODED rather than
+ * backslash-escaped. A bare link destination ends at the first ASCII
+ * whitespace character, so a space or a newline inside the href does not just
+ * truncate the URL — the remainder is re-read as a link title, or (for a
+ * newline) the inline link is terminated outright, leaving a structurally
+ * broken construct with URL fragments visible as prose. Backslash cannot
+ * rescue that: whitespace is not escapable in a bare destination. `%20` /
+ * `%0A` are the canonical URL encodings, so the href a client resolves is
+ * equivalent to the one the author wrote.
  */
 export function escapeLinkHref(href: string): string {
-  return href.replace(/\\/g, '\\\\').replace(/\(/g, '\\(').replace(/\)/g, '\\)')
+  return href
+    .replace(/\\/g, '\\\\')
+    .replace(/\(/g, '\\(')
+    .replace(/\)/g, '\\)')
+    .replace(
+      /[\x00-\x20\x7f]/g,
+      (c) => `%${c.charCodeAt(0).toString(16).toUpperCase().padStart(2, '0')}`,
+    )
 }
 
 /**

package/telegram-plugin/gateway/outbound-send-path.ts

@@ -43,6 +43,7 @@ import {
   isQuoteRejectionError,
   sendOptsHaveQuote,
 } from '../reply-quote.js'
+import { captureSpeechText } from './speech-capture.js'
 
 // ── send-orchestration façade imports (#2996 P2) ──
 // Pure/deterministic helpers are imported; stateful or side-effecting gateway
@@ -1376,6 +1377,12 @@ export async function sendReply(
   // plain-text TTS input); synthesis happens just before the send so a
   // voice-only reply can suppress the text chunk loop on success. Voice is
   // fully best-effort — every failure below falls back to the text reply.
+  // Raw-corpus capture (TTS redesign PR-0, flag-gated, off by default): `text`
+  // here IS Stage A's future input — capture it byte-for-byte BEFORE the
+  // resolve call, ahead of any TTS normalisation, so the redesign's property
+  // tests can validate against real markdown instead of synthetic fixtures
+  // only. See telegram-plugin/gateway/speech-capture.ts.
+  captureSpeechText(text)
   const voiceOutPlan = resolveVoiceOutPlan(access.voice_out, text)
   const configParseMode = access.parseMode ?? 'html'
   const format = (args.format as string | undefined) ?? configParseMode

package/telegram-plugin/gateway/speech-capture.ts

@@ -0,0 +1,158 @@
+/**
+ * Raw pre-normalisation speech-text capture (TTS normalisation redesign,
+ * PR-0 — `/tmp/claude-0/tts/out/fable-plan-v2.md` §9).
+ *
+ * Flag-gated on `SWITCHROOM_SPEECH_CAPTURE=1` (or `=true`), OFF by default:
+ * unset (or any other value) is a true no-op — `captureSpeechText` returns
+ * immediately without touching the filesystem, so the send path pays zero
+ * cost. When enabled, appends exactly one JSON line `{ts, text}` — where
+ * `text` is the EXACT string about to be passed to `resolveVoiceOutPlan`,
+ * byte-for-byte, no re-encoding — to `$TELEGRAM_STATE_DIR/speech-capture.jsonl`
+ * before that call (fable-plan-v2.md §0 C-4: that string IS Stage A's future
+ * input). NOT the same string the visible rich render displays — the render
+ * path additionally runs `computeEffectiveText`/`addParagraphSpacers` (U+00A0
+ * paragraph spacers) downstream of this capture point; this captures the
+ * plain pre-spacer prose Stage A will actually consume.
+ *
+ * Why this exists: `corpus.json` (the existing replay corpus) was captured
+ * downstream of the legacy pass-1 normaliser, so it carries zero tables,
+ * fences, pipes, or emoji (fable-plan-v2.md §0 C-3) — useless for validating
+ * the new Stage-A renderer beyond synthetic fixtures. `telegram/history.db`
+ * is also unusable: it stores text as Telegram echoed it back (rendered),
+ * not the pre-render markdown. Capture must happen in-process, here.
+ *
+ * Privacy: the capture file holds the full plaintext body of every outbound
+ * reply while the flag is on, so it is created `0o600` (owner-only) — NOT
+ * the default-umask `0o644` `appendFileSync` would otherwise produce, which
+ * would leave it world-readable in a directory where `access.json` (the chat
+ * allowlist) is deliberately `0o600`. Mode only applies at file CREATION
+ * (Node ignores `mode` on an append to an existing file), matching the
+ * established pattern in this codebase (`shown-ledger.ts`, `outbox.ts`).
+ *
+ * Write failures are swallowed — capture must never break a send — but never
+ * silently: the first write failure (and, rate-limited, subsequent ones)
+ * logs one stderr line, because a foreign-uid EACCES on this file (the #4371
+ * failure class: some other process touches it, it becomes root-owned, the
+ * agent uid EACCESes on every append thereafter) must be observable across a
+ * 7-day unattended capture window, not discovered after the fact from an
+ * empty corpus.
+ *
+ * Deliberately does NOT bound or rotate the capture file: the spec (§9 PR-0)
+ * describes a plain unbounded append for a time-boxed 7-day fleet-wide
+ * capture window (measured fleet-wide: ~2.9 MB over 7 days), after which the
+ * file is reviewed, secrets-scrubbed, and checked into the repo as a static
+ * fixture — it is not a long-lived production log.
+ *
+ * The enabled check reads `process.env` on every call (deliberately NOT
+ * cached at module scope, unlike `current-turn-map.ts`'s
+ * `EMISSION_AUTHORITY_ENABLED` kill-switch convention): the read is a single
+ * property lookup, not the per-turn state-store cost that convention exists
+ * to avoid, and caching it would make the flag un-togglable within a single
+ * test process — which the load-bearing "a capture write failure never
+ * breaks the reply" guarantee needs to exercise against the REAL `sendReply`
+ * wiring (see `send-reply-golden.test.ts`), not a synthetic call.
+ */
+
+import { appendFileSync } from 'node:fs'
+import { join } from 'node:path'
+
+export const SPEECH_CAPTURE_FILE_NAME = 'speech-capture.jsonl'
+
+/** File created owner-read-write only (see the Privacy note above). */
+const SPEECH_CAPTURE_FILE_MODE = 0o600
+
+export interface CaptureSpeechTextOptions {
+  /** Override the enabled flag (tests only); default reads
+   *  `SWITCHROOM_SPEECH_CAPTURE` from the environment. */
+  enabled?: boolean
+  /** Override the destination directory (tests only); default
+   *  `process.env.TELEGRAM_STATE_DIR`. */
+  stateDir?: string
+  /** Override the clock (tests only). */
+  now?: () => number
+}
+
+function envFlagEnabled(): boolean {
+  const v = process.env.SWITCHROOM_SPEECH_CAPTURE
+  return v === '1' || v === 'true'
+}
+
+function isCaptureEnabled(override: boolean | undefined): boolean {
+  if (override !== undefined) return override
+  return envFlagEnabled()
+}
+
+// One-shot "capture is live" log line, latched the first time capture
+// actually fires (not at module import — importing this module must stay a
+// true no-op regardless of whether the flag ends up used) — and a
+// rate-limited write-error log so a foreign-uid EACCES (#4371 class) is
+// observable within the window, not just discoverable from an empty corpus
+// after the fact.
+let loggedEnabledOnce = false
+// `null`, not `0`: a real failure at `Date.now() === 0` (or in a test driven
+// by `vi.setSystemTime(0)`) must still log — `0` is a valid past timestamp,
+// not "never logged", so it cannot double as the sentinel.
+let lastWriteErrorLoggedAt: number | null = null
+const WRITE_ERROR_LOG_INTERVAL_MS = 5 * 60_000
+
+function logEnabledOnce(): void {
+  if (loggedEnabledOnce) return
+  loggedEnabledOnce = true
+  try {
+    process.stderr.write(
+      `telegram gateway: speech-capture: enabled — writing $TELEGRAM_STATE_DIR/${SPEECH_CAPTURE_FILE_NAME}\n`,
+    )
+  } catch {
+    // Logging must never break the send path.
+  }
+}
+
+function logWriteErrorRateLimited(err: unknown, nowMs: number): void {
+  if (lastWriteErrorLoggedAt !== null && nowMs - lastWriteErrorLoggedAt < WRITE_ERROR_LOG_INTERVAL_MS) return
+  lastWriteErrorLoggedAt = nowMs
+  try {
+    const msg = err instanceof Error ? err.message : String(err)
+    process.stderr.write(
+      `telegram gateway: speech-capture: write failed (further failures rate-limited ` +
+        `${Math.round(WRITE_ERROR_LOG_INTERVAL_MS / 60_000)}m) err=${msg}\n`,
+    )
+  } catch {
+    // Logging must never break the send path.
+  }
+}
+
+/**
+ * Append one `{ts, text}` JSON line to the raw-corpus capture file when
+ * capture is enabled. `text` must be the EXACT string about to be passed to
+ * `resolveVoiceOutPlan` — no trimming, no re-encoding, byte-for-byte
+ * preservation of markdown (tables, fences, pipes, emoji, backticks, ...).
+ * Off by default; a true no-op when the flag is unset. Never throws.
+ */
+export function captureSpeechText(text: string, opts: CaptureSpeechTextOptions = {}): void {
+  if (!isCaptureEnabled(opts.enabled)) return
+  logEnabledOnce()
+  try {
+    const stateDir = opts.stateDir ?? process.env.TELEGRAM_STATE_DIR
+    if (stateDir == null || stateDir.length === 0) return
+    const now = opts.now ?? Date.now
+    const line = `${JSON.stringify({ ts: now(), text })}\n`
+    appendFileSync(join(stateDir, SPEECH_CAPTURE_FILE_NAME), line, {
+      encoding: 'utf8',
+      mode: SPEECH_CAPTURE_FILE_MODE,
+    })
+  } catch (err) {
+    // Capture must never break the send path — but never silently either
+    // (M2/#4371 class).
+    logWriteErrorRateLimited(err, Date.now())
+  }
+}
+
+/**
+ * Test-only: reset the one-shot enable log and the write-error rate limiter,
+ * so a test asserting stderr output is independent of import/call order
+ * versus every other test sharing this module instance.
+ */
+export function __resetSpeechCaptureLogStateForTests(): void {
+  loggedEnabledOnce = false
+  lastWriteErrorLoggedAt = null
+}

package/telegram-plugin/render/html-fold.ts

@@ -0,0 +1,354 @@
+// Raw-HTML dialect handling for the Bot API 10.1 rich-markdown render path.
+//
+// ── Why this module exists ────────────────────────────────────────────────
+// Agents habitually emit raw HTML tags in prose (`<b>`, `<i>`, `<a href>`).
+// Before this module those tags reached the wire byte-verbatim, which is two
+// distinct hazards at once:
+//
+//   1. Nothing establishes that Telegram's rich markdown parser accepts them.
+//      The wire-verified allowlist (raw `sendRichMessage` probes, 2026-08-13)
+//      covers exactly `<u>`, `<sub>`, `<sup>`, `<details>`/`<summary>` and
+//      `<aside>`/`<cite>`. `isParseEntitiesError` (`rich-send.ts`) matches
+//      `unsupported start tag` / `unclosed start tag`, so the wire CAN 400 on
+//      an unknown tag — and that fallback resends the body as PLAIN TEXT,
+//      where the reader then sees literal `<b>` markup.
+//   2. mdast `html` nodes degrade to `plain` in `parse.ts`, and `plain` is
+//      `escapeMarkdown`'d on render. `escapeMarkdown` escapes `=`, so
+//      `<a href="https://example.com">` shipped as `<a href\="…">` — an
+//      attribute no parser can read.
+//
+// ── The policy (three buckets, degrade by TYPE, never silently) ───────────
+//   • FOLD      — a tag with an exact native markdown equivalent is folded
+//                 into the IR node for that construct, so it renders as the
+//                 markdown the wire actually understands:
+//                   <b>/<strong>       -> bold          **…**
+//                   <i>/<em>           -> italic        *…*
+//                   <s>/<del>/<strike> -> strike        ~~…~~
+//                   <code>             -> code span     `…`
+//                   <a href="URL">     -> link          [label](URL)
+//                   <br>               -> a line break
+//                   <pre>              -> fenced code block ```…```
+//   • PASSTHROUGH — the wire-verified allowlist above is emitted RAW and
+//                 UNESCAPED (an IR `raw` inline), which is also what stops
+//                 `escapeMarkdown` from mangling their attributes. Requires a
+//                 MATCHED close: an unbalanced `<u>` emitted raw is exactly
+//                 the `unclosed start tag` 400 this module exists to prevent.
+//   • DEGRADE   — everything else, split by whether the token delimits
+//                 content. The discriminator is MATCHING, not the tag name:
+//                   – A MATCHED pair (`<marquee>…</marquee>`, `<div>…</div>`)
+//                     is markup wrapped around content. The markup is dropped
+//                     and the content kept; a block-level name additionally
+//                     leaves a hard line break, so `<li>one</li><li>two</li>`
+//                     cannot glue into `onetwo`.
+//                   – A comment, or a void / self-closing tag (`<hr/>`,
+//                     `<img …/>`), delimits nothing. Markup dropped; a
+//                     block-level one leaves the same separator.
+//                   – An UNMATCHED open/close marker is not markup at all — it
+//                     is PROSE. `<service>/<key>`, `<agent>`, "the `<b>` tag",
+//                     `run switchroom vault get <key>` are pervasive in this
+//                     project's own agent output, and DELETING them silently
+//                     mangles the agent's own reply. Such a token is kept as
+//                     LITERAL TEXT with `&`/`<`/`>` HTML-entity-escaped (see
+//                     `escapeHtmlLiteral`). Bot API rich markdown "can contain
+//                     arbitrary HTML … parsed as described in Rich HTML
+//                     style", and Rich HTML documents `&lt;`, `&gt;` and
+//                     `&amp;` among its supported named entities — so the
+//                     entity form is the DOCUMENTED way to ship a literal
+//                     angle bracket without tripping `unsupported start tag`.
+//
+// ── The invariant ─────────────────────────────────────────────────────────
+// CONTENT is never lost. Only MARKUP is dropped, and only where dropping it
+// loses no reader-visible text: a matched pair, a comment, a void tag.
+// Anything not demonstrably markup survives to the wire as escaped literal
+// text. (Known residual, NOT introduced by this module and not fixed here: a
+// `<` in prose that mdast hands over as a TEXT node rather than an `html` node
+// — the decoded `&lt;c&gt;`, or `response in <1s` — never reaches this module
+// and still ships unescaped. That is a `plain`-node / `escapeMarkdown`
+// concern.)
+//
+// This is a deterministic code-level guarantee, deliberately not a prompt
+// instruction to the agents that emit the tags.
+
+/** How a raw HTML token reads. `other` covers anything the tokenizer matched
+ *  but could not classify (it degrades like an unknown tag). */
+export type HtmlTagKind = "open" | "close" | "selfclose" | "comment" | "other";
+
+export interface HtmlTagInfo {
+  kind: HtmlTagKind;
+  /** Lowercased tag name (`""` for a comment). */
+  name: string;
+  /** The raw source bytes of the token, verbatim. */
+  raw: string;
+  /** Raw attribute text between the tag name and the closing `>`. */
+  attrs: string;
+}
+
+/** The IR construct a foldable tag maps onto. `pre` is BLOCK-level (a fenced
+ *  code block); every other target is inline. */
+export type HtmlFoldTarget =
+  | "bold"
+  | "italic"
+  | "strike"
+  | "code"
+  | "link"
+  | "break"
+  | "pre";
+
+/** Tags with an exact native markdown equivalent on the Bot API 10.1 rich
+ *  path. Folding them means the wire sees markdown it definitely parses
+ *  instead of HTML it may reject. */
+export const HTML_FOLD_TAGS: Readonly<Record<string, HtmlFoldTarget>> = {
+  b: "bold",
+  strong: "bold",
+  i: "italic",
+  em: "italic",
+  s: "strike",
+  del: "strike",
+  strike: "strike",
+  code: "code",
+  a: "link",
+  br: "break",
+  // `<pre>` is the one BLOCK-level fold. Telegram's own Rich HTML reference
+  // pairs `<pre><code class="language-…">` with the ```` ```lang ```` fence,
+  // so the fenced block is the exact native equivalent. Without it `<pre>`
+  // dropped its markup and the inner `<code>` folded to an INLINE span
+  // wrapping a newline — a construct Telegram will not parse, leaving the
+  // reader literal backticks around broken text.
+  pre: "pre",
+};
+
+/** HTML elements that carry no content of their own (void elements) plus the
+ *  XHTML self-closing spelling. Dropping such a token's markup cannot lose
+ *  text, so it degrades silently rather than surviving as literal prose.
+ *  `br` is excluded — it FOLDS to a line break. */
+export const HTML_VOID_TAGS: ReadonlySet<string> = new Set([
+  "area",
+  "base",
+  "col",
+  "embed",
+  "hr",
+  "img",
+  "input",
+  "link",
+  "meta",
+  "param",
+  "source",
+  "track",
+  "wbr",
+]);
+
+/** Structural / block-level element names. A degrade of one of these leaves a
+ *  hard line break behind so its neighbours do not glue together
+ *  (`<ul><li>one</li><li>two</li></ul>` -> `onetwo` was the defect). The
+ *  passthrough allowlist (`details`, `summary`, `aside`, `cite`) is
+ *  deliberately absent — those never degrade. */
+export const HTML_BLOCK_LEVEL_TAGS: ReadonlySet<string> = new Set([
+  "address",
+  "article",
+  "blockquote",
+  "center",
+  "dd",
+  "div",
+  "dl",
+  "dt",
+  "fieldset",
+  "figcaption",
+  "figure",
+  "footer",
+  "form",
+  "h1",
+  "h2",
+  "h3",
+  "h4",
+  "h5",
+  "h6",
+  "header",
+  "hr",
+  "li",
+  "main",
+  "nav",
+  "ol",
+  "p",
+  "section",
+  "table",
+  "tbody",
+  "td",
+  "tfoot",
+  "th",
+  "thead",
+  "tr",
+  "ul",
+]);
+
+/** True when a degraded token should leave a line break behind. */
+export function isBlockLevelTag(name: string): boolean {
+  return HTML_BLOCK_LEVEL_TAGS.has(name);
+}
+
+/** True for a tag that delimits no content of its own. */
+export function isVoidTag(name: string): boolean {
+  return HTML_VOID_TAGS.has(name);
+}
+
+/**
+ * HTML-entity-escape a run of literal text so its angle brackets reach the
+ * reader instead of being parsed as a tag (or 400ing as an unsupported one).
+ *
+ * Bot API rich markdown "can contain arbitrary HTML … parsed as described in
+ * Rich HTML style", and Rich HTML's supported named entities are documented as
+ * exactly `&lt; &gt; &amp; &quot; &apos; &nbsp; &hellip; &mdash; &ndash;
+ * &lsquo; &rsquo; &ldquo; &rdquo;` (https://core.telegram.org/bots/api). Only
+ * the first three are needed here. `&` is escaped FIRST so an already-entity
+ * -looking run cannot be double-decoded.
+ */
+export function escapeHtmlLiteral(text: string): string {
+  return text.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;");
+}
+
+/** The `class="language-xxx"` hint on a `<pre><code …>` block, or null. */
+const LANGUAGE_CLASS_RE = /(?:^|\s)class\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'>]+))/i;
+
+/** Pull the fenced-block language out of a `<code class="language-python">`
+ *  tag's attributes, or null. Telegram's reference spells the language hint
+ *  exactly this way; anything else yields a plain fence. */
+export function languageOf(tag: HtmlTagInfo): string | null {
+  const m = LANGUAGE_CLASS_RE.exec(tag.attrs);
+  if (m == null) return null;
+  const cls = m[1] ?? m[2] ?? m[3] ?? "";
+  const lang = cls
+    .split(/\s+/)
+    .map((c) => /^language-(.+)$/.exec(c)?.[1])
+    .find((c): c is string => c != null && c.length > 0);
+  // A fence info string cannot contain a backtick (it would close the fence).
+  return lang != null && !lang.includes("`") ? lang : null;
+}
+
+/** Tags PROVEN on the wire by raw `sendRichMessage` probes (2026-08-13) to
+ *  render as native typed nodes. These pass through raw and unescaped.
+ *  Deliberately an allowlist: an unprobed tag is not known-good syntax. */
+export const HTML_PASSTHROUGH_TAGS: ReadonlySet<string> = new Set([
+  "u",
+  "sub",
+  "sup",
+  "details",
+  "summary",
+  "aside",
+  "cite",
+]);
+
+/** Matches one HTML comment or one start/end tag. Attribute values containing
+ *  a raw `>` are not supported (they are invalid unquoted HTML and vanishingly
+ *  rare in agent prose); such a token simply degrades. */
+export const HTML_TOKEN_RE =
+  /<!--[\s\S]*?-->|<\/?[A-Za-z][A-Za-z0-9-]*(?:\s[^<>]*?)?\/?>/g;
+
+const TAG_RE = /^<(\/?)([A-Za-z][A-Za-z0-9-]*)((?:\s[^<>]*?)?)(\/?)>$/;
+const HREF_RE = /(?:^|\s)href\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'>]+))/i;
+
+/** Classify a single raw HTML token. */
+export function classifyHtmlTag(raw: string): HtmlTagInfo {
+  if (raw.startsWith("<!--")) return { kind: "comment", name: "", raw, attrs: "" };
+  const m = TAG_RE.exec(raw);
+  if (m == null) return { kind: "other", name: "", raw, attrs: "" };
+  const [, slash, name, attrs, selfClose] = m;
+  const kind: HtmlTagKind =
+    slash === "/" ? "close" : selfClose === "/" ? "selfclose" : "open";
+  return { kind, name: name.toLowerCase(), raw, attrs: attrs ?? "" };
+}
+
+/** Pull the `href` value out of a tag's attribute text, or null when absent
+ *  or empty. The ORIGINAL bytes are returned — never rewritten. */
+export function hrefOf(tag: HtmlTagInfo): string | null {
+  const m = HREF_RE.exec(tag.attrs);
+  if (m == null) return null;
+  const value = m[1] ?? m[2] ?? m[3] ?? "";
+  return value.length > 0 ? value : null;
+}
+
+/** True when this tag is emitted raw and unescaped (wire-verified allowlist). */
+export function isPassthroughTag(tag: HtmlTagInfo): boolean {
+  return tag.name.length > 0 && HTML_PASSTHROUGH_TAGS.has(tag.name);
+}
+
+/**
+ * Decide which HTML tokens in a DOCUMENT are balanced markup and which are
+ * bare prose, given every token in document order.
+ *
+ * Matching cannot be decided inside a single mdast `html` node: micromark
+ * splits `<details open>…\n\nbody\n\n</details>` into THREE nodes, so the open
+ * and close markers arrive in different streams. A document-level pass is the
+ * only place the question is answerable. Feeding it mdast `html` nodes (rather
+ * than the raw source) also means tags inside a code fence or a code span are
+ * never counted — they are not markup and must not balance anything.
+ *
+ * Standard HTML-ish matching: opens are pushed on a stack and a close pops to
+ * the nearest same-named open, leaving anything above it unmatched. Void and
+ * self-closing tags are balanced by definition and are not tracked.
+ *
+ * Returns the SOURCE OFFSETS of every token that has a partner; a token whose
+ * offset is absent is unmatched — prose, per the module policy above.
+ */
+export function matchedTagOffsets(
+  tokens: ReadonlyArray<{ tag: HtmlTagInfo; start: number }>,
+): Set<number> {
+  const matched = new Set<number>();
+  const stack: { name: string; start: number }[] = [];
+  for (const { tag, start } of tokens) {
+    if (tag.kind === "open" && !isVoidTag(tag.name)) {
+      stack.push({ name: tag.name, start });
+      continue;
+    }
+    if (tag.kind !== "close") continue;
+    for (let i = stack.length - 1; i >= 0; i--) {
+      if (stack[i].name !== tag.name) continue;
+      matched.add(stack[i].start);
+      matched.add(start);
+      stack.length = i;
+      break;
+    }
+  }
+  return matched;
+}
+
+/** A piece of a raw-HTML string: either an HTML token or a run of text. */
+export type HtmlPiece =
+  | { kind: "tag"; tag: HtmlTagInfo; start: number; end: number }
+  | { kind: "text"; text: string; start: number; end: number };
+
+/**
+ * Split a raw string into HTML tokens and the text runs between them.
+ * `base` is the absolute source offset the string starts at, so every piece
+ * carries usable UTF-16 offsets into the original markdown.
+ */
+export function tokenizeHtml(raw: string, base: number): HtmlPiece[] {
+  const pieces: HtmlPiece[] = [];
+  const re = new RegExp(HTML_TOKEN_RE.source, "g");
+  let last = 0;
+  let m: RegExpExecArray | null;
+  while ((m = re.exec(raw)) != null) {
+    if (m.index > last) {
+      pieces.push({
+        kind: "text",
+        text: raw.slice(last, m.index),
+        start: base + last,
+        end: base + m.index,
+      });
+    }
+    pieces.push({
+      kind: "tag",
+      tag: classifyHtmlTag(m[0]),
+      start: base + m.index,
+      end: base + m.index + m[0].length,
+    });
+    last = m.index + m[0].length;
+  }
+  if (last < raw.length) {
+    pieces.push({
+      kind: "text",
+      text: raw.slice(last),
+      start: base + last,
+      end: base + raw.length,
+    });
+  }
+  return pieces;
+}