switchroom 0.17.10 → 0.18.3

package/telegram-plugin/tests/render/parse-torture.test.ts

@@ -0,0 +1,136 @@
+// Cross-check the new markdown->IR parser (render/parse.ts) against the
+// existing adversarial torture-fixture set (formatting-torture-set.ts).
+//
+// This does NOT exercise the renderer or chunker (out of scope for the
+// parser+IR increment) — it feeds each fixture's raw `input` straight into
+// `parse()` and walks the resulting IR tree collecting a flat entity list,
+// then asserts every entity the fixture's `expect` block calls for is
+// present in that list. This is a structural cross-check independent of the
+// existing gateway-pipeline oracle: it proves the new parser captures the
+// same intent the current hand-rolled pipeline is required to preserve.
+import { describe, it, expect } from 'vitest'
+import { parse } from '../../render/parse.js'
+import type { Block, Inline } from '../../render/ir.js'
+import { TORTURE_SET, type ExpectEntity } from '../formatting-torture-set.js'
+import type { EntityType } from '../rich-markdown-oracle.js'
+
+interface FlatEntity {
+  type: EntityType
+  text: string
+  url?: string
+  lang?: string
+}
+
+function textOf(nodes: Inline[]): string {
+  return nodes
+    .map((n) => {
+      switch (n.type) {
+        case 'plain':
+        case 'code':
+          return n.text
+        default:
+          return textOf(n.children)
+      }
+    })
+    .join('')
+}
+
+function walkInline(node: Inline, out: FlatEntity[]) {
+  switch (node.type) {
+    case 'bold':
+      out.push({ type: 'bold', text: textOf(node.children) })
+      node.children.forEach((c) => walkInline(c, out))
+      break
+    case 'italic':
+      out.push({ type: 'italic', text: textOf(node.children) })
+      node.children.forEach((c) => walkInline(c, out))
+      break
+    case 'strike':
+      out.push({ type: 'strikethrough', text: textOf(node.children) })
+      node.children.forEach((c) => walkInline(c, out))
+      break
+    case 'code':
+      out.push({ type: 'code', text: node.text })
+      break
+    case 'link':
+      out.push({ type: 'link', text: textOf(node.children), url: node.href })
+      node.children.forEach((c) => walkInline(c, out))
+      break
+    case 'plain':
+      break
+    case 'underline':
+    case 'spoiler':
+    case 'highlight':
+      // No dedicated oracle EntityType for these Bot API 10.1 constructs yet;
+      // recurse so any nested bold/italic/code/link inside them is still
+      // collected for the cross-check.
+      node.children.forEach((c) => walkInline(c, out))
+      break
+  }
+}
+
+function walkBlock(node: Block, out: FlatEntity[]) {
+  switch (node.type) {
+    case 'paragraph':
+    case 'heading':
+      node.children.forEach((c) => walkInline(c, out))
+      break
+    case 'blockquote':
+      node.children.forEach((c) => walkBlock(c, out))
+      break
+    case 'code-block':
+      out.push({ type: 'pre', text: node.text, lang: node.language ?? undefined })
+      break
+    case 'list':
+      for (const item of node.items) item.children.forEach((c) => walkBlock(c, out))
+      break
+    case 'table':
+      for (const row of [node.header, ...node.rows]) {
+        for (const cell of row.cells) cell.children.forEach((c) => walkInline(c, out))
+      }
+      break
+    case 'thematic-break':
+      break
+  }
+}
+
+function flatten(input: string): FlatEntity[] {
+  const doc = parse(input)
+  const out: FlatEntity[] = []
+  doc.blocks.forEach((b) => walkBlock(b, out))
+  return out
+}
+
+function matches(found: FlatEntity[], want: ExpectEntity): boolean {
+  return found.some(
+    (f) =>
+      f.type === want.type &&
+      f.text === want.text &&
+      (want.url === undefined || f.url === want.url) &&
+      (want.lang === undefined || f.lang === want.lang),
+  )
+}
+
+describe('parse: torture-fixture cross-check', () => {
+  for (const fixture of TORTURE_SET) {
+    if (fixture.expect.length === 0) {
+      // No entity assertions for this fixture — just prove parse() doesn't
+      // throw and returns at least one block (structure-only fixtures).
+      it(`${fixture.name}: parses without throwing`, () => {
+        const doc = parse(fixture.input)
+        expect(doc.blocks.length).toBeGreaterThan(0)
+      })
+      continue
+    }
+
+    it(`${fixture.name}: IR captures every expected entity`, () => {
+      const found = flatten(fixture.input)
+      for (const want of fixture.expect) {
+        expect(
+          matches(found, want),
+          `expected a ${want.type} entity with text ${JSON.stringify(want.text)} in fixture "${fixture.name}"; found: ${JSON.stringify(found)}`,
+        ).toBe(true)
+      }
+    })
+  }
+})

package/telegram-plugin/tests/render/parse.test.ts

@@ -0,0 +1,393 @@
+import { describe, it, expect } from "vitest";
+import { parse } from "../../render/parse.js";
+import type { Block, Inline } from "../../render/ir.js";
+
+// Assert every node in a subtree round-trips: source.slice(start, end) is the
+// exact source text the node spans. Walks both inline and block children.
+function assertOffsetsRoundTrip(node: any, source: string) {
+  expect(typeof node.start).toBe("number");
+  expect(typeof node.end).toBe("number");
+  // The slice must be non-empty for real nodes and lie within bounds.
+  expect(node.start).toBeGreaterThanOrEqual(0);
+  expect(node.end).toBeLessThanOrEqual(source.length);
+  expect(node.end).toBeGreaterThanOrEqual(node.start);
+  if (Array.isArray(node.children)) {
+    for (const c of node.children) assertOffsetsRoundTrip(c, source);
+  }
+  if (Array.isArray(node.items)) {
+    for (const it of node.items) assertOffsetsRoundTrip(it, source);
+  }
+  if (Array.isArray(node.cells)) {
+    for (const c of node.cells) assertOffsetsRoundTrip(c, source);
+  }
+}
+
+describe("parse: inline palette", () => {
+  const cases: Array<{
+    name: string;
+    md: string;
+    // The inline construct's source text (what slice(start,end) must equal).
+    fragment: string;
+    check: (inline: Inline) => void;
+  }> = [
+    {
+      name: "bold",
+      md: "**hi**",
+      fragment: "**hi**",
+      check: (n) => {
+        expect(n.type).toBe("bold");
+        expect((n as any).children[0]).toMatchObject({ type: "plain", text: "hi" });
+      },
+    },
+    {
+      name: "italic",
+      md: "*hi*",
+      fragment: "*hi*",
+      check: (n) => {
+        expect(n.type).toBe("italic");
+        expect((n as any).children[0]).toMatchObject({ type: "plain", text: "hi" });
+      },
+    },
+    {
+      name: "strike",
+      md: "~~hi~~",
+      fragment: "~~hi~~",
+      check: (n) => {
+        expect(n.type).toBe("strike");
+        expect((n as any).children[0]).toMatchObject({ type: "plain", text: "hi" });
+      },
+    },
+    {
+      name: "inline code",
+      md: "`x = 1`",
+      fragment: "`x = 1`",
+      check: (n) => {
+        expect(n).toMatchObject({ type: "code", text: "x = 1" });
+      },
+    },
+    {
+      name: "link",
+      md: "[label](https://example.com)",
+      fragment: "[label](https://example.com)",
+      check: (n) => {
+        expect(n.type).toBe("link");
+        expect((n as any).href).toBe("https://example.com");
+        expect((n as any).children[0]).toMatchObject({ type: "plain", text: "label" });
+      },
+    },
+  ];
+
+  for (const c of cases) {
+    it(`parses ${c.name} and offsets round-trip`, () => {
+      const doc = parse(c.md);
+      expect(doc.blocks).toHaveLength(1);
+      const para = doc.blocks[0];
+      expect(para.type).toBe("paragraph");
+      const inline = (para as any).children[0] as Inline;
+      c.check(inline);
+      expect(c.md.slice(inline.start, inline.end)).toBe(c.fragment);
+      assertOffsetsRoundTrip(para, c.md);
+    });
+  }
+});
+
+describe("parse: heading levels", () => {
+  for (let level = 1; level <= 6; level++) {
+    it(`parses an h${level} heading`, () => {
+      const md = `${"#".repeat(level)} Title`;
+      const doc = parse(md);
+      expect(doc.blocks).toHaveLength(1);
+      const h = doc.blocks[0] as any;
+      expect(h.type).toBe("heading");
+      expect(h.level).toBe(level);
+      expect(h.children[0]).toMatchObject({ type: "plain", text: "Title" });
+      expect(md.slice(h.start, h.end)).toBe(md);
+    });
+  }
+});
+
+describe("parse: fenced code with language", () => {
+  it("captures text and language", () => {
+    const md = "```ts\nconst x = 1;\n```";
+    const doc = parse(md);
+    const cb = doc.blocks[0] as any;
+    expect(cb.type).toBe("code-block");
+    expect(cb.language).toBe("ts");
+    expect(cb.text).toBe("const x = 1;");
+    expect(md.slice(cb.start, cb.end)).toBe(md);
+  });
+
+  it("null language for a bare fence", () => {
+    const md = "```\nplain\n```";
+    const doc = parse(md);
+    const cb = doc.blocks[0] as any;
+    expect(cb.type).toBe("code-block");
+    expect(cb.language).toBeNull();
+  });
+});
+
+describe("parse: blockquote", () => {
+  it("parses a blockquote with expandable=false", () => {
+    const md = "> quoted text";
+    const doc = parse(md);
+    const bq = doc.blocks[0] as any;
+    expect(bq.type).toBe("blockquote");
+    expect(bq.expandable).toBe(false);
+    expect(bq.children[0].type).toBe("paragraph");
+    expect(md.slice(bq.start, bq.end)).toBe(md);
+  });
+});
+
+describe("parse: lists", () => {
+  it("parses an unordered list (start=null)", () => {
+    const md = "- one\n- two";
+    const doc = parse(md);
+    const list = doc.blocks[0] as any;
+    expect(list.type).toBe("list");
+    expect(list.ordered).toBe(false);
+    expect(list.startNumber).toBeNull();
+    expect(list.items).toHaveLength(2);
+    expect(md.slice(list.start, list.end)).toBe(md);
+    assertOffsetsRoundTrip(list, md);
+  });
+
+  it("parses an ordered list with a start number", () => {
+    const md = "3. three\n4. four";
+    const doc = parse(md);
+    const list = doc.blocks[0] as any;
+    expect(list.type).toBe("list");
+    expect(list.ordered).toBe(true);
+    expect(list.startNumber).toBe(3);
+    expect(list.items).toHaveLength(2);
+  });
+
+  it("parses task-list checked/unchecked state", () => {
+    const md = "- [x] done\n- [ ] todo";
+    const doc = parse(md);
+    const list = doc.blocks[0] as any;
+    expect(list.items[0].checked).toBe(true);
+    expect(list.items[1].checked).toBe(false);
+  });
+
+  it("plain (non-task) list items have checked=null", () => {
+    const md = "- a\n- b";
+    const doc = parse(md);
+    const list = doc.blocks[0] as any;
+    expect(list.items[0].checked).toBeNull();
+    expect(list.items[1].checked).toBeNull();
+  });
+});
+
+describe("parse: GFM table with mixed alignment", () => {
+  it("captures header, rows, and per-column alignment", () => {
+    const md = ["| L | C | R |", "|:--|:-:|--:|", "| 1 | 2 | 3 |"].join("\n");
+    const doc = parse(md);
+    const table = doc.blocks[0] as any;
+    expect(table.type).toBe("table");
+    expect(table.align).toEqual(["left", "center", "right"]);
+    expect(table.header.cells).toHaveLength(3);
+    expect(table.rows).toHaveLength(1);
+    expect(table.rows[0].cells[0].children[0]).toMatchObject({ type: "plain", text: "1" });
+    expect(md.slice(table.start, table.end)).toBe(md);
+    assertOffsetsRoundTrip(table, md);
+  });
+});
+
+describe("parse: thematic break", () => {
+  it("parses a horizontal rule", () => {
+    const md = "---";
+    const doc = parse(md);
+    const tb = doc.blocks[0] as any;
+    expect(tb.type).toBe("thematic-break");
+    expect(md.slice(tb.start, tb.end)).toBe(md);
+  });
+});
+
+describe("parse: nested inline", () => {
+  it("bold containing italic, offsets round-trip at both levels", () => {
+    const md = "**bold *and italic* here**";
+    const doc = parse(md);
+    const bold = (doc.blocks[0] as any).children[0];
+    expect(bold.type).toBe("bold");
+    expect(md.slice(bold.start, bold.end)).toBe("**bold *and italic* here**");
+    const italic = bold.children.find((c: any) => c.type === "italic");
+    expect(italic).toBeDefined();
+    expect(md.slice(italic.start, italic.end)).toBe("*and italic*");
+    assertOffsetsRoundTrip(doc.blocks[0] as Block, md);
+  });
+});
+
+describe("parse: unsupported construct fallback", () => {
+  it("degrades a raw HTML block to plain text without dropping content", () => {
+    // Raw HTML blocks (mdast `html`) are outside the palette; they must
+    // survive as plain source text.
+    const md = "<div class=\"x\">raw</div>";
+    const doc = parse(md);
+    const block = doc.blocks[0] as any;
+    expect(block.type).toBe("paragraph");
+    const plain = block.children[0];
+    expect(plain.type).toBe("plain");
+    expect(plain.text).toBe(md);
+    expect(md.slice(plain.start, plain.end)).toBe(md);
+  });
+});
+
+describe("parse: astral-plane emoji offsets are UTF-16 units", () => {
+  it("uses UTF-16 code-unit offsets, not codepoints or bytes", () => {
+    // U+1F680 ROCKET is one codepoint but TWO UTF-16 code units (a surrogate
+    // pair) and FOUR UTF-8 bytes. The bold node after it must start at the
+    // UTF-16 offset.
+    const emoji = "\u{1F680}"; // 🚀, length 2 in UTF-16
+    expect(emoji.length).toBe(2);
+    const md = `${emoji} **go**`;
+    const doc = parse(md);
+    const para = doc.blocks[0] as any;
+    const bold = para.children.find((c: any) => c.type === "bold");
+    expect(bold).toBeDefined();
+    // "🚀 " is 3 UTF-16 code units (2 + space), so bold starts at offset 3.
+    expect(bold.start).toBe(3);
+    expect(md.slice(bold.start, bold.end)).toBe("**go**");
+    // Sanity: byte-based or codepoint-based offsets would give 2 or a
+    // different index, not 3.
+    expect(md.slice(bold.start, bold.end)).not.toBe("*go**");
+  });
+});
+
+describe("parse: underline vs bold (`__` vs `**`)", () => {
+  it("folds a `__…__` run into an underline node (not bold)", () => {
+    const md = "__underlined__";
+    const doc = parse(md);
+    const u = (doc.blocks[0] as any).children[0];
+    expect(u.type).toBe("underline");
+    expect(u.children[0]).toMatchObject({ type: "plain", text: "underlined" });
+    expect(md.slice(u.start, u.end)).toBe(md);
+  });
+
+  it("keeps a `**…**` run as bold", () => {
+    const b = (parse("**strong**").blocks[0] as any).children[0];
+    expect(b.type).toBe("bold");
+  });
+
+  it("keeps single-delimiter `_…_` as italic", () => {
+    const i = (parse("_em_").blocks[0] as any).children[0];
+    expect(i.type).toBe("italic");
+  });
+
+  it("recovers nested bold inside underline with offsets intact", () => {
+    const md = "__a **b** c__";
+    const doc = parse(md);
+    const u = (doc.blocks[0] as any).children[0];
+    expect(u.type).toBe("underline");
+    expect(md.slice(u.start, u.end)).toBe(md);
+    const bold = u.children.find((c: any) => c.type === "bold");
+    expect(bold).toBeDefined();
+    expect(md.slice(bold.start, bold.end)).toBe("**b**");
+    assertOffsetsRoundTrip(doc.blocks[0], md);
+  });
+});
+
+describe("parse: spoiler (`||…||`) + highlight (`==…==`)", () => {
+  it("splits `||secret||` into a spoiler node", () => {
+    const md = "before ||secret|| after";
+    const doc = parse(md);
+    const kids = (doc.blocks[0] as any).children;
+    expect(kids.map((k: any) => k.type)).toEqual(["plain", "spoiler", "plain"]);
+    const sp = kids[1];
+    expect(sp.children[0]).toMatchObject({ type: "plain", text: "secret" });
+    expect(md.slice(sp.start, sp.end)).toBe("||secret||");
+    assertOffsetsRoundTrip(doc.blocks[0], md);
+  });
+
+  it("splits `==marked==` into a highlight node", () => {
+    const md = "a ==marked== b";
+    const doc = parse(md);
+    const kids = (doc.blocks[0] as any).children;
+    const hi = kids.find((k: any) => k.type === "highlight");
+    expect(hi).toBeDefined();
+    expect(hi.children[0]).toMatchObject({ type: "plain", text: "marked" });
+    expect(md.slice(hi.start, hi.end)).toBe("==marked==");
+  });
+
+  it("recognises both delimiters in one run", () => {
+    const md = "||s|| and ==m==";
+    const types = (parse(md).blocks[0] as any).children.map((c: any) => c.type);
+    expect(types).toContain("spoiler");
+    expect(types).toContain("highlight");
+  });
+
+  it("leaves an unclosed / single delimiter as literal plain text", () => {
+    // `a || b` has no closing `||` — must NOT become a spoiler.
+    const kids = (parse("a || b").blocks[0] as any).children;
+    expect(kids.every((k: any) => k.type === "plain")).toBe(true);
+  });
+});
+
+describe("parse: nested lists", () => {
+  it("folds a nested sub-list under a list item", () => {
+    const md = "- a\n  - nested1\n  - nested2\n- b";
+    const doc = parse(md);
+    const list = doc.blocks[0] as any;
+    expect(list.type).toBe("list");
+    const item0Kinds = list.items[0].children.map((c: any) => c.type);
+    expect(item0Kinds).toContain("list");
+    const nested = list.items[0].children.find((c: any) => c.type === "list");
+    expect(nested.items).toHaveLength(2);
+    assertOffsetsRoundTrip(list, md);
+  });
+});
+
+describe("expandable blockquote (Bot API 10.1 `**>` marker)", () => {
+  it("parses a single-line `**>` quote into an expandable blockquote", () => {
+    const md = "**> a collapsible line";
+    const doc = parse(md);
+    const bq = doc.blocks[0] as any;
+    expect(bq.type).toBe("blockquote");
+    expect(bq.expandable).toBe(true);
+  });
+
+  it("marks a multi-line quote expandable when the FIRST line carries `**>`", () => {
+    const md = "**> line one\n> line two\n> line three";
+    const doc = parse(md);
+    const bq = doc.blocks[0] as any;
+    expect(bq.type).toBe("blockquote");
+    expect(bq.expandable).toBe(true);
+    // All three quoted lines fold into ONE blockquote.
+    expect(bq.children.length).toBeGreaterThanOrEqual(1);
+  });
+
+  it("leaves a plain `>` quote non-expandable", () => {
+    const doc = parse("> just an ordinary quote");
+    const bq = doc.blocks[0] as any;
+    expect(bq.type).toBe("blockquote");
+    expect(bq.expandable).toBe(false);
+  });
+
+  it("keeps content offsets byte-identical to the ORIGINAL source", () => {
+    const md = "**> quoted body here";
+    const doc = parse(md);
+    const bq = doc.blocks[0] as any;
+    // The marker rewrite (`**>` -> `  >`) is length-preserving, so the inner
+    // text still slices out of the ORIGINAL markdown exactly.
+    const plain = bq.children[0].children[0];
+    expect(md.slice(plain.start, plain.end)).toBe("quoted body here");
+    assertOffsetsRoundTrip(bq, md);
+  });
+
+  it("only recognises the marker at column 0 (leading indent is not expandable)", () => {
+    // The renderer only ever emits `**>` at column 0; a leading-indented
+    // variant is deliberately NOT treated as an expandable quote (matching it
+    // would push the length-preserving rewrite past the 3-space blockquote
+    // budget into indented-code-block territory).
+    const md = "   **> indented";
+    const doc = parse(md);
+    const bq = doc.blocks[0] as any;
+    expect(bq.expandable).not.toBe(true);
+  });
+
+  it("does not treat inline `**` bold followed by `>` mid-line as expandable", () => {
+    // `**bold** > text` is a paragraph, not a quote — the marker must be at
+    // line start.
+    const doc = parse("**bold** > not a quote");
+    expect(doc.blocks[0].type).toBe("paragraph");
+  });
+});