safe-mdx 1.7.0 → 1.8.0

package/src/incremental-parse.test.ts

@@ -0,0 +1,315 @@
+// Verifies segment-cached parsing for streaming MDX without tying the API to React rendering.
+import { describe, expect, test } from 'vitest'
+
+import { mdxParse } from './parse.ts'
+import { createMdxProcessor, parseMarkdownIncremental, type SegmentCache } from './incremental-parse.ts'
+
+function nodeSummary(ast: any) {
+    return ast.children.map((node: any) => ({
+        type: node.type,
+        value: node.value,
+        name: node.name,
+        start: node.position?.start,
+        end: node.position?.end,
+    }))
+}
+
+describe('parseMarkdownIncremental', () => {
+    test('parses MDX into an mdast root and reports no errors', () => {
+        const cache: SegmentCache = new Map()
+        const result = parseMarkdownIncremental({
+            markdown: '# Hello\n\n<Alert>streaming</Alert>\n\nTail',
+            cache,
+            trailingNodes: 1,
+        })
+
+        expect(result.errors).toMatchInlineSnapshot(`[]`)
+        expect(nodeSummary(result.mdast)).toMatchInlineSnapshot(`
+          [
+            {
+              "end": {
+                "column": 8,
+                "line": 1,
+                "offset": 7,
+              },
+              "name": undefined,
+              "start": {
+                "column": 1,
+                "line": 1,
+                "offset": 0,
+              },
+              "type": "heading",
+              "value": undefined,
+            },
+            {
+              "end": {
+                "column": 25,
+                "line": 3,
+                "offset": 33,
+              },
+              "name": "Alert",
+              "start": {
+                "column": 1,
+                "line": 3,
+                "offset": 9,
+              },
+              "type": "mdxJsxFlowElement",
+              "value": undefined,
+            },
+            {
+              "end": {
+                "column": 5,
+                "line": 5,
+                "offset": 39,
+              },
+              "name": undefined,
+              "start": {
+                "column": 1,
+                "line": 5,
+                "offset": 35,
+              },
+              "type": "paragraph",
+              "value": undefined,
+            },
+          ]
+        `)
+        expect(Array.from(cache.keys())).toMatchInlineSnapshot(`
+          [
+            0,
+            7,
+          ]
+        `)
+    })
+
+    test('reuses cached stable segments', () => {
+        const cache: SegmentCache = new Map()
+        let parseCalls = 0
+        const parse = (markdown: string) => {
+            parseCalls++
+            return mdxParse(markdown)
+        }
+
+        parseMarkdownIncremental({
+            markdown: '# Title\n\nFirst paragraph\n\nSecond paragraph',
+            cache,
+            trailingNodes: 0,
+            parse,
+        })
+        parseMarkdownIncremental({
+            markdown: '# Title\n\nFirst paragraph\n\nSecond paragraph',
+            cache,
+            trailingNodes: 0,
+            parse,
+        })
+
+        expect(parseCalls).toMatchInlineSnapshot(`1`)
+        expect(Array.from(cache.entries()).map(([start, entry]) => ({ start, end: entry.end }))).toMatchInlineSnapshot(`
+          [
+            {
+              "end": 7,
+              "start": 0,
+            },
+            {
+              "end": 24,
+              "start": 7,
+            },
+            {
+              "end": 42,
+              "start": 24,
+            },
+          ]
+        `)
+    })
+
+    test('keeps positions correct after cached prefixes', () => {
+        const cache: SegmentCache = new Map()
+
+        parseMarkdownIncremental({
+            markdown: '# Title\n\nFirst paragraph\n\nLive',
+            cache,
+            trailingNodes: 1,
+        })
+        const result = parseMarkdownIncremental({
+            markdown: '# Title\n\nFirst paragraph\n\nLive tail\n\nNew paragraph',
+            cache,
+            trailingNodes: 1,
+        })
+
+        expect(result.errors).toMatchInlineSnapshot(`[]`)
+        expect(nodeSummary(result.mdast)).toMatchInlineSnapshot(`
+          [
+            {
+              "end": {
+                "column": 8,
+                "line": 1,
+                "offset": 7,
+              },
+              "name": undefined,
+              "start": {
+                "column": 1,
+                "line": 1,
+                "offset": 0,
+              },
+              "type": "heading",
+              "value": undefined,
+            },
+            {
+              "end": {
+                "column": 16,
+                "line": 3,
+                "offset": 24,
+              },
+              "name": undefined,
+              "start": {
+                "column": 1,
+                "line": 3,
+                "offset": 9,
+              },
+              "type": "paragraph",
+              "value": undefined,
+            },
+            {
+              "end": {
+                "column": 10,
+                "line": 5,
+                "offset": 35,
+              },
+              "name": undefined,
+              "start": {
+                "column": 1,
+                "line": 5,
+                "offset": 26,
+              },
+              "type": "paragraph",
+              "value": undefined,
+            },
+            {
+              "end": {
+                "column": 14,
+                "line": 7,
+                "offset": 50,
+              },
+              "name": undefined,
+              "start": {
+                "column": 1,
+                "line": 7,
+                "offset": 37,
+              },
+              "type": "paragraph",
+              "value": undefined,
+            },
+          ]
+        `)
+    })
+
+    test('returns partial cached AST instead of throwing on invalid live tail', () => {
+        const cache: SegmentCache = new Map()
+
+        parseMarkdownIncremental({
+            markdown: '# Stable\n\nDone\n\nTail',
+            cache,
+            trailingNodes: 1,
+        })
+        const result = parseMarkdownIncremental({
+            markdown: '# Stable\n\nDone\n\n<Card',
+            cache,
+            trailingNodes: 1,
+        })
+
+        expect(result.errors.map((error) => ({
+            message: error.message,
+            line: error.line,
+            column: error.column,
+            offset: error.offset,
+        }))).toMatchInlineSnapshot(`
+          [
+            {
+              "column": 6,
+              "line": 5,
+              "message": "Unexpected end of file in name, expected a name character such as letters, digits, \`$\`, or \`_\`; whitespace before attributes; or the end of the tag",
+              "offset": 21,
+            },
+          ]
+        `)
+        expect(nodeSummary(result.mdast)).toMatchInlineSnapshot(`
+          [
+            {
+              "end": {
+                "column": 9,
+                "line": 1,
+                "offset": 8,
+              },
+              "name": undefined,
+              "start": {
+                "column": 1,
+                "line": 1,
+                "offset": 0,
+              },
+              "type": "heading",
+              "value": undefined,
+            },
+            {
+              "end": {
+                "column": 5,
+                "line": 3,
+                "offset": 14,
+              },
+              "name": undefined,
+              "start": {
+                "column": 1,
+                "line": 3,
+                "offset": 10,
+              },
+              "type": "paragraph",
+              "value": undefined,
+            },
+          ]
+        `)
+    })
+
+    test('uses custom processor plugins', () => {
+        const cache: SegmentCache = new Map()
+        const processor = createMdxProcessor({
+            remarkPlugins: [remarkUppercaseText],
+        })
+
+        const result = parseMarkdownIncremental({
+            markdown: 'hello **world**',
+            cache,
+            processor,
+        })
+
+        expect(result.errors).toMatchInlineSnapshot(`[]`)
+        expect(textValues(result.mdast)).toMatchInlineSnapshot(`
+          [
+            "HELLO ",
+            "WORLD",
+          ]
+        `)
+    })
+})
+
+function remarkUppercaseText() {
+    return (tree: any) => {
+        walk(tree, (node) => {
+            if (node.type === 'text') {
+                node.value = node.value.toUpperCase()
+            }
+        })
+    }
+}
+
+function textValues(node: any): string[] {
+    const values: string[] = []
+    walk(node, (child) => {
+        if (child.type === 'text') values.push(child.value)
+    })
+    return values
+}
+
+function walk(node: any, visit: (node: any) => void) {
+    visit(node)
+    if (Array.isArray(node.children)) {
+        for (const child of node.children) walk(child, visit)
+    }
+}

package/src/incremental-parse.ts

@@ -0,0 +1,219 @@
+// Segment-cached markdown parsing for streaming MDX input. It reuses stable mdast nodes and reparses only the live tail.
+import type { Root, RootContent } from 'mdast'
+
+import { createMdxProcessor, mdxParse, type MdxProcessor } from './parse.ts'
+
+export { createMdxProcessor }
+
+export interface SegmentEntry {
+    len: number
+    hash: number
+    end: number
+    nodes: RootContent[]
+}
+
+export type SegmentCache = Map<number, SegmentEntry>
+
+export interface IncrementalParseError {
+    message: string
+    line?: number
+    column?: number
+    offset?: number
+    cause: ParseErrorCause
+}
+
+type ParseErrorCause = string | {
+    message?: string
+    place?: {
+        line?: number
+        column?: number
+        offset?: number
+    }
+}
+
+type MutableAstNode = {
+    position?: {
+        start?: Point
+        end?: Point
+    }
+    children?: MutableAstNode[]
+}
+
+type Point = {
+    line?: number
+    column?: number
+    offset?: number
+}
+
+export interface IncrementalParseOptions {
+    markdown: string
+    cache: SegmentCache
+    /** Number of unstable top-level nodes to keep out of the cache. Defaults to 2. */
+    trailingNodes?: number
+    /** Custom unified processor. Use `createMdxProcessor({ remarkPlugins })` for plugin customization. */
+    processor?: MdxProcessor
+    /** Fully custom parse hook. Takes precedence over `processor`. */
+    parse?: (markdown: string) => Root
+}
+
+export interface IncrementalParseResult {
+    mdast: Root
+    errors: IncrementalParseError[]
+}
+
+export function parseMarkdownIncremental({
+    markdown,
+    cache,
+    trailingNodes = 2,
+    processor,
+    parse,
+}: IncrementalParseOptions): IncrementalParseResult {
+    const children: RootContent[] = []
+    const errors: IncrementalParseError[] = []
+    const parseSlice = parse ?? ((code: string) => parseWithProcessor(code, processor))
+
+    for (let offset = 0; offset < markdown.length;) {
+        const entry = cache.get(offset)
+        if (entry) {
+            const slice = markdown.slice(offset, entry.end)
+            if (slice.length === entry.len && quickHash(slice) === entry.hash) {
+                children.push(...entry.nodes)
+                offset = entry.end
+                continue
+            }
+        }
+
+        const rest = markdown.slice(offset)
+        if (!rest) break
+        const basePoint = pointAtOffset(markdown, offset)
+
+        try {
+            const ast = parseSlice(rest)
+            children.push(...ast.children.map((node) => adjustNodePositions(node, basePoint)))
+        } catch (cause) {
+            errors.push(parseErrorFromCause(cause, basePoint))
+        }
+        break
+    }
+
+    const stableChildren = trailingNodes > 0 ? children.slice(0, -trailingNodes) : children
+    let segmentStart = 0
+    for (const node of stableChildren) {
+        const end = node.position?.end.offset
+        if (typeof end !== 'number' || end < segmentStart) continue
+
+        const slice = markdown.slice(segmentStart, end)
+        cache.set(segmentStart, {
+            len: slice.length,
+            hash: quickHash(slice),
+            end,
+            nodes: [node],
+        })
+        segmentStart = end
+    }
+
+    trimCache(cache)
+
+    return {
+        mdast: {
+            type: 'root',
+            children,
+            position: {
+                start: { line: 1, column: 1, offset: 0 },
+                end: pointAtOffset(markdown, markdown.length),
+            },
+        },
+        errors,
+    }
+}
+
+function parseWithProcessor(markdown: string, processor?: MdxProcessor): Root {
+    if (!processor) return mdxParse(markdown)
+
+    const file = processor.processSync(markdown)
+    const ast = file.data.ast
+    const isRoot = (value: any): value is Root => {
+        return value?.type === 'root' && Array.isArray(value?.children)
+    }
+    if (!isRoot(ast)) {
+        throw new Error('Processor did not expose mdast at file.data.ast')
+    }
+    return ast
+}
+
+function quickHash(s: string): number {
+    let h = 0x811c9dc5
+    for (let i = 0; i < s.length; i++) {
+        h ^= s.charCodeAt(i)
+        h = (h * 0x01000193) >>> 0
+    }
+    return h
+}
+
+function pointAtOffset(text: string, offset: number) {
+    let line = 1
+    let column = 1
+
+    for (let i = 0; i < offset; i++) {
+        if (text[i] === '\n') {
+            line++
+            column = 1
+        } else {
+            column++
+        }
+    }
+
+    return { line, column, offset }
+}
+
+function adjustNodePositions<T extends MutableAstNode>(node: T, basePoint: { line: number, column: number, offset: number }): T {
+    const position = node.position
+    if (position) {
+        adjustPoint(position.start, basePoint)
+        adjustPoint(position.end, basePoint)
+    }
+
+    if (Array.isArray(node.children)) {
+        node.children = node.children.map((child) => adjustNodePositions(child, basePoint))
+    }
+
+    return node
+}
+
+function adjustPoint(point: Point | undefined, basePoint: { line: number, column: number, offset: number }) {
+    if (!point) return
+
+    if (typeof point.offset === 'number') {
+        point.offset += basePoint.offset
+    }
+    if (typeof point.line === 'number') {
+        point.line += basePoint.line - 1
+    }
+    if (typeof point.column === 'number' && point.line === basePoint.line) {
+        point.column += basePoint.column - 1
+    }
+}
+
+function parseErrorFromCause(cause: ParseErrorCause, basePoint: { line: number, column: number, offset: number }): IncrementalParseError {
+    const place = typeof cause === 'string' ? undefined : cause.place
+    const point = place ? { ...place } : undefined
+    if (point) adjustPoint(point, basePoint)
+
+    return {
+        message: typeof cause === 'string' ? cause : cause.message ?? String(cause),
+        line: point?.line,
+        column: point?.column,
+        offset: point?.offset,
+        cause,
+    }
+}
+
+function trimCache(cache: SegmentCache) {
+    const maxCacheSize = 300
+    if (cache.size <= maxCacheSize) return
+
+    const keys = Array.from(cache.keys()).sort((a, b) => a - b)
+    for (const key of keys.slice(0, cache.size - maxCacheSize)) {
+        cache.delete(key)
+    }
+}

package/src/markdown-html.test.tsx

@@ -0,0 +1,144 @@
+// Tests for rendering markdown (not MDX) that contains raw HTML blocks.
+// The remarkHtmlToMdx plugin is used as a pre-processing step to convert
+// `html` AST nodes (produced by plain remark) into mdxJsx nodes before
+// MdastToJsx ever sees them. This way linkedom stays out of the main bundle.
+import dedent from 'dedent'
+import React from 'react'
+import { renderToStaticMarkup } from 'react-dom/server'
+import { remark } from 'remark'
+import remarkGfm from 'remark-gfm'
+import { expect, test, describe } from 'vitest'
+import type { Root } from 'mdast'
+import { MdastToJsx } from './safe-mdx.tsx'
+import { remarkHtmlToMdx } from './markdown.ts'
+import { validHtmlElements } from './html/valid-html-elements.ts'
+
+const components = {
+    Heading({ children, ...props }) {
+        return React.createElement('h1', props, children)
+    },
+}
+
+// convertTagName that only keeps standard HTML elements, drops everything else.
+// This mirrors what the old case 'html' did inside MdastToJsx.
+const onlyValidHtml = ({ tagName }: { tagName: string }) =>
+    validHtmlElements.has(tagName.toLowerCase()) ? tagName.toLowerCase() : ''
+
+// Parse with plain remark + remarkHtmlToMdx pre-processor, then render via MdastToJsx.
+// No html nodes remain by the time MdastToJsx sees the AST.
+function render(markdown: string) {
+    const processor = remark()
+        .use(remarkGfm)
+        .use(remarkHtmlToMdx, { convertTagName: onlyValidHtml })
+    const mdast = processor.parse(markdown) as Root
+    processor.runSync(mdast)
+    const visitor = new MdastToJsx({ markdown, mdast, components })
+    const result = visitor.run()
+    const html = renderToStaticMarkup(result)
+    return { errors: visitor.errors || [], html }
+}
+
+describe('remarkHtmlToMdx pre-processor with plain remark', () => {
+    test('renders a block-level HTML div', () => {
+        const { html, errors } = render(dedent`
+            # Title
+
+            <div class="box">hello world</div>
+
+            Some paragraph after.
+        `)
+        expect(errors).toMatchInlineSnapshot(`[]`)
+        expect(html).toMatchInlineSnapshot(`"<h1>Title</h1><div class="box">hello world</div><p>Some paragraph after.</p>"`)
+    })
+
+    test('inline HTML in a paragraph is split by remark into per-tag html nodes (known limitation)', () => {
+        // remark parses inline HTML at tag boundaries, so <strong>bold</strong> becomes:
+        //   html("<strong>") → empty element, text("bold"), html("</strong>") → dropped
+        // Block-level HTML (own line + blank lines) is the only reliably handled case.
+        const { html, errors } = render(
+            'Some text with <strong>bold</strong> and <em>italic</em> inline.'
+        )
+        expect(errors).toMatchInlineSnapshot(`[]`)
+        expect(html).toMatchInlineSnapshot(`"<p>Some text with <strong></strong>bold and <em></em>italic inline.</p>"`)
+    })
+
+    test('strips unknown custom elements but keeps their text children', () => {
+        // <callout> is not in validHtmlElements, so the tag wrapper is dropped
+        const { html, errors } = render(dedent`
+            <callout icon="💡">
+            Important note
+            </callout>
+        `)
+        expect(errors).toMatchInlineSnapshot(`[]`)
+        expect(html).toMatchInlineSnapshot(`"Important note"`)
+    })
+
+    test('renders self-closing void element hr', () => {
+        const { html, errors } = render(dedent`
+            Before
+
+            <hr>
+
+            After
+        `)
+        expect(errors).toMatchInlineSnapshot(`[]`)
+        expect(html).toMatchInlineSnapshot(`"<p>Before</p><hr/><p>After</p>"`)
+    })
+
+    test('renders HTML anchor with name attribute', () => {
+        const { html, errors } = render(dedent`
+            <a name="section-one"></a>
+
+            ## Section One
+        `)
+        expect(errors).toMatchInlineSnapshot(`[]`)
+        expect(html).toMatchInlineSnapshot(`"<p><a name="section-one"></a></p><h2>Section One</h2>"`)
+    })
+
+    test('converts class attribute to className', () => {
+        // Note: span without blank-line separation is treated as inline HTML by remark —
+        // the opening tag and text content become separate nodes (see inline limitation above).
+        const { html, errors } = render(dedent`
+            <span class="highlight">highlighted text</span>
+        `)
+        expect(errors).toMatchInlineSnapshot(`[]`)
+        expect(html).toMatchInlineSnapshot(`"<p><span class="highlight"></span>highlighted text</p>"`)
+    })
+
+    test('renders a table from raw HTML', () => {
+        const { html, errors } = render(dedent`
+            <table>
+            <tr><th>Name</th><th>Value</th></tr>
+            <tr><td>foo</td><td>bar</td></tr>
+            </table>
+        `)
+        expect(errors).toMatchInlineSnapshot(`[]`)
+        expect(html).toMatchInlineSnapshot(`"<table><tr><th>Name</th><th>Value</th></tr><tr><td>foo</td><td>bar</td></tr></table>"`)
+    })
+
+    test('mixed markdown and HTML block', () => {
+        const { html, errors } = render(dedent`
+            ## My Section
+
+            <div class="card">
+            Card content here
+            </div>
+
+            Back to **markdown**.
+        `)
+        expect(errors).toMatchInlineSnapshot(`[]`)
+        expect(html).toMatchInlineSnapshot(`"<h2>My Section</h2><div class="card">Card content here</div><p>Back to <strong>markdown</strong>.</p>"`)
+    })
+
+    test('without the plugin, html nodes are silently skipped by MdastToJsx', () => {
+        // Verify the old case 'html' path is gone — without the plugin, html nodes are dropped
+        const processor = remark().use(remarkGfm) // no remarkHtmlToMdx
+        const mdast = processor.parse(dedent`
+            <div class="box">hello world</div>
+        `) as Root
+        const visitor = new MdastToJsx({ markdown: '', mdast, components })
+        const result = visitor.run()
+        const html = renderToStaticMarkup(result)
+        expect(html).toMatchInlineSnapshot(`""`)
+    })
+})

package/src/markdown.ts

@@ -0,0 +1,4 @@
+// Pre-processing utilities for rendering plain markdown (not MDX) that contains raw HTML.
+// Import from 'safe-mdx/markdown' to keep linkedom out of the main bundle.
+export { remarkHtmlToMdx, parseHtmlToMdxAst } from './html/html-to-mdx-ast.ts'
+export type { RemarkHtmlToMdxOptions } from './html/html-to-mdx-ast.ts'