svelte-streamdown 3.1.2 → 4.0.0

package/dist/marked/index.js

@@ -106,21 +106,79 @@ export const lex = (markdown, extensions = []) => {
 export const createParseBlocksCache = () => ({
     content: '',
     raws: [],
-    keep: []
+    keep: [],
+    offsets: [0],
+    keptBefore: [0],
+    blocks: []
 });
 // Number of trailing rendered blocks that stay "live" (re-lexed every chunk).
 // 2 covers constructs that merge backward as they stream in — e.g. a paragraph
 // line becoming a table once its delimiter row arrives, or a setext heading.
 const SEAL_SLACK = 2;
-const blockTokensOf = (markdown, extensions) => new Lexer(getBlockOptions(extensions)).blockTokens(markdown, []);
+// Spot checks over the sealed prefix per append (see `appendable`). The sealed
+// region was compared byte-for-byte while it was live, so this only has to catch
+// a caller that swapped in a different, longer document.
+const SEAL_PROBES = 16;
+/**
+ * A Lexer used only to slice a document into top-level block raws.
+ *
+ * `Tokenizer.list()` recurses into every list item's text purely to decide
+ * whether the list is loose — `raw` is already final by then — and it marks that
+ * recursion by setting `state.top = false` first. `Tokenizer.blockquote()` also
+ * recurses, but its `raw` genuinely depends on the result (continuation lines are
+ * folded into a nested list/blockquote token), and it sets `state.top = true`.
+ * So dropping every nested call made with `top === false` removes the whole
+ * list-item subtree parse without changing a single raw.
+ */
+class SplitLexer extends Lexer {
+    blockTokens(src, tokens = [], lastParagraphClipped = false) {
+        if (!this.state.top) {
+            // Lexer.blockTokens() sets state.top = true on exit; the skip must too.
+            this.state.top = true;
+            return tokens;
+        }
+        return super.blockTokens(src, tokens, lastParagraphClipped);
+    }
+    /** Extensions (dl, table) tokenize cell/term text inline; raws never depend on it. */
+    inlineTokens(src, tokens = []) {
+        return tokens;
+    }
+}
+const blockTokensOf = (markdown, extensions) => new SplitLexer(getBlockOptions(extensions)).blockTokens(markdown, []);
+/**
+ * Is `markdown` an append to `cache.content`?
+ *
+ * The live region [offset, cache.content.length) is compared exactly — that is
+ * the only region whose segmentation can still change. The sealed prefix gets
+ * SEAL_PROBES sampled character comparisons instead of a full scan: a complete
+ * `markdown.startsWith(cache.content)` was 93% of parseBlocks' streaming cost
+ * (676 ms of 727 ms over a 100 KB / 5028-chunk stream) and is the single reason
+ * the append path was O(N) per chunk rather than O(tail).
+ *
+ * Note this compares source against source, not source against `raws`: marked's
+ * tokenizers normalize (e.g. the list tokenizer rewrites a trailing space as a
+ * newline), so raws are not always literal slices of the input.
+ */
+const appendable = (markdown, content, offset) => {
+    for (let i = offset; i < content.length; i++) {
+        if (markdown.charCodeAt(i) !== content.charCodeAt(i))
+            return false;
+    }
+    if (offset === 0)
+        return true;
+    const step = offset > SEAL_PROBES ? offset / SEAL_PROBES : 1;
+    for (let f = 0; f < offset; f += step) {
+        const i = f | 0;
+        if (markdown.charCodeAt(i) !== content.charCodeAt(i))
+            return false;
+    }
+    return markdown.charCodeAt(offset - 1) === content.charCodeAt(offset - 1);
+};
 export const parseBlocks = (markdown, extensions = [], cache) => {
-    if (cache &&
-        cache.content.length > 0 &&
-        markdown.length > cache.content.length &&
-        markdown.startsWith(cache.content)) {
+    if (cache && cache.content.length > 0 && markdown.length > cache.content.length) {
         // Append-only update: seal everything except the last SEAL_SLACK rendered
-        // blocks and re-lex only the tail. cache.raws concatenates exactly to
-        // cache.content (verified by length below), so summed lengths are offsets.
+        // blocks and re-lex only the tail. offsets[] are prefix sums over raws, so
+        // the seal point costs one array read instead of a running sum.
         let cut = cache.raws.length;
         let liveBlocks = 0;
         while (cut > 0 && liveBlocks < SEAL_SLACK) {
@@ -128,45 +186,72 @@ export const parseBlocks = (markdown, extensions = [], cache) => {
             if (cache.keep[cut])
                 liveBlocks++;
         }
-        let offset = 0;
-        for (let i = 0; i < cut; i++)
-            offset += cache.raws[i].length;
-        const tailTokens = blockTokensOf(markdown.slice(offset), extensions);
-        let tailLength = 0;
-        for (const token of tailTokens)
-            tailLength += token.raw.length;
-        // Contiguity guard: if the lexer normalized the tail (so raws no longer
-        // reconstruct the input), the offsets cannot be trusted — full reparse.
-        if (offset + tailLength === markdown.length) {
-            cache.raws.length = cut;
-            cache.keep.length = cut;
-            for (const token of tailTokens) {
-                cache.raws.push(token.raw);
-                cache.keep.push(token.type !== 'space' && token.type !== 'footnote');
+        const offset = cache.offsets[cut];
+        if (appendable(markdown, cache.content, offset)) {
+            const tailTokens = blockTokensOf(markdown.slice(offset), extensions);
+            let tailLength = 0;
+            for (const token of tailTokens)
+                tailLength += token.raw.length;
+            // Contiguity guard: if the lexer normalized the tail (so raws no longer
+            // reconstruct the input), the offsets cannot be trusted — full reparse.
+            if (offset + tailLength === markdown.length) {
+                const keptAtCut = cache.keptBefore[cut];
+                cache.raws.length = cut;
+                cache.keep.length = cut;
+                cache.offsets.length = cut + 1;
+                cache.keptBefore.length = cut + 1;
+                cache.blocks.length = keptAtCut;
+                let pos = offset;
+                let kept = keptAtCut;
+                for (const token of tailTokens) {
+                    const raw = token.raw;
+                    const keep = token.type !== 'space' && token.type !== 'footnote';
+                    cache.raws.push(raw);
+                    cache.keep.push(keep);
+                    pos += raw.length;
+                    cache.offsets.push(pos);
+                    if (keep) {
+                        cache.blocks.push(raw);
+                        kept++;
+                    }
+                    cache.keptBefore.push(kept);
+                }
+                cache.content = markdown;
+                // Copy out: callers (Svelte `$derived`, the perf harness) diff block
+                // lists by identity, so handing back the persistent array would read as
+                // "nothing changed". slice() is a memcpy with no per-element callback.
+                return cache.blocks.slice();
             }
-            cache.content = markdown;
-            return cache.raws.filter((_, i) => cache.keep[i]);
         }
     }
     // Full parse (first call, non-append update, or contiguity fallback).
     const tokens = blockTokensOf(markdown, extensions);
-    const blocks = [];
     if (cache) {
-        cache.raws = [];
-        cache.keep = [];
-        let total = 0;
+        cache.raws.length = 0;
+        cache.keep.length = 0;
+        cache.offsets.length = 1;
+        cache.keptBefore.length = 1;
+        cache.blocks.length = 0;
+        let pos = 0;
+        let kept = 0;
         for (const token of tokens) {
+            const raw = token.raw;
             const keep = token.type !== 'space' && token.type !== 'footnote';
-            cache.raws.push(token.raw);
+            cache.raws.push(raw);
             cache.keep.push(keep);
-            total += token.raw.length;
-            if (keep)
-                blocks.push(token.raw);
+            pos += raw.length;
+            cache.offsets.push(pos);
+            if (keep) {
+                cache.blocks.push(raw);
+                kept++;
+            }
+            cache.keptBefore.push(kept);
         }
         // Only trust the cache for future appends if raws reconstruct the input.
-        cache.content = total === markdown.length ? markdown : '';
-        return blocks;
+        cache.content = pos === markdown.length ? markdown : '';
+        return cache.blocks.slice();
     }
+    const blocks = [];
     for (const token of tokens) {
         if (token.type !== 'space' && token.type !== 'footnote')
             blocks.push(token.raw);

package/dist/marked/marked-br.js

@@ -2,6 +2,12 @@ export const markedBr = {
     name: 'br',
     level: 'inline',
     tokenizer(src) {
+        // marked dispatches every inline extension at every scan position; `start`
+        // only clips the text rule, it does not gate the tokenizer. This rule is
+        // anchored on a single literal character, so one charCodeAt rejects the
+        // ~3.6k non-matching dispatches per 100 KB before the regex engine runs.
+        if (src.charCodeAt(0) !== 60 /* < */)
+            return undefined;
         // Match HTML <br> tags (with or without closing slash, case insensitive)
         const match = src.match(/^<br\s*\/?>/i);
         if (match) {

package/dist/marked/marked-citations.js

@@ -6,6 +6,12 @@ export const markedCitations = {
         return i === -1 ? -1 : i;
     },
     tokenizer(src) {
+        // marked dispatches every inline extension at every scan position; `start`
+        // only clips the text rule, it does not gate the tokenizer. This rule is
+        // anchored on a single literal character, so one charCodeAt rejects the
+        // ~3.6k non-matching dispatches per 100 KB before the regex engine runs.
+        if (src.charCodeAt(0) !== 91 /* [ */)
+            return;
         // Match inline citations like [1], [ref], [1] [2], [ref] [ref2], etc.
         // Requires non-empty bracket contents and spaces between adjacent citation brackets
         const match = src.match(/^\[[^\][]+\](?:\s+\[[^\][]+\])*/);

package/dist/marked/marked-footnotes.js

@@ -34,9 +34,14 @@ export function markedFootnote() {
             name: 'footnote',
             level: 'block',
             tokenizer(src) {
-                const maps = ensureMaps(this);
+                // Match FIRST. ensureMaps() calls Svelte's getContext(), which throws
+                // (and is caught) whenever tokenizing happens outside component init —
+                // 3.3 us per call. Running it before the regex charged that to every
+                // block token in the document: 8.3 ms of the 13.0 ms it took to split a
+                // 100 KB document into blocks.
                 const match = footnoteRegex.exec(src);
                 if (match) {
+                    const maps = ensureMaps(this);
                     const [raw, label, text = ''] = match;
                     let content = text.split('\n').reduce((acc, curr) => {
                         return acc + '\n' + curr.replace(/^[ \t]+/, '');
@@ -66,9 +71,14 @@ export function markedFootnote() {
             name: 'footnoteRef',
             level: 'inline',
             tokenizer(src) {
-                const maps = ensureMaps(this);
+                // Same ordering rule as the block tokenizer above: this one is an inline
+                // extension, so it is dispatched at every inline scan position — so it also
+                // gets a single-character guard before the regex.
+                if (src.charCodeAt(0) !== 91 /* [ */)
+                    return;
                 const match = footnoteRefRegex.exec(src);
                 if (match) {
+                    const maps = ensureMaps(this);
                     const [raw, label] = match;
                     const footnote = maps.footnotes.get(label);
                     const token = {

package/dist/marked/marked-math.js

@@ -20,6 +20,8 @@ export const markedMath = [
         name: 'math',
         level: 'block',
         tokenizer(src) {
+            if (src.charCodeAt(0) !== 36 /* $ */)
+                return;
             const match = src.match(blockRule);
             if (match) {
                 // match[2] is multiline format, match[3] is single-line format
@@ -67,6 +69,9 @@ export const markedMath = [
             }
         },
         tokenizer(src) {
+            // inlineRule is anchored on `$`; skip the regex otherwise (see marked-br.ts).
+            if (src.charCodeAt(0) !== 36 /* $ */)
+                return;
             const match = src.match(inlineRule);
             if (match) {
                 const content = match[2];

package/dist/marked/marked-subsup.js

@@ -8,6 +8,12 @@ export const markedSub = {
         return i === -1 ? undefined : i;
     },
     tokenizer(src) {
+        // marked dispatches every inline extension at every scan position; `start`
+        // only clips the text rule, it does not gate the tokenizer. This rule is
+        // anchored on a single literal character, so one charCodeAt rejects the
+        // ~3.6k non-matching dispatches per 100 KB before the regex engine runs.
+        if (src.charCodeAt(0) !== 126 /* ~ */)
+            return;
         const match = src.match(subRule);
         if (match) {
             return {
@@ -27,6 +33,8 @@ export const markedSup = {
         return i === -1 ? undefined : i;
     },
     tokenizer(src) {
+        if (src.charCodeAt(0) !== 94 /* ^ */)
+            return;
         const match = src.match(supRule);
         if (match) {
             return {

package/dist/marked/marked-table.js

@@ -376,6 +376,19 @@ function processRows(headerRows, bodyRows, alignment, colCount, lexer, maxColspa
     }
     return tokens;
 }
+// Hoisted: this was `new RegExp(...)` inside the tokenizer, so a ~1 KB pattern was
+// concatenated and a RegExp constructed on every block-tokenizer dispatch — 3.3 ms
+// of the 13.0 ms it took to split a 100 KB document into blocks.
+const TABLE_WITH_ALIGN = new RegExp('^' +
+    '([^\\n ].*\\|.*\\n(?: *[^\\s].*\\n)*?)' + // Header
+    ' {0,3}(?:\\| *)?(:?-+:? *(?:\\| *:?-+:? *)*)(?:\\| *)?' + // Header Align
+    '(?:\\n((?:(?! *\\n| {0,3}((?:- *){3,}|(?:_ *){3,}|(?:\\* *){3,})' + // Body Cells
+    '(?:\\n+|$)| {0,3}#{1,6} | {0,3}>| {4}[^\\n]| {0,3}(?:`{3,}' +
+    '(?=[^`\\n]*\\n)|~{3,})[^\\n]*\\n| {0,3}(?:[*+-]|1[.)]) |' +
+    '<\\/?(?:address|article|aside|base|basefont|blockquote|body|' +
+    'caption|center|col|colgroup|dd|details|dialog|dir|div|dl|dt|fieldset|figcaption|figure|footer|form|frame|frameset|h[1-6]|head|header|hr|html|iframe|legend|li|link|main|menu|menuitem|meta|nav|noframes|ol|optgroup|option|p|param|section|source|summary|table|tbody|td|tfoot|th|thead|title|tr|track|ul)(?: +|\\n|\\/?>)|<(?:script|pre|style|textarea|!--)).*(?:\\n|$))*)\\n*|$)');
+// Tables written without a header alignment row (`|a|b|` rows only).
+const TABLE_NO_ALIGN = /^(\|.*\|(?:\n\|.*\|)*)/;
 const { detectFooter, maxColspan } = DEFAULT_OPTIONS;
 // Adds support for extended tables in marked with row spanning, column spanning,
 // multi-row headers, and column alignment
@@ -395,21 +408,11 @@ export const markedTable = {
     },
     tokenizer(src) {
         // Try to match table with header and alignment first
-        let regex = new RegExp('^' +
-            '([^\\n ].*\\|.*\\n(?: *[^\\s].*\\n)*?)' + // Header
-            ' {0,3}(?:\\| *)?(:?-+:? *(?:\\| *:?-+:? *)*)(?:\\| *)?' + // Header Align
-            '(?:\\n((?:(?! *\\n| {0,3}((?:- *){3,}|(?:_ *){3,}|(?:\\* *){3,})' + // Body Cells
-            '(?:\\n+|$)| {0,3}#{1,6} | {0,3}>| {4}[^\\n]| {0,3}(?:`{3,}' +
-            '(?=[^`\\n]*\\n)|~{3,})[^\\n]*\\n| {0,3}(?:[*+-]|1[.)]) |' +
-            '<\\/?(?:address|article|aside|base|basefont|blockquote|body|' +
-            'caption|center|col|colgroup|dd|details|dialog|dir|div|dl|dt|fieldset|figcaption|figure|footer|form|frame|frameset|h[1-6]|head|header|hr|html|iframe|legend|li|link|main|menu|menuitem|meta|nav|noframes|ol|optgroup|option|p|param|section|source|summary|table|tbody|td|tfoot|th|thead|title|tr|track|ul)(?: +|\\n|\\/?>)|<(?:script|pre|style|textarea|!--)).*(?:\\n|$))*)\\n*|$)');
-        let cap = regex.exec(src);
+        let cap = TABLE_WITH_ALIGN.exec(src);
         let hasHeaderAlignment = true;
         // If no match with header alignment, try table without header alignment
         if (!cap) {
-            // Simple regex for tables without header alignment
-            regex = /^(\|.*\|(?:\n\|.*\|)*)/;
-            cap = regex.exec(src);
+            cap = TABLE_NO_ALIGN.exec(src);
             hasHeaderAlignment = false;
         }
         if (!cap)

package/dist/theme.d.ts

@@ -42,7 +42,6 @@ export declare const theme: {
         header: string;
         buttons: string;
         language: string;
-        skeleton: string;
         pre: string;
         line: string;
     };
@@ -195,7 +194,6 @@ export declare const shadcnTheme: {
         header: string;
         buttons: string;
         language: string;
-        skeleton: string;
         pre: string;
         line: string;
     };
@@ -353,7 +351,6 @@ export declare const mergeTheme: (customTheme?: DeepPartialTheme, baseTheme?: "t
         header: string;
         buttons: string;
         language: string;
-        skeleton: string;
         pre: string;
         line: string;
     };

package/dist/theme.js

@@ -43,7 +43,6 @@ export const theme = {
         header: 'flex items-center justify-between bg-gray-100/80 p-2	 text-gray-600 text-xs',
         buttons: 'flex items-center gap-2',
         language: 'ml-1 font-mono lowercase',
-        skeleton: 'block rounded-md font-mono text-transparent bg-gray-200 scale-y-90 animate-pulse whitespace-nowrap',
         pre: 'overflow-x-auto font-mono p-0 bg-gray-100/40',
         line: 'block'
     },
@@ -196,7 +195,6 @@ export const shadcnTheme = {
         header: 'flex items-center justify-between bg-muted/80 px-2 py-1 text-muted-foreground text-xs',
         buttons: 'flex items-center gap-2',
         language: 'ml-1 font-mono lowercase',
-        skeleton: 'block rounded-md font-mono text-transparent bg-border/80 scale-y-90 w-fit animate-pulse whitespace-nowrap',
         pre: 'overflow-x-auto font-mono p-0 bg-muted/40',
         line: 'block '
     },

package/dist/utils/highlightThemes.d.ts

@@ -0,0 +1,4 @@
+import type { HighlightTheme } from '@tanstack/highlight/theme';
+export declare const DEFAULT_THEMES: Record<string, HighlightTheme>;
+/** Resolves the active theme: explicit key, else the first user theme, else the dark-mode default. */
+export declare function resolveHighlightTheme(key: string | undefined, extra: Record<string, HighlightTheme> | undefined, isDark: boolean): HighlightTheme;

package/dist/utils/highlightThemes.js

@@ -0,0 +1,14 @@
+import githubDark from '@tanstack/highlight/themes/github-dark';
+import githubLight from '@tanstack/highlight/themes/github-light';
+// ponytail: themes live apart from the highlighter so <Streamdown> never pulls in the
+// language scanners — those are only reachable from the opt-in <Code> component.
+export const DEFAULT_THEMES = {
+    'github-dark': githubDark,
+    'github-light': githubLight
+};
+/** Resolves the active theme: explicit key, else the first user theme, else the dark-mode default. */
+export function resolveHighlightTheme(key, extra, isDark) {
+    const themes = { ...DEFAULT_THEMES, ...extra };
+    const fallback = DEFAULT_THEMES[isDark ? 'github-dark' : 'github-light'];
+    return themes[key ?? (extra ? Object.keys(extra)[0] : fallback.name)] ?? fallback;
+}

package/dist/utils/hightlighter.svelte.d.ts

@@ -1,30 +1,7 @@
-import type { ThemedToken, ThemeRegistration } from 'shiki';
-import type { HighlighterCore } from 'shiki/core';
-import { SvelteMap } from 'svelte/reactivity';
-import { type LanguageInfo } from './bundledLanguages.js';
-export type Highlighter = HighlighterCore;
-declare class HighlighterManager {
-    loadedLanguages: SvelteMap<string, boolean | Promise<void>>;
-    highlighter: any;
-    customLanguages: Set<string>;
-    languageLoaders: Map<string, () => Promise<any>>;
-    additionalThemes: Record<string, ThemeRegistration>;
-    constructor(languages: LanguageInfo[], additionalThemes?: Record<string, ThemeRegistration>, additionalLanguages?: LanguageInfo[]);
-    private loadHighlighter;
-    private isThemeAvailable;
-    private loadLanguage;
-    private isLanguageSupported;
-    isReady(theme: string, language: string | undefined): boolean;
-    /**
-     * Ensures the highlighter is ready for the given theme and language.
-     */
-    load(theme: string, language: string | undefined): Promise<void>;
-    /**
-     * Highlights code synchronously. Must call isReady() first.
-     * Returns plaintext tokens for unsupported languages.
-     */
-    highlightCode(code: string, language: string | undefined, theme: string): ThemedToken[][];
-    static create(languages?: LanguageInfo[], additionalThemes?: Record<string, ThemeRegistration>, additionalLanguages?: LanguageInfo[]): HighlighterManager;
-}
-export { HighlighterManager };
+import { type HighlightToken, type LanguageDefinition } from '@tanstack/highlight';
+/**
+ * Tokenizes code into lines of tokens. Synchronous and isomorphic: works during SSR.
+ * Unknown languages fall back to plaintext.
+ */
+export declare function highlightLines(code: string, lang: string | undefined, extra?: LanguageDefinition[]): HighlightToken[][];
 export declare const languageExtensionMap: Record<string, string>;