svelte-streamdown 4.0.0 → 4.1.0

package/dist/marked/index.js

@@ -1,4 +1,4 @@
-import { Lexer } from 'marked';
+import { Lexer, Tokenizer } from 'marked';
 import { markedAlert } from './marked-alert.js';
 import { markedFootnote } from './marked-footnotes.js';
 import { markedMath } from './marked-math.js';
@@ -39,9 +39,42 @@ const DEFAULT_BLOCK_EXTENSIONS = [
     markedAlign,
     markedMdx
 ];
+class StreamdownTokenizer extends Tokenizer {
+    /**
+     * marked keeps the whole fence info string in `lang`, so ```ts title="x" {1}
+     * highlighted as plaintext, labelled the header with the entire string, downloaded
+     * as `file.txt` and — worst — missed `token.lang === 'mermaid'` (upstream d4ec6c0).
+     * Splitting here rather than walking lex()'s output costs one search per code
+     * token instead of a per-block tree walk, and it also reaches fences nested in
+     * lists and blockquotes, which a pass over the top-level tokens would not.
+     */
+    fences(src) {
+        const token = super.fences(src);
+        if (token?.lang) {
+            const end = token.lang.search(/\s/);
+            if (end !== -1) {
+                token.meta = token.lang.slice(end + 1).trim() || undefined;
+                token.lang = token.lang.slice(0, end);
+            }
+        }
+        return token;
+    }
+    /**
+     * `~x~` is a subscript here, not GFM strikethrough (marked-subsup.ts). The old
+     * subscript rule shadowed single-tilde del by matching anything; now that it
+     * rejects whitespace, del would inherit exactly the sentences 716a5f0 is about
+     * (`20~25°C and 30~35°C` → del('~25°C and 30~')). Only `~~` opens a del.
+     */
+    del(src, maskedSrc, prevChar) {
+        if (src.charCodeAt(0) !== 126 /* ~ */ || src.charCodeAt(1) !== 126)
+            return;
+        return super.del(src, maskedSrc, prevChar);
+    }
+}
 const parseExtensions = (...extensions) => {
     const options = {
         gfm: true,
+        tokenizer: new StreamdownTokenizer(),
         extensions: {
             block: [],
             inline: [],
@@ -71,9 +104,12 @@ const parseExtensions = (...extensions) => {
     });
     return options;
 };
-// Options objects are stateless and reusable across Lexer instances; cache them
-// per user-extension array (props are referentially stable across chunks) so the
-// hot path skips rebuilding ~20 tokenizer registrations on every streamed chunk.
+// Options objects are reusable across Lexer instances; cache them per
+// user-extension array (props are referentially stable across chunks) so the hot
+// path skips rebuilding ~20 tokenizer registrations on every streamed chunk. The
+// cached object carries one shared Tokenizer whose `lexer` back-pointer marked
+// re-stamps per Lexer construction — safe only because lexing is synchronous and
+// never re-entrant across documents.
 const DEFAULT_LEX_OPTIONS = parseExtensions(...DEFAULT_LEX_EXTENSIONS);
 const DEFAULT_BLOCK_OPTIONS = parseExtensions(...DEFAULT_BLOCK_EXTENSIONS);
 const lexOptionsCache = new WeakMap();
@@ -115,9 +151,10 @@ export const createParseBlocksCache = () => ({
 // 2 covers constructs that merge backward as they stream in — e.g. a paragraph
 // line becoming a table once its delimiter row arrives, or a setext heading.
 const SEAL_SLACK = 2;
-// Spot checks over the sealed prefix per append (see `appendable`). The sealed
-// region was compared byte-for-byte while it was live, so this only has to catch
-// a caller that swapped in a different, longer document.
+// Stride spot checks over the sealed prefix per append (see `appendable`), on
+// top of the per-block-start probes. The sealed region was compared
+// byte-for-byte while it was live, so this only has to catch a caller that
+// swapped in a different, longer document.
 const SEAL_PROBES = 16;
 /**
  * A Lexer used only to slice a document into top-level block raws.
@@ -149,23 +186,37 @@ const blockTokensOf = (markdown, extensions) => new SplitLexer(getBlockOptions(e
  * Is `markdown` an append to `cache.content`?
  *
  * The live region [offset, cache.content.length) is compared exactly — that is
- * the only region whose segmentation can still change. The sealed prefix gets
- * SEAL_PROBES sampled character comparisons instead of a full scan: a complete
- * `markdown.startsWith(cache.content)` was 93% of parseBlocks' streaming cost
- * (676 ms of 727 ms over a 100 KB / 5028-chunk stream) and is the single reason
- * the append path was O(N) per chunk rather than O(tail).
+ * the only region whose segmentation can still change. The sealed prefix is only
+ * sampled, never scanned: a complete `markdown.startsWith(cache.content)` was
+ * 93% of parseBlocks' streaming cost (676 ms of 727 ms over a 100 KB /
+ * 5028-chunk stream) and is the single reason the append path was O(N) per chunk
+ * rather than O(tail).
+ *
+ * Two families of samples, both O(sealed blocks) and not O(sealed characters):
+ * the first character of every sealed block (`cache.offsets` already holds those
+ * positions), which catches any edit that moves a block boundary or rewrites a
+ * block's first character — the shape a real out-of-band edit takes; plus
+ * SEAL_PROBES evenly spaced characters, which catch a wholesale document swap.
+ * A same-length edit in the middle of a sealed block can still slip through; see
+ * the streaming contract in the README.
  *
  * Note this compares source against source, not source against `raws`: marked's
  * tokenizers normalize (e.g. the list tokenizer rewrites a trailing space as a
  * newline), so raws are not always literal slices of the input.
  */
-const appendable = (markdown, content, offset) => {
+const appendable = (markdown, cache, cut, offset) => {
+    const content = cache.content;
     for (let i = offset; i < content.length; i++) {
         if (markdown.charCodeAt(i) !== content.charCodeAt(i))
             return false;
     }
     if (offset === 0)
         return true;
+    for (let b = 0; b < cut; b++) {
+        const i = cache.offsets[b];
+        if (markdown.charCodeAt(i) !== content.charCodeAt(i))
+            return false;
+    }
     const step = offset > SEAL_PROBES ? offset / SEAL_PROBES : 1;
     for (let f = 0; f < offset; f += step) {
         const i = f | 0;
@@ -187,7 +238,7 @@ export const parseBlocks = (markdown, extensions = [], cache) => {
                 liveBlocks++;
         }
         const offset = cache.offsets[cut];
-        if (appendable(markdown, cache.content, offset)) {
+        if (appendable(markdown, cache, cut, offset)) {
             const tailTokens = blockTokensOf(markdown.slice(offset), extensions);
             let tailLength = 0;
             for (const token of tailTokens)

package/dist/marked/marked-footnotes.js

@@ -1,8 +1,12 @@
 import {} from './index.js';
 import { StreamdownContext } from '../context.svelte.js';
 import { getContext } from 'svelte';
-const footnoteRegex = /^\[\^([^\]\n]+)\]:(?:[ \t]+|\n|$)([^\n]*(?:\n(?:[ \t]+[^\n]*)?)*)/;
-const footnoteRefRegex = /^\[\^([^\]\n]+)\]/;
+// Footnote identifiers are word characters, `-` and `:` only. `[^\]\n]+` turned
+// every regex character class written in prose (`[^\s]`, `[^,]`) into an empty
+// footnote marker (upstream 9f72224). `:` is kept for the completer's
+// `[^streamdown:footnote]` sentinel, which FootnoteRef.svelte renders as nothing.
+const footnoteRegex = /^\[\^([\w:-]{1,200})\]:(?:[ \t]+|\n|$)([^\n]*(?:\n(?:[ \t]+[^\n]*)?)*)/;
+const footnoteRefRegex = /^\[\^([\w:-]{1,200})\]/;
 const footNoteLastLineRegex = /^[ \t]*?[>\-*][ ]|[`]{3,}$|^[ \t]*?[|].+[|]$/;
 const safeGetContext = () => {
     try {

package/dist/marked/marked-math.js

@@ -4,6 +4,13 @@ const blockRule = /^(\$\$)(?:\n((?:\\[\s\S]|[^\\])+?)\n\1(?:\n|$)|([^$\n]+?)\1(?
 // Inline math: handles both single ($) and double ($$) dollar delimiters
 // Avoids matching currency by checking context and requiring proper content
 const inlineRule = /^(\${1,2})(?!\$)((?:[^$\n]|\\\$)*?)\1(?!\d)/;
+// LaTeX delimiters. LLMs emit these as often as dollars, so they produce the
+// same `math` token: `\(`/`\)` inline, `\[`/`\]` display. Only the block form
+// may span lines; an escaped backslash (`\\(`) is not a delimiter, which the
+// charCode dispatch below rejects before any regex runs.
+const inlineParenRule = /^\\\(([^\n]*?)\\\)/;
+const inlineBracketRule = /^\\\[([^\n]*?)\\\]/;
+const blockBracketRule = /^\\\[([\s\S]+?)\\\](?:\n|$)/;
 // Enhanced currency detection patterns
 const currencyPatterns = {
     // Simple price patterns: $123, $123.45, $1,234.56
@@ -20,6 +27,20 @@ export const markedMath = [
         name: 'math',
         level: 'block',
         tokenizer(src) {
+            if (src.charCodeAt(0) === 92 /* \ */) {
+                if (src.charCodeAt(1) !== 91 /* [ */)
+                    return;
+                const latex = src.match(blockBracketRule);
+                if (!latex)
+                    return;
+                return {
+                    type: 'math',
+                    isInline: false,
+                    displayMode: true,
+                    raw: latex[0],
+                    text: latex[1].trim()
+                };
+            }
             if (src.charCodeAt(0) !== 36 /* $ */)
                 return;
             const match = src.match(blockRule);
@@ -69,7 +90,25 @@ export const markedMath = [
             }
         },
         tokenizer(src) {
-            // inlineRule is anchored on `$`; skip the regex otherwise (see marked-br.ts).
+            // The rules are anchored on their opening delimiter; skip the regex
+            // otherwise (see marked-br.ts) — this runs at every inline scan position.
+            if (src.charCodeAt(0) === 92 /* \ */) {
+                const open = src.charCodeAt(1);
+                // `\\(` is an escaped backslash followed by a paren, never math.
+                const displayMode = open === 91; /* [ */
+                if (!displayMode && open !== 40 /* ( */)
+                    return;
+                const latex = src.match(displayMode ? inlineBracketRule : inlineParenRule);
+                if (!latex)
+                    return;
+                return {
+                    type: 'math',
+                    isInline: true,
+                    displayMode,
+                    raw: latex[0],
+                    text: latex[1].trim()
+                };
+            }
             if (src.charCodeAt(0) !== 36 /* $ */)
                 return;
             const match = src.match(inlineRule);

package/dist/marked/marked-subsup.js

@@ -1,5 +1,8 @@
-const subRule = /^~([^~\s](?:[^~]*[^~\s])?)~/; // ~text~
-const supRule = /^\^([^\^\s](?:[^\^]*[^\^\s])?)\^/; // ^text^
+// A sub/superscript is a single run, never a phrase: no whitespace anywhere in
+// it. The old rules only forbade whitespace at the edges, so `20~25°C and 30~35°C`
+// subscripted half the sentence (upstream 716a5f0).
+const subRule = /^~([^~\s]+)~/; // ~text~
+const supRule = /^\^([^\^\s]+)\^/; // ^text^
 export const markedSub = {
     name: 'sub',
     level: 'inline',
@@ -7,13 +10,23 @@ export const markedSub = {
         const i = src.indexOf('~');
         return i === -1 ? undefined : i;
     },
-    tokenizer(src) {
+    tokenizer(src, tokens) {
         // marked dispatches every inline extension at every scan position; `start`
         // only clips the text rule, it does not gate the tokenizer. This rule is
         // anchored on a single literal character, so one charCodeAt rejects the
         // ~3.6k non-matching dispatches per 100 KB before the regex engine runs.
         if (src.charCodeAt(0) !== 126 /* ~ */)
             return;
+        // A digit right before the opening `~` means a numeric range (`20~25°C`),
+        // not a subscript base — chemistry and indices always have a letter or a
+        // closing bracket there (`H~2~O`, `x~i+1~`). `src` starts at the marker, so
+        // the preceding character is the last one of the previous inline token.
+        const prevRaw = tokens[tokens.length - 1]?.raw;
+        if (prevRaw) {
+            const code = prevRaw.charCodeAt(prevRaw.length - 1);
+            if (code >= 48 && code <= 57)
+                return;
+        }
         const match = src.match(subRule);
         if (match) {
             return {

package/dist/utils/fence.d.ts

@@ -0,0 +1,16 @@
+/** An open code fence: the character it was opened with, and its run length. */
+export type OpenFence = {
+    char: string;
+    length: number;
+} | null;
+/**
+ * Fence tracking, one line at a time: returns the fence state after `line`.
+ * The completer's block scan and `hasUnclosedFence` both walk with this, so the
+ * "am I inside a fence" answer can never drift between them.
+ */
+export declare const trackFence: (line: string, open: OpenFence) => OpenFence;
+/**
+ * True when `raw` ends inside a fenced block — i.e. the fence is still being
+ * streamed. Only the last block of a live document can be in this state.
+ */
+export declare const hasUnclosedFence: (raw: string) => boolean;

package/dist/utils/fence.js

@@ -0,0 +1,39 @@
+// Up to 3 spaces of indentation per CommonMark; blockquote markers are stripped
+// too because a fence can be quoted ("> ```") inside a blockquote or an alert,
+// and so is a leading list marker, because "- ```js" opens a fence in the item.
+const FENCE_LINE = /^[ \t]{0,3}(?:>[ \t]*)*(?:(?:[-*+]|\d{1,9}[.)])[ \t]+)?(`{3,}|~{3,})(.*)$/;
+/**
+ * Fence tracking, one line at a time: returns the fence state after `line`.
+ * The completer's block scan and `hasUnclosedFence` both walk with this, so the
+ * "am I inside a fence" answer can never drift between them.
+ */
+export const trackFence = (line, open) => {
+    const match = FENCE_LINE.exec(line);
+    if (!match) {
+        return open;
+    }
+    const run = match[1];
+    const info = match[2];
+    if (open) {
+        // A closer uses the opener's character, is at least as long, and carries no
+        // info string — so a shorter fence line inside a longer block is content.
+        return run[0] === open.char && run.length >= open.length && info.trim() === '' ? null : open;
+    }
+    // A backtick fence's info string cannot contain a backtick (marked's own rule),
+    // so such a line opens nothing.
+    if (run[0] === '`' && info.includes('`')) {
+        return open;
+    }
+    return { char: run[0], length: run.length };
+};
+/**
+ * True when `raw` ends inside a fenced block — i.e. the fence is still being
+ * streamed. Only the last block of a live document can be in this state.
+ */
+export const hasUnclosedFence = (raw) => {
+    let open = null;
+    for (const line of raw.split('\n')) {
+        open = trackFence(line, open);
+    }
+    return open !== null;
+};

package/dist/utils/parse-incomplete-markdown.d.ts

@@ -1,3 +1,4 @@
+import { type OpenFence } from './fence.js';
 export interface Plugin {
     name: string;
     pattern?: RegExp;
@@ -25,13 +26,16 @@ interface ParseState {
     currentLine: number;
     context: 'normal' | 'list' | 'blockquote' | 'descriptionList';
     blockingContexts: Set<'code' | 'math' | 'center' | 'right'>;
+    /** Delimiter that closes the open math block: '$$' or '\]'. */
+    mathCloser?: '$$' | '\\]';
+    /** The fence still open at the end of the input, if any — what closes it. */
+    openFence?: OpenFence;
     lineContexts?: Array<{
         code: boolean;
         math: boolean;
         center: boolean;
         right: boolean;
     }>;
-    fenceInfo?: string;
     mdxUnclosedTags?: Array<{
         tagName: string;
         lineIndex: number;