svelte-streamdown 3.1.2 → 4.0.0

package/dist/utils/hightlighter.svelte.js

@@ -1,219 +1,34 @@
-import { SvelteMap } from 'svelte/reactivity';
-import { supportedLanguages, createLanguageSet, bundledLanguagesInfo } from './bundledLanguages.js';
-import { createHighlighterCore } from 'shiki/core';
-import { createJavaScriptRegexEngine } from 'shiki/engine/javascript';
-// Import default themes directly
-import githubDark from '@shikijs/themes/github-dark';
-import githubLight from '@shikijs/themes/github-light';
-// Default themes that are always loaded
-const DEFAULT_THEMES = {
-    'github-dark': githubDark,
-    'github-light': githubLight
-};
-class HighlighterManager {
-    loadedLanguages = new SvelteMap();
-    highlighter = $state(null);
-    customLanguages;
-    languageLoaders;
-    additionalThemes;
-    constructor(languages, additionalThemes, additionalLanguages) {
-        // Store additional themes (will be loaded with highlighter)
-        this.additionalThemes = additionalThemes || {};
-        // Merge languages: default + additional
-        const allLanguages = additionalLanguages ? [...languages, ...additionalLanguages] : languages;
-        this.languageLoaders = new Map();
-        allLanguages.forEach((l) => {
-            this.languageLoaders.set(l.id, l.import);
-            if (l.aliases) {
-                l.aliases.forEach((alias) => this.languageLoaders.set(alias, l.import));
-            }
-        });
-        // Build custom languages set for validation
-        if (additionalLanguages) {
-            const additionalSet = createLanguageSet(additionalLanguages);
-            this.customLanguages = new Set([...supportedLanguages, ...additionalSet]);
-        }
-        else {
-            this.customLanguages = supportedLanguages;
-        }
-        if (typeof window !== 'undefined') {
-            Object.assign(window, {
-                STREAMDOWN_HIGHLIGHTER: this
-            });
-        }
-    }
-    async loadHighlighter() {
-        if (this.highlighter instanceof Promise) {
-            return this.highlighter;
-        }
-        else if (!this.highlighter) {
-            this.highlighter = new Promise(async (resolve, reject) => {
-                try {
-                    const engine = createJavaScriptRegexEngine({ forgiving: true });
-                    // Load default themes + any additional themes immediately
-                    const allThemes = [
-                        ...Object.values(DEFAULT_THEMES),
-                        ...Object.values(this.additionalThemes)
-                    ];
-                    const highlighter = await createHighlighterCore({
-                        themes: allThemes,
-                        langs: [],
-                        engine
-                    });
-                    this.highlighter = highlighter;
-                    resolve(highlighter);
-                }
-                catch (error) {
-                    reject(error);
-                }
-            });
-            return this.highlighter;
-        }
-        else {
-            return this.highlighter;
-        }
-    }
-    isThemeAvailable(theme) {
-        return theme in DEFAULT_THEMES || theme in this.additionalThemes;
+import { createHighlighter, defaultHighlighter, allLanguages } from '@tanstack/highlight';
+const highlighters = new WeakMap();
+const getHighlighter = (extra) => {
+    if (!extra?.length)
+        return defaultHighlighter;
+    let highlighter = highlighters.get(extra);
+    if (!highlighter) {
+        highlighter = createHighlighter({ languages: [...allLanguages, ...extra] });
+        highlighters.set(extra, highlighter);
     }
-    async loadLanguage(language, highlighter) {
-        // Skip loading if language is not supported
-        if (!this.isLanguageSupported(language)) {
-            return;
-        }
-        const languageLoader = this.loadedLanguages.get(language);
-        if (languageLoader instanceof Promise) {
-            await languageLoader;
-        }
-        else if (!languageLoader) {
-            const loader = this.languageLoaders.get(language);
-            if (!loader) {
-                // Language not available, mark as failed
-                this.loadedLanguages.set(language, false);
-                return;
-            }
-            const languageLoaderPromise = loader()
-                .then((langModule) => {
-                const langObj = langModule.default || langModule;
-                return highlighter.loadLanguage(langObj);
-            })
-                .then(() => {
-                this.loadedLanguages.set(language, true);
-            })
-                .catch((err) => {
-                this.loadedLanguages.set(language, false);
-                throw err;
-            });
-            this.loadedLanguages.set(language, languageLoaderPromise);
-            await languageLoaderPromise;
-        }
-    }
-    isLanguageSupported = (language) => {
-        return this.customLanguages.has(language);
-    };
-    isReady(theme, language) {
-        // Check if theme is available
-        if (!this.isThemeAvailable(theme)) {
-            return false;
-        }
-        // For unsupported languages, we don't need to load anything (will use plaintext)
-        if (!language || !this.isLanguageSupported(language)) {
-            return !!this.highlighter && !(this.highlighter instanceof Promise);
-        }
-        return (!!this.highlighter &&
-            !(this.highlighter instanceof Promise) &&
-            this.loadedLanguages.get(language) === true);
-    }
-    /**
-     * Ensures the highlighter is ready for the given theme and language.
-     */
-    async load(theme, language) {
-        // For unsupported languages, only need to load highlighter (themes are pre-loaded)
-        if (!language || !this.isLanguageSupported(language)) {
-            if (this.highlighter !== null && !(this.highlighter instanceof Promise)) {
-                return;
-            }
-            await this.loadHighlighter();
-            return;
-        }
-        const languageLoader = this.loadedLanguages.get(language);
-        if (this.highlighter !== null &&
-            !(this.highlighter instanceof Promise) &&
-            languageLoader === true) {
-            return;
-        }
-        const highlighter = await this.loadHighlighter();
-        await this.loadLanguage(language, highlighter);
-    }
-    /**
-     * Highlights code synchronously. Must call isReady() first.
-     * Returns plaintext tokens for unsupported languages.
-     */
-    highlightCode(code, language, theme) {
-        try {
-            const highlighter = this.highlighter;
-            if (!highlighter || highlighter instanceof Promise) {
-                // Return plaintext tokens when highlighter is not ready
-                return code.split('\n').map((line) => [
-                    {
-                        content: line,
-                        color: undefined,
-                        bgColor: undefined
-                    }
-                ]);
-            }
-            // For unsupported languages, return plaintext tokens
-            if (!language || !this.isLanguageSupported(language)) {
-                return code.split('\n').map((line) => [
-                    {
-                        content: line,
-                        color: undefined,
-                        bgColor: undefined
-                    }
-                ]);
-            }
-            const tokens = highlighter.codeToTokensBase(code, {
-                lang: language,
-                theme
-            });
-            return tokens;
-        }
-        catch (error) {
-            // Return plaintext tokens on error
-            return code.split('\n').map((line) => [
-                {
-                    content: line,
-                    color: undefined,
-                    bgColor: undefined
-                }
-            ]);
-        }
-    }
-    static create(languages = bundledLanguagesInfo, additionalThemes, additionalLanguages) {
-        if (typeof window !== 'undefined' && 'STREAMDOWN_HIGHLIGHTER' in window) {
-            const previousHighlighter = window.STREAMDOWN_HIGHLIGHTER;
-            // Merge additional themes
-            if (additionalThemes) {
-                Object.assign(previousHighlighter.additionalThemes, additionalThemes);
-            }
-            // Merge additional languages with existing set
-            if (additionalLanguages) {
-                additionalLanguages.forEach((lang) => {
-                    previousHighlighter.languageLoaders.set(lang.id, lang.import);
-                    if (lang.aliases) {
-                        lang.aliases.forEach((alias) => previousHighlighter.languageLoaders.set(alias, lang.import));
-                    }
-                });
-                const additionalSet = createLanguageSet(additionalLanguages);
-                additionalSet.forEach((lang) => previousHighlighter.customLanguages.add(lang));
-            }
-            return previousHighlighter;
+    return highlighter;
+};
+/**
+ * Tokenizes code into lines of tokens. Synchronous and isomorphic: works during SSR.
+ * Unknown languages fall back to plaintext.
+ */
+export function highlightLines(code, lang, extra) {
+    const tokens = getHighlighter(extra).tokenize(code, { lang }).tokens;
+    const lines = [[]];
+    for (const token of tokens) {
+        const parts = token.value.split('\n');
+        for (let i = 0; i < parts.length; i++) {
+            if (i > 0)
+                lines.push([]);
+            // ponytail: empty segments carry no text, so skip them instead of emitting empty spans
+            if (parts[i])
+                lines[lines.length - 1].push({ ...token, value: parts[i] });
         }
-        return new HighlighterManager(languages, additionalThemes, additionalLanguages);
     }
+    return lines;
 }
-// Export the class for those who want to create their own instances
-export { HighlighterManager };
 export const languageExtensionMap = {
     '1c': '1c',
     '1c-query': '1cq',

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "svelte-streamdown",
-	"version": "3.1.2",
+	"version": "4.0.0",
 	"packageManager": "pnpm@10.32.1",
 	"repository": {
 		"type": "git",
@@ -94,13 +94,11 @@
 	},
 	"dependencies": {
 		"@floating-ui/dom": "^1.7.4",
-		"@shikijs/langs": "^3.17.1",
-		"@shikijs/themes": "^3.17.1",
+		"@tanstack/highlight": "^0.1.0",
 		"clsx": "^2.1.1",
 		"katex": "^0.16.22",
-		"marked": "^16.2.1",
+		"marked": "18.0.12",
 		"mermaid": "^11.15.0",
-		"shiki": "^3.13.0",
 		"tailwind-merge": "^3.3.1"
 	}
 }

package/dist/utils/bundledLanguages.d.ts

@@ -1,8 +0,0 @@
-export type LanguageInfo = {
-    id: string;
-    aliases?: string[];
-    import: () => Promise<any>;
-};
-export declare const bundledLanguagesInfo: LanguageInfo[];
-export declare function createLanguageSet(languages: LanguageInfo[]): Set<string>;
-export declare const supportedLanguages: Set<string>;

package/dist/utils/bundledLanguages.js

@@ -1,143 +0,0 @@
-export const bundledLanguagesInfo = [
-    // Web essentials
-    {
-        id: 'javascript',
-        aliases: ['js'],
-        import: () => import('@shikijs/langs/javascript')
-    },
-    {
-        id: 'typescript',
-        aliases: ['ts'],
-        import: () => import('@shikijs/langs/typescript')
-    },
-    {
-        id: 'html',
-        import: () => import('@shikijs/langs/html')
-    },
-    {
-        id: 'css',
-        import: () => import('@shikijs/langs/css')
-    },
-    {
-        id: 'json',
-        import: () => import('@shikijs/langs/json')
-    },
-    {
-        id: 'jsx',
-        import: () => import('@shikijs/langs/jsx')
-    },
-    {
-        id: 'tsx',
-        import: () => import('@shikijs/langs/tsx')
-    },
-    {
-        id: 'markdown',
-        aliases: ['md'],
-        import: () => import('@shikijs/langs/markdown')
-    },
-    {
-        id: 'yaml',
-        aliases: ['yml'],
-        import: () => import('@shikijs/langs/yaml')
-    },
-    {
-        id: 'xml',
-        import: () => import('@shikijs/langs/xml')
-    },
-    // Backend languages
-    {
-        id: 'python',
-        aliases: ['py'],
-        import: () => import('@shikijs/langs/python')
-    },
-    {
-        id: 'java',
-        import: () => import('@shikijs/langs/java')
-    },
-    {
-        id: 'go',
-        import: () => import('@shikijs/langs/go')
-    },
-    {
-        id: 'rust',
-        aliases: ['rs'],
-        import: () => import('@shikijs/langs/rust')
-    },
-    {
-        id: 'ruby',
-        aliases: ['rb'],
-        import: () => import('@shikijs/langs/ruby')
-    },
-    {
-        id: 'php',
-        import: () => import('@shikijs/langs/php')
-    },
-    {
-        id: 'c',
-        import: () => import('@shikijs/langs/c')
-    },
-    {
-        id: 'cpp',
-        aliases: ['c++'],
-        import: () => import('@shikijs/langs/cpp')
-    },
-    {
-        id: 'csharp',
-        aliases: ['c#', 'cs'],
-        import: () => import('@shikijs/langs/csharp')
-    },
-    {
-        id: 'sql',
-        import: () => import('@shikijs/langs/sql')
-    },
-    {
-        id: 'swift',
-        import: () => import('@shikijs/langs/swift')
-    },
-    {
-        id: 'kotlin',
-        aliases: ['kt', 'kts'],
-        import: () => import('@shikijs/langs/kotlin')
-    },
-    // Config/Shell
-    {
-        id: 'shellscript',
-        aliases: ['bash', 'sh', 'shell', 'zsh'],
-        import: () => import('@shikijs/langs/shellscript')
-    },
-    {
-        id: 'docker',
-        aliases: ['dockerfile'],
-        import: () => import('@shikijs/langs/docker')
-    },
-    {
-        id: 'toml',
-        import: () => import('@shikijs/langs/toml')
-    },
-    {
-        id: 'graphql',
-        aliases: ['gql'],
-        import: () => import('@shikijs/langs/graphql')
-    },
-    {
-        id: 'svelte',
-        import: () => import('@shikijs/langs/svelte')
-    },
-    {
-        id: 'vue',
-        import: () => import('@shikijs/langs/vue')
-    }
-];
-// Helper function to create a Set of all language IDs and aliases from language info
-export function createLanguageSet(languages) {
-    const set = new Set();
-    languages.forEach((lang) => {
-        set.add(lang.id);
-        if (lang.aliases) {
-            lang.aliases.forEach((alias) => set.add(alias));
-        }
-    });
-    return set;
-}
-// Create a Set of all supported language IDs and aliases for fast lookup (default set)
-export const supportedLanguages = createLanguageSet(bundledLanguagesInfo);