vocs 2.0.6 → 2.0.8

package/src/internal/mdx.ts

@@ -41,6 +41,7 @@ import { remarkVocsScope } from './remark-vocs-scope.js'
 import { remarkSandbox } from './sandbox.js'
 import * as ShikiTransformers from './shiki-transformers.js'
 import * as Snippets from './snippets.js'
+import * as InlineCache from './twoslash/inline-cache.js'
 import type { ExactPartial, UnionOmit } from './types.js'
 
 export { default as remarkFrontmatter } from 'remark-frontmatter'
@@ -121,6 +122,51 @@ export declare namespace remarkMermaid {
   type ReturnType = (tree: MdAst.Root) => void
 }
 
+/**
+ * Remark plugin that injects an ephemeral source-map comment into the body of
+ * each `twoslash` code block. The comment carries the block's source position
+ * (`{ path, from, to }`) so the inline types cache knows where to write the
+ * `// @twoslash-cache: ...` comment back to.
+ *
+ * The injected comment is stripped again by the `extractInlineCacheSourceMap`
+ * shiki transformer before twoslash/shiki run, so it never reaches the output
+ * or the on-disk file.
+ *
+ * Only top-level (non-indented) fences are supported, which covers the
+ * overwhelming majority of twoslash blocks.
+ */
+export function remarkInlineCache(): remarkInlineCache.ReturnType {
+  return (tree: MdAst.Root, file: VFile) => {
+    const raw = typeof file.value === 'string' ? file.value : file.value?.toString()
+    const filePath = file.path
+    if (!raw || !filePath) return
+
+    UnistUtil.visit(tree, 'code', (node) => {
+      if (!node.meta?.includes('twoslash')) return
+
+      const start = node.position?.start
+      const end = node.position?.end
+      if (start?.offset == null || end?.offset == null) return
+      // Only top-level fences (not nested in lists/blockquotes) are supported.
+      if (start.column !== 1) return
+
+      const newlineIndex = raw.indexOf('\n', start.offset)
+      if (newlineIndex === -1) return
+      const bodyStart = newlineIndex + 1
+
+      node.value = InlineCache.injectSourceMapComment(node.value, {
+        path: filePath,
+        from: bodyStart,
+        to: end.offset,
+      })
+    })
+  }
+}
+
+export declare namespace remarkInlineCache {
+  type ReturnType = (tree: MdAst.Root, file: VFile) => void
+}
+
 const extensions = ['.js', '.ts', '.tsx', '.jsx', '.mjs', '.cjs', '.md', '.mdx']
 const logger = createLogger(undefined, { allowClearScreen: false, prefix: '[vocs]' })
 
@@ -247,6 +293,11 @@ export function getCompileOptions(
           remarkSubheading,
           remarkVocsScope,
           ...(markdown?.remarkPlugins ?? []),
+          // Runs after snippet/include processing so the injected source-map
+          // comment doesn't interfere with snippet detection.
+          ...(twoslash && typeof twoslash === 'object' && InlineCache.enabled(twoslash.inlineCache)
+            ? [remarkInlineCache]
+            : []),
           remarkRestoreUnknownTextDirectives,
         ],
         recmaPlugins: [recmaMdxLayout(config), ...(markdown?.recmaPlugins ?? [])],
@@ -487,6 +538,11 @@ export function rehypeShiki(
       langs: mergedLangs,
       // TODO: infer `langs` for faster cold start.
       transformers: [
+        // Must run before the twoslash transformer so `this.meta.sourceMap` is
+        // available to the inline types cache.
+        twoslash && typeof twoslash === 'object' && InlineCache.enabled(twoslash.inlineCache)
+          ? ShikiTransformers.extractInlineCacheSourceMap()
+          : undefined,
         rootDir && srcDir ? ShikiTransformers.notationInclude({ srcDir, rootDir }) : undefined,
         twoslash
           ? ShikiTransformers.twoslash({
@@ -1062,7 +1118,10 @@ function getVirtualFiles(codeNodes: MdAst.Code[]): Map<string, string> {
   for (const node of codeNodes) {
     const fileName = getCodeFileName(node)
     if (!fileName) continue
-    virtualFiles.set(fileName, node.value)
+    // Strip any inline twoslash cache comment so it isn't inlined ahead of a
+    // host block's code when this virtual file is imported/included. The
+    // original node keeps its comment for its own rendering.
+    virtualFiles.set(fileName, InlineCache.stripInlineCacheComments(node.value))
   }
   return virtualFiles
 }

package/src/internal/shiki-transformers.ts

@@ -19,6 +19,7 @@ type TransformerTwoslashIndexOptions = TransformerTwoslashOptions
 
 import * as Snippets from './snippets.js'
 import * as TwoslashChecker from './twoslash/checker.js'
+import * as InlineCache from './twoslash/inline-cache.js'
 import * as Renderer from './twoslash/renderer.js'
 import * as TypesCache from './twoslash/types-cache.js'
 
@@ -240,7 +241,7 @@ let typescript: TS | undefined
 const twoslasherResetInterval = 200
 
 export function twoslash(options: twoslash.Options): ShikiTransformer {
-  const { cacheDir } = options
+  const { cacheDir, inlineCache } = options
   const {
     checkOnly = false,
     explicitTrigger = true,
@@ -252,7 +253,9 @@ export function twoslash(options: twoslash.Options): ShikiTransformer {
     renderer = Renderer.rich(),
     throws = true,
     twoslashOptions,
-    typesCache = TypesCache.fs({ dir: cacheDir ? path.join(cacheDir, 'twoslash') : undefined }),
+    typesCache = InlineCache.enabled(inlineCache)
+      ? InlineCache.getOrCreate().typesCache
+      : TypesCache.fs({ dir: cacheDir ? path.join(cacheDir, 'twoslash') : undefined }),
   } = options
 
   const lazyTwoslasher = (
@@ -304,8 +307,13 @@ export function twoslash(options: twoslash.Options): ShikiTransformer {
           executeOptions?: Parameters<TwoslashInstance>[2],
           meta?: Parameters<NonNullable<typeof typesCache.preprocess>>[3],
         ) {
-          const preprocessed = typesCache.preprocess?.(code, lang, executeOptions, meta)
-          return Renderer.normalizeCustomTagBlocks(preprocessed ?? code)
+          // Normalize before the inner preprocess so the cache key hashed on
+          // read matches the code hashed on write (which runs against the
+          // normalized output). Without this, snippets containing consecutive
+          // custom tag lines (`@log`/`@error`/`@warn`/`@annotate`) would never
+          // hit the cache.
+          const normalized = Renderer.normalizeCustomTagBlocks(code)
+          return typesCache.preprocess?.(normalized, lang, executeOptions, meta) ?? normalized
         },
       }
     : undefined
@@ -344,6 +352,34 @@ export declare namespace twoslash {
      */
     checkOnly?: boolean | undefined
     cacheDir?: string | undefined
+    /**
+     * Persist twoslash results inline in the markdown source as
+     * `// @twoslash-cache: ...` comments so the cache travels with the repo
+     * (e.g. for fresh clones and CI). Overridden by the `TWOSLASH_INLINE_CACHE`
+     * environment variable.
+     *
+     * @default false
+     */
+    inlineCache?: boolean | undefined
+  }
+}
+
+/**
+ * Shiki transformer that extracts the ephemeral source-map comment injected by
+ * the inline-cache remark plugin, exposing it on `this.meta.sourceMap` for the
+ * inline types cache and stripping it from the code before twoslash runs.
+ *
+ * Must run before the twoslash transformer.
+ */
+export function extractInlineCacheSourceMap(): ShikiTransformer {
+  return {
+    name: 'vocs:twoslash-inline-cache-source-map',
+    enforce: 'pre',
+    preprocess(code) {
+      const { code: stripped, sourceMap } = InlineCache.extractSourceMapComment(code)
+      this.meta.sourceMap = sourceMap
+      return stripped
+    },
   }
 }
 

package/src/internal/twoslash/file-patcher.ts

@@ -0,0 +1,60 @@
+import * as node_fs from 'node:fs'
+import MagicString from 'magic-string'
+
+/**
+ * Accumulates character-range patches per file and flushes them all at once.
+ *
+ * Used by the twoslash inline cache to write `// @twoslash-cache: ...` comments
+ * back into the original markdown source after a code block has been processed.
+ *
+ * Ported from `@shikijs/vitepress-twoslash`'s `FilePatcher`.
+ */
+export class FilePatcher {
+  private files = new Map<string, { content: string; patches: Map<string, string> } | null>()
+
+  /** Build a patch key from a character range. Omitting `to` denotes an insertion. */
+  static key(from: number, to?: number): string {
+    return `${from}${to ? `:${to}` : ''}`
+  }
+
+  /**
+   * Load a file's contents (cached for the lifetime of the patcher, until
+   * flushed via {@link patch}). Returns `null` if the file does not exist.
+   */
+  load(path: string): { content: string; patches: Map<string, string> } | null {
+    let file = this.files.get(path)
+    if (file === undefined) {
+      if (node_fs.existsSync(path)) {
+        const content = node_fs.readFileSync(path, { encoding: 'utf-8' })
+        file = { content, patches: new Map() }
+      } else {
+        file = null
+      }
+      this.files.set(path, file)
+    }
+    return file
+  }
+
+  /** Apply all queued patches for `path` and write the result back to disk. */
+  patch(path: string): void {
+    const file = this.files.get(path)
+    if (!file) return
+
+    if (file.patches.size) {
+      const s = new MagicString(file.content)
+
+      for (const [key, value] of file.patches) {
+        const [from, to] = key.split(':').map((x) => (x !== '' ? Number(x) : undefined))
+        if (from === undefined) continue
+
+        if (to !== undefined) s.update(from, to, value)
+        else s.appendRight(from, value)
+      }
+
+      const content = s.toString()
+      if (content !== file.content) node_fs.writeFileSync(path, content, { encoding: 'utf-8' })
+    }
+
+    this.files.delete(path)
+  }
+}

package/src/internal/twoslash/index.ts

@@ -1,3 +1,4 @@
 export * from './experimental_rust.js'
+export * as InlineCache from './inline-cache.js'
 export * as Renderer from './renderer.js'
 export * as TypesCache from './types-cache.js'

package/src/internal/twoslash/inline-cache.test.ts

@@ -0,0 +1,212 @@
+import * as fs from 'node:fs'
+import * as os from 'node:os'
+import * as path from 'node:path'
+import { afterEach, describe, expect, it } from 'vitest'
+import {
+  createInlineTypesCache,
+  extractSourceMapComment,
+  injectSourceMapComment,
+  stripInlineCacheComments,
+} from './inline-cache.js'
+
+describe('source map codec', () => {
+  it('round-trips a source map through inject/extract', () => {
+    const sourceMap = { path: '/docs/a.md', from: 15, to: 42 }
+    const injected = injectSourceMapComment('const a = 1', sourceMap)
+    expect(injected.startsWith('// @vocs-twoslash-source:')).toBe(true)
+
+    const result = extractSourceMapComment(injected)
+    expect(result.code).toBe('const a = 1')
+    expect(result.sourceMap).toEqual(sourceMap)
+  })
+
+  it('returns null source map when none present', () => {
+    const result = extractSourceMapComment('const a = 1')
+    expect(result.code).toBe('const a = 1')
+    expect(result.sourceMap).toBeNull()
+  })
+})
+
+describe('stripInlineCacheComments', () => {
+  it('removes all cache comments', () => {
+    const code = [
+      '// @twoslash-cache: {"v":1,"hash":"abc","data":"x"}',
+      'export const client = 1',
+    ].join('\n')
+    expect(stripInlineCacheComments(code)).toBe('export const client = 1')
+  })
+
+  it('leaves code without cache comments untouched', () => {
+    const code = 'export const client = 1\nconst a = 2'
+    expect(stripInlineCacheComments(code)).toBe(code)
+  })
+})
+
+describe('inline types cache', () => {
+  const tmpFiles: string[] = []
+
+  afterEach(() => {
+    for (const file of tmpFiles.splice(0)) fs.rmSync(file, { force: true })
+  })
+
+  function createMarkdown(body: string) {
+    const file = path.join(fs.mkdtempSync(path.join(os.tmpdir(), 'vocs-inline-cache-')), 'page.md')
+    const md = `\`\`\`ts twoslash\n${body}\n\`\`\`\n`
+    fs.writeFileSync(file, md, 'utf-8')
+    tmpFiles.push(file)
+    // body starts right after the first newline (the opening fence line).
+    const from = md.indexOf('\n') + 1
+    return { file, md, from, to: md.length }
+  }
+
+  // Pull the on-disk fence body the way shiki would see it (everything between
+  // the opening fence line and the closing fence).
+  function readBody(file: string, from: number) {
+    const content = fs.readFileSync(file, 'utf-8')
+    return content.slice(from, content.lastIndexOf('\n```'))
+  }
+
+  it('writes a cache comment back to the source and reads it on the next pass', () => {
+    const code = 'const a: string = "x"'
+    const { file, from, to } = createMarkdown(code)
+
+    const data = { nodes: [], code: 'compiled-output' }
+
+    // --- Pass 1: cache miss -> twoslash runs -> write back ---
+    {
+      const { typesCache, patcher } = createInlineTypesCache()
+      const meta = { sourceMap: { path: file, from, to } } as never
+
+      const preprocessed = typesCache.preprocess?.(code, 'ts', {}, meta)
+      expect(preprocessed).toBe(code)
+      expect(typesCache.read(code, 'ts', {}, meta)).toBeNull()
+
+      typesCache.write(code, data as never, 'ts', {}, meta)
+      patcher.patch(file)
+    }
+
+    const written = fs.readFileSync(file, 'utf-8')
+    expect(written).toContain('// @twoslash-cache:')
+
+    // --- Pass 2: cache hit -> twoslash skipped ---
+    {
+      const { typesCache } = createInlineTypesCache()
+      const body = readBody(file, from)
+      const meta = { sourceMap: { path: file, from, to } } as never
+
+      const preprocessed = typesCache.preprocess?.(body, 'ts', {}, meta)
+      // the cache comment is stripped before twoslash would run
+      expect(preprocessed).toBe(code)
+
+      const cached = typesCache.read(body, 'ts', {}, meta)
+      expect(cached).toEqual(data)
+    }
+  })
+
+  it('replaces an existing cache comment in place instead of appending a duplicate', () => {
+    const code = 'const a: string = "x"'
+    const { file, from, to } = createMarkdown(code)
+    const data = { nodes: [], code: 'compiled-output' }
+
+    // --- Pass 1: cold write ---
+    {
+      const { typesCache, patcher } = createInlineTypesCache()
+      const meta = { sourceMap: { path: file, from, to } } as never
+      typesCache.preprocess?.(code, 'ts', {}, meta)
+      typesCache.write(code, data as never, 'ts', {}, meta)
+      patcher.patch(file)
+    }
+
+    const cacheLines = () =>
+      fs
+        .readFileSync(file, 'utf-8')
+        .split('\n')
+        .filter((l) => l.includes('// @twoslash-cache:'))
+    expect(cacheLines()).toHaveLength(1)
+
+    // --- Pass 2: a writer with stale in-memory code (no cache comment, so the
+    // existing comment can't be located via `search`) must still replace the
+    // on-disk comment rather than append a second one. ---
+    {
+      const content = fs.readFileSync(file, 'utf-8')
+      const currentTo = content.lastIndexOf('\n```')
+      const { typesCache, patcher } = createInlineTypesCache()
+      const meta = { sourceMap: { path: file, from, to: currentTo } } as never
+      // Pass the original (comment-free) code to simulate a stale read.
+      typesCache.preprocess?.(code, 'ts', {}, meta)
+      typesCache.write(code, data as never, 'ts', {}, meta)
+      patcher.patch(file)
+    }
+
+    expect(cacheLines()).toHaveLength(1)
+  })
+
+  it('invalidates the cache when the hash no longer matches', () => {
+    const code = 'const a: string = "x"'
+    const { file, from, to } = createMarkdown(code)
+
+    // Seed a cache entry.
+    {
+      const { typesCache, patcher } = createInlineTypesCache()
+      const meta = { sourceMap: { path: file, from, to } } as never
+      typesCache.preprocess?.(code, 'ts', {}, meta)
+      typesCache.write(code, { nodes: [], code: 'x' } as never, 'ts', {}, meta)
+      patcher.patch(file)
+    }
+
+    // Read with different code -> hash mismatch -> miss.
+    {
+      const { typesCache } = createInlineTypesCache()
+      const body = readBody(file, from)
+      const meta = { sourceMap: { path: file, from, to } } as never
+      // The body still has the old cache line, but we validate against new code.
+      typesCache.preprocess?.(`${body}\nconst b = 2`, 'ts', {}, meta)
+      expect(typesCache.read(body, 'ts', {}, meta)).toBeNull()
+    }
+  })
+
+  it('ignoreCache skips reading existing cache', () => {
+    const code = 'const a: string = "x"'
+    const { file, from, to } = createMarkdown(code)
+
+    {
+      const { typesCache, patcher } = createInlineTypesCache()
+      const meta = { sourceMap: { path: file, from, to } } as never
+      typesCache.preprocess?.(code, 'ts', {}, meta)
+      typesCache.write(code, { nodes: [], code: 'x' } as never, 'ts', {}, meta)
+      patcher.patch(file)
+    }
+
+    {
+      const { typesCache } = createInlineTypesCache({ ignoreCache: true })
+      const body = readBody(file, from)
+      const meta = { sourceMap: { path: file, from, to } } as never
+      typesCache.preprocess?.(body, 'ts', {}, meta)
+      expect(typesCache.read(body, 'ts', {}, meta)).toBeNull()
+    }
+  })
+
+  it('remove mode strips the cache comment from the source', () => {
+    const code = 'const a: string = "x"'
+    const { file, from, to } = createMarkdown(code)
+
+    {
+      const { typesCache, patcher } = createInlineTypesCache()
+      const meta = { sourceMap: { path: file, from, to } } as never
+      typesCache.preprocess?.(code, 'ts', {}, meta)
+      typesCache.write(code, { nodes: [], code: 'x' } as never, 'ts', {}, meta)
+      patcher.patch(file)
+    }
+    expect(fs.readFileSync(file, 'utf-8')).toContain('// @twoslash-cache:')
+
+    {
+      const { typesCache, patcher } = createInlineTypesCache({ remove: true })
+      const body = readBody(file, from)
+      const meta = { sourceMap: { path: file, from, to } } as never
+      typesCache.preprocess?.(body, 'ts', {}, meta)
+      typesCache.write(body, { nodes: [], code: 'x' } as never, 'ts', {}, meta)
+      patcher.patch(file)
+    }
+    expect(fs.readFileSync(file, 'utf-8')).not.toContain('// @twoslash-cache:')
+  })
+})