polen 0.9.0-next.4 → 0.9.0-next.6

package/src/api/singletons/markdown/markdown.test.ts

@@ -0,0 +1,89 @@
+import { describe, expect, test } from 'vitest'
+import { parse } from './markdown.js'
+
+describe(`markdown parser with syntax highlighting`, () => {
+  test(`parse highlights code blocks`, async () => {
+    const markdown = `
+# Hello
+
+\`\`\`javascript
+const x = 42
+console.log(x)
+\`\`\`
+`
+    const result = await parse(markdown)
+    
+    expect(result).toContain(`<h1>Hello</h1>`)
+    expect(result).toContain(`<pre`)
+    expect(result).toContain(`shiki`)
+    expect(result).toContain(`console`)
+    expect(result).toContain(`42`)
+  })
+
+  // Note: parseSync cannot be used with async rehype plugins like Shiki
+  // This is a known limitation - syntax highlighting requires async processing
+
+  test(`parse supports GraphQL syntax`, async () => {
+    const markdown = `
+\`\`\`graphql
+type Query {
+  user(id: ID!): User
+}
+\`\`\`
+`
+    const result = await parse(markdown)
+    
+    expect(result).toContain(`type`)
+    expect(result).toContain(`Query`)
+    // Check that both ID and ! are present (they may be in separate spans)
+    expect(result).toContain(`> ID<`)
+    expect(result).toContain(`>!</`)
+  })
+
+  test(`parse handles inline code`, async () => {
+    const markdown = `This is \`inline code\` in a sentence.`
+    const result = await parse(markdown)
+    
+    expect(result).toContain(`<code>inline code</code>`)
+  })
+
+  test(`parse supports GitHub Flavored Markdown`, async () => {
+    const markdown = `
+| Column 1 | Column 2 |
+|----------|----------|
+| Cell 1   | Cell 2   |
+
+- [x] Task 1
+- [ ] Task 2
+`
+    const result = await parse(markdown)
+    
+    expect(result).toContain(`<table>`)
+    expect(result).toContain(`<input`)
+    expect(result).toContain(`checked`)
+  })
+
+  test(`parse handles code blocks without language`, async () => {
+    const markdown = `
+\`\`\`
+plain text without language
+\`\`\`
+`
+    const result = await parse(markdown)
+    
+    expect(result).toContain(`<pre`)
+    expect(result).toContain(`plain text without language`)
+  })
+
+  test(`parse preserves theme CSS variables`, async () => {
+    const markdown = `
+\`\`\`javascript
+const theme = "light"
+\`\`\`
+`
+    const result = await parse(markdown)
+    
+    expect(result).toContain(`--shiki-light`)
+    expect(result).toContain(`--shiki-dark`)
+  })
+})

package/src/api/singletons/markdown/markdown.ts

@@ -1,21 +1,43 @@
-import * as Remark from 'remark'
-import RemarkGfm from 'remark-gfm'
-import RemarkHtml from 'remark-html'
+import { unified } from 'unified'
+import remarkParse from 'remark-parse'
+import remarkGfm from 'remark-gfm'
+import remarkRehype from 'remark-rehype'
+import rehypeShiki from '@shikijs/rehype'
+import rehypeStringify from 'rehype-stringify'
 
-export const parse = async (content: string): Promise<string> => {
-  const result = await Remark.remark()
-    .use(RemarkGfm)
-    .use(RemarkHtml)
-    .process(content)
+// Create a processor with Shiki for syntax highlighting (async only)
+const createProcessorWithShiki = () => {
+  return unified()
+    .use(remarkParse)
+    .use(remarkGfm)
+    .use(remarkRehype)
+    .use(rehypeShiki, {
+      themes: {
+        light: `github-light`,
+        dark: `tokyo-night`,
+      },
+      defaultColor: false,
+      cssVariablePrefix: `--shiki-`,
+    })
+    .use(rehypeStringify)
+}
+
+// Create a processor without syntax highlighting for sync processing
+const createProcessorSync = () => {
+  return unified()
+    .use(remarkParse)
+    .use(remarkGfm)
+    .use(remarkRehype)
+    .use(rehypeStringify)
+}
 
+export const parse = async (content: string): Promise<string> => {
+  const result = await createProcessorWithShiki().process(content)
   return String(result)
 }
 
 export const parseSync = (content: string): string => {
-  const result = Remark.remark()
-    .use(RemarkGfm)
-    .use(RemarkHtml)
-    .processSync(content)
-
+  // Note: Syntax highlighting is not available in sync mode due to @shikijs/rehype being async-only
+  const result = createProcessorSync().processSync(content)
   return String(result)
 }

package/src/api/vite/plugins/pages.ts

@@ -10,6 +10,7 @@ import { superjson } from '#singletons/superjson'
 import mdx from '@mdx-js/rollup'
 import { Path, Str } from '@wollybeard/kit'
 import remarkGfm from 'remark-gfm'
+import rehypeShiki from '@shikijs/rehype'
 
 const _debug = debug.sub(`vite-plugin-pages`)
 
@@ -123,6 +124,22 @@ export const Pages = ({
       ...mdx({
         jsxImportSource: `polen/react`,
         remarkPlugins: [remarkGfm],
+        rehypePlugins: [
+          [
+            rehypeShiki,
+            {
+              themes: {
+                light: `github-light`,
+                dark: `tokyo-night`,
+              },
+              defaultColor: false,
+              cssVariablePrefix: `--shiki-`,
+              transformers: [
+                // Line numbers will be handled via CSS
+              ],
+            },
+          ],
+        ],
       }),
     },
 

package/src/lib/shiki/index.ts

@@ -0,0 +1 @@
+export * from './shiki.ts'

package/src/lib/shiki/shiki.test.ts

@@ -0,0 +1,107 @@
+import { expect, test } from 'vitest'
+import { getHighlighter, highlightCode } from './shiki.js'
+
+test(`getHighlighter returns singleton instance`, async () => {
+  const highlighter1 = await getHighlighter()
+  const highlighter2 = await getHighlighter()
+
+  expect(highlighter1).toBe(highlighter2)
+})
+
+test(`highlightCode generates HTML with syntax highlighting`, async () => {
+  const code = `const hello = "world"`
+  const result = await highlightCode({
+    code,
+    lang: `javascript`,
+    theme: `light`,
+  })
+
+  expect(result).toContain(`<pre`)
+  expect(result).toContain(`shiki`)
+  expect(result).toContain(`hello`)
+  expect(result).toContain(`world`)
+})
+
+test(`highlightCode supports TypeScript`, async () => {
+  const code = `interface User { name: string }`
+  const result = await highlightCode({
+    code,
+    lang: `typescript`,
+    theme: `light`,
+  })
+
+  expect(result).toContain(`interface`)
+  expect(result).toContain(`User`)
+  expect(result).toContain(`string`)
+})
+
+test(`highlightCode supports GraphQL`, async () => {
+  const code = `type Query { user: User }`
+  const result = await highlightCode({
+    code,
+    lang: `graphql`,
+    theme: `light`,
+  })
+
+  expect(result).toContain(`type`)
+  expect(result).toContain(`Query`)
+  expect(result).toContain(`User`)
+})
+
+test(`highlightCode handles unknown language gracefully`, async () => {
+  const code = `some random text`
+  // Use 'text' as fallback for unknown languages
+  const result = await highlightCode({
+    code,
+    lang: `text`,
+    theme: `light`,
+  })
+
+  // Should still return highlighted HTML
+  expect(result).toContain(`<pre`)
+  expect(result).toContain(`some random text`)
+})
+
+test(`highlightCode applies line numbers when requested`, async () => {
+  const code = `line 1\nline 2\nline 3`
+  const result = await highlightCode({
+    code,
+    lang: `text`,
+    theme: `light`,
+    showLineNumbers: true,
+  })
+
+  expect(result).toContain(`data-line-numbers="true"`)
+})
+
+test(`highlightCode applies line highlighting`, async () => {
+  const code = `line 1\nline 2\nline 3`
+  const result = await highlightCode({
+    code,
+    lang: `text`,
+    theme: `light`,
+    highlightLines: [2],
+  })
+
+  expect(result).toContain(`data-highlighted="true"`)
+})
+
+test(`highlightCode supports both light and dark themes`, async () => {
+  const code = `const x = 1`
+
+  const lightResult = await highlightCode({
+    code,
+    lang: `javascript`,
+    theme: `light`,
+  })
+
+  const darkResult = await highlightCode({
+    code,
+    lang: `javascript`,
+    theme: `dark`,
+  })
+
+  // Both should contain theme CSS variables
+  expect(lightResult).toContain(`--shiki-light`)
+  expect(darkResult).toContain(`--shiki-dark`)
+})

package/src/lib/shiki/shiki.ts

@@ -0,0 +1,161 @@
+import { createHighlighter, type Highlighter, type BundledTheme, type BundledLanguage } from 'shiki'
+import {
+  transformerNotationHighlight,
+  transformerNotationDiff,
+  transformerNotationFocus,
+  transformerRenderWhitespace,
+} from '@shikijs/transformers'
+
+export interface ShikiOptions {
+  themes?: {
+    light: BundledTheme
+    dark: BundledTheme
+  }
+  langs?: BundledLanguage[]
+  defaultTheme?: 'light' | 'dark'
+}
+
+const DEFAULT_THEMES = {
+  light: `github-light` as BundledTheme,
+  dark: `tokyo-night` as BundledTheme,
+}
+
+const DEFAULT_LANGS: BundledLanguage[] = [
+  `typescript`,
+  `javascript`,
+  `jsx`,
+  `tsx`,
+  `graphql`,
+  `json`,
+  `yaml`,
+  `markdown`,
+  `bash`,
+  `shell`,
+  `css`,
+  `html`,
+  `sql`,
+  `python`,
+  `rust`,
+  `go`,
+  `java`,
+  `csharp`,
+  `php`,
+  `ruby`,
+  `swift`,
+  `kotlin`,
+  `scala`,
+  `r`,
+  `matlab`,
+  `latex`,
+  `dockerfile`,
+  `makefile`,
+  `nginx`,
+  `apache`,
+  `xml`,
+  `toml`,
+  `ini`,
+  `diff`,
+]
+
+// Singleton highlighter instance
+let highlighterInstance: Highlighter | null = null
+let highlighterPromise: Promise<Highlighter> | null = null
+
+export async function getHighlighter(options: ShikiOptions = {}): Promise<Highlighter> {
+  if (highlighterInstance) {
+    return highlighterInstance
+  }
+
+  if (!highlighterPromise) {
+    const themes = options.themes || DEFAULT_THEMES
+    const langs = options.langs || DEFAULT_LANGS
+
+    highlighterPromise = createHighlighter({
+      themes: [themes.light, themes.dark],
+      langs,
+    }).then(highlighter => {
+      highlighterInstance = highlighter
+      return highlighter
+    })
+  }
+
+  return highlighterPromise
+}
+
+export interface CodeHighlightOptions {
+  code: string
+  lang?: string
+  theme?: 'light' | 'dark'
+  showLineNumbers?: boolean
+  highlightLines?: number[]
+  diffLines?: { add: number[], remove: number[] }
+  focusLines?: number[]
+  showInvisibles?: boolean
+}
+
+export async function highlightCode({
+  code,
+  lang = `text`,
+  theme = `light`,
+  showLineNumbers = false,
+  highlightLines = [],
+  diffLines,
+  focusLines = [],
+  showInvisibles = false,
+}: CodeHighlightOptions): Promise<string> {
+  const highlighter = await getHighlighter()
+  
+  const themes = {
+    light: DEFAULT_THEMES.light,
+    dark: DEFAULT_THEMES.dark,
+  }
+
+  const transformers = []
+
+  // Add line numbers transformer if needed
+  if (showLineNumbers) {
+    // Custom line numbers will be handled in CSS
+    transformers.push({
+      name: `line-numbers`,
+      pre(node: any) {
+        node.properties[`data-line-numbers`] = `true`
+      }
+    })
+  }
+
+  // Add highlight transformer
+  if (highlightLines.length > 0) {
+    transformers.push({
+      name: `highlight-lines`,
+      line(node: any, line: number) {
+        if (highlightLines.includes(line)) {
+          node.properties[`data-highlighted`] = `true`
+        }
+      }
+    })
+  }
+
+  // Add standard transformers
+  transformers.push(
+    transformerNotationHighlight(),
+    transformerNotationDiff(),
+    transformerNotationFocus(),
+  )
+
+  if (showInvisibles) {
+    transformers.push(transformerRenderWhitespace())
+  }
+
+  // Generate HTML with CSS variables for theme switching
+  const html = highlighter.codeToHtml(code, {
+    lang,
+    themes,
+    defaultColor: false,
+    transformers,
+  })
+
+  return html
+}
+
+// Re-export types
+export type { Highlighter, BundledTheme, BundledLanguage } from 'shiki'

package/src/template/components/CodeBlock.tsx

@@ -0,0 +1,73 @@
+import { highlightCode } from '#lib/shiki/index'
+import React, { useEffect, useState } from 'react'
+
+interface CodeBlockProps {
+  children: string
+  language?: string
+  className?: string
+  showLineNumbers?: boolean
+  highlightLines?: number[]
+  diffLines?: { add: number[]; remove: number[] }
+  focusLines?: number[]
+  showInvisibles?: boolean
+}
+
+export const CodeBlock: React.FC<CodeBlockProps> = ({
+  children,
+  language = `text`,
+  className = ``,
+  showLineNumbers = false,
+  highlightLines = [],
+  diffLines,
+  focusLines = [],
+  showInvisibles = false,
+}) => {
+  const [html, setHtml] = useState<string>(``)
+  const [isLoading, setIsLoading] = useState(true)
+
+  // TODO: Implement proper theme detection
+  // For now, we'll rely on CSS to handle theme switching
+  const theme = `light` // Default to light theme
+
+  useEffect(() => {
+    const renderCode = async () => {
+      try {
+        const output = await highlightCode({
+          code: children,
+          lang: language,
+          theme,
+          showLineNumbers,
+          highlightLines,
+          diffLines,
+          focusLines,
+          showInvisibles,
+        })
+
+        setHtml(output)
+      } catch (error) {
+        console.error(`Failed to highlight code:`, error)
+        // Fallback to plain text
+        setHtml(`<pre><code>${children}</code></pre>`)
+      } finally {
+        setIsLoading(false)
+      }
+    }
+
+    renderCode()
+  }, [children, language, showLineNumbers, highlightLines, diffLines, focusLines, showInvisibles])
+
+  if (isLoading) {
+    return (
+      <pre className={className}>
+        <code>{children}</code>
+      </pre>
+    )
+  }
+
+  return (
+    <div
+      className={`code-block ${className}`}
+      dangerouslySetInnerHTML={{ __html: html }}
+    />
+  )
+}

package/src/template/components/sidebar/Sidebar.tsx

@@ -1,17 +1,19 @@
 import type { FileRouter } from '#lib/file-router/index'
 import { Box } from '@radix-ui/themes'
+import type { BoxOwnProps, LayoutProps, MarginProps } from '@radix-ui/themes/props'
 import { Items } from './SidebarItem.tsx'
 
-interface SidebarProps {
-  items: FileRouter.Sidebar.Item[]
+interface SidebarProps extends LayoutProps, MarginProps, BoxOwnProps {
+  data: FileRouter.Sidebar.Item[]
+  style?: React.CSSProperties
 }
 
-export const Sidebar = ({ items }: SidebarProps) => {
+export const Sidebar = ({ data, ...props }: SidebarProps) => {
   return (
     <Box
       data-testid='sidebar'
       role='Sidebar'
-      flexShrink='0'
+      {...props}
     >
       <style>
         {`
@@ -20,7 +22,7 @@ export const Sidebar = ({ items }: SidebarProps) => {
           }
         `}
       </style>
-      <Items items={items} />
+      <Items items={data} />
     </Box>
   )
 }

package/src/template/entry.client.tsx

@@ -2,6 +2,7 @@
 // But then, we won't get it from the client manifest. But we could get it from the server manifest. Should we do that?
 // But then, that wouldn't work for SPA. Does that matter? Just put a conditional here e.g. if (import.meta.env.PROD) ...?
 import '@radix-ui/themes/styles.css'
+// import './styles/code-block.css' // TODO: Handle CSS in build process
 import { ReactDomClient } from '#dep/react-dom-client/index'
 import { StrictMode } from 'react'
 import { createBrowserRouter, RouterProvider } from 'react-router'