pretext-editor 0.1.0 → 0.2.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Pretext contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,107 @@
+# pretext-editor
+
+High-performance Canvas-virtualized code editor for React. Renders only visible
+lines via Canvas 2D, so it stays smooth even on very large files.
+
+## How it works
+
+| Layer | Technology |
+|---|---|
+| Rendering | Canvas 2D API — only visible lines are drawn each frame |
+| Text layout | [@chenglou/pretext](https://github.com/chenglou/pretext) |
+| Syntax highlighting | [Shiki](https://shiki.style/) with JavaScript regex engine (no WASM) |
+| Tokenization cache | [@tanstack/react-query](https://tanstack.com/query) — results cached, `useDeferredValue` keeps typing fast |
+| Keyboard / IME input | Hidden `<textarea>` captures all keyboard events and IME composition |
+
+## Installation
+
+```bash
+npm install @molang/pretext-editor
+```
+
+Wrap your app with `QueryClientProvider` once (skip if you already have it):
+
+```tsx
+import { QueryClient, QueryClientProvider } from '@tanstack/react-query'
+
+const queryClient = new QueryClient()
+
+function App() {
+  return (
+    <QueryClientProvider client={queryClient}>
+      <YourApp />
+    </QueryClientProvider>
+  )
+}
+```
+
+## Usage
+
+```tsx
+import { useState } from 'react'
+import { PretextEditor } from '@molang/pretext-editor'
+
+export function Editor() {
+  const [code, setCode] = useState('// hello world\n')
+
+  return (
+    <div style={{ height: 600 }}>
+      <PretextEditor
+        value={code}
+        onChange={setCode}
+        language="typescript"
+        fontSize={14}
+        fontFamily='Menlo, Monaco, "Courier New", monospace'
+        tabSize={4}
+      />
+    </div>
+  )
+}
+```
+
+The component fills its parent — give the parent an explicit height.
+
+## Props
+
+| Prop | Type | Default | Description |
+|---|---|---|---|
+| `value` | `string` | — | Document content (controlled) |
+| `onChange` | `(value: string) => void` | — | Called on every edit |
+| `language` | `string` | `undefined` | Shiki language id (`"typescript"`, `"go"`, `"python"` …). Omit to disable highlighting. |
+| `fontSize` | `number` | `14` | Font size in px |
+| `fontFamily` | `string` | `"monospace"` | CSS font-family string |
+| `tabSize` | `number` | `4` | Visual width of `\t` in spaces. Raw `\t` is preserved in content. |
+| `className` | `string` | — | Class on the scroll container |
+| `style` | `CSSProperties` | — | Inline style on the scroll container |
+
+## Supported languages
+
+Any language bundled with Shiki is supported. Common ones:
+
+`typescript` · `tsx` · `javascript` · `jsx` · `python` · `rust` · `go` ·
+`c` · `cpp` · `csharp` · `java` · `kotlin` · `swift` · `ruby` · `php` ·
+`css` · `scss` · `html` · `vue` · `svelte` · `json` · `yaml` · `toml` ·
+`markdown` · `bash` · `sql` · `graphql`
+
+Pass the extension → language id mapping helper if needed:
+
+```tsx
+import { extToLang } from '@molang/pretext-editor'
+
+const lang = extToLang('ts') // → "typescript"
+```
+
+## Keyboard shortcuts
+
+| Key | Action |
+|---|---|
+| Arrow keys | Move cursor |
+| Shift + Arrow | Extend selection |
+| Home / End | Jump to line start / end |
+| Ctrl/⌘ + ← / → | Jump to line start / end |
+| Ctrl/⌘ + A | Select all |
+| Ctrl/⌘ + C / X / V | Copy / Cut / Paste |
+| Ctrl/⌘ + Z | Undo |
+| Ctrl/⌘ + Shift+Z / Y | Redo |
+| Tab | Insert two spaces |
+| Backspace / Delete | Delete backward / forward |

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pretext-editor",
-  "version": "0.1.0",
+  "version": "0.2.0",
   "private": false,
   "description": "High-performance Canvas-virtualized text editor for large documents",
   "type": "module",
@@ -10,11 +10,13 @@
     "typecheck": "tsc --noEmit"
   },
   "peerDependencies": {
+    "@tanstack/react-query": ">=5.0.0",
     "react": ">=19.0.0",
     "react-dom": ">=19.0.0"
   },
   "dependencies": {
-    "@chenglou/pretext": "^0.0.8"
+    "@chenglou/pretext": "^0.0.8",
+    "shiki": "^4.2.0"
   },
   "devDependencies": {
     "@types/react": "^19.2.7",

package/src/core/renderer.ts

@@ -1,8 +1,12 @@
 import type { Cursor, Selection } from './document'
 import { isCollapsed, normalizeSelection } from './document'
 
+export type TokenSpan = { text: string; color: string }
+export type TokenizedLine = TokenSpan[]
+
 export const DEFAULT_FONT_SIZE = 14
 export const DEFAULT_FONT_FAMILY = 'monospace'
+export const DEFAULT_TAB_SIZE = 4
 export const PADDING_LEFT = 52
 export const PADDING_TOP = 8
 export const FONT_SIZE_TO_LINE_HEIGHT = (size: number): number => size + 8
@@ -11,10 +15,11 @@ const BG = '#1e1e1e'
 const FG = '#d4d4d4'
 const GUTTER_BG = '#1e1e1e'
 const GUTTER_FG = '#6e7681'
-const GUTTER_BORDER = '#2b2b2b'
 const CURSOR_COLOR = '#d4d4d4'
 const CURRENT_LINE_BG = 'rgba(255,255,255,0.04)'
 const SELECTION_BG = '#264f78'
+const INDENT_GUIDE = 'rgba(255,255,255,0.1)'
+const INDENT_GUIDE_ACTIVE = 'rgba(255,255,255,0.3)'
 
 export interface RenderOptions {
   canvas: HTMLCanvasElement
@@ -24,6 +29,124 @@ export interface RenderOptions {
   scrollTop: number
   fontSize?: number
   fontFamily?: string
+  tabSize?: number
+  tokenLines?: TokenizedLine[]
+}
+
+/** Count leading-whitespace depth in spaces (tabs expand to tabSize) */
+function indentDepth(text: string, tabSize: number): number {
+  let depth = 0
+  for (const ch of text) {
+    if (ch === ' ') depth++
+    else if (ch === '\t') depth = Math.ceil((depth + 1) / tabSize) * tabSize
+    else break
+  }
+  return depth
+}
+
+/** Detect the file's actual indent unit (minimum non-zero indent depth). */
+function detectIndentUnit(lines: string[], tabSize: number): number {
+  let unit = Infinity
+  for (const line of lines) {
+    if (line.trim() === '') continue
+    const d = indentDepth(line, tabSize)
+    if (d > 0) unit = Math.min(unit, d)
+  }
+  return isFinite(unit) ? unit : tabSize
+}
+
+function buildGuideData(
+  lines: string[],
+  tabSize: number,
+): { rawLevels: number[]; effectiveLevels: number[]; indentUnit: number } {
+  const indentUnit = detectIndentUnit(lines, tabSize)
+  const rawLevels = lines.map((l) =>
+    l.trim() === '' ? -1 : Math.floor(indentDepth(l, tabSize) / indentUnit),
+  )
+  const effectiveLevels = rawLevels.slice()
+  let prev = 0
+  for (let i = 0; i < effectiveLevels.length; i++) {
+    if (effectiveLevels[i] === -1) effectiveLevels[i] = prev
+    else prev = effectiveLevels[i]
+  }
+  let next = 0
+  for (let i = effectiveLevels.length - 1; i >= 0; i--) {
+    if (rawLevels[i] === -1) effectiveLevels[i] = Math.min(effectiveLevels[i], next)
+    else next = effectiveLevels[i]
+  }
+  return { rawLevels, effectiveLevels, indentUnit }
+}
+
+/**
+ * Walk upward from cursorLine, bracket-balance scanning, to find the innermost
+ * enclosing bracket pair ({}/[]/()). Returns the indent level of the content
+ * inside that pair. Falls back to the cursor line's own level when no enclosing
+ * bracket exists (mirrors Monaco's indentation-based fallback).
+ */
+function activeBracketLevel(
+  lines: string[],
+  cursorLine: number,
+  rawLevels: number[],
+  effectiveLevels: number[],
+): number {
+  const lvl = (i: number) => (rawLevels[i] === -1 ? (effectiveLevels[i] ?? 0) : rawLevels[i])
+  let depth = 0
+  for (let i = cursorLine; i >= 0; i--) {
+    const text = lines[i]
+    for (let j = text.length - 1; j >= 0; j--) {
+      const ch = text[j]
+      if (ch === '}' || ch === ']' || ch === ')') {
+        depth++
+      } else if (ch === '{' || ch === '[' || ch === '(') {
+        if (depth > 0) {
+          depth--
+        } else {
+          // innermost enclosing opener at line i — content level is the first non-empty line after it
+          for (let k = i + 1; k < lines.length && k <= i + 100; k++) {
+            if (lines[k].trim() !== '') return lvl(k)
+          }
+          return lvl(i) + 1
+        }
+      }
+    }
+  }
+  return lvl(cursorLine)
+}
+
+/** Measure text width, treating \t as tabSize spaces */
+function measureWithTabs(ctx: CanvasRenderingContext2D, text: string, tabWidth: number): number {
+  if (!text.includes('\t')) return ctx.measureText(text).width
+  const spaceW = ctx.measureText(' ').width
+  let w = 0
+  for (const ch of text) {
+    w += ch === '\t' ? spaceW * tabWidth : ctx.measureText(ch).width
+  }
+  return w
+}
+
+/** Draw text, treating \t as tabSize spaces */
+function fillTextWithTabs(
+  ctx: CanvasRenderingContext2D,
+  text: string,
+  x: number,
+  y: number,
+  tabWidth: number,
+): number {
+  if (!text.includes('\t')) {
+    ctx.fillText(text, x, y)
+    return x + ctx.measureText(text).width
+  }
+  const spaceW = ctx.measureText(' ').width
+  let xOff = x
+  for (const ch of text) {
+    if (ch === '\t') {
+      xOff += spaceW * tabWidth
+    } else {
+      ctx.fillText(ch, xOff, y)
+      xOff += ctx.measureText(ch).width
+    }
+  }
+  return xOff
 }
 
 export function renderCanvas({
@@ -34,6 +157,8 @@ export function renderCanvas({
   scrollTop,
   fontSize = DEFAULT_FONT_SIZE,
   fontFamily = DEFAULT_FONT_FAMILY,
+  tabSize = DEFAULT_TAB_SIZE,
+  tokenLines,
 }: RenderOptions): void {
   const ctx = canvas.getContext('2d')
   if (!ctx) return
@@ -55,6 +180,10 @@ export function renderCanvas({
 
   ctx.font = font
 
+  const spaceW = ctx.measureText(' ').width
+  const { rawLevels, effectiveLevels, indentUnit } = buildGuideData(lines, tabSize)
+  const activeGuideLevel = activeBracketLevel(lines, cursor.line, rawLevels, effectiveLevels)
+
   const hasSel = selection && !isCollapsed(selection)
   const [selStart, selEnd] = hasSel ? normalizeSelection(selection!) : [cursor, cursor]
 
@@ -68,16 +197,28 @@ export function renderCanvas({
       ctx.fillRect(0, y, w, lineHeight)
     }
 
-    // Selection highlight (drawn before text so text is readable on top)
+    // Indent guides — draw at every level strictly contained by this line's indent.
+    // Rule: guide g appears if rawLevel >= g (non-empty) or effectiveLevel >= g (empty).
+    // This means {-lines and }-lines at level g-1 are excluded naturally.
+    const rl = rawLevels[i]
+    const el = effectiveLevels[i] ?? 0
+    const maxG = rl === -1 ? el : rl
+    for (let g = 1; g <= maxG; g++) {
+      const gx = Math.floor(PADDING_LEFT + 4 + (g - 1) * indentUnit * spaceW)
+      ctx.fillStyle = g === activeGuideLevel ? INDENT_GUIDE_ACTIVE : INDENT_GUIDE
+      ctx.fillRect(gx, y, 1, lineHeight)
+    }
+
+    // Selection highlight
     if (hasSel && i >= selStart.line && i <= selEnd.line) {
       const colStart = i === selStart.line ? selStart.col : 0
       const colEnd = i === selEnd.line ? selEnd.col : lineText.length
 
       ctx.font = font
-      const xStart = PADDING_LEFT + 4 + ctx.measureText(lineText.slice(0, colStart)).width
+      const xStart = PADDING_LEFT + 4 + measureWithTabs(ctx, lineText.slice(0, colStart), tabSize)
       const xEnd =
         i === selEnd.line
-          ? PADDING_LEFT + 4 + ctx.measureText(lineText.slice(0, colEnd)).width
+          ? PADDING_LEFT + 4 + measureWithTabs(ctx, lineText.slice(0, colEnd), tabSize)
           : w
 
       ctx.fillStyle = SELECTION_BG
@@ -87,24 +228,32 @@ export function renderCanvas({
     // Gutter
     ctx.fillStyle = GUTTER_BG
     ctx.fillRect(0, y, PADDING_LEFT, lineHeight)
-    ctx.fillStyle = GUTTER_BORDER
-    ctx.fillRect(PADDING_LEFT - 1, y, 1, lineHeight)
 
     // Line number
     ctx.fillStyle = i === cursor.line ? FG : GUTTER_FG
     ctx.textAlign = 'right'
     ctx.textBaseline = 'top'
-    ctx.fillText(String(i + 1), PADDING_LEFT - 8, y + Math.floor((lineHeight - fontSize) / 2))
+    ctx.fillText(String(i + 1), PADDING_LEFT + 4 - 2 * spaceW, y + Math.floor((lineHeight - fontSize) / 2))
 
-    // Line text
-    ctx.fillStyle = FG
+    // Line text (syntax-highlighted or plain)
+    const tokenLine = tokenLines?.[i]
+    const textY = y + Math.floor((lineHeight - fontSize) / 2)
     ctx.textAlign = 'left'
-    ctx.fillText(lineText, PADDING_LEFT + 4, y + Math.floor((lineHeight - fontSize) / 2))
+    if (tokenLine && tokenLine.length > 0) {
+      let xOff = PADDING_LEFT + 4
+      for (const span of tokenLine) {
+        ctx.fillStyle = span.color
+        xOff = fillTextWithTabs(ctx, span.text, xOff, textY, tabSize)
+      }
+    } else {
+      ctx.fillStyle = FG
+      fillTextWithTabs(ctx, lineText, PADDING_LEFT + 4, textY, tabSize)
+    }
 
-    // Cursor (only when no active selection or selection is collapsed)
+    // Cursor
     if (i === cursor.line) {
       const textBefore = lineText.slice(0, cursor.col)
-      const cursorX = PADDING_LEFT + 4 + ctx.measureText(textBefore).width
+      const cursorX = PADDING_LEFT + 4 + measureWithTabs(ctx, textBefore, tabSize)
       ctx.fillStyle = CURSOR_COLOR
       ctx.fillRect(Math.floor(cursorX), y + 2, 2, lineHeight - 4)
     }
@@ -113,20 +262,21 @@ export function renderCanvas({
   ctx.restore()
 }
 
-/** Find the column index in `line` closest to pixel offset `targetX` */
+/** Find the column index in `line` closest to pixel offset `targetX`, tab-aware */
 export function colFromX(
   ctx: CanvasRenderingContext2D,
   line: string,
   targetX: number,
   fontSize: number,
   fontFamily: string,
+  tabSize: number = DEFAULT_TAB_SIZE,
 ): number {
   ctx.font = `${fontSize}px ${fontFamily}`
   let lo = 0
   let hi = line.length
   while (lo < hi) {
     const mid = (lo + hi + 1) >> 1
-    if (ctx.measureText(line.slice(0, mid)).width <= targetX) {
+    if (measureWithTabs(ctx, line.slice(0, mid), tabSize) <= targetX) {
       lo = mid
     } else {
       hi = mid - 1

package/src/core/tokenizer.ts

@@ -0,0 +1,62 @@
+import { createHighlighter, type Highlighter } from 'shiki'
+import { createJavaScriptRegexEngine } from 'shiki/engine/javascript'
+import type { BundledLanguage } from 'shiki'
+import type { TokenizedLine } from './renderer'
+
+const THEME = 'dark-plus'
+const DEFAULT_FG = '#d4d4d4'
+
+let _hlPromise: Promise<Highlighter> | null = null
+const loadedLangs = new Set<string>()
+
+const jsEngine = createJavaScriptRegexEngine()
+
+function getHighlighter(): Promise<Highlighter> {
+  if (!_hlPromise) {
+    _hlPromise = createHighlighter({
+      themes: [THEME],
+      langs: [],
+      engine: jsEngine,
+      // engine: createOnigurumaEngine(import('shiki/wasm')),
+    })
+  }
+  return _hlPromise
+}
+
+export async function tokenize(code: string, lang: string): Promise<TokenizedLine[]> {
+  const hl = await getHighlighter()
+  if (!loadedLangs.has(lang)) {
+    try {
+      await hl.loadLanguage(lang as unknown as BundledLanguage)
+      loadedLangs.add(lang)
+    } catch {
+      return []
+    }
+  }
+  try {
+    const result = hl.codeToTokens(code, { lang: lang as unknown as BundledLanguage, theme: THEME })
+    return result.tokens.map((line) =>
+      line.map((t) => ({ text: t.content, color: t.color ?? DEFAULT_FG })),
+    )
+  } catch {
+    return []
+  }
+}
+
+/** Map file extension → Shiki language id */
+export function extToLang(ext: string): string | undefined {
+  const map: Record<string, string> = {
+    ts: 'typescript', tsx: 'tsx', js: 'javascript', jsx: 'jsx',
+    py: 'python', rs: 'rust', go: 'go',
+    c: 'c', h: 'c', cpp: 'cpp', cc: 'cpp', cxx: 'cpp', cs: 'csharp',
+    java: 'java', kt: 'kotlin', swift: 'swift', rb: 'ruby', php: 'php',
+    lua: 'lua', r: 'r', dart: 'dart', scala: 'scala',
+    css: 'css', scss: 'scss', less: 'less',
+    html: 'html', htm: 'html', xml: 'xml', vue: 'vue', svelte: 'svelte',
+    json: 'json', jsonc: 'jsonc', yaml: 'yaml', yml: 'yaml', toml: 'toml',
+    md: 'markdown', mdx: 'mdx',
+    sh: 'bash', bash: 'bash', zsh: 'bash', fish: 'fish',
+    sql: 'sql', graphql: 'graphql',
+  }
+  return map[ext.toLowerCase()]
+}

package/src/index.ts

@@ -2,3 +2,5 @@ export { PretextEditor } from './react/PretextEditor'
 export type { PretextEditorProps } from './react/PretextEditor'
 export { fromString, toString, insert, deleteBackward, deleteForward, moveCursor, getSelectedText, deleteSelectedText, isCollapsed, normalizeSelection } from './core/document'
 export type { Doc, Cursor, Selection } from './core/document'
+export { extToLang } from './core/tokenizer'
+export type { TokenSpan, TokenizedLine } from './core/renderer'

package/src/react/PretextEditor.tsx

@@ -1,4 +1,7 @@
-import { useCallback, useEffect, useLayoutEffect, useRef, useState } from 'react'
+import { useCallback, useEffect, useLayoutEffect, useRef, useState, useDeferredValue } from 'react'
+import { useQuery } from '@tanstack/react-query'
+import { tokenize } from '../core/tokenizer'
+import type { TokenizedLine } from '../core/renderer'
 import {
   fromString,
   toString,
@@ -23,14 +26,17 @@ import {
   FONT_SIZE_TO_LINE_HEIGHT,
   DEFAULT_FONT_SIZE,
   DEFAULT_FONT_FAMILY,
+  DEFAULT_TAB_SIZE,
   colFromX,
 } from '../core/renderer'
 
 export interface PretextEditorProps {
   value: string
   onChange: (value: string) => void
+  language?: string
   fontSize?: number
   fontFamily?: string
+  tabSize?: number
   className?: string
   style?: React.CSSProperties
 }
@@ -40,8 +46,10 @@ type Snapshot = { doc: Doc; selAnchor: Cursor | null }
 export function PretextEditor({
   value,
   onChange,
+  language,
   fontSize = DEFAULT_FONT_SIZE,
   fontFamily = DEFAULT_FONT_FAMILY,
+  tabSize = DEFAULT_TAB_SIZE,
   className,
   style,
 }: PretextEditorProps): React.JSX.Element {
@@ -75,6 +83,19 @@ export function PretextEditor({
     setSelAnchor(null)
   }
 
+  // Syntax highlighting: tokenize with a deferred value so rapid typing doesn't queue many requests
+  const deferredValue = useDeferredValue(value)
+  const { data: tokenLines } = useQuery<TokenizedLine[]>({
+    queryKey: ['tokens', language, deferredValue],
+    queryFn: () => tokenize(deferredValue, language!),
+    enabled: !!language,
+    staleTime: Infinity,
+    gcTime: 60_000,
+    placeholderData: (prev) => prev,
+  })
+  const tokenLinesRef = useRef<TokenizedLine[] | undefined>(undefined)
+  tokenLinesRef.current = tokenLines
+
   const lineHeight = FONT_SIZE_TO_LINE_HEIGHT(fontSize)
   const totalHeight = Math.max(1, doc.lines.length) * lineHeight + PADDING_TOP * 2
 
@@ -126,13 +147,15 @@ export function PretextEditor({
       scrollTop: container.scrollTop,
       fontSize,
       fontFamily,
+      tabSize,
+      tokenLines: tokenLinesRef.current,
     })
-  }, [fontSize, fontFamily])
+  }, [fontSize, fontFamily, tabSize])
 
-  // Repaint after every state change
+  // Repaint after every state change or when syntax tokens arrive
   useLayoutEffect(() => {
     repaint()
-  }, [repaint, doc, selAnchor])
+  }, [repaint, doc, selAnchor, tokenLines])
 
   // REVIEW: useEffect used because DOM focus must run after mount
   useEffect(() => {
@@ -191,10 +214,10 @@ export function PretextEditor({
       )
       const textX = cssX - (PADDING_LEFT + 4)
       const col =
-        textX <= 0 ? 0 : colFromX(ctx, docRef.current.lines[line] ?? '', textX, fontSize, fontFamily)
+        textX <= 0 ? 0 : colFromX(ctx, docRef.current.lines[line] ?? '', textX, fontSize, fontFamily, tabSize)
       return { line, col }
     },
-    [lineHeight, fontSize, fontFamily],
+    [lineHeight, fontSize, fontFamily, tabSize],
   )
 
   // ---- Pointer events (click + drag selection) ----
@@ -236,7 +259,6 @@ export function PretextEditor({
   // ---- Keyboard ----
   const handleKeyDown = useCallback(
     (e: React.KeyboardEvent<HTMLTextAreaElement>) => {
-      console.log('[PretextEditor] keyDown:', e.key, 'composing=', isComposingRef.current)
       if (isComposingRef.current) return
       const ctrl = e.ctrlKey || e.metaKey
       const shift = e.shiftKey
@@ -438,7 +460,6 @@ export function PretextEditor({
       textareaRef.current = el
 
       const onBeforeInput = (e: InputEvent): void => {
-        console.log('[PretextEditor] beforeinput: data=', e.data, 'composing=', isComposingRef.current)
         if (isComposingRef.current || !e.data) return
         e.preventDefault()