sugar-high 1.3.0 → 2.0.0

package/README.md

@@ -3,7 +3,9 @@
 [![version][npm-version-badge]][npm]
 [![downloads][npm-downloads-badge]][npm]
 
-Super lightweight syntax highlighter for JavaScript and JSX—about **1 kB** minified and gzipped. Works in the browser or any JS runtime that can set HTML strings.
+Lightweight, zero-dependency syntax highlighting for JavaScript, popular programming languages,
+and formats commonly generated by coding agents. It runs in browsers and JavaScript runtimes and
+returns HTML without requiring a DOM.
 
 ![Sugar High preview](https://repository-images.githubusercontent.com/453236442/9aa2144a-3a4c-4a93-a87f-92ca6a37ded6)
 
@@ -13,47 +15,128 @@ Super lightweight syntax highlighter for JavaScript and JSX—about **1 kB** min
 npm install sugar-high
 ```
 
-## Usage
+## Highlight code
 
 ```js
 import { highlight } from 'sugar-high'
 
-const codeHTML = highlight(code)
+const html = highlight('const ready = true')
+```
+
+JavaScript, including JSX, is the default. Pass a canonical name for another built-in language:
 
-document.querySelector('pre > code').innerHTML = codeHTML
+```js
+highlight('print("hi")', { lang: 'python' })
+highlight('{"ready": true}', { lang: 'json' })
+highlight('+ added', { lang: 'diff' })
 ```
 
-### Language presets
+The `lang` option is typed and accepts canonical names only.
 
-The core highlighter targets JavaScript and JSX. For CSS, Rust, Python, and similar, import a preset from [`sugar-high/presets`](https://github.com/huozhi/sugar-high/tree/main/packages/sugar-high/lib/presets) and pass it into `highlight`:
+## Advanced: normalize extensions and aliases
+
+You do not need `lang()` when the language is already known. Use it only when input comes from a
+filename extension, Markdown fence, or another integration. It converts aliases to the canonical
+name expected by `highlight`:
 
 ```js
-import { highlight } from 'sugar-high'
-import { rust } from 'sugar-high/presets'
+import { lang } from 'sugar-high/lang'
 
-const html = highlight(source, { ...rust })
+lang('py')    // 'python'
+lang('bash')  // 'shell'
+lang('jsonc') // 'json'
+lang('.yml')  // 'yaml'
 ```
 
-Available presets: `c`, `css`, `diff`, `go`, `java`, `json`, `python`, and `rust`.
+If you already have a canonical name, pass it directly—`lang()` is not required.
+
+See [the API reference](../../docs/API.md#lang-mapping) for the complete mapping and normalization
+behavior.
 
-The `json` preset highlights quoted object keys as `property` tokens, keeping them distinct from string values:
+## Built-in languages
+
+`javascript`, `typescript`, `css`, `python`, `c`, `go`, `java`, `rust`, `json`, `diff`, `shell`,
+`cpp`, `csharp`, `sql`, `html`, `yaml`, `markdown`, `kotlin`, `swift`, `php`, `toml`, `powershell`,
+`dockerfile`, `graphql`, and `hcl`.
+
+Related dialects share one implementation: JavaScript includes JSX, TypeScript includes TSX, JSON
+includes JSONC comments, Shell includes sh/Bash/Zsh, and HCL includes Terraform.
+
+## Composable core
+
+Use `sugar-high/core` to separate configurable syntax parsing from HTML rendering. It does not
+include the built-in language registry:
 
 ```js
-import { json } from 'sugar-high/presets'
+import { parse, render } from 'sugar-high/core'
+
+const parsed = parse('select * from users', {
+  keywords: new Set(['select', 'from', 'where']),
+})
 
-const html = highlight('{"name": "Alice"}', json)
+const html = render(parsed, {
+  cx: { keyword: 'font-bold' },
+})
 ```
 
-For more language presets and syntax color themes, see **[sugar-high.vercel.app](https://sugar-high.vercel.app/)**.
+Use the default `sugar-high` export when you want built-in languages and one-step `highlight()`.
 
-## Styling
+## Customize tokens
+
+Use `cx` for a class map. It works well with utility CSS, CSS Modules, and global styles while
+preserving Sugar High's semantic token classes.
+
+```js
+highlight(source, {
+  lang: 'typescript',
+  cx: {
+    keyword: 'font-bold',
+    comment: 'italic opacity-60',
+  },
+})
+```
+
+Use `mark(token)` for conditional classes, inline styles, or custom attributes. It mutates the
+token and returns nothing:
+
+```js
+highlight(source, {
+  mark(token) {
+    if (token.type === 'comment' && token.value.includes('TODO')) {
+      token.className += ' text-orange-500'
+      token.properties['data-todo'] = true
+    }
+  },
+})
+```
+
+`cx` runs before `mark`, so `mark` receives the composed class name.
+
+## Highlight lines
+
+Use `markLine` to customize generated lines. Its index is zero-based:
 
-Each line is wrapped in `sh__line`. Set **CSS custom properties** `--sh-*` on an ancestor (for example `:root`) to pick colors—inspect the output or the example below for the variable names you need.
+```js
+highlight(source, {
+  markLine(line) {
+    if (line.index === 1) {
+      line.className += ' sh__line--highlighted'
+    }
+  },
+})
+```
+
+Style the class with CSS. The React package also provides the one-based
+`highlightLines={[1, [4, 7]]}` prop, while the Remark plugin reads ranges from fence metadata such
+as `{2,5-7}`.
 
-Example theme:
+## Styling
+
+Each line uses `.sh__line`. Token colors use CSS custom properties, so a theme can be embedded in
+your own stylesheet or scoped to any ancestor:
 
 ```css
-:root {
+.code {
   --sh-class: #2d5e9d;
   --sh-identifier: #354150;
   --sh-sign: #8996a3;
@@ -66,50 +149,52 @@ Example theme:
 }
 ```
 
-### Line numbers
-
-Use a `::before` counter on `.sh__line` for gutter numbers:
+Lines can be styled or numbered with ordinary CSS:
 
 ```css
 pre code {
-  counter-reset: sh-line-number;
+  counter-reset: line;
 }
 
-.sh__line::before {
-  counter-increment: sh-line-number 1;
-  content: counter(sh-line-number);
-  margin-right: 24px;
-  text-align: right;
-  color: #a4a4a4;
-}
-```
-
-### Highlighting a line
-
-Target a line with `.sh__line:nth-child(<n>)` (1-based):
-
-```css
 .sh__line {
   display: block;
-  padding: 0 12px;
-  margin: 0 -12px;
-  min-height: 1rem;
 }
 
-.sh__line:nth-child(5) {
-  background: #fff8c5;
+.sh__line::before {
+  counter-increment: line;
+  content: counter(line);
+  margin-right: 1.5rem;
+  color: #a4a4a4;
 }
 
+.sh__line:nth-child(5),
 .sh__line--highlighted {
   background: #fff8c5;
 }
 ```
 
-Use `.sh__line--highlighted` when you add highlight classes yourself (for example with `lineClassName`).
+## React
+
+[`@sugar-high/react`](https://sugar-high.vercel.app/react) provides a highlighted `<Code />` block
+and textarea-overlay `<Editor />` as separate composable exports.
+
+```tsx
+import { Code, Editor } from '@sugar-high/react'
+
+<Editor lang="typescript" value={source} onChange={setSource} />
+<Code lang="typescript" lineNumbers>{source}</Code>
+```
 
 ## Remark
 
-Use the [remark plugin](https://sugar-high.vercel.app/remark) to highlight fenced code blocks when processing Markdown with [remark](https://remark.js.org/). Details: [`packages/remark-sugar-high`](https://github.com/huozhi/sugar-high/tree/main/packages/remark-sugar-high).
+[`@sugar-high/remark`](https://sugar-high.vercel.app/remark) highlights fenced code blocks while
+processing Markdown. Fence aliases are normalized through the same `lang()` mapping.
+
+## API
+
+See [`docs/API.md`](https://github.com/huozhi/sugar-high/blob/main/docs/API.md) for package exports, the full language mapping, highlighting
+options, and lower-level functions. Upgrading from v1? Read the
+[v2 migration guide](https://github.com/huozhi/sugar-high/blob/main/docs/MIGRATION.md).
 
 ## License
 

package/lib/core.d.ts

@@ -0,0 +1,117 @@
+export type TokenType =
+  | 'identifier'
+  | 'keyword'
+  | 'string'
+  | 'class'
+  | 'property'
+  | 'entity'
+  | 'jsxliterals'
+  | 'sign'
+  | 'comment'
+  | 'break'
+  | 'space'
+
+export type ParsedToken = {
+  type: TokenType
+  value: string
+}
+
+export type MarkToken = {
+  type: TokenType
+  value: string
+  className: string
+  style: Record<string, string | number>
+  properties: Record<string, string | number | boolean>
+}
+
+export type ParsedLine = {
+  readonly index: number
+  readonly value: string
+  readonly tokens: readonly ParsedToken[]
+  readonly annotations: readonly string[]
+}
+
+export type AnnotateLine = {
+  readonly index: number
+  readonly value: string
+  readonly tokens: readonly ParsedToken[]
+  annotations: string[]
+}
+
+export type MarkLine = {
+  readonly index: number
+  readonly value: string
+  readonly tokens: readonly ParsedToken[]
+  readonly annotations: readonly string[]
+  className: string
+  style: Record<string, string | number>
+  properties: Record<string, string | number | boolean>
+}
+
+export type ParsedCode = {
+  readonly value: string
+  readonly lines: readonly ParsedLine[]
+}
+
+export type GeneratedProperties = {
+  className: string
+  style: Record<string, string | number>
+  [name: string]: unknown
+}
+
+export type GeneratedText = {
+  type: 'text'
+  value: string
+}
+
+export type GeneratedToken = {
+  type: 'element'
+  tokenType: TokenType
+  tagName: 'span'
+  children: GeneratedText[]
+  properties: GeneratedProperties
+}
+
+export type GeneratedLine = {
+  type: 'element'
+  tagName: 'span'
+  children: GeneratedToken[]
+  properties: GeneratedProperties
+}
+
+export type DisplayOptions = {
+  cx?: Partial<Record<TokenType, string>>
+  mark?: (token: MarkToken) => void
+  markLine?: (line: MarkLine) => void
+}
+
+export type ParseOptions = {
+  keywords?: Set<string>
+  typeKeywords?: Set<string>
+  onCommentStart?: (curr: string, next: string, index: number, code: string) => number | boolean
+  onCommentEnd?: (prev: string, curr: string, index: number, code: string) => number | boolean
+  onLiteral?: (curr: string, index: number, code: string) => number | null | undefined
+  onQuote?: (curr: string, index: number, code: string) => number | null | undefined
+  quotedKeys?: boolean
+  jsx?: boolean
+  regex?: boolean
+  templateStrings?: boolean
+  caseInsensitive?: boolean
+  typescript?: boolean
+  tokenize?: (code: string, options: ParseOptions) => Array<[number, string]>
+  /** Apply syntax-specific semantic annotations while parsing. */
+  annotateLine?: (line: AnnotateLine) => void
+}
+
+export function parse(code: string, options?: ParseOptions): ParsedCode
+export function render(parsed: ParsedCode, options?: DisplayOptions): string
+
+/** Low-level token API used by language presets and integrations. */
+export function tokenize(code: string, options?: ParseOptions): Array<[number, string]>
+/** Build renderable line nodes for HTML, React, and syntax-tree integrations. */
+export function generate(parsed: ParsedCode, options?: DisplayOptions): GeneratedLine[]
+
+export const SugarHigh: {
+  TokenTypes: { [key: number]: string }
+  TokenMap: Map<string, number>
+}

package/lib/core.js

@@ -0,0 +1,202 @@
+// @ts-check
+
+import {
+  assemble, generate, SugarHigh, toHtml, T_BREAK, T_CLASS, T_COMMENT, T_IDENTIFIER,
+  T_KEYWORD, T_PROPERTY, T_SIGN, T_SPACE, T_STRING,
+} from './shared.js'
+
+const signs = new Set('+-*/%=!&|^~?:.,;()[]{}<>#@\\'.split(''))
+const noComment = () => 0
+
+/** @param {string} value */
+const isWord = (value) => value === '_' || value === '$' || /[\p{L}\p{N}]/u.test(value)
+
+/** @param {string} code @param {number} index */
+function isQuotedKey(code, index) {
+  while (index < code.length && /\s/.test(code[index])) index++
+  return code[index] === ':'
+}
+
+/**
+ * General-purpose lexer for keyword, string, comment, and punctuation based languages.
+ * Complex language presets may supply `tokenize` to replace this stage.
+ * @param {string} code
+ * @param {HighlightOptions | undefined} options
+ * @returns {Array<[number, string]>}
+ */
+function tokenize(code, options) {
+  if (typeof options?.tokenize === 'function') return options.tokenize(code, options)
+
+  const keywords = options?.keywords || new Set()
+  const typeKeywords = options?.typeKeywords || new Set()
+  const onCommentStart = options?.onCommentStart || noComment
+  const onCommentEnd = options?.onCommentEnd || noComment
+  const normalize = options?.caseInsensitive
+    ? (value) => value.toLowerCase()
+    : (value) => value
+  /** @type {Array<[number, string]>} */
+  const tokens = []
+  let lastSignificant = ''
+
+  /** @param {number} type @param {string} value */
+  function append(type, value) {
+    if (!value) return
+    tokens.push([type, value])
+    if (type !== T_SPACE && type !== T_BREAK) lastSignificant = value
+  }
+
+  for (let i = 0; i < code.length;) {
+    const curr = code[i]
+    const next = code[i + 1]
+
+    const commentType = onCommentStart(curr, next, i, code)
+    if (commentType) {
+      const start = i++
+      while (i < code.length) {
+        if (onCommentEnd(code[i - 1], code[i], i, code) == commentType) {
+          i++
+          break
+        }
+        i++
+      }
+      append(T_COMMENT, code.slice(start, i))
+      continue
+    }
+
+    const literalLength = options?.onLiteral?.(curr, i, code)
+    if (literalLength) {
+      append(T_STRING, code.slice(i, i + literalLength))
+      i += literalLength
+      continue
+    }
+
+    if (typeof options?.onQuote === 'function' && curr === "'") {
+      const length = options.onQuote(curr, i, code)
+      if (typeof length === 'number' && length >= 1) {
+        append(T_IDENTIFIER, code.slice(i, i + length))
+        i += length
+        continue
+      }
+    }
+
+    if (curr === '"' || curr === "'" || (options?.templateStrings && curr === '`')) {
+      const quote = curr
+      const start = i++
+      while (i < code.length) {
+        if (code[i] === quote && code[i - 1] !== '\\') {
+          i++
+          break
+        }
+        i++
+      }
+      const value = code.slice(start, i)
+      append(options?.quotedKeys && isQuotedKey(code, i) ? T_PROPERTY : T_STRING, value)
+      continue
+    }
+
+    if (curr === '\n') {
+      append(T_BREAK, curr)
+      i++
+      continue
+    }
+
+    if (/[^\S\r\n]/.test(curr)) {
+      const start = i++
+      while (i < code.length && /[^\S\r\n]/.test(code[i])) i++
+      append(T_SPACE, code.slice(start, i))
+      continue
+    }
+
+    if (isWord(curr)) {
+      const start = i++
+      while (i < code.length && isWord(code[i])) i++
+      const value = code.slice(start, i)
+      const normalized = normalize(value)
+      const type = typeKeywords.has(normalized)
+        ? T_CLASS
+        : keywords.has(normalized)
+          ? T_KEYWORD
+          : lastSignificant === '.'
+            ? T_PROPERTY
+            : (/^\d/.test(value) || value === 'null' || /^\p{Lu}/u.test(value))
+              ? T_CLASS
+              : T_IDENTIFIER
+      append(type, value)
+      continue
+    }
+
+    if (signs.has(curr)) {
+      append(T_SIGN, curr)
+      i++
+      continue
+    }
+
+    append(T_STRING, curr)
+    i++
+  }
+
+  return tokens
+}
+
+/** @param {string} code @param {ParseOptions | undefined} options */
+function parse(code, options) {
+  const parsed = assemble(code, tokenize(code, options))
+  if (options?.annotateLine) {
+    for (const line of parsed.lines) options.annotateLine(line)
+  }
+  return parsed
+}
+
+/** @param {ParsedCode} parsed @param {DisplayOptions | undefined} options */
+function render(parsed, options) {
+  return toHtml(generate(parsed, options))
+}
+
+export { generate, parse, render, SugarHigh, tokenize }
+
+/**
+ * @typedef {Object} ParseOptions
+ * @property {Set<string>} [keywords]
+ * @property {Set<string>} [typeKeywords]
+ * @property {(curr: string, next: string, index: number, code: string) => number | boolean} [onCommentStart]
+ * @property {(prev: string, curr: string, index: number, code: string) => number | boolean} [onCommentEnd]
+ * @property {(curr: string, index: number, code: string) => number | null | undefined} [onLiteral]
+ * @property {(curr: string, index: number, code: string) => number | null | undefined} [onQuote]
+ * @property {boolean} [quotedKeys]
+ * @property {boolean} [caseInsensitive]
+ * @property {boolean} [templateStrings]
+ * @property {(code: string, options: ParseOptions) => Array<[number, string]>} [tokenize]
+ * @property {(line: AnnotateLine) => void} [annotateLine]
+ */
+
+/**
+ * @typedef {Object} DisplayOptions
+ * @property {Partial<Record<TokenType, string>>} [cx]
+ * @property {(token: MarkToken) => void} [mark]
+ * @property {(line: MarkLine) => void} [markLine]
+ */
+
+/**
+ * @typedef {'identifier' | 'keyword' | 'string' | 'class' | 'property' | 'entity' | 'jsxliterals' | 'sign' | 'comment' | 'break' | 'space'} TokenType
+ * @typedef {{
+ *   type: TokenType
+ *   value: string
+ *   className: string
+ *   style: Record<string, string | number>
+ *   properties: Record<string, string | number | boolean>
+ * }} MarkToken
+ * @typedef {{ type: TokenType, value: string }} ParsedToken
+ * @typedef {{
+ *   index: number
+ *   value: string
+ *   tokens: ParsedToken[]
+ *   annotations: string[]
+ * }} ParsedLine
+ * @typedef {ParsedLine} AnnotateLine
+ * @typedef {ParsedLine & {
+ *   className: string
+ *   style: Record<string, string | number>
+ *   properties: Record<string, string | number | boolean>
+ * }} MarkLine
+ * @typedef {{ value: string, lines: ParsedLine[] }} ParsedCode
+ */

package/lib/index.d.ts

@@ -1,27 +1,42 @@
-type HighlightOptions = {
-  keywords?: Set<string>
-  /**
-   * Highlighted as the `class` token type (e.g. built-in types). Checked before `keywords`.
-   */
-  typeKeywords?: Set<string>
-  onCommentStart?: (curr: string, next: string) => number | boolean
-  onCommentEnd?: (curr: string, prev: string) => number | boolean
-  /**
-   * At `code[i] === "'"`: return how many code units to consume from `i` as one token,
-   * or null/undefined or a number below 1 for default JS single-quoted string rules.
-   */
-  onQuote?: (curr: string, i: number, code: string) => number | null | undefined
-  /** Highlight quoted object keys followed by `:` as `property` tokens. */
-  quotedKeys?: boolean
-  lineClassName?: (line: string, index: number) => string | null | undefined
+import type {
+  DisplayOptions,
+  MarkLine,
+  MarkToken,
+  TokenType,
+} from './core.js'
+
+export type { DisplayOptions, MarkLine, MarkToken, TokenType }
+
+export type LanguageName =
+  | 'javascript'
+  | 'typescript'
+  | 'css'
+  | 'python'
+  | 'c'
+  | 'go'
+  | 'java'
+  | 'rust'
+  | 'json'
+  | 'diff'
+  | 'shell'
+  | 'cpp'
+  | 'csharp'
+  | 'sql'
+  | 'html'
+  | 'yaml'
+  | 'markdown'
+  | 'kotlin'
+  | 'swift'
+  | 'php'
+  | 'toml'
+  | 'powershell'
+  | 'dockerfile'
+  | 'graphql'
+  | 'hcl'
+
+export type HighlightOptions = DisplayOptions & {
+  /** Canonical language name. Fence and extension aliases must be normalized first. */
+  lang?: LanguageName
 }
 
 export function highlight(code: string, options?: HighlightOptions): string
-export function tokenize(code: string, options?: HighlightOptions): Array<[number, string]>
-export function generate(tokens: Array<[number, string]>, options?: Pick<HighlightOptions, 'lineClassName'>): Array<any>
-export const SugarHigh: {
-  TokenTypes: {
-    [key: number]: string
-  }
-  TokenMap: Map<string, number>
-}