sugar-high 2.3.1 → 2.4.1

package/README.md

@@ -3,9 +3,9 @@
 [![version][npm-version-badge]][npm]
 [![downloads][npm-downloads-badge]][npm]
 
-Lightweight, zero-dependency syntax highlighting for JavaScript, popular programming languages,
-and formats commonly generated by coding agents. It runs in browsers and JavaScript runtimes and
-returns HTML without requiring a DOM.
+Lightweight syntax highlighting for JavaScript, popular programming languages, and formats
+commonly generated by coding agents. It runs in browsers and JavaScript runtimes and returns HTML
+without requiring a DOM.
 
 ![Sugar High preview](https://repository-images.githubusercontent.com/453236442/9aa2144a-3a4c-4a93-a87f-92ca6a37ded6)
 
@@ -41,6 +41,14 @@ See [benchmark methodology and options](../../docs/BENCHMARK.md).
 npm install sugar-high
 ```
 
+## Agent skill
+
+Install Sugar High guidance for an AI coding agent with the Skills CLI:
+
+```sh
+npx skills add huozhi/sugar-high --skill sugar-high
+```
+
 ## Highlight code
 
 ```js
@@ -49,6 +57,28 @@ import { highlight } from 'sugar-high'
 const html = highlight('const ready = true')
 ```
 
+## Experimental WebGPU highlighting
+
+Use the opt-in `sugar-high/gpu` entry for asynchronous, language-agnostic highlighting powered by
+[`gpu-lexer`](https://gpu-lexer.vercel.app). It requires WebGPU and rejects when WebGPU is
+unavailable; importing any other Sugar High entry does not load the GPU model.
+
+Install the experimental lexer separately when using this entry:
+
+```sh
+npm install sugar-high gpu-lexer
+```
+
+```js
+import { highlight, parse } from 'sugar-high/gpu'
+
+const parsed = await parse(source)
+const html = await highlight(source)
+```
+
+GPU labels map onto Sugar High's existing token classes, themes, `cx`, `mark`, and `markLine`
+hooks. The model infers syntax without a language option and may differ from grammar-based output.
+
 JavaScript, including JSX, is the default. Pass a canonical name for another built-in language:
 
 ```js

package/lib/core.js

@@ -83,7 +83,9 @@ function tokenize(code, options) {
       const quote = curr
       const start = i++
       while (i < code.length) {
-        if (code[i] === quote && code[i - 1] !== '\\') {
+        if (code[i] === '\\') {
+          i++
+        } else if (code[i] === quote) {
           i++
           break
         }

package/lib/gpu.d.ts

@@ -0,0 +1,7 @@
+import type { DisplayOptions, ParsedCode } from './core.js'
+
+/** Parse source code asynchronously with gpu-lexer and return Sugar High's structured format. */
+export function parse(code: string): Promise<ParsedCode>
+
+/** Highlight source code asynchronously with WebGPU. */
+export function highlight(code: string, options?: DisplayOptions): Promise<string>

package/lib/gpu.js

@@ -0,0 +1,85 @@
+// @ts-check
+
+import { parse as parseWithGpu } from 'gpu-lexer'
+import {
+  assemble,
+  render,
+  T_CLASS,
+  T_COMMENT,
+  T_ENTITY,
+  T_IDENTIFIER,
+  T_KEYWORD,
+  T_SIGN,
+  T_SPACE,
+  T_STRING,
+  T_BREAK,
+} from './shared.js'
+
+const tokenTypes = Object.freeze({
+  plain: T_IDENTIFIER,
+  comment: T_COMMENT,
+  string: T_STRING,
+  number: T_CLASS,
+  keyword: T_KEYWORD,
+  type: T_CLASS,
+  function: T_ENTITY,
+  constant: T_CLASS,
+  operator: T_SIGN,
+})
+
+/**
+ * Preserve plain whitespace as Sugar High space and break tokens while mapping
+ * the GPU lexer's semantic labels onto the existing theme vocabulary.
+ * @param {Array<[number, string]>} tokens
+ * @param {string} value
+ */
+function appendPlain(tokens, value) {
+  for (const part of value.match(/\r\n|\r|\n|[^\S\r\n]+|[^\s\r\n]+/g) || []) {
+    const type = part === '\n' || part === '\r' || part === '\r\n'
+      ? T_BREAK
+      : /^\s/.test(part)
+        ? T_SPACE
+        : T_IDENTIFIER
+    tokens.push([type, part])
+  }
+}
+
+/**
+ * Parse source code with gpu-lexer and return Sugar High's structured format.
+ * GPU labels are adapted to the existing token and theme types.
+ * @param {string} code
+ * @returns {Promise<import('./core.js').ParsedCode>}
+ */
+async function parse(code) {
+  const spans = await parseWithGpu(code)
+  /** @type {Array<[number, string]>} */
+  const tokens = []
+  let cursor = 0
+
+  for (const span of spans) {
+    const start = Math.max(cursor, Math.min(code.length, span.start))
+    const end = Math.max(start, Math.min(code.length, span.end))
+    if (start > cursor) appendPlain(tokens, code.slice(cursor, start))
+    cursor = start
+    if (end === start) continue
+
+    const value = code.slice(start, end)
+    if (span.type === 'plain') appendPlain(tokens, value)
+    else tokens.push([tokenTypes[span.type] ?? T_IDENTIFIER, value])
+    cursor = end
+  }
+
+  if (cursor < code.length) appendPlain(tokens, code.slice(cursor))
+  return assemble(code, tokens)
+}
+
+/**
+ * Highlight source code asynchronously with WebGPU.
+ * @param {string} code
+ * @param {import('./core.js').DisplayOptions | undefined} options
+ */
+async function highlight(code, options) {
+  return render(await parse(code), options)
+}
+
+export { highlight, parse }

package/lib/presets/javascript-runtime.js

@@ -168,18 +168,11 @@ function isSign(ch) {
 }
 
 function isWord(chr) {
-  return /^[\w_]+$/.test(chr) || hasUnicode(chr)
+  return /^[\w$\u0080-\uffff]+$/.test(chr || '')
 }
 
 function isCls(str) {
-  const chr0 = str[0]
-  return isWord(chr0) &&
-    chr0 === chr0.toUpperCase() ||
-    str === 'null'
-}
-
-function hasUnicode(s) {
-  return /[^\u0000-\u007f]/.test(s);
+  return /^[0-9A-Z\p{Lu}]/u.test(str) || str === 'null'
 }
 
 function isAlpha(chr) {
@@ -187,11 +180,11 @@ function isAlpha(chr) {
 }
 
 function isIdentifierChar(chr) {
-  return isAlpha(chr) || hasUnicode(chr)
+  return /^[$_A-Za-z\u0080-\uffff]$/.test(chr || '')
 }
 
 function isIdentifier(str) {
-  return isIdentifierChar(str[0]) && (str.length === 1 || isWord(str.slice(1)))
+  return isIdentifierChar(str[0]) && isWord(str)
 }
 
 function isStrTemplateChr(chr) {
@@ -282,17 +275,17 @@ function tokenize(code, options) {
   let __jsxEnter = false
   /** @type {0 | 1 | 2} 0 = none; 1 = inside `<open`; 2 = inside `</close` */
   let __jsxTag = 0
-  let __jsxExpr = false
+  let __jsxExprDepth = 0
   let __jsxTagExpr = 0
 
   /** Nested `<open>…</open>` depth (content between tags, including nested elements). */
   let __jsxStack = 0
 
-  const __jsxChild = () => __jsxEnter && !__jsxExpr && !__jsxTag
+  const __jsxChild = () => __jsxEnter && !__jsxExprDepth && !__jsxTag
   // < __content__ >
-  const inJsxTag = () => __jsxTag && !__jsxChild()
+  const inJsxTag = () => __jsxTag
   // {'__content__'}
-  const inJsxLiterals = () => !__jsxTag && __jsxChild() && !__jsxExpr && __jsxStack > 0
+  const inJsxLiterals = () => __jsxChild() && __jsxStack
 
   /** @type {string | null} */
   let __strQuote = null
@@ -419,14 +412,12 @@ function tokenize(code, options) {
     if (isSingleQuotes(curr) && !inJsxLiterals() && !inStrTemplateLiterals()) {
       append()
       let isStringClose = false
-      if (prev !== `\\`) {
-        if (__strQuote && curr === __strQuote) {
-          __strQuote = null
-          isStringClose = true
-        } else if (!__strQuote) {
-          __strQuote = curr
-          __strTokenStart = tokens.length
-        }
+      if (__strQuote && curr === __strQuote) {
+        __strQuote = null
+        isStringClose = true
+      } else if (!__strQuote) {
+        __strQuote = curr
+        __strTokenStart = tokens.length
       }
 
       append(T_STRING, curr)
@@ -438,25 +429,14 @@ function tokenize(code, options) {
       continue
     }
 
-    if (!inStrTemplateLiterals()) {
-      if (prev !== '\\n' && isTemplateQuote(curr)) {
-        append()
-        append(T_STRING, curr)
-        __strTemplateQuoteStack++
-        continue
-      }
+    if (isTemplateQuote(curr)) {
+      append()
+      __strTemplateQuoteStack += inStrTemplateLiterals() ? -1 : 1
+      append(T_STRING, curr)
+      continue
     }
 
     if (inStrTemplateLiterals()) {
-      if (prev !== '\\n' && isTemplateQuote(curr)) {
-        if (__strTemplateQuoteStack > 0) {
-          append()
-          __strTemplateQuoteStack--
-          append(T_STRING, curr)
-          continue
-        }
-      }
-
       if (c_n === '${') {
         __strTemplateExprStack++
         append(T_STRING)
@@ -477,7 +457,7 @@ function tokenize(code, options) {
       if (curr === '{') {
         append()
         append(T_SIGN, curr)
-        __jsxExpr = true
+        __jsxExprDepth = 1
         continue
       }
     }
@@ -634,6 +614,8 @@ function tokenize(code, options) {
     // string quotation
     if (isQuotationChar || isStringTemplateLiterals || isSingleQuotes(__strQuote)) {
       current += curr
+      // Consume the escaped character before the next delimiter check.
+      if (curr === '\\') current += code[++i] || ''
     } else if (isRegexChar) {
       append()
       const [lastType, lastToken] = last
@@ -656,25 +638,25 @@ function tokenize(code, options) {
       __regexQuoteStart = true
       const start = i++
 
-      // end of line of end of file
-      const isEof = () => i >= code.length
-      const isEol = () => isEof() || code[i] === '\n'
-
       let foundClose = false
 
       // `/` is literal inside regex character classes, e.g. `[/]`.
       let inCharClass = false
 
       // traverse to find closing regex slash
-      for (; !isEol(); i++) {
+      for (; i < code.length && code[i] !== '\n'; i++) {
         const ch = code[i]
-        const escaped = code[i - 1] === '\\'
-        if (!escaped && ch === '[') inCharClass = true
-        if (!escaped && ch === ']') inCharClass = false
-        if (ch === '/' && !inCharClass && !escaped) {
+        if (ch === '\\') {
+          if (code[i + 1] === '\n') break
+          i++
+          continue
+        }
+        if (ch === '[') inCharClass = true
+        if (ch === ']') inCharClass = false
+        if (ch === '/' && !inCharClass) {
           foundClose = true
           // end of regex, append regex flags
-          while (start !== i && /^[a-z]$/.test(code[i + 1]) && !isEol()) {
+          while (/^[a-z]$/.test(code[i + 1])) {
             i++
           }
           break
@@ -730,11 +712,11 @@ function tokenize(code, options) {
         append()
       }
     } else {
-      if (__jsxExpr && curr === '}') {
+      if (__jsxExprDepth && curr === '}') {
         append()
         current = curr
         append()
-        __jsxExpr = false
+        __jsxExprDepth--
       } else if (
         // it's jsx literals and is not a jsx bracket
         (isJsxLiterals && !JSXBrackets.has(curr)) ||
@@ -761,6 +743,7 @@ function tokenize(code, options) {
         }
         else if (JSXBrackets.has(curr)) append()
       }
+      if (__jsxExprDepth && curr === '{') __jsxExprDepth++
     }
   }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sugar-high",
-  "version": "2.3.1",
+  "version": "2.4.1",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/huozhi/sugar-high.git",
@@ -18,6 +18,10 @@
       "types": "./lib/core.d.ts",
       "default": "./lib/core.js"
     },
+    "./gpu": {
+      "types": "./lib/gpu.d.ts",
+      "default": "./lib/gpu.js"
+    },
     "./lang": {
       "types": "./lib/lang.d.ts",
       "default": "./lib/lang.js"
@@ -34,9 +38,18 @@
   "license": "MIT",
   "devDependencies": {
     "@types/node": "22.12.0",
+    "gpu-lexer": "0.0.2",
     "typescript": "6.0.2",
     "vitest": "^3.0.2"
   },
+  "peerDependencies": {
+    "gpu-lexer": ">=0.0.2"
+  },
+  "peerDependenciesMeta": {
+    "gpu-lexer": {
+      "optional": true
+    }
+  },
   "scripts": {
     "test": "vitest",
     "build": "echo 'package requires no build'",