sugar-high 2.5.0 → 2.5.1

package/README.md

@@ -31,28 +31,6 @@ import { highlight } from 'sugar-high'
 const html = highlight('const ready = true')
 ```
 
-## Experimental WebGPU highlighting
-
-Use the opt-in `sugar-high/gpu` entry for asynchronous, language-agnostic highlighting powered by
-[`gpu-lexer`](https://gpu-lexer.vercel.app). It requires WebGPU and rejects when WebGPU is
-unavailable; importing any other Sugar High entry does not load the GPU model.
-
-Install the experimental lexer separately when using this entry:
-
-```sh
-npm install sugar-high gpu-lexer
-```
-
-```js
-import { highlight, parse } from 'sugar-high/gpu'
-
-const parsed = await parse(source)
-const html = await highlight(source)
-```
-
-GPU labels map onto Sugar High's existing token classes, themes, `cx`, `mark`, and `markLine`
-hooks. The model infers syntax without a language option and may differ from grammar-based output.
-
 JavaScript, including JSX, is the default. Pass a canonical name for another built-in language:
 
 ```js
@@ -241,6 +219,28 @@ See [`docs/API.md`](https://github.com/huozhi/sugar-high/blob/main/docs/API.md)
 options, and lower-level functions. Upgrading from v1? Read the
 [v2 migration guide](https://github.com/huozhi/sugar-high/blob/main/docs/MIGRATION.md).
 
+## Experimental WebGPU highlighting
+
+Use the opt-in `sugar-high/gpu` entry for asynchronous, language-agnostic highlighting powered by
+[`gpu-lexer`](https://gpu-lexer.vercel.app). It requires WebGPU and rejects when WebGPU is
+unavailable; importing any other Sugar High entry does not load the GPU model.
+
+Install the experimental lexer separately when using this entry:
+
+```sh
+npm install sugar-high gpu-lexer
+```
+
+```js
+import { highlight, parse } from 'sugar-high/gpu'
+
+const parsed = await parse(source)
+const html = await highlight(source)
+```
+
+GPU labels map onto Sugar High's existing token classes, themes, `cx`, `mark`, and `markLine`
+hooks. The model infers syntax without a language option and may differ from grammar-based output.
+
 ## Benchmarks
 
 Sugar High, PrismJS, and highlight.js highlighting the same generated TypeScript files:

package/lib/lang/css.js

@@ -36,17 +36,23 @@ const isHyphen = ([type, value]) => type === T_SIGN && value === '-'
  * @param {Array<[number, string]>} tokens
  */
 const mergeDashedNames = (tokens) => {
+  /** @type {Array<[number, string]>} */
+  const merged = []
   for (let index = 0; index < tokens.length; index++) {
     let firstWord = index
     let end = index
 
     if (isHyphen(tokens[end])) {
       while (tokens[end] && isHyphen(tokens[end])) end++
-      if (!tokens[end] || !isNameStart(tokens[end])) continue
+      if (!tokens[end] || !isNameStart(tokens[end])) {
+        merged.push(tokens[index])
+        continue
+      }
       firstWord = end++
     } else if (isNameStart(tokens[end])) {
       end++
     } else {
+      merged.push(tokens[index])
       continue
     }
 
@@ -62,10 +68,15 @@ const mergeDashedNames = (tokens) => {
       end++
     }
 
-    if (!dashed) continue
+    if (!dashed) {
+      merged.push(tokens[index])
+      continue
+    }
     const name = tokens.slice(index, end).map(([, value]) => value).join('')
-    tokens.splice(index, end - index, [tokens[firstWord][0], name])
+    merged.push([tokens[firstWord][0], name])
+    index = end - 1
   }
+  return merged
 }
 
 /** Return true when a colon belongs to a nested selector instead of a declaration. */
@@ -91,29 +102,37 @@ const opensBlock = (tokens, start) => {
  * @param {import('../core.js').ParseOptions} options
  */
 export const tokenize = (code, options) => {
-  const tokens = tokenizePlain(code, { ...options, tokenize: undefined })
-  mergeDashedNames(tokens)
+  const tokens = mergeDashedNames(tokenizePlain(code, { ...options, tokenize: undefined }))
+  /** @type {Array<[number, string]>} */
+  const output = []
   let blockDepth = 0
   let declarationStart = false
 
   for (let index = 0; index < tokens.length; index++) {
-    const [type, value] = tokens[index]
+    const token = tokens[index]
+    const [type, value] = token
 
     if (type === T_SIGN && value === '{') {
       blockDepth++
       declarationStart = true
+      output.push(token)
       continue
     }
     if (type === T_SIGN && value === '}') {
       blockDepth--
       declarationStart = false
+      output.push(token)
       continue
     }
     if (type === T_SIGN && value === ';') {
       declarationStart = blockDepth > 0
+      output.push(token)
+      continue
+    }
+    if (!declarationStart || isIgnored(type)) {
+      output.push(token)
       continue
     }
-    if (!declarationStart || isIgnored(type)) continue
 
     const propertyStart = index
     let propertyEnd = index
@@ -133,10 +152,13 @@ export const tokenize = (code, options) => {
         .slice(propertyStart, propertyEnd)
         .map(([, part]) => part)
         .join('')
-      tokens.splice(propertyStart, propertyEnd - propertyStart, [T_PROPERTY, property])
+      output.push([T_PROPERTY, property])
+      index = propertyEnd - 1
+    } else {
+      output.push(token)
     }
     declarationStart = false
   }
 
-  return tokens
+  return output
 }

package/lib/lang/embedded-html.js

@@ -35,6 +35,8 @@ function findTagEnd(code, start) {
 /** Find real script/style blocks while ignoring comments and quoted attributes. */
 function findEmbeddedBlocks(code) {
   const blocks = []
+  /** @type {Set<string>} */
+  const unclosed = new Set()
   let index = 0
   while (index < code.length) {
     if (code.startsWith('<!--', index)) {
@@ -57,10 +59,15 @@ function findEmbeddedBlocks(code) {
     }
 
     const tag = match[1].toLowerCase()
+    if (unclosed.has(tag)) {
+      index = openEnd
+      continue
+    }
     const closing = embeddedClosingTag(tag)
     closing.lastIndex = openEnd
     const closeMatch = closing.exec(code)
     if (!closeMatch) {
+      unclosed.add(tag)
       index = openEnd
       continue
     }
@@ -77,22 +84,29 @@ function findEmbeddedBlocks(code) {
  * @returns {Array<[number, string]>}
  */
 export function tokenizeEmbeddedHtml(code) {
+  const blocks = findEmbeddedBlocks(code)
+  if (!blocks.length) return tokenizeHtml(code)
+
   /** @type {Array<[number, string]>} */
   const tokens = []
+  /** @param {Array<[number, string]>} part */
+  const append = (part) => {
+    for (let index = 0; index < part.length; index++) tokens.push(part[index])
+  }
   let cursor = 0
 
-  for (const block of findEmbeddedBlocks(code)) {
+  for (const block of blocks) {
     const { start, openEnd, closeStart, end, tag } = block
 
-    tokens.push(...tokenizeHtml(code.slice(cursor, start)))
-    tokens.push(...tokenizeHtml(code.slice(start, openEnd)))
-    tokens.push(...(tag === 'style'
+    append(tokenizeHtml(code.slice(cursor, start)))
+    append(tokenizeHtml(code.slice(start, openEnd)))
+    append(tag === 'style'
       ? tokenizeCss(code.slice(openEnd, closeStart))
-      : tokenizeJavaScript(code.slice(openEnd, closeStart), { jsx: false })))
-    tokens.push(...tokenizeHtml(code.slice(closeStart, end)))
+      : tokenizeJavaScript(code.slice(openEnd, closeStart), { jsx: false }))
+    append(tokenizeHtml(code.slice(closeStart, end)))
     cursor = end
   }
 
-  tokens.push(...tokenizeHtml(code.slice(cursor)))
+  append(tokenizeHtml(code.slice(cursor)))
   return tokens
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sugar-high",
-  "version": "2.5.0",
+  "version": "2.5.1",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/huozhi/sugar-high.git",