sugar-high 2.1.0 → 2.2.1

package/README.md

@@ -56,8 +56,8 @@ behavior.
 ## Built-in languages
 
 `javascript`, `typescript`, `css`, `python`, `c`, `go`, `java`, `rust`, `json`, `diff`, `shell`,
-`cpp`, `csharp`, `sql`, `html`, `yaml`, `markdown`, `kotlin`, `swift`, `php`, `toml`, `powershell`,
-`dockerfile`, `graphql`, and `hcl`.
+`cpp`, `csharp`, `sql`, `html`, `yaml`, `markdown`, `plaintext`, `ruby`, `kotlin`, `swift`, `php`,
+`toml`, `powershell`, `dockerfile`, `graphql`, and `hcl`.
 
 Related dialects share one implementation: JavaScript includes JSX, TypeScript includes TSX, JSON
 includes JSONC comments, Shell includes sh/Bash/Zsh, and HCL includes Terraform.

package/lib/core.js

@@ -1,7 +1,7 @@
 // @ts-check
 
 import {
-  assemble, generate, SugarHigh, toHtml, T_BREAK, T_CLASS, T_COMMENT, T_IDENTIFIER,
+  assemble, generate, render, SugarHigh, T_BREAK, T_CLASS, T_COMMENT, T_IDENTIFIER,
   T_KEYWORD, T_PROPERTY, T_SIGN, T_SPACE, T_STRING,
 } from './shared.js'
 
@@ -147,11 +147,6 @@ function parse(code, options) {
   return parsed
 }
 
-/** @param {ParsedCode} parsed @param {DisplayOptions | undefined} options */
-function render(parsed, options) {
-  return toHtml(generate(parsed, options))
-}
-
 export { generate, parse, render, SugarHigh, tokenize }
 
 /**

package/lib/index.d.ts

@@ -25,6 +25,8 @@ export type LanguageName =
   | 'html'
   | 'yaml'
   | 'markdown'
+  | 'plaintext'
+  | 'ruby'
   | 'kotlin'
   | 'swift'
   | 'php'

package/lib/lang/plaintext.d.ts

@@ -0,0 +1 @@
+export function tokenize(code: string): Array<[number, string]>;

package/lib/lang/plaintext.js

@@ -0,0 +1,5 @@
+// @ts-check
+import { T_IDENTIFIER } from '../shared.js'
+
+/** @param {string} code */
+export const tokenize = (code) => [[T_IDENTIFIER, code]]

package/lib/lang/ruby.d.ts

@@ -0,0 +1,13 @@
+export const keywords: Set<string>;
+export function onCommentStart(
+  curr: string,
+  next: string,
+  index: number,
+  code: string,
+): 0 | 1 | 2;
+export function onCommentEnd(
+  prev: string,
+  curr: string,
+  index: number,
+  code: string,
+): 0 | 1 | 2;

package/lib/lang/ruby.js

@@ -0,0 +1,41 @@
+// @ts-check
+
+export const keywords = new Set([
+  'BEGIN', 'END', '__ENCODING__', '__FILE__', '__LINE__', 'alias', 'and', 'begin', 'break',
+  'case', 'class', 'def', 'defined', 'do', 'else', 'elsif', 'end', 'ensure', 'false', 'for', 'if',
+  'in', 'module', 'next', 'nil', 'not', 'or', 'redo', 'rescue', 'retry', 'return', 'self', 'super',
+  'then', 'true', 'undef', 'unless', 'until', 'when', 'while', 'yield',
+])
+
+/** @param {number} index @param {string} code */
+function isBlockCommentStart(index, code) {
+  if (index > 0 && code[index - 1] !== '\n') return false
+  return /^=begin(?:\s|$)/.test(code.slice(index))
+}
+
+/**
+ * @param {string} curr
+ * @param {string} _next
+ * @param {number} index
+ * @param {string} code
+ */
+export function onCommentStart(curr, _next, index, code) {
+  if (curr === '#') return 1
+  if (curr === '=' && isBlockCommentStart(index, code)) return 2
+  return 0
+}
+
+/**
+ * @param {string} _prev
+ * @param {string} curr
+ * @param {number} index
+ * @param {string} code
+ */
+export function onCommentEnd(_prev, curr, index, code) {
+  if (curr !== '\n') return 0
+
+  const lineStart = code.lastIndexOf('\n', index - 1) + 1
+  const line = code.slice(lineStart, index)
+  if (/^=end(?:\s|$)/.test(line)) return 2
+  return 1
+}

package/lib/lang.js

@@ -16,8 +16,10 @@ import * as javascript from './lang/javascript.js'
 import * as kotlin from './lang/kotlin.js'
 import * as markdown from './lang/markdown.js'
 import * as php from './lang/php.js'
+import * as plaintext from './lang/plaintext.js'
 import * as powershell from './lang/powershell.js'
 import * as python from './lang/python.js'
+import * as ruby from './lang/ruby.js'
 import * as rust from './lang/rust.js'
 import * as shell from './lang/shell.js'
 import * as sql from './lang/sql.js'
@@ -69,6 +71,8 @@ const languages = [
   { id: 'html', extension: 'html', aliases: ['htm', 'xml'], config: html },
   { id: 'yaml', extension: 'yaml', aliases: ['yml'], config: nonJavaScript(yaml) },
   { id: 'markdown', extension: 'md', aliases: ['md', 'mdx'], config: nonJavaScript(markdown) },
+  { id: 'plaintext', extension: 'txt', aliases: ['text', 'plain'], config: nonJavaScript(plaintext) },
+  { id: 'ruby', extension: 'rb', aliases: [], config: nonJavaScript(ruby) },
   { id: 'kotlin', extension: 'kt', aliases: ['kts'], config: nonJavaScript(kotlin) },
   { id: 'swift', extension: 'swift', aliases: [], config: nonJavaScript(swift) },
   { id: 'php', extension: 'php', aliases: [], config: nonJavaScript(php) },

package/lib/shared.js

@@ -64,6 +64,42 @@ function assemble(value, tokens) {
   return { value, lines }
 }
 
+/**
+ * @param {import('./core.js').ParsedLine} parsedLine
+ * @param {import('./core.js').DisplayOptions['markLine']} markLine
+ */
+function createLine(parsedLine, markLine) {
+  const line = {
+    index: parsedLine.index,
+    value: parsedLine.value,
+    tokens: parsedLine.tokens,
+    annotations: parsedLine.annotations,
+    className: `sh__line${parsedLine.annotations.map(annotation => ` sh__line--${annotation}`).join('')}`,
+    style: {},
+    properties: {},
+  }
+  markLine?.(line)
+  return line
+}
+
+/**
+ * @param {import('./core.js').ParsedToken} parsedToken
+ * @param {import('./core.js').DisplayOptions['cx']} cx
+ * @param {import('./core.js').DisplayOptions['mark']} mark
+ */
+function createToken({ type, value }, cx, mark) {
+  const extraClassName = cx?.[type]
+  const token = {
+    type,
+    value,
+    className: `sh__token--${type}${extraClassName ? ` ${extraClassName}` : ''}`,
+    style: { color: `var(--sh-${type})` },
+    properties: {},
+  }
+  mark?.(token)
+  return token
+}
+
 /**
  * @param {import('./core.js').ParsedCode} parsed
  * @param {import('./core.js').DisplayOptions | undefined} options
@@ -74,30 +110,13 @@ function generate(parsed, options) {
   const markLine = options?.markLine
 
   return parsed.lines.map((parsedLine) => {
-    const line = {
-      index: parsedLine.index,
-      value: parsedLine.value,
-      tokens: parsedLine.tokens,
-      annotations: parsedLine.annotations,
-      className: `sh__line${parsedLine.annotations.map(annotation => ` sh__line--${annotation}`).join('')}`,
-      style: {},
-      properties: {},
-    }
-    markLine?.(line)
+    const line = createLine(parsedLine, markLine)
 
     return {
       type: 'element',
       tagName: 'span',
-      children: parsedLine.tokens.map(({ type, value }) => {
-        const extraClassName = cx?.[type]
-        const token = {
-          type,
-          value,
-          className: `sh__token--${type}${extraClassName ? ` ${extraClassName}` : ''}`,
-          style: { color: `var(--sh-${type})` },
-          properties: {},
-        }
-        mark?.(token)
+      children: parsedLine.tokens.map((parsedToken) => {
+        const token = createToken(parsedToken, cx, mark)
         return {
           type: 'element',
           tokenType: token.type,
@@ -119,6 +138,46 @@ function generate(parsed, options) {
   })
 }
 
+/**
+ * Render parsed lines directly while preserving generate()'s markup and mutable display hooks.
+ * @param {import('./core.js').ParsedCode} parsed
+ * @param {import('./core.js').DisplayOptions | undefined} options
+ */
+function render(parsed, options) {
+  const cx = options?.cx
+  const mark = options?.mark
+  const markLine = options?.markLine
+
+  if (!cx && !mark && !markLine) {
+    return parsed.lines.map((line) => {
+      const className = `sh__line${line.annotations.map(annotation => ` sh__line--${annotation}`).join('')}`
+      const children = line.tokens.map(({ type, value }) => (
+        `<span class="sh__token--${type}" style="color:var(--sh-${type})">${encode(value)}</span>`
+      )).join('')
+      return `<span class="${encode(className)}">${children}</span>`
+    }).join('\n')
+  }
+
+  return parsed.lines.map((parsedLine) => {
+    const line = createLine(parsedLine, markLine)
+
+    const children = parsedLine.tokens.map((parsedToken) => {
+      const token = createToken(parsedToken, cx, mark)
+      return `<span ${attributes({
+        ...token.properties,
+        className: token.className,
+        style: token.style,
+      })}>${encode(token.value)}</span>`
+    }).join('')
+
+    return `<span ${attributes({
+      ...line.properties,
+      className: line.className,
+      style: line.style,
+    })}>${children}</span>`
+  }).join('\n')
+}
+
 const entities = { '&': '&amp;', '<': '&lt;', '>': '&gt;', '"': '&quot;', "'": '&#039;' }
 /** @param {string} value */
 const encode = (value) => value.replace(/[&<>"']/g, character => entities[character])
@@ -133,17 +192,7 @@ function attributes(values) {
   return `class="${encode(values.className || '')}"${style ? ` style="${encode(style)}"` : ''}${properties ? ` ${properties}` : ''}`
 }
 
-/** @param {Array<any>} lines */
-function toHtml(lines) {
-  return lines.map(line => {
-    const children = line.children.map(token => {
-      return `<${token.tagName} ${attributes(token.properties)}>${encode(token.children[0].value)}</${token.tagName}>`
-    }).join('')
-    return `<${line.tagName} ${attributes(line.properties)}>${children}</${line.tagName}>`
-  }).join('\n')
-}
-
 export {
-  assemble, encode, generate, SugarHigh, toHtml, TokenTypes, T_BREAK, T_CLASS, T_COMMENT, T_ENTITY, T_IDENTIFIER,
+  assemble, encode, generate, render, SugarHigh, TokenTypes, T_BREAK, T_CLASS, T_COMMENT, T_ENTITY, T_IDENTIFIER,
   T_JSX_LITERALS, T_KEYWORD, T_PROPERTY, T_SIGN, T_SPACE, T_STRING,
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sugar-high",
-  "version": "2.1.0",
+  "version": "2.2.1",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/huozhi/sugar-high.git",
@@ -40,6 +40,7 @@
   "scripts": {
     "test": "vitest",
     "build": "echo 'package requires no build'",
-    "benchmark": "node scripts/benchmark.mjs"
+    "benchmark": "node scripts/benchmark.mjs",
+    "benchmark:large": "node scripts/benchmark-large.mjs"
   }
 }