sugar-high 2.2.2 → 2.3.0

package/README.md

@@ -57,7 +57,7 @@ behavior.
 
 `javascript`, `typescript`, `css`, `python`, `c`, `go`, `java`, `rust`, `json`, `diff`, `shell`,
 `cpp`, `csharp`, `sql`, `html`, `yaml`, `markdown`, `plaintext`, `ruby`, `kotlin`, `swift`, `php`,
-`toml`, `powershell`, `dockerfile`, `graphql`, and `hcl`.
+`toml`, `powershell`, `dockerfile`, `graphql`, `hcl`, `zig`, and `lua`.
 
 Related dialects share one implementation: JavaScript includes JSX, TypeScript includes TSX, JSON
 includes JSONC comments, Shell includes sh/Bash/Zsh, and HCL includes Terraform.

package/lib/core.d.ts

@@ -89,7 +89,13 @@ export type ParseOptions = {
   keywords?: Set<string>
   typeKeywords?: Set<string>
   onCommentStart?: (curr: string, next: string, index: number, code: string) => number | boolean
-  onCommentEnd?: (prev: string, curr: string, index: number, code: string) => number | boolean
+  onCommentEnd?: (
+    prev: string,
+    curr: string,
+    index: number,
+    code: string,
+    start: number,
+  ) => number | boolean
   onLiteral?: (curr: string, index: number, code: string) => number | null | undefined
   onQuote?: (curr: string, index: number, code: string) => number | null | undefined
   quotedKeys?: boolean

package/lib/core.js

@@ -53,7 +53,7 @@ function tokenize(code, options) {
     if (commentType) {
       const start = i++
       while (i < code.length) {
-        if (onCommentEnd(code[i - 1], code[i], i, code) == commentType) {
+        if (onCommentEnd(code[i - 1], code[i], i, code, start) == commentType) {
           i++
           break
         }
@@ -154,7 +154,7 @@ export { generate, parse, render, SugarHigh, tokenize }
  * @property {Set<string>} [keywords]
  * @property {Set<string>} [typeKeywords]
  * @property {(curr: string, next: string, index: number, code: string) => number | boolean} [onCommentStart]
- * @property {(prev: string, curr: string, index: number, code: string) => number | boolean} [onCommentEnd]
+ * @property {(prev: string, curr: string, index: number, code: string, start: number) => number | boolean} [onCommentEnd]
  * @property {(curr: string, index: number, code: string) => number | null | undefined} [onLiteral]
  * @property {(curr: string, index: number, code: string) => number | null | undefined} [onQuote]
  * @property {boolean} [quotedKeys]

package/lib/index.d.ts

@@ -35,6 +35,8 @@ export type LanguageName =
   | 'dockerfile'
   | 'graphql'
   | 'hcl'
+  | 'zig'
+  | 'lua'
 
 export type HighlightOptions = DisplayOptions & {
   /** Canonical language name. Fence and extension aliases must be normalized first. */

package/lib/lang/lua.d.ts

@@ -0,0 +1,10 @@
+export const keywords: Set<string>;
+export function onCommentStart(curr: string, next: string, index: number, code: string): number;
+export function onCommentEnd(
+  prev: string,
+  curr: string,
+  index: number,
+  code: string,
+  start: number,
+): number;
+export function onLiteral(curr: string, index: number, code: string): number;

package/lib/lang/lua.js

@@ -0,0 +1,45 @@
+// @ts-check
+
+export const keywords = new Set([
+  'and', 'break', 'do', 'else', 'elseif', 'end', 'false', 'for', 'function', 'goto', 'if', 'in',
+  'local', 'nil', 'not', 'or', 'repeat', 'return', 'then', 'true', 'until', 'while',
+])
+
+/**
+ * @param {string} curr
+ * @param {string} next
+ * @param {number} index
+ * @param {string} code
+ */
+export function onCommentStart(curr, next, index, code) {
+  if (curr === '#' && index === 0 && next === '!') return 1
+  if (curr + next !== '--') return 0
+  return /^--\[=*\[/.test(code.slice(index)) ? 2 : 1
+}
+
+/**
+ * @param {string} _prev
+ * @param {string} curr
+ * @param {number} index
+ * @param {string} code
+ * @param {number} start
+ */
+export function onCommentEnd(_prev, curr, index, code, start) {
+  if (curr === '\n') return 1
+  if (curr !== ']') return 0
+  const opener = code.slice(start).match(/^--\[(=*)\[/)
+  if (!opener) return 0
+  const closing = `]${opener[1]}]`
+  if (code.slice(index - closing.length + 1, index + 1) === closing) return 2
+  return 0
+}
+
+/** @param {string} curr @param {number} index @param {string} code */
+export function onLiteral(curr, index, code) {
+  if (curr !== '[') return 0
+  const opener = code.slice(index).match(/^\[(=*)\[/)
+  if (!opener) return 0
+  const closing = `]${opener[1]}]`
+  const end = code.indexOf(closing, index + opener[0].length)
+  return end === -1 ? code.length - index : end + closing.length - index
+}

package/lib/lang/markdown.d.ts

@@ -1,4 +1,5 @@
 export const keywords: Set<never>;
+export function tokenize(code: string): Array<[number, string]>;
 export function annotateLine(line: any): void;
 import { onCommentEnd } from '../presets/plain-base.js';
 import { onCommentStart } from '../presets/plain-base.js';

package/lib/lang/markdown.js

@@ -1,8 +1,91 @@
 // @ts-check
+import { T_IDENTIFIER, T_SIGN } from '../shared.js'
 import { onCommentEnd, onCommentStart } from '../presets/plain-base.js'
 
 export const keywords = new Set([])
 
+/** @param {string} code */
+export const tokenize = (code) => {
+  /** @type {Array<[number, string]>} */
+  const tokens = []
+  let fence = ''
+  let fenceQuoted = false
+
+  /** @param {number} type @param {string} value */
+  const append = (type, value) => {
+    if (!value) return
+    const previous = tokens[tokens.length - 1]
+    if (previous?.[0] === type) previous[1] += value
+    else tokens.push([type, value])
+  }
+
+  for (const part of code.match(/[^\n]*(?:\n|$)/g) || []) {
+    if (!part) continue
+    const newline = part.endsWith('\n') ? '\n' : ''
+    const line = newline ? part.slice(0, -1) : part
+    /** @type {Array<[number, number]>} */
+    const ranges = []
+    const container = line.match(/^(?: {0,3}>[ \t]?)*/)?.[0] || ''
+    if (!fence || fenceQuoted) {
+      for (const match of container.matchAll(/>/g)) {
+        ranges.push([match.index, match.index + 1])
+      }
+    }
+
+    let start = container.length
+    let spaces = 0
+    while (spaces < 3 && line[start] === ' ') {
+      start++
+      spaces++
+    }
+    const fenceMarker = line.slice(start).match(/^(`{3,}|~{3,})/)?.[1]
+
+    if (fence) {
+      if (
+        fenceMarker?.[0] === fence[0] &&
+        fenceMarker.length >= fence.length &&
+        /^\s*$/.test(line.slice(start + fenceMarker.length))
+      ) {
+        ranges.push([start, start + fenceMarker.length])
+        fence = ''
+        fenceQuoted = false
+      }
+    } else if (fenceMarker) {
+      ranges.push([start, start + fenceMarker.length])
+      fence = fenceMarker
+      fenceQuoted = container.includes('>')
+    } else {
+      const prefix = line.slice(start).match(
+        /^(#{1,6}(?=\s)|(?:[-+*]|\d+[.)])(?=\s)|(?:(?:\*\s*){3,}|(?:-\s*){3,}|(?:_\s*){3,})$)/,
+      )?.[1]
+      if (prefix) ranges.push([start, start + prefix.length])
+
+      for (const match of line.matchAll(/(`+)(.*?)\1|(\*{1,3}|_{1,3}|~{2})(?=\S)(.*?\S)\3/g)) {
+        const marker = match[1] || match[3]
+        if (
+          line[match.index - 1] === '\\' ||
+          (marker[0] === '_' && /\w/.test(line[match.index - 1] || ''))
+        ) continue
+        ranges.push(
+          [match.index, match.index + marker.length],
+          [match.index + match[0].length - marker.length, match.index + match[0].length],
+        )
+      }
+    }
+
+    let offset = 0
+    for (const [rangeStart, rangeEnd] of ranges.sort((a, b) => a[0] - b[0])) {
+      if (rangeStart < offset) continue
+      append(T_IDENTIFIER, line.slice(offset, rangeStart))
+      append(T_SIGN, line.slice(rangeStart, rangeEnd))
+      offset = rangeEnd
+    }
+    append(T_IDENTIFIER, line.slice(offset) + newline)
+  }
+
+  return tokens
+}
+
 export const annotateLine = (line) => {
   let annotation = ''
   if (/^#{1,6}\s/.test(line.value)) annotation = 'markdown-heading'

package/lib/lang/zig.d.ts

@@ -0,0 +1,5 @@
+export const keywords: Set<string>;
+export const typeKeywords: Set<string>;
+export function onCommentStart(curr: string, next: string): number;
+export function onCommentEnd(prev: string, curr: string): number;
+export function onLiteral(curr: string, index: number, code: string): number;

package/lib/lang/zig.js

@@ -0,0 +1,29 @@
+// @ts-check
+
+export const keywords = new Set([
+  'addrspace', 'align', 'allowzero', 'and', 'anyframe', 'anytype', 'asm', 'async', 'await', 'break',
+  'callconv', 'catch', 'comptime', 'const', 'continue', 'defer', 'else', 'enum', 'errdefer', 'error',
+  'export', 'extern', 'false', 'fn', 'for', 'if', 'inline', 'linksection', 'noalias', 'noinline',
+  'nosuspend', 'null', 'opaque', 'or', 'orelse', 'packed', 'pub', 'resume', 'return', 'struct',
+  'suspend', 'switch', 'test', 'threadlocal', 'true', 'try', 'undefined', 'union', 'unreachable',
+  'usingnamespace', 'var', 'volatile', 'while',
+])
+
+export const typeKeywords = new Set([
+  'anyerror', 'anyopaque', 'bool', 'c_char', 'c_int', 'c_long', 'c_longdouble', 'c_longlong',
+  'c_short', 'c_uint', 'c_ulong', 'c_ulonglong', 'c_ushort', 'comptime_float', 'comptime_int',
+  'f16', 'f32', 'f64', 'f80', 'f128', 'i8', 'i16', 'i32', 'i64', 'i128', 'isize', 'noreturn',
+  'type', 'u8', 'u16', 'u32', 'u64', 'u128', 'usize', 'void',
+])
+
+export const onCommentStart = (curr, next) => curr + next === '//' ? 1 : 0
+export const onCommentEnd = (_prev, curr) => curr === '\n' ? 1 : 0
+
+/** @param {string} curr @param {number} index @param {string} code */
+export function onLiteral(curr, index, code) {
+  if (curr !== '\\' || code[index + 1] !== '\\') return 0
+  const lineStart = code.lastIndexOf('\n', index - 1) + 1
+  if (code.slice(lineStart, index).trim()) return 0
+  const lineEnd = code.indexOf('\n', index + 2)
+  return (lineEnd === -1 ? code.length : lineEnd) - index
+}

package/lib/lang.js

@@ -2,8 +2,8 @@
 
 import {
   c, cpp, csharp, css, diff, dockerfile, go, graphql, hcl, html, java, javascript, json,
-  kotlin, markdown, nonJavaScript, php, plaintext, powershell, python, ruby, rust, shell, sql,
-  swift, toml, typescript, yaml,
+  kotlin, lua, markdown, nonJavaScript, php, plaintext, powershell, python, ruby, rust, shell, sql,
+  swift, toml, typescript, yaml, zig,
 } from './presets/configs.js'
 
 /**
@@ -45,6 +45,8 @@ const languages = [
   { id: 'dockerfile', extension: 'dockerfile', aliases: ['docker'], config: nonJavaScript(dockerfile) },
   { id: 'graphql', extension: 'graphql', aliases: ['gql'], config: nonJavaScript(graphql) },
   { id: 'hcl', extension: 'hcl', aliases: ['terraform', 'tf'], config: nonJavaScript(hcl) },
+  { id: 'zig', extension: 'zig', aliases: [], config: nonJavaScript(zig) },
+  { id: 'lua', extension: 'lua', aliases: [], config: nonJavaScript(lua) },
 ]
 
 /** @param {string} value */

package/lib/presets/clike-base.js

@@ -2,11 +2,11 @@
 export const onCommentStart = (currentChar, nextChar) => {
   const pair = currentChar + nextChar
   if (pair === '//') return 1
-  if (pair === '/*') return 1
+  if (pair === '/*') return 2
   return 0
 }
 
 export const onCommentEnd = (prevChar, currChar) => {
   if (currChar === '\n') return 1
-  return prevChar + currChar === '*/' ? 1 : 0
+  return prevChar + currChar === '*/' ? 2 : 0
 }

package/lib/presets/configs.js

@@ -14,6 +14,7 @@ import * as java from '../lang/java.js'
 import * as javascript from '../lang/javascript.js'
 import * as json from '../lang/json.js'
 import * as kotlin from '../lang/kotlin.js'
+import * as lua from '../lang/lua.js'
 import * as markdown from '../lang/markdown.js'
 import * as php from '../lang/php.js'
 import * as plaintext from '../lang/plaintext.js'
@@ -27,6 +28,7 @@ import * as swift from '../lang/swift.js'
 import * as toml from '../lang/toml.js'
 import * as typescript from '../lang/typescript.js'
 import * as yaml from '../lang/yaml.js'
+import * as zig from '../lang/zig.js'
 
 /**
  * Disable JavaScript-only scanner modes for a non-JavaScript language.
@@ -71,6 +73,8 @@ function createConfigs() {
     dockerfile: nonJavaScript(dockerfile),
     graphql: nonJavaScript(graphql),
     hcl: nonJavaScript(hcl),
+    zig: nonJavaScript(zig),
+    lua: nonJavaScript(lua),
   }
 }
 
@@ -83,6 +87,6 @@ function configFor(name) {
 
 export {
   c, configFor, configs, cpp, csharp, css, diff, dockerfile, go, graphql, hcl, html, java,
-  javascript, json, kotlin, markdown, nonJavaScript, php, plaintext, powershell, python, ruby,
-  rust, shell, sql, swift, toml, typescript, yaml,
+  javascript, json, kotlin, lua, markdown, nonJavaScript, php, plaintext, powershell, python, ruby,
+  rust, shell, sql, swift, toml, typescript, yaml, zig,
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sugar-high",
-  "version": "2.2.2",
+  "version": "2.3.0",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/huozhi/sugar-high.git",