sugar-high 0.7.4 → 0.8.0

package/lib/index.d.ts

@@ -1,3 +1,14 @@
-export function highlight(code: string): string
-export function tokenize(code: string): Array<[number, string]>
+type HighlightOptions = {
+  keywords?: Set<string>
+  onCommentStart?: (curr: string, next: string) => boolean
+}
+
+export function highlight(code: string, options?: HighlightOptions): string
+export function tokenize(code: string, options?: HighlightOptions): Array<[number, string]>
 export function generate(tokens: Array<[number, string]>): Array<any>
+export const SugarHigh: {
+  TokenTypes: {
+    [key: number]: string
+  }
+  TokenMap: Map<string, number>
+}

package/lib/index.js

@@ -1,7 +1,7 @@
 // @ts-check
 
-const jsxBrackets = new Set(['<', '>', '{', '}', '[', ']'])
-const keywords = new Set([
+const JSXBrackets = new Set(['<', '>', '{', '}', '[', ']'])
+const Keywords_Js = new Set([
   'for',
   'do',
   'while',
@@ -45,7 +45,7 @@ const keywords = new Set([
   'static',
 ])
 
-const signs = new Set([
+const Signs = new Set([
   '+',
   '-',
   '*',
@@ -73,9 +73,13 @@ const signs = new Set([
   '#',
   '@',
   '\\',
-  ...jsxBrackets,
+  ...JSXBrackets,
 ])
 
+const DefaultOptions = {
+  keywords: Keywords_Js,
+  onCommentStart: isCommentStart_Js,
+}
 
 /**
  *
@@ -92,7 +96,7 @@ const signs = new Set([
  * 10 - space
  *
  */
-const types = /** @type {const} */ ([
+const TokenTypes = /** @type {const} */ ([
   'identifier',
   'keyword',
   'string',
@@ -117,14 +121,14 @@ const [
   T_COMMENT,
   T_BREAK,
   T_SPACE,
-] = /** @types {const} */ types.map((_, i) => i)
+] = /** @types {const} */ TokenTypes.map((_, i) => i)
 
 function isSpaces(str) {
   return /^[^\S\r\n]+$/g.test(str)
 }
 
 function isSign(ch) {
-  return signs.has(ch)
+  return Signs.has(ch)
 }
 
 function encode(str) {
@@ -175,20 +179,26 @@ function isStringQuotation(chr) {
   return isSingleQuotes(chr) || isStrTemplateChr(chr)
 }
 
-function isCommentStart(str) {
-  str = str.slice(0, 2)
+function isCommentStart_Js(curr, next) {
+  const str = curr + next
   return str === '//' || str === '/*'
 }
 
 function isRegexStart(str) {
-  return str[0] === '/' && !isCommentStart(str[0] + str[1])
+  return str[0] === '/' && !isCommentStart_Js(str[0], str[1])
 }
 
 /**
  * @param {string} code
+ * @param {{ keywords: Set<string> }} options
  * @return {Array<[number, string]>}
  */
-function tokenize(code) {
+function tokenize(code, options) {
+  const {
+    keywords,
+    onCommentStart,
+  } = { ...DefaultOptions, ...options }
+  
   let current = ''
   let type = -1
   /** @type {[number, string]} */
@@ -220,9 +230,11 @@ function tokenize(code) {
 
   /** @type {string | null} */
   let __strQuote = null
+  let __regexQuoteStart = false
   let __strTemplateExprStack = 0
   let __strTemplateQuoteStack = 0
   const inStringQuotes = () => __strQuote !== null
+  const inRegexQuotes = () => __regexQuoteStart
   const inStrTemplateLiterals = () => (__strTemplateQuoteStack > __strTemplateExprStack)
   const inStrTemplateExpr = () => __strTemplateQuoteStack > 0 && (__strTemplateQuoteStack === __strTemplateExprStack)
   const inStringContent = () => inStringQuotes() || inStrTemplateLiterals()
@@ -275,12 +287,20 @@ function tokenize(code) {
     }
   }
 
-  const append = (_type, _token) => {
-    if (_token) {
-      current = _token
+  /**
+   * 
+   * @param {number} type_ 
+   * @param {string} token_ 
+   */
+  const append = (type_, token_) => {
+    if (type_ || token_) {
+      const nType = TokenTypes[type_]
+    }
+    if (token_) {
+      current = token_
     }
     if (current) {
-      type = _type || classify(current)
+      type = type_ || classify(current)
       /** @type [number, string]  */
       const pair = [type, current]
       if (type !== T_SPACE && type !== T_BREAK) {
@@ -456,7 +476,13 @@ function tokenize(code) {
 
       // current and next char can form a jsx open or close tag
       if (curr === '<' && (next === '/' || isAlpha(next))) {
-        __jsxEnter = true
+        if (
+          !inStringContent() && 
+          !inJsxLiterals() &&
+          !inRegexQuotes()
+        ) {
+          __jsxEnter = true
+        }
       }
     }
 
@@ -487,6 +513,7 @@ function tokenize(code) {
         continue
       }
 
+      __regexQuoteStart = true
       const start = i++
 
       // end of line of end of file
@@ -494,17 +521,20 @@ function tokenize(code) {
       const isEol = () => isEof() || code[i] === '\n'
 
       let foundClose = false
-      // regex
+      
+      // traverse to find closing regex slash
       for (; !isEol(); i++) {
         if (code[i] === '/' && code[i - 1] !== '\\') {
           foundClose = true
-          // append regex flags
+          // end of regex, append regex flags
           while (start !== i && /^[a-z]$/.test(code[i + 1]) && !isEol()) {
             i++
           }
           break
         }
       }
+      __regexQuoteStart = false
+
       if (start !== i && foundClose) {
         // If current line is fully closed with string quotes or regex slashes,
         // add them to tokens
@@ -516,7 +546,7 @@ function tokenize(code) {
         append()
         i = start
       }
-    } else if (isCommentStart(c_n)) {
+    } else if (onCommentStart(curr, next)) {
       append()
       const start = i
       if (next === '/') {
@@ -551,9 +581,9 @@ function tokenize(code) {
         __jsxExpr = false
       } else if (
         // it's jsx literals and is not a jsx bracket
-        (isJsxLiterals && !jsxBrackets.has(curr)) ||
+        (isJsxLiterals && !JSXBrackets.has(curr)) ||
         // same type char as previous one in current token
-        ((isWord(curr) === isWord(current[current.length - 1]) || __jsxChild()) && !signs.has(curr))
+        ((isWord(curr) === isWord(current[current.length - 1]) || __jsxChild()) && !Signs.has(curr))
       ) {
         current += curr
       } else {
@@ -571,7 +601,7 @@ function tokenize(code) {
           append()
           i++
         }
-        else if (jsxBrackets.has(curr)) append()
+        else if (JSXBrackets.has(curr)) append()
       }
     }
   }
@@ -618,8 +648,8 @@ function generate(tokens) {
               value: value, // to encode
             }],
             properties: {
-              className: `sh__token--${types[type]}`,
-              style: `color: var(--sh-${types[type]})`,
+              className: `sh__token--${TokenTypes[type]}`,
+              style: `color: var(--sh-${TokenTypes[type]})`,
             },
           }
         ))
@@ -680,10 +710,11 @@ function toHtml(lines) {
 /**
  *
  * @param {string} code
+ * @param {{ keywords: Set<string> } | undefined} options
  * @returns {string}
  */
-function highlight(code) {
-  const tokens = tokenize(code)
+function highlight(code, options) {
+  const tokens = tokenize(code, options)
   const lines = generate(tokens)
   const output = toHtml(lines)
   return output
@@ -691,8 +722,8 @@ function highlight(code) {
 
 // namespace
 const SugarHigh = /** @type {const} */ {
-  TokenTypes: types,
-  TokenMap: new Map(types.map((type, i) => [type, i])),
+  TokenTypes,
+  TokenMap: new Map(TokenTypes.map((type, i) => [type, i])),
 }
 
 export {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sugar-high",
-  "version": "0.7.4",
+  "version": "0.8.0",
   "type": "module",
   "types": "./lib/index.d.ts",
   "exports": {