sugar-high 0.4.0 → 0.4.3

package/lib/index.mjs

@@ -144,8 +144,16 @@ function isIdentifier(str) {
   return isIdentifierChar(str[0]) && (str.length === 1 || isWord(str.slice(1)))
 }
 
+function isStrTemplateChr(chr) {
+  return chr === '`'
+}
+
+function isSingleQuotes(chr) {
+  return chr === '"' || chr === "'"
+}
+
 function isStringQuotation(chr) {
-  return chr === '"' || chr === "'" || chr === '`'
+  return isSingleQuotes(chr) || isStrTemplateChr(chr)
 }
 
 function isCommentStart(str) {
@@ -168,15 +176,28 @@ export function tokenize(code) {
   /** @type {Array<[number, string]>} */
   const tokens = []
 
-  // jsx.tag for entering open or closed tags
-  // jsx.close for is the closing sign of open or close tags
-  // jsx.child for entering children
-  // jsx.expr for entering {expression}
-  /** @type {{ tag: boolean; close: boolean; child: boolean; expr: boolean }} */
-  const jsx = { tag: false, close: false, child: false, expr: false }
+  let __jsxEnter = false
+  /**
+   * @type {0 | 1 | 2}
+   * 0 for not in jsx
+   * 1 for open jsx tag
+   * 2 for closing jsx tag
+   **/
+  let __jsxTag = 0
+  let __jsxExpr = false
+
+  // only match paired (open + close) tags, not self-closing tags
+  let __jsxStack = 0
+  const __jsxChild = () => __jsxEnter && !__jsxExpr && !__jsxTag
+  const inJsxTag = () => __jsxTag && !__jsxChild()
+  const inJsxLiterals = () => !__jsxTag && __jsxChild() && !__jsxExpr && __jsxStack > 0
 
-  const inJsxTag = () => jsx.tag || jsx.close
-  const inJsxLiterals = () => !inJsxTag() && jsx.child && !jsx.expr
+  /** @type {string | 0} */
+  let __strQuote = 0
+  let __strTemplateExprStack = 0
+  let __strTemplateQuoteStack = 0
+  const inStrTemplateLiterals = () => (__strTemplateQuoteStack > __strTemplateExprStack)
+  const inStrTemplateExpr = () => __strTemplateQuoteStack > 0 && (__strTemplateQuoteStack === __strTemplateExprStack)
 
   /**
    *
@@ -184,8 +205,11 @@ export function tokenize(code) {
    * @returns {number}
    */
   function classify(token) {
-    if (keywords.has(token)) {
-      return T_KEYWORD
+    const isJsxLiterals = inJsxLiterals()
+    if (isJsxLiterals) {
+      return T_JSX_LITERALS
+    } else if (keywords.has(token)) {
+      return last[1] === '.' ? T_IDENTIFIER : T_KEYWORD
     } else if (token === '\n') {
       return T_BREAK
     } else if (isSpaces(token)) {
@@ -193,15 +217,16 @@ export function tokenize(code) {
     } else if (token.split('').every(ch => signs.has(ch))) {
       return T_SIGN
     } else if (isCls(token)) {
-      return inJsxLiterals() ? T_JSX_LITERALS : inJsxTag() ? T_IDENTIFIER : T_CLS_NUMBER
+      return inJsxTag() ? T_IDENTIFIER : T_CLS_NUMBER
     } else {
-      return inJsxLiterals()
-        ? T_JSX_LITERALS
-        : isIdentifier(token) ? T_IDENTIFIER : T_STRING
+      return isIdentifier(token) && !inStrTemplateLiterals() && (__strQuote !== `"` && __strQuote !== `'`) ? T_IDENTIFIER : T_STRING
     }
   }
 
-  const append = (_type) => {
+  const append = (_type, _token) => {
+    if (_token) {
+      current = _token
+    }
     if (current) {
       type = _type || classify(current)
       /** @type [number, string]  */
@@ -219,37 +244,149 @@ export function tokenize(code) {
     const next = code[i + 1]
     const p_c = prev + curr // previous and current
     const c_n = curr + next // current and next
-    const isJsxLiterals = inJsxLiterals()
 
-    if (jsx.close && (last[1] === '>' || last[1] === '/>')) {
-      jsx.close = false
+    if (isSingleQuotes(curr)) {
+      append()
+      if (prev !== `\\`) {
+        if (__strQuote && curr === __strQuote) {
+          __strQuote = 0
+        } else if (!__strQuote) {
+          __strQuote = curr
+        }
+      }
+
+      append(T_STRING, curr)
+      continue
+    }
+
+
+    if (!inStrTemplateLiterals()) {
+      if (prev !== '\\n' && curr === '`') {
+        append()
+        append(T_SIGN, curr)
+        __strTemplateQuoteStack++
+        continue
+      }
+    }
+
+    if (inStrTemplateLiterals()) {
+      if (prev !== '\\n' && curr === '`') {
+        if (__strTemplateQuoteStack > 0) {
+          append()
+          __strTemplateQuoteStack--
+          append(T_SIGN, curr)
+          continue
+        }
+      }
+
+
+      if (c_n === '${') {
+        __strTemplateExprStack++
+        append(T_STRING)
+        append(T_SIGN, c_n)
+        i++
+        continue
+      }
+    }
+
+    if (inStrTemplateExpr() && curr === '}') {
+      append()
+      __strTemplateExprStack--
+      append(T_SIGN, curr)
+      continue
+    }
+
+    if (__jsxChild()) {
+      if (curr === '{') {
+        append()
+        append(T_SIGN, curr)
+        __jsxExpr = true
+        continue
+      }
     }
-    if (jsx.tag) {
-      const isOpenElementEnd = curr === '>'
-      const isCloseElementEnd = p_c === '/>'
-      jsx.child = !isCloseElementEnd
-      jsx.tag = !(isOpenElementEnd || isCloseElementEnd)
-      jsx.close = isOpenElementEnd || isCloseElementEnd
+
+    if (__jsxEnter) {
+      // <: open tag sign
+      if (!__jsxTag && curr === '<') {
+        append()
+        __jsxTag = 1
+        current = curr
+        if (next === '/') {
+          __jsxTag = 2
+          current = c_n
+          i++
+        }
+        append(T_SIGN)
+        continue
+      }
+      if (__jsxTag) {
+        // >: open tag close sign or closing tag closing sign
+        if (curr === '>' && prev !== '/') {
+          append()
+          if (__jsxTag === 1) {
+            __jsxTag = 0
+            __jsxStack++
+          } else {
+            __jsxTag = 0
+            __jsxEnter = false
+          }
+          append(T_SIGN, curr)
+          continue
+        }
+
+        // >: tag self close sign or close tag sign
+        if (c_n === '/>' || c_n === '</') {
+          append()
+          if (c_n === '/>') {
+            __jsxTag = 0
+          }
+          if (c_n === '</') {
+            __jsxStack--
+          }
+          current = c_n
+          i++
+          append(T_SIGN)
+          continue
+        }
+
+        // <: open tag sign
+        if (curr === '<') {
+          append()
+          current = curr
+          append(T_SIGN)
+          continue
+        }
+      }
     }
+
     // if it's not in a jsx tag declaration or a string, close child if next is jsx close tag
-    if (!jsx.tag && (curr === '<' && isIdentifierChar(next) || c_n === '</')) {
-      jsx.tag = true
-      jsx.child = false
+    if (!__jsxTag && (curr === '<' && isIdentifierChar(next) || c_n === '</')) {
+      __jsxTag = next === '/' ? 2 : 1
+
+      if (curr === '<' && next !== '/') {
+        __jsxEnter = true
+      }
     }
 
     const isQuotationChar = isStringQuotation(curr)
-    const isRegexChar = !jsx.tag && isRegexStart(c_n)
-    if (isQuotationChar || isRegexChar) {
+    const isStringTemplateLiterals = inStrTemplateLiterals()
+    const isRegexChar = !__jsxEnter && isRegexStart(c_n)
+    const isJsxLiterals = inJsxLiterals()
+
+    // string quotation
+    if (isQuotationChar || isStringTemplateLiterals || isSingleQuotes(__strQuote)) {
+      current += curr
+    } else if (isRegexChar) {
       append()
       const [lastType, lastToken] = last
       // Special cases that are not considered as regex:
-      // * (expr1) / expr2: `()` before `/` operator is still in expression
+      // * (expr1) / expr2: `)` before `/` operator is still in expression
       // * <non comment start>/ expr: non comment start before `/` is not regex
       if (
         isRegexChar &&
         lastType &&
         !(
-          (lastType === T_SIGN && !'()'.includes(lastToken)) ||
+          (lastType === T_SIGN && ')' !== lastToken) ||
           lastType === T_COMMENT
         )
       ) {
@@ -257,34 +394,23 @@ export function tokenize(code) {
         append()
         continue
       }
+
       const start = i++
-      const leftQuote = code[start]
-      let foundClose = false
 
       // end of line of end of file
       const isEof = () => i >= code.length
       const isEol = () => isEof() || code[i] === '\n'
-      // string quotation
-      if (isQuotationChar) {
-        for (; !isEof(); i++) {
-          // handle single quotes
-          if ((leftQuote === "'" || leftQuote === '"') && code[i] === '\n') break
-          if (isStringQuotation(code[i]) && code[i] === leftQuote) {
-            foundClose = true
-            break
-          }
-        }
-      } else {
-        // regex
-        for (; !isEol(); i++) {
-          if (code[i] === '/' && code[i - 1] !== '\\') {
-            foundClose = true
-            // append regex flags
-            while (start !== i && /^[a-z]$/.test(code[i + 1]) && !isEol()) {
-              i++
-            }
-            break
+
+      let foundClose = false
+      // regex
+      for (; !isEol(); i++) {
+        if (code[i] === '/' && code[i - 1] !== '\\') {
+          foundClose = true
+          // append regex flags
+          while (start !== i && /^[a-z]$/.test(code[i + 1]) && !isEol()) {
+            i++
           }
+          break
         }
       }
       if (start !== i && foundClose) {
@@ -317,25 +443,38 @@ export function tokenize(code) {
         )
       ) {
         current += curr
+        if (next === '<') {
+          append()
+        }
       } else {
         append()
         current = curr
         append()
       }
     } else {
-      if (
+      if (__jsxExpr && curr === '}') {
+        append()
+        current = curr
+        append()
+        __jsxExpr = false
+      } else if (
+        // it's jsx literals and is not a jsx bracket
         (isJsxLiterals && !jsxBrackets.has(curr)) ||
-        isWord(curr) === isWord(current[current.length - 1]) &&
-        !signs.has(curr)
+        // same type char as previous one in current token
+        ((isWord(curr) === isWord(current[current.length - 1]) || __jsxChild()) && !signs.has(curr))
       ) {
         current += curr
-        if (next === '<') {
-          append();
-        }
       } else {
+        if (p_c === '</') {
+          current = p_c
+        }
         append()
-        current = curr
-        if (c_n === '</' || c_n === '/>') {
+
+        if (p_c !== '</') {
+          current = curr
+
+        }
+        if ((c_n === '</' || c_n === '/>')) {
           current = c_n
           append()
           i++
@@ -343,13 +482,6 @@ export function tokenize(code) {
         else if (jsxBrackets.has(curr)) append()
       }
     }
-
-    if (jsx.child && curr === '{') {
-      jsx.expr = true
-    }
-    if (jsx.child && jsx.expr && curr === '}') {
-      jsx.expr = false
-    }
   }
 
   append()

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sugar-high",
-  "version": "0.4.0",
+  "version": "0.4.3",
   "type": "module",
   "exports": "./lib/index.mjs",
   "description": "Super lightweight JSX syntax highlighter",
@@ -16,9 +16,11 @@
     "build": "next build docs"
   },
   "devDependencies": {
-    "next": "12.1.4",
+    "codice": "^0.0.10",
+    "next": "12.1.6-canary.6",
     "react": "18.0.0",
     "react-dom": "18.0.0",
+    "sugar-high": "link:./",
     "vitest": "~0.6.0"
   }
 }