sugar-high 0.1.0 → 0.2.0

package/README.md

@@ -33,7 +33,7 @@ Then make your own theme with customized colors by token type and put in global
  * Class, number and null
  * sign
  * comment
- * 
+ * jsxliterals
  */
 .sh__class {
   color: #2d5e9d;
@@ -53,7 +53,9 @@ Then make your own theme with customized colors by token type and put in global
 .sh__comment {
   color: #a19595;
 }
-
+.sh__jsxliterals {
+  color: #03066e;
+}
 ```
 
 ### LICENSE

package/lib/index.mjs

@@ -74,7 +74,7 @@ const signs = new Set([
   '\\',
 ])
 
-const types = [
+export const types = [
   'identifier',
   'keyword',
   'string',
@@ -83,6 +83,7 @@ const types = [
   'comment',
   'break',
   'space',
+  'jsxliterals'
 ]
 
 /**
@@ -95,6 +96,7 @@ const types = [
  * 5 - comment
  * 6 - break
  * 7 - space
+ * 8 - jsx literals
  * 
  */
 const [
@@ -106,6 +108,7 @@ const [
   T_COMMENT,
   T_BREAK,
   T_SPACE,
+  T_JSX_LITERALS,
 ] = types.map((_, i) => i)
 
 function isSpaces(str) {
@@ -124,7 +127,11 @@ function encode(str) {
 }
 
 function isIdentifierChar(chr) {
-  return /[a-zA-Z0-9_$]/.test(chr)
+  return /^[\w_$]$/.test(chr)
+}
+
+function isIdentifier(str) {
+  return /[a-zA-Z_$]/.test(str[0]) && (str.length === 1 || /^[\w_$]+$/.test(str.slice(1)))
 }
 
 function isStringQuotation(chr) {
@@ -147,15 +154,9 @@ function isRegexStart(str) {
 export function tokenize(code) {
   let current = ''
   let type = -1
+  let last = [null, null]
   /** @type {Array<[number, string]>} */
   const tokens = []
-  // string.type = 0 for string or string template
-  // string.type = 1 for regex
-  const string = { entered: false, type: 0 }
-  
-  // comment.type = 0 for single line comments
-  // comment.type = 1 for multi-line comments
-  const comment = { entered: false, type: 0 }
 
   // jsx.tag for entering open or closed tag
   // jsx.child for entering children
@@ -166,25 +167,14 @@ export function tokenize(code) {
   const inJsxLiterals = () => !jsx.tag && jsx.child && !jsx.expr
 
   function classify(token) {
-    const [chr0, chr1] = [token[0], token[1]]
-    if (isCommentStart(chr0 + chr1)) {
-      return T_COMMENT
-    } else if (keywords.has(token)) {
+    const chr0 = token[0]
+    if (keywords.has(token)) {
       return T_KEYWORD
     } else if (token === '\n') {
       return T_BREAK
-    } else if (
-       (
-        // is quoted string
-        (isStringQuotation(chr0) && !isStringQuotation(chr1)) ||
-        // is regex
-        (!jsx.tag && isRegexStart(chr0 + chr1) && token[token.length - 1] === '/')
-      )
-    ) {
-      return T_STRING
-    } else if (token === ' ') {
+    } else if (isSpaces(token)) {
       return T_SPACE
-    } else if (signs.has(chr0)) {
+    } else if (token.split('').every(ch => signs.has(ch))) {
       return T_SIGN
     } else if (
       !inJsxLiterals() && 
@@ -196,14 +186,19 @@ export function tokenize(code) {
     ) {
       return T_CLS_NUMBER
     } else {
-      return T_IDENTIFIER
+      return isIdentifier(token) ? T_IDENTIFIER : inJsxLiterals() ? T_JSX_LITERALS : T_STRING
     }
   }
 
-  const append = () => {
+  const append = (_type) => {
     if (current) {
-      type = classify(current)
-      tokens.push([type, current])
+      type = _type || classify(current)
+      /** @type [number, string]  */
+      const pair = [type, current]
+      if (type !== T_SPACE && type !== T_BREAK) {
+        last = pair
+      }
+      tokens.push(pair)
     }
     current = ''
   }
@@ -222,48 +217,69 @@ export function tokenize(code) {
       jsx.tag = !(isOpenElementEnd || isCloseElementEnd)
     }
     // if it's not in a jsx tag declaration or a string, close child if next is jsx close tag
-    if (!jsx.tag && !string.entered && (curr === '<' && isIdentifierChar(next) || c_n === '</')) {
+    if (!jsx.tag && (curr === '<' && isIdentifierChar(next) || c_n === '</')) {
       jsx.tag = true
       jsx.child = false
     }
 
-    if (
-      !string.entered && 
-      (isStringQuotation(curr) || !jsx.tag && isRegexStart(c_n))
-    ) {
-      string.entered = true
-      string.type = isStringQuotation(curr) ? 0 : 1
+    const isQuotationChar = isStringQuotation(curr)
+    const isRegexChar = !jsx.tag && isRegexStart(c_n)
+    if (isQuotationChar || isRegexChar) {
       append()
-      current = curr
-    } else if (string.entered) {
-      current += curr
-      if (string.type === 0 && isStringQuotation(curr)) {
-        string.entered = false
-        append()
-      } else if (string.type === 1 && prev !== '\\' && curr === '/') {
-        string.entered = false
+      const [lastType, lastToken] = last
+      // Special cases that are not considered as regex:
+      // * (expr) / expr: `()[]` before `/` is still in expression
+      // * <non comment start>/ expr: non comment start before `/` is not regex
+      if (
+        isRegexChar && 
+        lastType && 
+        !(
+          (lastType === T_SIGN && !'()[]'.includes(lastToken)) || 
+          lastType === T_COMMENT
+        )
+      ) {
+        current = curr
         append()
+        continue
       }
-    } else if (
-      !comment.entered && 
-      isCommentStart(c_n)
-    ) {
-      comment.type = next === '/' ? 0 : 1
-      comment.entered = true
-      append()
-      current = c_n
-      i++
-    } else if (comment.entered) {
-      current += curr
-      if (comment.type === 0 && next === '\n') {
-        comment.entered = false
-        append()
-      } else if (comment.type === 1 && (c_n === '*/')) {
-        comment.entered = false
-        current += '/'
+      const start = i++
+      
+      const isInline = () => i < code.length && code[i] !== '\n'
+      // string quotation
+      if (isQuotationChar) {
+        for (; isInline(); i++) {
+          if (isStringQuotation(code[i])) {
+            break
+          }
+        }
+      } else {
+        // regex
+        for (; isInline(); i++) {
+          if (code[i] === '/' && code[i - 1] !== '\\') {
+            // append regex flags
+            while (start !== i && /^[a-z]$/.test(code[i + 1]) && isInline()) {
+              i++
+            };
+            break
+          }
+        }
+      }
+      if (start !== i) {
+        current = code.slice(start, i + 1)
+        append(T_STRING)
+      } else {
+        current = curr
         append()
-        i++
       }
+    } else if (isCommentStart(c_n)) {
+      const start = i
+      if (next === '/') {
+        for (; i < code.length && code[i] !== '\n'; i++);
+      } else {
+        for (; i < code.length && code[i - 1] + code[i] !== '*/'; i++);
+      }
+      current = code.slice(start, i + 1)
+      append(T_COMMENT)
     } else if (curr === ' ' || curr === '\n') {
       if (
         curr === ' ' &&

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sugar-high",
-  "version": "0.1.0",
+  "version": "0.2.0",
   "type": "module",
   "exports": "./lib/index.mjs",
   "description": "Super lightweight JSX syntax highlighter",
@@ -11,10 +11,12 @@
   ],
   "license": "MIT",
   "scripts": {
+    "test": "vitest run",
     "dev": "vite docs",
     "build": "vite build docs"
   },
   "devDependencies": {
-    "vite": "2.7.13"
+    "vite": "^2.8.6",
+    "vitest": "^0.5.9"
   }
 }