shaderkit 0.6.4 → 0.8.0

package/src/tokenizer.ts

@@ -1,89 +1,105 @@
-import { WGSL_KEYWORDS, WGSL_SYMBOLS, GLSL_KEYWORDS, GLSL_SYMBOLS } from './constants.js'
-
-export type TokenType = 'whitespace' | 'comment' | 'symbol' | 'bool' | 'float' | 'int' | 'identifier' | 'keyword'
-
-export interface Token<T = TokenType, V = string> {
-  type: T
-  value: V
-}
-
-// Checks for WGSL-specific `fn foo(`, `var bar =`, `let baz =`, `const qux =`
-const WGSL_REGEX = /\bfn\s+\w+\s*\(|\b(var|let|const)\s+\w+\s*[:=]/
-
-const FLOAT_REGEX = /((\d+\.\d*|\d*\.\d+)([eEpP][-+]?\d+)?|\d+[eEpP][-+]?\d+)[fFhH]?/y
-const INT_REGEX = /(0[xX][\w\d]+|\d+)[iIuU]?/y
-const BOOL_REGEX = /^(true|false)$/
-
-const ZERO = 48
-const NINE = 57
-const A = 65
-const Z = 90
-const LF = 10
-const CR = 13
-const TAB = 9
-const SPACE = 32
-const UNDERSCORE = 95
-const DOT = 46
-const SLASH = 47
-const STAR = 42
-const HASH = 35
-const AT = 64
-
-const isDigit = (c: number) => ZERO <= c && c <= NINE
-const isAlpha = (c: number) => ((c &= ~32), A <= c && c <= Z)
-const isLine = (c: number) => c === LF || c === CR
-const isSpace = (c: number) => isLine(c) || c === TAB || c === SPACE
-const isIdent = (c: number) => isAlpha(c) || isDigit(c) || c === UNDERSCORE
-const isMacro = (c: number) => c === HASH || c === AT
-
-// https://mrale.ph/blog/2016/11/23/making-less-dart-faster.html
-function matchAsPrefix(regex: RegExp, string: string, start: number): string | undefined {
-  regex.lastIndex = start
-  return regex.exec(string)?.[0]
-}
-
-/**
- * Tokenizes a string of GLSL or WGSL code.
- */
-export function tokenize(code: string, index: number = 0): Token[] {
-  const tokens: Token[] = []
-
-  let prev: number = -1
-  const [KEYWORDS, SYMBOLS] = WGSL_REGEX.test(code) ? [WGSL_KEYWORDS, WGSL_SYMBOLS] : [GLSL_KEYWORDS, GLSL_SYMBOLS]
-  while (index < code.length) {
-    let value = code[index]
-    const char = code.charCodeAt(index++)
-
-    if (isSpace(char)) {
-      while (isSpace(code.charCodeAt(index))) value += code[index++]
-      tokens.push({ type: 'whitespace', value })
-    } else if (isDigit(char) || (char === DOT && isDigit(code.charCodeAt(index)))) {
-      if ((value = matchAsPrefix(FLOAT_REGEX, code, index - 1)!)) {
-        index = FLOAT_REGEX.lastIndex
-        tokens.push({ type: 'float', value })
-      } else if ((value = matchAsPrefix(INT_REGEX, code, index - 1)!)) {
-        index = INT_REGEX.lastIndex
-        tokens.push({ type: 'int', value })
-      }
-    } else if (isIdent(char)) {
-      while (isIdent(code.charCodeAt(index))) value += code[index++]
-      if (BOOL_REGEX.test(value)) tokens.push({ type: 'bool', value })
-      else if (KEYWORDS.includes(isMacro(prev) ? String.fromCharCode(prev) + value : value))
-        tokens.push({ type: 'keyword', value })
-      else tokens.push({ type: 'identifier', value })
-    } else if (char === SLASH && (code.charCodeAt(index) === SLASH || code.charCodeAt(index) === STAR)) {
-      const terminator = code.charCodeAt(index) === STAR ? '*/' : '\n'
-      while (index < code.length && !value.endsWith(terminator)) value += code[index++]
-      tokens.push({ type: 'comment', value: value.trim() })
-    } else {
-      for (const symbol of SYMBOLS) {
-        if (symbol.length > value.length && code.startsWith(symbol, index - 1)) value = symbol
-      }
-      index += value.length - 1
-      tokens.push({ type: 'symbol', value })
-    }
-    prev = char
-  }
-
-  return tokens
-}
+import { WGSL_KEYWORDS, WGSL_SYMBOLS, GLSL_KEYWORDS, GLSL_SYMBOLS } from './constants.js'
+
+export type TokenType = 'whitespace' | 'comment' | 'symbol' | 'bool' | 'float' | 'int' | 'identifier' | 'keyword'
+
+export interface Token<T = TokenType, V = string> {
+  type: T
+  value: V
+}
+
+// Checks for WGSL-specific `fn foo(`, `var bar =`, `let baz =`, `const qux =`
+const WGSL_REGEX = /\bfn\s+\w+\s*\(|\b(var|let|const)\s+\w+\s*[:=]/
+
+const FLOAT_REGEX = /((\d+\.\d*|\d*\.\d+)([eEpP][-+]?\d+)?|\d+[eEpP][-+]?\d+)[fFhH]?/y
+const INT_REGEX = /(0[xX][\w\d]+|\d+)[iIuU]?/y
+const BOOL_REGEX = /^(true|false)$/
+
+const ZERO = 48
+const NINE = 57
+const A = 65
+const Z = 90
+const LF = 10
+const CR = 13
+const TAB = 9
+const SPACE = 32
+const UNDERSCORE = 95
+const DOT = 46
+const SLASH = 47
+const STAR = 42
+const HASH = 35
+const AT = 64
+
+const isDigit = (c: number): boolean => ZERO <= c && c <= NINE
+const isAlpha = (c: number): boolean => ((c &= ~32), A <= c && c <= Z)
+const isLine = (c: number): boolean => c === LF || c === CR
+const isSpace = (c: number): boolean => isLine(c) || c === TAB || c === SPACE
+const isIdent = (c: number): boolean => isAlpha(c) || isDigit(c) || c === UNDERSCORE
+const isMacro = (c: number): boolean => c === HASH || c === AT
+
+// https://mrale.ph/blog/2016/11/23/making-less-dart-faster.html
+function matchAsPrefix(regex: RegExp, string: string, start: number): string | undefined {
+  regex.lastIndex = start
+  return regex.exec(string)?.[0]
+}
+
+/**
+ * Tokenizes a string of GLSL or WGSL code.
+ */
+export function tokenize(code: string, index: number = 0): Token[] {
+  const [KEYWORDS, SYMBOLS] = WGSL_REGEX.test(code) ? [WGSL_KEYWORDS, WGSL_SYMBOLS] : [GLSL_KEYWORDS, GLSL_SYMBOLS]
+
+  const KEYWORDS_LIST = new Set(KEYWORDS)
+  const SYMBOLS_REGEX = new RegExp(
+    SYMBOLS.sort((a, b) => b.length - a.length)
+      .map((s) => s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'))
+      .join('|'),
+    'y',
+  )
+
+  const tokens: Token[] = []
+
+  let value: string | undefined
+  let start: number
+  let char: number
+
+  while (index < code.length) {
+    start = index
+    char = code.charCodeAt(index++)
+
+    if (isSpace(char)) {
+      while (isSpace(code.charCodeAt(index))) index++
+      value = code.slice(start, index)
+      tokens.push({ type: 'whitespace', value })
+    } else if (isDigit(char) || (char === DOT && isDigit(code.charCodeAt(index)))) {
+      if ((value = matchAsPrefix(FLOAT_REGEX, code, start))) {
+        index = FLOAT_REGEX.lastIndex
+        tokens.push({ type: 'float', value })
+      } else if ((value = matchAsPrefix(INT_REGEX, code, start))) {
+        index = INT_REGEX.lastIndex
+        tokens.push({ type: 'int', value })
+      }
+    } else if (isIdent(char)) {
+      while (isIdent(code.charCodeAt(index))) index++
+      value = code.slice(start, index)
+      if (BOOL_REGEX.test(value)) tokens.push({ type: 'bool', value })
+      else if (KEYWORDS_LIST.has(isMacro(code.charCodeAt(start - 1)) ? code[start - 1] + value : value))
+        tokens.push({ type: 'keyword', value })
+      else tokens.push({ type: 'identifier', value })
+    } else if (char === SLASH && code.charCodeAt(index) === SLASH) {
+      while (index < code.length && code.charCodeAt(index) !== LF) index++
+      value = code.slice(start, index)
+      index++ // consume LF
+      tokens.push({ type: 'comment', value })
+    } else if (char === SLASH && code.charCodeAt(index) === STAR) {
+      while (index < code.length && (code.charCodeAt(index - 2) !== STAR || code.charCodeAt(index - 1) !== SLASH))
+        index++
+      value = code.slice(start, index)
+      tokens.push({ type: 'comment', value })
+    } else if ((value = matchAsPrefix(SYMBOLS_REGEX, code, start))) {
+      index += value.length - 1
+      tokens.push({ type: 'symbol', value })
+    }
+  }
+
+  return tokens
+}

package/src/visitor.ts

@@ -1,131 +1,131 @@
-import { type AST } from './ast'
-
-export type Visitors = Partial<{
-  [K in AST['type']]:
-    | ((node: Extract<AST, { type: K }>, ancestors: AST[]) => void)
-    | {
-        enter?(node: Extract<AST, { type: K }>, ancestors: AST[]): void
-        exit?(node: Extract<AST, { type: K }>, ancestors: AST[]): void
-      }
-}>
-
-/**
- * Recurses through an [AST](https://en.wikipedia.org/wiki/Abstract_syntax_tree), calling a visitor object on matching nodes.
- */
-export function visit(node: AST, visitors: Visitors, ancestors: AST[] = []): void {
-  const parentAncestors = ancestors
-  const visitor = visitors[node.type]
-
-  // @ts-ignore
-  ;(visitor?.enter ?? visitor)?.(node, parentAncestors)
-
-  ancestors = [...ancestors, node]
-
-  switch (node.type) {
-    case 'ArraySpecifier':
-      visit(node.typeSpecifier, visitors, ancestors)
-      for (const dimension of node.dimensions) if (dimension) visit(dimension, visitors, ancestors)
-      break
-    case 'ExpressionStatement':
-      visit(node.expression, visitors, ancestors)
-      break
-    case 'BlockStatement':
-      for (const statement of node.body) visit(statement, visitors, ancestors)
-      break
-    case 'PreprocessorStatement':
-      if (node.value) for (const expression of node.value) visit(expression, visitors, ancestors)
-      break
-    case 'PrecisionQualifierStatement':
-      visit(node.typeSpecifier, visitors, ancestors)
-      break
-    case 'InvariantQualifierStatement':
-      visit(node.typeSpecifier, visitors, ancestors)
-      break
-    case 'ReturnStatement':
-      if (node.argument) visit(node.argument, visitors, ancestors)
-      break
-    case 'IfStatement':
-      visit(node.test, visitors, ancestors)
-      visit(node.consequent, visitors, ancestors)
-      if (node.alternate) visit(node.alternate, visitors, ancestors)
-      break
-    case 'SwitchStatement':
-      visit(node.discriminant, visitors, ancestors)
-      for (const kase of node.cases) visit(kase, visitors, ancestors)
-      break
-    case 'SwitchCase':
-      if (node.test) visit(node.test, visitors, ancestors)
-      for (const statement of node.consequent) visit(statement, visitors, ancestors)
-      break
-    case 'WhileStatement':
-    case 'DoWhileStatement':
-      visit(node.test, visitors, ancestors)
-      visit(node.body, visitors, ancestors)
-      break
-    case 'ForStatement':
-      if (node.init) visit(node.init, visitors, ancestors)
-      if (node.test) visit(node.test, visitors, ancestors)
-      if (node.update) visit(node.update, visitors, ancestors)
-      visit(node.body, visitors, ancestors)
-      break
-    case 'FunctionDeclaration':
-      visit(node.typeSpecifier, visitors, ancestors)
-      visit(node.id, visitors, ancestors)
-      if (node.body) visit(node.body, visitors, ancestors)
-      break
-    case 'FunctionParameter':
-      visit(node.typeSpecifier, visitors, ancestors)
-      if (node.id) visit(node.id, visitors, ancestors)
-      break
-    case 'VariableDeclaration':
-      for (const declaration of node.declarations) visit(declaration, visitors, ancestors)
-      break
-    case 'VariableDeclarator':
-      visit(node.typeSpecifier, visitors, ancestors)
-      visit(node.id, visitors, ancestors)
-      if (node.init) visit(node.init, visitors, ancestors)
-      break
-    case 'StructuredBufferDeclaration':
-      visit(node.typeSpecifier, visitors, ancestors)
-      for (const member of node.members) visit(member, visitors, ancestors)
-      if (node.id) visit(node.id, visitors, ancestors)
-      break
-    case 'StructDeclaration':
-      visit(node.id, visitors, ancestors)
-      for (const member of node.members) visit(member, visitors, ancestors)
-      break
-    case 'ArrayExpression':
-      visit(node.typeSpecifier, visitors, ancestors)
-      for (const element of node.elements) visit(element, visitors, ancestors)
-      break
-    case 'UnaryExpression':
-    case 'UpdateExpression':
-      visit(node.argument, visitors, ancestors)
-      break
-    case 'BinaryExpression':
-    case 'AssignmentExpression':
-    case 'LogicalExpression':
-      visit(node.left, visitors, ancestors)
-      visit(node.right, visitors, ancestors)
-      break
-    case 'MemberExpression':
-      visit(node.object, visitors, ancestors)
-      visit(node.property, visitors, ancestors)
-      break
-    case 'ConditionalExpression':
-      visit(node.test, visitors, ancestors)
-      visit(node.consequent, visitors, ancestors)
-      visit(node.alternate, visitors, ancestors)
-      break
-    case 'CallExpression':
-      visit(node.callee, visitors, ancestors)
-      for (const argument of node.arguments) visit(argument, visitors, ancestors)
-      break
-    case 'Program':
-      for (const statement of node.body) visit(statement, visitors, ancestors)
-      break
-  }
-
-  // @ts-ignore
-  visitor?.exit?.(node, parentAncestors)
-}
+import { type AST } from './ast'
+
+export type Visitors = Partial<{
+  [K in AST['type']]:
+    | ((node: Extract<AST, { type: K }>, ancestors: AST[]) => void)
+    | {
+        enter?(node: Extract<AST, { type: K }>, ancestors: AST[]): void
+        exit?(node: Extract<AST, { type: K }>, ancestors: AST[]): void
+      }
+}>
+
+/**
+ * Recurses through an [AST](https://en.wikipedia.org/wiki/Abstract_syntax_tree), calling a visitor object on matching nodes.
+ */
+export function visit(node: AST, visitors: Visitors, ancestors: AST[] = []): void {
+  const parentAncestors = ancestors
+  const visitor = visitors[node.type]
+
+  // @ts-ignore
+  ;(visitor?.enter ?? visitor)?.(node, parentAncestors)
+
+  ancestors = [...ancestors, node]
+
+  switch (node.type) {
+    case 'ArraySpecifier':
+      visit(node.typeSpecifier, visitors, ancestors)
+      for (const dimension of node.dimensions) if (dimension) visit(dimension, visitors, ancestors)
+      break
+    case 'ExpressionStatement':
+      visit(node.expression, visitors, ancestors)
+      break
+    case 'BlockStatement':
+      for (const statement of node.body) visit(statement, visitors, ancestors)
+      break
+    case 'PreprocessorStatement':
+      if (node.value) for (const expression of node.value) visit(expression, visitors, ancestors)
+      break
+    case 'PrecisionQualifierStatement':
+      visit(node.typeSpecifier, visitors, ancestors)
+      break
+    case 'InvariantQualifierStatement':
+      visit(node.typeSpecifier, visitors, ancestors)
+      break
+    case 'ReturnStatement':
+      if (node.argument) visit(node.argument, visitors, ancestors)
+      break
+    case 'IfStatement':
+      visit(node.test, visitors, ancestors)
+      visit(node.consequent, visitors, ancestors)
+      if (node.alternate) visit(node.alternate, visitors, ancestors)
+      break
+    case 'SwitchStatement':
+      visit(node.discriminant, visitors, ancestors)
+      for (const kase of node.cases) visit(kase, visitors, ancestors)
+      break
+    case 'SwitchCase':
+      if (node.test) visit(node.test, visitors, ancestors)
+      for (const statement of node.consequent) visit(statement, visitors, ancestors)
+      break
+    case 'WhileStatement':
+    case 'DoWhileStatement':
+      visit(node.test, visitors, ancestors)
+      visit(node.body, visitors, ancestors)
+      break
+    case 'ForStatement':
+      if (node.init) visit(node.init, visitors, ancestors)
+      if (node.test) visit(node.test, visitors, ancestors)
+      if (node.update) visit(node.update, visitors, ancestors)
+      visit(node.body, visitors, ancestors)
+      break
+    case 'FunctionDeclaration':
+      visit(node.typeSpecifier, visitors, ancestors)
+      visit(node.id, visitors, ancestors)
+      if (node.body) visit(node.body, visitors, ancestors)
+      break
+    case 'FunctionParameter':
+      visit(node.typeSpecifier, visitors, ancestors)
+      if (node.id) visit(node.id, visitors, ancestors)
+      break
+    case 'VariableDeclaration':
+      for (const declaration of node.declarations) visit(declaration, visitors, ancestors)
+      break
+    case 'VariableDeclarator':
+      visit(node.typeSpecifier, visitors, ancestors)
+      visit(node.id, visitors, ancestors)
+      if (node.init) visit(node.init, visitors, ancestors)
+      break
+    case 'StructuredBufferDeclaration':
+      visit(node.typeSpecifier, visitors, ancestors)
+      for (const member of node.members) visit(member, visitors, ancestors)
+      if (node.id) visit(node.id, visitors, ancestors)
+      break
+    case 'StructDeclaration':
+      visit(node.id, visitors, ancestors)
+      for (const member of node.members) visit(member, visitors, ancestors)
+      break
+    case 'ArrayExpression':
+      visit(node.typeSpecifier, visitors, ancestors)
+      for (const element of node.elements) visit(element, visitors, ancestors)
+      break
+    case 'UnaryExpression':
+    case 'UpdateExpression':
+      visit(node.argument, visitors, ancestors)
+      break
+    case 'BinaryExpression':
+    case 'AssignmentExpression':
+    case 'LogicalExpression':
+      visit(node.left, visitors, ancestors)
+      visit(node.right, visitors, ancestors)
+      break
+    case 'MemberExpression':
+      visit(node.object, visitors, ancestors)
+      visit(node.property, visitors, ancestors)
+      break
+    case 'ConditionalExpression':
+      visit(node.test, visitors, ancestors)
+      visit(node.consequent, visitors, ancestors)
+      visit(node.alternate, visitors, ancestors)
+      break
+    case 'CallExpression':
+      visit(node.callee, visitors, ancestors)
+      for (const argument of node.arguments) visit(argument, visitors, ancestors)
+      break
+    case 'Program':
+      for (const statement of node.body) visit(statement, visitors, ancestors)
+      break
+  }
+
+  // @ts-ignore
+  visitor?.exit?.(node, parentAncestors)
+}