squirreling 0.1.0

package/src/parse/tokenize.js

@@ -0,0 +1,332 @@
+/**
+ * @import { Token } from '../types.d.ts'
+ */
+
+const KEYWORDS = new Set([
+  'SELECT',
+  'FROM',
+  'WHERE',
+  'AND',
+  'OR',
+  'NOT',
+  'IS',
+  'GROUP',
+  'BY',
+  'HAVING',
+  'ORDER',
+  'ASC',
+  'DESC',
+  'LIMIT',
+  'OFFSET',
+  'AS',
+  'DISTINCT',
+  'TRUE',
+  'FALSE',
+  'NULL',
+  'LIKE',
+  'IN',
+  'BETWEEN',
+  'CASE',
+  'WHEN',
+  'THEN',
+  'ELSE',
+  'END',
+  'JOIN',
+  'INNER',
+  'LEFT',
+  'RIGHT',
+  'FULL',
+  'OUTER',
+  'ON',
+])
+
+/**
+ * @param {string} sql
+ * @returns {Token[]}
+ */
+export function tokenize(sql) {
+  /** @type {Token[]} */
+  const tokens = []
+  const { length } = sql
+  let i = 0
+
+  /**
+   * @returns {string}
+   */
+  function peek() {
+    if (i >= length) return ''
+    return sql[i]
+  }
+
+  /**
+   * @returns {string}
+   */
+  function nextChar() {
+    if (i >= length) return ''
+    const ch = sql[i]
+    i++
+    return ch
+  }
+
+  while (i < length) {
+    const ch = peek()
+
+    if (isWhitespace(ch)) {
+      nextChar()
+      continue
+    }
+
+    // line comment --
+    if (ch === '-' && i + 1 < length && sql[i + 1] === '-') {
+      while (i < length && sql[i] !== '\n') {
+        i++
+      }
+      continue
+    }
+
+    // block comment /* ... */
+    if (ch === '/' && i + 1 < length && sql[i + 1] === '*') {
+      i += 2
+      while (i < length) {
+        if (sql[i] === '*' && i + 1 < length && sql[i + 1] === '/') {
+          i += 2
+          break
+        }
+        i++
+      }
+      continue
+    }
+
+    const pos = i
+
+    // numbers
+    if (isDigit(ch)) {
+      let text = ''
+      while (isDigit(peek())) {
+        text += nextChar()
+      }
+      if (peek() === '.') {
+        text += nextChar()
+        while (isDigit(peek())) {
+          text += nextChar()
+        }
+      }
+      // exponent
+      if (peek() === 'e' || peek() === 'E') {
+        text += nextChar()
+        if (peek() === '+' || peek() === '-') {
+          text += nextChar()
+        }
+        while (isDigit(peek())) {
+          text += nextChar()
+        }
+      }
+      if (isAlpha(peek())) {
+        throw new Error('Invalid number at position ' + pos + ': ' + text + peek())
+      }
+      const num = parseFloat(text)
+      if (isNaN(num)) {
+        throw new Error('Invalid number at position ' + pos + ': ' + text)
+      }
+      tokens.push({
+        type: 'number',
+        value: text,
+        position: pos,
+        numericValue: num,
+      })
+      continue
+    }
+
+    // identifiers / keywords
+    if (isAlpha(ch)) {
+      let text = ''
+      while (isAlphaNumeric(peek())) {
+        text += nextChar()
+      }
+      const upper = text.toUpperCase()
+      if (KEYWORDS.has(upper)) {
+        tokens.push({
+          type: 'keyword',
+          value: upper,
+          originalValue: text,
+          position: pos,
+        })
+      } else {
+        tokens.push({
+          type: 'identifier',
+          value: text,
+          position: pos,
+        })
+      }
+      continue
+    }
+
+    // string literals: single quotes
+    if (ch === '\'') {
+      const quote = nextChar()
+      let text = ''
+      while (i <= length) {
+        if (i === length) {
+          throw new Error('Unterminated string literal starting at position ' + pos)
+        }
+        const c = nextChar()
+        if (c === quote) {
+          // check for escaped quote
+          if (peek() === quote) {
+            text += quote
+            nextChar()
+            continue
+          }
+          break
+        }
+        text += c
+      }
+      tokens.push({
+        type: 'string',
+        value: text,
+        position: pos,
+      })
+      continue
+    }
+
+    // quoted identifiers: double quotes
+    if (ch === '"') {
+      const quote = nextChar()
+      let text = ''
+      while (i <= length) {
+        if (i === length) {
+          throw new Error('Unterminated identifier starting at position ' + pos)
+        }
+        const c = nextChar()
+        if (c === quote) {
+          // check for escaped quote
+          if (peek() === quote) {
+            text += quote
+            nextChar()
+            continue
+          }
+          break
+        }
+        text += c
+      }
+      tokens.push({
+        type: 'identifier',
+        value: text,
+        position: pos,
+      })
+      continue
+    }
+
+    // two-character operators
+    if (ch === '<' || ch === '>' || ch === '!' || ch === '=') {
+      let op = nextChar()
+      if ((op === '<' || op === '>' || op === '!') && peek() === '=') {
+        op += nextChar()
+      } else if (op === '<' && peek() === '>') {
+        op += nextChar()
+      }
+      tokens.push({
+        type: 'operator',
+        value: op,
+        position: pos,
+      })
+      continue
+    }
+
+    // single-char operators
+    if (ch === '*' || ch === '+' || ch === '-' || ch === '/' || ch === '%') {
+      nextChar()
+      tokens.push({
+        type: 'operator',
+        value: ch,
+        position: pos,
+      })
+      continue
+    }
+
+    if (ch === ',') {
+      nextChar()
+      tokens.push({
+        type: 'comma',
+        value: ',',
+        position: pos,
+      })
+      continue
+    }
+
+    if (ch === '.') {
+      nextChar()
+      tokens.push({
+        type: 'dot',
+        value: '.',
+        position: pos,
+      })
+      continue
+    }
+
+    if (ch === '(' || ch === ')') {
+      nextChar()
+      tokens.push({
+        type: 'paren',
+        value: ch,
+        position: pos,
+      })
+      continue
+    }
+
+    if (ch === ';') {
+      nextChar()
+      tokens.push({
+        type: 'semicolon',
+        value: ';',
+        position: pos,
+      })
+      continue
+    }
+
+    if (tokens.length === 0) {
+      throw new Error('Expected SELECT at position ' + pos)
+    }
+    throw new Error('Unexpected character at position ' + pos + ': ' + ch)
+  }
+
+  tokens.push({
+    type: 'eof',
+    value: '',
+    position: length,
+  })
+
+  return tokens
+}
+
+/**
+ * @param {string} ch
+ * @returns {boolean}
+ */
+function isWhitespace(ch) {
+  return ch === ' ' || ch === '\t' || ch === '\n' || ch === '\r'
+}
+
+/**
+ * @param {string} ch
+ * @returns {boolean}
+ */
+function isDigit(ch) {
+  return ch >= '0' && ch <= '9'
+}
+
+/**
+ * @param {string} ch
+ * @returns {boolean}
+ */
+function isAlpha(ch) {
+  return ch >= 'a' && ch <= 'z' || ch >= 'A' && ch <= 'Z' || ch === '_' || ch === '$'
+}
+
+/**
+ * @param {string} ch
+ * @returns {boolean}
+ */
+function isAlphaNumeric(ch) {
+  return isAlpha(ch) || isDigit(ch)
+}

package/src/types.d.ts

@@ -0,0 +1,159 @@
+export type Row = Record<string, any>
+
+export type SqlPrimitive = string | number | bigint | boolean | null
+
+export interface SelectStatement {
+  distinct: boolean
+  columns: SelectColumn[]
+  from?: string
+  joins: JoinClause[]
+  where?: ExprNode
+  groupBy: ExprNode[]
+  orderBy: OrderByItem[]
+  limit?: number
+  offset?: number
+}
+
+export type TokenType =
+  | 'keyword'
+  | 'identifier'
+  | 'number'
+  | 'string'
+  | 'operator'
+  | 'comma'
+  | 'dot'
+  | 'paren'
+  | 'semicolon'
+  | 'eof'
+
+export interface Token {
+  type: TokenType
+  value: string
+  position: number
+  numericValue?: number
+  originalValue?: string
+}
+
+export type BinaryOp =
+  | 'AND'
+  | 'OR'
+  | '='
+  | '!='
+  | '<>'
+  | '<'
+  | '>'
+  | '<='
+  | '>='
+  | 'LIKE'
+
+export interface LiteralNode {
+  type: 'literal'
+  value: SqlPrimitive
+}
+
+export interface IdentifierNode {
+  type: 'identifier'
+  name: string
+}
+
+export interface UnaryNode {
+  type: 'unary'
+  op: 'NOT' | 'IS NULL' | 'IS NOT NULL' | '-'
+  argument: ExprNode
+}
+
+export interface BinaryNode {
+  type: 'binary'
+  op: BinaryOp
+  left: ExprNode
+  right: ExprNode
+}
+
+export interface FunctionNode {
+  type: 'function'
+  name: string
+  args: ExprNode[]
+}
+
+export interface CastNode {
+  type: 'cast'
+  expr: ExprNode
+  toType: string
+}
+
+export type ExprNode = LiteralNode | IdentifierNode | UnaryNode | BinaryNode | FunctionNode | CastNode
+
+export interface StarColumn {
+  kind: 'star'
+  alias?: string
+}
+
+export interface SimpleColumn {
+  kind: 'column'
+  column: string
+  alias?: string
+}
+
+export type AggregateFunc = 'COUNT' | 'SUM' | 'AVG' | 'MIN' | 'MAX'
+
+export type StringFunc = 'UPPER' | 'LOWER' | 'CONCAT' | 'LENGTH' | 'SUBSTRING' | 'TRIM'
+
+export interface AggregateArgStar {
+  kind: 'star'
+}
+
+export interface AggregateArgColumn {
+  kind: 'column'
+  column: string
+}
+
+export type AggregateArg = AggregateArgStar | AggregateArgColumn
+
+export interface AggregateColumn {
+  kind: 'aggregate'
+  func: AggregateFunc
+  arg: AggregateArg
+  alias?: string
+}
+
+export interface FunctionColumn {
+  kind: 'function'
+  func: StringFunc
+  args: ExprNode[]
+  alias?: string
+}
+
+export interface OperationColumn {
+  kind: 'operation'
+  expr: ExprNode
+  alias?: string
+}
+
+export type SelectColumn = StarColumn | SimpleColumn | AggregateColumn | FunctionColumn | OperationColumn
+
+export interface OrderByItem {
+  expr: ExprNode
+  direction: 'ASC' | 'DESC'
+}
+
+export type JoinType = 'INNER' | 'LEFT' | 'RIGHT' | 'FULL' | 'CROSS'
+
+export interface JoinClause {
+  type: JoinType
+  table: string
+  on?: ExprNode
+}
+
+export interface ParserState {
+  tokens: Token[]
+  pos: number
+}
+
+export interface ExprCursor {
+  current(): Token
+  peek(offset: number): Token
+  consume(): Token
+  match(type: TokenType, value?: string): boolean
+  expect(type: TokenType, value: string): Token
+  expectIdentifier(): Token
+}