squirreling 0.10.2 → 0.11.0

package/src/parse/tokenize.js

@@ -1,166 +1,90 @@
-import { invalidLiteralError, unexpectedCharError, unterminatedError } from '../validation/parseErrors.js'
+import { KEYWORDS } from '../validation/keywords.js'
+import { InvalidLiteralError, ParseError, UnexpectedCharError } from '../validation/parseErrors.js'
 
 /**
  * @import { Token } from '../types.d.ts'
  */
 
-const KEYWORDS = new Set([
-  'WITH',
-  'SELECT',
-  'FROM',
-  'WHERE',
-  'AND',
-  'OR',
-  'NOT',
-  'IS',
-  'GROUP',
-  'BY',
-  'HAVING',
-  'ORDER',
-  'ASC',
-  'DESC',
-  'NULLS',
-  'LIMIT',
-  'OFFSET',
-  'AS',
-  'ALL',
-  'DISTINCT',
-  'TRUE',
-  'FALSE',
-  'NULL',
-  'LIKE',
-  'IN',
-  'EXISTS',
-  'BETWEEN',
-  'CASE',
-  'WHEN',
-  'THEN',
-  'ELSE',
-  'END',
-  'JOIN',
-  'INNER',
-  'LEFT',
-  'RIGHT',
-  'FULL',
-  'OUTER',
-  'POSITIONAL',
-  'ON',
-  'INTERVAL',
-  'DAY',
-  'MONTH',
-  'YEAR',
-  'HOUR',
-  'MINUTE',
-  'SECOND',
-  'FILTER',
-])
+const NUMBER_REGEX = /^-?(?:\d+n|\d+(?:\.\d*)?(?:[eE][+-]?\d+)?)/
 
 /**
- * @param {string} sql
+ * @param {string} query
  * @returns {Token[]}
  */
-export function tokenizeSql(sql) {
+export function tokenizeSql(query) {
   /** @type {Token[]} */
   const tokens = []
-  const { length } = sql
-  let i = 0
+  const len = query.length
+  let i = 0 // current position in query string
 
   /**
    * @returns {string}
    */
   function peek() {
-    if (i >= length) return ''
-    return sql[i]
+    return query[i]
   }
 
   /**
    * @returns {string}
    */
   function nextChar() {
-    if (i >= length) return ''
-    const ch = sql[i]
-    i++
-    return ch
+    return query[i++]
   }
 
   /**
-   * @param {number} startPos
-   * @param {string} prefix
+   * @param {number} positionStart
    * @returns {Token}
    */
-  function parseNumber(startPos, prefix = '') {
-    let text = prefix
-    while (isDigit(peek())) {
-      text += nextChar()
+  function parseNumber(positionStart) {
+    const value = query.slice(i).match(NUMBER_REGEX)?.[0]
+    if (!value) {
+      throw new InvalidLiteralError({ expected: 'number', value: query[i] || 'eof', positionStart, positionEnd: i + 1 })
     }
-    if (peek() === '.') {
-      text += nextChar()
-      while (isDigit(peek())) {
-        text += nextChar()
-      }
+    i += value.length
+    const next = peek()
+    if (isAlpha(next) || next === '.') {
+      throw new InvalidLiteralError({ expected: 'number', value: value + next, positionStart, positionEnd: i + 1 })
     }
-    // exponent
-    if (peek() === 'e' || peek() === 'E') {
-      text += nextChar()
-      if (peek() === '+' || peek() === '-') {
-        text += nextChar()
-      }
-      while (isDigit(peek())) {
-        text += nextChar()
-      }
-    }
-    // bigint suffix
-    if (peek() === 'n') {
-      text += nextChar()
-      try {
-        return {
-          type: 'number',
-          value: text,
-          positionStart: startPos,
-          positionEnd: i,
-          numericValue: BigInt(text.slice(0, -1)),
-        }
-      } catch {
-        throw invalidLiteralError({ type: 'bigint', value: text.slice(0, -1), positionStart: startPos, positionEnd: i })
+    if (value.endsWith('n')) {
+      return {
+        type: 'number',
+        value,
+        positionStart,
+        positionEnd: i,
+        numericValue: BigInt(value.slice(0, -1)),
       }
     }
-    if (isAlpha(peek())) {
-      throw invalidLiteralError({ type: 'number', value: text + peek(), positionStart: startPos, positionEnd: i + 1 })
-    }
-    const num = parseFloat(text)
-    if (isNaN(num)) {
-      throw invalidLiteralError({ type: 'number', value: text, positionStart: startPos, positionEnd: i })
-    }
     return {
       type: 'number',
-      value: text,
-      positionStart: startPos,
+      value,
+      positionStart,
       positionEnd: i,
-      numericValue: num,
+      numericValue: Number(value),
     }
   }
 
-  while (i < length) {
+  while (i < len) {
+    const positionStart = i
     const ch = peek()
 
     if (isWhitespace(ch)) {
-      nextChar()
+      i++
       continue
     }
 
     // line comment --
-    if (ch === '-' && i + 1 < length && sql[i + 1] === '-') {
-      while (i < length && sql[i] !== '\n') {
+    if (ch === '-' && query[i + 1] === '-') {
+      while (i < len && query[i] !== '\n') {
         i++
       }
       continue
     }
 
     // block comment /* ... */
-    if (ch === '/' && i + 1 < length && sql[i + 1] === '*') {
+    if (ch === '/' && query[i + 1] === '*') {
       i += 2
-      while (i < length) {
-        if (sql[i] === '*' && i + 1 < length && sql[i + 1] === '/') {
+      while (i < len) {
+        if (query[i] === '*' && query[i + 1] === '/') {
           i += 2
           break
         }
@@ -169,10 +93,8 @@ export function tokenizeSql(sql) {
       continue
     }
 
-    const pos = i
-
     // negative numbers (when not subtraction)
-    if (ch === '-' && i + 1 < length && isDigit(sql[i + 1])) {
+    if (ch === '-' && isDigit(query[i + 1])) {
       const lastToken = tokens[tokens.length - 1]
       const isValueBefore = lastToken && (
         lastToken.type === 'identifier' ||
@@ -181,104 +103,75 @@ export function tokenizeSql(sql) {
         lastToken.type === 'paren' && lastToken.value === ')'
       )
       if (!isValueBefore) {
-        nextChar() // consume '-'
-        tokens.push(parseNumber(pos, '-'))
+        tokens.push(parseNumber(positionStart))
         continue
       }
     }
 
     // numbers
     if (isDigit(ch)) {
-      tokens.push(parseNumber(pos))
+      tokens.push(parseNumber(positionStart))
       continue
     }
 
     // identifiers / keywords
     if (isAlpha(ch)) {
-      let text = ''
+      let value = ''
       while (isAlphaNumeric(peek())) {
-        text += nextChar()
+        value += nextChar()
       }
-      const upper = text.toUpperCase()
+      const upper = value.toUpperCase()
       if (KEYWORDS.has(upper)) {
         tokens.push({
           type: 'keyword',
-          value: upper,
-          originalValue: text,
-          positionStart: pos,
+          value: upper, // uppercase for keywords
+          originalValue: value, // preserve user casing
+          positionStart,
           positionEnd: i,
         })
       } else {
         tokens.push({
           type: 'identifier',
-          value: text,
-          positionStart: pos,
+          value,
+          positionStart,
           positionEnd: i,
         })
       }
       continue
     }
 
-    // string literals: single quotes
-    if (ch === '\'') {
-      const quote = nextChar()
-      let text = ''
-      while (i <= length) {
-        if (i === length) {
-          throw unterminatedError({ type: 'string', positionStart: pos, positionEnd: length })
-        }
-        const c = nextChar()
-        if (c === quote) {
-          // check for escaped quote
-          if (peek() === quote) {
-            text += quote
-            nextChar()
-            continue
-          }
-          break
-        }
-        text += c
-      }
-      tokens.push({
-        type: 'string',
-        value: text,
-        positionStart: pos,
-        positionEnd: i,
-      })
-      continue
-    }
-
-    // quoted identifiers: double quotes
-    if (ch === '"') {
+    // string literals (single quotes) and quoted identifiers (double quotes)
+    if (ch === '\'' || ch === '"') {
+      const type = ch === '\'' ? 'string' : 'identifier'
       const quote = nextChar()
-      let text = ''
-      while (i <= length) {
-        if (i === length) {
-          throw unterminatedError({ type: 'identifier', positionStart: pos, positionEnd: length })
+      let value = ''
+      while (i <= len) {
+        if (i === len) {
+          throw new ParseError({
+            message: `Unterminated ${type} starting at position ${positionStart}`,
+            positionStart,
+            positionEnd: i,
+          })
         }
         const c = nextChar()
         if (c === quote) {
           // check for escaped quote
           if (peek() === quote) {
-            text += quote
-            nextChar()
+            value += quote
+            i++
             continue
           }
+          // end quote
           break
         }
-        text += c
+        value += c
       }
-      tokens.push({
-        type: 'identifier',
-        value: text,
-        positionStart: pos,
-        positionEnd: i,
-      })
+      tokens.push({ type, value, positionStart, positionEnd: i })
       continue
     }
 
-    // two-character operators
-    if (ch === '<' || ch === '>' || ch === '!' || ch === '=') {
+    // operators
+    if ('<>!=+-*/%'.includes(ch)) {
       let op = nextChar()
       if ((op === '<' || op === '>' || op === '!') && peek() === '=') {
         op += nextChar()
@@ -288,76 +181,64 @@ export function tokenizeSql(sql) {
       tokens.push({
         type: 'operator',
         value: op,
-        positionStart: pos,
-        positionEnd: i,
-      })
-      continue
-    }
-
-    // single-char operators
-    if (ch === '*' || ch === '+' || ch === '-' || ch === '/' || ch === '%') {
-      nextChar()
-      tokens.push({
-        type: 'operator',
-        value: ch,
-        positionStart: pos,
+        positionStart,
         positionEnd: i,
       })
       continue
     }
 
     if (ch === ',') {
-      nextChar()
+      i++
       tokens.push({
         type: 'comma',
         value: ',',
-        positionStart: pos,
+        positionStart,
         positionEnd: i,
       })
       continue
     }
 
     if (ch === '.') {
-      nextChar()
+      i++
       tokens.push({
         type: 'dot',
         value: '.',
-        positionStart: pos,
+        positionStart,
         positionEnd: i,
       })
       continue
     }
 
     if (ch === '(' || ch === ')') {
-      nextChar()
+      i++
       tokens.push({
         type: 'paren',
         value: ch,
-        positionStart: pos,
+        positionStart,
         positionEnd: i,
       })
       continue
     }
 
     if (ch === ';') {
-      nextChar()
+      i++
       tokens.push({
         type: 'semicolon',
         value: ';',
-        positionStart: pos,
+        positionStart,
         positionEnd: i,
       })
       continue
     }
 
-    throw unexpectedCharError({ char: ch, positionStart: pos, expectsSelect: !tokens.length })
+    throw new UnexpectedCharError({ char: ch, positionStart, expectsSelect: !tokens.length })
   }
 
   tokens.push({
     type: 'eof',
     value: '',
-    positionStart: length,
-    positionEnd: length,
+    positionStart: len,
+    positionEnd: len,
   })
 
   return tokens

package/src/plan/columns.js

@@ -1,35 +1,48 @@
+import { derivedAlias } from '../expression/alias.js'
+
+/**
+ * @import { AsyncDataSource, ExprNode, FromSubquery, FromTable, SelectStatement, Statement } from '../types.js'
+ */
+
 /**
- * @import { ExprNode, SelectStatement } from '../types.js'
+ * @param {FromTable | FromSubquery} from
+ * @returns {string}
  */
+export function fromAlias(from) {
+  return from.alias ?? (from.type === 'table' ? from.table : 'table')
+}
 
 /**
  * Extracts per-table column names needed from a SELECT statement with joins.
  * Returns a Map from table alias to column names, or undefined if all columns needed.
  *
- * @param {SelectStatement} select
+ * @param {object} options
+ * @param {SelectStatement} options.select
+ * @param {string[]} [options.parentColumns] - columns needed by the parent query
  * @returns {Map<string, string[] | undefined>}
  */
-export function extractColumns(select) {
+export function extractColumns({ select, parentColumns }) {
   /** @type {Map<string, string[] | undefined>} */
   const result = new Map()
 
   // Build alias list from FROM + JOINs
-  const fromAlias = select.from.kind === 'table'
-    ? select.from.alias ?? select.from.table
-    : select.from.alias
-  const aliases = [fromAlias]
+  const aliases = [fromAlias(select.from)]
   for (const join of select.joins) {
     aliases.push(join.alias ?? join.table)
   }
 
   // If any unqualified SELECT * exists, all tables need all columns
-  if (select.columns.some(col => col.kind === 'star' && !col.table)) {
-    /** @type {Map<string, string[] | undefined>} */
-    const result = new Map()
-    for (const alias of aliases) {
-      result.set(alias, undefined)
+  if (select.columns.some(col => col.type === 'star' && !col.table)) {
+    if (!parentColumns) {
+      /** @type {Map<string, string[] | undefined>} */
+      const result = new Map()
+      for (const alias of aliases) {
+        result.set(alias, undefined)
+      }
+      return result
     }
-    return result
+    // With parentColumns, fall through to collect internal clause columns
+    // and seed with what the parent needs
   }
 
   // Track per-table columns needed; undefined means all columns (table.*)
@@ -37,8 +50,11 @@ export function extractColumns(select) {
   const perTable = new Map(aliases.map(alias => [alias, new Set()]))
 
   // Collect all identifiers from all clauses
-  /** @type {Set<string>} */
-  const identifiers = new Set()
+  // For SELECT *, parent column names are real table columns, so seed them
+  // directly. For non-star queries, parent names may be aliases and are
+  // handled below by filtering derived columns and collecting from expressions.
+  const hasStar = select.columns.some(col => col.type === 'star' && !col.table)
+  const identifiers = new Set(hasStar ? parentColumns : undefined)
 
   // Collect ORDER BY identifiers, excluding SELECT aliases (their underlying
   // columns are already collected from select.columns expressions above)
@@ -46,10 +62,15 @@ export function extractColumns(select) {
   const selectAliases = new Set()
 
   for (const col of select.columns) {
-    if (col.kind === 'star' && col.table) {
+    if (col.type === 'star' && col.table) {
       // SELECT table.* means all columns needed
       perTable.set(col.table, undefined)
-    } else if (col.kind === 'derived') {
+    } else if (col.type === 'derived') {
+      // When parentColumns is set, skip columns the parent doesn't need
+      if (parentColumns) {
+        const outputName = col.alias ?? derivedAlias(col.expr)
+        if (!parentColumns.includes(outputName)) continue
+      }
       collectColumnsFromExpr(col.expr, identifiers)
       if (col.alias) {
         selectAliases.add(col.alias)
@@ -145,3 +166,80 @@ function collectColumnsFromExpr(expr, columns, aliases) {
   }
   // No columns: count(*), literal, interval, exists, not exists, subquery
 }
+
+/**
+ * Infers output columns for set-operation validation.
+ *
+ * @param {object} options
+ * @param {Statement} options.stmt
+ * @param {Map<string, string[]>} [options.cteColumns]
+ * @param {Record<string, AsyncDataSource>} [options.tables]
+ * @returns {string[]}
+ */
+export function inferStatementColumns({ stmt, cteColumns, tables }) {
+  if (stmt.type === 'with') {
+    return inferStatementColumns({ stmt: stmt.query, cteColumns, tables })
+  }
+  if (stmt.type === 'compound') {
+    return inferStatementColumns({ stmt: stmt.left, cteColumns, tables })
+  }
+
+  const sourceColumns = inferSelectSourceColumns({ select: stmt, cteColumns, tables })
+  /** @type {string[]} */
+  const result = []
+
+  for (const col of stmt.columns) {
+    if (col.type === 'star') {
+      result.push(...sourceColumns)
+    } else {
+      result.push(col.alias ?? derivedAlias(col.expr))
+    }
+  }
+
+  return result
+}
+
+/**
+ * Infers the source columns available before SELECT projection.
+ * Mirrors the column ordering used by join row materialization.
+ *
+ * @param {object} options
+ * @param {SelectStatement} options.select
+ * @param {Map<string, string[]>} [options.cteColumns]
+ * @param {Record<string, AsyncDataSource>} [options.tables]
+ * @returns {string[]}
+ */
+function inferSelectSourceColumns({ select, cteColumns, tables }) {
+  if (select.from.type === 'subquery') {
+    return inferStatementColumns({ stmt: select.from.query, cteColumns, tables })
+  }
+
+  if (!select.joins.length) {
+    return lookupTableColumns(select.from.table, cteColumns, tables)
+  }
+
+  // Collect all sources, then prefix each table's columns
+  /** @type {string[]} */
+  const result = []
+  const fromAlias = select.from.alias ?? select.from.table
+  for (const col of lookupTableColumns(select.from.table, cteColumns, tables)) {
+    result.push(`${fromAlias}.${col}`, col)
+  }
+  for (const join of select.joins) {
+    const alias = join.alias ?? join.table
+    for (const col of lookupTableColumns(join.table, cteColumns, tables)) {
+      result.push(`${alias}.${col}`, col)
+    }
+  }
+  return result
+}
+
+/**
+ * @param {string} table
+ * @param {Map<string, string[]>} [cteColumns]
+ * @param {Record<string, AsyncDataSource>} [tables]
+ * @returns {string[]}
+ */
+function lookupTableColumns(table, cteColumns, tables) {
+  return cteColumns?.get(table.toLowerCase()) ?? tables?.[table]?.columns ?? []
+}