squirreling 0.2.3 → 0.2.5

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "squirreling",
-  "version": "0.2.3",
+  "version": "0.2.5",
   "description": "Squirreling SQL Engine",
   "author": "Hyperparam",
   "homepage": "https://hyperparam.app",

package/src/execute/execute.js

@@ -1,13 +1,13 @@
-/**
- * @import { DataSource, ExecuteSqlOptions, FunctionColumn, FunctionNode, OrderByItem, RowSource, SelectStatement, SqlPrimitive } from '../types.js'
- */
-
 import { defaultAggregateAlias, evaluateAggregate } from './aggregates.js'
 import { evaluateExpr } from './expression.js'
 import { evaluateHavingExpr } from './having.js'
 import { parseSql } from '../parse/parse.js'
 import { createMemorySource, createRowAccessor } from '../backend/memory.js'
 
+/**
+ * @import { DataSource, ExecuteSqlOptions, ExprNode, OrderByItem, RowSource, SelectStatement, SqlPrimitive } from '../types.js'
+ */
+
 /**
  * Executes a SQL SELECT query against a data source
  *
@@ -21,21 +21,31 @@ export function executeSql({ source, query }) {
 }
 
 /**
- * Generates a default alias name for a string function
+ * Generates a default alias for a derived column expression
  *
- * @param {FunctionColumn} col - the function column definition
- * @returns {string} the generated alias (e.g., "upper_name", "concat_a_b")
+ * @param {ExprNode} expr - the expression node
+ * @returns {string} the generated alias
  */
-function defaultFunctionAlias(col) {
-  const base = col.func.toLowerCase()
-  // Try to extract column names from identifier arguments
-  const columnNames = col.args
-    .filter(arg => arg.type === 'identifier')
-    .map(arg => arg.name)
-  if (columnNames.length > 0) {
-    return base + '_' + columnNames.join('_')
+function defaultDerivedAlias(expr) {
+  if (expr.type === 'identifier') {
+    return expr.name
   }
-  return base
+  if (expr.type === 'function') {
+    const base = expr.name.toLowerCase()
+    // Try to extract column names from identifier arguments
+    const columnNames = expr.args
+      .filter(arg => arg.type === 'identifier')
+      .map(arg => arg.name)
+    if (columnNames.length > 0) {
+      return base + '_' + columnNames.join('_')
+    }
+    return base
+  }
+  if (expr.type === 'cast') return 'cast_expr'
+  if (expr.type === 'unary' && expr.argument.type === 'identifier') {
+    return expr.op === '-' ? 'neg_' + expr.argument.name : 'expr'
+  }
+  return 'expr'
 }
 
 /**
@@ -118,6 +128,24 @@ function applyOrderBy(rows, orderBy) {
       const dir = term.direction
       const av = evaluateExpr(term.expr, createRowAccessor(a))
       const bv = evaluateExpr(term.expr, createRowAccessor(b))
+
+      // Handle NULLS FIRST / NULLS LAST
+      const aIsNull = av == null
+      const bIsNull = bv == null
+
+      if (aIsNull || bIsNull) {
+        if (aIsNull && bIsNull) continue // both null, try next sort term
+
+        // Determine null ordering
+        const nullsFirst = term.nulls === 'LAST' ? false : true // default is NULLS FIRST
+
+        if (aIsNull) {
+          return nullsFirst ? -1 : 1
+        } else {
+          return nullsFirst ? 1 : -1
+        }
+      }
+
       const cmp = compareValues(av, bv)
       if (cmp !== 0) {
         return dir === 'DESC' ? -cmp : cmp
@@ -214,20 +242,9 @@ function evaluateSelectAst(select, dataSource) {
           continue
         }
 
-        if (col.kind === 'column') {
-          const name = col.column
-          const alias = col.alias ?? name
-          // Evaluate on first row of group (all rows have same value for GROUP BY columns)
-          resultRow[alias] = group[0]?.getCell(name)
-          continue
-        }
-
-        if (col.kind === 'function') {
-          // Evaluate function on the first row of the group
-          /** @type {FunctionNode} */
-          const funcNode = { type: 'function', name: col.func, args: col.args }
-          const alias = col.alias ?? defaultFunctionAlias(col)
-          const value = group.length > 0 ? evaluateExpr(funcNode, group[0]) : undefined
+        if (col.kind === 'derived') {
+          const alias = col.alias ?? defaultDerivedAlias(col.expr)
+          const value = group.length > 0 ? evaluateExpr(col.expr, group[0]) : undefined
           resultRow[alias] = value
           continue
         }
@@ -238,13 +255,6 @@ function evaluateSelectAst(select, dataSource) {
           resultRow[alias] = value
           continue
         }
-
-        if (col.kind === 'operation') {
-          const alias = col.alias ?? 'expr'
-          const value = group.length > 0 ? evaluateExpr(col.expr, group[0]) : undefined
-          resultRow[alias] = value
-          continue
-        }
       }
 
       // Apply HAVING filter before adding to projected results
@@ -269,18 +279,8 @@ function evaluateSelectAst(select, dataSource) {
           for (const key of keys) {
             outRow[key] = row.getCell(key)
           }
-        } else if (col.kind === 'column') {
-          const name = col.column
-          const alias = col.alias ?? name
-          outRow[alias] = row.getCell(name)
-        } else if (col.kind === 'function') {
-          /** @type {FunctionNode} */
-          const funcNode = { type: 'function', name: col.func, args: col.args }
-          const value = evaluateExpr(funcNode, row)
-          const alias = col.alias ?? defaultFunctionAlias(col)
-          outRow[alias] = value
-        } else if (col.kind === 'operation') {
-          const alias = col.alias ?? 'expr'
+        } else if (col.kind === 'derived') {
+          const alias = col.alias ?? defaultDerivedAlias(col.expr)
           const value = evaluateExpr(col.expr, row)
           outRow[alias] = value
         } else if (col.kind === 'aggregate') {

package/src/execute/expression.js

@@ -131,23 +131,23 @@ export function evaluateExpr(node, row) {
       return String(val).length
     }
 
-    if (funcName === 'SUBSTRING') {
+    if (funcName === 'SUBSTRING' || funcName === 'SUBSTR') {
       if (args.length < 2 || args.length > 3) {
-        throw new Error('SUBSTRING requires 2 or 3 arguments')
+        throw new Error(`${funcName} requires 2 or 3 arguments`)
       }
       const str = args[0]
       if (str == null) return null
       const strVal = String(str)
       const start = Number(args[1])
       if (!Number.isInteger(start) || start < 1) {
-        throw new Error('SUBSTRING start position must be a positive integer')
+        throw new Error(`${funcName} start position must be a positive integer`)
       }
       // SQL uses 1-based indexing
       const startIdx = start - 1
       if (args.length === 3) {
         const len = Number(args[2])
         if (!Number.isInteger(len) || len < 0) {
-          throw new Error('SUBSTRING length must be a non-negative integer')
+          throw new Error(`${funcName} length must be a non-negative integer`)
         }
         return strVal.substring(startIdx, startIdx + len)
       }
@@ -190,6 +190,24 @@ export function evaluateExpr(node, row) {
     throw new Error('Unsupported CAST to type ' + node.toType)
   }
 
+  // IN and NOT IN with value lists
+  if (node.type === 'in valuelist') {
+    const exprVal = evaluateExpr(node.expr, row)
+    for (const valueNode of node.values) {
+      const val = evaluateExpr(valueNode, row)
+      if (exprVal === val) return true
+    }
+    return false
+  }
+  if (node.type === 'not in valuelist') {
+    const exprVal = evaluateExpr(node.expr, row)
+    for (const valueNode of node.values) {
+      const val = evaluateExpr(valueNode, row)
+      if (exprVal === val) return false
+    }
+    return true
+  }
+
   // IN and NOT IN with subqueries
   if (node.type === 'in') {
     throw new Error('WHERE IN with subqueries is not yet supported.')

package/src/parse/expression.js

@@ -1,5 +1,7 @@
+import { isAggregateFunc, isStringFunc } from '../validation.js'
+
 /**
- * @import { ExprCursor, ExprNode, BinaryOp, SelectStatement } from '../types.js'
+ * @import { ExprCursor, ExprNode, BinaryOp } from '../types.js'
  */
 
 /**
@@ -11,13 +13,10 @@ export function parseExpression(c) {
 }
 
 /**
- * Exposed so SELECT list parsing can reuse the same notion of "primary"
- * for function arguments, etc.
- *
  * @param {ExprCursor} c
  * @returns {ExprNode}
  */
-export function parsePrimary(c) {
+function parsePrimary(c) {
   const tok = c.current()
 
   if (tok.type === 'paren' && tok.value === '(') {
@@ -30,10 +29,30 @@ export function parsePrimary(c) {
   if (tok.type === 'identifier') {
     const next = c.peek(1)
 
+    // CAST expression
+    if (tok.value === 'CAST' && next.type === 'paren' && next.value === '(') {
+      c.consume() // CAST
+      c.consume() // '('
+      const expr = parseExpression(c)
+      c.expect('keyword', 'AS')
+      const typeTok = c.expectIdentifier()
+      c.expect('paren', ')')
+      return {
+        type: 'cast',
+        expr,
+        toType: typeTok.value,
+      }
+    }
+
     // function call
     if (next.type === 'paren' && next.value === '(') {
       const funcName = tok.value
-      // TODO: validate function name
+
+      // validate function names
+      if (!isStringFunc(funcName) && !isAggregateFunc(funcName)) {
+        throw new Error(`Unknown function "${funcName}" at position ${tok.position}`)
+      }
+
       c.consume() // function name
       c.consume() // '('
 
@@ -281,28 +300,80 @@ function parseComparison(c) {
     if (nextTok.type === 'keyword' && nextTok.value === 'IN') {
       c.consume() // NOT
       c.consume() // IN
-      if (!c.parseSubquery) {
-        throw new Error('Subquery parsing not available in this context')
+
+      // Check if it's a subquery or a list of values by peeking ahead
+      // parseSubquery expects to consume the opening paren itself
+      const parenTok = c.current()
+      if (parenTok.type !== 'paren' || parenTok.value !== '(') {
+        throw new Error('Expected ( after IN')
       }
-      const subquery = c.parseSubquery()
-      return {
-        type: 'not in',
-        expr: left,
-        subquery,
+      const peekTok = c.peek(1)
+      if (peekTok.type === 'keyword' && peekTok.value === 'SELECT') {
+        // Subquery - let parseSubquery handle the parens
+        if (!c.parseSubquery) {
+          throw new Error('Subquery parsing not available in this context')
+        }
+        const subquery = c.parseSubquery()
+        return {
+          type: 'not in',
+          expr: left,
+          subquery,
+        }
+      } else {
+        // Parse list of values - we handle the parens
+        c.consume() // '('
+        /** @type {ExprNode[]} */
+        const values = []
+        while (true) {
+          values.push(parseExpression(c))
+          if (!c.match('comma')) break
+        }
+        c.expect('paren', ')')
+        return {
+          type: 'not in valuelist',
+          expr: left,
+          values,
+        }
       }
     }
   }
 
   if (tok.type === 'keyword' && tok.value === 'IN') {
     c.consume() // IN
-    if (!c.parseSubquery) {
-      throw new Error('Subquery parsing not available in this context')
+
+    // Check if it's a subquery or a list of values by peeking ahead
+    // parseSubquery expects to consume the opening paren itself
+    const parenTok = c.current()
+    if (parenTok.type !== 'paren' || parenTok.value !== '(') {
+      throw new Error('Expected ( after IN')
     }
-    const subquery = c.parseSubquery()
-    return {
-      type: 'in',
-      expr: left,
-      subquery,
+    const peekTok = c.peek(1)
+    if (peekTok.type === 'keyword' && peekTok.value === 'SELECT') {
+      // Subquery - let parseSubquery handle the parens
+      if (!c.parseSubquery) {
+        throw new Error('Subquery parsing not available in this context')
+      }
+      const subquery = c.parseSubquery()
+      return {
+        type: 'in',
+        expr: left,
+        subquery,
+      }
+    } else {
+      // Parse list of values - we handle the parens
+      c.consume() // '('
+      /** @type {ExprNode[]} */
+      const values = []
+      while (true) {
+        values.push(parseExpression(c))
+        if (!c.match('comma')) break
+      }
+      c.expect('paren', ')')
+      return {
+        type: 'in valuelist',
+        expr: left,
+        values,
+      }
     }
   }
 

package/src/parse/parse.js

@@ -1,11 +1,11 @@
+import { tokenize } from './tokenize.js'
+import { parseExpression } from './expression.js'
+import { isAggregateFunc } from '../validation.js'
+
 /**
- * @import { AggregateColumn, AggregateArg, AggregateFunc, ExprCursor, ExprNode, FromSubquery, JoinClause, JoinType, OrderByItem, ParserState, SelectStatement, SelectColumn, StringFunc, Token, TokenType } from '../types.js'
+ * @import { AggregateColumn, AggregateArg, AggregateFunc, ExprCursor, ExprNode, FromSubquery, JoinClause, JoinType, OrderByItem, ParserState, SelectStatement, SelectColumn, Token, TokenType } from '../types.js'
  */
 
-import { tokenize } from './tokenize.js'
-import { parseExpression, parsePrimary } from './expression.js'
-import { isAggregateFunc, isStringFunc } from '../validation.js'
-
 // Keywords that cannot be used as implicit aliases after a column
 const RESERVED_AFTER_COLUMN = new Set([
   'FROM',
@@ -112,6 +112,7 @@ function expectIdentifier(state) {
 
 /**
  * Creates an ExprCursor adapter for the ParserState.
+ *
  * @param {ParserState} state
  * @returns {ExprCursor}
  */
@@ -141,6 +142,20 @@ function parseSelectList(state) {
   const cols = []
   const tok = current(state)
 
+  // Check for qualified asterisk (table.*)
+  if (tok.type === 'identifier') {
+    const next = peekToken(state, 1)
+    const nextNext = peekToken(state, 2)
+    if (next.type === 'dot' && nextNext.type === 'operator' && nextNext.value === '*') {
+      const tableTok = consume(state) // consume table name
+      consume(state) // consume dot
+      consume(state) // consume asterisk
+      cols.push({ kind: 'star', table: tableTok.value })
+      return cols
+    }
+  }
+
+  // Check for unqualified asterisk (*)
   if (tok.type === 'operator' && tok.value === '*') {
     consume(state)
     cols.push({ kind: 'star' })
@@ -166,56 +181,20 @@ function parseSelectItem(state) {
     throw parseError(state, 'column name or expression')
   }
 
-  if (tok.type === 'identifier' && tok.value === 'CAST') {
-    expectIdentifier(state) // consume CAST
-    expect(state, 'paren', '(')
-    const cursor = createExprCursor(state)
-    const expr = parseExpression(cursor)
-    expect(state, 'keyword', 'AS')
-    const typeTok = expectIdentifier(state)
-    expect(state, 'paren', ')')
-    const alias = parseAs(state)
-    return {
-      kind: 'operation',
-      expr: { type: 'cast', expr, toType: typeTok.value },
-      alias,
-    }
-  }
-
-  if (tok.type === 'operator') {
-    // Handle SELECT expression AS alias
-    const cursor = createExprCursor(state)
-    const expr = parseExpression(cursor)
-    const alias = parseAs(state)
-    return { kind: 'operation', expr, alias }
-  }
-
   const next = peekToken(state, 1)
-  const upper = tok.value.toUpperCase()
-
   if (next.type === 'paren' && next.value === '(') {
-    expectIdentifier(state) // consume function name
+    const upper = tok.value.toUpperCase()
     if (isAggregateFunc(upper)) {
+      expectIdentifier(state) // consume function name
       return parseAggregateItem(state, upper)
     }
-    if (isStringFunc(upper)) {
-      return parseStringFunctionItem(state, upper)
-    }
-  }
-
-  consume(state)
-  let column = tok.value
-
-  // Handle dot notation (table.column)
-  if (current(state).type === 'dot') {
-    consume(state) // consume the dot
-    const columnTok = expectIdentifier(state)
-    column += '.' + columnTok.value
   }
 
+  // Delegate to expression parser
+  const cursor = createExprCursor(state)
+  const expr = parseExpression(cursor)
   const alias = parseAs(state)
-
-  return { kind: 'column', column, alias }
+  return { kind: 'derived', expr, alias }
 }
 
 /**
@@ -261,34 +240,6 @@ function parseAggregateItem(state, func) {
   return { kind: 'aggregate', func, arg, alias }
 }
 
-/**
- * @param {ParserState} state
- * @param {StringFunc} func
- * @returns {SelectColumn}
- */
-function parseStringFunctionItem(state, func) {
-  expect(state, 'paren', '(')
-
-  /** @type {ExprNode[]} */
-  const args = []
-
-  // Parse comma-separated arguments
-  if (current(state).type !== 'paren' || current(state).value !== ')') {
-    const cursor = createExprCursor(state)
-    while (true) {
-      const arg = parsePrimary(cursor)
-      args.push(arg)
-      if (!match(state, 'comma')) break
-    }
-  }
-
-  expect(state, 'paren', ')')
-
-  const alias = parseAs(state)
-
-  return { kind: 'function', func, args, alias }
-}
-
 /**
  * @param {ParserState} state
  * @returns {string | undefined}
@@ -481,9 +432,24 @@ function parseSelectInternal(state) {
       } else if (match(state, 'keyword', 'DESC')) {
         direction = 'DESC'
       }
+      /** @type {'FIRST' | 'LAST' | undefined} */
+      let nulls
+      if (match(state, 'keyword', 'NULLS')) {
+        const tok = current(state)
+        if (tok.type === 'identifier' && tok.value.toUpperCase() === 'FIRST') {
+          consume(state)
+          nulls = 'FIRST'
+        } else if (tok.type === 'identifier' && tok.value.toUpperCase() === 'LAST') {
+          consume(state)
+          nulls = 'LAST'
+        } else {
+          throw parseError(state, 'FIRST or LAST after NULLS')
+        }
+      }
       orderBy.push({
         expr,
         direction,
+        nulls,
       })
       if (!match(state, 'comma')) break
     }

package/src/parse/tokenize.js

@@ -16,6 +16,7 @@ const KEYWORDS = new Set([
   'ORDER',
   'ASC',
   'DESC',
+  'NULLS',
   'LIMIT',
   'OFFSET',
   'AS',

package/src/types.d.ts

@@ -34,26 +34,6 @@ export interface SelectStatement {
   offset?: number
 }
 
-export type TokenType =
-  | 'keyword'
-  | 'identifier'
-  | 'number'
-  | 'string'
-  | 'operator'
-  | 'comma'
-  | 'dot'
-  | 'paren'
-  | 'semicolon'
-  | 'eof'
-
-export interface Token {
-  type: TokenType
-  value: string
-  position: number
-  numericValue?: number
-  originalValue?: string
-}
-
 export type BinaryOp =
   | 'AND'
   | 'OR'
@@ -108,12 +88,18 @@ export interface BetweenNode {
   upper: ExprNode
 }
 
-export interface InNode {
+export interface InSubqueryNode {
   type: 'in' | 'not in'
   expr: ExprNode
   subquery: SelectStatement
 }
 
+export interface InValuesNode {
+  type: 'in valuelist' | 'not in valuelist'
+  expr: ExprNode
+  values: ExprNode[]
+}
+
 export interface ExistsNode {
   type: 'exists' | 'not exists'
   subquery: SelectStatement
@@ -127,23 +113,19 @@ export type ExprNode =
   | FunctionNode
   | CastNode
   | BetweenNode
-  | InNode
+  | InSubqueryNode
+  | InValuesNode
   | ExistsNode
 
 export interface StarColumn {
   kind: 'star'
-  alias?: string
-}
-
-export interface SimpleColumn {
-  kind: 'column'
-  column: string
+  table?: string
   alias?: string
 }
 
 export type AggregateFunc = 'COUNT' | 'SUM' | 'AVG' | 'MIN' | 'MAX'
 
-export type StringFunc = 'UPPER' | 'LOWER' | 'CONCAT' | 'LENGTH' | 'SUBSTRING' | 'TRIM'
+export type StringFunc = 'UPPER' | 'LOWER' | 'CONCAT' | 'LENGTH' | 'SUBSTRING' | 'SUBSTR' | 'TRIM'
 
 export interface AggregateArgStar {
   kind: 'star'
@@ -163,24 +145,18 @@ export interface AggregateColumn {
   alias?: string
 }
 
-export interface FunctionColumn {
-  kind: 'function'
-  func: StringFunc
-  args: ExprNode[]
-  alias?: string
-}
-
-export interface OperationColumn {
-  kind: 'operation'
+export interface DerivedColumn {
+  kind: 'derived'
   expr: ExprNode
   alias?: string
 }
 
-export type SelectColumn = StarColumn | SimpleColumn | AggregateColumn | FunctionColumn | OperationColumn
+export type SelectColumn = StarColumn | AggregateColumn | DerivedColumn
 
 export interface OrderByItem {
   expr: ExprNode
   direction: 'ASC' | 'DESC'
+  nulls?: 'FIRST' | 'LAST'
 }
 
 export type JoinType = 'INNER' | 'LEFT' | 'RIGHT' | 'FULL' | 'CROSS'
@@ -205,3 +181,24 @@ export interface ExprCursor {
   expectIdentifier(): Token
   parseSubquery?: () => SelectStatement
 }
+
+// Tokenizer types
+export type TokenType =
+  | 'keyword'
+  | 'identifier'
+  | 'number'
+  | 'string'
+  | 'operator'
+  | 'comma'
+  | 'dot'
+  | 'paren'
+  | 'semicolon'
+  | 'eof'
+
+export interface Token {
+  type: TokenType
+  value: string
+  position: number
+  numericValue?: number
+  originalValue?: string
+}

package/src/validation.js

@@ -13,5 +13,5 @@ export function isAggregateFunc(name) {
  * @returns {name is StringFunc}
  */
 export function isStringFunc(name) {
-  return ['UPPER', 'LOWER', 'CONCAT', 'LENGTH', 'SUBSTRING', 'TRIM'].includes(name)
+  return ['UPPER', 'LOWER', 'CONCAT', 'LENGTH', 'SUBSTRING', 'SUBSTR', 'TRIM'].includes(name)
 }