squirreling 0.10.0 → 0.10.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "squirreling",
-  "version": "0.10.0",
+  "version": "0.10.2",
   "description": "Squirreling Async SQL Engine",
   "author": "Hyperparam",
   "homepage": "https://hyperparam.app",
@@ -10,7 +10,9 @@
     "dataset",
     "hyperparam",
     "hyparquet",
-    "parquet"
+    "parquet",
+    "query",
+    "relational"
   ],
   "license": "MIT",
   "repository": {
@@ -37,11 +39,11 @@
     "test": "vitest run"
   },
   "devDependencies": {
-    "@types/node": "25.3.5",
-    "@vitest/coverage-v8": "4.0.18",
+    "@types/node": "25.5.0",
+    "@vitest/coverage-v8": "4.1.0",
     "eslint": "9.39.2",
-    "eslint-plugin-jsdoc": "62.7.1",
+    "eslint-plugin-jsdoc": "62.8.0",
     "typescript": "5.9.3",
-    "vitest": "4.0.18"
+    "vitest": "4.1.0"
   }
 }

package/src/ast.d.ts

@@ -0,0 +1,184 @@
+export type SqlPrimitive =
+  | string
+  | number
+  | bigint
+  | boolean
+  | Date
+  | null
+  | SqlPrimitive[]
+  | Record<string, any>
+
+export interface SelectStatement {
+  with?: WithClause
+  distinct: boolean
+  columns: SelectColumn[]
+  from: FromTable | FromSubquery
+  joins: JoinClause[]
+  where?: ExprNode
+  groupBy: ExprNode[]
+  having?: ExprNode
+  orderBy: OrderByItem[]
+  limit?: number
+  offset?: number
+}
+
+export interface WithClause {
+  ctes: CTEDefinition[]
+}
+
+export interface CTEDefinition {
+  name: string
+  query: SelectStatement
+}
+
+export interface FromTable extends AstBase {
+  kind: 'table'
+  table: string
+  alias?: string
+}
+
+export interface FromSubquery {
+  kind: 'subquery'
+  query: SelectStatement
+  alias: string
+}
+
+export type ArithmeticOp = '+' | '-' | '*' | '/' | '%'
+
+export type BinaryOp = 'AND' | 'OR' | 'LIKE' | ComparisonOp | ArithmeticOp
+
+export type ComparisonOp = '=' | '!=' | '<>' | '<' | '>' | '<=' | '>='
+
+export interface LiteralNode extends AstBase {
+  type: 'literal'
+  value: SqlPrimitive
+}
+
+export interface IdentifierNode extends AstBase {
+  type: 'identifier'
+  name: string
+}
+
+export interface UnaryNode extends AstBase {
+  type: 'unary'
+  op: 'NOT' | 'IS NULL' | 'IS NOT NULL' | '-'
+  argument: ExprNode
+}
+
+export interface BinaryNode extends AstBase {
+  type: 'binary'
+  op: BinaryOp
+  left: ExprNode
+  right: ExprNode
+}
+
+export interface FunctionNode extends AstBase {
+  type: 'function'
+  funcName: string
+  args: ExprNode[]
+  distinct?: boolean
+  filter?: ExprNode
+}
+
+export type CastType = 'TEXT' | 'STRING' | 'VARCHAR' | 'INTEGER' | 'INT' | 'BIGINT' | 'FLOAT' | 'REAL' | 'DOUBLE' | 'BOOLEAN' | 'BOOL'
+
+export interface CastNode extends AstBase {
+  type: 'cast'
+  expr: ExprNode
+  toType: CastType
+}
+
+export interface InSubqueryNode extends AstBase {
+  type: 'in'
+  expr: ExprNode
+  subquery: SelectStatement
+}
+
+export interface InValuesNode extends AstBase {
+  type: 'in valuelist'
+  expr: ExprNode
+  values: ExprNode[]
+}
+
+export interface ExistsNode extends AstBase {
+  type: 'exists' | 'not exists'
+  subquery: SelectStatement
+}
+
+export interface WhenClause extends AstBase {
+  condition: ExprNode
+  result: ExprNode
+}
+
+export interface CaseNode extends AstBase {
+  type: 'case'
+  caseExpr?: ExprNode
+  whenClauses: WhenClause[]
+  elseResult?: ExprNode
+}
+
+export interface SubqueryNode extends AstBase {
+  type: 'subquery'
+  subquery: SelectStatement
+}
+
+export type IntervalUnit = 'DAY' | 'MONTH' | 'YEAR' | 'HOUR' | 'MINUTE' | 'SECOND'
+
+export interface IntervalNode extends AstBase {
+  type: 'interval'
+  value: number
+  unit: IntervalUnit
+}
+
+export interface StarNode extends AstBase {
+  type: 'star'
+}
+
+export type ExprNode =
+  | LiteralNode
+  | IdentifierNode
+  | UnaryNode
+  | BinaryNode
+  | FunctionNode
+  | CastNode
+  | InSubqueryNode
+  | InValuesNode
+  | ExistsNode
+  | CaseNode
+  | SubqueryNode
+  | IntervalNode
+  | StarNode
+
+export interface StarColumn {
+  kind: 'star'
+  table?: string
+}
+
+export interface DerivedColumn {
+  kind: 'derived'
+  expr: ExprNode
+  alias?: string
+}
+
+export type SelectColumn = StarColumn | DerivedColumn
+
+export interface OrderByItem {
+  expr: ExprNode
+  direction: 'ASC' | 'DESC'
+  nulls?: 'FIRST' | 'LAST'
+}
+
+export type JoinType = 'INNER' | 'LEFT' | 'RIGHT' | 'FULL' | 'CROSS' | 'POSITIONAL'
+
+export interface JoinClause extends AstBase {
+  joinType: JoinType
+  table: string
+  alias?: string
+  on?: ExprNode
+}
+
+// All AST node derive from this base, which includes position info for error reporting and other purposes
+interface AstBase {
+  positionStart: number // start position in query (0-based, inclusive)
+  positionEnd: number // end position in query (0-based, exclusive)
+}

package/src/execute/execute.js

@@ -1,9 +1,9 @@
 import { memorySource } from '../backend/dataSource.js'
-import { tableNotFoundError } from '../executionErrors.js'
 import { derivedAlias } from '../expression/alias.js'
 import { evaluateExpr } from '../expression/evaluate.js'
 import { parseSql } from '../parse/parse.js'
 import { planSql } from '../plan/plan.js'
+import { tableNotFoundError } from '../validation/planErrors.js'
 import { executeHashAggregate, executeScalarAggregate } from './aggregates.js'
 import { executeHashJoin, executeNestedLoopJoin, executePositionalJoin } from './join.js'
 import { executeSort } from './sort.js'
@@ -46,7 +46,7 @@ export async function* executeSql({ tables, query, functions, signal }) {
  * @yields {AsyncRow}
  */
 export async function* executeSelect({ select, context }) {
-  const plan = planSql({ query: select, functions: context.functions })
+  const plan = planSql({ query: select, functions: context.functions, tables: context.tables })
   yield* executePlan({ plan, context })
 }
 
@@ -98,7 +98,7 @@ async function* executeScan(plan, context) {
   // check table
   const table = tables[plan.table]
   if (!table) {
-    throw tableNotFoundError({ tableName: plan.table })
+    throw tableNotFoundError({ table: plan.table, tables })
   }
   // check columns
   const missingColumn = plan.hints.columns?.find(col => !table.columns.includes(col))
@@ -119,7 +119,7 @@ async function* executeScan(plan, context) {
 
   // Apply WHERE if data source did not
   if (!appliedWhere && plan.hints.where) {
-    result = filterRows(result, plan.hints.where, context)
+    result = filterRows(result, plan.hints.where, context, plan.hints.limit)
   }
 
   // Apply LIMIT/OFFSET if data source did not
@@ -140,7 +140,7 @@ async function* executeScan(plan, context) {
 async function* executeCount(plan, { tables, signal }) {
   const table = tables[plan.table]
   if (!table) {
-    throw tableNotFoundError({ tableName: plan.table })
+    throw tableNotFoundError({ table: plan.table, tables })
   }
 
   // Use source numRows if available
@@ -174,15 +174,42 @@ async function* executeCount(plan, { tables, signal }) {
  * @param {AsyncIterable<AsyncRow>} rows
  * @param {ExprNode} condition
  * @param {ExecuteContext} context
+ * @param {number} [limit] - downstream LIMIT hint for chunk sizing
  * @yields {AsyncRow}
  */
-async function* filterRows(rows, condition, context) {
+async function* filterRows(rows, condition, context, limit) {
+  const MAX_CHUNK = 256
+  let chunkSize = limit ?? Infinity
   let rowIndex = 0
+
+  /** @type {{ row: AsyncRow, rowIndex: number }[]} */
+  let buffer = []
+
   for await (const row of rows) {
     if (context.signal?.aborted) return
     rowIndex++
-    const pass = await evaluateExpr({ node: condition, row, rowIndex, context })
-    if (pass) yield row
+    buffer.push({ row, rowIndex })
+
+    if (buffer.length >= chunkSize) {
+      const results = await Promise.all(buffer.map(b =>
+        evaluateExpr({ node: condition, row: b.row, rowIndex: b.rowIndex, context })
+      ))
+      for (let i = 0; i < buffer.length; i++) {
+        if (results[i]) yield buffer[i].row
+      }
+      buffer = []
+      chunkSize = Math.min(chunkSize * 2, MAX_CHUNK)
+    }
+  }
+
+  // Flush remaining rows
+  if (buffer.length > 0) {
+    const results = await Promise.all(buffer.map(b =>
+      evaluateExpr({ node: condition, row: b.row, rowIndex: b.rowIndex, context })
+    ))
+    for (let i = 0; i < buffer.length; i++) {
+      if (results[i]) yield buffer[i].row
+    }
   }
 }
 
@@ -275,17 +302,38 @@ async function* executeProject(plan, context) {
  */
 async function* executeDistinct(plan, context) {
   const { signal } = context
+  const MAX_CHUNK = 256
 
   /** @type {Set<string>} */
   const seen = new Set()
 
+  /** @type {AsyncRow[]} */
+  let buffer = []
+
   for await (const row of executePlan({ plan: plan.child, context })) {
     if (signal?.aborted) return
+    buffer.push(row)
 
-    const key = await stableRowKey(row.cells)
-    if (!seen.has(key)) {
-      seen.add(key)
-      yield row
+    if (buffer.length >= MAX_CHUNK) {
+      const keys = await Promise.all(buffer.map(r => stableRowKey(r.cells)))
+      for (let i = 0; i < buffer.length; i++) {
+        if (!seen.has(keys[i])) {
+          seen.add(keys[i])
+          yield buffer[i]
+        }
+      }
+      buffer = []
+    }
+  }
+
+  // Flush remaining
+  if (buffer.length > 0) {
+    const keys = await Promise.all(buffer.map(r => stableRowKey(r.cells)))
+    for (let i = 0; i < buffer.length; i++) {
+      if (!seen.has(keys[i])) {
+        seen.add(keys[i])
+        yield buffer[i]
+      }
     }
   }
 }

package/src/execute/utils.js

@@ -44,17 +44,22 @@ export function compareForTerm(a, b, term) {
  * @returns {Promise<Record<string, SqlPrimitive>[]>} array of all yielded values
  */
 export async function collect(asyncRows) {
-  /** @type {Record<string, SqlPrimitive>[]} */
-  const results = []
+  // Collect all rows first, then materialize cells concurrently
+  // This enables dataloader-style batching of cell accessors
+  /** @type {AsyncRow[]} */
+  const rows = []
   for await (const asyncRow of asyncRows) {
+    rows.push(asyncRow)
+  }
+  return Promise.all(rows.map(async asyncRow => {
+    const values = await Promise.all(asyncRow.columns.map(k => asyncRow.cells[k]()))
     /** @type {Record<string, SqlPrimitive>} */
     const item = {}
-    for (const key of asyncRow.columns) {
-      item[key] = await asyncRow.cells[key]()
+    for (let i = 0; i < asyncRow.columns.length; i++) {
+      item[asyncRow.columns[i]] = values[i]
     }
-    results.push(item)
-  }
-  return results
+    return item
+  }))
 }
 
 /**
@@ -79,11 +84,6 @@ export function stringify(value) {
  */
 export async function stableRowKey(cells) {
   const keys = Object.keys(cells).sort()
-  /** @type {string[]} */
-  const parts = []
-  for (const k of keys) {
-    const v = await cells[k]()
-    parts.push(k + ':' + stringify(v))
-  }
-  return parts.join('|')
+  const values = await Promise.all(keys.map(k => cells[k]()))
+  return keys.map((k, i) => k + ':' + stringify(values[i])).join('|')
 }

package/src/expression/alias.js

@@ -31,9 +31,9 @@ export function derivedAlias(expr) {
   if (expr.type === 'function') {
     // Handle aggregate functions with star (COUNT(*) -> count_all)
     if (expr.args.length === 1 && expr.args[0].type === 'star') {
-      return expr.name.toLowerCase() + '_all'
+      return expr.funcName.toLowerCase() + '_all'
     }
-    return expr.name.toLowerCase() + '_' + expr.args.map(derivedAlias).join('_')
+    return expr.funcName.toLowerCase() + '_' + expr.args.map(derivedAlias).join('_')
   }
   if (expr.type === 'interval') {
     return `interval_${expr.value}_${expr.unit.toLowerCase()}`

package/src/expression/evaluate.js

@@ -1,9 +1,10 @@
 import { executeSelect } from '../execute/execute.js'
 import { stringify } from '../execute/utils.js'
-import { columnNotFoundError, invalidContextError } from '../executionErrors.js'
-import { unknownFunctionError } from '../parseErrors.js'
-import { isAggregateFunc, isMathFunc, isRegexpFunc, isSpatialFunc, isStringFunc } from '../validation.js'
-import { aggregateError, argValueError, castError } from '../validationErrors.js'
+import { invalidContextError } from '../validation/executionErrors.js'
+import { aggregateError, argValueError, castError } from '../validation/expressionErrors.js'
+import { isAggregateFunc, isMathFunc, isRegexpFunc, isSpatialFunc, isStringFunc } from '../validation/functions.js'
+import { unknownFunctionError } from '../validation/parseErrors.js'
+import { columnNotFoundError } from '../validation/planErrors.js'
 import { derivedAlias } from './alias.js'
 import { applyBinaryOp } from './binary.js'
 import { applyIntervalToDate, dateTrunc, extractField } from './date.js'
@@ -99,7 +100,7 @@ export async function evaluateExpr({ node, row, rowIndex, rows, context }) {
 
   // Function calls
   if (node.type === 'function') {
-    const funcName = node.name.toUpperCase()
+    const funcName = node.funcName.toUpperCase()
 
     // Handle aggregate functions
     if (isAggregateFunc(funcName)) {
@@ -110,22 +111,17 @@ export async function evaluateExpr({ node, row, rowIndex, rows, context }) {
         if (row.columns.includes(alias)) {
           return row.cells[alias]()
         } else {
-          throw aggregateError({
-            funcName,
-            positionStart: node.positionStart,
-            positionEnd: node.positionEnd,
-          })
+          throw aggregateError(node)
         }
       }
 
       // Apply FILTER clause if present
       let filteredRows = rows
       if (node.filter) {
-        filteredRows = []
-        for (const row of rows) {
-          const passes = await evaluateExpr({ node: node.filter, row, context })
-          if (passes) filteredRows.push(row)
-        }
+        const passes = await Promise.all(rows.map(row =>
+          evaluateExpr({ node: node.filter, row, context })
+        ))
+        filteredRows = rows.filter((_, i) => passes[i])
       }
 
       const argNode = node.args[0]
@@ -135,23 +131,27 @@ export async function evaluateExpr({ node, row, rowIndex, rows, context }) {
           return filteredRows.length
         }
 
+        const values = await Promise.all(filteredRows.map(row =>
+          evaluateExpr({ node: argNode, row, context })
+        ))
         if (node.distinct) {
           const seen = new Set()
-          for (const row of filteredRows) {
-            const v = await evaluateExpr({ node: argNode, row, context })
+          for (const v of values) {
             if (v != null) seen.add(v)
           }
           return seen.size
         }
         let count = 0
-        for (const row of filteredRows) {
-          const v = await evaluateExpr({ node: argNode, row, context })
+        for (const v of values) {
           if (v != null) count++
         }
         return count
       }
 
       if (funcName === 'SUM' || funcName === 'AVG' || funcName === 'MIN' || funcName === 'MAX') {
+        const rawValues = await Promise.all(filteredRows.map(row =>
+          evaluateExpr({ node: argNode, row, context })
+        ))
         let sum = 0
         let count = 0
         /** @type {number | null} */
@@ -159,8 +159,7 @@ export async function evaluateExpr({ node, row, rowIndex, rows, context }) {
         /** @type {number | null} */
         let max = null
 
-        for (const row of filteredRows) {
-          const raw = await evaluateExpr({ node: argNode, row, context })
+        for (const raw of rawValues) {
           if (raw == null) continue
           const num = Number(raw)
           if (!Number.isFinite(num)) continue
@@ -183,9 +182,12 @@ export async function evaluateExpr({ node, row, rowIndex, rows, context }) {
       }
 
       if (funcName === 'STDDEV_SAMP' || funcName === 'STDDEV_POP') {
+        const rawValues = await Promise.all(filteredRows.map(row =>
+          evaluateExpr({ node: argNode, row, context })
+        ))
+        /** @type {number[]} */
         const values = []
-        for (const row of filteredRows) {
-          const raw = await evaluateExpr({ node: argNode, row, context })
+        for (const raw of rawValues) {
           if (raw == null) continue
           const num = Number(raw)
           if (!Number.isFinite(num)) continue
@@ -230,23 +232,11 @@ export async function evaluateExpr({ node, row, rowIndex, rows, context }) {
       : await Promise.all(node.args.map(arg => evaluateExpr({ node: arg, row, rowIndex, rows, context })))
 
     if (isStringFunc(funcName)) {
-      return evaluateStringFunc({
-        funcName,
-        args,
-        positionStart: node.positionStart,
-        positionEnd: node.positionEnd,
-        rowIndex,
-      })
+      return evaluateStringFunc({ funcName, node, args, rowIndex })
     }
 
     if (isRegexpFunc(funcName)) {
-      return evaluateRegexpFunc({
-        funcName,
-        args,
-        positionStart: node.positionStart,
-        positionEnd: node.positionEnd,
-        rowIndex,
-      })
+      return evaluateRegexpFunc({ funcName, node, args, rowIndex })
     }
 
     if (isMathFunc(funcName)) {
@@ -296,10 +286,8 @@ export async function evaluateExpr({ node, row, rowIndex, rows, context }) {
     if (funcName === 'JSON_OBJECT') {
       if (args.length % 2 !== 0) {
         throw argValueError({
-          funcName: 'JSON_OBJECT',
+          ...node,
           message: 'requires an even number of arguments (key-value pairs)',
-          positionStart: node.positionStart,
-          positionEnd: node.positionEnd,
           rowIndex,
         })
       }
@@ -310,10 +298,8 @@ export async function evaluateExpr({ node, row, rowIndex, rows, context }) {
         const value = args[i + 1]
         if (key == null) {
           throw argValueError({
-            funcName: 'JSON_OBJECT',
+            ...node,
             message: 'key cannot be null',
-            positionStart: node.positionStart,
-            positionEnd: node.positionEnd,
             hint: 'All keys must be non-null values.',
             rowIndex,
           })
@@ -360,10 +346,8 @@ export async function evaluateExpr({ node, row, rowIndex, rows, context }) {
           jsonArg = JSON.parse(jsonArg)
         } catch {
           throw argValueError({
-            funcName,
+            ...node,
             message: 'invalid JSON string',
-            positionStart: node.positionStart,
-            positionEnd: node.positionEnd,
             hint: 'First argument must be valid JSON.',
             rowIndex,
           })
@@ -371,10 +355,8 @@ export async function evaluateExpr({ node, row, rowIndex, rows, context }) {
       }
       if (typeof jsonArg !== 'object' || jsonArg instanceof Date) {
         throw argValueError({
-          funcName,
+          ...node,
           message: `first argument must be JSON string or object, got ${typeof jsonArg}`,
-          positionStart: node.positionStart,
-          positionEnd: node.positionEnd,
           rowIndex,
         })
       }
@@ -414,30 +396,20 @@ export async function evaluateExpr({ node, row, rowIndex, rows, context }) {
       }
     }
 
-    throw unknownFunctionError({
-      funcName,
-      positionStart: node.positionStart,
-      positionEnd: node.positionEnd,
-    })
+    throw unknownFunctionError(node)
   }
 
   if (node.type === 'cast') {
     const val = await evaluateExpr({ node: node.expr, row, rowIndex, rows, context })
     if (val == null) return null
-    const toType = node.toType.toUpperCase()
+    const { toType } = node
     if (toType === 'TEXT' || toType === 'STRING' || toType === 'VARCHAR') {
       if (typeof val === 'object') return stringify(val)
       return String(val)
     }
     // Can only cast primitives to other primitive types
     if (typeof val === 'object') {
-      throw castError({
-        toType: node.toType,
-        positionStart: node.positionStart,
-        positionEnd: node.positionEnd,
-        fromType: 'object',
-        rowIndex,
-      })
+      throw castError({ ...node, fromType: 'object', rowIndex })
     }
     if (toType === 'INTEGER' || toType === 'INT') {
       const num = Number(val)
@@ -455,12 +427,7 @@ export async function evaluateExpr({ node, row, rowIndex, rows, context }) {
     if (toType === 'BOOLEAN' || toType === 'BOOL') {
       return Boolean(val)
     }
-    throw castError({
-      toType: node.toType,
-      positionStart: node.positionStart,
-      positionEnd: node.positionEnd,
-      rowIndex,
-    })
+    throw castError({ ...node, rowIndex })
   }
 
   // IN and NOT IN with value lists