squirreling 0.9.5 → 0.10.1

package/README.md

@@ -91,6 +91,8 @@ Squirreling can work with any data source that implements the `AsyncDataSource`
 
 ```typescript
 interface AsyncDataSource {
+  numRows?: number
+  columns: string[]
   scan(options: ScanOptions): ScanResults
 }
 
@@ -113,6 +115,8 @@ The `scan()` method returns a `ScanResults` object containing a row stream and f
 
 ```typescript
 const customSource: AsyncDataSource = {
+  numRows: 1000000, // optional total row count for planning
+  columns: ['id', 'name', 'active'], // columns available in this source
   scan({ columns, where, limit, offset, signal }) {
     // Use hints to optimize your scan, or ignore them
     return {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "squirreling",
-  "version": "0.9.5",
+  "version": "0.10.1",
   "description": "Squirreling Async SQL Engine",
   "author": "Hyperparam",
   "homepage": "https://hyperparam.app",
@@ -10,7 +10,9 @@
     "dataset",
     "hyperparam",
     "hyparquet",
-    "parquet"
+    "parquet",
+    "query",
+    "relational"
   ],
   "license": "MIT",
   "repository": {
@@ -37,7 +39,7 @@
     "test": "vitest run"
   },
   "devDependencies": {
-    "@types/node": "25.3.3",
+    "@types/node": "25.4.0",
     "@vitest/coverage-v8": "4.0.18",
     "eslint": "9.39.2",
     "eslint-plugin-jsdoc": "62.7.1",

package/src/ast.d.ts

@@ -0,0 +1,184 @@
+export type SqlPrimitive =
+  | string
+  | number
+  | bigint
+  | boolean
+  | Date
+  | null
+  | SqlPrimitive[]
+  | Record<string, any>
+
+export interface SelectStatement {
+  with?: WithClause
+  distinct: boolean
+  columns: SelectColumn[]
+  from: FromTable | FromSubquery
+  joins: JoinClause[]
+  where?: ExprNode
+  groupBy: ExprNode[]
+  having?: ExprNode
+  orderBy: OrderByItem[]
+  limit?: number
+  offset?: number
+}
+
+export interface WithClause {
+  ctes: CTEDefinition[]
+}
+
+export interface CTEDefinition {
+  name: string
+  query: SelectStatement
+}
+
+export interface FromTable extends AstBase {
+  kind: 'table'
+  table: string
+  alias?: string
+}
+
+export interface FromSubquery {
+  kind: 'subquery'
+  query: SelectStatement
+  alias: string
+}
+
+export type ArithmeticOp = '+' | '-' | '*' | '/' | '%'
+
+export type BinaryOp = 'AND' | 'OR' | 'LIKE' | ComparisonOp | ArithmeticOp
+
+export type ComparisonOp = '=' | '!=' | '<>' | '<' | '>' | '<=' | '>='
+
+export interface LiteralNode extends AstBase {
+  type: 'literal'
+  value: SqlPrimitive
+}
+
+export interface IdentifierNode extends AstBase {
+  type: 'identifier'
+  name: string
+}
+
+export interface UnaryNode extends AstBase {
+  type: 'unary'
+  op: 'NOT' | 'IS NULL' | 'IS NOT NULL' | '-'
+  argument: ExprNode
+}
+
+export interface BinaryNode extends AstBase {
+  type: 'binary'
+  op: BinaryOp
+  left: ExprNode
+  right: ExprNode
+}
+
+export interface FunctionNode extends AstBase {
+  type: 'function'
+  funcName: string
+  args: ExprNode[]
+  distinct?: boolean
+  filter?: ExprNode
+}
+
+export type CastType = 'TEXT' | 'STRING' | 'VARCHAR' | 'INTEGER' | 'INT' | 'BIGINT' | 'FLOAT' | 'REAL' | 'DOUBLE' | 'BOOLEAN' | 'BOOL'
+
+export interface CastNode extends AstBase {
+  type: 'cast'
+  expr: ExprNode
+  toType: CastType
+}
+
+export interface InSubqueryNode extends AstBase {
+  type: 'in'
+  expr: ExprNode
+  subquery: SelectStatement
+}
+
+export interface InValuesNode extends AstBase {
+  type: 'in valuelist'
+  expr: ExprNode
+  values: ExprNode[]
+}
+
+export interface ExistsNode extends AstBase {
+  type: 'exists' | 'not exists'
+  subquery: SelectStatement
+}
+
+export interface WhenClause extends AstBase {
+  condition: ExprNode
+  result: ExprNode
+}
+
+export interface CaseNode extends AstBase {
+  type: 'case'
+  caseExpr?: ExprNode
+  whenClauses: WhenClause[]
+  elseResult?: ExprNode
+}
+
+export interface SubqueryNode extends AstBase {
+  type: 'subquery'
+  subquery: SelectStatement
+}
+
+export type IntervalUnit = 'DAY' | 'MONTH' | 'YEAR' | 'HOUR' | 'MINUTE' | 'SECOND'
+
+export interface IntervalNode extends AstBase {
+  type: 'interval'
+  value: number
+  unit: IntervalUnit
+}
+
+export interface StarNode extends AstBase {
+  type: 'star'
+}
+
+export type ExprNode =
+  | LiteralNode
+  | IdentifierNode
+  | UnaryNode
+  | BinaryNode
+  | FunctionNode
+  | CastNode
+  | InSubqueryNode
+  | InValuesNode
+  | ExistsNode
+  | CaseNode
+  | SubqueryNode
+  | IntervalNode
+  | StarNode
+
+export interface StarColumn {
+  kind: 'star'
+  table?: string
+}
+
+export interface DerivedColumn {
+  kind: 'derived'
+  expr: ExprNode
+  alias?: string
+}
+
+export type SelectColumn = StarColumn | DerivedColumn
+
+export interface OrderByItem {
+  expr: ExprNode
+  direction: 'ASC' | 'DESC'
+  nulls?: 'FIRST' | 'LAST'
+}
+
+export type JoinType = 'INNER' | 'LEFT' | 'RIGHT' | 'FULL' | 'CROSS' | 'POSITIONAL'
+
+export interface JoinClause extends AstBase {
+  joinType: JoinType
+  table: string
+  alias?: string
+  on?: ExprNode
+}
+
+// All AST node derive from this base, which includes position info for error reporting and other purposes
+interface AstBase {
+  positionStart: number // start position in query (0-based, inclusive)
+  positionEnd: number // end position in query (0-based, exclusive)
+}

package/src/backend/dataSource.js

@@ -6,27 +6,50 @@
  * Creates an async row accessor that wraps a plain JavaScript object
  *
  * @param {Record<string, SqlPrimitive>} obj - the plain object
+ * @param {string[]} columns - list of column names (keys in the object)
  * @returns {AsyncRow} a row accessor interface
  */
-export function asyncRow(obj) {
+export function asyncRow(obj, columns) {
   /** @type {AsyncCells} */
   const cells = {}
-  for (const [key, value] of Object.entries(obj)) {
-    cells[key] = () => Promise.resolve(value)
+  for (const key of columns) {
+    cells[key] = () => Promise.resolve(obj[key])
   }
-  return { columns: Object.keys(obj), cells }
+  return { columns, cells }
 }
 
 /**
  * Creates an async memory-backed data source from an array of plain objects
  *
- * @param {Record<string, SqlPrimitive>[]} data - array of plain objects
+ * @param {Object} options
+ * @param {Record<string, SqlPrimitive>[]} options.data - array of plain objects
+ * @param {string[]} [options.columns] - optional list of column names (if not provided, inferred from first row)
  * @returns {AsyncDataSource} an async data source interface
  */
-export function memorySource(data) {
+export function memorySource({ data, columns }) {
+  if (!columns) {
+    // Columns not provided, infer from data
+    if (!data.length) {
+      throw new Error('Unknown columns: data is empty and no columns provided')
+    }
+    columns = Object.keys(data[0])
+    // Check first 1000 rows for consistent columns
+    for (let i = 1; i < data.length && i < 1000; i++) {
+      const rowColumns = Object.keys(data[i])
+      const missing = columns.find(col => !rowColumns.includes(col))
+      if (missing) {
+        throw new Error(`Inconsistent data, column "${missing}" not found in row ${i}`)
+      }
+      const extra = rowColumns.find(col => !columns.includes(col))
+      if (extra) {
+        throw new Error(`Inconsistent data, unexpected column "${extra}" found in row ${i}`)
+      }
+    }
+  }
   return {
     numRows: data.length,
-    scan({ where, limit, offset, signal }) {
+    columns,
+    scan({ columns: scanColumns, where, limit, offset, signal }) {
       // Only apply offset and limit if no where clause
       const start = !where ? offset ?? 0 : 0
       const end = !where && limit !== undefined ? start + limit : data.length
@@ -34,7 +57,7 @@ export function memorySource(data) {
         rows: (async function* () {
           for (let i = start; i < end && i < data.length; i++) {
             if (signal?.aborted) break
-            yield asyncRow(data[i])
+            yield asyncRow(data[i], scanColumns ?? columns)
           }
         })(),
         appliedWhere: false,
@@ -53,6 +76,7 @@ export function cachedDataSource(source) {
   /** @type {Map<string, Promise<SqlPrimitive>>} */
   const cache = new Map()
   return {
+    ...source,
     scan(options) {
       // Does re-run the scan, but cache avoids re-computing expensive async cells
       // TODO: check cache first to avoid re-scanning when possible

package/src/execute/aggregates.js

@@ -91,7 +91,11 @@ export async function* executeHashAggregate(plan, context) {
 
     // Apply HAVING filter
     if (plan.having) {
-      const havingRow = { ...group[0], ...asyncRow }
+      /** @type {AsyncRow} */
+      const havingRow = {
+        columns: [...group[0].columns, ...asyncRow.columns],
+        cells: { ...group[0].cells, ...asyncRow.cells },
+      }
       const passes = await evaluateExpr({
         node: plan.having,
         row: havingRow,
@@ -125,7 +129,11 @@ export async function* executeScalarAggregate(plan, context) {
 
   // Apply HAVING filter
   if (plan.having) {
-    const havingRow = { ...group[0], ...asyncRow }
+    /** @type {AsyncRow} */
+    const havingRow = {
+      columns: [...group[0].columns, ...asyncRow.columns],
+      cells: { ...group[0].cells, ...asyncRow.cells },
+    }
     const passes = await evaluateExpr({
       node: plan.having,
       row: havingRow,

package/src/execute/execute.js

@@ -1,9 +1,9 @@
 import { memorySource } from '../backend/dataSource.js'
-import { tableNotFoundError } from '../executionErrors.js'
 import { derivedAlias } from '../expression/alias.js'
 import { evaluateExpr } from '../expression/evaluate.js'
 import { parseSql } from '../parse/parse.js'
 import { planSql } from '../plan/plan.js'
+import { tableNotFoundError } from '../validation/planErrors.js'
 import { executeHashAggregate, executeScalarAggregate } from './aggregates.js'
 import { executeHashJoin, executeNestedLoopJoin, executePositionalJoin } from './join.js'
 import { executeSort } from './sort.js'
@@ -26,11 +26,11 @@ export async function* executeSql({ tables, query, functions, signal }) {
   // Normalize tables: convert arrays to AsyncDataSource
   /** @type {Record<string, AsyncDataSource>} */
   const normalizedTables = {}
-  for (const [name, source] of Object.entries(tables)) {
-    if (Array.isArray(source)) {
-      normalizedTables[name] = memorySource(source)
+  for (const [name, data] of Object.entries(tables)) {
+    if (Array.isArray(data)) {
+      normalizedTables[name] = memorySource({ data })
     } else {
-      normalizedTables[name] = source
+      normalizedTables[name] = data
     }
   }
 
@@ -46,7 +46,7 @@ export async function* executeSql({ tables, query, functions, signal }) {
  * @yields {AsyncRow}
  */
 export async function* executeSelect({ select, context }) {
-  const plan = planSql({ query: select, functions: context.functions })
+  const plan = planSql({ query: select, functions: context.functions, tables: context.tables })
   yield* executePlan({ plan, context })
 }
 
@@ -95,16 +95,19 @@ export async function* executePlan({ plan, context }) {
  */
 async function* executeScan(plan, context) {
   const { tables, signal } = context
-  const dataSource = tables[plan.table]
-  if (dataSource === undefined) {
-    throw tableNotFoundError({ tableName: plan.table })
+  // check table
+  const table = tables[plan.table]
+  if (!table) {
+    throw tableNotFoundError({ table: plan.table, tables })
   }
-
-  const scanResult = dataSource.scan({ ...plan.hints, signal })
-  if (!scanResult.rows) {
-    throw new Error(`Data source "${plan.table}" scan() must return a ScanResults object with { rows, appliedWhere, appliedLimitOffset }`)
+  // check columns
+  const missingColumn = plan.hints.columns?.find(col => !table.columns.includes(col))
+  if (missingColumn) {
+    throw new Error(`Column "${missingColumn}" not found. Available columns: ${table.columns.join(', ') || '[]'}`)
   }
-  const { rows, appliedWhere, appliedLimitOffset } = scanResult
+
+  // do the scan
+  const { rows, appliedWhere, appliedLimitOffset } = table.scan({ ...plan.hints, signal })
 
   // Applied limit/offset without applied where is invalid
   const hasLimitOffset = plan.hints.limit !== undefined || plan.hints.offset // 0 offset is noop
@@ -135,17 +138,17 @@ async function* executeScan(plan, context) {
  * @yields {AsyncRow}
  */
 async function* executeCount(plan, { tables, signal }) {
-  const dataSource = tables[plan.table]
-  if (dataSource === undefined) {
-    throw tableNotFoundError({ tableName: plan.table })
+  const table = tables[plan.table]
+  if (!table) {
+    throw tableNotFoundError({ table: plan.table, tables })
   }
 
   // Use source numRows if available
-  let count = dataSource.numRows
-  if (dataSource.numRows === undefined) {
+  let count = table.numRows
+  if (table.numRows === undefined) {
     // Fall back to counting rows via scan
     count = 0
-    const { rows } = dataSource.scan({ signal })
+    const { rows } = table.scan({ signal })
     // eslint-disable-next-line no-unused-vars
     for await (const _ of rows) {
       if (signal?.aborted) return

package/src/execute/join.js

@@ -203,16 +203,16 @@ export async function* executeHashJoin(plan, context) {
 /**
  * Creates a NULL-filled row with the given column names
  *
- * @param {string[]} columnNames
+ * @param {string[]} columns
  * @returns {AsyncRow}
  */
-function createNullRow(columnNames) {
+function createNullRow(columns) {
   /** @type {AsyncCells} */
   const cells = {}
-  for (const col of columnNames) {
+  for (const col of columns) {
     cells[col] = () => Promise.resolve(null)
   }
-  return { columns: columnNames, cells }
+  return { columns, cells }
 }
 
 /**
@@ -234,6 +234,7 @@ function mergeRows(leftRow, rightRow, leftTable, rightTable) {
     // Skip already-prefixed keys (from previous joins)
     if (!key.includes('.')) {
       const alias = `${leftTable}.${key}`
+      columns.push(alias)
       cells[alias] = cell
     }
     // Also keep unqualified name for convenience
@@ -244,9 +245,9 @@ function mergeRows(leftRow, rightRow, leftTable, rightTable) {
   // Add right table columns with prefix
   for (const [key, cell] of Object.entries(rightRow.cells)) {
     if (!key.includes('.')) {
-      cells[`${rightTable}.${key}`] = cell
-    } else {
-      cells[key] = cell
+      const alias = `${rightTable}.${key}`
+      columns.push(alias)
+      cells[alias] = cell
     }
     // Unqualified name (overwrites if same name exists in left table)
     columns.push(key)

package/src/expression/alias.js

@@ -31,9 +31,9 @@ export function derivedAlias(expr) {
   if (expr.type === 'function') {
     // Handle aggregate functions with star (COUNT(*) -> count_all)
     if (expr.args.length === 1 && expr.args[0].type === 'star') {
-      return expr.name.toLowerCase() + '_all'
+      return expr.funcName.toLowerCase() + '_all'
     }
-    return expr.name.toLowerCase() + '_' + expr.args.map(derivedAlias).join('_')
+    return expr.funcName.toLowerCase() + '_' + expr.args.map(derivedAlias).join('_')
   }
   if (expr.type === 'interval') {
     return `interval_${expr.value}_${expr.unit.toLowerCase()}`

package/src/expression/evaluate.js

@@ -1,9 +1,10 @@
 import { executeSelect } from '../execute/execute.js'
 import { stringify } from '../execute/utils.js'
-import { columnNotFoundError, invalidContextError } from '../executionErrors.js'
-import { unknownFunctionError } from '../parseErrors.js'
-import { isAggregateFunc, isMathFunc, isRegexpFunc, isSpatialFunc, isStringFunc } from '../validation.js'
-import { aggregateError, argValueError, castError } from '../validationErrors.js'
+import { invalidContextError } from '../validation/executionErrors.js'
+import { aggregateError, argValueError, castError } from '../validation/expressionErrors.js'
+import { isAggregateFunc, isMathFunc, isRegexpFunc, isSpatialFunc, isStringFunc } from '../validation/functions.js'
+import { unknownFunctionError } from '../validation/parseErrors.js'
+import { columnNotFoundError } from '../validation/planErrors.js'
 import { derivedAlias } from './alias.js'
 import { applyBinaryOp } from './binary.js'
 import { applyIntervalToDate, dateTrunc, extractField } from './date.js'
@@ -47,7 +48,7 @@ export async function evaluateExpr({ node, row, rowIndex, rows, context }) {
     // Unknown identifier
     throw columnNotFoundError({
       columnName: node.name,
-      availableColumns: Object.keys(row.cells),
+      availableColumns: row.columns,
       positionStart: node.positionStart,
       positionEnd: node.positionEnd,
       rowIndex,
@@ -99,7 +100,7 @@ export async function evaluateExpr({ node, row, rowIndex, rows, context }) {
 
   // Function calls
   if (node.type === 'function') {
-    const funcName = node.name.toUpperCase()
+    const funcName = node.funcName.toUpperCase()
 
     // Handle aggregate functions
     if (isAggregateFunc(funcName)) {
@@ -110,11 +111,7 @@ export async function evaluateExpr({ node, row, rowIndex, rows, context }) {
         if (row.columns.includes(alias)) {
           return row.cells[alias]()
         } else {
-          throw aggregateError({
-            funcName,
-            positionStart: node.positionStart,
-            positionEnd: node.positionEnd,
-          })
+          throw aggregateError(node)
         }
       }
 
@@ -225,26 +222,16 @@ export async function evaluateExpr({ node, row, rowIndex, rows, context }) {
     }
 
     /** @type {SqlPrimitive[]} */
-    const args = await Promise.all(node.args.map(arg => evaluateExpr({ node: arg, row, rowIndex, rows, context })))
+    const args = node.args.length === 1
+      ? [await evaluateExpr({ node: node.args[0], row, rowIndex, rows, context })]
+      : await Promise.all(node.args.map(arg => evaluateExpr({ node: arg, row, rowIndex, rows, context })))
 
     if (isStringFunc(funcName)) {
-      return evaluateStringFunc({
-        funcName,
-        args,
-        positionStart: node.positionStart,
-        positionEnd: node.positionEnd,
-        rowIndex,
-      })
+      return evaluateStringFunc({ funcName, node, args, rowIndex })
     }
 
     if (isRegexpFunc(funcName)) {
-      return evaluateRegexpFunc({
-        funcName,
-        args,
-        positionStart: node.positionStart,
-        positionEnd: node.positionEnd,
-        rowIndex,
-      })
+      return evaluateRegexpFunc({ funcName, node, args, rowIndex })
     }
 
     if (isMathFunc(funcName)) {
@@ -294,10 +281,8 @@ export async function evaluateExpr({ node, row, rowIndex, rows, context }) {
     if (funcName === 'JSON_OBJECT') {
       if (args.length % 2 !== 0) {
         throw argValueError({
-          funcName: 'JSON_OBJECT',
+          ...node,
           message: 'requires an even number of arguments (key-value pairs)',
-          positionStart: node.positionStart,
-          positionEnd: node.positionEnd,
           rowIndex,
         })
       }
@@ -308,10 +293,8 @@ export async function evaluateExpr({ node, row, rowIndex, rows, context }) {
         const value = args[i + 1]
         if (key == null) {
           throw argValueError({
-            funcName: 'JSON_OBJECT',
+            ...node,
             message: 'key cannot be null',
-            positionStart: node.positionStart,
-            positionEnd: node.positionEnd,
             hint: 'All keys must be non-null values.',
             rowIndex,
           })
@@ -358,10 +341,8 @@ export async function evaluateExpr({ node, row, rowIndex, rows, context }) {
           jsonArg = JSON.parse(jsonArg)
         } catch {
           throw argValueError({
-            funcName,
+            ...node,
             message: 'invalid JSON string',
-            positionStart: node.positionStart,
-            positionEnd: node.positionEnd,
             hint: 'First argument must be valid JSON.',
             rowIndex,
           })
@@ -369,10 +350,8 @@ export async function evaluateExpr({ node, row, rowIndex, rows, context }) {
       }
       if (typeof jsonArg !== 'object' || jsonArg instanceof Date) {
         throw argValueError({
-          funcName,
+          ...node,
           message: `first argument must be JSON string or object, got ${typeof jsonArg}`,
-          positionStart: node.positionStart,
-          positionEnd: node.positionEnd,
           rowIndex,
         })
       }
@@ -412,30 +391,20 @@ export async function evaluateExpr({ node, row, rowIndex, rows, context }) {
       }
     }
 
-    throw unknownFunctionError({
-      funcName,
-      positionStart: node.positionStart,
-      positionEnd: node.positionEnd,
-    })
+    throw unknownFunctionError(node)
   }
 
   if (node.type === 'cast') {
     const val = await evaluateExpr({ node: node.expr, row, rowIndex, rows, context })
     if (val == null) return null
-    const toType = node.toType.toUpperCase()
+    const { toType } = node
     if (toType === 'TEXT' || toType === 'STRING' || toType === 'VARCHAR') {
       if (typeof val === 'object') return stringify(val)
       return String(val)
     }
     // Can only cast primitives to other primitive types
     if (typeof val === 'object') {
-      throw castError({
-        toType: node.toType,
-        positionStart: node.positionStart,
-        positionEnd: node.positionEnd,
-        fromType: 'object',
-        rowIndex,
-      })
+      throw castError({ ...node, fromType: 'object', rowIndex })
     }
     if (toType === 'INTEGER' || toType === 'INT') {
       const num = Number(val)
@@ -453,12 +422,7 @@ export async function evaluateExpr({ node, row, rowIndex, rows, context }) {
     if (toType === 'BOOLEAN' || toType === 'BOOL') {
       return Boolean(val)
     }
-    throw castError({
-      toType: node.toType,
-      positionStart: node.positionStart,
-      positionEnd: node.positionEnd,
-      rowIndex,
-    })
+    throw castError({ ...node, rowIndex })
   }
 
   // IN and NOT IN with value lists