squirreling 0.3.1 → 0.4.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "squirreling",
-  "version": "0.3.1",
+  "version": "0.4.0",
   "description": "Squirreling SQL Engine",
   "author": "Hyperparam",
   "homepage": "https://hyperparam.app",

package/src/backend/dataSource.js

@@ -1,20 +1,18 @@
 /**
- * @import { AsyncDataSource, AsyncRow } from '../types.js'
+ * @import { AsyncDataSource, AsyncRow, SqlPrimitive } from '../types.js'
  */
 
 
 /**
  * Wraps an async generator of plain objects into an AsyncDataSource
  *
- * @param {AsyncGenerator<Record<string, any>>} gen
+ * @param {AsyncGenerator<AsyncRow>} gen
  * @returns {AsyncDataSource}
  */
 export function generatorSource(gen) {
   return {
     async *getRows() {
-      for await (const row of gen) {
-        yield asyncRow(row)
-      }
+      yield* gen
     },
   }
 }
@@ -22,24 +20,22 @@ export function generatorSource(gen) {
 /**
  * Creates an async row accessor that wraps a plain JavaScript object
  *
- * @param {Record<string, any>} obj - the plain object
+ * @param {Record<string, SqlPrimitive>} obj - the plain object
  * @returns {AsyncRow} a row accessor interface
  */
-export function asyncRow(obj) {
-  return {
-    getCell(name) {
-      return obj[name]
-    },
-    getKeys() {
-      return Object.keys(obj)
-    },
+function asyncRow(obj) {
+  /** @type {AsyncRow} */
+  const row = {}
+  for (const [key, value] of Object.entries(obj)) {
+    row[key] = () => Promise.resolve(value)
   }
+  return row
 }
 
 /**
  * Creates an async memory-backed data source from an array of plain objects
  *
- * @param {Record<string, any>[]} data - array of plain objects
+ * @param {Record<string, SqlPrimitive>[]} data - array of plain objects
  * @returns {AsyncDataSource} an async data source interface
  */
 export function memorySource(data) {
@@ -51,3 +47,40 @@ export function memorySource(data) {
     },
   }
 }
+
+/**
+ * Wraps a data source that caches all accessed rows in memory
+ * @param {AsyncDataSource} source
+ * @returns {AsyncDataSource}
+ */
+export function cachedDataSource(source) {
+  /** @type {Map<string, Promise<SqlPrimitive>>} */
+  const cache = new Map()
+  return {
+    /**
+     * @returns {AsyncGenerator<AsyncRow>}
+     */
+    async *getRows() {
+      let index = 0
+      for await (const row of source.getRows()) {
+        const rowIndex = index
+        /** @type {AsyncRow} */
+        const out = {}
+        for (const [key, cell] of Object.entries(row)) {
+          // Wrap the cell to cache accesses
+          out[key] = () => {
+            const cacheKey = `${rowIndex}:${key}`
+            let value = cache.get(cacheKey)
+            if (!value) {
+              value = cell()
+              cache.set(cacheKey, value)
+            }
+            return value
+          }
+        }
+        yield out
+        index++
+      }
+    },
+  }
+}

package/src/execute/aggregates.js

@@ -3,19 +3,21 @@ import { evaluateExpr } from './expression.js'
 /**
  * Evaluates an aggregate function over a set of rows
  *
- * @import { AggregateColumn, ExprNode, AsyncRow } from '../types.js'
- * @param {AggregateColumn} col - aggregate column definition
- * @param {AsyncRow[]} rows - rows to aggregate
+ * @import { AggregateColumn, AsyncDataSource, ExprNode, AsyncRow } from '../types.js'
+ * @param {Object} options
+ * @param {AggregateColumn} options.col - aggregate column definition
+ * @param {AsyncRow[]} options.rows - rows to aggregate
+ * @param {Record<string, AsyncDataSource>} options.tables
  * @returns {Promise<number | null>} aggregated result
  */
-export async function evaluateAggregate(col, rows) {
+export async function evaluateAggregate({ col, rows, tables }) {
   const { arg, func } = col
 
   if (func === 'COUNT') {
     if (arg.kind === 'star') return rows.length
     let count = 0
-    for (let i = 0; i < rows.length; i += 1) {
-      const v = await evaluateExpr({ node: arg.expr, row: rows[i] })
+    for (const row of rows) {
+      const v = await evaluateExpr({ node: arg.expr, row, tables })
       if (v !== null && v !== undefined) {
         count += 1
       }
@@ -34,8 +36,8 @@ export async function evaluateAggregate(col, rows) {
     /** @type {number | null} */
     let max = null
 
-    for (let i = 0; i < rows.length; i += 1) {
-      const raw = await evaluateExpr({ node: arg.expr, row: rows[i] })
+    for (const row of rows) {
+      const raw = await evaluateExpr({ node: arg.expr, row, tables })
       if (raw == null) continue
       const num = Number(raw)
       if (!Number.isFinite(num)) continue
@@ -80,20 +82,17 @@ export function defaultAggregateAlias(col) {
 export function defaultAggregateAliasExpr(expr) {
   if (expr.type === 'identifier') {
     return expr.name
-  }
-  if (expr.type === 'literal') {
+  } else if (expr.type === 'literal') {
     return String(expr.value)
-  }
-  if (expr.type === 'cast') {
+  } else if (expr.type === 'cast') {
     return defaultAggregateAliasExpr(expr.expr) + '_as_' + expr.toType
-  }
-  if (expr.type === 'unary') {
+  } else if (expr.type === 'unary') {
     return expr.op + '_' + defaultAggregateAliasExpr(expr.argument)
-  }
-  if (expr.type === 'binary') {
+  } else if (expr.type === 'binary') {
     return defaultAggregateAliasExpr(expr.left) + '_' + expr.op + '_' + defaultAggregateAliasExpr(expr.right)
-  }
-  if (expr.type === 'function') {
+  } else if (expr.type === 'function') {
     return expr.name.toLowerCase() + '_' + expr.args.map(defaultAggregateAliasExpr).join('_')
+  } else {
+    return 'expr'
   }
 }

package/src/execute/execute.js

@@ -1,6 +1,6 @@
 import { evaluateExpr } from './expression.js'
 import { parseSql } from '../parse/parse.js'
-import { asyncRow, generatorSource, memorySource } from '../backend/dataSource.js'
+import { generatorSource, memorySource } from '../backend/dataSource.js'
 import { defaultAggregateAlias, evaluateAggregate } from './aggregates.js'
 import { evaluateHavingExpr } from './having.js'
 
@@ -12,7 +12,7 @@ import { evaluateHavingExpr } from './having.js'
  * Executes a SQL SELECT query against named data sources
  *
  * @param {ExecuteSqlOptions} options - the execution options
- * @returns {AsyncGenerator<Record<string, any>>} async generator yielding result rows
+ * @returns {AsyncGenerator<AsyncRow>} async generator yielding result rows
  */
 export async function* executeSql({ tables, query }) {
   const select = parseSql(query)
@@ -44,7 +44,7 @@ export async function* executeSql({ tables, query }) {
  *
  * @param {SelectStatement} select
  * @param {Record<string, AsyncDataSource>} tables
- * @returns {AsyncGenerator<Record<string, any>>} async generator yielding result rows
+ * @returns {AsyncGenerator<AsyncRow>}
  */
 export async function* executeSelect(select, tables) {
   /** @type {AsyncDataSource} */
@@ -96,15 +96,15 @@ function defaultDerivedAlias(expr) {
 /**
  * Creates a stable string key for a row to enable deduplication
  *
- * @param {Record<string, any>} row
- * @returns {string} a stable string representation of the row
+ * @param {AsyncRow} row
+ * @returns {Promise<string>} a stable string representation of the row
  */
-function stableRowKey(row) {
+async function stableRowKey(row) {
   const keys = Object.keys(row).sort()
   /** @type {string[]} */
   const parts = []
   for (const k of keys) {
-    const v = row[k]
+    const v = await row[k]()
     parts.push(k + ':' + JSON.stringify(v))
   }
   return parts.join('|')
@@ -138,18 +138,18 @@ function compareValues(a, b) {
 /**
  * Applies DISTINCT filtering to remove duplicate rows
  *
- * @param {Record<string, any>[]} rows - The input rows
- * @param {boolean} distinct - Whether to apply deduplication
- * @returns {Record<string, any>[]} The deduplicated rows
+ * @param {AsyncRow[]} rows - the input rows
+ * @param {boolean} distinct - whether to apply deduplication
+ * @returns {Promise<AsyncRow[]>} the deduplicated rows
  */
-function applyDistinct(rows, distinct) {
+async function applyDistinct(rows, distinct) {
   if (!distinct) return rows
   /** @type {Set<string>} */
   const seen = new Set()
-  /** @type {Record<string, any>[]} */
+  /** @type {AsyncRow[]} */
   const result = []
   for (const row of rows) {
-    const key = stableRowKey(row)
+    const key = await stableRowKey(row)
     if (seen.has(key)) continue
     seen.add(key)
     result.push(row)
@@ -221,10 +221,10 @@ async function sortRowSources(rows, orderBy, tables) {
 /**
  * Applies ORDER BY sorting to rows
  *
- * @param {Record<string, any>[]} rows - the input rows
+ * @param {AsyncRow[]} rows - the input rows
  * @param {OrderByItem[]} orderBy - the sort specifications
  * @param {Record<string, AsyncDataSource>} tables
- * @returns {Promise<Record<string, any>[]>} the sorted rows
+ * @returns {Promise<AsyncRow[]>} the sorted rows
  */
 async function applyOrderBy(rows, orderBy, tables) {
   if (!orderBy.length) return rows
@@ -236,7 +236,7 @@ async function applyOrderBy(rows, orderBy, tables) {
     /** @type {SqlPrimitive[]} */
     const rowValues = []
     for (const term of orderBy) {
-      const value = await evaluateExpr({ node: term.expr, row: asyncRow(row), tables })
+      const value = await evaluateExpr({ node: term.expr, row, tables })
       rowValues.push(value)
     }
     evaluatedValues.push(rowValues)
@@ -282,10 +282,10 @@ async function applyOrderBy(rows, orderBy, tables) {
 /**
  * Evaluates a select with a resolved FROM data source
  *
- * @param {SelectStatement} select - the parsed SQL AST
- * @param {AsyncDataSource} dataSource - the async data source
+ * @param {SelectStatement} select
+ * @param {AsyncDataSource} dataSource
  * @param {Record<string, AsyncDataSource>} tables
- * @returns {AsyncGenerator<Record<string, any>>} async generator yielding result rows
+ * @returns {AsyncGenerator<AsyncRow>}
  */
 async function* evaluateSelectAst(select, dataSource, tables) {
   // SQL priority: from, where, group by, having, select, order by, offset, limit
@@ -310,7 +310,7 @@ async function* evaluateSelectAst(select, dataSource, tables) {
  * @param {SelectStatement} select
  * @param {AsyncDataSource} dataSource
  * @param {Record<string, AsyncDataSource>} tables
- * @returns {AsyncGenerator<Record<string, any>>}
+ * @returns {AsyncGenerator<AsyncRow>}
  */
 async function* evaluateStreaming(select, dataSource, tables) {
   let rowsYielded = 0
@@ -337,17 +337,16 @@ async function* evaluateStreaming(select, dataSource, tables) {
     }
 
     // SELECT projection
-    /** @type {Record<string, any>} */
+    /** @type {AsyncRow} */
     const outRow = {}
     for (const col of select.columns) {
       if (col.kind === 'star') {
-        const keys = row.getKeys()
-        for (const key of keys) {
-          outRow[key] = row.getCell(key)
+        for (const [key, cell] of Object.entries(row)) {
+          outRow[key] = cell
         }
       } else if (col.kind === 'derived') {
         const alias = col.alias ?? defaultDerivedAlias(col.expr)
-        outRow[alias] = await evaluateExpr({ node: col.expr, row, tables })
+        outRow[alias] = () => evaluateExpr({ node: col.expr, row, tables })
       } else if (col.kind === 'aggregate') {
         throw new Error(
           'Aggregate functions require GROUP BY or will act on the whole dataset; add GROUP BY or remove aggregates'
@@ -357,7 +356,7 @@ async function* evaluateStreaming(select, dataSource, tables) {
 
     // DISTINCT: skip duplicate rows
     if (seen) {
-      const key = stableRowKey(outRow)
+      const key = await stableRowKey(outRow)
       if (seen.has(key)) continue
       seen.add(key)
       // OFFSET applies to distinct rows
@@ -383,7 +382,7 @@ async function* evaluateStreaming(select, dataSource, tables) {
  * @param {Record<string, AsyncDataSource>} tables
  * @param {boolean} hasAggregate
  * @param {boolean} useGrouping
- * @returns {AsyncGenerator<Record<string, any>>}
+ * @returns {AsyncGenerator<AsyncRow>}
  */
 async function* evaluateBuffered(select, dataSource, tables, hasAggregate, useGrouping) {
   // Step 1: Collect all rows from data source
@@ -409,7 +408,7 @@ async function* evaluateBuffered(select, dataSource, tables, hasAggregate, useGr
   }
 
   // Step 3: Projection (grouping vs non-grouping)
-  /** @type {Record<string, any>[]} */
+  /** @type {AsyncRow[]} */
   let projected = []
 
   if (useGrouping) {
@@ -446,15 +445,14 @@ async function* evaluateBuffered(select, dataSource, tables, hasAggregate, useGr
     }
 
     for (const group of groups) {
-      /** @type {Record<string, any>} */
+      /** @type {AsyncRow} */
       const resultRow = {}
       for (const col of select.columns) {
         if (col.kind === 'star') {
           const firstRow = group[0]
           if (firstRow) {
-            const keys = firstRow.getKeys()
-            for (const key of keys) {
-              resultRow[key] = firstRow.getCell(key)
+            for (const [key, cell] of Object.entries(firstRow)) {
+              resultRow[key] = cell
             }
           }
           continue
@@ -463,18 +461,16 @@ async function* evaluateBuffered(select, dataSource, tables, hasAggregate, useGr
         if (col.kind === 'derived') {
           const alias = col.alias ?? defaultDerivedAlias(col.expr)
           if (group.length > 0) {
-            const value = await evaluateExpr({ node: col.expr, row: group[0], tables })
-            resultRow[alias] = value
+            resultRow[alias] = () => evaluateExpr({ node: col.expr, row: group[0], tables })
           } else {
-            resultRow[alias] = undefined
+            delete resultRow[alias]
           }
           continue
         }
 
         if (col.kind === 'aggregate') {
           const alias = col.alias ?? defaultAggregateAlias(col)
-          const value = await evaluateAggregate(col, group)
-          resultRow[alias] = value
+          resultRow[alias] = () => evaluateAggregate({ col, rows: group, tables })
           continue
         }
       }
@@ -503,18 +499,16 @@ async function* evaluateBuffered(select, dataSource, tables, hasAggregate, useGr
     }
 
     for (const row of rowsToProject) {
-      /** @type {Record<string, any>} */
+      /** @type {AsyncRow} */
       const outRow = {}
       for (const col of select.columns) {
         if (col.kind === 'star') {
-          const keys = row.getKeys()
-          for (const key of keys) {
-            outRow[key] = row.getCell(key)
+          for (const [key, cell] of Object.entries(row)) {
+            outRow[key] = cell
           }
         } else if (col.kind === 'derived') {
           const alias = col.alias ?? defaultDerivedAlias(col.expr)
-          const value = await evaluateExpr({ node: col.expr, row, tables })
-          outRow[alias] = value
+          outRow[alias] = () => evaluateExpr({ node: col.expr, row, tables })
         }
       }
       projected.push(outRow)
@@ -522,7 +516,7 @@ async function* evaluateBuffered(select, dataSource, tables, hasAggregate, useGr
   }
 
   // Step 4: DISTINCT
-  projected = applyDistinct(projected, select.distinct)
+  projected = await applyDistinct(projected, select.distinct)
 
   // Step 5: ORDER BY (final sort for grouped queries)
   projected = await applyOrderBy(projected, select.orderBy, tables)

package/src/execute/expression.js

@@ -1,5 +1,4 @@
 import { executeSelect } from './execute.js'
-import { collect } from './utils.js'
 
 /**
  * @import { ExprNode, AsyncRow, SqlPrimitive, AsyncDataSource } from '../types.js'
@@ -11,7 +10,7 @@ import { collect } from './utils.js'
  * @param {Object} params
  * @param {ExprNode} params.node - The expression node to evaluate
  * @param {AsyncRow} params.row - The data row to evaluate against
- * @param {Record<string, AsyncDataSource>} [params.tables]
+ * @param {Record<string, AsyncDataSource>} params.tables
  * @returns {Promise<SqlPrimitive>} The result of the evaluation
  */
 export async function evaluateExpr({ node, row, tables }) {
@@ -20,17 +19,19 @@ export async function evaluateExpr({ node, row, tables }) {
   }
 
   if (node.type === 'identifier') {
-    return row.getCell(node.name)
+    return row[node.name]?.()
   }
 
   // Scalar subquery - returns a single value
   if (node.type === 'subquery') {
-    const results = await collect(executeSelect(node.subquery, tables))
-    if (results.length === 0) return null
-    // Return the first column of the first row
-    const firstRow = results[0]
+    const gen = executeSelect(node.subquery, tables)
+    const first = await gen.next() // Start the generator
+    gen.return() // Stop further execution
+    if (first.done) return null
+    /** @type {AsyncRow} */
+    const firstRow = first.value
     const firstKey = Object.keys(firstRow)[0]
-    return firstRow[firstKey]
+    return firstRow[firstKey]()
   }
 
   // Unary operators
@@ -238,29 +239,37 @@ export async function evaluateExpr({ node, row, tables }) {
   // IN and NOT IN with subqueries
   if (node.type === 'in') {
     const exprVal = await evaluateExpr({ node: node.expr, row, tables })
-    const results = await collect(executeSelect(node.subquery, tables))
-    if (results.length === 0) return false
-    const firstKey = Object.keys(results[0])[0]
-    const values = results.map(r => r[firstKey])
+    const results = executeSelect(node.subquery, tables)
+    /** @type {SqlPrimitive[]} */
+    const values = []
+    for await (const resRow of results) {
+      const firstKey = Object.keys(resRow)[0]
+      const val = await resRow[firstKey]()
+      values.push(val)
+    }
     return values.includes(exprVal)
   }
   if (node.type === 'not in') {
     const exprVal = await evaluateExpr({ node: node.expr, row, tables })
-    const results = await collect(executeSelect(node.subquery, tables))
-    if (results.length === 0) return true
-    const firstKey = Object.keys(results[0])[0]
-    const values = results.map(r => r[firstKey])
+    const results = executeSelect(node.subquery, tables)
+    /** @type {SqlPrimitive[]} */
+    const values = []
+    for await (const resRow of results) {
+      const firstKey = Object.keys(resRow)[0]
+      const val = await resRow[firstKey]()
+      values.push(val)
+    }
     return !values.includes(exprVal)
   }
 
   // EXISTS and NOT EXISTS with subqueries
   if (node.type === 'exists') {
-    const results = await collect(executeSelect(node.subquery, tables))
-    return results.length > 0
+    const results = await executeSelect(node.subquery, tables).next()
+    return results.done === false
   }
   if (node.type === 'not exists') {
-    const results = await collect(executeSelect(node.subquery, tables))
-    return results.length === 0
+    const results = await executeSelect(node.subquery, tables).next()
+    return results.done === true
   }
 
   // CASE expressions

package/src/execute/having.js

@@ -8,32 +8,17 @@ import { evaluateExpr } from './expression.js'
 /**
  * Creates a context for evaluating HAVING expressions
  *
- * @param {Record<string, any>} resultRow - the aggregated result row
+ * @param {AsyncRow} resultRow - the aggregated result row
  * @param {AsyncRow[]} group - the group of rows
  * @returns {AsyncRow} a context row for HAVING evaluation
  */
 function createHavingContext(resultRow, group) {
   // Include the first row of the group (for GROUP BY columns)
   const firstRow = group[0]
-  /** @type {Record<string, any>} */
-  const context = {}
   if (firstRow) {
-    const keys = firstRow.getKeys()
-    for (const key of keys) {
-      context[key] = firstRow.getCell(key)
-    }
-  }
-  // Merge with result row (which has aggregates computed)
-  Object.assign(context, resultRow)
-
-  // Return a Row accessor wrapping the context
-  return {
-    getCell(name) {
-      return context[name]
-    },
-    getKeys() {
-      return Object.keys(context)
-    },
+    return { ...firstRow, ...resultRow }
+  } else {
+    return resultRow
   }
 }
 
@@ -41,7 +26,7 @@ function createHavingContext(resultRow, group) {
  * Evaluates a HAVING expression with support for aggregate functions
  *
  * @param {ExprNode} expr - the HAVING expression
- * @param {Record<string, any>} row - the aggregated result row
+ * @param {AsyncRow} row - the aggregated result row
  * @param {AsyncRow[]} group - the group of rows for re-evaluating aggregates
  * @param {Record<string, AsyncDataSource>} tables
  * @returns {Promise<boolean>} whether the HAVING condition is satisfied

package/src/execute/utils.js

@@ -1,13 +1,19 @@
 /**
- * Collects all results from an async generator into an array
+ * Collects and materialize all results from an async row generator into an array
  *
- * @template T
- * @param {AsyncGenerator<T>} asyncGen - the async generator
- * @returns {Promise<T[]>} array of all yielded values
+ * @import {AsyncRow, SqlPrimitive} from '../types.js'
+ * @param {AsyncGenerator<AsyncRow>} asyncRows
+ * @returns {Promise<Record<string, SqlPrimitive>[]>} array of all yielded values
  */
-export async function collect(asyncGen) {
+export async function collect(asyncRows) {
+  /** @type {Record<string, SqlPrimitive>[]} */
   const results = []
-  for await (const item of asyncGen) {
+  for await (const asyncRow of asyncRows) {
+    /** @type {Record<string, SqlPrimitive>} */
+    const item = {}
+    for (const [key, cell] of Object.entries(asyncRow)) {
+      item[key] = await cell()
+    }
     results.push(item)
   }
   return results

package/src/index.d.ts

@@ -1,4 +1,4 @@
-import type { ExecuteSqlOptions, SelectStatement } from './types.js'
+import type { AsyncRow, ExecuteSqlOptions, SelectStatement, SqlPrimitive } from './types.js'
 
 /**
  * Executes a SQL SELECT query against an array of data rows
@@ -8,12 +8,12 @@ import type { ExecuteSqlOptions, SelectStatement } from './types.js'
  * @param options.query - SQL query string
  * @returns async generator yielding rows matching the query
  */
-export function executeSql(options: ExecuteSqlOptions): AsyncGenerator<Record<string, any>>
+export function executeSql(options: ExecuteSqlOptions): AsyncGenerator<AsyncRow>
 
 /**
  * Parses a SQL query string into an abstract syntax tree
  *
- * @param sql - SQL query string to parse
+ * @param query - SQL query string to parse
  * @returns parsed SQL select statement
  */
 export function parseSql(query: string): SelectStatement
@@ -24,4 +24,4 @@ export function parseSql(query: string): SelectStatement
  * @param asyncGen - the async generator
  * @returns array of all yielded values
  */
-export function collect<T>(asyncGen: AsyncGenerator<T>): Promise<T[]>
+export function collect<T>(asyncGen: AsyncGenerator<AsyncRow>): Promise<Record<string, SqlPrimitive>[]>

package/src/types.d.ts

@@ -6,15 +6,13 @@
 export interface AsyncDataSource {
   getRows(): AsyncIterable<AsyncRow>
 }
-export interface AsyncRow {
-  getCell(name: string): any
-  getKeys(): string[]
-}
+export type AsyncRow = Record<string, AsyncCell>
+export type AsyncCell = () => Promise<SqlPrimitive>
 
-export type RawData = Record<string, any>[]
+export type Row = Record<string, SqlPrimitive>[]
 
 export interface ExecuteSqlOptions {
-  tables: Record<string, RawData | AsyncDataSource>
+  tables: Record<string, Row | AsyncDataSource>
   query: string
 }
 
@@ -23,7 +21,7 @@ export type SqlPrimitive = string | number | bigint | boolean | null
 export interface SelectStatement {
   distinct: boolean
   columns: SelectColumn[]
-  from?: string | FromSubquery
+  from: string | FromSubquery
   joins: JoinClause[]
   where?: ExprNode
   groupBy: ExprNode[]