squirreling 0.3.0 → 0.4.0

package/README.md

@@ -7,10 +7,10 @@
 [![minzipped](https://img.shields.io/bundlephobia/minzip/squirreling)](https://www.npmjs.com/package/squirreling)
 [![workflow status](https://github.com/hyparam/squirreling/actions/workflows/ci.yml/badge.svg)](https://github.com/hyparam/squirreling/actions)
 [![mit license](https://img.shields.io/badge/License-MIT-orange.svg)](https://opensource.org/licenses/MIT)
-![coverage](https://img.shields.io/badge/Coverage-90-darkred)
+![coverage](https://img.shields.io/badge/Coverage-93-darkred)
 [![dependencies](https://img.shields.io/badge/Dependencies-0-blueviolet)](https://www.npmjs.com/package/squirreling?activeTab=dependencies)
 
-Squirreling is a streaming async SQL engine for JavaScript. It is designed to provide efficient streaming of results from pluggable backend for highly efficient retrieval of data for browser applications.
+Squirreling is a streaming async SQL engine for JavaScript. It is designed to provide efficient streaming of results from pluggable backends for highly efficient retrieval of data for browser applications.
 
 ## Features
 
@@ -22,6 +22,8 @@ Squirreling is a streaming async SQL engine for JavaScript. It is designed to pr
 - Constant memory usage for simple queries with LIMIT
 - Robust error handling and validation designed for LLM tool use
 - In-memory data option for simple use cases
+- Select only
+- No joins (yet)
 
 ## Usage
 
@@ -39,18 +41,19 @@ const users = [
 ]
 
 // Process rows as they arrive (streaming)
-for await (const user of executeSql({
+for await (const { cnt } of executeSql({
   tables: { users },
-  query: 'SELECT * FROM users WHERE active = TRUE LIMIT 100',
+  query: 'SELECT count(*) as cnt FROM users WHERE active = TRUE LIMIT 10',
 })) {
-  console.log(user.name)
+  console.log('Count', cnt)
 }
 ```
 
 There is an exported helper function `collect` to gather all rows into an array if needed:
 
 ```javascript
-import { collect } from 'squirreling'
+import { collect, executeSql } from 'squirreling'
+
 const allUsers = await collect(executeSql({
   tables: { users },
   query: 'SELECT * FROM users',

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "squirreling",
-  "version": "0.3.0",
+  "version": "0.4.0",
   "description": "Squirreling SQL Engine",
   "author": "Hyperparam",
   "homepage": "https://hyperparam.app",

package/src/backend/dataSource.js

@@ -0,0 +1,86 @@
+/**
+ * @import { AsyncDataSource, AsyncRow, SqlPrimitive } from '../types.js'
+ */
+
+
+/**
+ * Wraps an async generator of plain objects into an AsyncDataSource
+ *
+ * @param {AsyncGenerator<AsyncRow>} gen
+ * @returns {AsyncDataSource}
+ */
+export function generatorSource(gen) {
+  return {
+    async *getRows() {
+      yield* gen
+    },
+  }
+}
+
+/**
+ * Creates an async row accessor that wraps a plain JavaScript object
+ *
+ * @param {Record<string, SqlPrimitive>} obj - the plain object
+ * @returns {AsyncRow} a row accessor interface
+ */
+function asyncRow(obj) {
+  /** @type {AsyncRow} */
+  const row = {}
+  for (const [key, value] of Object.entries(obj)) {
+    row[key] = () => Promise.resolve(value)
+  }
+  return row
+}
+
+/**
+ * Creates an async memory-backed data source from an array of plain objects
+ *
+ * @param {Record<string, SqlPrimitive>[]} data - array of plain objects
+ * @returns {AsyncDataSource} an async data source interface
+ */
+export function memorySource(data) {
+  return {
+    async *getRows() {
+      for (const item of data) {
+        yield asyncRow(item)
+      }
+    },
+  }
+}
+
+/**
+ * Wraps a data source that caches all accessed rows in memory
+ * @param {AsyncDataSource} source
+ * @returns {AsyncDataSource}
+ */
+export function cachedDataSource(source) {
+  /** @type {Map<string, Promise<SqlPrimitive>>} */
+  const cache = new Map()
+  return {
+    /**
+     * @returns {AsyncGenerator<AsyncRow>}
+     */
+    async *getRows() {
+      let index = 0
+      for await (const row of source.getRows()) {
+        const rowIndex = index
+        /** @type {AsyncRow} */
+        const out = {}
+        for (const [key, cell] of Object.entries(row)) {
+          // Wrap the cell to cache accesses
+          out[key] = () => {
+            const cacheKey = `${rowIndex}:${key}`
+            let value = cache.get(cacheKey)
+            if (!value) {
+              value = cell()
+              cache.set(cacheKey, value)
+            }
+            return value
+          }
+        }
+        yield out
+        index++
+      }
+    },
+  }
+}

package/src/execute/aggregates.js

@@ -3,19 +3,21 @@ import { evaluateExpr } from './expression.js'
 /**
  * Evaluates an aggregate function over a set of rows
  *
- * @import { AggregateColumn, ExprNode, RowSource } from '../types.js'
- * @param {AggregateColumn} col - aggregate column definition
- * @param {RowSource[]} rows - rows to aggregate
+ * @import { AggregateColumn, AsyncDataSource, ExprNode, AsyncRow } from '../types.js'
+ * @param {Object} options
+ * @param {AggregateColumn} options.col - aggregate column definition
+ * @param {AsyncRow[]} options.rows - rows to aggregate
+ * @param {Record<string, AsyncDataSource>} options.tables
  * @returns {Promise<number | null>} aggregated result
  */
-export async function evaluateAggregate(col, rows) {
+export async function evaluateAggregate({ col, rows, tables }) {
   const { arg, func } = col
 
   if (func === 'COUNT') {
     if (arg.kind === 'star') return rows.length
     let count = 0
-    for (let i = 0; i < rows.length; i += 1) {
-      const v = await evaluateExpr({ node: arg.expr, row: rows[i] })
+    for (const row of rows) {
+      const v = await evaluateExpr({ node: arg.expr, row, tables })
       if (v !== null && v !== undefined) {
         count += 1
       }
@@ -34,8 +36,8 @@ export async function evaluateAggregate(col, rows) {
     /** @type {number | null} */
     let max = null
 
-    for (let i = 0; i < rows.length; i += 1) {
-      const raw = await evaluateExpr({ node: arg.expr, row: rows[i] })
+    for (const row of rows) {
+      const raw = await evaluateExpr({ node: arg.expr, row, tables })
       if (raw == null) continue
       const num = Number(raw)
       if (!Number.isFinite(num)) continue
@@ -80,20 +82,17 @@ export function defaultAggregateAlias(col) {
 export function defaultAggregateAliasExpr(expr) {
   if (expr.type === 'identifier') {
     return expr.name
-  }
-  if (expr.type === 'literal') {
+  } else if (expr.type === 'literal') {
     return String(expr.value)
-  }
-  if (expr.type === 'cast') {
+  } else if (expr.type === 'cast') {
     return defaultAggregateAliasExpr(expr.expr) + '_as_' + expr.toType
-  }
-  if (expr.type === 'unary') {
+  } else if (expr.type === 'unary') {
     return expr.op + '_' + defaultAggregateAliasExpr(expr.argument)
-  }
-  if (expr.type === 'binary') {
+  } else if (expr.type === 'binary') {
     return defaultAggregateAliasExpr(expr.left) + '_' + expr.op + '_' + defaultAggregateAliasExpr(expr.right)
-  }
-  if (expr.type === 'function') {
+  } else if (expr.type === 'function') {
     return expr.name.toLowerCase() + '_' + expr.args.map(defaultAggregateAliasExpr).join('_')
+  } else {
+    return 'expr'
   }
 }

package/src/execute/execute.js

@@ -1,19 +1,18 @@
 import { evaluateExpr } from './expression.js'
 import { parseSql } from '../parse/parse.js'
-import { createAsyncMemorySource, createRowAccessor } from '../backend/memory.js'
+import { generatorSource, memorySource } from '../backend/dataSource.js'
 import { defaultAggregateAlias, evaluateAggregate } from './aggregates.js'
 import { evaluateHavingExpr } from './having.js'
-import { collect } from './utils.js'
 
 /**
- * @import { AsyncDataSource, ExecuteSqlOptions, ExprNode, OrderByItem, RowSource, SelectStatement, SqlPrimitive } from '../types.js'
+ * @import { AsyncDataSource, ExecuteSqlOptions, ExprNode, OrderByItem, AsyncRow, SelectStatement, SqlPrimitive } from '../types.js'
  */
 
 /**
  * Executes a SQL SELECT query against named data sources
  *
  * @param {ExecuteSqlOptions} options - the execution options
- * @returns {AsyncGenerator<Record<string, any>>} async generator yielding result rows
+ * @returns {AsyncGenerator<AsyncRow>} async generator yielding result rows
  */
 export async function* executeSql({ tables, query }) {
   const select = parseSql(query)
@@ -31,7 +30,7 @@ export async function* executeSql({ tables, query }) {
   const normalizedTables = {}
   for (const [name, source] of Object.entries(tables)) {
     if (Array.isArray(source)) {
-      normalizedTables[name] = createAsyncMemorySource(source)
+      normalizedTables[name] = memorySource(source)
     } else {
       normalizedTables[name] = source
     }
@@ -45,7 +44,7 @@ export async function* executeSql({ tables, query }) {
  *
  * @param {SelectStatement} select
  * @param {Record<string, AsyncDataSource>} tables
- * @returns {AsyncGenerator<Record<string, any>>} async generator yielding result rows
+ * @returns {AsyncGenerator<AsyncRow>}
  */
 export async function* executeSelect(select, tables) {
   /** @type {AsyncDataSource} */
@@ -60,8 +59,7 @@ export async function* executeSelect(select, tables) {
     dataSource = table
   } else {
     // Nested subquery - recursively resolve
-    const derivedData = await collect(executeSelect(select.from.query, tables))
-    dataSource = createAsyncMemorySource(derivedData)
+    dataSource = generatorSource(executeSelect(select.from.query, tables))
   }
 
   yield* evaluateSelectAst(select, dataSource, tables)
@@ -98,15 +96,15 @@ function defaultDerivedAlias(expr) {
 /**
  * Creates a stable string key for a row to enable deduplication
  *
- * @param {Record<string, any>} row
- * @returns {string} a stable string representation of the row
+ * @param {AsyncRow} row
+ * @returns {Promise<string>} a stable string representation of the row
  */
-function stableRowKey(row) {
+async function stableRowKey(row) {
   const keys = Object.keys(row).sort()
   /** @type {string[]} */
   const parts = []
   for (const k of keys) {
-    const v = row[k]
+    const v = await row[k]()
     parts.push(k + ':' + JSON.stringify(v))
   }
   return parts.join('|')
@@ -140,18 +138,18 @@ function compareValues(a, b) {
 /**
  * Applies DISTINCT filtering to remove duplicate rows
  *
- * @param {Record<string, any>[]} rows - The input rows
- * @param {boolean} distinct - Whether to apply deduplication
- * @returns {Record<string, any>[]} The deduplicated rows
+ * @param {AsyncRow[]} rows - the input rows
+ * @param {boolean} distinct - whether to apply deduplication
+ * @returns {Promise<AsyncRow[]>} the deduplicated rows
  */
-function applyDistinct(rows, distinct) {
+async function applyDistinct(rows, distinct) {
   if (!distinct) return rows
   /** @type {Set<string>} */
   const seen = new Set()
-  /** @type {Record<string, any>[]} */
+  /** @type {AsyncRow[]} */
   const result = []
   for (const row of rows) {
-    const key = stableRowKey(row)
+    const key = await stableRowKey(row)
     if (seen.has(key)) continue
     seen.add(key)
     result.push(row)
@@ -162,13 +160,13 @@ function applyDistinct(rows, distinct) {
 /**
  * Applies ORDER BY sorting to RowSource array (before projection)
  *
- * @param {RowSource[]} rows - the input row sources
+ * @param {AsyncRow[]} rows - the input row sources
  * @param {OrderByItem[]} orderBy - the sort specifications
  * @param {Record<string, AsyncDataSource>} tables
- * @returns {Promise<RowSource[]>} the sorted row sources
+ * @returns {Promise<AsyncRow[]>} the sorted row sources
  */
 async function sortRowSources(rows, orderBy, tables) {
-  if (!orderBy?.length) return rows
+  if (!orderBy.length) return rows
 
   // Pre-evaluate ORDER BY expressions for all rows
   /** @type {SqlPrimitive[][]} */
@@ -223,13 +221,13 @@ async function sortRowSources(rows, orderBy, tables) {
 /**
  * Applies ORDER BY sorting to rows
  *
- * @param {Record<string, any>[]} rows - the input rows
+ * @param {AsyncRow[]} rows - the input rows
  * @param {OrderByItem[]} orderBy - the sort specifications
  * @param {Record<string, AsyncDataSource>} tables
- * @returns {Promise<Record<string, any>[]>} the sorted rows
+ * @returns {Promise<AsyncRow[]>} the sorted rows
  */
 async function applyOrderBy(rows, orderBy, tables) {
-  if (!orderBy?.length) return rows
+  if (!orderBy.length) return rows
 
   // Pre-evaluate ORDER BY expressions for all rows
   /** @type {SqlPrimitive[][]} */
@@ -238,7 +236,7 @@ async function applyOrderBy(rows, orderBy, tables) {
     /** @type {SqlPrimitive[]} */
     const rowValues = []
     for (const term of orderBy) {
-      const value = await evaluateExpr({ node: term.expr, row: createRowAccessor(row), tables })
+      const value = await evaluateExpr({ node: term.expr, row, tables })
       rowValues.push(value)
     }
     evaluatedValues.push(rowValues)
@@ -284,22 +282,17 @@ async function applyOrderBy(rows, orderBy, tables) {
 /**
  * Evaluates a select with a resolved FROM data source
  *
- * @param {SelectStatement} select - the parsed SQL AST
- * @param {AsyncDataSource} dataSource - the async data source
+ * @param {SelectStatement} select
+ * @param {AsyncDataSource} dataSource
  * @param {Record<string, AsyncDataSource>} tables
- * @returns {AsyncGenerator<Record<string, any>>} async generator yielding result rows
+ * @returns {AsyncGenerator<AsyncRow>}
  */
 async function* evaluateSelectAst(select, dataSource, tables) {
   // SQL priority: from, where, group by, having, select, order by, offset, limit
 
   const hasAggregate = select.columns.some(col => col.kind === 'aggregate')
-  const useGrouping = hasAggregate || select.groupBy?.length > 0
-
-  // Determine if we need to buffer (collect all rows first)
-  const needsBuffering =
-    select.orderBy.length > 0 ||
-    select.distinct ||
-    useGrouping
+  const useGrouping = hasAggregate || select.groupBy.length > 0
+  const needsBuffering = useGrouping || select.orderBy.length > 0
 
   if (needsBuffering) {
     // BUFFERING PATH: Collect all rows, process, then yield
@@ -311,52 +304,49 @@ async function* evaluateSelectAst(select, dataSource, tables) {
 }
 
 /**
- * Streaming evaluation for simple queries (no ORDER BY, DISTINCT, or GROUP BY)
+ * Streaming evaluation for simple queries (no ORDER BY or GROUP BY)
+ * Supports DISTINCT by tracking seen row keys without buffering full rows
  *
  * @param {SelectStatement} select
  * @param {AsyncDataSource} dataSource
  * @param {Record<string, AsyncDataSource>} tables
- * @returns {AsyncGenerator<Record<string, any>>}
+ * @returns {AsyncGenerator<AsyncRow>}
  */
 async function* evaluateStreaming(select, dataSource, tables) {
   let rowsYielded = 0
   let rowsSkipped = 0
   const offset = select.offset ?? 0
   const limit = select.limit ?? Infinity
+  if (limit <= 0) return
+
+  // For DISTINCT, track seen row keys
+  /** @type {Set<string> | undefined} */
+  const seen = select.distinct ? new Set() : undefined
 
   for await (const row of dataSource.getRows()) {
     // WHERE filter
     if (select.where) {
-      const passes = await evaluateExpr({ node: select.where, row, tables })
-
-      if (!passes) {
-        continue
-      }
+      const pass = await evaluateExpr({ node: select.where, row, tables })
+      if (!pass) continue
     }
 
-    // OFFSET handling
-    if (rowsSkipped < offset) {
+    // For non-DISTINCT queries, we can skip rows before projection (optimization)
+    if (!seen && rowsSkipped < offset) {
       rowsSkipped++
       continue
     }
 
-    // LIMIT handling
-    if (rowsYielded >= limit) {
-      break
-    }
-
     // SELECT projection
-    /** @type {Record<string, any>} */
+    /** @type {AsyncRow} */
     const outRow = {}
     for (const col of select.columns) {
       if (col.kind === 'star') {
-        const keys = row.getKeys()
-        for (const key of keys) {
-          outRow[key] = row.getCell(key)
+        for (const [key, cell] of Object.entries(row)) {
+          outRow[key] = cell
         }
       } else if (col.kind === 'derived') {
         const alias = col.alias ?? defaultDerivedAlias(col.expr)
-        outRow[alias] = await evaluateExpr({ node: col.expr, row, tables })
+        outRow[alias] = () => evaluateExpr({ node: col.expr, row, tables })
       } else if (col.kind === 'aggregate') {
         throw new Error(
           'Aggregate functions require GROUP BY or will act on the whole dataset; add GROUP BY or remove aggregates'
@@ -364,31 +354,46 @@ async function* evaluateStreaming(select, dataSource, tables) {
       }
     }
 
+    // DISTINCT: skip duplicate rows
+    if (seen) {
+      const key = await stableRowKey(outRow)
+      if (seen.has(key)) continue
+      seen.add(key)
+      // OFFSET applies to distinct rows
+      if (rowsSkipped < offset) {
+        rowsSkipped++
+        continue
+      }
+    }
+
     yield outRow
     rowsYielded++
+    if (rowsYielded >= limit) {
+      break
+    }
   }
 }
 
 /**
- * Buffered evaluation for complex queries (with ORDER BY, DISTINCT, or GROUP BY)
+ * Buffered evaluation for complex queries (with ORDER BY or GROUP BY)
  *
  * @param {SelectStatement} select
  * @param {AsyncDataSource} dataSource
  * @param {Record<string, AsyncDataSource>} tables
  * @param {boolean} hasAggregate
  * @param {boolean} useGrouping
- * @returns {AsyncGenerator<Record<string, any>>}
+ * @returns {AsyncGenerator<AsyncRow>}
  */
 async function* evaluateBuffered(select, dataSource, tables, hasAggregate, useGrouping) {
   // Step 1: Collect all rows from data source
-  /** @type {RowSource[]} */
+  /** @type {AsyncRow[]} */
   const working = []
   for await (const row of dataSource.getRows()) {
     working.push(row)
   }
 
   // Step 2: WHERE clause filtering
-  /** @type {RowSource[]} */
+  /** @type {AsyncRow[]} */
   const filtered = []
 
   for (const row of working) {
@@ -403,16 +408,16 @@ async function* evaluateBuffered(select, dataSource, tables, hasAggregate, useGr
   }
 
   // Step 3: Projection (grouping vs non-grouping)
-  /** @type {Record<string, any>[]} */
+  /** @type {AsyncRow[]} */
   let projected = []
 
   if (useGrouping) {
     // Grouping due to GROUP BY or aggregate functions
-    /** @type {RowSource[][]} */
+    /** @type {AsyncRow[][]} */
     const groups = []
 
-    if (select.groupBy?.length) {
-      /** @type {Map<string, RowSource[]>} */
+    if (select.groupBy.length) {
+      /** @type {Map<string, AsyncRow[]>} */
       const map = new Map()
       for (const row of filtered) {
         /** @type {string[]} */
@@ -440,15 +445,14 @@ async function* evaluateBuffered(select, dataSource, tables, hasAggregate, useGr
     }
 
     for (const group of groups) {
-      /** @type {Record<string, any>} */
+      /** @type {AsyncRow} */
       const resultRow = {}
       for (const col of select.columns) {
         if (col.kind === 'star') {
           const firstRow = group[0]
           if (firstRow) {
-            const keys = firstRow.getKeys()
-            for (const key of keys) {
-              resultRow[key] = firstRow.getCell(key)
+            for (const [key, cell] of Object.entries(firstRow)) {
+              resultRow[key] = cell
             }
           }
           continue
@@ -457,18 +461,16 @@ async function* evaluateBuffered(select, dataSource, tables, hasAggregate, useGr
         if (col.kind === 'derived') {
           const alias = col.alias ?? defaultDerivedAlias(col.expr)
           if (group.length > 0) {
-            const value = await evaluateExpr({ node: col.expr, row: group[0], tables })
-            resultRow[alias] = value
+            resultRow[alias] = () => evaluateExpr({ node: col.expr, row: group[0], tables })
           } else {
-            resultRow[alias] = undefined
+            delete resultRow[alias]
           }
           continue
         }
 
         if (col.kind === 'aggregate') {
           const alias = col.alias ?? defaultAggregateAlias(col)
-          const value = await evaluateAggregate(col, group)
-          resultRow[alias] = value
+          resultRow[alias] = () => evaluateAggregate({ col, rows: group, tables })
           continue
         }
       }
@@ -487,19 +489,26 @@ async function* evaluateBuffered(select, dataSource, tables, hasAggregate, useGr
     // Sort before projection so ORDER BY can access columns not in SELECT
     const sorted = await sortRowSources(filtered, select.orderBy, tables)
 
-    for (const row of sorted) {
-      /** @type {Record<string, any>} */
+    // OPTIMIZATION: For non-DISTINCT queries, apply OFFSET/LIMIT before projection
+    // to avoid reading expensive cells for rows that won't be in the final result
+    let rowsToProject = sorted
+    if (!select.distinct) {
+      const start = select.offset ?? 0
+      const end = select.limit ? start + select.limit : sorted.length
+      rowsToProject = sorted.slice(start, end)
+    }
+
+    for (const row of rowsToProject) {
+      /** @type {AsyncRow} */
       const outRow = {}
       for (const col of select.columns) {
         if (col.kind === 'star') {
-          const keys = row.getKeys()
-          for (const key of keys) {
-            outRow[key] = row.getCell(key)
+          for (const [key, cell] of Object.entries(row)) {
+            outRow[key] = cell
           }
         } else if (col.kind === 'derived') {
           const alias = col.alias ?? defaultDerivedAlias(col.expr)
-          const value = await evaluateExpr({ node: col.expr, row, tables })
-          outRow[alias] = value
+          outRow[alias] = () => evaluateExpr({ node: col.expr, row, tables })
         }
       }
       projected.push(outRow)
@@ -507,17 +516,25 @@ async function* evaluateBuffered(select, dataSource, tables, hasAggregate, useGr
   }
 
   // Step 4: DISTINCT
-  projected = applyDistinct(projected, select.distinct)
+  projected = await applyDistinct(projected, select.distinct)
 
   // Step 5: ORDER BY (final sort for grouped queries)
   projected = await applyOrderBy(projected, select.orderBy, tables)
 
   // Step 6: OFFSET and LIMIT
-  const start = select.offset ?? 0
-  const end = select.limit ? start + select.limit : projected.length
-
-  // Step 7: Yield results
-  for (let i = start; i < end && i < projected.length; i++) {
-    yield projected[i]
+  // For non-DISTINCT, non-grouping queries, OFFSET/LIMIT was already applied before projection
+  if (select.distinct || useGrouping) {
+    const start = select.offset ?? 0
+    const end = select.limit ? start + select.limit : projected.length
+
+    // Step 7: Yield results
+    for (let i = start; i < end && i < projected.length; i++) {
+      yield projected[i]
+    }
+  } else {
+    // Already limited, yield all projected rows
+    for (const row of projected) {
+      yield row
+    }
   }
 }

package/src/execute/expression.js

@@ -1,8 +1,7 @@
 import { executeSelect } from './execute.js'
-import { collect } from './utils.js'
 
 /**
- * @import { ExprNode, RowSource, SqlPrimitive, AsyncDataSource } from '../types.js'
+ * @import { ExprNode, AsyncRow, SqlPrimitive, AsyncDataSource } from '../types.js'
  */
 
 /**
@@ -10,8 +9,8 @@ import { collect } from './utils.js'
  *
  * @param {Object} params
  * @param {ExprNode} params.node - The expression node to evaluate
- * @param {RowSource} params.row - The data row to evaluate against
- * @param {Record<string, AsyncDataSource>} [params.tables]
+ * @param {AsyncRow} params.row - The data row to evaluate against
+ * @param {Record<string, AsyncDataSource>} params.tables
  * @returns {Promise<SqlPrimitive>} The result of the evaluation
  */
 export async function evaluateExpr({ node, row, tables }) {
@@ -20,7 +19,19 @@ export async function evaluateExpr({ node, row, tables }) {
   }
 
   if (node.type === 'identifier') {
-    return row.getCell(node.name)
+    return row[node.name]?.()
+  }
+
+  // Scalar subquery - returns a single value
+  if (node.type === 'subquery') {
+    const gen = executeSelect(node.subquery, tables)
+    const first = await gen.next() // Start the generator
+    gen.return() // Stop further execution
+    if (first.done) return null
+    /** @type {AsyncRow} */
+    const firstRow = first.value
+    const firstKey = Object.keys(firstRow)[0]
+    return firstRow[firstKey]()
   }
 
   // Unary operators
@@ -228,29 +239,37 @@ export async function evaluateExpr({ node, row, tables }) {
   // IN and NOT IN with subqueries
   if (node.type === 'in') {
     const exprVal = await evaluateExpr({ node: node.expr, row, tables })
-    const results = await collect(executeSelect(node.subquery, tables))
-    if (results.length === 0) return false
-    const firstKey = Object.keys(results[0])[0]
-    const values = results.map(r => r[firstKey])
+    const results = executeSelect(node.subquery, tables)
+    /** @type {SqlPrimitive[]} */
+    const values = []
+    for await (const resRow of results) {
+      const firstKey = Object.keys(resRow)[0]
+      const val = await resRow[firstKey]()
+      values.push(val)
+    }
     return values.includes(exprVal)
   }
   if (node.type === 'not in') {
     const exprVal = await evaluateExpr({ node: node.expr, row, tables })
-    const results = await collect(executeSelect(node.subquery, tables))
-    if (results.length === 0) return true
-    const firstKey = Object.keys(results[0])[0]
-    const values = results.map(r => r[firstKey])
+    const results = executeSelect(node.subquery, tables)
+    /** @type {SqlPrimitive[]} */
+    const values = []
+    for await (const resRow of results) {
+      const firstKey = Object.keys(resRow)[0]
+      const val = await resRow[firstKey]()
+      values.push(val)
+    }
     return !values.includes(exprVal)
   }
 
   // EXISTS and NOT EXISTS with subqueries
   if (node.type === 'exists') {
-    const results = await collect(executeSelect(node.subquery, tables))
-    return results.length > 0
+    const results = await executeSelect(node.subquery, tables).next()
+    return results.done === false
   }
   if (node.type === 'not exists') {
-    const results = await collect(executeSelect(node.subquery, tables))
-    return results.length === 0
+    const results = await executeSelect(node.subquery, tables).next()
+    return results.done === true
   }
 
   // CASE expressions

package/src/execute/having.js

@@ -1,5 +1,5 @@
 /**
- * @import { AggregateFunc, AsyncDataSource, ExprNode, RowSource, SqlPrimitive } from '../types.js'
+ * @import { AggregateFunc, AsyncDataSource, ExprNode, AsyncRow, SqlPrimitive } from '../types.js'
  */
 
 import { isAggregateFunc } from '../validation.js'
@@ -8,32 +8,17 @@ import { evaluateExpr } from './expression.js'
 /**
  * Creates a context for evaluating HAVING expressions
  *
- * @param {Record<string, any>} resultRow - the aggregated result row
- * @param {RowSource[]} group - the group of rows
- * @returns {RowSource} a context row for HAVING evaluation
+ * @param {AsyncRow} resultRow - the aggregated result row
+ * @param {AsyncRow[]} group - the group of rows
+ * @returns {AsyncRow} a context row for HAVING evaluation
  */
 function createHavingContext(resultRow, group) {
   // Include the first row of the group (for GROUP BY columns)
   const firstRow = group[0]
-  /** @type {Record<string, any>} */
-  const context = {}
   if (firstRow) {
-    const keys = firstRow.getKeys()
-    for (const key of keys) {
-      context[key] = firstRow.getCell(key)
-    }
-  }
-  // Merge with result row (which has aggregates computed)
-  Object.assign(context, resultRow)
-
-  // Return a Row accessor wrapping the context
-  return {
-    getCell(name) {
-      return context[name]
-    },
-    getKeys() {
-      return Object.keys(context)
-    },
+    return { ...firstRow, ...resultRow }
+  } else {
+    return resultRow
   }
 }
 
@@ -41,8 +26,8 @@ function createHavingContext(resultRow, group) {
  * Evaluates a HAVING expression with support for aggregate functions
  *
  * @param {ExprNode} expr - the HAVING expression
- * @param {Record<string, any>} row - the aggregated result row
- * @param {RowSource[]} group - the group of rows for re-evaluating aggregates
+ * @param {AsyncRow} row - the aggregated result row
+ * @param {AsyncRow[]} group - the group of rows for re-evaluating aggregates
  * @param {Record<string, AsyncDataSource>} tables
  * @returns {Promise<boolean>} whether the HAVING condition is satisfied
  */
@@ -129,8 +114,8 @@ export async function evaluateHavingExpr(expr, row, group, tables) {
  * Evaluates a value in a HAVING expression
  *
  * @param {ExprNode} expr
- * @param {RowSource} context - the context row
- * @param {RowSource[]} group - the group of rows
+ * @param {AsyncRow} context - the context row
+ * @param {AsyncRow[]} group - the group of rows
  * @param {Record<string, AsyncDataSource>} tables
  * @returns {Promise<SqlPrimitive>} the evaluated value
  */
@@ -155,7 +140,7 @@ function evaluateHavingValue(expr, context, group, tables) {
  *
  * @param {AggregateFunc} funcName - aggregate function name
  * @param {ExprNode[]} args - function arguments
- * @param {RowSource[]} group - the group of rows
+ * @param {AsyncRow[]} group - the group of rows
  * @param {Record<string, AsyncDataSource>} tables
  * @returns {Promise<SqlPrimitive>} the aggregate result
  */

package/src/execute/utils.js

@@ -1,13 +1,19 @@
 /**
- * Collects all results from an async generator into an array
+ * Collects and materialize all results from an async row generator into an array
  *
- * @template T
- * @param {AsyncGenerator<T>} asyncGen - the async generator
- * @returns {Promise<T[]>} array of all yielded values
+ * @import {AsyncRow, SqlPrimitive} from '../types.js'
+ * @param {AsyncGenerator<AsyncRow>} asyncRows
+ * @returns {Promise<Record<string, SqlPrimitive>[]>} array of all yielded values
  */
-export async function collect(asyncGen) {
+export async function collect(asyncRows) {
+  /** @type {Record<string, SqlPrimitive>[]} */
   const results = []
-  for await (const item of asyncGen) {
+  for await (const asyncRow of asyncRows) {
+    /** @type {Record<string, SqlPrimitive>} */
+    const item = {}
+    for (const [key, cell] of Object.entries(asyncRow)) {
+      item[key] = await cell()
+    }
     results.push(item)
   }
   return results

package/src/index.d.ts

@@ -1,4 +1,4 @@
-import type { ExecuteSqlOptions, SelectStatement } from './types.js'
+import type { AsyncRow, ExecuteSqlOptions, SelectStatement, SqlPrimitive } from './types.js'
 
 /**
  * Executes a SQL SELECT query against an array of data rows
@@ -8,12 +8,12 @@ import type { ExecuteSqlOptions, SelectStatement } from './types.js'
  * @param options.query - SQL query string
  * @returns async generator yielding rows matching the query
  */
-export function executeSql(options: ExecuteSqlOptions): AsyncGenerator<Record<string, any>>
+export function executeSql(options: ExecuteSqlOptions): AsyncGenerator<AsyncRow>
 
 /**
  * Parses a SQL query string into an abstract syntax tree
  *
- * @param sql - SQL query string to parse
+ * @param query - SQL query string to parse
  * @returns parsed SQL select statement
  */
 export function parseSql(query: string): SelectStatement
@@ -24,4 +24,4 @@ export function parseSql(query: string): SelectStatement
  * @param asyncGen - the async generator
  * @returns array of all yielded values
  */
-export function collect<T>(asyncGen: AsyncGenerator<T>): Promise<T[]>
+export function collect<T>(asyncGen: AsyncGenerator<AsyncRow>): Promise<Record<string, SqlPrimitive>[]>

package/src/parse/expression.js

@@ -20,6 +20,17 @@ function parsePrimary(c) {
   const tok = c.current()
 
   if (tok.type === 'paren' && tok.value === '(') {
+    // Peek ahead to see if this is a scalar subquery
+    const nextTok = c.peek(1)
+    if (nextTok.type === 'keyword' && nextTok.value === 'SELECT') {
+      // It's a scalar subquery
+      const subquery = c.parseSubquery()
+      return {
+        type: 'subquery',
+        subquery,
+      }
+    }
+    // Regular grouped expression
     c.consume()
     const expr = parseExpression(c)
     c.expect('paren', ')')
@@ -132,9 +143,6 @@ function parsePrimary(c) {
     }
     if (tok.value === 'EXISTS') {
       c.consume() // EXISTS
-      if (!c.parseSubquery) {
-        throw new Error('Subquery parsing not available in this context')
-      }
       const subquery = c.parseSubquery()
       return {
         type: 'exists',

package/src/types.d.ts

@@ -4,17 +4,15 @@
  * Provides an async iterator over rows.
  */
 export interface AsyncDataSource {
-  getRows(): AsyncIterable<RowSource>
-}
-export interface RowSource {
-  getCell(name: string): any
-  getKeys(): string[]
+  getRows(): AsyncIterable<AsyncRow>
 }
+export type AsyncRow = Record<string, AsyncCell>
+export type AsyncCell = () => Promise<SqlPrimitive>
 
-export type RawData = Record<string, any>[]
+export type Row = Record<string, SqlPrimitive>[]
 
 export interface ExecuteSqlOptions {
-  tables: Record<string, RawData | AsyncDataSource>
+  tables: Record<string, Row | AsyncDataSource>
   query: string
 }
 
@@ -23,7 +21,7 @@ export type SqlPrimitive = string | number | bigint | boolean | null
 export interface SelectStatement {
   distinct: boolean
   columns: SelectColumn[]
-  from?: string | FromSubquery
+  from: string | FromSubquery
   joins: JoinClause[]
   where?: ExprNode
   groupBy: ExprNode[]
@@ -122,6 +120,11 @@ export interface CaseNode {
   elseResult?: ExprNode
 }
 
+export interface SubqueryNode {
+  type: 'subquery'
+  subquery: SelectStatement
+}
+
 export type ExprNode =
   | LiteralNode
   | IdentifierNode
@@ -134,6 +137,7 @@ export type ExprNode =
   | InValuesNode
   | ExistsNode
   | CaseNode
+  | SubqueryNode
 
 export interface StarColumn {
   kind: 'star'
@@ -197,7 +201,7 @@ export interface ExprCursor {
   match(type: TokenType, value?: string): boolean
   expect(type: TokenType, value: string): Token
   expectIdentifier(): Token
-  parseSubquery?: () => SelectStatement
+  parseSubquery: () => SelectStatement
 }
 
 // Tokenizer types

package/src/backend/memory.js

@@ -1,36 +0,0 @@
-/**
- * @import { AsyncDataSource, RowSource } from '../types.js'
- */
-
-/**
- * Creates a row accessor that wraps a plain JavaScript object
- *
- * @param {Record<string, any>} obj - the plain object
- * @returns {RowSource} a row accessor interface
- */
-export function createRowAccessor(obj) {
-  return {
-    getCell(name) {
-      return obj[name]
-    },
-    getKeys() {
-      return Object.keys(obj)
-    },
-  }
-}
-
-/**
- * Creates an async memory-backed data source from an array of plain objects
- *
- * @param {Record<string, any>[]} data - array of plain objects
- * @returns {AsyncDataSource} an async data source interface
- */
-export function createAsyncMemorySource(data) {
-  return {
-    async *getRows() {
-      for (const item of data) {
-        yield createRowAccessor(item)
-      }
-    },
-  }
-}