squirreling 0.2.6 → 0.3.0

package/README.md

@@ -7,30 +7,53 @@
 [![minzipped](https://img.shields.io/bundlephobia/minzip/squirreling)](https://www.npmjs.com/package/squirreling)
 [![workflow status](https://github.com/hyparam/squirreling/actions/workflows/ci.yml/badge.svg)](https://github.com/hyparam/squirreling/actions)
 [![mit license](https://img.shields.io/badge/License-MIT-orange.svg)](https://opensource.org/licenses/MIT)
-![coverage](https://img.shields.io/badge/Coverage-89-darkred)
+![coverage](https://img.shields.io/badge/Coverage-90-darkred)
 [![dependencies](https://img.shields.io/badge/Dependencies-0-blueviolet)](https://www.npmjs.com/package/squirreling?activeTab=dependencies)
 
-Squirreling is a lightweight SQL engine for JavaScript applications, designed to provide efficient and easy-to-use database functionalities in the browser.
+Squirreling is a streaming async SQL engine for JavaScript. It is designed to provide efficient streaming of results from pluggable backend for highly efficient retrieval of data for browser applications.
 
 ## Features
 
 - Lightweight and fast
-- Easy to integrate with JavaScript applications
+- Easy to integrate with frontend applications
+- Lets you move query execution closer to your users
 - Supports standard SQL queries
-- In-memory database for quick data access
-- Robust error handling and validation
+- Async streaming for large datasets
+- Constant memory usage for simple queries with LIMIT
+- Robust error handling and validation designed for LLM tool use
+- In-memory data option for simple use cases
 
 ## Usage
 
+Squirreling returns an async generator, allowing you to process rows one at a time without loading everything into memory.
+
 ```javascript
 import { executeSql } from 'squirreling'
 
-const source = [
-  { id: 1, name: 'Alice' },
-  { id: 2, name: 'Bob' },
+// In-memory table
+const users = [
+  { id: 1, name: 'Alice', active: true },
+  { id: 2, name: 'Bob', active: false },
+  { id: 3, name: 'Charlie', active: true },
+  // ...more rows
 ]
 
-const result = executeSql({ source, query: 'SELECT UPPER(name) AS name_upper FROM users' })
-console.log(result)
-// Output: [ { name_upper: 'ALICE' }, { name_upper: 'BOB' } ]
+// Process rows as they arrive (streaming)
+for await (const user of executeSql({
+  tables: { users },
+  query: 'SELECT * FROM users WHERE active = TRUE LIMIT 100',
+})) {
+  console.log(user.name)
+}
+```
+
+There is an exported helper function `collect` to gather all rows into an array if needed:
+
+```javascript
+import { collect } from 'squirreling'
+const allUsers = await collect(executeSql({
+  tables: { users },
+  query: 'SELECT * FROM users',
+}))
+console.log(allUsers)
 ```

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "squirreling",
-  "version": "0.2.6",
+  "version": "0.3.0",
   "description": "Squirreling SQL Engine",
   "author": "Hyperparam",
   "homepage": "https://hyperparam.app",
@@ -38,10 +38,10 @@
   },
   "devDependencies": {
     "@types/node": "24.10.1",
-    "@vitest/coverage-v8": "4.0.13",
+    "@vitest/coverage-v8": "4.0.14",
     "eslint": "9.39.1",
     "eslint-plugin-jsdoc": "61.4.1",
     "typescript": "5.9.3",
-    "vitest": "4.0.13"
+    "vitest": "4.0.14"
   }
 }

package/src/backend/memory.js

@@ -1,5 +1,5 @@
 /**
- * @import { DataSource, RowSource } from '../types.js'
+ * @import { AsyncDataSource, RowSource } from '../types.js'
  */
 
 /**
@@ -20,18 +20,17 @@ export function createRowAccessor(obj) {
 }
 
 /**
- * Creates a memory-backed data source from an array of plain objects
+ * Creates an async memory-backed data source from an array of plain objects
  *
  * @param {Record<string, any>[]} data - array of plain objects
- * @returns {DataSource} a data source interface
+ * @returns {AsyncDataSource} an async data source interface
  */
-export function createMemorySource(data) {
+export function createAsyncMemorySource(data) {
   return {
-    getNumRows() {
-      return data.length
-    },
-    getRow(index) {
-      return createRowAccessor(data[index])
+    async *getRows() {
+      for (const item of data) {
+        yield createRowAccessor(item)
+      }
     },
   }
 }

package/src/execute/aggregates.js

@@ -6,16 +6,16 @@ import { evaluateExpr } from './expression.js'
  * @import { AggregateColumn, ExprNode, RowSource } from '../types.js'
  * @param {AggregateColumn} col - aggregate column definition
  * @param {RowSource[]} rows - rows to aggregate
- * @returns {number | null} aggregated result
+ * @returns {Promise<number | null>} aggregated result
  */
-export function evaluateAggregate(col, rows) {
+export async function evaluateAggregate(col, rows) {
   const { arg, func } = col
 
   if (func === 'COUNT') {
     if (arg.kind === 'star') return rows.length
     let count = 0
     for (let i = 0; i < rows.length; i += 1) {
-      const v = evaluateExpr(arg.expr, rows[i])
+      const v = await evaluateExpr({ node: arg.expr, row: rows[i] })
       if (v !== null && v !== undefined) {
         count += 1
       }
@@ -35,7 +35,7 @@ export function evaluateAggregate(col, rows) {
     let max = null
 
     for (let i = 0; i < rows.length; i += 1) {
-      const raw = evaluateExpr(arg.expr, rows[i])
+      const raw = await evaluateExpr({ node: arg.expr, row: rows[i] })
       if (raw == null) continue
       const num = Number(raw)
       if (!Number.isFinite(num)) continue

package/src/execute/execute.js

@@ -1,23 +1,70 @@
-import { defaultAggregateAlias, evaluateAggregate } from './aggregates.js'
 import { evaluateExpr } from './expression.js'
-import { evaluateHavingExpr } from './having.js'
 import { parseSql } from '../parse/parse.js'
-import { createMemorySource, createRowAccessor } from '../backend/memory.js'
+import { createAsyncMemorySource, createRowAccessor } from '../backend/memory.js'
+import { defaultAggregateAlias, evaluateAggregate } from './aggregates.js'
+import { evaluateHavingExpr } from './having.js'
+import { collect } from './utils.js'
 
 /**
- * @import { DataSource, ExecuteSqlOptions, ExprNode, OrderByItem, RowSource, SelectStatement, SqlPrimitive } from '../types.js'
+ * @import { AsyncDataSource, ExecuteSqlOptions, ExprNode, OrderByItem, RowSource, SelectStatement, SqlPrimitive } from '../types.js'
  */
 
 /**
- * Executes a SQL SELECT query against a data source
+ * Executes a SQL SELECT query against named data sources
  *
  * @param {ExecuteSqlOptions} options - the execution options
- * @returns {Record<string, any>[]} the result rows matching the query
+ * @returns {AsyncGenerator<Record<string, any>>} async generator yielding result rows
  */
-export function executeSql({ source, query }) {
+export async function* executeSql({ tables, query }) {
   const select = parseSql(query)
-  const dataSource = Array.isArray(source) ? createMemorySource(source) : source
-  return evaluateSelectAst(select, dataSource)
+
+  // Check for unsupported operations
+  if (select.joins.length) {
+    throw new Error('JOIN is not supported')
+  }
+  if (!select.from) {
+    throw new Error('FROM clause is required')
+  }
+
+  // Normalize tables: convert arrays to AsyncDataSource
+  /** @type {Record<string, AsyncDataSource>} */
+  const normalizedTables = {}
+  for (const [name, source] of Object.entries(tables)) {
+    if (Array.isArray(source)) {
+      normalizedTables[name] = createAsyncMemorySource(source)
+    } else {
+      normalizedTables[name] = source
+    }
+  }
+
+  yield* executeSelect(select, normalizedTables)
+}
+
+/**
+ * Executes a SELECT query against the provided tables
+ *
+ * @param {SelectStatement} select
+ * @param {Record<string, AsyncDataSource>} tables
+ * @returns {AsyncGenerator<Record<string, any>>} async generator yielding result rows
+ */
+export async function* executeSelect(select, tables) {
+  /** @type {AsyncDataSource} */
+  let dataSource
+
+  if (typeof select.from === 'string') {
+    const table = tables[select.from]
+    if (table === undefined) {
+      throw new Error(`Table "${select.from}" not found`)
+    }
+
+    dataSource = table
+  } else {
+    // Nested subquery - recursively resolve
+    const derivedData = await collect(executeSelect(select.from.query, tables))
+    dataSource = createAsyncMemorySource(derivedData)
+  }
+
+  yield* evaluateSelectAst(select, dataSource, tables)
 }
 
 /**
@@ -112,32 +159,108 @@ function applyDistinct(rows, distinct) {
   return result
 }
 
+/**
+ * Applies ORDER BY sorting to RowSource array (before projection)
+ *
+ * @param {RowSource[]} rows - the input row sources
+ * @param {OrderByItem[]} orderBy - the sort specifications
+ * @param {Record<string, AsyncDataSource>} tables
+ * @returns {Promise<RowSource[]>} the sorted row sources
+ */
+async function sortRowSources(rows, orderBy, tables) {
+  if (!orderBy?.length) return rows
+
+  // Pre-evaluate ORDER BY expressions for all rows
+  /** @type {SqlPrimitive[][]} */
+  const evaluatedValues = []
+  for (const row of rows) {
+    /** @type {SqlPrimitive[]} */
+    const rowValues = []
+    for (const term of orderBy) {
+      const value = await evaluateExpr({ node: term.expr, row, tables })
+      rowValues.push(value)
+    }
+    evaluatedValues.push(rowValues)
+  }
+
+  // Create index array and sort it
+  const indices = rows.map((_, i) => i)
+  indices.sort((aIdx, bIdx) => {
+    for (let termIdx = 0; termIdx < orderBy.length; termIdx++) {
+      const term = orderBy[termIdx]
+      const dir = term.direction
+      const av = evaluatedValues[aIdx][termIdx]
+      const bv = evaluatedValues[bIdx][termIdx]
+
+      // Handle NULLS FIRST / NULLS LAST
+      const aIsNull = av == null
+      const bIsNull = bv == null
+
+      if (aIsNull || bIsNull) {
+        if (aIsNull && bIsNull) continue
+
+        const nullsFirst = term.nulls === 'LAST' ? false : true
+
+        if (aIsNull) {
+          return nullsFirst ? -1 : 1
+        } else {
+          return nullsFirst ? 1 : -1
+        }
+      }
+
+      const cmp = compareValues(av, bv)
+      if (cmp !== 0) {
+        return dir === 'DESC' ? -cmp : cmp
+      }
+    }
+    return 0
+  })
+
+  // Return sorted rows
+  return indices.map(i => rows[i])
+}
+
 /**
  * Applies ORDER BY sorting to rows
  *
  * @param {Record<string, any>[]} rows - the input rows
  * @param {OrderByItem[]} orderBy - the sort specifications
- * @returns {Record<string, any>[]} the sorted rows
+ * @param {Record<string, AsyncDataSource>} tables
+ * @returns {Promise<Record<string, any>[]>} the sorted rows
  */
-function applyOrderBy(rows, orderBy) {
+async function applyOrderBy(rows, orderBy, tables) {
   if (!orderBy?.length) return rows
 
-  const sorted = rows.slice()
-  sorted.sort((a, b) => {
+  // Pre-evaluate ORDER BY expressions for all rows
+  /** @type {SqlPrimitive[][]} */
+  const evaluatedValues = []
+  for (const row of rows) {
+    /** @type {SqlPrimitive[]} */
+    const rowValues = []
     for (const term of orderBy) {
+      const value = await evaluateExpr({ node: term.expr, row: createRowAccessor(row), tables })
+      rowValues.push(value)
+    }
+    evaluatedValues.push(rowValues)
+  }
+
+  // Create index array and sort it
+  const indices = rows.map((_, i) => i)
+  indices.sort((aIdx, bIdx) => {
+    for (let termIdx = 0; termIdx < orderBy.length; termIdx++) {
+      const term = orderBy[termIdx]
       const dir = term.direction
-      const av = evaluateExpr(term.expr, createRowAccessor(a))
-      const bv = evaluateExpr(term.expr, createRowAccessor(b))
+      const av = evaluatedValues[aIdx][termIdx]
+      const bv = evaluatedValues[bIdx][termIdx]
 
       // Handle NULLS FIRST / NULLS LAST
       const aIsNull = av == null
       const bIsNull = bv == null
 
       if (aIsNull || bIsNull) {
-        if (aIsNull && bIsNull) continue // both null, try next sort term
+        if (aIsNull && bIsNull) continue
 
-        // Determine null ordering
-        const nullsFirst = term.nulls === 'LAST' ? false : true // default is NULLS FIRST
+        const nullsFirst = term.nulls === 'LAST' ? false : true
 
         if (aIsNull) {
           return nullsFirst ? -1 : 1
@@ -154,45 +277,134 @@ function applyOrderBy(rows, orderBy) {
     return 0
   })
 
-  return sorted
+  // Return sorted rows
+  return indices.map(i => rows[i])
 }
 
 /**
- * Evaluates a parsed SELECT AST against data rows
+ * Evaluates a select with a resolved FROM data source
  *
  * @param {SelectStatement} select - the parsed SQL AST
- * @param {DataSource} dataSource - the data source
- * @returns {Record<string, any>[]} the filtered, projected, and sorted result rows
+ * @param {AsyncDataSource} dataSource - the async data source
+ * @param {Record<string, AsyncDataSource>} tables
+ * @returns {AsyncGenerator<Record<string, any>>} async generator yielding result rows
  */
-function evaluateSelectAst(select, dataSource) {
-  // Check for unsupported JOIN operations
-  if (select.joins.length) {
-    throw new Error('JOIN is not supported')
-  }
+async function* evaluateSelectAst(select, dataSource, tables) {
+  // SQL priority: from, where, group by, having, select, order by, offset, limit
+
+  const hasAggregate = select.columns.some(col => col.kind === 'aggregate')
+  const useGrouping = hasAggregate || select.groupBy?.length > 0
 
-  // Check for unsupported subquery in FROM clause
-  if (typeof select.from !== 'string') {
-    throw new Error('Subquery in FROM clause is not supported')
+  // Determine if we need to buffer (collect all rows first)
+  const needsBuffering =
+    select.orderBy.length > 0 ||
+    select.distinct ||
+    useGrouping
+
+  if (needsBuffering) {
+    // BUFFERING PATH: Collect all rows, process, then yield
+    yield* evaluateBuffered(select, dataSource, tables, hasAggregate, useGrouping)
+  } else {
+    // STREAMING PATH: Yield rows one by one
+    yield* evaluateStreaming(select, dataSource, tables)
   }
+}
 
-  // SQL priority: from, where, group by, having, select, order by, offset, limit
+/**
+ * Streaming evaluation for simple queries (no ORDER BY, DISTINCT, or GROUP BY)
+ *
+ * @param {SelectStatement} select
+ * @param {AsyncDataSource} dataSource
+ * @param {Record<string, AsyncDataSource>} tables
+ * @returns {AsyncGenerator<Record<string, any>>}
+ */
+async function* evaluateStreaming(select, dataSource, tables) {
+  let rowsYielded = 0
+  let rowsSkipped = 0
+  const offset = select.offset ?? 0
+  const limit = select.limit ?? Infinity
+
+  for await (const row of dataSource.getRows()) {
+    // WHERE filter
+    if (select.where) {
+      const passes = await evaluateExpr({ node: select.where, row, tables })
+
+      if (!passes) {
+        continue
+      }
+    }
 
-  // WHERE clause filtering
+    // OFFSET handling
+    if (rowsSkipped < offset) {
+      rowsSkipped++
+      continue
+    }
+
+    // LIMIT handling
+    if (rowsYielded >= limit) {
+      break
+    }
+
+    // SELECT projection
+    /** @type {Record<string, any>} */
+    const outRow = {}
+    for (const col of select.columns) {
+      if (col.kind === 'star') {
+        const keys = row.getKeys()
+        for (const key of keys) {
+          outRow[key] = row.getCell(key)
+        }
+      } else if (col.kind === 'derived') {
+        const alias = col.alias ?? defaultDerivedAlias(col.expr)
+        outRow[alias] = await evaluateExpr({ node: col.expr, row, tables })
+      } else if (col.kind === 'aggregate') {
+        throw new Error(
+          'Aggregate functions require GROUP BY or will act on the whole dataset; add GROUP BY or remove aggregates'
+        )
+      }
+    }
+
+    yield outRow
+    rowsYielded++
+  }
+}
+
+/**
+ * Buffered evaluation for complex queries (with ORDER BY, DISTINCT, or GROUP BY)
+ *
+ * @param {SelectStatement} select
+ * @param {AsyncDataSource} dataSource
+ * @param {Record<string, AsyncDataSource>} tables
+ * @param {boolean} hasAggregate
+ * @param {boolean} useGrouping
+ * @returns {AsyncGenerator<Record<string, any>>}
+ */
+async function* evaluateBuffered(select, dataSource, tables, hasAggregate, useGrouping) {
+  // Step 1: Collect all rows from data source
   /** @type {RowSource[]} */
   const working = []
-  const length = dataSource.getNumRows()
-  for (let i = 0; i < length; i++) {
-    const row = dataSource.getRow(i)
-    if (!select.where || evaluateExpr(select.where, row)) {
-      working.push(row)
-    }
+  for await (const row of dataSource.getRows()) {
+    working.push(row)
   }
 
-  const hasAggregate = select.columns.some(col => col.kind === 'aggregate')
-  const useGrouping = hasAggregate || select.groupBy?.length > 0
+  // Step 2: WHERE clause filtering
+  /** @type {RowSource[]} */
+  const filtered = []
+
+  for (const row of working) {
+    if (select.where) {
+      const passes = await evaluateExpr({ node: select.where, row, tables })
 
+      if (!passes) {
+        continue
+      }
+    }
+    filtered.push(row)
+  }
+
+  // Step 3: Projection (grouping vs non-grouping)
   /** @type {Record<string, any>[]} */
-  const projected = []
+  let projected = []
 
   if (useGrouping) {
     // Grouping due to GROUP BY or aggregate functions
@@ -202,11 +414,11 @@ function evaluateSelectAst(select, dataSource) {
     if (select.groupBy?.length) {
       /** @type {Map<string, RowSource[]>} */
       const map = new Map()
-      for (const row of working) {
+      for (const row of filtered) {
         /** @type {string[]} */
         const keyParts = []
         for (const expr of select.groupBy) {
-          const v = evaluateExpr(expr, row)
+          const v = await evaluateExpr({ node: expr, row, tables })
           keyParts.push(JSON.stringify(v))
         }
         const key = keyParts.join('|')
@@ -219,7 +431,7 @@ function evaluateSelectAst(select, dataSource) {
         group.push(row)
       }
     } else {
-      groups.push(working)
+      groups.push(filtered)
     }
 
     const hasStar = select.columns.some(col => col.kind === 'star')
@@ -244,14 +456,18 @@ function evaluateSelectAst(select, dataSource) {
 
         if (col.kind === 'derived') {
           const alias = col.alias ?? defaultDerivedAlias(col.expr)
-          const value = group.length > 0 ? evaluateExpr(col.expr, group[0]) : undefined
-          resultRow[alias] = value
+          if (group.length > 0) {
+            const value = await evaluateExpr({ node: col.expr, row: group[0], tables })
+            resultRow[alias] = value
+          } else {
+            resultRow[alias] = undefined
+          }
           continue
         }
 
         if (col.kind === 'aggregate') {
           const alias = col.alias ?? defaultAggregateAlias(col)
-          const value = evaluateAggregate(col, group)
+          const value = await evaluateAggregate(col, group)
           resultRow[alias] = value
           continue
         }
@@ -259,9 +475,7 @@ function evaluateSelectAst(select, dataSource) {
 
       // Apply HAVING filter before adding to projected results
       if (select.having) {
-        // For HAVING, we need to evaluate aggregates in the context of the group
-        // Create a special row context that includes both the group data and aggregate values
-        if (!evaluateHavingExpr(select.having, resultRow, group)) {
+        if (!await evaluateHavingExpr(select.having, resultRow, group, tables)) {
           continue
         }
       }
@@ -270,7 +484,10 @@ function evaluateSelectAst(select, dataSource) {
     }
   } else {
     // No grouping, simple projection
-    for (const row of working) {
+    // Sort before projection so ORDER BY can access columns not in SELECT
+    const sorted = await sortRowSources(filtered, select.orderBy, tables)
+
+    for (const row of sorted) {
       /** @type {Record<string, any>} */
       const outRow = {}
       for (const col of select.columns) {
@@ -281,29 +498,26 @@ function evaluateSelectAst(select, dataSource) {
           }
         } else if (col.kind === 'derived') {
           const alias = col.alias ?? defaultDerivedAlias(col.expr)
-          const value = evaluateExpr(col.expr, row)
+          const value = await evaluateExpr({ node: col.expr, row, tables })
           outRow[alias] = value
-        } else if (col.kind === 'aggregate') {
-          throw new Error(
-            'Aggregate functions require GROUP BY or will act on the whole dataset; add GROUP BY or remove aggregates'
-          )
         }
       }
       projected.push(outRow)
     }
   }
 
-  let result = projected
+  // Step 4: DISTINCT
+  projected = applyDistinct(projected, select.distinct)
 
-  result = applyDistinct(result, select.distinct)
-  result = applyOrderBy(result, select.orderBy)
+  // Step 5: ORDER BY (final sort for grouped queries)
+  projected = await applyOrderBy(projected, select.orderBy, tables)
 
-  if (typeof select.offset === 'number' && select.offset > 0) {
-    result = result.slice(select.offset)
-  }
-  if (typeof select.limit === 'number') {
-    result = result.slice(0, select.limit)
-  }
+  // Step 6: OFFSET and LIMIT
+  const start = select.offset ?? 0
+  const end = select.limit ? start + select.limit : projected.length
 
-  return result
+  // Step 7: Yield results
+  for (let i = start; i < end && i < projected.length; i++) {
+    yield projected[i]
+  }
 }

package/src/execute/expression.js

@@ -1,13 +1,20 @@
+import { executeSelect } from './execute.js'
+import { collect } from './utils.js'
 
 /**
- * Evaluates an expression node against a row of data
+ * @import { ExprNode, RowSource, SqlPrimitive, AsyncDataSource } from '../types.js'
+ */
+
+/**
+ * Evaluates an expression node against a row of data (async version)
  *
- * @import { ExprNode, RowSource, SqlPrimitive } from '../types.js'
- * @param {ExprNode} node - The expression node to evaluate
- * @param {RowSource} row - The data row to evaluate against
- * @returns {SqlPrimitive} The result of the evaluation
+ * @param {Object} params
+ * @param {ExprNode} params.node - The expression node to evaluate
+ * @param {RowSource} params.row - The data row to evaluate against
+ * @param {Record<string, AsyncDataSource>} [params.tables]
+ * @returns {Promise<SqlPrimitive>} The result of the evaluation
  */
-export function evaluateExpr(node, row) {
+export async function evaluateExpr({ node, row, tables }) {
   if (node.type === 'literal') {
     return node.value
   }
@@ -19,16 +26,16 @@ export function evaluateExpr(node, row) {
   // Unary operators
   if (node.type === 'unary') {
     if (node.op === 'NOT') {
-      return !evaluateExpr(node.argument, row)
+      return !await evaluateExpr({ node: node.argument, row, tables })
     }
     if (node.op === 'IS NULL') {
-      return evaluateExpr(node.argument, row) == null
+      return await evaluateExpr({ node: node.argument, row, tables }) == null
     }
     if (node.op === 'IS NOT NULL') {
-      return evaluateExpr(node.argument, row) != null
+      return await evaluateExpr({ node: node.argument, row, tables }) != null
     }
     if (node.op === '-') {
-      const val = evaluateExpr(node.argument, row)
+      const val = await evaluateExpr({ node: node.argument, row, tables })
       if (val == null) return null
       return -Number(val)
     }
@@ -37,19 +44,19 @@ export function evaluateExpr(node, row) {
   // Binary operators
   if (node.type === 'binary') {
     if (node.op === 'AND') {
-      const leftVal = evaluateExpr(node.left, row)
+      const leftVal = await evaluateExpr({ node: node.left, row, tables })
       if (!leftVal) return false
-      return Boolean(evaluateExpr(node.right, row))
+      return Boolean(await evaluateExpr({ node: node.right, row, tables }))
     }
 
     if (node.op === 'OR') {
-      const leftVal = evaluateExpr(node.left, row)
+      const leftVal = await evaluateExpr({ node: node.left, row, tables })
       if (leftVal) return true
-      return Boolean(evaluateExpr(node.right, row))
+      return Boolean(await evaluateExpr({ node: node.right, row, tables }))
     }
 
-    const left = evaluateExpr(node.left, row)
-    const right = evaluateExpr(node.right, row)
+    const left = await evaluateExpr({ node: node.left, row, tables })
+    const right = await evaluateExpr({ node: node.right, row, tables })
 
     // In SQL, NULL comparisons with =, !=, <> always return false (unknown)
     // You must use IS NULL or IS NOT NULL to check for NULL
@@ -83,9 +90,9 @@ export function evaluateExpr(node, row) {
 
   // BETWEEN and NOT BETWEEN
   if (node.type === 'between' || node.type === 'not between') {
-    const expr = evaluateExpr(node.expr, row)
-    const lower = evaluateExpr(node.lower, row)
-    const upper = evaluateExpr(node.upper, row)
+    const expr = await evaluateExpr({ node: node.expr, row, tables })
+    const lower = await evaluateExpr({ node: node.lower, row, tables })
+    const upper = await evaluateExpr({ node: node.upper, row, tables })
 
     // If any value is NULL, return false (SQL behavior)
     if (expr == null || lower == null || upper == null) {
@@ -99,7 +106,7 @@ export function evaluateExpr(node, row) {
   // Function calls
   if (node.type === 'function') {
     const funcName = node.name.toUpperCase()
-    const args = node.args.map(arg => evaluateExpr(arg, row))
+    const args = await Promise.all(node.args.map(arg => evaluateExpr({ node: arg, row, tables })))
 
     if (funcName === 'UPPER') {
       if (args.length !== 1) throw new Error('UPPER requires exactly 1 argument')
@@ -161,11 +168,21 @@ export function evaluateExpr(node, row) {
       return String(val).trim()
     }
 
+    if (funcName === 'REPLACE') {
+      if (args.length !== 3) throw new Error('REPLACE requires exactly 3 arguments')
+      const str = args[0]
+      const searchStr = args[1]
+      const replaceStr = args[2]
+      // SQL REPLACE returns NULL if any argument is NULL
+      if (str == null || searchStr == null || replaceStr == null) return null
+      return String(str).replaceAll(String(searchStr), String(replaceStr))
+    }
+
     throw new Error('Unsupported function ' + funcName)
   }
 
   if (node.type === 'cast') {
-    const val = evaluateExpr(node.expr, row)
+    const val = await evaluateExpr({ node: node.expr, row, tables })
     if (val == null) return null
     const toType = node.toType.toUpperCase()
     if (toType === 'INTEGER' || toType === 'INT') {
@@ -192,17 +209,17 @@ export function evaluateExpr(node, row) {
 
   // IN and NOT IN with value lists
   if (node.type === 'in valuelist') {
-    const exprVal = evaluateExpr(node.expr, row)
+    const exprVal = await evaluateExpr({ node: node.expr, row, tables })
     for (const valueNode of node.values) {
-      const val = evaluateExpr(valueNode, row)
+      const val = await evaluateExpr({ node: valueNode, row, tables })
       if (exprVal === val) return true
     }
     return false
   }
   if (node.type === 'not in valuelist') {
-    const exprVal = evaluateExpr(node.expr, row)
+    const exprVal = await evaluateExpr({ node: node.expr, row, tables })
     for (const valueNode of node.values) {
-      const val = evaluateExpr(valueNode, row)
+      const val = await evaluateExpr({ node: valueNode, row, tables })
       if (exprVal === val) return false
     }
     return true
@@ -210,45 +227,57 @@ export function evaluateExpr(node, row) {
 
   // IN and NOT IN with subqueries
   if (node.type === 'in') {
-    throw new Error('WHERE IN with subqueries is not yet supported.')
+    const exprVal = await evaluateExpr({ node: node.expr, row, tables })
+    const results = await collect(executeSelect(node.subquery, tables))
+    if (results.length === 0) return false
+    const firstKey = Object.keys(results[0])[0]
+    const values = results.map(r => r[firstKey])
+    return values.includes(exprVal)
   }
   if (node.type === 'not in') {
-    throw new Error('WHERE NOT IN with subqueries is not yet supported.')
+    const exprVal = await evaluateExpr({ node: node.expr, row, tables })
+    const results = await collect(executeSelect(node.subquery, tables))
+    if (results.length === 0) return true
+    const firstKey = Object.keys(results[0])[0]
+    const values = results.map(r => r[firstKey])
+    return !values.includes(exprVal)
   }
 
   // EXISTS and NOT EXISTS with subqueries
   if (node.type === 'exists') {
-    throw new Error('WHERE EXISTS with subqueries is not yet supported.')
+    const results = await collect(executeSelect(node.subquery, tables))
+    return results.length > 0
   }
   if (node.type === 'not exists') {
-    throw new Error('WHERE NOT EXISTS with subqueries is not yet supported.')
+    const results = await collect(executeSelect(node.subquery, tables))
+    return results.length === 0
   }
 
   // CASE expressions
   if (node.type === 'case') {
     // For simple CASE: evaluate the case expression once
-    const caseValue = node.caseExpr ? evaluateExpr(node.caseExpr, row) : undefined
+    const caseValue = node.caseExpr ? await evaluateExpr({ node: node.caseExpr, row, tables }) : undefined
 
     // Iterate through WHEN clauses
     for (const whenClause of node.whenClauses) {
       let conditionResult
       if (caseValue !== undefined) {
         // Simple CASE: compare caseValue with condition
-        const whenValue = evaluateExpr(whenClause.condition, row)
+        const whenValue = await evaluateExpr({ node: whenClause.condition, row, tables })
         conditionResult = caseValue === whenValue
       } else {
         // Searched CASE: evaluate condition as boolean
-        conditionResult = evaluateExpr(whenClause.condition, row)
+        conditionResult = await evaluateExpr({ node: whenClause.condition, row, tables })
       }
 
       if (conditionResult) {
-        return evaluateExpr(whenClause.result, row)
+        return evaluateExpr({ node: whenClause.result, row, tables })
       }
     }
 
     // No WHEN clause matched, return ELSE result or NULL
     if (node.elseResult) {
-      return evaluateExpr(node.elseResult, row)
+      return evaluateExpr({ node: node.elseResult, row, tables })
     }
     return null
   }

package/src/execute/having.js

@@ -1,5 +1,5 @@
 /**
- * @import { AggregateFunc, ExprNode, RowSource, SqlPrimitive } from '../types.js'
+ * @import { AggregateFunc, AsyncDataSource, ExprNode, RowSource, SqlPrimitive } from '../types.js'
  */
 
 import { isAggregateFunc } from '../validation.js'
@@ -43,9 +43,10 @@ function createHavingContext(resultRow, group) {
  * @param {ExprNode} expr - the HAVING expression
  * @param {Record<string, any>} row - the aggregated result row
  * @param {RowSource[]} group - the group of rows for re-evaluating aggregates
- * @returns {boolean} whether the HAVING condition is satisfied
+ * @param {Record<string, AsyncDataSource>} tables
+ * @returns {Promise<boolean>} whether the HAVING condition is satisfied
  */
-export function evaluateHavingExpr(expr, row, group) {
+export async function evaluateHavingExpr(expr, row, group, tables) {
   const context = createHavingContext(row, group)
 
   // For HAVING, we need special handling of aggregate functions
@@ -54,13 +55,13 @@ export function evaluateHavingExpr(expr, row, group) {
     const funcName = expr.name.toUpperCase()
     if (isAggregateFunc(funcName)) {
       // Evaluate aggregate function on the group
-      return Boolean(evaluateAggregateFunction(funcName, expr.args, group))
+      return Boolean(await evaluateAggregateFunction(funcName, expr.args, group, tables))
     }
   }
 
   if (expr.type === 'binary') {
-    const left = evaluateHavingValue(expr.left, context, group)
-    const right = evaluateHavingValue(expr.right, context, group)
+    const left = await evaluateHavingValue(expr.left, context, group, tables)
+    const right = await evaluateHavingValue(expr.right, context, group, tables)
 
     if (expr.op === 'AND') {
       return Boolean(left && right)
@@ -96,20 +97,20 @@ export function evaluateHavingExpr(expr, row, group) {
 
   if (expr.type === 'unary') {
     if (expr.op === 'NOT') {
-      return !evaluateHavingExpr(expr.argument, context, group)
+      return !await evaluateHavingExpr(expr.argument, context, group, tables)
     }
     if (expr.op === 'IS NULL') {
-      return evaluateHavingValue(expr.argument, context, group) == null
+      return await evaluateHavingValue(expr.argument, context, group, tables) == null
     }
     if (expr.op === 'IS NOT NULL') {
-      return evaluateHavingValue(expr.argument, context, group) != null
+      return await evaluateHavingValue(expr.argument, context, group, tables) != null
     }
   }
 
   if (expr.type === 'between' || expr.type === 'not between') {
-    const exprVal = evaluateHavingValue(expr.expr, context, group)
-    const lower = evaluateHavingValue(expr.lower, context, group)
-    const upper = evaluateHavingValue(expr.upper, context, group)
+    const exprVal = await evaluateHavingValue(expr.expr, context, group, tables)
+    const lower = await evaluateHavingValue(expr.lower, context, group, tables)
+    const upper = await evaluateHavingValue(expr.upper, context, group, tables)
 
     // If any value is NULL, return false (SQL behavior)
     if (exprVal == null || lower == null || upper == null) {
@@ -121,7 +122,7 @@ export function evaluateHavingExpr(expr, row, group) {
   }
 
   // For other expression types, use the context row
-  return Boolean(evaluateExpr(expr, context))
+  return Boolean(await evaluateExpr({ node: expr, row: context, tables }))
 }
 
 /**
@@ -130,22 +131,23 @@ export function evaluateHavingExpr(expr, row, group) {
  * @param {ExprNode} expr
  * @param {RowSource} context - the context row
  * @param {RowSource[]} group - the group of rows
- * @returns {SqlPrimitive} the evaluated value
+ * @param {Record<string, AsyncDataSource>} tables
+ * @returns {Promise<SqlPrimitive>} the evaluated value
  */
-function evaluateHavingValue(expr, context, group) {
+function evaluateHavingValue(expr, context, group, tables) {
   if (expr.type === 'function') {
     const funcName = expr.name.toUpperCase()
     if (isAggregateFunc(funcName)) {
-      return evaluateAggregateFunction(funcName, expr.args, group)
+      return evaluateAggregateFunction(funcName, expr.args, group, tables)
     }
   }
 
   // For binary expressions, we need to use evaluateHavingExpr to properly handle aggregates
   if (expr.type === 'binary' || expr.type === 'unary' || expr.type === 'between' || expr.type === 'not between') {
-    return evaluateHavingExpr(expr, context, group)
+    return evaluateHavingExpr(expr, context, group, tables)
   }
 
-  return evaluateExpr(expr, context)
+  return evaluateExpr({ node: expr, row: context, tables })
 }
 
 /**
@@ -154,9 +156,10 @@ function evaluateHavingValue(expr, context, group) {
  * @param {AggregateFunc} funcName - aggregate function name
  * @param {ExprNode[]} args - function arguments
  * @param {RowSource[]} group - the group of rows
- * @returns {SqlPrimitive} the aggregate result
+ * @param {Record<string, AsyncDataSource>} tables
+ * @returns {Promise<SqlPrimitive>} the aggregate result
  */
-function evaluateAggregateFunction(funcName, args, group) {
+async function evaluateAggregateFunction(funcName, args, group, tables) {
   if (funcName === 'COUNT') {
     if (args.length === 1 && args[0].type === 'identifier' && args[0].name === '*') {
       return group.length
@@ -164,7 +167,7 @@ function evaluateAggregateFunction(funcName, args, group) {
     // COUNT(column) - count non-null values
     let count = 0
     for (const row of group) {
-      const val = evaluateExpr(args[0], row)
+      const val = await evaluateExpr({ node: args[0], row, tables })
       if (val != null) count++
     }
     return count
@@ -173,7 +176,7 @@ function evaluateAggregateFunction(funcName, args, group) {
   if (funcName === 'SUM') {
     let sum = 0
     for (const row of group) {
-      const val = evaluateExpr(args[0], row)
+      const val = await evaluateExpr({ node: args[0], row, tables })
       if (val != null) sum += Number(val)
     }
     return sum
@@ -183,7 +186,7 @@ function evaluateAggregateFunction(funcName, args, group) {
     let sum = 0
     let count = 0
     for (const row of group) {
-      const val = evaluateExpr(args[0], row)
+      const val = await evaluateExpr({ node: args[0], row, tables })
       if (val != null) {
         sum += Number(val)
         count++
@@ -195,7 +198,7 @@ function evaluateAggregateFunction(funcName, args, group) {
   if (funcName === 'MIN') {
     let min = null
     for (const row of group) {
-      const val = evaluateExpr(args[0], row)
+      const val = await evaluateExpr({ node: args[0], row, tables })
       if (val != null && (min == null || val < min)) {
         min = val
       }
@@ -206,7 +209,7 @@ function evaluateAggregateFunction(funcName, args, group) {
   if (funcName === 'MAX') {
     let max = null
     for (const row of group) {
-      const val = evaluateExpr(args[0], row)
+      const val = await evaluateExpr({ node: args[0], row, tables })
       if (val != null && (max == null || val > max)) {
         max = val
       }

package/src/execute/utils.js

@@ -0,0 +1,14 @@
+/**
+ * Collects all results from an async generator into an array
+ *
+ * @template T
+ * @param {AsyncGenerator<T>} asyncGen - the async generator
+ * @returns {Promise<T[]>} array of all yielded values
+ */
+export async function collect(asyncGen) {
+  const results = []
+  for await (const item of asyncGen) {
+    results.push(item)
+  }
+  return results
+}

package/src/index.d.ts

@@ -4,11 +4,11 @@ import type { ExecuteSqlOptions, SelectStatement } from './types.js'
  * Executes a SQL SELECT query against an array of data rows
  *
  * @param options
- * @param options.source - source data as a list of objects or a DataSource
- * @param options.sql - SQL query string
- * @returns rows matching the query
+ * @param options.tables - source data as a list of objects or an AsyncDataSource
+ * @param options.query - SQL query string
+ * @returns async generator yielding rows matching the query
  */
-export function executeSql(options: ExecuteSqlOptions): Record<string, any>[]
+export function executeSql(options: ExecuteSqlOptions): AsyncGenerator<Record<string, any>>
 
 /**
  * Parses a SQL query string into an abstract syntax tree
@@ -17,3 +17,11 @@ export function executeSql(options: ExecuteSqlOptions): Record<string, any>[]
  * @returns parsed SQL select statement
  */
 export function parseSql(query: string): SelectStatement
+
+/**
+ * Collects all results from an async generator into an array
+ *
+ * @param asyncGen - the async generator
+ * @returns array of all yielded values
+ */
+export function collect<T>(asyncGen: AsyncGenerator<T>): Promise<T[]>

package/src/index.js

@@ -1,2 +1,3 @@
 export { executeSql } from './execute/execute.js'
 export { parseSql } from './parse/parse.js'
+export { collect } from './execute/utils.js'

package/src/types.d.ts

@@ -1,26 +1,25 @@
+
+/**
+ * Async data source for streaming SQL execution.
+ * Provides an async iterator over rows.
+ */
+export interface AsyncDataSource {
+  getRows(): AsyncIterable<RowSource>
+}
 export interface RowSource {
   getCell(name: string): any
   getKeys(): string[]
 }
 
-export interface DataSource {
-  getNumRows(): number
-  getRow(index: number): RowSource
-}
+export type RawData = Record<string, any>[]
 
 export interface ExecuteSqlOptions {
-  source: Record<string, any>[] | DataSource
+  tables: Record<string, RawData | AsyncDataSource>
   query: string
 }
 
 export type SqlPrimitive = string | number | bigint | boolean | null
 
-export interface FromSubquery {
-  kind: 'subquery'
-  query: SelectStatement
-  alias: string
-}
-
 export interface SelectStatement {
   distinct: boolean
   columns: SelectColumn[]
@@ -34,6 +33,12 @@ export interface SelectStatement {
   offset?: number
 }
 
+export interface FromSubquery {
+  kind: 'subquery'
+  query: SelectStatement
+  alias: string
+}
+
 export type BinaryOp =
   | 'AND'
   | 'OR'
@@ -138,7 +143,7 @@ export interface StarColumn {
 
 export type AggregateFunc = 'COUNT' | 'SUM' | 'AVG' | 'MIN' | 'MAX'
 
-export type StringFunc = 'UPPER' | 'LOWER' | 'CONCAT' | 'LENGTH' | 'SUBSTRING' | 'SUBSTR' | 'TRIM'
+export type StringFunc = 'UPPER' | 'LOWER' | 'CONCAT' | 'LENGTH' | 'SUBSTRING' | 'SUBSTR' | 'TRIM' | 'REPLACE'
 
 export interface AggregateArgStar {
   kind: 'star'

package/src/validation.js

@@ -13,5 +13,5 @@ export function isAggregateFunc(name) {
  * @returns {name is StringFunc}
  */
 export function isStringFunc(name) {
-  return ['UPPER', 'LOWER', 'CONCAT', 'LENGTH', 'SUBSTRING', 'SUBSTR', 'TRIM'].includes(name)
+  return ['UPPER', 'LOWER', 'CONCAT', 'LENGTH', 'SUBSTRING', 'SUBSTR', 'TRIM', 'REPLACE'].includes(name)
 }