squirreling 0.3.1 → 0.4.1

package/README.md

@@ -7,7 +7,7 @@
 [![minzipped](https://img.shields.io/bundlephobia/minzip/squirreling)](https://www.npmjs.com/package/squirreling)
 [![workflow status](https://github.com/hyparam/squirreling/actions/workflows/ci.yml/badge.svg)](https://github.com/hyparam/squirreling/actions)
 [![mit license](https://img.shields.io/badge/License-MIT-orange.svg)](https://opensource.org/licenses/MIT)
-![coverage](https://img.shields.io/badge/Coverage-93-darkred)
+![coverage](https://img.shields.io/badge/Coverage-95-darkred)
 [![dependencies](https://img.shields.io/badge/Dependencies-0-blueviolet)](https://www.npmjs.com/package/squirreling?activeTab=dependencies)
 
 Squirreling is a streaming async SQL engine for JavaScript. It is designed to provide efficient streaming of results from pluggable backends for highly efficient retrieval of data for browser applications.
@@ -22,8 +22,8 @@ Squirreling is a streaming async SQL engine for JavaScript. It is designed to pr
 - Constant memory usage for simple queries with LIMIT
 - Robust error handling and validation designed for LLM tool use
 - In-memory data option for simple use cases
+- Late materialization for efficiency
 - Select only
-- No joins (yet)
 
 ## Usage
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "squirreling",
-  "version": "0.3.1",
+  "version": "0.4.1",
   "description": "Squirreling SQL Engine",
   "author": "Hyperparam",
   "homepage": "https://hyperparam.app",

package/src/backend/dataSource.js

@@ -1,20 +1,18 @@
 /**
- * @import { AsyncDataSource, AsyncRow } from '../types.js'
+ * @import { AsyncDataSource, AsyncRow, SqlPrimitive } from '../types.js'
  */
 
 
 /**
  * Wraps an async generator of plain objects into an AsyncDataSource
  *
- * @param {AsyncGenerator<Record<string, any>>} gen
+ * @param {AsyncGenerator<AsyncRow>} gen
  * @returns {AsyncDataSource}
  */
 export function generatorSource(gen) {
   return {
     async *getRows() {
-      for await (const row of gen) {
-        yield asyncRow(row)
-      }
+      yield* gen
     },
   }
 }
@@ -22,24 +20,22 @@ export function generatorSource(gen) {
 /**
  * Creates an async row accessor that wraps a plain JavaScript object
  *
- * @param {Record<string, any>} obj - the plain object
+ * @param {Record<string, SqlPrimitive>} obj - the plain object
  * @returns {AsyncRow} a row accessor interface
  */
-export function asyncRow(obj) {
-  return {
-    getCell(name) {
-      return obj[name]
-    },
-    getKeys() {
-      return Object.keys(obj)
-    },
+function asyncRow(obj) {
+  /** @type {AsyncRow} */
+  const row = {}
+  for (const [key, value] of Object.entries(obj)) {
+    row[key] = () => Promise.resolve(value)
   }
+  return row
 }
 
 /**
  * Creates an async memory-backed data source from an array of plain objects
  *
- * @param {Record<string, any>[]} data - array of plain objects
+ * @param {Record<string, SqlPrimitive>[]} data - array of plain objects
  * @returns {AsyncDataSource} an async data source interface
  */
 export function memorySource(data) {
@@ -51,3 +47,40 @@ export function memorySource(data) {
     },
   }
 }
+
+/**
+ * Wraps a data source that caches all accessed rows in memory
+ * @param {AsyncDataSource} source
+ * @returns {AsyncDataSource}
+ */
+export function cachedDataSource(source) {
+  /** @type {Map<string, Promise<SqlPrimitive>>} */
+  const cache = new Map()
+  return {
+    /**
+     * @yields {AsyncRow}
+     */
+    async *getRows() {
+      let index = 0
+      for await (const row of source.getRows()) {
+        const rowIndex = index
+        /** @type {AsyncRow} */
+        const out = {}
+        for (const [key, cell] of Object.entries(row)) {
+          // Wrap the cell to cache accesses
+          out[key] = () => {
+            const cacheKey = `${rowIndex}:${key}`
+            let value = cache.get(cacheKey)
+            if (!value) {
+              value = cell()
+              cache.set(cacheKey, value)
+            }
+            return value
+          }
+        }
+        yield out
+        index++
+      }
+    },
+  }
+}

package/src/execute/aggregates.js

@@ -1,21 +1,24 @@
 import { evaluateExpr } from './expression.js'
+import { defaultDerivedAlias } from './utils.js'
 
 /**
  * Evaluates an aggregate function over a set of rows
  *
- * @import { AggregateColumn, ExprNode, AsyncRow } from '../types.js'
- * @param {AggregateColumn} col - aggregate column definition
- * @param {AsyncRow[]} rows - rows to aggregate
+ * @import { AggregateColumn, AsyncDataSource, ExprNode, AsyncRow } from '../types.js'
+ * @param {Object} options
+ * @param {AggregateColumn} options.col - aggregate column definition
+ * @param {AsyncRow[]} options.rows - rows to aggregate
+ * @param {Record<string, AsyncDataSource>} options.tables
  * @returns {Promise<number | null>} aggregated result
  */
-export async function evaluateAggregate(col, rows) {
+export async function evaluateAggregate({ col, rows, tables }) {
   const { arg, func } = col
 
   if (func === 'COUNT') {
     if (arg.kind === 'star') return rows.length
     let count = 0
-    for (let i = 0; i < rows.length; i += 1) {
-      const v = await evaluateExpr({ node: arg.expr, row: rows[i] })
+    for (const row of rows) {
+      const v = await evaluateExpr({ node: arg.expr, row, tables })
       if (v !== null && v !== undefined) {
         count += 1
       }
@@ -34,8 +37,8 @@ export async function evaluateAggregate(col, rows) {
     /** @type {number | null} */
     let max = null
 
-    for (let i = 0; i < rows.length; i += 1) {
-      const raw = await evaluateExpr({ node: arg.expr, row: rows[i] })
+    for (const row of rows) {
+      const raw = await evaluateExpr({ node: arg.expr, row, tables })
       if (raw == null) continue
       const num = Number(raw)
       if (!Number.isFinite(num)) continue
@@ -70,30 +73,5 @@ export async function evaluateAggregate(col, rows) {
 export function defaultAggregateAlias(col) {
   const base = col.func.toLowerCase()
   if (col.arg.kind === 'star') return base + '_all'
-  return base + '_' + defaultAggregateAliasExpr(col.arg.expr)
-}
-
-/**
- * @param {ExprNode} expr
- * @returns {string}
- */
-export function defaultAggregateAliasExpr(expr) {
-  if (expr.type === 'identifier') {
-    return expr.name
-  }
-  if (expr.type === 'literal') {
-    return String(expr.value)
-  }
-  if (expr.type === 'cast') {
-    return defaultAggregateAliasExpr(expr.expr) + '_as_' + expr.toType
-  }
-  if (expr.type === 'unary') {
-    return expr.op + '_' + defaultAggregateAliasExpr(expr.argument)
-  }
-  if (expr.type === 'binary') {
-    return defaultAggregateAliasExpr(expr.left) + '_' + expr.op + '_' + defaultAggregateAliasExpr(expr.right)
-  }
-  if (expr.type === 'function') {
-    return expr.name.toLowerCase() + '_' + expr.args.map(defaultAggregateAliasExpr).join('_')
-  }
+  return base + '_' + defaultDerivedAlias(col.arg.expr)
 }

package/src/execute/execute.js

@@ -1,8 +1,10 @@
-import { evaluateExpr } from './expression.js'
+import { generatorSource, memorySource } from '../backend/dataSource.js'
 import { parseSql } from '../parse/parse.js'
-import { asyncRow, generatorSource, memorySource } from '../backend/dataSource.js'
 import { defaultAggregateAlias, evaluateAggregate } from './aggregates.js'
+import { evaluateExpr } from './expression.js'
 import { evaluateHavingExpr } from './having.js'
+import { executeJoins } from './join.js'
+import { defaultDerivedAlias } from './utils.js'
 
 /**
  * @import { AsyncDataSource, ExecuteSqlOptions, ExprNode, OrderByItem, AsyncRow, SelectStatement, SqlPrimitive } from '../types.js'
@@ -12,15 +14,12 @@ import { evaluateHavingExpr } from './having.js'
  * Executes a SQL SELECT query against named data sources
  *
  * @param {ExecuteSqlOptions} options - the execution options
- * @returns {AsyncGenerator<Record<string, any>>} async generator yielding result rows
+ * @yields {AsyncRow} async generator yielding result rows
  */
 export async function* executeSql({ tables, query }) {
   const select = parseSql(query)
 
   // Check for unsupported operations
-  if (select.joins.length) {
-    throw new Error('JOIN is not supported')
-  }
   if (!select.from) {
     throw new Error('FROM clause is required')
   }
@@ -44,67 +43,47 @@ export async function* executeSql({ tables, query }) {
  *
  * @param {SelectStatement} select
  * @param {Record<string, AsyncDataSource>} tables
- * @returns {AsyncGenerator<Record<string, any>>} async generator yielding result rows
+ * @yields {AsyncRow}
  */
 export async function* executeSelect(select, tables) {
   /** @type {AsyncDataSource} */
   let dataSource
-
-  if (typeof select.from === 'string') {
-    const table = tables[select.from]
-    if (table === undefined) {
-      throw new Error(`Table "${select.from}" not found`)
+  /** @type {string} */
+  let fromTableName
+
+  if (select.from.kind === 'table') {
+    // Use alias for column prefixing, but look up the actual table name
+    fromTableName = select.from.alias ?? select.from.table
+    dataSource = tables[select.from.table]
+    if (dataSource === undefined) {
+      throw new Error(`Table "${select.from.table}" not found`)
     }
-
-    dataSource = table
   } else {
     // Nested subquery - recursively resolve
+    fromTableName = select.from.alias
     dataSource = generatorSource(executeSelect(select.from.query, tables))
   }
 
-  yield* evaluateSelectAst(select, dataSource, tables)
-}
-
-/**
- * Generates a default alias for a derived column expression
- *
- * @param {ExprNode} expr - the expression node
- * @returns {string} the generated alias
- */
-function defaultDerivedAlias(expr) {
-  if (expr.type === 'identifier') {
-    return expr.name
-  }
-  if (expr.type === 'function') {
-    const base = expr.name.toLowerCase()
-    // Try to extract column names from identifier arguments
-    const columnNames = expr.args
-      .filter(arg => arg.type === 'identifier')
-      .map(arg => arg.name)
-    if (columnNames.length > 0) {
-      return base + '_' + columnNames.join('_')
-    }
-    return base
-  }
-  if (expr.type === 'cast') return 'cast_expr'
-  if (expr.type === 'unary' && expr.argument.type === 'identifier') {
-    return expr.op === '-' ? 'neg_' + expr.argument.name : 'expr'
+  // Execute JOINs if present
+  if (select.joins.length) {
+    dataSource = await executeJoins(dataSource, select.joins, fromTableName, tables)
   }
-  return 'expr'
+
+  yield* evaluateSelectAst(select, dataSource, tables)
 }
 
 /**
  * Creates a stable string key for a row to enable deduplication
  *
- * @param {Record<string, any>} row
- * @returns {string} a stable string representation of the row
+ * @param {AsyncRow} row
+ * @returns {Promise<string>} a stable string representation of the row
  */
-function stableRowKey(row) {
+async function stableRowKey(row) {
   const keys = Object.keys(row).sort()
   /** @type {string[]} */
   const parts = []
   for (const k of keys) {
-    const v = row[k]
+    const v = await row[k]()
     parts.push(k + ':' + JSON.stringify(v))
   }
   return parts.join('|')
@@ -128,28 +107,28 @@ function compareValues(a, b) {
     return 0
   }
 
-  const as = String(a)
-  const bs = String(b)
-  if (as < bs) return -1
-  if (as > bs) return 1
+  const aa = String(a)
+  const bb = String(b)
+  if (aa < bb) return -1
+  if (aa > bb) return 1
   return 0
 }
 
 /**
  * Applies DISTINCT filtering to remove duplicate rows
  *
- * @param {Record<string, any>[]} rows - The input rows
- * @param {boolean} distinct - Whether to apply deduplication
- * @returns {Record<string, any>[]} The deduplicated rows
+ * @param {AsyncRow[]} rows - the input rows
+ * @param {boolean} distinct - whether to apply deduplication
+ * @returns {Promise<AsyncRow[]>} the deduplicated rows
  */
-function applyDistinct(rows, distinct) {
+async function applyDistinct(rows, distinct) {
   if (!distinct) return rows
   /** @type {Set<string>} */
   const seen = new Set()
-  /** @type {Record<string, any>[]} */
+  /** @type {AsyncRow[]} */
   const result = []
   for (const row of rows) {
-    const key = stableRowKey(row)
+    const key = await stableRowKey(row)
     if (seen.has(key)) continue
     seen.add(key)
     result.push(row)
@@ -221,10 +200,10 @@ async function sortRowSources(rows, orderBy, tables) {
 /**
  * Applies ORDER BY sorting to rows
  *
- * @param {Record<string, any>[]} rows - the input rows
+ * @param {AsyncRow[]} rows - the input rows
  * @param {OrderByItem[]} orderBy - the sort specifications
  * @param {Record<string, AsyncDataSource>} tables
- * @returns {Promise<Record<string, any>[]>} the sorted rows
+ * @returns {Promise<AsyncRow[]>} the sorted rows
  */
 async function applyOrderBy(rows, orderBy, tables) {
   if (!orderBy.length) return rows
@@ -236,7 +215,7 @@ async function applyOrderBy(rows, orderBy, tables) {
     /** @type {SqlPrimitive[]} */
     const rowValues = []
     for (const term of orderBy) {
-      const value = await evaluateExpr({ node: term.expr, row: asyncRow(row), tables })
+      const value = await evaluateExpr({ node: term.expr, row, tables })
       rowValues.push(value)
     }
     evaluatedValues.push(rowValues)
@@ -282,10 +261,10 @@ async function applyOrderBy(rows, orderBy, tables) {
 /**
  * Evaluates a select with a resolved FROM data source
  *
- * @param {SelectStatement} select - the parsed SQL AST
- * @param {AsyncDataSource} dataSource - the async data source
+ * @param {SelectStatement} select
+ * @param {AsyncDataSource} dataSource
  * @param {Record<string, AsyncDataSource>} tables
- * @returns {AsyncGenerator<Record<string, any>>} async generator yielding result rows
+ * @yields {AsyncRow}
  */
 async function* evaluateSelectAst(select, dataSource, tables) {
   // SQL priority: from, where, group by, having, select, order by, offset, limit
@@ -310,7 +289,7 @@ async function* evaluateSelectAst(select, dataSource, tables) {
  * @param {SelectStatement} select
  * @param {AsyncDataSource} dataSource
  * @param {Record<string, AsyncDataSource>} tables
- * @returns {AsyncGenerator<Record<string, any>>}
+ * @yields {AsyncRow}
  */
 async function* evaluateStreaming(select, dataSource, tables) {
   let rowsYielded = 0
@@ -337,17 +316,16 @@ async function* evaluateStreaming(select, dataSource, tables) {
     }
 
     // SELECT projection
-    /** @type {Record<string, any>} */
+    /** @type {AsyncRow} */
     const outRow = {}
     for (const col of select.columns) {
       if (col.kind === 'star') {
-        const keys = row.getKeys()
-        for (const key of keys) {
-          outRow[key] = row.getCell(key)
+        for (const [key, cell] of Object.entries(row)) {
+          outRow[key] = cell
         }
       } else if (col.kind === 'derived') {
         const alias = col.alias ?? defaultDerivedAlias(col.expr)
-        outRow[alias] = await evaluateExpr({ node: col.expr, row, tables })
+        outRow[alias] = () => evaluateExpr({ node: col.expr, row, tables })
       } else if (col.kind === 'aggregate') {
         throw new Error(
           'Aggregate functions require GROUP BY or will act on the whole dataset; add GROUP BY or remove aggregates'
@@ -357,7 +335,7 @@ async function* evaluateStreaming(select, dataSource, tables) {
 
     // DISTINCT: skip duplicate rows
     if (seen) {
-      const key = stableRowKey(outRow)
+      const key = await stableRowKey(outRow)
       if (seen.has(key)) continue
       seen.add(key)
       // OFFSET applies to distinct rows
@@ -383,7 +361,7 @@ async function* evaluateStreaming(select, dataSource, tables) {
  * @param {Record<string, AsyncDataSource>} tables
  * @param {boolean} hasAggregate
  * @param {boolean} useGrouping
- * @returns {AsyncGenerator<Record<string, any>>}
+ * @yields {AsyncRow}
  */
 async function* evaluateBuffered(select, dataSource, tables, hasAggregate, useGrouping) {
   // Step 1: Collect all rows from data source
@@ -409,7 +387,7 @@ async function* evaluateBuffered(select, dataSource, tables, hasAggregate, useGr
   }
 
   // Step 3: Projection (grouping vs non-grouping)
-  /** @type {Record<string, any>[]} */
+  /** @type {AsyncRow[]} */
   let projected = []
 
   if (useGrouping) {
@@ -446,15 +424,14 @@ async function* evaluateBuffered(select, dataSource, tables, hasAggregate, useGr
     }
 
     for (const group of groups) {
-      /** @type {Record<string, any>} */
+      /** @type {AsyncRow} */
       const resultRow = {}
       for (const col of select.columns) {
         if (col.kind === 'star') {
           const firstRow = group[0]
           if (firstRow) {
-            const keys = firstRow.getKeys()
-            for (const key of keys) {
-              resultRow[key] = firstRow.getCell(key)
+            for (const [key, cell] of Object.entries(firstRow)) {
+              resultRow[key] = cell
             }
           }
           continue
@@ -463,18 +440,16 @@ async function* evaluateBuffered(select, dataSource, tables, hasAggregate, useGr
         if (col.kind === 'derived') {
           const alias = col.alias ?? defaultDerivedAlias(col.expr)
           if (group.length > 0) {
-            const value = await evaluateExpr({ node: col.expr, row: group[0], tables })
-            resultRow[alias] = value
+            resultRow[alias] = () => evaluateExpr({ node: col.expr, row: group[0], tables })
           } else {
-            resultRow[alias] = undefined
+            delete resultRow[alias]
           }
           continue
         }
 
         if (col.kind === 'aggregate') {
           const alias = col.alias ?? defaultAggregateAlias(col)
-          const value = await evaluateAggregate(col, group)
-          resultRow[alias] = value
+          resultRow[alias] = () => evaluateAggregate({ col, rows: group, tables })
           continue
         }
       }
@@ -503,18 +478,16 @@ async function* evaluateBuffered(select, dataSource, tables, hasAggregate, useGr
     }
 
     for (const row of rowsToProject) {
-      /** @type {Record<string, any>} */
+      /** @type {AsyncRow} */
       const outRow = {}
       for (const col of select.columns) {
         if (col.kind === 'star') {
-          const keys = row.getKeys()
-          for (const key of keys) {
-            outRow[key] = row.getCell(key)
+          for (const [key, cell] of Object.entries(row)) {
+            outRow[key] = cell
           }
         } else if (col.kind === 'derived') {
           const alias = col.alias ?? defaultDerivedAlias(col.expr)
-          const value = await evaluateExpr({ node: col.expr, row, tables })
-          outRow[alias] = value
+          outRow[alias] = () => evaluateExpr({ node: col.expr, row, tables })
         }
       }
       projected.push(outRow)
@@ -522,7 +495,7 @@ async function* evaluateBuffered(select, dataSource, tables, hasAggregate, useGr
   }
 
   // Step 4: DISTINCT
-  projected = applyDistinct(projected, select.distinct)
+  projected = await applyDistinct(projected, select.distinct)
 
   // Step 5: ORDER BY (final sort for grouped queries)
   projected = await applyOrderBy(projected, select.orderBy, tables)

package/src/execute/expression.js

@@ -1,5 +1,4 @@
 import { executeSelect } from './execute.js'
-import { collect } from './utils.js'
 
 /**
  * @import { ExprNode, AsyncRow, SqlPrimitive, AsyncDataSource } from '../types.js'
@@ -11,7 +10,7 @@ import { collect } from './utils.js'
  * @param {Object} params
  * @param {ExprNode} params.node - The expression node to evaluate
  * @param {AsyncRow} params.row - The data row to evaluate against
- * @param {Record<string, AsyncDataSource>} [params.tables]
+ * @param {Record<string, AsyncDataSource>} params.tables
  * @returns {Promise<SqlPrimitive>} The result of the evaluation
  */
 export async function evaluateExpr({ node, row, tables }) {
@@ -20,17 +19,30 @@ export async function evaluateExpr({ node, row, tables }) {
   }
 
   if (node.type === 'identifier') {
-    return row.getCell(node.name)
+    // Try exact match first (handles both qualified and unqualified names)
+    if (row[node.name]) {
+      return row[node.name]()
+    }
+    // For qualified names like 'users.id', also try just the column part
+    if (node.name.includes('.')) {
+      const colName = node.name.split('.').pop()
+      if (colName && row[colName]) {
+        return row[colName]()
+      }
+    }
+    return undefined
   }
 
   // Scalar subquery - returns a single value
   if (node.type === 'subquery') {
-    const results = await collect(executeSelect(node.subquery, tables))
-    if (results.length === 0) return null
-    // Return the first column of the first row
-    const firstRow = results[0]
+    const gen = executeSelect(node.subquery, tables)
+    const first = await gen.next() // Start the generator
+    gen.return(undefined) // Stop further execution
+    if (!first.value) return null
+    /** @type {AsyncRow} */
+    const firstRow = first.value
     const firstKey = Object.keys(firstRow)[0]
-    return firstRow[firstKey]
+    return firstRow[firstKey]()
   }
 
   // Unary operators
@@ -238,35 +250,43 @@ export async function evaluateExpr({ node, row, tables }) {
   // IN and NOT IN with subqueries
   if (node.type === 'in') {
     const exprVal = await evaluateExpr({ node: node.expr, row, tables })
-    const results = await collect(executeSelect(node.subquery, tables))
-    if (results.length === 0) return false
-    const firstKey = Object.keys(results[0])[0]
-    const values = results.map(r => r[firstKey])
+    const results = executeSelect(node.subquery, tables)
+    /** @type {SqlPrimitive[]} */
+    const values = []
+    for await (const resRow of results) {
+      const firstKey = Object.keys(resRow)[0]
+      const val = await resRow[firstKey]()
+      values.push(val)
+    }
     return values.includes(exprVal)
   }
   if (node.type === 'not in') {
     const exprVal = await evaluateExpr({ node: node.expr, row, tables })
-    const results = await collect(executeSelect(node.subquery, tables))
-    if (results.length === 0) return true
-    const firstKey = Object.keys(results[0])[0]
-    const values = results.map(r => r[firstKey])
+    const results = executeSelect(node.subquery, tables)
+    /** @type {SqlPrimitive[]} */
+    const values = []
+    for await (const resRow of results) {
+      const firstKey = Object.keys(resRow)[0]
+      const val = await resRow[firstKey]()
+      values.push(val)
+    }
     return !values.includes(exprVal)
   }
 
   // EXISTS and NOT EXISTS with subqueries
   if (node.type === 'exists') {
-    const results = await collect(executeSelect(node.subquery, tables))
-    return results.length > 0
+    const results = await executeSelect(node.subquery, tables).next()
+    return results.done === false
   }
   if (node.type === 'not exists') {
-    const results = await collect(executeSelect(node.subquery, tables))
-    return results.length === 0
+    const results = await executeSelect(node.subquery, tables).next()
+    return results.done === true
   }
 
   // CASE expressions
   if (node.type === 'case') {
     // For simple CASE: evaluate the case expression once
-    const caseValue = node.caseExpr ? await evaluateExpr({ node: node.caseExpr, row, tables }) : undefined
+    const caseValue = node.caseExpr && await evaluateExpr({ node: node.caseExpr, row, tables })
 
     // Iterate through WHEN clauses
     for (const whenClause of node.whenClauses) {