squirreling 0.4.0 → 0.4.2

package/README.md

@@ -7,7 +7,7 @@
 [![minzipped](https://img.shields.io/bundlephobia/minzip/squirreling)](https://www.npmjs.com/package/squirreling)
 [![workflow status](https://github.com/hyparam/squirreling/actions/workflows/ci.yml/badge.svg)](https://github.com/hyparam/squirreling/actions)
 [![mit license](https://img.shields.io/badge/License-MIT-orange.svg)](https://opensource.org/licenses/MIT)
-![coverage](https://img.shields.io/badge/Coverage-93-darkred)
+![coverage](https://img.shields.io/badge/Coverage-95-darkred)
 [![dependencies](https://img.shields.io/badge/Dependencies-0-blueviolet)](https://www.npmjs.com/package/squirreling?activeTab=dependencies)
 
 Squirreling is a streaming async SQL engine for JavaScript. It is designed to provide efficient streaming of results from pluggable backends for highly efficient retrieval of data for browser applications.
@@ -22,8 +22,8 @@ Squirreling is a streaming async SQL engine for JavaScript. It is designed to pr
 - Constant memory usage for simple queries with LIMIT
 - Robust error handling and validation designed for LLM tool use
 - In-memory data option for simple use cases
+- Late materialization for efficiency
 - Select only
-- No joins (yet)
 
 ## Usage
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "squirreling",
-  "version": "0.4.0",
+  "version": "0.4.2",
   "description": "Squirreling SQL Engine",
   "author": "Hyperparam",
   "homepage": "https://hyperparam.app",
@@ -38,10 +38,10 @@
   },
   "devDependencies": {
     "@types/node": "24.10.1",
-    "@vitest/coverage-v8": "4.0.14",
+    "@vitest/coverage-v8": "4.0.15",
     "eslint": "9.39.1",
     "eslint-plugin-jsdoc": "61.4.1",
     "typescript": "5.9.3",
-    "vitest": "4.0.14"
+    "vitest": "4.0.15"
   }
 }

package/src/backend/dataSource.js

@@ -58,7 +58,7 @@ export function cachedDataSource(source) {
   const cache = new Map()
   return {
     /**
-     * @returns {AsyncGenerator<AsyncRow>}
+     * @yields {AsyncRow}
      */
     async *getRows() {
       let index = 0

package/src/execute/aggregates.js

@@ -1,4 +1,5 @@
 import { evaluateExpr } from './expression.js'
+import { defaultDerivedAlias } from './utils.js'
 
 /**
  * Evaluates an aggregate function over a set of rows
@@ -72,27 +73,5 @@ export async function evaluateAggregate({ col, rows, tables }) {
 export function defaultAggregateAlias(col) {
   const base = col.func.toLowerCase()
   if (col.arg.kind === 'star') return base + '_all'
-  return base + '_' + defaultAggregateAliasExpr(col.arg.expr)
-}
-
-/**
- * @param {ExprNode} expr
- * @returns {string}
- */
-export function defaultAggregateAliasExpr(expr) {
-  if (expr.type === 'identifier') {
-    return expr.name
-  } else if (expr.type === 'literal') {
-    return String(expr.value)
-  } else if (expr.type === 'cast') {
-    return defaultAggregateAliasExpr(expr.expr) + '_as_' + expr.toType
-  } else if (expr.type === 'unary') {
-    return expr.op + '_' + defaultAggregateAliasExpr(expr.argument)
-  } else if (expr.type === 'binary') {
-    return defaultAggregateAliasExpr(expr.left) + '_' + expr.op + '_' + defaultAggregateAliasExpr(expr.right)
-  } else if (expr.type === 'function') {
-    return expr.name.toLowerCase() + '_' + expr.args.map(defaultAggregateAliasExpr).join('_')
-  } else {
-    return 'expr'
-  }
+  return base + '_' + defaultDerivedAlias(col.arg.expr)
 }

package/src/execute/execute.js

@@ -1,8 +1,10 @@
-import { evaluateExpr } from './expression.js'
-import { parseSql } from '../parse/parse.js'
 import { generatorSource, memorySource } from '../backend/dataSource.js'
+import { parseSql } from '../parse/parse.js'
 import { defaultAggregateAlias, evaluateAggregate } from './aggregates.js'
+import { evaluateExpr } from './expression.js'
 import { evaluateHavingExpr } from './having.js'
+import { executeJoins } from './join.js'
+import { compareForTerm, defaultDerivedAlias } from './utils.js'
 
 /**
  * @import { AsyncDataSource, ExecuteSqlOptions, ExprNode, OrderByItem, AsyncRow, SelectStatement, SqlPrimitive } from '../types.js'
@@ -12,15 +14,12 @@ import { evaluateHavingExpr } from './having.js'
  * Executes a SQL SELECT query against named data sources
  *
  * @param {ExecuteSqlOptions} options - the execution options
- * @returns {AsyncGenerator<AsyncRow>} async generator yielding result rows
+ * @yields {AsyncRow} async generator yielding result rows
  */
 export async function* executeSql({ tables, query }) {
   const select = parseSql(query)
 
   // Check for unsupported operations
-  if (select.joins.length) {
-    throw new Error('JOIN is not supported')
-  }
   if (!select.from) {
     throw new Error('FROM clause is required')
   }
@@ -44,53 +43,33 @@ export async function* executeSql({ tables, query }) {
  *
  * @param {SelectStatement} select
  * @param {Record<string, AsyncDataSource>} tables
- * @returns {AsyncGenerator<AsyncRow>}
+ * @yields {AsyncRow}
  */
 export async function* executeSelect(select, tables) {
   /** @type {AsyncDataSource} */
   let dataSource
-
-  if (typeof select.from === 'string') {
-    const table = tables[select.from]
-    if (table === undefined) {
-      throw new Error(`Table "${select.from}" not found`)
+  /** @type {string} */
+  let fromTableName
+
+  if (select.from.kind === 'table') {
+    // Use alias for column prefixing, but look up the actual table name
+    fromTableName = select.from.alias ?? select.from.table
+    dataSource = tables[select.from.table]
+    if (dataSource === undefined) {
+      throw new Error(`Table "${select.from.table}" not found`)
     }
-
-    dataSource = table
   } else {
     // Nested subquery - recursively resolve
+    fromTableName = select.from.alias
     dataSource = generatorSource(executeSelect(select.from.query, tables))
   }
 
-  yield* evaluateSelectAst(select, dataSource, tables)
-}
-
-/**
- * Generates a default alias for a derived column expression
- *
- * @param {ExprNode} expr - the expression node
- * @returns {string} the generated alias
- */
-function defaultDerivedAlias(expr) {
-  if (expr.type === 'identifier') {
-    return expr.name
-  }
-  if (expr.type === 'function') {
-    const base = expr.name.toLowerCase()
-    // Try to extract column names from identifier arguments
-    const columnNames = expr.args
-      .filter(arg => arg.type === 'identifier')
-      .map(arg => arg.name)
-    if (columnNames.length > 0) {
-      return base + '_' + columnNames.join('_')
-    }
-    return base
-  }
-  if (expr.type === 'cast') return 'cast_expr'
-  if (expr.type === 'unary' && expr.argument.type === 'identifier') {
-    return expr.op === '-' ? 'neg_' + expr.argument.name : 'expr'
+  // Execute JOINs if present
+  if (select.joins.length) {
+    dataSource = await executeJoins(dataSource, select.joins, fromTableName, tables)
   }
-  return 'expr'
+
+  yield* evaluateSelectAst(select, dataSource, tables)
 }
 
 /**
@@ -110,31 +89,6 @@ async function stableRowKey(row) {
   return parts.join('|')
 }
 
-/**
- * Compares two SQL values for sorting
- *
- * @param {SqlPrimitive} a
- * @param {SqlPrimitive} b
- * @returns {number} negative if a < b, positive if a > b, 0 if equal
- */
-function compareValues(a, b) {
-  if (a === b) return 0
-  if (a == null) return -1
-  if (b == null) return 1
-
-  if (typeof a === 'number' && typeof b === 'number') {
-    if (a < b) return -1
-    if (a > b) return 1
-    return 0
-  }
-
-  const as = String(a)
-  const bs = String(b)
-  if (as < bs) return -1
-  if (as > bs) return 1
-  return 0
-}
-
 /**
  * Applies DISTINCT filtering to remove duplicate rows
  *
@@ -156,127 +110,89 @@ async function applyDistinct(rows, distinct) {
   }
   return result
 }
-
 /**
- * Applies ORDER BY sorting to RowSource array (before projection)
- *
- * @param {AsyncRow[]} rows - the input row sources
- * @param {OrderByItem[]} orderBy - the sort specifications
- * @param {Record<string, AsyncDataSource>} tables
- * @returns {Promise<AsyncRow[]>} the sorted row sources
- */
-async function sortRowSources(rows, orderBy, tables) {
-  if (!orderBy.length) return rows
-
-  // Pre-evaluate ORDER BY expressions for all rows
-  /** @type {SqlPrimitive[][]} */
-  const evaluatedValues = []
-  for (const row of rows) {
-    /** @type {SqlPrimitive[]} */
-    const rowValues = []
-    for (const term of orderBy) {
-      const value = await evaluateExpr({ node: term.expr, row, tables })
-      rowValues.push(value)
-    }
-    evaluatedValues.push(rowValues)
-  }
-
-  // Create index array and sort it
-  const indices = rows.map((_, i) => i)
-  indices.sort((aIdx, bIdx) => {
-    for (let termIdx = 0; termIdx < orderBy.length; termIdx++) {
-      const term = orderBy[termIdx]
-      const dir = term.direction
-      const av = evaluatedValues[aIdx][termIdx]
-      const bv = evaluatedValues[bIdx][termIdx]
-
-      // Handle NULLS FIRST / NULLS LAST
-      const aIsNull = av == null
-      const bIsNull = bv == null
-
-      if (aIsNull || bIsNull) {
-        if (aIsNull && bIsNull) continue
-
-        const nullsFirst = term.nulls === 'LAST' ? false : true
-
-        if (aIsNull) {
-          return nullsFirst ? -1 : 1
-        } else {
-          return nullsFirst ? 1 : -1
-        }
-      }
-
-      const cmp = compareValues(av, bv)
-      if (cmp !== 0) {
-        return dir === 'DESC' ? -cmp : cmp
-      }
-    }
-    return 0
-  })
-
-  // Return sorted rows
-  return indices.map(i => rows[i])
-}
-
-/**
- * Applies ORDER BY sorting to rows
+ * Applies ORDER BY sorting to rows using multi-pass lazy evaluation.
+ * Secondary ORDER BY columns are only evaluated for rows that tie on
+ * previous columns, reducing expensive cell evaluations.
  *
  * @param {AsyncRow[]} rows - the input rows
  * @param {OrderByItem[]} orderBy - the sort specifications
  * @param {Record<string, AsyncDataSource>} tables
  * @returns {Promise<AsyncRow[]>} the sorted rows
  */
-async function applyOrderBy(rows, orderBy, tables) {
+async function sortRows(rows, orderBy, tables) {
   if (!orderBy.length) return rows
 
-  // Pre-evaluate ORDER BY expressions for all rows
-  /** @type {SqlPrimitive[][]} */
-  const evaluatedValues = []
-  for (const row of rows) {
-    /** @type {SqlPrimitive[]} */
-    const rowValues = []
-    for (const term of orderBy) {
-      const value = await evaluateExpr({ node: term.expr, row, tables })
-      rowValues.push(value)
-    }
-    evaluatedValues.push(rowValues)
-  }
-
-  // Create index array and sort it
-  const indices = rows.map((_, i) => i)
-  indices.sort((aIdx, bIdx) => {
-    for (let termIdx = 0; termIdx < orderBy.length; termIdx++) {
-      const term = orderBy[termIdx]
-      const dir = term.direction
-      const av = evaluatedValues[aIdx][termIdx]
-      const bv = evaluatedValues[bIdx][termIdx]
+  // Cache for evaluated values: evaluatedValues[rowIdx][colIdx]
+  /** @type {(SqlPrimitive | undefined)[][]} */
+  const evaluatedValues = rows.map(() => Array(orderBy.length))
 
-      // Handle NULLS FIRST / NULLS LAST
-      const aIsNull = av == null
-      const bIsNull = bv == null
+  // Start with all indices in one group
+  /** @type {number[][]} */
+  let groups = [rows.map((_, i) => i)]
 
-      if (aIsNull || bIsNull) {
-        if (aIsNull && bIsNull) continue
+  // Process each ORDER BY column incrementally
+  for (let orderByIdx = 0; orderByIdx < orderBy.length; orderByIdx++) {
+    const term = orderBy[orderByIdx]
+    /** @type {number[][]} */
+    const nextGroups = []
 
-        const nullsFirst = term.nulls === 'LAST' ? false : true
+    for (const group of groups) {
+      // Single-element groups don't need sorting or evaluation
+      if (group.length <= 1) {
+        nextGroups.push(group)
+        continue
+      }
 
-        if (aIsNull) {
-          return nullsFirst ? -1 : 1
-        } else {
-          return nullsFirst ? 1 : -1
+      // Evaluate this column for all rows in the group
+      for (const idx of group) {
+        if (evaluatedValues[idx][orderByIdx] === undefined) {
+          evaluatedValues[idx][orderByIdx] = await evaluateExpr({
+            node: term.expr,
+            row: rows[idx],
+            tables,
+          })
         }
       }
 
-      const cmp = compareValues(av, bv)
-      if (cmp !== 0) {
-        return dir === 'DESC' ? -cmp : cmp
+      // Sort the group by this column
+      group.sort((aIdx, bIdx) => {
+        const av = evaluatedValues[aIdx][orderByIdx]
+        const bv = evaluatedValues[bIdx][orderByIdx]
+        return compareForTerm(av, bv, term)
+      })
+
+      // Split into sub-groups based on ties (for next column)
+      if (orderByIdx < orderBy.length - 1) {
+        /** @type {number[]} */
+        let currentSubGroup = [group[0]]
+        for (let i = 1; i < group.length; i++) {
+          const prevIdx = group[i - 1]
+          const currIdx = group[i]
+          const prevVal = evaluatedValues[prevIdx][orderByIdx]
+          const currVal = evaluatedValues[currIdx][orderByIdx]
+
+          if (compareForTerm(prevVal, currVal, term) === 0) {
+            // Same value, extend current sub-group
+            currentSubGroup.push(currIdx)
+          } else {
+            // Different value, start new sub-group
+            nextGroups.push(currentSubGroup)
+            currentSubGroup = [currIdx]
+          }
+        }
+        nextGroups.push(currentSubGroup)
+      } else {
+        // Last column, no need to split
+        nextGroups.push(group)
       }
     }
-    return 0
-  })
 
-  // Return sorted rows
-  return indices.map(i => rows[i])
+    groups = nextGroups
+  }
+
+  // Flatten groups to get final sorted indices
+  return groups.flat().map(i => rows[i])
 }
 
 /**
@@ -285,7 +201,7 @@ async function applyOrderBy(rows, orderBy, tables) {
  * @param {SelectStatement} select
  * @param {AsyncDataSource} dataSource
  * @param {Record<string, AsyncDataSource>} tables
- * @returns {AsyncGenerator<AsyncRow>}
+ * @yields {AsyncRow}
  */
 async function* evaluateSelectAst(select, dataSource, tables) {
   // SQL priority: from, where, group by, having, select, order by, offset, limit
@@ -310,7 +226,7 @@ async function* evaluateSelectAst(select, dataSource, tables) {
  * @param {SelectStatement} select
  * @param {AsyncDataSource} dataSource
  * @param {Record<string, AsyncDataSource>} tables
- * @returns {AsyncGenerator<AsyncRow>}
+ * @yields {AsyncRow}
  */
 async function* evaluateStreaming(select, dataSource, tables) {
   let rowsYielded = 0
@@ -382,7 +298,7 @@ async function* evaluateStreaming(select, dataSource, tables) {
  * @param {Record<string, AsyncDataSource>} tables
  * @param {boolean} hasAggregate
  * @param {boolean} useGrouping
- * @returns {AsyncGenerator<AsyncRow>}
+ * @yields {AsyncRow}
  */
 async function* evaluateBuffered(select, dataSource, tables, hasAggregate, useGrouping) {
   // Step 1: Collect all rows from data source
@@ -487,7 +403,7 @@ async function* evaluateBuffered(select, dataSource, tables, hasAggregate, useGr
   } else {
     // No grouping, simple projection
     // Sort before projection so ORDER BY can access columns not in SELECT
-    const sorted = await sortRowSources(filtered, select.orderBy, tables)
+    const sorted = await sortRows(filtered, select.orderBy, tables)
 
     // OPTIMIZATION: For non-DISTINCT queries, apply OFFSET/LIMIT before projection
     // to avoid reading expensive cells for rows that won't be in the final result
@@ -519,7 +435,9 @@ async function* evaluateBuffered(select, dataSource, tables, hasAggregate, useGr
   projected = await applyDistinct(projected, select.distinct)
 
   // Step 5: ORDER BY (final sort for grouped queries)
-  projected = await applyOrderBy(projected, select.orderBy, tables)
+  if (useGrouping) {
+    projected = await sortRows(projected, select.orderBy, tables)
+  }
 
   // Step 6: OFFSET and LIMIT
   // For non-DISTINCT, non-grouping queries, OFFSET/LIMIT was already applied before projection

package/src/execute/expression.js

@@ -19,15 +19,26 @@ export async function evaluateExpr({ node, row, tables }) {
   }
 
   if (node.type === 'identifier') {
-    return row[node.name]?.()
+    // Try exact match first (handles both qualified and unqualified names)
+    if (row[node.name]) {
+      return row[node.name]()
+    }
+    // For qualified names like 'users.id', also try just the column part
+    if (node.name.includes('.')) {
+      const colName = node.name.split('.').pop()
+      if (colName && row[colName]) {
+        return row[colName]()
+      }
+    }
+    return undefined
   }
 
   // Scalar subquery - returns a single value
   if (node.type === 'subquery') {
     const gen = executeSelect(node.subquery, tables)
     const first = await gen.next() // Start the generator
-    gen.return() // Stop further execution
-    if (first.done) return null
+    gen.return(undefined) // Stop further execution
+    if (!first.value) return null
     /** @type {AsyncRow} */
     const firstRow = first.value
     const firstKey = Object.keys(firstRow)[0]
@@ -99,21 +110,6 @@ export async function evaluateExpr({ node, row, tables }) {
     }
   }
 
-  // BETWEEN and NOT BETWEEN
-  if (node.type === 'between' || node.type === 'not between') {
-    const expr = await evaluateExpr({ node: node.expr, row, tables })
-    const lower = await evaluateExpr({ node: node.lower, row, tables })
-    const upper = await evaluateExpr({ node: node.upper, row, tables })
-
-    // If any value is NULL, return false (SQL behavior)
-    if (expr == null || lower == null || upper == null) {
-      return false
-    }
-
-    const isBetween = expr >= lower && expr <= upper
-    return node.type === 'between' ? isBetween : !isBetween
-  }
-
   // Function calls
   if (node.type === 'function') {
     const funcName = node.name.toUpperCase()
@@ -189,6 +185,11 @@ export async function evaluateExpr({ node, row, tables }) {
       return String(str).replaceAll(String(searchStr), String(replaceStr))
     }
 
+    if (funcName === 'RANDOM' || funcName === 'RAND') {
+      if (args.length !== 0) throw new Error(`${funcName} takes no arguments`)
+      return Math.random()
+    }
+
     throw new Error('Unsupported function ' + funcName)
   }
 
@@ -227,16 +228,7 @@ export async function evaluateExpr({ node, row, tables }) {
     }
     return false
   }
-  if (node.type === 'not in valuelist') {
-    const exprVal = await evaluateExpr({ node: node.expr, row, tables })
-    for (const valueNode of node.values) {
-      const val = await evaluateExpr({ node: valueNode, row, tables })
-      if (exprVal === val) return false
-    }
-    return true
-  }
-
-  // IN and NOT IN with subqueries
+  // IN with subqueries
   if (node.type === 'in') {
     const exprVal = await evaluateExpr({ node: node.expr, row, tables })
     const results = executeSelect(node.subquery, tables)
@@ -249,18 +241,6 @@ export async function evaluateExpr({ node, row, tables }) {
     }
     return values.includes(exprVal)
   }
-  if (node.type === 'not in') {
-    const exprVal = await evaluateExpr({ node: node.expr, row, tables })
-    const results = executeSelect(node.subquery, tables)
-    /** @type {SqlPrimitive[]} */
-    const values = []
-    for await (const resRow of results) {
-      const firstKey = Object.keys(resRow)[0]
-      const val = await resRow[firstKey]()
-      values.push(val)
-    }
-    return !values.includes(exprVal)
-  }
 
   // EXISTS and NOT EXISTS with subqueries
   if (node.type === 'exists') {
@@ -275,7 +255,7 @@ export async function evaluateExpr({ node, row, tables }) {
   // CASE expressions
   if (node.type === 'case') {
     // For simple CASE: evaluate the case expression once
-    const caseValue = node.caseExpr ? await evaluateExpr({ node: node.caseExpr, row, tables }) : undefined
+    const caseValue = node.caseExpr && await evaluateExpr({ node: node.caseExpr, row, tables })
 
     // Iterate through WHEN clauses
     for (const whenClause of node.whenClauses) {

package/src/execute/having.js

@@ -1,26 +1,9 @@
-/**
- * @import { AggregateFunc, AsyncDataSource, ExprNode, AsyncRow, SqlPrimitive } from '../types.js'
- */
-
 import { isAggregateFunc } from '../validation.js'
 import { evaluateExpr } from './expression.js'
 
 /**
- * Creates a context for evaluating HAVING expressions
- *
- * @param {AsyncRow} resultRow - the aggregated result row
- * @param {AsyncRow[]} group - the group of rows
- * @returns {AsyncRow} a context row for HAVING evaluation
+ * @import { AggregateFunc, AsyncDataSource, ExprNode, AsyncRow, SqlPrimitive } from '../types.js'
  */
-function createHavingContext(resultRow, group) {
-  // Include the first row of the group (for GROUP BY columns)
-  const firstRow = group[0]
-  if (firstRow) {
-    return { ...firstRow, ...resultRow }
-  } else {
-    return resultRow
-  }
-}
 
 /**
  * Evaluates a HAVING expression with support for aggregate functions
@@ -32,7 +15,8 @@ function createHavingContext(resultRow, group) {
  * @returns {Promise<boolean>} whether the HAVING condition is satisfied
  */
 export async function evaluateHavingExpr(expr, row, group, tables) {
-  const context = createHavingContext(row, group)
+  // Having context
+  const context = { ...group[0] ?? {}, ...row }
 
   // For HAVING, we need special handling of aggregate functions
   // They need to be re-evaluated against the group
@@ -92,20 +76,6 @@ export async function evaluateHavingExpr(expr, row, group, tables) {
     }
   }
 
-  if (expr.type === 'between' || expr.type === 'not between') {
-    const exprVal = await evaluateHavingValue(expr.expr, context, group, tables)
-    const lower = await evaluateHavingValue(expr.lower, context, group, tables)
-    const upper = await evaluateHavingValue(expr.upper, context, group, tables)
-
-    // If any value is NULL, return false (SQL behavior)
-    if (exprVal == null || lower == null || upper == null) {
-      return false
-    }
-
-    const isBetween = exprVal >= lower && exprVal <= upper
-    return expr.type === 'between' ? isBetween : !isBetween
-  }
-
   // For other expression types, use the context row
   return Boolean(await evaluateExpr({ node: expr, row: context, tables }))
 }
@@ -128,7 +98,7 @@ function evaluateHavingValue(expr, context, group, tables) {
   }
 
   // For binary expressions, we need to use evaluateHavingExpr to properly handle aggregates
-  if (expr.type === 'binary' || expr.type === 'unary' || expr.type === 'between' || expr.type === 'not between') {
+  if (expr.type === 'binary' || expr.type === 'unary') {
     return evaluateHavingExpr(expr, context, group, tables)
   }