squirreling 0.3.0 → 0.3.1

package/README.md

@@ -7,10 +7,10 @@
 [![minzipped](https://img.shields.io/bundlephobia/minzip/squirreling)](https://www.npmjs.com/package/squirreling)
 [![workflow status](https://github.com/hyparam/squirreling/actions/workflows/ci.yml/badge.svg)](https://github.com/hyparam/squirreling/actions)
 [![mit license](https://img.shields.io/badge/License-MIT-orange.svg)](https://opensource.org/licenses/MIT)
-![coverage](https://img.shields.io/badge/Coverage-90-darkred)
+![coverage](https://img.shields.io/badge/Coverage-93-darkred)
 [![dependencies](https://img.shields.io/badge/Dependencies-0-blueviolet)](https://www.npmjs.com/package/squirreling?activeTab=dependencies)
 
-Squirreling is a streaming async SQL engine for JavaScript. It is designed to provide efficient streaming of results from pluggable backend for highly efficient retrieval of data for browser applications.
+Squirreling is a streaming async SQL engine for JavaScript. It is designed to provide efficient streaming of results from pluggable backends for highly efficient retrieval of data for browser applications.
 
 ## Features
 
@@ -22,6 +22,8 @@ Squirreling is a streaming async SQL engine for JavaScript. It is designed to pr
 - Constant memory usage for simple queries with LIMIT
 - Robust error handling and validation designed for LLM tool use
 - In-memory data option for simple use cases
+- Select only
+- No joins (yet)
 
 ## Usage
 
@@ -39,18 +41,19 @@ const users = [
 ]
 
 // Process rows as they arrive (streaming)
-for await (const user of executeSql({
+for await (const { cnt } of executeSql({
   tables: { users },
-  query: 'SELECT * FROM users WHERE active = TRUE LIMIT 100',
+  query: 'SELECT count(*) as cnt FROM users WHERE active = TRUE LIMIT 10',
 })) {
-  console.log(user.name)
+  console.log('Count', cnt)
 }
 ```
 
 There is an exported helper function `collect` to gather all rows into an array if needed:
 
 ```javascript
-import { collect } from 'squirreling'
+import { collect, executeSql } from 'squirreling'
+
 const allUsers = await collect(executeSql({
   tables: { users },
   query: 'SELECT * FROM users',

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "squirreling",
-  "version": "0.3.0",
+  "version": "0.3.1",
   "description": "Squirreling SQL Engine",
   "author": "Hyperparam",
   "homepage": "https://hyperparam.app",

package/src/backend/dataSource.js

@@ -0,0 +1,53 @@
+/**
+ * @import { AsyncDataSource, AsyncRow } from '../types.js'
+ */
+
+
+/**
+ * Wraps an async generator of plain objects into an AsyncDataSource
+ *
+ * @param {AsyncGenerator<Record<string, any>>} gen
+ * @returns {AsyncDataSource}
+ */
+export function generatorSource(gen) {
+  return {
+    async *getRows() {
+      for await (const row of gen) {
+        yield asyncRow(row)
+      }
+    },
+  }
+}
+
+/**
+ * Creates an async row accessor that wraps a plain JavaScript object
+ *
+ * @param {Record<string, any>} obj - the plain object
+ * @returns {AsyncRow} a row accessor interface
+ */
+export function asyncRow(obj) {
+  return {
+    getCell(name) {
+      return obj[name]
+    },
+    getKeys() {
+      return Object.keys(obj)
+    },
+  }
+}
+
+/**
+ * Creates an async memory-backed data source from an array of plain objects
+ *
+ * @param {Record<string, any>[]} data - array of plain objects
+ * @returns {AsyncDataSource} an async data source interface
+ */
+export function memorySource(data) {
+  return {
+    async *getRows() {
+      for (const item of data) {
+        yield asyncRow(item)
+      }
+    },
+  }
+}

package/src/execute/aggregates.js

@@ -3,9 +3,9 @@ import { evaluateExpr } from './expression.js'
 /**
  * Evaluates an aggregate function over a set of rows
  *
- * @import { AggregateColumn, ExprNode, RowSource } from '../types.js'
+ * @import { AggregateColumn, ExprNode, AsyncRow } from '../types.js'
  * @param {AggregateColumn} col - aggregate column definition
- * @param {RowSource[]} rows - rows to aggregate
+ * @param {AsyncRow[]} rows - rows to aggregate
  * @returns {Promise<number | null>} aggregated result
  */
 export async function evaluateAggregate(col, rows) {

package/src/execute/execute.js

@@ -1,12 +1,11 @@
 import { evaluateExpr } from './expression.js'
 import { parseSql } from '../parse/parse.js'
-import { createAsyncMemorySource, createRowAccessor } from '../backend/memory.js'
+import { asyncRow, generatorSource, memorySource } from '../backend/dataSource.js'
 import { defaultAggregateAlias, evaluateAggregate } from './aggregates.js'
 import { evaluateHavingExpr } from './having.js'
-import { collect } from './utils.js'
 
 /**
- * @import { AsyncDataSource, ExecuteSqlOptions, ExprNode, OrderByItem, RowSource, SelectStatement, SqlPrimitive } from '../types.js'
+ * @import { AsyncDataSource, ExecuteSqlOptions, ExprNode, OrderByItem, AsyncRow, SelectStatement, SqlPrimitive } from '../types.js'
  */
 
 /**
@@ -31,7 +30,7 @@ export async function* executeSql({ tables, query }) {
   const normalizedTables = {}
   for (const [name, source] of Object.entries(tables)) {
     if (Array.isArray(source)) {
-      normalizedTables[name] = createAsyncMemorySource(source)
+      normalizedTables[name] = memorySource(source)
     } else {
       normalizedTables[name] = source
     }
@@ -60,8 +59,7 @@ export async function* executeSelect(select, tables) {
     dataSource = table
   } else {
     // Nested subquery - recursively resolve
-    const derivedData = await collect(executeSelect(select.from.query, tables))
-    dataSource = createAsyncMemorySource(derivedData)
+    dataSource = generatorSource(executeSelect(select.from.query, tables))
   }
 
   yield* evaluateSelectAst(select, dataSource, tables)
@@ -162,13 +160,13 @@ function applyDistinct(rows, distinct) {
 /**
  * Applies ORDER BY sorting to RowSource array (before projection)
  *
- * @param {RowSource[]} rows - the input row sources
+ * @param {AsyncRow[]} rows - the input row sources
  * @param {OrderByItem[]} orderBy - the sort specifications
  * @param {Record<string, AsyncDataSource>} tables
- * @returns {Promise<RowSource[]>} the sorted row sources
+ * @returns {Promise<AsyncRow[]>} the sorted row sources
  */
 async function sortRowSources(rows, orderBy, tables) {
-  if (!orderBy?.length) return rows
+  if (!orderBy.length) return rows
 
   // Pre-evaluate ORDER BY expressions for all rows
   /** @type {SqlPrimitive[][]} */
@@ -229,7 +227,7 @@ async function sortRowSources(rows, orderBy, tables) {
  * @returns {Promise<Record<string, any>[]>} the sorted rows
  */
 async function applyOrderBy(rows, orderBy, tables) {
-  if (!orderBy?.length) return rows
+  if (!orderBy.length) return rows
 
   // Pre-evaluate ORDER BY expressions for all rows
   /** @type {SqlPrimitive[][]} */
@@ -238,7 +236,7 @@ async function applyOrderBy(rows, orderBy, tables) {
     /** @type {SqlPrimitive[]} */
     const rowValues = []
     for (const term of orderBy) {
-      const value = await evaluateExpr({ node: term.expr, row: createRowAccessor(row), tables })
+      const value = await evaluateExpr({ node: term.expr, row: asyncRow(row), tables })
       rowValues.push(value)
     }
     evaluatedValues.push(rowValues)
@@ -293,13 +291,8 @@ async function* evaluateSelectAst(select, dataSource, tables) {
   // SQL priority: from, where, group by, having, select, order by, offset, limit
 
   const hasAggregate = select.columns.some(col => col.kind === 'aggregate')
-  const useGrouping = hasAggregate || select.groupBy?.length > 0
-
-  // Determine if we need to buffer (collect all rows first)
-  const needsBuffering =
-    select.orderBy.length > 0 ||
-    select.distinct ||
-    useGrouping
+  const useGrouping = hasAggregate || select.groupBy.length > 0
+  const needsBuffering = useGrouping || select.orderBy.length > 0
 
   if (needsBuffering) {
     // BUFFERING PATH: Collect all rows, process, then yield
@@ -311,7 +304,8 @@ async function* evaluateSelectAst(select, dataSource, tables) {
 }
 
 /**
- * Streaming evaluation for simple queries (no ORDER BY, DISTINCT, or GROUP BY)
+ * Streaming evaluation for simple queries (no ORDER BY or GROUP BY)
+ * Supports DISTINCT by tracking seen row keys without buffering full rows
  *
  * @param {SelectStatement} select
  * @param {AsyncDataSource} dataSource
@@ -323,28 +317,25 @@ async function* evaluateStreaming(select, dataSource, tables) {
   let rowsSkipped = 0
   const offset = select.offset ?? 0
   const limit = select.limit ?? Infinity
+  if (limit <= 0) return
+
+  // For DISTINCT, track seen row keys
+  /** @type {Set<string> | undefined} */
+  const seen = select.distinct ? new Set() : undefined
 
   for await (const row of dataSource.getRows()) {
     // WHERE filter
     if (select.where) {
-      const passes = await evaluateExpr({ node: select.where, row, tables })
-
-      if (!passes) {
-        continue
-      }
+      const pass = await evaluateExpr({ node: select.where, row, tables })
+      if (!pass) continue
     }
 
-    // OFFSET handling
-    if (rowsSkipped < offset) {
+    // For non-DISTINCT queries, we can skip rows before projection (optimization)
+    if (!seen && rowsSkipped < offset) {
       rowsSkipped++
       continue
     }
 
-    // LIMIT handling
-    if (rowsYielded >= limit) {
-      break
-    }
-
     // SELECT projection
     /** @type {Record<string, any>} */
     const outRow = {}
@@ -364,13 +355,28 @@ async function* evaluateStreaming(select, dataSource, tables) {
       }
     }
 
+    // DISTINCT: skip duplicate rows
+    if (seen) {
+      const key = stableRowKey(outRow)
+      if (seen.has(key)) continue
+      seen.add(key)
+      // OFFSET applies to distinct rows
+      if (rowsSkipped < offset) {
+        rowsSkipped++
+        continue
+      }
+    }
+
     yield outRow
     rowsYielded++
+    if (rowsYielded >= limit) {
+      break
+    }
   }
 }
 
 /**
- * Buffered evaluation for complex queries (with ORDER BY, DISTINCT, or GROUP BY)
+ * Buffered evaluation for complex queries (with ORDER BY or GROUP BY)
  *
  * @param {SelectStatement} select
  * @param {AsyncDataSource} dataSource
@@ -381,14 +387,14 @@ async function* evaluateStreaming(select, dataSource, tables) {
  */
 async function* evaluateBuffered(select, dataSource, tables, hasAggregate, useGrouping) {
   // Step 1: Collect all rows from data source
-  /** @type {RowSource[]} */
+  /** @type {AsyncRow[]} */
   const working = []
   for await (const row of dataSource.getRows()) {
     working.push(row)
   }
 
   // Step 2: WHERE clause filtering
-  /** @type {RowSource[]} */
+  /** @type {AsyncRow[]} */
   const filtered = []
 
   for (const row of working) {
@@ -408,11 +414,11 @@ async function* evaluateBuffered(select, dataSource, tables, hasAggregate, useGr
 
   if (useGrouping) {
     // Grouping due to GROUP BY or aggregate functions
-    /** @type {RowSource[][]} */
+    /** @type {AsyncRow[][]} */
     const groups = []
 
-    if (select.groupBy?.length) {
-      /** @type {Map<string, RowSource[]>} */
+    if (select.groupBy.length) {
+      /** @type {Map<string, AsyncRow[]>} */
       const map = new Map()
       for (const row of filtered) {
         /** @type {string[]} */
@@ -487,7 +493,16 @@ async function* evaluateBuffered(select, dataSource, tables, hasAggregate, useGr
     // Sort before projection so ORDER BY can access columns not in SELECT
     const sorted = await sortRowSources(filtered, select.orderBy, tables)
 
-    for (const row of sorted) {
+    // OPTIMIZATION: For non-DISTINCT queries, apply OFFSET/LIMIT before projection
+    // to avoid reading expensive cells for rows that won't be in the final result
+    let rowsToProject = sorted
+    if (!select.distinct) {
+      const start = select.offset ?? 0
+      const end = select.limit ? start + select.limit : sorted.length
+      rowsToProject = sorted.slice(start, end)
+    }
+
+    for (const row of rowsToProject) {
       /** @type {Record<string, any>} */
       const outRow = {}
       for (const col of select.columns) {
@@ -513,11 +528,19 @@ async function* evaluateBuffered(select, dataSource, tables, hasAggregate, useGr
   projected = await applyOrderBy(projected, select.orderBy, tables)
 
   // Step 6: OFFSET and LIMIT
-  const start = select.offset ?? 0
-  const end = select.limit ? start + select.limit : projected.length
-
-  // Step 7: Yield results
-  for (let i = start; i < end && i < projected.length; i++) {
-    yield projected[i]
+  // For non-DISTINCT, non-grouping queries, OFFSET/LIMIT was already applied before projection
+  if (select.distinct || useGrouping) {
+    const start = select.offset ?? 0
+    const end = select.limit ? start + select.limit : projected.length
+
+    // Step 7: Yield results
+    for (let i = start; i < end && i < projected.length; i++) {
+      yield projected[i]
+    }
+  } else {
+    // Already limited, yield all projected rows
+    for (const row of projected) {
+      yield row
+    }
   }
 }

package/src/execute/expression.js

@@ -2,7 +2,7 @@ import { executeSelect } from './execute.js'
 import { collect } from './utils.js'
 
 /**
- * @import { ExprNode, RowSource, SqlPrimitive, AsyncDataSource } from '../types.js'
+ * @import { ExprNode, AsyncRow, SqlPrimitive, AsyncDataSource } from '../types.js'
  */
 
 /**
@@ -10,7 +10,7 @@ import { collect } from './utils.js'
  *
  * @param {Object} params
  * @param {ExprNode} params.node - The expression node to evaluate
- * @param {RowSource} params.row - The data row to evaluate against
+ * @param {AsyncRow} params.row - The data row to evaluate against
  * @param {Record<string, AsyncDataSource>} [params.tables]
  * @returns {Promise<SqlPrimitive>} The result of the evaluation
  */
@@ -23,6 +23,16 @@ export async function evaluateExpr({ node, row, tables }) {
     return row.getCell(node.name)
   }
 
+  // Scalar subquery - returns a single value
+  if (node.type === 'subquery') {
+    const results = await collect(executeSelect(node.subquery, tables))
+    if (results.length === 0) return null
+    // Return the first column of the first row
+    const firstRow = results[0]
+    const firstKey = Object.keys(firstRow)[0]
+    return firstRow[firstKey]
+  }
+
   // Unary operators
   if (node.type === 'unary') {
     if (node.op === 'NOT') {

package/src/execute/having.js

@@ -1,5 +1,5 @@
 /**
- * @import { AggregateFunc, AsyncDataSource, ExprNode, RowSource, SqlPrimitive } from '../types.js'
+ * @import { AggregateFunc, AsyncDataSource, ExprNode, AsyncRow, SqlPrimitive } from '../types.js'
  */
 
 import { isAggregateFunc } from '../validation.js'
@@ -9,8 +9,8 @@ import { evaluateExpr } from './expression.js'
  * Creates a context for evaluating HAVING expressions
  *
  * @param {Record<string, any>} resultRow - the aggregated result row
- * @param {RowSource[]} group - the group of rows
- * @returns {RowSource} a context row for HAVING evaluation
+ * @param {AsyncRow[]} group - the group of rows
+ * @returns {AsyncRow} a context row for HAVING evaluation
  */
 function createHavingContext(resultRow, group) {
   // Include the first row of the group (for GROUP BY columns)
@@ -42,7 +42,7 @@ function createHavingContext(resultRow, group) {
  *
  * @param {ExprNode} expr - the HAVING expression
  * @param {Record<string, any>} row - the aggregated result row
- * @param {RowSource[]} group - the group of rows for re-evaluating aggregates
+ * @param {AsyncRow[]} group - the group of rows for re-evaluating aggregates
  * @param {Record<string, AsyncDataSource>} tables
  * @returns {Promise<boolean>} whether the HAVING condition is satisfied
  */
@@ -129,8 +129,8 @@ export async function evaluateHavingExpr(expr, row, group, tables) {
  * Evaluates a value in a HAVING expression
  *
  * @param {ExprNode} expr
- * @param {RowSource} context - the context row
- * @param {RowSource[]} group - the group of rows
+ * @param {AsyncRow} context - the context row
+ * @param {AsyncRow[]} group - the group of rows
  * @param {Record<string, AsyncDataSource>} tables
  * @returns {Promise<SqlPrimitive>} the evaluated value
  */
@@ -155,7 +155,7 @@ function evaluateHavingValue(expr, context, group, tables) {
  *
  * @param {AggregateFunc} funcName - aggregate function name
  * @param {ExprNode[]} args - function arguments
- * @param {RowSource[]} group - the group of rows
+ * @param {AsyncRow[]} group - the group of rows
  * @param {Record<string, AsyncDataSource>} tables
  * @returns {Promise<SqlPrimitive>} the aggregate result
  */

package/src/parse/expression.js

@@ -20,6 +20,17 @@ function parsePrimary(c) {
   const tok = c.current()
 
   if (tok.type === 'paren' && tok.value === '(') {
+    // Peek ahead to see if this is a scalar subquery
+    const nextTok = c.peek(1)
+    if (nextTok.type === 'keyword' && nextTok.value === 'SELECT') {
+      // It's a scalar subquery
+      const subquery = c.parseSubquery()
+      return {
+        type: 'subquery',
+        subquery,
+      }
+    }
+    // Regular grouped expression
     c.consume()
     const expr = parseExpression(c)
     c.expect('paren', ')')
@@ -132,9 +143,6 @@ function parsePrimary(c) {
     }
     if (tok.value === 'EXISTS') {
       c.consume() // EXISTS
-      if (!c.parseSubquery) {
-        throw new Error('Subquery parsing not available in this context')
-      }
       const subquery = c.parseSubquery()
       return {
         type: 'exists',

package/src/types.d.ts

@@ -4,9 +4,9 @@
  * Provides an async iterator over rows.
  */
 export interface AsyncDataSource {
-  getRows(): AsyncIterable<RowSource>
+  getRows(): AsyncIterable<AsyncRow>
 }
-export interface RowSource {
+export interface AsyncRow {
   getCell(name: string): any
   getKeys(): string[]
 }
@@ -122,6 +122,11 @@ export interface CaseNode {
   elseResult?: ExprNode
 }
 
+export interface SubqueryNode {
+  type: 'subquery'
+  subquery: SelectStatement
+}
+
 export type ExprNode =
   | LiteralNode
   | IdentifierNode
@@ -134,6 +139,7 @@ export type ExprNode =
   | InValuesNode
   | ExistsNode
   | CaseNode
+  | SubqueryNode
 
 export interface StarColumn {
   kind: 'star'
@@ -197,7 +203,7 @@ export interface ExprCursor {
   match(type: TokenType, value?: string): boolean
   expect(type: TokenType, value: string): Token
   expectIdentifier(): Token
-  parseSubquery?: () => SelectStatement
+  parseSubquery: () => SelectStatement
 }
 
 // Tokenizer types

package/src/backend/memory.js

@@ -1,36 +0,0 @@
-/**
- * @import { AsyncDataSource, RowSource } from '../types.js'
- */
-
-/**
- * Creates a row accessor that wraps a plain JavaScript object
- *
- * @param {Record<string, any>} obj - the plain object
- * @returns {RowSource} a row accessor interface
- */
-export function createRowAccessor(obj) {
-  return {
-    getCell(name) {
-      return obj[name]
-    },
-    getKeys() {
-      return Object.keys(obj)
-    },
-  }
-}
-
-/**
- * Creates an async memory-backed data source from an array of plain objects
- *
- * @param {Record<string, any>[]} data - array of plain objects
- * @returns {AsyncDataSource} an async data source interface
- */
-export function createAsyncMemorySource(data) {
-  return {
-    async *getRows() {
-      for (const item of data) {
-        yield createRowAccessor(item)
-      }
-    },
-  }
-}