squirreling 0.11.5 → 0.12.0

package/README.md

@@ -34,20 +34,23 @@ const users = [
 ]
 
 // Squirreling return types
+interface QueryResults {
+  rows: () => AsyncGenerator<AsyncRow>
+}
 interface AsyncRow {
   columns: string[]
   cells: Record<string, AsyncCell>
 }
 type AsyncCell = () => Promise<SqlPrimitive>
 
-// Returns an AsyncIterable of rows with async cell loading
-const asyncRows: AsyncIterable<AsyncRow> = executeSql({
+// Returns a QueryResults object with streaming rows
+const { rows } = executeSql({
   tables: { users },
   query: 'SELECT * FROM users',
 })
 
 // Process rows as they arrive (streaming)
-for await (const { cells } of asyncRows) {
+for await (const { cells } of rows()) {
   console.log(`User id=${await cells.id()}, name=${await cells.name()}`)
 }
 ```
@@ -105,7 +108,7 @@ interface ScanOptions {
 }
 
 interface ScanResults {
-  rows: AsyncIterable<AsyncRow> // async iterable of rows
+  rows(): AsyncIterable<AsyncRow> // async iterable of rows
   appliedWhere: boolean // WHERE filter applied at scan time?
   appliedLimitOffset: boolean // LIMIT and OFFSET applied at scan time?
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "squirreling",
-  "version": "0.11.5",
+  "version": "0.12.0",
   "description": "Squirreling Async SQL Engine",
   "author": "Hyperparam",
   "homepage": "https://hyperparam.app",

package/src/backend/dataSource.js

@@ -1,5 +1,5 @@
 /**
- * @import { AsyncCells, AsyncDataSource, AsyncRow, SqlPrimitive } from '../types.js'
+ * @import { AsyncCells, AsyncDataSource, AsyncRow, ScanColumnOptions, SqlPrimitive } from '../types.js'
  */
 
 /**
@@ -55,12 +55,12 @@ export function memorySource({ data, columns }) {
       const start = !where ? offset ?? 0 : 0
       const end = !where && limit !== undefined ? start + limit : data.length
       return {
-        rows: (async function* () {
+        async *rows() {
           for (let i = start; i < end && i < data.length; i++) {
             if (signal?.aborted) break
             yield asyncRow(data[i], scanColumns ?? columns)
           }
-        })(),
+        },
         appliedWhere: false,
         appliedLimitOffset: !where,
       }
@@ -92,9 +92,9 @@ export function cachedDataSource(source) {
       const indexOffset = appliedLimitOffset && options.offset ? options.offset : 0
 
       return {
-        rows: (async function* () {
+        async *rows() {
           let index = 0
-          for await (const row of rows) {
+          for await (const row of rows()) {
             if (options.signal?.aborted) break
             const rowIndex = index + indexOffset
             /** @type {AsyncCells} */
@@ -115,10 +115,41 @@ export function cachedDataSource(source) {
             yield { columns: row.columns, cells }
             index++
           }
-        })(),
+        },
         appliedWhere,
         appliedLimitOffset,
       }
     },
+    ...source.scanColumn && {
+      /**
+       * @param {ScanColumnOptions} options
+       * @returns {AsyncIterable<ArrayLike<SqlPrimitive>>}
+       */
+      scanColumn(options) {
+        const inner = source.scanColumn(options)
+        const indexOffset = options.offset ?? 0
+        return (async function* () {
+          let chunkStart = 0
+          for await (const chunk of inner) {
+            if (options.signal?.aborted) break
+            /** @type {SqlPrimitive[]} */
+            const cached = new Array(chunk.length)
+            for (let i = 0; i < chunk.length; i++) {
+              const cacheKey = `${chunkStart + i + indexOffset}:${options.column}`
+              const existing = cache.get(cacheKey)
+              if (existing) {
+                cached[i] = await existing
+              } else {
+                const value = chunk[i]
+                cache.set(cacheKey, Promise.resolve(value))
+                cached[i] = value
+              }
+            }
+            yield cached
+            chunkStart += chunk.length
+          }
+        })()
+      },
+    },
   }
 }

package/src/execute/aggregates.js

@@ -4,7 +4,7 @@ import { executePlan } from './execute.js'
 import { keyify } from './utils.js'
 
 /**
- * @import { AsyncCells, AsyncDataSource, AsyncRow, DerivedColumn, ExecuteContext, SelectColumn, SqlPrimitive } from '../types.js'
+ * @import { AsyncCells, AsyncDataSource, AsyncRow, DerivedColumn, ExecuteContext, QueryResults, SelectColumn, SqlPrimitive } from '../types.js'
  * @import { HashAggregateNode, ScalarAggregateNode } from '../plan/types.js'
  */
 
@@ -55,52 +55,58 @@ function projectAggregateColumns(selectColumns, group, context) {
  *
  * @param {HashAggregateNode} plan
  * @param {ExecuteContext} context
- * @yields {AsyncRow}
+ * @returns {QueryResults}
  */
-export async function* executeHashAggregate(plan, context) {
-  // Collect all rows
-  /** @type {AsyncRow[]} */
-  const allRows = []
-  for await (const row of executePlan({ plan: plan.child, context })) {
-    if (context.signal?.aborted) return
-    allRows.push(row)
-  }
+export function executeHashAggregate(plan, context) {
+  const child = executePlan({ plan: plan.child, context })
+  return {
+    maxRows: child.maxRows,
+    async *rows () {
+      // Collect all rows
+      /** @type {AsyncRow[]} */
+      const allRows = []
+      for await (const row of child.rows()) {
+        if (context.signal?.aborted) return
+        allRows.push(row)
+      }
 
-  // Group rows by GROUP BY keys
-  /** @type {Map<any, AsyncRow[]>} */
-  const groups = new Map()
+      // Group rows by GROUP BY keys
+      /** @type {Map<any, AsyncRow[]>} */
+      const groups = new Map()
 
-  for (const row of allRows) {
-    const key = keyify(...await Promise.all(plan.groupBy.map(expr => evaluateExpr({ node: expr, row, context }))))
-    let group = groups.get(key)
-    if (!group) {
-      group = []
-      groups.set(key, group)
-    }
-    group.push(row)
-  }
+      for (const row of allRows) {
+        const key = keyify(...await Promise.all(plan.groupBy.map(expr => evaluateExpr({ node: expr, row, context }))))
+        let group = groups.get(key)
+        if (!group) {
+          group = []
+          groups.set(key, group)
+        }
+        group.push(row)
+      }
 
-  // Yield one row per group
-  for (const group of groups.values()) {
-    const asyncRow = projectAggregateColumns(plan.columns, group, context)
+      // Yield one row per group
+      for (const group of groups.values()) {
+        const asyncRow = projectAggregateColumns(plan.columns, group, context)
 
-    // Apply HAVING filter
-    if (plan.having) {
-      /** @type {AsyncRow} */
-      const havingRow = {
-        columns: [...group[0].columns, ...asyncRow.columns],
-        cells: { ...group[0].cells, ...asyncRow.cells },
-      }
-      const passes = await evaluateExpr({
-        node: plan.having,
-        row: havingRow,
-        rows: group,
-        context,
-      })
-      if (!passes) continue
-    }
+        // Apply HAVING filter
+        if (plan.having) {
+          /** @type {AsyncRow} */
+          const havingRow = {
+            columns: [...group[0].columns, ...asyncRow.columns],
+            cells: { ...group[0].cells, ...asyncRow.cells },
+          }
+          const passes = await evaluateExpr({
+            node: plan.having,
+            row: havingRow,
+            rows: group,
+            context,
+          })
+          if (!passes) continue
+        }
 
-    yield asyncRow
+        yield asyncRow
+      }
+    },
   }
 }
 
@@ -109,43 +115,54 @@ export async function* executeHashAggregate(plan, context) {
  *
  * @param {ScalarAggregateNode} plan
  * @param {ExecuteContext} context
- * @yields {AsyncRow}
+ * @returns {QueryResults}
  */
-export async function* executeScalarAggregate(plan, context) {
+export function executeScalarAggregate(plan, context) {
   // Fast path: use scanColumn when available
   const fast = tryColumnScanAggregate(plan, context)
   if (fast) {
-    yield* fast
-    return
+    return {
+      numRows: 1,
+      maxRows: 1,
+      rows: fast,
+    }
   }
 
-  // Collect all rows into single group
-  /** @type {AsyncRow[]} */
-  const group = []
-  for await (const row of executePlan({ plan: plan.child, context })) {
-    if (context.signal?.aborted) return
-    group.push(row)
-  }
+  const child = executePlan({ plan: plan.child, context })
+  return {
+    numRows: plan.having ? undefined : 1,
+    maxRows: 1,
+    async *rows () {
+      // Collect all rows into single group
+      /** @type {AsyncRow[]} */
+      const group = []
+      for await (const row of child.rows()) {
+        if (context.signal?.aborted) return
+        group.push(row)
+      }
 
-  const asyncRow = projectAggregateColumns(plan.columns, group, context)
+      const asyncRow = projectAggregateColumns(plan.columns, group, context)
 
-  // Apply HAVING filter
-  if (plan.having) {
-    /** @type {AsyncRow} */
-    const havingRow = {
-      columns: [...group[0].columns, ...asyncRow.columns],
-      cells: { ...group[0].cells, ...asyncRow.cells },
-    }
-    const passes = await evaluateExpr({
-      node: plan.having,
-      row: havingRow,
-      rows: group,
-      context,
-    })
-    if (!passes) return
-  }
+      // Apply HAVING filter
+      if (plan.having) {
+        const baseRow = group[0] ?? { columns: [], cells: {} }
+        /** @type {AsyncRow} */
+        const havingRow = {
+          columns: [...baseRow.columns, ...asyncRow.columns],
+          cells: { ...baseRow.cells, ...asyncRow.cells },
+        }
+        const passes = await evaluateExpr({
+          node: plan.having,
+          row: havingRow,
+          rows: group,
+          context,
+        })
+        if (!passes) return
+      }
 
-  yield asyncRow
+      yield asyncRow
+    },
+  }
 }
 
 /**
@@ -163,7 +180,7 @@ export async function* executeScalarAggregate(plan, context) {
  *
  * @param {ScalarAggregateNode} plan
  * @param {ExecuteContext} context
- * @returns {AsyncGenerator<AsyncRow> | undefined}
+ * @returns {(() => AsyncGenerator<AsyncRow>) | undefined}
  */
 function tryColumnScanAggregate(plan, { tables, signal }) {
   // No HAVING support in fast path
@@ -188,7 +205,7 @@ function tryColumnScanAggregate(plan, { tables, signal }) {
     specs.push(spec)
   }
 
-  return (async function* () {
+  return async function* () {
     /** @type {string[]} */
     const columns = []
     /** @type {AsyncCells} */
@@ -200,7 +217,7 @@ function tryColumnScanAggregate(plan, { tables, signal }) {
     }
 
     yield { columns, cells }
-  })()
+  }
 }
 
 /**