squirreling 0.12.26 → 0.13.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "squirreling",
-  "version": "0.12.26",
+  "version": "0.13.0",
   "description": "Squirreling Async SQL Engine",
   "author": "Hyperparam",
   "homepage": "https://hyperparam.app",
@@ -42,7 +42,7 @@
     "@types/node": "26.1.0",
     "@vitest/coverage-v8": "4.1.9",
     "eslint": "9.39.4",
-    "eslint-plugin-jsdoc": "63.0.10",
+    "eslint-plugin-jsdoc": "63.0.11",
     "typescript": "6.0.3",
     "vitest": "4.1.9"
   }

package/src/execute/accumulator.js

@@ -0,0 +1,89 @@
+import { keyify } from './utils.js'
+
+/**
+ * @import { SqlPrimitive } from '../types.js'
+ */
+
+/**
+ * Incremental state for one streamable aggregate. Shared by the streaming
+ * aggregate executor and the scanColumn fast path so COUNT/COUNTIF/SUM/AVG/
+ * MIN/MAX fold semantics live in one place.
+ *
+ * @typedef {{
+ *   count: number,
+ *   sum: number,
+ *   min: SqlPrimitive,
+ *   max: SqlPrimitive,
+ *   seen: Set<unknown> | null,
+ * }} Accumulator
+ */
+
+/**
+ * @param {string} funcName
+ * @param {boolean} [distinct]
+ * @returns {Accumulator}
+ */
+export function newAccumulator(funcName, distinct) {
+  return {
+    count: 0,
+    sum: 0,
+    min: null,
+    max: null,
+    seen: funcName === 'COUNT' && distinct ? new Set() : null,
+  }
+}
+
+/**
+ * Folds one value into an accumulator, matching the buffered semantics in
+ * evaluate.js: COUNT counts non-null, COUNTIF counts truthy, MIN/MAX compare
+ * raw values, SUM/AVG only accumulate finite numbers.
+ *
+ * @param {string} funcName
+ * @param {Accumulator} acc
+ * @param {SqlPrimitive} value
+ */
+export function updateAccumulator(funcName, acc, value) {
+  switch (funcName) {
+  case 'COUNT':
+    if (value == null) break
+    if (acc.seen) acc.seen.add(keyify(value))
+    else acc.count++
+    break
+  case 'COUNTIF':
+    if (value) acc.count++
+    break
+  case 'MIN':
+    if (value != null && (acc.min === null || value < acc.min)) acc.min = value
+    break
+  case 'MAX':
+    if (value != null && (acc.max === null || value > acc.max)) acc.max = value
+    break
+  default: { // SUM, AVG
+    if (value == null) break
+    const num = Number(value)
+    if (Number.isFinite(num)) {
+      acc.sum += num
+      acc.count++
+    }
+  }
+  }
+}
+
+/**
+ * Reduces an accumulator to its final aggregate value.
+ *
+ * @param {string} funcName
+ * @param {Accumulator} acc
+ * @returns {SqlPrimitive}
+ */
+export function finalizeAccumulator(funcName, acc) {
+  switch (funcName) {
+  case 'COUNT': return acc.seen ? acc.seen.size : acc.count
+  case 'COUNTIF': return acc.count
+  case 'SUM': return acc.count === 0 ? null : acc.sum
+  case 'AVG': return acc.count === 0 ? null : acc.sum / acc.count
+  case 'MIN': return acc.min
+  case 'MAX': return acc.max
+  default: return null
+  }
+}

package/src/execute/aggregates.js

@@ -1,7 +1,9 @@
 import { derivedAlias } from '../expression/alias.js'
 import { evaluateExpr } from '../expression/evaluate.js'
+import { finalizeAccumulator, newAccumulator, updateAccumulator } from './accumulator.js'
 import { executePlan, selectColumnNames } from './execute.js'
 import { sortEntriesByTerms } from './sort.js'
+import { planStreamingAggregates, streamingHashAggregateRows, streamingScalarAggregateRows } from './streamingAggregate.js'
 import { keyify } from './utils.js'
 import { yieldToEventLoop } from './yield.js'
 
@@ -80,6 +82,14 @@ function aggregateContextRow(group, aggregateRow) {
  */
 export function executeHashAggregate(plan, context) {
   const child = executePlan({ plan: plan.child, context })
+  const streaming = planStreamingAggregates(plan, child.columns)
+  if (streaming) {
+    return {
+      columns: selectColumnNames(plan.columns, child.columns),
+      maxRows: child.maxRows,
+      rows: streamingHashAggregateRows({ plan, streaming, child, context }),
+    }
+  }
   return {
     columns: selectColumnNames(plan.columns, child.columns),
     maxRows: child.maxRows,
@@ -91,7 +101,7 @@ export function executeHashAggregate(plan, context) {
       for await (const row of child.rows()) {
         if (++collectCount % YIELD_INTERVAL === 0) {
           await yieldToEventLoop()
-          if (context.signal?.aborted) return
+          context.signal?.throwIfAborted()
         }
         allRows.push(row)
       }
@@ -111,7 +121,7 @@ export function executeHashAggregate(plan, context) {
       for (let chunkStart = 0; chunkStart < allRows.length; chunkStart += YIELD_INTERVAL) {
         if (chunkStart > 0) {
           await yieldToEventLoop()
-          if (context.signal?.aborted) return
+          context.signal?.throwIfAborted()
         }
         const chunkEnd = Math.min(chunkStart + YIELD_INTERVAL, allRows.length)
         const chunkLen = chunkEnd - chunkStart
@@ -196,6 +206,15 @@ export function executeScalarAggregate(plan, context) {
   }
 
   const child = executePlan({ plan: plan.child, context })
+  const streaming = planStreamingAggregates(plan, child.columns)
+  if (streaming) {
+    return {
+      columns: selectColumnNames(plan.columns, child.columns),
+      numRows: plan.having ? undefined : 1,
+      maxRows: 1,
+      rows: streamingScalarAggregateRows({ plan, streaming, child, context }),
+    }
+  }
   return {
     columns: selectColumnNames(plan.columns, child.columns),
     numRows: plan.having ? undefined : 1,
@@ -208,7 +227,7 @@ export function executeScalarAggregate(plan, context) {
       for await (const row of child.rows()) {
         if (++collectCount % YIELD_INTERVAL === 0) {
           await yieldToEventLoop()
-          if (context.signal?.aborted) return
+          context.signal?.throwIfAborted()
         }
         group.push(row)
       }
@@ -344,17 +363,6 @@ function extractColumnAggSpec({ expr, alias }) {
   }
 }
 
-/**
- * @typedef {{
- *   spec: ColumnAggSpec,
- *   count: number,
- *   sum: number,
- *   min: SqlPrimitive,
- *   max: SqlPrimitive,
- *   seen: Set<unknown> | null,
- * }} AggAccumulator
- */
-
 /**
  * Scans a column once and computes every aggregate over it in a single pass.
  * All specs share the one scanColumn walk, so MIN(x)/MAX(x)/AVG(x) decode x once.
@@ -371,41 +379,15 @@ async function scanColumnGroup({ table, specs, limit, offset, signal }) {
   const { column } = specs[0]
   const values = table.scanColumn({ column, limit, offset, signal })
 
-  /** @type {AggAccumulator[]} */
-  const accs = specs.map(spec => /** @type {AggAccumulator} */ ({
-    spec,
-    count: 0,
-    sum: 0,
-    min: null,
-    max: null,
-    seen: spec.funcName === 'COUNT' && spec.distinct ? new Set() : null,
-  }))
+  const accs = specs.map(spec => ({ spec, acc: newAccumulator(spec.funcName, spec.distinct) }))
 
   for await (const chunk of values) {
     signal?.throwIfAborted()
     for (let i = 0; i < chunk.length; i++) {
       const v = chunk[i]
       if (v == null) continue
-      for (const acc of accs) {
-        switch (acc.spec.funcName) {
-        case 'COUNT':
-          if (acc.seen) acc.seen.add(keyify(v))
-          else acc.count++
-          break
-        case 'MIN':
-          if (acc.min === null || v < acc.min) acc.min = v
-          break
-        case 'MAX':
-          if (acc.max === null || v > acc.max) acc.max = v
-          break
-        default: { // SUM, AVG
-          const num = Number(v)
-          if (Number.isFinite(num)) {
-            acc.sum += num
-            acc.count++
-          }
-        }
-        }
+      for (const { spec, acc } of accs) {
+        updateAccumulator(spec.funcName, acc, v)
       }
     }
   }
@@ -413,25 +395,8 @@ async function scanColumnGroup({ table, specs, limit, offset, signal }) {
 
   /** @type {Map<string, SqlPrimitive>} */
   const result = new Map()
-  for (const acc of accs) {
-    result.set(acc.spec.alias, finalizeAgg(acc))
+  for (const { spec, acc } of accs) {
+    result.set(spec.alias, finalizeAccumulator(spec.funcName, acc))
   }
   return result
 }
-
-/**
- * Reduces one accumulator to its final aggregate value.
- *
- * @param {AggAccumulator} acc
- * @returns {SqlPrimitive}
- */
-function finalizeAgg(acc) {
-  switch (acc.spec.funcName) {
-  case 'COUNT': return acc.seen ? acc.seen.size : acc.count
-  case 'SUM': return acc.count === 0 ? null : acc.sum
-  case 'AVG': return acc.count === 0 ? null : acc.sum / acc.count
-  case 'MIN': return acc.min
-  case 'MAX': return acc.max
-  default: return null
-  }
-}

package/src/execute/execute.js

@@ -174,7 +174,7 @@ function executeUnnest(plan, context) {
       const value = await evaluateExpr({ node: plan.args[0], row, rowIndex: 1, context })
       if (!Array.isArray(value)) return
       for (const element of value) {
-        if (context.signal?.aborted) return
+        context.signal?.throwIfAborted()
         yield {
           columns,
           cells: { [columnName]: () => Promise.resolve(element) },
@@ -209,7 +209,7 @@ function executeJsonEach(plan, context) {
       }
       if (Array.isArray(parsed)) {
         for (let i = 0; i < parsed.length; i++) {
-          if (context.signal?.aborted) return
+          context.signal?.throwIfAborted()
           const k = i
           const v = parsed[i]
           yield {
@@ -224,7 +224,7 @@ function executeJsonEach(plan, context) {
       }
       if (typeof parsed === 'object' && parsed !== null) {
         for (const [k, v] of Object.entries(parsed)) {
-          if (context.signal?.aborted) return
+          context.signal?.throwIfAborted()
           yield {
             columns,
             cells: {
@@ -294,7 +294,7 @@ function executeScan(plan, context) {
       async *rows() {
         const columns = [column]
         for await (const chunk of chunks) {
-          if (signal?.aborted) return
+          signal?.throwIfAborted()
           for (let i = 0; i < chunk.length; i++) {
             const value = chunk[i]
             yield {
@@ -303,6 +303,9 @@ function executeScan(plan, context) {
             }
           }
         }
+        // A data source may end its stream cooperatively on abort; surface
+        // the abort so a truncated scan is not mistaken for a complete one
+        signal?.throwIfAborted()
       },
     }
   }
@@ -335,6 +338,10 @@ function executeScan(plan, context) {
       }
 
       yield* result
+
+      // A data source may end its stream cooperatively on abort; surface
+      // the abort so a truncated scan is not mistaken for a complete one
+      signal?.throwIfAborted()
     },
   }
 }
@@ -420,10 +427,10 @@ async function* filterRows(rows, condition, context, limit) {
   let buffer = []
 
   for await (const row of rows) {
-    if (context.signal?.aborted) return
+    context.signal?.throwIfAborted()
     if (++innerCount % YIELD_INTERVAL === 0) {
       await yieldToEventLoop()
-      if (context.signal?.aborted) return
+      context.signal?.throwIfAborted()
     }
     rowIndex++
     buffer.push({ row, rowIndex })
@@ -466,10 +473,10 @@ async function* limitRows(rows, limit = Infinity, offset = 0, signal) {
   let yielded = 0
   let innerCount = 0
   for await (const row of rows) {
-    if (signal?.aborted) return
+    signal?.throwIfAborted()
     if (++innerCount % YIELD_INTERVAL === 0) {
       await yieldToEventLoop()
-      if (signal?.aborted) return
+      signal?.throwIfAborted()
     }
     if (skipped < offset) {
       skipped++
@@ -521,10 +528,10 @@ function executeProject(plan, context) {
       let innerCount = 0
 
       for await (const row of child.rows()) {
-        if (context.signal?.aborted) return
+        context.signal?.throwIfAborted()
         if (++innerCount % YIELD_INTERVAL === 0) {
           await yieldToEventLoop()
-          if (context.signal?.aborted) return
+          context.signal?.throwIfAborted()
         }
         rowIndex++
         const currentRowIndex = rowIndex
@@ -614,10 +621,10 @@ function executeDistinct(plan, context) {
       let innerCount = 0
 
       for await (const row of child.rows()) {
-        if (signal?.aborted) return
+        signal?.throwIfAborted()
         if (++innerCount % YIELD_INTERVAL === 0) {
           await yieldToEventLoop()
-          if (signal?.aborted) return
+          signal?.throwIfAborted()
         }
         buffer.push(row)
 
@@ -701,10 +708,10 @@ function executeSetOperation(plan, context) {
           const seen = new Set()
           let count = 0
           for await (const row of left.rows()) {
-            if (signal?.aborted) return
+            signal?.throwIfAborted()
             if (++count % YIELD_INTERVAL === 0) {
               await yieldToEventLoop()
-              if (signal?.aborted) return
+              signal?.throwIfAborted()
             }
             const key = await stableRowKey(row)
             if (!seen.has(key)) {
@@ -713,10 +720,10 @@ function executeSetOperation(plan, context) {
             }
           }
           for await (const row of right.rows()) {
-            if (signal?.aborted) return
+            signal?.throwIfAborted()
             if (++count % YIELD_INTERVAL === 0) {
               await yieldToEventLoop()
-              if (signal?.aborted) return
+              signal?.throwIfAborted()
             }
             const key = await stableRowKey(row)
             if (!seen.has(key)) {
@@ -739,10 +746,10 @@ function executeSetOperation(plan, context) {
         const rightKeys = new Map()
         let tick = 0
         for await (const row of right.rows()) {
-          if (signal?.aborted) return
+          signal?.throwIfAborted()
           if (++tick % YIELD_INTERVAL === 0) {
             await yieldToEventLoop()
-            if (signal?.aborted) return
+            signal?.throwIfAborted()
           }
           const key = await stableRowKey(row)
           rightKeys.set(key, (rightKeys.get(key) ?? 0) + 1)
@@ -751,10 +758,10 @@ function executeSetOperation(plan, context) {
         if (plan.all) {
           // INTERSECT ALL: yield each left row that matches, consuming right counts
           for await (const row of left.rows()) {
-            if (signal?.aborted) return
+            signal?.throwIfAborted()
             if (++tick % YIELD_INTERVAL === 0) {
               await yieldToEventLoop()
-              if (signal?.aborted) return
+              signal?.throwIfAborted()
             }
             const key = await stableRowKey(row)
             const count = rightKeys.get(key)
@@ -767,10 +774,10 @@ function executeSetOperation(plan, context) {
           // INTERSECT: yield deduplicated rows present in both
           const seen = new Set()
           for await (const row of left.rows()) {
-            if (signal?.aborted) return
+            signal?.throwIfAborted()
             if (++tick % YIELD_INTERVAL === 0) {
               await yieldToEventLoop()
-              if (signal?.aborted) return
+              signal?.throwIfAborted()
             }
             const key = await stableRowKey(row)
             if (rightKeys.has(key) && !seen.has(key)) {
@@ -794,10 +801,10 @@ function executeSetOperation(plan, context) {
         const rightKeys = new Map()
         let tick = 0
         for await (const row of right.rows()) {
-          if (signal?.aborted) return
+          signal?.throwIfAborted()
           if (++tick % YIELD_INTERVAL === 0) {
             await yieldToEventLoop()
-            if (signal?.aborted) return
+            signal?.throwIfAborted()
           }
           const key = await stableRowKey(row)
           rightKeys.set(key, (rightKeys.get(key) ?? 0) + 1)
@@ -806,10 +813,10 @@ function executeSetOperation(plan, context) {
         if (plan.all) {
           // EXCEPT ALL: yield left rows, consuming right counts
           for await (const row of left.rows()) {
-            if (signal?.aborted) return
+            signal?.throwIfAborted()
             if (++tick % YIELD_INTERVAL === 0) {
               await yieldToEventLoop()
-              if (signal?.aborted) return
+              signal?.throwIfAborted()
             }
             const key = await stableRowKey(row)
             const count = rightKeys.get(key)
@@ -823,10 +830,10 @@ function executeSetOperation(plan, context) {
           // EXCEPT: yield deduplicated left rows not in right
           const seen = new Set()
           for await (const row of left.rows()) {
-            if (signal?.aborted) return
+            signal?.throwIfAborted()
             if (++tick % YIELD_INTERVAL === 0) {
               await yieldToEventLoop()
-              if (signal?.aborted) return
+              signal?.throwIfAborted()
             }
             const key = await stableRowKey(row)
             if (!rightKeys.has(key) && !seen.has(key)) {

package/src/execute/join.js

@@ -56,7 +56,7 @@ export function executeNestedLoopJoin(plan, context) {
       /** @type {AsyncRow[]} */
       const innerRows = []
       for await (const row of inner.rows()) {
-        if (context.signal?.aborted) return
+        context.signal?.throwIfAborted()
         innerRows.push(row)
       }
 
@@ -69,7 +69,7 @@ export function executeNestedLoopJoin(plan, context) {
 
       let innerCount = 0
       for await (const outerRow of outer.rows()) {
-        if (context.signal?.aborted) return
+        context.signal?.throwIfAborted()
 
         if (!outerCols) {
           outerCols = outerRow.columns
@@ -80,7 +80,7 @@ export function executeNestedLoopJoin(plan, context) {
         for (const innerRow of innerRows) {
           if (++innerCount % YIELD_INTERVAL === 0) {
             await yieldToEventLoop()
-            if (context.signal?.aborted) return
+            context.signal?.throwIfAborted()
           }
           const tempMerged = merge(outerRow, innerRow)
           const matches = await evaluateExpr({
@@ -101,7 +101,7 @@ export function executeNestedLoopJoin(plan, context) {
         }
       }
 
-      if (context.signal?.aborted) return
+      context.signal?.throwIfAborted()
 
       // Unmatched inner rows for outer joins on the buffered side
       if (matchedInnerRows) {
@@ -134,7 +134,7 @@ function executeLateralJoin(plan, context) {
       const rightTable = plan.rightAlias
 
       for await (const leftRow of left.rows()) {
-        if (context.signal?.aborted) return
+        context.signal?.throwIfAborted()
 
         // When nested inside a correlated subquery, preserve the enclosing
         // outer row so UNNEST args can reference its columns (e.g. o.arr).
@@ -146,7 +146,7 @@ function executeLateralJoin(plan, context) {
 
         let hasMatch = false
         for await (const rightRow of right.rows()) {
-          if (context.signal?.aborted) return
+          context.signal?.throwIfAborted()
           const merged = mergeRows(leftRow, rightRow, leftTable, rightTable)
           const matches = plan.condition === undefined
             ? true
@@ -199,10 +199,10 @@ export function executePositionalJoin(plan, context) {
       while (true) {
         const [leftResult, rightResult] = await Promise.all([leftRows.next(), rightRows.next()])
         if (leftResult.done && rightResult.done) return
-        if (signal?.aborted) return
+        signal?.throwIfAborted()
         if (++tick % YIELD_INTERVAL === 0) {
           await yieldToEventLoop()
-          if (signal?.aborted) return
+          signal?.throwIfAborted()
         }
         if (!leftResult.done && !leftCols.length) leftCols = leftResult.value.columns
         if (!rightResult.done && !rightCols.length) rightCols = rightResult.value.columns
@@ -266,10 +266,10 @@ export function executeHashJoin(plan, context) {
       let buildCols = []
       let innerCount = 0
       for await (const buildRow of build.rows()) {
-        if (context.signal?.aborted) return
+        context.signal?.throwIfAborted()
         if (++innerCount % YIELD_INTERVAL === 0) {
           await yieldToEventLoop()
-          if (context.signal?.aborted) return
+          context.signal?.throwIfAborted()
         }
         if (!buildCols.length) {
           buildCols = buildRow.columns
@@ -297,7 +297,7 @@ export function executeHashJoin(plan, context) {
 
       // Probe phase: stream the other side
       for await (const probeRow of probe.rows()) {
-        if (context.signal?.aborted) return
+        context.signal?.throwIfAborted()
 
         if (!probeCols) {
           probeCols = probeRow.columns
@@ -314,7 +314,7 @@ export function executeHashJoin(plan, context) {
             for (const buildRow of candidates) {
               if (++innerCount % YIELD_INTERVAL === 0) {
                 await yieldToEventLoop()
-                if (context.signal?.aborted) return
+                context.signal?.throwIfAborted()
               }
               const merged = merge(probeRow, buildRow)
               if (residual) {
@@ -333,7 +333,7 @@ export function executeHashJoin(plan, context) {
         }
       }
 
-      if (context.signal?.aborted) return
+      context.signal?.throwIfAborted()
 
       // Unmatched build rows for outer joins on the build side
       if (buildRows && matchedBuildRows) {

package/src/execute/sort.js

@@ -4,16 +4,21 @@ import { executePlan } from './execute.js'
 import { compareForTerm } from './utils.js'
 
 /**
- * @import { AsyncRow, ExecuteContext, OrderByItem, QueryResults, SqlPrimitive } from '../types.js'
+ * @import { AsyncRow, ExecuteContext, ExprNode, OrderByItem, QueryResults, SqlPrimitive } from '../types.js'
  * @import { SortNode } from '../plan/types.js'
  */
 
 const MAX_CHUNK = 256
 
 /**
+ * When exprs is set, exprs[i] is evaluated for ORDER BY term i instead of the
+ * term's own expression, so callers can pre-substitute per-entry values (for
+ * example, finalized aggregates) while keeping tie-aware term evaluation.
+ *
  * @typedef {{
  *   row: AsyncRow,
  *   rows?: AsyncRow[],
+ *   exprs?: ExprNode[],
  * }} SortEntry
  */
 
@@ -63,7 +68,7 @@ export async function sortEntriesByTerms({ entries, orderBy, context, cacheValue
         const chunk = missing.slice(start, start + chunkSize)
         const values = await Promise.all(chunk.map(idx =>
           evaluateExpr({
-            node: term.expr,
+            node: entries[idx].exprs?.[orderByIdx] ?? term.expr,
             row: entries[idx].row,
             rows: entries[idx].rows,
             context,
@@ -141,7 +146,7 @@ export function executeSort(plan, context) {
       /** @type {SortEntry[]} */
       let entries = []
       for await (const row of child.rows()) {
-        if (context.signal?.aborted) return
+        context.signal?.throwIfAborted()
         entries.push({ row })
         if (entries.length >= bufferLimit) {
           // Sort keys are cached on the rows, so survivors of one truncation