squirreling 0.12.25 → 0.12.27

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "squirreling",
-  "version": "0.12.25",
+  "version": "0.12.27",
   "description": "Squirreling Async SQL Engine",
   "author": "Hyperparam",
   "homepage": "https://hyperparam.app",
@@ -39,7 +39,7 @@
     "test": "vitest run"
   },
   "devDependencies": {
-    "@types/node": "26.0.1",
+    "@types/node": "26.1.0",
     "@vitest/coverage-v8": "4.1.9",
     "eslint": "9.39.4",
     "eslint-plugin-jsdoc": "63.0.10",

package/src/execute/accumulator.js

@@ -0,0 +1,89 @@
+import { keyify } from './utils.js'
+
+/**
+ * @import { SqlPrimitive } from '../types.js'
+ */
+
+/**
+ * Incremental state for one streamable aggregate. Shared by the streaming
+ * aggregate executor and the scanColumn fast path so COUNT/COUNTIF/SUM/AVG/
+ * MIN/MAX fold semantics live in one place.
+ *
+ * @typedef {{
+ *   count: number,
+ *   sum: number,
+ *   min: SqlPrimitive,
+ *   max: SqlPrimitive,
+ *   seen: Set<unknown> | null,
+ * }} Accumulator
+ */
+
+/**
+ * @param {string} funcName
+ * @param {boolean} [distinct]
+ * @returns {Accumulator}
+ */
+export function newAccumulator(funcName, distinct) {
+  return {
+    count: 0,
+    sum: 0,
+    min: null,
+    max: null,
+    seen: funcName === 'COUNT' && distinct ? new Set() : null,
+  }
+}
+
+/**
+ * Folds one value into an accumulator, matching the buffered semantics in
+ * evaluate.js: COUNT counts non-null, COUNTIF counts truthy, MIN/MAX compare
+ * raw values, SUM/AVG only accumulate finite numbers.
+ *
+ * @param {string} funcName
+ * @param {Accumulator} acc
+ * @param {SqlPrimitive} value
+ */
+export function updateAccumulator(funcName, acc, value) {
+  switch (funcName) {
+  case 'COUNT':
+    if (value == null) break
+    if (acc.seen) acc.seen.add(keyify(value))
+    else acc.count++
+    break
+  case 'COUNTIF':
+    if (value) acc.count++
+    break
+  case 'MIN':
+    if (value != null && (acc.min === null || value < acc.min)) acc.min = value
+    break
+  case 'MAX':
+    if (value != null && (acc.max === null || value > acc.max)) acc.max = value
+    break
+  default: { // SUM, AVG
+    if (value == null) break
+    const num = Number(value)
+    if (Number.isFinite(num)) {
+      acc.sum += num
+      acc.count++
+    }
+  }
+  }
+}
+
+/**
+ * Reduces an accumulator to its final aggregate value.
+ *
+ * @param {string} funcName
+ * @param {Accumulator} acc
+ * @returns {SqlPrimitive}
+ */
+export function finalizeAccumulator(funcName, acc) {
+  switch (funcName) {
+  case 'COUNT': return acc.seen ? acc.seen.size : acc.count
+  case 'COUNTIF': return acc.count
+  case 'SUM': return acc.count === 0 ? null : acc.sum
+  case 'AVG': return acc.count === 0 ? null : acc.sum / acc.count
+  case 'MIN': return acc.min
+  case 'MAX': return acc.max
+  default: return null
+  }
+}

package/src/execute/aggregates.js

@@ -1,7 +1,9 @@
 import { derivedAlias } from '../expression/alias.js'
 import { evaluateExpr } from '../expression/evaluate.js'
+import { finalizeAccumulator, newAccumulator, updateAccumulator } from './accumulator.js'
 import { executePlan, selectColumnNames } from './execute.js'
 import { sortEntriesByTerms } from './sort.js'
+import { planStreamingAggregates, streamingHashAggregateRows, streamingScalarAggregateRows } from './streamingAggregate.js'
 import { keyify } from './utils.js'
 import { yieldToEventLoop } from './yield.js'
 
@@ -80,6 +82,14 @@ function aggregateContextRow(group, aggregateRow) {
  */
 export function executeHashAggregate(plan, context) {
   const child = executePlan({ plan: plan.child, context })
+  const streaming = planStreamingAggregates(plan, child.columns)
+  if (streaming) {
+    return {
+      columns: selectColumnNames(plan.columns, child.columns),
+      maxRows: child.maxRows,
+      rows: streamingHashAggregateRows({ plan, streaming, child, context }),
+    }
+  }
   return {
     columns: selectColumnNames(plan.columns, child.columns),
     maxRows: child.maxRows,
@@ -196,6 +206,15 @@ export function executeScalarAggregate(plan, context) {
   }
 
   const child = executePlan({ plan: plan.child, context })
+  const streaming = planStreamingAggregates(plan, child.columns)
+  if (streaming) {
+    return {
+      columns: selectColumnNames(plan.columns, child.columns),
+      numRows: plan.having ? undefined : 1,
+      maxRows: 1,
+      rows: streamingScalarAggregateRows({ plan, streaming, child, context }),
+    }
+  }
   return {
     columns: selectColumnNames(plan.columns, child.columns),
     numRows: plan.having ? undefined : 1,
@@ -344,17 +363,6 @@ function extractColumnAggSpec({ expr, alias }) {
   }
 }
 
-/**
- * @typedef {{
- *   spec: ColumnAggSpec,
- *   count: number,
- *   sum: number,
- *   min: SqlPrimitive,
- *   max: SqlPrimitive,
- *   seen: Set<unknown> | null,
- * }} AggAccumulator
- */
-
 /**
  * Scans a column once and computes every aggregate over it in a single pass.
  * All specs share the one scanColumn walk, so MIN(x)/MAX(x)/AVG(x) decode x once.
@@ -371,41 +379,15 @@ async function scanColumnGroup({ table, specs, limit, offset, signal }) {
   const { column } = specs[0]
   const values = table.scanColumn({ column, limit, offset, signal })
 
-  /** @type {AggAccumulator[]} */
-  const accs = specs.map(spec => /** @type {AggAccumulator} */ ({
-    spec,
-    count: 0,
-    sum: 0,
-    min: null,
-    max: null,
-    seen: spec.funcName === 'COUNT' && spec.distinct ? new Set() : null,
-  }))
+  const accs = specs.map(spec => ({ spec, acc: newAccumulator(spec.funcName, spec.distinct) }))
 
   for await (const chunk of values) {
     signal?.throwIfAborted()
     for (let i = 0; i < chunk.length; i++) {
       const v = chunk[i]
       if (v == null) continue
-      for (const acc of accs) {
-        switch (acc.spec.funcName) {
-        case 'COUNT':
-          if (acc.seen) acc.seen.add(keyify(v))
-          else acc.count++
-          break
-        case 'MIN':
-          if (acc.min === null || v < acc.min) acc.min = v
-          break
-        case 'MAX':
-          if (acc.max === null || v > acc.max) acc.max = v
-          break
-        default: { // SUM, AVG
-          const num = Number(v)
-          if (Number.isFinite(num)) {
-            acc.sum += num
-            acc.count++
-          }
-        }
-        }
+      for (const { spec, acc } of accs) {
+        updateAccumulator(spec.funcName, acc, v)
       }
     }
   }
@@ -413,25 +395,8 @@ async function scanColumnGroup({ table, specs, limit, offset, signal }) {
 
   /** @type {Map<string, SqlPrimitive>} */
   const result = new Map()
-  for (const acc of accs) {
-    result.set(acc.spec.alias, finalizeAgg(acc))
+  for (const { spec, acc } of accs) {
+    result.set(spec.alias, finalizeAccumulator(spec.funcName, acc))
   }
   return result
 }
-
-/**
- * Reduces one accumulator to its final aggregate value.
- *
- * @param {AggAccumulator} acc
- * @returns {SqlPrimitive}
- */
-function finalizeAgg(acc) {
-  switch (acc.spec.funcName) {
-  case 'COUNT': return acc.seen ? acc.seen.size : acc.count
-  case 'SUM': return acc.count === 0 ? null : acc.sum
-  case 'AVG': return acc.count === 0 ? null : acc.sum / acc.count
-  case 'MIN': return acc.min
-  case 'MAX': return acc.max
-  default: return null
-  }
-}

package/src/execute/join.js

@@ -24,43 +24,65 @@ export function executeNestedLoopJoin(plan, context) {
   }
   const left = executePlan({ plan: plan.left, context })
   const right = executePlan({ plan: plan.right, context })
+  // Buffer the smaller side when both sizes are known, streaming the larger
+  // side through the outer loop. Swapping reorders output rows, which SQL
+  // leaves unspecified.
+  const leftSize = left.numRows ?? left.maxRows
+  const rightSize = right.numRows ?? right.maxRows
+  const swap = leftSize !== undefined && rightSize !== undefined && leftSize < rightSize
   return {
     columns: mergeColumnNames(left.columns, right.columns, plan.leftAlias, plan.rightAlias),
     async *rows() {
       const leftTable = plan.leftAlias
       const rightTable = plan.rightAlias
+      const inner = swap ? left : right
+      const outer = swap ? right : left
+      // Which sides must also emit their unmatched rows
+      const innerOuter = plan.joinType === 'FULL' || plan.joinType === (swap ? 'LEFT' : 'RIGHT')
+      const outerOuter = plan.joinType === 'FULL' || plan.joinType === (swap ? 'RIGHT' : 'LEFT')
+
+      /**
+       * @param {AsyncRow} outerRow
+       * @param {AsyncRow} innerRow
+       * @returns {AsyncRow}
+       */
+      function merge(outerRow, innerRow) {
+        return swap
+          ? mergeRows(innerRow, outerRow, leftTable, rightTable)
+          : mergeRows(outerRow, innerRow, leftTable, rightTable)
+      }
 
-      // Buffer right rows
+      // Buffer the inner side
       /** @type {AsyncRow[]} */
-      const rightRows = []
-      for await (const row of right.rows()) {
+      const innerRows = []
+      for await (const row of inner.rows()) {
         if (context.signal?.aborted) return
-        rightRows.push(row)
+        innerRows.push(row)
       }
 
-      const rightCols = rightRows.length ? rightRows[0].columns : []
+      const innerCols = innerRows.length ? innerRows[0].columns : []
 
       /** @type {string[] | undefined} */
-      let leftCols = undefined
+      let outerCols = undefined
       /** @type {Set<AsyncRow> | undefined} */
-      const matchedRightRows = plan.joinType === 'RIGHT' || plan.joinType === 'FULL' ? new Set() : undefined
+      const matchedInnerRows = innerOuter ? new Set() : undefined
 
       let innerCount = 0
-      for await (const leftRow of left.rows()) {
+      for await (const outerRow of outer.rows()) {
         if (context.signal?.aborted) return
 
-        if (!leftCols) {
-          leftCols = leftRow.columns
+        if (!outerCols) {
+          outerCols = outerRow.columns
         }
 
         let hasMatch = false
 
-        for (const rightRow of rightRows) {
+        for (const innerRow of innerRows) {
           if (++innerCount % YIELD_INTERVAL === 0) {
             await yieldToEventLoop()
             if (context.signal?.aborted) return
           }
-          const tempMerged = mergeRows(leftRow, rightRow, leftTable, rightTable)
+          const tempMerged = merge(outerRow, innerRow)
           const matches = await evaluateExpr({
             node: plan.condition,
             row: tempMerged,
@@ -69,25 +91,23 @@ export function executeNestedLoopJoin(plan, context) {
 
           if (matches) {
             hasMatch = true
-            matchedRightRows?.add(rightRow)
+            matchedInnerRows?.add(innerRow)
             yield tempMerged
           }
         }
 
-        if (!hasMatch && (plan.joinType === 'LEFT' || plan.joinType === 'FULL')) {
-          const nullRight = createNullRow(rightCols)
-          yield mergeRows(leftRow, nullRight, leftTable, rightTable)
+        if (!hasMatch && outerOuter) {
+          yield merge(outerRow, createNullRow(innerCols))
         }
       }
 
       if (context.signal?.aborted) return
 
-      // Unmatched right rows for RIGHT/FULL joins
-      if (matchedRightRows) {
-        for (const rightRow of rightRows) {
-          if (!matchedRightRows.has(rightRow)) {
-            const nullLeft = createNullRow(leftCols ?? [])
-            yield mergeRows(nullLeft, rightRow, leftTable, rightTable)
+      // Unmatched inner rows for outer joins on the buffered side
+      if (matchedInnerRows) {
+        for (const innerRow of innerRows) {
+          if (!matchedInnerRows.has(innerRow)) {
+            yield merge(createNullRow(outerCols ?? []), innerRow)
           }
         }
       }
@@ -167,29 +187,27 @@ export function executePositionalJoin(plan, context) {
       const leftTable = plan.leftAlias
       const rightTable = plan.rightAlias
 
-      // Buffer both sides (required for positional join)
-      /** @type {AsyncRow[]} */
-      const leftRows = []
-      for await (const row of left.rows()) {
-        if (signal?.aborted) return
-        leftRows.push(row)
-      }
-
-      /** @type {AsyncRow[]} */
-      const rightRows = []
-      for await (const row of right.rows()) {
+      // Zip both sides in lockstep without buffering either; the shorter
+      // side is padded with NULL rows until the longer side is exhausted
+      const leftRows = left.rows()
+      const rightRows = right.rows()
+      /** @type {string[]} */
+      let leftCols = []
+      /** @type {string[]} */
+      let rightCols = []
+      let tick = 0
+      while (true) {
+        const [leftResult, rightResult] = await Promise.all([leftRows.next(), rightRows.next()])
+        if (leftResult.done && rightResult.done) return
         if (signal?.aborted) return
-        rightRows.push(row)
-      }
-
-      const maxLen = Math.max(leftRows.length, rightRows.length)
-      const leftCols = leftRows[0]?.columns ?? []
-      const rightCols = rightRows[0]?.columns ?? []
-
-      for (let i = 0; i < maxLen; i++) {
-        if (signal?.aborted) return
-        const leftRow = leftRows[i] ?? createNullRow(leftCols)
-        const rightRow = rightRows[i] ?? createNullRow(rightCols)
+        if (++tick % YIELD_INTERVAL === 0) {
+          await yieldToEventLoop()
+          if (signal?.aborted) return
+        }
+        if (!leftResult.done && !leftCols.length) leftCols = leftResult.value.columns
+        if (!rightResult.done && !rightCols.length) rightCols = rightResult.value.columns
+        const leftRow = leftResult.done ? createNullRow(leftCols) : leftResult.value
+        const rightRow = rightResult.done ? createNullRow(rightCols) : rightResult.value
         yield mergeRows(leftRow, rightRow, leftTable, rightTable)
       }
     },
@@ -206,26 +224,59 @@ export function executePositionalJoin(plan, context) {
 export function executeHashJoin(plan, context) {
   const left = executePlan({ plan: plan.left, context })
   const right = executePlan({ plan: plan.right, context })
+  // Build the hash table on the smaller side when both sizes are known,
+  // so joining a small table against a large one buffers the small one.
+  // Swapping reorders output rows, which SQL leaves unspecified.
+  const leftSize = left.numRows ?? left.maxRows
+  const rightSize = right.numRows ?? right.maxRows
+  const swap = leftSize !== undefined && rightSize !== undefined && leftSize < rightSize
   return {
     columns: mergeColumnNames(left.columns, right.columns, plan.leftAlias, plan.rightAlias),
     async *rows() {
       const leftTable = plan.leftAlias
       const rightTable = plan.rightAlias
-      const { leftKeys, rightKeys, residual } = plan
-
-      // Buffer right rows and build hash map
-      /** @type {AsyncRow[]} */
-      const rightRows = []
-      for await (const row of right.rows()) {
-        if (context.signal?.aborted) return
-        rightRows.push(row)
+      const { residual } = plan
+      const build = swap ? left : right
+      const probe = swap ? right : left
+      const buildKeys = swap ? plan.leftKeys : plan.rightKeys
+      const probeKeys = swap ? plan.rightKeys : plan.leftKeys
+      // Which sides must also emit their unmatched rows
+      const buildOuter = plan.joinType === 'FULL' || plan.joinType === (swap ? 'LEFT' : 'RIGHT')
+      const probeOuter = plan.joinType === 'FULL' || plan.joinType === (swap ? 'RIGHT' : 'LEFT')
+
+      /**
+       * @param {AsyncRow} probeRow
+       * @param {AsyncRow} buildRow
+       * @returns {AsyncRow}
+       */
+      function merge(probeRow, buildRow) {
+        return swap
+          ? mergeRows(buildRow, probeRow, leftTable, rightTable)
+          : mergeRows(probeRow, buildRow, leftTable, rightTable)
       }
 
+      // Build phase: stream one side into the hash map. The full row list is
+      // only retained when unmatched build rows must be emitted afterwards;
+      // otherwise rows with NULL join keys are released immediately.
       /** @type {Map<string | number | bigint | boolean, AsyncRow[]>} */
       const hashMap = new Map()
-      for (const rightRow of rightRows) {
+      /** @type {AsyncRow[] | undefined} */
+      const buildRows = buildOuter ? [] : undefined
+      /** @type {string[]} */
+      let buildCols = []
+      let innerCount = 0
+      for await (const buildRow of build.rows()) {
+        if (context.signal?.aborted) return
+        if (++innerCount % YIELD_INTERVAL === 0) {
+          await yieldToEventLoop()
+          if (context.signal?.aborted) return
+        }
+        if (!buildCols.length) {
+          buildCols = buildRow.columns
+        }
+        buildRows?.push(buildRow)
         const keyValues = await Promise.all(
-          rightKeys.map(node => evaluateExpr({ node, row: rightRow, context }))
+          buildKeys.map(node => evaluateExpr({ node, row: buildRow, context }))
         )
         // SQL semantics: NULL never equals anything, so a row with any NULL
         // join key is excluded from the hash table.
@@ -236,65 +287,59 @@ export function executeHashJoin(plan, context) {
           bucket = []
           hashMap.set(key, bucket)
         }
-        bucket.push(rightRow)
+        bucket.push(buildRow)
       }
 
-      // Get column info for NULL row generation
-      const rightCols = rightRows.length ? rightRows[0].columns : []
-
       /** @type {string[] | undefined} */
-      let leftCols
+      let probeCols
       /** @type {Set<AsyncRow> | undefined} */
-      const matchedRightRows = plan.joinType === 'RIGHT' || plan.joinType === 'FULL' ? new Set() : undefined
+      const matchedBuildRows = buildOuter ? new Set() : undefined
 
-      // Probe phase: stream left rows
-      let innerCount = 0
-      for await (const leftRow of left.rows()) {
+      // Probe phase: stream the other side
+      for await (const probeRow of probe.rows()) {
         if (context.signal?.aborted) return
 
-        if (!leftCols) {
-          leftCols = leftRow.columns
+        if (!probeCols) {
+          probeCols = probeRow.columns
         }
 
         const keyValues = await Promise.all(
-          leftKeys.map(node => evaluateExpr({ node, row: leftRow, context }))
+          probeKeys.map(node => evaluateExpr({ node, row: probeRow, context }))
         )
         let matched = false
         if (!keyValues.some(v => v == null)) {
           const key = keyify(...keyValues)
           const candidates = hashMap.get(key)
           if (candidates?.length) {
-            for (const rightRow of candidates) {
+            for (const buildRow of candidates) {
               if (++innerCount % YIELD_INTERVAL === 0) {
                 await yieldToEventLoop()
                 if (context.signal?.aborted) return
               }
-              const merged = mergeRows(leftRow, rightRow, leftTable, rightTable)
+              const merged = merge(probeRow, buildRow)
               if (residual) {
                 const ok = await evaluateExpr({ node: residual, row: merged, context })
                 if (!ok) continue
               }
               matched = true
-              matchedRightRows?.add(rightRow)
+              matchedBuildRows?.add(buildRow)
               yield merged
             }
           }
         }
 
-        if (!matched && (plan.joinType === 'LEFT' || plan.joinType === 'FULL')) {
-          const nullRight = createNullRow(rightCols)
-          yield mergeRows(leftRow, nullRight, leftTable, rightTable)
+        if (!matched && probeOuter) {
+          yield merge(probeRow, createNullRow(buildCols))
         }
       }
 
       if (context.signal?.aborted) return
 
-      // Unmatched right rows for RIGHT/FULL joins
-      if (matchedRightRows) {
-        for (const rightRow of rightRows) {
-          if (!matchedRightRows.has(rightRow)) {
-            const nullLeft = createNullRow(leftCols ?? [])
-            yield mergeRows(nullLeft, rightRow, leftTable, rightTable)
+      // Unmatched build rows for outer joins on the build side
+      if (buildRows && matchedBuildRows) {
+        for (const buildRow of buildRows) {
+          if (!matchedBuildRows.has(buildRow)) {
+            yield merge(createNullRow(probeCols ?? []), buildRow)
           }
         }
       }

package/src/execute/sort.js

@@ -4,16 +4,21 @@ import { executePlan } from './execute.js'
 import { compareForTerm } from './utils.js'
 
 /**
- * @import { AsyncRow, ExecuteContext, OrderByItem, QueryResults, SqlPrimitive } from '../types.js'
+ * @import { AsyncRow, ExecuteContext, ExprNode, OrderByItem, QueryResults, SqlPrimitive } from '../types.js'
  * @import { SortNode } from '../plan/types.js'
  */
 
 const MAX_CHUNK = 256
 
 /**
+ * When exprs is set, exprs[i] is evaluated for ORDER BY term i instead of the
+ * term's own expression, so callers can pre-substitute per-entry values (for
+ * example, finalized aggregates) while keeping tie-aware term evaluation.
+ *
  * @typedef {{
  *   row: AsyncRow,
  *   rows?: AsyncRow[],
+ *   exprs?: ExprNode[],
  * }} SortEntry
  */
 
@@ -63,7 +68,7 @@ export async function sortEntriesByTerms({ entries, orderBy, context, cacheValue
         const chunk = missing.slice(start, start + chunkSize)
         const values = await Promise.all(chunk.map(idx =>
           evaluateExpr({
-            node: term.expr,
+            node: entries[idx].exprs?.[orderByIdx] ?? term.expr,
             row: entries[idx].row,
             rows: entries[idx].rows,
             context,
@@ -124,25 +129,47 @@ export async function sortEntriesByTerms({ entries, orderBy, context, cacheValue
  */
 export function executeSort(plan, context) {
   const child = executePlan({ plan: plan.child, context })
+  const { topK } = plan
+  // With a LIMIT bound pushed into the sort, keep at most this many buffered
+  // rows: periodically sort and discard everything past topK, so memory is
+  // bounded by the limit instead of the input size.
+  const bufferLimit = topK === undefined ? Infinity : Math.max(topK * 2, 1024)
   return {
     columns: child.columns,
-    numRows: child.numRows,
-    maxRows: child.maxRows,
+    numRows: topK === undefined || child.numRows === undefined
+      ? child.numRows
+      : Math.min(child.numRows, topK),
+    maxRows: topK === undefined
+      ? child.maxRows
+      : Math.min(child.maxRows ?? Infinity, topK),
     async *rows() {
-      // Buffer all rows
-      /** @type {AsyncRow[]} */
-      const rows = []
+      /** @type {SortEntry[]} */
+      let entries = []
       for await (const row of child.rows()) {
         if (context.signal?.aborted) return
-        rows.push(row)
+        entries.push({ row })
+        if (entries.length >= bufferLimit) {
+          // Sort keys are cached on the rows, so survivors of one truncation
+          // are not re-evaluated by the next
+          const sorted = await sortEntriesByTerms({
+            entries,
+            orderBy: plan.orderBy,
+            context,
+            cacheValues: true,
+          })
+          entries = sorted.slice(0, topK).map(({ row }) => ({ row }))
+        }
       }
 
-      const sortedRows = await sortEntriesByTerms({
-        entries: rows.map(row => ({ row })),
+      let sortedRows = await sortEntriesByTerms({
+        entries,
         orderBy: plan.orderBy,
         context,
         cacheValues: true,
       })
+      if (topK !== undefined && sortedRows.length > topK) {
+        sortedRows = sortedRows.slice(0, topK)
+      }
 
       // Yield sorted rows
       for (const { row } of sortedRows) {