squirreling 0.16.0 → 0.16.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "squirreling",
-  "version": "0.16.0",
+  "version": "0.16.1",
   "description": "Squirreling Async SQL Engine",
   "author": "Hyperparam",
   "homepage": "https://hyperparam.app",
@@ -40,10 +40,10 @@
   },
   "devDependencies": {
     "@types/node": "26.2.0",
-    "@vitest/coverage-v8": "4.1.10",
+    "@vitest/coverage-v8": "4.1.11",
     "eslint": "9.39.4",
     "eslint-plugin-jsdoc": "64.2.1",
     "typescript": "7.0.2",
-    "vitest": "4.1.10"
+    "vitest": "4.1.11"
   }
 }

package/src/backend/batch.js

@@ -17,6 +17,35 @@ export function selectedRowCount(selection) {
   return selection.indices.length
 }
 
+/**
+ * Recognizes promises and thenables without relying on realm identity.
+ *
+ * @template T
+ * @param {T | PromiseLike<T>} value
+ * @returns {value is PromiseLike<T>}
+ */
+export function isPromiseLike(value) {
+  if (value === null || typeof value !== 'object' && typeof value !== 'function') return false
+  return typeof Reflect.get(value, 'then') === 'function'
+}
+
+/**
+ * Avoids a promise boundary when every column is already loaded synchronously.
+ *
+ * @param {ColumnResult[]} results
+ * @returns {ColumnVector[] | Promise<ColumnVector[]>}
+ */
+export function resolveColumnResults(results) {
+  if (results.some(isPromiseLike)) return Promise.all(results)
+  /** @type {ColumnVector[]} */
+  const vectors = []
+  for (const result of results) {
+    if (isPromiseLike(result)) throw new Error('Unexpected asynchronous column result')
+    vectors.push(result)
+  }
+  return vectors
+}
+
 /**
  * Creates base-aligned ordinal values for the rows in a selection.
  *
@@ -158,8 +187,8 @@ export function readBatchColumn({ batch, columnIndex, selection = batch.selectio
       : undefined,
   })
   const validated = validateColumnResult(result, selectedRowCount(selection))
-  if (validated instanceof Promise) {
-    const settled = validated.then(function cacheResolved(vector) {
+  if (isPromiseLike(validated)) {
+    const settled = Promise.resolve(validated).then(function cacheResolved(vector) {
       cache.pending.delete(signal)
       cache.resolved ??= vector
       return vector
@@ -215,8 +244,8 @@ function selectionIndexAt(selection, index) {
  * @returns {ColumnResult}
  */
 function validateColumnResult(result, expectedLength) {
-  if (result instanceof Promise) {
-    return result.then(function validateResolvedVector(vector) {
+  if (isPromiseLike(result)) {
+    return Promise.resolve(result).then(function validateResolvedVector(vector) {
       return validateVectorLength(vector, expectedLength)
     })
   }

package/src/backend/batchAdapters.js

@@ -1,8 +1,8 @@
-import { readBatchColumn, selectVector, selectedRowCount, valueAt } from './batch.js'
+import { readBatchColumn, resolveColumnResults, selectVector, selectedRowCount, valueAt } from './batch.js'
 import { yieldToEventLoop } from '../execute/yield.js'
 
 /**
- * @import { AsyncBatch, AsyncCells, AsyncRow, ColumnResult, ColumnVector, RowsToBatchesOptions, SqlPrimitive } from '../types.js'
+ * @import { AsyncBatch, AsyncCells, AsyncRow, RowsToBatchesOptions, SqlPrimitive } from '../types.js'
  */
 
 const DEFAULT_BATCH_ROWS = 1024
@@ -21,24 +21,19 @@ export async function* rowsToBatches(rows, columnNames, options) {
   if (!Number.isInteger(batchRows) || batchRows <= 0) {
     throw new RangeError(`batchRows must be a positive integer, got ${batchRows}`)
   }
-  let values = makeValueBuffers(columnNames.length)
-  let rowCount = 0
+  /** @type {AsyncRow[]} */
+  let bufferedRows = []
 
   for await (const row of rows) {
     options?.signal?.throwIfAborted()
-    for (let columnIndex = 0; columnIndex < columnNames.length; columnIndex++) {
-      const name = columnNames[columnIndex]
-      values[columnIndex].push(await readRowCell(row, name))
-    }
-    rowCount++
-    if (rowCount === batchRows) {
-      yield loadedBatch(values, rowCount)
-      values = makeValueBuffers(columnNames.length)
-      rowCount = 0
+    bufferedRows.push(row)
+    if (bufferedRows.length === batchRows) {
+      yield await materializeRows(bufferedRows, columnNames)
+      bufferedRows = []
     }
   }
 
-  if (rowCount > 0) yield loadedBatch(values, rowCount)
+  if (bufferedRows.length > 0) yield await materializeRows(bufferedRows, columnNames)
   options?.signal?.throwIfAborted()
 }
 
@@ -100,7 +95,7 @@ export async function collectBatches(batches, names, signal) {
     const results = batch.columns.map(function readColumn(_column, columnIndex) {
       return readBatchColumn({ batch, columnIndex, signal })
     })
-    const vectors = await resolveColumns(results)
+    const vectors = await resolveColumnResults(results)
     const rowCount = selectedRowCount(batch.selection)
     for (let rowIndex = 0; rowIndex < rowCount; rowIndex++) {
       if (signal && rowIndex % 4000 === 0) {
@@ -133,6 +128,27 @@ async function readRowCell(row, name) {
   return await cell()
 }
 
+/**
+ * @param {AsyncRow[]} rows
+ * @param {string[]} columnNames
+ * @returns {Promise<AsyncBatch>}
+ */
+async function materializeRows(rows, columnNames) {
+  const values = makeValueBuffers(columnNames.length)
+  /** @type {Promise<void>[]} */
+  const pendingReads = []
+  for (let rowIndex = 0; rowIndex < rows.length; rowIndex++) {
+    for (let columnIndex = 0; columnIndex < columnNames.length; columnIndex++) {
+      const name = columnNames[columnIndex]
+      pendingReads.push(readRowCell(rows[rowIndex], name).then(function storeValue(value) {
+        values[columnIndex][rowIndex] = value
+      }))
+    }
+  }
+  await Promise.all(pendingReads)
+  return loadedBatch(values, rows.length)
+}
+
 /**
  * @param {number} count
  * @returns {SqlPrimitive[][]}
@@ -157,22 +173,3 @@ function loadedBatch(values, rowCount) {
     }),
   }
 }
-
-/**
- * Avoids a promise boundary when every column is already loaded synchronously.
- *
- * @param {ColumnResult[]} results
- * @returns {ColumnVector[] | Promise<ColumnVector[]>}
- */
-function resolveColumns(results) {
-  if (results.some(function isPromise(result) { return result instanceof Promise })) {
-    return Promise.all(results)
-  }
-  /** @type {ColumnVector[]} */
-  const vectors = []
-  for (const result of results) {
-    if (result instanceof Promise) throw new Error('Unexpected asynchronous column result')
-    vectors.push(result)
-  }
-  return vectors
-}

package/src/execute/batches.js

@@ -1,10 +1,10 @@
-import { readBatchColumn, selectBatch, selectedRowCount, selectionOrdinals, valueAt } from '../backend/batch.js'
+import { isPromiseLike, readBatchColumn, resolveColumnResults, selectBatch, selectedRowCount, selectionOrdinals, valueAt } from '../backend/batch.js'
 import { keyify } from './utils.js'
 import { yieldToEventLoop } from './yield.js'
 
 /**
  * @import { BatchProjection, CompiledBatchExpression } from '../internalTypes.js'
- * @import { AsyncBatch, BatchColumn, ColumnResult, ColumnVector, ReadColumn, RowSelection, SqlPrimitive } from '../types.js'
+ * @import { AsyncBatch, BatchColumn, ColumnVector, ReadColumn, RowSelection, SqlPrimitive } from '../types.js'
  */
 
 const INITIAL_FILTER_WINDOW_ROWS = 256
@@ -95,7 +95,7 @@ export async function* filterBatches(batches, expression, signal, targetRows) {
         signal,
         rowOffset: rowOffset + start,
       })
-      const predicate = result instanceof Promise ? await result : result
+      const predicate = isPromiseLike(result) ? await result : result
       const { selection, selectedCount } = predicateSelection(predicate, windowRowCount)
       start = end
       matchedRows += selectedCount
@@ -138,8 +138,8 @@ export async function* distinctBatches(batches, signal) {
     const results = batch.columns.map(function readColumn(_column, columnIndex) {
       return readBatchColumn({ batch, columnIndex, signal })
     })
-    const resolved = resolveVectors(results)
-    const vectors = resolved instanceof Promise ? await resolved : resolved
+    const resolved = resolveColumnResults(results)
+    const vectors = isPromiseLike(resolved) ? await resolved : resolved
     const rowCount = selectedRowCount(batch.selection)
     const indices = new Uint32Array(rowCount)
     /** @type {SqlPrimitive[]} */
@@ -255,20 +255,3 @@ function predicateSelection(predicate, rowCount) {
     selectedCount,
   }
 }
-
-/**
- * @param {ColumnResult[]} results
- * @returns {ColumnVector[] | Promise<ColumnVector[]>}
- */
-function resolveVectors(results) {
-  if (results.some(function isPromise(result) { return result instanceof Promise })) {
-    return Promise.all(results)
-  }
-  /** @type {ColumnVector[]} */
-  const vectors = []
-  for (const result of results) {
-    if (result instanceof Promise) throw new Error('Unexpected asynchronous column result')
-    vectors.push(result)
-  }
-  return vectors
-}

package/src/execute/execute.js

@@ -12,6 +12,7 @@ import { executeHashAggregate, executeScalarAggregate } from './aggregates.js'
 import { batchResult } from './batchResults.js'
 import { distinctBatches, filterBatches, limitBatches, projectExpressionBatches } from './batches.js'
 import { executeHashJoin, executeNestedLoopJoin, executePositionalJoin } from './join.js'
+import { referencesRowScope } from './rowScope.js'
 import { normalizeScanColumnResult } from './scanColumn.js'
 import { executeSort } from './sort.js'
 import { addBounds, minBounds, stableRowKey } from './utils.js'
@@ -720,26 +721,6 @@ function compileUnscopedBatchExpression(expression, columns, context) {
     : compileBatchExpression(expression, columns)
 }
 
-/**
- * Returns whether an expression reads a qualified identifier from row scope.
- *
- * @param {ExprNode} expression
- * @param {string[]} columns
- * @param {ExecuteContext} context
- * @returns {boolean}
- */
-function referencesRowScope(expression, columns, context) {
-  /** @type {IdentifierNode[]} */
-  const identifiers = []
-  collectColumnsFromExpr(expression, identifiers)
-  return identifiers.some(function scopedIdentifier(identifier) {
-    return Boolean(identifier.prefix && (
-      context.outerAliases?.has(identifier.prefix) ||
-      context.scope?.includes(identifier.prefix) && columns.includes(identifier.prefix)
-    ))
-  })
-}
-
 /**
  * Executes a filter operation (WHERE clause)
  *

package/src/execute/rowScope.js

@@ -0,0 +1,27 @@
+import { collectColumnsFromExpr } from '../plan/columns.js'
+
+/**
+ * @import { ExecuteContext, ExprNode, IdentifierNode } from '../types.js'
+ */
+
+/**
+ * Returns whether an expression reads a qualified identifier from row scope.
+ * A current-scope prefix is ambiguous only when it is also a child column and
+ * could therefore mean struct-field access.
+ *
+ * @param {ExprNode} expression
+ * @param {readonly string[]} columns
+ * @param {ExecuteContext} context
+ * @returns {boolean}
+ */
+export function referencesRowScope(expression, columns, context) {
+  /** @type {IdentifierNode[]} */
+  const identifiers = []
+  collectColumnsFromExpr(expression, identifiers)
+  return identifiers.some(function scopedIdentifier(identifier) {
+    return Boolean(identifier.prefix && (
+      context.outerAliases?.has(identifier.prefix) ||
+      context.scope?.includes(identifier.prefix) && columns.includes(identifier.prefix)
+    ))
+  })
+}

package/src/execute/streamingAggregate.js

@@ -5,6 +5,7 @@ import { evaluateAll, evaluateExpr } from '../expression/evaluate.js'
 import { collectColumnsFromExpr } from '../plan/columns.js'
 import { isAggregateFunc } from '../validation/functions.js'
 import { finalizeAccumulator, newAccumulator, updateAccumulator } from './accumulator.js'
+import { referencesRowScope } from './rowScope.js'
 import { sortEntriesByTerms } from './sort.js'
 import { keyify } from './utils.js'
 import { yieldToEventLoop } from './yield.js'
@@ -437,7 +438,7 @@ function compileBatchAggregateInputs(groupBy, specs, columns, context) {
   /** @type {CompiledBatchExpression[]} */
   const keys = []
   for (const expression of groupBy) {
-    if (referencesRowScope(expression, context)) return undefined
+    if (referencesRowScope(expression, columns, context)) return undefined
     const key = compileBatchExpression(expression, columns)
     if (!key) return undefined
     keys.push(key)
@@ -449,8 +450,8 @@ function compileBatchAggregateInputs(groupBy, specs, columns, context) {
   const args = []
   for (const spec of specs) {
     if (spec.node.filter && !spec.star) return undefined
-    if (spec.node.filter && referencesRowScope(spec.node.filter, context)) return undefined
-    if (!spec.star && referencesRowScope(spec.node.args[0], context)) return undefined
+    if (spec.node.filter && referencesRowScope(spec.node.filter, columns, context)) return undefined
+    if (!spec.star && referencesRowScope(spec.node.args[0], columns, context)) return undefined
     const filter = spec.node.filter
       ? compileBatchExpression(spec.node.filter, columns)
       : undefined
@@ -468,25 +469,6 @@ function compileBatchAggregateInputs(groupBy, specs, columns, context) {
   }
 }
 
-/**
- * Returns whether an expression reads a qualified identifier whose table
- * scope the batch compiler cannot distinguish from struct-field access.
- *
- * @param {ExprNode} expression
- * @param {ExecuteContext} context
- * @returns {boolean}
- */
-function referencesRowScope(expression, context) {
-  /** @type {IdentifierNode[]} */
-  const identifiers = []
-  collectColumnsFromExpr(expression, identifiers)
-  return identifiers.some(function isScopedReference(identifier) {
-    return Boolean(identifier.prefix && (
-      context.scope?.includes(identifier.prefix) || context.outerAliases?.has(identifier.prefix)
-    ))
-  })
-}
-
 /**
  * Resolves one set of compiled expressions against a batch.
  *

package/src/expression/batch.js

@@ -1,4 +1,4 @@
-import { composeSelections, readBatchColumn, selectVector, selectedRowCount, valueAt } from '../backend/batch.js'
+import { composeSelections, isPromiseLike, readBatchColumn, resolveColumnResults, selectVector, selectedRowCount, valueAt } from '../backend/batch.js'
 import { isPlainObject, sqlEquals } from '../execute/utils.js'
 import { yieldToEventLoop } from '../execute/yield.js'
 import { isStringFunc } from '../validation/functions.js'
@@ -126,8 +126,8 @@ function compileKernelEvaluator(node, columns) {
       const results = dependencies.map(function readDependency(columnIndex) {
         return readBatchColumn({ batch, columnIndex, selection, signal })
       })
-      const vectors = resolveVectors(results)
-      if (vectors instanceof Promise) {
+      const vectors = resolveColumnResults(results)
+      if (isPromiseLike(vectors)) {
         return vectors.then(function evaluateResolved(resolved) {
           signal?.throwIfAborted()
           return evaluateKernel(kernel, resolved, selection, signal, rowOffset, rowOrdinals)
@@ -640,20 +640,3 @@ function readsIdentifier(node) {
   if (node.type === 'function') return node.args.some(readsIdentifier)
   return false
 }
-
-/**
- * @param {ColumnResult[]} results
- * @returns {ColumnVector[] | Promise<ColumnVector[]>}
- */
-function resolveVectors(results) {
-  if (results.some(function isPromise(result) { return result instanceof Promise })) {
-    return Promise.all(results)
-  }
-  /** @type {ColumnVector[]} */
-  const vectors = []
-  for (const result of results) {
-    if (result instanceof Promise) throw new Error('Unexpected asynchronous column result')
-    vectors.push(result)
-  }
-  return vectors
-}