squirreling 0.16.13 → 0.16.14

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "squirreling",
-  "version": "0.16.13",
+  "version": "0.16.14",
   "description": "Squirreling Async SQL Engine",
   "author": "Hyperparam",
   "homepage": "https://hyperparam.app",
@@ -43,7 +43,7 @@
     "@types/node": "26.6.5",
     "@vitest/coverage-v8": "5.0.5",
     "eslint": "10.13.0",
-    "eslint-plugin-jsdoc": "65.4.0",
+    "eslint-plugin-jsdoc": "65.4.1",
     "globals": "17.13.0",
     "typescript": "7.0.2",
     "vitest": "5.0.5"

package/src/execute/aggregates.js

@@ -193,6 +193,18 @@ export function executeHashAggregate(plan, context) {
  * @returns {QueryResults}
  */
 export function executeScalarAggregate(plan, context) {
+  if (context.deferScans && plan.child.type === 'Scan') {
+    const child = executeScan(plan.child, context)
+    return {
+      columns: selectColumnNames(plan.columns, child.columns),
+      numRows: plan.having ? undefined : 1,
+      maxRows: 1,
+      async *rows() {
+        context.signal?.throwIfAborted()
+        yield* executeScalarAggregate(plan, { ...context, deferScans: false }).rows()
+      },
+    }
+  }
   // Fast path: use scanColumn when available
   const columnScan = tryColumnScanAggregate(plan, context)
   if (columnScan?.rows) {

package/src/execute/dynamicFilter.js

@@ -0,0 +1,131 @@
+/**
+ * @import { ExprNode, SqlPrimitive } from '../types.js'
+ * @import { QueryPlan } from '../plan/types.js'
+ */
+
+// Bound filter memory and match the session-index pruning limit.
+const MAX_DYNAMIC_FILTER_KEYS = 256
+
+/**
+ * Bounded, per-column key sets. Once a column exceeds the cap, release it and
+ * stop collecting that column. Unsupported values disable that column too.
+ *
+ * @param {number} count
+ * @param {boolean} [hash] - preserve the hash join's serialized key equivalences
+ * @returns {{ add(values: SqlPrimitive[]): void, apply(plan: QueryPlan, keys: ExprNode[]): QueryPlan }}
+ */
+export function dynamicFilter(count, hash = false) {
+  /** @type {(Set<SqlPrimitive> | undefined)[]} */
+  const sets = Array.from({ length: count }, () => new Set())
+  let active = true
+  return {
+    /**
+     * @param {SqlPrimitive[]} values
+     * @returns {void}
+     */
+    add(values) {
+      if (!active) return
+      // Hash tuple serialization uses delimiters and serializes objects. Do
+      // not prune when those encodings can collide across column boundaries.
+      if (hash && values.some(value => typeof value === 'object' && value !== null ||
+          typeof value === 'string' && (value.includes('|') || ['[', '{', '"'].includes(value[0]) || value === 'null'))) {
+        sets.fill(undefined)
+        active = false
+        return
+      }
+      for (let i = 0; i < sets.length; i++) {
+        const set = sets[i]
+        const value = values[i]
+        if (!set || value == null) continue
+        if (!['string', 'number', 'bigint', 'boolean'].includes(typeof value) ||
+            typeof value === 'number' && !Number.isFinite(value)) {
+          sets[i] = undefined
+          continue
+        }
+        set.add(value)
+        if (hash && count > 1) {
+          if (typeof value === 'boolean') set.add(String(value))
+          if (value === 'true' || value === 'false') set.add(value === 'true')
+        }
+        if (set.size > MAX_DYNAMIC_FILTER_KEYS) sets[i] = undefined
+      }
+      active = sets.some(Boolean)
+    },
+    /**
+     * @param {QueryPlan} plan
+     * @param {ExprNode[]} keys
+     * @returns {QueryPlan}
+     */
+    apply(plan, keys) {
+      for (let i = 0; i < keys.length; i++) {
+        const set = sets[i]
+        if (set?.size) plan = pushFilter(plan, keys[i], [...set])
+      }
+      return plan
+    },
+  }
+}
+
+/**
+ * Only traverse operators that preserve the meaning of a physical column.
+ * Limits, aggregates, windows and projections are deliberate barriers.
+ *
+ * @param {QueryPlan} plan
+ * @param {ExprNode} key
+ * @param {SqlPrimitive[]} values
+ * @returns {QueryPlan}
+ */
+function pushFilter(plan, key, values) {
+  if (key?.type !== 'identifier') return plan
+  if (plan.type === 'Scan') {
+    if (key.prefix && key.prefix !== (plan.alias ?? plan.table) ||
+        plan.hints.limit !== undefined || plan.hints.offset || plan.topK) return plan
+    const position = { positionStart: key.positionStart, positionEnd: key.positionEnd }
+    /** @type {ExprNode} */
+    const filter = {
+      type: 'in valuelist', expr: key, ...position,
+      values: values.map(value => ({ type: 'literal', value, ...position })),
+    }
+    /** @type {ExprNode} */
+    const where = plan.hints.where
+      ? { type: 'binary', op: 'AND', left: plan.hints.where, right: filter, ...position }
+      : filter
+    return { ...plan, hints: { ...plan.hints, where } }
+  }
+  if (plan.type === 'Filter' || plan.type === 'Sort' && !plan.topK || plan.type === 'Distinct') {
+    return { ...plan, child: pushFilter(plan.child, key, values) }
+  }
+  if (plan.type === 'HashJoin' && key.prefix) {
+    // Restrict only preserved inputs of outer joins; restricting an optional
+    // input could turn a formerly matched row into a new null-extended row.
+    const left = plan.joinType === 'INNER' || plan.joinType === 'LEFT'
+      ? pushFilter(plan.left, key, values) : plan.left
+    const right = plan.joinType === 'INNER' || plan.joinType === 'RIGHT'
+      ? pushFilter(plan.right, key, values) : plan.right
+    return { ...plan, left, right }
+  }
+  return plan
+}
+
+/**
+ * Selectivity heuristic when exact filtered cardinalities are unavailable.
+ * Count conjuncts so a date plus a LIKE wins over a lone part-type predicate.
+ *
+ * @param {QueryPlan} plan
+ * @returns {number}
+ */
+export function filterScore(plan) {
+  if (plan.type === 'Scan') return conjuncts(plan.hints.where)
+  if (plan.type === 'Filter') return conjuncts(plan.condition) + filterScore(plan.child)
+  if (plan.type === 'HashJoin' && plan.joinType === 'INNER') return Math.max(filterScore(plan.left), filterScore(plan.right))
+  return 0
+}
+
+/**
+ * @param {ExprNode} expr
+ * @returns {number}
+ */
+function conjuncts(expr) {
+  return !expr ? 0 : expr.type === 'binary' && expr.op === 'AND'
+    ? conjuncts(expr.left) + conjuncts(expr.right) : 1
+}

package/src/execute/execute.js

@@ -1,5 +1,5 @@
 import { selectedRowCount } from '../backend/batch.js'
-import { batchesToRows } from '../backend/batchAdapters.js'
+import { batchesToRows, rowsToBatches } from '../backend/batchAdapters.js'
 import { dataSourceColumns, memorySource } from '../backend/dataSource.js'
 import { derivedAlias } from '../expression/alias.js'
 import { compileBatchExpression } from '../expression/batch.js'
@@ -318,6 +318,37 @@ export function executeScan(plan, context, existingColumnResult) {
   const { tables, signal } = context
   const table = validateTable({ ...plan, tables })
   validateScan({ ...plan, tables })
+  if (context.deferScans && !existingColumnResult) {
+    const size = computeScanRows(table.numRows, plan.hints.limit, plan.hints.offset)
+    const columns = plan.hints.columns ?? dataSourceColumns(table)
+    /** @type {QueryResults} */
+    const deferred = {
+      columns,
+      numRows: plan.hints.where ? undefined : size,
+      maxRows: size,
+      async *rows() {
+        context.signal?.throwIfAborted()
+        yield* executeScan(plan, { ...context, deferScans: false }).rows()
+      },
+    }
+    // Preserve native batch execution above a deferred input. Legacy fallback
+    // remains at the source boundary if a source declines its column path.
+    if (table.prepareScan && table.schema || table.scanColumn && plan.hints.columns?.length === 1) {
+      deferred.batches = async function* deferredBatches() {
+        context.signal?.throwIfAborted()
+        const result = executeScan(plan, { ...context, deferScans: false })
+        if (result.batches) {
+          // Prepared sources may return extra fields or a different order.
+          // Align vectors with the metadata exposed before opening the scan.
+          const indexes = columns.map(column => result.columns.indexOf(column))
+          for await (const batch of result.batches()) {
+            yield { ...batch, columns: indexes.map(index => batch.columns[index]) }
+          }
+        } else yield* rowsToBatches(result.rows(), columns, { signal })
+      }
+    }
+    return deferred
+  }
   const { where, residual } = scanFilter(plan, dataSourceColumns(table))
   const hints = { ...plan.hints, where }
   if (residual) {

package/src/execute/join.js

@@ -1,6 +1,7 @@
 import { evaluateExpr } from '../expression/evaluate.js'
 import { keyify, maxBounds } from './utils.js'
 import { executePlan } from './execute.js'
+import { dynamicFilter, filterScore } from './dynamicFilter.js'
 import { yieldToEventLoop } from './yield.js'
 
 /**
@@ -222,14 +223,18 @@ export function executePositionalJoin(plan, context) {
  * @returns {QueryResults}
  */
 export function executeHashJoin(plan, context) {
-  const left = executePlan({ plan: plan.left, context })
-  const right = executePlan({ plan: plan.right, context })
-  // Build the hash table on the smaller side when both sizes are known,
-  // so joining a small table against a large one buffers the small one.
+  const left = executePlan({ plan: plan.left, context: { ...context, deferScans: true } })
+  const right = executePlan({ plan: plan.right, context: { ...context, deferScans: true } })
+  // Prefer exact sizes, then predicate selectivity, then upper bounds.
+  // A filtered scan's table size is only an upper bound, not its cardinality.
   // Swapping reorders output rows, which SQL leaves unspecified.
   const leftSize = left.numRows ?? left.maxRows
   const rightSize = right.numRows ?? right.maxRows
-  const swap = leftSize !== undefined && rightSize !== undefined && leftSize < rightSize
+  const swap = left.numRows !== undefined && right.numRows !== undefined
+    ? leftSize < rightSize
+    : filterScore(plan.left) !== filterScore(plan.right)
+      ? filterScore(plan.left) > filterScore(plan.right)
+      : leftSize !== undefined && rightSize !== undefined && leftSize < rightSize
   return {
     columns: mergeColumnNames(left.columns, right.columns, plan.leftAlias, plan.rightAlias),
     async *rows() {
@@ -237,7 +242,7 @@ export function executeHashJoin(plan, context) {
       const rightTable = plan.rightAlias
       const { residual } = plan
       const build = swap ? left : right
-      const probe = swap ? right : left
+      let probe = swap ? right : left
       const buildKeys = swap ? plan.leftKeys : plan.rightKeys
       const probeKeys = swap ? plan.rightKeys : plan.leftKeys
       // Which sides must also emit their unmatched rows
@@ -260,6 +265,7 @@ export function executeHashJoin(plan, context) {
       // otherwise rows with NULL join keys are released immediately.
       /** @type {Map<string | number | bigint | boolean, AsyncRow[]>} */
       const hashMap = new Map()
+      const filters = dynamicFilter(buildKeys.length, true)
       /** @type {AsyncRow[] | undefined} */
       const buildRows = buildOuter ? [] : undefined
       /** @type {string[]} */
@@ -281,6 +287,7 @@ export function executeHashJoin(plan, context) {
         // SQL semantics: NULL never equals anything, so a row with any NULL
         // join key is excluded from the hash table.
         if (keyValues.some(v => v == null)) continue
+        if (!probeOuter) filters.add(keyValues)
         const key = keyify(...keyValues)
         let bucket = hashMap.get(key)
         if (!bucket) {
@@ -295,8 +302,14 @@ export function executeHashJoin(plan, context) {
       /** @type {Set<AsyncRow> | undefined} */
       const matchedBuildRows = buildOuter ? new Set() : undefined
 
+      if (!probeOuter && !hashMap.size && !buildOuter) return
+      if (!probeOuter && hashMap.size) {
+        const filtered = filters.apply(swap ? plan.right : plan.left, probeKeys)
+        probe = executePlan({ plan: filtered, context: { ...context, deferScans: true } })
+      }
+
       // Probe phase: stream the other side
-      for await (const probeRow of probe.rows()) {
+      for await (const probeRow of !probeOuter && !hashMap.size ? [] : probe.rows()) {
         context.signal?.throwIfAborted()
 
         if (!probeCols) {

package/src/execute/semiJoin.js

@@ -1,5 +1,6 @@
 import { evaluateExpr } from '../expression/evaluate.js'
 import { executePlan } from './execute.js'
+import { dynamicFilter } from './dynamicFilter.js'
 import { sqlEquals } from './utils.js'
 import { yieldToEventLoop } from './yield.js'
 
@@ -19,7 +20,7 @@ import { yieldToEventLoop } from './yield.js'
  */
 export function executeSemiJoin(plan, context) {
   if (plan.keys) return executeExistsJoin(plan, context)
-  const left = executePlan({ plan: plan.left, context })
+  const left = executePlan({ plan: plan.left, context: { ...context, deferScans: true } })
   return {
     columns: left.columns,
     maxRows: left.maxRows,
@@ -28,29 +29,30 @@ export function executeSemiJoin(plan, context) {
       const values = new Map()
       let hasNull = false
       let count = 0
-      let built = false
       let rowIndex = 0
-      for await (const row of left.rows()) {
+      const filters = dynamicFilter(1)
+      for await (const inner of executePlan({ plan: plan.right, context }).rows()) {
+        context.signal?.throwIfAborted()
+        if (++count % 4000 === 0) await yieldToEventLoop()
+        const value = await inner.cells[inner.columns[0]]()
+        if (value == null) hasNull = true
+        else {
+          const type = typeof value
+          let group = values.get(type)
+          if (!group) values.set(type, group = new Set())
+          group.add(value)
+          if (!plan.not) filters.add([value])
+        }
+      }
+      if (!plan.not && !values.size || plan.not && hasNull) return
+      const probe = executePlan({
+        plan: plan.not ? plan.left : filters.apply(plan.left, [plan.expr]),
+        context: { ...context, deferScans: true },
+      })
+      for await (const row of probe.rows()) {
         context.signal?.throwIfAborted()
         if (++rowIndex % 4000 === 0) await yieldToEventLoop()
         const value = await evaluateExpr({ node: plan.expr, row, rowIndex, context })
-        if (!built) {
-          for await (const inner of executePlan({ plan: plan.right, context }).rows()) {
-            if (++count % 4000 === 0) {
-              await yieldToEventLoop()
-              context.signal?.throwIfAborted()
-            }
-            const value = await inner.cells[inner.columns[0]]()
-            if (value == null) hasNull = true
-            else {
-              const type = typeof value
-              let group = values.get(type)
-              if (!group) values.set(type, group = new Set())
-              group.add(value)
-            }
-          }
-          built = true
-        }
         const matched = count === 0 ? false : value == null ? null : contains(values, value) ? true : hasNull ? null : false
         if (matched !== null && matched !== plan.not) yield row
       }
@@ -87,36 +89,39 @@ function contains(values, value) {
  * @returns {QueryResults}
  */
 function executeExistsJoin(plan, context) {
-  const left = executePlan({ plan: plan.left, context })
+  const left = executePlan({ plan: plan.left, context: { ...context, deferScans: true } })
   return {
     columns: left.columns,
     maxRows: left.maxRows,
     async *rows() {
       /** @type {KeyTrie} */
       const root = { groups: new Map() }
-      let built = false
       let count = 0
-      for await (const row of left.rows()) {
+      const filters = dynamicFilter(plan.keys?.length ?? 0)
+      for await (const inner of executePlan({ plan: plan.right, context }).rows()) {
         context.signal?.throwIfAborted()
         if (++count % 4000 === 0) await yieldToEventLoop()
-        if (!built) {
-          for await (const inner of executePlan({ plan: plan.right, context }).rows()) {
-            context.signal?.throwIfAborted()
-            if (++count % 4000 === 0) await yieldToEventLoop()
-            const values = await Promise.all(inner.columns.map(col => inner.cells[col]()))
-            if (values.some(value => value == null)) continue
-            let node = root
-            for (const value of values) {
-              const type = typeof value
-              let group = node.groups.get(type)
-              if (!group) node.groups.set(type, group = new Map())
-              let child = group.get(value)
-              if (!child) group.set(value, child = { groups: new Map() })
-              node = child
-            }
-          }
-          built = true
+        const values = await Promise.all(inner.columns.map(col => inner.cells[col]()))
+        if (values.some(value => value == null)) continue
+        if (!plan.not) filters.add(values)
+        let node = root
+        for (const value of values) {
+          const type = typeof value
+          let group = node.groups.get(type)
+          if (!group) node.groups.set(type, group = new Map())
+          let child = group.get(value)
+          if (!child) group.set(value, child = { groups: new Map() })
+          node = child
         }
+      }
+      if (!plan.not && !root.groups.size) return
+      const probe = executePlan({
+        plan: plan.not ? plan.left : filters.apply(plan.left, plan.keys ?? []),
+        context: { ...context, deferScans: true },
+      })
+      for await (const row of probe.rows()) {
+        context.signal?.throwIfAborted()
+        if (++count % 4000 === 0) await yieldToEventLoop()
         const values = await Promise.all((plan.keys ?? []).map(node => evaluateExpr({ node, row, context })))
         const matched = !values.some(value => value == null) && matchesKeys(root, values, 0)
         if (matched !== plan.not) yield row

package/src/types.d.ts

@@ -199,6 +199,8 @@ export interface PlanSqlOptions {
 
 // executePlan(plan, context)
 export interface ExecuteContext {
+  // Delay opening join input scans until their rows are consumed.
+  deferScans?: boolean
   tables: Record<string, AsyncDataSource>
   functions?: Record<string, UserDefinedFunction>
   signal?: AbortSignal