squirreling 0.7.9 → 0.8.0

package/src/execute/join.js

@@ -1,197 +1,218 @@
+import { evaluateExpr } from '../expression/evaluate.js'
 import { missingClauseError } from '../parseErrors.js'
-import { evaluateExpr } from './expression.js'
-import { resolveTableSource } from './tableSource.js'
 import { stringify } from './utils.js'
+import { executePlan } from './execute.js'
 
 /**
- * @import { AsyncRow, AsyncDataSource, JoinClause, ExprNode, AsyncCells, UserDefinedFunction, WithClause } from '../types.js'
+ * @import { AsyncCells, AsyncRow } from '../types.js'
+ * @import { ExecuteContext, HashJoinNode, NestedLoopJoinNode, PositionalJoinNode } from '../plan/types.js'
  */
 
 /**
- * Executes JOIN operations against a base data source
+ * Executes a nested loop join operation
  *
- * @param {Object} options
- * @param {AsyncDataSource} options.leftSource - the left side of the join (FROM table)
- * @param {JoinClause[]} options.joins - array of join clauses to execute
- * @param {string} options.leftTable - name of the left table (for column prefixing)
- * @param {Record<string, AsyncDataSource>} options.tables - all available tables
- * @param {WithClause} [options.withClause] - WITH clause containing CTE definitions
- * @param {Record<string, UserDefinedFunction>} [options.functions]
- * @param {Function} [options.executeSelectFn] - function to execute SELECT for CTEs (passed to avoid circular dep)
- * @param {AbortSignal} [options.signal]
- * @returns {Promise<AsyncDataSource>} data source yielding joined rows
+ * @param {NestedLoopJoinNode} plan
+ * @param {ExecuteContext} context
+ * @yields {AsyncRow}
  */
-export async function executeJoins({ leftSource, joins, leftTable, tables, withClause, functions, executeSelectFn, signal }) {
-  let currentLeftTable = leftTable
-
-  // Single join optimization: stream left rows without buffering
-  if (joins.length === 1) {
-    const join = joins[0]
-    const rightSource = resolveTableSource(join.table, tables, withClause, executeSelectFn, functions, signal)
-
-    // Buffer right rows for hash index (required for hash join)
-    /** @type {AsyncRow[]} */
-    const rightRows = []
-    for await (const row of rightSource.scan({})) {
-      rightRows.push(row)
-    }
+export async function* executeNestedLoopJoin(plan, context) {
+  const { tables, functions, signal } = context
+  const leftTable = plan.leftAlias
+  const rightTable = plan.rightAlias
 
-    // Use alias for column prefixing if present
-    const rightTable = join.alias ?? join.table
-
-    // Return streaming data source - left rows stream through without buffering
-    return {
-      async *scan(options) {
-        const { signal } = options
-        if (join.joinType === 'POSITIONAL') {
-          yield* positionalJoin({
-            leftRows: leftSource.scan(options),
-            rightRows,
-            leftTable: currentLeftTable,
-            rightTable,
-            signal,
-          })
-        } else {
-          yield* hashJoin({
-            leftRows: leftSource.scan(options), // Stream directly, not buffered
-            rightRows,
-            join,
-            leftTable: currentLeftTable,
-            rightTable,
-            tables,
-            functions,
-            signal,
-          })
-        }
-      },
-    }
+  if (!plan.condition) {
+    throw missingClauseError({
+      missing: 'ON condition',
+      context: 'JOIN',
+    })
   }
 
-  // Multiple joins: buffer intermediate results, stream final join
+  // Buffer right rows
   /** @type {AsyncRow[]} */
-  let leftRows = []
-  for await (const row of leftSource.scan({})) {
-    leftRows.push(row)
+  const rightRows = []
+  for await (const row of executePlan(plan.right, context)) {
+    if (signal?.aborted) return
+    rightRows.push(row)
   }
 
-  // Process all but the last join, buffering intermediate results
-  for (let i = 0; i < joins.length - 1; i++) {
-    const join = joins[i]
-    const rightSource = resolveTableSource(join.table, tables, withClause, executeSelectFn, functions, signal)
+  const rightCols = rightRows.length ? rightRows[0].columns : []
+  const rightPrefixedCols = prefixColumns(rightCols, rightTable)
+
+  /** @type {string[] | null} */
+  let leftPrefixedCols = null
+  /** @type {Set<AsyncRow> | null} */
+  const matchedRightRows = plan.joinType === 'RIGHT' || plan.joinType === 'FULL' ? new Set() : null
+
+  for await (const leftRow of executePlan(plan.left, context)) {
+    if (signal?.aborted) break
 
-    /** @type {AsyncRow[]} */
-    const rightRows = []
-    for await (const row of rightSource.scan({})) {
-      rightRows.push(row)
+    if (!leftPrefixedCols) {
+      leftPrefixedCols = prefixColumns(leftRow.columns, leftTable)
     }
 
-    // Use alias for column prefixing if present
-    const rightTable = join.alias ?? join.table
-
-    // Collect intermediate results into array for next join
-    /** @type {AsyncRow[]} */
-    const newLeftRows = []
-    const joined = join.joinType === 'POSITIONAL'
-      ? positionalJoin({
-        leftRows,
-        rightRows,
-        leftTable: currentLeftTable,
-        rightTable,
-      })
-      : hashJoin({
-        leftRows,
-        rightRows,
-        join,
-        leftTable: currentLeftTable,
-        rightTable,
+    let hasMatch = false
+
+    for (const rightRow of rightRows) {
+      const tempMerged = mergeRows(leftRow, rightRow, leftTable, rightTable)
+      const matches = await evaluateExpr({
+        node: plan.condition,
+        row: tempMerged,
         tables,
         functions,
+        signal,
       })
-    for await (const row of joined) {
-      newLeftRows.push(row)
+
+      if (matches) {
+        hasMatch = true
+        if (matchedRightRows) matchedRightRows.add(rightRow)
+        yield tempMerged
+      }
+    }
+
+    if (!hasMatch && (plan.joinType === 'LEFT' || plan.joinType === 'FULL')) {
+      const nullRight = createNullRow(rightPrefixedCols)
+      yield mergeRows(leftRow, nullRight, leftTable, rightTable)
     }
-    leftRows = newLeftRows
+  }
 
-    // After join, the "left" table for the next join includes all joined tables
-    currentLeftTable = `${currentLeftTable}_${rightTable}`
+  // Unmatched right rows for RIGHT/FULL joins
+  if (matchedRightRows) {
+    for (const rightRow of rightRows) {
+      if (!matchedRightRows.has(rightRow)) {
+        const nullLeft = createNullRow(leftPrefixedCols || [])
+        yield mergeRows(nullLeft, rightRow, leftTable, rightTable)
+      }
+    }
   }
+}
 
-  // Final join: stream the results
-  const join = joins[joins.length - 1]
-  const rightSource = resolveTableSource(join.table, tables, withClause, executeSelectFn, functions, signal)
+/**
+ * Executes a positional join operation
+ *
+ * @param {PositionalJoinNode} plan
+ * @param {ExecuteContext} context
+ * @yields {AsyncRow}
+ */
+export async function* executePositionalJoin(plan, context) {
+  const { signal } = context
+  const leftTable = plan.leftAlias
+  const rightTable = plan.rightAlias
+
+  // Buffer both sides (required for positional join)
+  /** @type {AsyncRow[]} */
+  const leftRows = []
+  for await (const row of executePlan(plan.left, context)) {
+    if (signal?.aborted) return
+    leftRows.push(row)
+  }
 
   /** @type {AsyncRow[]} */
   const rightRows = []
-  for await (const row of rightSource.scan({})) {
+  for await (const row of executePlan(plan.right, context)) {
+    if (signal?.aborted) return
     rightRows.push(row)
   }
 
-  // Use alias for column prefixing if present
-  const rightTable = join.alias ?? join.table
-
-  return {
-    async *scan(options) {
-      const { signal } = options
-      if (join.joinType === 'POSITIONAL') {
-        yield* positionalJoin({
-          leftRows,
-          rightRows,
-          leftTable: currentLeftTable,
-          rightTable,
-          signal,
-        })
-      } else {
-        yield* hashJoin({
-          leftRows,
-          rightRows,
-          join,
-          leftTable: currentLeftTable,
-          rightTable,
-          tables,
-          functions,
-          signal,
-        })
-      }
-    },
+  const maxLen = Math.max(leftRows.length, rightRows.length)
+  const leftCols = leftRows[0]?.columns ?? []
+  const rightCols = rightRows[0]?.columns ?? []
+  const leftPrefixedCols = prefixColumns(leftCols, leftTable)
+  const rightPrefixedCols = prefixColumns(rightCols, rightTable)
+
+  for (let i = 0; i < maxLen; i++) {
+    if (signal?.aborted) return
+    const leftRow = leftRows[i] ?? createNullRow(leftPrefixedCols)
+    const rightRow = rightRows[i] ?? createNullRow(rightPrefixedCols)
+    yield mergeRows(leftRow, rightRow, leftTable, rightTable)
   }
 }
 
 /**
- * Checks if an expression references a specific table.
- * Returns true if the expression is an identifier prefixed with the table name.
+ * Executes a hash join operation
  *
- * @param {ExprNode} expr
- * @param {string} tableName
- * @returns {boolean}
+ * @param {HashJoinNode} plan
+ * @param {ExecuteContext} context
+ * @yields {AsyncRow}
  */
-function exprReferencesTable(expr, tableName) {
-  return expr.type === 'identifier' && expr.name.startsWith(`${tableName}.`)
-}
+export async function* executeHashJoin(plan, context) {
+  const { tables, functions, signal } = context
+  const leftTable = plan.leftAlias
+  const rightTable = plan.rightAlias
 
-/**
- * Extracts the join key expressions from an ON condition.
- * Handles both `left.col = right.col` and `right.col = left.col` orderings.
- *
- * @param {ExprNode} onCondition
- * @param {string} leftTable
- * @param {string} rightTable
- * @returns {{ leftKey: ExprNode, rightKey: ExprNode } | undefined}
- */
-function extractJoinKeys(onCondition, leftTable, rightTable) {
-  if (onCondition.type === 'binary' && onCondition.op === '=') {
-    const { left, right } = onCondition
+  // Buffer right rows and build hash map
+  /** @type {AsyncRow[]} */
+  const rightRows = []
+  for await (const row of executePlan(plan.right, context)) {
+    if (signal?.aborted) return
+    rightRows.push(row)
+  }
 
-    // Check if keys are swapped (right table referenced in left position)
-    const leftRefsRight = exprReferencesTable(left, rightTable)
-    const rightRefsLeft = exprReferencesTable(right, leftTable)
+  /** @type {Map<string, AsyncRow[]>} */
+  const hashMap = new Map()
+  for (const rightRow of rightRows) {
+    const keyValue = await evaluateExpr({
+      node: plan.rightKey,
+      row: rightRow,
+      tables,
+      functions,
+      signal,
+    })
+    if (keyValue == null) continue
+    const keyStr = stringify(keyValue)
+    let bucket = hashMap.get(keyStr)
+    if (!bucket) {
+      bucket = []
+      hashMap.set(keyStr, bucket)
+    }
+    bucket.push(rightRow)
+  }
 
-    if (leftRefsRight && rightRefsLeft) {
-      // Keys are swapped, return them in correct order
-      return { leftKey: right, rightKey: left }
+  // Get column info for NULL row generation
+  const rightCols = rightRows.length ? rightRows[0].columns : []
+  const rightPrefixedCols = prefixColumns(rightCols, rightTable)
+
+  /** @type {string[] | null} */
+  let leftPrefixedCols = null
+  /** @type {Set<AsyncRow> | null} */
+  const matchedRightRows = plan.joinType === 'RIGHT' || plan.joinType === 'FULL' ? new Set() : null
+
+  // Probe phase: stream left rows
+  for await (const leftRow of executePlan(plan.left, context)) {
+    if (signal?.aborted) break
+
+    if (!leftPrefixedCols) {
+      leftPrefixedCols = prefixColumns(leftRow.columns, leftTable)
     }
 
-    // Default: assume left operand is for left table
-    return { leftKey: left, rightKey: right }
+    const keyValue = await evaluateExpr({
+      node: plan.leftKey,
+      row: leftRow,
+      tables,
+      functions,
+      signal,
+    })
+    const keyStr = stringify(keyValue)
+    const matchingRightRows = hashMap.get(keyStr)
+
+    if (matchingRightRows?.length) {
+      for (const rightRow of matchingRightRows) {
+        if (matchedRightRows) matchedRightRows.add(rightRow)
+        yield mergeRows(leftRow, rightRow, leftTable, rightTable)
+      }
+    } else if (plan.joinType === 'LEFT' || plan.joinType === 'FULL') {
+      const nullRight = createNullRow(rightPrefixedCols)
+      yield mergeRows(leftRow, nullRight, leftTable, rightTable)
+    }
+  }
+
+  // Unmatched right rows for RIGHT/FULL joins
+  if (matchedRightRows) {
+    for (const rightRow of rightRows) {
+      if (!matchedRightRows.has(rightRow)) {
+        const nullLeft = createNullRow(leftPrefixedCols || [])
+        yield mergeRows(nullLeft, rightRow, leftTable, rightTable)
+      }
+    }
   }
 }
 
@@ -252,187 +273,12 @@ function mergeRows(leftRow, rightRow, leftTable, rightTable) {
 }
 
 /**
- * Performs a positional join between left and right row sets.
- * Matches rows by their index position (row 0 with row 0, row 1 with row 1, etc.).
- * When tables have different lengths, the shorter table is padded with NULLs.
+ * Prefixes column names with table alias, keeping already-prefixed columns as-is
  *
- * @param {Object} params
- * @param {AsyncIterable<AsyncRow>|AsyncRow[]} params.leftRows - rows from left table
- * @param {AsyncRow[]} params.rightRows - rows from right table (must be buffered)
- * @param {string} params.leftTable - name of left table (for column prefixing)
- * @param {string} params.rightTable - name of right table (for column prefixing, may be alias)
- * @param {AbortSignal} [params.signal] - abort signal for cancellation
- * @yields {AsyncRow} joined rows
+ * @param {string[]} cols
+ * @param {string} table
+ * @returns {string[]}
  */
-async function* positionalJoin({ leftRows, rightRows, leftTable, rightTable, signal }) {
-  // Buffer left rows if streaming
-  /** @type {AsyncRow[]} */
-  const leftArr = []
-  for await (const row of leftRows) {
-    if (signal?.aborted) return
-    leftArr.push(row)
-  }
-
-  const maxLen = Math.max(leftArr.length, rightRows.length)
-
-  // Get column info for NULL row creation
-  const leftCols = leftArr[0]?.columns ?? []
-  const rightCols = rightRows[0]?.columns ?? []
-  const leftPrefixedCols = leftCols.flatMap(col =>
-    col.includes('.') ? [col] : [`${leftTable}.${col}`, col]
-  )
-  const rightPrefixedCols = rightCols.flatMap(col =>
-    col.includes('.') ? [col] : [`${rightTable}.${col}`, col]
-  )
-
-  for (let i = 0; i < maxLen; i++) {
-    if (signal?.aborted) return
-    const leftRow = leftArr[i] ?? createNullRow(leftPrefixedCols)
-    const rightRow = rightRows[i] ?? createNullRow(rightPrefixedCols)
-    yield mergeRows(leftRow, rightRow, leftTable, rightTable)
-  }
-}
-
-/**
- * Performs a hash join between left and right row sets (streaming).
- * Yields rows as they are found instead of buffering all results.
- *
- * @param {Object} params
- * @param {AsyncIterable<AsyncRow>|AsyncRow[]} params.leftRows - rows from left table (can stream)
- * @param {AsyncRow[]} params.rightRows - rows from right table (must be buffered for hash index)
- * @param {JoinClause} params.join - join specification
- * @param {string} params.leftTable - name of left table (for column prefixing)
- * @param {string} params.rightTable - name of right table (for column prefixing, may be alias)
- * @param {Record<string, AsyncDataSource>} params.tables - all tables for expression evaluation
- * @param {Record<string, UserDefinedFunction>} [params.functions]
- * @param {AbortSignal} [params.signal] - abort signal for cancellation
- * @yields {AsyncRow} joined rows
- */
-async function* hashJoin({ leftRows, rightRows, join, leftTable, rightTable, tables, functions, signal }) {
-  const { joinType, on: onCondition } = join
-
-  if (!onCondition) {
-    throw missingClauseError({
-      missing: 'ON condition',
-      context: 'JOIN',
-    })
-  }
-
-  const keys = extractJoinKeys(onCondition, leftTable, rightTable)
-
-  // Get column names for NULL row generation (right side is always buffered)
-  const rightCols = rightRows.length ? rightRows[0].columns : []
-  const rightPrefixedCols = rightCols.flatMap(col =>
-    col.includes('.') ? [col] : [`${rightTable}.${col}`, col]
-  )
-
-  // Track left column info - captured from first row during iteration
-  /** @type {string[]|null} */
-  let leftPrefixedCols = null
-
-  if (keys) {
-    // Hash join: build hash map on right table
-    /** @type {Map<string, AsyncRow[]>} */
-    const hashMap = new Map()
-
-    // BUILD PHASE: Index right rows by join key
-    // Skip null keys - SQL semantics: NULL != NULL
-    for (const rightRow of rightRows) {
-      const keyValue = await evaluateExpr({ node: keys.rightKey, row: rightRow, tables, functions })
-      if (keyValue == null) continue // NULL keys never match
-      const keyStr = stringify(keyValue)
-      let bucket = hashMap.get(keyStr)
-      if (!bucket) {
-        bucket = []
-        hashMap.set(keyStr, bucket)
-      }
-      bucket.push(rightRow)
-    }
-
-    // Track which right rows matched (only needed for RIGHT/FULL joins)
-    /** @type {Set<AsyncRow>|null} */
-    const matchedRightRows = joinType === 'RIGHT' || joinType === 'FULL' ? new Set() : null
-
-    // PROBE PHASE: Stream through left rows, yield matches immediately
-    for await (const leftRow of leftRows) {
-      if (signal?.aborted) break
-      // Capture left column info from first row (for NULL row generation)
-      if (!leftPrefixedCols) {
-        leftPrefixedCols = leftRow.columns.flatMap(col =>
-          col.includes('.') ? [col] : [`${leftTable}.${col}`, col]
-        )
-      }
-
-      const keyValue = await evaluateExpr({ node: keys.leftKey, row: leftRow, tables, functions })
-      const keyStr = stringify(keyValue)
-
-      const matchingRightRows = hashMap.get(keyStr)
-
-      if (matchingRightRows && matchingRightRows.length > 0) {
-        for (const rightRow of matchingRightRows) {
-          if (matchedRightRows) matchedRightRows.add(rightRow)
-          yield mergeRows(leftRow, rightRow, leftTable, rightTable)
-        }
-      } else if (joinType === 'LEFT' || joinType === 'FULL') {
-        const nullRight = createNullRow(rightPrefixedCols)
-        yield mergeRows(leftRow, nullRight, leftTable, rightTable)
-      }
-      // INNER join with no match: don't yield anything
-    }
-
-    // UNMATCHED PHASE: Handle unmatched right rows for RIGHT/FULL joins
-    if (matchedRightRows) {
-      for (const rightRow of rightRows) {
-        if (!matchedRightRows.has(rightRow)) {
-          // Use empty array if left table was empty (no rows to derive columns from)
-          const nullLeft = createNullRow(leftPrefixedCols || [])
-          yield mergeRows(nullLeft, rightRow, leftTable, rightTable)
-        }
-      }
-    }
-  } else {
-    // Fallback to nested loop for complex ON conditions
-    // Left rows stream through, right rows are iterated for each left row
-    /** @type {Set<AsyncRow>|null} */
-    const matchedRightRows = joinType === 'RIGHT' || joinType === 'FULL' ? new Set() : null
-
-    for await (const leftRow of leftRows) {
-      if (signal?.aborted) break
-      // Capture left column info from first row (for NULL row generation)
-      if (!leftPrefixedCols) {
-        leftPrefixedCols = leftRow.columns.flatMap(col =>
-          col.includes('.') ? [col] : [`${leftTable}.${col}`, col]
-        )
-      }
-
-      let hasMatch = false
-
-      for (const rightRow of rightRows) {
-        const tempMerged = mergeRows(leftRow, rightRow, leftTable, rightTable)
-        const matches = await evaluateExpr({ node: onCondition, row: tempMerged, tables, functions })
-
-        if (matches) {
-          hasMatch = true
-          if (matchedRightRows) matchedRightRows.add(rightRow)
-          yield tempMerged
-        }
-      }
-
-      if (!hasMatch && (joinType === 'LEFT' || joinType === 'FULL')) {
-        const nullRight = createNullRow(rightPrefixedCols)
-        yield mergeRows(leftRow, nullRight, leftTable, rightTable)
-      }
-    }
-
-    // Handle unmatched right rows for RIGHT/FULL joins
-    if (matchedRightRows) {
-      for (const rightRow of rightRows) {
-        if (!matchedRightRows.has(rightRow)) {
-          // Use empty array if left table was empty (no rows to derive columns from)
-          const nullLeft = createNullRow(leftPrefixedCols || [])
-          yield mergeRows(nullLeft, rightRow, leftTable, rightTable)
-        }
-      }
-    }
-  }
+function prefixColumns(cols, table) {
+  return cols.flatMap(col => col.includes('.') ? [col] : [`${table}.${col}`, col])
 }

package/src/execute/sort.js

@@ -0,0 +1,99 @@
+import { evaluateExpr } from '../expression/evaluate.js'
+import { executePlan } from './execute.js'
+import { compareForTerm } from './utils.js'
+
+/**
+ * @import { AsyncRow, SqlPrimitive } from '../types.js'
+ * @import { ExecuteContext, SortNode } from '../plan/types.js'
+ */
+
+/**
+ * Executes a sort operation (ORDER BY)
+ *
+ * @param {SortNode} plan
+ * @param {ExecuteContext} context
+ * @yields {AsyncRow}
+ */
+export async function* executeSort(plan, context) {
+  const { tables, functions, signal } = context
+
+  // Buffer all rows
+  /** @type {AsyncRow[]} */
+  const rows = []
+  for await (const row of executePlan(plan.child, context)) {
+    if (signal?.aborted) return
+    rows.push(row)
+  }
+
+  if (rows.length === 0) return
+
+  // Multi-pass lazy sorting
+  /** @type {(SqlPrimitive | undefined)[][]} */
+  const evaluatedValues = rows.map(() => Array(plan.orderBy.length))
+
+  /** @type {number[][]} */
+  let groups = [rows.map((_, i) => i)]
+
+  for (let orderByIdx = 0; orderByIdx < plan.orderBy.length; orderByIdx++) {
+    const term = plan.orderBy[orderByIdx]
+    /** @type {number[][]} */
+    const nextGroups = []
+
+    for (const group of groups) {
+      if (group.length <= 1) {
+        nextGroups.push(group)
+        continue
+      }
+
+      // Evaluate this column for all rows in the group
+      for (const idx of group) {
+        if (evaluatedValues[idx][orderByIdx] === undefined) {
+          evaluatedValues[idx][orderByIdx] = await evaluateExpr({
+            node: term.expr,
+            row: rows[idx],
+            tables,
+            functions,
+            aliases: plan.aliases,
+            signal,
+          })
+        }
+      }
+
+      // Sort the group by this column
+      group.sort((aIdx, bIdx) => {
+        const av = evaluatedValues[aIdx][orderByIdx]
+        const bv = evaluatedValues[bIdx][orderByIdx]
+        return compareForTerm(av, bv, term)
+      })
+
+      // Split into sub-groups based on ties
+      if (orderByIdx < plan.orderBy.length - 1) {
+        /** @type {number[]} */
+        let currentSubGroup = [group[0]]
+        for (let i = 1; i < group.length; i++) {
+          const prevIdx = group[i - 1]
+          const currIdx = group[i]
+          const prevVal = evaluatedValues[prevIdx][orderByIdx]
+          const currVal = evaluatedValues[currIdx][orderByIdx]
+
+          if (compareForTerm(prevVal, currVal, term) === 0) {
+            currentSubGroup.push(currIdx)
+          } else {
+            nextGroups.push(currentSubGroup)
+            currentSubGroup = [currIdx]
+          }
+        }
+        nextGroups.push(currentSubGroup)
+      } else {
+        nextGroups.push(group)
+      }
+    }
+
+    groups = nextGroups
+  }
+
+  // Yield sorted rows
+  for (const idx of groups.flat()) {
+    yield rows[idx]
+  }
+}