squirreling 0.4.0 → 0.4.2

package/src/execute/join.js

@@ -0,0 +1,357 @@
+import { evaluateExpr } from './expression.js'
+
+/**
+ * @import { AsyncRow, AsyncDataSource, JoinClause, ExprNode } from '../types.js'
+ */
+
+/**
+ * Executes JOIN operations against a base data source
+ *
+ * @param {AsyncDataSource} leftSource - the left side of the join (FROM table)
+ * @param {JoinClause[]} joins - array of join clauses to execute
+ * @param {string} leftTableName - name of the left table (for column prefixing)
+ * @param {Record<string, AsyncDataSource>} tables - all available tables
+ * @returns {Promise<AsyncDataSource>} data source yielding joined rows
+ */
+export async function executeJoins(leftSource, joins, leftTableName, tables) {
+  let currentLeftTable = leftTableName
+
+  // Single join optimization: stream left rows without buffering
+  if (joins.length === 1) {
+    const join = joins[0]
+    const rightSource = tables[join.table]
+    if (rightSource === undefined) {
+      throw new Error(`Table "${join.table}" not found`)
+    }
+
+    // Buffer right rows for hash index (required for hash join)
+    /** @type {AsyncRow[]} */
+    const rightRows = []
+    for await (const row of rightSource.getRows()) {
+      rightRows.push(row)
+    }
+
+    // Use alias for column prefixing if present
+    const rightTableName = join.alias ?? join.table
+
+    // Return streaming data source - left rows stream through without buffering
+    return {
+      async *getRows() {
+        yield* hashJoin({
+          leftRows: leftSource.getRows(), // Stream directly, not buffered
+          rightRows,
+          join,
+          leftTable: currentLeftTable,
+          rightTable: rightTableName,
+          tables,
+        })
+      },
+    }
+  }
+
+  // Multiple joins: buffer intermediate results, stream final join
+  /** @type {AsyncRow[]} */
+  let leftRows = []
+  for await (const row of leftSource.getRows()) {
+    leftRows.push(row)
+  }
+
+  // Process all but the last join, buffering intermediate results
+  for (let i = 0; i < joins.length - 1; i++) {
+    const join = joins[i]
+    const rightSource = tables[join.table]
+    if (rightSource === undefined) {
+      throw new Error(`Table "${join.table}" not found`)
+    }
+
+    /** @type {AsyncRow[]} */
+    const rightRows = []
+    for await (const row of rightSource.getRows()) {
+      rightRows.push(row)
+    }
+
+    // Use alias for column prefixing if present
+    const rightTableName = join.alias ?? join.table
+
+    // Collect intermediate results into array for next join
+    /** @type {AsyncRow[]} */
+    const newLeftRows = []
+    const joined = hashJoin({
+      leftRows,
+      rightRows,
+      join,
+      leftTable: currentLeftTable,
+      rightTable: rightTableName,
+      tables,
+    })
+    for await (const row of joined) {
+      newLeftRows.push(row)
+    }
+    leftRows = newLeftRows
+
+    // After join, the "left" table for the next join includes all joined tables
+    currentLeftTable = `${currentLeftTable}_${rightTableName}`
+  }
+
+  // Final join: stream the results
+  const lastJoin = joins[joins.length - 1]
+  const rightSource = tables[lastJoin.table]
+  if (rightSource === undefined) {
+    throw new Error(`Table "${lastJoin.table}" not found`)
+  }
+
+  /** @type {AsyncRow[]} */
+  const rightRows = []
+  for await (const row of rightSource.getRows()) {
+    rightRows.push(row)
+  }
+
+  // Use alias for column prefixing if present
+  const lastRightTableName = lastJoin.alias ?? lastJoin.table
+
+  return {
+    async *getRows() {
+      yield* hashJoin({
+        leftRows,
+        rightRows,
+        join: lastJoin,
+        leftTable: currentLeftTable,
+        rightTable: lastRightTableName,
+        tables,
+      })
+    },
+  }
+}
+
+/**
+ * Checks if an expression references a specific table.
+ * Returns true if the expression is an identifier prefixed with the table name.
+ *
+ * @param {ExprNode} expr
+ * @param {string} tableName
+ * @returns {boolean}
+ */
+function exprReferencesTable(expr, tableName) {
+  return expr.type === 'identifier' && expr.name.startsWith(`${tableName}.`)
+}
+
+/**
+ * Extracts the join key expressions from an ON condition.
+ * Handles both `left.col = right.col` and `right.col = left.col` orderings.
+ *
+ * @param {ExprNode} onCondition
+ * @param {string} leftTable
+ * @param {string} rightTable
+ * @returns {{ leftKey: ExprNode, rightKey: ExprNode } | undefined}
+ */
+function extractJoinKeys(onCondition, leftTable, rightTable) {
+  if (onCondition.type === 'binary' && onCondition.op === '=') {
+    const { left, right } = onCondition
+
+    // Check if keys are swapped (right table referenced in left position)
+    const leftRefsRight = exprReferencesTable(left, rightTable)
+    const rightRefsLeft = exprReferencesTable(right, leftTable)
+
+    if (leftRefsRight && rightRefsLeft) {
+      // Keys are swapped, return them in correct order
+      return { leftKey: right, rightKey: left }
+    }
+
+    // Default: assume left operand is for left table
+    return { leftKey: left, rightKey: right }
+  }
+}
+
+/**
+ * Creates a NULL-filled row with the given column names
+ *
+ * @param {string[]} columnNames
+ * @returns {AsyncRow}
+ */
+function createNullRow(columnNames) {
+  /** @type {AsyncRow} */
+  const row = {}
+  for (const col of columnNames) {
+    row[col] = () => Promise.resolve(null)
+  }
+  return row
+}
+
+/**
+ * Merges two rows into one, prefixing columns with table names
+ *
+ * @param {AsyncRow} leftRow
+ * @param {AsyncRow} rightRow
+ * @param {string} leftTable
+ * @param {string} rightTable
+ * @returns {AsyncRow}
+ */
+function mergeRows(leftRow, rightRow, leftTable, rightTable) {
+  /** @type {AsyncRow} */
+  const merged = {}
+
+  // Add left table columns with prefix
+  for (const [key, cell] of Object.entries(leftRow)) {
+    // Skip already-prefixed keys (from previous joins)
+    if (!key.includes('.')) {
+      merged[`${leftTable}.${key}`] = cell
+    } else {
+      merged[key] = cell
+    }
+    // Also keep unqualified name for convenience (may be overwritten if ambiguous)
+    merged[key] = cell
+  }
+
+  // Add right table columns with prefix
+  for (const [key, cell] of Object.entries(rightRow)) {
+    if (!key.includes('.')) {
+      merged[`${rightTable}.${key}`] = cell
+    } else {
+      merged[key] = cell
+    }
+    // Unqualified name (overwrites if same name exists in left table)
+    merged[key] = cell
+  }
+
+  return merged
+}
+
+/**
+ * Performs a hash join between left and right row sets (streaming).
+ * Yields rows as they are found instead of buffering all results.
+ *
+ * @param {Object} params
+ * @param {AsyncIterable<AsyncRow>|AsyncRow[]} params.leftRows - rows from left table (can stream)
+ * @param {AsyncRow[]} params.rightRows - rows from right table (must be buffered for hash index)
+ * @param {JoinClause} params.join - join specification
+ * @param {string} params.leftTable - name of left table (for column prefixing)
+ * @param {string} params.rightTable - name of right table (for column prefixing, may be alias)
+ * @param {Record<string, AsyncDataSource>} params.tables - all tables for expression evaluation
+ * @yields {AsyncRow} joined rows
+ */
+async function* hashJoin({ leftRows, rightRows, join, leftTable, rightTable, tables }) {
+  const { joinType, on: onCondition } = join
+
+  if (!onCondition) {
+    throw new Error('JOIN requires ON condition')
+  }
+
+  const keys = extractJoinKeys(onCondition, leftTable, rightTable)
+
+  // Get column names for NULL row generation (right side is always buffered)
+  const rightCols = rightRows.length ? Object.keys(rightRows[0]) : []
+  const rightPrefixedCols = rightCols.flatMap(col =>
+    col.includes('.') ? [col] : [`${rightTable}.${col}`, col]
+  )
+
+  // Track left column info - captured from first row during iteration
+  /** @type {string[]|null} */
+  let leftPrefixedCols = null
+
+  if (keys) {
+    // Hash join: build hash map on right table
+    /** @type {Map<string, AsyncRow[]>} */
+    const hashMap = new Map()
+
+    // BUILD PHASE: Index right rows by join key
+    // Skip null keys - SQL semantics: NULL != NULL
+    for (const rightRow of rightRows) {
+      const keyValue = await evaluateExpr({ node: keys.rightKey, row: rightRow, tables })
+      if (keyValue == null) continue // NULL keys never match
+      const keyStr = JSON.stringify(keyValue)
+
+      let bucket = hashMap.get(keyStr)
+      if (!bucket) {
+        bucket = []
+        hashMap.set(keyStr, bucket)
+      }
+      bucket.push(rightRow)
+    }
+
+    // Track which right rows matched (only needed for RIGHT/FULL joins)
+    /** @type {Set<AsyncRow>|null} */
+    const matchedRightRows = joinType === 'RIGHT' || joinType === 'FULL' ? new Set() : null
+
+    // PROBE PHASE: Stream through left rows, yield matches immediately
+    for await (const leftRow of leftRows) {
+      // Capture left column info from first row (for NULL row generation)
+      if (!leftPrefixedCols) {
+        const leftCols = Object.keys(leftRow)
+        leftPrefixedCols = leftCols.flatMap(col =>
+          col.includes('.') ? [col] : [`${leftTable}.${col}`, col]
+        )
+      }
+
+      const keyValue = await evaluateExpr({ node: keys.leftKey, row: leftRow, tables })
+      const keyStr = JSON.stringify(keyValue)
+
+      const matchingRightRows = hashMap.get(keyStr)
+
+      if (matchingRightRows && matchingRightRows.length > 0) {
+        for (const rightRow of matchingRightRows) {
+          if (matchedRightRows) matchedRightRows.add(rightRow)
+          yield mergeRows(leftRow, rightRow, leftTable, rightTable)
+        }
+      } else if (joinType === 'LEFT' || joinType === 'FULL') {
+        const nullRight = createNullRow(rightPrefixedCols)
+        yield mergeRows(leftRow, nullRight, leftTable, rightTable)
+      }
+      // INNER join with no match: don't yield anything
+    }
+
+    // UNMATCHED PHASE: Handle unmatched right rows for RIGHT/FULL joins
+    if (matchedRightRows) {
+      for (const rightRow of rightRows) {
+        if (!matchedRightRows.has(rightRow)) {
+          // Use empty array if left table was empty (no rows to derive columns from)
+          const nullLeft = createNullRow(leftPrefixedCols || [])
+          yield mergeRows(nullLeft, rightRow, leftTable, rightTable)
+        }
+      }
+    }
+  } else {
+    // Fallback to nested loop for complex ON conditions
+    // Left rows stream through, right rows are iterated for each left row
+    /** @type {Set<AsyncRow>|null} */
+    const matchedRightRows = joinType === 'RIGHT' || joinType === 'FULL' ? new Set() : null
+
+    for await (const leftRow of leftRows) {
+      // Capture left column info from first row (for NULL row generation)
+      if (!leftPrefixedCols) {
+        const leftCols = Object.keys(leftRow)
+        leftPrefixedCols = leftCols.flatMap(col =>
+          col.includes('.') ? [col] : [`${leftTable}.${col}`, col]
+        )
+      }
+
+      let hasMatch = false
+
+      for (const rightRow of rightRows) {
+        const tempMerged = mergeRows(leftRow, rightRow, leftTable, rightTable)
+        const matches = await evaluateExpr({ node: onCondition, row: tempMerged, tables })
+
+        if (matches) {
+          hasMatch = true
+          if (matchedRightRows) matchedRightRows.add(rightRow)
+          yield tempMerged
+        }
+      }
+
+      if (!hasMatch && (joinType === 'LEFT' || joinType === 'FULL')) {
+        const nullRight = createNullRow(rightPrefixedCols)
+        yield mergeRows(leftRow, nullRight, leftTable, rightTable)
+      }
+    }
+
+    // Handle unmatched right rows for RIGHT/FULL joins
+    if (matchedRightRows) {
+      for (const rightRow of rightRows) {
+        if (!matchedRightRows.has(rightRow)) {
+          // Use empty array if left table was empty (no rows to derive columns from)
+          const nullLeft = createNullRow(leftPrefixedCols || [])
+          yield mergeRows(nullLeft, rightRow, leftTable, rightTable)
+        }
+      }
+    }
+  }
+}

package/src/execute/utils.js

@@ -1,7 +1,44 @@
+/**
+ * @import {AsyncRow, ExprNode, OrderByItem, SqlPrimitive} from '../types.js'
+ */
+
+/**
+ * Compares two values for a single ORDER BY term, handling nulls and direction
+ *
+ * @param {SqlPrimitive} a
+ * @param {SqlPrimitive} b
+ * @param {OrderByItem} term
+ * @returns {number} comparison result
+ */
+export function compareForTerm(a, b, term) {
+  const aIsNull = a == null
+  const bIsNull = b == null
+
+  if (aIsNull || bIsNull) {
+    if (aIsNull && bIsNull) return 0
+    const nullsFirst = term.nulls !== 'LAST'
+    if (aIsNull) return nullsFirst ? -1 : 1
+    return nullsFirst ? 1 : -1
+  }
+
+  // Compare non-null values
+  if (a === b) return 0
+
+  let cmp
+  if (typeof a === 'number' && typeof b === 'number') {
+    cmp = a < b ? -1 : a > b ? 1 : 0
+  } else {
+    const aa = String(a)
+    const bb = String(b)
+    cmp = aa < bb ? -1 : aa > bb ? 1 : 0
+  }
+
+  return term.direction === 'DESC' ? -cmp : cmp
+}
+
 /**
  * Collects and materialize all results from an async row generator into an array
  *
- * @import {AsyncRow, SqlPrimitive} from '../types.js'
  * @param {AsyncGenerator<AsyncRow>} asyncRows
  * @returns {Promise<Record<string, SqlPrimitive>[]>} array of all yielded values
  */
@@ -18,3 +55,35 @@ export async function collect(asyncRows) {
   }
   return results
 }
+
+/**
+ * Generates a default alias for a derived column expression
+ *
+ * @param {ExprNode} expr - the expression node
+ * @returns {string} the generated alias
+ */
+export function defaultDerivedAlias(expr) {
+  if (expr.type === 'identifier') {
+    // For qualified names like 'users.name', use just the column part as alias
+    if (expr.name.includes('.')) {
+      return expr.name.split('.').pop()
+    }
+    return expr.name
+  }
+  if (expr.type === 'literal') {
+    return String(expr.value)
+  }
+  if (expr.type === 'cast') {
+    return defaultDerivedAlias(expr.expr) + '_as_' + expr.toType
+  }
+  if (expr.type === 'unary') {
+    return expr.op + '_' + defaultDerivedAlias(expr.argument)
+  }
+  if (expr.type === 'binary') {
+    return defaultDerivedAlias(expr.left) + '_' + expr.op + '_' + defaultDerivedAlias(expr.right)
+  }
+  if (expr.type === 'function') {
+    return expr.name.toLowerCase() + '_' + expr.args.map(defaultDerivedAlias).join('_')
+  }
+  return 'expr'
+}

package/src/index.d.ts

@@ -1,4 +1,5 @@
 import type { AsyncRow, ExecuteSqlOptions, SelectStatement, SqlPrimitive } from './types.js'
+export type { AsyncDataSource, AsyncRow, SqlPrimitive } from './types.js'
 
 /**
  * Executes a SQL SELECT query against an array of data rows

package/src/parse/expression.js

@@ -1,7 +1,7 @@
 import { isAggregateFunc, isStringFunc } from '../validation.js'
 
 /**
- * @import { ExprCursor, ExprNode, BinaryOp } from '../types.js'
+ * @import { BinaryOp, ExprCursor, ExprNode, WhenClause } from '../types.js'
  */
 
 /**
@@ -153,7 +153,7 @@ function parsePrimary(c) {
       c.consume() // CASE
 
       // Check if it's simple CASE (CASE expr WHEN ...) or searched CASE (CASE WHEN ...)
-      /** @type {import('../types.js').ExprNode | undefined} */
+      /** @type {ExprNode | undefined} */
       let caseExpr
       const nextTok = c.current()
       if (nextTok.type !== 'keyword' || nextTok.value !== 'WHEN') {
@@ -162,7 +162,7 @@ function parsePrimary(c) {
       }
 
       // Parse WHEN clauses
-      /** @type {import('../types.js').WhenClause[]} */
+      /** @type {WhenClause[]} */
       const whenClauses = []
       while (c.match('keyword', 'WHEN')) {
         const condition = parseExpression(c)
@@ -176,7 +176,7 @@ function parsePrimary(c) {
       }
 
       // Parse optional ELSE clause
-      /** @type {import('../types.js').ExprNode | undefined} */
+      /** @type {ExprNode | undefined} */
       let elseResult
       if (c.match('keyword', 'ELSE')) {
         elseResult = parseExpression(c)
@@ -301,7 +301,26 @@ function parseComparison(c) {
     }
   }
 
-  // LIKE
+  // [NOT] LIKE
+  if (tok.type === 'keyword' && tok.value === 'NOT') {
+    const nextTok = c.peek(1)
+    if (nextTok.type === 'keyword' && nextTok.value === 'LIKE') {
+      c.consume() // NOT
+      c.consume() // LIKE
+      const right = parsePrimary(c)
+      return {
+        type: 'unary',
+        op: 'NOT',
+        argument: {
+          type: 'binary',
+          op: 'LIKE',
+          left,
+          right,
+        },
+      }
+    }
+  }
+
   if (tok.type === 'keyword' && tok.value === 'LIKE') {
     c.consume()
     const right = parsePrimary(c)
@@ -313,7 +332,7 @@ function parseComparison(c) {
     }
   }
 
-  // [NOT] BETWEEN
+  // [NOT] BETWEEN - convert to range comparison
   if (tok.type === 'keyword' && tok.value === 'NOT') {
     const nextTok = c.peek(1)
     if (nextTok.type === 'keyword' && nextTok.value === 'BETWEEN') {
@@ -322,11 +341,12 @@ function parseComparison(c) {
       const lower = parsePrimary(c)
       c.expect('keyword', 'AND')
       const upper = parsePrimary(c)
+      // NOT BETWEEN -> expr < lower OR expr > upper
       return {
-        type: 'not between',
-        expr: left,
-        lower,
-        upper,
+        type: 'binary',
+        op: 'OR',
+        left: { type: 'binary', op: '<', left, right: lower },
+        right: { type: 'binary', op: '>', left, right: upper },
       }
     }
   }
@@ -336,11 +356,12 @@ function parseComparison(c) {
     const lower = parsePrimary(c)
     c.expect('keyword', 'AND')
     const upper = parsePrimary(c)
+    // BETWEEN -> expr >= lower AND expr <= upper
     return {
-      type: 'between',
-      expr: left,
-      lower,
-      upper,
+      type: 'binary',
+      op: 'AND',
+      left: { type: 'binary', op: '>=', left, right: lower },
+      right: { type: 'binary', op: '<=', left, right: upper },
     }
   }
 
@@ -365,9 +386,13 @@ function parseComparison(c) {
         }
         const subquery = c.parseSubquery()
         return {
-          type: 'not in',
-          expr: left,
-          subquery,
+          type: 'unary',
+          op: 'NOT',
+          argument: {
+            type: 'in',
+            expr: left,
+            subquery,
+          },
         }
       } else {
         // Parse list of values - we handle the parens
@@ -380,9 +405,13 @@ function parseComparison(c) {
         }
         c.expect('paren', ')')
         return {
-          type: 'not in valuelist',
-          expr: left,
-          values,
+          type: 'unary',
+          op: 'NOT',
+          argument: {
+            type: 'in valuelist',
+            expr: left,
+            values,
+          },
         }
       }
     }