squirreling 0.16.10 → 0.16.12

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "squirreling",
-  "version": "0.16.10",
+  "version": "0.16.12",
   "description": "Squirreling Async SQL Engine",
   "author": "Hyperparam",
   "homepage": "https://hyperparam.app",

package/src/execute/aggregates.js

@@ -5,6 +5,7 @@ import { finalizeAccumulator, newAccumulator, updateAccumulator } from './accumu
 import { executePlan, executeScan, selectColumnNames } from './execute.js'
 import { foldEvaluatedRows } from './fold.js'
 import { normalizeScanColumnResult } from './scanColumn.js'
+import { scanFilter } from './scanFilter.js'
 import { sortEntriesByTerms } from './sort.js'
 import { planStreamingAggregates, streamingHashAggregateRows, streamingScalarAggregateRows } from './streamingAggregate.js'
 import { keyify } from './utils.js'
@@ -289,10 +290,12 @@ function tryColumnScanAggregate(plan, { tables, signal }) {
   // Child must be a direct table scan
   if (plan.child.type !== 'Scan') return
   const scanNode = plan.child
-  const { limit, offset, where } = scanNode.hints
+  const { limit, offset } = scanNode.hints
 
   const table = tables[scanNode.table]
   if (!table?.scanColumn) return
+  const { where, residual } = scanFilter(scanNode, dataSourceColumns(table))
+  if (residual) return
 
   // COUNT(*) needs a physical column whose filtered chunk lengths can be
   // counted. Prefer a predicate/projection column, then any table column.

package/src/execute/execute.js

@@ -14,6 +14,7 @@ import { distinctBatches, filterBatches, limitBatches, projectExpressionBatches
 import { executeHashJoin, executeNestedLoopJoin, executePositionalJoin } from './join.js'
 import { referencesRowScope } from './rowScope.js'
 import { normalizeScanColumnResult } from './scanColumn.js'
+import { scanFilter } from './scanFilter.js'
 import { executeSort } from './sort.js'
 import { executeSemiJoin } from './semiJoin.js'
 import { addBounds, minBounds, stableRowKey } from './utils.js'
@@ -317,35 +318,51 @@ export function executeScan(plan, context, existingColumnResult) {
   const { tables, signal } = context
   const table = validateTable({ ...plan, tables })
   validateScan({ ...plan, tables })
-  const hasLimitOffset = plan.hints.limit !== undefined || plan.hints.offset // 0 offset is noop
+  const { where, residual } = scanFilter(plan, dataSourceColumns(table))
+  const hints = { ...plan.hints, where }
+  if (residual) {
+    // Keep range and Top-K outside the scan until the engine residual runs.
+    delete hints.limit
+    delete hints.offset
+  }
+  const hasLimitOffset = hints.limit !== undefined || hints.offset // 0 offset is noop
   const scanContext = { ...context, scope: [plan.alias ?? plan.table] }
 
   if (!existingColumnResult && table.prepareScan && table.schema) {
-    const prepared = table.prepareScan(scanRequest(plan, table.schema, scanContext))
-    return executePreparedScan({ plan, prepared, context: scanContext, table })
+    // Derive column demands from the original predicate, including fields
+    // needed by engine residuals, but send only the source-local predicate.
+    const request = scanRequest(plan, table.schema, scanContext)
+    request.filter = where
+    if (residual) {
+      delete request.limit
+      delete request.offset
+      delete request.topK
+    }
+    const prepared = table.prepareScan(request)
+    return finish(executePreparedScan({ plan: { ...plan, hints }, prepared, context: scanContext, table }))
   }
 
   // Fast path: single column scan. As with scan(), hints the source did not
   // apply are handled by the engine over the returned column values.
-  const scanColumnOptions = plan.hints.columns?.length === 1
-    ? plan.hints.where
+  const scanColumnOptions = hints.columns?.length === 1
+    ? hints.where
       // Do not push a filtered range until WHERE is known to be applied: an
       // older source may ignore WHERE but eagerly apply LIMIT/OFFSET.
-      ? { column: plan.hints.columns[0], where: plan.hints.where, signal }
-      : { column: plan.hints.columns[0], ...plan.hints, signal }
+      ? { column: hints.columns[0], where: hints.where, signal }
+      : { column: hints.columns[0], ...hints, signal }
     : undefined
   const columnResult = existingColumnResult ?? (table.scanColumn && scanColumnOptions
     ? normalizeScanColumnResult(table.scanColumn(scanColumnOptions), scanColumnOptions)
     : undefined)
   if (columnResult && scanColumnOptions) {
-    const column = plan.hints.columns[0]
-    const scanRows = computeScanRows(table.numRows, plan.hints.limit, plan.hints.offset)
+    const column = hints.columns[0]
+    const scanRows = computeScanRows(table.numRows, hints.limit, hints.offset)
     const columns = [column]
-    const appliedLimitOffset = plan.hints.where
+    const appliedLimitOffset = hints.where
       ? false
       : columnResult.appliedLimitOffset
-    const residualFilter = plan.hints.where && !columnResult.appliedWhere
-      ? compileBatchExpression(plan.hints.where, columns)
+    const residualFilter = hints.where && !columnResult.appliedWhere
+      ? compileBatchExpression(hints.where, columns)
       : undefined
 
     /** @returns {AsyncIterable<AsyncBatch>} */
@@ -353,28 +370,28 @@ export function executeScan(plan, context, existingColumnResult) {
       /** @type {AsyncIterable<AsyncBatch>} */
       let batches = columnBatches(columnResult.chunks(), signal)
       if (residualFilter) {
-        const targetRows = plan.hints.limit === undefined
+        const targetRows = hints.limit === undefined
           ? undefined
-          : plan.hints.limit + (plan.hints.offset ?? 0)
+          : hints.limit + (hints.offset ?? 0)
         batches = filterBatches(batches, residualFilter, signal, targetRows)
       }
       if (!appliedLimitOffset && hasLimitOffset) {
-        batches = limitBatches(batches, plan.hints.limit, plan.hints.offset, signal)
+        batches = limitBatches(batches, hints.limit, hints.offset, signal)
       }
       return batches
     }
 
-    const canUseBatches = !plan.hints.where || columnResult.appliedWhere || residualFilter !== undefined
+    const canUseBatches = !hints.where || columnResult.appliedWhere || residualFilter !== undefined
     if (canUseBatches) {
-      return batchResult({
+      return finish(batchResult({
         columns,
-        numRows: plan.hints.where ? undefined : scanRows,
+        numRows: hints.where ? undefined : scanRows,
         maxRows: scanRows,
         batches: makeBatches,
         signal,
-      })
+      }))
     }
-    return {
+    return finish({
       columns,
       numRows: undefined,
       maxRows: scanRows,
@@ -395,10 +412,10 @@ export function executeScan(plan, context, existingColumnResult) {
           }
         })()
 
-        result = filterRows(result, plan.hints.where, scanContext, plan.hints.limit)
+        result = filterRows(result, hints.where, scanContext, hints.limit)
         // Filtered LIMIT/OFFSET was intentionally not passed to scanColumn.
         if (!appliedLimitOffset && hasLimitOffset) {
-          result = limitRows(result, plan.hints.limit, plan.hints.offset, signal)
+          result = limitRows(result, hints.limit, hints.offset, signal)
         }
 
         yield* result
@@ -406,7 +423,7 @@ export function executeScan(plan, context, existingColumnResult) {
         // the abort so a truncated scan is not mistaken for a complete one
         signal?.throwIfAborted()
       },
-    }
+    })
   }
 
   if (!table.scan) {
@@ -414,30 +431,30 @@ export function executeScan(plan, context, existingColumnResult) {
   }
 
   // do the scan
-  const scanResult = table.scan({ ...plan.hints, signal })
+  const scanResult = table.scan({ ...hints, signal })
   const { appliedWhere, appliedLimitOffset } = scanResult
 
   // Applied limit/offset without applied where is invalid
-  if (!appliedWhere && appliedLimitOffset && plan.hints.where && hasLimitOffset) {
+  if (!appliedWhere && appliedLimitOffset && hints.where && hasLimitOffset) {
     throw new Error(`Data source "${plan.table}" applied limit/offset without applying where`)
   }
 
-  const scanRows = computeScanRows(table.numRows, plan.hints.limit, plan.hints.offset)
-  return {
-    columns: plan.hints.columns ?? dataSourceColumns(table),
-    numRows: !plan.hints.where ? scanRows : undefined,
+  const scanRows = computeScanRows(table.numRows, hints.limit, hints.offset)
+  return finish({
+    columns: hints.columns ?? dataSourceColumns(table),
+    numRows: !hints.where ? scanRows : undefined,
     maxRows: scanRows,
     async *rows() {
       let result = scanResult.rows()
 
       // Apply WHERE if data source did not
-      if (!appliedWhere && plan.hints.where) {
-        result = filterRows(result, plan.hints.where, scanContext, plan.hints.limit)
+      if (!appliedWhere && hints.where) {
+        result = filterRows(result, hints.where, scanContext, hints.limit)
       }
 
       // Apply LIMIT/OFFSET if data source did not
       if (!appliedLimitOffset && hasLimitOffset) {
-        result = limitRows(result, plan.hints.limit, plan.hints.offset, signal)
+        result = limitRows(result, hints.limit, hints.offset, signal)
       }
 
       yield* result
@@ -446,6 +463,24 @@ export function executeScan(plan, context, existingColumnResult) {
       // the abort so a truncated scan is not mistaken for a complete one
       signal?.throwIfAborted()
     },
+  })
+
+  /**
+   * Apply engine-only predicates and ranges after any source execution path.
+   * @param {QueryResults} source
+   * @returns {QueryResults}
+   */
+  function finish(source) {
+    if (!residual) return source
+    const { limit, offset } = plan.hints
+    return {
+      columns: source.columns,
+      maxRows: computeScanRows(source.maxRows, limit, offset),
+      async *rows() {
+        const filtered = filterRows(source.rows(), residual, scanContext, limit)
+        yield* limitRows(filtered, limit, offset, signal)
+      },
+    }
   }
 }
 

package/src/execute/join.js

@@ -60,7 +60,7 @@ export function executeNestedLoopJoin(plan, context) {
         innerRows.push(row)
       }
 
-      const innerCols = innerRows.length ? innerRows[0].columns : []
+      const innerCols = innerRows.length ? innerRows[0].columns : inner.columns
 
       /** @type {string[] | undefined} */
       let outerCols = undefined
@@ -107,7 +107,7 @@ export function executeNestedLoopJoin(plan, context) {
       if (matchedInnerRows) {
         for (const innerRow of innerRows) {
           if (!matchedInnerRows.has(innerRow)) {
-            yield merge(createNullRow(outerCols ?? []), innerRow)
+            yield merge(createNullRow(outerCols ?? outer.columns), innerRow)
           }
         }
       }
@@ -329,7 +329,7 @@ export function executeHashJoin(plan, context) {
         }
 
         if (!matched && probeOuter) {
-          yield merge(probeRow, createNullRow(buildCols))
+          yield merge(probeRow, createNullRow(buildCols.length ? buildCols : build.columns))
         }
       }
 
@@ -339,7 +339,7 @@ export function executeHashJoin(plan, context) {
       if (buildRows && matchedBuildRows) {
         for (const buildRow of buildRows) {
           if (!matchedBuildRows.has(buildRow)) {
-            yield merge(createNullRow(probeCols ?? []), buildRow)
+            yield merge(createNullRow(probeCols ?? probe.columns), buildRow)
           }
         }
       }

package/src/execute/scanFilter.js

@@ -0,0 +1,81 @@
+/**
+ * @import { ExprNode } from '../types.js'
+ * @import { ScanNode } from '../plan/types.js'
+ */
+
+/**
+ * Translate local conjuncts into physical column names. Keep subqueries and
+ * references outside this scan in the engine. Match scope names exactly, as
+ * expression evaluation does, including quoted identifiers and case.
+ *
+ * @param {ScanNode} plan
+ * @param {string[]} columns
+ * @returns {{ where?: ExprNode, residual?: ExprNode }}
+ */
+export function scanFilter(plan, columns) {
+  const scope = plan.alias ?? plan.table
+
+  /**
+   * @param {ExprNode} node
+   * @returns {{ where?: ExprNode, residual?: ExprNode }}
+   */
+  function split(node) {
+    if (!node) return {}
+    if (node.type === 'binary' && node.op === 'AND') {
+      const left = split(node.left)
+      const right = split(node.right)
+      return {
+        where: combine(left.where, right.where),
+        residual: combine(left.residual, right.residual),
+      }
+    }
+    let safe = true
+    /**
+     * @param {ExprNode} expr
+     * @returns {ExprNode}
+     */
+    function rewrite(expr) {
+      if (!expr) return expr
+      if (expr.type === 'identifier') {
+        if (expr.prefix && expr.prefix !== scope || !columns.includes(expr.name) ||
+            expr.prefix && columns.includes(`${expr.prefix}.${expr.name}`)) {
+          safe = false
+          return expr
+        }
+        if (!expr.prefix) return expr
+        const bare = { ...expr }
+        delete bare.prefix
+        return bare
+      }
+      if (expr.type === 'literal' || expr.type === 'interval') return expr
+      if (expr.type === 'binary') return { ...expr, left: rewrite(expr.left), right: rewrite(expr.right) }
+      if (expr.type === 'unary') return { ...expr, argument: rewrite(expr.argument) }
+      if (expr.type === 'cast') return { ...expr, expr: rewrite(expr.expr) }
+      if (expr.type === 'function') return { ...expr, args: expr.args.map(rewrite), ...expr.filter ? { filter: rewrite(expr.filter) } : {} }
+      if (expr.type === 'in valuelist') return { ...expr, expr: rewrite(expr.expr), values: expr.values.map(rewrite) }
+      if (expr.type === 'subscript') return { ...expr, expr: rewrite(expr.expr), index: rewrite(expr.index) }
+      if (expr.type === 'case') return {
+        ...expr,
+        caseExpr: rewrite(expr.caseExpr),
+        whenClauses: expr.whenClauses.map(w => ({ ...w, condition: rewrite(w.condition), result: rewrite(w.result) })),
+        elseResult: rewrite(expr.elseResult),
+      }
+      safe = false
+      return expr
+    }
+    const where = rewrite(node)
+    return safe ? { where } : { residual: node }
+  }
+  return split(plan.hints.where)
+}
+
+/**
+ * @param {ExprNode} left
+ * @param {ExprNode} right
+ * @returns {ExprNode}
+ */
+function combine(left, right) {
+  if (!left) return right
+  if (!right) return left
+  return { type: 'binary', op: 'AND', left, right, positionStart: left.positionStart, positionEnd: right.positionEnd }
+}

package/src/execute/semiJoin.js

@@ -4,6 +4,7 @@ import { sqlEquals } from './utils.js'
 import { yieldToEventLoop } from './yield.js'
 
 /**
+ * @import { KeyTrie } from '../internalTypes.js'
  * @import { ExecuteContext, QueryResults, SqlPrimitive } from '../types.js'
  * @import { SemiJoinNode } from '../plan/types.js'
  */
@@ -17,6 +18,7 @@ import { yieldToEventLoop } from './yield.js'
  * @returns {QueryResults}
  */
 export function executeSemiJoin(plan, context) {
+  if (plan.keys) return executeExistsJoin(plan, context)
   const left = executePlan({ plan: plan.left, context })
   return {
     columns: left.columns,
@@ -75,3 +77,73 @@ function contains(values, value) {
   }
   return false
 }
+
+/**
+ * Builds a tuple membership trie once. Type partitions preserve the evaluator's
+ * coercive equality without making same-type primitive lookups linear.
+ *
+ * @param {SemiJoinNode} plan
+ * @param {ExecuteContext} context
+ * @returns {QueryResults}
+ */
+function executeExistsJoin(plan, context) {
+  const left = executePlan({ plan: plan.left, context })
+  return {
+    columns: left.columns,
+    maxRows: left.maxRows,
+    async *rows() {
+      /** @type {KeyTrie} */
+      const root = { groups: new Map() }
+      let built = false
+      let count = 0
+      for await (const row of left.rows()) {
+        context.signal?.throwIfAborted()
+        if (++count % 4000 === 0) await yieldToEventLoop()
+        if (!built) {
+          for await (const inner of executePlan({ plan: plan.right, context }).rows()) {
+            context.signal?.throwIfAborted()
+            if (++count % 4000 === 0) await yieldToEventLoop()
+            const values = await Promise.all(inner.columns.map(col => inner.cells[col]()))
+            if (values.some(value => value == null)) continue
+            let node = root
+            for (const value of values) {
+              const type = typeof value
+              let group = node.groups.get(type)
+              if (!group) node.groups.set(type, group = new Map())
+              let child = group.get(value)
+              if (!child) group.set(value, child = { groups: new Map() })
+              node = child
+            }
+          }
+          built = true
+        }
+        const values = await Promise.all((plan.keys ?? []).map(node => evaluateExpr({ node, row, context })))
+        const matched = !values.some(value => value == null) && matchesKeys(root, values, 0)
+        if (matched !== plan.not) yield row
+      }
+    },
+  }
+}
+
+/**
+ * @param {KeyTrie} node
+ * @param {SqlPrimitive[]} values
+ * @param {number} index
+ * @returns {boolean}
+ */
+function matchesKeys(node, values, index) {
+  if (index === values.length) return true
+  const value = values[index]
+  const type = typeof value
+  if (type !== 'object' && !Number.isNaN(value)) {
+    const child = node.groups.get(type)?.get(value)
+    if (child && matchesKeys(child, values, index + 1)) return true
+  }
+  for (const [valueType, group] of node.groups) {
+    if (valueType === type && type !== 'object') continue
+    for (const [candidate, child] of group) {
+      if (sqlEquals(value, candidate) && matchesKeys(child, values, index + 1)) return true
+    }
+  }
+  return false
+}

package/src/expression/evaluate.js

@@ -713,7 +713,11 @@ export async function evaluateExpr({ node, row, rowIndex, rows, context }) {
   // IN with subqueries, same three-valued logic as the value-list form
   if (node.type === 'in') {
     const exprVal = await evaluateExpr({ node: node.expr, row, rowIndex, rows, context })
-    const subResult = executeStatement({ query: node.subquery, context })
+    const outerScope = context.scope
+    const subContext = outerScope
+      ? { ...context, outerRow: row, outerAliases: new Set(outerScope) }
+      : context
+    const subResult = executeStatement({ query: node.subquery, context: subContext, outerScope })
     let sawNull = false
     let innerCount = 0
     for await (const resRow of subResult.rows()) {

package/src/expression/regexp.js

@@ -16,7 +16,22 @@ import { ArgValueError } from '../validation/executionErrors.js'
  * @returns {SqlPrimitive}
  */
 export function evaluateRegexpFunc({ funcName, node, args, rowIndex }) {
-  if (funcName === 'REGEXP_SUBSTR' || funcName === 'REGEXP_EXTRACT') {
+  if (funcName === 'REGEXP_EXTRACT') {
+    const [str, pattern] = args
+    if (str == null || pattern == null || args.length >= 3 && args[2] == null) return null
+    const group = args.length >= 3 ? Number(args[2]) : 0
+    if (!Number.isInteger(group) || group < 0) {
+      throw new ArgValueError({
+        ...node,
+        message: `group must be a non-negative integer, got ${args[2]}`,
+        rowIndex,
+      })
+    }
+    const regex = compileRegexp(node, String(pattern), '', rowIndex)
+    return regex.exec(String(str))?.[group] ?? null
+  }
+
+  if (funcName === 'REGEXP_SUBSTR') {
     const str = args[0]
     const pattern = args[1]
     if (str == null || pattern == null) return null
@@ -51,17 +66,7 @@ export function evaluateRegexpFunc({ funcName, node, args, rowIndex }) {
       }
     }
 
-    // Create regex
-    let regex
-    try {
-      regex = new RegExp(patternStr, 'g')
-    } catch (/** @type {any} */ error) {
-      throw new ArgValueError({
-        ...node,
-        message: `invalid regex pattern: ${error.message}`,
-        rowIndex,
-      })
-    }
+    const regex = compileRegexp(node, patternStr, 'g', rowIndex)
 
     // Search from position (convert to 0-based)
     const searchStr = strVal.substring(position - 1)
@@ -74,6 +79,8 @@ export function evaluateRegexpFunc({ funcName, node, args, rowIndex }) {
       if (count === occurrence) {
         return match[0]
       }
+      // RegExp.exec does not advance after an empty match.
+      if (match[0] === '') regex.lastIndex++
     }
 
     return null
@@ -120,17 +127,7 @@ export function evaluateRegexpFunc({ funcName, node, args, rowIndex }) {
       }
     }
 
-    // Create regex
-    let regex
-    try {
-      regex = new RegExp(patternStr, 'g')
-    } catch (/** @type {any} */ error) {
-      throw new ArgValueError({
-        ...node,
-        message: `invalid regex pattern: ${error.message}`,
-        rowIndex,
-      })
-    }
+    const regex = compileRegexp(node, patternStr, 'g', rowIndex)
 
     // If position > 1, preserve the prefix
     const prefix = strVal.substring(0, position - 1)
@@ -174,15 +171,7 @@ export function evaluateRegexpLike({ node, args, rowIndex, cache }) {
   const patternString = String(pattern)
   let regex = cache?.pattern === patternString ? cache.regex : undefined
   if (!regex) {
-    try {
-      regex = new RegExp(patternString)
-    } catch (/** @type {any} */ error) {
-      throw new ArgValueError({
-        ...node,
-        message: `invalid regex pattern: ${error.message}`,
-        rowIndex,
-      })
-    }
+    regex = compileRegexp(node, patternString, '', rowIndex)
     if (cache) {
       cache.pattern = patternString
       cache.regex = regex
@@ -194,3 +183,24 @@ export function evaluateRegexpLike({ node, args, rowIndex, cache }) {
   // '[object Object]', making REGEXP_LIKE a silent never-match on them.
   return regex.test(typeof string === 'object' ? stringify(string) : String(string))
 }
+
+/**
+ * Compile a regex and report invalid patterns as SQL argument errors.
+ *
+ * @param {FunctionNode} node
+ * @param {string} pattern
+ * @param {string} flags
+ * @param {number} [rowIndex]
+ * @returns {RegExp}
+ */
+function compileRegexp(node, pattern, flags, rowIndex) {
+  try {
+    return new RegExp(pattern, flags)
+  } catch (/** @type {any} */ error) {
+    throw new ArgValueError({
+      ...node,
+      message: `invalid regex pattern: ${error.message}`,
+      rowIndex,
+    })
+  }
+}

package/src/internalTypes.d.ts

@@ -1,5 +1,9 @@
 import type { ColumnReadRequest, ColumnResult, ColumnVector, SqlPrimitive } from './types.js'
 
+export interface KeyTrie {
+  groups: Map<string, Map<SqlPrimitive, KeyTrie>>
+}
+
 export interface CompiledBatchExpression {
   evaluate(context: ColumnReadRequest): ColumnResult
 }

package/src/plan/plan.js

@@ -8,7 +8,7 @@ import { validateNoIdentifiers, validateScan, validateTableRefs } from '../valid
 import { collectColumnsFromExpr, collectScopeColumns, extractColumns, fromAlias, inferSelectSourceColumns, inferStatementColumns, statementScope, tableFunctionColumnNames } from './columns.js'
 
 /**
- * @import { AsyncDataSource, DerivedColumn, ExprNode, FromFunction, IdentifierNode, JoinClause, OrderByItem, PlanSqlOptions, ScanOptions, SelectColumn, SelectStatement, SetOperationStatement, Statement, WindowFunctionNode } from '../types.js'
+ * @import { AsyncDataSource, DerivedColumn, ExprNode, FromFunction, FromSubquery, FromTable, IdentifierNode, JoinClause, OrderByItem, PlanSqlOptions, ScanOptions, SelectColumn, SelectStatement, SetOperationStatement, Statement, WindowFunctionNode } from '../types.js'
  * @import { HashJoinNode, QueryPlan, SemiJoinNode, TableFunctionNode, WindowSpec } from './types.js'
  */
 
@@ -243,6 +243,20 @@ function planSelect({ select, ctePlans, cteColumns, tables, parentColumns, outer
     for (const conjunct of conjuncts) {
       const not = conjunct.type === 'unary' && conjunct.op === 'NOT'
       const expr = not ? conjunct.argument : conjunct
+      if (expr.type === 'exists' || expr.type === 'not exists') {
+        const keys = decorrelateExists(expr.subquery, scopeAliases, tables, cteColumns)
+        if (keys) {
+          const right = planStatement({ stmt: keys.query, tables, ctePlans, cteColumns })
+          semiJoins.push({
+            type: 'SemiJoin',
+            expr: keys.left[0],
+            keys: keys.left,
+            not: not !== (expr.type === 'not exists'),
+            right: { type: 'Subquery', child: right, scope: statementScope(keys.query) ?? [] },
+          })
+          continue
+        }
+      }
       if (expr.type !== 'in') {
         residual = andPredicate(residual, conjunct)
         continue
@@ -253,6 +267,23 @@ function planSelect({ select, ctePlans, cteColumns, tables, parentColumns, outer
         tables, cteColumns, outerAliases: new Set(scopeAliases),
       })
       if (correlated.length) {
+        // Positive correlated IN is EXISTS with an extra equality. NOT IN
+        // needs three-valued membership semantics and stays on the fallback.
+        // Unqualified operands must stay in the outer scope: moving them
+        // into the subquery could bind them to an inner column instead.
+        const sub = expr.subquery
+        const column = sub.type === 'select' && sub.columns.length === 1 ? sub.columns[0] : undefined
+        if (!not && sub.type === 'select' && column?.type === 'derived' &&
+            column.expr.type === 'identifier' && expr.expr.type === 'identifier' && expr.expr.prefix) {
+          /** @type {ExprNode} */
+          const equality = { type: 'binary', op: '=', left: column.expr, right: expr.expr, positionStart: expr.positionStart, positionEnd: expr.positionEnd }
+          const keys = decorrelateExists({ ...sub, where: andPredicate(sub.where, equality) }, scopeAliases, tables, cteColumns)
+          if (keys) {
+            const right = planStatement({ stmt: keys.query, tables, ctePlans, cteColumns })
+            semiJoins.push({ type: 'SemiJoin', expr: keys.left[0], keys: keys.left, not: false, right: { type: 'Subquery', child: right, scope: statementScope(keys.query) ?? [] } })
+            continue
+          }
+        }
         residual = andPredicate(residual, conjunct)
         continue
       }
@@ -279,7 +310,6 @@ function planSelect({ select, ctePlans, cteColumns, tables, parentColumns, outer
   /** @type {QueryPlan} */
   let plan = planFrom({ select, ctePlans, cteColumns, hints, tables, outerScope })
   pruneAggregateColumns(plan, subqueryNeeds)
-  const directFrom = plan.type === 'Scan' && plan.hints === hints
 
   // Add JOINs
   if (select.joins.length) {
@@ -291,9 +321,8 @@ function planSelect({ select, ctePlans, cteColumns, tables, parentColumns, outer
 
   // Add WHERE filter when the scan didn't receive it
   let { where } = select
-  if (directFrom && select.joins.length === 1 && !outerScope?.length &&
-      !ctePlans?.has(select.joins[0].table?.toLowerCase())) {
-    where = pushJoinPredicates(plan, where, tables)
+  if (select.joins.length && !outerScope?.length) {
+    where = pushJoinPredicates(plan, where, select, tables, cteColumns)
   }
   if (where && !isOwnScan) {
     plan = { type: 'Filter', condition: where, child: plan }
@@ -876,26 +905,34 @@ function collectConjuncts(node, out) {
 }
 
 /**
- * Push each table-local conjunct into fresh scans of a direct INNER
- * equijoin. Scan execution enforces hints a source cannot apply. Unresolved
- * and cross-table conjuncts remain residual predicates. Predicate evaluation
- * order and error short-circuiting are not guaranteed.
+ * Push local ON and WHERE conjuncts according to which side is preserved.
+ * Only recurse through this SELECT's join chain, never through shared CTEs
+ * or derived-table boundaries. Scan hints are copied rather than mutated.
  *
  * @param {QueryPlan} plan
  * @param {ExprNode | undefined} where
+ * @param {SelectStatement} select
  * @param {Record<string, AsyncDataSource>} [tables]
+ * @param {Map<string, string[]>} [cteColumns]
+ * @param {((aliases: string[]) => boolean)[]} [inherited] - proofs required by ancestor filters or inner joins
  * @returns {ExprNode | undefined}
  */
-function pushJoinPredicates(plan, where, tables) {
-  if (!tables || plan.type !== 'HashJoin' || plan.joinType !== 'INNER' ||
-      plan.left.type !== 'Scan' || plan.right.type !== 'Scan') return where
-  const scans = [plan.left, plan.right]
-  if (scans.some(scan => scan.hints.where || scan.hints.limit !== undefined || scan.hints.offset)) return where
-  const aliases = scans.map(scan => scan.alias ?? scan.table)
-  if (aliases[0] === aliases[1]) return where
-  const columns = scans.map(scan => dataSourceColumns(tables[scan.table]))
-  // A prefix that also names a struct column is not an unambiguous alias.
-  if (columns.some(names => aliases.some(alias => names.includes(alias)))) return where
+function pushJoinPredicates(plan, where, select, tables, cteColumns, inherited = []) {
+  if (!tables || plan.type !== 'HashJoin' && plan.type !== 'NestedLoopJoin' ||
+      select.from?.type === 'function' || select.joins.some(join => join.fromFunction)) return where
+  const last = select.joins[select.joins.length - 1]
+  const leftSelect = { ...select, joins: select.joins.slice(0, -1) }
+  const leftSources = [select.from, ...leftSelect.joins.map(joinSource)]
+  const rightSource = joinSource(last)
+  const sources = [...leftSources, rightSource]
+  const aliases = sources.map(source => fromAlias(source))
+  const columns = sources.map(source => source?.type === 'subquery'
+    ? inferStatementColumns({ stmt: source.query, tables, cteColumns })
+    : source?.type === 'table'
+      ? cteColumns?.get(source.table?.toLowerCase()) ?? dataSourceColumns(tables[source.table])
+      : [])
+  if (new Set(aliases).size !== aliases.length ||
+      columns.some(names => aliases.some(alias => names.includes(alias)))) return where
 
   /**
    * @param {ExprNode} expr
@@ -907,15 +944,14 @@ function pushJoinPredicates(plan, where, tables) {
     const matches = columns.map((names, i) => names.includes(expr.name) ? i : -1).filter(i => i >= 0)
     const index = expr.prefix ? aliases.indexOf(expr.prefix) : matches.length === 1 ? matches[0] : -1
     if (index < 0 || !columns[index].includes(expr.name)) return 'unsafe'
-    return index === 0 ? 'left' : 'right'
+    return index < leftSources.length ? 'left' : 'right'
   }
 
-  // Avoid changing the evaluation of unresolved or unqualified ON keys.
-  if (plan.leftKeys.some(key => owner(key) !== 'left') || plan.rightKeys.some(key => owner(key) !== 'right')) return where
+  // Do not change the interpretation of ambiguous or misoriented hash keys.
+  if (plan.type === 'HashJoin' && (plan.leftKeys.some(key => owner(key) !== 'left') ||
+      plan.rightKeys.some(key => owner(key) !== 'right'))) return where
+
   /**
-   * Resolve every reference in a conjunct independently. Subqueries and
-   * windows have their own scope and must remain above the join.
-   *
    * @param {ExprNode} expr
    * @returns {'left' | 'right' | 'literal' | 'unsafe'}
    */
@@ -942,51 +978,147 @@ function pushJoinPredicates(plan, where, tables) {
     return target
   }
 
+  /** @type {ExprNode[]} */
+  const whereConjuncts = []
+  if (where) collectConjuncts(where, whereConjuncts)
   /**
-   * @param {ExprNode[]} conjuncts
-   * @returns {Array<'left' | 'right' | 'residual'>}
+   * Capture this scope so unqualified references remain unambiguous when
+   * proofs are passed down a join chain with fewer visible sources.
+   * @param {ExprNode} expr
+   * @returns {(missing: string[]) => boolean}
    */
-  function classify(conjuncts) {
-    return conjuncts.map(function target(expr) {
-      const side = expressionOwner(expr)
-      return side === 'left' || side === 'right' ? side : 'residual'
+  function rejectionProof(expr) {
+    return missing => rejectsNulls(expr, column => {
+      if (column.type !== 'identifier') return false
+      const matches = columns.map((names, i) => names.includes(column.name) ? i : -1).filter(i => i >= 0)
+      const index = column.prefix ? aliases.indexOf(column.prefix) : matches.length === 1 ? matches[0] : -1
+      return index >= 0 && columns[index].includes(column.name) && missing.includes(aliases[index])
     })
   }
+  const proofs = [...inherited, ...whereConjuncts.map(rejectionProof)]
+  const rejectsLeft = proofs.some(proof => proof(aliases.slice(0, -1)))
+  const rejectsRight = proofs.some(proof => proof(aliases.slice(-1)))
+  if (plan.joinType === 'FULL') {
+    if (rejectsLeft && rejectsRight) plan.joinType = 'INNER'
+    else if (rejectsLeft) plan.joinType = 'LEFT'
+    else if (rejectsRight) plan.joinType = 'RIGHT'
+  } else if (plan.joinType === 'LEFT' && rejectsRight || plan.joinType === 'RIGHT' && rejectsLeft) {
+    plan.joinType = 'INNER'
+  }
 
+  const { joinType } = plan
+  /** @type {ExprNode | undefined} */
+  let leftWhere
+  /** @type {ExprNode | undefined} */
+  let rightWhere
   /**
    * @param {ExprNode[]} conjuncts
-   * @param {Array<'left' | 'right' | 'residual'>} targets
+   * @param {boolean} on
    * @returns {ExprNode | undefined}
    */
-  function push(conjuncts, targets) {
+  function push(conjuncts, on) {
     /** @type {ExprNode | undefined} */
     let residual
-    for (let i = 0; i < conjuncts.length; i++) {
-      const target = targets[i]
-      if (target === 'residual') {
-        residual = andPredicate(residual, conjuncts[i])
-      } else {
-        const scan = target === 'left' ? scans[0] : scans[1]
-        scan.hints = { ...scan.hints, where: andPredicate(scan.hints.where, conjuncts[i]) }
-      }
+    for (const expr of conjuncts) {
+      const side = expressionOwner(expr)
+      const canLeft = joinType === 'INNER' || joinType === (on ? 'RIGHT' : 'LEFT')
+      const canRight = joinType === 'INNER' || joinType === (on ? 'LEFT' : 'RIGHT')
+      if (side === 'left' && canLeft) leftWhere = andPredicate(leftWhere, expr)
+      else if (side === 'right' && canRight) rightWhere = andPredicate(rightWhere, expr)
+      else residual = andPredicate(residual, expr)
     }
     return residual
   }
 
   /** @type {ExprNode[]} */
   const onConjuncts = []
-  if (plan.residual) collectConjuncts(plan.residual, onConjuncts)
-  const onTargets = classify(onConjuncts)
-  const onResidual = push(onConjuncts, onTargets)
-  if (onResidual) plan.residual = onResidual
-  else delete plan.residual
+  const condition = plan.type === 'HashJoin' ? plan.residual : plan.condition
+  if (condition) collectConjuncts(condition, onConjuncts)
+  const onResidual = push(onConjuncts, true)
+  if (plan.type === 'HashJoin') {
+    if (onResidual) plan.residual = onResidual
+    else delete plan.residual
+  } else if (condition) {
+    // Nested-loop joins still need a condition when every conjunct was pushed.
+    plan.condition = onResidual ?? {
+      type: 'literal', value: true,
+      positionStart: condition.positionStart, positionEnd: condition.positionEnd,
+    }
+  }
+  const residual = push(whereConjuncts, false)
+  if (leftSelect.joins.length) {
+    // An inner join requires its ON condition to succeed, so it can reject
+    // NULL rows from an earlier outer join. An outer join's own ON cannot.
+    const leftProofs = joinType === 'INNER' || joinType === 'LEFT' ? [...proofs] : []
+    if (joinType === 'INNER') {
+      if (condition) leftProofs.push(rejectionProof(condition))
+      if (plan.type === 'HashJoin') leftProofs.push(...plan.leftKeys.map(rejectionProof))
+    }
+    leftWhere = pushJoinPredicates(plan.left, leftWhere, leftSelect, tables, cteColumns, leftProofs)
+  }
+  plan.left = filterJoinInput(plan.left, leftWhere)
+  plan.right = filterJoinInput(plan.right, rightWhere)
+  return residual
+}
 
-  if (!where) return undefined
-  /** @type {ExprNode[]} */
-  const whereConjuncts = []
-  collectConjuncts(where, whereConjuncts)
-  const whereTargets = classify(whereConjuncts)
-  return push(whereConjuncts, whereTargets)
+/**
+ * @param {JoinClause} join
+ * @returns {FromSubquery | FromTable}
+ */
+function joinSource(join) {
+  return join.subquery
+    ? { ...join.subquery, alias: join.alias }
+    : { ...join, type: 'table' }
+}
+
+/**
+ * Keep LIMIT, projection and shared CTE plans intact. A filter outside these
+ * boundaries still reduces the rows entering the join.
+ *
+ * @param {QueryPlan} plan
+ * @param {ExprNode | undefined} where
+ * @returns {QueryPlan}
+ */
+function filterJoinInput(plan, where) {
+  if (!where) return plan
+  if (plan.type === 'Scan' && plan.hints.limit === undefined && !plan.hints.offset) {
+    return { ...plan, hints: { ...plan.hints, where: andPredicate(plan.hints.where, where) } }
+  }
+  return { type: 'Filter', condition: where, child: plan }
+}
+
+/**
+ * Conservative proof for an expression with references to the side being
+ * NULL-extended. Unknown functions (including COALESCE) and CASE
+ * are deliberately not assumed to propagate NULL.
+ *
+ * @param {ExprNode} expr
+ * @param {(expr: ExprNode) => boolean} isMissing
+ * @returns {boolean}
+ */
+function nullWhenMissing(expr, isMissing) {
+  if (expr.type === 'identifier') return isMissing(expr)
+  if (expr.type === 'literal') return expr.value == null
+  if (expr.type === 'cast') return nullWhenMissing(expr.expr, isMissing)
+  if (expr.type === 'unary') return ['NOT', '+', '-'].includes(expr.op) && nullWhenMissing(expr.argument, isMissing)
+  if (expr.type === 'in valuelist') return nullWhenMissing(expr.expr, isMissing)
+  if (expr.type === 'binary' && ['=', '==', '!=', '<>', '<', '>', '<=', '>=', 'LIKE', '+', '-', '*', '/', '%', '||'].includes(expr.op)) {
+    return nullWhenMissing(expr.left, isMissing) || nullWhenMissing(expr.right, isMissing)
+  }
+  return false
+}
+
+/**
+ * @param {ExprNode} expr
+ * @param {(expr: ExprNode) => boolean} isMissing
+ * @returns {boolean}
+ */
+function rejectsNulls(expr, isMissing) {
+  if (nullWhenMissing(expr, isMissing)) return true
+  if (expr.type === 'unary' && expr.op === 'IS NOT NULL') return nullWhenMissing(expr.argument, isMissing)
+  if (expr.type === 'binary' && expr.op === 'AND') return rejectsNulls(expr.left, isMissing) || rejectsNulls(expr.right, isMissing)
+  if (expr.type === 'binary' && expr.op === 'OR') return rejectsNulls(expr.left, isMissing) && rejectsNulls(expr.right, isMissing)
+  return false
 }
 
 /**
@@ -1224,3 +1356,54 @@ function isAllCountStar(columns) {
     !col.expr.filter
   )
 }
+
+/**
+ * Pulls plain equality correlations out of a row-preserving EXISTS query.
+ * Limits and aggregation must stay on the per-outer-row execution path.
+ *
+ * @param {Statement} query
+ * @param {string[]} outerScope
+ * @param {Record<string, AsyncDataSource>} [tables]
+ * @param {Map<string, string[]>} [cteColumns]
+ * @returns {{ query: SelectStatement, left: IdentifierNode[] } | undefined}
+ */
+function decorrelateExists(query, outerScope, tables, cteColumns) {
+  if (query.type !== 'select' || query.limit !== undefined || query.offset ||
+      query.groupBy.length || query.having ||
+      query.columns.some(col => col.type === 'derived' && (findAggregate(col.expr) || col.expr.type === 'window'))) return
+  const scope = statementScope(query) ?? []
+  /** @type {IdentifierNode[]} */
+  const left = []
+  /** @type {DerivedColumn[]} */
+  const columns = []
+  /** @type {ExprNode[]} */
+  const conjuncts = []
+  if (query.where) collectConjuncts(query.where, conjuncts)
+  /** @type {ExprNode | undefined} */
+  let where
+  for (const term of conjuncts) {
+    const previousKeys = columns.length
+    if (term.type === 'binary' && term.op === '=' && term.left.type === 'identifier' && term.right.type === 'identifier') {
+      for (const [inner, outer] of [[term.left, term.right], [term.right, term.left]]) {
+        if (inner.prefix && scope.includes(inner.prefix) && outer.prefix &&
+            outerScope.includes(outer.prefix) && !scope.includes(outer.prefix)) {
+          left.push(outer)
+          columns.push({ type: 'derived', expr: inner, alias: `__exists_key_${columns.length}`, positionStart: inner.positionStart, positionEnd: inner.positionEnd })
+          break
+        }
+      }
+      if (columns.length > previousKeys) continue
+    }
+    where = andPredicate(where, term)
+  }
+  if (!left.length) return
+  // Check every remaining clause, including nested subqueries and projections,
+  // before discarding the projection and ordering that EXISTS does not read.
+  /** @type {IdentifierNode[]} */
+  const remaining = []
+  collectColumnsFromExpr({ type: 'subquery', subquery: { ...query, where }, positionStart: query.positionStart, positionEnd: query.positionEnd }, remaining, undefined, {
+    tables, cteColumns, outerAliases: new Set(outerScope),
+  })
+  if (remaining.length) return
+  return { query: { ...query, columns, where, distinct: false, orderBy: [] }, left }
+}

package/src/plan/types.d.ts

@@ -107,6 +107,8 @@ export interface SemiJoinNode {
   left: QueryPlan
   right: QueryPlan
   expr: ExprNode
+  // EXISTS equality keys; NOT uses anti-join rather than NOT IN null semantics.
+  keys?: ExprNode[]
   not: boolean
 }
 

package/src/types.d.ts

@@ -145,6 +145,7 @@ export interface ScanTopK {
 export interface ScanRequest {
   columns: readonly ColumnDemand[]
   topK?: ScanTopK
+  /** Source-local predicate: identifiers use bare physical column names. */
   filter?: ExprNode
   limit?: number
   offset?: number
@@ -271,7 +272,8 @@ export interface ScanResults {
  */
 export interface ScanOptions {
   columns?: string[] // columns needed (undefined means all columns)
-  where?: ExprNode // where clause
+  /** Source-local predicate: identifiers use bare physical column names. */
+  where?: ExprNode
   // important: only apply limit/offset if where is fully applied by the data source
   // otherwise, the data source must return at least enough rows to ensure the engine
   // can apply limit/offset correctly after filtering
@@ -285,6 +287,7 @@ export interface ScanOptions {
  */
 export interface ScanColumnOptions {
   column: string
+  /** Source-local predicate: identifiers use bare physical column names. */
   where?: ExprNode
   limit?: number
   offset?: number

package/src/validation/functions.js

@@ -131,7 +131,7 @@ export const FUNCTION_SIGNATURES = {
   SPLIT_PART: { min: 3, max: 3, signature: 'string, delimiter, index' },
   STRING_SPLIT: { min: 2, max: 2, signature: 'string, delimiter' },
   REGEXP_SUBSTR: { min: 2, max: 4, signature: 'string, pattern[, position[, occurrence]]' },
-  REGEXP_EXTRACT: { min: 2, max: 4, signature: 'string, pattern[, position[, occurrence]]' },
+  REGEXP_EXTRACT: { min: 2, max: 3, signature: 'string, pattern[, group]' },
   REGEXP_REPLACE: { min: 3, max: 5, signature: 'string, pattern, replacement[, position[, occurrence]]' },
   REGEXP_MATCHES: { min: 2, max: 2, signature: 'string, pattern' },
   REGEXP_LIKE: { min: 2, max: 2, signature: 'string, pattern' },