squirreling 0.16.9 → 0.16.11

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "squirreling",
-  "version": "0.16.9",
+  "version": "0.16.11",
   "description": "Squirreling Async SQL Engine",
   "author": "Hyperparam",
   "homepage": "https://hyperparam.app",
@@ -41,11 +41,11 @@
   "devDependencies": {
     "@eslint/js": "10.0.1",
     "@types/node": "26.6.5",
-    "@vitest/coverage-v8": "5.0.4",
-    "eslint": "10.12.0",
-    "eslint-plugin-jsdoc": "65.3.4",
+    "@vitest/coverage-v8": "5.0.5",
+    "eslint": "10.13.0",
+    "eslint-plugin-jsdoc": "65.4.0",
     "globals": "17.13.0",
     "typescript": "7.0.2",
-    "vitest": "5.0.4"
+    "vitest": "5.0.5"
   }
 }

package/src/execute/execute.js

@@ -15,6 +15,7 @@ import { executeHashJoin, executeNestedLoopJoin, executePositionalJoin } from '.
 import { referencesRowScope } from './rowScope.js'
 import { normalizeScanColumnResult } from './scanColumn.js'
 import { executeSort } from './sort.js'
+import { executeSemiJoin } from './semiJoin.js'
 import { addBounds, minBounds, stableRowKey } from './utils.js'
 import { executeWindow } from './window.js'
 import { yieldToEventLoop } from './yield.js'
@@ -108,6 +109,8 @@ export function executePlan({ plan, context }) {
     return executeFilter(plan, context)
   } else if (plan.type === 'Project') {
     return executeProject(plan, context)
+  } else if (plan.type === 'SemiJoin') {
+    return executeSemiJoin(plan, context)
   } else if (plan.type === 'HashJoin') {
     return executeHashJoin(plan, context)
   } else if (plan.type === 'NestedLoopJoin') {

package/src/execute/join.js

@@ -60,7 +60,7 @@ export function executeNestedLoopJoin(plan, context) {
         innerRows.push(row)
       }
 
-      const innerCols = innerRows.length ? innerRows[0].columns : []
+      const innerCols = innerRows.length ? innerRows[0].columns : inner.columns
 
       /** @type {string[] | undefined} */
       let outerCols = undefined
@@ -107,7 +107,7 @@ export function executeNestedLoopJoin(plan, context) {
       if (matchedInnerRows) {
         for (const innerRow of innerRows) {
           if (!matchedInnerRows.has(innerRow)) {
-            yield merge(createNullRow(outerCols ?? []), innerRow)
+            yield merge(createNullRow(outerCols ?? outer.columns), innerRow)
           }
         }
       }
@@ -329,7 +329,7 @@ export function executeHashJoin(plan, context) {
         }
 
         if (!matched && probeOuter) {
-          yield merge(probeRow, createNullRow(buildCols))
+          yield merge(probeRow, createNullRow(buildCols.length ? buildCols : build.columns))
         }
       }
 
@@ -339,7 +339,7 @@ export function executeHashJoin(plan, context) {
       if (buildRows && matchedBuildRows) {
         for (const buildRow of buildRows) {
           if (!matchedBuildRows.has(buildRow)) {
-            yield merge(createNullRow(probeCols ?? []), buildRow)
+            yield merge(createNullRow(probeCols ?? probe.columns), buildRow)
           }
         }
       }

package/src/execute/semiJoin.js

@@ -0,0 +1,149 @@
+import { evaluateExpr } from '../expression/evaluate.js'
+import { executePlan } from './execute.js'
+import { sqlEquals } from './utils.js'
+import { yieldToEventLoop } from './yield.js'
+
+/**
+ * @import { KeyTrie } from '../internalTypes.js'
+ * @import { ExecuteContext, QueryResults, SqlPrimitive } from '../types.js'
+ * @import { SemiJoinNode } from '../plan/types.js'
+ */
+
+/**
+ * Executes WHERE IN / NOT IN with one build of the right input per iteration.
+ * The membership set belongs to this operator, never to the context or AST.
+ *
+ * @param {SemiJoinNode} plan
+ * @param {ExecuteContext} context
+ * @returns {QueryResults}
+ */
+export function executeSemiJoin(plan, context) {
+  if (plan.keys) return executeExistsJoin(plan, context)
+  const left = executePlan({ plan: plan.left, context })
+  return {
+    columns: left.columns,
+    maxRows: left.maxRows,
+    async *rows() {
+      /** @type {Map<string, Set<SqlPrimitive>>} */
+      const values = new Map()
+      let hasNull = false
+      let count = 0
+      let built = false
+      let rowIndex = 0
+      for await (const row of left.rows()) {
+        context.signal?.throwIfAborted()
+        if (++rowIndex % 4000 === 0) await yieldToEventLoop()
+        const value = await evaluateExpr({ node: plan.expr, row, rowIndex, context })
+        if (!built) {
+          for await (const inner of executePlan({ plan: plan.right, context }).rows()) {
+            if (++count % 4000 === 0) {
+              await yieldToEventLoop()
+              context.signal?.throwIfAborted()
+            }
+            const value = await inner.cells[inner.columns[0]]()
+            if (value == null) hasNull = true
+            else {
+              const type = typeof value
+              let group = values.get(type)
+              if (!group) values.set(type, group = new Set())
+              group.add(value)
+            }
+          }
+          built = true
+        }
+        const matched = count === 0 ? false : value == null ? null : contains(values, value) ? true : hasNull ? null : false
+        if (matched !== null && matched !== plan.not) yield row
+      }
+    },
+  }
+}
+
+/**
+ * Same-type primitives use a Set. Cross-type values and Dates retain the
+ * evaluator's SQL equality, including coercion and timestamp comparison.
+ *
+ * @param {Map<string, Set<SqlPrimitive>>} values
+ * @param {SqlPrimitive} value
+ * @returns {boolean}
+ */
+function contains(values, value) {
+  const type = typeof value
+  if (type !== 'object' && !(typeof value === 'number' && Number.isNaN(value)) && values.get(type)?.has(value)) return true
+  for (const [valueType, group] of values) {
+    if (valueType === type && type !== 'object') continue
+    for (const candidate of group) {
+      if (sqlEquals(value, candidate)) return true
+    }
+  }
+  return false
+}
+
+/**
+ * Builds a tuple membership trie once. Type partitions preserve the evaluator's
+ * coercive equality without making same-type primitive lookups linear.
+ *
+ * @param {SemiJoinNode} plan
+ * @param {ExecuteContext} context
+ * @returns {QueryResults}
+ */
+function executeExistsJoin(plan, context) {
+  const left = executePlan({ plan: plan.left, context })
+  return {
+    columns: left.columns,
+    maxRows: left.maxRows,
+    async *rows() {
+      /** @type {KeyTrie} */
+      const root = { groups: new Map() }
+      let built = false
+      let count = 0
+      for await (const row of left.rows()) {
+        context.signal?.throwIfAborted()
+        if (++count % 4000 === 0) await yieldToEventLoop()
+        if (!built) {
+          for await (const inner of executePlan({ plan: plan.right, context }).rows()) {
+            context.signal?.throwIfAborted()
+            if (++count % 4000 === 0) await yieldToEventLoop()
+            const values = await Promise.all(inner.columns.map(col => inner.cells[col]()))
+            if (values.some(value => value == null)) continue
+            let node = root
+            for (const value of values) {
+              const type = typeof value
+              let group = node.groups.get(type)
+              if (!group) node.groups.set(type, group = new Map())
+              let child = group.get(value)
+              if (!child) group.set(value, child = { groups: new Map() })
+              node = child
+            }
+          }
+          built = true
+        }
+        const values = await Promise.all((plan.keys ?? []).map(node => evaluateExpr({ node, row, context })))
+        const matched = !values.some(value => value == null) && matchesKeys(root, values, 0)
+        if (matched !== plan.not) yield row
+      }
+    },
+  }
+}
+
+/**
+ * @param {KeyTrie} node
+ * @param {SqlPrimitive[]} values
+ * @param {number} index
+ * @returns {boolean}
+ */
+function matchesKeys(node, values, index) {
+  if (index === values.length) return true
+  const value = values[index]
+  const type = typeof value
+  if (type !== 'object' && !Number.isNaN(value)) {
+    const child = node.groups.get(type)?.get(value)
+    if (child && matchesKeys(child, values, index + 1)) return true
+  }
+  for (const [valueType, group] of node.groups) {
+    if (valueType === type && type !== 'object') continue
+    for (const [candidate, child] of group) {
+      if (sqlEquals(value, candidate) && matchesKeys(child, values, index + 1)) return true
+    }
+  }
+  return false
+}

package/src/expression/evaluate.js

@@ -713,7 +713,11 @@ export async function evaluateExpr({ node, row, rowIndex, rows, context }) {
   // IN with subqueries, same three-valued logic as the value-list form
   if (node.type === 'in') {
     const exprVal = await evaluateExpr({ node: node.expr, row, rowIndex, rows, context })
-    const subResult = executeStatement({ query: node.subquery, context })
+    const outerScope = context.scope
+    const subContext = outerScope
+      ? { ...context, outerRow: row, outerAliases: new Set(outerScope) }
+      : context
+    const subResult = executeStatement({ query: node.subquery, context: subContext, outerScope })
     let sawNull = false
     let innerCount = 0
     for await (const resRow of subResult.rows()) {

package/src/expression/regexp.js

@@ -16,7 +16,22 @@ import { ArgValueError } from '../validation/executionErrors.js'
  * @returns {SqlPrimitive}
  */
 export function evaluateRegexpFunc({ funcName, node, args, rowIndex }) {
-  if (funcName === 'REGEXP_SUBSTR' || funcName === 'REGEXP_EXTRACT') {
+  if (funcName === 'REGEXP_EXTRACT') {
+    const [str, pattern] = args
+    if (str == null || pattern == null || args.length >= 3 && args[2] == null) return null
+    const group = args.length >= 3 ? Number(args[2]) : 0
+    if (!Number.isInteger(group) || group < 0) {
+      throw new ArgValueError({
+        ...node,
+        message: `group must be a non-negative integer, got ${args[2]}`,
+        rowIndex,
+      })
+    }
+    const regex = compileRegexp(node, String(pattern), '', rowIndex)
+    return regex.exec(String(str))?.[group] ?? null
+  }
+
+  if (funcName === 'REGEXP_SUBSTR') {
     const str = args[0]
     const pattern = args[1]
     if (str == null || pattern == null) return null
@@ -51,17 +66,7 @@ export function evaluateRegexpFunc({ funcName, node, args, rowIndex }) {
       }
     }
 
-    // Create regex
-    let regex
-    try {
-      regex = new RegExp(patternStr, 'g')
-    } catch (/** @type {any} */ error) {
-      throw new ArgValueError({
-        ...node,
-        message: `invalid regex pattern: ${error.message}`,
-        rowIndex,
-      })
-    }
+    const regex = compileRegexp(node, patternStr, 'g', rowIndex)
 
     // Search from position (convert to 0-based)
     const searchStr = strVal.substring(position - 1)
@@ -74,6 +79,8 @@ export function evaluateRegexpFunc({ funcName, node, args, rowIndex }) {
       if (count === occurrence) {
         return match[0]
       }
+      // RegExp.exec does not advance after an empty match.
+      if (match[0] === '') regex.lastIndex++
     }
 
     return null
@@ -120,17 +127,7 @@ export function evaluateRegexpFunc({ funcName, node, args, rowIndex }) {
       }
     }
 
-    // Create regex
-    let regex
-    try {
-      regex = new RegExp(patternStr, 'g')
-    } catch (/** @type {any} */ error) {
-      throw new ArgValueError({
-        ...node,
-        message: `invalid regex pattern: ${error.message}`,
-        rowIndex,
-      })
-    }
+    const regex = compileRegexp(node, patternStr, 'g', rowIndex)
 
     // If position > 1, preserve the prefix
     const prefix = strVal.substring(0, position - 1)
@@ -174,15 +171,7 @@ export function evaluateRegexpLike({ node, args, rowIndex, cache }) {
   const patternString = String(pattern)
   let regex = cache?.pattern === patternString ? cache.regex : undefined
   if (!regex) {
-    try {
-      regex = new RegExp(patternString)
-    } catch (/** @type {any} */ error) {
-      throw new ArgValueError({
-        ...node,
-        message: `invalid regex pattern: ${error.message}`,
-        rowIndex,
-      })
-    }
+    regex = compileRegexp(node, patternString, '', rowIndex)
     if (cache) {
       cache.pattern = patternString
       cache.regex = regex
@@ -194,3 +183,24 @@ export function evaluateRegexpLike({ node, args, rowIndex, cache }) {
   // '[object Object]', making REGEXP_LIKE a silent never-match on them.
   return regex.test(typeof string === 'object' ? stringify(string) : String(string))
 }
+
+/**
+ * Compile a regex and report invalid patterns as SQL argument errors.
+ *
+ * @param {FunctionNode} node
+ * @param {string} pattern
+ * @param {string} flags
+ * @param {number} [rowIndex]
+ * @returns {RegExp}
+ */
+function compileRegexp(node, pattern, flags, rowIndex) {
+  try {
+    return new RegExp(pattern, flags)
+  } catch (/** @type {any} */ error) {
+    throw new ArgValueError({
+      ...node,
+      message: `invalid regex pattern: ${error.message}`,
+      rowIndex,
+    })
+  }
+}

package/src/internalTypes.d.ts

@@ -1,5 +1,9 @@
 import type { ColumnReadRequest, ColumnResult, ColumnVector, SqlPrimitive } from './types.js'
 
+export interface KeyTrie {
+  groups: Map<string, Map<SqlPrimitive, KeyTrie>>
+}
+
 export interface CompiledBatchExpression {
   evaluate(context: ColumnReadRequest): ColumnResult
 }

package/src/plan/plan.js

@@ -8,8 +8,8 @@ import { validateNoIdentifiers, validateScan, validateTableRefs } from '../valid
 import { collectColumnsFromExpr, collectScopeColumns, extractColumns, fromAlias, inferSelectSourceColumns, inferStatementColumns, statementScope, tableFunctionColumnNames } from './columns.js'
 
 /**
- * @import { AsyncDataSource, DerivedColumn, ExprNode, FromFunction, IdentifierNode, JoinClause, OrderByItem, PlanSqlOptions, ScanOptions, SelectColumn, SelectStatement, SetOperationStatement, Statement, WindowFunctionNode } from '../types.js'
- * @import { HashJoinNode, QueryPlan, TableFunctionNode, WindowSpec } from './types.js'
+ * @import { AsyncDataSource, DerivedColumn, ExprNode, FromFunction, FromSubquery, FromTable, IdentifierNode, JoinClause, OrderByItem, PlanSqlOptions, ScanOptions, SelectColumn, SelectStatement, SetOperationStatement, Statement, WindowFunctionNode } from '../types.js'
+ * @import { HashJoinNode, QueryPlan, SemiJoinNode, TableFunctionNode, WindowSpec } from './types.js'
  */
 
 /**
@@ -230,9 +230,77 @@ function planSelect({ select, ctePlans, cteColumns, tables, parentColumns, outer
   if (hints.columns?.length === 0 && select.from?.type === 'subquery') {
     hints.columns = undefined
   }
+  // A WHERE membership test is a semi-join, not a row-by-row expression
+  // subquery. Keep the original SELECT for column demand analysis above.
+  /** @type {Omit<SemiJoinNode, 'left'>[]} */
+  const semiJoins = []
+  if (select.where && !outerScope?.length) {
+    /** @type {ExprNode[]} */
+    const conjuncts = []
+    collectConjuncts(select.where, conjuncts)
+    /** @type {ExprNode | undefined} */
+    let residual
+    for (const conjunct of conjuncts) {
+      const not = conjunct.type === 'unary' && conjunct.op === 'NOT'
+      const expr = not ? conjunct.argument : conjunct
+      if (expr.type === 'exists' || expr.type === 'not exists') {
+        const keys = decorrelateExists(expr.subquery, scopeAliases, tables, cteColumns)
+        if (keys) {
+          const right = planStatement({ stmt: keys.query, tables, ctePlans, cteColumns })
+          semiJoins.push({
+            type: 'SemiJoin',
+            expr: keys.left[0],
+            keys: keys.left,
+            not: not !== (expr.type === 'not exists'),
+            right: { type: 'Subquery', child: right, scope: statementScope(keys.query) ?? [] },
+          })
+          continue
+        }
+      }
+      if (expr.type !== 'in') {
+        residual = andPredicate(residual, conjunct)
+        continue
+      }
+      /** @type {IdentifierNode[]} */
+      const correlated = []
+      collectColumnsFromExpr({ type: 'subquery', subquery: expr.subquery, positionStart: expr.positionStart, positionEnd: expr.positionEnd }, correlated, undefined, {
+        tables, cteColumns, outerAliases: new Set(scopeAliases),
+      })
+      if (correlated.length) {
+        // Positive correlated IN is EXISTS with an extra equality. NOT IN
+        // needs three-valued membership semantics and stays on the fallback.
+        // Unqualified operands must stay in the outer scope: moving them
+        // into the subquery could bind them to an inner column instead.
+        const sub = expr.subquery
+        const column = sub.type === 'select' && sub.columns.length === 1 ? sub.columns[0] : undefined
+        if (!not && sub.type === 'select' && column?.type === 'derived' &&
+            column.expr.type === 'identifier' && expr.expr.type === 'identifier' && expr.expr.prefix) {
+          /** @type {ExprNode} */
+          const equality = { type: 'binary', op: '=', left: column.expr, right: expr.expr, positionStart: expr.positionStart, positionEnd: expr.positionEnd }
+          const keys = decorrelateExists({ ...sub, where: andPredicate(sub.where, equality) }, scopeAliases, tables, cteColumns)
+          if (keys) {
+            const right = planStatement({ stmt: keys.query, tables, ctePlans, cteColumns })
+            semiJoins.push({ type: 'SemiJoin', expr: keys.left[0], keys: keys.left, not: false, right: { type: 'Subquery', child: right, scope: statementScope(keys.query) ?? [] } })
+            continue
+          }
+        }
+        residual = andPredicate(residual, conjunct)
+        continue
+      }
+      const right = planStatement({ stmt: expr.subquery, tables, ctePlans, cteColumns })
+      const scope = statementScope(expr.subquery)
+      semiJoins.push({
+        type: 'SemiJoin',
+        expr: expr.expr,
+        not,
+        right: scope ? { type: 'Subquery', child: right, scope } : right,
+      })
+    }
+    select = { ...select, where: residual }
+  }
   if (!select.joins.length) {
     hints.where = select.where
-    if (!needsBuffering && !select.distinct) {
+    if (!needsBuffering && !select.distinct && !semiJoins.length) {
       hints.limit = select.limit
       hints.offset = select.offset
     }
@@ -242,7 +310,6 @@ function planSelect({ select, ctePlans, cteColumns, tables, parentColumns, outer
   /** @type {QueryPlan} */
   let plan = planFrom({ select, ctePlans, cteColumns, hints, tables, outerScope })
   pruneAggregateColumns(plan, subqueryNeeds)
-  const directFrom = plan.type === 'Scan' && plan.hints === hints
 
   // Add JOINs
   if (select.joins.length) {
@@ -254,14 +321,15 @@ function planSelect({ select, ctePlans, cteColumns, tables, parentColumns, outer
 
   // Add WHERE filter when the scan didn't receive it
   let { where } = select
-  if (directFrom && select.joins.length === 1 && !outerScope?.length &&
-      !ctePlans?.has(select.joins[0].table?.toLowerCase())) {
-    where = pushJoinPredicates(plan, where, tables)
+  if (select.joins.length && !outerScope?.length) {
+    where = pushJoinPredicates(plan, where, select, tables, cteColumns)
   }
   if (where && !isOwnScan) {
     plan = { type: 'Filter', condition: where, child: plan }
   }
 
+  for (const semiJoin of semiJoins) plan = { ...semiJoin, left: plan }
+
   if (useGrouping) {
     // Aggregation path: GROUP BY or scalar aggregate
     // HAVING is integrated into aggregate nodes for access to group context
@@ -346,7 +414,7 @@ function planSelect({ select, ctePlans, cteColumns, tables, parentColumns, outer
       plan = { type: 'Distinct', child: plan }
     }
 
-    if (!(isOwnScan && !needsBuffering && !select.distinct) && (select.limit !== undefined || select.offset)) {
+    if (!(isOwnScan && !needsBuffering && !select.distinct && !semiJoins.length) && (select.limit !== undefined || select.offset)) {
       if (select.limit !== undefined) pushLimitIntoSort(plan, select.limit, select.offset)
       plan = { type: 'Limit', limit: select.limit, offset: select.offset, child: plan }
     }
@@ -837,27 +905,34 @@ function collectConjuncts(node, out) {
 }
 
 /**
- * Push total, table-local comparisons into fresh scans of a direct INNER
- * equijoin. Scan execution already enforces hints a source cannot apply.
- * Validate ON before moving anything; an unsafe ON also blocks WHERE
- * pushdown. Safe ON terms can move even when WHERE must remain above the
- * join, since ON already excludes those pairs before WHERE is evaluated.
+ * Push local ON and WHERE conjuncts according to which side is preserved.
+ * Only recurse through this SELECT's join chain, never through shared CTEs
+ * or derived-table boundaries. Scan hints are copied rather than mutated.
  *
  * @param {QueryPlan} plan
  * @param {ExprNode | undefined} where
+ * @param {SelectStatement} select
  * @param {Record<string, AsyncDataSource>} [tables]
+ * @param {Map<string, string[]>} [cteColumns]
+ * @param {((aliases: string[]) => boolean)[]} [inherited] - proofs required by ancestor filters or inner joins
  * @returns {ExprNode | undefined}
  */
-function pushJoinPredicates(plan, where, tables) {
-  if (!tables || plan.type !== 'HashJoin' || plan.joinType !== 'INNER' ||
-      plan.left.type !== 'Scan' || plan.right.type !== 'Scan') return where
-  const scans = [plan.left, plan.right]
-  if (scans.some(scan => scan.hints.where || scan.hints.limit !== undefined || scan.hints.offset)) return where
-  const aliases = scans.map(scan => scan.alias ?? scan.table)
-  if (aliases[0] === aliases[1]) return where
-  const columns = scans.map(scan => dataSourceColumns(tables[scan.table]))
-  // A prefix that also names a struct column is not an unambiguous alias.
-  if (columns.some(names => aliases.some(alias => names.includes(alias)))) return where
+function pushJoinPredicates(plan, where, select, tables, cteColumns, inherited = []) {
+  if (!tables || plan.type !== 'HashJoin' && plan.type !== 'NestedLoopJoin' ||
+      select.from?.type === 'function' || select.joins.some(join => join.fromFunction)) return where
+  const last = select.joins[select.joins.length - 1]
+  const leftSelect = { ...select, joins: select.joins.slice(0, -1) }
+  const leftSources = [select.from, ...leftSelect.joins.map(joinSource)]
+  const rightSource = joinSource(last)
+  const sources = [...leftSources, rightSource]
+  const aliases = sources.map(source => fromAlias(source))
+  const columns = sources.map(source => source?.type === 'subquery'
+    ? inferStatementColumns({ stmt: source.query, tables, cteColumns })
+    : source?.type === 'table'
+      ? cteColumns?.get(source.table?.toLowerCase()) ?? dataSourceColumns(tables[source.table])
+      : [])
+  if (new Set(aliases).size !== aliases.length ||
+      columns.some(names => aliases.some(alias => names.includes(alias)))) return where
 
   /**
    * @param {ExprNode} expr
@@ -865,67 +940,185 @@ function pushJoinPredicates(plan, where, tables) {
    */
   function owner(expr) {
     if (expr.type === 'literal') return 'literal'
-    if (expr.type !== 'identifier' || !expr.prefix) return 'unsafe'
-    const index = aliases.indexOf(expr.prefix)
+    if (expr.type !== 'identifier') return 'unsafe'
+    const matches = columns.map((names, i) => names.includes(expr.name) ? i : -1).filter(i => i >= 0)
+    const index = expr.prefix ? aliases.indexOf(expr.prefix) : matches.length === 1 ? matches[0] : -1
     if (index < 0 || !columns[index].includes(expr.name)) return 'unsafe'
-    return index === 0 ? 'left' : 'right'
+    return index < leftSources.length ? 'left' : 'right'
   }
 
-  // Avoid changing the evaluation of unresolved or unqualified ON keys.
-  if (plan.leftKeys.some(key => owner(key) !== 'left') || plan.rightKeys.some(key => owner(key) !== 'right')) return where
+  // Do not change the interpretation of ambiguous or misoriented hash keys.
+  if (plan.type === 'HashJoin' && (plan.leftKeys.some(key => owner(key) !== 'left') ||
+      plan.rightKeys.some(key => owner(key) !== 'right'))) return where
+
   /**
-   * @param {ExprNode[]} conjuncts
-   * @returns {Array<'left' | 'right' | 'residual'> | undefined}
+   * @param {ExprNode} expr
+   * @returns {'left' | 'right' | 'literal' | 'unsafe'}
    */
-  function classify(conjuncts) {
-    /** @type {Array<'left' | 'right' | 'residual'>} */
-    const targets = []
-    for (const expr of conjuncts) {
-      if (expr.type !== 'binary' || !['=', '==', '!=', '<>', '<', '<=', '>', '>='].includes(expr.op)) return undefined
-      const left = owner(expr.left)
-      const right = owner(expr.right)
-      if (left === 'unsafe' || right === 'unsafe') return undefined
-      // Only column/literal terms move; cross-table comparisons remain residual.
-      targets.push(left !== 'literal' && right === 'literal' ? left : right !== 'literal' && left === 'literal' ? right : 'residual')
+  function expressionOwner(expr) {
+    if (!expr) return 'literal'
+    if (expr.type === 'identifier' || expr.type === 'literal') return owner(expr)
+    /** @type {ExprNode[]} */
+    let children
+    if (expr.type === 'binary') children = [expr.left, expr.right]
+    else if (expr.type === 'unary') children = [expr.argument]
+    else if (expr.type === 'function' && !expr.filter) children = expr.args
+    else if (expr.type === 'cast') children = [expr.expr]
+    else if (expr.type === 'in valuelist') children = [expr.expr, ...expr.values]
+    else if (expr.type === 'subscript') children = [expr.expr, expr.index]
+    else if (expr.type === 'case') children = [expr.caseExpr, ...expr.whenClauses.flatMap(w => [w.condition, w.result]), expr.elseResult]
+    else return 'unsafe'
+    /** @type {'left' | 'right' | 'literal'} */
+    let target = 'literal'
+    for (const child of children) {
+      const side = expressionOwner(child)
+      if (side === 'unsafe' || side !== 'literal' && target !== 'literal' && side !== target) return 'unsafe'
+      if (side !== 'literal') target = side
     }
-    return targets
+    return target
   }
 
+  /** @type {ExprNode[]} */
+  const whereConjuncts = []
+  if (where) collectConjuncts(where, whereConjuncts)
+  /**
+   * Capture this scope so unqualified references remain unambiguous when
+   * proofs are passed down a join chain with fewer visible sources.
+   * @param {ExprNode} expr
+   * @returns {(missing: string[]) => boolean}
+   */
+  function rejectionProof(expr) {
+    return missing => rejectsNulls(expr, column => {
+      if (column.type !== 'identifier') return false
+      const matches = columns.map((names, i) => names.includes(column.name) ? i : -1).filter(i => i >= 0)
+      const index = column.prefix ? aliases.indexOf(column.prefix) : matches.length === 1 ? matches[0] : -1
+      return index >= 0 && columns[index].includes(column.name) && missing.includes(aliases[index])
+    })
+  }
+  const proofs = [...inherited, ...whereConjuncts.map(rejectionProof)]
+  const rejectsLeft = proofs.some(proof => proof(aliases.slice(0, -1)))
+  const rejectsRight = proofs.some(proof => proof(aliases.slice(-1)))
+  if (plan.joinType === 'FULL') {
+    if (rejectsLeft && rejectsRight) plan.joinType = 'INNER'
+    else if (rejectsLeft) plan.joinType = 'LEFT'
+    else if (rejectsRight) plan.joinType = 'RIGHT'
+  } else if (plan.joinType === 'LEFT' && rejectsRight || plan.joinType === 'RIGHT' && rejectsLeft) {
+    plan.joinType = 'INNER'
+  }
+
+  const { joinType } = plan
+  /** @type {ExprNode | undefined} */
+  let leftWhere
+  /** @type {ExprNode | undefined} */
+  let rightWhere
   /**
    * @param {ExprNode[]} conjuncts
-   * @param {Array<'left' | 'right' | 'residual'>} targets
+   * @param {boolean} on
    * @returns {ExprNode | undefined}
    */
-  function push(conjuncts, targets) {
+  function push(conjuncts, on) {
     /** @type {ExprNode | undefined} */
     let residual
-    for (let i = 0; i < conjuncts.length; i++) {
-      const target = targets[i]
-      if (target === 'residual') {
-        residual = andPredicate(residual, conjuncts[i])
-      } else {
-        const scan = target === 'left' ? scans[0] : scans[1]
-        scan.hints = { ...scan.hints, where: andPredicate(scan.hints.where, conjuncts[i]) }
-      }
+    for (const expr of conjuncts) {
+      const side = expressionOwner(expr)
+      const canLeft = joinType === 'INNER' || joinType === (on ? 'RIGHT' : 'LEFT')
+      const canRight = joinType === 'INNER' || joinType === (on ? 'LEFT' : 'RIGHT')
+      if (side === 'left' && canLeft) leftWhere = andPredicate(leftWhere, expr)
+      else if (side === 'right' && canRight) rightWhere = andPredicate(rightWhere, expr)
+      else residual = andPredicate(residual, expr)
     }
     return residual
   }
 
   /** @type {ExprNode[]} */
   const onConjuncts = []
-  if (plan.residual) collectConjuncts(plan.residual, onConjuncts)
-  const onTargets = classify(onConjuncts)
-  if (!onTargets) return where
-  const onResidual = push(onConjuncts, onTargets)
-  if (onResidual) plan.residual = onResidual
-  else delete plan.residual
-
-  if (!where) return undefined
-  /** @type {ExprNode[]} */
-  const whereConjuncts = []
-  collectConjuncts(where, whereConjuncts)
-  const whereTargets = classify(whereConjuncts)
-  return whereTargets ? push(whereConjuncts, whereTargets) : where
+  const condition = plan.type === 'HashJoin' ? plan.residual : plan.condition
+  if (condition) collectConjuncts(condition, onConjuncts)
+  const onResidual = push(onConjuncts, true)
+  if (plan.type === 'HashJoin') {
+    if (onResidual) plan.residual = onResidual
+    else delete plan.residual
+  } else if (condition) {
+    // Nested-loop joins still need a condition when every conjunct was pushed.
+    plan.condition = onResidual ?? {
+      type: 'literal', value: true,
+      positionStart: condition.positionStart, positionEnd: condition.positionEnd,
+    }
+  }
+  const residual = push(whereConjuncts, false)
+  if (leftSelect.joins.length) {
+    // An inner join requires its ON condition to succeed, so it can reject
+    // NULL rows from an earlier outer join. An outer join's own ON cannot.
+    const leftProofs = joinType === 'INNER' || joinType === 'LEFT' ? [...proofs] : []
+    if (joinType === 'INNER') {
+      if (condition) leftProofs.push(rejectionProof(condition))
+      if (plan.type === 'HashJoin') leftProofs.push(...plan.leftKeys.map(rejectionProof))
+    }
+    leftWhere = pushJoinPredicates(plan.left, leftWhere, leftSelect, tables, cteColumns, leftProofs)
+  }
+  plan.left = filterJoinInput(plan.left, leftWhere)
+  plan.right = filterJoinInput(plan.right, rightWhere)
+  return residual
+}
+
+/**
+ * @param {JoinClause} join
+ * @returns {FromSubquery | FromTable}
+ */
+function joinSource(join) {
+  return join.subquery
+    ? { ...join.subquery, alias: join.alias }
+    : { ...join, type: 'table' }
+}
+
+/**
+ * Keep LIMIT, projection and shared CTE plans intact. A filter outside these
+ * boundaries still reduces the rows entering the join.
+ *
+ * @param {QueryPlan} plan
+ * @param {ExprNode | undefined} where
+ * @returns {QueryPlan}
+ */
+function filterJoinInput(plan, where) {
+  if (!where) return plan
+  if (plan.type === 'Scan' && plan.hints.limit === undefined && !plan.hints.offset) {
+    return { ...plan, hints: { ...plan.hints, where: andPredicate(plan.hints.where, where) } }
+  }
+  return { type: 'Filter', condition: where, child: plan }
+}
+
+/**
+ * Conservative proof for an expression with references to the side being
+ * NULL-extended. Unknown functions (including COALESCE) and CASE
+ * are deliberately not assumed to propagate NULL.
+ *
+ * @param {ExprNode} expr
+ * @param {(expr: ExprNode) => boolean} isMissing
+ * @returns {boolean}
+ */
+function nullWhenMissing(expr, isMissing) {
+  if (expr.type === 'identifier') return isMissing(expr)
+  if (expr.type === 'literal') return expr.value == null
+  if (expr.type === 'cast') return nullWhenMissing(expr.expr, isMissing)
+  if (expr.type === 'unary') return ['NOT', '+', '-'].includes(expr.op) && nullWhenMissing(expr.argument, isMissing)
+  if (expr.type === 'in valuelist') return nullWhenMissing(expr.expr, isMissing)
+  if (expr.type === 'binary' && ['=', '==', '!=', '<>', '<', '>', '<=', '>=', 'LIKE', '+', '-', '*', '/', '%', '||'].includes(expr.op)) {
+    return nullWhenMissing(expr.left, isMissing) || nullWhenMissing(expr.right, isMissing)
+  }
+  return false
+}
+
+/**
+ * @param {ExprNode} expr
+ * @param {(expr: ExprNode) => boolean} isMissing
+ * @returns {boolean}
+ */
+function rejectsNulls(expr, isMissing) {
+  if (nullWhenMissing(expr, isMissing)) return true
+  if (expr.type === 'unary' && expr.op === 'IS NOT NULL') return nullWhenMissing(expr.argument, isMissing)
+  if (expr.type === 'binary' && expr.op === 'AND') return rejectsNulls(expr.left, isMissing) || rejectsNulls(expr.right, isMissing)
+  if (expr.type === 'binary' && expr.op === 'OR') return rejectsNulls(expr.left, isMissing) && rejectsNulls(expr.right, isMissing)
+  return false
 }
 
 /**
@@ -1163,3 +1356,54 @@ function isAllCountStar(columns) {
     !col.expr.filter
   )
 }
+
+/**
+ * Pulls plain equality correlations out of a row-preserving EXISTS query.
+ * Limits and aggregation must stay on the per-outer-row execution path.
+ *
+ * @param {Statement} query
+ * @param {string[]} outerScope
+ * @param {Record<string, AsyncDataSource>} [tables]
+ * @param {Map<string, string[]>} [cteColumns]
+ * @returns {{ query: SelectStatement, left: IdentifierNode[] } | undefined}
+ */
+function decorrelateExists(query, outerScope, tables, cteColumns) {
+  if (query.type !== 'select' || query.limit !== undefined || query.offset ||
+      query.groupBy.length || query.having ||
+      query.columns.some(col => col.type === 'derived' && (findAggregate(col.expr) || col.expr.type === 'window'))) return
+  const scope = statementScope(query) ?? []
+  /** @type {IdentifierNode[]} */
+  const left = []
+  /** @type {DerivedColumn[]} */
+  const columns = []
+  /** @type {ExprNode[]} */
+  const conjuncts = []
+  if (query.where) collectConjuncts(query.where, conjuncts)
+  /** @type {ExprNode | undefined} */
+  let where
+  for (const term of conjuncts) {
+    const previousKeys = columns.length
+    if (term.type === 'binary' && term.op === '=' && term.left.type === 'identifier' && term.right.type === 'identifier') {
+      for (const [inner, outer] of [[term.left, term.right], [term.right, term.left]]) {
+        if (inner.prefix && scope.includes(inner.prefix) && outer.prefix &&
+            outerScope.includes(outer.prefix) && !scope.includes(outer.prefix)) {
+          left.push(outer)
+          columns.push({ type: 'derived', expr: inner, alias: `__exists_key_${columns.length}`, positionStart: inner.positionStart, positionEnd: inner.positionEnd })
+          break
+        }
+      }
+      if (columns.length > previousKeys) continue
+    }
+    where = andPredicate(where, term)
+  }
+  if (!left.length) return
+  // Check every remaining clause, including nested subqueries and projections,
+  // before discarding the projection and ordering that EXISTS does not read.
+  /** @type {IdentifierNode[]} */
+  const remaining = []
+  collectColumnsFromExpr({ type: 'subquery', subquery: { ...query, where }, positionStart: query.positionStart, positionEnd: query.positionEnd }, remaining, undefined, {
+    tables, cteColumns, outerAliases: new Set(outerScope),
+  })
+  if (remaining.length) return
+  return { query: { ...query, columns, where, distinct: false, orderBy: [] }, left }
+}

package/src/plan/types.d.ts

@@ -11,6 +11,7 @@ export type QueryPlan =
   | LimitNode
   | HashAggregateNode
   | ScalarAggregateNode
+  | SemiJoinNode
   | HashJoinNode
   | NestedLoopJoinNode
   | PositionalJoinNode
@@ -100,6 +101,17 @@ export interface ScalarAggregateNode {
 }
 
 // Join nodes
+// WHERE IN / NOT IN: retain left rows using membership in the right output.
+export interface SemiJoinNode {
+  type: 'SemiJoin'
+  left: QueryPlan
+  right: QueryPlan
+  expr: ExprNode
+  // EXISTS equality keys; NOT uses anti-join rather than NOT IN null semantics.
+  keys?: ExprNode[]
+  not: boolean
+}
+
 export interface HashJoinNode {
   type: 'HashJoin'
   joinType: JoinType

package/src/validation/functions.js

@@ -131,7 +131,7 @@ export const FUNCTION_SIGNATURES = {
   SPLIT_PART: { min: 3, max: 3, signature: 'string, delimiter, index' },
   STRING_SPLIT: { min: 2, max: 2, signature: 'string, delimiter' },
   REGEXP_SUBSTR: { min: 2, max: 4, signature: 'string, pattern[, position[, occurrence]]' },
-  REGEXP_EXTRACT: { min: 2, max: 4, signature: 'string, pattern[, position[, occurrence]]' },
+  REGEXP_EXTRACT: { min: 2, max: 3, signature: 'string, pattern[, group]' },
   REGEXP_REPLACE: { min: 3, max: 5, signature: 'string, pattern, replacement[, position[, occurrence]]' },
   REGEXP_MATCHES: { min: 2, max: 2, signature: 'string, pattern' },
   REGEXP_LIKE: { min: 2, max: 2, signature: 'string, pattern' },