squirreling 0.16.9 → 0.16.10

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "squirreling",
-  "version": "0.16.9",
+  "version": "0.16.10",
   "description": "Squirreling Async SQL Engine",
   "author": "Hyperparam",
   "homepage": "https://hyperparam.app",
@@ -41,11 +41,11 @@
   "devDependencies": {
     "@eslint/js": "10.0.1",
     "@types/node": "26.6.5",
-    "@vitest/coverage-v8": "5.0.4",
-    "eslint": "10.12.0",
-    "eslint-plugin-jsdoc": "65.3.4",
+    "@vitest/coverage-v8": "5.0.5",
+    "eslint": "10.13.0",
+    "eslint-plugin-jsdoc": "65.4.0",
     "globals": "17.13.0",
     "typescript": "7.0.2",
-    "vitest": "5.0.4"
+    "vitest": "5.0.5"
   }
 }

package/src/execute/execute.js

@@ -15,6 +15,7 @@ import { executeHashJoin, executeNestedLoopJoin, executePositionalJoin } from '.
 import { referencesRowScope } from './rowScope.js'
 import { normalizeScanColumnResult } from './scanColumn.js'
 import { executeSort } from './sort.js'
+import { executeSemiJoin } from './semiJoin.js'
 import { addBounds, minBounds, stableRowKey } from './utils.js'
 import { executeWindow } from './window.js'
 import { yieldToEventLoop } from './yield.js'
@@ -108,6 +109,8 @@ export function executePlan({ plan, context }) {
     return executeFilter(plan, context)
   } else if (plan.type === 'Project') {
     return executeProject(plan, context)
+  } else if (plan.type === 'SemiJoin') {
+    return executeSemiJoin(plan, context)
   } else if (plan.type === 'HashJoin') {
     return executeHashJoin(plan, context)
   } else if (plan.type === 'NestedLoopJoin') {

package/src/execute/semiJoin.js

@@ -0,0 +1,77 @@
+import { evaluateExpr } from '../expression/evaluate.js'
+import { executePlan } from './execute.js'
+import { sqlEquals } from './utils.js'
+import { yieldToEventLoop } from './yield.js'
+
+/**
+ * @import { ExecuteContext, QueryResults, SqlPrimitive } from '../types.js'
+ * @import { SemiJoinNode } from '../plan/types.js'
+ */
+
+/**
+ * Executes WHERE IN / NOT IN with one build of the right input per iteration.
+ * The membership set belongs to this operator, never to the context or AST.
+ *
+ * @param {SemiJoinNode} plan
+ * @param {ExecuteContext} context
+ * @returns {QueryResults}
+ */
+export function executeSemiJoin(plan, context) {
+  const left = executePlan({ plan: plan.left, context })
+  return {
+    columns: left.columns,
+    maxRows: left.maxRows,
+    async *rows() {
+      /** @type {Map<string, Set<SqlPrimitive>>} */
+      const values = new Map()
+      let hasNull = false
+      let count = 0
+      let built = false
+      let rowIndex = 0
+      for await (const row of left.rows()) {
+        context.signal?.throwIfAborted()
+        if (++rowIndex % 4000 === 0) await yieldToEventLoop()
+        const value = await evaluateExpr({ node: plan.expr, row, rowIndex, context })
+        if (!built) {
+          for await (const inner of executePlan({ plan: plan.right, context }).rows()) {
+            if (++count % 4000 === 0) {
+              await yieldToEventLoop()
+              context.signal?.throwIfAborted()
+            }
+            const value = await inner.cells[inner.columns[0]]()
+            if (value == null) hasNull = true
+            else {
+              const type = typeof value
+              let group = values.get(type)
+              if (!group) values.set(type, group = new Set())
+              group.add(value)
+            }
+          }
+          built = true
+        }
+        const matched = count === 0 ? false : value == null ? null : contains(values, value) ? true : hasNull ? null : false
+        if (matched !== null && matched !== plan.not) yield row
+      }
+    },
+  }
+}
+
+/**
+ * Same-type primitives use a Set. Cross-type values and Dates retain the
+ * evaluator's SQL equality, including coercion and timestamp comparison.
+ *
+ * @param {Map<string, Set<SqlPrimitive>>} values
+ * @param {SqlPrimitive} value
+ * @returns {boolean}
+ */
+function contains(values, value) {
+  const type = typeof value
+  if (type !== 'object' && !(typeof value === 'number' && Number.isNaN(value)) && values.get(type)?.has(value)) return true
+  for (const [valueType, group] of values) {
+    if (valueType === type && type !== 'object') continue
+    for (const candidate of group) {
+      if (sqlEquals(value, candidate)) return true
+    }
+  }
+  return false
+}

package/src/plan/plan.js

@@ -9,7 +9,7 @@ import { collectColumnsFromExpr, collectScopeColumns, extractColumns, fromAlias,
 
 /**
  * @import { AsyncDataSource, DerivedColumn, ExprNode, FromFunction, IdentifierNode, JoinClause, OrderByItem, PlanSqlOptions, ScanOptions, SelectColumn, SelectStatement, SetOperationStatement, Statement, WindowFunctionNode } from '../types.js'
- * @import { HashJoinNode, QueryPlan, TableFunctionNode, WindowSpec } from './types.js'
+ * @import { HashJoinNode, QueryPlan, SemiJoinNode, TableFunctionNode, WindowSpec } from './types.js'
  */
 
 /**
@@ -230,9 +230,46 @@ function planSelect({ select, ctePlans, cteColumns, tables, parentColumns, outer
   if (hints.columns?.length === 0 && select.from?.type === 'subquery') {
     hints.columns = undefined
   }
+  // A WHERE membership test is a semi-join, not a row-by-row expression
+  // subquery. Keep the original SELECT for column demand analysis above.
+  /** @type {Omit<SemiJoinNode, 'left'>[]} */
+  const semiJoins = []
+  if (select.where && !outerScope?.length) {
+    /** @type {ExprNode[]} */
+    const conjuncts = []
+    collectConjuncts(select.where, conjuncts)
+    /** @type {ExprNode | undefined} */
+    let residual
+    for (const conjunct of conjuncts) {
+      const not = conjunct.type === 'unary' && conjunct.op === 'NOT'
+      const expr = not ? conjunct.argument : conjunct
+      if (expr.type !== 'in') {
+        residual = andPredicate(residual, conjunct)
+        continue
+      }
+      /** @type {IdentifierNode[]} */
+      const correlated = []
+      collectColumnsFromExpr({ type: 'subquery', subquery: expr.subquery, positionStart: expr.positionStart, positionEnd: expr.positionEnd }, correlated, undefined, {
+        tables, cteColumns, outerAliases: new Set(scopeAliases),
+      })
+      if (correlated.length) {
+        residual = andPredicate(residual, conjunct)
+        continue
+      }
+      const right = planStatement({ stmt: expr.subquery, tables, ctePlans, cteColumns })
+      const scope = statementScope(expr.subquery)
+      semiJoins.push({
+        type: 'SemiJoin',
+        expr: expr.expr,
+        not,
+        right: scope ? { type: 'Subquery', child: right, scope } : right,
+      })
+    }
+    select = { ...select, where: residual }
+  }
   if (!select.joins.length) {
     hints.where = select.where
-    if (!needsBuffering && !select.distinct) {
+    if (!needsBuffering && !select.distinct && !semiJoins.length) {
       hints.limit = select.limit
       hints.offset = select.offset
     }
@@ -262,6 +299,8 @@ function planSelect({ select, ctePlans, cteColumns, tables, parentColumns, outer
     plan = { type: 'Filter', condition: where, child: plan }
   }
 
+  for (const semiJoin of semiJoins) plan = { ...semiJoin, left: plan }
+
   if (useGrouping) {
     // Aggregation path: GROUP BY or scalar aggregate
     // HAVING is integrated into aggregate nodes for access to group context
@@ -346,7 +385,7 @@ function planSelect({ select, ctePlans, cteColumns, tables, parentColumns, outer
       plan = { type: 'Distinct', child: plan }
     }
 
-    if (!(isOwnScan && !needsBuffering && !select.distinct) && (select.limit !== undefined || select.offset)) {
+    if (!(isOwnScan && !needsBuffering && !select.distinct && !semiJoins.length) && (select.limit !== undefined || select.offset)) {
       if (select.limit !== undefined) pushLimitIntoSort(plan, select.limit, select.offset)
       plan = { type: 'Limit', limit: select.limit, offset: select.offset, child: plan }
     }
@@ -837,11 +876,10 @@ function collectConjuncts(node, out) {
 }
 
 /**
- * Push total, table-local comparisons into fresh scans of a direct INNER
- * equijoin. Scan execution already enforces hints a source cannot apply.
- * Validate ON before moving anything; an unsafe ON also blocks WHERE
- * pushdown. Safe ON terms can move even when WHERE must remain above the
- * join, since ON already excludes those pairs before WHERE is evaluated.
+ * Push each table-local conjunct into fresh scans of a direct INNER
+ * equijoin. Scan execution enforces hints a source cannot apply. Unresolved
+ * and cross-table conjuncts remain residual predicates. Predicate evaluation
+ * order and error short-circuiting are not guaranteed.
  *
  * @param {QueryPlan} plan
  * @param {ExprNode | undefined} where
@@ -865,30 +903,54 @@ function pushJoinPredicates(plan, where, tables) {
    */
   function owner(expr) {
     if (expr.type === 'literal') return 'literal'
-    if (expr.type !== 'identifier' || !expr.prefix) return 'unsafe'
-    const index = aliases.indexOf(expr.prefix)
+    if (expr.type !== 'identifier') return 'unsafe'
+    const matches = columns.map((names, i) => names.includes(expr.name) ? i : -1).filter(i => i >= 0)
+    const index = expr.prefix ? aliases.indexOf(expr.prefix) : matches.length === 1 ? matches[0] : -1
     if (index < 0 || !columns[index].includes(expr.name)) return 'unsafe'
     return index === 0 ? 'left' : 'right'
   }
 
   // Avoid changing the evaluation of unresolved or unqualified ON keys.
   if (plan.leftKeys.some(key => owner(key) !== 'left') || plan.rightKeys.some(key => owner(key) !== 'right')) return where
+  /**
+   * Resolve every reference in a conjunct independently. Subqueries and
+   * windows have their own scope and must remain above the join.
+   *
+   * @param {ExprNode} expr
+   * @returns {'left' | 'right' | 'literal' | 'unsafe'}
+   */
+  function expressionOwner(expr) {
+    if (!expr) return 'literal'
+    if (expr.type === 'identifier' || expr.type === 'literal') return owner(expr)
+    /** @type {ExprNode[]} */
+    let children
+    if (expr.type === 'binary') children = [expr.left, expr.right]
+    else if (expr.type === 'unary') children = [expr.argument]
+    else if (expr.type === 'function' && !expr.filter) children = expr.args
+    else if (expr.type === 'cast') children = [expr.expr]
+    else if (expr.type === 'in valuelist') children = [expr.expr, ...expr.values]
+    else if (expr.type === 'subscript') children = [expr.expr, expr.index]
+    else if (expr.type === 'case') children = [expr.caseExpr, ...expr.whenClauses.flatMap(w => [w.condition, w.result]), expr.elseResult]
+    else return 'unsafe'
+    /** @type {'left' | 'right' | 'literal'} */
+    let target = 'literal'
+    for (const child of children) {
+      const side = expressionOwner(child)
+      if (side === 'unsafe' || side !== 'literal' && target !== 'literal' && side !== target) return 'unsafe'
+      if (side !== 'literal') target = side
+    }
+    return target
+  }
+
   /**
    * @param {ExprNode[]} conjuncts
-   * @returns {Array<'left' | 'right' | 'residual'> | undefined}
+   * @returns {Array<'left' | 'right' | 'residual'>}
    */
   function classify(conjuncts) {
-    /** @type {Array<'left' | 'right' | 'residual'>} */
-    const targets = []
-    for (const expr of conjuncts) {
-      if (expr.type !== 'binary' || !['=', '==', '!=', '<>', '<', '<=', '>', '>='].includes(expr.op)) return undefined
-      const left = owner(expr.left)
-      const right = owner(expr.right)
-      if (left === 'unsafe' || right === 'unsafe') return undefined
-      // Only column/literal terms move; cross-table comparisons remain residual.
-      targets.push(left !== 'literal' && right === 'literal' ? left : right !== 'literal' && left === 'literal' ? right : 'residual')
-    }
-    return targets
+    return conjuncts.map(function target(expr) {
+      const side = expressionOwner(expr)
+      return side === 'left' || side === 'right' ? side : 'residual'
+    })
   }
 
   /**
@@ -915,7 +977,6 @@ function pushJoinPredicates(plan, where, tables) {
   const onConjuncts = []
   if (plan.residual) collectConjuncts(plan.residual, onConjuncts)
   const onTargets = classify(onConjuncts)
-  if (!onTargets) return where
   const onResidual = push(onConjuncts, onTargets)
   if (onResidual) plan.residual = onResidual
   else delete plan.residual
@@ -925,7 +986,7 @@ function pushJoinPredicates(plan, where, tables) {
   const whereConjuncts = []
   collectConjuncts(where, whereConjuncts)
   const whereTargets = classify(whereConjuncts)
-  return whereTargets ? push(whereConjuncts, whereTargets) : where
+  return push(whereConjuncts, whereTargets)
 }
 
 /**

package/src/plan/types.d.ts

@@ -11,6 +11,7 @@ export type QueryPlan =
   | LimitNode
   | HashAggregateNode
   | ScalarAggregateNode
+  | SemiJoinNode
   | HashJoinNode
   | NestedLoopJoinNode
   | PositionalJoinNode
@@ -100,6 +101,15 @@ export interface ScalarAggregateNode {
 }
 
 // Join nodes
+// WHERE IN / NOT IN: retain left rows using membership in the right output.
+export interface SemiJoinNode {
+  type: 'SemiJoin'
+  left: QueryPlan
+  right: QueryPlan
+  expr: ExprNode
+  not: boolean
+}
+
 export interface HashJoinNode {
   type: 'HashJoin'
   joinType: JoinType