squirreling 0.16.5 → 0.16.7

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "squirreling",
-  "version": "0.16.5",
+  "version": "0.16.7",
   "description": "Squirreling Async SQL Engine",
   "author": "Hyperparam",
   "homepage": "https://hyperparam.app",
@@ -40,12 +40,12 @@
   },
   "devDependencies": {
     "@eslint/js": "10.0.1",
-    "@types/node": "26.5.0",
-    "@vitest/coverage-v8": "5.0.0",
-    "eslint": "10.10.0",
-    "eslint-plugin-jsdoc": "64.3.6",
+    "@types/node": "26.6.2",
+    "@vitest/coverage-v8": "5.0.2",
+    "eslint": "10.11.0",
+    "eslint-plugin-jsdoc": "64.5.4",
     "globals": "17.12.0",
     "typescript": "7.0.2",
-    "vitest": "5.0.0"
+    "vitest": "5.0.2"
   }
 }

package/src/execute/execute.js

@@ -21,7 +21,7 @@ import { yieldToEventLoop } from './yield.js'
 
 /**
  * @import { BatchProjection, CompiledBatchExpression } from '../internalTypes.js'
- * @import { AsyncBatch, AsyncCells, AsyncDataSource, AsyncRow, ColumnDemand, ColumnVector, DerivedColumn, ExecuteContext, ExecuteSqlOptions, ExprNode, IdentifierNode, PreparedScan, QueryResults, RelationSchema, ScanRequest, SelectColumn, SqlPrimitive, Statement } from '../types.js'
+ * @import { AsyncBatch, AsyncCells, AsyncDataSource, AsyncRow, ColumnDemand, ColumnVector, DerivedColumn, ExecuteContext, ExecuteSqlOptions, ExprNode, IdentifierNode, NumericArray, PreparedScan, QueryResults, RelationSchema, ScanColumnResults, ScanRequest, ScanTopK, SelectColumn, SqlPrimitive, Statement } from '../types.js'
  * @import { CountNode, DistinctNode, FilterNode, LimitNode, ProjectNode, QueryPlan, ScanNode, SetOperationNode, TableFunctionNode } from '../plan/types.js'
  */
 
@@ -277,7 +277,7 @@ export function selectColumnNames(selectColumns, childColumns) {
 /**
  * @param {ScanNode} plan
  * @param {ExecuteContext} context
- * @param {import('../types.js').ScanColumnResults} [existingColumnResult]
+ * @param {ScanColumnResults} [existingColumnResult]
  * @returns {QueryResults}
  */
 export function executeScan(plan, context, existingColumnResult) {
@@ -539,12 +539,27 @@ function scanRequest(plan, schema, context) {
       mode: predicate ? 'required' : 'deferred',
     }
   })
-  return {
+  /** @type {ScanRequest} */
+  const request = {
     columns,
     filter: plan.hints.where,
     limit: plan.hints.limit,
     offset: plan.hints.offset,
   }
+  if (plan.topK) {
+    /** @type {ScanTopK['orderBy'][number][]} */
+    const orderBy = []
+    for (const term of plan.topK.orderBy) {
+      const { expr } = term
+      if (expr.type !== 'identifier' ||
+          expr.prefix && expr.prefix !== (plan.alias ?? plan.table)) return request
+      const field = schema.fields.find(field => field.name === expr.name)
+      if (!field) return request
+      orderBy.push({ field: field.id, direction: term.direction, nulls: term.nulls ?? 'FIRST' })
+    }
+    if (orderBy.length) request.topK = { orderBy, limit: plan.topK.limit }
+  }
+  return request
 }
 
 /**
@@ -1016,7 +1031,7 @@ function vectorFromChunk(chunk) {
 
 /**
  * @param {ArrayLike<SqlPrimitive>} values
- * @returns {values is import('../types.js').NumericArray}
+ * @returns {values is NumericArray}
  */
 function isNumericArray(values) {
   return values instanceof Int8Array

package/src/execute/sort.js

@@ -128,8 +128,16 @@ export async function sortEntriesByTerms({ entries, orderBy, context, cacheValue
  * @returns {QueryResults}
  */
 export function executeSort(plan, context) {
-  const child = executePlan({ plan: plan.child, context })
   const { topK } = plan
+  let childPlan = plan.child
+  // Only a directly adjacent scan can prune candidates for this sort. Never
+  // cross filters, joins, aggregates or windows, or change an existing scan
+  // slice. Copy the node: CTE plans can be shared by other consumers.
+  if (Number.isSafeInteger(topK) && topK > 0 && childPlan.type === 'Scan' &&
+      childPlan.hints.limit === undefined && !childPlan.hints.offset) {
+    childPlan = { ...childPlan, topK: { orderBy: plan.orderBy, limit: topK } }
+  }
+  const child = executePlan({ plan: childPlan, context })
   // With a LIMIT bound pushed into the sort, keep at most this many buffered
   // rows: periodically sort and discard everything past topK, so memory is
   // bounded by the limit instead of the input size.

package/src/index.d.ts

@@ -30,6 +30,7 @@ export type {
   ScanOptions,
   ScanProperties,
   ScanRequest,
+  ScanTopK,
   ScanResidual,
   ScanResults,
   SelectStatement,

package/src/plan/plan.js

@@ -1,3 +1,4 @@
+import { dataSourceColumns } from '../backend/dataSource.js'
 import { derivedAlias } from '../expression/alias.js'
 import { parseSql } from '../parse/parse.js'
 import { findAggregate } from '../validation/aggregates.js'
@@ -238,6 +239,7 @@ function planSelect({ select, ctePlans, cteColumns, tables, parentColumns, outer
   /** @type {QueryPlan} */
   let plan = planFrom({ select, ctePlans, cteColumns, hints, tables, outerScope })
   pruneAggregateColumns(plan, subqueryNeeds)
+  const directFrom = plan.type === 'Scan' && plan.hints === hints
 
   // Add JOINs
   if (select.joins.length) {
@@ -248,8 +250,13 @@ function planSelect({ select, ctePlans, cteColumns, tables, parentColumns, outer
   const isOwnScan = plan.type === 'Scan' && plan.hints === hints
 
   // Add WHERE filter when the scan didn't receive it
-  if (select.where && !isOwnScan) {
-    plan = { type: 'Filter', condition: select.where, child: plan }
+  let { where } = select
+  if (directFrom && select.joins.length === 1 && !outerScope?.length &&
+      !ctePlans?.has(select.joins[0].table?.toLowerCase())) {
+    where = pushJoinPredicates(plan, where, tables)
+  }
+  if (where && !isOwnScan) {
+    plan = { type: 'Filter', condition: where, child: plan }
   }
 
   if (useGrouping) {
@@ -826,6 +833,109 @@ function collectConjuncts(node, out) {
   out.push(node)
 }
 
+/**
+ * Push total, table-local comparisons into fresh scans of a direct INNER
+ * equijoin. Scan execution already enforces hints a source cannot apply.
+ * Validate ON before moving anything; an unsafe ON also blocks WHERE
+ * pushdown. Safe ON terms can move even when WHERE must remain above the
+ * join, since ON already excludes those pairs before WHERE is evaluated.
+ *
+ * @param {QueryPlan} plan
+ * @param {ExprNode | undefined} where
+ * @param {Record<string, AsyncDataSource>} [tables]
+ * @returns {ExprNode | undefined}
+ */
+function pushJoinPredicates(plan, where, tables) {
+  if (!tables || plan.type !== 'HashJoin' || plan.joinType !== 'INNER' ||
+      plan.left.type !== 'Scan' || plan.right.type !== 'Scan') return where
+  const scans = [plan.left, plan.right]
+  if (scans.some(scan => scan.hints.where || scan.hints.limit !== undefined || scan.hints.offset)) return where
+  const aliases = scans.map(scan => scan.alias ?? scan.table)
+  if (aliases[0] === aliases[1]) return where
+  const columns = scans.map(scan => dataSourceColumns(tables[scan.table]))
+  // A prefix that also names a struct column is not an unambiguous alias.
+  if (columns.some(names => aliases.some(alias => names.includes(alias)))) return where
+
+  /**
+   * @param {ExprNode} expr
+   * @returns {'left' | 'right' | 'literal' | 'unsafe'}
+   */
+  function owner(expr) {
+    if (expr.type === 'literal') return 'literal'
+    if (expr.type !== 'identifier' || !expr.prefix) return 'unsafe'
+    const index = aliases.indexOf(expr.prefix)
+    if (index < 0 || !columns[index].includes(expr.name)) return 'unsafe'
+    return index === 0 ? 'left' : 'right'
+  }
+
+  // Avoid changing the evaluation of unresolved or unqualified ON keys.
+  if (plan.leftKeys.some(key => owner(key) !== 'left') || plan.rightKeys.some(key => owner(key) !== 'right')) return where
+  /**
+   * @param {ExprNode[]} conjuncts
+   * @returns {Array<'left' | 'right' | 'residual'> | undefined}
+   */
+  function classify(conjuncts) {
+    /** @type {Array<'left' | 'right' | 'residual'>} */
+    const targets = []
+    for (const expr of conjuncts) {
+      if (expr.type !== 'binary' || !['=', '==', '!=', '<>', '<', '<=', '>', '>='].includes(expr.op)) return undefined
+      const left = owner(expr.left)
+      const right = owner(expr.right)
+      if (left === 'unsafe' || right === 'unsafe') return undefined
+      // Only column/literal terms move; cross-table comparisons remain residual.
+      targets.push(left !== 'literal' && right === 'literal' ? left : right !== 'literal' && left === 'literal' ? right : 'residual')
+    }
+    return targets
+  }
+
+  /**
+   * @param {ExprNode[]} conjuncts
+   * @param {Array<'left' | 'right' | 'residual'>} targets
+   * @returns {ExprNode | undefined}
+   */
+  function push(conjuncts, targets) {
+    /** @type {ExprNode | undefined} */
+    let residual
+    for (let i = 0; i < conjuncts.length; i++) {
+      const target = targets[i]
+      if (target === 'residual') {
+        residual = andPredicate(residual, conjuncts[i])
+      } else {
+        const scan = target === 'left' ? scans[0] : scans[1]
+        scan.hints = { ...scan.hints, where: andPredicate(scan.hints.where, conjuncts[i]) }
+      }
+    }
+    return residual
+  }
+
+  /** @type {ExprNode[]} */
+  const onConjuncts = []
+  if (plan.residual) collectConjuncts(plan.residual, onConjuncts)
+  const onTargets = classify(onConjuncts)
+  if (!onTargets) return where
+  const onResidual = push(onConjuncts, onTargets)
+  if (onResidual) plan.residual = onResidual
+  else delete plan.residual
+
+  if (!where) return undefined
+  /** @type {ExprNode[]} */
+  const whereConjuncts = []
+  collectConjuncts(where, whereConjuncts)
+  const whereTargets = classify(whereConjuncts)
+  return whereTargets ? push(whereConjuncts, whereTargets) : where
+}
+
+/**
+ * @param {ExprNode | undefined} left
+ * @param {ExprNode} right
+ * @returns {ExprNode}
+ */
+function andPredicate(left, right) {
+  return left
+    ? { type: 'binary', op: 'AND', left, right, positionStart: left.positionStart, positionEnd: right.positionEnd }
+    : right
+}
+
 /**
  * Returns the (leftKey, rightKey) pair for an equi conjunct, oriented so
  * leftKey references the left input and rightKey references the right input.

package/src/plan/types.d.ts

@@ -25,6 +25,9 @@ export interface ScanNode {
   table: string
   alias?: string
   hints: ScanOptions
+  // Execution-local hint from an immediately enclosing bounded Sort. Kept
+  // separate from scan LIMIT/OFFSET, which select an unordered input slice.
+  topK?: { orderBy: OrderByItem[], limit: number }
 }
 
 // Source for FROM-less SELECT like `SELECT 1`. Yields exactly one empty row.

package/src/types.d.ts

@@ -126,8 +126,25 @@ export interface ColumnDemand {
   mode: 'required' | 'deferred'
 }
 
+/**
+ * Optional candidate-pruning hint for an ordered slice. A source may ignore
+ * it, or omit rows proven unable to enter the first `limit` rows after the
+ * scan's filter. Preserve all boundary ties and the relative input order of
+ * retained rows. Return an unordered superset of the winners, not an arbitrary
+ * `limit` rows. The engine retains the final sort and slice.
+ */
+export interface ScanTopK {
+  orderBy: readonly {
+    field: number
+    direction: 'ASC' | 'DESC'
+    nulls: 'FIRST' | 'LAST'
+  }[]
+  limit: number // Includes the outer OFFSET.
+}
+
 export interface ScanRequest {
   columns: readonly ColumnDemand[]
+  topK?: ScanTopK
   filter?: ExprNode
   limit?: number
   offset?: number