squirreling 0.15.2 → 0.16.0

package/src/expression/evaluate.js

@@ -1,5 +1,5 @@
 import { executeStatement } from '../execute/execute.js'
-import { isPlainObject, keyify, sqlEquals, stringify } from '../execute/utils.js'
+import { isPlainObject, keyify, sqlEquals } from '../execute/utils.js'
 import { yieldToEventLoop } from '../execute/yield.js'
 import { ArgValueError, ExecutionError } from '../validation/executionErrors.js'
 import { isAggregateFunc, isMathFunc, isRegexpFunc, isSpatialFunc, isStringFunc } from '../validation/functions.js'
@@ -7,9 +7,10 @@ import { UnknownFunctionError } from '../validation/parseErrors.js'
 import { ColumnNotFoundError } from '../validation/tables.js'
 import { derivedAlias } from './alias.js'
 import { applyBinaryOp } from './binary.js'
-import { applyIntervalToDate, dateDiff, dateTrunc, extractField, toDate } from './date.js'
+import { applyIntervalToDate, dateDiff, dateTrunc, extractField } from './date.js'
 import { evaluateMathFunc } from './math.js'
 import { evaluateRegexpFunc } from './regexp.js'
+import { applyCast, evaluateJsonExtract } from './scalar.js'
 import { evaluateSpatialFunc } from '../spatial/spatial.js'
 import { evaluateStringFunc } from './strings.js'
 
@@ -74,6 +75,13 @@ export async function evaluateExpr({ node, row, rowIndex, rows, context }) {
       if (qualified in row.cells) {
         return row.cells[qualified]()
       }
+      if (context.scope?.includes(node.prefix) && node.name in row.cells) {
+        return row.cells[node.name]()
+      }
+      if (context.outerRow && context.outerAliases?.has(node.prefix)) {
+        if (qualified in context.outerRow.cells) return context.outerRow.cells[qualified]()
+        if (node.name in context.outerRow.cells) return context.outerRow.cells[node.name]()
+      }
       const prefix = node.prefix + '.'
       const prefixedColumns = row.columns.filter(col => col.startsWith(prefix))
       if (prefixedColumns.length === 1) {
@@ -93,10 +101,6 @@ export async function evaluateExpr({ node, row, rowIndex, rows, context }) {
           return value[node.name]
         }
       }
-      // Check outer row for correlated subquery references
-      if (context.outerRow && context.outerAliases?.has(node.prefix) && node.name in context.outerRow.cells) {
-        return context.outerRow.cells[node.name]()
-      }
       // Standalone `FROM UNNEST(...) AS alias` row has a single bare column;
       // `alias.field` should struct-access that column's element.
       if (context.scope?.includes(node.prefix) && row.columns.length === 1) {
@@ -167,7 +171,9 @@ export async function evaluateExpr({ node, row, rowIndex, rows, context }) {
       if (val == null) return null
       return -val
     }
-    if (node.op === 'NOT') return !val
+    // NOT UNKNOWN is UNKNOWN: JS ! would flip null to true and return rows
+    // SQL excludes, e.g. WHERE NOT (nullable < 300)
+    if (node.op === 'NOT') return val == null ? null : !val
     if (node.op === 'IS NULL') return val == null
     if (node.op === 'IS NOT NULL') return val != null
   }
@@ -186,9 +192,11 @@ export async function evaluateExpr({ node, row, rowIndex, rows, context }) {
 
     const left = await evaluateExpr({ node: node.left, row, rowIndex, rows, context })
 
-    // Short-circuit evaluation for AND and OR
-    if (node.op === 'AND' && !left) return false
-    if (node.op === 'OR' && left) return true
+    // Short-circuit evaluation for AND and OR. A null left cannot
+    // short-circuit AND: NULL AND FALSE is false but NULL AND TRUE is null,
+    // so the right side decides
+    if (node.op === 'AND' && left != null && !left) return false
+    if (node.op === 'OR' && left != null && Boolean(left)) return true
 
     const right = await evaluateExpr({ node: node.right, row, rowIndex, rows, context })
     return applyBinaryOp(node.op, left, right)
@@ -665,60 +673,7 @@ export async function evaluateExpr({ node, row, rowIndex, rows, context }) {
     }
 
     if (funcName === 'JSON_VALUE' || funcName === 'JSON_QUERY' || funcName === 'JSON_EXTRACT' || funcName === 'JSON_EXTRACT_STRING') {
-      let jsonArg = args[0]
-      const pathArg = args[1]
-      if (jsonArg == null || pathArg == null) return null
-
-      // Parse JSON if string, otherwise use directly
-      if (typeof jsonArg === 'string') {
-        try {
-          jsonArg = JSON.parse(jsonArg)
-        } catch {
-          throw new ArgValueError({
-            ...node,
-            message: 'invalid JSON string',
-            hint: 'First argument must be valid JSON.',
-            rowIndex,
-          })
-        }
-      }
-      if (typeof jsonArg !== 'object' || jsonArg instanceof Date) {
-        throw new ArgValueError({
-          ...node,
-          message: `first argument must be JSON string or object, got ${typeof jsonArg}`,
-          rowIndex,
-        })
-      }
-
-      // Parse path ("$.foo.bar[0].baz" or "foo.bar[0]")
-      const path = String(pathArg)
-      const normalizedPath = path.startsWith('$') ? path.slice(1) : path
-
-      // Navigate the path
-      let current = jsonArg
-      const segments = normalizedPath.match(/\.?([^.[]+)|\[(\d+)\]/g) || []
-      for (const segment of segments) {
-        if (current == null) return null
-        if (segment.startsWith('[')) {
-          // Array index access
-          const index = parseInt(segment.slice(1, -1), 10)
-          if (!Array.isArray(current)) return null
-          current = current[index]
-        } else {
-          // Property access
-          const key = segment.startsWith('.') ? segment.slice(1) : segment
-          if (typeof current !== 'object' || Array.isArray(current)) return null
-          current = current[key]
-        }
-      }
-
-      if (current == null) return null
-      // JSON_EXTRACT_STRING returns text: unquoted scalars, JSON text for objects and arrays
-      if (funcName === 'JSON_EXTRACT_STRING') {
-        if (typeof current === 'object') return JSON.stringify(current)
-        return String(current)
-      }
-      return current
+      return evaluateJsonExtract({ funcName, node, args, rowIndex })
     }
 
     // Check user-defined functions (case-insensitive lookup)
@@ -735,60 +690,31 @@ export async function evaluateExpr({ node, row, rowIndex, rows, context }) {
 
   if (node.type === 'cast') {
     const val = await evaluateExpr({ node: node.expr, row, rowIndex, rows, context })
-    if (val == null) return null
-    const { toType } = node
-    if (toType === 'TEXT' || toType === 'STRING' || toType === 'VARCHAR') {
-      if (typeof val === 'object') return stringify(val)
-      return String(val)
-    }
-    // Can only cast primitives (and Dates) to other primitive types
-    if (typeof val === 'object' && !(val instanceof Date)) {
-      if (node.tryCast) return null
-      throw new ExecutionError({ message: `Cannot CAST object to ${toType}`, rowIndex, ...node })
-    }
-    if (toType === 'INTEGER' || toType === 'INT') {
-      const num = Number(val)
-      if (isNaN(num)) return null
-      return Math.trunc(num)
-    }
-    if (toType === 'BIGINT') {
-      if (typeof val === 'bigint') return val
-      const num = Number(val)
-      // NaN and Infinity have no bigint representation
-      if (!isFinite(num)) return null
-      return BigInt(Math.trunc(num))
-    }
-    if (toType === 'FLOAT' || toType === 'REAL' || toType === 'DOUBLE') {
-      const num = Number(val)
-      if (isNaN(num)) return null
-      return num
-    }
-    if (toType === 'BOOLEAN' || toType === 'BOOL') {
-      return Boolean(val)
-    }
-    if (toType === 'TIMESTAMP') {
-      if (typeof val === 'number' || typeof val === 'bigint') {
-        const date = new Date(Number(val))
-        if (isNaN(date.getTime())) return null
-        return date
-      }
-      return toDate(val)
-    }
+    return applyCast(node, val, rowIndex)
   }
 
-  // IN and NOT IN with value lists
+  // IN and NOT IN with value lists. IN is a chain of OR'd equalities, so it
+  // inherits their three-valued logic: a null on either side of any equality
+  // makes a non-match UNKNOWN rather than false, and NOT above it must not
+  // turn that into true
   if (node.type === 'in valuelist') {
     const exprVal = await evaluateExpr({ node: node.expr, row, rowIndex, rows, context })
+    let sawNull = exprVal == null
     for (const valueNode of node.values) {
       const val = await evaluateExpr({ node: valueNode, row, rowIndex, rows, context })
-      if (sqlEquals(exprVal, val)) return true
+      if (val == null) {
+        sawNull = true
+      } else if (exprVal != null && sqlEquals(exprVal, val)) {
+        return true
+      }
     }
-    return false
+    return sawNull ? null : false
   }
-  // IN with subqueries
+  // IN with subqueries, same three-valued logic as the value-list form
   if (node.type === 'in') {
     const exprVal = await evaluateExpr({ node: node.expr, row, rowIndex, rows, context })
     const subResult = executeStatement({ query: node.subquery, context })
+    let sawNull = false
     let innerCount = 0
     for await (const resRow of subResult.rows()) {
       if (++innerCount % YIELD_INTERVAL === 0) {
@@ -796,9 +722,13 @@ export async function evaluateExpr({ node, row, rowIndex, rows, context }) {
         context.signal?.throwIfAborted()
       }
       const value = await resRow.cells[resRow.columns[0]]()
-      if (sqlEquals(exprVal, value)) return true
+      if (exprVal == null || value == null) {
+        sawNull = true
+      } else if (exprVal != null && sqlEquals(exprVal, value)) {
+        return true
+      }
     }
-    return false
+    return sawNull ? null : false
   }
 
   // EXISTS and NOT EXISTS with subqueries

package/src/expression/scalar.js

@@ -0,0 +1,105 @@
+import { stringify } from '../execute/utils.js'
+import { ArgValueError, ExecutionError } from '../validation/executionErrors.js'
+import { toDate } from './date.js'
+
+/**
+ * @import { CastNode, FunctionNode, SqlPrimitive } from '../types.js'
+ */
+
+/**
+ * Applies a scalar CAST without requiring a row evaluator.
+ *
+ * @param {CastNode} node
+ * @param {SqlPrimitive} value
+ * @param {number} [rowIndex]
+ * @returns {SqlPrimitive}
+ */
+export function applyCast(node, value, rowIndex) {
+  if (value == null) return null
+  const { toType } = node
+  if (toType === 'TEXT' || toType === 'STRING' || toType === 'VARCHAR') {
+    return typeof value === 'object' ? stringify(value) : String(value)
+  }
+  if (typeof value === 'object' && !(value instanceof Date)) {
+    if (node.tryCast) return null
+    throw new ExecutionError({ message: `Cannot CAST object to ${toType}`, rowIndex, ...node })
+  }
+  if (toType === 'INTEGER' || toType === 'INT') {
+    const number = Number(value)
+    return isNaN(number) ? null : Math.trunc(number)
+  }
+  if (toType === 'BIGINT') {
+    if (typeof value === 'bigint') return value
+    const number = Number(value)
+    return isFinite(number) ? BigInt(Math.trunc(number)) : null
+  }
+  if (toType === 'FLOAT' || toType === 'REAL' || toType === 'DOUBLE') {
+    const number = Number(value)
+    return isNaN(number) ? null : number
+  }
+  if (toType === 'BOOLEAN' || toType === 'BOOL') return Boolean(value)
+  if (typeof value === 'number' || typeof value === 'bigint') {
+    const date = new Date(Number(value))
+    return isNaN(date.getTime()) ? null : date
+  }
+  return toDate(value)
+}
+
+/**
+ * Applies the JSON extraction functions shared by row and batch evaluation.
+ *
+ * @param {object} options
+ * @param {string} options.funcName
+ * @param {FunctionNode} options.node
+ * @param {SqlPrimitive[]} options.args
+ * @param {number} [options.rowIndex]
+ * @returns {SqlPrimitive}
+ */
+export function evaluateJsonExtract({ funcName, node, args, rowIndex }) {
+  let json = args[0]
+  const pathArg = args[1]
+  if (json == null || pathArg == null) return null
+
+  if (typeof json === 'string') {
+    try {
+      json = JSON.parse(json)
+    } catch {
+      throw new ArgValueError({
+        ...node,
+        message: 'invalid JSON string',
+        hint: 'First argument must be valid JSON.',
+        rowIndex,
+      })
+    }
+  }
+  if (typeof json !== 'object' || json instanceof Date) {
+    throw new ArgValueError({
+      ...node,
+      message: `first argument must be JSON string or object, got ${typeof json}`,
+      rowIndex,
+    })
+  }
+
+  const path = String(pathArg)
+  const normalized = path.startsWith('$') ? path.slice(1) : path
+  /** @type {any} */
+  let current = json
+  const segments = normalized.match(/\.?([^.[]+)|\[(\d+)\]/g) || []
+  for (const segment of segments) {
+    if (current == null) return null
+    if (segment.startsWith('[')) {
+      if (!Array.isArray(current)) return null
+      current = current[parseInt(segment.slice(1, -1), 10)]
+    } else {
+      if (typeof current !== 'object' || Array.isArray(current)) return null
+      const key = segment.startsWith('.') ? segment.slice(1) : segment
+      current = current[key]
+    }
+  }
+
+  if (current == null) return null
+  if (funcName === 'JSON_EXTRACT_STRING') {
+    return typeof current === 'object' ? JSON.stringify(current) : String(current)
+  }
+  return current
+}

package/src/index.d.ts

@@ -1,19 +1,38 @@
-import type { AsyncDataSource, AsyncRow, ExecuteContext, ExecuteSqlOptions, ExprNode, ParseSqlOptions, PlanSqlOptions, QueryPlan, QueryResults, SqlPrimitive, Statement, Token } from './types.js'
+import type { AsyncBatch, AsyncDataSource, AsyncRow, ColumnResult, ColumnVector, ExecuteContext, ExecuteSqlOptions, ExprNode, ParseSqlOptions, PlanSqlOptions, QueryPlan, QueryResults, ReadBatchColumnOptions, RowsToBatchesOptions, RowSelection, SqlPrimitive, Statement, Token } from './types.js'
 export type {
+  AsyncBatch,
   AsyncCells,
   AsyncDataSource,
   AsyncRow,
+  BatchColumn,
+  ColumnDemand,
+  ColumnReadRequest,
+  ColumnResult,
+  ColumnVector,
   ExecuteContext,
   ExecuteSqlOptions,
   ExprNode,
+  Field,
+  NumericArray,
   ParseSqlOptions,
   PlanSqlOptions,
+  PreparedScan,
+  PrepareScan,
   QueryPlan,
   QueryResults,
+  ReadBatchColumnOptions,
+  ReadColumn,
+  RelationSchema,
+  RowsToBatchesOptions,
+  RowSelection,
   ScanOptions,
+  ScanProperties,
+  ScanRequest,
+  ScanResidual,
   ScanResults,
   SelectStatement,
   SetOperationStatement,
+  SqlType,
   SqlPrimitive,
   Statement,
   Token,
@@ -96,6 +115,30 @@ export function asyncRow(row: Record<string, SqlPrimitive>, columns: string[]):
 
 export function cachedDataSource(source: AsyncDataSource): AsyncDataSource
 
+export function selectedRowCount(selection: RowSelection): number
+
+export function composeSelections(outer: RowSelection, inner: RowSelection): RowSelection
+
+export function valueAt(vector: ColumnVector, index: number): SqlPrimitive
+
+export function selectVector(vector: ColumnVector, selection: RowSelection): ColumnVector
+
+export function readBatchColumn(options: ReadBatchColumnOptions): ColumnResult
+
+export function selectBatch(batch: AsyncBatch, selection: RowSelection): AsyncBatch
+
+export function rowsToBatches(
+  rows: AsyncIterable<AsyncRow>,
+  columns: string[],
+  options?: RowsToBatchesOptions,
+): AsyncIterable<AsyncBatch>
+
+export function batchesToRows(
+  batches: AsyncIterable<AsyncBatch>,
+  columns: string[],
+  signal?: AbortSignal,
+): AsyncIterable<AsyncRow>
+
 /**
  * Generates a default alias for a derived column expression.
  * Useful for generating column names pre-execution.

package/src/index.js

@@ -5,4 +5,13 @@ export { planSql } from './plan/plan.js'
 export { tokenizeSql } from './parse/tokenize.js'
 export { collect } from './execute/utils.js'
 export { asyncRow, cachedDataSource } from './backend/dataSource.js'
+export {
+  composeSelections,
+  readBatchColumn,
+  selectBatch,
+  selectedRowCount,
+  selectVector,
+  valueAt,
+} from './backend/batch.js'
+export { batchesToRows, rowsToBatches } from './backend/batchAdapters.js'
 export { derivedAlias } from './expression/alias.js'

package/src/internalTypes.d.ts

@@ -0,0 +1,28 @@
+import type { ColumnReadRequest, ColumnResult, ColumnVector, SqlPrimitive } from './types.js'
+
+export interface CompiledBatchExpression {
+  evaluate(context: ColumnReadRequest): ColumnResult
+}
+
+export type ValueKernel = (
+  vectors: ColumnVector[],
+  rowIndex: number,
+  streamRowIndex: number,
+) => SqlPrimitive
+
+export interface CompileState {
+  dependencies: number[]
+  dependencyPositions: Map<number, number>
+  columns: readonly string[]
+}
+
+export interface BatchAggregateInputs {
+  keys: CompiledBatchExpression[]
+  filters: (CompiledBatchExpression | undefined)[]
+  args: (CompiledBatchExpression | undefined)[]
+}
+
+export type BatchProjection =
+  | { type: 'column', columnIndex: number }
+  | { type: 'constant', value: SqlPrimitive }
+  | { type: 'expression', expression: CompiledBatchExpression }