squirreling 0.12.23 → 0.12.25

package/README.md

@@ -12,6 +12,8 @@
 
 Squirreling is a streaming async SQL engine in pure JavaScript. Built for the browser from the ground up: streaming input and output, pluggable data sources, and lazy async cell evaluation. This makes Squirreling ideal for querying data from network sources, APIs, or LLMs where latency and cost matter.
 
+> Part of **[HypStack](https://hypstack.ai/)**, an open-source stack for AI observability.
+
 - **Standard SQL**: Full SQL support for querying data (read-only)
 - **Async UDFs**: User-defined functions can call APIs or models
 - **Tiny**: 13 kb bundle, zero dependencies, instant startup

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "squirreling",
-  "version": "0.12.23",
+  "version": "0.12.25",
   "description": "Squirreling Async SQL Engine",
   "author": "Hyperparam",
   "homepage": "https://hyperparam.app",
@@ -39,11 +39,11 @@
     "test": "vitest run"
   },
   "devDependencies": {
-    "@types/node": "25.9.1",
-    "@vitest/coverage-v8": "4.1.7",
+    "@types/node": "26.0.1",
+    "@vitest/coverage-v8": "4.1.9",
     "eslint": "9.39.4",
-    "eslint-plugin-jsdoc": "63.0.0",
+    "eslint-plugin-jsdoc": "63.0.10",
     "typescript": "6.0.3",
-    "vitest": "4.1.7"
+    "vitest": "4.1.9"
   }
 }

package/src/ast.d.ts

@@ -212,6 +212,7 @@ export interface JoinClause extends AstBase {
   on?: ExprNode
   using?: string[]
   fromFunction?: FromFunction
+  subquery?: FromSubquery
 }
 
 // All AST node derive from this base, which includes position info for error reporting and other purposes

package/src/backend/dataSource.js

@@ -1,5 +1,5 @@
 /**
- * @import { AsyncCells, AsyncDataSource, AsyncRow, ScanColumnOptions, SqlPrimitive } from '../types.js'
+ * @import { AsyncCells, AsyncDataSource, AsyncRow, SqlPrimitive } from '../types.js'
  */
 
 /**
@@ -71,13 +71,15 @@ export function memorySource({ data, columns }) {
 }
 
 /**
- * Wraps a data source that caches all accessed rows in memory
+ * Wraps a data source, memoizing accessed cells. The row cache is a WeakMap
+ * keyed on row identity so entries are collectible once the row is unreachable,
+ * keeping a streaming scan O(1) instead of O(rows).
  * @param {AsyncDataSource} source
  * @returns {AsyncDataSource}
  */
 export function cachedDataSource(source) {
-  /** @type {Map<string, Promise<SqlPrimitive>>} */
-  const cache = new Map()
+  /** @type {WeakMap<object, Map<string, Promise<SqlPrimitive>>>} */
+  const cache = new WeakMap()
   return {
     ...source,
     scan(options) {
@@ -90,68 +92,38 @@ export function cachedDataSource(source) {
         return { rows, appliedWhere, appliedLimitOffset }
       }
 
-      // Adjust index when source applied offset so cache keys match original rows
-      const indexOffset = appliedLimitOffset && options.offset ? options.offset : 0
-
       return {
         async *rows() {
-          let index = 0
           for await (const row of rows()) {
             if (options.signal?.aborted) break
-            const rowIndex = index + indexOffset
+            const anchor = row.resolved ?? row
+            let rowCache = cache.get(anchor)
+            if (!rowCache) {
+              rowCache = new Map()
+              cache.set(anchor, rowCache)
+            }
             /** @type {AsyncCells} */
             const cells = {}
             for (const key of row.columns) {
               const cell = row.cells[key]
-              // Wrap the cell to cache accesses
               cells[key] = () => {
-                const cacheKey = `${rowIndex}:${key}`
-                let value = cache.get(cacheKey)
+                let value = rowCache.get(key)
                 if (!value) {
                   value = cell()
-                  cache.set(cacheKey, value)
+                  rowCache.set(key, value)
                 }
                 return value
               }
             }
-            yield { columns: row.columns, cells }
-            index++
+            // Preserve resolved so downstream fast paths still apply.
+            yield { columns: row.columns, cells, resolved: row.resolved }
           }
         },
         appliedWhere,
         appliedLimitOffset,
       }
     },
-    ...source.scanColumn && {
-      /**
-       * @param {ScanColumnOptions} options
-       * @returns {AsyncIterable<ArrayLike<SqlPrimitive>>}
-       */
-      scanColumn(options) {
-        const inner = source.scanColumn(options)
-        const indexOffset = options.offset ?? 0
-        return (async function* () {
-          let chunkStart = 0
-          for await (const chunk of inner) {
-            if (options.signal?.aborted) break
-            /** @type {SqlPrimitive[]} */
-            const cached = new Array(chunk.length)
-            for (let i = 0; i < chunk.length; i++) {
-              const cacheKey = `${chunkStart + i + indexOffset}:${options.column}`
-              const existing = cache.get(cacheKey)
-              if (existing) {
-                cached[i] = await existing
-              } else {
-                const value = chunk[i]
-                cache.set(cacheKey, Promise.resolve(value))
-                cached[i] = value
-              }
-            }
-            yield cached
-            chunkStart += chunk.length
-          }
-        })()
-      },
-    },
+    // scanColumn passes through from ...source unwrapped: column values are
+    // already materialized, so caching them only retained memory unboundedly.
   }
 }

package/src/execute/aggregates.js

@@ -95,6 +95,7 @@ export function executeHashAggregate(plan, context) {
         }
         allRows.push(row)
       }
+      context.signal?.throwIfAborted()
 
       // Group rows by GROUP BY keys.
       // Each chunk dispatches all per-row key evaluations in parallel so
@@ -211,6 +212,7 @@ export function executeScalarAggregate(plan, context) {
         }
         group.push(row)
       }
+      context.signal?.throwIfAborted()
 
       const asyncRow = projectAggregateColumns(plan.columns, group, context)
 
@@ -282,9 +284,36 @@ function tryColumnScanAggregate(plan, { tables, signal }) {
     /** @type {AsyncCells} */
     const cells = {}
 
+    // Group specs by column so each column is scanned at most once no matter how
+    // many aggregates read it (e.g. MIN(x), MAX(x), AVG(x) share one pass).
+    /** @type {Map<string, ColumnAggSpec[]>} */
+    const specsByColumn = new Map()
+    for (const spec of specs) {
+      const group = specsByColumn.get(spec.column)
+      if (group) group.push(spec)
+      else specsByColumn.set(spec.column, [spec])
+    }
+
+    // Each column's single pass is computed once and shared by all its cells;
+    // a column is only scanned if one of its aggregates is actually read.
+    /** @type {Map<string, Promise<Map<string, SqlPrimitive>>>} */
+    const passes = new Map()
+    /**
+     * @param {string} column
+     * @returns {Promise<Map<string, SqlPrimitive>>}
+     */
+    function scanOnce(column) {
+      let pass = passes.get(column)
+      if (!pass) {
+        pass = scanColumnGroup({ table, specs: specsByColumn.get(column) ?? [], limit, offset, signal })
+        passes.set(column, pass)
+      }
+      return pass
+    }
+
     for (const spec of specs) {
       columns.push(spec.alias)
-      cells[spec.alias] = () => scanColumnAggregate({ table, spec, limit, offset, signal })
+      cells[spec.alias] = async () => (await scanOnce(spec.column)).get(spec.alias) ?? null
     }
 
     yield { columns, cells }
@@ -316,68 +345,93 @@ function extractColumnAggSpec({ expr, alias }) {
 }
 
 /**
- * Scans a single column and computes an aggregate value.
+ * @typedef {{
+ *   spec: ColumnAggSpec,
+ *   count: number,
+ *   sum: number,
+ *   min: SqlPrimitive,
+ *   max: SqlPrimitive,
+ *   seen: Set<unknown> | null,
+ * }} AggAccumulator
+ */
+
+/**
+ * Scans a column once and computes every aggregate over it in a single pass.
+ * All specs share the one scanColumn walk, so MIN(x)/MAX(x)/AVG(x) decode x once.
  *
  * @param {Object} options
  * @param {AsyncDataSource} options.table
- * @param {ColumnAggSpec} options.spec
+ * @param {ColumnAggSpec[]} options.specs - aggregates over the same column
  * @param {number} [options.limit]
  * @param {number} [options.offset]
  * @param {AbortSignal} [options.signal]
- * @returns {Promise<SqlPrimitive>}
+ * @returns {Promise<Map<string, SqlPrimitive>>} alias → aggregate value
  */
-async function scanColumnAggregate({ table, spec, limit, offset, signal }) {
-  const values = table.scanColumn({ column: spec.column, limit, offset, signal })
-
-  if (spec.funcName === 'COUNT' && spec.distinct) {
-    const seen = new Set()
-    for await (const chunk of values) {
-      if (signal?.aborted) return
-      for (let i = 0; i < chunk.length; i++) {
-        const v = chunk[i]
-        if (v == null) continue
-        seen.add(keyify(v))
-      }
-    }
-    return seen.size
-  }
-
-  if (spec.funcName === 'COUNT') {
-    let count = 0
-    for await (const chunk of values) {
-      if (signal?.aborted) return
-      for (let i = 0; i < chunk.length; i++) {
-        if (chunk[i] != null) count++
-      }
-    }
-    return count
-  }
-
-  // SUM, AVG, MIN, MAX
-  let sum = 0
-  let count = 0
-  /** @type {SqlPrimitive} */
-  let min = null
-  /** @type {SqlPrimitive} */
-  let max = null
+async function scanColumnGroup({ table, specs, limit, offset, signal }) {
+  const { column } = specs[0]
+  const values = table.scanColumn({ column, limit, offset, signal })
+
+  /** @type {AggAccumulator[]} */
+  const accs = specs.map(spec => /** @type {AggAccumulator} */ ({
+    spec,
+    count: 0,
+    sum: 0,
+    min: null,
+    max: null,
+    seen: spec.funcName === 'COUNT' && spec.distinct ? new Set() : null,
+  }))
 
   for await (const chunk of values) {
-    if (signal?.aborted) return
+    signal?.throwIfAborted()
     for (let i = 0; i < chunk.length; i++) {
       const v = chunk[i]
       if (v == null) continue
-      if (min === null || v < min) min = v
-      if (max === null || v > max) max = v
-      const num = Number(v)
-      if (!Number.isFinite(num)) continue
-      sum += num
-      count++
+      for (const acc of accs) {
+        switch (acc.spec.funcName) {
+        case 'COUNT':
+          if (acc.seen) acc.seen.add(keyify(v))
+          else acc.count++
+          break
+        case 'MIN':
+          if (acc.min === null || v < acc.min) acc.min = v
+          break
+        case 'MAX':
+          if (acc.max === null || v > acc.max) acc.max = v
+          break
+        default: { // SUM, AVG
+          const num = Number(v)
+          if (Number.isFinite(num)) {
+            acc.sum += num
+            acc.count++
+          }
+        }
+        }
+      }
     }
   }
+  signal?.throwIfAborted()
 
-  if (spec.funcName === 'SUM') return count === 0 ? null : sum
-  if (spec.funcName === 'AVG') return count === 0 ? null : sum / count
-  if (spec.funcName === 'MIN') return min
-  if (spec.funcName === 'MAX') return max
-  return null
+  /** @type {Map<string, SqlPrimitive>} */
+  const result = new Map()
+  for (const acc of accs) {
+    result.set(acc.spec.alias, finalizeAgg(acc))
+  }
+  return result
+}
+
+/**
+ * Reduces one accumulator to its final aggregate value.
+ *
+ * @param {AggAccumulator} acc
+ * @returns {SqlPrimitive}
+ */
+function finalizeAgg(acc) {
+  switch (acc.spec.funcName) {
+  case 'COUNT': return acc.seen ? acc.seen.size : acc.count
+  case 'SUM': return acc.count === 0 ? null : acc.sum
+  case 'AVG': return acc.count === 0 ? null : acc.sum / acc.count
+  case 'MIN': return acc.min
+  case 'MAX': return acc.max
+  default: return null
+  }
 }

package/src/execute/execute.js

@@ -356,16 +356,20 @@ function executeCount(plan, context) {
     numRows: 1,
     maxRows: 1,
     async *rows() {
-      // Use source numRows if available
-      const countPromise = table.numRows !== undefined ? Promise.resolve(table.numRows) : (async () => {
+      const countPromise = (async () => {
+        signal?.throwIfAborted()
+        // Use source numRows if available
+        if (table.numRows !== undefined) return table.numRows
+
         // Fall back to counting rows via scan
         let count = 0
         const { rows } = table.scan({ signal })
         // eslint-disable-next-line no-unused-vars
         for await (const _ of rows()) {
-          if (signal?.aborted) return
+          signal?.throwIfAborted()
           count++
         }
+        signal?.throwIfAborted()
         return count
       })()
 

package/src/execute/join.js

@@ -47,7 +47,7 @@ export function executeNestedLoopJoin(plan, context) {
 
       let innerCount = 0
       for await (const leftRow of left.rows()) {
-        if (context.signal?.aborted) break
+        if (context.signal?.aborted) return
 
         if (!leftCols) {
           leftCols = leftRow.columns
@@ -80,6 +80,8 @@ export function executeNestedLoopJoin(plan, context) {
         }
       }
 
+      if (context.signal?.aborted) return
+
       // Unmatched right rows for RIGHT/FULL joins
       if (matchedRightRows) {
         for (const rightRow of rightRows) {
@@ -248,7 +250,7 @@ export function executeHashJoin(plan, context) {
       // Probe phase: stream left rows
       let innerCount = 0
       for await (const leftRow of left.rows()) {
-        if (context.signal?.aborted) break
+        if (context.signal?.aborted) return
 
         if (!leftCols) {
           leftCols = leftRow.columns
@@ -285,6 +287,8 @@ export function executeHashJoin(plan, context) {
         }
       }
 
+      if (context.signal?.aborted) return
+
       // Unmatched right rows for RIGHT/FULL joins
       if (matchedRightRows) {
         for (const rightRow of rightRows) {

package/src/expression/binary.js

@@ -55,7 +55,7 @@ export function applyBinaryOp(op, a, b) {
       .replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
       .replace(/%/g, '.*')
       .replace(/_/g, '.')
-    const regex = new RegExp(`^${regexPattern}$`, 'i')
+    const regex = new RegExp(`^${regexPattern}$`, 'is')
     return regex.test(str)
   }
 

package/src/parse/extractTables.js

@@ -59,6 +59,8 @@ function walkStatement(stmt, cteScope, refs) {
   for (const j of stmt.joins) {
     if (j.fromFunction) {
       for (const a of j.fromFunction.args) walkExpr(a, cteScope, refs)
+    } else if (j.subquery) {
+      walkStatement(j.subquery.query, cteScope, refs)
     } else if (!cteScope.has(j.table.toLowerCase())) {
       refs.add(j.table)
     }

package/src/parse/joins.js

@@ -2,11 +2,11 @@ import { expectNoAggregate } from '../validation/aggregates.js'
 import { isTableFunction, validateFunctionArgs } from '../validation/functions.js'
 import { ParseError } from '../validation/parseErrors.js'
 import { parseExpression } from './expression.js'
-import { isTableFunctionStart, parseFromFunction, parseTableAlias, tableFunctionColumnCount, tableFunctionDefaultColumns } from './parse.js'
+import { isTableFunctionStart, parseFromFunction, parseStatement, parseTableAlias, tableFunctionColumnCount, tableFunctionDefaultColumns } from './parse.js'
 import { consume, current, expect, match } from './state.js'
 
 /**
- * @import { ExprNode, FromFunction, JoinClause, JoinType, ParserState } from '../types.js'
+ * @import { ExprNode, FromFunction, FromSubquery, JoinClause, JoinType, ParserState } from '../types.js'
  */
 
 /**
@@ -218,9 +218,42 @@ export function parseJoins(state) {
       })
     }
 
-    // Parse table name and optional alias
-    const tableTok = expect(state, 'identifier')
-    const tableAlias = parseTableAlias(state)
+    // Subquery on the right side: JOIN (SELECT ...) AS alias ON ...
+    const rightTok = current(state)
+    /** @type {FromSubquery | undefined} */
+    let subquery
+    let tableName
+    /** @type {string | undefined} */
+    let tableAlias
+    let endPos
+    if (rightTok.type === 'paren' && rightTok.value === '(') {
+      consume(state)
+      const query = parseStatement(state)
+      expect(state, 'paren', ')')
+      tableAlias = parseTableAlias(state)
+      if (!tableAlias) {
+        throw new ParseError({
+          message: 'Subquery in JOIN must have an alias',
+          positionStart: rightTok.positionStart,
+          positionEnd: state.lastPos,
+        })
+      }
+      endPos = state.lastPos
+      subquery = {
+        type: 'subquery',
+        query,
+        alias: tableAlias,
+        positionStart: rightTok.positionStart,
+        positionEnd: endPos,
+      }
+      tableName = tableAlias
+    } else {
+      // Parse table name and optional alias
+      const tableTok = expect(state, 'identifier')
+      tableName = tableTok.value
+      tableAlias = parseTableAlias(state)
+      endPos = tableTok.positionEnd
+    }
 
     // Parse ON condition or USING column list (not for POSITIONAL joins)
     /** @type {ExprNode | undefined} */
@@ -246,12 +279,13 @@ export function parseJoins(state) {
 
     joins.push({
       joinType,
-      table: tableTok.value,
+      table: tableName,
       alias: tableAlias,
       on: condition,
       using,
+      subquery,
       positionStart: tok.positionStart,
-      positionEnd: tableTok.positionEnd,
+      positionEnd: endPos,
     })
   }
 

package/src/plan/columns.js

@@ -420,6 +420,10 @@ export function inferSelectSourceColumns({ select, cteColumns, tables }) {
         for (const col of tableFunctionColumnNames(join.fromFunction)) {
           result.push(`${joinAlias}.${col}`)
         }
+      } else if (join.subquery) {
+        for (const col of inferStatementColumns({ stmt: join.subquery.query, cteColumns, tables })) {
+          result.push(`${joinAlias}.${col}`)
+        }
       } else {
         for (const col of lookupTableColumns(join.table, cteColumns, tables)) {
           result.push(`${joinAlias}.${col}`)
@@ -446,6 +450,10 @@ export function inferSelectSourceColumns({ select, cteColumns, tables }) {
       for (const col of tableFunctionColumnNames(join.fromFunction)) {
         result.push(`${joinAlias}.${col}`)
       }
+    } else if (join.subquery) {
+      for (const col of inferStatementColumns({ stmt: join.subquery.query, cteColumns, tables })) {
+        result.push(`${joinAlias}.${col}`)
+      }
     } else {
       for (const col of lookupTableColumns(join.table, cteColumns, tables)) {
         result.push(`${joinAlias}.${col}`)

package/src/plan/plan.js

@@ -447,18 +447,47 @@ function planJoin({ left, joins, leftTable, ctePlans, cteColumns, perTableColumn
       continue
     }
 
-    const ctePlan = ctePlans?.get(join.table.toLowerCase())
-    /** @type {ScanOptions} */
-    const rightHints = {}
-    if (!ctePlan) {
-      rightHints.columns = perTableColumns.get(rightTable)
-      validateScan({ ...join, hints: rightHints, tables })
+    /** @type {QueryPlan} */
+    let rightScan
+    if (join.subquery) {
+      // Subquery on the right side of the join (derived table). Mirror the
+      // FROM-clause subquery handling: plan the inner statement, push down the
+      // columns the outer query needs, and wrap in the inner scope so
+      // correlated subqueries inside resolve against the right aliases.
+      let subColumns = perTableColumns.get(rightTable)
+      // Empty array means no columns referenced, but the derived table still
+      // needs its own columns. Treat empty as unrestricted.
+      if (subColumns?.length === 0) subColumns = undefined
+      const subPlan = planStatement({
+        stmt: join.subquery.query,
+        ctePlans,
+        cteColumns,
+        tables,
+        outerScope,
+        parentColumns: subColumns?.map(name => ({ type: 'identifier', name, positionStart: 0, positionEnd: 0 })),
+      })
+      const availableColumns = inferStatementColumns({ stmt: join.subquery.query, cteColumns, tables })
+      if (subColumns && availableColumns.length) {
+        const missingColumn = subColumns.find(col => !availableColumns.includes(col))
+        if (missingColumn) {
+          throw new ColumnNotFoundError({ missingColumn, availableColumns, ...join.subquery })
+        }
+      }
+      const innerScope = statementScope(join.subquery.query)
+      rightScan = innerScope ? { type: 'Subquery', scope: innerScope, child: subPlan } : subPlan
     } else {
-      // For CTE joins, use CTE column metadata for hints
-      rightHints.columns = perTableColumns.get(rightTable) ?? cteColumns?.get(join.table.toLowerCase())
+      const ctePlan = ctePlans?.get(join.table.toLowerCase())
+      /** @type {ScanOptions} */
+      const rightHints = {}
+      if (!ctePlan) {
+        rightHints.columns = perTableColumns.get(rightTable)
+        validateScan({ ...join, hints: rightHints, tables })
+      } else {
+        // For CTE joins, use CTE column metadata for hints
+        rightHints.columns = perTableColumns.get(rightTable) ?? cteColumns?.get(join.table.toLowerCase())
+      }
+      rightScan = ctePlan ?? { type: 'Scan', table: join.table, hints: rightHints }
     }
-    /** @type {QueryPlan} */
-    const rightScan = ctePlan ?? { type: 'Scan', table: join.table, hints: rightHints }
 
     if (join.joinType === 'POSITIONAL') {
       plan = { type: 'PositionalJoin', leftAlias: currentLeftTable, rightAlias: rightTable, left: plan, right: rightScan }