squirreling 0.9.1 → 0.9.2

package/README.md

@@ -47,8 +47,8 @@ const asyncRows: AsyncIterable<AsyncRow> = executeSql({
 })
 
 // Process rows as they arrive (streaming)
-for await (const { id, name } of asyncRows) {
-  console.log(`User id=${await id()}, name=${await name()}`)
+for await (const { cells } of asyncRows) {
+  console.log(`User id=${await cells.id()}, name=${await cells.name()}`)
 }
 ```
 
@@ -95,7 +95,7 @@ interface AsyncDataSource {
 }
 
 interface ScanOptions {
-  columns?: string[]
+  columns?: string[] // columns to scan (undefined means all)
   where?: ExprNode
   limit?: number
   offset?: number
@@ -128,11 +128,12 @@ const customSource: AsyncDataSource = {
 
 Squirreling mostly follows the SQL standard. The following features are supported:
 
-- `SELECT` statements with `WHERE`, `ORDER BY`, `LIMIT`, `OFFSET`
+- `SELECT` statements with `DISTINCT`, `WHERE`, `ORDER BY`, `LIMIT`, `OFFSET`
 - `WITH` clause for Common Table Expressions (CTEs)
 - Subqueries in `SELECT`, `FROM`, and `WHERE` clauses
-- `JOIN` operations: `INNER JOIN`, `LEFT JOIN`, `RIGHT JOIN`, `FULL JOIN`, `POSITIONAL JOIN`
+- `JOIN` operations: `INNER JOIN`, `LEFT JOIN`, `RIGHT JOIN`, `FULL JOIN`, `CROSS JOIN`, `POSITIONAL JOIN`
 - `GROUP BY` and `HAVING` clauses
+- Expressions: `CASE`, `CAST`, `BETWEEN`, `IN`, `LIKE`, `IS NULL`, `IS NOT NULL`
 
 ### Quoting
 
@@ -142,7 +143,7 @@ Squirreling mostly follows the SQL standard. The following features are supporte
 
 ### Functions
 
-- Aggregate: `COUNT`, `SUM`, `AVG`, `MIN`, `MAX`, `JSON_ARRAYAGG`
+- Aggregate: `COUNT`, `SUM`, `AVG`, `MIN`, `MAX`, `STDDEV_POP`, `STDDEV_SAMP`, `JSON_ARRAYAGG`
 - String: `CONCAT`, `SUBSTRING`, `REPLACE`, `LENGTH`, `UPPER`, `LOWER`, `TRIM`, `LEFT`, `RIGHT`, `INSTR`
 - Math: `ABS`, `SIGN`, `CEIL`, `FLOOR`, `ROUND`, `MOD`, `RAND`, `RANDOM`, `LN`, `LOG10`, `EXP`, `POWER`, `SQRT`
 - Trig: `SIN`, `COS`, `TAN`, `COT`, `ASIN`, `ACOS`, `ATAN`, `ATAN2`, `DEGREES`, `RADIANS`, `PI`

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "squirreling",
-  "version": "0.9.1",
+  "version": "0.9.2",
   "description": "Squirreling Async SQL Engine",
   "author": "Hyperparam",
   "homepage": "https://hyperparam.app",
@@ -37,10 +37,10 @@
     "test": "vitest run"
   },
   "devDependencies": {
-    "@types/node": "25.2.3",
+    "@types/node": "25.3.0",
     "@vitest/coverage-v8": "4.0.18",
     "eslint": "9.39.2",
-    "eslint-plugin-jsdoc": "62.6.0",
+    "eslint-plugin-jsdoc": "62.6.1",
     "typescript": "5.9.3",
     "vitest": "4.0.18"
   }

package/src/backend/dataSource.js

@@ -25,6 +25,7 @@ function asyncRow(obj) {
  */
 export function memorySource(data) {
   return {
+    numRows: data.length,
     scan({ where, limit, offset, signal }) {
       // Only apply offset and limit if no where clause
       const start = !where ? offset ?? 0 : 0

package/src/execute/execute.js

@@ -11,7 +11,7 @@ import { stableRowKey } from './utils.js'
 
 /**
  * @import { AsyncCells, AsyncDataSource, AsyncRow, ExecuteContext, ExecuteSqlOptions, ExprNode, SelectStatement } from '../types.js'
- * @import { DistinctNode, FilterNode, LimitNode, ProjectNode, QueryPlan, ScanNode } from '../plan/types.js'
+ * @import { CountNode, DistinctNode, FilterNode, LimitNode, ProjectNode, QueryPlan, ScanNode } from '../plan/types.js'
  */
 
 /**
@@ -61,6 +61,8 @@ export async function* executeSelect({ select, context }) {
 export async function* executePlan({ plan, context }) {
   if (plan.type === 'Scan') {
     yield* executeScan(plan, context)
+  } else if (plan.type === 'Count') {
+    yield* executeCount(plan, context)
   } else if (plan.type === 'Filter') {
     yield* executeFilter(plan, context)
   } else if (plan.type === 'Project') {
@@ -105,15 +107,15 @@ async function* executeScan(plan, context) {
   const { rows, appliedWhere, appliedLimitOffset } = scanResult
 
   // Applied limit/offset without applied where is invalid
-  const hasLimitOffset = plan.hints?.limit !== undefined || plan.hints?.offset // 0 offset is noop
-  if (!appliedWhere && appliedLimitOffset && plan.hints?.where && hasLimitOffset) {
+  const hasLimitOffset = plan.hints.limit !== undefined || plan.hints.offset // 0 offset is noop
+  if (!appliedWhere && appliedLimitOffset && plan.hints.where && hasLimitOffset) {
     throw new Error(`Data source "${plan.table}" applied limit/offset without applying where`)
   }
 
   let result = rows
 
   // Apply WHERE if data source did not
-  if (!appliedWhere && plan.hints?.where) {
+  if (!appliedWhere && plan.hints.where) {
     result = filterRows(result, plan.hints.where, context)
   }
 
@@ -125,6 +127,44 @@ async function* executeScan(plan, context) {
   yield* result
 }
 
+/**
+ * Executes a Count node using numRows when available, falling back to scan
+ *
+ * @param {CountNode} plan
+ * @param {ExecuteContext} context
+ * @yields {AsyncRow}
+ */
+async function* executeCount(plan, { tables, signal }) {
+  const dataSource = tables[plan.table]
+  if (dataSource === undefined) {
+    throw tableNotFoundError({ tableName: plan.table })
+  }
+
+  // Use source numRows if available
+  let count = dataSource.numRows
+  if (dataSource.numRows === undefined) {
+    // Fall back to counting rows via scan
+    count = 0
+    const { rows } = dataSource.scan({ signal })
+    // eslint-disable-next-line no-unused-vars
+    for await (const _ of rows) {
+      if (signal?.aborted) return
+      count++
+    }
+  }
+
+  /** @type {string[]} */
+  const columns = []
+  /** @type {AsyncCells} */
+  const cells = {}
+  for (const col of plan.columns) {
+    const alias = col.alias ?? derivedAlias(col.expr)
+    columns.push(alias)
+    cells[alias] = () => Promise.resolve(count)
+  }
+  yield { columns, cells }
+}
+
 /**
  * Filters rows by a condition
  *

package/src/expression/alias.js

@@ -30,7 +30,7 @@ export function derivedAlias(expr) {
   }
   if (expr.type === 'function') {
     // Handle aggregate functions with star (COUNT(*) -> count_all)
-    if (expr.args.length === 1 && expr.args[0].type === 'identifier' && expr.args[0].name === '*') {
+    if (expr.args.length === 1 && expr.args[0].type === 'star') {
       return expr.name.toLowerCase() + '_all'
     }
     return expr.name.toLowerCase() + '_' + expr.args.map(derivedAlias).join('_')

package/src/expression/evaluate.js

@@ -127,14 +127,13 @@ export async function evaluateExpr({ node, row, rowIndex, rows, context }) {
         }
       }
 
-      // Handle COUNT(*) special case
-      if (node.args.length === 1 && node.args[0].type === 'identifier' && funcName === 'COUNT' && node.args[0].name === '*') {
-        return filteredRows.length
-      }
-
       const argNode = node.args[0]
-
       if (funcName === 'COUNT') {
+        // COUNT(*) special case
+        if (argNode.type === 'star') {
+          return filteredRows.length
+        }
+
         if (node.distinct) {
           const seen = new Set()
           for (const row of filteredRows) {

package/src/expression/regexp.js

@@ -10,8 +10,8 @@ import { argValueError } from '../validationErrors.js'
  * @param {Object} options
  * @param {string} options.funcName - Uppercase function name
  * @param {SqlPrimitive[]} options.args - Function arguments
- * @param {number} [options.positionStart] - Start position in SQL string for error reporting
- * @param {number} [options.positionEnd] - End position in SQL string for error reporting
+ * @param {number} options.positionStart - Start position in SQL string for error reporting
+ * @param {number} options.positionEnd - End position in SQL string for error reporting
  * @param {number} [options.rowIndex] - Row number for error reporting
  * @returns {SqlPrimitive}
  */

package/src/expression/strings.js

@@ -10,8 +10,8 @@ import { argValueError } from '../validationErrors.js'
  * @param {Object} options
  * @param {StringFunc} options.funcName - Uppercase function name
  * @param {SqlPrimitive[]} options.args - Function arguments
- * @param {number} [options.positionStart] - Start position for error reporting
- * @param {number} [options.positionEnd] - End position for error reporting
+ * @param {number} options.positionStart - Start position for error reporting
+ * @param {number} options.positionEnd - End position for error reporting
  * @param {number} [options.rowIndex] - Row index for error reporting
  * @returns {SqlPrimitive}
  */

package/src/parse/expression.js

@@ -194,6 +194,8 @@ export function parsePrimary(state) {
         throw missingClauseError({
           missing: 'at least one WHEN clause',
           context: 'CASE expression',
+          positionStart,
+          positionEnd: state.lastPos,
         })
       }
 

package/src/parse/functions.js

@@ -38,8 +38,7 @@ export function parseFunctionCall(state, funcName, positionStart) {
         const starTok = current(state)
         consume(state)
         args.push({
-          type: 'identifier',
-          name: '*',
+          type: 'star',
           positionStart: starTok.positionStart,
           positionEnd: state.lastPos,
         })
@@ -74,7 +73,7 @@ export function parseFunctionCall(state, funcName, positionStart) {
 
   // Validate star argument at parse time (only COUNT supports *)
   const funcNameUpper = funcName.toUpperCase()
-  const hasStar = args.length === 1 && args[0].type === 'identifier' && args[0].name === '*'
+  const hasStar = args.length === 1 && args[0].type === 'star'
   if (hasStar && isAggregateFunc(funcNameUpper) && funcNameUpper !== 'COUNT') {
     throw new ParseError({
       message: `${funcName} cannot be applied to "*"`,

package/src/parseErrors.js

@@ -139,8 +139,8 @@ export function argCountParseError({ funcName, expected, received, positionStart
  * @param {Object} options
  * @param {string} options.missing - What is missing (e.g., 'WHEN clause', 'FROM clause', 'ON condition')
  * @param {string} options.context - Where it's missing from (e.g., 'CASE expression', 'SELECT statement', 'JOIN')
- * @param {number} [options.positionStart] - Start position in query
- * @param {number} [options.positionEnd] - End position in query
+ * @param {number} options.positionStart - Start position in query
+ * @param {number} options.positionEnd - End position in query
  * @returns {ParseError}
  */
 export function missingClauseError({ missing, context, positionStart, positionEnd }) {

package/src/plan/columns.js

@@ -10,11 +10,13 @@
  * @returns {Map<string, string[] | undefined>}
  */
 export function extractColumns(select) {
+  /** @type {Map<string, string[] | undefined>} */
+  const result = new Map()
+
   // Build alias list from FROM + JOINs
   const fromAlias = select.from.kind === 'table'
     ? select.from.alias ?? select.from.table
     : select.from.alias
-  /** @type {string[]} */
   const aliases = [fromAlias]
   for (const join of select.joins) {
     aliases.push(join.alias ?? join.table)
@@ -30,20 +32,18 @@ export function extractColumns(select) {
     return result
   }
 
-  // Track which tables need all columns (SELECT table.*)
-  /** @type {Set<string>} */
-  const allColumnsNeeded = new Set()
-  for (const col of select.columns) {
-    if (col.kind === 'star' && col.table) {
-      allColumnsNeeded.add(col.table)
-    }
-  }
+  // Track per-table columns needed; undefined means all columns (table.*)
+  /** @type {Map<string, Set<string> | undefined>} */
+  const perTable = new Map(aliases.map(alias => [alias, new Set()]))
 
   // Collect all identifiers from all clauses
   /** @type {Set<string>} */
   const identifiers = new Set()
   for (const col of select.columns) {
-    if (col.kind === 'derived') {
+    if (col.kind === 'star' && col.table) {
+      // SELECT table.* means all columns needed
+      perTable.set(col.table, undefined)
+    } else if (col.kind === 'derived') {
       collectColumnsFromExpr(col.expr, identifiers)
     }
   }
@@ -59,15 +59,6 @@ export function extractColumns(select) {
     collectColumnsFromExpr(join.on, identifiers)
   }
 
-  // Initialize per-table sets (skip tables needing all columns)
-  /** @type {Map<string, Set<string>>} */
-  const perTable = new Map()
-  for (const alias of aliases) {
-    if (!allColumnsNeeded.has(alias)) {
-      perTable.set(alias, new Set())
-    }
-  }
-
   // Partition identifiers by table prefix
   for (const name of identifiers) {
     const dotIndex = name.indexOf('.')
@@ -76,27 +67,19 @@ export function extractColumns(select) {
       const tablePrefix = name.substring(0, dotIndex)
       const columnName = name.substring(dotIndex + 1)
       const set = perTable.get(tablePrefix)
-      if (set) {
-        set.add(columnName)
-      }
+      if (set) set.add(columnName)
     } else {
       // Unqualified: add to all tables (ambiguous)
       for (const [, set] of perTable) {
-        set.add(name)
+        if (set) set.add(name)
       }
     }
   }
 
   // Build result map: convert Sets to arrays, undefined for all-columns tables
-  /** @type {Map<string, string[] | undefined>} */
-  const result = new Map()
   for (const alias of aliases) {
-    if (allColumnsNeeded.has(alias)) {
-      result.set(alias, undefined)
-    } else {
-      const set = perTable.get(alias)
-      result.set(alias, set ? [...set] : undefined)
-    }
+    const set = perTable.get(alias)
+    result.set(alias, set ? [...set] : undefined)
   }
   return result
 }
@@ -104,15 +87,13 @@ export function extractColumns(select) {
 /**
  * Recursively collects column names (identifiers) from an expression
  *
- * @param {ExprNode | undefined} expr
+ * @param {ExprNode} expr
  * @param {Set<string>} columns
  */
 function collectColumnsFromExpr(expr, columns) {
   if (!expr) return
-  if (expr.type === 'identifier' && expr.name !== '*') {
+  if (expr.type === 'identifier') {
     columns.add(expr.name)
-  } else if (expr.type === 'literal') {
-    // No columns
   } else if (expr.type === 'binary') {
     collectColumnsFromExpr(expr.left, columns)
     collectColumnsFromExpr(expr.right, columns)
@@ -122,6 +103,7 @@ function collectColumnsFromExpr(expr, columns) {
     for (const arg of expr.args) {
       collectColumnsFromExpr(arg, columns)
     }
+    collectColumnsFromExpr(expr.filter, columns)
   } else if (expr.type === 'cast') {
     collectColumnsFromExpr(expr.expr, columns)
   } else if (expr.type === 'in valuelist') {
@@ -131,9 +113,6 @@ function collectColumnsFromExpr(expr, columns) {
     }
   } else if (expr.type === 'in') {
     collectColumnsFromExpr(expr.expr, columns)
-    // Subquery columns are from a different scope, don't collect
-  } else if (expr.type === 'exists' || expr.type === 'not exists') {
-    // Subquery columns are from a different scope, don't collect
   } else if (expr.type === 'case') {
     if (expr.caseExpr) {
       collectColumnsFromExpr(expr.caseExpr, columns)
@@ -146,4 +125,5 @@ function collectColumnsFromExpr(expr, columns) {
       collectColumnsFromExpr(expr.elseResult, columns)
     }
   }
+  // No columns: count(*), literal, interval, exists, not exists, subquery
 }

package/src/plan/plan.js

@@ -3,7 +3,7 @@ import { findAggregate } from '../validation.js'
 import { extractColumns } from './columns.js'
 
 /**
- * @import { ExprNode, JoinClause, PlanSqlOptions, ScanOptions, SelectStatement } from '../types.js'
+ * @import { ExprNode, DerivedColumn, JoinClause, PlanSqlOptions, ScanOptions, SelectColumn, SelectStatement } from '../types.js'
  * @import { QueryPlan } from './types.d.ts'
  */
 
@@ -51,11 +51,19 @@ function planSelect({ select, ctePlans }) {
     ? select.from.alias ?? select.from.table
     : select.from.alias
 
-  // Determine per-table column hints for pushdown
+  // Determine scan hints for direct table scans (WHERE and LIMIT/OFFSET are
+  // included so they are only applied to fresh scans, not CTE/subquery plans)
   /** @type {ScanOptions} */
   const hints = {}
   const perTableColumns = extractColumns(select)
   hints.columns = perTableColumns.get(sourceAlias)
+  if (!select.joins.length) {
+    hints.where = select.where
+    if (!needsBuffering && !select.distinct) {
+      hints.limit = select.limit
+      hints.offset = select.offset
+    }
+  }
 
   // Start with the data source (FROM clause)
   /** @type {QueryPlan} */
@@ -66,27 +74,24 @@ function planSelect({ select, ctePlans }) {
     plan = planJoin({ left: plan, joins: select.joins, leftTable: sourceAlias, ctePlans, perTableColumns })
   }
 
-  // Delegate WHERE and LIMIT/OFFSET to scan when plan is a direct table scan
-  if (plan.type === 'Scan') {
-    plan.hints.where = select.where
-    if (!needsBuffering && !select.distinct) {
-      plan.hints.limit = select.limit
-      plan.hints.offset = select.offset
-    }
-  }
+  // Whether FROM resolved to our own direct table scan
+  const isOwnScan = plan.type === 'Scan' && plan.hints === hints
 
-  // Add WHERE filter when scan can't handle it (JOINs, subqueries, CTEs)
-  const isScan = plan.type === 'Scan'
-  if (select.where && !isScan) {
+  // Add WHERE filter when the scan didn't receive it
+  if (select.where && !isOwnScan) {
     plan = { type: 'Filter', condition: select.where, child: plan }
   }
 
   if (useGrouping) {
     // Aggregation path: GROUP BY or scalar aggregate
     // HAVING is integrated into aggregate nodes for access to group context
-    plan = select.groupBy.length
-      ? { type: 'HashAggregate', groupBy: select.groupBy, columns: select.columns, having: select.having, child: plan }
-      : { type: 'ScalarAggregate', columns: select.columns, having: select.having, child: plan }
+    if (select.groupBy.length) {
+      plan = { type: 'HashAggregate', groupBy: select.groupBy, columns: select.columns, having: select.having, child: plan }
+    } else if (!select.having && !select.where && plan.type === 'Scan' && isOwnScan && isAllCountStar(select.columns)) {
+      plan = { type: 'Count', table: plan.table, columns: select.columns }
+    } else {
+      plan = { type: 'ScalarAggregate', columns: select.columns, having: select.having, child: plan }
+    }
 
     // ORDER BY (after aggregation)
     if (select.orderBy.length) {
@@ -99,7 +104,7 @@ function planSelect({ select, ctePlans }) {
     }
 
     // LIMIT/OFFSET
-    if (select.limit !== undefined || select.offset !== undefined) {
+    if (select.limit !== undefined || select.offset) {
       plan = { type: 'Limit', limit: select.limit, offset: select.offset, child: plan }
     }
   } else {
@@ -124,13 +129,18 @@ function planSelect({ select, ctePlans }) {
     // DISTINCT needs to come after projection but before LIMIT
     // However, for streaming distinct we need to project first
     // So the order is: Sort -> Project -> Distinct -> Limit
-    plan = { type: 'Project', columns: select.columns, child: plan }
+
+    // Fast path for SELECT *
+    const isPassthrough = select.columns.length === 1 && select.columns[0].kind === 'star'
+    if (!isPassthrough) {
+      plan = { type: 'Project', columns: select.columns, child: plan }
+    }
 
     if (select.distinct) {
       plan = { type: 'Distinct', child: plan }
     }
 
-    if (!(isScan && !needsBuffering && !select.distinct) && (select.limit !== undefined || select.offset !== undefined)) {
+    if (!(isOwnScan && !needsBuffering && !select.distinct) && (select.limit !== undefined || select.offset)) {
       plan = { type: 'Limit', limit: select.limit, offset: select.offset, child: plan }
     }
   }
@@ -302,3 +312,22 @@ function extractSimpleJoinKeys({ condition, leftTable, rightTable }) {
 
   return { leftKey: left, rightKey: right }
 }
+
+/**
+ * Checks if every SELECT column is a plain COUNT(*).
+ *
+ * @param {SelectColumn[]} columns
+ * @returns {columns is DerivedColumn[]}
+ */
+function isAllCountStar(columns) {
+  if (columns.length === 0) return false
+  return columns.every(col =>
+    col.kind === 'derived' &&
+    col.expr.type === 'function' &&
+    col.expr.name.toUpperCase() === 'COUNT' &&
+    col.expr.args.length === 1 &&
+    col.expr.args[0].type === 'star' &&
+    !col.expr.distinct &&
+    !col.expr.filter
+  )
+}

package/src/plan/types.d.ts

@@ -1,7 +1,8 @@
-import { ExprNode, JoinType, OrderByItem, ScanOptions, SelectColumn } from '../types.js'
+import { DerivedColumn, ExprNode, JoinType, OrderByItem, ScanOptions, SelectColumn } from '../types.js'
 
 export type QueryPlan =
   | ScanNode
+  | CountNode
   | FilterNode
   | ProjectNode
   | SortNode
@@ -20,6 +21,13 @@ export interface ScanNode {
   hints: ScanOptions
 }
 
+// Count node for COUNT(*) optimization
+export interface CountNode {
+  type: 'Count'
+  table: string
+  columns: DerivedColumn[]
+}
+
 // Single-child nodes
 export interface FilterNode {
   type: 'Filter'

package/src/types.d.ts

@@ -42,6 +42,7 @@ export type Row = Record<string, SqlPrimitive>[]
  * Async data source for streaming SQL execution.
  */
 export interface AsyncDataSource {
+  numRows?: number
   scan(options: ScanOptions): ScanResults
 }
 
@@ -214,6 +215,10 @@ export interface IntervalNode extends ExprNodeBase {
   unit: IntervalUnit
 }
 
+export interface StarNode extends ExprNodeBase {
+  type: 'star'
+}
+
 export type ExprNode =
   | LiteralNode
   | IdentifierNode
@@ -227,6 +232,7 @@ export type ExprNode =
   | CaseNode
   | SubqueryNode
   | IntervalNode
+  | StarNode
 
 export type AggregateFunc = 'COUNT' | 'SUM' | 'AVG' | 'MIN' | 'MAX' | 'JSON_ARRAYAGG' | 'STDDEV_SAMP' | 'STDDEV_POP'
 

package/src/validationErrors.js

@@ -99,7 +99,7 @@ export function argValueError({ funcName, message, positionStart, positionEnd, h
  */
 export function aggregateError({ funcName, positionStart, positionEnd }) {
   return new ExecutionError({
-    message: `Aggregate function ${funcName} must exist in a GROUP BY clause or be part of an aggregate SELECT list`,
+    message: `Aggregate function ${funcName} is not available in this context`,
     positionStart,
     positionEnd,
   })