squirreling 0.7.10 → 0.9.0

package/README.md

@@ -85,6 +85,45 @@ const rows = await collect(executeSql({
 
 Because Squirreling uses lazy cell evaluation, the `AI_SCORE` function only executes for cells that are actually materialized. Combined with `LIMIT` or `WHERE`, you can efficiently query expensive operations.
 
+### Custom Data Sources
+
+Squirreling can work with any data source that implements the `AsyncDataSource` interface.
+
+```typescript
+interface AsyncDataSource {
+  scan(options: ScanOptions): ScanResults
+}
+
+interface ScanOptions {
+  columns?: string[]
+  where?: ExprNode
+  limit?: number
+  offset?: number
+  signal?: AbortSignal
+}
+
+interface ScanResults {
+  rows: AsyncIterable<AsyncRow> // async iterable of rows
+  appliedWhere: boolean // WHERE filter applied at scan time?
+  appliedLimitOffset: boolean // LIMIT and OFFSET applied at scan time?
+}
+```
+
+The `scan()` method returns a `ScanResults` object containing a row stream and flags indicating which query hints were applied by the data source. This allows optional push down optimizations like filtering, limiting, and offsetting at the data source level when possible. Set `appliedWhere` or `appliedLimitOffset` to `true` if the data source handled them, `false` if the engine should apply them.
+
+```typescript
+const customSource: AsyncDataSource = {
+  scan({ columns, where, limit, offset, signal }) {
+    // Use hints to optimize your scan, or ignore them
+    return {
+      rows: fetchAllRows({ columns, signal }),
+      appliedWhere: false, // source returned all rows, engine will filter
+      appliedLimitOffset: false, // source returned all rows, engine will limit/skip
+    }
+  },
+}
+```
+
 ## Supported SQL Syntax
 
 Squirreling mostly follows the SQL standard. The following features are supported:
@@ -95,6 +134,12 @@ Squirreling mostly follows the SQL standard. The following features are supporte
 - `JOIN` operations: `INNER JOIN`, `LEFT JOIN`, `RIGHT JOIN`, `FULL JOIN`, `POSITIONAL JOIN`
 - `GROUP BY` and `HAVING` clauses
 
+### Quoting
+
+- Single quotes for string literals: `'hello world'`
+- Double quotes for identifiers with spaces or special characters: `"column name"`
+- Escape quotes by doubling: `'can''t'` or `"col""name"`
+
 ### Functions
 
 - Aggregate: `COUNT`, `SUM`, `AVG`, `MIN`, `MAX`, `JSON_ARRAYAGG`

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "squirreling",
-  "version": "0.7.10",
-  "description": "Squirreling SQL Engine",
+  "version": "0.9.0",
+  "description": "Squirreling Async SQL Engine",
   "author": "Hyperparam",
   "homepage": "https://hyperparam.app",
   "keywords": [
@@ -37,10 +37,10 @@
     "test": "vitest run"
   },
   "devDependencies": {
-    "@types/node": "25.2.0",
+    "@types/node": "25.2.3",
     "@vitest/coverage-v8": "4.0.18",
     "eslint": "9.39.2",
-    "eslint-plugin-jsdoc": "62.5.0",
+    "eslint-plugin-jsdoc": "62.5.5",
     "typescript": "5.9.3",
     "vitest": "4.0.18"
   }

package/src/backend/dataSource.js

@@ -1,24 +1,7 @@
 /**
- * @import { AsyncCells, AsyncDataSource, AsyncRow, ScanOptions, SqlPrimitive } from '../types.js'
+ * @import { AsyncCells, AsyncDataSource, AsyncRow, ScanOptions, ScanResults, SqlPrimitive } from '../types.js'
  */
 
-/**
- * Wraps an async generator of plain objects into an AsyncDataSource
- *
- * @param {AsyncGenerator<AsyncRow>} gen
- * @returns {AsyncDataSource}
- */
-export function generatorSource(gen) {
-  return {
-    async *scan({ signal }) {
-      for await (const row of gen) {
-        if (signal?.aborted) break
-        yield row
-      }
-    },
-  }
-}
-
 /**
  * Creates an async row accessor that wraps a plain JavaScript object
  *
@@ -42,10 +25,19 @@ function asyncRow(obj) {
  */
 export function memorySource(data) {
   return {
-    async *scan({ signal }) {
-      for (const item of data) {
-        if (signal?.aborted) break
-        yield asyncRow(item)
+    scan({ where, limit, offset, signal }) {
+      // Only apply offset and limit if no where clause
+      const start = !where ? offset ?? 0 : 0
+      const end = !where && limit !== undefined ? start + limit : data.length
+      return {
+        rows: (async function* () {
+          for (let i = start; i < end && i < data.length; i++) {
+            if (signal?.aborted) break
+            yield asyncRow(data[i])
+          }
+        })(),
+        appliedWhere: false,
+        appliedLimitOffset: !where,
       }
     },
   }
@@ -60,33 +52,46 @@ export function cachedDataSource(source) {
   /** @type {Map<string, Promise<SqlPrimitive>>} */
   const cache = new Map()
   return {
-    /**
-     * @param {ScanOptions} options
-     * @yields {AsyncRow}
-     */
-    async *scan(options) {
-      const { signal } = options
-      let index = 0
-      for await (const row of source.scan(options)) {
-        if (signal?.aborted) break
-        const rowIndex = index
-        /** @type {AsyncCells} */
-        const cells = {}
-        for (const key of row.columns) {
-          const cell = row.cells[key]
-          // Wrap the cell to cache accesses
-          cells[key] = () => {
-            const cacheKey = `${rowIndex}:${key}`
-            let value = cache.get(cacheKey)
-            if (!value) {
-              value = cell()
-              cache.set(cacheKey, value)
+    scan(options) {
+      // Does re-run the scan, but cache avoids re-computing expensive async cells
+      // TODO: check cache first to avoid re-scanning when possible
+      const { rows, appliedWhere, appliedLimitOffset } = source.scan(options)
+
+      // Applied where clause changes which rows are returned so can't be cached
+      if (appliedWhere && options.where) {
+        return { rows, appliedWhere, appliedLimitOffset }
+      }
+
+      // Adjust index when source applied offset so cache keys match original rows
+      const indexOffset = appliedLimitOffset && options.offset ? options.offset : 0
+
+      return {
+        rows: (async function* () {
+          let index = 0
+          for await (const row of rows) {
+            if (options.signal?.aborted) break
+            const rowIndex = index + indexOffset
+            /** @type {AsyncCells} */
+            const cells = {}
+            for (const key of row.columns) {
+              const cell = row.cells[key]
+              // Wrap the cell to cache accesses
+              cells[key] = () => {
+                const cacheKey = `${rowIndex}:${key}`
+                let value = cache.get(cacheKey)
+                if (!value) {
+                  value = cell()
+                  cache.set(cacheKey, value)
+                }
+                return value
+              }
             }
-            return value
+            yield { columns: row.columns, cells }
+            index++
           }
-        }
-        yield { columns: row.columns, cells }
-        index++
+        })(),
+        appliedWhere,
+        appliedLimitOffset,
       }
     },
   }

package/src/execute/aggregates.js

@@ -0,0 +1,138 @@
+import { evaluateExpr } from '../expression/evaluate.js'
+import { defaultDerivedAlias, stringify } from './utils.js'
+import { executePlan } from './execute.js'
+
+/**
+ * @import { AsyncCells, AsyncRow, ExecuteContext, SelectColumn } from '../types.js'
+ * @import { HashAggregateNode, ScalarAggregateNode } from '../plan/types.js'
+ */
+
+/**
+ * Projects aggregate columns from a group of rows
+ *
+ * @param {SelectColumn[]} selectColumns
+ * @param {AsyncRow[]} group
+ * @param {ExecuteContext} context
+ * @returns {AsyncRow}
+ */
+function projectAggregateColumns(selectColumns, group, context) {
+  /** @type {string[]} */
+  const columns = []
+  /** @type {AsyncCells} */
+  const cells = {}
+
+  for (const col of selectColumns) {
+    if (col.kind === 'star') {
+      const firstRow = group[0]
+      if (firstRow) {
+        for (const key of firstRow.columns) {
+          columns.push(key)
+          cells[key] = firstRow.cells[key]
+        }
+      }
+    } else if (col.kind === 'derived') {
+      const alias = col.alias ?? defaultDerivedAlias(col.expr)
+      columns.push(alias)
+      cells[alias] = () => evaluateExpr({
+        node: col.expr,
+        row: group[0] ?? { columns: [], cells: {} },
+        rows: group,
+        context,
+      })
+    }
+  }
+
+  return { columns, cells }
+}
+
+/**
+ * Executes a hash aggregate operation (GROUP BY)
+ *
+ * @param {HashAggregateNode} plan
+ * @param {ExecuteContext} context
+ * @yields {AsyncRow}
+ */
+export async function* executeHashAggregate(plan, context) {
+  // Collect all rows
+  /** @type {AsyncRow[]} */
+  const allRows = []
+  for await (const row of executePlan({ plan: plan.child, context })) {
+    if (context.signal?.aborted) return
+    allRows.push(row)
+  }
+
+  // Group rows by GROUP BY keys
+  /** @type {Map<string, AsyncRow[]>} */
+  const groupMap = new Map()
+  /** @type {AsyncRow[][]} */
+  const groups = []
+
+  for (const row of allRows) {
+    /** @type {string[]} */
+    const keyParts = []
+    for (const expr of plan.groupBy) {
+      const v = await evaluateExpr({ node: expr, row, context })
+      keyParts.push(stringify(v))
+    }
+    const key = keyParts.join('|')
+    let group = groupMap.get(key)
+    if (!group) {
+      group = []
+      groupMap.set(key, group)
+      groups.push(group)
+    }
+    group.push(row)
+  }
+
+  // Yield one row per group
+  for (const group of groups) {
+    const asyncRow = projectAggregateColumns(plan.columns, group, context)
+
+    // Apply HAVING filter
+    if (plan.having) {
+      const havingRow = { ...group[0], ...asyncRow }
+      const passes = await evaluateExpr({
+        node: plan.having,
+        row: havingRow,
+        rows: group,
+        context,
+      })
+      if (!passes) continue
+    }
+
+    yield asyncRow
+  }
+}
+
+/**
+ * Executes a scalar aggregate operation (no GROUP BY, whole table aggregate)
+ *
+ * @param {ScalarAggregateNode} plan
+ * @param {ExecuteContext} context
+ * @yields {AsyncRow}
+ */
+export async function* executeScalarAggregate(plan, context) {
+  // Collect all rows into single group
+  /** @type {AsyncRow[]} */
+  const group = []
+  for await (const row of executePlan({ plan: plan.child, context })) {
+    if (context.signal?.aborted) return
+    group.push(row)
+  }
+
+  const asyncRow = projectAggregateColumns(plan.columns, group, context)
+
+  // Apply HAVING filter
+  if (plan.having) {
+    const havingRow = { ...group[0], ...asyncRow }
+    const passes = await evaluateExpr({
+      node: plan.having,
+      row: havingRow,
+      rows: group,
+      context,
+    })
+    if (!passes) return
+  }
+
+  yield asyncRow
+}