squirreling 0.6.1 → 0.7.1

package/README.md

@@ -19,6 +19,8 @@ Squirreling is a streaming async SQL engine for JavaScript. It is designed to pr
 - Lets you move query execution closer to your users
 - Supports standard SQL queries
 - Async streaming for large datasets
+- Native javascript Promises, AsyncGenerators, AbortSignals
+- Async user-defined functions (UDFs)
 - Constant memory usage for simple queries with LIMIT
 - Robust error handling and validation designed for LLM tool use
 - In-memory data option for simple use cases
@@ -27,12 +29,12 @@ Squirreling is a streaming async SQL engine for JavaScript. It is designed to pr
 
 ## Usage
 
-Squirreling returns an async generator, allowing you to process rows one at a time without loading everything into memory.
+Squirreling returns an AsyncGenerator of AsyncRows, allowing you to process rows one at a time without loading everything into memory. AsyncRows are made up of AsyncCells, allowing for late materialization of values.
 
 ```typescript
 import { executeSql } from 'squirreling'
 
-// In-memory table
+// Input table (in-memory for this example)
 const users = [
   { id: 1, name: 'Alice', active: true },
   { id: 2, name: 'Bob', active: false },
@@ -40,35 +42,37 @@ const users = [
   // ...more rows
 ]
 
+// Squirreling return types
 interface AsyncRow {
   columns: string[]
   cells: Record<string, AsyncCell>
 }
 type AsyncCell = () => Promise<SqlPrimitive>
 
-// Returns an async iterable of rows with async cells
+// Returns an AsyncIterable of rows with async cell loading
 const asyncRows: AsyncIterable<AsyncRow> = executeSql({
   tables: { users },
-  query: 'SELECT count(*) as cnt FROM users WHERE active = TRUE LIMIT 10',
+  query: 'SELECT * FROM users',
 })
 
 // Process rows as they arrive (streaming)
-for await (const { cnt } of asyncRows) {
-  console.log('Count', await cnt())
+for await (const { id, name } of asyncRows) {
+  console.log(`User id=${await id()}, name=${await name()}`)
 }
 ```
 
-There is an exported helper function `collect` to gather all rows into an array if needed:
+Squirreling exports a helper function `collect` to gather all rows into an array:
 
 ```javascript
 import { collect, executeSql } from 'squirreling'
 
 // Collect all rows and cells into a materialized array
-const allUsers: Record<string, SqlPrimitive>[] = await collect(executeSql({
+const rows: Record<string, SqlPrimitive>[] = await collect(executeSql({
   tables: { users },
-  query: 'SELECT * FROM users',
+  query: 'SELECT active, count(*) as cnt FROM users GROUP BY active',
 }))
-console.log(allUsers)
+console.log(`Collected rows:`, rows)
+// Collected rows: [ { active: true, cnt: 2 }, { active: false, cnt: 1 } ]
 ```
 
 ## Supported SQL Features
@@ -77,9 +81,14 @@ console.log(allUsers)
 - Subqueries in `SELECT`, `FROM`, and `WHERE` clauses
 - `JOIN` operations: `INNER JOIN`, `LEFT JOIN`, `RIGHT JOIN`, `FULL JOIN`
 - `GROUP BY` and `HAVING` clauses
-- Aggregate functions: `COUNT`, `SUM`, `AVG`, `MIN`, `MAX`, `JSON_ARRAYAGG`
-- String functions: `CONCAT`, `SUBSTRING`, `LENGTH`, `UPPER`, `LOWER`
-- Math functions: `ABS`, `CEIL`, `FLOOR`, `ROUND`, `MOD`, `RAND`, `RANDOM`, `LN`, `LOG10`, `EXP`, `POWER`, `SQRT`, `SIN`, `COS`, `TAN`, `COT`, `ASIN`, `ACOS`, `ATAN`, `ATAN2`, `DEGREES`, `RADIANS`, `PI`
-- Date functions: `CURRENT_DATE`, `CURRENT_TIME`, `CURRENT_TIMESTAMP`, `INTERVAL`
-- Json functions: `JSON_VALUE`, `JSON_QUERY`, `JSON_OBJECT`
-- Basic expressions and arithmetic operations
+
+### Functions
+
+- Aggregate: `COUNT`, `SUM`, `AVG`, `MIN`, `MAX`, `JSON_ARRAYAGG`
+- String: `CONCAT`, `SUBSTRING`, `REPLACE`, `LENGTH`, `UPPER`, `LOWER`, `TRIM`, `LEFT`, `RIGHT`, `INSTR`
+- Math: `ABS`, `CEIL`, `FLOOR`, `ROUND`, `MOD`, `RAND`, `RANDOM`, `LN`, `LOG10`, `EXP`, `POWER`, `SQRT`
+- Trig: `SIN`, `COS`, `TAN`, `COT`, `ASIN`, `ACOS`, `ATAN`, `ATAN2`, `DEGREES`, `RADIANS`, `PI`
+- Date: `CURRENT_DATE`, `CURRENT_TIME`, `CURRENT_TIMESTAMP`, `INTERVAL`
+- Json: `JSON_VALUE`, `JSON_QUERY`, `JSON_OBJECT`
+- Regex: `REGEXP_SUBSTR`, `REGEXP_REPLACE`
+- User-defined functions (UDFs)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "squirreling",
-  "version": "0.6.1",
+  "version": "0.7.1",
   "description": "Squirreling SQL Engine",
   "author": "Hyperparam",
   "homepage": "https://hyperparam.app",

package/src/execute/execute.js

@@ -9,7 +9,7 @@ import { executeJoins } from './join.js'
 import { compareForTerm, defaultDerivedAlias, stringify } from './utils.js'
 
 /**
- * @import { AsyncCells, AsyncDataSource, AsyncRow, ExecuteSqlOptions, OrderByItem, QueryHints, SelectStatement, SqlPrimitive } from '../types.js'
+ * @import { AsyncCells, AsyncDataSource, AsyncRow, ExecuteSqlOptions, OrderByItem, QueryHints, SelectStatement, SqlPrimitive, UserDefinedFunction } from '../types.js'
  */
 
 /**
@@ -18,8 +18,8 @@ import { compareForTerm, defaultDerivedAlias, stringify } from './utils.js'
  * @param {ExecuteSqlOptions} options - the execution options
  * @yields {AsyncRow} async generator yielding result rows
  */
-export async function* executeSql({ tables, query, signal }) {
-  const select = typeof query === 'string' ? parseSql(query) : query
+export async function* executeSql({ tables, query, functions, signal }) {
+  const select = typeof query === 'string' ? parseSql({ query, functions }) : query
 
   // Check for unsupported operations
   if (!select.from) {
@@ -40,47 +40,44 @@ export async function* executeSql({ tables, query, signal }) {
     }
   }
 
-  yield* executeSelect({ select, tables: normalizedTables, signal })
+  yield* executeSelect({ select, tables: normalizedTables, functions, signal })
 }
 
-/**
- * @typedef {Object} ExecuteSelectOptions
- * @property {SelectStatement} select
- * @property {Record<string, AsyncDataSource>} tables
- * @property {AbortSignal} [signal]
- */
-
 /**
  * Executes a SELECT query against the provided tables
  *
- * @param {ExecuteSelectOptions} options
+ * @param {Object} options
+ * @param {SelectStatement} options.select
+ * @param {Record<string, AsyncDataSource>} options.tables
+ * @param {Record<string, UserDefinedFunction>} [options.functions]
+ * @param {AbortSignal} [options.signal]
  * @yields {AsyncRow}
  */
-export async function* executeSelect({ select, tables, signal }) {
+export async function* executeSelect({ select, tables, functions, signal }) {
   /** @type {AsyncDataSource} */
   let dataSource
   /** @type {string} */
-  let fromTableName
+  let leftTable
 
   if (select.from.kind === 'table') {
     // Use alias for column prefixing, but look up the actual table name
-    fromTableName = select.from.alias ?? select.from.table
+    leftTable = select.from.alias ?? select.from.table
     dataSource = tables[select.from.table]
     if (dataSource === undefined) {
       throw tableNotFoundError({ tableName: select.from.table })
     }
   } else {
     // Nested subquery - recursively resolve
-    fromTableName = select.from.alias
-    dataSource = generatorSource(executeSelect({ select: select.from.query, tables, signal }))
+    leftTable = select.from.alias
+    dataSource = generatorSource(executeSelect({ select: select.from.query, tables, functions, signal }))
   }
 
   // Execute JOINs if present
   if (select.joins.length) {
-    dataSource = await executeJoins(dataSource, select.joins, fromTableName, tables)
+    dataSource = await executeJoins({ leftSource: dataSource, joins: select.joins, leftTable, tables, functions })
   }
 
-  yield* evaluateSelectAst({ select, dataSource, tables, signal })
+  yield* evaluateSelectAst({ select, dataSource, tables, functions, signal })
 }
 
 /**
@@ -121,17 +118,20 @@ async function applyDistinct(rows, distinct) {
   }
   return result
 }
+
 /**
  * Applies ORDER BY sorting to rows using multi-pass lazy evaluation.
  * Secondary ORDER BY columns are only evaluated for rows that tie on
  * previous columns, reducing expensive cell evaluations.
  *
- * @param {AsyncRow[]} rows - the input rows
- * @param {OrderByItem[]} orderBy - the sort specifications
- * @param {Record<string, AsyncDataSource>} tables
+ * @param {Object} options
+ * @param {AsyncRow[]} options.rows - the input rows
+ * @param {OrderByItem[]} options.orderBy - the sort specifications
+ * @param {Record<string, AsyncDataSource>} options.tables
+ * @param {Record<string, UserDefinedFunction>} [options.functions]
  * @returns {Promise<AsyncRow[]>} the sorted rows
  */
-async function sortRows(rows, orderBy, tables) {
+async function sortRows({ rows, orderBy, tables, functions }) {
   if (!orderBy.length) return rows
 
   // Cache for evaluated values: evaluatedValues[rowIdx][colIdx]
@@ -162,6 +162,7 @@ async function sortRows(rows, orderBy, tables) {
             node: term.expr,
             row: rows[idx],
             tables,
+            functions,
           })
         }
       }
@@ -206,21 +207,18 @@ async function sortRows(rows, orderBy, tables) {
   return groups.flat().map(i => rows[i])
 }
 
-/**
- * @typedef {Object} EvaluateSelectAstOptions
- * @property {SelectStatement} select
- * @property {AsyncDataSource} dataSource
- * @property {Record<string, AsyncDataSource>} tables
- * @property {AbortSignal} [signal]
- */
-
 /**
  * Evaluates a select with a resolved FROM data source
  *
- * @param {EvaluateSelectAstOptions} options
+ * @param {Object} options
+ * @param {SelectStatement} options.select
+ * @param {AsyncDataSource} options.dataSource
+ * @param {Record<string, AsyncDataSource>} options.tables
+ * @param {Record<string, UserDefinedFunction>} [options.functions]
+ * @param {AbortSignal} [options.signal]
  * @yields {AsyncRow}
  */
-async function* evaluateSelectAst({ select, dataSource, tables, signal }) {
+async function* evaluateSelectAst({ select, dataSource, tables, functions, signal }) {
   // SQL priority: from, where, group by, having, select, order by, offset, limit
 
   const hasAggregate = select.columns.some(col => col.kind === 'derived' && containsAggregate(col.expr))
@@ -229,29 +227,26 @@ async function* evaluateSelectAst({ select, dataSource, tables, signal }) {
 
   if (needsBuffering) {
     // BUFFERING PATH: Collect all rows, process, then yield
-    yield* evaluateBuffered({ select, dataSource, tables, hasAggregate, useGrouping, signal })
+    yield* evaluateBuffered({ select, dataSource, tables, functions, hasAggregate, useGrouping, signal })
   } else {
     // STREAMING PATH: Yield rows one by one
-    yield* evaluateStreaming({ select, dataSource, tables, signal })
+    yield* evaluateStreaming({ select, dataSource, tables, functions, signal })
   }
 }
 
-/**
- * @typedef {Object} EvaluateStreamingOptions
- * @property {SelectStatement} select
- * @property {AsyncDataSource} dataSource
- * @property {Record<string, AsyncDataSource>} tables
- * @property {AbortSignal} [signal]
- */
-
 /**
  * Streaming evaluation for simple queries (no ORDER BY or GROUP BY)
  * Supports DISTINCT by tracking seen row keys without buffering full rows
  *
- * @param {EvaluateStreamingOptions} options
+ * @param {Object} options
+ * @param {SelectStatement} options.select
+ * @param {AsyncDataSource} options.dataSource
+ * @param {Record<string, AsyncDataSource>} options.tables
+ * @param {Record<string, UserDefinedFunction>} [options.functions]
+ * @param {AbortSignal} [options.signal]
  * @yields {AsyncRow}
  */
-async function* evaluateStreaming({ select, dataSource, tables, signal }) {
+async function* evaluateStreaming({ select, dataSource, tables, functions, signal }) {
   let rowsYielded = 0
   let rowsSkipped = 0
   let rowIndex = 0
@@ -276,7 +271,7 @@ async function* evaluateStreaming({ select, dataSource, tables, signal }) {
     rowIndex++
     // WHERE filter
     if (select.where) {
-      const pass = await evaluateExpr({ node: select.where, row, tables, rowIndex })
+      const pass = await evaluateExpr({ node: select.where, row, tables, functions, rowIndex })
       if (!pass) continue
     }
 
@@ -301,7 +296,7 @@ async function* evaluateStreaming({ select, dataSource, tables, signal }) {
       } else if (col.kind === 'derived') {
         const alias = col.alias ?? defaultDerivedAlias(col.expr)
         columns.push(alias)
-        cells[alias] = () => evaluateExpr({ node: col.expr, row, tables, rowIndex: currentRowIndex })
+        cells[alias] = () => evaluateExpr({ node: col.expr, row, tables, functions, rowIndex: currentRowIndex })
       }
     }
 
@@ -325,23 +320,20 @@ async function* evaluateStreaming({ select, dataSource, tables, signal }) {
   }
 }
 
-/**
- * @typedef {Object} EvaluateBufferedOptions
- * @property {SelectStatement} select
- * @property {AsyncDataSource} dataSource
- * @property {Record<string, AsyncDataSource>} tables
- * @property {boolean} hasAggregate
- * @property {boolean} useGrouping
- * @property {AbortSignal} [signal]
- */
-
 /**
  * Buffered evaluation for complex queries (with ORDER BY or GROUP BY)
  *
- * @param {EvaluateBufferedOptions} options
+ * @param {Object} options
+ * @param {SelectStatement} options.select
+ * @param {AsyncDataSource} options.dataSource
+ * @param {Record<string, AsyncDataSource>} options.tables
+ * @param {Record<string, UserDefinedFunction>} [options.functions]
+ * @param {boolean} options.hasAggregate
+ * @param {boolean} options.useGrouping
+ * @param {AbortSignal} [options.signal]
  * @yields {AsyncRow}
  */
-async function* evaluateBuffered({ select, dataSource, tables, hasAggregate, useGrouping, signal }) {
+async function* evaluateBuffered({ select, dataSource, tables, functions, hasAggregate, useGrouping, signal }) {
   // Build hints for data source optimization
   // Note: limit/offset not passed here since buffering needs all rows for sorting/grouping
   /** @type {QueryHints} */
@@ -365,7 +357,7 @@ async function* evaluateBuffered({ select, dataSource, tables, hasAggregate, use
     const row = working[i]
     const rowIndex = i + 1 // 1-based
     if (select.where) {
-      const passes = await evaluateExpr({ node: select.where, row, tables, rowIndex })
+      const passes = await evaluateExpr({ node: select.where, row, tables, functions, rowIndex })
 
       if (!passes) {
         continue
@@ -390,7 +382,7 @@ async function* evaluateBuffered({ select, dataSource, tables, hasAggregate, use
         /** @type {string[]} */
         const keyParts = []
         for (const expr of select.groupBy) {
-          const v = await evaluateExpr({ node: expr, row, tables })
+          const v = await evaluateExpr({ node: expr, row, tables, functions })
           keyParts.push(stringify(v))
         }
         const key = keyParts.join('|')
@@ -435,7 +427,7 @@ async function* evaluateBuffered({ select, dataSource, tables, hasAggregate, use
           columns.push(alias)
           // Pass group to evaluateExpr so it can handle aggregate functions within expressions
           // For empty groups, still provide an empty row context for aggregates to return appropriate values
-          cells[alias] = () => evaluateExpr({ node: col.expr, row: group[0] ?? { columns: [], cells: {} }, tables, rows: group })
+          cells[alias] = () => evaluateExpr({ node: col.expr, row: group[0] ?? { columns: [], cells: {} }, tables, functions, rows: group })
           continue
         }
       }
@@ -443,7 +435,7 @@ async function* evaluateBuffered({ select, dataSource, tables, hasAggregate, use
 
       // Apply HAVING filter before adding to projected results
       if (select.having) {
-        if (!await evaluateHavingExpr(select.having, asyncRow, group, tables)) {
+        if (!await evaluateHavingExpr({ expr: select.having, row: asyncRow, group, tables, functions })) {
           continue
         }
       }
@@ -453,7 +445,7 @@ async function* evaluateBuffered({ select, dataSource, tables, hasAggregate, use
   } else {
     // No grouping, simple projection
     // Sort before projection so ORDER BY can access columns not in SELECT
-    const sorted = await sortRows(filtered, select.orderBy, tables)
+    const sorted = await sortRows({ rows: filtered, orderBy: select.orderBy, tables, functions })
 
     // OPTIMIZATION: For non-DISTINCT queries, apply OFFSET/LIMIT before projection
     // to avoid reading expensive cells for rows that won't be in the final result
@@ -477,7 +469,7 @@ async function* evaluateBuffered({ select, dataSource, tables, hasAggregate, use
         } else if (col.kind === 'derived') {
           const alias = col.alias ?? defaultDerivedAlias(col.expr)
           columns.push(alias)
-          cells[alias] = () => evaluateExpr({ node: col.expr, row, tables })
+          cells[alias] = () => evaluateExpr({ node: col.expr, row, tables, functions })
         }
       }
       projected.push({ columns, cells })
@@ -489,7 +481,7 @@ async function* evaluateBuffered({ select, dataSource, tables, hasAggregate, use
 
   // Step 5: ORDER BY (final sort for grouped queries)
   if (useGrouping) {
-    projected = await sortRows(projected, select.orderBy, tables)
+    projected = await sortRows({ rows: projected, orderBy: select.orderBy, tables, functions })
   }
 
   // Step 6: OFFSET and LIMIT