squirreling 0.4.3 → 0.4.4

package/README.md

@@ -29,7 +29,7 @@ Squirreling is a streaming async SQL engine for JavaScript. It is designed to pr
 
 Squirreling returns an async generator, allowing you to process rows one at a time without loading everything into memory.
 
-```javascript
+```typescript
 import { executeSql } from 'squirreling'
 
 // In-memory table
@@ -40,12 +40,18 @@ const users = [
   // ...more rows
 ]
 
-// Process rows as they arrive (streaming)
-for await (const { cnt } of executeSql({
+type AsyncRow = Record<string, AsyncCell>
+type AsyncCell = () => Promise<SqlPrimitive>
+
+// Returns an async iterable of rows with async cells
+const asyncRows: AsyncIterable<AsyncRow> = executeSql({
   tables: { users },
   query: 'SELECT count(*) as cnt FROM users WHERE active = TRUE LIMIT 10',
-})) {
-  console.log('Count', cnt)
+})
+
+// Process rows as they arrive (streaming)
+for await (const { cnt } of asyncRows) {
+  console.log('Count', await cnt())
 }
 ```
 
@@ -54,7 +60,8 @@ There is an exported helper function `collect` to gather all rows into an array
 ```javascript
 import { collect, executeSql } from 'squirreling'
 
-const allUsers = await collect(executeSql({
+// Collect all rows and cells into a materialized array
+const allUsers: Record<string, SqlPrimitive>[] = await collect(executeSql({
   tables: { users },
   query: 'SELECT * FROM users',
 }))

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "squirreling",
-  "version": "0.4.3",
+  "version": "0.4.4",
   "description": "Squirreling SQL Engine",
   "author": "Hyperparam",
   "homepage": "https://hyperparam.app",

package/src/execute/columns.js

@@ -0,0 +1,107 @@
+/**
+ * @import { ExprNode, SelectStatement, SelectColumn } from '../types.js'
+ */
+
+/**
+ * Extracts column names needed from a SELECT statement.
+ *
+ * @param {SelectStatement} select
+ * @returns {string[] | undefined} array of column names, or undefined if all columns needed
+ */
+export function extractColumns(select) {
+  // If any column is SELECT *, we need all columns
+  if (select.columns.some(col => col.kind === 'star')) {
+    return undefined
+  }
+
+  /** @type {Set<string>} */
+  const columns = new Set()
+
+  // Columns from SELECT list
+  for (const col of select.columns) {
+    collectColumnsFromSelectColumn(col, columns)
+  }
+
+  // Columns from WHERE
+  collectColumnsFromExpr(select.where, columns)
+
+  // Columns from ORDER BY
+  for (const item of select.orderBy) {
+    collectColumnsFromExpr(item.expr, columns)
+  }
+
+  // Columns from GROUP BY
+  for (const expr of select.groupBy) {
+    collectColumnsFromExpr(expr, columns)
+  }
+
+  // Columns from HAVING
+  collectColumnsFromExpr(select.having, columns)
+
+  return [...columns]
+}
+
+/**
+ * Collects column names from a SELECT column
+ *
+ * @param {SelectColumn} col
+ * @param {Set<string>} columns
+ */
+function collectColumnsFromSelectColumn(col, columns) {
+  if (col.kind === 'derived') {
+    collectColumnsFromExpr(col.expr, columns)
+  } else if (col.kind === 'aggregate') {
+    if (col.arg.kind === 'expression') {
+      collectColumnsFromExpr(col.arg.expr, columns)
+    }
+    // 'star' aggregate (COUNT(*)) doesn't reference specific columns
+  }
+  // 'star' columns handled separately (returns undefined for all columns)
+}
+
+/**
+ * Recursively collects column names (identifiers) from an expression
+ *
+ * @param {ExprNode | undefined} expr
+ * @param {Set<string>} columns
+ */
+function collectColumnsFromExpr(expr, columns) {
+  if (!expr) return
+  if (expr.type === 'identifier') {
+    columns.add(expr.name)
+  } else if (expr.type === 'literal') {
+    // No columns
+  } else if (expr.type === 'binary') {
+    collectColumnsFromExpr(expr.left, columns)
+    collectColumnsFromExpr(expr.right, columns)
+  } else if (expr.type === 'unary') {
+    collectColumnsFromExpr(expr.argument, columns)
+  } else if (expr.type === 'function') {
+    for (const arg of expr.args) {
+      collectColumnsFromExpr(arg, columns)
+    }
+  } else if (expr.type === 'cast') {
+    collectColumnsFromExpr(expr.expr, columns)
+  } else if (expr.type === 'in valuelist') {
+    collectColumnsFromExpr(expr.expr, columns)
+    for (const val of expr.values) {
+      collectColumnsFromExpr(val, columns)
+    }
+  } else if (expr.type === 'in') {
+    collectColumnsFromExpr(expr.expr, columns)
+    // Subquery columns are from a different scope, don't collect
+  } else if (expr.type === 'exists' || expr.type === 'not exists') {
+    // Subquery columns are from a different scope, don't collect
+  } else if (expr.type === 'case') {
+    if (expr.caseExpr) {
+      collectColumnsFromExpr(expr.caseExpr, columns)
+    }
+    for (const when of expr.whenClauses) {
+      collectColumnsFromExpr(when.condition, columns)
+      collectColumnsFromExpr(when.result, columns)
+    }
+    if (expr.elseResult) {
+      collectColumnsFromExpr(expr.elseResult, columns)
+    }
+  }
+}

package/src/execute/execute.js

@@ -5,9 +5,10 @@ import { evaluateExpr } from './expression.js'
 import { evaluateHavingExpr } from './having.js'
 import { executeJoins } from './join.js'
 import { compareForTerm, defaultDerivedAlias } from './utils.js'
+import { extractColumns } from './columns.js'
 
 /**
- * @import { AsyncDataSource, ExecuteSqlOptions, ExprNode, OrderByItem, AsyncRow, SelectStatement, SqlPrimitive } from '../types.js'
+ * @import { AsyncDataSource, AsyncRow, ExecuteSqlOptions, ExprNode, OrderByItem, QueryHints, SelectStatement, SqlPrimitive } from '../types.js'
  */
 
 /**
@@ -239,7 +240,16 @@ async function* evaluateStreaming(select, dataSource, tables) {
   /** @type {Set<string> | undefined} */
   const seen = select.distinct ? new Set() : undefined
 
-  for await (const row of dataSource.getRows()) {
+  // hints for data source optimization
+  /** @type {QueryHints} */
+  const hints = {
+    columns: extractColumns(select),
+    where: select.where,
+    limit: select.limit,
+    offset: select.offset,
+  }
+
+  for await (const row of dataSource.getRows(hints)) {
     // WHERE filter
     if (select.where) {
       const pass = await evaluateExpr({ node: select.where, row, tables })
@@ -301,10 +311,18 @@ async function* evaluateStreaming(select, dataSource, tables) {
  * @yields {AsyncRow}
  */
 async function* evaluateBuffered(select, dataSource, tables, hasAggregate, useGrouping) {
+  // Build hints for data source optimization
+  // Note: limit/offset not passed here since buffering needs all rows for sorting/grouping
+  /** @type {QueryHints} */
+  const hints = {
+    where: select.where,
+    columns: extractColumns(select),
+  }
+
   // Step 1: Collect all rows from data source
   /** @type {AsyncRow[]} */
   const working = []
-  for await (const row of dataSource.getRows()) {
+  for await (const row of dataSource.getRows(hints)) {
     working.push(row)
   }
 

package/src/index.d.ts

@@ -1,4 +1,4 @@
-import type { AsyncRow, ExecuteSqlOptions, SelectStatement, SqlPrimitive } from './types.js'
+import type { AsyncDataSource, AsyncRow, ExecuteSqlOptions, SelectStatement, SqlPrimitive } from './types.js'
 export type { AsyncDataSource, AsyncRow, SqlPrimitive } from './types.js'
 
 /**
@@ -26,3 +26,5 @@ export function parseSql(query: string): SelectStatement
  * @returns array of all yielded values
  */
 export function collect<T>(asyncGen: AsyncGenerator<AsyncRow>): Promise<Record<string, SqlPrimitive>[]>
+
+export function cachedDataSource(source: AsyncDataSource): AsyncDataSource

package/src/index.js

@@ -1,3 +1,4 @@
 export { executeSql } from './execute/execute.js'
 export { parseSql } from './parse/parse.js'
 export { collect } from './execute/utils.js'
+export { cachedDataSource } from './backend/dataSource.js'

package/src/types.d.ts

@@ -1,10 +1,21 @@
 
+/**
+ * Hints passed to data sources for query optimization.
+ * All hints are optional and "best effort" - sources may ignore them.
+ */
+export interface QueryHints {
+  columns?: string[] // columns needed
+  where?: ExprNode // where clause
+  limit?: number
+  offset?: number
+}
+
 /**
  * Async data source for streaming SQL execution.
  * Provides an async iterator over rows.
  */
 export interface AsyncDataSource {
-  getRows(): AsyncIterable<AsyncRow>
+  getRows(hints?: QueryHints): AsyncIterable<AsyncRow>
 }
 export type AsyncRow = Record<string, AsyncCell>
 export type AsyncCell = () => Promise<SqlPrimitive>
@@ -43,17 +54,9 @@ export interface FromSubquery {
   alias: string
 }
 
-export type BinaryOp =
-  | 'AND'
-  | 'OR'
-  | '='
-  | '!='
-  | '<>'
-  | '<'
-  | '>'
-  | '<='
-  | '>='
-  | 'LIKE'
+export type BinaryOp = 'AND' | 'OR' | 'LIKE' | ComparisonOp
+
+export type ComparisonOp = '=' | '!=' | '<>' | '<' | '>' | '<=' | '>='
 
 export interface LiteralNode {
   type: 'literal'