squirreling 0.4.3 → 0.4.5

package/README.md

@@ -29,7 +29,7 @@ Squirreling is a streaming async SQL engine for JavaScript. It is designed to pr
 
 Squirreling returns an async generator, allowing you to process rows one at a time without loading everything into memory.
 
-```javascript
+```typescript
 import { executeSql } from 'squirreling'
 
 // In-memory table
@@ -40,12 +40,18 @@ const users = [
   // ...more rows
 ]
 
-// Process rows as they arrive (streaming)
-for await (const { cnt } of executeSql({
+type AsyncRow = Record<string, AsyncCell>
+type AsyncCell = () => Promise<SqlPrimitive>
+
+// Returns an async iterable of rows with async cells
+const asyncRows: AsyncIterable<AsyncRow> = executeSql({
   tables: { users },
   query: 'SELECT count(*) as cnt FROM users WHERE active = TRUE LIMIT 10',
-})) {
-  console.log('Count', cnt)
+})
+
+// Process rows as they arrive (streaming)
+for await (const { cnt } of asyncRows) {
+  console.log('Count', await cnt())
 }
 ```
 
@@ -54,7 +60,8 @@ There is an exported helper function `collect` to gather all rows into an array
 ```javascript
 import { collect, executeSql } from 'squirreling'
 
-const allUsers = await collect(executeSql({
+// Collect all rows and cells into a materialized array
+const allUsers: Record<string, SqlPrimitive>[] = await collect(executeSql({
   tables: { users },
   query: 'SELECT * FROM users',
 }))

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "squirreling",
-  "version": "0.4.3",
+  "version": "0.4.5",
   "description": "Squirreling SQL Engine",
   "author": "Hyperparam",
   "homepage": "https://hyperparam.app",

package/src/execute/columns.js

@@ -0,0 +1,107 @@
+/**
+ * @import { ExprNode, SelectStatement, SelectColumn } from '../types.js'
+ */
+
+/**
+ * Extracts column names needed from a SELECT statement.
+ *
+ * @param {SelectStatement} select
+ * @returns {string[] | undefined} array of column names, or undefined if all columns needed
+ */
+export function extractColumns(select) {
+  // If any column is SELECT *, we need all columns
+  if (select.columns.some(col => col.kind === 'star')) {
+    return undefined
+  }
+
+  /** @type {Set<string>} */
+  const columns = new Set()
+
+  // Columns from SELECT list
+  for (const col of select.columns) {
+    collectColumnsFromSelectColumn(col, columns)
+  }
+
+  // Columns from WHERE
+  collectColumnsFromExpr(select.where, columns)
+
+  // Columns from ORDER BY
+  for (const item of select.orderBy) {
+    collectColumnsFromExpr(item.expr, columns)
+  }
+
+  // Columns from GROUP BY
+  for (const expr of select.groupBy) {
+    collectColumnsFromExpr(expr, columns)
+  }
+
+  // Columns from HAVING
+  collectColumnsFromExpr(select.having, columns)
+
+  return [...columns]
+}
+
+/**
+ * Collects column names from a SELECT column
+ *
+ * @param {SelectColumn} col
+ * @param {Set<string>} columns
+ */
+function collectColumnsFromSelectColumn(col, columns) {
+  if (col.kind === 'derived') {
+    collectColumnsFromExpr(col.expr, columns)
+  } else if (col.kind === 'aggregate') {
+    if (col.arg.kind === 'expression') {
+      collectColumnsFromExpr(col.arg.expr, columns)
+    }
+    // 'star' aggregate (COUNT(*)) doesn't reference specific columns
+  }
+  // 'star' columns handled separately (returns undefined for all columns)
+}
+
+/**
+ * Recursively collects column names (identifiers) from an expression
+ *
+ * @param {ExprNode | undefined} expr
+ * @param {Set<string>} columns
+ */
+function collectColumnsFromExpr(expr, columns) {
+  if (!expr) return
+  if (expr.type === 'identifier') {
+    columns.add(expr.name)
+  } else if (expr.type === 'literal') {
+    // No columns
+  } else if (expr.type === 'binary') {
+    collectColumnsFromExpr(expr.left, columns)
+    collectColumnsFromExpr(expr.right, columns)
+  } else if (expr.type === 'unary') {
+    collectColumnsFromExpr(expr.argument, columns)
+  } else if (expr.type === 'function') {
+    for (const arg of expr.args) {
+      collectColumnsFromExpr(arg, columns)
+    }
+  } else if (expr.type === 'cast') {
+    collectColumnsFromExpr(expr.expr, columns)
+  } else if (expr.type === 'in valuelist') {
+    collectColumnsFromExpr(expr.expr, columns)
+    for (const val of expr.values) {
+      collectColumnsFromExpr(val, columns)
+    }
+  } else if (expr.type === 'in') {
+    collectColumnsFromExpr(expr.expr, columns)
+    // Subquery columns are from a different scope, don't collect
+  } else if (expr.type === 'exists' || expr.type === 'not exists') {
+    // Subquery columns are from a different scope, don't collect
+  } else if (expr.type === 'case') {
+    if (expr.caseExpr) {
+      collectColumnsFromExpr(expr.caseExpr, columns)
+    }
+    for (const when of expr.whenClauses) {
+      collectColumnsFromExpr(when.condition, columns)
+      collectColumnsFromExpr(when.result, columns)
+    }
+    if (expr.elseResult) {
+      collectColumnsFromExpr(expr.elseResult, columns)
+    }
+  }
+}

package/src/execute/execute.js

@@ -5,9 +5,10 @@ import { evaluateExpr } from './expression.js'
 import { evaluateHavingExpr } from './having.js'
 import { executeJoins } from './join.js'
 import { compareForTerm, defaultDerivedAlias } from './utils.js'
+import { extractColumns } from './columns.js'
 
 /**
- * @import { AsyncDataSource, ExecuteSqlOptions, ExprNode, OrderByItem, AsyncRow, SelectStatement, SqlPrimitive } from '../types.js'
+ * @import { AsyncDataSource, AsyncRow, ExecuteSqlOptions, ExprNode, OrderByItem, QueryHints, SelectStatement, SqlPrimitive } from '../types.js'
  */
 
 /**
@@ -239,7 +240,16 @@ async function* evaluateStreaming(select, dataSource, tables) {
   /** @type {Set<string> | undefined} */
   const seen = select.distinct ? new Set() : undefined
 
-  for await (const row of dataSource.getRows()) {
+  // hints for data source optimization
+  /** @type {QueryHints} */
+  const hints = {
+    columns: extractColumns(select),
+    where: select.where,
+    limit: select.limit,
+    offset: select.offset,
+  }
+
+  for await (const row of dataSource.getRows(hints)) {
     // WHERE filter
     if (select.where) {
       const pass = await evaluateExpr({ node: select.where, row, tables })
@@ -301,10 +311,18 @@ async function* evaluateStreaming(select, dataSource, tables) {
  * @yields {AsyncRow}
  */
 async function* evaluateBuffered(select, dataSource, tables, hasAggregate, useGrouping) {
+  // Build hints for data source optimization
+  // Note: limit/offset not passed here since buffering needs all rows for sorting/grouping
+  /** @type {QueryHints} */
+  const hints = {
+    where: select.where,
+    columns: extractColumns(select),
+  }
+
   // Step 1: Collect all rows from data source
   /** @type {AsyncRow[]} */
   const working = []
-  for await (const row of dataSource.getRows()) {
+  for await (const row of dataSource.getRows(hints)) {
     working.push(row)
   }
 

package/src/execute/utils.js

@@ -24,8 +24,9 @@ export function compareForTerm(a, b, term) {
   // Compare non-null values
   if (a === b) return 0
 
+  const primitives = ['number', 'bigint', 'boolean', 'string']
   let cmp
-  if (typeof a === 'number' && typeof b === 'number') {
+  if (primitives.includes(typeof a) && primitives.includes(typeof b)) {
     cmp = a < b ? -1 : a > b ? 1 : 0
   } else {
     const aa = String(a)

package/src/index.d.ts

@@ -1,4 +1,4 @@
-import type { AsyncRow, ExecuteSqlOptions, SelectStatement, SqlPrimitive } from './types.js'
+import type { AsyncDataSource, AsyncRow, ExecuteSqlOptions, SelectStatement, SqlPrimitive } from './types.js'
 export type { AsyncDataSource, AsyncRow, SqlPrimitive } from './types.js'
 
 /**
@@ -26,3 +26,5 @@ export function parseSql(query: string): SelectStatement
  * @returns array of all yielded values
  */
 export function collect<T>(asyncGen: AsyncGenerator<AsyncRow>): Promise<Record<string, SqlPrimitive>[]>
+
+export function cachedDataSource(source: AsyncDataSource): AsyncDataSource

package/src/index.js

@@ -1,3 +1,4 @@
 export { executeSql } from './execute/execute.js'
 export { parseSql } from './parse/parse.js'
 export { collect } from './execute/utils.js'
+export { cachedDataSource } from './backend/dataSource.js'

package/src/parse/comparison.js

@@ -1,32 +1,33 @@
 import { isBinaryOp } from '../validation.js'
-import { parseExpression, parsePrimary } from './expression.js'
+import { parseExpression, parsePrimary, parseSubquery } from './expression.js'
+import { consume, current, expect, match, peekToken } from './state.js'
 
 /**
- * @import { ExprCursor, ExprNode } from '../types.js'
+ * @import { ExprNode, ParserState } from '../types.js'
  */
 
 /**
- * @param {ExprCursor} c
+ * @param {ParserState} state
  * @returns {ExprNode}
  */
-export function parseComparison(c) {
-  const left = parsePrimary(c)
-  const tok = c.current()
+export function parseComparison(state) {
+  const left = parsePrimary(state)
+  const tok = current(state)
 
   // IS [NOT] NULL
   if (tok.type === 'keyword' && tok.value === 'IS') {
-    c.consume()
-    const notToken = c.current()
+    consume(state)
+    const notToken = current(state)
     if (notToken.type === 'keyword' && notToken.value === 'NOT') {
-      c.consume()
-      c.expect('keyword', 'NULL')
+      consume(state)
+      expect(state, 'keyword', 'NULL')
       return {
         type: 'unary',
         op: 'IS NOT NULL',
         argument: left,
       }
     }
-    c.expect('keyword', 'NULL')
+    expect(state, 'keyword', 'NULL')
     return {
       type: 'unary',
       op: 'IS NULL',
@@ -36,11 +37,11 @@ export function parseComparison(c) {
 
   // [NOT] LIKE
   if (tok.type === 'keyword' && tok.value === 'NOT') {
-    const nextTok = c.peek(1)
+    const nextTok = peekToken(state, 1)
     if (nextTok.type === 'keyword' && nextTok.value === 'LIKE') {
-      c.consume() // NOT
-      c.consume() // LIKE
-      const right = parsePrimary(c)
+      consume(state) // NOT
+      consume(state) // LIKE
+      const right = parsePrimary(state)
       return {
         type: 'unary',
         op: 'NOT',
@@ -55,8 +56,8 @@ export function parseComparison(c) {
   }
 
   if (tok.type === 'keyword' && tok.value === 'LIKE') {
-    c.consume()
-    const right = parsePrimary(c)
+    consume(state)
+    const right = parsePrimary(state)
     return {
       type: 'binary',
       op: 'LIKE',
@@ -67,13 +68,13 @@ export function parseComparison(c) {
 
   // [NOT] BETWEEN - convert to range comparison
   if (tok.type === 'keyword' && tok.value === 'NOT') {
-    const nextTok = c.peek(1)
+    const nextTok = peekToken(state, 1)
     if (nextTok.type === 'keyword' && nextTok.value === 'BETWEEN') {
-      c.consume() // NOT
-      c.consume() // BETWEEN
-      const lower = parsePrimary(c)
-      c.expect('keyword', 'AND')
-      const upper = parsePrimary(c)
+      consume(state) // NOT
+      consume(state) // BETWEEN
+      const lower = parsePrimary(state)
+      expect(state, 'keyword', 'AND')
+      const upper = parsePrimary(state)
       // NOT BETWEEN -> expr < lower OR expr > upper
       return {
         type: 'binary',
@@ -85,10 +86,10 @@ export function parseComparison(c) {
   }
 
   if (tok.type === 'keyword' && tok.value === 'BETWEEN') {
-    c.consume()
-    const lower = parsePrimary(c)
-    c.expect('keyword', 'AND')
-    const upper = parsePrimary(c)
+    consume(state)
+    const lower = parsePrimary(state)
+    expect(state, 'keyword', 'AND')
+    const upper = parsePrimary(state)
     // BETWEEN -> expr >= lower AND expr <= upper
     return {
       type: 'binary',
@@ -100,21 +101,21 @@ export function parseComparison(c) {
 
   // [NOT] IN
   if (tok.type === 'keyword' && tok.value === 'NOT') {
-    const nextTok = c.peek(1)
+    const nextTok = peekToken(state, 1)
     if (nextTok.type === 'keyword' && nextTok.value === 'IN') {
-      c.consume() // NOT
-      c.consume() // IN
+      consume(state) // NOT
+      consume(state) // IN
 
       // Check if it's a subquery or a list of values by peeking ahead
       // parseSubquery expects to consume the opening paren itself
-      const parenTok = c.current()
+      const parenTok = current(state)
       if (parenTok.type !== 'paren' || parenTok.value !== '(') {
         throw new Error('Expected ( after IN')
       }
-      const peekTok = c.peek(1)
+      const peekTok = peekToken(state, 1)
       if (peekTok.type === 'keyword' && peekTok.value === 'SELECT') {
         // Subquery - let parseSubquery handle the parens
-        const subquery = c.parseSubquery()
+        const subquery = parseSubquery(state)
         return {
           type: 'unary',
           op: 'NOT',
@@ -126,14 +127,14 @@ export function parseComparison(c) {
         }
       } else {
         // Parse list of values - we handle the parens
-        c.consume() // '('
+        consume(state) // '('
         /** @type {ExprNode[]} */
         const values = []
         while (true) {
-          values.push(parseExpression(c))
-          if (!c.match('comma')) break
+          values.push(parseExpression(state))
+          if (!match(state, 'comma')) break
         }
-        c.expect('paren', ')')
+        expect(state, 'paren', ')')
         return {
           type: 'unary',
           op: 'NOT',
@@ -148,18 +149,18 @@ export function parseComparison(c) {
   }
 
   if (tok.type === 'keyword' && tok.value === 'IN') {
-    c.consume() // IN
+    consume(state) // IN
 
     // Check if it's a subquery or a list of values by peeking ahead
     // parseSubquery expects to consume the opening paren itself
-    const parenTok = c.current()
+    const parenTok = current(state)
     if (parenTok.type !== 'paren' || parenTok.value !== '(') {
       throw new Error('Expected ( after IN')
     }
-    const peekTok = c.peek(1)
+    const peekTok = peekToken(state, 1)
     if (peekTok.type === 'keyword' && peekTok.value === 'SELECT') {
       // Subquery - let parseSubquery handle the parens
-      const subquery = c.parseSubquery()
+      const subquery = parseSubquery(state)
       return {
         type: 'in',
         expr: left,
@@ -167,14 +168,14 @@ export function parseComparison(c) {
       }
     } else {
       // Parse list of values - we handle the parens
-      c.consume() // '('
+      consume(state) // '('
       /** @type {ExprNode[]} */
       const values = []
       while (true) {
-        values.push(parseExpression(c))
-        if (!c.match('comma')) break
+        values.push(parseExpression(state))
+        if (!match(state, 'comma')) break
       }
-      c.expect('paren', ')')
+      expect(state, 'paren', ')')
       return {
         type: 'in valuelist',
         expr: left,
@@ -184,8 +185,8 @@ export function parseComparison(c) {
   }
 
   if (tok.type === 'operator' && isBinaryOp(tok.value)) {
-    c.consume()
-    const right = parsePrimary(c)
+    consume(state)
+    const right = parsePrimary(state)
     return {
       type: 'binary',
       op: tok.value,

package/src/parse/expression.js

@@ -1,54 +1,56 @@
 import { isAggregateFunc, isStringFunc } from '../validation.js'
 import { parseComparison } from './comparison.js'
+import { parseSelectInternal } from './parse.js'
+import { consume, current, expect, expectIdentifier, match, peekToken } from './state.js'
 
 /**
- * @import { ExprCursor, ExprNode, WhenClause } from '../types.js'
+ * @import { ExprNode, ParserState, SelectStatement, WhenClause } from '../types.js'
  */
 
 /**
- * @param {ExprCursor} c
+ * @param {ParserState} state
  * @returns {ExprNode}
  */
-export function parseExpression(c) {
-  return parseOr(c)
+export function parseExpression(state) {
+  return parseOr(state)
 }
 
 /**
- * @param {ExprCursor} c
+ * @param {ParserState} state
  * @returns {ExprNode}
  */
-export function parsePrimary(c) {
-  const tok = c.current()
+export function parsePrimary(state) {
+  const tok = current(state)
 
   if (tok.type === 'paren' && tok.value === '(') {
     // Peek ahead to see if this is a scalar subquery
-    const nextTok = c.peek(1)
+    const nextTok = peekToken(state, 1)
     if (nextTok.type === 'keyword' && nextTok.value === 'SELECT') {
       // It's a scalar subquery
-      const subquery = c.parseSubquery()
+      const subquery = parseSubquery(state)
       return {
         type: 'subquery',
         subquery,
       }
     }
     // Regular grouped expression
-    c.consume()
-    const expr = parseExpression(c)
-    c.expect('paren', ')')
+    consume(state)
+    const expr = parseExpression(state)
+    expect(state, 'paren', ')')
     return expr
   }
 
   if (tok.type === 'identifier') {
-    const next = c.peek(1)
+    const next = peekToken(state, 1)
 
     // CAST expression
     if (tok.value === 'CAST' && next.type === 'paren' && next.value === '(') {
-      c.consume() // CAST
-      c.consume() // '('
-      const expr = parseExpression(c)
-      c.expect('keyword', 'AS')
-      const typeTok = c.expectIdentifier()
-      c.expect('paren', ')')
+      consume(state) // CAST
+      consume(state) // '('
+      const expr = parseExpression(state)
+      expect(state, 'keyword', 'AS')
+      const typeTok = expectIdentifier(state)
+      expect(state, 'paren', ')')
       return {
         type: 'cast',
         expr,
@@ -65,30 +67,30 @@ export function parsePrimary(c) {
         throw new Error(`Unknown function "${funcName}" at position ${tok.position}`)
       }
 
-      c.consume() // function name
-      c.consume() // '('
+      consume(state) // function name
+      consume(state) // '('
 
       /** @type {ExprNode[]} */
       const args = []
 
-      if (c.current().type !== 'paren' || c.current().value !== ')') {
+      if (current(state).type !== 'paren' || current(state).value !== ')') {
         while (true) {
           // Handle COUNT(*) - treat * as a special identifier
-          if (c.current().type === 'operator' && c.current().value === '*') {
-            c.consume()
+          if (current(state).type === 'operator' && current(state).value === '*') {
+            consume(state)
             args.push({
               type: 'identifier',
               name: '*',
             })
           } else {
-            const arg = parseExpression(c)
+            const arg = parseExpression(state)
             args.push(arg)
           }
-          if (!c.match('comma')) break
+          if (!match(state, 'comma')) break
         }
       }
 
-      c.expect('paren', ')')
+      expect(state, 'paren', ')')
 
       return {
         type: 'function',
@@ -97,13 +99,13 @@ export function parsePrimary(c) {
       }
     }
 
-    c.consume()
+    consume(state)
     let name = tok.value
 
     // table.column
-    if (c.current().type === 'dot') {
-      c.consume()
-      const columnTok = c.expectIdentifier()
+    if (current(state).type === 'dot') {
+      consume(state)
+      const columnTok = expectIdentifier(state)
       name = name + '.' + columnTok.value
     }
 
@@ -114,7 +116,7 @@ export function parsePrimary(c) {
   }
 
   if (tok.type === 'number') {
-    c.consume()
+    consume(state)
     return {
       type: 'literal',
       value: tok.numericValue ?? null,
@@ -122,7 +124,7 @@ export function parsePrimary(c) {
   }
 
   if (tok.type === 'string') {
-    c.consume()
+    consume(state)
     return {
       type: 'literal',
       value: tok.value,
@@ -131,44 +133,44 @@ export function parsePrimary(c) {
 
   if (tok.type === 'keyword') {
     if (tok.value === 'TRUE') {
-      c.consume()
+      consume(state)
       return { type: 'literal', value: true }
     }
     if (tok.value === 'FALSE') {
-      c.consume()
+      consume(state)
       return { type: 'literal', value: false }
     }
     if (tok.value === 'NULL') {
-      c.consume()
+      consume(state)
       return { type: 'literal', value: null }
     }
     if (tok.value === 'EXISTS') {
-      c.consume() // EXISTS
-      const subquery = c.parseSubquery()
+      consume(state) // EXISTS
+      const subquery = parseSubquery(state)
       return {
         type: 'exists',
         subquery,
       }
     }
     if (tok.value === 'CASE') {
-      c.consume() // CASE
+      consume(state) // CASE
 
       // Check if it's simple CASE (CASE expr WHEN ...) or searched CASE (CASE WHEN ...)
       /** @type {ExprNode | undefined} */
       let caseExpr
-      const nextTok = c.current()
+      const nextTok = current(state)
       if (nextTok.type !== 'keyword' || nextTok.value !== 'WHEN') {
         // Simple CASE: parse the case expression
-        caseExpr = parseExpression(c)
+        caseExpr = parseExpression(state)
       }
 
       // Parse WHEN clauses
       /** @type {WhenClause[]} */
       const whenClauses = []
-      while (c.match('keyword', 'WHEN')) {
-        const condition = parseExpression(c)
-        c.expect('keyword', 'THEN')
-        const result = parseExpression(c)
+      while (match(state, 'keyword', 'WHEN')) {
+        const condition = parseExpression(state)
+        expect(state, 'keyword', 'THEN')
+        const result = parseExpression(state)
         whenClauses.push({ condition, result })
       }
 
@@ -179,11 +181,11 @@ export function parsePrimary(c) {
       // Parse optional ELSE clause
       /** @type {ExprNode | undefined} */
       let elseResult
-      if (c.match('keyword', 'ELSE')) {
-        elseResult = parseExpression(c)
+      if (match(state, 'keyword', 'ELSE')) {
+        elseResult = parseExpression(state)
       }
 
-      c.expect('keyword', 'END')
+      expect(state, 'keyword', 'END')
 
       return {
         type: 'case',
@@ -195,8 +197,8 @@ export function parsePrimary(c) {
   }
 
   if (tok.type === 'operator' && tok.value === '-') {
-    c.consume()
-    const argument = parsePrimary(c)
+    consume(state)
+    const argument = parsePrimary(state)
     return {
       type: 'unary',
       op: '-',
@@ -209,13 +211,13 @@ export function parsePrimary(c) {
 }
 
 /**
- * @param {ExprCursor} c
+ * @param {ParserState} state
  * @returns {ExprNode}
  */
-function parseOr(c) {
-  let node = parseAnd(c)
-  while (c.match('keyword', 'OR')) {
-    const right = parseAnd(c)
+function parseOr(state) {
+  let node = parseAnd(state)
+  while (match(state, 'keyword', 'OR')) {
+    const right = parseAnd(state)
     node = {
       type: 'binary',
       op: 'OR',
@@ -227,13 +229,13 @@ function parseOr(c) {
 }
 
 /**
- * @param {ExprCursor} c
+ * @param {ParserState} state
  * @returns {ExprNode}
  */
-function parseAnd(c) {
-  let node = parseNot(c)
-  while (c.match('keyword', 'AND')) {
-    const right = parseNot(c)
+function parseAnd(state) {
+  let node = parseNot(state)
+  while (match(state, 'keyword', 'AND')) {
+    const right = parseNot(state)
     node = {
       type: 'binary',
       op: 'AND',
@@ -245,27 +247,40 @@ function parseAnd(c) {
 }
 
 /**
- * @param {ExprCursor} c
+ * @param {ParserState} state
  * @returns {ExprNode}
  */
-function parseNot(c) {
-  if (c.match('keyword', 'NOT')) {
+function parseNot(state) {
+  if (match(state, 'keyword', 'NOT')) {
     // Check for NOT EXISTS
-    const nextTok = c.current()
+    const nextTok = current(state)
     if (nextTok.type === 'keyword' && nextTok.value === 'EXISTS') {
-      c.consume() // EXISTS
-      const subquery = c.parseSubquery()
+      consume(state) // EXISTS
+      const subquery = parseSubquery(state)
       return {
         type: 'not exists',
         subquery,
       }
     }
-    const argument = parseNot(c)
+    const argument = parseNot(state)
     return {
       type: 'unary',
       op: 'NOT',
       argument,
     }
   }
-  return parseComparison(c)
+  return parseComparison(state)
+}
+
+/**
+ * Creates an ExprCursor adapter for the ParserState.
+ *
+ * @param {ParserState} state
+ * @returns {SelectStatement}
+ */
+export function parseSubquery(state) {
+  expect(state, 'paren', '(')
+  const query = parseSelectInternal(state)
+  expect(state, 'paren', ')')
+  return query
 }

package/src/parse/joins.js

@@ -0,0 +1,77 @@
+import { parseExpression } from './expression.js'
+import { parseTableAlias } from './parse.js'
+import { consume, current, expect, expectIdentifier, match } from './state.js'
+
+/**
+ * @import { JoinClause, JoinType, ParserState } from '../types.js'
+ * @param {ParserState} state
+ * @returns {JoinClause[]}
+ */
+export function parseJoins(state) {
+  /** @type {JoinClause[]} */
+  const joins = []
+
+  while (true) {
+    const tok = current(state)
+
+    // Check for join keywords
+    /** @type {JoinType} */
+    let joinType = 'INNER'
+
+    if (tok.type === 'keyword') {
+      if (tok.value === 'INNER') {
+        consume(state)
+        joinType = 'INNER'
+      } else if (tok.value === 'LEFT') {
+        consume(state)
+        if (match(state, 'keyword', 'OUTER')) {
+          // LEFT OUTER JOIN
+        }
+        joinType = 'LEFT'
+      } else if (tok.value === 'RIGHT') {
+        consume(state)
+        if (match(state, 'keyword', 'OUTER')) {
+          // RIGHT OUTER JOIN
+        }
+        joinType = 'RIGHT'
+      } else if (tok.value === 'FULL') {
+        consume(state)
+        if (match(state, 'keyword', 'OUTER')) {
+          // FULL OUTER JOIN
+        }
+        joinType = 'FULL'
+      } else if (tok.value === 'JOIN') {
+        // Just JOIN (defaults to INNER)
+        consume(state)
+      } else {
+        // Not a join keyword, stop parsing joins
+        break
+      }
+
+      // If we consumed a join type keyword (INNER/LEFT/RIGHT/FULL), expect JOIN
+      if (tok.value !== 'JOIN') {
+        expect(state, 'keyword', 'JOIN')
+      }
+    } else {
+      // No more joins
+      break
+    }
+
+    // Parse table name and optional alias
+    const tableName = expectIdentifier(state).value
+    const tableAlias = parseTableAlias(state)
+
+    // Parse ON condition
+    expect(state, 'keyword', 'ON')
+    const condition = parseExpression(state)
+
+    joins.push({
+      joinType,
+      table: tableName,
+      alias: tableAlias,
+      on: condition,
+    })
+  }
+
+  return joins
+}

package/src/parse/parse.js

@@ -1,39 +1,13 @@
 import { tokenize } from './tokenize.js'
 import { parseExpression } from './expression.js'
-import { isAggregateFunc } from '../validation.js'
+import { RESERVED_AFTER_COLUMN, RESERVED_AFTER_TABLE, isAggregateFunc } from '../validation.js'
+import { consume, current, expect, expectIdentifier, match, parseError, peekToken } from './state.js'
+import { parseJoins } from './joins.js'
 
 /**
- * @import { AggregateColumn, AggregateArg, AggregateFunc, ExprCursor, ExprNode, FromSubquery, FromTable, JoinClause, JoinType, OrderByItem, ParserState, SelectStatement, SelectColumn, Token, TokenType } from '../types.js'
+ * @import { AggregateColumn, AggregateArg, AggregateFunc, ExprNode, FromSubquery, FromTable, OrderByItem, ParserState, SelectStatement, SelectColumn } from '../types.js'
  */
 
-// Keywords that cannot be used as implicit aliases after a column
-const RESERVED_AFTER_COLUMN = new Set([
-  'FROM',
-  'WHERE',
-  'GROUP',
-  'HAVING',
-  'ORDER',
-  'LIMIT',
-  'OFFSET',
-])
-
-// Keywords that cannot be used as table aliases
-const RESERVED_AFTER_TABLE = new Set([
-  'WHERE',
-  'GROUP',
-  'HAVING',
-  'ORDER',
-  'LIMIT',
-  'OFFSET',
-  'JOIN',
-  'INNER',
-  'LEFT',
-  'RIGHT',
-  'FULL',
-  'CROSS',
-  'ON',
-])
-
 /**
  * @param {string} query
  * @returns {SelectStatement}
@@ -52,104 +26,6 @@ export function parseSql(query) {
   return select
 }
 
-/**
- * @param {ParserState} state
- * @returns {Token}
- */
-function current(state) {
-  return state.tokens[state.pos]
-}
-
-/**
- * @param {ParserState} state
- * @param {number} offset
- * @returns {Token}
- */
-function peekToken(state, offset) {
-  const idx = state.pos + offset
-  if (idx >= state.tokens.length) {
-    return state.tokens[state.tokens.length - 1]
-  }
-  return state.tokens[idx]
-}
-
-/**
- * @param {ParserState} state
- * @returns {Token}
- */
-function consume(state) {
-  const tok = current(state)
-  if (state.pos < state.tokens.length - 1) {
-    state.pos += 1
-  }
-  return tok
-}
-
-/**
- * @param {ParserState} state
- * @param {TokenType} type
- * @param {string} [value]
- * @returns {boolean}
- */
-function match(state, type, value) {
-  const tok = current(state)
-  if (tok.type !== type) return false
-  if (typeof value === 'string' && tok.value !== value) return false
-  consume(state)
-  return true
-}
-
-/**
- * @param {ParserState} state
- * @param {TokenType} type
- * @param {string} value
- * @returns {Token}
- */
-function expect(state, type, value) {
-  const tok = current(state)
-  if (tok.type !== type || tok.value !== value) {
-    throw parseError(state, value)
-  }
-  consume(state)
-  return tok
-}
-
-/**
- * @param {ParserState} state
- * @returns {Token}
- */
-function expectIdentifier(state) {
-  const tok = current(state)
-  if (tok.type !== 'identifier') {
-    throw parseError(state, 'identifier')
-  }
-  consume(state)
-  return tok
-}
-
-/**
- * Creates an ExprCursor adapter for the ParserState.
- *
- * @param {ParserState} state
- * @returns {ExprCursor}
- */
-function createExprCursor(state) {
-  return {
-    current: () => current(state),
-    peek: (offset) => peekToken(state, offset),
-    consume: () => consume(state),
-    match: (type, value) => match(state, type, value),
-    expect: (type, value) => expect(state, type, value),
-    expectIdentifier: () => expectIdentifier(state),
-    parseSubquery: () => {
-      expect(state, 'paren', '(')
-      const query = parseSelectInternal(state)
-      expect(state, 'paren', ')')
-      return query
-    },
-  }
-}
-
 /**
  * @param {ParserState} state
  * @returns {SelectColumn[]}
@@ -208,8 +84,7 @@ function parseSelectItem(state) {
   }
 
   // Delegate to expression parser
-  const cursor = createExprCursor(state)
-  const expr = parseExpression(cursor)
+  const expr = parseExpression(state)
   const alias = parseAs(state)
   return { kind: 'derived', expr, alias }
 }
@@ -239,8 +114,7 @@ function parseAggregateItem(state, func) {
       quantifier = 'distinct'
     }
 
-    const cursor = createExprCursor(state)
-    const expr = parseExpression(cursor)
+    const expr = parseExpression(state)
     arg = {
       kind: 'expression',
       expr,
@@ -260,7 +134,7 @@ function parseAggregateItem(state, func) {
  * @param {ParserState} state
  * @returns {string | undefined}
  */
-function parseTableAlias(state) {
+export function parseTableAlias(state) {
   // Check for explicit AS keyword
   if (match(state, 'keyword', 'AS')) {
     const aliasTok = expectIdentifier(state)
@@ -302,86 +176,12 @@ function parseAs(state) {
   }
 }
 
-/**
- * @param {ParserState} state
- * @returns {JoinClause[]}
- */
-function parseJoins(state) {
-  /** @type {JoinClause[]} */
-  const joins = []
-
-  while (true) {
-    const tok = current(state)
-
-    // Check for join keywords
-    /** @type {JoinType} */
-    let joinType = 'INNER'
-
-    if (tok.type === 'keyword') {
-      if (tok.value === 'INNER') {
-        consume(state)
-        joinType = 'INNER'
-      } else if (tok.value === 'LEFT') {
-        consume(state)
-        if (match(state, 'keyword', 'OUTER')) {
-          // LEFT OUTER JOIN
-        }
-        joinType = 'LEFT'
-      } else if (tok.value === 'RIGHT') {
-        consume(state)
-        if (match(state, 'keyword', 'OUTER')) {
-          // RIGHT OUTER JOIN
-        }
-        joinType = 'RIGHT'
-      } else if (tok.value === 'FULL') {
-        consume(state)
-        if (match(state, 'keyword', 'OUTER')) {
-          // FULL OUTER JOIN
-        }
-        joinType = 'FULL'
-      } else if (tok.value === 'JOIN') {
-        // Just JOIN (defaults to INNER)
-        consume(state)
-      } else {
-        // Not a join keyword, stop parsing joins
-        break
-      }
-
-      // If we consumed a join type keyword (INNER/LEFT/RIGHT/FULL), expect JOIN
-      if (tok.value !== 'JOIN') {
-        expect(state, 'keyword', 'JOIN')
-      }
-    } else {
-      // No more joins
-      break
-    }
-
-    // Parse table name and optional alias
-    const tableName = expectIdentifier(state).value
-    const tableAlias = parseTableAlias(state)
-
-    // Parse ON condition
-    expect(state, 'keyword', 'ON')
-    const cursor = createExprCursor(state)
-    const condition = parseExpression(cursor)
-
-    joins.push({
-      joinType,
-      table: tableName,
-      alias: tableAlias,
-      on: condition,
-    })
-  }
-
-  return joins
-}
-
 /**
  * Parses a subquery in parentheses with an alias
  * @param {ParserState} state
  * @returns {FromSubquery}
  */
-function parseSubquery(state) {
+function parseFromSubquery(state) {
   expect(state, 'paren', '(')
   const query = parseSelectInternal(state)
   expect(state, 'paren', ')')
@@ -394,7 +194,7 @@ function parseSubquery(state) {
  * @param {ParserState} state
  * @returns {SelectStatement}
  */
-function parseSelectInternal(state) {
+export function parseSelectInternal(state) {
   expect(state, 'keyword', 'SELECT')
 
   let distinct = false
@@ -412,7 +212,7 @@ function parseSelectInternal(state) {
   const tok = current(state)
   if (tok.type === 'paren' && tok.value === '(') {
     // Subquery: SELECT * FROM (SELECT ...) AS alias
-    from = parseSubquery(state)
+    from = parseFromSubquery(state)
   } else {
     // Simple table name: SELECT * FROM users
     const table = expectIdentifier(state).value
@@ -436,29 +236,27 @@ function parseSelectInternal(state) {
   /** @type {number | undefined} */
   let offset
 
-  const cursor = createExprCursor(state)
-
   if (match(state, 'keyword', 'WHERE')) {
-    where = parseExpression(cursor)
+    where = parseExpression(state)
   }
 
   if (match(state, 'keyword', 'GROUP')) {
     expect(state, 'keyword', 'BY')
     while (true) {
-      const expr = parseExpression(cursor)
+      const expr = parseExpression(state)
       groupBy.push(expr)
       if (!match(state, 'comma')) break
     }
   }
 
   if (match(state, 'keyword', 'HAVING')) {
-    having = parseExpression(cursor)
+    having = parseExpression(state)
   }
 
   if (match(state, 'keyword', 'ORDER')) {
     expect(state, 'keyword', 'BY')
     while (true) {
-      const expr = parseExpression(cursor)
+      const expr = parseExpression(state)
       /** @type {'ASC' | 'DESC'} */
       let direction = 'ASC'
       if (match(state, 'keyword', 'ASC')) {
@@ -544,17 +342,3 @@ function parseSelectInternal(state) {
     offset,
   }
 }
-
-/**
- * Helper function to create consistent parser error messages.
- * @param {ParserState} state
- * @param {string} expected - Description of what was expected
- * @returns {Error}
- */
-function parseError(state, expected) {
-  const tok = current(state)
-  const prevToken = state.tokens[state.pos - 1]
-  const after = prevToken ? ` after "${prevToken.originalValue ?? prevToken.value}"` : ''
-  const found = tok.type === 'eof' ? 'end of query' : `"${tok.originalValue ?? tok.value}"`
-  return new Error(`Expected ${expected}${after} but found ${found} at position ${tok.position}`)
-}

package/src/parse/state.js

@@ -0,0 +1,92 @@
+/**
+ * @import { ParserState, Token, TokenType } from '../types.js'
+ */
+
+/**
+ * @param {ParserState} state
+ * @returns {Token}
+ */
+export function current(state) {
+  return state.tokens[state.pos]
+}
+
+/**
+ * @param {ParserState} state
+ * @param {number} offset
+ * @returns {Token}
+ */
+export function peekToken(state, offset) {
+  const idx = state.pos + offset
+  if (idx >= state.tokens.length) {
+    return state.tokens[state.tokens.length - 1]
+  }
+  return state.tokens[idx]
+}
+
+/**
+ * @param {ParserState} state
+ * @returns {Token}
+ */
+export function consume(state) {
+  const tok = current(state)
+  if (state.pos < state.tokens.length - 1) {
+    state.pos += 1
+  }
+  return tok
+}
+
+/**
+ * @param {ParserState} state
+ * @param {TokenType} type
+ * @param {string} [value]
+ * @returns {boolean}
+ */
+export function match(state, type, value) {
+  const tok = current(state)
+  if (tok.type !== type) return false
+  if (typeof value === 'string' && tok.value !== value) return false
+  consume(state)
+  return true
+}
+
+/**
+ * @param {ParserState} state
+ * @param {TokenType} type
+ * @param {string} value
+ * @returns {Token}
+ */
+export function expect(state, type, value) {
+  const tok = current(state)
+  if (tok.type !== type || tok.value !== value) {
+    throw parseError(state, value)
+  }
+  consume(state)
+  return tok
+}
+
+/**
+ * @param {ParserState} state
+ * @returns {Token}
+ */
+export function expectIdentifier(state) {
+  const tok = current(state)
+  if (tok.type !== 'identifier') {
+    throw parseError(state, 'identifier')
+  }
+  consume(state)
+  return tok
+}
+
+/**
+ * Helper function to create consistent parser error messages.
+ * @param {ParserState} state
+ * @param {string} expected - Description of what was expected
+ * @returns {Error}
+ */
+export function parseError(state, expected) {
+  const tok = current(state)
+  const prevToken = state.tokens[state.pos - 1]
+  const after = prevToken ? ` after "${prevToken.originalValue ?? prevToken.value}"` : ''
+  const found = tok.type === 'eof' ? 'end of query' : `"${tok.originalValue ?? tok.value}"`
+  return new Error(`Expected ${expected}${after} but found ${found} at position ${tok.position}`)
+}

package/src/types.d.ts

@@ -1,10 +1,21 @@
 
+/**
+ * Hints passed to data sources for query optimization.
+ * All hints are optional and "best effort" - sources may ignore them.
+ */
+export interface QueryHints {
+  columns?: string[] // columns needed
+  where?: ExprNode // where clause
+  limit?: number
+  offset?: number
+}
+
 /**
  * Async data source for streaming SQL execution.
  * Provides an async iterator over rows.
  */
 export interface AsyncDataSource {
-  getRows(): AsyncIterable<AsyncRow>
+  getRows(hints?: QueryHints): AsyncIterable<AsyncRow>
 }
 export type AsyncRow = Record<string, AsyncCell>
 export type AsyncCell = () => Promise<SqlPrimitive>
@@ -43,17 +54,9 @@ export interface FromSubquery {
   alias: string
 }
 
-export type BinaryOp =
-  | 'AND'
-  | 'OR'
-  | '='
-  | '!='
-  | '<>'
-  | '<'
-  | '>'
-  | '<='
-  | '>='
-  | 'LIKE'
+export type BinaryOp = 'AND' | 'OR' | 'LIKE' | ComparisonOp
+
+export type ComparisonOp = '=' | '!=' | '<>' | '<' | '>' | '<=' | '>='
 
 export interface LiteralNode {
   type: 'literal'
@@ -194,16 +197,6 @@ export interface ParserState {
   pos: number
 }
 
-export interface ExprCursor {
-  current(): Token
-  peek(offset: number): Token
-  consume(): Token
-  match(type: TokenType, value?: string): boolean
-  expect(type: TokenType, value: string): Token
-  expectIdentifier(): Token
-  parseSubquery: () => SelectStatement
-}
-
 // Tokenizer types
 export type TokenType =
   | 'keyword'

package/src/validation.js

@@ -23,3 +23,14 @@ export function isStringFunc(name) {
 export function isBinaryOp(op) {
   return ['=', '!=', '<>', '<', '>', '<=', '>='].includes(op)
 }
+
+// Keywords that cannot be used as implicit aliases after a column
+export const RESERVED_AFTER_COLUMN = new Set([
+  'FROM', 'WHERE', 'GROUP', 'HAVING', 'ORDER', 'LIMIT', 'OFFSET',
+])
+
+// Keywords that cannot be used as table aliases
+export const RESERVED_AFTER_TABLE = new Set([
+  'WHERE', 'GROUP', 'HAVING', 'ORDER', 'LIMIT', 'OFFSET', 'JOIN', 'INNER',
+  'LEFT', 'RIGHT', 'FULL', 'CROSS', 'ON',
+])