squirreling 0.10.1 → 0.10.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "squirreling",
-  "version": "0.10.1",
+  "version": "0.10.2",
   "description": "Squirreling Async SQL Engine",
   "author": "Hyperparam",
   "homepage": "https://hyperparam.app",
@@ -39,11 +39,11 @@
     "test": "vitest run"
   },
   "devDependencies": {
-    "@types/node": "25.4.0",
-    "@vitest/coverage-v8": "4.0.18",
+    "@types/node": "25.5.0",
+    "@vitest/coverage-v8": "4.1.0",
     "eslint": "9.39.2",
-    "eslint-plugin-jsdoc": "62.7.1",
+    "eslint-plugin-jsdoc": "62.8.0",
     "typescript": "5.9.3",
-    "vitest": "4.0.18"
+    "vitest": "4.1.0"
   }
 }

package/src/execute/execute.js

@@ -119,7 +119,7 @@ async function* executeScan(plan, context) {
 
   // Apply WHERE if data source did not
   if (!appliedWhere && plan.hints.where) {
-    result = filterRows(result, plan.hints.where, context)
+    result = filterRows(result, plan.hints.where, context, plan.hints.limit)
   }
 
   // Apply LIMIT/OFFSET if data source did not
@@ -174,15 +174,42 @@ async function* executeCount(plan, { tables, signal }) {
  * @param {AsyncIterable<AsyncRow>} rows
  * @param {ExprNode} condition
  * @param {ExecuteContext} context
+ * @param {number} [limit] - downstream LIMIT hint for chunk sizing
  * @yields {AsyncRow}
  */
-async function* filterRows(rows, condition, context) {
+async function* filterRows(rows, condition, context, limit) {
+  const MAX_CHUNK = 256
+  let chunkSize = limit ?? Infinity
   let rowIndex = 0
+
+  /** @type {{ row: AsyncRow, rowIndex: number }[]} */
+  let buffer = []
+
   for await (const row of rows) {
     if (context.signal?.aborted) return
     rowIndex++
-    const pass = await evaluateExpr({ node: condition, row, rowIndex, context })
-    if (pass) yield row
+    buffer.push({ row, rowIndex })
+
+    if (buffer.length >= chunkSize) {
+      const results = await Promise.all(buffer.map(b =>
+        evaluateExpr({ node: condition, row: b.row, rowIndex: b.rowIndex, context })
+      ))
+      for (let i = 0; i < buffer.length; i++) {
+        if (results[i]) yield buffer[i].row
+      }
+      buffer = []
+      chunkSize = Math.min(chunkSize * 2, MAX_CHUNK)
+    }
+  }
+
+  // Flush remaining rows
+  if (buffer.length > 0) {
+    const results = await Promise.all(buffer.map(b =>
+      evaluateExpr({ node: condition, row: b.row, rowIndex: b.rowIndex, context })
+    ))
+    for (let i = 0; i < buffer.length; i++) {
+      if (results[i]) yield buffer[i].row
+    }
   }
 }
 
@@ -275,17 +302,38 @@ async function* executeProject(plan, context) {
  */
 async function* executeDistinct(plan, context) {
   const { signal } = context
+  const MAX_CHUNK = 256
 
   /** @type {Set<string>} */
   const seen = new Set()
 
+  /** @type {AsyncRow[]} */
+  let buffer = []
+
   for await (const row of executePlan({ plan: plan.child, context })) {
     if (signal?.aborted) return
+    buffer.push(row)
 
-    const key = await stableRowKey(row.cells)
-    if (!seen.has(key)) {
-      seen.add(key)
-      yield row
+    if (buffer.length >= MAX_CHUNK) {
+      const keys = await Promise.all(buffer.map(r => stableRowKey(r.cells)))
+      for (let i = 0; i < buffer.length; i++) {
+        if (!seen.has(keys[i])) {
+          seen.add(keys[i])
+          yield buffer[i]
+        }
+      }
+      buffer = []
+    }
+  }
+
+  // Flush remaining
+  if (buffer.length > 0) {
+    const keys = await Promise.all(buffer.map(r => stableRowKey(r.cells)))
+    for (let i = 0; i < buffer.length; i++) {
+      if (!seen.has(keys[i])) {
+        seen.add(keys[i])
+        yield buffer[i]
+      }
     }
   }
 }

package/src/execute/utils.js

@@ -44,17 +44,22 @@ export function compareForTerm(a, b, term) {
  * @returns {Promise<Record<string, SqlPrimitive>[]>} array of all yielded values
  */
 export async function collect(asyncRows) {
-  /** @type {Record<string, SqlPrimitive>[]} */
-  const results = []
+  // Collect all rows first, then materialize cells concurrently
+  // This enables dataloader-style batching of cell accessors
+  /** @type {AsyncRow[]} */
+  const rows = []
   for await (const asyncRow of asyncRows) {
+    rows.push(asyncRow)
+  }
+  return Promise.all(rows.map(async asyncRow => {
+    const values = await Promise.all(asyncRow.columns.map(k => asyncRow.cells[k]()))
     /** @type {Record<string, SqlPrimitive>} */
     const item = {}
-    for (const key of asyncRow.columns) {
-      item[key] = await asyncRow.cells[key]()
+    for (let i = 0; i < asyncRow.columns.length; i++) {
+      item[asyncRow.columns[i]] = values[i]
     }
-    results.push(item)
-  }
-  return results
+    return item
+  }))
 }
 
 /**
@@ -79,11 +84,6 @@ export function stringify(value) {
  */
 export async function stableRowKey(cells) {
   const keys = Object.keys(cells).sort()
-  /** @type {string[]} */
-  const parts = []
-  for (const k of keys) {
-    const v = await cells[k]()
-    parts.push(k + ':' + stringify(v))
-  }
-  return parts.join('|')
+  const values = await Promise.all(keys.map(k => cells[k]()))
+  return keys.map((k, i) => k + ':' + stringify(values[i])).join('|')
 }

package/src/expression/evaluate.js

@@ -118,11 +118,10 @@ export async function evaluateExpr({ node, row, rowIndex, rows, context }) {
       // Apply FILTER clause if present
       let filteredRows = rows
       if (node.filter) {
-        filteredRows = []
-        for (const row of rows) {
-          const passes = await evaluateExpr({ node: node.filter, row, context })
-          if (passes) filteredRows.push(row)
-        }
+        const passes = await Promise.all(rows.map(row =>
+          evaluateExpr({ node: node.filter, row, context })
+        ))
+        filteredRows = rows.filter((_, i) => passes[i])
       }
 
       const argNode = node.args[0]
@@ -132,23 +131,27 @@ export async function evaluateExpr({ node, row, rowIndex, rows, context }) {
           return filteredRows.length
         }
 
+        const values = await Promise.all(filteredRows.map(row =>
+          evaluateExpr({ node: argNode, row, context })
+        ))
         if (node.distinct) {
           const seen = new Set()
-          for (const row of filteredRows) {
-            const v = await evaluateExpr({ node: argNode, row, context })
+          for (const v of values) {
             if (v != null) seen.add(v)
           }
           return seen.size
         }
         let count = 0
-        for (const row of filteredRows) {
-          const v = await evaluateExpr({ node: argNode, row, context })
+        for (const v of values) {
           if (v != null) count++
         }
         return count
       }
 
       if (funcName === 'SUM' || funcName === 'AVG' || funcName === 'MIN' || funcName === 'MAX') {
+        const rawValues = await Promise.all(filteredRows.map(row =>
+          evaluateExpr({ node: argNode, row, context })
+        ))
         let sum = 0
         let count = 0
         /** @type {number | null} */
@@ -156,8 +159,7 @@ export async function evaluateExpr({ node, row, rowIndex, rows, context }) {
         /** @type {number | null} */
         let max = null
 
-        for (const row of filteredRows) {
-          const raw = await evaluateExpr({ node: argNode, row, context })
+        for (const raw of rawValues) {
           if (raw == null) continue
           const num = Number(raw)
           if (!Number.isFinite(num)) continue
@@ -180,9 +182,12 @@ export async function evaluateExpr({ node, row, rowIndex, rows, context }) {
       }
 
       if (funcName === 'STDDEV_SAMP' || funcName === 'STDDEV_POP') {
+        const rawValues = await Promise.all(filteredRows.map(row =>
+          evaluateExpr({ node: argNode, row, context })
+        ))
+        /** @type {number[]} */
         const values = []
-        for (const row of filteredRows) {
-          const raw = await evaluateExpr({ node: argNode, row, context })
+        for (const raw of rawValues) {
           if (raw == null) continue
           const num = Number(raw)
           if (!Number.isFinite(num)) continue