squirreling 0.10.1 → 0.10.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "squirreling",
-  "version": "0.10.1",
+  "version": "0.10.3",
   "description": "Squirreling Async SQL Engine",
   "author": "Hyperparam",
   "homepage": "https://hyperparam.app",
@@ -39,11 +39,11 @@
     "test": "vitest run"
   },
   "devDependencies": {
-    "@types/node": "25.4.0",
-    "@vitest/coverage-v8": "4.0.18",
+    "@types/node": "25.5.0",
+    "@vitest/coverage-v8": "4.1.0",
     "eslint": "9.39.2",
-    "eslint-plugin-jsdoc": "62.7.1",
+    "eslint-plugin-jsdoc": "62.8.0",
     "typescript": "5.9.3",
-    "vitest": "4.0.18"
+    "vitest": "4.1.0"
   }
 }

package/src/execute/execute.js

@@ -119,7 +119,7 @@ async function* executeScan(plan, context) {
 
   // Apply WHERE if data source did not
   if (!appliedWhere && plan.hints.where) {
-    result = filterRows(result, plan.hints.where, context)
+    result = filterRows(result, plan.hints.where, context, plan.hints.limit)
   }
 
   // Apply LIMIT/OFFSET if data source did not
@@ -174,15 +174,42 @@ async function* executeCount(plan, { tables, signal }) {
  * @param {AsyncIterable<AsyncRow>} rows
  * @param {ExprNode} condition
  * @param {ExecuteContext} context
+ * @param {number} [limit] - downstream LIMIT hint for chunk sizing
  * @yields {AsyncRow}
  */
-async function* filterRows(rows, condition, context) {
+async function* filterRows(rows, condition, context, limit) {
+  const MAX_CHUNK = 256
+  let chunkSize = limit ?? Infinity
   let rowIndex = 0
+
+  /** @type {{ row: AsyncRow, rowIndex: number }[]} */
+  let buffer = []
+
   for await (const row of rows) {
     if (context.signal?.aborted) return
     rowIndex++
-    const pass = await evaluateExpr({ node: condition, row, rowIndex, context })
-    if (pass) yield row
+    buffer.push({ row, rowIndex })
+
+    if (buffer.length >= chunkSize) {
+      const results = await Promise.all(buffer.map(b =>
+        evaluateExpr({ node: condition, row: b.row, rowIndex: b.rowIndex, context })
+      ))
+      for (let i = 0; i < buffer.length; i++) {
+        if (results[i]) yield buffer[i].row
+      }
+      buffer = []
+      chunkSize = Math.min(chunkSize * 2, MAX_CHUNK)
+    }
+  }
+
+  // Flush remaining rows
+  if (buffer.length > 0) {
+    const results = await Promise.all(buffer.map(b =>
+      evaluateExpr({ node: condition, row: b.row, rowIndex: b.rowIndex, context })
+    ))
+    for (let i = 0; i < buffer.length; i++) {
+      if (results[i]) yield buffer[i].row
+    }
   }
 }
 
@@ -275,17 +302,38 @@ async function* executeProject(plan, context) {
  */
 async function* executeDistinct(plan, context) {
   const { signal } = context
+  const MAX_CHUNK = 256
 
   /** @type {Set<string>} */
   const seen = new Set()
 
+  /** @type {AsyncRow[]} */
+  let buffer = []
+
   for await (const row of executePlan({ plan: plan.child, context })) {
     if (signal?.aborted) return
+    buffer.push(row)
 
-    const key = await stableRowKey(row.cells)
-    if (!seen.has(key)) {
-      seen.add(key)
-      yield row
+    if (buffer.length >= MAX_CHUNK) {
+      const keys = await Promise.all(buffer.map(r => stableRowKey(r.cells)))
+      for (let i = 0; i < buffer.length; i++) {
+        if (!seen.has(keys[i])) {
+          seen.add(keys[i])
+          yield buffer[i]
+        }
+      }
+      buffer = []
+    }
+  }
+
+  // Flush remaining
+  if (buffer.length > 0) {
+    const keys = await Promise.all(buffer.map(r => stableRowKey(r.cells)))
+    for (let i = 0; i < buffer.length; i++) {
+      if (!seen.has(keys[i])) {
+        seen.add(keys[i])
+        yield buffer[i]
+      }
     }
   }
 }

package/src/execute/utils.js

@@ -44,17 +44,22 @@ export function compareForTerm(a, b, term) {
  * @returns {Promise<Record<string, SqlPrimitive>[]>} array of all yielded values
  */
 export async function collect(asyncRows) {
-  /** @type {Record<string, SqlPrimitive>[]} */
-  const results = []
+  // Collect all rows first, then materialize cells concurrently
+  // This enables dataloader-style batching of cell accessors
+  /** @type {AsyncRow[]} */
+  const rows = []
   for await (const asyncRow of asyncRows) {
+    rows.push(asyncRow)
+  }
+  return Promise.all(rows.map(async asyncRow => {
+    const values = await Promise.all(asyncRow.columns.map(k => asyncRow.cells[k]()))
     /** @type {Record<string, SqlPrimitive>} */
     const item = {}
-    for (const key of asyncRow.columns) {
-      item[key] = await asyncRow.cells[key]()
+    for (let i = 0; i < asyncRow.columns.length; i++) {
+      item[asyncRow.columns[i]] = values[i]
     }
-    results.push(item)
-  }
-  return results
+    return item
+  }))
 }
 
 /**
@@ -79,11 +84,6 @@ export function stringify(value) {
  */
 export async function stableRowKey(cells) {
   const keys = Object.keys(cells).sort()
-  /** @type {string[]} */
-  const parts = []
-  for (const k of keys) {
-    const v = await cells[k]()
-    parts.push(k + ':' + stringify(v))
-  }
-  return parts.join('|')
+  const values = await Promise.all(keys.map(k => cells[k]()))
+  return keys.map((k, i) => k + ':' + stringify(values[i])).join('|')
 }

package/src/expression/evaluate.js

@@ -118,11 +118,10 @@ export async function evaluateExpr({ node, row, rowIndex, rows, context }) {
       // Apply FILTER clause if present
       let filteredRows = rows
       if (node.filter) {
-        filteredRows = []
-        for (const row of rows) {
-          const passes = await evaluateExpr({ node: node.filter, row, context })
-          if (passes) filteredRows.push(row)
-        }
+        const passes = await Promise.all(rows.map(row =>
+          evaluateExpr({ node: node.filter, row, context })
+        ))
+        filteredRows = rows.filter((_, i) => passes[i])
       }
 
       const argNode = node.args[0]
@@ -132,23 +131,27 @@ export async function evaluateExpr({ node, row, rowIndex, rows, context }) {
           return filteredRows.length
         }
 
+        const values = await Promise.all(filteredRows.map(row =>
+          evaluateExpr({ node: argNode, row, context })
+        ))
         if (node.distinct) {
           const seen = new Set()
-          for (const row of filteredRows) {
-            const v = await evaluateExpr({ node: argNode, row, context })
+          for (const v of values) {
             if (v != null) seen.add(v)
           }
           return seen.size
         }
         let count = 0
-        for (const row of filteredRows) {
-          const v = await evaluateExpr({ node: argNode, row, context })
+        for (const v of values) {
           if (v != null) count++
         }
         return count
       }
 
       if (funcName === 'SUM' || funcName === 'AVG' || funcName === 'MIN' || funcName === 'MAX') {
+        const rawValues = await Promise.all(filteredRows.map(row =>
+          evaluateExpr({ node: argNode, row, context })
+        ))
         let sum = 0
         let count = 0
         /** @type {number | null} */
@@ -156,8 +159,7 @@ export async function evaluateExpr({ node, row, rowIndex, rows, context }) {
         /** @type {number | null} */
         let max = null
 
-        for (const row of filteredRows) {
-          const raw = await evaluateExpr({ node: argNode, row, context })
+        for (const raw of rawValues) {
           if (raw == null) continue
           const num = Number(raw)
           if (!Number.isFinite(num)) continue
@@ -180,9 +182,12 @@ export async function evaluateExpr({ node, row, rowIndex, rows, context }) {
       }
 
       if (funcName === 'STDDEV_SAMP' || funcName === 'STDDEV_POP') {
+        const rawValues = await Promise.all(filteredRows.map(row =>
+          evaluateExpr({ node: argNode, row, context })
+        ))
+        /** @type {number[]} */
         const values = []
-        for (const row of filteredRows) {
-          const raw = await evaluateExpr({ node: argNode, row, context })
+        for (const raw of rawValues) {
           if (raw == null) continue
           const num = Number(raw)
           if (!Number.isFinite(num)) continue

package/src/index.d.ts

@@ -76,6 +76,8 @@ export function tokenizeSql(sql: string): Token[]
  */
 export function collect<T>(asyncGen: AsyncGenerator<AsyncRow>): Promise<Record<string, SqlPrimitive>[]>
 
+export function asyncRow(row: Record<string, SqlPrimitive>, columns: string[]): AsyncRow
+
 export function cachedDataSource(source: AsyncDataSource): AsyncDataSource
 
 /**

package/src/index.js

@@ -3,5 +3,5 @@ export { parseSql } from './parse/parse.js'
 export { planSql } from './plan/plan.js'
 export { tokenizeSql } from './parse/tokenize.js'
 export { collect } from './execute/utils.js'
-export { cachedDataSource } from './backend/dataSource.js'
+export { asyncRow, cachedDataSource } from './backend/dataSource.js'
 export { derivedAlias } from './expression/alias.js'

package/src/spatial/bbox.js

@@ -1,11 +1,11 @@
 /**
- * @import { BBox, SimpleGeometry } from './geometry.js'
+ * @import { BoundingBox, SimpleGeometry } from './geometry.js'
  */
 
 export const EPSILON = 1e-10
 export const EPSILON_SQ = EPSILON * EPSILON
 
-/** @type {WeakMap<SimpleGeometry, BBox>} */
+/** @type {WeakMap<SimpleGeometry, BoundingBox>} */
 const bboxCache = new WeakMap()
 
 /**
@@ -26,7 +26,7 @@ export function bboxOverlap(a, b) {
  * Results are cached per geometry object.
  *
  * @param {SimpleGeometry} geom
- * @returns {BBox}
+ * @returns {BoundingBox}
  */
 export function bbox(geom) {
   let b = bboxCache.get(geom)

package/src/spatial/geometry.d.ts

@@ -15,7 +15,7 @@ export type Geometry =
  */
 export type SimpleGeometry = Point | LineString | Polygon
 
-export interface BBox {
+export interface BoundingBox {
   minX: number
   minY: number
   maxX: number

package/src/spatial/index.d.ts

@@ -0,0 +1,6 @@
+import type { BoundingBox, Geometry, SimpleGeometry } from './geometry.js'
+
+export function decompose(geom: Geometry): SimpleGeometry[]
+export function bbox(geom: SimpleGeometry): BoundingBox
+export function bboxOverlap(a: SimpleGeometry, b: SimpleGeometry): boolean
+export function parseWkt(wkt: string): Geometry | null

package/src/spatial/index.js

@@ -0,0 +1,3 @@
+export { decompose } from './spatial.js'
+export { bbox, bboxOverlap } from './bbox.js'
+export { parseWkt } from './wkt.js'

package/src/spatial/spatial.js

@@ -68,6 +68,25 @@ export function evaluateSpatialFunc({ funcName, args }) {
   }
 }
 
+/**
+ * Decompose Multi* and GeometryCollection into simple geometries.
+ *
+ * @param {Geometry} geom
+ * @returns {SimpleGeometry[]}
+ */
+export function decompose(geom) {
+  if (geom.type === 'MultiPoint') {
+    return geom.coordinates.map(c => ({ type: 'Point', coordinates: c }))
+  } else if (geom.type === 'MultiLineString') {
+    return geom.coordinates.map(c => ({ type: 'LineString', coordinates: c }))
+  } else if (geom.type === 'MultiPolygon') {
+    return geom.coordinates.map(c => ({ type: 'Polygon', coordinates: c }))
+  } else if (geom.type === 'GeometryCollection') {
+    return geom.geometries.flatMap(decompose)
+  }
+  return [geom]
+}
+
 /**
  * Normalize a geometry value. Accepts GeoJSON objects.
  * Returns null if the value is not a valid geometry.
@@ -90,10 +109,6 @@ function toGeometry(val) {
   return null
 }
 
-// ============================================================================
-// Minimum distance between geometries
-// ============================================================================
-
 /**
  * Get all line segments from a geometry.
  *
@@ -165,35 +180,6 @@ function stDWithin(a, b, distance) {
   return false
 }
 
-// ============================================================================
-// Spatial predicate dispatch - decompose to primitive type pairs
-// ============================================================================
-
-/**
- * Decompose Multi* and GeometryCollection into simple geometries.
- *
- * @param {Geometry} geom
- * @returns {SimpleGeometry[]}
- */
-function decompose(geom) {
-  switch (geom.type) {
-  case 'MultiPoint':
-    return geom.coordinates.map(c => ({ type: 'Point', coordinates: c }))
-  case 'MultiLineString':
-    return geom.coordinates.map(c => ({ type: 'LineString', coordinates: c }))
-  case 'MultiPolygon':
-    return geom.coordinates.map(c => ({ type: 'Polygon', coordinates: c }))
-  case 'GeometryCollection':
-    return geom.geometries.flatMap(decompose)
-  default:
-    return [geom]
-  }
-}
-
-// ============================================================================
-// ST_Contains
-// ============================================================================
-
 /**
  * @param {SimpleGeometry[]} a
  * @param {SimpleGeometry[]} b
@@ -204,10 +190,6 @@ function stContains(a, b) {
   return b.every(pb => a.some(pa => pairContainment(pa, pb) !== 'OUTSIDE'))
 }
 
-// ============================================================================
-// ST_ContainsProperly
-// ============================================================================
-
 /**
  * @param {SimpleGeometry[]} a
  * @param {SimpleGeometry[]} b
@@ -218,10 +200,6 @@ function stContainsProperly(a, b) {
   return b.every(pb => a.some(pa => pairContainment(pa, pb) === 'INSIDE'))
 }
 
-// ============================================================================
-// ST_Touches
-// ============================================================================
-
 /**
  * @param {SimpleGeometry[]} a
  * @param {SimpleGeometry[]} b
@@ -239,10 +217,6 @@ function stTouches(a, b) {
   return intersects
 }
 
-// ============================================================================
-// ST_Overlaps
-// ============================================================================
-
 /**
  * @param {SimpleGeometry[]} a
  * @param {SimpleGeometry[]} b
@@ -281,10 +255,6 @@ function geometryDimension(parts) {
   return max
 }
 
-// ============================================================================
-// ST_Equals
-// ============================================================================
-
 /**
  * @param {SimpleGeometry[]} a
  * @param {SimpleGeometry[]} b
@@ -310,10 +280,6 @@ function stEquals(a, b) {
   return true
 }
 
-// ============================================================================
-// ST_Crosses
-// ============================================================================
-
 /**
  * @param {SimpleGeometry[]} a
  * @param {SimpleGeometry[]} b