squirreling 0.4.1 → 0.4.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "squirreling",
-  "version": "0.4.1",
+  "version": "0.4.2",
   "description": "Squirreling SQL Engine",
   "author": "Hyperparam",
   "homepage": "https://hyperparam.app",
@@ -38,10 +38,10 @@
   },
   "devDependencies": {
     "@types/node": "24.10.1",
-    "@vitest/coverage-v8": "4.0.14",
+    "@vitest/coverage-v8": "4.0.15",
     "eslint": "9.39.1",
     "eslint-plugin-jsdoc": "61.4.1",
     "typescript": "5.9.3",
-    "vitest": "4.0.14"
+    "vitest": "4.0.15"
   }
 }

package/src/execute/execute.js

@@ -4,7 +4,7 @@ import { defaultAggregateAlias, evaluateAggregate } from './aggregates.js'
 import { evaluateExpr } from './expression.js'
 import { evaluateHavingExpr } from './having.js'
 import { executeJoins } from './join.js'
-import { defaultDerivedAlias } from './utils.js'
+import { compareForTerm, defaultDerivedAlias } from './utils.js'
 
 /**
  * @import { AsyncDataSource, ExecuteSqlOptions, ExprNode, OrderByItem, AsyncRow, SelectStatement, SqlPrimitive } from '../types.js'
@@ -89,31 +89,6 @@ async function stableRowKey(row) {
   return parts.join('|')
 }
 
-/**
- * Compares two SQL values for sorting
- *
- * @param {SqlPrimitive} a
- * @param {SqlPrimitive} b
- * @returns {number} negative if a < b, positive if a > b, 0 if equal
- */
-function compareValues(a, b) {
-  if (a === b) return 0
-  if (a == null) return -1
-  if (b == null) return 1
-
-  if (typeof a === 'number' && typeof b === 'number') {
-    if (a < b) return -1
-    if (a > b) return 1
-    return 0
-  }
-
-  const aa = String(a)
-  const bb = String(b)
-  if (aa < bb) return -1
-  if (aa > bb) return 1
-  return 0
-}
-
 /**
  * Applies DISTINCT filtering to remove duplicate rows
  *
@@ -135,127 +110,89 @@ async function applyDistinct(rows, distinct) {
   }
   return result
 }
-
 /**
- * Applies ORDER BY sorting to RowSource array (before projection)
- *
- * @param {AsyncRow[]} rows - the input row sources
- * @param {OrderByItem[]} orderBy - the sort specifications
- * @param {Record<string, AsyncDataSource>} tables
- * @returns {Promise<AsyncRow[]>} the sorted row sources
- */
-async function sortRowSources(rows, orderBy, tables) {
-  if (!orderBy.length) return rows
-
-  // Pre-evaluate ORDER BY expressions for all rows
-  /** @type {SqlPrimitive[][]} */
-  const evaluatedValues = []
-  for (const row of rows) {
-    /** @type {SqlPrimitive[]} */
-    const rowValues = []
-    for (const term of orderBy) {
-      const value = await evaluateExpr({ node: term.expr, row, tables })
-      rowValues.push(value)
-    }
-    evaluatedValues.push(rowValues)
-  }
-
-  // Create index array and sort it
-  const indices = rows.map((_, i) => i)
-  indices.sort((aIdx, bIdx) => {
-    for (let termIdx = 0; termIdx < orderBy.length; termIdx++) {
-      const term = orderBy[termIdx]
-      const dir = term.direction
-      const av = evaluatedValues[aIdx][termIdx]
-      const bv = evaluatedValues[bIdx][termIdx]
-
-      // Handle NULLS FIRST / NULLS LAST
-      const aIsNull = av == null
-      const bIsNull = bv == null
-
-      if (aIsNull || bIsNull) {
-        if (aIsNull && bIsNull) continue
-
-        const nullsFirst = term.nulls === 'LAST' ? false : true
-
-        if (aIsNull) {
-          return nullsFirst ? -1 : 1
-        } else {
-          return nullsFirst ? 1 : -1
-        }
-      }
-
-      const cmp = compareValues(av, bv)
-      if (cmp !== 0) {
-        return dir === 'DESC' ? -cmp : cmp
-      }
-    }
-    return 0
-  })
-
-  // Return sorted rows
-  return indices.map(i => rows[i])
-}
-
-/**
- * Applies ORDER BY sorting to rows
+ * Applies ORDER BY sorting to rows using multi-pass lazy evaluation.
+ * Secondary ORDER BY columns are only evaluated for rows that tie on
+ * previous columns, reducing expensive cell evaluations.
  *
  * @param {AsyncRow[]} rows - the input rows
  * @param {OrderByItem[]} orderBy - the sort specifications
  * @param {Record<string, AsyncDataSource>} tables
  * @returns {Promise<AsyncRow[]>} the sorted rows
  */
-async function applyOrderBy(rows, orderBy, tables) {
+async function sortRows(rows, orderBy, tables) {
   if (!orderBy.length) return rows
 
-  // Pre-evaluate ORDER BY expressions for all rows
-  /** @type {SqlPrimitive[][]} */
-  const evaluatedValues = []
-  for (const row of rows) {
-    /** @type {SqlPrimitive[]} */
-    const rowValues = []
-    for (const term of orderBy) {
-      const value = await evaluateExpr({ node: term.expr, row, tables })
-      rowValues.push(value)
-    }
-    evaluatedValues.push(rowValues)
-  }
-
-  // Create index array and sort it
-  const indices = rows.map((_, i) => i)
-  indices.sort((aIdx, bIdx) => {
-    for (let termIdx = 0; termIdx < orderBy.length; termIdx++) {
-      const term = orderBy[termIdx]
-      const dir = term.direction
-      const av = evaluatedValues[aIdx][termIdx]
-      const bv = evaluatedValues[bIdx][termIdx]
+  // Cache for evaluated values: evaluatedValues[rowIdx][colIdx]
+  /** @type {(SqlPrimitive | undefined)[][]} */
+  const evaluatedValues = rows.map(() => Array(orderBy.length))
 
-      // Handle NULLS FIRST / NULLS LAST
-      const aIsNull = av == null
-      const bIsNull = bv == null
+  // Start with all indices in one group
+  /** @type {number[][]} */
+  let groups = [rows.map((_, i) => i)]
 
-      if (aIsNull || bIsNull) {
-        if (aIsNull && bIsNull) continue
+  // Process each ORDER BY column incrementally
+  for (let orderByIdx = 0; orderByIdx < orderBy.length; orderByIdx++) {
+    const term = orderBy[orderByIdx]
+    /** @type {number[][]} */
+    const nextGroups = []
 
-        const nullsFirst = term.nulls === 'LAST' ? false : true
+    for (const group of groups) {
+      // Single-element groups don't need sorting or evaluation
+      if (group.length <= 1) {
+        nextGroups.push(group)
+        continue
+      }
 
-        if (aIsNull) {
-          return nullsFirst ? -1 : 1
-        } else {
-          return nullsFirst ? 1 : -1
+      // Evaluate this column for all rows in the group
+      for (const idx of group) {
+        if (evaluatedValues[idx][orderByIdx] === undefined) {
+          evaluatedValues[idx][orderByIdx] = await evaluateExpr({
+            node: term.expr,
+            row: rows[idx],
+            tables,
+          })
         }
       }
 
-      const cmp = compareValues(av, bv)
-      if (cmp !== 0) {
-        return dir === 'DESC' ? -cmp : cmp
+      // Sort the group by this column
+      group.sort((aIdx, bIdx) => {
+        const av = evaluatedValues[aIdx][orderByIdx]
+        const bv = evaluatedValues[bIdx][orderByIdx]
+        return compareForTerm(av, bv, term)
+      })
+
+      // Split into sub-groups based on ties (for next column)
+      if (orderByIdx < orderBy.length - 1) {
+        /** @type {number[]} */
+        let currentSubGroup = [group[0]]
+        for (let i = 1; i < group.length; i++) {
+          const prevIdx = group[i - 1]
+          const currIdx = group[i]
+          const prevVal = evaluatedValues[prevIdx][orderByIdx]
+          const currVal = evaluatedValues[currIdx][orderByIdx]
+
+          if (compareForTerm(prevVal, currVal, term) === 0) {
+            // Same value, extend current sub-group
+            currentSubGroup.push(currIdx)
+          } else {
+            // Different value, start new sub-group
+            nextGroups.push(currentSubGroup)
+            currentSubGroup = [currIdx]
+          }
+        }
+        nextGroups.push(currentSubGroup)
+      } else {
+        // Last column, no need to split
+        nextGroups.push(group)
       }
     }
-    return 0
-  })
 
-  // Return sorted rows
-  return indices.map(i => rows[i])
+    groups = nextGroups
+  }
+
+  // Flatten groups to get final sorted indices
+  return groups.flat().map(i => rows[i])
 }
 
 /**
@@ -466,7 +403,7 @@ async function* evaluateBuffered(select, dataSource, tables, hasAggregate, useGr
   } else {
     // No grouping, simple projection
     // Sort before projection so ORDER BY can access columns not in SELECT
-    const sorted = await sortRowSources(filtered, select.orderBy, tables)
+    const sorted = await sortRows(filtered, select.orderBy, tables)
 
     // OPTIMIZATION: For non-DISTINCT queries, apply OFFSET/LIMIT before projection
     // to avoid reading expensive cells for rows that won't be in the final result
@@ -498,7 +435,9 @@ async function* evaluateBuffered(select, dataSource, tables, hasAggregate, useGr
   projected = await applyDistinct(projected, select.distinct)
 
   // Step 5: ORDER BY (final sort for grouped queries)
-  projected = await applyOrderBy(projected, select.orderBy, tables)
+  if (useGrouping) {
+    projected = await sortRows(projected, select.orderBy, tables)
+  }
 
   // Step 6: OFFSET and LIMIT
   // For non-DISTINCT, non-grouping queries, OFFSET/LIMIT was already applied before projection

package/src/execute/expression.js

@@ -110,21 +110,6 @@ export async function evaluateExpr({ node, row, tables }) {
     }
   }
 
-  // BETWEEN and NOT BETWEEN
-  if (node.type === 'between' || node.type === 'not between') {
-    const expr = await evaluateExpr({ node: node.expr, row, tables })
-    const lower = await evaluateExpr({ node: node.lower, row, tables })
-    const upper = await evaluateExpr({ node: node.upper, row, tables })
-
-    // If any value is NULL, return false (SQL behavior)
-    if (expr == null || lower == null || upper == null) {
-      return false
-    }
-
-    const isBetween = expr >= lower && expr <= upper
-    return node.type === 'between' ? isBetween : !isBetween
-  }
-
   // Function calls
   if (node.type === 'function') {
     const funcName = node.name.toUpperCase()
@@ -200,6 +185,11 @@ export async function evaluateExpr({ node, row, tables }) {
       return String(str).replaceAll(String(searchStr), String(replaceStr))
     }
 
+    if (funcName === 'RANDOM' || funcName === 'RAND') {
+      if (args.length !== 0) throw new Error(`${funcName} takes no arguments`)
+      return Math.random()
+    }
+
     throw new Error('Unsupported function ' + funcName)
   }
 
@@ -238,16 +228,7 @@ export async function evaluateExpr({ node, row, tables }) {
     }
     return false
   }
-  if (node.type === 'not in valuelist') {
-    const exprVal = await evaluateExpr({ node: node.expr, row, tables })
-    for (const valueNode of node.values) {
-      const val = await evaluateExpr({ node: valueNode, row, tables })
-      if (exprVal === val) return false
-    }
-    return true
-  }
-
-  // IN and NOT IN with subqueries
+  // IN with subqueries
   if (node.type === 'in') {
     const exprVal = await evaluateExpr({ node: node.expr, row, tables })
     const results = executeSelect(node.subquery, tables)
@@ -260,18 +241,6 @@ export async function evaluateExpr({ node, row, tables }) {
     }
     return values.includes(exprVal)
   }
-  if (node.type === 'not in') {
-    const exprVal = await evaluateExpr({ node: node.expr, row, tables })
-    const results = executeSelect(node.subquery, tables)
-    /** @type {SqlPrimitive[]} */
-    const values = []
-    for await (const resRow of results) {
-      const firstKey = Object.keys(resRow)[0]
-      const val = await resRow[firstKey]()
-      values.push(val)
-    }
-    return !values.includes(exprVal)
-  }
 
   // EXISTS and NOT EXISTS with subqueries
   if (node.type === 'exists') {

package/src/execute/having.js

@@ -76,20 +76,6 @@ export async function evaluateHavingExpr(expr, row, group, tables) {
     }
   }
 
-  if (expr.type === 'between' || expr.type === 'not between') {
-    const exprVal = await evaluateHavingValue(expr.expr, context, group, tables)
-    const lower = await evaluateHavingValue(expr.lower, context, group, tables)
-    const upper = await evaluateHavingValue(expr.upper, context, group, tables)
-
-    // If any value is NULL, return false (SQL behavior)
-    if (exprVal == null || lower == null || upper == null) {
-      return false
-    }
-
-    const isBetween = exprVal >= lower && exprVal <= upper
-    return expr.type === 'between' ? isBetween : !isBetween
-  }
-
   // For other expression types, use the context row
   return Boolean(await evaluateExpr({ node: expr, row: context, tables }))
 }
@@ -112,7 +98,7 @@ function evaluateHavingValue(expr, context, group, tables) {
   }
 
   // For binary expressions, we need to use evaluateHavingExpr to properly handle aggregates
-  if (expr.type === 'binary' || expr.type === 'unary' || expr.type === 'between' || expr.type === 'not between') {
+  if (expr.type === 'binary' || expr.type === 'unary') {
     return evaluateHavingExpr(expr, context, group, tables)
   }
 

package/src/execute/utils.js

@@ -1,7 +1,44 @@
+/**
+ * @import {AsyncRow, ExprNode, OrderByItem, SqlPrimitive} from '../types.js'
+ */
+
+/**
+ * Compares two values for a single ORDER BY term, handling nulls and direction
+ *
+ * @param {SqlPrimitive} a
+ * @param {SqlPrimitive} b
+ * @param {OrderByItem} term
+ * @returns {number} comparison result
+ */
+export function compareForTerm(a, b, term) {
+  const aIsNull = a == null
+  const bIsNull = b == null
+
+  if (aIsNull || bIsNull) {
+    if (aIsNull && bIsNull) return 0
+    const nullsFirst = term.nulls !== 'LAST'
+    if (aIsNull) return nullsFirst ? -1 : 1
+    return nullsFirst ? 1 : -1
+  }
+
+  // Compare non-null values
+  if (a === b) return 0
+
+  let cmp
+  if (typeof a === 'number' && typeof b === 'number') {
+    cmp = a < b ? -1 : a > b ? 1 : 0
+  } else {
+    const aa = String(a)
+    const bb = String(b)
+    cmp = aa < bb ? -1 : aa > bb ? 1 : 0
+  }
+
+  return term.direction === 'DESC' ? -cmp : cmp
+}
+
 /**
  * Collects and materialize all results from an async row generator into an array
  *
- * @import {AsyncRow, ExprNode, SqlPrimitive} from '../types.js'
  * @param {AsyncGenerator<AsyncRow>} asyncRows
  * @returns {Promise<Record<string, SqlPrimitive>[]>} array of all yielded values
  */

package/src/index.d.ts

@@ -1,4 +1,5 @@
 import type { AsyncRow, ExecuteSqlOptions, SelectStatement, SqlPrimitive } from './types.js'
+export type { AsyncDataSource, AsyncRow, SqlPrimitive } from './types.js'
 
 /**
  * Executes a SQL SELECT query against an array of data rows

package/src/parse/expression.js

@@ -301,7 +301,26 @@ function parseComparison(c) {
     }
   }
 
-  // LIKE
+  // [NOT] LIKE
+  if (tok.type === 'keyword' && tok.value === 'NOT') {
+    const nextTok = c.peek(1)
+    if (nextTok.type === 'keyword' && nextTok.value === 'LIKE') {
+      c.consume() // NOT
+      c.consume() // LIKE
+      const right = parsePrimary(c)
+      return {
+        type: 'unary',
+        op: 'NOT',
+        argument: {
+          type: 'binary',
+          op: 'LIKE',
+          left,
+          right,
+        },
+      }
+    }
+  }
+
   if (tok.type === 'keyword' && tok.value === 'LIKE') {
     c.consume()
     const right = parsePrimary(c)
@@ -313,7 +332,7 @@ function parseComparison(c) {
     }
   }
 
-  // [NOT] BETWEEN
+  // [NOT] BETWEEN - convert to range comparison
   if (tok.type === 'keyword' && tok.value === 'NOT') {
     const nextTok = c.peek(1)
     if (nextTok.type === 'keyword' && nextTok.value === 'BETWEEN') {
@@ -322,11 +341,12 @@ function parseComparison(c) {
       const lower = parsePrimary(c)
       c.expect('keyword', 'AND')
       const upper = parsePrimary(c)
+      // NOT BETWEEN -> expr < lower OR expr > upper
       return {
-        type: 'not between',
-        expr: left,
-        lower,
-        upper,
+        type: 'binary',
+        op: 'OR',
+        left: { type: 'binary', op: '<', left, right: lower },
+        right: { type: 'binary', op: '>', left, right: upper },
       }
     }
   }
@@ -336,11 +356,12 @@ function parseComparison(c) {
     const lower = parsePrimary(c)
     c.expect('keyword', 'AND')
     const upper = parsePrimary(c)
+    // BETWEEN -> expr >= lower AND expr <= upper
     return {
-      type: 'between',
-      expr: left,
-      lower,
-      upper,
+      type: 'binary',
+      op: 'AND',
+      left: { type: 'binary', op: '>=', left, right: lower },
+      right: { type: 'binary', op: '<=', left, right: upper },
     }
   }
 
@@ -365,9 +386,13 @@ function parseComparison(c) {
         }
         const subquery = c.parseSubquery()
         return {
-          type: 'not in',
-          expr: left,
-          subquery,
+          type: 'unary',
+          op: 'NOT',
+          argument: {
+            type: 'in',
+            expr: left,
+            subquery,
+          },
         }
       } else {
         // Parse list of values - we handle the parens
@@ -380,9 +405,13 @@ function parseComparison(c) {
         }
         c.expect('paren', ')')
         return {
-          type: 'not in valuelist',
-          expr: left,
-          values,
+          type: 'unary',
+          op: 'NOT',
+          argument: {
+            type: 'in valuelist',
+            expr: left,
+            values,
+          },
         }
       }
     }

package/src/types.d.ts

@@ -90,21 +90,14 @@ export interface CastNode {
   toType: string
 }
 
-export interface BetweenNode {
-  type: 'between' | 'not between'
-  expr: ExprNode
-  lower: ExprNode
-  upper: ExprNode
-}
-
 export interface InSubqueryNode {
-  type: 'in' | 'not in'
+  type: 'in'
   expr: ExprNode
   subquery: SelectStatement
 }
 
 export interface InValuesNode {
-  type: 'in valuelist' | 'not in valuelist'
+  type: 'in valuelist'
   expr: ExprNode
   values: ExprNode[]
 }
@@ -138,7 +131,6 @@ export type ExprNode =
   | BinaryNode
   | FunctionNode
   | CastNode
-  | BetweenNode
   | InSubqueryNode
   | InValuesNode
   | ExistsNode

package/src/validation.js

@@ -13,5 +13,5 @@ export function isAggregateFunc(name) {
  * @returns {name is StringFunc}
  */
 export function isStringFunc(name) {
-  return ['UPPER', 'LOWER', 'CONCAT', 'LENGTH', 'SUBSTRING', 'SUBSTR', 'TRIM', 'REPLACE'].includes(name)
+  return ['UPPER', 'LOWER', 'CONCAT', 'LENGTH', 'SUBSTRING', 'SUBSTR', 'TRIM', 'REPLACE', 'RANDOM', 'RAND'].includes(name)
 }