squirreling 0.12.9 → 0.12.10

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "squirreling",
-  "version": "0.12.9",
+  "version": "0.12.10",
   "description": "Squirreling Async SQL Engine",
   "author": "Hyperparam",
   "homepage": "https://hyperparam.app",

package/src/execute/aggregates.js

@@ -1,6 +1,7 @@
 import { derivedAlias } from '../expression/alias.js'
 import { evaluateExpr } from '../expression/evaluate.js'
 import { executePlan, selectColumnNames } from './execute.js'
+import { sortEntriesByTerms } from './sort.js'
 import { keyify } from './utils.js'
 
 /**
@@ -50,6 +51,22 @@ function projectAggregateColumns(selectColumns, group, context) {
   return { columns, cells }
 }
 
+/**
+ * Builds the row visible to post-aggregation expressions such as HAVING and
+ * grouped ORDER BY: source group columns plus aggregate output aliases.
+ *
+ * @param {AsyncRow[]} group
+ * @param {AsyncRow} aggregateRow
+ * @returns {AsyncRow}
+ */
+function aggregateContextRow(group, aggregateRow) {
+  const baseRow = group[0] ?? { columns: [], cells: {} }
+  return {
+    columns: [...baseRow.columns, ...aggregateRow.columns],
+    cells: { ...baseRow.cells, ...aggregateRow.cells },
+  }
+}
+
 /**
  * Executes a hash aggregate operation (GROUP BY)
  *
@@ -85,27 +102,42 @@ export function executeHashAggregate(plan, context) {
         group.push(row)
       }
 
-      // Yield one row per group
+      /** @type {{ row: AsyncRow, group: AsyncRow[], contextRow: AsyncRow }[]} */
+      const aggregateRows = []
+
       for (const group of groups.values()) {
         const asyncRow = projectAggregateColumns(plan.columns, group, context)
+        const contextRow = aggregateContextRow(group, asyncRow)
 
         // Apply HAVING filter
         if (plan.having) {
-          /** @type {AsyncRow} */
-          const havingRow = {
-            columns: [...group[0].columns, ...asyncRow.columns],
-            cells: { ...group[0].cells, ...asyncRow.cells },
-          }
           const passes = await evaluateExpr({
             node: plan.having,
-            row: havingRow,
+            row: contextRow,
             rows: group,
             context,
           })
           if (!passes) continue
         }
 
-        yield asyncRow
+        aggregateRows.push({ row: asyncRow, group, contextRow })
+      }
+
+      if (plan.orderBy?.length) {
+        const sortedRows = await sortEntriesByTerms({
+          entries: aggregateRows.map((aggregateRow, idx) => ({
+            row: aggregateRow.contextRow,
+            rows: aggregateRow.group,
+            idx,
+          })),
+          orderBy: plan.orderBy,
+          context,
+        })
+        aggregateRows.splice(0, aggregateRows.length, ...sortedRows.map(({ idx }) => aggregateRows[idx]))
+      }
+
+      for (const { row } of aggregateRows) {
+        yield row
       }
     },
   }

package/src/execute/sort.js

@@ -4,12 +4,117 @@ import { executePlan } from './execute.js'
 import { compareForTerm } from './utils.js'
 
 /**
- * @import { AsyncRow, ExecuteContext, QueryResults, SqlPrimitive } from '../types.js'
+ * @import { AsyncRow, ExecuteContext, OrderByItem, QueryResults, SqlPrimitive } from '../types.js'
  * @import { SortNode } from '../plan/types.js'
  */
 
 const MAX_CHUNK = 256
 
+/**
+ * @typedef {{
+ *   row: AsyncRow,
+ *   rows?: AsyncRow[],
+ * }} SortEntry
+ */
+
+/**
+ * Sorts rows by ORDER BY terms while evaluating async sort keys in concurrent
+ * chunks and delaying later terms until earlier terms tie.
+ *
+ * @template {SortEntry} T
+ * @param {{
+ *   entries: T[],
+ *   orderBy: OrderByItem[],
+ *   context: ExecuteContext,
+ *   cacheValues?: boolean,
+ * }} options
+ * @returns {Promise<T[]>}
+ */
+export async function sortEntriesByTerms({ entries, orderBy, context, cacheValues = false }) {
+  if (entries.length === 0) return []
+
+  /** @type {(SqlPrimitive | undefined)[][]} */
+  const evaluatedValues = entries.map(() => Array(orderBy.length))
+
+  /** @type {number[][]} */
+  let groups = [entries.map((_, i) => i)]
+
+  for (let orderByIdx = 0; orderByIdx < orderBy.length; orderByIdx++) {
+    const term = orderBy[orderByIdx]
+    /** @type {number[][]} */
+    const nextGroups = []
+
+    for (const group of groups) {
+      if (group.length <= 1) {
+        nextGroups.push(group)
+        continue
+      }
+
+      const alias = derivedAlias(term.expr)
+      /** @type {number[]} */
+      const missing = []
+      for (const idx of group) {
+        if (evaluatedValues[idx][orderByIdx] === undefined) missing.push(idx)
+      }
+      let chunkSize = 1
+      let start = 0
+      while (start < missing.length) {
+        if (context.signal?.aborted) return []
+        const chunk = missing.slice(start, start + chunkSize)
+        const values = await Promise.all(chunk.map(idx =>
+          evaluateExpr({
+            node: term.expr,
+            row: entries[idx].row,
+            rows: entries[idx].rows,
+            context,
+          })
+        ))
+        for (let i = 0; i < chunk.length; i++) {
+          const idx = chunk[i]
+          const value = values[i]
+          evaluatedValues[idx][orderByIdx] = value
+          if (cacheValues && !(alias in entries[idx].row.cells)) {
+            entries[idx].row.cells[alias] = () => Promise.resolve(value)
+          }
+        }
+        start += chunk.length
+        chunkSize = Math.min(chunkSize * 2, MAX_CHUNK)
+      }
+
+      group.sort((aIdx, bIdx) => {
+        const av = evaluatedValues[aIdx][orderByIdx]
+        const bv = evaluatedValues[bIdx][orderByIdx]
+        return compareForTerm(av, bv, term)
+      })
+
+      if (orderByIdx < orderBy.length - 1) {
+        /** @type {number[]} */
+        let currentSubGroup = [group[0]]
+        for (let i = 1; i < group.length; i++) {
+          const prevIdx = group[i - 1]
+          const currIdx = group[i]
+          const prevVal = evaluatedValues[prevIdx][orderByIdx]
+          const currVal = evaluatedValues[currIdx][orderByIdx]
+
+          if (compareForTerm(prevVal, currVal, term) === 0) {
+            currentSubGroup.push(currIdx)
+          } else {
+            nextGroups.push(currentSubGroup)
+            currentSubGroup = [currIdx]
+          }
+        }
+        nextGroups.push(currentSubGroup)
+      } else {
+        nextGroups.push(group)
+      }
+    }
+
+    groups = nextGroups
+  }
+
+  return groups.flat().map(idx => entries[idx])
+}
+
 /**
  * Executes a sort operation (ORDER BY)
  *
@@ -32,92 +137,16 @@ export function executeSort(plan, context) {
         rows.push(row)
       }
 
-      if (rows.length === 0) return
-
-      // Multi-pass lazy sorting
-      /** @type {(SqlPrimitive | undefined)[][]} */
-      const evaluatedValues = rows.map(() => Array(plan.orderBy.length))
-
-      /** @type {number[][]} */
-      let groups = [rows.map((_, i) => i)]
-
-      for (let orderByIdx = 0; orderByIdx < plan.orderBy.length; orderByIdx++) {
-        const term = plan.orderBy[orderByIdx]
-        /** @type {number[][]} */
-        const nextGroups = []
-
-        for (const group of groups) {
-          if (group.length <= 1) {
-            nextGroups.push(group)
-            continue
-          }
-
-          // Evaluate this column for all rows in the group, in parallel
-          // chunks that double up to MAX_CHUNK so a slow UDF doesn't serialize.
-          // Cache each value back into the row so downstream projection can
-          // reuse it instead of re-invoking the expression.
-          const alias = derivedAlias(term.expr)
-          /** @type {number[]} */
-          const missing = []
-          for (const idx of group) {
-            if (evaluatedValues[idx][orderByIdx] === undefined) missing.push(idx)
-          }
-          let chunkSize = 1
-          let start = 0
-          while (start < missing.length) {
-            if (context.signal?.aborted) return
-            const chunk = missing.slice(start, start + chunkSize)
-            const values = await Promise.all(chunk.map(idx =>
-              evaluateExpr({ node: term.expr, row: rows[idx], context })
-            ))
-            for (let i = 0; i < chunk.length; i++) {
-              const idx = chunk[i]
-              const value = values[i]
-              evaluatedValues[idx][orderByIdx] = value
-              if (!(alias in rows[idx].cells)) {
-                rows[idx].cells[alias] = () => Promise.resolve(value)
-              }
-            }
-            start += chunk.length
-            chunkSize = Math.min(chunkSize * 2, MAX_CHUNK)
-          }
-
-          // Sort the group by this column
-          group.sort((aIdx, bIdx) => {
-            const av = evaluatedValues[aIdx][orderByIdx]
-            const bv = evaluatedValues[bIdx][orderByIdx]
-            return compareForTerm(av, bv, term)
-          })
-
-          // Split into sub-groups based on ties
-          if (orderByIdx < plan.orderBy.length - 1) {
-            /** @type {number[]} */
-            let currentSubGroup = [group[0]]
-            for (let i = 1; i < group.length; i++) {
-              const prevIdx = group[i - 1]
-              const currIdx = group[i]
-              const prevVal = evaluatedValues[prevIdx][orderByIdx]
-              const currVal = evaluatedValues[currIdx][orderByIdx]
-
-              if (compareForTerm(prevVal, currVal, term) === 0) {
-                currentSubGroup.push(currIdx)
-              } else {
-                nextGroups.push(currentSubGroup)
-                currentSubGroup = [currIdx]
-              }
-            }
-            nextGroups.push(currentSubGroup)
-          } else {
-            nextGroups.push(group)
-          }
-        }
-
-        groups = nextGroups
-      }
+      const sortedRows = await sortEntriesByTerms({
+        entries: rows.map(row => ({ row })),
+        orderBy: plan.orderBy,
+        context,
+        cacheValues: true,
+      })
 
       // Yield sorted rows
-      for (const idx of groups.flat()) {
-        yield rows[idx]
+      for (const { row } of sortedRows) {
+        yield row
       }
     },
   }

package/src/execute/utils.js

@@ -121,6 +121,19 @@ export function maxBounds(a, b) {
   return a ?? b
 }
 
+/**
+ * Returns true for plain object SqlPrimitive values, excluding null, arrays, and Dates.
+ *
+ * @param {SqlPrimitive} value
+ * @returns {value is Record<string, SqlPrimitive>}
+ */
+export function isPlainObject(value) {
+  return value != null
+    && typeof value === 'object'
+    && !Array.isArray(value)
+    && !(value instanceof Date)
+}
+
 /**
  * @param {SqlPrimitive} value
  * @returns {string}

package/src/expression/evaluate.js

@@ -1,5 +1,5 @@
 import { executeStatement } from '../execute/execute.js'
-import { keyify, stringify } from '../execute/utils.js'
+import { isPlainObject, keyify, stringify } from '../execute/utils.js'
 import { ArgValueError, ExecutionError } from '../validation/executionErrors.js'
 import { isAggregateFunc, isMathFunc, isRegexpFunc, isSpatialFunc, isStringFunc } from '../validation/functions.js'
 import { UnknownFunctionError } from '../validation/parseErrors.js'
@@ -39,6 +39,14 @@ export async function evaluateExpr({ node, row, rowIndex, rows, context }) {
       if (qualified in row.cells) {
         return row.cells[qualified]()
       }
+      const prefix = node.prefix + '.'
+      const prefixedColumns = row.columns.filter(col => col.startsWith(prefix))
+      if (prefixedColumns.length === 1) {
+        const value = await row.cells[prefixedColumns[0]]()
+        if (isPlainObject(value) && Object.prototype.hasOwnProperty.call(value, node.name)) {
+          return value[node.name]
+        }
+      }
       // Check outer row for correlated subquery references
       if (context.outerRow && context.outerAliases?.has(node.prefix) && node.name in context.outerRow.cells) {
         return context.outerRow.cells[node.name]()
@@ -473,6 +481,23 @@ export async function evaluateExpr({ node, row, rowIndex, rows, context }) {
     if (funcName === 'ARRAY_LENGTH' || funcName === 'CARDINALITY') {
       const arr = args[0]
       if (!Array.isArray(arr)) return null
+      if (funcName === 'ARRAY_LENGTH' && args.length === 2) {
+        const dim = args[1]
+        if (typeof dim !== 'number' && typeof dim !== 'bigint') return null
+        const d = Number(dim)
+        if (!Number.isInteger(d) || d < 1) return null
+        let level = arr
+        for (let i = 1; i < d; i++) {
+          if (!Array.isArray(level) || level.length === 0) return null
+          const first = level[0]
+          if (!Array.isArray(first)) return null
+          for (const item of level) {
+            if (!Array.isArray(item) || item.length !== first.length) return null
+          }
+          level = first
+        }
+        return level.length
+      }
       return arr.length
     }
 

package/src/parse/primary.js

@@ -157,6 +157,16 @@ export function parsePrimary(state) {
     if (match(state, 'dot')) {
       prefix = name
       name = expect(state, 'identifier').value
+    } else if (match(state, 'bracket', '[')) {
+      // table['column'] — string subscript is equivalent to dot access
+      const fieldTok = current(state)
+      if (fieldTok.type !== 'string') {
+        throw parseError(state, 'string literal')
+      }
+      consume(state)
+      expect(state, 'bracket', ']')
+      prefix = name
+      name = fieldTok.value
     }
 
     return {

package/src/plan/plan.js

@@ -7,7 +7,7 @@ import { validateNoIdentifiers, validateScan, validateTableRefs } from '../valid
 import { extractColumns, fromAlias, inferSelectSourceColumns, inferStatementColumns, tableFunctionColumnNames } from './columns.js'
 
 /**
- * @import { AsyncDataSource, ExprNode, DerivedColumn, IdentifierNode, JoinClause, PlanSqlOptions, ScanOptions, SelectColumn, SelectStatement, SetOperationStatement, Statement, WindowFunctionNode } from '../types.js'
+ * @import { AsyncDataSource, ExprNode, DerivedColumn, IdentifierNode, JoinClause, OrderByItem, PlanSqlOptions, ScanOptions, SelectColumn, SelectStatement, SetOperationStatement, Statement, WindowFunctionNode } from '../types.js'
  * @import { QueryPlan, WindowSpec } from './types.d.ts'
  */
 
@@ -177,6 +177,7 @@ function planSelect({ select, ctePlans, cteColumns, tables, parentColumns, outer
     }
     return col
   })
+  const orderBy = resolveOrderByAliases(select.orderBy, aliases)
 
   // Validate qualified references in other clauses
   validateTableRefs(select.where, scopeTables)
@@ -235,7 +236,16 @@ function planSelect({ select, ctePlans, cteColumns, tables, parentColumns, outer
       const groupBy = aliases.size > 0
         ? select.groupBy.map(expr => resolveAliases(expr, aliases))
         : select.groupBy
-      plan = { type: 'HashAggregate', groupBy, columns, having: select.having, child: plan }
+      /** @type {QueryPlan} */
+      const aggregatePlan = {
+        type: 'HashAggregate',
+        groupBy,
+        columns,
+        having: select.having,
+        child: plan,
+      }
+      if (orderBy.length) aggregatePlan.orderBy = orderBy
+      plan = aggregatePlan
     } else if (!select.having && !select.where && plan.type === 'Scan' && isOwnScan && isAllCountStar(select.columns)) {
       plan = { type: 'Count', table: plan.table, columns: select.columns }
     } else {
@@ -243,8 +253,8 @@ function planSelect({ select, ctePlans, cteColumns, tables, parentColumns, outer
     }
 
     // ORDER BY (after aggregation)
-    if (select.orderBy.length) {
-      plan = { type: 'Sort', orderBy: select.orderBy, child: plan }
+    if (orderBy.length && !select.groupBy.length) {
+      plan = { type: 'Sort', orderBy, child: plan }
     }
 
     // DISTINCT
@@ -267,10 +277,7 @@ function planSelect({ select, ctePlans, cteColumns, tables, parentColumns, outer
 
     // ORDER BY (before projection so it can access all columns)
     // Resolve SELECT aliases in ORDER BY expressions at plan time
-    if (select.orderBy.length) {
-      const orderBy = aliases.size > 0
-        ? select.orderBy.map(term => ({ ...term, expr: resolveAliases(term.expr, aliases) }))
-        : select.orderBy
+    if (orderBy.length) {
       plan = { type: 'Sort', orderBy, child: plan }
     }
 
@@ -469,6 +476,19 @@ function planJoin({ left, joins, leftTable, ctePlans, cteColumns, perTableColumn
   return plan
 }
 
+/**
+ * Recursively replaces identifier nodes in ORDER BY terms that match SELECT
+ * aliases with their aliased expressions.
+ *
+ * @param {OrderByItem[]} orderBy
+ * @param {Map<string, ExprNode>} aliases
+ * @returns {OrderByItem[]}
+ */
+function resolveOrderByAliases(orderBy, aliases) {
+  if (!aliases.size) return orderBy
+  return orderBy.map(term => ({ ...term, expr: resolveAliases(term.expr, aliases) }))
+}
+
 /**
  * Recursively replaces identifier nodes that match SELECT aliases
  * with their aliased expressions.
@@ -492,7 +512,8 @@ function resolveAliases(node, aliases) {
   }
   if (node.type === 'function') {
     const args = node.args.map(arg => resolveAliases(arg, aliases))
-    return { ...node, args }
+    if (!node.filter) return { ...node, args }
+    return { ...node, args, filter: resolveAliases(node.filter, aliases) }
   }
   if (node.type === 'cast') {
     return { ...node, expr: resolveAliases(node.expr, aliases) }

package/src/plan/types.d.ts

@@ -67,6 +67,7 @@ export interface HashAggregateNode {
   type: 'HashAggregate'
   groupBy: ExprNode[]
   columns: SelectColumn[]
+  orderBy?: OrderByItem[]
   having?: ExprNode
   child: QueryPlan
 }

package/src/validation/functions.js

@@ -180,7 +180,7 @@ export const FUNCTION_SIGNATURES = {
   ARRAY_AGG: { min: 1, max: 1, signature: 'expression' },
 
   // Array functions
-  ARRAY_LENGTH: { min: 1, max: 1, signature: 'array' },
+  ARRAY_LENGTH: { min: 1, max: 2, signature: 'array[, dimension]' },
   ARRAY_POSITION: { min: 2, max: 2, signature: 'array, element' },
   ARRAY_SORT: { min: 1, max: 1, signature: 'array' },
   CARDINALITY: { min: 1, max: 1, signature: 'array' },