squirreling 0.12.26 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +2 -2
- package/src/execute/accumulator.js +89 -0
- package/src/execute/aggregates.js +27 -62
- package/src/execute/execute.js +35 -28
- package/src/execute/join.js +13 -13
- package/src/execute/sort.js +8 -3
- package/src/execute/streamingAggregate.js +598 -0
- package/src/execute/window.js +8 -8
- package/src/expression/evaluate.js +1 -1
- package/src/index.d.ts +1 -1
- package/src/plan/columns.js +1 -1
- package/src/plan/plan.js +39 -1
|
@@ -0,0 +1,598 @@
|
|
|
1
|
+
import { derivedAlias } from '../expression/alias.js'
|
|
2
|
+
import { evaluateAll, evaluateExpr } from '../expression/evaluate.js'
|
|
3
|
+
import { collectColumnsFromExpr } from '../plan/columns.js'
|
|
4
|
+
import { isAggregateFunc } from '../validation/functions.js'
|
|
5
|
+
import { finalizeAccumulator, newAccumulator, updateAccumulator } from './accumulator.js'
|
|
6
|
+
import { sortEntriesByTerms } from './sort.js'
|
|
7
|
+
import { keyify } from './utils.js'
|
|
8
|
+
import { yieldToEventLoop } from './yield.js'
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* @import { AsyncCells, AsyncRow, ExecuteContext, ExprNode, FunctionNode, IdentifierNode, QueryResults, SelectColumn, SqlPrimitive } from '../types.js'
|
|
12
|
+
* @import { HashAggregateNode, ScalarAggregateNode } from '../plan/types.js'
|
|
13
|
+
* @import { Accumulator } from './accumulator.js'
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
// Accumulate rows in chunks of this size so aborts can fire and async cells overlap
|
|
17
|
+
const CHUNK_SIZE = 4000
|
|
18
|
+
|
|
19
|
+
// Aggregate functions whose state can be accumulated one row at a time with
|
|
20
|
+
// bounded memory. Aggregates outside this set (MEDIAN, ARRAY_AGG, STDDEV, ...)
|
|
21
|
+
// need the full value set, so their queries buffer rows instead.
|
|
22
|
+
const STREAMABLE_FUNCS = new Set(['COUNT', 'COUNTIF', 'SUM', 'AVG', 'MIN', 'MAX'])
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* Specs are keyed by aggregate node identity, so two aggregates that differ
|
|
26
|
+
* only in FILTER accumulate separately even though their derived aliases match.
|
|
27
|
+
*
|
|
28
|
+
* @typedef {{
|
|
29
|
+
* node: FunctionNode,
|
|
30
|
+
* funcName: string,
|
|
31
|
+
* star: boolean,
|
|
32
|
+
* }} StreamingAggSpec
|
|
33
|
+
*/
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* @typedef {{
|
|
37
|
+
* firstRow: AsyncRow | undefined,
|
|
38
|
+
* keyValues: SqlPrimitive[],
|
|
39
|
+
* accumulators: Accumulator[],
|
|
40
|
+
* }} StreamingGroup
|
|
41
|
+
*/
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* The streaming plan for an aggregate node: which aggregate calls to
|
|
45
|
+
* accumulate, which expression nodes are group key references (substituted
|
|
46
|
+
* from the group's key values), and whether any expression still needs a
|
|
47
|
+
* representative row from the group. When needsRow is false, no input rows
|
|
48
|
+
* are retained, so memory is bounded by the number of groups — plus, for
|
|
49
|
+
* COUNT(DISTINCT ...), each group's set of distinct values — even for
|
|
50
|
+
* high-cardinality GROUP BY.
|
|
51
|
+
*
|
|
52
|
+
* @typedef {{
|
|
53
|
+
* specs: StreamingAggSpec[],
|
|
54
|
+
* keyRefs: Map<ExprNode, number>,
|
|
55
|
+
* needsRow: boolean,
|
|
56
|
+
* }} StreamingAggPlan
|
|
57
|
+
*/
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* Structural signature of an expression node, ignoring source positions, so
|
|
61
|
+
* an expression repeated in SELECT and GROUP BY compares equal.
|
|
62
|
+
*
|
|
63
|
+
* @param {ExprNode} node
|
|
64
|
+
* @returns {string}
|
|
65
|
+
*/
|
|
66
|
+
function exprSig(node) {
|
|
67
|
+
return JSON.stringify(node, (key, value) => {
|
|
68
|
+
if (key === 'positionStart' || key === 'positionEnd') return undefined
|
|
69
|
+
// JSON.stringify throws on BigInt literal values; wrap so 1n !== '1n'
|
|
70
|
+
if (typeof value === 'bigint') return { bigint: value.toString() }
|
|
71
|
+
return value
|
|
72
|
+
})
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* Extracts the aggregate calls an aggregate node needs so they can be
|
|
77
|
+
* computed incrementally, without buffering the group's rows. Returns
|
|
78
|
+
* undefined when any expression needs a buffered group: an aggregate outside
|
|
79
|
+
* STREAMABLE_FUNCS, an aggregate over a non-scalar argument, or a subquery.
|
|
80
|
+
* Also returns undefined when an aggregate references a column the child
|
|
81
|
+
* does not produce: projection pushdown prunes columns whose output cells
|
|
82
|
+
* are never read, and only the buffered path defers evaluation of those cells.
|
|
83
|
+
*
|
|
84
|
+
* @param {Pick<HashAggregateNode, 'columns' | 'having'> & Partial<Pick<HashAggregateNode, 'orderBy' | 'groupBy'>>} plan
|
|
85
|
+
* @param {string[]} [childColumns] - columns produced by the child plan
|
|
86
|
+
* @returns {StreamingAggPlan | undefined}
|
|
87
|
+
*/
|
|
88
|
+
export function planStreamingAggregates({ columns, having, orderBy, groupBy }, childColumns) {
|
|
89
|
+
const groupExprs = groupBy ?? []
|
|
90
|
+
const groupSigs = groupExprs.map(exprSig)
|
|
91
|
+
/** @type {StreamingAggSpec[]} */
|
|
92
|
+
const specs = []
|
|
93
|
+
/** @type {Map<ExprNode, number>} */
|
|
94
|
+
const keyRefs = new Map()
|
|
95
|
+
let needsRow = false
|
|
96
|
+
|
|
97
|
+
/**
|
|
98
|
+
* Index of the group key the expression structurally matches, or -1.
|
|
99
|
+
* Only exact matches qualify: a bare identifier and a qualified group key
|
|
100
|
+
* (or vice versa) can resolve to different columns in a join, so mixed
|
|
101
|
+
* qualification falls back to evaluating against the representative row.
|
|
102
|
+
*
|
|
103
|
+
* @param {ExprNode} node
|
|
104
|
+
* @returns {number}
|
|
105
|
+
*/
|
|
106
|
+
function matchGroupKey(node) {
|
|
107
|
+
const signature = exprSig(node)
|
|
108
|
+
for (let i = 0; i < groupExprs.length; i++) {
|
|
109
|
+
if (groupSigs[i] === signature) return i
|
|
110
|
+
}
|
|
111
|
+
return -1
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/**
|
|
115
|
+
* Walks an expression collecting streamable aggregate calls and group key
|
|
116
|
+
* references. Returns false if the expression cannot be evaluated from
|
|
117
|
+
* precomputed values plus a representative row. `lazy` marks positions the
|
|
118
|
+
* evaluator can skip (short-circuited AND/OR right sides, CASE branches,
|
|
119
|
+
* and select or sort expressions of a group HAVING may reject): an
|
|
120
|
+
* aggregate there may never be evaluated by the buffered path, so
|
|
121
|
+
* accumulating it eagerly could evaluate expressions the query never asks
|
|
122
|
+
* for, and the query falls back to buffered aggregation. A FILTER-less
|
|
123
|
+
* star aggregate is exempt: accumulating it evaluates nothing against
|
|
124
|
+
* input rows, so eager accumulation is unobservable.
|
|
125
|
+
*
|
|
126
|
+
* @param {ExprNode} node
|
|
127
|
+
* @param {boolean} lazy
|
|
128
|
+
* @returns {boolean}
|
|
129
|
+
*/
|
|
130
|
+
function walk(node, lazy) {
|
|
131
|
+
const keyIndex = matchGroupKey(node)
|
|
132
|
+
if (keyIndex >= 0) {
|
|
133
|
+
keyRefs.set(node, keyIndex)
|
|
134
|
+
return true
|
|
135
|
+
}
|
|
136
|
+
switch (node.type) {
|
|
137
|
+
case 'literal':
|
|
138
|
+
case 'interval':
|
|
139
|
+
return true
|
|
140
|
+
case 'identifier':
|
|
141
|
+
case 'star':
|
|
142
|
+
// resolves against the group's representative row
|
|
143
|
+
needsRow = true
|
|
144
|
+
return true
|
|
145
|
+
case 'unary':
|
|
146
|
+
return walk(node.argument, lazy)
|
|
147
|
+
case 'binary':
|
|
148
|
+
if (node.op === 'AND' || node.op === 'OR') {
|
|
149
|
+
// the right side is skipped when the left side short-circuits
|
|
150
|
+
return walk(node.left, lazy) && walk(node.right, true)
|
|
151
|
+
}
|
|
152
|
+
return walk(node.left, lazy) && walk(node.right, lazy)
|
|
153
|
+
case 'cast':
|
|
154
|
+
return walk(node.expr, lazy)
|
|
155
|
+
case 'case':
|
|
156
|
+
// only the first WHEN condition is always evaluated; later
|
|
157
|
+
// conditions, results, and ELSE run only when reached
|
|
158
|
+
return (!node.caseExpr || walk(node.caseExpr, lazy)) &&
|
|
159
|
+
node.whenClauses.every((w, i) => walk(w.condition, lazy || i > 0) && walk(w.result, true)) &&
|
|
160
|
+
(!node.elseResult || walk(node.elseResult, true))
|
|
161
|
+
case 'in valuelist':
|
|
162
|
+
// values after the first are skipped once an earlier value matches
|
|
163
|
+
return walk(node.expr, lazy) && node.values.every((v, i) => walk(v, lazy || i > 0))
|
|
164
|
+
case 'function': {
|
|
165
|
+
const funcName = node.funcName.toUpperCase()
|
|
166
|
+
if (!isAggregateFunc(funcName)) {
|
|
167
|
+
return node.args.every(arg => walk(arg, lazy))
|
|
168
|
+
}
|
|
169
|
+
if (!STREAMABLE_FUNCS.has(funcName)) return false
|
|
170
|
+
const star = node.args[0]?.type === 'star'
|
|
171
|
+
if (lazy && !(star && !node.filter)) return false
|
|
172
|
+
if (!star && !node.args.every(arg => isScalarExpr(arg))) return false
|
|
173
|
+
if (node.filter && !isScalarExpr(node.filter)) return false
|
|
174
|
+
if (!specs.some(spec => spec.node === node)) {
|
|
175
|
+
specs.push({ node, funcName, star })
|
|
176
|
+
}
|
|
177
|
+
return true
|
|
178
|
+
}
|
|
179
|
+
default:
|
|
180
|
+
// subqueries, EXISTS, IN (subquery), window functions
|
|
181
|
+
return false
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
// HAVING can reject a group before its output cells are ever read, so
|
|
186
|
+
// with HAVING present the buffered path may never evaluate SELECT or
|
|
187
|
+
// ORDER BY aggregates; treat those positions as lazy
|
|
188
|
+
const rejectable = Boolean(having)
|
|
189
|
+
for (const col of columns) {
|
|
190
|
+
if (col.type === 'star') {
|
|
191
|
+
needsRow = true
|
|
192
|
+
continue
|
|
193
|
+
}
|
|
194
|
+
if (!walk(col.expr, rejectable)) return
|
|
195
|
+
}
|
|
196
|
+
if (having && !walk(having, false)) return
|
|
197
|
+
const orderTerms = orderBy ?? []
|
|
198
|
+
for (let i = 0; i < orderTerms.length; i++) {
|
|
199
|
+
// the sorter evaluates later terms only to break ties on earlier terms
|
|
200
|
+
if (!walk(orderTerms[i].expr, rejectable || i > 0)) return
|
|
201
|
+
}
|
|
202
|
+
if (childColumns && !specsResolvable(specs, childColumns)) return
|
|
203
|
+
return { specs, keyRefs, needsRow }
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
/**
|
|
207
|
+
* Reports whether every identifier the streaming path evaluates eagerly
|
|
208
|
+
* (aggregate arguments and FILTER conditions) can resolve against the
|
|
209
|
+
* child's output columns, using the same exact-then-suffix matching as
|
|
210
|
+
* identifier evaluation. Anything unresolvable means projection pushdown
|
|
211
|
+
* pruned the column, so the query must buffer instead.
|
|
212
|
+
*
|
|
213
|
+
* @param {StreamingAggSpec[]} specs
|
|
214
|
+
* @param {string[]} childColumns
|
|
215
|
+
* @returns {boolean}
|
|
216
|
+
*/
|
|
217
|
+
function specsResolvable(specs, childColumns) {
|
|
218
|
+
/** @type {IdentifierNode[]} */
|
|
219
|
+
const identifiers = []
|
|
220
|
+
for (const spec of specs) {
|
|
221
|
+
collectColumnsFromExpr(spec.node, identifiers)
|
|
222
|
+
}
|
|
223
|
+
return identifiers.every(({ prefix, name }) => {
|
|
224
|
+
if (childColumns.includes(prefix ? `${prefix}.${name}` : name)) return true
|
|
225
|
+
if (!prefix) return childColumns.some(col => col.endsWith('.' + name))
|
|
226
|
+
// a qualified name may also resolve as struct access on a base column,
|
|
227
|
+
// or fall back to the bare column part
|
|
228
|
+
return childColumns.includes(prefix) ||
|
|
229
|
+
childColumns.some(col => col.endsWith('.' + prefix)) ||
|
|
230
|
+
childColumns.includes(name)
|
|
231
|
+
})
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
/**
|
|
235
|
+
* Reports whether an expression is a plain scalar over row values: no
|
|
236
|
+
* aggregates and no subqueries, so it can be evaluated per input row.
|
|
237
|
+
*
|
|
238
|
+
* @param {ExprNode} node
|
|
239
|
+
* @returns {boolean}
|
|
240
|
+
*/
|
|
241
|
+
function isScalarExpr(node) {
|
|
242
|
+
switch (node.type) {
|
|
243
|
+
case 'literal':
|
|
244
|
+
case 'identifier':
|
|
245
|
+
case 'star':
|
|
246
|
+
case 'interval':
|
|
247
|
+
return true
|
|
248
|
+
case 'unary':
|
|
249
|
+
return isScalarExpr(node.argument)
|
|
250
|
+
case 'binary':
|
|
251
|
+
return isScalarExpr(node.left) && isScalarExpr(node.right)
|
|
252
|
+
case 'cast':
|
|
253
|
+
return isScalarExpr(node.expr)
|
|
254
|
+
case 'case':
|
|
255
|
+
return (!node.caseExpr || isScalarExpr(node.caseExpr)) &&
|
|
256
|
+
node.whenClauses.every(w => isScalarExpr(w.condition) && isScalarExpr(w.result)) &&
|
|
257
|
+
(!node.elseResult || isScalarExpr(node.elseResult))
|
|
258
|
+
case 'in valuelist':
|
|
259
|
+
return isScalarExpr(node.expr) && node.values.every(v => isScalarExpr(v))
|
|
260
|
+
case 'function':
|
|
261
|
+
return !isAggregateFunc(node.funcName.toUpperCase()) && node.args.every(arg => isScalarExpr(arg))
|
|
262
|
+
default:
|
|
263
|
+
return false
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
/**
|
|
268
|
+
* Replaces each precomputed node in an expression with its value as a
|
|
269
|
+
* literal: aggregate calls (keyed by node identity) and group key references,
|
|
270
|
+
* so the rest of the expression can be evaluated against a representative
|
|
271
|
+
* row. Nodes without precomputed values are returned unchanged.
|
|
272
|
+
*
|
|
273
|
+
* @param {ExprNode} node
|
|
274
|
+
* @param {Map<ExprNode, SqlPrimitive>} values - computed value per substituted node
|
|
275
|
+
* @returns {ExprNode}
|
|
276
|
+
*/
|
|
277
|
+
function substituteValues(node, values) {
|
|
278
|
+
if (values.has(node)) {
|
|
279
|
+
// group keys over missing columns evaluate to undefined; preserve it
|
|
280
|
+
// so streaming output matches the buffered path's evaluation result
|
|
281
|
+
// eslint-disable-next-line no-extra-parens
|
|
282
|
+
const value = /** @type {SqlPrimitive} */ (values.get(node))
|
|
283
|
+
return {
|
|
284
|
+
type: 'literal',
|
|
285
|
+
value,
|
|
286
|
+
positionStart: node.positionStart,
|
|
287
|
+
positionEnd: node.positionEnd,
|
|
288
|
+
}
|
|
289
|
+
}
|
|
290
|
+
switch (node.type) {
|
|
291
|
+
case 'unary': {
|
|
292
|
+
const argument = substituteValues(node.argument, values)
|
|
293
|
+
return argument === node.argument ? node : { ...node, argument }
|
|
294
|
+
}
|
|
295
|
+
case 'binary': {
|
|
296
|
+
const left = substituteValues(node.left, values)
|
|
297
|
+
const right = substituteValues(node.right, values)
|
|
298
|
+
return left === node.left && right === node.right ? node : { ...node, left, right }
|
|
299
|
+
}
|
|
300
|
+
case 'cast': {
|
|
301
|
+
const expr = substituteValues(node.expr, values)
|
|
302
|
+
return expr === node.expr ? node : { ...node, expr }
|
|
303
|
+
}
|
|
304
|
+
case 'case': {
|
|
305
|
+
const caseExpr = node.caseExpr && substituteValues(node.caseExpr, values)
|
|
306
|
+
const whenClauses = node.whenClauses.map(w => {
|
|
307
|
+
const condition = substituteValues(w.condition, values)
|
|
308
|
+
const result = substituteValues(w.result, values)
|
|
309
|
+
return condition === w.condition && result === w.result ? w : { ...w, condition, result }
|
|
310
|
+
})
|
|
311
|
+
const elseResult = node.elseResult && substituteValues(node.elseResult, values)
|
|
312
|
+
return { ...node, caseExpr, whenClauses, elseResult }
|
|
313
|
+
}
|
|
314
|
+
case 'in valuelist': {
|
|
315
|
+
const expr = substituteValues(node.expr, values)
|
|
316
|
+
const valueNodes = node.values.map(v => substituteValues(v, values))
|
|
317
|
+
return { ...node, expr, values: valueNodes }
|
|
318
|
+
}
|
|
319
|
+
case 'function': {
|
|
320
|
+
const args = node.args.map(arg => substituteValues(arg, values))
|
|
321
|
+
return args.every((arg, i) => arg === node.args[i]) ? node : { ...node, args }
|
|
322
|
+
}
|
|
323
|
+
default:
|
|
324
|
+
return node
|
|
325
|
+
}
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
/**
|
|
329
|
+
* Folds one chunk of rows into the group accumulators. Group keys, FILTER
|
|
330
|
+
* conditions, and aggregate arguments are each evaluated across the whole
|
|
331
|
+
* chunk so async cells overlap; the chunk is released afterwards.
|
|
332
|
+
*
|
|
333
|
+
* @param {object} options
|
|
334
|
+
* @param {AsyncRow[]} options.chunk
|
|
335
|
+
* @param {ExprNode[]} options.groupBy
|
|
336
|
+
* @param {StreamingAggSpec[]} options.specs
|
|
337
|
+
* @param {Map<unknown, StreamingGroup>} options.groups
|
|
338
|
+
* @param {boolean} options.needsRow - retain each group's first row?
|
|
339
|
+
* @param {ExecuteContext} options.context
|
|
340
|
+
* @returns {Promise<void>}
|
|
341
|
+
*/
|
|
342
|
+
async function accumulateChunk({ chunk, groupBy, specs, groups, needsRow, context }) {
|
|
343
|
+
/** @type {SqlPrimitive[][] | undefined} */
|
|
344
|
+
let keyColumns
|
|
345
|
+
if (groupBy.length) {
|
|
346
|
+
keyColumns = await Promise.all(groupBy.map(expr => evaluateAll(expr, chunk, context)))
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
/** @type {(SqlPrimitive[] | undefined)[]} */
|
|
350
|
+
const filters = new Array(specs.length)
|
|
351
|
+
/** @type {(SqlPrimitive[] | undefined)[]} */
|
|
352
|
+
const args = new Array(specs.length)
|
|
353
|
+
for (let s = 0; s < specs.length; s++) {
|
|
354
|
+
const { node, star } = specs[s]
|
|
355
|
+
if (node.filter) {
|
|
356
|
+
const passes = await evaluateAll(node.filter, chunk, context)
|
|
357
|
+
filters[s] = passes
|
|
358
|
+
if (!star) {
|
|
359
|
+
// The buffered path filters the group before evaluating arguments,
|
|
360
|
+
// so only evaluate the argument for rows that pass the FILTER
|
|
361
|
+
/** @type {AsyncRow[]} */
|
|
362
|
+
const passingRows = []
|
|
363
|
+
/** @type {number[]} */
|
|
364
|
+
const passingIndices = []
|
|
365
|
+
for (let j = 0; j < chunk.length; j++) {
|
|
366
|
+
if (passes[j]) {
|
|
367
|
+
passingRows.push(chunk[j])
|
|
368
|
+
passingIndices.push(j)
|
|
369
|
+
}
|
|
370
|
+
}
|
|
371
|
+
const values = await evaluateAll(node.args[0], passingRows, context)
|
|
372
|
+
const spread = new Array(chunk.length).fill(null)
|
|
373
|
+
for (let k = 0; k < passingIndices.length; k++) {
|
|
374
|
+
spread[passingIndices[k]] = values[k]
|
|
375
|
+
}
|
|
376
|
+
args[s] = spread
|
|
377
|
+
}
|
|
378
|
+
} else {
|
|
379
|
+
args[s] = star ? undefined : await evaluateAll(node.args[0], chunk, context)
|
|
380
|
+
}
|
|
381
|
+
}
|
|
382
|
+
|
|
383
|
+
for (let j = 0; j < chunk.length; j++) {
|
|
384
|
+
const key = keyColumns
|
|
385
|
+
? keyColumns.length === 1 ? keyify(keyColumns[0][j]) : keyify(...keyColumns.map(c => c[j]))
|
|
386
|
+
: true
|
|
387
|
+
let group = groups.get(key)
|
|
388
|
+
if (!group) {
|
|
389
|
+
group = {
|
|
390
|
+
firstRow: needsRow ? chunk[j] : undefined,
|
|
391
|
+
keyValues: keyColumns ? keyColumns.map(c => c[j]) : [],
|
|
392
|
+
accumulators: specs.map(spec => newAccumulator(spec.funcName, spec.node.distinct)),
|
|
393
|
+
}
|
|
394
|
+
groups.set(key, group)
|
|
395
|
+
}
|
|
396
|
+
for (let s = 0; s < specs.length; s++) {
|
|
397
|
+
const filter = filters[s]
|
|
398
|
+
if (filter && !filter[j]) continue
|
|
399
|
+
const spec = specs[s]
|
|
400
|
+
if (spec.star && spec.funcName === 'COUNT') {
|
|
401
|
+
group.accumulators[s].count++
|
|
402
|
+
} else {
|
|
403
|
+
const arg = args[s]
|
|
404
|
+
updateAccumulator(spec.funcName, group.accumulators[s], arg ? arg[j] : null)
|
|
405
|
+
}
|
|
406
|
+
}
|
|
407
|
+
}
|
|
408
|
+
}
|
|
409
|
+
|
|
410
|
+
/**
|
|
411
|
+
* Consumes the child rows into per-group accumulators, holding at most one
|
|
412
|
+
* chunk of rows at a time. Throws when aborted so partial accumulators are
|
|
413
|
+
* never finalized into results.
|
|
414
|
+
*
|
|
415
|
+
* @param {object} options
|
|
416
|
+
* @param {QueryResults} options.child
|
|
417
|
+
* @param {ExprNode[]} options.groupBy
|
|
418
|
+
* @param {StreamingAggSpec[]} options.specs
|
|
419
|
+
* @param {boolean} options.needsRow
|
|
420
|
+
* @param {ExecuteContext} options.context
|
|
421
|
+
* @returns {Promise<Map<unknown, StreamingGroup>>}
|
|
422
|
+
*/
|
|
423
|
+
async function accumulateGroups({ child, groupBy, specs, needsRow, context }) {
|
|
424
|
+
/** @type {Map<unknown, StreamingGroup>} */
|
|
425
|
+
const groups = new Map()
|
|
426
|
+
/** @type {AsyncRow[]} */
|
|
427
|
+
let chunk = []
|
|
428
|
+
for await (const row of child.rows()) {
|
|
429
|
+
chunk.push(row)
|
|
430
|
+
if (chunk.length >= CHUNK_SIZE) {
|
|
431
|
+
await accumulateChunk({ chunk, groupBy, specs, groups, needsRow, context })
|
|
432
|
+
chunk = []
|
|
433
|
+
await yieldToEventLoop()
|
|
434
|
+
context.signal?.throwIfAborted()
|
|
435
|
+
}
|
|
436
|
+
}
|
|
437
|
+
if (chunk.length) {
|
|
438
|
+
await accumulateChunk({ chunk, groupBy, specs, groups, needsRow, context })
|
|
439
|
+
}
|
|
440
|
+
context.signal?.throwIfAborted()
|
|
441
|
+
return groups
|
|
442
|
+
}
|
|
443
|
+
|
|
444
|
+
/**
|
|
445
|
+
* Builds a group's output row by substituting the group's finalized
|
|
446
|
+
* aggregate and group key values into the select expressions and evaluating
|
|
447
|
+
* them against the group's representative row (an empty row when no
|
|
448
|
+
* expression needs one).
|
|
449
|
+
*
|
|
450
|
+
* @param {object} options
|
|
451
|
+
* @param {SelectColumn[]} options.selectColumns
|
|
452
|
+
* @param {StreamingAggSpec[]} options.specs
|
|
453
|
+
* @param {Map<ExprNode, number>} options.keyRefs
|
|
454
|
+
* @param {StreamingGroup} options.group
|
|
455
|
+
* @param {ExecuteContext} options.context
|
|
456
|
+
* @returns {{ outputRow: AsyncRow, values: Map<ExprNode, SqlPrimitive> }}
|
|
457
|
+
*/
|
|
458
|
+
function finalizeGroup({ selectColumns, specs, keyRefs, group, context }) {
|
|
459
|
+
const firstRow = group.firstRow ?? { columns: [], cells: {} }
|
|
460
|
+
|
|
461
|
+
/** @type {Map<ExprNode, SqlPrimitive>} */
|
|
462
|
+
const values = new Map()
|
|
463
|
+
for (let s = 0; s < specs.length; s++) {
|
|
464
|
+
values.set(specs[s].node, finalizeAccumulator(specs[s].funcName, group.accumulators[s]))
|
|
465
|
+
}
|
|
466
|
+
for (const [node, keyIndex] of keyRefs) {
|
|
467
|
+
values.set(node, group.keyValues[keyIndex])
|
|
468
|
+
}
|
|
469
|
+
|
|
470
|
+
/** @type {string[]} */
|
|
471
|
+
const columns = []
|
|
472
|
+
/** @type {AsyncCells} */
|
|
473
|
+
const cells = {}
|
|
474
|
+
for (const col of selectColumns) {
|
|
475
|
+
if (col.type === 'star') {
|
|
476
|
+
if (group.firstRow) {
|
|
477
|
+
const prefix = col.table ? `${col.table}.` : undefined
|
|
478
|
+
for (const key of firstRow.columns) {
|
|
479
|
+
if (prefix && !key.startsWith(prefix)) continue
|
|
480
|
+
const dotIndex = key.indexOf('.')
|
|
481
|
+
const outputKey = prefix ? key.substring(prefix.length) : dotIndex >= 0 ? key.substring(dotIndex + 1) : key
|
|
482
|
+
columns.push(outputKey)
|
|
483
|
+
cells[outputKey] = firstRow.cells[key]
|
|
484
|
+
}
|
|
485
|
+
}
|
|
486
|
+
} else {
|
|
487
|
+
const alias = col.alias ?? derivedAlias(col.expr)
|
|
488
|
+
const expr = substituteValues(col.expr, values)
|
|
489
|
+
columns.push(alias)
|
|
490
|
+
cells[alias] = () => evaluateExpr({ node: expr, row: firstRow, context })
|
|
491
|
+
}
|
|
492
|
+
}
|
|
493
|
+
/** @type {AsyncRow} */
|
|
494
|
+
const outputRow = { columns, cells }
|
|
495
|
+
return { outputRow, values }
|
|
496
|
+
}
|
|
497
|
+
|
|
498
|
+
/**
|
|
499
|
+
* Builds the row visible to HAVING and grouped ORDER BY: the group's
|
|
500
|
+
* representative columns plus the select output aliases, mirroring the
|
|
501
|
+
* buffered aggregate context row.
|
|
502
|
+
*
|
|
503
|
+
* @param {StreamingGroup} group
|
|
504
|
+
* @param {AsyncRow} outputRow
|
|
505
|
+
* @returns {AsyncRow}
|
|
506
|
+
*/
|
|
507
|
+
function groupContextRow(group, outputRow) {
|
|
508
|
+
const firstRow = group.firstRow ?? { columns: [], cells: {} }
|
|
509
|
+
return {
|
|
510
|
+
columns: [...firstRow.columns, ...outputRow.columns],
|
|
511
|
+
cells: { ...firstRow.cells, ...outputRow.cells },
|
|
512
|
+
}
|
|
513
|
+
}
|
|
514
|
+
|
|
515
|
+
/**
|
|
516
|
+
* Streaming GROUP BY execution: accumulates aggregates incrementally instead
|
|
517
|
+
* of buffering every input row, then applies HAVING and grouped ORDER BY
|
|
518
|
+
* against the finalized aggregate values.
|
|
519
|
+
*
|
|
520
|
+
* @param {object} options
|
|
521
|
+
* @param {HashAggregateNode} options.plan
|
|
522
|
+
* @param {StreamingAggPlan} options.streaming
|
|
523
|
+
* @param {QueryResults} options.child
|
|
524
|
+
* @param {ExecuteContext} options.context
|
|
525
|
+
* @returns {() => AsyncGenerator<AsyncRow>}
|
|
526
|
+
*/
|
|
527
|
+
export function streamingHashAggregateRows({ plan, streaming, child, context }) {
|
|
528
|
+
const { specs, keyRefs, needsRow } = streaming
|
|
529
|
+
return async function* () {
|
|
530
|
+
const groups = await accumulateGroups({ child, groupBy: plan.groupBy, specs, needsRow, context })
|
|
531
|
+
const { orderBy, having } = plan
|
|
532
|
+
|
|
533
|
+
// Without ORDER BY, groups finalize and yield one at a time so output
|
|
534
|
+
// rows are never all held at once; sorting needs the full set below.
|
|
535
|
+
/** @type {{ row: AsyncRow, exprs: ExprNode[], outputRow: AsyncRow }[] | undefined} */
|
|
536
|
+
const entries = orderBy?.length ? [] : undefined
|
|
537
|
+
for (const group of groups.values()) {
|
|
538
|
+
const { outputRow, values } = finalizeGroup({ selectColumns: plan.columns, specs, keyRefs, group, context })
|
|
539
|
+
if (having) {
|
|
540
|
+
const passes = await evaluateExpr({
|
|
541
|
+
node: substituteValues(having, values),
|
|
542
|
+
row: groupContextRow(group, outputRow),
|
|
543
|
+
context,
|
|
544
|
+
})
|
|
545
|
+
if (!passes) continue
|
|
546
|
+
}
|
|
547
|
+
if (entries && orderBy) {
|
|
548
|
+
entries.push({
|
|
549
|
+
row: groupContextRow(group, outputRow),
|
|
550
|
+
exprs: orderBy.map(term => substituteValues(term.expr, values)),
|
|
551
|
+
outputRow,
|
|
552
|
+
})
|
|
553
|
+
} else {
|
|
554
|
+
yield outputRow
|
|
555
|
+
}
|
|
556
|
+
}
|
|
557
|
+
|
|
558
|
+
if (entries && orderBy) {
|
|
559
|
+
// The shared sorter evaluates later ORDER BY terms only within ties
|
|
560
|
+
// on earlier terms, so expensive sort keys are skipped when possible
|
|
561
|
+
const sorted = await sortEntriesByTerms({ entries, orderBy, context })
|
|
562
|
+
for (const { outputRow } of sorted) {
|
|
563
|
+
yield outputRow
|
|
564
|
+
}
|
|
565
|
+
}
|
|
566
|
+
}
|
|
567
|
+
}
|
|
568
|
+
|
|
569
|
+
/**
|
|
570
|
+
* Streaming scalar aggregate execution: the whole input is one group,
|
|
571
|
+
* accumulated incrementally with bounded memory.
|
|
572
|
+
*
|
|
573
|
+
* @param {object} options
|
|
574
|
+
* @param {ScalarAggregateNode} options.plan
|
|
575
|
+
* @param {StreamingAggPlan} options.streaming
|
|
576
|
+
* @param {QueryResults} options.child
|
|
577
|
+
* @param {ExecuteContext} options.context
|
|
578
|
+
* @returns {() => AsyncGenerator<AsyncRow>}
|
|
579
|
+
*/
|
|
580
|
+
export function streamingScalarAggregateRows({ plan, streaming, child, context }) {
|
|
581
|
+
const { specs, keyRefs, needsRow } = streaming
|
|
582
|
+
return async function* () {
|
|
583
|
+
const groups = await accumulateGroups({ child, groupBy: [], specs, needsRow, context })
|
|
584
|
+
/** @type {StreamingGroup} */
|
|
585
|
+
const group = groups.get(true) ?? { firstRow: undefined, keyValues: [], accumulators: specs.map(spec => newAccumulator(spec.funcName, spec.node.distinct)) }
|
|
586
|
+
|
|
587
|
+
const { outputRow, values } = finalizeGroup({ selectColumns: plan.columns, specs, keyRefs, group, context })
|
|
588
|
+
if (plan.having) {
|
|
589
|
+
const passes = await evaluateExpr({
|
|
590
|
+
node: substituteValues(plan.having, values),
|
|
591
|
+
row: groupContextRow(group, outputRow),
|
|
592
|
+
context,
|
|
593
|
+
})
|
|
594
|
+
if (!passes) return
|
|
595
|
+
}
|
|
596
|
+
yield outputRow
|
|
597
|
+
}
|
|
598
|
+
}
|
package/src/execute/window.js
CHANGED
|
@@ -42,7 +42,7 @@ export function executeWindow(plan, context) {
|
|
|
42
42
|
for await (const row of child.rows()) {
|
|
43
43
|
if (++i % YIELD_INTERVAL === 0) {
|
|
44
44
|
await yieldToEventLoop()
|
|
45
|
-
|
|
45
|
+
context.signal?.throwIfAborted()
|
|
46
46
|
}
|
|
47
47
|
const cells = { ...row.cells }
|
|
48
48
|
for (const w of plan.windows) {
|
|
@@ -69,7 +69,7 @@ export function executeWindow(plan, context) {
|
|
|
69
69
|
for await (const row of child.rows()) {
|
|
70
70
|
if (++collectCount % YIELD_INTERVAL === 0) {
|
|
71
71
|
await yieldToEventLoop()
|
|
72
|
-
|
|
72
|
+
context.signal?.throwIfAborted()
|
|
73
73
|
}
|
|
74
74
|
rows.push(row)
|
|
75
75
|
}
|
|
@@ -81,14 +81,14 @@ export function executeWindow(plan, context) {
|
|
|
81
81
|
|
|
82
82
|
for (let w = 0; w < plan.windows.length; w++) {
|
|
83
83
|
await computeWindow(plan.windows[w], rows, windowValues[w], context)
|
|
84
|
-
|
|
84
|
+
context.signal?.throwIfAborted()
|
|
85
85
|
}
|
|
86
86
|
|
|
87
87
|
let emitCount = 0
|
|
88
88
|
for (let i = 0; i < rows.length; i++) {
|
|
89
89
|
if (++emitCount % YIELD_INTERVAL === 0) {
|
|
90
90
|
await yieldToEventLoop()
|
|
91
|
-
|
|
91
|
+
context.signal?.throwIfAborted()
|
|
92
92
|
}
|
|
93
93
|
const row = rows[i]
|
|
94
94
|
const cells = { ...row.cells }
|
|
@@ -122,7 +122,7 @@ async function computeWindow(spec, rows, output, context) {
|
|
|
122
122
|
for (let chunkStart = 0; chunkStart < rows.length; chunkStart += YIELD_INTERVAL) {
|
|
123
123
|
if (chunkStart > 0) {
|
|
124
124
|
await yieldToEventLoop()
|
|
125
|
-
|
|
125
|
+
context.signal?.throwIfAborted()
|
|
126
126
|
}
|
|
127
127
|
const chunkEnd = Math.min(chunkStart + YIELD_INTERVAL, rows.length)
|
|
128
128
|
const chunkKeys = await Promise.all(
|
|
@@ -142,7 +142,7 @@ async function computeWindow(spec, rows, output, context) {
|
|
|
142
142
|
}
|
|
143
143
|
|
|
144
144
|
for (const bucket of partitions.values()) {
|
|
145
|
-
|
|
145
|
+
context.signal?.throwIfAborted()
|
|
146
146
|
|
|
147
147
|
// Order within the partition. Empty ORDER BY → input order.
|
|
148
148
|
/** @type {number[]} */
|
|
@@ -153,7 +153,7 @@ async function computeWindow(spec, rows, output, context) {
|
|
|
153
153
|
for (let chunkStart = 0; chunkStart < bucket.length; chunkStart += YIELD_INTERVAL) {
|
|
154
154
|
if (chunkStart > 0) {
|
|
155
155
|
await yieldToEventLoop()
|
|
156
|
-
|
|
156
|
+
context.signal?.throwIfAborted()
|
|
157
157
|
}
|
|
158
158
|
const chunkEnd = Math.min(chunkStart + YIELD_INTERVAL, bucket.length)
|
|
159
159
|
const chunkValues = await Promise.all(
|
|
@@ -205,7 +205,7 @@ async function applyWindowFunction(spec, ordered, rows, output, context) {
|
|
|
205
205
|
for (let k = 0; k < ordered.length; k++) {
|
|
206
206
|
if (++tick % YIELD_INTERVAL === 0) {
|
|
207
207
|
await yieldToEventLoop()
|
|
208
|
-
|
|
208
|
+
context.signal?.throwIfAborted()
|
|
209
209
|
}
|
|
210
210
|
const idx = ordered[k]
|
|
211
211
|
const row = rows[idx]
|
|
@@ -29,7 +29,7 @@ const YIELD_INTERVAL = 4000
|
|
|
29
29
|
* @param {ExecuteContext} context
|
|
30
30
|
* @returns {Promise<SqlPrimitive[]>}
|
|
31
31
|
*/
|
|
32
|
-
async function evaluateAll(node, rows, context) {
|
|
32
|
+
export async function evaluateAll(node, rows, context) {
|
|
33
33
|
/** @type {SqlPrimitive[]} */
|
|
34
34
|
const results = new Array(rows.length)
|
|
35
35
|
/** @type {Promise<SqlPrimitive>[]} */
|
package/src/index.d.ts
CHANGED
|
@@ -28,7 +28,7 @@ export type {
|
|
|
28
28
|
* @param options.tables - source data as a list of objects or an AsyncDataSource
|
|
29
29
|
* @param options.query - SQL query string
|
|
30
30
|
* @param options.functions - user-defined functions available in the SQL context
|
|
31
|
-
* @param options.signal - AbortSignal to cancel the query
|
|
31
|
+
* @param options.signal - AbortSignal to cancel the query; an aborted query rejects with the signal's reason
|
|
32
32
|
* @returns async generator yielding rows matching the query
|
|
33
33
|
*/
|
|
34
34
|
export function executeSql(options: ExecuteSqlOptions): QueryResults
|
package/src/plan/columns.js
CHANGED
|
@@ -226,7 +226,7 @@ export function extractColumns({ select, parentColumns }) {
|
|
|
226
226
|
* @param {IdentifierNode[]} columns
|
|
227
227
|
* @param {Set<string>} [aliases] - aliases to exclude from columns
|
|
228
228
|
*/
|
|
229
|
-
function collectColumnsFromExpr(expr, columns, aliases) {
|
|
229
|
+
export function collectColumnsFromExpr(expr, columns, aliases) {
|
|
230
230
|
if (!expr) return
|
|
231
231
|
if (expr.type === 'identifier') {
|
|
232
232
|
if (expr.prefix || !aliases?.has(expr.name)) {
|