sqllang 0.0.3 → 0.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/index.js +129 -11
- package/dist/esm/index.js +129 -11
- package/dist/esm/index.min.js +6 -6
- package/package.json +1 -1
- package/src/execute.js +126 -12
- package/src/index.js +28 -2
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "sqllang",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.4",
|
|
4
4
|
"description": "SQL over your own data: query arrays of rows, columnar tables, nested documents and JSON-text columns with a SQL dialect — joins, CTEs, window functions, subqueries. Compiles to JSON Logic and runs on json-logic-engine.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/cjs/index.js",
|
package/src/execute.js
CHANGED
|
@@ -26,6 +26,67 @@ const internal = {
|
|
|
26
26
|
// One output row from `[name, value, name, value, ...]`. Compiled, it is a single
|
|
27
27
|
// object literal, so a whole select list costs one call per row. Not
|
|
28
28
|
// deterministic: each row needs its own object, never a folded shared one.
|
|
29
|
+
// A one-table GROUP BY as one function: [alias, where | null, keys, aggregates], each
|
|
30
|
+
// aggregate { kind, star, arg, filter }. The loop, the group lookup and the running
|
|
31
|
+
// totals are one piece of code. Rows come in as `context.$scan`, the group table as
|
|
32
|
+
// `context.$grp`. COUNT, SUM, AVG, MIN and MAX only: the incremental ones.
|
|
33
|
+
GROUPSCAN: {
|
|
34
|
+
method: () => { throw new SqlError('GROUPSCAN runs compiled only') },
|
|
35
|
+
compile: ([alias, where, keys, aggs], bs) => {
|
|
36
|
+
const raw = (text) => bs.compile([text])
|
|
37
|
+
// `alias.column` reads straight off the row, with the same guard the engine puts on
|
|
38
|
+
// a column read; anything else reads the context, so the row is stored there too.
|
|
39
|
+
const simple = (logic) => {
|
|
40
|
+
const v = logic !== null && typeof logic === 'object' ? logic.val : null
|
|
41
|
+
return Array.isArray(v) && v.length === 2 && v[0] === alias && typeof v[1] === 'string' ? v[1] : null
|
|
42
|
+
}
|
|
43
|
+
const read = (logic) => {
|
|
44
|
+
const column = simple(logic)
|
|
45
|
+
return column === null ? logic : raw(`(typeof (p = (r?.[${JSON.stringify(column)}] ?? null)) === "function" ? null : p)`)
|
|
46
|
+
}
|
|
47
|
+
const exprs = [...keys, ...aggs.flatMap((a) => [a.arg, a.filter])].filter((e) => e !== null)
|
|
48
|
+
const needsContext = where !== null || exprs.some((e) => simple(e) === null)
|
|
49
|
+
|
|
50
|
+
let res = raw('(() => { const rows = context["$scan"], grp = context["$grp"], root = grp.root, open = grp.open; let single = null, p, prev; const ints = []; for (let i = 0; i < rows.length; i++) { ')
|
|
51
|
+
res = bs.compile`${res}if ((i & 1023) === 0) ${sync(tick)}(Math.min(1024, rows.length - i)); const r = rows[i]; `
|
|
52
|
+
if (needsContext) res = bs.compile`${res}context[${alias}] = r; `
|
|
53
|
+
if (where !== null) res = bs.compile`${res}if (!${sync(isTrue)}(${where})) continue; `
|
|
54
|
+
const fresh = raw(`{ ${aggs.map((_, j) => `c${j}: 0, s${j}: 0, n${j}: 0, b${j}: null`).join(', ')}${aggs.length ? ', ' : ''}first: null }`)
|
|
55
|
+
if (!keys.length) {
|
|
56
|
+
res = bs.compile`${res}if (single === null) { single = ${fresh}; open(single, r) } const g = single; `
|
|
57
|
+
} else {
|
|
58
|
+
const key = (logic) => bs.compile`(p = ${read(logic)}) === undefined ? null : (p !== null && typeof p === "object") ? ${sync(keyOf)}(p) : p`
|
|
59
|
+
if (keys.length === 1) {
|
|
60
|
+
// One key: small non-negative integers index an array, which V8 reads faster than a Map.
|
|
61
|
+
res = bs.compile`${res}let g; { const k = ${key(keys[0])}; if (typeof k === "number" && k >= 0 && k < 1048576 && (k | 0) === k) { g = ints[k]; if (g === undefined) { g = ${fresh}; open(g, r); ints[k] = g } } else { g = root.get(k); if (g === undefined) { g = ${fresh}; open(g, r); root.set(k, g) } } } `
|
|
62
|
+
}
|
|
63
|
+
res = bs.compile`${res}${raw(keys.length === 1 ? '' : 'let level = root; ')}`
|
|
64
|
+
for (let d = 0; keys.length > 1 && d < keys.length - 1; d++) {
|
|
65
|
+
res = bs.compile`${res}{ const k = ${key(keys[d])}; let next = level.get(k); if (next === undefined) { next = new Map(); level.set(k, next) } level = next } `
|
|
66
|
+
}
|
|
67
|
+
if (keys.length > 1) res = bs.compile`${res}let g; { const k = ${key(keys[keys.length - 1])}; g = level.get(k); if (g === undefined) { g = ${fresh}; open(g, r); level.set(k, g) } } `
|
|
68
|
+
}
|
|
69
|
+
aggs.forEach((a, j) => {
|
|
70
|
+
let body
|
|
71
|
+
if (a.star) body = raw(`g.c${j}++; `)
|
|
72
|
+
else {
|
|
73
|
+
const kindCode = a.kind === SUM || a.kind === AVG
|
|
74
|
+
? bs.compile`${raw('const x = typeof v === "number" ? v : ')}${sync(toNumber)}(v); if (x !== null) ${raw(`{ g.s${j} += x; g.n${j}++ }`)} `
|
|
75
|
+
: a.kind === MIN || a.kind === MAX
|
|
76
|
+
? bs.compile`${raw(`if (g.b${j} === null || (typeof v === "number" && typeof g.b${j} === "number" ? v ${a.kind === MIN ? '<' : '>'} g.b${j} : `)}${sync(sortCompare)}(v, ${raw(`g.b${j}`)}, false, null) ${raw(a.kind === MIN ? '< 0' : '> 0')})) ${raw(`g.b${j} = v; `)}`
|
|
77
|
+
: raw('')
|
|
78
|
+
body = bs.compile`${raw('const v = ')}${read(a.arg)}; if (v !== null && v !== undefined) { ${raw(`g.c${j}++; `)}${kindCode} } `
|
|
79
|
+
}
|
|
80
|
+
if (a.filter !== null) body = bs.compile`if (${sync(isTrue)}(${read(a.filter)})) { ${body} } `
|
|
81
|
+
res = bs.compile`${res}{ ${body} } `
|
|
82
|
+
})
|
|
83
|
+
return bs.compile`${res}} })()`
|
|
84
|
+
},
|
|
85
|
+
traverse: false,
|
|
86
|
+
deterministic: false
|
|
87
|
+
},
|
|
88
|
+
// (Both scans declare their own `prev`, the engine's scratch variable, so the snippets inside
|
|
89
|
+
// them use a local instead of the enclosing function's, which a closure would keep in memory.)
|
|
29
90
|
// A whole single-table scan as one function: [alias, where | null, row]. Compiled, the
|
|
30
91
|
// loop, the filter and the projection are one piece of code, with the rows passed in
|
|
31
92
|
// as `context.$scan`. Only used compiled; the interpreted path runs the loop itself.
|
|
@@ -33,7 +94,7 @@ const internal = {
|
|
|
33
94
|
method: () => { throw new SqlError('SCAN runs compiled only') },
|
|
34
95
|
compile: ([alias, where, row], bs) => {
|
|
35
96
|
const skip = where === null ? bs.compile`` : bs.compile`if (!${sync(isTrue)}(${where})) continue;`
|
|
36
|
-
return bs.compile`(() => { const rows = context["$scan"]; const out = []; for (let i = 0; i < rows.length; i++) { if ((i & 1023) === 0) ${sync(tick)}(Math.min(1024, rows.length - i)); context[${alias}] = rows[i]; ${skip} out.push(${row}) } return out })()`
|
|
97
|
+
return bs.compile`(() => { let prev; const rows = context["$scan"]; const out = []; for (let i = 0; i < rows.length; i++) { if ((i & 1023) === 0) ${sync(tick)}(Math.min(1024, rows.length - i)); context[${alias}] = rows[i]; ${skip} out.push(${row}) } return out })()`
|
|
37
98
|
},
|
|
38
99
|
traverse: false,
|
|
39
100
|
deterministic: false
|
|
@@ -420,8 +481,11 @@ function extend (env, aliases, alias, row) {
|
|
|
420
481
|
|
|
421
482
|
function runSelect (p, state, outer, keep = null) {
|
|
422
483
|
const base = outer ? Object.create(outer) : {}
|
|
423
|
-
const projector = makeProjector(p, state)
|
|
424
484
|
const orderFns = p.orderBy ? p.orderBy.map((o) => fnOf(state, o.expr)) : null
|
|
485
|
+
// A plain one-table scan runs as one compiled function, which has its own copy of
|
|
486
|
+
// the select list, so the projector need not build (and eval) another.
|
|
487
|
+
const scan = orderFns === null && !p.grouped && p.having === null && scannable(p, state)
|
|
488
|
+
const projector = makeProjector(p, state, scan)
|
|
425
489
|
// DISTINCT must see every row before the first k are known, so it sorts fully.
|
|
426
490
|
const top = orderFns && keep !== null && !p.distinct ? topK(keep, comparatorFor(p.orderBy)) : null
|
|
427
491
|
|
|
@@ -445,8 +509,8 @@ function runSelect (p, state, outer, keep = null) {
|
|
|
445
509
|
items = top.best
|
|
446
510
|
} else if (orderFns === null) {
|
|
447
511
|
// No ORDER BY: nothing needs the `{ row, keys }` wrapper, so keep the rows.
|
|
448
|
-
plain = scanRows(p, state, base)
|
|
449
|
-
|
|
512
|
+
if (scan) plain = scanRows(p, state, base)
|
|
513
|
+
else {
|
|
450
514
|
plain = []
|
|
451
515
|
streamRows(p, state, base, (env) => { plain.push(projector.project(env)) })
|
|
452
516
|
}
|
|
@@ -462,7 +526,9 @@ function runSelect (p, state, outer, keep = null) {
|
|
|
462
526
|
} else {
|
|
463
527
|
let envs
|
|
464
528
|
if (single && p.grouped) {
|
|
465
|
-
envs =
|
|
529
|
+
envs = groupScannable(p, state)
|
|
530
|
+
? groupScan(p, state, base)
|
|
531
|
+
: group(p, state, base, (visit) => streamRows(p, state, base, visit))
|
|
466
532
|
} else {
|
|
467
533
|
envs = [base]
|
|
468
534
|
for (const step of p.steps) envs = joinStep(step, envs, base, state)
|
|
@@ -535,11 +601,54 @@ function sourceRows (p, state, base) {
|
|
|
535
601
|
return source.kind === 'expr' ? toRows(fnOf(state, source.expr)(base), false) : materialize(source, state)
|
|
536
602
|
}
|
|
537
603
|
|
|
604
|
+
const groupLogics = new WeakMap()
|
|
605
|
+
|
|
606
|
+
/** Whether a one-table GROUP BY can run as one compiled function. */
|
|
607
|
+
function groupScannable (p, state) {
|
|
608
|
+
return !state.interpreted && p.aggregates.every((spec) => INCREMENTAL.has(spec.name) && !spec.distinct && !spec.orderBy)
|
|
609
|
+
}
|
|
610
|
+
|
|
611
|
+
/** The groups of a one-table GROUP BY, as the contexts `group()` would give. */
|
|
612
|
+
function groupScan (p, state, base) {
|
|
613
|
+
let logic = groupLogics.get(p)
|
|
614
|
+
if (!logic) {
|
|
615
|
+
const aggs = p.aggregates.map((spec) => ({
|
|
616
|
+
kind: KIND[spec.name], star: spec.star, arg: spec.star ? null : spec.args[0], filter: spec.filter
|
|
617
|
+
}))
|
|
618
|
+
groupLogics.set(p, logic = { GROUPSCAN: [p.steps[0].alias, p.where, p.groupBy ?? [], aggs] })
|
|
619
|
+
}
|
|
620
|
+
const alias = p.steps[0].alias
|
|
621
|
+
const aggs = p.aggregates.map((spec) => ({ incremental: true, kind: KIND[spec.name] }))
|
|
622
|
+
const list = []
|
|
623
|
+
const open = (g, row) => {
|
|
624
|
+
g.first = snapshot(base)
|
|
625
|
+
g.first[alias] = row
|
|
626
|
+
list.push(g)
|
|
627
|
+
checkRows(list.length)
|
|
628
|
+
}
|
|
629
|
+
const env = snapshot(base)
|
|
630
|
+
env.$scan = sourceRows(p, state, base)
|
|
631
|
+
env.$grp = { root: new Map(), open }
|
|
632
|
+
fnOf(state, logic)(env)
|
|
633
|
+
if (!(p.groupBy && p.groupBy.length) && list.length === 0) list.push({ first: base, accs: newAccumulators(aggs) })
|
|
634
|
+
// The generated code keeps each total as a field on its group; `finish` reads accumulator objects.
|
|
635
|
+
for (let i = 0; i < list.length; i++) {
|
|
636
|
+
const g = list[i]
|
|
637
|
+
if (g.accs) continue
|
|
638
|
+
g.accs = aggs.map((_, j) => ({ count: g[`c${j}`], sum: g[`s${j}`], numeric: g[`n${j}`], best: g[`b${j}`] }))
|
|
639
|
+
}
|
|
640
|
+
return finishGroups(list, aggs)
|
|
641
|
+
}
|
|
642
|
+
|
|
538
643
|
const scanLogics = new WeakMap()
|
|
539
644
|
|
|
540
|
-
/**
|
|
645
|
+
/** Whether a one-table select can run as one compiled scan. */
|
|
646
|
+
function scannable (p, state) {
|
|
647
|
+
return !state.interpreted && p.row !== null && p.steps.length === 1 && p.steps[0].join === 'FROM' && !p.windows.length
|
|
648
|
+
}
|
|
649
|
+
|
|
650
|
+
/** The projected rows of a one-table scan, run as one compiled function. */
|
|
541
651
|
function scanRows (p, state, base) {
|
|
542
|
-
if (state.interpreted || !p.row) return null
|
|
543
652
|
let logic = scanLogics.get(p)
|
|
544
653
|
if (!logic) scanLogics.set(p, logic = { SCAN: [p.steps[0].alias, p.where, p.row] })
|
|
545
654
|
const env = snapshot(base)
|
|
@@ -570,10 +679,13 @@ function orderKeys (p, env, row, projector, orderFns) {
|
|
|
570
679
|
}
|
|
571
680
|
|
|
572
681
|
/** Builds output rows from contexts, and collects the column list as it goes. */
|
|
573
|
-
function makeProjector (p, state) {
|
|
682
|
+
function makeProjector (p, state, skipBuild = false) {
|
|
574
683
|
const cols = p.columns
|
|
575
|
-
const exprFns = cols.map((c) => (c.kind === 'expr' ? fnOf(state, c.expr) : null))
|
|
576
684
|
const hasStar = cols.some((c) => c.kind === 'star')
|
|
685
|
+
// No `*` and no `__proto__`: the whole select list is one compiled object literal.
|
|
686
|
+
const rowFn = !skipBuild && !hasStar && p.row ? fnOf(state, p.row) : null
|
|
687
|
+
// Per-column functions are only the fallback for that, so they are built only when needed.
|
|
688
|
+
const exprFns = skipBuild || rowFn ? [] : cols.map((c) => (c.kind === 'expr' ? fnOf(state, c.expr) : null))
|
|
577
689
|
const columnSet = new Set()
|
|
578
690
|
const columns = []
|
|
579
691
|
const addColumn = (name) => { if (!columnSet.has(name)) { columnSet.add(name); columns.push(name) } }
|
|
@@ -581,9 +693,6 @@ function makeProjector (p, state) {
|
|
|
581
693
|
// Names are fixed per query; only `__proto__` needs the careful path.
|
|
582
694
|
const plain = cols.map((c) => c.kind === 'expr' && c.name !== '__proto__')
|
|
583
695
|
|
|
584
|
-
// No `*` and no `__proto__`: the whole select list is one compiled object literal.
|
|
585
|
-
const rowFn = !hasStar && p.row ? fnOf(state, p.row) : null
|
|
586
|
-
|
|
587
696
|
const project = rowFn ?? ((env) => {
|
|
588
697
|
const out = {}
|
|
589
698
|
for (let j = 0; j < cols.length; j++) {
|
|
@@ -1232,6 +1341,11 @@ function group (p, state, base, each) {
|
|
|
1232
1341
|
})
|
|
1233
1342
|
}
|
|
1234
1343
|
|
|
1344
|
+
return finishGroups(list, aggs)
|
|
1345
|
+
}
|
|
1346
|
+
|
|
1347
|
+
/** One context per group: its first row, with the finished aggregates as `$agg`. */
|
|
1348
|
+
function finishGroups (list, aggs) {
|
|
1235
1349
|
const out = new Array(list.length)
|
|
1236
1350
|
for (let gi = 0; gi < list.length; gi++) {
|
|
1237
1351
|
const g = list[gi]
|
package/src/index.js
CHANGED
|
@@ -47,14 +47,40 @@ export function prepare (sql, options = {}) {
|
|
|
47
47
|
* @returns {object[] & { columns: string[] }}
|
|
48
48
|
*/
|
|
49
49
|
export function query (sql, data, options = {}) {
|
|
50
|
-
|
|
50
|
+
const engine = options.engine ?? defaultEngine
|
|
51
|
+
return asRows(last(executePlan(planFor(sql, engine, options), data, { ...options, engine }).results))
|
|
51
52
|
}
|
|
52
53
|
|
|
53
54
|
/**
|
|
54
55
|
* Run a script and get every statement's result, and the tables it created.
|
|
55
56
|
*/
|
|
56
57
|
export function execute (sql, data, options = {}) {
|
|
57
|
-
|
|
58
|
+
const engine = options.engine ?? defaultEngine
|
|
59
|
+
return executePlan(planFor(sql, engine, options), data, { ...options, engine })
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
// The same SQL text compiles to the same plan, and a plan carries the functions the
|
|
63
|
+
// engine built for it, so `query()` keeps the most recent ones. `limits` shape the
|
|
64
|
+
// plan at compile time, so a call that passes them compiles its own.
|
|
65
|
+
const plans = new WeakMap()
|
|
66
|
+
const PLAN_CACHE_SIZE = 256
|
|
67
|
+
const PLAN_CACHE_MAX_SQL = 8192
|
|
68
|
+
|
|
69
|
+
function planFor (sql, engine, options) {
|
|
70
|
+
const fresh = () => compile(sql, { functions: engine.sqlFunctions, limits: options.limits })
|
|
71
|
+
if (options.limits || typeof sql !== 'string' || sql.length > PLAN_CACHE_MAX_SQL) return fresh()
|
|
72
|
+
let cache = plans.get(engine)
|
|
73
|
+
if (!cache) plans.set(engine, cache = new Map())
|
|
74
|
+
let plan = cache.get(sql)
|
|
75
|
+
if (plan !== undefined) {
|
|
76
|
+
cache.delete(sql) // least recently used first
|
|
77
|
+
cache.set(sql, plan)
|
|
78
|
+
return plan
|
|
79
|
+
}
|
|
80
|
+
plan = fresh()
|
|
81
|
+
cache.set(sql, plan)
|
|
82
|
+
if (cache.size > PLAN_CACHE_SIZE) cache.delete(cache.keys().next().value)
|
|
83
|
+
return plan
|
|
58
84
|
}
|
|
59
85
|
|
|
60
86
|
function last (results) {
|