querylens 0.1.0

package/dist/src/engine/storage/policy.js

@@ -0,0 +1,360 @@
+/**
+ * Buffer replacement, behind one interface, with two implementations.
+ *
+ * This exists because **textbook LRU is not what real databases do**, and
+ * plan.md §2 makes honesty about ground truth a top-line goal. Postgres avoids
+ * an LRU list precisely because maintaining one needs a lock on every page
+ * access; it uses clock sweep instead. InnoDB uses LRU but inserts at a
+ * midpoint so a big scan cannot flush the hot set. Teaching "eviction = LRU"
+ * would plant exactly the misconception §12 warns about.
+ */
+/** Postgres caps `usage_count` at 5. Same here. */
+export const USAGE_COUNT_CAP = 5;
+function frame(state, frameId) {
+    return state.frames.find((f) => f.frameId === frameId);
+}
+/* -------------------------------------------------------------------------- */
+const lru = {
+    name: 'lru',
+    onAccess(state, frameId) {
+        const f = frame(state, frameId);
+        if (!f)
+            return;
+        state.now += 1;
+        f.lastUsedAt = state.now;
+        f.usageCount = Math.min(f.usageCount + 1, USAGE_COUNT_CAP);
+    },
+    selectVictim(state) {
+        let victim;
+        for (const f of state.frames) {
+            if (f.pinCount > 0)
+                continue;
+            if (!victim || f.lastUsedAt < victim.lastUsedAt)
+                victim = f;
+        }
+        return victim ? { frameId: victim.frameId } : null;
+    },
+};
+/**
+ * Postgres's clock sweep. The hand walks the frames in a circle; each pass
+ * decrements `usage_count`, and the first frame found with
+ * `refcount == 0 && usage_count == 0` is the victim. Frequently used pages
+ * survive several passes without any list to maintain — which is the whole
+ * point, since maintaining one would need a lock on every access.
+ */
+const clock = {
+    name: 'clock',
+    onAccess(state, frameId) {
+        const f = frame(state, frameId);
+        if (!f)
+            return;
+        state.now += 1;
+        f.lastUsedAt = state.now;
+        f.usageCount = Math.min(f.usageCount + 1, USAGE_COUNT_CAP);
+    },
+    selectVictim(state, emit) {
+        const n = state.frames.length;
+        if (n === 0)
+            return null;
+        if (state.frames.every((f) => f.pinCount > 0))
+            return null;
+        // Bounded by construction: every full pass strictly decreases the total
+        // usage count of unpinned frames, and at least one is unpinned.
+        const maxSteps = n * (USAGE_COUNT_CAP + 1) + n;
+        for (let step = 0; step < maxSteps; step++) {
+            const f = state.frames[state.hand];
+            const at = state.hand;
+            state.hand = (state.hand + 1) % n;
+            if (f.pinCount > 0)
+                continue;
+            if (f.usageCount === 0)
+                return { frameId: f.frameId, clockHand: state.hand };
+            f.usageCount -= 1;
+            emit(`Clock sweep: frame ${at} was used recently, so it gets a second chance — usage count drops to ${f.usageCount} and the hand moves on.`, { stage: 'buffer', action: 'sweep', frameId: at, usageCountAfter: f.usageCount, clockHand: state.hand });
+        }
+        return null;
+    },
+};
+/* -------------------------------------------------------------------------- */
+/**
+ * FIFO: evict whichever page has been resident longest, no matter how heavily
+ * it has been used since. The textbook baseline clock sweep improves on — and
+ * the one that shows Bélády's anomaly, where *adding* frames can add misses.
+ */
+const fifo = {
+    name: 'fifo',
+    onAccess() {
+        // A page's age under FIFO is when it was loaded, not when it was touched.
+    },
+    selectVictim(state) {
+        let victim;
+        for (const f of state.frames) {
+            if (f.pinCount > 0)
+                continue;
+            if (!victim || f.loadedAt < victim.loadedAt)
+                victim = f;
+        }
+        return victim ? { frameId: victim.frameId } : null;
+    },
+};
+/**
+ * Second-chance: FIFO with a reference bit. Walk the queue oldest-first; a
+ * frame whose bit is set is forgiven once — the bit clears and it goes to the
+ * back — so the first frame reached with a clear bit is the victim. This is
+ * clock sweep's ancestor: the same idea before the circular hand and the
+ * multi-bit counter. The forgiveness passes emit `sweep` events, the same kind
+ * clock uses, so the panel animates them identically.
+ */
+const secondChance = {
+    name: 'second-chance',
+    onAccess(state, frameId) {
+        const f = frame(state, frameId);
+        if (!f)
+            return;
+        state.now += 1;
+        f.lastUsedAt = state.now;
+        f.usageCount = Math.min(f.usageCount + 1, USAGE_COUNT_CAP);
+    },
+    selectVictim(state, emit) {
+        if (state.frames.every((f) => f.pinCount > 0))
+            return null;
+        // Every forgiveness clears one bit and cannot recur for that frame in the
+        // same call, so a clear bit is reached within one pass over the unpinned
+        // frames. The bound doubles that for headroom, matching `clock` above.
+        const unpinned = state.frames.filter((f) => f.pinCount === 0).length;
+        const maxSteps = unpinned * 2 + 1;
+        for (let step = 0; step < maxSteps; step++) {
+            let oldest;
+            for (const f of state.frames) {
+                if (f.pinCount > 0)
+                    continue;
+                if (!oldest || f.loadedAt < oldest.loadedAt)
+                    oldest = f;
+            }
+            if (!oldest)
+                return null;
+            if (oldest.usageCount === 0)
+                return { frameId: oldest.frameId };
+            oldest.usageCount = 0;
+            state.now += 1;
+            oldest.loadedAt = state.now; // to the back of the queue
+            emit(`Second chance: frame ${oldest.frameId} was used since it loaded, so it survives this pass — its reference bit clears and it moves to the back of the queue.`, {
+                stage: 'buffer',
+                action: 'sweep',
+                frameId: oldest.frameId,
+                usageCountAfter: 0,
+                clockHand: state.hand,
+            });
+        }
+        return null;
+    },
+};
+/**
+ * Bélády's MIN / OPT: evict the resident page whose next use is farthest in
+ * the future (a page never used again wins outright). It cannot be implemented
+ * for real — nothing knows the future — but it is the optimal bound every
+ * other policy is measured against, and seeing it makes "how good is LRU here?"
+ * concrete. With no `lookahead` on the state it degrades to FIFO.
+ */
+function nextUseDistance(future, cursor, pageId) {
+    if (!future)
+        return Number.POSITIVE_INFINITY;
+    for (let k = cursor + 1; k < future.length; k++) {
+        if (future[k] === pageId)
+            return k;
+    }
+    return Number.POSITIVE_INFINITY;
+}
+const optimal = {
+    name: 'optimal',
+    onAccess() {
+        // MIN needs no per-access bookkeeping — the whole decision is the future.
+    },
+    selectVictim(state) {
+        let victim;
+        let victimNextUse = -1;
+        for (const f of state.frames) {
+            if (f.pinCount > 0 || f.pageId === null)
+                continue;
+            const nextUse = nextUseDistance(state.lookahead, state.lookaheadCursor, f.pageId);
+            // Farthest next use wins; on a tie (typically two pages never used
+            // again) evict the one loaded earliest, which makes the no-lookahead
+            // fallback exactly FIFO.
+            if (nextUse > victimNextUse ||
+                (nextUse === victimNextUse && victim !== undefined && f.loadedAt < victim.loadedAt)) {
+                victim = f;
+                victimNextUse = nextUse;
+            }
+        }
+        return victim ? { frameId: victim.frameId } : null;
+    },
+};
+/**
+ * InnoDB's midpoint-insertion LRU. The buffer pool's LRU list is split into a
+ * *young* (hot) sublist and an *old* sublist; a page read from disk lands at
+ * the head of the *old* sublist, and only reaches the young sublist if it is
+ * touched again. A sequential scan touches each page once, so its pages ride
+ * through the old sublist and evict one another without ever displacing the
+ * hot set — the flaw in plain LRU that this exists to fix (plan.md §7.5).
+ *
+ * Modelled without InnoDB's `innodb_old_blocks_time` delay: `usageCount` is a
+ * 1-bit "which sublist" flag — 0 = old, 1 = young — and one re-access promotes.
+ * Eviction takes the least-recently-used frame in the old sublist, falling
+ * back to the overall LRU frame only when every unpinned frame is young.
+ */
+const midpoint = {
+    onLoad(state, frameId) {
+        const f = frame(state, frameId);
+        if (!f)
+            return;
+        state.now += 1;
+        f.lastUsedAt = state.now; // recency, but it stays in the old sublist
+    },
+    name: 'midpoint',
+    onAccess(state, frameId) {
+        const f = frame(state, frameId);
+        if (!f)
+            return;
+        state.now += 1;
+        f.lastUsedAt = state.now;
+        f.usageCount = 1; // a re-access promotes it to the young sublist
+    },
+    selectVictim(state) {
+        let old;
+        let overall;
+        for (const f of state.frames) {
+            if (f.pinCount > 0)
+                continue;
+            if (!overall || f.lastUsedAt < overall.lastUsedAt)
+                overall = f;
+            if (f.usageCount === 0 && (!old || f.lastUsedAt < old.lastUsedAt))
+                old = f;
+        }
+        const victim = old ?? overall;
+        return victim ? { frameId: victim.frameId } : null;
+    },
+};
+/**
+ * LRU-K with K = 2 (O'Neil, O'Neil and Weikum, 1993). Plain LRU evicts whatever was touched least recently, so a
+ * single sequential scan can push out the pages that were genuinely hot. LRU-K evicts by the time of a page's K-th
+ * most recent reference instead. A page referenced only once has no K-th reference at all, so it is the first to go,
+ * and a page that has been touched twice survives a one-pass scan. That is the scan resistance this policy exists for.
+ */
+const LRU_K = 2;
+const lruK = {
+    name: 'lru-k',
+    onAccess(state, frameId) {
+        const f = frame(state, frameId);
+        if (!f)
+            return;
+        state.now += 1;
+        f.lastUsedAt = state.now;
+        f.usageCount = Math.min(f.usageCount + 1, USAGE_COUNT_CAP);
+        if (f.pageId === null)
+            return;
+        const history = (state.history ??= {});
+        const times = [...(history[f.pageId] ?? []), state.now].slice(-LRU_K);
+        history[f.pageId] = times;
+    },
+    selectVictim(state) {
+        let victim;
+        let victimKth = Number.POSITIVE_INFINITY;
+        for (const f of state.frames) {
+            if (f.pinCount > 0)
+                continue;
+            const times = f.pageId === null ? [] : (state.history?.[f.pageId] ?? []);
+            // A page with fewer than K references has no K-th reference: treat it as older than anything, so it goes first.
+            const kth = times.length >= LRU_K ? times[0] : Number.NEGATIVE_INFINITY;
+            if (!victim ||
+                kth < victimKth ||
+                (kth === victimKth && f.lastUsedAt < victim.lastUsedAt)) {
+                victim = f;
+                victimKth = kth;
+            }
+        }
+        return victim ? { frameId: victim.frameId } : null;
+    },
+};
+/**
+ * 2Q (Johnson and Shasha, 1994). A page read once waits in a small FIFO, A1in. Only a page read again after it has
+ * left that FIFO, its id remembered in the ghost list, is promoted to the LRU of hot pages, Am. A one-pass scan
+ * therefore fills A1in and evicts only itself. A1in holds a quarter of the frames, the ghost list half as many ids.
+ */
+const twoQ = {
+    name: 'two-q',
+    onAccess(state, frameId) {
+        const f = frame(state, frameId);
+        if (!f)
+            return;
+        // A re-reference while the page is still in A1in does not promote it: the paper's correlated-reference rule.
+        if (state.queues?.[frameId] === 'a1in')
+            return;
+        state.now += 1;
+        f.lastUsedAt = state.now;
+        f.usageCount = Math.min(f.usageCount + 1, USAGE_COUNT_CAP);
+    },
+    onLoad(state, frameId) {
+        const f = frame(state, frameId);
+        if (!f)
+            return;
+        state.now += 1;
+        f.lastUsedAt = state.now;
+        f.usageCount = Math.min(f.usageCount + 1, USAGE_COUNT_CAP);
+        const ghost = state.ghost ?? [];
+        const remembered = f.pageId !== null && ghost.includes(f.pageId);
+        if (remembered)
+            state.ghost = ghost.filter((p) => p !== f.pageId);
+        (state.queues ??= {})[frameId] = remembered ? 'am' : 'a1in';
+    },
+    selectVictim(state) {
+        const queues = state.queues ?? {};
+        const candidates = state.frames.filter((f) => f.pinCount === 0);
+        const oldestFirst = (a, b) => a.lastUsedAt - b.lastUsedAt;
+        const a1in = candidates.filter((f) => queues[f.frameId] === 'a1in').sort(oldestFirst);
+        const am = candidates.filter((f) => queues[f.frameId] !== 'a1in').sort(oldestFirst);
+        const kin = Math.max(1, Math.floor(state.frames.length / 4));
+        const victim = a1in.length > kin || am.length === 0 ? a1in[0] : am[0];
+        if (!victim)
+            return null;
+        if (queues[victim.frameId] === 'a1in' && victim.pageId !== null) {
+            const kout = Math.max(1, Math.floor(state.frames.length / 2));
+            state.ghost = [...(state.ghost ?? []), victim.pageId].slice(-kout);
+        }
+        return { frameId: victim.frameId };
+    },
+};
+const POLICIES = {
+    lru,
+    clock,
+    fifo,
+    'second-chance': secondChance,
+    optimal,
+    midpoint,
+    'lru-k': lruK,
+    'two-q': twoQ,
+};
+export function policyFor(name) {
+    return POLICIES[name];
+}
+/** Display order for policy pickers. */
+export const REPLACEMENT_POLICY_NAMES = [
+    'lru',
+    'clock',
+    'fifo',
+    'second-chance',
+    'optimal',
+    'midpoint',
+    'lru-k',
+    'two-q',
+];
+/** Short label for each policy, for `<select>`s and column headers. */
+export const REPLACEMENT_POLICY_LABELS = {
+    lru: 'LRU',
+    clock: 'Clock sweep',
+    fifo: 'FIFO',
+    'second-chance': 'Second-chance',
+    optimal: 'Optimal (Bélády)',
+    midpoint: 'Midpoint LRU (InnoDB)',
+    'lru-k': 'LRU-2 (scan-resistant)',
+    'two-q': '2Q (scan-resistant)',
+};

package/dist/src/engine/subquery.js

@@ -0,0 +1,88 @@
+/**
+ * Resolves an `IN (subquery)` predicate (plan.md §25.4 C3 slice c, closing C3) exactly once, before the query
+ * that names it is planned — the one place `runQuery.ts`'s SELECT, DELETE and UPDATE paths all share.
+ *
+ * `mapChildren` (`parser/ast.ts`) already rebuilds every other `Expr` kind from its children, but it does not
+ * know how to turn a `subquery` into `items` — that needs a real sub-run of the engine, not a pure tree
+ * transform — so this walks the same shape `mapChildren` would (`and`/`or`/`not`, the only kinds that can nest
+ * another predicate) by hand, stopping to resolve any `in` node that carries a `subquery`.
+ */
+/** Every distinct value `column` holds across `rows`, in first-seen order — `null` counted like any other value. */
+function distinctColumnValues(rows, column) {
+    const seen = new Set();
+    const values = [];
+    for (const row of rows) {
+        const value = row[column] ?? null;
+        const key = value === null ? '\u0000null' : `${typeof value}:${String(value)}`;
+        if (seen.has(key))
+            continue;
+        seen.add(key);
+        values.push(value);
+    }
+    return values;
+}
+/** A resolved value, as the literal `Expr` an ordinary `IN (1, 2, 3)` would already have held. */
+function literalFor(value, span) {
+    return {
+        kind: 'literal',
+        value,
+        raw: value === null ? 'NULL' : typeof value === 'string' ? `'${value}'` : String(value),
+        span,
+    };
+}
+/**
+ * Runs one `IN`'s subquery for real — its own complete, independent sub-run of the engine (its own buffer pool,
+ * its own disk reads), exactly the way `explain.ts`'s `explainResult` already runs an inner query: padding the
+ * text with leading spaces rather than slicing them away keeps every character offset the sub-run reports (a
+ * parse error's span, say) pointing at the right place in the *outer* query's own editor. Not interleaved into
+ * the outer query's own step-by-step trace — the subquery's cost is real and already paid by the time this
+ * returns, but it is reported as one resolved fact (`emit.ts`'s WHERE-clause narration names the count), not a
+ * second parallel playback.
+ */
+function resolveOne(expr, subquery, sql, run) {
+    const inner = run(' '.repeat(subquery.span.from) + sql.slice(subquery.span.from, subquery.span.to));
+    if (inner.error) {
+        return {
+            ok: false,
+            error: {
+                message: `The IN subquery failed: ${inner.error.message}`,
+                ...(inner.error.hint ? { hint: inner.error.hint } : {}),
+                from: subquery.span.from,
+                to: subquery.span.to,
+            },
+        };
+    }
+    const values = distinctColumnValues(inner.rows, subquery.column.name);
+    const items = values.map((value) => literalFor(value, subquery.span));
+    return { ok: true, expr: { ...expr, items } };
+}
+/** `resolveOne`, recursively, for every `IN (subquery)` anywhere in `expr` — under any nesting of `AND`/`OR`/`NOT`. */
+function resolveInSubqueries(expr, sql, run) {
+    if (expr.kind === 'in' && expr.subquery)
+        return resolveOne(expr, expr.subquery, sql, run);
+    if (expr.kind === 'and' || expr.kind === 'or') {
+        const left = resolveInSubqueries(expr.left, sql, run);
+        if (!left.ok)
+            return left;
+        const right = resolveInSubqueries(expr.right, sql, run);
+        if (!right.ok)
+            return right;
+        return { ok: true, expr: { ...expr, left: left.expr, right: right.expr } };
+    }
+    if (expr.kind === 'not') {
+        const operand = resolveInSubqueries(expr.operand, sql, run);
+        if (!operand.ok)
+            return operand;
+        return { ok: true, expr: { ...expr, operand: operand.expr } };
+    }
+    // Every other kind's own children are arithmetic/columns/literals — `parsePredicate`'s grammar never lets a
+    // predicate (so never an `IN (subquery)`) nest inside one of those.
+    return { ok: true, expr };
+}
+/** `where`, with every `IN (subquery)` it contains resolved — or the first error one of them hit. `undefined` in, `undefined` out. */
+export function resolveWhereSubqueries(where, sql, run) {
+    if (!where)
+        return { ok: true, where };
+    const resolved = resolveInSubqueries(where, sql, run);
+    return resolved.ok ? { ok: true, where: resolved.expr } : resolved;
+}

package/dist/src/engine/trace.js

@@ -0,0 +1,17 @@
+/**
+ * Trace buffer: an `emit` callback threaded down into every operator. The
+ * query runs to completion and the array is returned. Simplest thing that
+ * works, and it works because playback is pre-computed anyway (plan.md §7.8).
+ */
+export function createTracer() {
+    const events = [];
+    const emit = (label, body, options) => {
+        const event = { ...body, id: events.length, label };
+        if (options?.predictable)
+            event.predictable = options.predictable;
+        if (options?.dwell !== undefined)
+            event.dwell = options.dwell;
+        events.push(event);
+    };
+    return { emit, drain: () => events.slice() };
+}

package/dist/src/engine/types.js

@@ -0,0 +1,39 @@
+/**
+ * The single contract between the engine and the UI.
+ *
+ * Rules for everything in `src/engine`:
+ *   - pure and synchronous (no DOM, no React, no async, no I/O)
+ *   - every value here must be structurally cloneable, so the engine stays
+ *     worker-ready even though v0 runs it on the main thread
+ */
+export const STAGES = [
+    'parse',
+    'plan',
+    'optimize',
+    'execute',
+    'buffer',
+    'index',
+    'lock',
+    'result',
+];
+export const DEFAULT_ENGINE_OPTIONS = {
+    bufferFrames: 8,
+    rowsPerPage: 4,
+    policy: 'lru',
+};
+/**
+ * The stages this build of the engine actually executes. Everything else in
+ * `STAGES` is scaffolding the UI must show as unbuilt — plan.md §2 makes
+ * honesty about ground truth a top-line goal, and a pipeline that looks
+ * complete when it is not breaks that. Grows one milestone at a time.
+ */
+export const IMPLEMENTED_STAGES = [
+    'parse',
+    'plan',
+    'optimize',
+    'buffer',
+    'index',
+    'execute',
+    'lock',
+    'result',
+];

package/dist/src/engine/value.js

@@ -0,0 +1,80 @@
+/**
+ * Arithmetic on `SqlValue`s (plan.md §25.4 B1b) — pure, and free of the parser
+ * and the executor so that both can use it: the executor to evaluate
+ * `price * qty`, the parser and optimizer to fold `-5` or `2 + 3` into a
+ * literal (a folded literal is what lets `WHERE id = 2 + 3` become an index
+ * lookup). The rules follow SQLite where it is well-defined, and PostgreSQL
+ * where SQLite is permissive — each choice is a ledger entry or a comment.
+ */
+const NUMERIC_TEXT = /^\s*[-+]?(\d+\.?\d*|\.\d+)(e[-+]?\d+)?\s*$/i;
+/**
+ * A value as a number, or `null` when it is not one.
+ *
+ * NULL stays NULL. A boolean is 0 or 1. A string that *is* a number (`'5'`,
+ * `' 2.5 '`) is that number. Any other string is not a number, so it yields
+ * NULL — the arithmetic result is then NULL. SQLite reads such text as 0,
+ * silently (ledger: `arithmetic-on-text`); NULL says "this is not a number"
+ * without inventing a value.
+ */
+export function toNumber(value) {
+    if (value === null)
+        return null;
+    if (typeof value === 'number')
+        return value;
+    if (typeof value === 'boolean')
+        return value ? 1 : 0;
+    return NUMERIC_TEXT.test(value) ? Number(value) : null;
+}
+/** `-0` is not a value SQL users ever see; keep it out of results and out of `toEqual`. */
+const clean = (n) => (n === 0 ? 0 : n);
+/**
+ * `left op right`. NULL in, NULL out. `x / 0` and `x % 0` are NULL (SQLite's
+ * rule — PostgreSQL raises an error, which this engine has no channel for at
+ * run time). `/` between two whole numbers is **integer division**, truncating
+ * toward zero (`7 / 2` is 3, `-7 / 2` is -3), in both SQLite and PostgreSQL;
+ * anything else divides exactly. Numbers here carry no int/real distinction,
+ * so a REAL column holding 3.0 divides as the integer 3 — ledger:
+ * `division-of-whole-reals`.
+ */
+export function arithmetic(op, left, right) {
+    const a = toNumber(left);
+    const b = toNumber(right);
+    if (a === null || b === null)
+        return null;
+    switch (op) {
+        case '+':
+            return clean(a + b);
+        case '-':
+            return clean(a - b);
+        case '*':
+            return clean(a * b);
+        case '/':
+            if (b === 0)
+                return null;
+            return clean(Number.isInteger(a) && Number.isInteger(b) ? Math.trunc(a / b) : a / b);
+        case '%':
+            return b === 0 ? null : clean(a % b);
+    }
+}
+/** Unary minus. NULL stays NULL; a non-number is NULL. */
+export function negate(value) {
+    const n = toNumber(value);
+    return n === null ? null : clean(-n);
+}
+/**
+ * A total order over `SqlValue`, `-1`/`0`/`1`: NULL sorts before everything and equals only NULL, two numbers
+ * compare numerically, anything else compares as text (a boolean as `'0'`/`'1'`) — an **index's** order, not SQL's
+ * `ORDER BY` (`exec/sort.ts`'s own `compareAsc` sorts NULLs *last*, Postgres's default there, deliberately the
+ * opposite rule). Shared by a table's own B+Tree indexes (`index/btree.ts`'s `compareScalar`), a clustered table's
+ * physical layout (`storage/page.ts`), and `ANALYZE`'s histogram and correlation statistics (plan.md §25.4 C1) —
+ * kept here once so none of the three can quietly drift from another.
+ */
+export function compareValues(a, b) {
+    if (a === null || b === null)
+        return a === b ? 0 : a === null ? -1 : 1;
+    if (typeof a === 'number' && typeof b === 'number')
+        return a - b;
+    const sa = typeof a === 'boolean' ? String(Number(a)) : String(a);
+    const sb = typeof b === 'boolean' ? String(Number(b)) : String(b);
+    return sa < sb ? -1 : sa > sb ? 1 : 0;
+}