querylens 0.1.0

package/dist/src/engine/stats.js

@@ -0,0 +1,118 @@
+/**
+ * `ANALYZE` (plan.md §25.4 C1) — a real scan of a real table, producing the statistics a query planner actually
+ * reads instead of assuming every value in a column is equally likely and every column independent of every
+ * other. Shaped after PostgreSQL's own `pg_stats`: distinct count, null fraction, most-common values (with their
+ * real frequencies), an equi-depth histogram over what is left, and the column's physical/logical correlation.
+ *
+ * Pure and synchronous, on the same rules as the rest of `src/engine`. A full scan, not a sample — a real
+ * `ANALYZE` samples a fraction of the table for speed and lives with the noise that introduces; a toy table is
+ * small enough to just look at every row, so this engine's statistics are as accurate as a scan gets. What they
+ * are *not* is fresh forever: `Table.stats` is a snapshot of one `ANALYZE` run, and stays exactly that — even
+ * once wrong — until `ANALYZE` runs again (plan.md §25.4 C2 is where that staleness is made to matter).
+ */
+import { compareValues } from "./value.js";
+/** How many of a column's most frequent values get their own entry, PostgreSQL's own default. */
+const MOST_COMMON_LIMIT = 5;
+/** Buckets in the equi-depth histogram over whatever `mostCommonValues` does not already cover. */
+const HISTOGRAM_BUCKETS = 10;
+/** Below this many non-MCV rows, a histogram is noise, not signal — left empty instead. */
+const MIN_ROWS_FOR_HISTOGRAM = 20;
+/** `sorted`'s `buckets`-quantile boundaries: `n + 1` values bounding `n` roughly-equal-depth buckets. */
+function equiDepthBounds(sorted, buckets) {
+    const n = Math.max(1, Math.min(buckets, sorted.length - 1));
+    const bounds = [];
+    for (let i = 0; i <= n; i++) {
+        const at = Math.round((i * (sorted.length - 1)) / n);
+        bounds.push(sorted[at]);
+    }
+    return bounds;
+}
+/** Pearson correlation between two equal-length number series — 0 when either has no variation to correlate. */
+function pearson(xs, ys) {
+    const n = xs.length;
+    const meanX = xs.reduce((s, x) => s + x, 0) / n;
+    const meanY = ys.reduce((s, y) => s + y, 0) / n;
+    let num = 0;
+    let denX = 0;
+    let denY = 0;
+    for (let i = 0; i < n; i++) {
+        const dx = xs[i] - meanX;
+        const dy = ys[i] - meanY;
+        num += dx * dy;
+        denX += dx * dx;
+        denY += dy * dy;
+    }
+    return denX === 0 || denY === 0 ? 0 : num / Math.sqrt(denX * denY);
+}
+/**
+ * How much `values`' order tracks physical row order: each value's *rank* among all of them (tied values share
+ * the average of their positions, same as Spearman's), correlated against its row index. `compareValues`'s
+ * NULL-first order decides ties among NULLs the same way it decides everything else here — there is no second
+ * rule for them.
+ */
+function correlationOf(values) {
+    const n = values.length;
+    if (n < 2)
+        return 0;
+    const byValue = values.map((_, i) => i).sort((a, b) => compareValues(values[a], values[b]));
+    const rank = new Array(n);
+    let i = 0;
+    while (i < n) {
+        let j = i;
+        while (j + 1 < n && compareValues(values[byValue[j + 1]], values[byValue[i]]) === 0)
+            j += 1;
+        const averageRank = (i + j) / 2;
+        for (let k = i; k <= j; k++)
+            rank[byValue[k]] = averageRank;
+        i = j + 1;
+    }
+    const physicalIndex = Array.from({ length: n }, (_, i) => i);
+    return pearson(physicalIndex, rank);
+}
+function analyzeColumn(rows, column) {
+    const values = rows.map((row) => row[column] ?? null);
+    const nonNull = values.filter((v) => v !== null);
+    const nullFraction = values.length === 0 ? 0 : (values.length - nonNull.length) / values.length;
+    const counts = new Map();
+    for (const v of nonNull)
+        counts.set(v, (counts.get(v) ?? 0) + 1);
+    // "Most common" means it actually repeats — a column of all-distinct values (a key) has none, honestly.
+    const byFrequency = [...counts.entries()]
+        .filter(([, frequency]) => frequency > 1)
+        .sort((a, b) => b[1] - a[1] || compareValues(a[0], b[0]));
+    const mostCommonValues = byFrequency.slice(0, MOST_COMMON_LIMIT).map(([value, frequency]) => ({ value, frequency }));
+    const covered = new Set(mostCommonValues.map((m) => m.value));
+    const remaining = nonNull.filter((v) => !covered.has(v)).sort(compareValues);
+    const histogramBounds = remaining.length < MIN_ROWS_FOR_HISTOGRAM ? [] : equiDepthBounds(remaining, HISTOGRAM_BUCKETS);
+    return {
+        nDistinct: counts.size,
+        nullFraction,
+        mostCommonValues,
+        histogramBounds,
+        correlation: correlationOf(values),
+    };
+}
+/** The table's statistics, as of right now — what `ANALYZE <table>` computes. */
+export function analyzeTable(table) {
+    const columns = {};
+    for (const column of table.columns)
+        columns[column] = analyzeColumn(table.rows, column);
+    return { rowCount: table.rows.length, columns };
+}
+function describeValue(value) {
+    if (value === null)
+        return 'NULL';
+    return typeof value === 'string' ? `'${value}'` : String(value);
+}
+/** `ANALYZE`'s own result, one row per column — what the Playground's result table (and the CLI) already know
+ *  how to render, so a learner opens the statistics the same way they open any other query's answer. */
+export function statsToRows(stats) {
+    return Object.entries(stats.columns).map(([column, s]) => ({
+        column,
+        n_distinct: s.nDistinct,
+        null_frac: Math.round(s.nullFraction * 1000) / 1000,
+        most_common: s.mostCommonValues.length === 0 ? null : s.mostCommonValues.map((m) => `${describeValue(m.value)} (${String(m.frequency)})`).join(', '),
+        histogram_buckets: Math.max(0, s.histogramBounds.length - 1),
+        correlation: Math.round(s.correlation * 1000) / 1000,
+    }));
+}

package/dist/src/engine/storage/bufferPool.js

@@ -0,0 +1,194 @@
+/**
+ * The buffer pool: a fixed set of frames standing between the executor and
+ * "disk". Every page the engine reads goes through `fetch`, which is the only
+ * place a disk read can happen — so the hit/miss numbers in the UI are exactly
+ * the ones the engine experienced.
+ */
+import { policyFor } from "./policy.js";
+import { evictionPrompt } from "../predict.js";
+/** Playback pacing weight — a disk read should *feel* slower than a hit. */
+export const DISK_READ_DWELL = 2.5;
+function emptyState(frameCount) {
+    return {
+        frames: Array.from({ length: frameCount }, (_, frameId) => ({
+            frameId,
+            pageId: null,
+            usageCount: 0,
+            pinCount: 0,
+            lastUsedAt: -1,
+            loadedAt: -1,
+        })),
+        now: 0,
+        hand: 0,
+        lookaheadCursor: 0,
+    };
+}
+/** How the `evict` event narrates the choice, per policy. */
+function evictionReason(policy) {
+    switch (policy) {
+        case 'lru':
+            return 'LRU picked it because it was used least recently.';
+        case 'clock':
+            return 'Clock sweep picked it because the hand reached it with a usage count of zero.';
+        case 'fifo':
+            return 'FIFO picked it because it has been resident longest — first in, first out, however much it was used.';
+        case 'second-chance':
+            return 'Second-chance picked it: the oldest frame whose reference bit was already clear.';
+        case 'optimal':
+            return "The optimal policy picked it because its next use is farthest away — the choice no real policy can make, since it needs the future.";
+        case 'midpoint':
+            return 'Midpoint LRU picked the least-recently-used page in the old sublist — a page read once and never touched again, which is exactly what it is built to evict first.';
+        case 'two-q':
+            return '2Q picked it: the oldest page in the small queue of pages read only once, which is where a one-pass scan leaves its pages.';
+        case 'lru-k':
+            return 'LRU-2 picked it because its second-most-recent reference is the oldest — a page touched only once has no second reference at all, so it goes first.';
+    }
+}
+/**
+ * `pageCount` covers every page in the database — heap pages first, then index
+ * pages. The pool manages frames, not contents: what a page *holds* is the
+ * caller's business, which is why heap and index pages can share one pool and
+ * compete for the same frames.
+ */
+export function createBufferPool(pageCount, options, emit, 
+/**
+ * The full page-reference string, when the caller has one (`runBufferTrace`
+ * does; `runQuery` does not — its page accesses emerge from execution). Only
+ * the `optimal` policy reads it; every other policy ignores it.
+ */
+lookahead) {
+    const state = emptyState(options.bufferFrames);
+    if (lookahead !== undefined)
+        state.lookahead = lookahead;
+    const policy = policyFor(options.policy);
+    const stats = { hits: 0, misses: 0, evictions: 0, writes: 0 };
+    // Which access `fetch` is resolving, so `optimal` can look past it. Advances
+    // once per successful `fetch`, staying aligned with the reference index.
+    let accessIndex = -1;
+    // Only the first eviction asks. After that the learner has the rule, and a
+    // prompt on every one of them would be nagging rather than teaching.
+    let evictionAsked = false;
+    // Pages a DELETE (or a future write) has modified since they were read —
+    // resident or not, since a page can be marked dirty and evicted later in
+    // the same query. Cleared once the writeback happens.
+    const dirty = new Set();
+    const resident = (pageId) => state.frames.find((f) => f.pageId === pageId);
+    /**
+     * A free frame, or the policy's eviction victim — shared between a disk
+     * read's miss and a fresh write, since both need "a frame for this page"
+     * and both may have to evict for it under the same policy. `incoming` is
+     * narration only: what the emitted `evict` event says is displacing the
+     * victim.
+     */
+    function claimFrame(incoming, forWrite) {
+        const free = state.frames.find((f) => f.pageId === null);
+        if (free)
+            return free;
+        const choice = policy.selectVictim(state, emit);
+        if (choice === null)
+            return null; // every frame pinned
+        const victimId = choice.frameId;
+        const victim = state.frames.find((f) => f.frameId === victimId);
+        const evicted = victim.pageId;
+        stats.evictions += 1;
+        // A clean page's eviction is free — the disk already holds the same
+        // bytes. A dirty one costs a write first, charged and narrated here,
+        // at the moment it is actually paid, not when the page was marked.
+        const writtenBack = dirty.delete(evicted);
+        if (writtenBack)
+            stats.writes += 1;
+        // The eviction question is asked once regardless of which path triggers
+        // it — a write can just as easily be the first eviction a trace shows.
+        const prompt = evictionAsked ? null : evictionPrompt(state.frames, victimId, incoming, options.policy);
+        evictionAsked = true;
+        emit(`The pool is full, so page ${evicted} is evicted from frame ${victimId} to make room ${forWrite ? 'for a freshly written page' : ''}. ${writtenBack
+            ? `Page ${evicted} is dirty — a row was deleted from it — so evicting it costs a write back to disk first. `
+            : ''}${evictionReason(options.policy)}`, {
+            stage: 'buffer',
+            action: 'evict',
+            frameId: victimId,
+            victimPageId: evicted,
+            pageId: incoming,
+            ...(choice.clockHand === undefined ? {} : { clockHand: choice.clockHand }),
+            ...(writtenBack ? { writtenBack } : {}),
+        }, prompt ? { predictable: prompt } : undefined);
+        victim.pageId = null;
+        victim.usageCount = 0;
+        victim.loadedAt = -1;
+        return victim;
+    }
+    function markDirty(pageId) {
+        if (Number.isInteger(pageId) && pageId >= 0 && pageId < pageCount)
+            dirty.add(pageId);
+    }
+    function fetch(pageId) {
+        if (!Number.isInteger(pageId) || pageId < 0 || pageId >= pageCount)
+            return null;
+        accessIndex += 1;
+        state.lookaheadCursor = accessIndex;
+        const hitFrame = resident(pageId);
+        if (hitFrame) {
+            stats.hits += 1;
+            hitFrame.pinCount += 1;
+            policy.onAccess(state, hitFrame.frameId);
+            emit(`Page ${pageId} is already in frame ${hitFrame.frameId} — a buffer hit. No disk read.`, { stage: 'buffer', action: 'hit', pageId, frameId: hitFrame.frameId });
+            return { frameId: hitFrame.frameId, hit: true };
+        }
+        stats.misses += 1;
+        emit(`Page ${pageId} is not in the pool — a buffer miss.`, {
+            stage: 'buffer',
+            action: 'miss',
+            pageId,
+        });
+        const target = claimFrame(pageId, false);
+        if (!target)
+            return null;
+        // Bump the clock on every load, not only in `onAccess` — FIFO and
+        // second-chance order frames by `loadedAt`, and FIFO's `onAccess` is a
+        // no-op, so nothing else would advance it.
+        state.now += 1;
+        target.pageId = pageId;
+        target.usageCount = 0;
+        target.loadedAt = state.now;
+        target.pinCount += 1;
+        // A load is a miss, not a hit: `onLoad` (default `onAccess`) lets a policy
+        // treat the two differently — midpoint-LRU inserts at the old sublist.
+        (policy.onLoad ?? policy.onAccess)(state, target.frameId);
+        emit(`Read page ${pageId} from disk into frame ${target.frameId}. This is the expensive part — everything else is memory.`, { stage: 'buffer', action: 'load', pageId, frameId: target.frameId }, { dwell: DISK_READ_DWELL });
+        return { frameId: target.frameId, hit: false };
+    }
+    /**
+     * A page a caller is *producing* — a `Sort` run or merge-pass output — not
+     * reading off disk. Always a fresh page id (a `Sort` operator never reuses
+     * one within a run), so unlike `fetch` there is no hit path: it either
+     * takes a free frame or evicts, exactly the way a disk read does, and the
+     * two kinds of page genuinely compete for the same pool.
+     */
+    function write(pageId) {
+        if (!Number.isInteger(pageId) || pageId < 0 || pageId >= pageCount)
+            return null;
+        stats.writes += 1;
+        const target = claimFrame(pageId, true);
+        if (!target)
+            return null;
+        state.now += 1;
+        target.pageId = pageId;
+        target.usageCount = 0;
+        target.loadedAt = state.now;
+        target.pinCount += 1;
+        (policy.onLoad ?? policy.onAccess)(state, target.frameId);
+        emit(`Write page ${pageId} into frame ${target.frameId} — a fresh page, not a disk read.`, {
+            stage: 'buffer',
+            action: 'write',
+            pageId,
+            frameId: target.frameId,
+        });
+        return { frameId: target.frameId, hit: false };
+    }
+    function unpin(frameId) {
+        const f = state.frames.find((x) => x.frameId === frameId);
+        if (f && f.pinCount > 0)
+            f.pinCount -= 1;
+    }
+    return { fetch, write, markDirty, unpin, state, stats };
+}

package/dist/src/engine/storage/index.js

@@ -0,0 +1,3 @@
+export { buildHeap, pageById, pageOfRow, rowsOnPage } from "./page.js";
+export { createBufferPool, DISK_READ_DWELL } from "./bufferPool.js";
+export { policyFor, REPLACEMENT_POLICY_LABELS, REPLACEMENT_POLICY_NAMES, USAGE_COUNT_CAP, } from "./policy.js";

package/dist/src/engine/storage/page.js

@@ -0,0 +1,46 @@
+/**
+ * Pages. Real engines use ~4–16 KB pages holding hundreds of rows; we hold a
+ * handful, because a page you can see the contents of teaches more than a
+ * realistic one you cannot. The disclaimer banner covers the difference.
+ */
+import { compareValues } from "../value.js";
+function compareRows(a, b, columns) {
+    for (const column of columns) {
+        const cmp = compareValues(a[column] ?? null, b[column] ?? null);
+        if (cmp !== 0)
+            return cmp;
+    }
+    return 0;
+}
+export function buildHeap(table, rowsPerPage, firstPageId = 0) {
+    const pages = [];
+    const firstRowIndex = [];
+    // A CLUSTERED (index-organized) table (plan.md §25.4 B3e) is stored in clustering-key order, not insertion order
+    // — its whole reason to exist: rows near each other in key order sit near each other on disk, so a range on that
+    // key reads pages in sequence instead of wherever insertion happened to leave them.
+    const rows = table.clusteredKey ? [...table.rows].sort((a, b) => compareRows(a, b, table.clusteredKey)) : table.rows;
+    for (let i = 0; i < rows.length; i += rowsPerPage) {
+        firstRowIndex.push(i);
+        pages.push({
+            pageId: firstPageId + pages.length,
+            table: table.name,
+            rows: rows.slice(i, i + rowsPerPage),
+        });
+    }
+    // An empty table still has one (empty) page, so a scan has something to read.
+    if (pages.length === 0) {
+        firstRowIndex.push(0);
+        pages.push({ pageId: firstPageId, table: table.name, rows: [] });
+    }
+    return { table: table.name, pages, firstRowIndex };
+}
+/** Which page holds the row at `rowIndex`. */
+export function pageOfRow(heap, rowIndex, rowsPerPage) {
+    return heap.pages[Math.floor(rowIndex / rowsPerPage)];
+}
+export function pageById(heap, pageId) {
+    return heap.pages.find((p) => p.pageId === pageId);
+}
+export function rowsOnPage(page) {
+    return page.rows;
+}