slopkit 0.3.0 → 0.4.0-rc.0

package/dist/test/states.d.ts

@@ -1,10 +1,7 @@
-/**
- * Checks an app's oracle runs on its state tables and screens (`slopkit/states`), for what the types can't rule out.
- * Each returns the problems found, empty when there are none, so a test asserts `toEqual([])` and reads every problem
- * at once. They check rules, never an app's expected texts: those stay the app's hand-written oracle.
- */
-import type { Line, Screen, Subject } from "../states.ts";
-/** At least one sample state of every kind, for the oracle and the gallery. */
+import type { Update } from "../machine.ts";
+import type { Line, Meaning, Screen, Subject } from "../states.ts";
+import type { Model } from "../sync.ts";
+/** At least one sample state of every kind, for the tests and the gallery. */
 export type Samples<S extends {
     kind: string;
 }> = {
@@ -36,3 +33,43 @@ export declare function screenProblems<A>(screen: Screen<A>, parts: readonly Lin
  * that must vary most last, or pass a limit that covers it.
  */
 export declare function bounded<T>(choices: readonly (readonly T[])[], limit: number): T[][];
+/** What is under way when a view is drawn: the server's jobs, the page's sync model, and the clock. */
+export type World = {
+    now: number;
+    /** The server's jobs (`slopkit/server/jobs`); finished ones may be listed too. */
+    jobs?: readonly {
+        kind: string;
+        key: string;
+        state: "due" | "running" | "done" | "failed";
+        dueAt: string;
+    }[];
+    /** The page's sync model: its commands and its copy. */
+    model?: Model<any, any, any, any>;
+};
+/**
+ * A reached view's problems by the kit's rules: every line has text with no broken value; every screen keeps its parts
+ * (`screenProblems`); and, given what is under way (`world`), every working line's owed work is: a job running or due by
+ * now, a command not yet settled, or the page's copy loading, pulling, pushing or writing. Takes any value an app's view
+ * returns and finds the kit's lines and screens in it. Run it on every view a test reaches, not only on samples.
+ */
+export declare function viewProblems(shown: unknown, options: {
+    order: readonly Meaning[];
+    world?: World;
+}): string[];
+/** Where a replay ends (`replayProblems`): the last model, every effect, and each reached view's problems. */
+export type ReplayChecked<M, E> = {
+    model: M;
+    effects: E[];
+    problems: string[];
+};
+/**
+ * Replay `messages` through `update` from `initial` and run `viewProblems` on the view of every model reached: the
+ * initial one (step 0) and the one after each message (step n). `world` is drawn from each reached model (its sync
+ * model, its clock, the jobs the test supplies), never a snapshot. Returns the last model, every effect, and the
+ * problems, each naming its step.
+ */
+export declare function replayProblems<M, Msg, E>(update: Update<M, Msg, E>, initial: M, messages: readonly Msg[], options: {
+    view: (model: M) => unknown;
+    order: readonly Meaning[];
+    world?: (model: M, step: number) => World;
+}): ReplayChecked<M, E>;

package/dist/test/states.js

@@ -1,3 +1,10 @@
+/**
+ * Checks an app's tests run on its state tables, screens and every view they reach (`slopkit/states`), for what the
+ * types can't rule out. Each returns the problems found, empty when there are none, so a test asserts `toEqual([])`
+ * and reads every problem at once. They check rules, never an app's expected texts: those stay the app's own tests.
+ */
+import { isLine, isScreen } from "../states.js";
+import { deriveHistoryState, reachable, settled } from "../sync.js";
 const BROKEN = /undefined|null|NaN|\[object |Invalid Date/;
 /** Every sample drawn through its table, in table order. */
 export function lines(subject, samples) {
@@ -103,3 +110,148 @@ export function bounded(choices, limit) {
     }
     return out;
 }
+/** Every Line and Screen inside a value: objects, arrays, Maps and Sets, each visited once. */
+// oxlint-disable-next-line anti-slop/no-unknown-parameters -- an app's view is any shape; only the kit's classes are read
+function found(value) {
+    const drawn = [];
+    const screens = [];
+    const seen = new Set();
+    // oxlint-disable-next-line anti-slop/no-unknown-parameters -- walks any shape, keeping only the kit's classes
+    const visit = (each) => {
+        // oxlint-disable-next-line anti-slop/no-runtime-typeof -- only objects can hold a Line or a Screen
+        if (typeof each !== "object" || each === null || seen.has(each)) {
+            return;
+        }
+        seen.add(each);
+        if (isLine(each)) {
+            drawn.push(each);
+            return;
+        }
+        if (isScreen(each)) {
+            screens.push(each);
+        }
+        const inner = each instanceof Map
+            ? [...each.keys(), ...each.values()]
+            : each instanceof Set || Array.isArray(each)
+                ? each
+                : Object.values(each);
+        for (const item of inner) {
+            visit(item);
+        }
+    };
+    visit(value);
+    return { lines: drawn, screens };
+}
+/** A pull in flight, or one the engine would start now: the page visible, the connection letting it, a pull owed. */
+function pullUnderWay(model, now) {
+    if (model.pull.kind === "pulling") {
+        return true;
+    }
+    const connection = model.connection.kind;
+    const canPull = model.visible && connection !== "signed-out" && connection !== "incompatible";
+    const owed = model.restart !== null ||
+        model.target > model.replica.revision ||
+        deriveHistoryState(model).kind === "partial";
+    // As the engine starts one: an owed pull right away unless it is in backoff, any pull once its retry is due.
+    const due = model.pull.nextAt <= now;
+    return canPull && (due || (owed && model.pull.failures === 0));
+}
+/** A push in flight, or one the engine would start now: the connection live or stale and no backoff pending. */
+function pushUnderWay(model, now) {
+    return (model.push.kind === "pushing" ||
+        (reachable(model.connection) && model.push.kind === "idle" && model.push.nextAt <= now));
+}
+function syncUnderWay(model, now) {
+    const unsent = model.commands.some((each) => each.state.kind !== "done");
+    return (model.boot.kind === "loading" ||
+        model.saved.writing > 0 ||
+        model.push.kind === "pushing" ||
+        pullUnderWay(model, now) ||
+        (unsent && pushUnderWay(model, now)));
+}
+function owedName(owed) {
+    if ("job" in owed) {
+        return `job ${owed.job.kind}:${owed.job.key}`;
+    }
+    return "command" in owed ? `command ${owed.command.client}:${owed.command.id}` : "sync";
+}
+function underWay(owed, world) {
+    if ("job" in owed) {
+        return (world.jobs ?? []).some((job) => job.kind === owed.job.kind &&
+            job.key === owed.job.key &&
+            (job.state === "running" || (job.state === "due" && Date.parse(job.dueAt) <= world.now)));
+    }
+    const model = world.model;
+    if (model === undefined) {
+        return false;
+    }
+    if ("command" in owed) {
+        const command = model.commands.find((each) => each.client === owed.command.client && each.id === owed.command.id);
+        if (command === undefined || settled(model, command)) {
+            return false;
+        }
+        const state = command.state;
+        // Sent or about to be: a push in flight or one the engine would start now. Answered or parked: a pull that brings it.
+        if (state.kind === "saving" || state.kind === "sending") {
+            return true;
+        }
+        return state.kind === "queued" || (state.kind === "unknown" && state.parked !== true)
+            ? pushUnderWay(model, world.now)
+            : pullUnderWay(model, world.now);
+    }
+    return syncUnderWay(model, world.now);
+}
+/**
+ * A reached view's problems by the kit's rules: every line has text with no broken value; every screen keeps its parts
+ * (`screenProblems`); and, given what is under way (`world`), every working line's owed work is: a job running or due by
+ * now, a command not yet settled, or the page's copy loading, pulling, pushing or writing. Takes any value an app's view
+ * returns and finds the kit's lines and screens in it. Run it on every view a test reaches, not only on samples.
+ */
+export function viewProblems(
+// oxlint-disable-next-line anti-slop/no-unknown-parameters -- an app's view is any shape; only the kit's classes are read
+shown, options) {
+    const problems = [];
+    const { lines: drawn, screens } = found(shown);
+    for (const line of drawn) {
+        if (line.text.trim() === "") {
+            problems.push(`${line.subject}.${line.kind}: no text`);
+        }
+        else if (BROKEN.test(line.text)) {
+            problems.push(`${line.subject}.${line.kind}: "${line.text}" shows a broken value`);
+        }
+        const world = options.world;
+        if (world !== undefined && line.owed !== null && !underWay(line.owed, world)) {
+            problems.push(`${line.subject}.${line.kind} says "${line.text}" (working) but owes ${owedName(line.owed)}, which isn't under way`);
+        }
+    }
+    for (const screen of screens) {
+        problems.push(...screenProblems(screen, screen.parts, options.order));
+    }
+    return problems;
+}
+/**
+ * Replay `messages` through `update` from `initial` and run `viewProblems` on the view of every model reached: the
+ * initial one (step 0) and the one after each message (step n). `world` is drawn from each reached model (its sync
+ * model, its clock, the jobs the test supplies), never a snapshot. Returns the last model, every effect, and the
+ * problems, each naming its step.
+ */
+export function replayProblems(update, initial, messages, options) {
+    const effects = [];
+    const problems = [];
+    let model = initial;
+    const check = (step) => {
+        const world = options.world?.(model, step);
+        problems.push(...viewProblems(options.view(model), {
+            order: options.order,
+            ...(world === undefined ? {} : { world }),
+        }).map((problem) => `step ${step}: ${problem}`));
+    };
+    check(0);
+    for (const [index, msg] of messages.entries()) {
+        const [next, produced] = update(model, msg);
+        model = next;
+        effects.push(...produced);
+        check(index + 1);
+    }
+    return { model, effects, problems };
+}

package/dist/web/replica.d.ts

@@ -19,6 +19,13 @@ export declare function indexedReplica<T, P, S, C, O>(options: {
         store: string;
         convert: (value: unknown) => Pending<C, O> | undefined;
     };
+    /**
+     * By table: an earlier release's row, as this one reads it (a field it added, given). A table whose row shape
+     * changed needs one, or the copy starts over; a table whose shape didn't is kept as it is. A lift may be handed any
+     * other release's row, a newer one's too (a rollback): it parses the shapes it knows strictly and throws on others,
+     * which starts the copy over.
+     */
+    lifts?: Readonly<Record<string, (row: unknown) => unknown>>;
 }): StoragePort<T, S, C, O> & {
     forget: () => Promise<void>;
 };

package/dist/web/replica.js

@@ -19,8 +19,13 @@
  *   so it never takes part of a history for all of it. A delta takes the patch's history only when the disk is at the
  *   cursor the patch moved from, and a complete copy at a revision is never narrowed to a partial one. The server's facts are not rows: facts an older page kept that no longer fit load as null and
  *   never cost the copy; facts that change at the same revision are written too.
- * - A version upgrade starts the replica over (it comes back from the server) but keeps the outbox and the generation:
- *   an intent on disk is never erased by an upgrade. One this version can't read stays on disk and is counted.
+ * - An upgrade keeps the outbox and the generation (an intent on disk is never erased; one this version can't read
+ *   stays on disk and is counted) and the copy where it can: a table whose row shape is unchanged is kept as it is, one
+ *   whose shape changed goes through the app's lift (`lifts`), a table that is new starts empty. The copy is rewritten
+ *   under this release's shapes in one transaction, kept only if every table then reads, keeps its dataset and
+ *   revision (taps made on it carry it) but no server facts, and is marked lifted: it loads owing a real reset, so the
+ *   first contact replaces it whole; offline it shows what it had. A changed table with no lift, a row that still
+ *   doesn't read, or a table this release doesn't have starts the copy over.
  * - A newer version opening in another tab closes this page's connection; its writes then fail as `Replaced`.
  *
  * `factory` is the browser's `indexedDB`, or any implementation of the same API (the headless simulation passes an
@@ -39,6 +44,18 @@ const partialSchema = object({
     held: count(),
     total: count(),
 });
+/** The replica tables a checkpoint's row shapes name (each line `"<name>":<shape>`). */
+function tablesOf(point) {
+    const written = typeof point === "object" && point !== null ? Reflect.get(point, "shapes") : undefined;
+    const found = new Set();
+    for (const line of typeof written === "string" ? written.split("\n") : []) {
+        const name = /^("(?:[^"\\]|\\.)*"):/.exec(line)?.[1];
+        if (name !== undefined) {
+            found.add(String(JSON.parse(name)));
+        }
+    }
+    return found;
+}
 export function indexedReplica(options) {
     const names = options.tables.map((lens) => lens.name);
     const shapes = options.tables.map((lens) => lens.shape).join("\n");
@@ -70,7 +87,10 @@ export function indexedReplica(options) {
             opening.onsuccess = () => resolve(opening.result);
             opening.onerror = () => reject(opening.error ?? new Error(`can't open ${options.name}`));
         });
-        const missing = [...names, "meta", "outbox"].some((name) => !probe.objectStoreNames.contains(name));
+        // A store missing, or one this release doesn't have (a table dropped without a bump): the upgrade runs while the
+        // checkpoint still names a dropped table, so its rows are deleted, never moved into the outbox.
+        const missing = [...names, "meta", "outbox"].some((name) => !probe.objectStoreNames.contains(name)) ||
+            [...probe.objectStoreNames].some((name) => !names.includes(name) && name !== "meta" && name !== "outbox");
         const current = probe.version;
         probe.close();
         if (current >= options.version && !missing) {
@@ -81,52 +101,62 @@ export function indexedReplica(options) {
     const open = () => {
         opened ??= layout()
             .then((version) => openDatabase(options.factory, options.name, version, (database, _from, upgrade) => {
-            // Replica tables start over (the server has them); every other store an earlier version kept may hold work
-            // nobody else has, so its records move into the outbox: converted by `legacy`, else kept as they were under
-            // their own key, counted unreadable. Nothing is dropped unseen.
+            // Replica tables stay for the load to lift; one the copy held that this release dropped goes (the server has
+            // it). Every other store an earlier version kept may hold work nobody else has, so its records move into the
+            // outbox: converted by `legacy`, else kept as they were under their own key, counted unreadable. Nothing is
+            // dropped unseen.
             const legacy = options.legacy;
             const own = new Set(["meta", "outbox"]);
             if (!database.objectStoreNames.contains("outbox")) {
                 database.createObjectStore("outbox");
             }
-            if (database.objectStoreNames.contains("meta")) {
-                upgrade.objectStore("meta").delete("checkpoint");
-            }
-            else {
+            const kept = database.objectStoreNames.contains("meta");
+            if (!kept) {
                 database.createObjectStore("meta");
             }
             const outbox = upgrade.objectStore("outbox");
             // A snapshot: deleting a store changes the live list being walked.
             const existing = [...database.objectStoreNames];
-            for (const name of existing) {
-                if (own.has(name)) {
-                    continue;
-                }
-                if (names.includes(name)) {
-                    database.deleteObjectStore(name);
-                    continue;
-                }
-                const cursor = upgrade.objectStore(name).openCursor();
-                let at = 0;
-                cursor.onsuccess = () => {
-                    const current = cursor.result;
-                    if (current === null) {
-                        database.deleteObjectStore(name);
-                        return;
+            const walk = (replicated) => {
+                for (const name of existing) {
+                    if (own.has(name) || names.includes(name)) {
+                        continue;
                     }
-                    const value = current.value;
-                    const command = legacy?.store === name ? legacy.convert(value) : undefined;
-                    if (command === undefined) {
-                        outbox.put(value, ["legacy", name, at++]);
-                    }
-                    else {
-                        outbox.put(command, [command.client, command.id]);
+                    if (replicated.has(name)) {
+                        database.deleteObjectStore(name);
+                        continue;
                     }
-                    current.continue();
-                };
+                    const cursor = upgrade.objectStore(name).openCursor();
+                    let at = 0;
+                    cursor.onsuccess = () => {
+                        const current = cursor.result;
+                        if (current === null) {
+                            database.deleteObjectStore(name);
+                            return;
+                        }
+                        const value = current.value;
+                        const command = legacy?.store === name ? legacy.convert(value) : undefined;
+                        if (command === undefined) {
+                            outbox.put(value, ["legacy", name, at++]);
+                        }
+                        else {
+                            outbox.put(command, [command.client, command.id]);
+                        }
+                        current.continue();
+                    };
+                }
+            };
+            if (kept) {
+                const point = upgrade.objectStore("meta").get("checkpoint");
+                point.onsuccess = () => walk(tablesOf(point.result));
+            }
+            else {
+                walk(new Set());
             }
             for (const name of names) {
-                database.createObjectStore(name);
+                if (!database.objectStoreNames.contains(name)) {
+                    database.createObjectStore(name);
+                }
             }
         }))
             .then((database) => {
@@ -183,7 +213,97 @@ export function indexedReplica(options) {
         }
         return at;
     };
-    return {
+    /**
+     * A copy an earlier release wrote under other row shapes, as this release reads it: each table whose shape changed
+     * through its lift, the others as they are, a table that is new empty. Undefined when any table can't be read.
+     */
+    const lift = (written, stored) => {
+        const partial = written.endsWith("\npartial");
+        const before = new Set((partial ? written.slice(0, -"\npartial".length) : written).split("\n"));
+        const prefixes = options.tables.map((lens) => `${JSON.stringify(lens.name)}:`);
+        // A table this release doesn't have: a newer release's copy, or a table renamed. Neither is this one's to lift.
+        if ([...before].some((line) => !prefixes.some((prefix) => line.startsWith(prefix)))) {
+            return undefined;
+        }
+        let restored = options.empty;
+        for (const [index, lens] of options.tables.entries()) {
+            const rows = stored[index] ?? [];
+            const prefix = prefixes[index] ?? "";
+            const each = options.lifts?.[lens.name];
+            let read;
+            if (before.has(lens.shape)) {
+                read = rows;
+            }
+            else if (![...before].some((line) => line.startsWith(prefix))) {
+                read = [];
+            }
+            else if (each === undefined) {
+                return undefined;
+            }
+            else {
+                try {
+                    read = rows.map((entry) => ({ key: entry.key, row: each(entry.row) }));
+                }
+                catch {
+                    return undefined;
+                }
+            }
+            restored = restored === undefined ? undefined : lens.restore(restored, read);
+        }
+        return restored === undefined ? undefined : { tables: restored, partial };
+    };
+    /**
+     * Write a lifted copy in place of the one it came from, in one transaction, if the disk still holds that one: its
+     * dataset and revision, marked lifted, under this release's shapes. False when the disk moved on or refused.
+     */
+    const rewrite = async (from, tables, history) => {
+        const database = await db();
+        const transaction = database.transaction([...names, "meta"], "readwrite");
+        const done = committed(transaction);
+        const meta = transaction.objectStore("meta");
+        try {
+            await guard(transaction);
+            const point = await request(meta.get("checkpoint"));
+            const still = typeof point === "object" &&
+                point !== null &&
+                Reflect.get(point, "dataset") === from.dataset &&
+                Reflect.get(point, "revision") === from.revision &&
+                Reflect.get(point, "shapes") === from.shapes;
+            if (!still) {
+                transaction.abort();
+                await done.catch(() => undefined);
+                return false;
+            }
+            for (const name of names) {
+                transaction.objectStore(name).clear();
+            }
+            for (const lens of options.tables) {
+                for (const op of lens.snapshot(tables)) {
+                    if (!("drop" in op)) {
+                        transaction.objectStore(op.table).put(op.row, op.key);
+                    }
+                }
+            }
+            const checkpoint = {
+                dataset: from.dataset,
+                revision: from.revision,
+                server: null,
+                lifted: true,
+                ...(history.kind === "partial" ? { history, shapes: partialShapes } : { shapes }),
+            };
+            meta.put(checkpoint, "checkpoint");
+            options.interfere?.("commit", transaction);
+            await done;
+            return true;
+        }
+        catch {
+            await done.catch(() => undefined);
+            return false;
+        }
+    };
+    /** A load that lost a lift to another tab reads again once: that tab's rewrite is readable now. */
+    let again = false;
+    const port = {
         async load() {
             const database = await db();
             const transaction = database.transaction([...names, "meta", "outbox"], "readonly");
@@ -221,6 +341,41 @@ export function indexedReplica(options) {
                 server: null,
                 history: COMPLETE,
             };
+            const written = typeof point === "object" && point !== null ? Reflect.get(point, "shapes") : undefined;
+            const held = typeof point === "object" && point !== null ? Reflect.get(point, "dataset") : null;
+            // Another release's copy, and this one can lift it: rewritten under this release's shapes, claiming no history.
+            if (typeof written === "string" && !ours(written) && typeof held === "string") {
+                const up = lift(written, tables);
+                const partial = check(partialSchema, Reflect.get(point ?? {}, "history"));
+                const history = up?.partial === true && partial.ok ? partial.value : COMPLETE;
+                const revision = typeof disk === "number" ? disk : 0;
+                const wrote = up !== undefined &&
+                    (!up.partial || partial.ok) &&
+                    (await rewrite({ dataset: held, revision, shapes: written }, up.tables, history));
+                if (up !== undefined && !wrote && !again) {
+                    again = true;
+                    try {
+                        return await port.load();
+                    }
+                    finally {
+                        again = false;
+                    }
+                }
+                if (up !== undefined && wrote) {
+                    loaded = {
+                        dataset: held,
+                        revision,
+                        shapes: history.kind === "partial" ? partialShapes : shapes,
+                    };
+                    discarded = null;
+                    return {
+                        replica: { dataset: held, revision, tables: up.tables, server: null, history },
+                        commands,
+                        unreadable,
+                        restart: "paged",
+                    };
+                }
+            }
             const at = checkpointOf(point);
             const raw = typeof point === "object" && point !== null ? Reflect.get(point, "dataset") : null;
             discarded = typeof raw === "string" && raw !== at.dataset ? raw : null;
@@ -249,6 +404,7 @@ export function indexedReplica(options) {
             if (restored === undefined || at.dataset === null) {
                 return { replica: empty, commands, unreadable };
             }
+            const lifted = typeof point === "object" && point !== null && Reflect.get(point, "lifted") === true;
             return {
                 replica: {
                     dataset: at.dataset,
@@ -259,6 +415,7 @@ export function indexedReplica(options) {
                 },
                 commands,
                 unreadable,
+                ...(lifted ? { restart: "paged" } : {}),
             };
         },
         async commit(commit) {
@@ -285,13 +442,22 @@ export function indexedReplica(options) {
                     throw new Replaced();
                 }
                 const stored = checkpointOf(point);
-                const plan = commitPlan(stored, {
-                    ...target,
-                    reset: commit.reset,
-                    base: commit.base,
-                    replaces: commit.replaces,
-                    ...(commit.from === undefined ? {} : { from: commit.from }),
-                });
+                const liftedOnDisk = typeof point === "object" && point !== null && Reflect.get(point, "lifted") === true;
+                // The reset a lifted copy owes (a pull since 0, at or past its revision) replaces it whole even when the
+                // server hasn't moved: the disk claims the revision, not the rows. A repair write (base at the revision) can't.
+                const plan = liftedOnDisk &&
+                    commit.reset &&
+                    commit.base === 0 &&
+                    held === commit.replica.dataset &&
+                    commit.replica.revision >= stored.revision
+                    ? "whole"
+                    : commitPlan(stored, {
+                        ...target,
+                        reset: commit.reset,
+                        base: commit.base,
+                        replaces: commit.replaces,
+                        ...(commit.from === undefined ? {} : { from: commit.from }),
+                    });
                 // The disk holds a dataset this page never saw: the server replaced the one this page is on.
                 if (plan === "stale") {
                     transaction.abort();
@@ -346,8 +512,12 @@ export function indexedReplica(options) {
                 const history = plan === "whole" || plan === "adopt"
                     ? (commit.replica.history ?? COMPLETE)
                     : stored.history;
+                // A lifted copy stays marked until a whole write replaces it: a delta or new facts don't make it trusted.
+                const lifted = plan !== "whole" && liftedOnDisk;
+                // A lifted copy keeps no server facts either until the reset: they'd vouch for rows the server never sent.
                 const checkpoint = {
                     ...target,
+                    ...(lifted ? { server: null, lifted: true } : {}),
                     ...(history.kind === "partial" ? { history, shapes: partialShapes } : { shapes }),
                 };
                 if (plan !== "skip" || facts) {
@@ -463,4 +633,5 @@ export function indexedReplica(options) {
             await done;
         },
     };
+    return port;
 }