slopkit 0.1.2 → 0.3.0

package/dist/conformance/paging-storage.test.js

@@ -0,0 +1,255 @@
+import { describe, expect, test } from "vitest";
+import { createClient, Replaced } from "../client.js";
+import { POLL_MS } from "../sync.js";
+import { copy, database, deferred, patch, pendingSchema, rowsLens, serverSchema, sync, values, } from "./fixtures.js";
+import { indexedReplica as legacyReplica } from "./legacy/replica-012.js";
+import { all, complete, firstPage, history, lastPage, middle, newest, oldest, paged, partialCopy, requestedPage, secondPage, } from "./paging-fixtures.js";
+function commit(replica, from, rows = [], reset = false) {
+    return {
+        replica,
+        from,
+        reset,
+        base: replica.revision,
+        replaces: "dataset-a",
+        ops: rows.map((item) => ({ table: "rows", key: item.id, row: item })),
+    };
+}
+const legacyRowsLens = {
+    ...rowsLens,
+    shape: '"rows":object(strict,{"id":string(0,100000),"value":number(true,-1.7976931348623157e+308,1.7976931348623157e+308),"deleted":boolean,"revision":number(true,0,1.7976931348623157e+308)})',
+};
+function oldPort(stored) {
+    return legacyReplica({
+        factory: stored.factory,
+        name: stored.name,
+        version: 1,
+        tables: [legacyRowsLens],
+        empty: { rows: new Map() },
+        server: serverSchema,
+        pending: pendingSchema,
+    });
+}
+function runtime(stored, network, name = "page-a", clock) {
+    return createClient({
+        sync,
+        client: name,
+        ports: {
+            storage: stored.port,
+            network,
+            now: () => clock?.now ?? 0,
+            fingerprint: async () => "fixture-fingerprint",
+        },
+    });
+}
+const failedPush = async () => ({
+    ok: false,
+    failure: { kind: "offline", message: "no commands in this history" },
+});
+const unavailable = {
+    ok: false,
+    failure: { kind: "offline", message: "download paused" },
+};
+describe("paged checkpoints", () => {
+    test("12: executor persists each page and a reload requests the stored cursor, not the first page", async () => {
+        const stored = database();
+        const firstRequests = [];
+        const first = runtime(stored, {
+            push: failedPush,
+            pull: async (request) => {
+                firstRequests.push(request);
+                return firstRequests.length === 1
+                    ? { ok: true, value: { patch: paged(10, newest, firstPage, true), receipts: [] } }
+                    : unavailable;
+            },
+        });
+        first.start();
+        await first.settle();
+        expect(first.faults).toStrictEqual([]);
+        expect(firstRequests[0]).toMatchObject({ since: 0, page: null });
+        const disk = await stored.port.load();
+        expect(history(disk.replica)).toStrictEqual(partialCopy().history);
+        expect(values(disk.replica)).toStrictEqual(values(copy(10, newest)));
+        const reopened = database({ factory: stored.factory, name: stored.name });
+        const requests = [];
+        const second = runtime(reopened, {
+            push: failedPush,
+            pull: async (request) => {
+                requests.push(request);
+                const cursor = requestedPage(request);
+                return {
+                    ok: true,
+                    value: {
+                        patch: cursor === firstPage.next
+                            ? paged(10, middle, secondPage)
+                            : cursor === secondPage.next
+                                ? paged(10, oldest, lastPage)
+                                : patch(10, [], false),
+                        receipts: [],
+                    },
+                };
+            },
+        }, "page-b");
+        second.start();
+        await second.settle();
+        expect(requests[0]).toMatchObject({ since: 10, page: firstPage.next, verify: false });
+        // Two remaining pages, at most one separate verification pull.
+        expect(requests.length).toBeLessThanOrEqual(3);
+        expect(values((await reopened.port.load()).replica)).toStrictEqual(values(copy(10, all)));
+        expect(history((await reopened.port.load()).replica)).toStrictEqual(complete);
+        expect(second.faults).toStrictEqual([]);
+    });
+    test("13: an already-open 0.1.2 tab is fenced by a newer partial checkpoint; reopening reads no partial rows", async () => {
+        const stored = database();
+        const old = oldPort(stored);
+        await old.load();
+        await old.commit(commit(copy(9, all), null, [], true));
+        const current = database({ factory: stored.factory, name: stored.name });
+        await current.port.load();
+        await current.port.commit(commit(partialCopy(10), null, [], true));
+        await expect(old.commit(commit(copy(11, all), null, [], true))).rejects.toBeInstanceOf(Replaced);
+        const reloaded = oldPort(stored);
+        const unread = await reloaded.load();
+        expect(unread.replica.dataset).toBeNull();
+        expect(values(unread.replica)).toStrictEqual([]);
+        // This is the old release's legacy whole pull, not a paged response.
+        const whole = patch(10, all, true);
+        await reloaded.commit(commit(copy(whole.revision, whole.rows.rows), null, [], true));
+        const modern = database({ factory: stored.factory, name: stored.name });
+        const restored = await modern.port.load();
+        expect(values(restored.replica)).toStrictEqual(values(copy(10, all)));
+        expect(history(restored.replica)).toStrictEqual(complete);
+    });
+    test("13 compatibility control: the released adapter reads and writes whole checkpoints unchanged", async () => {
+        const stored = database();
+        const old = oldPort(stored);
+        await old.load();
+        await old.commit(commit(copy(10, all), null, [], true));
+        const reopen = oldPort(stored);
+        expect(values((await reopen.load()).replica)).toStrictEqual(values(copy(10, all)));
+    });
+    test("13 compatibility control: a new complete checkpoint stays readable by the released adapter", async () => {
+        const stored = database();
+        await stored.port.load();
+        await stored.port.commit(commit(partialCopy(10, all, lastPage), null, [], true));
+        const old = await oldPort(stored).load();
+        expect(old.replica.dataset).toBe("dataset-a");
+        expect(old.replica.revision).toBe(10);
+        expect(values(old.replica)).toStrictEqual(values(copy(10, all)));
+    });
+    test("14: a native aborted same-revision page commit leaves a whole write owed and repairs the cursor", async () => {
+        let abort = false;
+        const stored = database({
+            interfere: (what, transaction) => {
+                if (what === "commit" && abort) {
+                    abort = false;
+                    transaction.abort();
+                }
+            },
+        });
+        await stored.port.load();
+        await stored.port.commit(commit(partialCopy(), null, [], true));
+        let replies = 0;
+        const clock = { now: 0 };
+        const app = runtime(stored, {
+            push: failedPush,
+            pull: async () => {
+                replies++;
+                if (replies === 1) {
+                    abort = true;
+                    return { ok: true, value: { patch: paged(10, middle, secondPage), receipts: [] } };
+                }
+                return unavailable;
+            },
+        }, "page-a", clock);
+        app.start();
+        await app.settle();
+        expect(app.model().storage.kind).toBe("failed");
+        expect(history((await stored.port.load()).replica)).toStrictEqual(partialCopy().history);
+        clock.now = POLL_MS + 1;
+        app.dispatch({ type: "tick", now: clock.now });
+        await app.settle();
+        const recovered = await stored.port.load();
+        expect(values(recovered.replica)).toStrictEqual(values(copy(10, [...newest, ...middle])));
+        expect(history(recovered.replica)).toStrictEqual(partialCopy(10, [], secondPage).history);
+        expect(app.faults).toStrictEqual([]);
+    });
+    test("15: different cursors at one revision never advance disk coverage without its rows", async () => {
+        const first = database();
+        const second = database({ factory: first.factory, name: first.name });
+        await Promise.all([first.port.load(), second.port.load()]);
+        await first.port.commit(commit(partialCopy(), null, [], true));
+        await second.port.commit(commit(partialCopy(10, [...newest, ...middle], secondPage), firstPage.next, middle));
+        const disk = await database({ factory: first.factory, name: first.name }).port.load();
+        expect(values(disk.replica)).toStrictEqual(values(copy(10, [...newest, ...middle])));
+        expect(history(disk.replica)).toStrictEqual(partialCopy(10, [], secondPage).history);
+        // A stale page arriving after completion may not narrow durable coverage.
+        await second.port.commit(commit(partialCopy(10, all, lastPage), secondPage.next, oldest));
+        await first.port.commit(commit(partialCopy(), null, [], true));
+        const finished = await database({ factory: first.factory, name: first.name }).port.load();
+        expect(values(finished.replica)).toStrictEqual(values(copy(10, all)));
+        expect(history(finished.replica)).toStrictEqual(complete);
+    });
+    test("15: two production clients at different cursors and revisions do not ping-pong whole writes", async () => {
+        const stored = database();
+        await stored.port.load();
+        await stored.port.commit(commit(partialCopy(), null, [], true));
+        const tab = database({ factory: stored.factory, name: stored.name });
+        const enter = deferred();
+        const release = deferred();
+        let firstCalls = 0;
+        let writes = 0;
+        const original = stored.port.commit;
+        stored.port.commit = async (value) => {
+            writes++;
+            if (writes > 8) {
+                throw new Error("busy whole-write loop");
+            }
+            return await original(value);
+        };
+        const first = runtime(stored, {
+            push: failedPush,
+            pull: async () => {
+                firstCalls++;
+                if (firstCalls === 1) {
+                    enter.resolve();
+                    await release.promise;
+                    return { ok: true, value: { patch: paged(10, middle, secondPage), receipts: [] } };
+                }
+                if (firstCalls === 2) {
+                    return { ok: true, value: { patch: paged(11, oldest, lastPage), receipts: [] } };
+                }
+                return { ok: true, value: { patch: patch(11, []), receipts: [] } };
+            },
+        });
+        first.start();
+        await enter.promise;
+        let secondCalls = 0;
+        const second = runtime(tab, {
+            push: failedPush,
+            pull: async () => {
+                secondCalls++;
+                return {
+                    ok: true,
+                    value: {
+                        patch: secondCalls === 1
+                            ? paged(11, [...middle, ...oldest], { ...lastPage, from: firstPage.next })
+                            : patch(11, []),
+                        receipts: [],
+                    },
+                };
+            },
+        }, "page-b");
+        second.start();
+        await second.settle();
+        release.resolve();
+        await first.settle();
+        expect(writes).toBeLessThanOrEqual(3);
+        for (const app of [first, second]) {
+            expect(history(app.model().replica)).toStrictEqual(complete);
+            expect(app.faults).toStrictEqual([]);
+        }
+        const disk = await database({ factory: stored.factory, name: stored.name }).port.load();
+        expect(values(disk.replica)).toStrictEqual(values(copy(11, all)));
+        expect(history(disk.replica)).toStrictEqual(complete);
+    });
+});

package/dist/conformance/paging.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/conformance/paging.test.js

@@ -0,0 +1,209 @@
+import { describe, expect, test } from "vitest";
+import { POLL_MS, settled } from "../sync.js";
+import { copy, patch, pending, row, sync, values } from "./fixtures.js";
+import { all, assertCopy, complete, deliver, firstPage, history, lastPage, loaded, middle, newest, oldest, paged, partialCopy, Reference, secondPage, } from "./paging-fixtures.js";
+describe("paged first copy", () => {
+    test("1, 2, 5: edited and tombstoned anchors, changes on both sides, duplicates and late pages converge", () => {
+        const reference = new Reference(10, all);
+        let model = loaded();
+        [model] = deliver(model, paged(10, newest, firstPage, true), 0);
+        assertCopy(model, reference);
+        const changes = [
+            row("f", 65, 11),
+            row("e", 15, 11),
+            row("z", 70, 11),
+            row("a", 5, 11),
+            row("b", 20, 11, true),
+        ];
+        reference.write(11, changes);
+        const page = { ...secondPage, held: 4, total: 6 };
+        const reply = paged(11, [...changes, ...middle], page);
+        [model] = deliver(model, reply, 10);
+        assertCopy(model, reference);
+        [model] = deliver(model, reply, 10);
+        assertCopy(model, reference);
+        expect(history(model.replica)).toMatchObject({ next: secondPage.next, through: "30" });
+        const deletion = row("c", 30, 12, true);
+        reference.write(12, [deletion]);
+        [model] = deliver(model, paged(12, [deletion], {
+            ...page,
+            from: secondPage.next,
+            next: secondPage.next,
+            held: 3,
+            total: 5,
+        }), 11);
+        assertCopy(model, reference);
+        [model] = deliver(model, reply, 10);
+        assertCopy(model, reference);
+        expect(model.replica.tables.rows.has("c")).toBe(false);
+        [model] = deliver(model, paged(12, [row("e", 15, 11), row("a", 5, 11)], { ...lastPage, held: 5, total: 5 }), 12);
+        assertCopy(model, reference);
+        expect(history(model.replica)).toStrictEqual(complete);
+        expect(values(model.replica)).toStrictEqual([
+            ["a", 5, 11],
+            ["d", 40, 10],
+            ["e", 15, 11],
+            ["f", 65, 11],
+            ["z", 70, 11],
+        ]);
+    });
+    test("2: equal-revision pages merge; an unexpected from applies current rows but cannot advance the cursor", () => {
+        let model = loaded(partialCopy());
+        [model] = deliver(model, paged(10, middle, { ...secondPage, from: "another-cursor" }));
+        expect(values(model.replica)).toHaveLength(4);
+        expect(history(model.replica)).toStrictEqual(partialCopy().history);
+        [model] = deliver(model, paged(10, middle, secondPage));
+        expect(history(model.replica)).toStrictEqual(partialCopy(10, [], secondPage).history);
+        [model] = deliver(model, paged(10, oldest, lastPage));
+        assertCopy(model, new Reference(10, all));
+    });
+    test("3: a dataset reset starts its own history and refuses old-dataset intents", () => {
+        let model = loaded(partialCopy());
+        model = {
+            ...model,
+            commands: [
+                {
+                    ...pending("page-a", 1),
+                    dataset: "dataset-a",
+                    state: { kind: "unknown", failure: "lost" },
+                },
+            ],
+        };
+        const replacement = [row("new-dataset", 900, 1)];
+        [model] = deliver(model, paged(1, replacement, { from: null, next: "b:900", through: "900", held: 1, total: 2 }, true, "dataset-b"));
+        expect(values(model.replica)).toStrictEqual([["new-dataset", 900, 1]]);
+        expect(history(model.replica)).toMatchObject({ kind: "partial", next: "b:900" });
+        expect(model.commands[0]?.state).toMatchObject({ kind: "done", outcome: { kind: "failed" } });
+    });
+    test("4: after a verify mismatch an in-flight delta cannot become a copy with a gap", () => {
+        const wholeCopy = { ...copy(10, all), history: complete };
+        let model = loaded(wholeCopy);
+        [model] = sync.update(model, {
+            type: "checked",
+            dataset: "dataset-a",
+            revision: 10,
+            match: false,
+            now: 1,
+        });
+        expect(model.replica.revision).toBe(0);
+        const [after, effects] = deliver(model, patch(11, [row("new", 80, 11)]), 10);
+        expect(after.replica.revision).toBe(0);
+        expect(values(after.replica)).toStrictEqual([]);
+        // The outstanding reset is sufficient; no second concurrent pull is required.
+        expect((after.pull.kind === "pulling" && after.pull.since === 0) ||
+            effects.some((effect) => effect.kind === "pull" && effect.since === 0)).toBe(true);
+    });
+    test("4: a same-dataset gap and a non-reset foreign dataset are dropped until a real reset", () => {
+        for (const reply of [
+            patch(12, [row("gap", 1, 12)]),
+            patch(1, [row("foreign", 2, 1)], false, "dataset-b"),
+        ]) {
+            const model = loaded(partialCopy());
+            const [after] = deliver(model, reply, 11);
+            expect(values(after.replica)).toStrictEqual(values(model.replica));
+            expect(after.replica.dataset).toBe("dataset-a");
+            expect(after.replica.revision).toBe(10);
+        }
+    });
+    test("6: one round trip exposes newest rows and the exact partial-history claim", () => {
+        const [model] = deliver(loaded(), paged(10, newest, firstPage, true), 0);
+        expect(values(model.replica)).toStrictEqual([
+            ["e", 50, 10],
+            ["f", 60, 10],
+        ]);
+        expect(history(model.replica)).toStrictEqual(partialCopy().history);
+    });
+    test("7: partial history owes back-to-back pulls carrying next, no-progress backs off and eventually resumes", () => {
+        let [model, effects] = deliver(loaded(), paged(10, newest, firstPage, true), 0);
+        expect(effects.filter((effect) => effect.kind === "pull")).toMatchObject([
+            { since: 10, page: firstPage.next, verify: false },
+        ]);
+        [model, effects] = deliver(model, paged(10, newest, firstPage, true), 10);
+        expect(effects.filter((effect) => effect.kind === "pull")).toStrictEqual([]);
+        expect(model.pull).toMatchObject({ kind: "idle", failures: 1 });
+        [model, effects] = sync.update(model, { type: "tick", now: model.now + POLL_MS - 1 });
+        expect(effects.filter((effect) => effect.kind === "pull")).toStrictEqual([]);
+        [model, effects] = sync.update(model, { type: "tick", now: model.now + 1 });
+        expect(effects.filter((effect) => effect.kind === "pull")).toMatchObject([
+            { page: firstPage.next },
+        ]);
+        [model, effects] = deliver(model, paged(10, newest, firstPage, true), 10);
+        expect(effects.filter((effect) => effect.kind === "pull")).toStrictEqual([]);
+        expect(model.pull).toMatchObject({
+            kind: "idle",
+            failures: 2,
+            nextAt: model.now + 2 * POLL_MS,
+        });
+        [model, effects] = sync.update(model, { type: "tick", now: model.now + 2 * POLL_MS });
+        expect(effects.filter((effect) => effect.kind === "pull")).toMatchObject([
+            { page: firstPage.next },
+        ]);
+        [model, effects] = deliver(model, paged(10, middle, secondPage));
+        expect(effects.filter((effect) => effect.kind === "pull")).toMatchObject([
+            { page: secondPage.next },
+        ]);
+        [model] = deliver(model, paged(10, oldest, lastPage));
+        expect(history(model.replica)).toStrictEqual(complete);
+        assertCopy(model, new Reference(10, all));
+    });
+    test("8: verification is postponed on partial history, then requested when complete", () => {
+        const [, firstEffects] = deliver(loaded(), paged(10, newest, firstPage, true), 0);
+        expect(firstEffects.filter((effect) => effect.kind === "check")).toStrictEqual([]);
+        const [model, effects] = sync.update(sync.initial("page-a", 0), {
+            type: "loaded",
+            replica: partialCopy(),
+            commands: [],
+            unreadable: 0,
+            now: 0,
+        });
+        expect(effects.filter((effect) => effect.kind === "pull")).toMatchObject([
+            { verify: false, page: firstPage.next },
+        ]);
+        const [middleModel, during] = deliver(model, paged(10, middle, secondPage));
+        expect(during.filter((effect) => effect.kind === "check")).toStrictEqual([]);
+        const [, finished] = deliver(middleModel, paged(10, oldest, lastPage));
+        expect(finished.some((effect) => effect.kind === "check" || (effect.kind === "pull" && effect.verify))).toBe(true);
+    });
+    test("9: an old server cannot certify partial history; the recovery request omits page and asks since zero", () => {
+        const [model, effects] = deliver(loaded(partialCopy()), patch(11, [row("f", 61, 11)]), 10);
+        expect(history(model.replica)).toStrictEqual(partialCopy().history);
+        const pulls = effects.filter((effect) => effect.kind === "pull");
+        expect(pulls).toHaveLength(1);
+        expect(pulls[0]).toMatchObject({ since: 0 });
+        expect(Object.hasOwn(pulls[0] ?? {}, "page")).toBe(false);
+        const [whole] = deliver(model, patch(11, [row("f", 61, 11), ...all.filter((item) => item.id !== "f")], true), 0);
+        expect(history(whole.replica)).toStrictEqual(complete);
+        expect(whole.replica.tables.rows.size).toBe(6);
+    });
+    test("10: a delta below through never deepens the coverage claim", () => {
+        const [model] = deliver(loaded(partialCopy()), paged(11, [row("a", 5, 11)], { ...firstPage, from: "unrelated", held: 2, total: 6 }));
+        expect(model.replica.tables.rows.get("a")).toStrictEqual(row("a", 5, 11));
+        expect(history(model.replica)).toStrictEqual(partialCopy().history);
+    });
+    test("11: a paged reset that settles a command holds the row named by its receipt", () => {
+        const model = { ...loaded(), commands: [pending("page-a", 1)] };
+        const effectRow = row("effect-below-page", 1, 10);
+        const [after] = sync.update(model, {
+            type: "pulled",
+            since: 0,
+            patch: paged(10, [...newest, effectRow], { ...firstPage, total: 7 }, true),
+            receipts: [{ id: 1, revision: 10, outcome: { kind: "ok", resource: effectRow.id } }],
+            now: 1,
+        });
+        const command = after.commands[0];
+        expect(command).toBeDefined();
+        if (command === undefined) {
+            throw new Error("missing command");
+        }
+        expect(settled(after, command)).toBe(true);
+        expect(after.replica.tables.rows.get(effectRow.id)).toStrictEqual(effectRow);
+        expect(history(after.replica)).toStrictEqual({ ...partialCopy().history, total: 7 });
+    });
+    test("compatibility: a whole legacy reply is a complete copy and late replies stay harmless", () => {
+        let model = loaded();
+        [model] = deliver(model, patch(10, all, true), 0);
+        expect(values(model.replica)).toStrictEqual(values(copy(10, all)));
+        [model] = deliver(model, patch(9, [row("f", 999, 9)], true), 0);
+        expect(values(model.replica)).toStrictEqual(values(copy(10, all)));
+    });
+});

package/dist/conformance/pull.d.ts

@@ -0,0 +1,74 @@
+/** Synthetic rows an application's adapter maps to its own paged and unpaged tables. Order is (value, id), descending. */
+export type PullRow = {
+    id: string;
+    value: number;
+    revision: number;
+    deleted: boolean;
+};
+export type PullWrite = {
+    rows: Omit<PullRow, "revision">[];
+    unpaged?: Omit<PullRow, "revision">[];
+    receipt?: {
+        client: string;
+        id: number;
+        resource: string;
+    };
+};
+export type PullRequest = {
+    client: string;
+    dataset: string | null;
+    since: number;
+    verify: boolean;
+    page?: string | null;
+};
+export type PullPage = {
+    from: string | null;
+    next: string | null;
+    through: string | null;
+    held: number;
+    total: number;
+};
+export type PullReply = {
+    patch: {
+        dataset: string;
+        revision: number;
+        reset: boolean;
+        rows: {
+            rows: PullRow[];
+            unpaged: PullRow[];
+        };
+        page?: PullPage;
+    };
+    receipts: {
+        id: number;
+        revision: number;
+        outcome: {
+            resource: string;
+        };
+    }[];
+    /** Exact encoded response size, before normalization by the application's adapter. */
+    bytes: number;
+};
+export type PullLimits = {
+    firstRows: number;
+    pageRows: number;
+    maxRows: number;
+    maxBytes: number;
+};
+export type PullFixture = {
+    pull: (request: PullRequest) => Promise<PullReply>;
+    /** One committed transaction, including any receipt. The adapter stamps the returned revision on all written rows. */
+    write: (changes: PullWrite) => Promise<{
+        dataset: string;
+        revision: number;
+    }>;
+    /** A new dataset, with no rows, receipts or usable cursors from the previous dataset. */
+    replace: () => Promise<void>;
+    close: () => Promise<void>;
+};
+export type PullConformance = {
+    name: string;
+    create: (limits: PullLimits) => Promise<PullFixture>;
+};
+/** Registers black-box pull requirements against a fresh real application store for each case. */
+export declare function pullConformance(input: PullConformance): void;