slopkit 0.1.1 → 0.2.0

package/dist/sync.js

@@ -1,5 +1,93 @@
 import { unreachable } from "./machine.js";
 import { check } from "./schema.js";
+/** A key a chunk removed, in a table's layer of changes. */
+const GONE = Symbol("gone");
+/**
+ * A table's rows after a chunk, without copying the rows it already had: the chunk's changes are a layer over them, so
+ * merging a chunk costs its own size, not the table's. Layers merge when a newer one is as big as the one below it, and
+ * fold into a plain map once the top one holds half as many keys as it, so lookups pass a logarithmic number of layers
+ * and a changed row is copied a logarithmic number of times. Immutable: a published value never changes.
+ */
+class Layered {
+    size;
+    base;
+    /** Newest first. */
+    layers;
+    constructor(base, layers, size) {
+        this.base = base;
+        this.layers = layers;
+        this.size = size;
+    }
+    find(key) {
+        for (const each of this.layers) {
+            if (each.has(key)) {
+                return each.get(key);
+            }
+        }
+        return this.base.get(key);
+    }
+    get(key) {
+        const value = this.find(key);
+        return value === GONE ? undefined : value;
+    }
+    // Rows are never undefined.
+    has(key) {
+        return this.get(key) !== undefined;
+    }
+    // The base's keys in place, then new keys in the order they first came.
+    *entries() {
+        for (const key of this.base.keys()) {
+            const value = this.get(key);
+            if (value !== undefined) {
+                yield [key, value];
+            }
+        }
+        const seen = new Set();
+        for (let at = this.layers.length - 1; at >= 0; at--) {
+            for (const key of this.layers[at]?.keys() ?? []) {
+                const value = this.get(key);
+                if (!seen.has(key) && !this.base.has(key) && value !== undefined) {
+                    seen.add(key);
+                    yield [key, value];
+                }
+            }
+        }
+    }
+    *keys() {
+        for (const [key] of this.entries()) {
+            yield key;
+        }
+    }
+    *values() {
+        for (const [, value] of this.entries()) {
+            yield value;
+        }
+    }
+    [Symbol.iterator]() {
+        return this.entries();
+    }
+    // oxlint-disable-next-line anti-slop/no-unknown-parameters -- Map's receiver is whatever the caller passes
+    forEach(each, receiver) {
+        for (const [key, value] of this.entries()) {
+            each.call(receiver, value, key, this);
+        }
+    }
+}
+/** `rows` with `changes` over it, as a new value; `rows` is left as it was. */
+function overlay(rows, changes) {
+    let size = rows.size;
+    for (const [key, value] of changes) {
+        size += (value === GONE ? 0 : 1) - (rows.has(key) ? 1 : 0);
+    }
+    const base = rows instanceof Layered ? rows.base : rows;
+    const below = rows instanceof Layered ? [...rows.layers] : [];
+    let top = changes;
+    while (below[0] !== undefined && below[0].size <= top.size) {
+        top = new Map([...(below.shift() ?? []), ...top]);
+    }
+    const layered = new Layered(base, [top, ...below], size);
+    return top.size * 2 < base.size ? layered : new Map(layered);
+}
 export function table(def) {
     const gone = (row) => def.gone?.(row) === true;
     return {
@@ -15,16 +103,11 @@ export function table(def) {
             if (changed.length === 0 && !reset) {
                 return tables;
             }
-            const next = new Map(reset ? [] : def.get(tables));
+            const changes = new Map();
             for (const row of changed) {
-                if (gone(row)) {
-                    next.delete(def.key(row));
-                }
-                else {
-                    next.set(def.key(row), row);
-                }
+                changes.set(def.key(row), gone(row) ? GONE : row);
             }
-            return def.set(tables, next);
+            return def.set(tables, overlay(reset ? new Map() : def.get(tables), changes));
         },
         snapshot: (tables) => [...def.get(tables).entries()].map(([key, row]) => ({ table: def.name, key, row })),
         restore: (tables, values) => {
@@ -40,6 +123,18 @@ export function table(def) {
         },
     };
 }
+const COMPLETE = { kind: "complete" };
+const historyOf = (replica) => replica.history ?? COMPLETE;
+/** The history a page leaves: complete without one (an old server's whole copy) or once no older page is left. */
+const covered = (page) => page === undefined || page.next === null
+    ? COMPLETE
+    : {
+        kind: "partial",
+        next: page.next,
+        through: page.through,
+        held: page.held,
+        total: page.total,
+    };
 /** The socket must be heard this often (the server pokes every 20 s); quieter, the page polls. */
 export const SOCKET_DEADLINE_MS = 45_000;
 /** Polling while the socket can't be trusted or work is in flight, and the first retry after a failure. */
@@ -47,40 +142,96 @@ export const POLL_MS = 4000;
 const BACKOFF_MAX_MS = 30_000;
 /** Storage gets this long to load, or to keep an intent, before the page goes on without it. */
 export const STORAGE_DEADLINE_MS = 3000;
+/** Commands per push unless the spec says otherwise; the rest wait for the next push. */
+export const PUSH_MAX = 50;
 /** Settled commands stay drawn this long. */
 export const DONE_KEEP_MS = 60_000;
 /** Whether the server can be asked now. */
 export function reachable(connection) {
     return connection.kind === "live" || connection.kind === "stale";
 }
-/** Whether the replica holds this command's effects. */
+/**
+ * Whether the replica holds this command's effects: it is done, and the copy is at its revision and either complete or
+ * holding the rows its receipt names (a pull reply carried them). One that failed on the page has none.
+ */
 export function settled(model, pending) {
-    return pending.state.kind === "done" && model.replica.revision >= pending.state.revision;
+    return (pending.state.kind === "done" &&
+        (pending.state.revision === 0 ||
+            (model.replica.revision >= pending.state.revision &&
+                (historyOf(model.replica).kind === "complete" ||
+                    model.carried.has(`${pending.client}:${pending.id}`)))));
 }
-/** Whether what the page shows is also on the disk. */
+/**
+ * Whether what the page shows is also on the disk: by revision only, so tabs at different cursors never chase each
+ * other's writes. A failed write is owed again even at the same revision (a page that failed to land).
+ */
 export function saved(model) {
-    return (model.saved.dataset === model.replica.dataset && model.saved.revision >= model.replica.revision);
+    return (model.saved.dataset === model.replica.dataset &&
+        model.saved.revision >= model.replica.revision &&
+        !model.saved.owed);
 }
 /**
  * What a commit does to the disk, decided inside the disk's own transaction from what it holds: nothing when the disk
- * already has this or newer (another tab got there), the patch alone when the disk is at least at its base, else the
- * whole replica, so a skipped or failed patch is never covered by a later checkpoint. A page may replace the disk's
+ * already has this or newer (another tab got there; a complete copy at the same revision is newer than any page), the
+ * patch alone when the disk is at least at its base, else the whole replica, so a skipped or failed patch is never
+ * covered by a later checkpoint. A delta keeps the disk's own history, and takes the patch's only when the disk is at
+ * the cursor the patch moved from (`adopt`): coverage never lands without its rows. A page may replace the disk's
  * dataset only with one it moved to from that very dataset (`replaces`): a tab still on a dataset the server has since
  * replaced is `stale` and never writes it back.
  */
 export function commitPlan(stored, commit) {
     const same = stored.dataset === commit.dataset;
+    const disk = stored.history ?? COMPLETE;
     if (!same && stored.dataset !== null && stored.dataset !== commit.replaces) {
         return "stale";
     }
-    if (same && stored.revision >= commit.revision) {
+    if (same &&
+        (stored.revision > commit.revision ||
+            (stored.revision === commit.revision && disk.kind === "complete"))) {
         return "skip";
     }
     if (!commit.reset && same && stored.revision >= commit.base) {
-        return "delta";
+        return disk.kind === "partial" && disk.next === commit.from ? "adopt" : "delta";
     }
     return "whole";
 }
+/**
+ * The copy's history: loading while storage loads, none without a dataset, else partial or complete. Partial is a claim
+ * beside the numbers ("history from <through>, loading older · held of total"), and totals cover only `through` and
+ * above. Complete says nothing about storage: that is `deriveDurabilityState`, its own subject.
+ */
+export function deriveHistoryState(model) {
+    if (model.boot.kind === "loading") {
+        return { kind: "loading" };
+    }
+    if (model.replica.dataset === null) {
+        return { kind: "none" };
+    }
+    const history = historyOf(model.replica);
+    return history.kind === "complete"
+        ? history
+        : { kind: "partial", through: history.through, held: history.held, total: history.total };
+}
+/**
+ * The copy's durability at the model's clock: loading, replaced by another tab, a failing store (lost while the page
+ * shows more than the disk holds), unsaved past the storage deadline, commands kept that this version can't read, or ok.
+ */
+export function deriveDurabilityState(model) {
+    const behind = model.saved.behindSince;
+    if (model.boot.kind === "loading") {
+        return { kind: "loading" };
+    }
+    if (model.storage.kind === "replaced") {
+        return { kind: "replaced" };
+    }
+    if (model.storage.kind === "failed") {
+        return { kind: behind === null ? "failing" : "lost", reason: model.storage.reason };
+    }
+    if (behind !== null && model.now - behind >= STORAGE_DEADLINE_MS) {
+        return { kind: "unsaved" };
+    }
+    return model.unreadable > 0 ? { kind: "unreadable", count: model.unreadable } : { kind: "ok" };
+}
 /**
  * Storage after a write that worked: a failure that only a keep or a drop had (the replica's own writes aren't
  * failing) is over; a replaced copy stays replaced.
@@ -114,11 +265,16 @@ function failedConnection(connection, failure) {
     }
 }
 export function createSync(spec) {
+    const pushMax = spec.pushMax ?? PUSH_MAX;
+    if (!Number.isSafeInteger(pushMax) || pushMax < 1) {
+        throw new Error(`sync: pushMax must be a positive integer, got ${pushMax}`);
+    }
     const emptyReplica = () => ({
         dataset: null,
         revision: 0,
         tables: spec.empty,
         server: null,
+        history: COMPLETE,
     });
     function initial(client, now) {
         return {
@@ -134,14 +290,17 @@ export function createSync(spec) {
                 writing: 0,
                 failures: 0,
                 nextAt: now,
+                owed: false,
             },
             target: 0,
+            restart: null,
             reconciledAt: null,
             connection: { kind: "starting" },
             socket: { open: false, heardAt: 0 },
             pull: { kind: "idle", nextAt: now, failures: 0 },
             push: { kind: "idle", nextAt: now, failures: 0 },
             commands: [],
+            carried: new Set(),
             unreadable: 0,
             visible: true,
             verify: { kind: "pending", tries: 0 },
@@ -149,22 +308,37 @@ export function createSync(spec) {
         };
     }
     /**
-     * The one merge: a patch applied to a replica, with its disk operations, or undefined when it is older than the
-     * replica (a late reply). A reset or another dataset replaces everything.
+     * The one merge: a patch applied to a replica, with its disk operations; undefined when it is older than the replica
+     * (a late reply), `gap` when it can't follow from it. Only a reset (or the answer to `since` 0, which holds
+     * everything) replaces the copy, with its own history. A delta applies only to the same dataset with
+     * `since ≤ copy.revision ≤ reply.revision` (equal only with a page or new server facts); it moves the cursor only when
+     * it answers the copy's own `next`, and a reply without a page never changes the history.
      */
-    function merge(replica, patch) {
-        const reset = patch.reset || patch.dataset !== replica.dataset;
-        if (!reset && patch.revision <= replica.revision) {
-            // Nothing newer, but the server's own facts (version, settings) may have moved.
-            if (patch.revision === replica.revision &&
-                JSON.stringify(patch.server) !== JSON.stringify(replica.server)) {
-                return { replica: { ...replica, server: patch.server }, ops: [], reset: false };
+    function merge(replica, patch, since) {
+        const reset = patch.reset || since === 0;
+        const page = patch.page;
+        const was = historyOf(replica);
+        let history = was;
+        if (reset) {
+            // A reset older than what we hold of the same dataset is a late reply.
+            if (patch.dataset === replica.dataset && patch.revision < replica.revision) {
+                return undefined;
             }
-            return undefined;
+            history = covered(page);
+        }
+        else if (patch.dataset !== replica.dataset || since > replica.revision) {
+            return "gap";
         }
-        // A reset older than what we hold of the same dataset is a late reply.
-        if (reset && patch.dataset === replica.dataset && patch.revision < replica.revision) {
-            return undefined;
+        else if (patch.revision < replica.revision ||
+            (patch.revision === replica.revision && (page === undefined || was.kind === "complete"))) {
+            // Nothing newer, but the server's own facts (version, settings) may have moved.
+            return patch.revision === replica.revision &&
+                JSON.stringify(patch.server) !== JSON.stringify(replica.server)
+                ? { replica: { ...replica, server: patch.server }, ops: [], reset: false }
+                : undefined;
+        }
+        else if (page !== undefined && was.kind === "partial" && page.from === was.next) {
+            history = covered(page);
         }
         let tables = reset ? spec.empty : replica.tables;
         const ops = [];
@@ -173,7 +347,13 @@ export function createSync(spec) {
             ops.push(...lens.ops(patch.rows));
         }
         return {
-            replica: { dataset: patch.dataset, revision: patch.revision, server: patch.server, tables },
+            replica: {
+                dataset: patch.dataset,
+                revision: patch.revision,
+                server: patch.server,
+                tables,
+                history,
+            },
             ops,
             reset,
         };
@@ -190,19 +370,36 @@ export function createSync(spec) {
             const behind = next.replica.revision < next.target;
             const unsettled = next.commands.some((each) => each.state.kind === "done" && !settled(next, each));
             const polling = next.connection.kind !== "live" || unsettled || (spec.outstanding?.(next.replica) ?? false);
-            const verifying = next.verify.kind === "pending";
-            if (((behind || verifying) && (failures === 0 || due)) || (polling && due)) {
+            const history = historyOf(next.replica);
+            const whole = next.restart === "whole";
+            // Only a complete copy is checked: a fingerprint of everything can't certify part of it.
+            const verifying = next.verify.kind === "pending" && (history.kind === "complete" || whole);
+            // A partial copy owes its next page, back to back, until it is complete.
+            const owed = behind || verifying || history.kind === "partial" || next.restart !== null;
+            const since = next.restart === null ? next.replica.revision : 0;
+            if ((owed && (failures === 0 || due)) || (polling && due)) {
+                const current = next;
+                // Commands waiting on an outcome or on their effects, oldest first, at most a push's worth: their receipts
+                // come back with the rows they name; the rest in the next pull.
+                const commands = current.commands
+                    .filter((each) => each.state.kind === "sending" ||
+                    each.state.kind === "unknown" ||
+                    (each.state.kind === "done" && !settled(current, each)))
+                    .slice(0, pushMax)
+                    .map((each) => ({ client: each.client, id: each.id }));
                 out.push({
                     kind: "pull",
                     client: next.client,
                     dataset: next.replica.dataset,
-                    since: next.replica.revision,
+                    since,
                     verify: verifying,
+                    ...(whole ? {} : { page: history.kind === "partial" && since > 0 ? history.next : null }),
+                    commands,
                 });
                 next = {
                     ...next,
-                    pull: { kind: "pulling", since: next.replica.revision, failures },
-                    verify: next.verify.kind === "pending"
+                    pull: { kind: "pulling", since, failures },
+                    verify: verifying && next.verify.kind === "pending"
                         ? { kind: "requested", tries: next.verify.tries }
                         : next.verify,
                 };
@@ -212,12 +409,14 @@ export function createSync(spec) {
             const first = next.commands.find((each) => each.state.kind === "queued" || each.state.kind === "unknown");
             if (first !== undefined) {
                 // One push per client, dataset and tapped-against revision: the server judges each by what it was made on.
+                // At most `pushMax`, oldest first; the rest go in the next push.
                 const tapped = (each) => each.since ?? next.replica.revision;
                 const ids = new Set(next.commands
                     .filter((each) => each.client === first.client &&
                     each.dataset === first.dataset &&
                     tapped(each) === tapped(first) &&
                     (each.state.kind === "queued" || each.state.kind === "unknown"))
+                    .slice(0, pushMax)
                     .map((each) => each.id));
                 const mine = (each) => each.client === first.client && ids.has(each.id);
                 out.push({
@@ -347,20 +546,60 @@ export function createSync(spec) {
                 ];
             }
             case "pulled": {
-                const merged = merge(at.replica, msg.patch);
+                const result = merge(at.replica, msg.patch, msg.since);
+                const merged = result === "gap" ? undefined : result;
                 const replica = merged?.replica ?? at.replica;
                 const effects = [];
+                const was = historyOf(at.replica);
+                const history = historyOf(replica);
+                const failures = at.pull.kind === "pulling" ? at.pull.failures : 0;
+                // An old server answering a partial copy (with rows or without): only its whole copy completes it.
+                const old = was.kind === "partial" &&
+                    msg.patch.page === undefined &&
+                    !msg.patch.reset &&
+                    msg.since > 0 &&
+                    result !== "gap";
+                // A partial copy this reply didn't move on (the same page again, a repeated first page at the same revision, a
+                // reply it couldn't take) is a pull that failed, for backoff: never a busy loop on a server that doesn't move.
+                const stuck = was.kind === "partial" &&
+                    !old &&
+                    (merged === undefined ||
+                        (msg.patch.page !== undefined &&
+                            history.kind === "partial" &&
+                            history.next === was.next &&
+                            replica.dataset === at.replica.dataset &&
+                            !(merged.reset && replica.revision > at.replica.revision)));
+                // Receipts that came with an applied reply came with the rows they name.
+                const carried = new Set(merged?.reset === true ? [] : at.carried);
+                if (merged !== undefined) {
+                    for (const each of msg.receipts) {
+                        if (each.revision <= replica.revision) {
+                            carried.add(`${each.client ?? at.client}:${each.id}`);
+                        }
+                    }
+                }
                 let next = {
                     ...at,
                     replica,
+                    carried,
                     target: msg.patch.dataset === at.replica.dataset
                         ? Math.max(at.target, msg.patch.revision)
                         : msg.patch.revision,
+                    // A reply it couldn't follow asks a real reset; an old server's reply to a partial copy, its whole copy.
+                    restart: result === "gap"
+                        ? (at.restart ?? "paged")
+                        : merged?.reset === true
+                            ? null
+                            : old
+                                ? "whole"
+                                : at.restart,
                     reconciledAt: now,
                     connection: at.socket.open && now - at.socket.heardAt < SOCKET_DEADLINE_MS
                         ? { kind: "live" }
                         : { kind: "stale" },
-                    pull: { kind: "idle", nextAt: now + POLL_MS, failures: 0 },
+                    pull: stuck
+                        ? { kind: "idle", nextAt: now + backoff(failures + 1), failures: failures + 1 }
+                        : { kind: "idle", nextAt: now + POLL_MS, failures: 0 },
                 };
                 if (merged !== undefined && next.storage.kind !== "replaced") {
                     effects.push({
@@ -370,12 +609,15 @@ export function createSync(spec) {
                         reset: merged.reset,
                         base: msg.since,
                         replaces: at.saved.dataset,
+                        from: was.kind === "partial" ? was.next : null,
                     });
                     next = { ...next, saved: { ...next.saved, writing: next.saved.writing + 1 } };
                 }
                 if (at.verify.kind === "requested") {
-                    // The fingerprint describes the server at the reply's revision: comparable only if the replica is there.
+                    // The fingerprint describes the server at the reply's revision: comparable only if the replica is there, and
+                    // whole (a fingerprint of everything never certifies part of it).
                     if (msg.patch.checksum !== undefined &&
+                        history.kind === "complete" &&
                         replica.revision === msg.patch.revision &&
                         replica.dataset === msg.patch.dataset) {
                         effects.push({ kind: "check", replica, checksum: msg.patch.checksum });
@@ -384,13 +626,21 @@ export function createSync(spec) {
                         next = { ...next, verify: { kind: "pending", tries: at.verify.tries } };
                     }
                 }
-                const [answered, more] = answer(next, next.client, msg.receipts);
+                // Receipts by client: an earlier page's commands are answered too.
+                let answered = next;
+                const more = [];
+                for (const client of new Set(msg.receipts.map((each) => each.client ?? at.client))) {
+                    const [after, told] = answer(answered, client, msg.receipts.filter((each) => (each.client ?? at.client) === client));
+                    answered = after;
+                    more.push(...told);
+                }
                 // Intents tapped against a store the server no longer has never go to its replacement (the server refuses
                 // them too). One that may already have run there says so.
                 const replaced = answered.commands.map((each) => {
                     if (each.state.kind === "done" ||
                         each.dataset === null ||
-                        each.dataset === msg.patch.dataset) {
+                        replica.dataset === null ||
+                        each.dataset === replica.dataset) {
                         return each;
                     }
                     const done = fail(each, each.state.kind === "unknown"
@@ -466,10 +716,12 @@ export function createSync(spec) {
             }
             case "committed": {
                 const same = msg.dataset === at.saved.dataset;
+                // A delta the disk took under its own history doesn't repair a failed write: the whole one stays owed.
+                const owed = at.saved.owed && msg.whole === false;
                 return [
                     {
                         ...at,
-                        storage: at.storage.kind === "replaced" ? at.storage : { kind: "ok" },
+                        storage: at.storage.kind === "replaced" || owed ? at.storage : { kind: "ok" },
                         saved: {
                             dataset: msg.dataset,
                             revision: same ? Math.max(at.saved.revision, msg.revision) : msg.revision,
@@ -477,6 +729,7 @@ export function createSync(spec) {
                             writing: Math.max(0, at.saved.writing - 1),
                             failures: 0,
                             nextAt: now,
+                            owed,
                         },
                     },
                     [],
@@ -493,6 +746,7 @@ export function createSync(spec) {
                             writing: Math.max(0, at.saved.writing - 1),
                             failures,
                             nextAt: now + backoff(failures),
+                            owed: true,
                         },
                     },
                     [{ kind: "telemetry", name: "replica.save-failed", attributes: { reason: msg.reason } }],
@@ -677,6 +931,12 @@ export function createSync(spec) {
         let marked = since === next.saved.behindSince
             ? next
             : { ...next, saved: { ...next.saved, behindSince: since } };
+        // Commands that left the page leave what was carried for them.
+        const kept = new Set(marked.commands.map((each) => `${each.client}:${each.id}`));
+        const carried = [...marked.carried].filter((key) => kept.has(key));
+        if (carried.length < marked.carried.size) {
+            marked = { ...marked, carried: new Set(carried) };
+        }
         // A command that resolves only now (its result came later than its receipt) is drawn its full time from now.
         const resolved = spec.resolved;
         if (resolved !== undefined) {