slopkit 0.1.2 → 0.3.0

package/dist/sync.js

@@ -1,5 +1,93 @@
 import { unreachable } from "./machine.js";
 import { check } from "./schema.js";
+/** A key a chunk removed, in a table's layer of changes. */
+const GONE = Symbol("gone");
+/**
+ * A table's rows after a chunk, without copying the rows it already had: the chunk's changes are a layer over them, so
+ * merging a chunk costs its own size, not the table's. Layers merge when a newer one is as big as the one below it, and
+ * fold into a plain map once the top one holds half as many keys as it, so lookups pass a logarithmic number of layers
+ * and a changed row is copied a logarithmic number of times. Immutable: a published value never changes.
+ */
+class Layered {
+    size;
+    base;
+    /** Newest first. */
+    layers;
+    constructor(base, layers, size) {
+        this.base = base;
+        this.layers = layers;
+        this.size = size;
+    }
+    find(key) {
+        for (const each of this.layers) {
+            if (each.has(key)) {
+                return each.get(key);
+            }
+        }
+        return this.base.get(key);
+    }
+    get(key) {
+        const value = this.find(key);
+        return value === GONE ? undefined : value;
+    }
+    // Rows are never undefined.
+    has(key) {
+        return this.get(key) !== undefined;
+    }
+    // The base's keys in place, then new keys in the order they first came.
+    *entries() {
+        for (const key of this.base.keys()) {
+            const value = this.get(key);
+            if (value !== undefined) {
+                yield [key, value];
+            }
+        }
+        const seen = new Set();
+        for (let at = this.layers.length - 1; at >= 0; at--) {
+            for (const key of this.layers[at]?.keys() ?? []) {
+                const value = this.get(key);
+                if (!seen.has(key) && !this.base.has(key) && value !== undefined) {
+                    seen.add(key);
+                    yield [key, value];
+                }
+            }
+        }
+    }
+    *keys() {
+        for (const [key] of this.entries()) {
+            yield key;
+        }
+    }
+    *values() {
+        for (const [, value] of this.entries()) {
+            yield value;
+        }
+    }
+    [Symbol.iterator]() {
+        return this.entries();
+    }
+    // oxlint-disable-next-line anti-slop/no-unknown-parameters -- Map's receiver is whatever the caller passes
+    forEach(each, receiver) {
+        for (const [key, value] of this.entries()) {
+            each.call(receiver, value, key, this);
+        }
+    }
+}
+/** `rows` with `changes` over it, as a new value; `rows` is left as it was. */
+function overlay(rows, changes) {
+    let size = rows.size;
+    for (const [key, value] of changes) {
+        size += (value === GONE ? 0 : 1) - (rows.has(key) ? 1 : 0);
+    }
+    const base = rows instanceof Layered ? rows.base : rows;
+    const below = rows instanceof Layered ? [...rows.layers] : [];
+    let top = changes;
+    while (below[0] !== undefined && below[0].size <= top.size) {
+        top = new Map([...(below.shift() ?? []), ...top]);
+    }
+    const layered = new Layered(base, [top, ...below], size);
+    return top.size * 2 < base.size ? layered : new Map(layered);
+}
 export function table(def) {
     const gone = (row) => def.gone?.(row) === true;
     return {
@@ -15,16 +103,11 @@ export function table(def) {
             if (changed.length === 0 && !reset) {
                 return tables;
             }
-            const next = new Map(reset ? [] : def.get(tables));
+            const changes = new Map();
             for (const row of changed) {
-                if (gone(row)) {
-                    next.delete(def.key(row));
-                }
-                else {
-                    next.set(def.key(row), row);
-                }
+                changes.set(def.key(row), gone(row) ? GONE : row);
             }
-            return def.set(tables, next);
+            return def.set(tables, overlay(reset ? new Map() : def.get(tables), changes));
         },
         snapshot: (tables) => [...def.get(tables).entries()].map(([key, row]) => ({ table: def.name, key, row })),
         restore: (tables, values) => {
@@ -40,6 +123,18 @@ export function table(def) {
         },
     };
 }
+const COMPLETE = { kind: "complete" };
+const historyOf = (replica) => replica.history ?? COMPLETE;
+/** The history a page leaves: complete without one (an old server's whole copy) or once no older page is left. */
+const covered = (page) => page === undefined || page.next === null
+    ? COMPLETE
+    : {
+        kind: "partial",
+        next: page.next,
+        through: page.through,
+        held: page.held,
+        total: page.total,
+    };
 /** The socket must be heard this often (the server pokes every 20 s); quieter, the page polls. */
 export const SOCKET_DEADLINE_MS = 45_000;
 /** Polling while the socket can't be trusted or work is in flight, and the first retry after a failure. */
@@ -49,40 +144,128 @@ const BACKOFF_MAX_MS = 30_000;
 export const STORAGE_DEADLINE_MS = 3000;
 /** Commands per push unless the spec says otherwise; the rest wait for the next push. */
 export const PUSH_MAX = 50;
+/** UTF-8 bytes of one push request unless the spec says otherwise (48 KiB). */
+export const PUSH_BYTES = 49_152;
+/** Commands the outbox admits unless the spec says otherwise; updates to kept ones always are. */
+export const OUTBOX_MAX = 500;
 /** Settled commands stay drawn this long. */
 export const DONE_KEEP_MS = 60_000;
+/** A string's length in UTF-8 bytes, as it goes on the wire. */
+function utf8(text) {
+    let bytes = 0;
+    // By code point: a surrogate pair is one character of four bytes, a lone surrogate encodes as three.
+    for (const char of text) {
+        const code = char.codePointAt(0) ?? 0;
+        bytes += code < 0x80 ? 1 : code < 0x8_00 ? 2 : code < 0x1_00_00 ? 3 : 4;
+    }
+    return bytes;
+}
+/** A push's bytes as `call` encodes it: this envelope, each command's bytes, and a comma between commands. */
+const envelope = (client, dataset, since) => utf8(JSON.stringify({ client, dataset, since, commands: [] }));
+const bytesOf = (pending) => utf8(JSON.stringify({ id: pending.id, command: pending.command }));
+/**
+ * Where the outbox stands: full once it holds `max` commands, by this page's count or, failing that, by the store's
+ * last refusal (other tabs' commands count too). Unreadable kept commands take room as well.
+ */
+export function deriveOutboxState(model) {
+    const finished = model.commands.filter((each) => each.state.kind === "done").length;
+    const waiting = model.commands.length - finished + model.unreadable;
+    if (waiting + finished >= model.outbox.max) {
+        return { kind: "full", waiting, finished, max: model.outbox.max };
+    }
+    return model.outbox.full ?? { kind: "ok" };
+}
+/** The command a push's schema path names (`request.commands[2].command.value` names 2), if it names one. */
+function commandAt(path) {
+    const found = /^request\.commands\[(\d+)\](?:$|[.[])/.exec(path ?? "");
+    return found?.[1] === undefined ? undefined : Number(found[1]);
+}
 /** Whether the server can be asked now. */
 export function reachable(connection) {
     return connection.kind === "live" || connection.kind === "stale";
 }
-/** Whether the replica holds this command's effects. */
+/**
+ * Whether the replica holds this command's effects: it is done, and the copy is at its revision and either complete or
+ * holding the rows its receipt names (a pull reply carried them). One that failed on the page has none.
+ */
 export function settled(model, pending) {
-    return pending.state.kind === "done" && model.replica.revision >= pending.state.revision;
+    return (pending.state.kind === "done" &&
+        (pending.state.revision === 0 ||
+            (model.replica.revision >= pending.state.revision &&
+                (historyOf(model.replica).kind === "complete" ||
+                    model.carried.has(`${pending.client}:${pending.id}`)))));
 }
-/** Whether what the page shows is also on the disk. */
+/**
+ * Whether what the page shows is also on the disk: by revision only, so tabs at different cursors never chase each
+ * other's writes. A failed write is owed again even at the same revision (a page that failed to land).
+ */
 export function saved(model) {
-    return (model.saved.dataset === model.replica.dataset && model.saved.revision >= model.replica.revision);
+    return (model.saved.dataset === model.replica.dataset &&
+        model.saved.revision >= model.replica.revision &&
+        !model.saved.owed);
 }
 /**
  * What a commit does to the disk, decided inside the disk's own transaction from what it holds: nothing when the disk
- * already has this or newer (another tab got there), the patch alone when the disk is at least at its base, else the
- * whole replica, so a skipped or failed patch is never covered by a later checkpoint. A page may replace the disk's
+ * already has this or newer (another tab got there; a complete copy at the same revision is newer than any page), the
+ * patch alone when the disk is at least at its base, else the whole replica, so a skipped or failed patch is never
+ * covered by a later checkpoint. A delta keeps the disk's own history, and takes the patch's only when the disk is at
+ * the cursor the patch moved from (`adopt`): coverage never lands without its rows. A page may replace the disk's
  * dataset only with one it moved to from that very dataset (`replaces`): a tab still on a dataset the server has since
  * replaced is `stale` and never writes it back.
  */
 export function commitPlan(stored, commit) {
     const same = stored.dataset === commit.dataset;
+    const disk = stored.history ?? COMPLETE;
     if (!same && stored.dataset !== null && stored.dataset !== commit.replaces) {
         return "stale";
     }
-    if (same && stored.revision >= commit.revision) {
+    if (same &&
+        (stored.revision > commit.revision ||
+            (stored.revision === commit.revision && disk.kind === "complete"))) {
         return "skip";
     }
     if (!commit.reset && same && stored.revision >= commit.base) {
-        return "delta";
+        return disk.kind === "partial" && disk.next === commit.from ? "adopt" : "delta";
     }
     return "whole";
 }
+/**
+ * The copy's history: loading while storage loads, none without a dataset, else partial or complete. Partial is a claim
+ * beside the numbers ("history from <through>, loading older · held of total"), and totals cover only `through` and
+ * above. Complete says nothing about storage: that is `deriveDurabilityState`, its own subject.
+ */
+export function deriveHistoryState(model) {
+    if (model.boot.kind === "loading") {
+        return { kind: "loading" };
+    }
+    if (model.replica.dataset === null) {
+        return { kind: "none" };
+    }
+    const history = historyOf(model.replica);
+    return history.kind === "complete"
+        ? history
+        : { kind: "partial", through: history.through, held: history.held, total: history.total };
+}
+/**
+ * The copy's durability at the model's clock: loading, replaced by another tab, a failing store (lost while the page
+ * shows more than the disk holds), unsaved past the storage deadline, commands kept that this version can't read, or ok.
+ */
+export function deriveDurabilityState(model) {
+    const behind = model.saved.behindSince;
+    if (model.boot.kind === "loading") {
+        return { kind: "loading" };
+    }
+    if (model.storage.kind === "replaced") {
+        return { kind: "replaced" };
+    }
+    if (model.storage.kind === "failed") {
+        return { kind: behind === null ? "failing" : "lost", reason: model.storage.reason };
+    }
+    if (behind !== null && model.now - behind >= STORAGE_DEADLINE_MS) {
+        return { kind: "unsaved" };
+    }
+    return model.unreadable > 0 ? { kind: "unreadable", count: model.unreadable } : { kind: "ok" };
+}
 /**
  * Storage after a write that worked: a failure that only a keep or a drop had (the replica's own writes aren't
  * failing) is over; a replaced copy stays replaced.
@@ -100,7 +283,8 @@ function failedConnection(connection, failure) {
         case "signed-out": {
             return { kind: "signed-out" };
         }
-        case "invalid": {
+        case "invalid":
+        case "unreadable": {
             return { kind: "incompatible", failure: failure.message };
         }
         case "forbidden":
@@ -117,14 +301,23 @@ function failedConnection(connection, failure) {
 }
 export function createSync(spec) {
     const pushMax = spec.pushMax ?? PUSH_MAX;
-    if (!Number.isSafeInteger(pushMax) || pushMax < 1) {
-        throw new Error(`sync: pushMax must be a positive integer, got ${pushMax}`);
+    const pushBytes = spec.pushBytes ?? PUSH_BYTES;
+    const outboxMax = spec.outboxMax ?? OUTBOX_MAX;
+    for (const [name, value] of [
+        ["pushMax", pushMax],
+        ["pushBytes", pushBytes],
+        ["outboxMax", outboxMax],
+    ]) {
+        if (!Number.isSafeInteger(value) || value < 1) {
+            throw new Error(`sync: ${name} must be a positive integer, got ${value}`);
+        }
     }
     const emptyReplica = () => ({
         dataset: null,
         revision: 0,
         tables: spec.empty,
         server: null,
+        history: COMPLETE,
     });
     function initial(client, now) {
         return {
@@ -140,14 +333,19 @@ export function createSync(spec) {
                 writing: 0,
                 failures: 0,
                 nextAt: now,
+                owed: false,
             },
             target: 0,
+            restart: null,
             reconciledAt: null,
             connection: { kind: "starting" },
             socket: { open: false, heardAt: 0 },
             pull: { kind: "idle", nextAt: now, failures: 0 },
             push: { kind: "idle", nextAt: now, failures: 0 },
             commands: [],
+            outbox: { max: outboxMax, full: null },
+            asking: 0,
+            carried: new Set(),
             unreadable: 0,
             visible: true,
             verify: { kind: "pending", tries: 0 },
@@ -155,22 +353,37 @@ export function createSync(spec) {
         };
     }
     /**
-     * The one merge: a patch applied to a replica, with its disk operations, or undefined when it is older than the
-     * replica (a late reply). A reset or another dataset replaces everything.
+     * The one merge: a patch applied to a replica, with its disk operations; undefined when it is older than the replica
+     * (a late reply), `gap` when it can't follow from it. Only a reset (or the answer to `since` 0, which holds
+     * everything) replaces the copy, with its own history. A delta applies only to the same dataset with
+     * `since ≤ copy.revision ≤ reply.revision` (equal only with a page or new server facts); it moves the cursor only when
+     * it answers the copy's own `next`, and a reply without a page never changes the history.
      */
-    function merge(replica, patch) {
-        const reset = patch.reset || patch.dataset !== replica.dataset;
-        if (!reset && patch.revision <= replica.revision) {
-            // Nothing newer, but the server's own facts (version, settings) may have moved.
-            if (patch.revision === replica.revision &&
-                JSON.stringify(patch.server) !== JSON.stringify(replica.server)) {
-                return { replica: { ...replica, server: patch.server }, ops: [], reset: false };
+    function merge(replica, patch, since) {
+        const reset = patch.reset || since === 0;
+        const page = patch.page;
+        const was = historyOf(replica);
+        let history = was;
+        if (reset) {
+            // A reset older than what we hold of the same dataset is a late reply.
+            if (patch.dataset === replica.dataset && patch.revision < replica.revision) {
+                return undefined;
             }
-            return undefined;
+            history = covered(page);
         }
-        // A reset older than what we hold of the same dataset is a late reply.
-        if (reset && patch.dataset === replica.dataset && patch.revision < replica.revision) {
-            return undefined;
+        else if (patch.dataset !== replica.dataset || since > replica.revision) {
+            return "gap";
+        }
+        else if (patch.revision < replica.revision ||
+            (patch.revision === replica.revision && (page === undefined || was.kind === "complete"))) {
+            // Nothing newer, but the server's own facts (version, settings) may have moved.
+            return patch.revision === replica.revision &&
+                JSON.stringify(patch.server) !== JSON.stringify(replica.server)
+                ? { replica: { ...replica, server: patch.server }, ops: [], reset: false }
+                : undefined;
+        }
+        else if (page !== undefined && was.kind === "partial" && page.from === was.next) {
+            history = covered(page);
         }
         let tables = reset ? spec.empty : replica.tables;
         const ops = [];
@@ -179,12 +392,51 @@ export function createSync(spec) {
             ops.push(...lens.ops(patch.rows));
         }
         return {
-            replica: { dataset: patch.dataset, revision: patch.revision, server: patch.server, tables },
+            replica: {
+                dataset: patch.dataset,
+                revision: patch.revision,
+                server: patch.server,
+                tables,
+                history,
+            },
             ops,
             reset,
         };
     }
     const ready = (model) => model.boot.kind === "ready";
+    const keep = (pending) => ({ kind: "keep", pending, max: outboxMax });
+    /** Waiting to be sent: queued, or unknown and not parked (asked again under its key). */
+    const sendable = (each) => each.state.kind === "queued" || (each.state.kind === "unknown" && each.state.parked !== true);
+    const tooLarge = (bytes) => `this is too large to send: ${bytes} bytes, where a push holds at most ${pushBytes}`;
+    /** A command that failed on the page: done with that outcome, never sent, and its kept intent dropped. */
+    function fail(pending, message, now) {
+        return {
+            ...pending,
+            at: now,
+            state: { kind: "done", outcome: spec.refused(pending.command, message), revision: 0 },
+        };
+    }
+    /**
+     * A command the server refused by name, or one too large to send: final if no attempt of it may have acted; else
+     * parked, since a refusal proves only that attempt didn't act. Kept either way, and passed over by every push.
+     */
+    function refuse(pending, doubt, message, now, effects) {
+        if (doubt) {
+            const parked = {
+                ...pending,
+                state: { kind: "unknown", failure: message, parked: true },
+            };
+            effects.push(keep(parked), {
+                kind: "telemetry",
+                name: "command.parked",
+                attributes: { reason: message },
+            });
+            return parked;
+        }
+        const done = fail(pending, message, now);
+        effects.push(keep(done), { kind: "answered", pending: done });
+        return done;
+    }
     /** Start what is due: the obligations are read off the model after every message, never remembered. */
     function schedule(model, effects) {
         let next = model;
@@ -196,56 +448,103 @@ export function createSync(spec) {
             const behind = next.replica.revision < next.target;
             const unsettled = next.commands.some((each) => each.state.kind === "done" && !settled(next, each));
             const polling = next.connection.kind !== "live" || unsettled || (spec.outstanding?.(next.replica) ?? false);
-            const verifying = next.verify.kind === "pending";
-            if (((behind || verifying) && (failures === 0 || due)) || (polling && due)) {
+            const history = historyOf(next.replica);
+            const whole = next.restart === "whole";
+            // Only a complete copy is checked: a fingerprint of everything can't certify part of it.
+            const verifying = next.verify.kind === "pending" && (history.kind === "complete" || whole);
+            // A partial copy owes its next page, back to back, until it is complete.
+            const owed = behind || verifying || history.kind === "partial" || next.restart !== null;
+            const since = next.restart === null ? next.replica.revision : 0;
+            if ((owed && (failures === 0 || due)) || (polling && due)) {
+                const current = next;
+                // Commands waiting on an outcome or on their effects, at most a push's worth: their receipts come back with
+                // the rows they name. When more wait, each pull names the next ones round, so every one is asked about in
+                // turn (a parked command has no other way to settle).
+                const waiting = current.commands
+                    .filter((each) => each.state.kind === "sending" ||
+                    each.state.kind === "unknown" ||
+                    (each.state.kind === "done" && !settled(current, each)))
+                    .map((each) => ({ client: each.client, id: each.id }));
+                const turn = waiting.length > pushMax ? current.asking % waiting.length : 0;
+                const commands = [...waiting.slice(turn), ...waiting.slice(0, turn)].slice(0, pushMax);
                 out.push({
                     kind: "pull",
                     client: next.client,
                     dataset: next.replica.dataset,
-                    since: next.replica.revision,
+                    since,
                     verify: verifying,
+                    ...(whole ? {} : { page: history.kind === "partial" && since > 0 ? history.next : null }),
+                    commands,
                 });
                 next = {
                     ...next,
-                    pull: { kind: "pulling", since: next.replica.revision, failures },
-                    verify: next.verify.kind === "pending"
+                    asking: waiting.length > pushMax ? turn + pushMax : 0,
+                    pull: { kind: "pulling", since, failures },
+                    verify: verifying && next.verify.kind === "pending"
                         ? { kind: "requested", tries: next.verify.tries }
                         : next.verify,
                 };
             }
         }
         if (next.push.kind === "idle" && now >= next.push.nextAt && reachable(next.connection)) {
-            const first = next.commands.find((each) => each.state.kind === "queued" || each.state.kind === "unknown");
+            const tapped = (each) => each.since ?? next.replica.revision;
+            let first = next.commands.find(sendable);
+            // A kept command too large to send even alone (an older release kept it, or the bound was lowered) is refused,
+            // or parked if an attempt may have acted, and passed over: it never strands the commands behind it.
+            while (first !== undefined) {
+                const alone = envelope(first.client, first.dataset, tapped(first)) + bytesOf(first);
+                if (alone <= pushBytes) {
+                    break;
+                }
+                const large = first;
+                const told = [];
+                const withdrawn = refuse(large, large.state.kind === "unknown", tooLarge(alone), now, told);
+                out.push(...told);
+                next = {
+                    ...next,
+                    commands: next.commands.map((each) => (each === large ? withdrawn : each)),
+                };
+                first = next.commands.find(sendable);
+            }
             if (first !== undefined) {
                 // One push per client, dataset and tapped-against revision: the server judges each by what it was made on.
-                // At most `pushMax`, oldest first; the rest go in the next push.
-                const tapped = (each) => each.since ?? next.replica.revision;
-                const ids = new Set(next.commands
-                    .filter((each) => each.client === first.client &&
-                    each.dataset === first.dataset &&
-                    tapped(each) === tapped(first) &&
-                    (each.state.kind === "queued" || each.state.kind === "unknown"))
-                    .slice(0, pushMax)
-                    .map((each) => each.id));
-                const mine = (each) => each.client === first.client && ids.has(each.id);
+                // Oldest first, at most `pushMax` commands and `pushBytes` bytes: it stops at the first command that doesn't
+                // fit, never skipping ahead to a smaller one, and the rest go in the next push.
+                const lead = first;
+                const batch = [];
+                let bytes = envelope(lead.client, lead.dataset, tapped(lead));
+                for (const each of next.commands) {
+                    if (each.client !== lead.client ||
+                        each.dataset !== lead.dataset ||
+                        tapped(each) !== tapped(lead) ||
+                        !sendable(each)) {
+                        continue;
+                    }
+                    const size = bytesOf(each) + (batch.length > 0 ? 1 : 0);
+                    if (batch.length >= pushMax || bytes + size > pushBytes) {
+                        break;
+                    }
+                    bytes += size;
+                    batch.push(each);
+                }
+                const sent = new Set(batch);
                 out.push({
                     kind: "push",
-                    client: first.client,
-                    dataset: first.dataset,
-                    since: tapped(first),
-                    commands: next.commands
-                        .filter(mine)
-                        .map((each) => ({ id: each.id, command: each.command })),
+                    client: lead.client,
+                    dataset: lead.dataset,
+                    since: tapped(lead),
+                    commands: batch.map((each) => ({ id: each.id, command: each.command })),
                 });
                 next = {
                     ...next,
                     push: {
                         kind: "pushing",
-                        client: first.client,
-                        ids: [...ids],
+                        client: lead.client,
+                        ids: batch.map((each) => each.id),
+                        was: batch.map((each) => each.state),
                         failures: next.push.failures,
                     },
-                    commands: next.commands.map((each) => mine(each) ? { ...each, state: { kind: "sending" } } : each),
+                    commands: next.commands.map((each) => sent.has(each) ? { ...each, state: { kind: "sending" } } : each),
                 };
             }
         }
@@ -289,19 +588,16 @@ export function createSync(spec) {
                 at: model.now,
                 state: { kind: "done", outcome: receipt.outcome, revision: receipt.revision },
             };
-            effects.push({ kind: "keep", pending: done }, { kind: "answered", pending: done }, { kind: "telemetry", name: "command.done", attributes: { ms: model.now - each.at } });
+            effects.push(keep(done), { kind: "answered", pending: done }, { kind: "telemetry", name: "command.done", attributes: { ms: model.now - each.at } });
             return done;
         });
         const target = Math.max(model.target, ...receipts.map((each) => each.revision));
         return [{ ...model, commands, target }, effects];
     }
-    /** A command that failed on the page: done with that outcome, never sent, and its kept intent dropped. */
-    function fail(pending, message, now) {
-        return {
-            ...pending,
-            at: now,
-            state: { kind: "done", outcome: spec.refused(pending.command, message), revision: 0 },
-        };
+    /** Each command's state before the push in flight for `client`; unknown when this page can't tell. */
+    function prior(model, client) {
+        const pushing = model.push.kind === "pushing" && model.push.client === client ? model.push : null;
+        return (each) => pushing?.was[pushing.ids.indexOf(each.id)] ?? { kind: "unknown", failure: "sent before" };
     }
     function reduce(model, msg) {
         const now = msg.now;
@@ -355,20 +651,60 @@ export function createSync(spec) {
                 ];
             }
             case "pulled": {
-                const merged = merge(at.replica, msg.patch);
+                const result = merge(at.replica, msg.patch, msg.since);
+                const merged = result === "gap" ? undefined : result;
                 const replica = merged?.replica ?? at.replica;
                 const effects = [];
+                const was = historyOf(at.replica);
+                const history = historyOf(replica);
+                const failures = at.pull.kind === "pulling" ? at.pull.failures : 0;
+                // An old server answering a partial copy (with rows or without): only its whole copy completes it.
+                const old = was.kind === "partial" &&
+                    msg.patch.page === undefined &&
+                    !msg.patch.reset &&
+                    msg.since > 0 &&
+                    result !== "gap";
+                // A partial copy this reply didn't move on (the same page again, a repeated first page at the same revision, a
+                // reply it couldn't take) is a pull that failed, for backoff: never a busy loop on a server that doesn't move.
+                const stuck = was.kind === "partial" &&
+                    !old &&
+                    (merged === undefined ||
+                        (msg.patch.page !== undefined &&
+                            history.kind === "partial" &&
+                            history.next === was.next &&
+                            replica.dataset === at.replica.dataset &&
+                            !(merged.reset && replica.revision > at.replica.revision)));
+                // Receipts that came with an applied reply came with the rows they name.
+                const carried = new Set(merged?.reset === true ? [] : at.carried);
+                if (merged !== undefined) {
+                    for (const each of msg.receipts) {
+                        if (each.revision <= replica.revision) {
+                            carried.add(`${each.client ?? at.client}:${each.id}`);
+                        }
+                    }
+                }
                 let next = {
                     ...at,
                     replica,
+                    carried,
                     target: msg.patch.dataset === at.replica.dataset
                         ? Math.max(at.target, msg.patch.revision)
                         : msg.patch.revision,
+                    // A reply it couldn't follow asks a real reset; an old server's reply to a partial copy, its whole copy.
+                    restart: result === "gap"
+                        ? (at.restart ?? "paged")
+                        : merged?.reset === true
+                            ? null
+                            : old
+                                ? "whole"
+                                : at.restart,
                     reconciledAt: now,
                     connection: at.socket.open && now - at.socket.heardAt < SOCKET_DEADLINE_MS
                         ? { kind: "live" }
                         : { kind: "stale" },
-                    pull: { kind: "idle", nextAt: now + POLL_MS, failures: 0 },
+                    pull: stuck
+                        ? { kind: "idle", nextAt: now + backoff(failures + 1), failures: failures + 1 }
+                        : { kind: "idle", nextAt: now + POLL_MS, failures: 0 },
                 };
                 if (merged !== undefined && next.storage.kind !== "replaced") {
                     effects.push({
@@ -378,12 +714,15 @@ export function createSync(spec) {
                         reset: merged.reset,
                         base: msg.since,
                         replaces: at.saved.dataset,
+                        from: was.kind === "partial" ? was.next : null,
                     });
                     next = { ...next, saved: { ...next.saved, writing: next.saved.writing + 1 } };
                 }
                 if (at.verify.kind === "requested") {
-                    // The fingerprint describes the server at the reply's revision: comparable only if the replica is there.
+                    // The fingerprint describes the server at the reply's revision: comparable only if the replica is there, and
+                    // whole (a fingerprint of everything never certifies part of it).
                     if (msg.patch.checksum !== undefined &&
+                        history.kind === "complete" &&
                         replica.revision === msg.patch.revision &&
                         replica.dataset === msg.patch.dataset) {
                         effects.push({ kind: "check", replica, checksum: msg.patch.checksum });
@@ -392,30 +731,39 @@ export function createSync(spec) {
                         next = { ...next, verify: { kind: "pending", tries: at.verify.tries } };
                     }
                 }
-                const [answered, more] = answer(next, next.client, msg.receipts);
+                // Receipts by client: an earlier page's commands are answered too.
+                let answered = next;
+                const more = [];
+                for (const client of new Set(msg.receipts.map((each) => each.client ?? at.client))) {
+                    const [after, told] = answer(answered, client, msg.receipts.filter((each) => (each.client ?? at.client) === client));
+                    answered = after;
+                    more.push(...told);
+                }
                 // Intents tapped against a store the server no longer has never go to its replacement (the server refuses
                 // them too). One that may already have run there says so.
                 const replaced = answered.commands.map((each) => {
                     if (each.state.kind === "done" ||
                         each.dataset === null ||
-                        each.dataset === msg.patch.dataset) {
+                        replica.dataset === null ||
+                        each.dataset === replica.dataset) {
                         return each;
                     }
                     const done = fail(each, each.state.kind === "unknown"
                         ? "the server's data was replaced; whether this ran before is unknown"
                         : "the server's data was replaced since this was tapped, so it wasn't sent", now);
-                    more.push({ kind: "keep", pending: done }, { kind: "answered", pending: done });
+                    more.push(keep(done), { kind: "answered", pending: done });
                     return done;
                 });
                 return [{ ...answered, commands: replaced }, [...effects, ...more]];
             }
             case "pullFailed": {
                 const failures = (at.pull.kind === "pulling" ? at.pull.failures : 0) + 1;
+                const wait = Math.max(backoff(failures), msg.failure.retryAfterMs ?? 0);
                 return [
                     {
                         ...at,
                         connection: failedConnection(at.connection, msg.failure),
-                        pull: { kind: "idle", nextAt: now + backoff(failures), failures },
+                        pull: { kind: "idle", nextAt: now + wait, failures },
                         verify: at.verify.kind === "requested"
                             ? { kind: "pending", tries: at.verify.tries }
                             : at.verify,
@@ -426,7 +774,8 @@ export function createSync(spec) {
             case "pushed": {
                 const ids = new Set(msg.ids);
                 const [answered, effects] = answer(at, msg.client, msg.receipts, ids);
-                // A command the reply has no receipt for is still acting elsewhere: asked again.
+                // A command the reply has no receipt for is still acting elsewhere: asked again. The disk holds its in-flight
+                // mark, so a reload knows it too.
                 const commands = answered.commands.map((each) => each.client === msg.client && ids.has(each.id) && each.state.kind === "sending"
                     ? { ...each, state: { kind: "unknown", failure: "the server is still on it" } }
                     : each);
@@ -442,42 +791,108 @@ export function createSync(spec) {
                 ];
             }
             case "pushFailed": {
-                // Refused before anything acted: final. Anything else may have acted (an unreadable reply included), so it is
-                // unknown and asked again under the same key, which answers with the first outcome.
-                const final = msg.failure.kind === "forbidden" || msg.failure.kind === "conflict";
+                const failure = msg.failure;
                 const ids = new Set(msg.ids);
+                const before = prior(at, msg.client);
                 const effects = [];
+                // The server's own refusal of a request it never ran (its schema is checked before the handler, and a handler
+                // never refuses after anything acted): this attempt didn't act, though an earlier one may have.
+                const refused = failure.refused === true &&
+                    (failure.kind === "invalid" ||
+                        failure.kind === "forbidden" ||
+                        failure.kind === "conflict");
+                // A schema refusal naming one of this push's commands is that command's alone.
+                const index = refused && failure.kind === "invalid" ? commandAt(failure.path) : undefined;
+                const named = index === undefined ? undefined : msg.ids[index];
                 const commands = at.commands.map((each) => {
-                    if (each.client !== msg.client || !ids.has(each.id) || each.state.kind === "done") {
+                    if (each.client !== msg.client || !ids.has(each.id) || each.state.kind !== "sending") {
                         return each;
                     }
-                    if (!final) {
-                        return { ...each, state: { kind: "unknown", failure: msg.failure.message } };
+                    const was = before(each);
+                    // An attempt whose outcome was lost may have acted: no later refusal makes it final.
+                    const doubt = was.kind === "unknown";
+                    if (named !== undefined) {
+                        return each.id === named
+                            ? refuse(each, doubt, failure.message, now, effects)
+                            : { ...each, state: was };
                     }
-                    const done = fail(each, msg.failure.message, now);
-                    effects.push({ kind: "keep", pending: done }, { kind: "answered", pending: done });
-                    return done;
+                    // A kit forbidden or conflict: nothing in this attempt acted. Final only for a command no attempt of which
+                    // may have acted.
+                    if (refused && failure.kind !== "invalid") {
+                        return doubt
+                            ? { ...each, state: was }
+                            : refuse(each, false, failure.message, now, effects);
+                    }
+                    // An envelope refusal (a page and server of different releases): nothing acted, and the page stops. The disk
+                    // holds the in-flight mark: a command never uncertain is kept as it was again.
+                    if (refused) {
+                        const restored = { ...each, state: was };
+                        if (!doubt) {
+                            effects.push(keep(restored));
+                        }
+                        return restored;
+                    }
+                    // Anything else may have acted (a lost reply, an unreadable one, a proxy's answer): asked again, same key.
+                    // The disk holds its in-flight mark.
+                    return { ...each, state: { kind: "unknown", failure: failure.message } };
                 });
-                const failures = (at.push.kind === "pushing" ? at.push.failures : 0) + 1;
+                const telemetry = {
+                    kind: "telemetry",
+                    name: "push.failed",
+                    attributes: {
+                        kind: failure.kind,
+                        refused: failure.refused === true,
+                        path: failure.path ?? null,
+                    },
+                };
+                const pushing = at.push.kind === "pushing" ? at.push.failures : 0;
+                // One command refused by name: the rest go again at once, without it.
+                if (named !== undefined) {
+                    return [
+                        { ...at, commands, push: { kind: "idle", nextAt: now, failures: pushing } },
+                        [...effects, telemetry],
+                    ];
+                }
+                const failures = pushing + 1;
                 return [
                     {
                         ...at,
                         commands,
-                        connection: failedConnection(at.connection, msg.failure),
+                        connection: failedConnection(at.connection, failure),
+                        push: {
+                            kind: "idle",
+                            nextAt: now + Math.max(backoff(failures), failure.retryAfterMs ?? 0),
+                            failures,
+                        },
+                    },
+                    [...effects, telemetry],
+                ];
+            }
+            case "unsent": {
+                // Never sent: each command is what it was before, and goes again once the disk keeps its mark.
+                const ids = new Set(msg.ids);
+                const before = prior(at, msg.client);
+                const failures = (at.push.kind === "pushing" ? at.push.failures : 0) + 1;
+                return [
+                    {
+                        ...at,
+                        commands: at.commands.map((each) => each.client === msg.client && ids.has(each.id) && each.state.kind === "sending"
+                            ? { ...each, state: before(each) }
+                            : each),
+                        storage: msg.replaced ? { kind: "replaced" } : { kind: "failed", reason: msg.reason },
                         push: { kind: "idle", nextAt: now + backoff(failures), failures },
                     },
-                    [
-                        ...effects,
-                        { kind: "telemetry", name: "push.failed", attributes: { kind: msg.failure.kind } },
-                    ],
+                    [{ kind: "telemetry", name: "push.unsent", attributes: { reason: msg.reason } }],
                 ];
             }
             case "committed": {
                 const same = msg.dataset === at.saved.dataset;
+                // A delta the disk took under its own history doesn't repair a failed write: the whole one stays owed.
+                const owed = at.saved.owed && msg.whole === false;
                 return [
                     {
                         ...at,
-                        storage: at.storage.kind === "replaced" ? at.storage : { kind: "ok" },
+                        storage: at.storage.kind === "replaced" || owed ? at.storage : { kind: "ok" },
                         saved: {
                             dataset: msg.dataset,
                             revision: same ? Math.max(at.saved.revision, msg.revision) : msg.revision,
@@ -485,6 +900,7 @@ export function createSync(spec) {
                             writing: Math.max(0, at.saved.writing - 1),
                             failures: 0,
                             nextAt: now,
+                            owed,
                         },
                     },
                     [],
@@ -501,16 +917,57 @@ export function createSync(spec) {
                             writing: Math.max(0, at.saved.writing - 1),
                             failures,
                             nextAt: now + backoff(failures),
+                            owed: true,
                         },
                     },
                     [{ kind: "telemetry", name: "replica.save-failed", attributes: { reason: msg.reason } }],
                 ];
             }
             case "kept": {
-                const commands = at.commands.map((each) => each.client === msg.client && each.id === msg.id && each.state.kind === "saving"
-                    ? { ...each, state: { kind: "queued" } }
-                    : each);
-                return [{ ...at, commands, storage: working(at) }, []];
+                let admitted = false;
+                const commands = at.commands.map((each) => {
+                    if (each.client !== msg.client || each.id !== msg.id || each.state.kind !== "saving") {
+                        return each;
+                    }
+                    admitted = true;
+                    return { ...each, state: { kind: "queued" } };
+                });
+                return [
+                    {
+                        ...at,
+                        commands,
+                        storage: working(at),
+                        outbox: admitted ? { ...at.outbox, full: null } : at.outbox,
+                    },
+                    [],
+                ];
+            }
+            case "full": {
+                // Not a storage failure: the store answered, with no room. The tap is answered and not retained.
+                const gone = at.commands.find((each) => each.client === msg.client && each.id === msg.id && each.state.kind === "saving");
+                if (gone === undefined) {
+                    return [{ ...at, storage: working(at) }, []];
+                }
+                const refused = fail(gone, "this device's outbox is full, so it wasn't kept or sent", now);
+                return [
+                    {
+                        ...at,
+                        commands: at.commands.filter((each) => each !== gone),
+                        storage: working(at),
+                        outbox: {
+                            ...at.outbox,
+                            full: { kind: "full", waiting: msg.waiting, finished: msg.finished, max: msg.max },
+                        },
+                    },
+                    [
+                        { kind: "answered", pending: refused },
+                        {
+                            kind: "telemetry",
+                            name: "commands.full",
+                            attributes: { waiting: msg.waiting, finished: msg.finished, max: msg.max },
+                        },
+                    ],
+                ];
             }
             case "keepFailed": {
                 const effects = [
@@ -528,7 +985,7 @@ export function createSync(spec) {
             }
             case "dropped": {
                 if (msg.failure === null) {
-                    return [{ ...at, storage: working(at) }, []];
+                    return [{ ...at, storage: working(at), outbox: { ...at.outbox, full: null } }, []];
                 }
                 return [
                     {
@@ -625,9 +1082,15 @@ export function createSync(spec) {
                 if (joined) {
                     return [at, []];
                 }
+                // A tap no push could carry is answered now, never kept.
+                const alone = envelope(at.client, pending.dataset, at.replica.revision) + bytesOf(pending);
+                if (alone > pushBytes) {
+                    return [numbered, [{ kind: "answered", pending: fail(pending, tooLarge(alone), now) }]];
+                }
+                // Admitted by the store, inside the keep's own transaction: a full outbox answers `full`, and the tap goes.
                 return [
                     { ...numbered, commands: [...at.commands, pending] },
-                    [{ kind: "keep", pending: { ...pending, state: { kind: "queued" } } }],
+                    [keep({ ...pending, state: { kind: "queued" } })],
                 ];
             }
             case "dismiss": {
@@ -685,6 +1148,12 @@ export function createSync(spec) {
         let marked = since === next.saved.behindSince
             ? next
             : { ...next, saved: { ...next.saved, behindSince: since } };
+        // Commands that left the page leave what was carried for them.
+        const kept = new Set(marked.commands.map((each) => `${each.client}:${each.id}`));
+        const carried = [...marked.carried].filter((key) => kept.has(key));
+        if (carried.length < marked.carried.size) {
+            marked = { ...marked, carried: new Set(carried) };
+        }
         // A command that resolves only now (its result came later than its receipt) is drawn its full time from now.
         const resolved = spec.resolved;
         if (resolved !== undefined) {