slopkit 0.1.1 → 0.2.0

package/dist/server/pull.js

@@ -0,0 +1,233 @@
+/**
+ * The server half of pull, over SQLite: the app's history table read newest first in pages by a keyset order, its
+ * other tables whole, and the receipts of the caller's commands with the rows they name, assembled in one read
+ * transaction at one revision and bounded in rows and encoded bytes. `sync` is the client half.
+ *
+ * Invariants it owns:
+ * - **One revision.** The delta, the page, the counts, the coverage boundary and the receipts' rows are read in one
+ *   transaction (`pull` is synchronous and opens its own unless the caller has one open), so every row in a reply is
+ *   the store's row at the reply's revision.
+ * - **Keyset cursors.** A cursor is the page's last row by its order values and unique key, with the dataset and the
+ *   order's `version`: never an offset, and never looked up from that row again, so an anchor edited or tombstoned
+ *   since skips nothing. A cursor of another dataset or order version, or one that doesn't parse, gets a paged reset.
+ * - **Truthful coverage.** `held` counts the live rows at or above the cursor and `total` every live row. `through` is
+ *   the lowest order group held whole: a page ending inside a tie leaves that group out until it lands whole, and it
+ *   is null while no group is whole.
+ * - **Every paging-aware reply is bounded** by `limits`: the history's rows, the other tables' and the receipts' rows
+ *   together within `maxRows`, the reply as `respond` builds it within `maxBytes` encoded. A delta (with its receipts'
+ *   rows and the other tables) that can't fit by itself is answered as a paged reset; otherwise the older page shrinks
+ *   to the rows and bytes it leaves, and with no room at all the cursor stays where it was. A reset carries one row of history at least, then as many receipts as fit, then more rows; one
+ *   that can't fit even so (the other tables alone are over the caps) is never sent: `pull` throws, a server fault
+ *   the page sees as unavailable and retries.
+ * - **Receipts carry their rows.** A row a receipt names is in the reply that carries the receipt, even below the page,
+ *   so a command settles only on a copy that holds its effects. A paging client names the commands it waits on (any of
+ *   its pages', at most a push's worth); a receipt left out for room is asked for again by its next pull.
+ * - **Old clients keep the old protocol.** A request without `page` gets the whole reply as before: everything on a
+ *   reset, the changes otherwise, without page metadata and unbounded.
+ *
+ * The history table has a `revision` column (the revision that last wrote the row), keeps tombstones (`live` tells
+ * them apart), and has an index on its order columns, which are never null. The order is descending in every column;
+ * its last column is the unique key that receipts name.
+ *
+ * A fingerprint for `verify` (asked only by a complete copy) is the app's: read what it covers in the same transaction,
+ * by opening one around `pull`, and keep its few bytes in mind under `maxBytes`, which measures `respond`'s reply.
+ */
+import { Buffer } from "node:buffer";
+const marks = (columns) => columns.map(() => "?").join(", ");
+const sizeOf = (tables) => Object.values(tables).reduce((sum, rows) => sum + rows.length, 0);
+const tally = (row) => Number(row?.n ?? 0);
+/** The largest whole number from `low` to `high` that `fits`, or undefined (`fits` holds for every one below it). */
+function largest(low, high, fits) {
+    let found;
+    let from = low;
+    let to = high;
+    while (from <= to) {
+        const at = Math.floor((from + to) / 2);
+        if (fits(at)) {
+            found = at;
+            from = at + 1;
+        }
+        else {
+            to = at - 1;
+        }
+    }
+    return found;
+}
+/** A row's values in these order columns, which are never null: what a cursor holds. */
+const values = (row, columns) => columns.map((column) => {
+    const value = row[column];
+    if (typeof value !== "string" && typeof value !== "number") {
+        throw new TypeError(`pull: order column ${column} must be text or a number`);
+    }
+    return value;
+});
+export function createPull(options) {
+    const { db, limits, paged } = options;
+    for (const [name, value] of Object.entries(limits)) {
+        if (!Number.isSafeInteger(value) || value < 1) {
+            throw new Error(`pull: ${name} must be a positive integer, got ${value}`);
+        }
+    }
+    if (limits.firstRows > limits.maxRows || limits.pageRows > limits.maxRows) {
+        throw new Error("pull: a page must fit in maxRows");
+    }
+    const order = paged.order;
+    const unique = order.at(-1);
+    if (unique === undefined) {
+        throw new Error("pull: the paged order needs at least its key column");
+    }
+    const key = unique;
+    const group = order.length > 1 ? order.slice(0, -1) : order;
+    const live = `from ${paged.table} where ${paged.live}`;
+    const newest = `order by ${order.map((column) => `${column} desc`).join(", ")} limit ?`;
+    const sql = {
+        changes: db.prepare(`select * from ${paged.table} where revision > ?`),
+        live: db.prepare(`select * ${live}`),
+        top: db.prepare(`select * ${live} ${newest}`),
+        older: db.prepare(`select * ${live} and (${order.join(", ")}) < (${marks(order)}) ${newest}`),
+        total: db.prepare(`select count(*) as n ${live}`),
+        held: db.prepare(`select count(*) as n ${live} and (${order.join(", ")}) >= (${marks(order)})`),
+        split: db.prepare(`select 1 as n ${live} and (${group.join(", ")}) = (${marks(group)}) and ${key} < ? limit 1`),
+        above: db.prepare(`select ${group.join(", ")} ${live} and (${group.join(", ")}) > (${marks(group)}) order by ${group.map((column) => `${column} asc`).join(", ")} limit 1`),
+        named: db.prepare(`select * from ${paged.table} where ${key} = ?`),
+    };
+    const encode = (dataset, row) => JSON.stringify([dataset, paged.version, ...values(row, order)]);
+    /** A cursor's order values, or undefined for one this history can't honour. */
+    function decode(text, dataset) {
+        let parsed;
+        try {
+            parsed = JSON.parse(text);
+        }
+        catch {
+            return undefined;
+        }
+        if (!Array.isArray(parsed) ||
+            parsed.length !== order.length + 2 ||
+            parsed[0] !== dataset ||
+            parsed[1] !== paged.version) {
+            return undefined;
+        }
+        const at = parsed.slice(2);
+        return at.every((value) => typeof value === "string" || (typeof value === "number" && Number.isFinite(value)))
+            ? at.filter((value) => typeof value === "string" || typeof value === "number")
+            : undefined;
+    }
+    /** Where a page ending at `last` leaves the copy, at this revision. */
+    function place(dataset, from, last, more) {
+        const total = tally(sql.total.get());
+        if (last === undefined || !more) {
+            return { from, next: null, through: null, held: total, total };
+        }
+        const at = values(last, group);
+        const split = sql.split.get(...at, ...values(last, [key])) !== undefined;
+        const above = split ? sql.above.get(...at) : undefined;
+        return {
+            from,
+            next: encode(dataset, last),
+            through: split
+                ? above === undefined
+                    ? null
+                    : paged.through(Object.values(above))
+                : paged.through(at),
+            held: tally(sql.held.get(...values(last, order))),
+            total,
+        };
+    }
+    function assemble(request, receipts) {
+        const dataset = options.dataset();
+        const revision = options.revision();
+        const { since } = request;
+        const cursor = typeof request.page === "string" ? decode(request.page, dataset) : null;
+        const reset = request.dataset !== dataset ||
+            since <= 0 ||
+            since > revision ||
+            options.known?.(since) === false ||
+            cursor === undefined;
+        const answer = (pulled) => {
+            const reply = options.respond(pulled);
+            return { reply, bytes: Buffer.byteLength(JSON.stringify(reply)), pulled };
+        };
+        const base = { dataset, revision };
+        // A client from before paging: the whole reply, as it always was.
+        if (request.page === undefined) {
+            const after = reset ? 0 : since;
+            return answer({
+                ...base,
+                reset,
+                rows: (after === 0 ? sql.live.all() : sql.changes.all(after)).map(paged.read),
+                unpaged: options.unpaged(after),
+                receipts: [...receipts(after)],
+            });
+        }
+        /**
+         * A reply: `size` rows of `page` (fetched with one row more, to know if older ones are left), the first `count` of
+         * `kept` receipts with the rows they name, and `extra` history rows beside them; once each, as the store holds
+         * them now. Says whether it fits the caps, and how many rows it carries.
+         */
+        const reply = (fixed, page, kept, extra) => {
+            const named = kept.map((each) => options.receipts.names(each).flatMap((name) => sql.named.all(name)));
+            return (size, count = kept.length) => {
+                const rows = new Map();
+                for (const row of [...page.slice(0, size), ...extra, ...named.slice(0, count).flat()]) {
+                    rows.set(row[key], rows.get(row[key]) ?? row);
+                }
+                const made = answer({
+                    ...base,
+                    ...fixed,
+                    rows: [...rows.values()].map(paged.read),
+                    receipts: kept.slice(0, count),
+                    page: place(dataset, fixed.from, page[size - 1] ?? fixed.at, page.length > size),
+                });
+                const carried = rows.size + sizeOf(fixed.unpaged);
+                return { made, carried, fits: carried <= limits.maxRows && made.bytes <= limits.maxBytes };
+            };
+        };
+        if (!reset) {
+            const unpaged = options.unpaged(since);
+            const room = cursor === null ? 0 : limits.pageRows;
+            const page = cursor === null ? [] : sql.older.all(...cursor, room + 1);
+            // The cursor as the row it stands for: a reply with no older row leaves the copy there.
+            const at = cursor === null
+                ? {}
+                : {
+                    at: Object.fromEntries(order.map((column, index) => [column, cursor[index] ?? null])),
+                };
+            const build = reply({ reset: false, from: request.page, unpaged, ...at }, page, receipts(since), sql.changes.all(since));
+            const alone = build(0);
+            // Only a delta (with its receipts' rows and the other tables) that can't fit by itself starts the copy over; the
+            // older page takes the rows and bytes it leaves.
+            if (alone.fits) {
+                const most = Math.min(room, page.length, limits.maxRows - alone.carried);
+                return build(largest(1, most, (size) => build(size).fits) ?? 0).made;
+            }
+        }
+        // Everything, from the newest page: one row of it at least, then as many receipts (with their rows) as fit, then
+        // as many more of the first rows. A receipt left out is asked for again by the next pull.
+        const kept = receipts(0);
+        const page = sql.top.all(limits.firstRows + 1);
+        const least = Math.min(1, page.length);
+        const build = reply({ reset: true, from: null, unpaged: options.unpaged(0) }, page, kept, []);
+        const count = largest(0, kept.length, (at) => build(least, at).fits);
+        if (count === undefined) {
+            throw new Error(`pull: the reply can't fit its caps (${limits.maxRows} rows, ${limits.maxBytes} bytes), even with one row of history`);
+        }
+        const most = Math.min(limits.firstRows, page.length);
+        return build(largest(least, most, (at) => build(at, count).fits) ?? least, count).made;
+    }
+    /** The reply to one request, its encoded size, and what it carries before `respond` shaped it. */
+    function pull(request, caller) {
+        const own = !db.isTransaction;
+        if (own) {
+            db.exec("begin");
+        }
+        try {
+            return assemble(request, (since) => options.receipts.read(request.commands === undefined ? { since } : { since, commands: request.commands }, caller));
+        }
+        finally {
+            if (own) {
+                db.exec("commit");
+            }
+        }
+    }
+    return { pull };
+}

package/dist/sync.d.ts

@@ -5,11 +5,22 @@
  * headless simulation over real stores and in the browser alike.
  *
  * Invariants it owns:
- * - **The replica only moves forward.** A pull applies only if newer (or a reset for another dataset). The disk commit
- *   is decided again against what the disk holds, inside its own transaction (`commitPlan`), so another tab or a
- *   failed write can't make the disk go back or skip a patch.
+ * - **The replica only moves forward.** A delta applies only to the same dataset with `since ≤ copy.revision ≤
+ *   reply.revision`; only a reset replaces the copy (a late one is dropped), and a reply it can't follow asks for a
+ *   real reset instead of leaving a gap. The disk commit is decided again against what the disk holds, inside its own
+ *   transaction (`commitPlan`), so another tab or a failed write can't make the disk go back or skip a patch.
+ * - **A copy never claims more history than it holds.** It is complete or partial (`History`): every row it holds is
+ *   the server's row at its revision, and a partial copy holds every row at or above its cursor, so every row at or
+ *   above `through`. The first reply draws the newest rows at once; the cursor moves only when a reply answers the
+ *   copy's own `next`, a reply without a page never changes it, and a row merged below `through` never deepens it.
+ *   The disk takes a page's history only from the cursor it moved from, in the transaction that writes its rows.
+ * - **A partial copy owes its next page**, back to back, until it is complete; a reply that doesn't move it on counts
+ *   as a failed pull, for backoff. An old server's reply to a partial copy asks for its whole copy instead.
+ * - **Only a complete copy is checked.** No fingerprint is asked for or compared while the copy is partial.
+ * - **A chunk costs its own size.** Merging a reply doesn't copy the tables it lands in (`Layered`).
  * - **Observed is not saved.** `saved` is what the disk is known to hold; a failed commit is retried whole until the
- *   disk catches up, and until then the page says so.
+ *   disk catches up (a later page the disk takes as a delta under its own history doesn't), and until then the page
+ *   says so.
  * - **A pull is owed while the server is ahead.** `target` is the newest revision the server is known to have (pokes,
  *   which also beat every 20 s, push replies, receipts); while the replica is behind it a pull is in flight or
  *   scheduled, whatever the socket says. Liveness, the known head and the last reconciliation are separate facts.
@@ -17,7 +28,8 @@
  *   earlier page keep theirs.
  * - **Nothing is sent before it is kept.** An intent the device can't keep fails at once, visibly, and is never sent.
  * - **Every command reaches an outcome, and is settled only once the replica holds its effects** (a receipt names the
- *   revision that made it visible); until then a pull is owed.
+ *   revision that made it visible; on a partial copy, only a pull reply that carried the receipt, with the rows it
+ *   names, settles it); until then a pull is owed, and names the commands it waits on, any page's.
  * - **Every waiting state ends.** Pulling, pushing, retrying and saving have deadlines or backoff; storage that is slow
  *   to load or keep is said (the page goes on from the server meanwhile), and its late answer still counts: a slow
  *   write is not a failed one.
@@ -65,12 +77,39 @@ export declare function table<T, P, R>(def: {
     get: (tables: T) => ReadonlyMap<Key, R>;
     set: (tables: T, rows: ReadonlyMap<Key, R>) => T;
 }): Lens<T, P>;
+/**
+ * How much of the server's history a copy holds. Complete: every row. Partial: every row at or above the keyset cursor
+ * `next` (the server's opaque cursor of the next older page), so every row at or above `through` (the fully covered
+ * boundary, as the app formats it; null while no order group is whole); `held` of `total` is progress, not a claim.
+ */
+export type History = {
+    kind: "complete";
+} | {
+    kind: "partial";
+    next: string;
+    through: string | null;
+    held: number;
+    total: number;
+};
+/**
+ * A paged reply's place in the history: `from` is the cursor it answered (null on a reset's first page), `next` the
+ * next older one (null once complete), `through`, `held` and `total` as in `History`, at the reply's revision.
+ */
+export type Page = {
+    from: string | null;
+    next: string | null;
+    through: string | null;
+    held: number;
+    total: number;
+};
 export type Replica<T, S> = {
     dataset: string | null;
     revision: number;
     tables: T;
     /** What the server says about itself, read with every pull. */
     server: S | null;
+    /** How much of the history it holds; absent (a copy from before paging) is complete. */
+    history?: History;
 };
 /** A pull reply as sync sees it: rows changed since the pull's `since`, or everything (`reset`). */
 export type Patch<P, S> = {
@@ -80,12 +119,18 @@ export type Patch<P, S> = {
     rows: P;
     server: S;
     checksum?: string;
+    /** A paging server's place in the history: absent from an old server, whose reset is the whole copy. */
+    page?: Page;
 };
-/** A command's outcome as the server recorded it, and the revision at which its effects are visible. */
+/**
+ * A command's outcome as the server recorded it, and the revision at which its effects are visible. `client` names an
+ * earlier page's command (absent: the pull's or push's own client).
+ */
 export type Receipt<O> = {
     id: number;
     outcome: O;
     revision: number;
+    client?: string;
 };
 export type CommandState<O> = {
     kind: "saving";
@@ -160,8 +205,18 @@ export type Model<T, S, C, O> = {
         writing: number;
         failures: number;
         nextAt: number;
+        /**
+         * A write failed and only a whole one (or one the disk took whole) repairs it: a later page landing as a delta
+         * doesn't, since the disk still lacks the failed page's rows.
+         */
+        owed: boolean;
     };
     target: number;
+    /**
+     * A reset this copy asks for: `paged` after a reply it couldn't apply (another dataset without a reset, a gap),
+     * `whole` (no `page`, as an old server understands) after an old server answered a partial copy.
+     */
+    restart: "paged" | "whole" | null;
     /** When a pull last succeeded: freshness, apart from the socket's liveness. */
     reconciledAt: number | null;
     connection: Connection;
@@ -189,6 +244,8 @@ export type Model<T, S, C, O> = {
         failures: number;
     };
     commands: readonly Pending<C, O>[];
+    /** Commands (`client:id`) whose receipt came in a pull reply this copy applied, so with the rows it names. */
+    carried: ReadonlySet<string>;
     /** Commands an earlier page kept that this version can't read: left on disk, and said. */
     unreadable: number;
     visible: boolean;
@@ -242,11 +299,15 @@ export type Msg<T, P, S, C, O> = {
     failure: Failure;
     now: number;
 }
-/** The disk holds the replica at `revision` (perhaps newer than asked, from another tab). */
+/**
+ * The disk holds the replica at `revision` (perhaps newer than asked, from another tab); `whole: false` when it took
+ * the patch as a delta under its own, other history.
+ */
  | {
     type: "committed";
     dataset: string;
     revision: number;
+    whole?: boolean;
     now: number;
 } | {
     type: "commitFailed";
@@ -311,6 +372,8 @@ export type Commit<T, S> = {
     reset: boolean;
     /** The revision the patch starts from: the disk takes it as a delta only if it is at least there. */
     base: number;
+    /** The cursor the copy held before this patch: the disk adopts the patch's history only from that same cursor. */
+    from?: string | null;
     /** The dataset this page last knew the disk to hold: the only one it may replace with another. */
     replaces: string | null;
 };
@@ -322,6 +385,13 @@ export type Effect<T, S, C, O> = {
     dataset: string | null;
     since: number;
     verify: boolean;
+    /** The older page wanted (null: none, but a paged reset is understood); absent asks an old server's whole reply. */
+    page?: string | null;
+    /** Commands still waiting on an outcome or on their effects, any page's: their receipts come with their rows. */
+    commands: {
+        client: string;
+        id: number;
+    }[];
 } | {
     kind: "push";
     client: string;
@@ -378,6 +448,8 @@ export type Spec<T, P, S, C, O> = {
      * until it resolves or the person dismisses it. Default: every outcome is resolved.
      */
     resolved?: (pending: Pending<C, O>, model: Model<T, S, C, O>) => boolean;
+    /** The most commands one push carries: the push endpoint's own bound. Default `PUSH_MAX`. */
+    pushMax?: number;
 };
 /** The socket must be heard this often (the server pokes every 20 s); quieter, the page polls. */
 export declare const SOCKET_DEADLINE_MS = 45000;
@@ -385,37 +457,92 @@ export declare const SOCKET_DEADLINE_MS = 45000;
 export declare const POLL_MS = 4000;
 /** Storage gets this long to load, or to keep an intent, before the page goes on without it. */
 export declare const STORAGE_DEADLINE_MS = 3000;
+/** Commands per push unless the spec says otherwise; the rest wait for the next push. */
+export declare const PUSH_MAX = 50;
 /** Settled commands stay drawn this long. */
 export declare const DONE_KEEP_MS = 60000;
 /** Whether the server can be asked now. */
 export declare function reachable(connection: Connection): boolean;
-/** Whether the replica holds this command's effects. */
+/**
+ * Whether the replica holds this command's effects: it is done, and the copy is at its revision and either complete or
+ * holding the rows its receipt names (a pull reply carried them). One that failed on the page has none.
+ */
 export declare function settled<T, S, C, O>(model: Model<T, S, C, O>, pending: Pending<C, O>): boolean;
-/** Whether what the page shows is also on the disk. */
+/**
+ * Whether what the page shows is also on the disk: by revision only, so tabs at different cursors never chase each
+ * other's writes. A failed write is owed again even at the same revision (a page that failed to land).
+ */
 export declare function saved<T, S, C, O>(model: Model<T, S, C, O>): boolean;
 /**
  * What a commit does to the disk, decided inside the disk's own transaction from what it holds: nothing when the disk
- * already has this or newer (another tab got there), the patch alone when the disk is at least at its base, else the
- * whole replica, so a skipped or failed patch is never covered by a later checkpoint. A page may replace the disk's
+ * already has this or newer (another tab got there; a complete copy at the same revision is newer than any page), the
+ * patch alone when the disk is at least at its base, else the whole replica, so a skipped or failed patch is never
+ * covered by a later checkpoint. A delta keeps the disk's own history, and takes the patch's only when the disk is at
+ * the cursor the patch moved from (`adopt`): coverage never lands without its rows. A page may replace the disk's
  * dataset only with one it moved to from that very dataset (`replaces`): a tab still on a dataset the server has since
  * replaced is `stale` and never writes it back.
  */
 export declare function commitPlan(stored: {
     dataset: string | null;
     revision: number;
+    history?: History;
 }, commit: {
     dataset: string | null;
     revision: number;
     reset: boolean;
     base: number;
     replaces: string | null;
-}): "skip" | "delta" | "whole" | "stale";
+    from?: string | null;
+}): "skip" | "delta" | "adopt" | "whole" | "stale";
+/** How much history the page holds, as a closed subject for the app's state table. */
+export type HistoryState = {
+    kind: "loading";
+} | {
+    kind: "none";
+} | {
+    kind: "complete";
+} | {
+    kind: "partial";
+    through: string | null;
+    held: number;
+    total: number;
+};
+/**
+ * The copy's history: loading while storage loads, none without a dataset, else partial or complete. Partial is a claim
+ * beside the numbers ("history from <through>, loading older · held of total"), and totals cover only `through` and
+ * above. Complete says nothing about storage: that is `deriveDurabilityState`, its own subject.
+ */
+export declare function deriveHistoryState<T, S, C, O>(model: Model<T, S, C, O>): HistoryState;
+/** Whether the page's copy is kept on this device, as a closed subject apart from how much history it holds. */
+export type DurabilityState = {
+    kind: "loading";
+} | {
+    kind: "unsaved";
+} | {
+    kind: "lost";
+    reason: string;
+} | {
+    kind: "failing";
+    reason: string;
+} | {
+    kind: "replaced";
+} | {
+    kind: "unreadable";
+    count: number;
+} | {
+    kind: "ok";
+};
+/**
+ * The copy's durability at the model's clock: loading, replaced by another tab, a failing store (lost while the page
+ * shows more than the disk holds), unsaved past the storage deadline, commands kept that this version can't read, or ok.
+ */
+export declare function deriveDurabilityState<T, S, C, O>(model: Model<T, S, C, O>): DurabilityState;
 export declare function createSync<T, P, S, C, O>(spec: Spec<T, P, S, C, O>): {
     initial: (client: string, now: number) => Model<T, S, C, O>;
     update: (model: Model<T, S, C, O>, msg: Msg<T, P, S, C, O>) => readonly [Model<T, S, C, O>, readonly Effect<T, S, C, O>[]];
-    merge: (replica: Replica<T, S>, patch: Patch<P, S>) => {
+    merge: (replica: Replica<T, S>, patch: Patch<P, S>, since: number) => {
         replica: Replica<T, S>;
         ops: Op[];
         reset: boolean;
-    } | undefined;
+    } | "gap" | undefined;
 };