slopkit 0.1.2 → 0.3.0

package/DEPS.md

@@ -29,7 +29,8 @@ seeded network's PRNG and scheduler.
 ## Budgets
 
 - **Kit client core**, gated by `pack:check` on the packed package: `schema`, `contract`, `sync`, `client`, `machine`,
-  `money`, `digest`, `states` and `web/replica`, minified and gzipped together, ≤ 10,730 bytes (10,726 measured).
+  `money`, `digest`, `states` and `web/replica`, minified and gzipped together, ≤ 13,951 bytes (13,951 measured; the
+  paged first copy added 1,470, refused pushes and the bounded outbox 1,755).
   `cron` is browser-safe but outside the core; an app that ships it counts it in its own budget. The budget grows only
   with a new entry or guarantee, by its measured size.
 - **Hello app** (preact, signals, and `sync`, `client`, `web/replica`, `schema`, `contract`, `states`): ≤ 17 KB gzip

package/README.md

@@ -94,6 +94,57 @@ export default defineConfig({
 });
 ```
 
+push handlers return per-command outcomes as ledger receipts. once any command in a request has acted, never throw
+an `invalid`, `forbidden` or `conflict` refusal for the whole request: a refusal proves only that attempt did not act,
+not that an earlier delivery did not. request schemas are checked before the handler runs.
+
+a push is at most `PUSH_BYTES` (48 KiB) of UTF-8 json, within `serveWeb`'s default `bodyLimit` (64 KiB): never set
+`bodyLimit` below the page's `pushBytes`. the outbox admits at most `OUTBOX_MAX` (500) commands across tabs: place
+`deriveOutboxState(model)` in its own table (`waiting` needs a connection, `finished` needs dismissing), and a
+command that is `unknown` and `parked` as needing attention. a `Failure` of kind `unreadable` is a reply the page
+couldn't read: it may have acted.
+
+### sync in pages
+
+a device with no copy draws its newest rows after one round trip, whatever the history's size, then pulls older pages
+back to back until it holds everything. the server half is `slopkit/server/pull`: one history table paged newest
+first by a keyset order, the other tables whole, receipts with the rows they name, all in one read transaction and
+within your caps.
+
+```ts
+import { createPull } from "slopkit/server/pull";
+
+const pulls = createPull({
+  db, dataset: () => store.dataset(), revision: () => store.revision(),
+  // newest first by (date, id), descending; id is unique. bump version when the order changes
+  paged: { table: "transactions", order: ["date", "id"], version: 1, live: "deleted = 0", read: toTransaction,
+    through: ([date]) => String(date) },
+  unpaged: (since) => ({ accounts: store.accounts(since) }), // whole on every reset
+  // the commands the page names (any of its pages'), or an old page's own receipts since `since`
+  receipts: { read: ({ since, commands }, viewer: Viewer) => commands === undefined
+      ? ledger.receipts(viewer.principal, viewer.client, since).map((each) => ({ ...each, client: viewer.client }))
+      : ledger.receiptsOf(viewer.principal, commands),
+    names: (each) => [each.receipt.transaction] },
+  limits: { firstRows: 200, pageRows: 2000, maxRows: 5000, maxBytes: 2_000_000 },
+  // your endpoint's reply: the caps measure its json
+  respond: ({ rows, unpaged, receipts, ...rest }) => ({ ...rest, transactions: rows, ...unpaged, commands: receipts }),
+});
+
+route(pull, (request, caller) => pulls.pull(request, viewerOf(request, caller)).reply);
+```
+
+the page's pull request carries `page` (the engine fills it in; a request without it gets the old whole reply), and
+the copy's history is its own subject: place `deriveHistoryState(model)` (loading, none, partial, complete) and
+`deriveDurabilityState(model)` (the storage states) in two tables and compose both. partial is a claim beside the
+numbers, "history from <through>, loading older · held of total", and totals cover only `through` and above. run the
+pull contract against your own pull in your tests:
+
+```ts
+import { pullConformance } from "slopkit/conformance/pull";
+
+pullConformance({ name: "my app", create: async (limits) => fixtureOverMyRealStore(limits) });
+```
+
 the shared lint rules go on top of ultracite's presets (`oxlint.config.ts`):
 
 ```ts

package/dist/client.d.ts

@@ -7,7 +7,7 @@
  * the app's own hooks (`answered`, for example opening or closing a provider's window, and `telemetry`).
  */
 import type { Result } from "./contract.ts";
-import type { Commit, createSync, Model, Msg, Patch, Pending, Receipt, Replica } from "./sync.ts";
+import type { Commit, createSync, Model, Msg, OutboxFull, Patch, Pending, Receipt, Replica } from "./sync.ts";
 /** The disk was forgotten or moved to another dataset under this page (another tab): its writes stop, and it reloads. */
 export declare class Replaced extends Error {
     constructor();
@@ -19,17 +19,34 @@ export type StoragePort<T, S, C, O> = {
         commands: Pending<C, O>[];
         unreadable: number;
     }>;
-    /** Commit a patch, decided against the disk inside its transaction; returns what the disk now holds. */
+    /**
+     * Commit a patch, decided against the disk inside its transaction; returns what the disk now holds, and `whole:
+     * false` when it took the patch as a delta under its own history (so it lacks rows of this page's copy).
+     */
     commit: (commit: Commit<T, S>) => Promise<{
         dataset: string;
         revision: number;
+        whole?: boolean;
     }>;
-    /** Write one command (an intent, or its outcome) under its key. */
-    keep: (pending: Pending<C, O>) => Promise<void>;
+    /**
+     * Write one command (an intent, or its outcome) under its key. A key the outbox doesn't hold yet is admitted only
+     * while it holds fewer than `max` commands (default `OUTBOX_MAX`), decided inside the write's own transaction so
+     * every tab shares the bound; otherwise nothing is written and the store's counts come back. A kept key always
+     * updates, and nothing kept is ever dropped for room.
+     */
+    keep: (pending: Pending<C, O>, max?: number) => Promise<void | OutboxFull>;
     drop: (keys: {
         client: string;
         id: number;
     }[]) => Promise<void>;
+    /**
+     * Journal a push before it is sent: these kept commands move to `sending`, all in one write (a key not kept is left
+     * alone). A command loaded so is uncertain, since that attempt may have acted.
+     */
+    mark: (keys: {
+        client: string;
+        id: number;
+    }[]) => Promise<void>;
 };
 export type NetworkPort<P, S, C, O> = {
     pull: (request: {
@@ -37,6 +54,16 @@ export type NetworkPort<P, S, C, O> = {
         dataset: string | null;
         since: number;
         verify: boolean;
+        /**
+         * The older page wanted: null for none (a paged reset is understood), the copy's `next` while partial. Absent asks
+         * for an old server's whole reply.
+         */
+        page?: string | null;
+        /** Commands waiting on an outcome or on their effects, any page's: their receipts come with the rows they name. */
+        commands?: {
+            client: string;
+            id: number;
+        }[];
     }) => Promise<Result<{
         patch: Patch<P, S>;
         receipts: Receipt<O>[];

package/dist/client.js

@@ -37,6 +37,10 @@ export function createClient(options) {
                     dataset: effect.dataset,
                     since,
                     verify: effect.verify,
+                    // An old server's whole copy is asked as an old client asks it: with neither.
+                    ...(effect.page === undefined
+                        ? {}
+                        : { page: effect.page, commands: effect.commands }),
                 })
                     .then((result) => result.ok
                     ? { type: "pulled", since, ...result.value, now: now() }
@@ -49,7 +53,7 @@ export function createClient(options) {
             case "push": {
                 const ids = effect.commands.map((each) => each.id);
                 const client = effect.client;
-                return track(ports.network
+                const send = () => ports.network
                     .push({
                     client,
                     dataset: effect.dataset,
@@ -64,6 +68,15 @@ export function createClient(options) {
                     ids,
                     failure: { kind: "offline", message: reason(error) },
                     now: now(),
+                }));
+                // Journalled before it is sent: a page that dies mid-push leaves these commands uncertain on disk.
+                return track(ports.storage.mark(ids.map((id) => ({ client, id }))).then(send, (error) => ({
+                    type: "unsent",
+                    client,
+                    ids,
+                    reason: reason(error),
+                    replaced: error instanceof Replaced,
+                    now: now(),
                 })), dispatch);
             }
             case "commit": {
@@ -76,7 +89,17 @@ export function createClient(options) {
             }
             case "keep": {
                 const { client, id } = effect.pending;
-                return track(ports.storage.keep(effect.pending).then(() => ({ type: "kept", client, id, now: now() }), (error) => ({
+                return track(ports.storage.keep(effect.pending, effect.max).then((kept) => kept === undefined
+                    ? { type: "kept", client, id, now: now() }
+                    : {
+                        type: "full",
+                        client,
+                        id,
+                        waiting: kept.waiting,
+                        finished: kept.finished,
+                        max: kept.max,
+                        now: now(),
+                    }, (error) => ({
                     type: "keepFailed",
                     client,
                     id,

package/dist/conformance/client.test.js

@@ -13,7 +13,7 @@ function peer() {
     const sent = [];
     const pulls = [];
     let loseNextPush = false;
-    let invalidNextPush = false;
+    let unreadableNextPush = false;
     let rejectPull = false;
     let unavailable = false;
     const network = {
@@ -57,11 +57,11 @@ function peer() {
                 loseNextPush = false;
                 return { ok: false, failure: { kind: "offline", message: "reply lost after execution" } };
             }
-            if (invalidNextPush) {
-                invalidNextPush = false;
+            if (unreadableNextPush) {
+                unreadableNextPush = false;
                 return {
                     ok: false,
-                    failure: { kind: "invalid", message: "unreadable reply after execution" },
+                    failure: { kind: "unreadable", message: "unreadable reply after execution" },
                 };
             }
             return { ok: true, value: { receipts: answers, revision } };
@@ -75,8 +75,8 @@ function peer() {
         loseReply: () => {
             loseNextPush = true;
         },
-        invalidReply: () => {
-            invalidNextPush = true;
+        unreadableReply: () => {
+            unreadableNextPush = true;
         },
         failPull: () => {
             rejectPull = true;
@@ -315,7 +315,7 @@ describe("independent client executor conformance", () => {
         const remote = peer();
         const app = client(stored, remote.network, "page-a", { now: 0 });
         await start(app, stored);
-        remote.invalidReply();
+        remote.unreadableReply();
         app.runtime.command({ name: "put", value: 125 });
         await app.runtime.settle();
         expect(remote.effects()).toBe(1);

package/dist/conformance/fixtures.d.ts

@@ -1,7 +1,7 @@
 import type { NetworkPort } from "../client.ts";
 import type { Infer, Schema } from "../schema.ts";
 import type { Key, Pending, Replica } from "../sync.ts";
-declare const rowSchema: Schema<{
+export declare const rowSchema: Schema<{
     deleted: boolean;
     id: string;
     revision: number;
@@ -14,7 +14,7 @@ export type Tables = {
 export type PatchRows = {
     rows: Row[];
 };
-declare const serverSchema: Schema<{
+export declare const serverSchema: Schema<{
     build: string;
 }>;
 export type Server = Infer<typeof serverSchema>;
@@ -40,6 +40,58 @@ export declare const outcomeSchema: Schema<({
     kind: "unknown";
 })>;
 export type Outcome = Infer<typeof outcomeSchema>;
+export declare const pendingSchema: Schema<{
+    at: number;
+    client: string;
+    command: {
+        name: "put";
+        value: number;
+    };
+    dataset: string | null;
+    id: number;
+    since?: number;
+    state: ({
+        kind: "saving";
+    } & {
+        kind: "saving";
+    }) | ({
+        kind: "queued";
+    } & {
+        kind: "queued";
+    }) | ({
+        kind: "sending";
+    } & {
+        kind: "sending";
+    }) | ({
+        failure: string;
+        kind: "unknown";
+        parked?: true;
+    } & {
+        kind: "unknown";
+    }) | ({
+        kind: "done";
+        outcome: ({
+            kind: "ok";
+            resource: string;
+        } & {
+            kind: "ok";
+        }) | ({
+            kind: "failed";
+            reason: string;
+        } & {
+            kind: "failed";
+        }) | ({
+            kind: "unknown";
+            reason: string;
+        } & {
+            kind: "unknown";
+        });
+        revision: number;
+    } & {
+        kind: "done";
+    });
+}>;
+export declare const rowsLens: import("../sync.ts").Lens<Tables, PatchRows>;
 export declare const sync: {
     initial: (client: string, now: number) => import("../sync.ts").Model<Tables, {
         build: string;
@@ -147,7 +199,7 @@ export declare const sync: {
         build: string;
     }>, patch: import("../sync.ts").Patch<PatchRows, {
         build: string;
-    }>) => {
+    }>, since: number) => "gap" | {
         replica: Replica<Tables, {
             build: string;
         }>;

package/dist/conformance/fixtures.js

@@ -3,20 +3,20 @@ import { IDBFactory } from "fake-indexeddb";
 import { boolean, literal, nullable, number, object, optional, string, variant, } from "../schema.js";
 import { createSync, table } from "../sync.js";
 import { indexedReplica } from "../web/replica.js";
-const rowSchema = object({
+export const rowSchema = object({
     id: string(),
     value: number({ integer: true }),
     deleted: boolean(),
     revision: number({ integer: true, min: 0 }),
 }, { strict: true });
-const serverSchema = object({ build: string() }, { strict: true });
+export const serverSchema = object({ build: string() }, { strict: true });
 const commandSchema = object({ name: literal("put"), value: number({ integer: true }) }, { strict: true });
 export const outcomeSchema = variant("kind", {
     ok: object({ kind: literal("ok"), resource: string() }, { strict: true }),
     failed: object({ kind: literal("failed"), reason: string() }, { strict: true }),
     unknown: object({ kind: literal("unknown"), reason: string() }, { strict: true }),
 });
-const pendingSchema = object({
+export const pendingSchema = object({
     client: string(),
     id: number({ integer: true, min: 1 }),
     dataset: nullable(string()),
@@ -27,7 +27,11 @@ const pendingSchema = object({
         saving: object({ kind: literal("saving") }),
         queued: object({ kind: literal("queued") }),
         sending: object({ kind: literal("sending") }),
-        unknown: object({ kind: literal("unknown"), failure: string() }),
+        unknown: object({
+            kind: literal("unknown"),
+            failure: string(),
+            parked: optional(literal(true)),
+        }),
         done: object({
             kind: literal("done"),
             outcome: outcomeSchema,
@@ -36,7 +40,7 @@ const pendingSchema = object({
     }),
 });
 const empty = { rows: new Map() };
-const rowsLens = table({
+export const rowsLens = table({
     name: "rows",
     schema: rowSchema,
     rows: (patch) => patch.rows,

package/dist/conformance/legacy/replica-012.d.ts

@@ -0,0 +1,52 @@
+import type { Schema } from "../../schema.ts";
+import type { Lens, Op, Pending } from "../../sync.ts";
+type Replica<T, S> = {
+    dataset: string | null;
+    revision: number;
+    tables: T;
+    server: S | null;
+};
+type StoragePort<T, S, C, O> = {
+    load: () => Promise<{
+        replica: Replica<T, S>;
+        commands: Pending<C, O>[];
+        unreadable: number;
+    }>;
+    commit: (commit: {
+        replica: Replica<T, S>;
+        ops: Op[];
+        reset: boolean;
+        base: number;
+        replaces: string | null;
+    }) => Promise<{
+        dataset: string;
+        revision: number;
+    }>;
+    keep: (pending: Pending<C, O>) => Promise<void>;
+    drop: (keys: {
+        client: string;
+        id: number;
+    }[]) => Promise<void>;
+};
+export declare function indexedReplica<T, P, S, C, O>(options: {
+    factory: IDBFactory;
+    name: string;
+    /** The object stores' layout: bump when a table is added or renamed. Row shapes are checked on their own. */
+    version: number;
+    tables: readonly Lens<T, P>[];
+    empty: T;
+    server: Schema<S>;
+    pending: Schema<Pending<C, O>>;
+    interfere?: (what: "commit" | "keep" | "drop", transaction: IDBTransaction) => void;
+    /**
+     * An outbox an earlier release kept in another store: on upgrade each record is moved into the outbox, converted to
+     * a command (or kept as it was, under its own key, to be counted unreadable). Never dropped.
+     */
+    legacy?: {
+        store: string;
+        convert: (value: unknown) => Pending<C, O> | undefined;
+    };
+}): StoragePort<T, S, C, O> & {
+    forget: () => Promise<void>;
+};
+export {};