slopkit 0.2.0 → 0.4.0-rc.0

package/dist/server/store.js

@@ -0,0 +1,347 @@
+/**
+ * The replicated store's core: a home's SQLite file (Node's built-in SQLite, WAL), owned by the one server under the
+ * home's lock, that a replica pulls from. An app's store extends `SyncStore` (or holds one) and adds its own tables and
+ * queries; the mechanics every replicated store needs are here.
+ *
+ * - **Nothing opens a store it can't read.** A store from a newer release, or older than the schema this release
+ *   starts from, is refused before anything is written to it.
+ * - **Migrations are append only** and each runs in its own transaction, foreign keys checked at its end; a store with
+ *   data is first copied whole (`<path>.schema-<n>.bak`, the oldest kept), so a rollback can put it back.
+ * - **Every write is stamped:** the rows a transaction writes to a replicated table carry the revision `stamp()` made
+ *   in it (a trigger refuses any other), and nothing replicated is deleted (tombstones). Each revision is logged, so a
+ *   page holding one this history never had (a restore's lost future) is told apart from one that is behind.
+ * - **A restore is told apart:** the newest revision served is kept beside the store (`<path>.head`), written before
+ *   the reply carrying it leaves; a store that opens behind it went back, and takes a new dataset.
+ * - **Retention never deletes a key:** receipts retire, old revisions are forgotten (a page holding one starts over),
+ *   finished jobs beyond the newest are pruned.
+ */
+import { randomBytes } from "node:crypto";
+import { closeSync, existsSync, fsyncSync, openSync, readFileSync, renameSync, writeSync, } from "node:fs";
+import { dirname } from "node:path";
+import { DatabaseSync } from "node:sqlite";
+import { check, number, object, string } from "../schema.js";
+import { Jobs } from "./jobs.js";
+import { Ledger } from "./ledger.js";
+/**
+ * The core tables every replicated store has, before the app's own: `meta`, the command ledger's `commands`
+ * (`slopkit/server/ledger`), `jobs` (`slopkit/server/jobs`) and the `revisions` log. Never edited once shipped.
+ */
+export const CORE_TABLES = `create table meta(key text primary key, value text not null);
+   create table commands(
+     principal text not null check (length(principal) > 0),
+     client text not null, id integer not null check (id > 0),
+     name text not null, args text not null,
+     state text not null check (state in ('accepted', 'done', 'retired')),
+     outcome text check ((state = 'done') = (outcome is not null)),
+     revision integer check ((state = 'accepted') = (revision is null)),
+     created_at text not null,
+     primary key (principal, client, id)) strict;
+   create index commands_revision on commands(principal, client, revision);
+   create trigger commands_never_deleted before delete on commands
+     begin select raise(abort, 'a command key is kept forever: retire it'); end;
+   create table jobs(
+     id integer primary key,
+     kind text not null check (length(kind) > 0),
+     key text not null,
+     state text not null check (state in ('due', 'running', 'done', 'failed')),
+     due_at text not null, deadline_at text not null,
+     attempts integer not null default 0 check (attempts >= 0),
+     note text,
+     result text check ((state = 'done') = (result is not null)),
+     who text check ((state = 'failed') = (who is not null) and (who is null or who in ('you', 'us', 'gone'))),
+     reason text check ((state = 'failed') = (reason is not null)),
+     asked_by text not null default '[]',
+     created_at text not null, updated_at text not null);
+   create unique index jobs_open on jobs(kind, key) where state in ('due', 'running');
+   create table revisions(revision integer primary key, at text not null) strict;
+   insert into meta(key, value) values ('revision', '1'), ('history_floor', '1'),
+     ('dataset', lower(hex(randomblob(12))));`;
+/** A replicated table's triggers: every write carries the revision stamped in its own transaction; no delete. */
+function stamped(table) {
+    const revision = "(select cast(value as integer) from meta where key = 'revision')";
+    const stamping = "(select cast(value as integer) from meta where key = 'stamping')";
+    return [
+        ...["insert", "update"].map((event) => `create trigger ${table}_stamped_${event} before ${event} on ${table}
+     when new.revision is not ${revision} or ${revision} is not ${stamping}
+     begin select raise(abort, 'a replicated row must carry a revision stamped in its own transaction'); end;`),
+        `create trigger ${table}_never_deleted before delete on ${table}
+     begin select raise(abort, 'a replicated row is never deleted: tombstone it'); end;`,
+    ].join("\n");
+}
+/** The store schema a release writes: what `version --json` reports and a rollback compares. */
+export function schemaVersion(schema) {
+    return schema.baseVersion + schema.migrations.length;
+}
+const DAY = 86_400_000;
+/** What `<path>.head` holds: the newest revision of a dataset that left the server. */
+const HEAD = object({ dataset: string(), revision: number() });
+/** Retention reads no receipt or result: these stand in for an app's own codecs. */
+const OPAQUE = {
+    encode: (value) => JSON.stringify(value),
+    decode: (text) => text,
+};
+/** A safe name for a table: it is spliced into the trigger SQL. */
+const TABLE = /^[a-z_][a-z0-9_]*$/;
+export class SyncStore {
+    db;
+    path;
+    now;
+    /**
+     * Set when the store opened older than what it had already served (restored from a backup), or a caller proved it
+     * went back: the dataset and revision it had served, and its revision then. It took a new dataset, so every replica
+     * starts over and no intent made against the lost history runs here.
+     */
+    restored;
+    head;
+    servedHead;
+    depth = 0;
+    constructor(path, options) {
+        const { schema } = options;
+        for (const table of schema.replicated) {
+            if (!TABLE.test(table)) {
+                throw new Error(`not a table name: ${table}`);
+            }
+        }
+        this.path = path;
+        this.now = options.now ?? Date.now;
+        this.db = new DatabaseSync(path);
+        const version = this.userVersion();
+        const latest = schemaVersion(schema);
+        // A file at version 0 is new only while it holds nothing: one with tables is from before any release could say.
+        const empty = this.db.prepare("select count(*) as n from sqlite_master").get()?.n === 0;
+        const refused = version > latest
+            ? `The store is from a newer version (schema ${version}); update ${options.name}`
+            : (version > 0 || !empty) && version < schema.baseVersion
+                ? `The store is at schema ${version}, from before ${schema.baseVersion}, the schema this release starts from: it can't be opened`
+                : undefined;
+        if (refused !== undefined) {
+            this.db.close();
+            throw new Error(refused);
+        }
+        // FULL: a commit is on the disk before it is acknowledged.
+        this.db.exec("pragma journal_mode = wal; pragma synchronous = full; pragma foreign_keys = on;");
+        this.migrate(schema, version);
+        this.head = `${path}.head`;
+        this.restored = this.checkHead();
+    }
+    userVersion() {
+        return Number(this.db.prepare("pragma user_version").get()?.user_version ?? 0);
+    }
+    migrate(schema, version) {
+        if (version === 0) {
+            this.transaction(() => {
+                this.db.exec(CORE_TABLES);
+                this.db.exec(schema.base);
+                for (const table of schema.replicated) {
+                    this.db.exec(stamped(table));
+                }
+                this.db.exec(`pragma user_version = ${schema.baseVersion}`);
+            });
+        }
+        const from = Math.max(version, schema.baseVersion);
+        const latest = schemaVersion(schema);
+        const backup = `${this.path}.schema-${from}.bak`;
+        if (version > 0 && version < latest && !existsSync(backup)) {
+            this.db.prepare("vacuum into ?").run(backup);
+        }
+        for (let next = from; next < latest; next++) {
+            // A rebuild drops tables other tables reference; foreign keys are checked once it is done.
+            this.db.exec("pragma foreign_keys = off");
+            try {
+                this.transaction(() => {
+                    this.db.exec(schema.migrations[next - schema.baseVersion] ?? "");
+                    this.db.exec(`pragma user_version = ${next + 1}`);
+                    const broken = this.db.prepare("pragma foreign_key_check").all();
+                    if (broken.length > 0) {
+                        throw new Error(`Migration ${next + 1} leaves ${broken.length} broken references`);
+                    }
+                });
+            }
+            finally {
+                this.db.exec("pragma foreign_keys = on");
+            }
+        }
+        if (schema.indexes !== undefined) {
+            this.db.exec(schema.indexes);
+        }
+    }
+    /** The head beside the store: a store that opens behind what it served went back, and takes a new dataset. */
+    checkHead() {
+        let head;
+        try {
+            head = JSON.parse(readFileSync(this.head, "utf-8"));
+        }
+        catch {
+            return undefined;
+        }
+        const read = check(HEAD, head);
+        if (!read.ok) {
+            return undefined;
+        }
+        const { dataset, revision: served } = read.value;
+        if (dataset !== this.dataset() || this.revision() >= served) {
+            return undefined;
+        }
+        const replaced = { dataset, served, revision: this.revision() };
+        this.setMeta("dataset", randomBytes(12).toString("hex"));
+        return replaced;
+    }
+    /** A caller proved this store went back: it holds `seen` of this very dataset, more than the store has. */
+    wentBack(seen) {
+        if (seen <= this.revision()) {
+            return false;
+        }
+        this.restored = { dataset: this.dataset(), served: seen, revision: this.revision() };
+        this.setMeta("dataset", randomBytes(12).toString("hex"));
+        return true;
+    }
+    /** Record that `revision` of this dataset left the server (a pull or push reply), before the reply does. */
+    served(revision) {
+        const dataset = this.dataset();
+        const last = this.servedHead;
+        if (last !== undefined && last.dataset === dataset && last.revision >= revision) {
+            return;
+        }
+        const temporary = `${this.head}.tmp`;
+        const fd = openSync(temporary, "w", 0o600);
+        try {
+            const bytes = Buffer.from(JSON.stringify({ dataset, revision }), "utf-8");
+            for (let at = 0; at < bytes.length;) {
+                at += writeSync(fd, bytes, at, bytes.length - at);
+            }
+            fsyncSync(fd);
+        }
+        finally {
+            closeSync(fd);
+        }
+        renameSync(temporary, this.head);
+        // The rename itself on the disk too.
+        const dir = openSync(dirname(this.head), "r");
+        try {
+            fsyncSync(dir);
+        }
+        finally {
+            closeSync(dir);
+        }
+        this.servedHead = { dataset, revision };
+    }
+    close() {
+        if (this.db.isOpen) {
+            this.db.close();
+        }
+    }
+    /** All of `body` or none of it. Nested calls join the outer transaction: a failure anywhere undoes all of it. */
+    transaction(body) {
+        if (this.depth > 0) {
+            return body();
+        }
+        this.db.exec("begin immediate");
+        this.depth++;
+        try {
+            const result = body();
+            this.db.exec("delete from meta where key = 'stamping'");
+            this.db.exec("commit");
+            return result;
+        }
+        catch (error) {
+            this.db.exec("rollback");
+            throw error;
+        }
+        finally {
+            this.depth--;
+        }
+    }
+    meta(key) {
+        const value = this.db.prepare("select value from meta where key = ?").get(key)?.value;
+        return typeof value === "string" ? value : undefined;
+    }
+    setMeta(key, value) {
+        this.db
+            .prepare("insert into meta(key, value) values (?, ?) on conflict(key) do update set value = excluded.value")
+            .run(key, value);
+    }
+    /** This store's identity: a replica of another store starts over. */
+    dataset() {
+        let dataset = this.meta("dataset");
+        if (dataset === undefined) {
+            dataset = randomBytes(12).toString("hex");
+            this.setMeta("dataset", dataset);
+        }
+        return dataset;
+    }
+    revision() {
+        return Number(this.meta("revision") ?? 0);
+    }
+    /** The store's clock as stored: now, or `at` (milliseconds). */
+    iso(at) {
+        return new Date(at ?? this.now()).toISOString();
+    }
+    /**
+     * Whether `revision` is one of this history: none (0), the head, at or before the floor (the revisions before they
+     * were logged), or one this store stamped and still remembers. A page holding another came from a history this store
+     * doesn't have (a restore's lost future), or is older than the log keeps: it starts over, and its taps don't run.
+     */
+    known(revision) {
+        if (revision === 0 || revision === this.revision()) {
+            return true;
+        }
+        if (revision <= Number(this.meta("history_floor") ?? 0)) {
+            return true;
+        }
+        return (this.db.prepare("select 1 from revisions where revision = ?").get(revision) !== undefined);
+    }
+    /**
+     * The next revision, for rows written in the current transaction: past the last, and at least the clock in
+     * microseconds (plus a random part below a millisecond), so two histories forked from one copy never stamp the same
+     * number while the clock moves forward. Each is logged (`known`).
+     */
+    stamp() {
+        if (!this.db.isTransaction) {
+            throw new Error("A revision is stamped inside the transaction that writes its rows");
+        }
+        const next = Math.max(this.revision() + 1, this.now() * 1000 + Math.floor(Math.random() * 1000));
+        if (!Number.isSafeInteger(next)) {
+            throw new TypeError(`Revision ${next} is past a safe integer`);
+        }
+        this.db.prepare("insert into revisions(revision, at) values (?, ?)").run(next, this.iso());
+        this.setMeta("revision", String(next));
+        this.setMeta("stamping", String(next));
+        return next;
+    }
+    /** The command ledger (`slopkit/server/ledger`) on this store: its transaction and its stamp. */
+    createLedger(codec) {
+        return new Ledger({
+            db: this.db,
+            transaction: (body) => this.transaction(body),
+            stamp: () => this.stamp(),
+            encode: codec.encode,
+            decode: codec.decode,
+            ...(codec.recover === undefined ? {} : { recover: codec.recover }),
+        });
+    }
+    /** Jobs (`slopkit/server/jobs`) on this store. */
+    createJobs(codec) {
+        return new Jobs({
+            db: this.db,
+            transaction: (body) => this.transaction(body),
+            encode: codec.encode,
+            decode: codec.decode,
+            now: this.now,
+        });
+    }
+    /**
+     * Retention, which never deletes a key: receipts accepted more than `receiptsMs` ago retire (their outcome dropped, never acting
+     * again), logged revisions older than `revisionsMs` are forgotten except the head (a page holding one starts over),
+     * and finished jobs beyond the newest `jobsKept` of each kind are pruned.
+     */
+    retain(options = {}) {
+        const now = this.now();
+        this.transaction(() => {
+            this.createLedger(OPAQUE).retire(this.iso(now - (options.receiptsMs ?? 90 * DAY)));
+            this.db
+                .prepare("delete from revisions where at < ? and revision <> ?")
+                .run(this.iso(now - (options.revisionsMs ?? 400 * DAY)), this.revision());
+        });
+        this.createJobs(OPAQUE).prune(options.jobsKept ?? 200);
+    }
+}

package/dist/server/sync.d.ts

@@ -0,0 +1,158 @@
+/**
+ * The server's half of sync, over a replicated store (`slopkit/server/store`) and the app's sync contract
+ * (`slopkit/api`): pull and push handlers, their routes with telemetry, the poke socket, and `serveSync`, which serves
+ * them all on the kit's web server. An app supplies its tables, its commands and what its server says of itself.
+ *
+ * - **A push acts only on the history it was tapped against:** one naming another dataset, or a revision this store
+ *   doesn't know, runs none of its commands.
+ * - **Authorization before the ledger, every time,** replays included; then each command runs once through the ledger,
+ *   in order, and its receipt carries the revision where its effects are.
+ * - **Served before replied:** the store records each reply's revision (`served`) before the reply leaves.
+ * - **The socket only hints:** a poke carries the store's revision when it moves and as a heartbeat; the page pulls
+ *   whatever it hears.
+ * - **Telemetry is written, never trusted:** the page's events go to the log under the caller's identity.
+ */
+import type { KitOutcome, PullReply, PullRequest, PushReply, PushRequest, SyncApi } from "../api.ts";
+import type { CommandKey, Ledger } from "./ledger.ts";
+import type { Logger } from "./log.ts";
+import type { PullLimits, PullOptions } from "./pull.ts";
+import type { SyncStore } from "./store.ts";
+import { serveWeb } from "./web.ts";
+import type { Caller, RawRoute, Route, Socket, WebOptions } from "./web.ts";
+export type { KitOutcome } from "../api.ts";
+/** The kit's own refusal messages, said the same in every app. */
+export declare const KIT_ANSWERS: {
+    replaced: string;
+    history: string;
+    forbidden: (name: string) => string;
+    reused: (id: number) => string;
+};
+/**
+ * Nothing when the api's outcome union admits the kit's own answers, else `never`: where the handlers meet the api,
+ * an outcome schema that would refuse them doesn't compile.
+ */
+type Admits<O> = KitOutcome extends O ? unknown : never;
+/** How a command acts: only in the store, inside its claim's transaction, or elsewhere after it. */
+export type Act<K> = {
+    local: () => K;
+} | {
+    local: () => K | undefined;
+    remote: () => Promise<K>;
+};
+/** One command a push ran: its key, the command, what the ledger kept, and whether it acted just now. */
+export type Ran<C, K> = {
+    key: CommandKey;
+    command: C;
+    kept: K;
+    fresh: boolean;
+};
+/** Who asks, as the handlers read it: a `Caller`, or anything carrying its identities and whether a person. */
+export type Who = {
+    identities: readonly string[];
+    person: boolean;
+};
+/** What `serveSync` and the routes need of the handlers. */
+export type Sync<R, S, C, O, V extends Who = Caller> = {
+    pull: (request: PullRequest, caller: V) => Promise<PullReply<R, S, O>>;
+    push: (request: PushRequest<C>, caller: V) => Promise<PushReply<O>>;
+    revision: () => number;
+    subscribe: (listener: () => void) => () => void;
+};
+/** A row read from SQLite, as `createPull` hands it to `read`. */
+type HistoryOptions<R> = PullOptions<R, Record<string, readonly unknown[]>, never, never, never>["paged"];
+/**
+ * Whose commands these are: every person is the one owner (single user by design), so a person is one principal
+ * whichever login they came through; each service or key is its own.
+ */
+export declare function principalOf(caller: Who): string;
+export declare function syncHandlers<R extends Record<string, {
+    id: string | number;
+}>, S, C extends {
+    name: string;
+}, O, K extends {
+    kind: string;
+}, H extends keyof R & string, V extends Who = Caller>(options: {
+    api: SyncApi<R, S, C, O>;
+    store: SyncStore;
+    ledger: Ledger<K>;
+    log: Logger;
+    /** What the server says of itself, with every pull. */
+    server: (caller: V) => S;
+    /** The history: the table paged newest first (`slopkit/server/pull`), and its row key in the api. */
+    history: HistoryOptions<R[H]> & {
+        row: H;
+    };
+    /** The other row keys' rows changed after `since`, or every live row for 0. */
+    unpaged: (since: number) => {
+        [Key in Exclude<keyof R, H>]: R[Key][];
+    };
+    /** Every live row of every table (what a complete copy holds), for a verifying pull's checksum. */
+    everything: () => {
+        [Key in keyof R]: readonly R[Key][];
+    };
+    /** The history rows (by key) a receipt names: they come in the reply that carries it. */
+    names: (kept: K) => readonly (string | number)[];
+    limits: PullLimits;
+    act: (command: C, caller: V, key: CommandKey) => Act<K>;
+    /** A kept receipt as callers see it (a link as its row now); `(kept) => kept` when they are the same. */
+    outcome: (kept: K) => O;
+    /** Commands only a person's login may run. */
+    personOnly?: (command: C) => boolean;
+    principal?: (caller: V) => string;
+    /** After a push's commands ran, before its reply: what each did. */
+    pushed?: (ran: Ran<C, K>[], caller: V) => void;
+}): Sync<R, S, C, O | KitOutcome, V> & {
+    changed: () => void;
+};
+/**
+ * The sync contract's routes: pull, push and telemetry. Each telemetry event is one `page` log line at `info`, its
+ * attributes the event's own with the kit's (`client`, `at`, `person`) set last, so a page can't claim another sender.
+ */
+export declare function syncRoutes<R, S, C, O>(api: SyncApi<R, S, C, O>, handlers: Pick<Sync<R, S, C, NoInfer<O> | KitOutcome>, "pull" | "push"> & {
+    page: Logger;
+} & Admits<NoInfer<O>>): Route[];
+/**
+ * The poke socket: one poke with the revision on open, one for the changes within `gatherMs` (carrying the revision
+ * when it is sent), and a beat every `beatMs`, so a lost poke is made up for and a quiet socket is a dead one.
+ */
+export declare function pokes<R, S, C, O>(api: SyncApi<R, S, C, O>, options: {
+    revision: () => number;
+    subscribe: (listener: () => void) => () => void;
+    beatMs?: number;
+    gatherMs?: number;
+}): Socket;
+/** The install manifest's fields an app sets. */
+export type Manifest = {
+    id: string;
+    name: string;
+    short_name: string;
+    description?: string;
+    start_url: string;
+    scope: string;
+    display: "standalone" | "fullscreen" | "minimal-ui" | "browser";
+    background_color: string;
+    theme_color: string;
+    icons: {
+        src: string;
+        sizes: string;
+        type: string;
+        purpose?: string;
+    }[];
+};
+/**
+ * An app's web server: `serveWeb` with the sync routes, the poke socket, the session cookie `<name>_session` and the
+ * install manifest made from the name (black, `/icon.svg`), each field overridable. Extra routes are served beside them.
+ */
+/** What `serveSync` takes: `serveWeb`'s options, less what it makes from these. */
+export type SyncServerOptions<R, S, C, O> = Omit<WebOptions, "routes" | "socket" | "cookie" | "manifest"> & {
+    name: string;
+    api: SyncApi<R, S, C, O>;
+    sync: Sync<R, S, C, O | KitOutcome>;
+    /** Where the page's telemetry is written. */
+    page: Logger;
+    routes?: (Route | RawRoute)[];
+    manifest?: Partial<Manifest>;
+};
+export declare function serveSync<R, S, C, O>(options: SyncServerOptions<R, S, C, NoInfer<O>> & {
+    api: SyncApi<R, S, C, O>;
+} & Admits<NoInfer<O>>): Promise<Awaited<ReturnType<typeof serveWeb>>>;