slopkit 0.2.0 → 0.3.0

package/dist/contract.js

@@ -3,7 +3,7 @@
  * server (which checks requests and its own replies) and the page (which checks replies, since a proxy in between can
  * answer anything, such as an HTML login page). Failures are a closed set of kinds; a caller handles each by kind.
  */
-import { check, object, string } from "./schema.js";
+import { check, object, optional, string } from "./schema.js";
 export function endpoint(method, path, request, response, options = {}) {
     return { method, path, request, response, person: options.person ?? false };
 }
@@ -16,8 +16,25 @@ export const STATUS = {
     unavailable: 503,
     offline: 503,
 };
+const KINDS = [
+    "signed-out",
+    "forbidden",
+    "invalid",
+    "conflict",
+    "upstream",
+    "unavailable",
+    "offline",
+];
+/** A body that was due as JSON and wasn't. */
+const UNREADABLE = Symbol("unreadable");
 /** A failure's body, as `serveWeb` answers it: its message is shown when there is one. */
 const ERROR_BODY = object({ message: string() });
+/** The kit's own error body: a kind it can send, its message, and the schema path of a refused request. */
+const KIT_BODY = object({ kind: string(), message: string(), path: optional(string()) });
+/**
+ * A status's kind when no kit body says otherwise. A wrong endpoint or method and a body past the server's limit are
+ * `invalid` (no command path: incompatible); a timeout or rate limit is `unavailable`, retried.
+ */
 function kindOf(status) {
     if (status === 401) {
         return "signed-out";
@@ -28,6 +45,9 @@ function kindOf(status) {
     if (status === 409) {
         return "conflict";
     }
+    if (status === 408 || status === 429) {
+        return "unavailable";
+    }
     if (status === 502 || status === 504) {
         return "upstream";
     }
@@ -36,6 +56,21 @@ function kindOf(status) {
     }
     return "invalid";
 }
+/** The longest a `Retry-After` defers the next try: a bad header can delay sync, never stop it. */
+export const RETRY_AFTER_MAX_MS = 3_600_000;
+/**
+ * `Retry-After` as milliseconds from now, capped at `RETRY_AFTER_MAX_MS`: delay seconds or an HTTP date; undefined
+ * when absent or unreadable.
+ */
+function retryAfter(value) {
+    if (value === null) {
+        return undefined;
+    }
+    const at = /^\d+$/.test(value.trim())
+        ? Number(value.trim()) * 1000
+        : Date.parse(value) - Date.now();
+    return Number.isNaN(at) ? undefined : Math.min(RETRY_AFTER_MAX_MS, Math.max(0, at));
+}
 /** GET requests carry their input as one `q` query parameter of JSON; POST as the body. */
 export function encodeUrl(route, request) {
     if (route.method === "POST") {
@@ -44,8 +79,9 @@ export function encodeUrl(route, request) {
     return `${route.path}?q=${encodeURIComponent(JSON.stringify(request))}`;
 }
 /**
- * Call an endpoint from a browser (or any fetch). Every outcome is a `Result`: thrown errors, non-JSON answers (a
- * proxy's login page means signed out), error statuses and replies that break the schema all become a `Failure`.
+ * Call an endpoint from a browser (or any fetch). Every outcome is a `Result`: thrown errors, redirects and successful
+ * HTML pages (a proxy's login page means signed out), error statuses, and successes that aren't JSON or break the
+ * schema (`unreadable`) all become a `Failure`. An error is `refused` only with the kit's own body.
  */
 export async function call(route, request, options = {}) {
     const send = options.fetch ?? fetch;
@@ -70,29 +106,60 @@ export async function call(route, request, options = {}) {
             failure: { kind: "offline", message: error instanceof Error ? error.message : String(error) },
         };
     }
-    // A redirect (an opaque one, in a browser) or an HTML page is a proxy sending the browser to sign in.
+    // A redirect (an opaque one, in a browser) or a successful HTML page is a proxy sending the browser to sign in. An
+    // HTML error page keeps its status's kind.
     const type = response.headers.get("content-type") ?? "";
     if (response.type === "opaqueredirect" ||
         (response.status >= 300 && response.status < 400) ||
-        type.includes("text/html")) {
+        (response.ok && type.includes("text/html"))) {
         return { ok: false, failure: { kind: "signed-out", message: "the login expired" } };
     }
     const body = type.includes("application/json")
-        ? await response.json().catch(() => undefined)
-        : undefined;
+        ? await response.json().catch(() => UNREADABLE)
+        : UNREADABLE;
     if (!response.ok) {
+        const kit = check(KIT_BODY, body);
+        const named = kit.ok ? KINDS.find((each) => each === kit.value.kind) : undefined;
+        // Only the kit's own body, naming a kind its status carries, is the server's refusal; anything else is a status.
+        const own = kit.ok && named !== undefined && STATUS[named] === response.status;
+        const kind = own ? named : kindOf(response.status);
+        const after = kind === "unavailable" ? retryAfter(response.headers.get("retry-after")) : undefined;
+        if (own) {
+            const path = kit.value.path;
+            return {
+                ok: false,
+                failure: {
+                    kind,
+                    message: kit.value.message,
+                    refused: true,
+                    ...(path === undefined ? {} : { path }),
+                    ...(after === undefined ? {} : { retryAfterMs: after }),
+                },
+            };
+        }
         const error = check(ERROR_BODY, body);
         return {
             ok: false,
             failure: {
-                kind: kindOf(response.status),
+                kind,
                 message: error.ok ? error.value.message : `HTTP ${response.status}`,
+                ...(after === undefined ? {} : { retryAfterMs: after }),
             },
         };
     }
+    // A success the page can't read may still have acted: unreadable, never a refusal.
+    if (body === UNREADABLE) {
+        return {
+            ok: false,
+            failure: { kind: "unreadable", message: `reply: not JSON (HTTP ${response.status})` },
+        };
+    }
     const parsed = check(route.response, body);
     if (!parsed.ok) {
-        return { ok: false, failure: { kind: "invalid", message: `reply: ${parsed.error.message}` } };
+        return {
+            ok: false,
+            failure: { kind: "unreadable", message: `reply: ${parsed.error.message}` },
+        };
     }
     return { ok: true, value: parsed.value };
 }

package/dist/server/pull.js

@@ -19,6 +19,10 @@
  *   to the rows and bytes it leaves, and with no room at all the cursor stays where it was. A reset carries one row of history at least, then as many receipts as fit, then more rows; one
  *   that can't fit even so (the other tables alone are over the caps) is never sent: `pull` throws, a server fault
  *   the page sees as unavailable and retries.
+ * - **A reply costs about one encoding.** Each candidate is measured, not built: the reply as `respond` makes it with no
+ *   rows, plus each row's own encoding (every row read and encoded once) and a comma between rows; counts run once per
+ *   reply, and a candidate's coverage is read from the rows it fetched. The chosen reply is built once, and checked: one
+ *   whose bytes aren't its rows' (a `respond` that reshapes rows) is chosen again by building each candidate whole.
  * - **Receipts carry their rows.** A row a receipt names is in the reply that carries the receipt, even below the page,
  *   so a command settles only on a copy that holds its effects. A paging client names the commands it waits on (any of
  *   its pages', at most a push's worth); a receipt left out for room is asked for again by its next pull.
@@ -92,6 +96,8 @@ export function createPull(options) {
         named: db.prepare(`select * from ${paged.table} where ${key} = ?`),
     };
     const encode = (dataset, row) => JSON.stringify([dataset, paged.version, ...values(row, order)]);
+    /** Whether a row is in the order group `at`. */
+    const same = (row, at) => values(row, group).every((value, index) => value === at[index]);
     /** A cursor's order values, or undefined for one this history can't honour. */
     function decode(text, dataset) {
         let parsed;
@@ -112,27 +118,6 @@ export function createPull(options) {
             ? at.filter((value) => typeof value === "string" || typeof value === "number")
             : undefined;
     }
-    /** Where a page ending at `last` leaves the copy, at this revision. */
-    function place(dataset, from, last, more) {
-        const total = tally(sql.total.get());
-        if (last === undefined || !more) {
-            return { from, next: null, through: null, held: total, total };
-        }
-        const at = values(last, group);
-        const split = sql.split.get(...at, ...values(last, [key])) !== undefined;
-        const above = split ? sql.above.get(...at) : undefined;
-        return {
-            from,
-            next: encode(dataset, last),
-            through: split
-                ? above === undefined
-                    ? null
-                    : paged.through(Object.values(above))
-                : paged.through(at),
-            held: tally(sql.held.get(...values(last, order))),
-            total,
-        };
-    }
     function assemble(request, receipts) {
         const dataset = options.dataset();
         const revision = options.revision();
@@ -159,60 +144,166 @@ export function createPull(options) {
                 receipts: [...receipts(after)],
             });
         }
+        // Counted once per reply: every candidate page is placed from these and from the rows it was read with.
+        let total;
+        const count = () => (total ??= tally(sql.total.get()));
+        const above = new Map();
+        /** The lowest live order group above `at` (whole, so its values are the coverage boundary), once per group. */
+        const higher = (at) => {
+            const id = JSON.stringify(at);
+            if (!above.has(id)) {
+                const found = sql.above.get(...at);
+                above.set(id, found === undefined ? null : values(found, group));
+            }
+            return above.get(id) ?? null;
+        };
+        /** Each row read and encoded once, by key: a row is the same everywhere in one reply (one transaction). */
+        const encoded = new Map();
+        const row = (each) => {
+            let found = encoded.get(each[key]);
+            if (found === undefined) {
+                const value = paged.read(each);
+                found = { value, bytes: Buffer.byteLength(JSON.stringify(value)) };
+                encoded.set(each[key], found);
+            }
+            return found;
+        };
         /**
-         * A reply: `size` rows of `page` (fetched with one row more, to know if older ones are left), the first `count` of
-         * `kept` receipts with the rows they name, and `extra` history rows beside them; once each, as the store holds
-         * them now. Says whether it fits the caps, and how many rows it carries.
+         * A reply: `size` rows of `page` (fetched with one row more, to know if older ones are left, and live rows strictly
+         * below the cursor `fixed.at`, or the newest ones), the first `count` of `kept` receipts with the rows they name,
+         * and `extra` history rows beside them; once each, as the store holds them now. `held` counts the rows at or
+         * above the cursor (none on a reset). `measure` says whether one fits the caps without building it: the reply as
+         * `respond` makes it with no rows, plus each row's own encoded bytes and a comma between rows. `make` builds it.
          */
-        const reply = (fixed, page, kept, extra) => {
+        const reply = (fixed, page, kept, extra, held) => {
             const named = kept.map((each) => options.receipts.names(each).flatMap((name) => sql.named.all(name)));
-            return (size, count = kept.length) => {
-                const rows = new Map();
-                for (const row of [...page.slice(0, size), ...extra, ...named.slice(0, count).flat()]) {
-                    rows.set(row[key], rows.get(row[key]) ?? row);
+            /**
+             * Where a page of `size` rows leaves the copy: its last row (or the cursor), whether a live row of its group
+             * lies below it (the next row fetched), the group above it (in the page, else one query), and `held`.
+             */
+            const place = (size) => {
+                const last = page[size - 1] ?? fixed.at;
+                const next = page[size];
+                if (last === undefined || next === undefined) {
+                    return { from: fixed.from, next: null, through: null, held: count(), total: count() };
                 }
-                const made = answer({
-                    ...base,
-                    ...fixed,
-                    rows: [...rows.values()].map(paged.read),
-                    receipts: kept.slice(0, count),
-                    page: place(dataset, fixed.from, page[size - 1] ?? fixed.at, page.length > size),
-                });
-                const carried = rows.size + sizeOf(fixed.unpaged);
-                return { made, carried, fits: carried <= limits.maxRows && made.bytes <= limits.maxBytes };
+                const at = values(last, group);
+                let through = at;
+                // A page ending inside a group leaves it out: the boundary is the lowest whole group above.
+                if (same(next, at)) {
+                    const inPage = page.slice(0, Math.max(0, size - 1)).findLast((each) => !same(each, at));
+                    through = inPage === undefined ? higher(at) : values(inPage, group);
+                }
+                return {
+                    from: fixed.from,
+                    next: encode(dataset, last),
+                    through: through === null ? null : paged.through(through),
+                    held: held() + size,
+                    total: count(),
+                };
+            };
+            const rows = (size, taken) => {
+                const out = new Map();
+                for (const each of [...page.slice(0, size), ...extra, ...named.slice(0, taken).flat()]) {
+                    out.set(each[key], out.get(each[key]) ?? each);
+                }
+                return [...out.values()];
+            };
+            const pulled = (size, taken, carried) => ({
+                ...base,
+                ...fixed,
+                rows: carried,
+                receipts: kept.slice(0, taken),
+                page: place(size),
+            });
+            const fits = (carried, bytes) => carried + sizeOf(fixed.unpaged) <= limits.maxRows && bytes <= limits.maxBytes;
+            return {
+                /** The rows fetched for the page, one more than it may carry when older ones are left. */
+                fetched: page.length,
+                receipts: kept.length,
+                /** The reply's encoded bytes and whether it fits, from its rows' own encodings. */
+                measure: (size, taken = kept.length) => {
+                    const carried = rows(size, taken);
+                    let bytes = answer(pulled(size, taken, [])).bytes + Math.max(0, carried.length - 1);
+                    for (const each of carried) {
+                        bytes += row(each).bytes;
+                    }
+                    return { bytes, fits: fits(carried.length, bytes) };
+                },
+                /** The reply itself, and whether it fits, encoded whole. */
+                make: (size, taken = kept.length) => {
+                    const carried = rows(size, taken);
+                    const made = answer(pulled(size, taken, carried.map((each) => row(each).value)));
+                    return { made, fits: fits(carried.length, made.bytes) };
+                },
             };
         };
-        if (!reset) {
-            const unpaged = options.unpaged(since);
-            const room = cursor === null ? 0 : limits.pageRows;
-            const page = cursor === null ? [] : sql.older.all(...cursor, room + 1);
-            // The cursor as the row it stands for: a reply with no older row leaves the copy there.
-            const at = cursor === null
-                ? {}
-                : {
-                    at: Object.fromEntries(order.map((column, index) => [column, cursor[index] ?? null])),
-                };
-            const build = reply({ reset: false, from: request.page, unpaged, ...at }, page, receipts(since), sql.changes.all(since));
-            const alone = build(0);
-            // Only a delta (with its receipts' rows and the other tables) that can't fit by itself starts the copy over; the
-            // older page takes the rows and bytes it leaves.
-            if (alone.fits) {
-                const most = Math.min(room, page.length, limits.maxRows - alone.carried);
-                return build(largest(1, most, (size) => build(size).fits) ?? 0).made;
+        /** A delta's reply, and the newest page's: each read once, when first asked for. */
+        let delta;
+        let fresh;
+        const forDelta = () => {
+            if (delta === undefined) {
+                const unpaged = options.unpaged(since);
+                const room = cursor === null || cursor === undefined ? 0 : limits.pageRows;
+                const page = room === 0 ? [] : sql.older.all(...(cursor ?? []), room + 1);
+                let held;
+                // The cursor as the row it stands for: a reply with no older row leaves the copy there.
+                const at = cursor === null || cursor === undefined
+                    ? {}
+                    : {
+                        at: Object.fromEntries(order.map((column, index) => [column, cursor[index] ?? null])),
+                    };
+                delta = reply({ reset: false, from: request.page ?? null, unpaged, ...at }, page, receipts(since), sql.changes.all(since), 
+                // The live rows at or above the cursor: with the page's own rows, every row at or above its last.
+                () => (held ??= cursor ? tally(sql.held.get(...cursor)) : 0));
             }
+            return delta;
+        };
+        const forReset = () => {
+            fresh ??= reply({ reset: true, from: null, unpaged: options.unpaged(0) }, sql.top.all(limits.firstRows + 1), receipts(0), [], () => 0);
+            return fresh;
+        };
+        /**
+         * The reply's shape by `fits`: a delta that fits by itself takes the older page's rows and bytes it leaves;
+         * otherwise everything, from the newest page: one row of it at least, then as many receipts (with their rows) as
+         * fit, then as many more of the first rows. A receipt left out is asked for again by the next pull.
+         */
+        const choose = (fits) => {
+            if (!reset) {
+                const built = forDelta();
+                // Only a delta (with its receipts' rows and the other tables) that can't fit by itself starts the copy over.
+                if (fits(built, 0)) {
+                    const room = cursor === null ? 0 : limits.pageRows;
+                    const most = Math.min(room, built.fetched);
+                    return {
+                        built,
+                        size: largest(1, most, (size) => fits(built, size)) ?? 0,
+                        receipts: built.receipts,
+                    };
+                }
+            }
+            const built = forReset();
+            const least = Math.min(1, built.fetched);
+            const kept = largest(0, built.receipts, (at) => fits(built, least, at));
+            if (kept === undefined) {
+                throw new Error(`pull: the reply can't fit its caps (${limits.maxRows} rows, ${limits.maxBytes} bytes), even with one row of history`);
+            }
+            const most = Math.min(limits.firstRows, built.fetched);
+            return {
+                built,
+                size: largest(least, most, (at) => fits(built, at, kept)) ?? least,
+                receipts: kept,
+            };
+        };
+        // Measured from each row's own encoding, then built once. A reply whose encoding isn't its rows' (a `respond`
+        // that reshapes rows) is chosen again by building each candidate whole.
+        const measured = choose((built, size, taken) => built.measure(size, taken).fits);
+        const made = measured.built.make(measured.size, measured.receipts);
+        if (made.made.bytes === measured.built.measure(measured.size, measured.receipts).bytes) {
+            return made.made;
         }
-        // Everything, from the newest page: one row of it at least, then as many receipts (with their rows) as fit, then
-        // as many more of the first rows. A receipt left out is asked for again by the next pull.
-        const kept = receipts(0);
-        const page = sql.top.all(limits.firstRows + 1);
-        const least = Math.min(1, page.length);
-        const build = reply({ reset: true, from: null, unpaged: options.unpaged(0) }, page, kept, []);
-        const count = largest(0, kept.length, (at) => build(least, at).fits);
-        if (count === undefined) {
-            throw new Error(`pull: the reply can't fit its caps (${limits.maxRows} rows, ${limits.maxBytes} bytes), even with one row of history`);
-        }
-        const most = Math.min(limits.firstRows, page.length);
-        return build(largest(least, most, (at) => build(at, count).fits) ?? least, count).made;
+        const exact = choose((built, size, taken) => built.make(size, taken).fits);
+        return exact.built.make(exact.size, exact.receipts).made;
     }
     /** The reply to one request, its encoded size, and what it carries before `respond` shaped it. */
     function pull(request, caller) {

package/dist/server/web.d.ts

@@ -1,4 +1,4 @@
-import type { Endpoint, Failure } from "../contract.ts";
+import type { Endpoint, RefusalKind } from "../contract.ts";
 import type { WebConfig } from "./config.ts";
 import type { Logger } from "./log.ts";
 /** Who asked: their identities, whether a person (not a service token), and how they signed in. */
@@ -8,10 +8,14 @@ export type Caller = {
     login: "access" | "oidc";
     request: string;
 };
-/** A handler's refusal, answered as a typed failure. */
+/**
+ * A handler's refusal, answered as a typed failure: the kit's error body, so the page can tell it from a proxy's.
+ * `path` names where a request broke its schema (`request.commands[1].command.value`).
+ */
 export declare class Refusal extends Error {
-    readonly kind: Failure["kind"];
-    constructor(kind: Failure["kind"], message: string);
+    readonly kind: RefusalKind;
+    readonly path: string | undefined;
+    constructor(kind: RefusalKind, message: string, path?: string);
 }
 /** A route: an endpoint and its handler, typed together and run untyped by the server. */
 export type Route = {
@@ -39,7 +43,7 @@ export type WebOptions = {
     socket?: Socket;
     /** The session cookie's name. */
     cookie: string;
-    /** Request bodies past this are refused; default 64 KB. */
+    /** Request bodies past this many bytes are refused; default 64 KiB. Give a push endpoint `PUSH_BYTES` or more. */
     bodyLimit?: number;
 };
 export declare function serveWeb(options: WebOptions): Promise<{

package/dist/server/web.js

@@ -5,7 +5,7 @@
  * from the page's own Origin. Responses carry a strict CSP and no framing.
  *
  * Routes are contract endpoints: the request is checked against its schema before the handler runs and the reply
- * after, so neither side can drift from the contract. Failures answer `{ kind, message }` with the kind's status.
+ * after, so neither side can drift from the contract. Failures answer `{ kind, message, path? }` with the kind's status.
  * The socket is a hint channel: the app sends small messages (pokes, pings); nothing correct depends on it.
  * Every API request is logged with an id, status, time and whether a person or a service asked.
  */
@@ -32,12 +32,17 @@ const SECURITY = {
     "x-frame-options": "DENY",
     "content-security-policy": "default-src 'self'; connect-src 'self'; img-src 'self' data:; style-src 'self'; script-src 'self'; manifest-src 'self'; worker-src 'self'; frame-ancestors 'none'; base-uri 'none'; form-action 'self'",
 };
-/** A handler's refusal, answered as a typed failure. */
+/**
+ * A handler's refusal, answered as a typed failure: the kit's error body, so the page can tell it from a proxy's.
+ * `path` names where a request broke its schema (`request.commands[1].command.value`).
+ */
 export class Refusal extends Error {
     kind;
-    constructor(kind, message) {
+    path;
+    constructor(kind, message, path) {
         super(message);
         this.kind = kind;
+        this.path = path;
     }
 }
 export function route(endpoint, handle) {
@@ -52,7 +57,7 @@ export function route(endpoint, handle) {
             }
             catch (error) {
                 if (error instanceof Invalid) {
-                    throw new Refusal("invalid", error.message);
+                    throw new Refusal("invalid", error.message, error.path);
                 }
                 throw error;
             }
@@ -90,18 +95,30 @@ function send(request, response, status, body, headers = {}) {
     });
     response.end(request.method === "HEAD" ? undefined : bytes);
 }
-function failure(request, response, kind, message) {
-    send(request, response, STATUS[kind], JSON.stringify({ kind, message }), {
-        "content-type": "application/json",
-    });
+function failure(request, response, kind, message, path) {
+    send(request, response, STATUS[kind], JSON.stringify({ kind, message, ...(path === undefined ? {} : { path }) }), { "content-type": "application/json" });
 }
+/**
+ * The body as JSON: its bytes collected and counted against `limit`, then decoded once, so a character split between
+ * chunks arrives whole. Bytes that aren't UTF-8 are refused, never replaced.
+ */
 async function readBody(request, limit) {
-    let text = "";
+    const chunks = [];
+    let size = 0;
     for await (const chunk of request) {
-        text += String(chunk);
-        if (text.length > limit) {
+        const bytes = Buffer.isBuffer(chunk) ? chunk : Buffer.from(String(chunk));
+        size += bytes.length;
+        if (size > limit) {
             throw new Refusal("invalid", "the request is too large");
         }
+        chunks.push(bytes);
+    }
+    let text;
+    try {
+        text = new TextDecoder("utf-8", { fatal: true }).decode(Buffer.concat(chunks));
+    }
+    catch {
+        throw new Refusal("invalid", "the request is not UTF-8");
     }
     try {
         return text ? JSON.parse(text) : {};
@@ -212,7 +229,7 @@ export async function serveWeb(options) {
                     reason: message,
                 });
             }
-            failure(request, response, kind, error instanceof Refusal ? message : "the server failed; see its log");
+            failure(request, response, kind, error instanceof Refusal ? message : "the server failed; see its log", error instanceof Refusal ? error.path : undefined);
         }
         log(status < 400 ? "info" : "warn", "request", {
             request: caller.request,