slicetest 0.2.0 → 0.4.0

package/dist/record.js

@@ -0,0 +1,201 @@
+import { existsSync } from "node:fs";
+import { writeFile } from "node:fs/promises";
+import http from "node:http";
+import path from "node:path";
+import { stringify } from "yaml";
+import { mask } from "./trace.js";
+export const SESSION_ENV = "SLICETEST_RECORD_SESSION";
+/** Requests a browser makes on its own, not part of what the scenario is about. */
+const ASSET = /\.(ico|png|jpe?g|gif|webp|avif|svg|css|js|mjs|map|woff2?|ttf|eot)(\?|$)/i;
+/** Hop-by-hop headers, and the ones fetch rewrites. */
+const DROP_REQUEST = new Set(["host", "connection", "keep-alive", "transfer-encoding", "upgrade", "proxy-connection", "content-length", "accept-encoding"]);
+const DROP_RESPONSE = new Set(["connection", "keep-alive", "transfer-encoding", "content-encoding", "content-length"]);
+export async function runSession(ctx, session) {
+    const exchanges = [];
+    const target = ctx.app.url;
+    const proxy = http.createServer(async (req, res) => {
+        const chunks = [];
+        for await (const c of req)
+            chunks.push(c);
+        const body = Buffer.concat(chunks);
+        const headers = new Headers();
+        for (const [k, v] of Object.entries(req.headers)) {
+            if (v === undefined || DROP_REQUEST.has(k))
+                continue;
+            for (const value of Array.isArray(v) ? v : [v])
+                headers.append(k, value);
+        }
+        try {
+            const upstream = await fetch(new URL(req.url ?? "/", target), {
+                method: req.method,
+                headers,
+                body: body.length && req.method !== "GET" && req.method !== "HEAD" ? body : undefined,
+                redirect: "manual",
+            });
+            const payload = Buffer.from(await upstream.arrayBuffer());
+            const out = {};
+            upstream.headers.forEach((v, k) => {
+                if (!DROP_RESPONSE.has(k) && k !== "set-cookie")
+                    out[k] = v;
+            });
+            const cookies = upstream.headers.getSetCookie();
+            if (cookies.length)
+                out["set-cookie"] = cookies;
+            res.writeHead(upstream.status, out).end(payload);
+            exchanges.push({
+                method: req.method ?? "GET",
+                path: req.url ?? "/",
+                contentType: req.headers["content-type"],
+                body: body.toString("utf8"),
+                status: upstream.status,
+                responseType: upstream.headers.get("content-type") ?? undefined,
+                response: payload.toString("utf8"),
+            });
+        }
+        catch (e) {
+            res.writeHead(502, { "content-type": "text/plain" }).end(`slicetest record: the app didn't answer: ${e.message}\n`);
+        }
+    });
+    await new Promise((resolve, reject) => {
+        proxy.once("error", reject);
+        proxy.listen(session.port, "127.0.0.1", resolve);
+    });
+    const url = `http://127.0.0.1:${proxy.address().port}`;
+    await writeFile(session.stateFile, JSON.stringify({ url, app: target }));
+    while (!existsSync(session.stopFile))
+        await new Promise((r) => setTimeout(r, 200));
+    await new Promise((resolve) => proxy.close(() => resolve()));
+    proxy.closeAllConnections();
+    const calls = Object.fromEntries(session.stubs.map((name) => [name, ctx.stub(name).calls()]));
+    const changes = await ctx.db.changes();
+    const { yaml, summary } = buildScenario(exchanges, calls, changes, { name: path.basename(session.out).replace(/\.scenario\.ya?ml$/, "") });
+    await writeFile(session.out, yaml);
+    await writeFile(session.stateFile, JSON.stringify({ url, done: true, summary }));
+}
+/** The recorded session as a YAML scenario, with a comment header of what to review. */
+export function buildScenario(exchanges, calls, changes, { name = "recorded session", now = new Date() } = {}) {
+    const kept = exchanges.filter((x) => !ASSET.test(x.path));
+    const steps = [];
+    const notes = [];
+    // Stubs first: the answers the services gave, so the replay doesn't need them.
+    const unanswered = [];
+    let stubRoutes = 0;
+    for (const [stub, list] of Object.entries(calls)) {
+        const groups = new Map();
+        for (const c of list) {
+            if (!c.response || c.response.status === 501) {
+                unanswered.push(`${stub}: ${c.method} ${c.path}`);
+                continue;
+            }
+            const key = `${c.method} ${c.path}`;
+            groups.set(key, [...(groups.get(key) ?? []), c]);
+        }
+        for (const [on, group] of groups) {
+            const replies = group.map((c) => ({ status: c.response.status, ...(c.response.body ? { body: parse(c.response.body, c.response.headers["content-type"]) } : {}) }));
+            const same = replies.every((r) => JSON.stringify(r) === JSON.stringify(replies[0]));
+            steps.push({ stub, on, ...(same ? { reply: replies[0] } : { sequence: replies }) });
+            stubRoutes++;
+        }
+    }
+    if (unanswered.length)
+        notes.push(`These calls got no answer while recording (register a route, or give the stub an upstream / autoReply): ${unanswered.join(", ")}`);
+    // Values from earlier responses that later requests use (ids in paths, tokens in bodies) become captures.
+    const captured = new Map();
+    kept.forEach((x, i) => {
+        let reqPath = x.path;
+        let reqBody = x.body;
+        for (const [value, variable] of captured) {
+            reqPath = reqPath.split(value).join(`{{${variable}}}`);
+            reqBody = reqBody.split(value).join(`{{${variable}}}`);
+        }
+        const [method, pathOnly] = [x.method, reqPath];
+        const step = { request: `${method} ${pathOnly}` };
+        if (x.body) {
+            if (/json/i.test(x.contentType ?? ""))
+                step.json = parse(reqBody, "application/json");
+            else if (/x-www-form-urlencoded/i.test(x.contentType ?? ""))
+                step.form = Object.fromEntries(new URLSearchParams(reqBody));
+            else
+                step.body = reqBody;
+        }
+        const expectation = { status: x.status };
+        const json = /json/i.test(x.responseType ?? "") ? parse(x.response, "application/json") : undefined;
+        if (json !== undefined && typeof json === "object")
+            expectation.json = loosen(json);
+        else if (!/html/i.test(x.responseType ?? "") && x.response && x.response.length <= 200)
+            expectation.text = x.response;
+        step.expect = expectation;
+        // Capture top-level values that show up again later.
+        if (json && typeof json === "object" && !Array.isArray(json)) {
+            const later = kept.slice(i + 1).map((y) => `${y.path}\n${y.body}`).join("\n");
+            const capture = {};
+            for (const [key, value] of Object.entries(json)) {
+                if ((typeof value !== "string" && typeof value !== "number") || String(value).length < 3 || !later.includes(String(value)))
+                    continue;
+                let variable = key.replace(/\W/g, "_");
+                while ([...captured.values()].includes(variable))
+                    variable += "_";
+                capture[variable] = `json.${key}`;
+                captured.set(String(value), variable);
+            }
+            if (Object.keys(capture).length)
+                step.capture = capture;
+        }
+        steps.push(step);
+    });
+    for (const [stub, list] of Object.entries(calls)) {
+        const byCall = new Map();
+        for (const c of list)
+            if (c.response && c.response.status !== 501)
+                byCall.set(`${c.method} ${c.path}`, (byCall.get(`${c.method} ${c.path}`) ?? 0) + 1);
+        for (const [call, times] of byCall)
+            steps.push({ received: stub, call, times });
+    }
+    const tables = Object.keys(changes);
+    const counts = Object.fromEntries(Object.entries(changes).map(([table, c]) => [
+        table,
+        Object.fromEntries(Object.entries({ inserted: c.inserted.length, updated: c.updated.length, deleted: c.deleted.length }).filter(([, n]) => n > 0)),
+    ]));
+    steps.push({ changes: counts });
+    const skipped = exchanges.length - kept.length;
+    if (skipped)
+        notes.push(`${skipped} request(s) for static files (scripts, styles, images) were left out.`);
+    if (kept.length === 0)
+        notes.push("No requests to the app were recorded; send them to the proxy URL printed when recording started.");
+    const header = [
+        `# Recorded with \`npx slicetest record\` on ${now.toISOString().slice(0, 10)}. Review it before committing:`,
+        "# - give the scenario a name that says what it checks, and split it if it checks several things;",
+        "# - dates and UUIDs in responses are matched by type only, and `changes` only counts rows;",
+        "#   tighten what matters (see https://github.com/revo1290/slicetest#yaml-scenarios).",
+        ...notes.map((n) => `# - ${n}`),
+        "# yaml-language-server: $schema=https://unpkg.com/slicetest/schema/scenario.schema.json",
+        "",
+    ].join("\n");
+    const yaml = header + stringify({ scenarios: [{ name, steps }] }, { lineWidth: 0 });
+    const summary = { requests: kept.length, skippedAssets: skipped, stubs: stubRoutes, tables, unanswered };
+    return { yaml, summary };
+}
+function parse(text, contentType) {
+    if (/json/i.test(contentType ?? "")) {
+        try {
+            return JSON.parse(text);
+        }
+        catch {
+            return text;
+        }
+    }
+    return text;
+}
+/** Dates and UUIDs change from run to run: expect their type instead of their value. */
+function loosen(value) {
+    const walk = (v) => {
+        if (v === "[date]" || v === "[uuid]")
+            return { $type: "string" };
+        if (Array.isArray(v))
+            return v.map(walk);
+        if (v && typeof v === "object")
+            return Object.fromEntries(Object.entries(v).map(([k, x]) => [k, walk(x)]));
+        return v;
+    };
+    return walk(mask(value));
+}

package/dist/recording.d.ts

@@ -0,0 +1,43 @@
+import type { RecordedCall, StubResponse } from "./stub.js";
+/** One recorded exchange with the real service, as stored in the recordings file. */
+export interface Recording {
+    request: {
+        method: string;
+        path: string;
+        query?: Record<string, string>;
+        json?: unknown;
+        body?: string;
+    };
+    response: {
+        status: number;
+        headers?: Record<string, string>;
+        json?: unknown;
+        body?: string;
+    };
+}
+/**
+ * Replays recorded exchanges for calls no route matches and, in record mode,
+ * forwards the calls it has no recording for to the real service and records
+ * the answer. Identical requests replay their recordings in order within a
+ * scenario; the last one repeats.
+ */
+export declare class Recorder {
+    #private;
+    readonly name: string;
+    readonly file: string;
+    readonly upstream: string;
+    readonly recording: boolean;
+    private constructor();
+    static load(name: string, file: string, upstream: string, recording: boolean): Promise<Recorder>;
+    /** What recording a call gets, or undefined to leave it unanswered. */
+    answer(call: RecordedCall): Promise<StubResponse | undefined>;
+    /** Why a call went unanswered, for the stub's 501 reply. */
+    hint(): string;
+    /** Start of a scenario: identical requests replay from their first recording again. */
+    reset(): void;
+    /** Recordings made by this worker, to be merged into the file when the run ends. */
+    added(): Recording[];
+}
+export declare function readRecordings(file: string): Promise<Recording[]>;
+/** Append new recordings to `file`, skipping exact duplicates, keeping the order they were made in. */
+export declare function mergeRecordings(file: string, upstream: string, added: Recording[]): Promise<void>;

package/dist/recording.js

@@ -0,0 +1,148 @@
+import { readFile, writeFile, mkdir } from "node:fs/promises";
+import path from "node:path";
+import { isDeepStrictEqual } from "node:util";
+import YAML from "yaml";
+/**
+ * Response headers worth keeping. Everything else (dates, cookies, rate-limit
+ * counters, request ids) is noise in a committed file, or a secret.
+ */
+const KEPT_HEADERS = ["content-type", "location", "retry-after", "link", "etag"];
+/** Request headers not forwarded to the real service. */
+const HOP_HEADERS = ["host", "connection", "content-length", "accept-encoding", "transfer-encoding", "keep-alive"];
+/**
+ * Replays recorded exchanges for calls no route matches and, in record mode,
+ * forwards the calls it has no recording for to the real service and records
+ * the answer. Identical requests replay their recordings in order within a
+ * scenario; the last one repeats.
+ */
+export class Recorder {
+    name;
+    file;
+    upstream;
+    recording;
+    #entries;
+    #added = [];
+    #seen = new Map();
+    constructor(name, file, upstream, recording, entries) {
+        this.name = name;
+        this.file = file;
+        this.upstream = upstream;
+        this.recording = recording;
+        this.#entries = entries;
+    }
+    static async load(name, file, upstream, recording) {
+        return new Recorder(name, file, upstream, recording, await readRecordings(file));
+    }
+    /** What recording a call gets, or undefined to leave it unanswered. */
+    async answer(call) {
+        const request = requestOf(call);
+        const key = keyOf(request);
+        const matches = this.#entries.filter((e) => keyOf(e.request) === key);
+        const n = this.#seen.get(key) ?? 0;
+        this.#seen.set(key, n + 1);
+        if (matches.length > 0 && (!this.recording || n < matches.length))
+            return toResponse(matches[Math.min(n, matches.length - 1)]);
+        if (!this.recording)
+            return undefined;
+        const entry = { request, response: await this.#forward(call) };
+        this.#entries.push(entry);
+        this.#added.push(entry);
+        return toResponse(entry);
+    }
+    /** Why a call went unanswered, for the stub's 501 reply. */
+    hint() {
+        return `no recording in ${this.file} either; run with SLICETEST_RECORD=${this.name} to record it from ${this.upstream}`;
+    }
+    /** Start of a scenario: identical requests replay from their first recording again. */
+    reset() {
+        this.#seen.clear();
+    }
+    /** Recordings made by this worker, to be merged into the file when the run ends. */
+    added() {
+        return this.#added;
+    }
+    async #forward(call) {
+        const url = new URL(this.upstream);
+        // Keep a path prefix on the upstream (e.g. https://api.example.com/v2) in front of the call's path.
+        url.pathname = url.pathname.replace(/\/$/, "") + call.path;
+        url.search = call.query.toString();
+        const headers = new Headers();
+        for (const [k, v] of Object.entries(call.headers)) {
+            if (v !== undefined && !HOP_HEADERS.includes(k))
+                headers.set(k, Array.isArray(v) ? v.join(", ") : v);
+        }
+        let res;
+        try {
+            res = await fetch(url, { method: call.method, headers, body: ["GET", "HEAD"].includes(call.method) ? undefined : call.body });
+        }
+        catch (e) {
+            throw new Error(`slicetest: recording stub ${this.name}: ${call.method} ${url} failed: ${e.message}`);
+        }
+        const text = await res.text();
+        const kept = Object.fromEntries(KEPT_HEADERS.flatMap((h) => (res.headers.has(h) ? [[h, res.headers.get(h)]] : [])));
+        const json = parse(text);
+        return {
+            status: res.status,
+            ...(Object.keys(kept).length ? { headers: kept } : {}),
+            ...(json !== undefined ? { json } : text ? { body: text } : {}),
+        };
+    }
+}
+function requestOf(call) {
+    const query = Object.fromEntries([...call.query.entries()].sort(([a], [b]) => a.localeCompare(b)));
+    return {
+        method: call.method,
+        path: call.path,
+        ...(Object.keys(query).length ? { query } : {}),
+        ...(call.json !== undefined ? { json: call.json } : call.body ? { body: call.body } : {}),
+    };
+}
+function keyOf(r) {
+    return JSON.stringify([r.method.toUpperCase(), r.path, sortKeys(r.query ?? {}), sortKeys(r.json ?? null), r.body ?? ""]);
+}
+function sortKeys(v) {
+    if (Array.isArray(v))
+        return v.map(sortKeys);
+    if (v && typeof v === "object")
+        return Object.fromEntries(Object.entries(v).sort(([a], [b]) => a.localeCompare(b)).map(([k, x]) => [k, sortKeys(x)]));
+    return v;
+}
+function toResponse(e) {
+    return { status: e.response.status, headers: e.response.headers, body: e.response.json !== undefined ? e.response.json : (e.response.body ?? "") };
+}
+function parse(text) {
+    if (!text)
+        return undefined;
+    try {
+        return JSON.parse(text);
+    }
+    catch {
+        return undefined;
+    }
+}
+export async function readRecordings(file) {
+    let text;
+    try {
+        text = await readFile(file, "utf8");
+    }
+    catch {
+        return [];
+    }
+    const doc = YAML.parse(text);
+    if (doc == null)
+        return [];
+    if (!Array.isArray(doc) || !doc.every((e) => e?.request?.method && e?.request?.path && typeof e?.response?.status === "number")) {
+        throw new Error(`slicetest: ${file} is not a recordings file: expected a list of { request: { method, path }, response: { status } }`);
+    }
+    return doc;
+}
+/** Append new recordings to `file`, skipping exact duplicates, keeping the order they were made in. */
+export async function mergeRecordings(file, upstream, added) {
+    const entries = await readRecordings(file);
+    for (const e of added)
+        if (!entries.some((x) => isDeepStrictEqual(x, e)))
+            entries.push(e);
+    await mkdir(path.dirname(file), { recursive: true });
+    const header = `# Recorded by slicetest from ${upstream}. Review before committing: request bodies are stored as sent.\n`;
+    await writeFile(file, header + YAML.stringify(entries, { lineWidth: 0 }));
+}

package/dist/runtime.d.ts

@@ -1,8 +1,11 @@
 import { App } from "./app.js";
 import type { ResolvedOptions } from "./config.js";
+import { Dependency } from "./containers.js";
 import { Db } from "./db.js";
 import { HttpClient } from "./http.js";
+import { Mailbox } from "./mail.js";
 import { Stub } from "./stub.js";
+import { type MaskOptions, type Trace } from "./trace.js";
 export interface ScenarioContext {
     http: HttpClient;
     db: Db;
@@ -10,6 +13,15 @@ export interface ScenarioContext {
     app: App;
     /** A process from the `services` option: its `url`, `logs()` and `waitForLog()`. */
     service: (name: string) => App;
+    /** A container from the `containers` option: its `host`, `port`, `address` and `exec()`. */
+    container: (name: string) => Dependency;
+    /**
+     * What the scenario did so far: requests to the app, calls to stubs and database changes,
+     * with timestamps and UUIDs masked. `expect(await trace()).toMatchSnapshot()`.
+     */
+    trace: (opts?: MaskOptions) => Promise<Trace>;
+    /** Mail the app sent during the scenario. Needs `mail: true` in the config. */
+    mail: Mailbox;
 }
 /** Everything one test file needs: its own database, stub servers and app process. */
 export declare class Runtime {
@@ -22,6 +34,10 @@ export declare class Runtime {
     private readonly vars;
     private readonly specs;
     private readonly coverageDir?;
+    private readonly recorders;
+    private readonly recordDir?;
+    readonly containers: Map<string, Dependency>;
+    readonly mailbox?: Mailbox | undefined;
     private constructor();
     get http(): HttpClient;
     static start(opts: ResolvedOptions, shared: {
@@ -29,6 +45,7 @@ export declare class Runtime {
         template: string;
         prefix: string;
         coverageDir?: string;
+        recordDir?: string;
     }): Promise<Runtime>;
     context(): ScenarioContext;
     beforeScenario(): Promise<void>;

package/dist/runtime.js

@@ -2,11 +2,15 @@ import { randomUUID } from "node:crypto";
 import { writeFile } from "node:fs/promises";
 import path from "node:path";
 import { App } from "./app.js";
+import { Dependency } from "./containers.js";
 import { Db, formatChanges } from "./db.js";
 import { engineFor } from "./drivers/index.js";
 import { formatHistory, HttpClient } from "./http.js";
+import { Mailbox } from "./mail.js";
 import { OpenApiSpec } from "./openapi.js";
+import { Recorder } from "./recording.js";
 import { Stub } from "./stub.js";
+import { buildTrace, mask } from "./trace.js";
 /** Everything one test file needs: its own database, stub servers and app process. */
 export class Runtime {
     app;
@@ -17,12 +21,16 @@ export class Runtime {
     vars;
     specs;
     coverageDir;
+    recorders;
+    recordDir;
+    containers;
+    mailbox;
     #http;
     /** Responses from the app that don't match its OpenAPI spec, this scenario. */
     #contract = [];
     /** Documented responses seen in this file, for the run's coverage report. */
     #covered = new Set();
-    constructor(app, services, db, stubs, opts, vars, specs, coverageDir) {
+    constructor(app, services, db, stubs, opts, vars, specs, coverageDir, recorders = new Map(), recordDir, containers = new Map(), mailbox) {
         this.app = app;
         this.services = services;
         this.db = db;
@@ -31,6 +39,10 @@ export class Runtime {
         this.vars = vars;
         this.specs = specs;
         this.coverageDir = coverageDir;
+        this.recorders = recorders;
+        this.recordDir = recordDir;
+        this.containers = containers;
+        this.mailbox = mailbox;
         this.#http = this.#client();
     }
     #client() {
@@ -56,11 +68,14 @@ export class Runtime {
         return this.#http;
     }
     static async start(opts, shared) {
-        const engine = engineFor(opts);
+        const engine = await engineFor(opts);
         const url = await ensureWorkerDatabase(engine, shared.adminUrl, shared.template, shared.prefix);
         const stubs = new Map();
+        const recorders = new Map();
+        const containers = new Map();
         const services = new Map();
         let db;
+        let mailbox;
         try {
             // Loaded first: a broken spec should fail before anything is started.
             const specs = {
@@ -71,16 +86,37 @@ export class Runtime {
                 specs.stubs.set(name, await OpenApiSpec.load(path.resolve(opts.root, file), file));
             for (const name of opts.stubs)
                 stubs.set(name, await Stub.start(name));
-            for (const name of opts.openapi.autoReply) {
-                const spec = specs.stubs.get(name);
-                stubs.get(name).fallback((call) => spec.exampleResponse(call.method, call.path));
+            for (const [name, r] of Object.entries(opts.recordings))
+                recorders.set(name, await Recorder.load(name, path.resolve(opts.root, r.file), r.upstream, r.record));
+            // A registered route wins, then a recording of the real service, then an answer made up from its spec.
+            for (const [name, stub] of stubs) {
+                const recorder = recorders.get(name);
+                const spec = opts.openapi.autoReply.includes(name) ? specs.stubs.get(name) : undefined;
+                if (!recorder && !spec)
+                    continue;
+                stub.fallback(async (call) => (await recorder?.answer(call)) ?? spec?.exampleResponse(call.method, call.path), recorder?.hint());
             }
             db = await Db.connect(await engine.driver(url), url, {
                 schemas: opts.db.schemas,
                 keep: opts.db.keep,
                 seedFile: opts.db.seed && path.resolve(opts.root, opts.db.seed),
             });
+            const started = await Promise.allSettled(Object.entries(opts.containers).map(async ([name, c]) => containers.set(name, await Dependency.start(name, c))));
+            const failed = started.find((r) => r.status === "rejected");
+            if (failed)
+                throw failed.reason;
             const vars = { "db.url": url };
+            if (engine.name === "sqlite")
+                vars["db.path"] = (await import("./drivers/sqlite.js")).sqlitePath(url);
+            if (opts.mail) {
+                mailbox = await Mailbox.start();
+                Object.assign(vars, { "mail.host": mailbox.host, "mail.port": String(mailbox.port), "mail.url": mailbox.url });
+            }
+            for (const [name, c] of containers) {
+                vars[`container.${name}`] = c.address;
+                vars[`container.${name}.host`] = c.host;
+                vars[`container.${name}.port`] = String(c.port);
+            }
             for (const [name, stub] of stubs)
                 vars[`stub.${name}`] = stub.url;
             for (const [name, service] of Object.entries(opts.services)) {
@@ -90,16 +126,19 @@ export class Runtime {
                 vars[`service.${name}.port`] = String(started.port);
             }
             const app = await App.start(opts.app, opts.root, vars);
-            return new Runtime(app, services, db, stubs, opts, vars, specs, shared.coverageDir);
+            return new Runtime(app, services, db, stubs, opts, vars, specs, shared.coverageDir, recorders, shared.recordDir, containers, mailbox);
         }
         catch (e) {
             await Promise.all([...services.values()].map((s) => s.stop()));
             await db?.close();
             await Promise.all([...stubs.values()].map((s) => s.close()));
+            await Promise.allSettled([...containers.values()].map((c) => c.stop()));
+            await mailbox?.close();
             throw e;
         }
     }
     context() {
+        const mailbox = this.mailbox;
         return {
             http: this.http,
             db: this.db,
@@ -118,6 +157,18 @@ export class Runtime {
                 }
                 return service;
             },
+            container: (name) => {
+                const c = this.containers.get(name);
+                if (!c)
+                    throw new Error(`slicetest: unknown container "${name}". Declared containers: ${[...this.containers.keys()].join(", ") || "(none)"}`);
+                return c;
+            },
+            trace: async (opts) => mask(buildTrace(this.http.history, this.stubs.values(), await this.db.changesSinceStart(), this.mailbox), opts),
+            get mail() {
+                if (!mailbox)
+                    throw new Error("slicetest: mail is off. Add `mail: true` to the config and point the app's SMTP settings at {{mail.host}} / {{mail.port}}.");
+                return mailbox;
+            },
         };
     }
     /** The app and every service, for checks that apply to all of them. */
@@ -141,6 +192,10 @@ export class Runtime {
         await this.db.reset();
         for (const stub of this.stubs.values())
             stub.reset();
+        for (const recorder of this.recorders.values())
+            recorder.reset();
+        this.mailbox?.reset();
+        await Promise.all([...this.containers.values()].map((c) => c.reset()));
         this.http.reset();
         for (const p of this.#processes())
             p.beginScenario();
@@ -195,6 +250,7 @@ export class Runtime {
             return [
                 ...calls.map((c) => `  ${s.name}: ${c.method} ${c.path}${c.query.size ? `?${c.query}` : ""}`),
                 `    registered on ${s.name}: ${routes.length ? routes.join(", ") : "(none)"}`,
+                ...(this.recorders.has(s.name) ? [`    ${this.recorders.get(s.name).hint()}`] : []),
             ];
         });
     }
@@ -221,6 +277,8 @@ export class Runtime {
         catch (e) {
             sections.push(`database changes during this scenario: unavailable (${e.message})`);
         }
+        if (this.mailbox)
+            sections.push(this.mailbox.describe());
         const logs = this.app.scenarioLogs();
         sections.push(logs ? `app output during this scenario:\n${logs}` : "app output during this scenario: (none)");
         for (const [name, service] of this.services) {
@@ -233,11 +291,18 @@ export class Runtime {
         if (this.coverageDir && this.#covered.size > 0) {
             await writeFile(path.join(this.coverageDir, `${process.pid}-${randomUUID()}.json`), JSON.stringify([...this.#covered])).catch(() => { });
         }
+        for (const [name, recorder] of this.recorders) {
+            if (this.recordDir && recorder.added().length > 0) {
+                await writeFile(path.join(this.recordDir, `${name}.${process.pid}-${randomUUID()}.json`), JSON.stringify(recorder.added())).catch(() => { });
+            }
+        }
         const results = await Promise.allSettled([
             this.app.stop(),
             ...[...this.services.values()].map((s) => s.stop()),
             this.db.close(),
             ...[...this.stubs.values()].map((s) => s.close()),
+            ...[...this.containers.values()].map((c) => c.stop()),
+            this.mailbox?.close(),
         ]);
         const failed = results.find((r) => r.status === "rejected");
         if (failed)

package/dist/stub.d.ts

@@ -81,7 +81,7 @@ export declare class Stub {
      * Return undefined to leave a call unanswered (it then fails the scenario as usual).
      * Kept across scenarios, unlike routes.
      */
-    fallback(respond: ((call: RecordedCall) => StubResponse | undefined) | undefined): this;
+    fallback(respond: ((call: RecordedCall) => StubResponse | undefined | Promise<StubResponse | undefined>) | undefined, hint?: string): this;
     close(): Promise<void>;
 }
 /** `expected` is contained in `actual`: objects compare key by key, arrays element-wise. */

package/dist/stub.js

@@ -10,6 +10,8 @@ export class Stub {
     #routes = [];
     #calls = [];
     #fallback;
+    /** Appended to the 501 answer for a call nothing could answer. */
+    #hint;
     url = "";
     constructor(name) {
         this.name = name;
@@ -86,8 +88,9 @@ export class Stub {
      * Return undefined to leave a call unanswered (it then fails the scenario as usual).
      * Kept across scenarios, unlike routes.
      */
-    fallback(respond) {
+    fallback(respond, hint) {
         this.#fallback = respond;
+        this.#hint = hint;
         return this;
     }
     async close() {
@@ -125,9 +128,17 @@ export class Stub {
             break;
         }
         if (!route) {
-            const out = this.#fallback?.(call);
+            let out;
+            try {
+                out = await this.#fallback?.(call);
+            }
+            catch (e) {
+                res.writeHead(502, { "content-type": "text/plain" }).end(e.message);
+                return;
+            }
             if (!out) {
-                res.writeHead(501, { "content-type": "text/plain" }).end(`slicetest: no stub for ${call.method} ${call.path}`);
+                const hint = this.#hint ? ` (${this.#hint})` : "";
+                res.writeHead(501, { "content-type": "text/plain" }).end(`slicetest: no stub for ${call.method} ${call.path}${hint}`);
                 return;
             }
             call.matched = true;