slicetest 0.1.0 → 0.3.0

package/dist/runtime.js

@@ -1,50 +1,128 @@
+import { randomUUID } from "node:crypto";
+import { writeFile } from "node:fs/promises";
 import path from "node:path";
-import pg from "pg";
 import { App } from "./app.js";
-import { Db, withDatabase } from "./db.js";
+import { Dependency } from "./containers.js";
+import { Db, formatChanges } from "./db.js";
+import { engineFor } from "./drivers/index.js";
 import { formatHistory, HttpClient } from "./http.js";
+import { OpenApiSpec } from "./openapi.js";
+import { Recorder } from "./recording.js";
 import { Stub } from "./stub.js";
+import { buildTrace, mask } from "./trace.js";
 /** Everything one test file needs: its own database, stub servers and app process. */
 export class Runtime {
     app;
+    services;
     db;
     stubs;
     opts;
     vars;
-    #logMark = 0;
+    specs;
+    coverageDir;
+    recorders;
+    recordDir;
+    containers;
     #http;
-    constructor(app, db, stubs, opts, vars) {
+    /** Responses from the app that don't match its OpenAPI spec, this scenario. */
+    #contract = [];
+    /** Documented responses seen in this file, for the run's coverage report. */
+    #covered = new Set();
+    constructor(app, services, db, stubs, opts, vars, specs, coverageDir, recorders = new Map(), recordDir, containers = new Map()) {
         this.app = app;
+        this.services = services;
         this.db = db;
         this.stubs = stubs;
         this.opts = opts;
         this.vars = vars;
-        this.#http = new HttpClient(app.url, opts.http);
+        this.specs = specs;
+        this.coverageDir = coverageDir;
+        this.recorders = recorders;
+        this.recordDir = recordDir;
+        this.containers = containers;
+        this.#http = this.#client();
+    }
+    #client() {
+        const http = new HttpClient(this.app.url, this.opts.http);
+        const spec = this.specs.app;
+        if (spec) {
+            http.onResponse((res) => {
+                try {
+                    const path = res.url.split("?")[0];
+                    const key = spec.responseKey(res.method, path, res.status);
+                    if (key)
+                        this.#covered.add(key);
+                    this.#contract.push(...spec.checkResponse(res.method, path, message(res)));
+                }
+                catch (e) {
+                    this.#contract.push(`${res.method} ${res.url}: couldn't check against ${spec.file}: ${e.message}`);
+                }
+            });
+        }
+        return http;
     }
     get http() {
         return this.#http;
     }
     static async start(opts, shared) {
-        const url = await ensureWorkerDatabase(shared.adminUrl, shared.template, shared.prefix);
+        const engine = await engineFor(opts);
+        const url = await ensureWorkerDatabase(engine, shared.adminUrl, shared.template, shared.prefix);
         const stubs = new Map();
+        const recorders = new Map();
+        const containers = new Map();
+        const services = new Map();
         let db;
         try {
+            // Loaded first: a broken spec should fail before anything is started.
+            const specs = {
+                app: opts.openapi.app ? await OpenApiSpec.load(path.resolve(opts.root, opts.openapi.app), opts.openapi.app) : undefined,
+                stubs: new Map(),
+            };
+            for (const [name, file] of Object.entries(opts.openapi.stubs))
+                specs.stubs.set(name, await OpenApiSpec.load(path.resolve(opts.root, file), file));
             for (const name of opts.stubs)
                 stubs.set(name, await Stub.start(name));
-            db = await Db.connect(url, {
+            for (const [name, r] of Object.entries(opts.recordings))
+                recorders.set(name, await Recorder.load(name, path.resolve(opts.root, r.file), r.upstream, r.record));
+            // A registered route wins, then a recording of the real service, then an answer made up from its spec.
+            for (const [name, stub] of stubs) {
+                const recorder = recorders.get(name);
+                const spec = opts.openapi.autoReply.includes(name) ? specs.stubs.get(name) : undefined;
+                if (!recorder && !spec)
+                    continue;
+                stub.fallback(async (call) => (await recorder?.answer(call)) ?? spec?.exampleResponse(call.method, call.path), recorder?.hint());
+            }
+            db = await Db.connect(await engine.driver(url), url, {
                 schemas: opts.db.schemas,
                 keep: opts.db.keep,
                 seedFile: opts.db.seed && path.resolve(opts.root, opts.db.seed),
             });
+            const started = await Promise.allSettled(Object.entries(opts.containers).map(async ([name, c]) => containers.set(name, await Dependency.start(name, c))));
+            const failed = started.find((r) => r.status === "rejected");
+            if (failed)
+                throw failed.reason;
             const vars = { "db.url": url };
+            for (const [name, c] of containers) {
+                vars[`container.${name}`] = c.address;
+                vars[`container.${name}.host`] = c.host;
+                vars[`container.${name}.port`] = String(c.port);
+            }
             for (const [name, stub] of stubs)
                 vars[`stub.${name}`] = stub.url;
+            for (const [name, service] of Object.entries(opts.services)) {
+                const started = await App.start(service, opts.root, vars, `service.${name}`);
+                services.set(name, started);
+                vars[`service.${name}`] = started.url;
+                vars[`service.${name}.port`] = String(started.port);
+            }
             const app = await App.start(opts.app, opts.root, vars);
-            return new Runtime(app, db, stubs, opts, vars);
+            return new Runtime(app, services, db, stubs, opts, vars, specs, shared.coverageDir, recorders, shared.recordDir, containers);
         }
         catch (e) {
+            await Promise.all([...services.values()].map((s) => s.stop()));
             await db?.close();
             await Promise.all([...stubs.values()].map((s) => s.close()));
+            await Promise.allSettled([...containers.values()].map((c) => c.stop()));
             throw e;
         }
     }
@@ -60,29 +138,90 @@ export class Runtime {
                 }
                 return stub;
             },
+            service: (name) => {
+                const service = this.services.get(name);
+                if (!service) {
+                    throw new Error(`slicetest: unknown service "${name}". Declared services: ${[...this.services.keys()].join(", ") || "(none)"}`);
+                }
+                return service;
+            },
+            container: (name) => {
+                const c = this.containers.get(name);
+                if (!c)
+                    throw new Error(`slicetest: unknown container "${name}". Declared containers: ${[...this.containers.keys()].join(", ") || "(none)"}`);
+                return c;
+            },
+            trace: async (opts) => mask(buildTrace(this.http.history, this.stubs.values(), await this.db.changesSinceStart()), opts),
         };
     }
+    /** The app and every service, for checks that apply to all of them. */
+    #processes() {
+        return [this.app, ...this.services.values()];
+    }
     async beforeScenario() {
-        // A crash already failed the scenario that caused it; give the next one a fresh app.
+        // A crash already failed the scenario that caused it; give the next one a fresh process.
+        for (const [name, service] of this.services) {
+            if (!service.exited)
+                continue;
+            await service.stop();
+            this.services.set(name, await App.start(this.opts.services[name], this.opts.root, this.vars, `service.${name}`, service.port));
+        }
         if (this.app.exited) {
             await this.app.stop();
             this.app = await App.start(this.opts.app, this.opts.root, this.vars);
-            this.#http = new HttpClient(this.app.url, this.opts.http);
+            this.#http = this.#client();
         }
+        this.#contract = [];
         await this.db.reset();
         for (const stub of this.stubs.values())
             stub.reset();
+        for (const recorder of this.recorders.values())
+            recorder.reset();
+        await Promise.all([...this.containers.values()].map((c) => c.reset()));
         this.http.reset();
-        this.#logMark = this.app.mark();
+        for (const p of this.#processes())
+            p.beginScenario();
     }
     /** Failures that the scenario body can't see on its own. */
     async afterScenario() {
-        await this.app.settle();
+        await Promise.all(this.#processes().map((p) => p.settle()));
         this.#assertAlive();
         const unmatched = this.#unmatched();
         if (unmatched.length > 0) {
             throw new Error(`slicetest: the app called stubbed services with no matching route:\n${unmatched.join("\n")}`);
         }
+        const contract = this.#contractViolations();
+        if (contract.length > 0) {
+            throw new Error(`slicetest: traffic doesn't match the OpenAPI spec:\n${contract.map((c) => `  ${c}`).join("\n")}`);
+        }
+    }
+    /** The app's responses against its spec, and its calls to stubs (and the stubs' replies) against theirs. */
+    #contractViolations() {
+        const out = this.#contract.map((c) => `app: ${c}`);
+        for (const [name, spec] of this.specs.stubs) {
+            for (const call of this.stubs.get(name).calls()) {
+                try {
+                    out.push(...this.#checkStubCall(name, spec, call));
+                }
+                catch (e) {
+                    out.push(`app → ${name}: ${call.method} ${call.path}: couldn't check against ${spec.file}: ${e.message}`);
+                }
+            }
+        }
+        return [...new Set(out)];
+    }
+    #checkStubCall(name, spec, call) {
+        if (!call.matched)
+            return []; // already reported as an unmatched call
+        const req = { contentType: header(call.headers["content-type"]), body: call.json ?? call.body, query: call.query };
+        const out = spec.checkRequest(call.method, call.path, req).map((c) => `app → ${name}: ${c}`);
+        const res = call.response;
+        if (res) {
+            const body = res.body ? (parseJson(res.body) ?? res.body) : "";
+            const problems = spec.checkResponse(call.method, call.path, { status: res.status, contentType: res.headers["content-type"], body });
+            out.push(...problems.map((c) => `stub ${name} reply (the real service wouldn't answer this way): ${c}`));
+        }
+        return out;
     }
     #unmatched() {
         return [...this.stubs.values()].flatMap((s) => {
@@ -93,53 +232,93 @@ export class Runtime {
             return [
                 ...calls.map((c) => `  ${s.name}: ${c.method} ${c.path}${c.query.size ? `?${c.query}` : ""}`),
                 `    registered on ${s.name}: ${routes.length ? routes.join(", ") : "(none)"}`,
+                ...(this.recorders.has(s.name) ? [`    ${this.recorders.get(s.name).hint()}`] : []),
             ];
         });
     }
     /** What happened during the current scenario, printed when it fails. */
-    diagnostics() {
+    async diagnostics() {
         const sections = [];
-        const exit = this.app.exited;
-        if (exit)
-            sections.push(`the app exited (code ${exit.code}, signal ${exit.signal}); it will be restarted for the next scenario`);
+        for (const p of this.#processes()) {
+            const exit = p.exited;
+            if (exit)
+                sections.push(`the ${p.label} exited (code ${exit.code}, signal ${exit.signal}); it will be restarted for the next scenario`);
+        }
         const unmatched = this.#unmatched();
         if (unmatched.length > 0)
             sections.push(`stub calls with no matching route:\n${unmatched.join("\n")}`);
+        const contract = this.#contractViolations();
+        if (contract.length > 0)
+            sections.push(`OpenAPI mismatches:\n${contract.map((c) => `  ${c}`).join("\n")}`);
         if (this.http.history.length > 0)
             sections.push(`requests to the app:\n${formatHistory(this.http.history)}`);
-        const logs = this.app.logs(this.#logMark);
+        try {
+            const changes = formatChanges(await this.db.changesSinceStart());
+            sections.push(changes ? `database changes during this scenario:\n${changes}` : "database changes during this scenario: (none)");
+        }
+        catch (e) {
+            sections.push(`database changes during this scenario: unavailable (${e.message})`);
+        }
+        const logs = this.app.scenarioLogs();
         sections.push(logs ? `app output during this scenario:\n${logs}` : "app output during this scenario: (none)");
+        for (const [name, service] of this.services) {
+            const out = service.scenarioLogs();
+            sections.push(out ? `service ${name} output during this scenario:\n${out}` : `service ${name} output during this scenario: (none)`);
+        }
         return sections.join("\n\n");
     }
     async stop() {
+        if (this.coverageDir && this.#covered.size > 0) {
+            await writeFile(path.join(this.coverageDir, `${process.pid}-${randomUUID()}.json`), JSON.stringify([...this.#covered])).catch(() => { });
+        }
+        for (const [name, recorder] of this.recorders) {
+            if (this.recordDir && recorder.added().length > 0) {
+                await writeFile(path.join(this.recordDir, `${name}.${process.pid}-${randomUUID()}.json`), JSON.stringify(recorder.added())).catch(() => { });
+            }
+        }
         const results = await Promise.allSettled([
             this.app.stop(),
+            ...[...this.services.values()].map((s) => s.stop()),
             this.db.close(),
             ...[...this.stubs.values()].map((s) => s.close()),
+            ...[...this.containers.values()].map((c) => c.stop()),
         ]);
         const failed = results.find((r) => r.status === "rejected");
         if (failed)
             throw failed.reason;
     }
     #assertAlive() {
-        const exit = this.app.exited;
-        if (exit) {
-            throw new Error(`slicetest: app process exited (code ${exit.code}, signal ${exit.signal})\n${this.app.logs()}`);
+        for (const p of this.#processes()) {
+            const exit = p.exited;
+            if (exit)
+                throw new Error(`slicetest: ${p.label} process exited (code ${exit.code}, signal ${exit.signal})\n${p.logs()}`);
         }
     }
 }
 /** One database per vitest worker, cloned from the migrated template and reused across its test files. */
-async function ensureWorkerDatabase(adminUrl, template, prefix) {
+async function ensureWorkerDatabase(engine, adminUrl, template, prefix) {
     const name = `${prefix}_w${process.env.VITEST_POOL_ID ?? process.pid}`;
-    const admin = new pg.Client({ connectionString: adminUrl });
-    await admin.connect();
+    const admin = await engine.admin(adminUrl);
     try {
-        const { rowCount } = await admin.query("SELECT 1 FROM pg_database WHERE datname = $1", [name]);
-        if (!rowCount)
-            await admin.query(`CREATE DATABASE "${name}" TEMPLATE "${template}"`);
+        if (!(await admin.databases(name)).includes(name))
+            await admin.clone(template, name);
+        return admin.urlFor(name);
     }
     finally {
-        await admin.end();
+        await admin.close();
+    }
+}
+function message(res) {
+    return { status: res.status, contentType: res.headers.get("content-type") ?? undefined, body: res.json ?? res.text };
+}
+function header(v) {
+    return Array.isArray(v) ? v[0] : v;
+}
+function parseJson(text) {
+    try {
+        return JSON.parse(text);
+    }
+    catch {
+        return undefined;
     }
-    return withDatabase(adminUrl, name);
 }

package/dist/scenario.js

@@ -15,8 +15,8 @@ function define(register) {
         if (task.concurrent) {
             throw new Error("slicetest: scenarios share one app and database per file, so they can't run concurrently. Remove .concurrent / sequence.concurrent.");
         }
-        onTestFailed(() => {
-            console.error(`--- slicetest ---\n${runtime.diagnostics()}\n-----------------`);
+        onTestFailed(async () => {
+            console.error(`--- slicetest ---\n${await runtime.diagnostics()}\n-----------------`);
         });
         await runtime.beforeScenario();
         await body(runtime.context());

package/dist/stub.d.ts

@@ -11,6 +11,14 @@ export interface RecordedCall {
     params: Record<string, string>;
     /** Whether a registered route answered this call. */
     matched: boolean;
+    /** What the stub answered, once it has. */
+    response?: {
+        status: number;
+        headers: Record<string, string>;
+        body: string;
+    };
+    /** Answered by the fallback (e.g. an example from the provider's OpenAPI spec), not a registered route. */
+    fallback?: boolean;
 }
 export interface StubResponse {
     status?: number;
@@ -68,6 +76,12 @@ export declare class Stub {
     /** Human-readable list of registered routes, for diagnostics. */
     describeRoutes(): string[];
     reset(): void;
+    /**
+     * Answer calls that no registered route matches, instead of failing with 501.
+     * Return undefined to leave a call unanswered (it then fails the scenario as usual).
+     * Kept across scenarios, unlike routes.
+     */
+    fallback(respond: ((call: RecordedCall) => StubResponse | undefined | Promise<StubResponse | undefined>) | undefined, hint?: string): this;
     close(): Promise<void>;
 }
 /** `expected` is contained in `actual`: objects compare key by key, arrays element-wise. */

package/dist/stub.js

@@ -9,6 +9,9 @@ export class Stub {
     #server;
     #routes = [];
     #calls = [];
+    #fallback;
+    /** Appended to the 501 answer for a call nothing could answer. */
+    #hint;
     url = "";
     constructor(name) {
         this.name = name;
@@ -80,6 +83,16 @@ export class Stub {
         this.#routes = [];
         this.#calls = [];
     }
+    /**
+     * Answer calls that no registered route matches, instead of failing with 501.
+     * Return undefined to leave a call unanswered (it then fails the scenario as usual).
+     * Kept across scenarios, unlike routes.
+     */
+    fallback(respond, hint) {
+        this.#fallback = respond;
+        this.#hint = hint;
+        return this;
+    }
     async close() {
         this.#server.closeAllConnections();
         await new Promise((resolve) => this.#server.close(resolve));
@@ -115,7 +128,22 @@ export class Stub {
             break;
         }
         if (!route) {
-            res.writeHead(501, { "content-type": "text/plain" }).end(`slicetest: no stub for ${call.method} ${call.path}`);
+            let out;
+            try {
+                out = await this.#fallback?.(call);
+            }
+            catch (e) {
+                res.writeHead(502, { "content-type": "text/plain" }).end(e.message);
+                return;
+            }
+            if (!out) {
+                const hint = this.#hint ? ` (${this.#hint})` : "";
+                res.writeHead(501, { "content-type": "text/plain" }).end(`slicetest: no stub for ${call.method} ${call.path}${hint}`);
+                return;
+            }
+            call.matched = true;
+            call.fallback = true;
+            this.#send(call, res, out);
             return;
         }
         call.matched = true;
@@ -128,19 +156,27 @@ export class Stub {
             return;
         }
         try {
-            const out = typeof route.respond === "function" ? await route.respond(call) : route.respond;
-            const raw = out.body === undefined || typeof out.body === "string" || out.body instanceof Uint8Array;
-            const headers = { ...out.headers };
-            if (!raw && !Object.keys(headers).some((h) => h.toLowerCase() === "content-type")) {
-                headers["content-type"] = "application/json";
-            }
-            res.writeHead(out.status ?? 200, headers);
-            res.end(raw ? out.body : JSON.stringify(out.body));
+            this.#send(call, res, typeof route.respond === "function" ? await route.respond(call) : route.respond);
         }
         catch (e) {
             res.writeHead(500).end(`slicetest: stub handler threw: ${e}`);
         }
     }
+    #send(call, res, out) {
+        const raw = out.body === undefined || typeof out.body === "string" || out.body instanceof Uint8Array;
+        const headers = { ...out.headers };
+        if (!raw && !Object.keys(headers).some((h) => h.toLowerCase() === "content-type")) {
+            headers["content-type"] = "application/json";
+        }
+        const payload = raw ? out.body : JSON.stringify(out.body);
+        call.response = {
+            status: out.status ?? 200,
+            headers: Object.fromEntries(Object.entries(headers).map(([k, v]) => [k.toLowerCase(), v])),
+            body: payload === undefined ? "" : typeof payload === "string" ? payload : Buffer.from(payload).toString("utf8"),
+        };
+        res.writeHead(out.status ?? 200, headers);
+        res.end(payload);
+    }
 }
 function compilePath(path) {
     if (typeof path !== "string" || !path.includes("/:"))

package/dist/trace.d.ts

@@ -0,0 +1,35 @@
+import type { Changes } from "./db.js";
+import type { HttpResponse } from "./http.js";
+import type { Stub } from "./stub.js";
+/**
+ * Everything one scenario did at the three boundaries: the requests it made to
+ * the app, the calls the app made to stubs, and the rows it changed. Meant for
+ * `expect(await trace()).toMatchSnapshot()`: resets make ids deterministic,
+ * and `mask` replaces the values that aren't (timestamps, UUIDs).
+ */
+export interface Trace {
+    http: {
+        request: string;
+        status: number;
+        body?: unknown;
+    }[];
+    stubs: Record<string, {
+        request: string;
+        body?: unknown;
+        status?: number;
+    }[]>;
+    db: Changes;
+}
+export interface MaskOptions {
+    /** Keys whose values are always replaced, e.g. `["token", "password_hash"]`. Matched at any depth. */
+    keys?: string[];
+    /** Also mask strings matching these patterns, e.g. `[/^tok_\w+$/]`. */
+    patterns?: RegExp[];
+}
+/**
+ * A copy of `value` with the parts that change from run to run replaced by
+ * placeholders: dates (`Date` objects and ISO strings) by `[date]`, UUIDs by
+ * `[uuid]`, and the values of `keys` / strings matching `patterns` by `[masked]`.
+ */
+export declare function mask<T>(value: T, opts?: MaskOptions): T;
+export declare function buildTrace(history: readonly HttpResponse[], stubs: Iterable<Stub>, db: Changes): Trace;

package/dist/trace.js

@@ -0,0 +1,57 @@
+const ISO_DATE = /^\d{4}-\d{2}-\d{2}([T ]\d{2}:\d{2}(:\d{2}(\.\d+)?)?(Z|[+-]\d{2}:?\d{2})?)?$/;
+const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
+/**
+ * A copy of `value` with the parts that change from run to run replaced by
+ * placeholders: dates (`Date` objects and ISO strings) by `[date]`, UUIDs by
+ * `[uuid]`, and the values of `keys` / strings matching `patterns` by `[masked]`.
+ */
+export function mask(value, opts = {}) {
+    const keys = new Set(opts.keys ?? []);
+    const walk = (v) => {
+        if (v instanceof Date)
+            return "[date]";
+        if (typeof v === "bigint")
+            return v.toString();
+        if (typeof v === "string") {
+            if (UUID.test(v))
+                return "[uuid]";
+            if (ISO_DATE.test(v))
+                return "[date]";
+            if (opts.patterns?.some((p) => ((p.lastIndex = 0), p.test(v))))
+                return "[masked]";
+            return v;
+        }
+        if (Array.isArray(v))
+            return v.map(walk);
+        if (v && typeof v === "object") {
+            return Object.fromEntries(Object.entries(v).map(([k, x]) => [k, keys.has(k) && x != null ? "[masked]" : walk(x)]));
+        }
+        return v;
+    };
+    return walk(value);
+}
+export function buildTrace(history, stubs, db) {
+    const trace = {
+        http: history.map((r) => {
+            const url = new URL(r.url, "http://app");
+            const body = r.json !== undefined ? r.json : r.text || undefined;
+            return { request: `${r.method} ${url.pathname}${url.search}`, status: r.status, ...(body !== undefined ? { body } : {}) };
+        }),
+        stubs: {},
+        db,
+    };
+    for (const stub of stubs) {
+        const calls = stub.calls();
+        if (calls.length === 0)
+            continue;
+        trace.stubs[stub.name] = calls.map((c) => {
+            const body = c.json !== undefined ? c.json : c.body || undefined;
+            return {
+                request: `${c.method} ${c.path}${c.query.size ? `?${c.query}` : ""}`,
+                ...(body !== undefined ? { body } : {}),
+                ...(c.response ? { status: c.response.status } : {}),
+            };
+        });
+    }
+    return trace;
+}

package/dist/yaml-runtime.js

@@ -1,4 +1,5 @@
 import { expect } from "vitest";
+import { formatChanges } from "./db.js";
 import "./matchers.js";
 import { scenario } from "./scenario.js";
 /** Registers the scenarios of one parsed YAML file. Called from the module the plugin generates for it. */
@@ -15,7 +16,8 @@ export function defineYamlScenarios(doc) {
 async function runSteps(doc, sc, steps, ctx, vars) {
     for (const [i, step] of steps.entries()) {
         try {
-            await runStep(step, ctx, vars);
+            // A log step waits by itself; `within` is its timeout.
+            await retry("within" in step && !("log" in step) ? step.within : undefined, () => runStep(step, ctx, vars));
         }
         catch (e) {
             const label = step.name ?? describeStep(step);
@@ -39,9 +41,73 @@ function describeStep(step) {
         return `insert ${step.insert}`;
     if ("db" in step)
         return `db ${step.db}`;
+    if ("changes" in step)
+        return "changes";
+    if ("checkpoint" in step)
+        return "checkpoint";
+    if ("snapshot" in step)
+        return "snapshot";
+    if ("log" in step)
+        return `log ${step.from ? `from ${step.from} ` : ""}/${step.log}/`;
     return "sql";
 }
+/** Run `fn` until it passes or `within` ms have passed, then rethrow its last error. */
+async function retry(within, fn) {
+    if (!within)
+        return fn();
+    const deadline = Date.now() + within;
+    for (;;) {
+        try {
+            return await fn();
+        }
+        catch (e) {
+            if (Date.now() >= deadline) {
+                if (e instanceof Error)
+                    e.message = `${e.message}\n(still failing after retrying for ${within}ms)`;
+                throw e;
+            }
+            await new Promise((r) => setTimeout(r, 50));
+        }
+    }
+}
 async function runStep(step, ctx, vars) {
+    if ("checkpoint" in step) {
+        await ctx.db.checkpoint();
+        return;
+    }
+    if ("snapshot" in step) {
+        expect(await ctx.trace({ keys: step.mask })).toMatchSnapshot();
+        return;
+    }
+    if ("log" in step) {
+        const target = step.from ? ctx.service(step.from) : ctx.app;
+        await target.waitForLog(new RegExp(interpolate(step.log, vars)), step.within ?? 5000);
+        return;
+    }
+    if ("changes" in step) {
+        const actual = await ctx.db.changes();
+        const expected = interpolate(step.changes, vars);
+        const unexpected = Object.keys(actual).filter((t) => !(t in expected));
+        if (unexpected.length) {
+            throw new Error(`unexpected changes in ${unexpected.join(", ")}:\n${formatChanges(Object.fromEntries(unexpected.map((t) => [t, actual[t]])))}`);
+        }
+        for (const [table, spec] of Object.entries(expected)) {
+            const got = actual[table] ?? { inserted: [], updated: [], deleted: [] };
+            for (const kind of ["inserted", "updated", "deleted"]) {
+                const want = spec[kind];
+                if (want === undefined)
+                    continue;
+                const rows = kind === "updated" ? got.updated.map((u) => u.after) : got[kind];
+                if (typeof want === "number") {
+                    if (rows.length !== want)
+                        throw new Error(`expected ${want} row(s) ${kind} in ${table}, got ${rows.length}\n${formatChanges({ [table]: got }) || "  (no changes)"}`);
+                }
+                else
+                    check(rows, want, vars, `${kind} rows of ${table}`);
+            }
+        }
+        return;
+    }
     if ("stub" in step) {
         const [method, path] = splitCall(step.on);
         let route = ctx.stub(step.stub).on(method, interpolate(path, vars), conditions(step.when, vars));