slicetest 0.1.0 → 0.2.0

package/dist/stub.d.ts

@@ -11,6 +11,14 @@ export interface RecordedCall {
     params: Record<string, string>;
     /** Whether a registered route answered this call. */
     matched: boolean;
+    /** What the stub answered, once it has. */
+    response?: {
+        status: number;
+        headers: Record<string, string>;
+        body: string;
+    };
+    /** Answered by the fallback (e.g. an example from the provider's OpenAPI spec), not a registered route. */
+    fallback?: boolean;
 }
 export interface StubResponse {
     status?: number;
@@ -68,6 +76,12 @@ export declare class Stub {
     /** Human-readable list of registered routes, for diagnostics. */
     describeRoutes(): string[];
     reset(): void;
+    /**
+     * Answer calls that no registered route matches, instead of failing with 501.
+     * Return undefined to leave a call unanswered (it then fails the scenario as usual).
+     * Kept across scenarios, unlike routes.
+     */
+    fallback(respond: ((call: RecordedCall) => StubResponse | undefined) | undefined): this;
     close(): Promise<void>;
 }
 /** `expected` is contained in `actual`: objects compare key by key, arrays element-wise. */

package/dist/stub.js

@@ -9,6 +9,7 @@ export class Stub {
     #server;
     #routes = [];
     #calls = [];
+    #fallback;
     url = "";
     constructor(name) {
         this.name = name;
@@ -80,6 +81,15 @@ export class Stub {
         this.#routes = [];
         this.#calls = [];
     }
+    /**
+     * Answer calls that no registered route matches, instead of failing with 501.
+     * Return undefined to leave a call unanswered (it then fails the scenario as usual).
+     * Kept across scenarios, unlike routes.
+     */
+    fallback(respond) {
+        this.#fallback = respond;
+        return this;
+    }
     async close() {
         this.#server.closeAllConnections();
         await new Promise((resolve) => this.#server.close(resolve));
@@ -115,7 +125,14 @@ export class Stub {
             break;
         }
         if (!route) {
-            res.writeHead(501, { "content-type": "text/plain" }).end(`slicetest: no stub for ${call.method} ${call.path}`);
+            const out = this.#fallback?.(call);
+            if (!out) {
+                res.writeHead(501, { "content-type": "text/plain" }).end(`slicetest: no stub for ${call.method} ${call.path}`);
+                return;
+            }
+            call.matched = true;
+            call.fallback = true;
+            this.#send(call, res, out);
             return;
         }
         call.matched = true;
@@ -128,19 +145,27 @@ export class Stub {
             return;
         }
         try {
-            const out = typeof route.respond === "function" ? await route.respond(call) : route.respond;
-            const raw = out.body === undefined || typeof out.body === "string" || out.body instanceof Uint8Array;
-            const headers = { ...out.headers };
-            if (!raw && !Object.keys(headers).some((h) => h.toLowerCase() === "content-type")) {
-                headers["content-type"] = "application/json";
-            }
-            res.writeHead(out.status ?? 200, headers);
-            res.end(raw ? out.body : JSON.stringify(out.body));
+            this.#send(call, res, typeof route.respond === "function" ? await route.respond(call) : route.respond);
         }
         catch (e) {
             res.writeHead(500).end(`slicetest: stub handler threw: ${e}`);
         }
     }
+    #send(call, res, out) {
+        const raw = out.body === undefined || typeof out.body === "string" || out.body instanceof Uint8Array;
+        const headers = { ...out.headers };
+        if (!raw && !Object.keys(headers).some((h) => h.toLowerCase() === "content-type")) {
+            headers["content-type"] = "application/json";
+        }
+        const payload = raw ? out.body : JSON.stringify(out.body);
+        call.response = {
+            status: out.status ?? 200,
+            headers: Object.fromEntries(Object.entries(headers).map(([k, v]) => [k.toLowerCase(), v])),
+            body: payload === undefined ? "" : typeof payload === "string" ? payload : Buffer.from(payload).toString("utf8"),
+        };
+        res.writeHead(out.status ?? 200, headers);
+        res.end(payload);
+    }
 }
 function compilePath(path) {
     if (typeof path !== "string" || !path.includes("/:"))

package/dist/yaml-runtime.js

@@ -1,4 +1,5 @@
 import { expect } from "vitest";
+import { formatChanges } from "./db.js";
 import "./matchers.js";
 import { scenario } from "./scenario.js";
 /** Registers the scenarios of one parsed YAML file. Called from the module the plugin generates for it. */
@@ -15,7 +16,8 @@ export function defineYamlScenarios(doc) {
 async function runSteps(doc, sc, steps, ctx, vars) {
     for (const [i, step] of steps.entries()) {
         try {
-            await runStep(step, ctx, vars);
+            // A log step waits by itself; `within` is its timeout.
+            await retry("within" in step && !("log" in step) ? step.within : undefined, () => runStep(step, ctx, vars));
         }
         catch (e) {
             const label = step.name ?? describeStep(step);
@@ -39,9 +41,67 @@ function describeStep(step) {
         return `insert ${step.insert}`;
     if ("db" in step)
         return `db ${step.db}`;
+    if ("changes" in step)
+        return "changes";
+    if ("checkpoint" in step)
+        return "checkpoint";
+    if ("log" in step)
+        return `log ${step.from ? `from ${step.from} ` : ""}/${step.log}/`;
     return "sql";
 }
+/** Run `fn` until it passes or `within` ms have passed, then rethrow its last error. */
+async function retry(within, fn) {
+    if (!within)
+        return fn();
+    const deadline = Date.now() + within;
+    for (;;) {
+        try {
+            return await fn();
+        }
+        catch (e) {
+            if (Date.now() >= deadline) {
+                if (e instanceof Error)
+                    e.message = `${e.message}\n(still failing after retrying for ${within}ms)`;
+                throw e;
+            }
+            await new Promise((r) => setTimeout(r, 50));
+        }
+    }
+}
 async function runStep(step, ctx, vars) {
+    if ("checkpoint" in step) {
+        await ctx.db.checkpoint();
+        return;
+    }
+    if ("log" in step) {
+        const target = step.from ? ctx.service(step.from) : ctx.app;
+        await target.waitForLog(new RegExp(interpolate(step.log, vars)), step.within ?? 5000);
+        return;
+    }
+    if ("changes" in step) {
+        const actual = await ctx.db.changes();
+        const expected = interpolate(step.changes, vars);
+        const unexpected = Object.keys(actual).filter((t) => !(t in expected));
+        if (unexpected.length) {
+            throw new Error(`unexpected changes in ${unexpected.join(", ")}:\n${formatChanges(Object.fromEntries(unexpected.map((t) => [t, actual[t]])))}`);
+        }
+        for (const [table, spec] of Object.entries(expected)) {
+            const got = actual[table] ?? { inserted: [], updated: [], deleted: [] };
+            for (const kind of ["inserted", "updated", "deleted"]) {
+                const want = spec[kind];
+                if (want === undefined)
+                    continue;
+                const rows = kind === "updated" ? got.updated.map((u) => u.after) : got[kind];
+                if (typeof want === "number") {
+                    if (rows.length !== want)
+                        throw new Error(`expected ${want} row(s) ${kind} in ${table}, got ${rows.length}\n${formatChanges({ [table]: got }) || "  (no changes)"}`);
+                }
+                else
+                    check(rows, want, vars, `${kind} rows of ${table}`);
+            }
+        }
+        return;
+    }
     if ("stub" in step) {
         const [method, path] = splitCall(step.on);
         let route = ctx.stub(step.stub).on(method, interpolate(path, vars), conditions(step.when, vars));

package/dist/yaml.d.ts

@@ -19,7 +19,7 @@ export interface YamlScenario {
     timeout?: number;
     steps: Step[];
 }
-export type Step = (StubStep | RequestStep | InsertStep | SqlStep | DbStep | ReceivedStep) & {
+export type Step = (StubStep | RequestStep | InsertStep | SqlStep | DbStep | ReceivedStep | ChangesStep | CheckpointStep | LogStep) & {
     line: number;
     name?: string;
 };
@@ -76,6 +76,7 @@ export interface SqlStep {
         count?: number;
     };
     capture?: Record<string, string>;
+    within?: number;
 }
 export interface DbStep {
     db: string;
@@ -86,6 +87,8 @@ export interface DbStep {
         count?: number;
     };
     capture?: Record<string, string>;
+    /** Retry for up to this many milliseconds, for effects the app applies asynchronously. */
+    within?: number;
 }
 export interface ReceivedStep {
     received: string;
@@ -93,6 +96,32 @@ export interface ReceivedStep {
     when?: Conditions;
     /** Exact number of matching calls. Default: at least one. */
     times?: number;
+    within?: number;
+}
+/** Expected rows per table: a count, or a list of subset rows. */
+export type ChangeSpec = {
+    inserted?: number | unknown[];
+    updated?: number | unknown[];
+    deleted?: number | unknown[];
+};
+/**
+ * The database changed exactly in these tables since the scenario started (or
+ * the last checkpoint). Tables that aren't listed must be unchanged.
+ */
+export interface ChangesStep {
+    changes: Record<string, ChangeSpec>;
+    within?: number;
+}
+/** Wait until the app (or `from:` a service) prints a line matching `log` (a regex) during the scenario. */
+export interface LogStep {
+    log: string;
+    from?: string;
+    /** Milliseconds to wait. Default 5000. */
+    within?: number;
+}
+/** Later `changes` steps only see what happens after this step. */
+export interface CheckpointStep {
+    checkpoint: true;
 }
 export declare class YamlScenarioError extends Error {
 }

package/dist/yaml.js

@@ -3,9 +3,12 @@ const KINDS = {
     stub: ["on", "when", "reply", "sequence", "networkError", "times", "delay"],
     request: ["headers", "query", "json", "form", "body", "follow", "expect", "capture"],
     insert: ["rows", "capture"],
-    sql: ["params", "expect", "capture"],
-    db: ["where", "orderBy", "expect", "capture"],
-    received: ["call", "when", "times"],
+    sql: ["params", "expect", "capture", "within"],
+    db: ["where", "orderBy", "expect", "capture", "within"],
+    received: ["call", "when", "times", "within"],
+    changes: ["within"],
+    log: ["from", "within"],
+    checkpoint: [],
 };
 const EXPECT_KEYS = {
     request: ["status", "headers", "json", "text"],
@@ -13,6 +16,7 @@ const EXPECT_KEYS = {
     db: ["rows", "count"],
 };
 const CONDITION_KEYS = ["query", "headers", "json", "body"];
+const CHANGE_KEYS = ["inserted", "updated", "deleted"];
 const RESPONSE_KEYS = ["status", "headers", "body"];
 const SCENARIO_KEYS = ["name", "steps", "each", "skip", "only", "timeout"];
 const CALL = /^([A-Za-z]+|\*)\s+(\/\S*)$/;
@@ -88,7 +92,26 @@ function parseStep(node, fail, lineOf) {
         if (!allowed.has(key))
             fail(at(key), `unknown key "${key}" in a ${kind} step (allowed: ${[...allowed].join(", ")})`);
     }
-    if (typeof raw[kind] !== "string" || !raw[kind])
+    if (kind === "changes") {
+        const tables = raw.changes;
+        if (!tables || typeof tables !== "object" || Array.isArray(tables))
+            fail(at(kind), "`changes:` must map table names to { inserted, updated, deleted }");
+        for (const [table, spec] of Object.entries(tables)) {
+            if (!spec || typeof spec !== "object" || Array.isArray(spec))
+                fail(at(kind), `changes of "${table}" must be a mapping such as { inserted: 1 }`);
+            for (const [k, v] of Object.entries(spec)) {
+                if (!CHANGE_KEYS.includes(k))
+                    fail(at(kind), `unknown key "${k}" in changes of "${table}" (allowed: ${CHANGE_KEYS.join(", ")})`);
+                if (!Array.isArray(v) && !(typeof v === "number" && v >= 0))
+                    fail(at(kind), `${table}.${k} must be a count or a list of rows`);
+            }
+        }
+    }
+    else if (kind === "checkpoint") {
+        if (raw.checkpoint !== true)
+            fail(at(kind), "use `checkpoint: true`");
+    }
+    else if (typeof raw[kind] !== "string" || !raw[kind])
         fail(at(kind), `\`${kind}:\` must be a non-empty string`);
     const keysOf = (key, allowedKeys) => {
         const v = raw[key];
@@ -105,6 +128,8 @@ function parseStep(node, fail, lineOf) {
         if (raw[key] !== undefined && !CALL.test(String(raw[key])))
             fail(at(key), `\`${key}\` must look like "POST /path", got "${raw[key]}"`);
     };
+    if (raw.within !== undefined && (typeof raw.within !== "number" || raw.within <= 0))
+        fail(at("within"), "`within` must be a positive number of milliseconds");
     const number = (key) => {
         if (raw[key] !== undefined && (typeof raw[key] !== "number" || raw[key] < 0))
             fail(at(key), `\`${key}\` must be a non-negative number`);
@@ -142,6 +167,16 @@ function parseStep(node, fail, lineOf) {
             if (!raw.rows || typeof raw.rows !== "object")
                 fail(at("rows"), "an insert step needs `rows` (a mapping or a list of mappings)");
             break;
+        case "log":
+            try {
+                new RegExp(raw.log);
+            }
+            catch (e) {
+                fail(at("log"), `\`log\` must be a regular expression: ${e.message}`);
+            }
+            if (raw.from !== undefined && typeof raw.from !== "string")
+                fail(at("from"), "`from` must be a service name");
+            break;
         case "received":
             call("call");
             number("times");

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "slicetest",
-  "version": "0.1.0",
-  "description": "Framework-agnostic slice tests: drive a real app process over HTTP with a real Postgres and stubbed outbound APIs.",
+  "version": "0.2.0",
+  "description": "Framework-agnostic slice tests: drive a real app process over HTTP with a real Postgres and stubbed outbound APIs, checked against OpenAPI contracts.",
   "keywords": [
     "testing",
     "integration-testing",
@@ -14,7 +14,9 @@
     "vitest-plugin",
     "yaml",
     "testcontainers",
-    "language-agnostic"
+    "language-agnostic",
+    "openapi",
+    "contract-testing"
   ],
   "author": "pon-bok",
   "repository": {
@@ -52,7 +54,7 @@
     "build": "tsc -p tsconfig.build.json",
     "typecheck": "tsc --noEmit",
     "test": "vitest run",
-    "test:dist": "npm run build && vitest run --config examples/vitest.dist.config.ts && node dist/cli.js --config examples/slicetest.config.yaml",
+    "test:dist": "npm run build && tsc -p examples/tsconfig.dist.json && vitest run --config examples/vitest.dist.config.ts && node dist/cli.js --config examples/slicetest.config.yaml",
     "prepublishOnly": "npm run typecheck && npm run build"
   },
   "peerDependencies": {
@@ -60,6 +62,8 @@
   },
   "dependencies": {
     "@testcontainers/postgresql": "^12.2.0",
+    "ajv": "^8.20.0",
+    "ajv-formats": "^3.0.1",
     "pg": "^8.23.0",
     "testcontainers": "^12.2.0",
     "yaml": "^2.9.1"

package/schema/scenario.schema.json

@@ -277,6 +277,11 @@
               "additionalProperties": {
                 "type": "string"
               }
+            },
+            "within": {
+              "type": "integer",
+              "minimum": 1,
+              "description": "Retry this step for up to this many milliseconds, for effects the app applies asynchronously"
             }
           }
         },
@@ -325,6 +330,11 @@
               "additionalProperties": {
                 "type": "string"
               }
+            },
+            "within": {
+              "type": "integer",
+              "minimum": 1,
+              "description": "Retry this step for up to this many milliseconds, for effects the app applies asynchronously"
             }
           }
         },
@@ -368,6 +378,127 @@
               "type": "integer",
               "minimum": 0,
               "description": "Exact count; default: at least once"
+            },
+            "within": {
+              "type": "integer",
+              "minimum": 1,
+              "description": "Retry this step for up to this many milliseconds, for effects the app applies asynchronously"
+            }
+          }
+        },
+        {
+          "type": "object",
+          "required": [
+            "changes"
+          ],
+          "additionalProperties": false,
+          "properties": {
+            "changes": {
+              "type": "object",
+              "description": "The database changed exactly in these tables since the scenario started (or the last checkpoint). Unlisted tables must be unchanged.",
+              "additionalProperties": {
+                "type": "object",
+                "additionalProperties": false,
+                "properties": {
+                  "inserted": {
+                    "oneOf": [
+                      {
+                        "type": "integer",
+                        "minimum": 0
+                      },
+                      {
+                        "type": "array",
+                        "items": {
+                          "type": "object"
+                        }
+                      }
+                    ],
+                    "description": "Count, or subset rows"
+                  },
+                  "updated": {
+                    "oneOf": [
+                      {
+                        "type": "integer",
+                        "minimum": 0
+                      },
+                      {
+                        "type": "array",
+                        "items": {
+                          "type": "object"
+                        }
+                      }
+                    ],
+                    "description": "Count, or subset rows (after the update)"
+                  },
+                  "deleted": {
+                    "oneOf": [
+                      {
+                        "type": "integer",
+                        "minimum": 0
+                      },
+                      {
+                        "type": "array",
+                        "items": {
+                          "type": "object"
+                        }
+                      }
+                    ],
+                    "description": "Count, or subset rows"
+                  }
+                }
+              }
+            },
+            "name": {
+              "type": "string",
+              "description": "Label shown when this step fails"
+            },
+            "within": {
+              "type": "integer",
+              "minimum": 1,
+              "description": "Retry this step for up to this many milliseconds, for effects the app applies asynchronously"
+            }
+          }
+        },
+        {
+          "type": "object",
+          "required": [
+            "checkpoint"
+          ],
+          "additionalProperties": false,
+          "properties": {
+            "checkpoint": {
+              "const": true,
+              "description": "Later `changes` steps only see what happens after this step"
+            },
+            "name": {
+              "type": "string",
+              "description": "Label shown when this step fails"
+            }
+          }
+        },
+        {
+          "type": "object",
+          "required": [
+            "log"
+          ],
+          "additionalProperties": false,
+          "properties": {
+            "log": {
+              "type": "string",
+              "description": "Regular expression; waits until the app (or `from:` a service) prints a matching line during this scenario"
+            },
+            "from": {
+              "type": "string",
+              "description": "Service name (from the `services` option). Default: the app"
+            },
+            "within": {
+              "type": "integer",
+              "minimum": 1,
+              "description": "Milliseconds to wait. Default 5000"
+            },
+            "name": {
+              "type": "string",
+              "description": "Label shown when this step fails"
             }
           }
         }
@@ -402,4 +533,4 @@
       "additionalProperties": true
     }
   }
-}
+}