slicetest 0.8.0 → 0.10.0

package/dist/webhook.js

@@ -1,7 +1,7 @@
 import { createHmac, randomUUID } from "node:crypto";
 import { encodeForm } from "./form.js";
 /** The providers `signWebhook` knows by name. */
-export const WEBHOOK_PROVIDERS = ["stripe", "github", "slack", "shopify", "standard", "line", "paddle", "linear", "gitlab", "zoom", "twitch", "twilio"];
+export const WEBHOOK_PROVIDERS = ["stripe", "github", "slack", "shopify", "standard", "line", "paddle", "linear", "gitlab", "zoom", "twitch", "twilio", "typeform", "notion", "bitbucket", "intercom"];
 const hmac = (algorithm, key, data) => createHmac(algorithm, key).update(data);
 /**
  * The headers a provider sends with `body`, signed with `secret` the way its
@@ -9,7 +9,7 @@ const hmac = (algorithm, key, data) => createHmac(algorithm, key).update(data);
  * `X-Hub-Signature-256`, Slack's `v0` signature, Shopify's base64 HMAC,
  * Standard Webhooks (Svix, Resend, Clerk, …), LINE's `X-Line-Signature`,
  * Paddle Billing's `Paddle-Signature`, Linear, GitLab's token, Zoom's `v0`
- * signature and Twitch EventSub.
+ * signature, Twitch EventSub, Typeform, Notion, Bitbucket and Intercom.
  */
 export function signWebhook(body, opts) {
     const now = Math.floor(Date.now() / 1000);
@@ -67,6 +67,14 @@ export function signWebhook(body, opts) {
                 "twitch-eventsub-message-signature": `sha256=${hmac("sha256", secret, `${id}${at}${body}`).digest("hex")}`,
             };
         }
+        case "typeform":
+            return { "typeform-signature": `sha256=${hmac("sha256", secret, body).digest("base64")}` };
+        case "notion":
+            return { "x-notion-signature": `sha256=${hmac("sha256", secret, body).digest("hex")}` };
+        case "bitbucket":
+            return { "x-hub-signature": `sha256=${hmac("sha256", secret, body).digest("hex")}` };
+        case "intercom":
+            return { "x-hub-signature": `sha1=${hmac("sha1", secret, body).digest("hex")}` };
         case "twilio": {
             // Twilio's validateRequest: the URL, then each POST parameter's name and value, sorted by name; HMAC-SHA1, base64.
             if (!opts.url)

package/dist/yaml-runtime.d.ts

@@ -1,11 +1,12 @@
 import "./provided.js";
 import "./matchers.js";
-import type { YamlFile } from "./yaml.js";
+import { lookup, type YamlFile } from "./yaml.js";
+export { lookup };
 type Vars = Record<string, unknown>;
 /** Registers the scenarios of one parsed YAML file. Called from the module the plugin generates for it. */
 export declare function defineYamlScenarios(doc: YamlFile): void;
-/** `json.items[0].id` or `json.items.0.id` */
-export declare function lookup(obj: unknown, path: string): unknown;
+/** `expected` (with matchers) contained in `actual`; keys set to `{ $absent: true }` must not be there. */
+export declare function checkSubset(actual: unknown, expected: unknown, what: string): void;
 /**
  * Replace `{{name}}` in every string. A string that is only `{{name}}` takes
  * the variable's own type, so `id: "{{pollId}}"` stays a number.
@@ -17,4 +18,3 @@ export declare function interpolate(value: unknown, vars: Vars): unknown;
  * `{ $format: uuid }` become Vitest asymmetric matchers. Several `$` keys must all hold.
  */
 export declare function toMatchers(value: unknown): unknown;
-export {};

package/dist/yaml-runtime.js

@@ -1,3 +1,4 @@
+import { waitBudget } from "./deadline.js";
 import { expect, inject } from "vitest";
 import { recordYamlFailure } from "./ci.js";
 import "./provided.js";
@@ -8,16 +9,19 @@ import { signWebhook, webhookBody } from "./webhook.js";
 import { readFile } from "node:fs/promises";
 import { parse as parseYaml } from "yaml";
 import path from "node:path";
+import { isIPv4, isIPv6 } from "node:net";
 import { graphqlErrors } from "./graphql.js";
 import { schemaProblems } from "./schema.js";
 import { sse } from "./stub.js";
+import { lookup, scenarioTitle } from "./yaml.js";
+export { lookup };
 /** Registers the scenarios of one parsed YAML file. Called from the module the plugin generates for it. */
 export function defineYamlScenarios(doc) {
     for (const sc of doc.scenarios) {
         const register = sc.only ? scenario.only : sc.skip ? scenario.skip : scenario;
         const rows = sc.each ?? [undefined];
         rows.forEach((row, i) => {
-            const title = row ? interpolateTitle(sc.name, row, i) : sc.name;
+            const title = row ? scenarioTitle(sc.name, row, i) : sc.name;
             register(title, (ctx) => {
                 seq = 0;
                 return runSteps(doc, sc, [...doc.setup, ...sc.steps], ctx, { ...row });
@@ -95,9 +99,10 @@ function describeStep(step) {
     return "sql";
 }
 /** Run `fn` until it passes or `within` ms have passed, then rethrow its last error. */
-async function retry(within, fn, every = 50) {
-    if (!within)
+async function retry(wanted, fn, every = 50) {
+    if (!wanted)
         return fn();
+    const { ms: within, note } = waitBudget(wanted);
     const deadline = Date.now() + within;
     for (;;) {
         try {
@@ -106,7 +111,7 @@ async function retry(within, fn, every = 50) {
         catch (e) {
             if (Date.now() >= deadline) {
                 if (e instanceof Error)
-                    e.message = `${e.message}\n(still failing after retrying for ${within}ms)`;
+                    e.message = `${e.message}\n(still failing after retrying for ${within}ms${note})`;
                 throw e;
             }
             await new Promise((r) => setTimeout(r, every));
@@ -145,8 +150,12 @@ async function runStep(step, ctx, vars, base) {
     }
     if ("mail" in step) {
         const filter = Object.fromEntries(Object.entries(interpolate(step.mail, vars)).map(([k, v]) => {
-            const re = v && typeof v === "object" && "$regex" in v ? new RegExp(String(v.$regex)) : undefined;
-            return [k, re ?? String(v)];
+            if (!v || typeof v !== "object")
+                return [k, String(v)];
+            const { $regex, $options, ...rest } = v;
+            if ($regex === undefined || Object.keys(rest).length)
+                throw new Error(`mail ${k}: takes a string or { $regex, $options }, got ${JSON.stringify(v)}`);
+            return [k, regexOf($regex, $options)];
         }));
         const box = ctx.mail;
         await retry(step.within ?? 5000, async () => {
@@ -235,7 +244,7 @@ async function runStep(step, ctx, vars, base) {
             follow: step.follow,
         });
         verifyResponse(res, step.expect, vars, base);
-        capture(step.capture, { status: res.status, json: res.json, text: res.text, headers: Object.fromEntries(res.headers) }, vars);
+        capture(step.capture, { status: res.status, json: res.json, text: res.text, headers: Object.fromEntries(res.headers), events: res.events }, vars);
         return;
     }
     if ("request" in step) {
@@ -264,7 +273,9 @@ async function runStep(step, ctx, vars, base) {
             body = signed.body;
             const url = new URL(path, ctx.http.baseUrl);
             for (const [k, v] of Object.entries(opts.query ?? {}))
-                url.searchParams.set(k, String(v));
+                for (const item of [v].flat())
+                    if (item !== null && item !== undefined)
+                        url.searchParams.append(k, String(item));
             opts.headers = { "content-type": signed.type, ...signWebhook(signed.body, { url: url.href, ...hook }), ...opts.headers };
         }
         const e = step.expect;
@@ -296,7 +307,7 @@ async function runStep(step, ctx, vars, base) {
             }
         }
         verify(res);
-        capture(step.capture, { status: res.status, json: res.json, text: res.text, headers: Object.fromEntries(res.headers) }, vars);
+        capture(step.capture, { status: res.status, json: res.json, text: res.text, headers: Object.fromEntries(res.headers), events: res.events }, vars);
         return;
     }
     if ("insert" in step) {
@@ -356,11 +367,34 @@ function expectRows(e, rows, vars, table) {
 }
 /** Subset comparison with Vitest's diff: objects may have extra keys, arrays must have the same length. */
 function check(actual, expected, vars, what) {
-    const want = toMatchers(interpolate(expected, vars));
+    checkSubset(actual, interpolate(expected, vars), what);
+}
+/** `expected` (with matchers) contained in `actual`; keys set to `{ $absent: true }` must not be there. */
+export function checkSubset(actual, expected, what) {
+    const absent = [];
+    const want = withoutAbsent(toMatchers(expected), [], absent);
     if (want && typeof want === "object" && !isAsymmetric(want))
         expect(actual, what).toMatchObject(want);
     else
         expect(actual, what).toEqual(want);
+    for (const p of absent) {
+        const v = lookup(actual, p);
+        if (v !== undefined)
+            throw new Error(`${what}: expected no ${p}, got ${JSON.stringify(v)}`);
+    }
+}
+/** `toMatchObject` never calls a matcher for a key that is missing, so `$absent` keys are taken out and checked apart. */
+function withoutAbsent(v, at, out) {
+    if (Array.isArray(v))
+        return v.map((x, i) => withoutAbsent(x, [...at, String(i)], out));
+    if (!v || typeof v !== "object" || isAsymmetric(v))
+        return v;
+    return Object.fromEntries(Object.entries(v).flatMap(([k, x]) => {
+        if (x !== ABSENT)
+            return [[k, withoutAbsent(x, [...at, k], out)]];
+        out.push([...at, k].join("."));
+        return [];
+    }));
 }
 function conditions(when, vars) {
     if (!when)
@@ -464,14 +498,6 @@ function splitCall(s) {
     const i = s.search(/\s/);
     return [s.slice(0, i).toUpperCase(), s.slice(i).trim()];
 }
-/** `json.items[0].id` or `json.items.0.id` */
-export function lookup(obj, path) {
-    return path
-        .replace(/\[(\d+)\]/g, ".$1")
-        .split(".")
-        .filter(Boolean)
-        .reduce((cur, key) => (cur == null ? undefined : cur[key]), obj);
-}
 const WHOLE = /^\{\{\s*([^{}]+?)\s*\}\}$/;
 const PART = /\{\{\s*([^{}]+?)\s*\}\}/g;
 /**
@@ -548,21 +574,46 @@ const FORMATS = {
     email: /^[^\s@]+@[^\s@]+\.[^\s@]+$/,
     date: /^\d{4}-\d{2}-\d{2}$/,
     "date-time": /^\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}(:\d{2}(\.\d+)?)?(Z|[+-]\d{2}:?\d{2})?$/i,
+    time: /^([01]\d|2[0-3]):[0-5]\d(:[0-5]\d(\.\d+)?)?(Z|[+-]\d{2}:?\d{2})?$/i,
     uri: /^[a-z][a-z0-9+.-]*:\/\/\S+$/i,
     integer: /^-?\d+$/,
+    ipv4: (v) => isIPv4(v),
+    ipv6: (v) => isIPv6(v),
+    hostname: /^(?=.{1,253}$)[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?(\.[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?)*$/i,
+    // Crockford's base32: no I, L, O or U.
+    ulid: /^[0-7][0-9A-HJKMNP-TV-Z]{25}$/i,
+    // Three base64url parts whose first is a JSON header with `alg`; the signature isn't checked.
+    jwt: (v) => {
+        const parts = v.split(".");
+        if (parts.length !== 3 || !parts.slice(0, 2).every((p) => /^[A-Za-z0-9_-]+$/.test(p)) || !/^[A-Za-z0-9_-]*$/.test(parts[2]))
+            return false;
+        try {
+            return typeof JSON.parse(Buffer.from(parts[0], "base64url").toString()).alg === "string";
+        }
+        catch {
+            return false;
+        }
+    },
 };
-const MATCHERS = ["$type", "$regex", "$contains", "$any", "$gt", "$gte", "$lt", "$lte", "$len", "$not", "$oneOf", "$format"];
+/** `{ $absent: true }`: the key isn't there (a value of `null` is there). */
+const ABSENT = matcher("$absent", (v) => v === undefined);
+const MATCHERS = ["$absent", "$type", "$regex", "$options", "$contains", "$any", "$gt", "$gte", "$lt", "$lte", "$closeTo", "$len", "$not", "$oneOf", "$format"];
 /** An asymmetric matcher Vitest's `toEqual` / `toMatchObject` call, with a readable name in diffs. */
 function matcher(name, test) {
     return { asymmetricMatch: test, toAsymmetricMatcher: () => name, toString: () => name };
 }
-/** Numbers compare as numbers; strings (ISO dates, versions) compare as strings; anything else never matches. */
+/** A decimal as Postgres `numeric` and MySQL `DECIMAL` columns come back, to keep their precision: `"12.50"`. */
+const DECIMAL = /^-?\d+(\.\d+)?$/;
+/**
+ * Numbers compare as numbers, also with decimal strings from `numeric` / `DECIMAL` columns; strings
+ * (ISO dates, versions) compare as strings; anything else never matches.
+ */
 function compare(op, arg, holds) {
     if (typeof arg !== "number" && typeof arg !== "string")
         throw new Error(`${op} takes a number or a string, got ${JSON.stringify(arg)}`);
     return matcher(`${op} ${JSON.stringify(arg)}`, (v) => {
         if (typeof arg === "number")
-            return typeof v === "number" && holds(v - arg);
+            return (typeof v === "number" || (typeof v === "string" && DECIMAL.test(v))) && holds(Number(v) - arg);
         return typeof v === "string" && holds(v < arg ? -1 : v > arg ? 1 : 0);
     });
 }
@@ -576,11 +627,20 @@ function subsetEquals(expected, actual) {
     if (Array.isArray(expected))
         return Array.isArray(actual) && actual.length === expected.length && expected.every((e, i) => subsetEquals(e, actual[i]));
     if (expected && typeof expected === "object") {
-        return !!actual && typeof actual === "object" && Object.entries(expected).every(([k, v]) => k in actual && subsetEquals(v, actual[k]));
+        return !!actual && typeof actual === "object" && Object.entries(expected).every(([k, v]) => (v === ABSENT ? actual[k] === undefined : k in actual && subsetEquals(v, actual[k])));
     }
     return Object.is(expected, actual) || expected === actual;
 }
-function single(key, arg) {
+function regexOf(pattern, flags) {
+    try {
+        return new RegExp(String(pattern), flags === undefined ? undefined : String(flags));
+    }
+    catch (e) {
+        const bad = flags !== undefined && /flags/i.test(e.message);
+        throw new Error(bad ? `$options ${JSON.stringify(flags)} has a flag JavaScript doesn't know (use i, m, s, u, g...)` : `$regex ${JSON.stringify(pattern)} isn't a valid regular expression: ${e.message}`);
+    }
+}
+function single(key, arg, options) {
     switch (key) {
         case "$type":
             if (arg === "null")
@@ -591,11 +651,15 @@ function single(key, arg) {
                 return TYPES[arg];
             throw new Error(`$type must be one of ${Object.keys(TYPES).join(", ")}, integer, null`);
         case "$regex":
-            return expect.stringMatching(new RegExp(String(arg)));
+            return expect.stringMatching(regexOf(arg, options));
         case "$contains":
             return matcher(`$contains ${JSON.stringify(arg)}`, (v) => typeof v === "string" ? v.includes(String(arg)) : Array.isArray(v) ? v.some((item) => equalsMatcher(toMatchers(arg), item)) : false);
         case "$any":
             return expect.anything();
+        case "$absent":
+            if (arg !== true)
+                throw new Error(`$absent takes true, got ${JSON.stringify(arg)}`);
+            return ABSENT;
         case "$gt":
             return compare(key, arg, (c) => c > 0);
         case "$gte":
@@ -604,6 +668,13 @@ function single(key, arg) {
             return compare(key, arg, (c) => c < 0);
         case "$lte":
             return compare(key, arg, (c) => c <= 0);
+        case "$closeTo": {
+            // `$closeTo: 9.99` allows ±0.005 (two decimals, like toBeCloseTo); `$closeTo: [9.99, 0.1]` sets the tolerance.
+            const [target, tolerance = 0.005] = Array.isArray(arg) ? arg : [arg];
+            if (typeof target !== "number" || typeof tolerance !== "number" || !(tolerance >= 0))
+                throw new Error(`$closeTo takes a number or [number, tolerance], got ${JSON.stringify(arg)}`);
+            return matcher(`$closeTo ${JSON.stringify(arg)}`, (v) => (typeof v === "number" || (typeof v === "string" && DECIMAL.test(v))) && Math.abs(Number(v) - target) <= tolerance + Number.EPSILON);
+        }
         case "$len": {
             const want = toMatchers(arg);
             return matcher(`$len ${JSON.stringify(arg)}`, (v) => (typeof v === "string" || Array.isArray(v)) && equalsMatcher(want, v.length));
@@ -619,10 +690,10 @@ function single(key, arg) {
             return matcher(`$oneOf ${JSON.stringify(arg)}`, (v) => options.some((o) => equalsMatcher(o, v)));
         }
         case "$format": {
-            const re = FORMATS[String(arg)];
-            if (!re)
+            const f = Object.hasOwn(FORMATS, String(arg)) ? FORMATS[String(arg)] : undefined;
+            if (!f)
                 throw new Error(`$format must be one of ${Object.keys(FORMATS).join(", ")}`);
-            return matcher(`$format ${arg}`, (v) => typeof v === "string" && re.test(v));
+            return matcher(`$format ${arg}`, (v) => typeof v === "string" && (typeof f === "function" ? f(v) : f.test(v)));
         }
         default:
             throw new Error(`unknown matcher ${key} (expected one of ${MATCHERS.join(", ")})`);
@@ -640,7 +711,12 @@ export function toMatchers(value) {
         return value;
     const keys = Object.keys(value);
     if (keys.length > 0 && keys.every((k) => k.startsWith("$"))) {
-        const parts = keys.map((k) => single(k, value[k]));
+        const given = value;
+        // `$options` is the flags of the `$regex` next to it, not a matcher of its own.
+        if ("$options" in given && !("$regex" in given))
+            throw new Error("$options belongs with $regex, e.g. { $regex: \"^ab\", $options: \"i\" }");
+        const own = keys.filter((k) => k !== "$options");
+        const parts = own.map((k) => single(k, given[k], given.$options));
         if (parts.length === 1)
             return parts[0];
         return matcher(keys.map((k) => `${k} ${JSON.stringify(value[k])}`).join(", "), (v) => parts.every((p) => equalsMatcher(p, v)));
@@ -650,25 +726,52 @@ export function toMatchers(value) {
 function isAsymmetric(v) {
     return !!v && typeof v === "object" && typeof v.asymmetricMatch === "function";
 }
-function interpolateTitle(name, row, index) {
-    return name.replace(PART, (m, key) => {
-        if (key.trim() === "#")
-            return String(index);
-        const v = lookup(row, key);
-        return v === undefined ? m : typeof v === "object" ? JSON.stringify(v) : String(v);
-    });
-}
 function verifyResponse(res, e, vars, base) {
     if (e?.status !== undefined)
         expect(res).toHaveStatus(interpolate(e.status, vars));
     if (e?.headers !== undefined) {
-        const expected = Object.fromEntries(Object.entries(e.headers).map(([k, v]) => [k.toLowerCase(), v]));
-        check(Object.fromEntries(res.headers), expected, vars, "response headers");
+        // `null`: the response must not send that header (`x-powered-by: null`), as in `cookies`.
+        const entries = Object.entries(e.headers).map(([k, v]) => [k.toLowerCase(), v]);
+        for (const [name] of entries.filter(([, v]) => v === null)) {
+            const sent = res.headers.get(name);
+            if (sent !== null)
+                throw new Error(`${res.method} ${res.url}: expected no ${name} header, got ${JSON.stringify(sent)}`);
+        }
+        check(Object.fromEntries(res.headers), Object.fromEntries(entries.filter(([, v]) => v !== null)), vars, "response headers");
     }
     if (e?.json !== undefined)
         check(res.json, e.json, vars, "response JSON");
     if (e?.text !== undefined)
         check(res.text, e.text, vars, "response text");
+    if (e?.events !== undefined) {
+        if (!res.events)
+            throw new Error(`${res.method} ${res.url}: expected server-sent events, but the response is ${res.headers.get("content-type") ?? "without a content type"}, not text/event-stream`);
+        const wanted = toMatchers(interpolate(e.events, vars));
+        let at = 0;
+        for (const [i, want] of wanted.entries()) {
+            const found = res.events.findIndex((ev, j) => j >= at && subsetEquals(want, ev));
+            if (found < 0) {
+                const shown = res.events.slice(0, 20).map((ev, j) => `  ${j}: ${JSON.stringify(ev).slice(0, 200)}`).join("\n");
+                throw new Error(`${res.method} ${res.url}: expected event ${i + 1} of ${wanted.length}, ${JSON.stringify(e.events[i])}, ${at > 0 ? `after event ${at - 1}` : "in the stream"}; ${res.events.length} event(s) arrived:\n${shown || "  (none)"}${res.events.length > 20 ? "\n  …" : ""}`);
+            }
+            at = found + 1;
+        }
+    }
+    if (e?.cookies !== undefined) {
+        for (const [name, attrs] of Object.entries(e.cookies)) {
+            if (attrs === null)
+                expect(res).not.toSetCookie(name);
+            else
+                expect(res).toSetCookie(name, toMatchers(interpolate(attrs, vars)));
+        }
+    }
+    if (e?.duration !== undefined) {
+        const want = interpolate(e.duration, vars);
+        if (typeof want === "number")
+            expect(res).toRespondWithin(want);
+        else
+            check(res.durationMs, want, vars, `${res.method} ${res.url} response time (${res.durationMs}ms)`);
+    }
     if (e?.schema !== undefined) {
         const problems = schemaProblems(e.schema, res.json, base);
         if (problems.length)

package/dist/yaml.d.ts

@@ -4,6 +4,10 @@
  * and validates a file (in the Vitest main process, where line numbers are
  * known); yaml-runtime.ts runs the result inside a worker.
  */
+/** `json.items[0].id` or `json.items.0.id` */
+export declare function lookup(obj: unknown, path: string): unknown;
+/** The title a row of `each` runs under: `{{field}}` from the row, `{{#}}` its index. */
+export declare function scenarioTitle(name: string, row: Record<string, unknown>, index: number): string;
 export interface YamlFile {
     file: string;
     /** Absolute path, for CI annotations. */
@@ -132,7 +136,10 @@ export interface RequestStep {
     /** `queries`: at most this many SQL statements (not counting BEGIN/COMMIT), with `db.queries` on. */
     /** `schema`: a JSON Schema for the response JSON, inline or `file#/pointer` relative to the scenario file. */
     expect?: {
-        status?: number;
+        events?: Record<string, unknown>[];
+        cookies?: Record<string, Record<string, unknown> | null>;
+        duration?: number | Record<string, unknown>;
+        status?: number | string | (number | string)[];
         statuses?: Record<string, number>;
         headers?: Record<string, unknown>;
         json?: unknown;
@@ -156,7 +163,10 @@ export interface SubmitStep {
     headers?: Record<string, string>;
     follow?: boolean;
     expect?: {
-        status?: number;
+        events?: Record<string, unknown>[];
+        cookies?: Record<string, Record<string, unknown> | null>;
+        duration?: number | Record<string, unknown>;
+        status?: number | string | (number | string)[];
         headers?: Record<string, unknown>;
         json?: unknown;
         text?: unknown;

package/dist/yaml.js

@@ -1,5 +1,28 @@
 import { isMap, isSeq, LineCounter, parseDocument } from "yaml";
 import { WEBHOOK_PROVIDERS as PROVIDERS } from "./webhook.js";
+/**
+ * YAML scenarios: the same things a TypeScript scenario can do, written as
+ * data so teams that don't use JavaScript can write them. This module parses
+ * and validates a file (in the Vitest main process, where line numbers are
+ * known); yaml-runtime.ts runs the result inside a worker.
+ */
+/** `json.items[0].id` or `json.items.0.id` */
+export function lookup(obj, path) {
+    return path
+        .replace(/\[(\d+)\]/g, ".$1")
+        .split(".")
+        .filter(Boolean)
+        .reduce((cur, key) => (cur == null ? undefined : cur[key]), obj);
+}
+/** The title a row of `each` runs under: `{{field}}` from the row, `{{#}}` its index. */
+export function scenarioTitle(name, row, index) {
+    return name.replace(/\{\{\s*([^{}]+?)\s*\}\}/g, (m, key) => {
+        if (key.trim() === "#")
+            return String(index);
+        const v = lookup(row, key);
+        return v === undefined ? m : typeof v === "object" ? JSON.stringify(v) : String(v);
+    });
+}
 const KINDS = {
     stub: ["on", "graphql", "when", "reply", "sequence", "networkError", "times", "delay", "optional"],
     request: ["headers", "query", "json", "form", "multipart", "body", "graphql", "follow", "auth", "webhook", "concurrency", "expect", "capture", "within", "every", "timeout"],
@@ -20,8 +43,8 @@ const KINDS = {
     use: ["with"],
 };
 const EXPECT_KEYS = {
-    request: ["status", "statuses", "headers", "json", "text", "queries", "schema"],
-    submit: ["status", "headers", "json", "text", "schema"],
+    request: ["status", "statuses", "headers", "json", "text", "queries", "schema", "duration", "cookies", "events"],
+    submit: ["status", "headers", "json", "text", "schema", "duration", "cookies", "events"],
     sql: ["rows", "count"],
     db: ["rows", "count"],
 };
@@ -275,6 +298,18 @@ function parseStep(node, fail, lineOf) {
     };
     if (raw.within !== undefined && (typeof raw.within !== "number" || raw.within <= 0))
         fail(at("within"), "`within` must be a positive number of milliseconds");
+    const events = raw.expect?.events;
+    if (events !== undefined && !(Array.isArray(events) && events.every((e) => e && typeof e === "object" && !Array.isArray(e)))) {
+        fail(at("expect"), "`expect.events` is a list of events, each a subset of { event, data, id }: [{ event: message_stop }]");
+    }
+    const cookies = raw.expect?.cookies;
+    if (cookies !== undefined && (!cookies || typeof cookies !== "object" || Array.isArray(cookies) || Object.values(cookies).some((c) => c !== null && (typeof c !== "object" || Array.isArray(c))))) {
+        fail(at("expect"), "`expect.cookies` maps cookie names to attributes ({ sid: { httpOnly: true } }), {} for any value, or null for a cookie that must not be set");
+    }
+    const duration = raw.expect?.duration;
+    if (duration !== undefined && !((typeof duration === "number" && duration > 0) || (duration && typeof duration === "object" && !Array.isArray(duration)))) {
+        fail(at("expect"), "`expect.duration` is a number of milliseconds (at most), or a matcher such as { $lt: 200 }");
+    }
     const number = (key) => {
         if (raw[key] !== undefined && (typeof raw[key] !== "number" || raw[key] < 0))
             fail(at(key), `\`${key}\` must be a non-negative number`);
@@ -404,6 +439,14 @@ function parseStep(node, fail, lineOf) {
                 if (q !== undefined && raw.concurrency !== undefined)
                     fail(at("expect"), "`expect.queries` can't be used with `concurrency`");
             }
+            {
+                const status = raw.expect?.status;
+                // A template (`{{status}}` from `each`) is only known when the step runs.
+                const ok = (s) => (typeof s === "number" && Number.isInteger(s)) || (typeof s === "string" && (s.includes("{{") || /^(\d{3}|[1-5]xx)$/.test(s)));
+                if (status !== undefined && !(Array.isArray(status) ? status.length > 0 && status.every(ok) : ok(status))) {
+                    fail(at("expect"), `\`expect.status\` is a code (201), a class (2xx) or a list of them, got ${JSON.stringify(status)}`);
+                }
+            }
             if (raw.concurrency !== undefined && raw.capture !== undefined)
                 fail(at("capture"), "`capture` can't be used with `concurrency`: there is more than one response");
             {
@@ -411,8 +454,8 @@ function parseStep(node, fail, lineOf) {
                 if (statuses !== undefined) {
                     if (raw.concurrency === undefined)
                         fail(at("expect"), "`expect.statuses` needs `concurrency`");
-                    if (!statuses || typeof statuses !== "object" || Object.entries(statuses).some(([k, v]) => !/^\d{3}$/.test(k) || !Number.isInteger(v))) {
-                        fail(at("expect"), "`expect.statuses` maps status codes to counts, e.g. { 201: 1, 409: 9 }");
+                    if (!statuses || typeof statuses !== "object" || Object.entries(statuses).some(([k, v]) => !/^(\d{3}|[1-5]xx)$/i.test(k) || !Number.isInteger(v))) {
+                        fail(at("expect"), "`expect.statuses` maps status codes or classes to counts, e.g. { 201: 1, 409: 9 } or { 2xx: 1, 4xx: 9 }");
                     }
                 }
             }

package/docs/guarantees.md

@@ -0,0 +1,92 @@
+# What slicetest's checks do and don't guarantee
+
+Each check below catches a defined kind of mistake. None of them proves the app is correct. For every one: what it catches, an example, and what it can't catch, with an example. This is read from the implementation (`src/openapi.ts`, `src/recording.ts`, `src/db.ts`, `src/trace.ts`) and its tests, not measured on real projects.
+
+## OpenAPI: your app's responses
+
+With `openapi: openapi.yaml`, every response the app gives to `http` during a scenario is checked against the spec.
+
+| Catches | Can't catch |
+|---|---|
+| A status the spec doesn't document for that operation (`GET /polls/{id} responded 500, which openapi.yaml doesn't document`). | A response with the right shape and the wrong value: `{ "total": 100 }` where it should be `120` passes if `total` is a number. |
+| A JSON body that breaks the schema: wrong type, missing required property, enum mismatch (`/id must be integer`). | Extra fields, unless the schema says `additionalProperties: false`: JSON Schema allows them by default. |
+| A content type the response doesn't list. | Response headers. Only the body and its content type are checked. |
+| A path or method that isn't in the spec at all. | A body that isn't JSON (HTML, CSV, files): only its content type is compared. A response without a content type isn't held to a body schema. |
+| | A spec that is wrong or incomplete. The check compares the app with the spec; if both are wrong the same way, it passes. A spec the app generates for itself (`openapi.fromApp`) often misses error responses and nullable fields, which shows up as failures, but it can also describe a bug as the contract. |
+
+## OpenAPI: the services you stub
+
+With `stubs: [{ name, openapi }]`, the app's calls to the stub and the stub's replies are checked against the provider's spec. What this gives is narrower than a stub that can't lie: **a reply you wrote can't contradict what the spec documents**.
+
+| Catches | Can't catch |
+|---|---|
+| A stub reply with a status or body the spec doesn't document (`200 { ok: true }` where the spec documents `202 { messageId }`). | The real provider behaving differently from its published spec: undocumented errors, outdated spec, fields it stopped sending. |
+| A required query parameter the app didn't send; a missing required body; a body that breaks the request schema; a wrong request content type. | Required request headers (authorization), path and cookie parameters: only query parameters and the body are checked. |
+| An operation that isn't in the spec (the call fails the scenario). | Whether the values make sense to the provider: a well-formed `customer: "cus_nope"` that doesn't exist, a currency it doesn't support, rate limits, idempotency. |
+| | A stub reply without a content type: its body isn't compared. |
+
+`autoReply` answers calls from the spec's examples or values built from its schemas. Those are valid per the spec, not necessarily what your code needs: register routes for the replies a scenario depends on.
+
+## Recording and replay
+
+A recorded stub replays what the real service answered **at the time of recording**, for requests identical in method, path, query and body. Credential-named values (`api_key`, `password`, …) are redacted before the comparison, so requests that differ only in those match.
+
+| Catches | Can't catch |
+|---|---|
+| The app's request changed: no recording matches, the call gets a 501 (unless `autoReply` answers from the spec instead) and the scenario fails with a hint to record again. | The real service changing after the recording: a renamed field, a new error, a removed endpoint. Replays are offline, so nothing notices. |
+| Replies that contradict the provider's spec, when the stub also has one. | Recordings that were never right: recorded with a wrong account, a sandbox that behaves unlike production. |
+| | Secrets stored in places the redaction doesn't look at. Request headers are never stored and well-known credential names in the query or body become `[redacted]`; everything else is stored as sent. Review the file before committing it. |
+
+To keep recordings honest, re-record on a schedule (delete the entries, run once with `SLICETEST_RECORD`) and review the diff, or check the real service with a separate contract test. slicetest doesn't do either on its own.
+
+## `trace()` and snapshots
+
+`expect(await trace()).toMatchSnapshot()` pins the requests to the app and their responses, the calls to stubs with the replies' statuses, the mail, and the database changes, with dates and UUIDs masked (the exact fields are below).
+
+| Catches | Can't catch |
+|---|---|
+| A change in what the trace holds, as a diff in review: a new outbound call, a different status or response body, an extra row, a column written differently. The trace has, per request to the app: method, path and query, status and response body; per stub call: method, path and query, request body and the reply's status; the mail; the database changes. | Whether the pinned behavior is right. A snapshot records what the app does today, bugs included; the first run and every `-u` accept whatever it shows. |
+| | Anything not in the trace: request bodies and headers sent to the app, response headers (a cookie attribute), reply bodies and headers of a stub, logs, files, other stores. Only the last 20 requests to the app are kept. Masked values (dates, UUIDs) aren't checked. |
+
+A snapshot update is a change to what the tests expect, so review it like a change to an assertion. The same goes for scenarios produced by `slicetest gen` and `slicetest record`: they start from the spec or from what the app did, so a misunderstanding in either is copied into the expected values.
+
+## `db.changes()`
+
+A diff of the database between the start of the scenario (after reset and seed, or the last `checkpoint()`) and now, matched by primary key.
+
+| Catches | Can't catch |
+|---|---|
+| A write to a table the test didn't list, when asserted with `toEqual`; which columns an update changed. | A row written and deleted again inside the scenario: it's a net diff, only the end state counts. |
+| An insert that should have happened and didn't. | Tables outside `db.schemas` (default `public`), tables in `db.keep`, migration bookkeeping tables, and columns in `db.ignoreChanges` / `ignore`. |
+| | A table the app creates while a scenario runs: it is emptied from the next reset on, but `db.changes()` of the scenario that created it doesn't list it (the list is read at each reset). |
+| | Writes to anything but the database: Redis, files, a queue. |
+| | Reads. A query that returns the wrong rows changes nothing. |
+
+A table without a primary key reports an update as one deleted row plus one inserted row.
+
+## OpenAPI coverage
+
+The report counts **documented responses of your app's spec**: each `METHOD /path` + status key the spec lists (`200`, `404`, `4XX`, `default`) is covered when a scenario's `http` request got a response that falls under it, crediting only the most specific key: the exact status, else `4XX`, else `default`. A 404 on an operation that documents both `404` and `default` covers `404` only. The percentage is covered keys over all keys in the spec.
+
+| Counts | Doesn't count |
+|---|---|
+| A `404` on `GET /polls/{id}` that some scenario produced. | Lines or branches of your code, or business rules. 100% means every documented response was seen once, not that the behavior behind each was tested. |
+| | How many different inputs reached a response, or whether the scenario that produced it asserted anything about it. |
+| | Responses to requests that didn't go through the scenario's `http` client, and undocumented responses (those fail the scenario instead). |
+
+A run filtered by file, `-t`, `--tag` or a shard prints its coverage marked as partial and doesn't enforce `minCoverage`.
+
+## Unexpected stub calls and unused routes
+
+| Catches | Can't catch |
+|---|---|
+| A call to a stub that no registered route (or recording, or `autoReply`) answers: the scenario fails with the closest route and why it differed. | A call the app should have made and didn't, unless a route is registered and `strictStubs` is on (or `toHaveReceived` is asserted). |
+| With `strictStubs`, a registered route the app never called. | Calls to hosts no stub covers, unless `offline: true` refuses them. A client that ignores proxy settings goes around `hosts` and `offline`. |
+
+## Tests written by an AI agent
+
+An agent that misreads a requirement tends to misread it in the code and in the test, and the test passes. Contract checks and snapshots don't catch that: they compare the app with the spec and with itself. Before merging generated scenarios:
+
+- Review the expected values against the acceptance criteria, not against the app's output.
+- Treat snapshot updates (`-u`) and `gen` / `record` output as code changes to be reviewed.
+- Prefer assertions on what the requirement states (`db.changes()` with the exact rows, `toHaveReceived` with the exact payload) over `toMatchSnapshot` alone.