slicetest 0.4.0 → 0.5.0

package/dist/webhook.js

@@ -0,0 +1,52 @@
+import { createHmac, randomUUID } from "node:crypto";
+const hmac = (algorithm, key, data) => createHmac(algorithm, key).update(data);
+/**
+ * The headers a provider sends with `body`, signed with `secret` the way its
+ * SDK verifies them: Stripe's `Stripe-Signature`, GitHub's
+ * `X-Hub-Signature-256`, Slack's `v0` signature, Shopify's base64 HMAC and
+ * Standard Webhooks (Svix, Resend, Clerk, …).
+ */
+export function signWebhook(body, opts) {
+    const now = Math.floor(Date.now() / 1000);
+    const ts = opts.timestamp ?? (opts.stale ? now - 600 : now);
+    const secret = opts.invalidSignature ? `${opts.secret}-not-the-real-secret` : opts.secret;
+    const p = opts.provider;
+    if (typeof p === "object") {
+        if (!p.header)
+            throw new Error("slicetest: a custom webhook provider needs `header`, e.g. { header: \"X-Signature\", prefix: \"sha256=\" }");
+        return { [p.header]: `${p.prefix ?? ""}${hmac(p.algorithm ?? "sha256", secret, body).digest(p.encoding ?? "hex")}` };
+    }
+    switch (p) {
+        case "stripe":
+            return { "stripe-signature": `t=${ts},v1=${hmac("sha256", secret, `${ts}.${body}`).digest("hex")}` };
+        case "github":
+            return {
+                "x-hub-signature-256": `sha256=${hmac("sha256", secret, body).digest("hex")}`,
+                "x-github-event": opts.event ?? "ping",
+                "x-github-delivery": randomUUID(),
+            };
+        case "slack":
+            return { "x-slack-request-timestamp": String(ts), "x-slack-signature": `v0=${hmac("sha256", secret, `v0:${ts}:${body}`).digest("hex")}` };
+        case "shopify":
+            return {
+                "x-shopify-hmac-sha256": hmac("sha256", secret, body).digest("base64"),
+                ...(opts.event ? { "x-shopify-topic": opts.event } : {}),
+                "x-shopify-webhook-id": randomUUID(),
+            };
+        case "standard": {
+            const id = opts.id ?? `msg_${randomUUID().replace(/-/g, "")}`;
+            const key = secret.startsWith("whsec_") ? Buffer.from(secret.slice(6), "base64") : Buffer.from(secret);
+            return { "webhook-id": id, "webhook-timestamp": String(ts), "webhook-signature": `v1,${hmac("sha256", key, `${id}.${ts}.${body}`).digest("base64")}` };
+        }
+        default:
+            throw new Error(`slicetest: unknown webhook provider ${JSON.stringify(p)} (expected stripe, github, slack, shopify, standard or { header, prefix, encoding })`);
+    }
+}
+/** The bytes to send and their content type: objects as JSON, URLSearchParams as a form (Slack commands), strings as-is. */
+export function webhookBody(payload) {
+    if (typeof payload === "string")
+        return { body: payload, type: "application/json" };
+    if (payload instanceof URLSearchParams)
+        return { body: payload.toString(), type: "application/x-www-form-urlencoded" };
+    return { body: JSON.stringify(payload ?? {}), type: "application/json" };
+}

package/dist/yaml-runtime.js

@@ -4,6 +4,7 @@ import "./provided.js";
 import { formatChanges } from "./db.js";
 import "./matchers.js";
 import { scenario } from "./scenario.js";
+import { signWebhook, webhookBody } from "./webhook.js";
 /** Registers the scenarios of one parsed YAML file. Called from the module the plugin generates for it. */
 export function defineYamlScenarios(doc) {
     for (const sc of doc.scenarios) {
@@ -47,6 +48,10 @@ function describeStep(step) {
         return `received ${step.received}${step.call ? ` ${step.call}` : ""}`;
     if ("insert" in step)
         return `insert ${step.insert}`;
+    if ("make" in step)
+        return `make ${step.make}`;
+    if ("chaos" in step)
+        return `chaos ${step.chaos}`;
     if ("db" in step)
         return `db ${step.db}`;
     if ("changes" in step)
@@ -159,16 +164,22 @@ async function runStep(step, ctx, vars) {
     if ("request" in step) {
         const [method, rawPath] = splitCall(step.request);
         const path = interpolate(rawPath, vars);
+        const headers = interpolate(step.headers, vars);
         const opts = {
-            headers: interpolate(step.headers, vars),
+            headers: step.auth ? { ...ctx.auth.header(step.auth === true ? {} : interpolate(step.auth, vars)), ...headers } : headers,
             query: interpolate(step.query, vars),
             follow: step.follow,
         };
-        const body = step.json !== undefined
+        let body = step.json !== undefined
             ? interpolate(step.json, vars)
             : step.form !== undefined
                 ? ctx.http.form(interpolate(step.form, vars))
                 : interpolate(step.body, vars);
+        if (step.webhook) {
+            const signed = webhookBody(body);
+            body = signed.body;
+            opts.headers = { "content-type": signed.type, ...signWebhook(signed.body, interpolate(step.webhook, vars)), ...opts.headers };
+        }
         const e = step.expect;
         const verify = (res) => {
             if (e?.status !== undefined)
@@ -190,7 +201,18 @@ async function runStep(step, ctx, vars) {
             all.forEach(verify);
             return;
         }
-        const res = await ctx.http.request(method, path, body, opts);
+        const max = e?.queries === undefined ? undefined : interpolate(e.queries, vars);
+        let res;
+        const queries = await (max === undefined ? undefined : ctx.db.queries(async () => (res = await ctx.http.request(method, path, body, opts))));
+        if (!queries)
+            res = await ctx.http.request(method, path, body, opts);
+        if (queries && max !== undefined) {
+            const ran = queries.withoutTransactions();
+            if (ran.length > max) {
+                const shapes = ran.shapes().map((s) => `  ${s.count > 1 ? `×${s.count} ` : ""}${s.sql}`);
+                throw new Error(`expected at most ${max} SQL statements for ${method} ${path}, the app ran ${ran.length}:\n${shapes.join("\n")}`);
+            }
+        }
         verify(res);
         capture(step.capture, { status: res.status, json: res.json, text: res.text, headers: Object.fromEntries(res.headers) }, vars);
         return;
@@ -200,6 +222,22 @@ async function runStep(step, ctx, vars) {
         capture(step.capture, { rows, row: rows[0] }, vars);
         return;
     }
+    if ("chaos" in step) {
+        const { chaos, name: _, ...opts } = step;
+        ctx.stub(chaos).chaos(opts);
+        return;
+    }
+    if ("make" in step) {
+        const given = interpolate(step.rows, vars) ?? {};
+        const rows = [];
+        if (Array.isArray(given))
+            for (const r of given)
+                rows.push(await ctx.db.make(step.make, r));
+        else
+            rows.push(...(await ctx.db.makeMany(step.make, step.count ?? 1, given)));
+        capture(step.capture, { rows, row: rows[0] }, vars);
+        return;
+    }
     if ("sql" in step) {
         const rows = await ctx.db.query(step.sql, interpolate(step.params, vars) ?? []);
         expectRows(step.expect, rows, vars);

package/dist/yaml.d.ts

@@ -21,7 +21,7 @@ export interface YamlScenario {
     timeout?: number;
     steps: Step[];
 }
-export type Step = (StubStep | RequestStep | InsertStep | SqlStep | DbStep | ReceivedStep | ChangesStep | CheckpointStep | LogStep | SnapshotStep | MailStep) & {
+export type Step = (StubStep | RequestStep | InsertStep | MakeStep | ChaosStep | SqlStep | DbStep | ReceivedStep | ChangesStep | CheckpointStep | LogStep | SnapshotStep | MailStep) & {
     line: number;
     name?: string;
 };
@@ -57,14 +57,31 @@ export interface RequestStep {
     form?: Record<string, unknown>;
     body?: string;
     follow?: boolean;
+    /** Send `Authorization: Bearer` with a token from the `auth` issuer: `true`, or the claims, e.g. { sub: u1, roles: [admin] }. */
+    auth?: true | Record<string, unknown>;
+    /** Sign the body as this provider would deliver a webhook: { provider: stripe, secret: whsec_test, event, stale, invalidSignature }. */
+    webhook?: {
+        provider: string | {
+            header: string;
+            algorithm?: string;
+            encoding?: "hex" | "base64";
+            prefix?: string;
+        };
+        secret: string;
+        event?: string;
+        stale?: boolean;
+        invalidSignature?: boolean;
+    };
     /** Send the request this many times at once. `expect` applies to every response; `statuses` counts them. */
     concurrency?: number;
+    /** `queries`: at most this many SQL statements (not counting BEGIN/COMMIT), with `db.queries` on. */
     expect?: {
         status?: number;
         statuses?: Record<string, number>;
         headers?: Record<string, unknown>;
         json?: unknown;
         text?: unknown;
+        queries?: number;
     };
     capture?: Record<string, string>;
 }
@@ -73,6 +90,24 @@ export interface InsertStep {
     rows: Record<string, unknown> | Record<string, unknown>[];
     capture?: Record<string, string>;
 }
+/** Rows built by `db.make()`: only the columns given are fixed, the rest is filled from the schema. */
+export interface MakeStep {
+    make: string;
+    rows?: Record<string, unknown> | Record<string, unknown>[];
+    /** Make this many rows, each with `rows` (a mapping). */
+    count?: number;
+    capture?: Record<string, string>;
+}
+/** Faults injected into a stub's answers for the rest of the scenario, like `stub(name).chaos()`. */
+export interface ChaosStep {
+    chaos: string;
+    failFirst?: number;
+    errorRate?: number;
+    statuses?: number[];
+    networkErrorRate?: number;
+    latency?: number | [number, number];
+    seed?: number;
+}
 export interface SqlStep {
     sql: string;
     params?: unknown[];

package/dist/yaml.js

@@ -1,7 +1,7 @@
 import { isMap, isSeq, LineCounter, parseDocument } from "yaml";
 const KINDS = {
     stub: ["on", "when", "reply", "sequence", "networkError", "times", "delay"],
-    request: ["headers", "query", "json", "form", "body", "follow", "concurrency", "expect", "capture"],
+    request: ["headers", "query", "json", "form", "body", "follow", "auth", "webhook", "concurrency", "expect", "capture"],
     insert: ["rows", "capture"],
     sql: ["params", "expect", "capture", "within"],
     db: ["where", "orderBy", "expect", "capture", "within"],
@@ -11,13 +11,17 @@ const KINDS = {
     checkpoint: [],
     snapshot: ["mask"],
     mail: ["times", "within", "capture"],
+    make: ["rows", "count", "capture"],
+    chaos: ["failFirst", "errorRate", "statuses", "networkErrorRate", "latency", "seed"],
 };
 const EXPECT_KEYS = {
-    request: ["status", "statuses", "headers", "json", "text"],
+    request: ["status", "statuses", "headers", "json", "text", "queries"],
     sql: ["rows", "count"],
     db: ["rows", "count"],
 };
 const CONDITION_KEYS = ["query", "headers", "json", "body"];
+const WEBHOOK_KEYS = ["provider", "secret", "event", "stale", "invalidSignature"];
+const WEBHOOK_PROVIDERS = ["stripe", "github", "slack", "shopify", "standard"];
 const MAIL_KEYS = ["to", "from", "subject", "text", "html"];
 const CHANGE_KEYS = ["inserted", "updated", "deleted"];
 const RESPONSE_KEYS = ["status", "headers", "body"];
@@ -181,8 +185,31 @@ function parseStep(node, fail, lineOf) {
             call("request", REQUEST);
             if (["json", "form", "body"].filter((k) => raw[k] !== undefined).length > 1)
                 fail(node, "use only one of json / form / body");
+            if (raw.webhook !== undefined) {
+                const w = raw.webhook;
+                if (!w || typeof w !== "object" || Array.isArray(w))
+                    fail(at("webhook"), "`webhook` must be a mapping such as { provider: stripe, secret: whsec_test }");
+                for (const k of Object.keys(w))
+                    if (!WEBHOOK_KEYS.includes(k))
+                        fail(at("webhook"), `unknown key "${k}" in webhook (allowed: ${WEBHOOK_KEYS.join(", ")})`);
+                if (typeof w.secret !== "string" || !w.secret)
+                    fail(at("webhook"), "`webhook.secret` is required: the signing secret the app is configured with");
+                const p = w.provider;
+                if (!(typeof p === "string" ? WEBHOOK_PROVIDERS.includes(p) : p && typeof p === "object" && typeof p.header === "string")) {
+                    fail(at("webhook"), `\`webhook.provider\` must be one of ${WEBHOOK_PROVIDERS.join(", ")} or { header, prefix, encoding }`);
+                }
+            }
+            if (raw.auth !== undefined && raw.auth !== true && !(raw.auth && typeof raw.auth === "object" && !Array.isArray(raw.auth)))
+                fail(at("auth"), "`auth` must be true or the token's claims, e.g. { sub: u1, roles: [admin] }");
             if (raw.concurrency !== undefined && !(Number.isInteger(raw.concurrency) && raw.concurrency >= 1))
                 fail(at("concurrency"), "`concurrency` must be a positive whole number");
+            {
+                const q = raw.expect?.queries;
+                if (q !== undefined && !(Number.isInteger(q) && q >= 0))
+                    fail(at("expect"), "`expect.queries` is the most SQL statements the request may run, e.g. 3");
+                if (q !== undefined && raw.concurrency !== undefined)
+                    fail(at("expect"), "`expect.queries` can't be used with `concurrency`");
+            }
             if (raw.concurrency !== undefined && raw.capture !== undefined)
                 fail(at("capture"), "`capture` can't be used with `concurrency`: there is more than one response");
             {
@@ -200,6 +227,33 @@ function parseStep(node, fail, lineOf) {
             if (!raw.rows || typeof raw.rows !== "object")
                 fail(at("rows"), "an insert step needs `rows` (a mapping or a list of mappings)");
             break;
+        case "chaos":
+            for (const k of ["errorRate", "networkErrorRate"]) {
+                if (raw[k] !== undefined && !(typeof raw[k] === "number" && raw[k] >= 0 && raw[k] <= 1))
+                    fail(at(k), `\`${k}\` must be a number between 0 and 1`);
+            }
+            for (const k of ["failFirst", "seed"])
+                if (raw[k] !== undefined && !Number.isInteger(raw[k]))
+                    fail(at(k), `\`${k}\` must be a whole number`);
+            if (raw.statuses !== undefined && !(Array.isArray(raw.statuses) && raw.statuses.length > 0 && raw.statuses.every((s) => Number.isInteger(s) && s >= 400 && s <= 599))) {
+                fail(at("statuses"), "`statuses` must be a list of 4xx/5xx codes, e.g. [503]");
+            }
+            {
+                const l = raw.latency;
+                if (l !== undefined && !(typeof l === "number" || (Array.isArray(l) && l.length === 2 && l.every((x) => typeof x === "number"))))
+                    fail(at("latency"), "`latency` must be milliseconds or [min, max]");
+            }
+            break;
+        case "make":
+            if (raw.rows !== undefined && (!raw.rows || typeof raw.rows !== "object"))
+                fail(at("rows"), "`rows` must be a mapping or a list of mappings");
+            if (raw.count !== undefined) {
+                if (!(Number.isInteger(raw.count) && raw.count >= 1))
+                    fail(at("count"), "`count` must be a positive whole number");
+                if (Array.isArray(raw.rows))
+                    fail(at("count"), "use either `count` or a list of `rows`");
+            }
+            break;
         case "log":
             try {
                 new RegExp(raw.log);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "slicetest",
-  "version": "0.4.0",
+  "version": "0.5.0",
   "description": "Framework-agnostic slice tests: drive a real app process over HTTP with a real Postgres, MySQL or SQLite and stubbed outbound APIs, checked against OpenAPI contracts.",
   "keywords": [
     "testing",
@@ -18,7 +18,13 @@
     "testcontainers",
     "language-agnostic",
     "openapi",
-    "contract-testing"
+    "contract-testing",
+    "jwt",
+    "oidc",
+    "webhooks",
+    "chaos-testing",
+    "n-plus-one",
+    "test-data-factory"
   ],
   "author": "pon-bok",
   "repository": {

package/schema/scenario.schema.json

@@ -180,6 +180,81 @@
             "follow": {
               "type": "boolean"
             },
+            "auth": {
+              "description": "Send Authorization: Bearer with a token from the auth issuer: true, or the token's claims",
+              "oneOf": [
+                {
+                  "const": true
+                },
+                {
+                  "type": "object"
+                }
+              ]
+            },
+            "webhook": {
+              "type": "object",
+              "description": "Sign the body as this provider delivers webhooks",
+              "required": [
+                "provider",
+                "secret"
+              ],
+              "additionalProperties": false,
+              "properties": {
+                "provider": {
+                  "oneOf": [
+                    {
+                      "enum": [
+                        "stripe",
+                        "github",
+                        "slack",
+                        "shopify",
+                        "standard"
+                      ]
+                    },
+                    {
+                      "type": "object",
+                      "required": [
+                        "header"
+                      ],
+                      "additionalProperties": false,
+                      "properties": {
+                        "header": {
+                          "type": "string"
+                        },
+                        "algorithm": {
+                          "type": "string"
+                        },
+                        "encoding": {
+                          "enum": [
+                            "hex",
+                            "base64"
+                          ]
+                        },
+                        "prefix": {
+                          "type": "string"
+                        }
+                      }
+                    }
+                  ]
+                },
+                "secret": {
+                  "type": "string",
+                  "description": "The signing secret the app is configured with"
+                },
+                "event": {
+                  "type": "string",
+                  "description": "Event type, e.g. X-GitHub-Event"
+                },
+                "stale": {
+                  "type": "boolean",
+                  "description": "Signed ten minutes ago"
+                },
+                "invalidSignature": {
+                  "type": "boolean",
+                  "description": "Signed with a different secret"
+                }
+              }
+            },
             "concurrency": {
               "type": "integer",
               "minimum": 1,
@@ -212,6 +287,11 @@
                     "minimum": 0
                   },
                   "description": "With `concurrency`: how many responses have each status, e.g. { 201: 1, 409: 9 }"
+                },
+                "queries": {
+                  "type": "integer",
+                  "minimum": 0,
+                  "description": "At most this many SQL statements for this request, not counting BEGIN/COMMIT (needs db.queries)"
                 }
               }
             },
@@ -255,6 +335,106 @@
             }
           }
         },
+        {
+          "type": "object",
+          "required": [
+            "make"
+          ],
+          "additionalProperties": false,
+          "properties": {
+            "make": {
+              "type": "string",
+              "description": "Table to make rows in: required columns and parent rows are filled from the schema"
+            },
+            "name": {
+              "type": "string",
+              "description": "Label shown when this step fails"
+            },
+            "rows": {
+              "type": [
+                "object",
+                "array"
+              ],
+              "description": "Columns to set: a mapping, or a list of mappings for several rows"
+            },
+            "count": {
+              "type": "integer",
+              "minimum": 1,
+              "description": "Make this many rows, each with `rows`"
+            },
+            "capture": {
+              "type": "object",
+              "description": "Save values as variables for later steps, e.g. { orderId: row.id }",
+              "additionalProperties": {
+                "type": "string"
+              }
+            }
+          }
+        },
+        {
+          "type": "object",
+          "required": [
+            "chaos"
+          ],
+          "additionalProperties": false,
+          "properties": {
+            "chaos": {
+              "type": "string",
+              "description": "Stub to inject faults into, for the rest of the scenario"
+            },
+            "name": {
+              "type": "string",
+              "description": "Label shown when this step fails"
+            },
+            "failFirst": {
+              "type": "integer",
+              "minimum": 0,
+              "description": "Fail the first n calls, then answer normally"
+            },
+            "errorRate": {
+              "type": "number",
+              "minimum": 0,
+              "maximum": 1,
+              "description": "Share of calls answered with an error status"
+            },
+            "statuses": {
+              "type": "array",
+              "minItems": 1,
+              "items": {
+                "type": "integer",
+                "minimum": 400,
+                "maximum": 599
+              },
+              "description": "Error statuses to pick from (default 500, 502, 503)"
+            },
+            "networkErrorRate": {
+              "type": "number",
+              "minimum": 0,
+              "maximum": 1,
+              "description": "Share of calls whose connection is dropped"
+            },
+            "latency": {
+              "oneOf": [
+                {
+                  "type": "number"
+                },
+                {
+                  "type": "array",
+                  "items": {
+                    "type": "number"
+                  },
+                  "minItems": 2,
+                  "maxItems": 2
+                }
+              ],
+              "description": "Extra delay in ms, or [min, max]"
+            },
+            "seed": {
+              "type": "integer",
+              "description": "Seed for the random faults"
+            }
+          }
+        },
         {
           "type": "object",
           "required": [